20
bili_login.py
Normal file
20
bili_login.py
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
import asyncio
|
||||||
|
from bilibili_api import login
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
print("请使用 Bilibili 手机 App 扫描二维码登录")
|
||||||
|
# 实例化二维码登录类
|
||||||
|
qr = login.QRLogin()
|
||||||
|
# 获取二维码
|
||||||
|
demo = qr.show_qrcode()
|
||||||
|
# 等待登录
|
||||||
|
credential = await qr.login()
|
||||||
|
|
||||||
|
print("\n登录成功!请将以下信息填入 config.toml 的 [bilibili] 部分:")
|
||||||
|
print(f"sessdata = \"{credential.sessdata}\"")
|
||||||
|
print(f"bili_jct = \"{credential.bili_jct}\"")
|
||||||
|
print(f"buvid3 = \"{credential.buvid3}\"")
|
||||||
|
print(f"dedeuserid = \"{credential.dedeuserid}\"")
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
asyncio.run(main())
|
||||||
23
config.toml
23
config.toml
@@ -75,3 +75,26 @@ client_key_path = "ca/key.pem"
|
|||||||
image_height = 1920
|
image_height = 1920
|
||||||
# 图片宽度
|
# 图片宽度
|
||||||
image_width = 1080
|
image_width = 1080
|
||||||
|
|
||||||
|
# 线程管理配置
|
||||||
|
[threading]
|
||||||
|
# 主线程池最大工作线程数 (1-100)
|
||||||
|
max_workers = 10
|
||||||
|
# 客户端线程池最大工作线程数 (1-50)
|
||||||
|
client_max_workers = 5
|
||||||
|
# 线程名称前缀
|
||||||
|
thread_name_prefix = "NeoBot-Thread"
|
||||||
|
|
||||||
|
# Bilibili 配置
|
||||||
|
[bilibili]
|
||||||
|
sessdata = "38140b76%2C1787735191%2Cf39c3%2A21CjDklI7Qvv-0Hsw7aux5cNxgEfNMeYwkTS0OoqZdyK9btBgYoDWbNY1vWb6mSixWvOkSVkUwYzRyb1FRcUJzaEtidkcxNVNMMzdvdTdKQl84aGdLSnJ6THZIT3c5dFhkbWRUVnJCWi1WZnpMR0FtQl96R0RzaHJZV3RQUGtLWGJNc09jZG9STnh3IIEC"
|
||||||
|
bili_jct = "2f0fe1768ab257630e554a82c3f01fe2"
|
||||||
|
buvid3 = "5AA3B81B-5CC0-2DAD-4DA6-B6741BA2F77D49525infoc"
|
||||||
|
dedeuserid = ""
|
||||||
|
|
||||||
|
# 本地文件服务器配置
|
||||||
|
# 用于下载远程文件到本地并提供本地访问,解决 NapCat 无法直接访问某些远程资源的问题
|
||||||
|
[local_file_server]
|
||||||
|
enabled = true # 是否启用
|
||||||
|
host = "101.36.126.55" # 监听地址
|
||||||
|
port = 3003 # 监听端口
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
import tomllib
|
import tomllib
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
from .config_models import ConfigModel, NapCatWSModel, BotModel, RedisModel, DockerModel, ImageManagerModel, MySQLModel, ReverseWSModel
|
from .config_models import ConfigModel, NapCatWSModel, BotModel, RedisModel, DockerModel, ImageManagerModel, MySQLModel, ReverseWSModel, ThreadingModel, BilibiliModel, LocalFileServerModel
|
||||||
from .utils.logger import ModuleLogger
|
from .utils.logger import ModuleLogger
|
||||||
from .utils.exceptions import ConfigError, ConfigNotFoundError, ConfigValidationError
|
from .utils.exceptions import ConfigError, ConfigNotFoundError, ConfigValidationError
|
||||||
|
|
||||||
@@ -136,6 +136,27 @@ class Config:
|
|||||||
"""
|
"""
|
||||||
return self._model.reverse_ws
|
return self._model.reverse_ws
|
||||||
|
|
||||||
|
@property
|
||||||
|
def threading(self) -> ThreadingModel:
|
||||||
|
"""
|
||||||
|
获取线程管理配置
|
||||||
|
"""
|
||||||
|
return self._model.threading
|
||||||
|
|
||||||
|
@property
|
||||||
|
def bilibili(self) -> BilibiliModel:
|
||||||
|
"""
|
||||||
|
获取 Bilibili 配置
|
||||||
|
"""
|
||||||
|
return self._model.bilibili
|
||||||
|
|
||||||
|
@property
|
||||||
|
def local_file_server(self) -> LocalFileServerModel:
|
||||||
|
"""
|
||||||
|
获取本地文件服务器配置
|
||||||
|
"""
|
||||||
|
return self._model.local_file_server
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# 实例化全局配置对象
|
# 实例化全局配置对象
|
||||||
|
|||||||
@@ -79,14 +79,32 @@ class ImageManagerModel(BaseModel):
|
|||||||
image_width: int = 1080
|
image_width: int = 1080
|
||||||
|
|
||||||
|
|
||||||
class ReverseWSModel(BaseModel):
|
class ThreadingModel(BaseModel):
|
||||||
"""
|
"""
|
||||||
对应 `config.toml` 中的 `[reverse_ws]` 配置块。
|
对应 `config.toml` 中的 `[threading]` 配置块。
|
||||||
"""
|
"""
|
||||||
enabled: bool = False
|
max_workers: int = Field(default=10, ge=1, le=100)
|
||||||
|
client_max_workers: int = Field(default=5, ge=1, le=50)
|
||||||
|
thread_name_prefix: str = "NeoBot-Thread"
|
||||||
|
|
||||||
|
|
||||||
|
class BilibiliModel(BaseModel):
|
||||||
|
"""
|
||||||
|
对应 `config.toml` 中的 `[bilibili]` 配置块。
|
||||||
|
"""
|
||||||
|
sessdata: Optional[str] = None
|
||||||
|
bili_jct: Optional[str] = None
|
||||||
|
buvid3: Optional[str] = None
|
||||||
|
dedeuserid: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class LocalFileServerModel(BaseModel):
|
||||||
|
"""
|
||||||
|
对应 `config.toml` 中的 `[local_file_server]` 配置块。
|
||||||
|
"""
|
||||||
|
enabled: bool = True
|
||||||
host: str = "0.0.0.0"
|
host: str = "0.0.0.0"
|
||||||
port: int = 3002
|
port: int = 3003
|
||||||
token: Optional[str] = None
|
|
||||||
|
|
||||||
|
|
||||||
class ConfigModel(BaseModel):
|
class ConfigModel(BaseModel):
|
||||||
@@ -100,5 +118,8 @@ class ConfigModel(BaseModel):
|
|||||||
docker: DockerModel
|
docker: DockerModel
|
||||||
image_manager: ImageManagerModel
|
image_manager: ImageManagerModel
|
||||||
reverse_ws: ReverseWSModel
|
reverse_ws: ReverseWSModel
|
||||||
|
threading: ThreadingModel = Field(default_factory=ThreadingModel)
|
||||||
|
bilibili: BilibiliModel = Field(default_factory=BilibiliModel)
|
||||||
|
local_file_server: LocalFileServerModel = Field(default_factory=LocalFileServerModel)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from .mysql_manager import MySQLManager
|
|||||||
from .browser_manager import BrowserManager
|
from .browser_manager import BrowserManager
|
||||||
from .image_manager import ImageManager
|
from .image_manager import ImageManager
|
||||||
from .reverse_ws_manager import ReverseWSManager
|
from .reverse_ws_manager import ReverseWSManager
|
||||||
|
from .thread_manager import thread_manager
|
||||||
|
|
||||||
# --- 实例化所有单例管理器 ---
|
# --- 实例化所有单例管理器 ---
|
||||||
|
|
||||||
@@ -40,6 +41,9 @@ image_manager = ImageManager()
|
|||||||
# 反向 WebSocket 管理器
|
# 反向 WebSocket 管理器
|
||||||
reverse_ws_manager = ReverseWSManager()
|
reverse_ws_manager = ReverseWSManager()
|
||||||
|
|
||||||
|
# 线程管理器
|
||||||
|
thread_manager.start()
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"permission_manager",
|
"permission_manager",
|
||||||
"command_manager",
|
"command_manager",
|
||||||
@@ -50,4 +54,5 @@ __all__ = [
|
|||||||
"browser_manager",
|
"browser_manager",
|
||||||
"image_manager",
|
"image_manager",
|
||||||
"reverse_ws_manager",
|
"reverse_ws_manager",
|
||||||
|
"thread_manager",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -11,15 +11,12 @@ from websockets.server import WebSocketServerProtocol
|
|||||||
from typing import Dict, Any, Optional, Set
|
from typing import Dict, Any, Optional, Set
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import uuid
|
import uuid
|
||||||
import random
|
import threading
|
||||||
|
|
||||||
from ..config_loader import global_config
|
|
||||||
from ..utils.logger import ModuleLogger
|
from ..utils.logger import ModuleLogger
|
||||||
from ..utils.exceptions import WebSocketError, WebSocketConnectionError
|
|
||||||
from ..utils.error_codes import ErrorCode, create_error_response
|
from ..utils.error_codes import ErrorCode, create_error_response
|
||||||
from .command_manager import matcher
|
from .command_manager import matcher
|
||||||
from models.events.factory import EventFactory
|
from models.events.factory import EventFactory
|
||||||
from .redis_manager import redis_manager
|
|
||||||
from ..bot import Bot
|
from ..bot import Bot
|
||||||
from ..ws import ReverseWSClient as _ReverseWSClient
|
from ..ws import ReverseWSClient as _ReverseWSClient
|
||||||
|
|
||||||
@@ -82,6 +79,18 @@ class ReverseWSManager:
|
|||||||
# 正在处理的事件ID集合(用于防止重复处理)
|
# 正在处理的事件ID集合(用于防止重复处理)
|
||||||
self._processing_events: Dict[str, Set[str]] = {} # client_id: set of event_ids
|
self._processing_events: Dict[str, Set[str]] = {} # client_id: set of event_ids
|
||||||
|
|
||||||
|
# 线程安全锁
|
||||||
|
self._clients_lock = threading.RLock()
|
||||||
|
self._bots_lock = threading.RLock()
|
||||||
|
self._pending_requests_lock = threading.RLock()
|
||||||
|
self._load_lock = threading.RLock()
|
||||||
|
self._health_lock = threading.RLock()
|
||||||
|
self._processed_events_lock = threading.RLock()
|
||||||
|
self._processed_messages_lock = threading.RLock()
|
||||||
|
self._processing_events_lock = threading.RLock()
|
||||||
|
self._message_locks_lock = threading.RLock()
|
||||||
|
self._message_lock_times_lock = threading.RLock()
|
||||||
|
|
||||||
async def start(self, host: str = "0.0.0.0", port: int = 3002) -> None:
|
async def start(self, host: str = "0.0.0.0", port: int = 3002) -> None:
|
||||||
"""
|
"""
|
||||||
启动反向 WebSocket 服务端。
|
启动反向 WebSocket 服务端。
|
||||||
@@ -184,6 +193,7 @@ class ReverseWSManager:
|
|||||||
current_time = datetime.now()
|
current_time = datetime.now()
|
||||||
|
|
||||||
# 清理过期的事件ID(按客户端)
|
# 清理过期的事件ID(按客户端)
|
||||||
|
with self._processed_events_lock:
|
||||||
for client_id, events in list(self._processed_events.items()):
|
for client_id, events in list(self._processed_events.items()):
|
||||||
expired_events = [
|
expired_events = [
|
||||||
event_id for event_id, timestamp in events.items()
|
event_id for event_id, timestamp in events.items()
|
||||||
@@ -195,17 +205,20 @@ class ReverseWSManager:
|
|||||||
del self._processed_events[client_id]
|
del self._processed_events[client_id]
|
||||||
|
|
||||||
# 清理过期的消息锁
|
# 清理过期的消息锁
|
||||||
|
with self._message_lock_times_lock:
|
||||||
expired_locks = [
|
expired_locks = [
|
||||||
lock_key for lock_key, timestamp in self._message_lock_times.items()
|
lock_key for lock_key, timestamp in self._message_lock_times.items()
|
||||||
if (current_time - timestamp).total_seconds() > self._lock_ttl
|
if (current_time - timestamp).total_seconds() > self._lock_ttl
|
||||||
]
|
]
|
||||||
for lock_key in expired_locks:
|
for lock_key in expired_locks:
|
||||||
|
with self._message_locks_lock:
|
||||||
if lock_key in self._message_locks:
|
if lock_key in self._message_locks:
|
||||||
del self._message_locks[lock_key]
|
del self._message_locks[lock_key]
|
||||||
if lock_key in self._message_lock_times:
|
if lock_key in self._message_lock_times:
|
||||||
del self._message_lock_times[lock_key]
|
del self._message_lock_times[lock_key]
|
||||||
|
|
||||||
# 清理过期的消息内容(按客户端)
|
# 清理过期的消息内容(按客户端)
|
||||||
|
with self._processed_messages_lock:
|
||||||
for client_id, messages in list(self._processed_messages.items()):
|
for client_id, messages in list(self._processed_messages.items()):
|
||||||
expired_messages = [
|
expired_messages = [
|
||||||
msg_key for msg_key, timestamp in messages.items()
|
msg_key for msg_key, timestamp in messages.items()
|
||||||
@@ -228,22 +241,30 @@ class ReverseWSManager:
|
|||||||
Args:
|
Args:
|
||||||
client_id: 客户端 ID
|
client_id: 客户端 ID
|
||||||
"""
|
"""
|
||||||
|
with self._clients_lock:
|
||||||
if client_id in self.clients:
|
if client_id in self.clients:
|
||||||
del self.clients[client_id]
|
del self.clients[client_id]
|
||||||
|
with self._clients_lock:
|
||||||
if client_id in self.client_self_ids:
|
if client_id in self.client_self_ids:
|
||||||
del self.client_self_ids[client_id]
|
del self.client_self_ids[client_id]
|
||||||
|
with self._load_lock:
|
||||||
if client_id in self._client_load:
|
if client_id in self._client_load:
|
||||||
del self._client_load[client_id]
|
del self._client_load[client_id]
|
||||||
|
with self._health_lock:
|
||||||
if client_id in self._client_health:
|
if client_id in self._client_health:
|
||||||
del self._client_health[client_id]
|
del self._client_health[client_id]
|
||||||
|
with self._bots_lock:
|
||||||
if client_id in self.bots:
|
if client_id in self.bots:
|
||||||
del self.bots[client_id]
|
del self.bots[client_id]
|
||||||
|
|
||||||
# 清理该客户端的防重复数据
|
# 清理该客户端的防重复数据
|
||||||
|
with self._processed_events_lock:
|
||||||
if client_id in self._processed_events:
|
if client_id in self._processed_events:
|
||||||
del self._processed_events[client_id]
|
del self._processed_events[client_id]
|
||||||
|
with self._processed_messages_lock:
|
||||||
if client_id in self._processed_messages:
|
if client_id in self._processed_messages:
|
||||||
del self._processed_messages[client_id]
|
del self._processed_messages[client_id]
|
||||||
|
with self._processing_events_lock:
|
||||||
if client_id in self._processing_events:
|
if client_id in self._processing_events:
|
||||||
del self._processing_events[client_id]
|
del self._processing_events[client_id]
|
||||||
|
|
||||||
@@ -266,11 +287,13 @@ class ReverseWSManager:
|
|||||||
event_key = f"{event_data.get('post_type')}:{event_id}"
|
event_key = f"{event_data.get('post_type')}:{event_id}"
|
||||||
|
|
||||||
# 检查客户端是否已连接
|
# 检查客户端是否已连接
|
||||||
|
with self._clients_lock:
|
||||||
if client_id not in self.clients:
|
if client_id not in self.clients:
|
||||||
self.logger.debug(f"_on_event: 客户端已断开, client_id={client_id}")
|
self.logger.debug(f"_on_event: 客户端已断开, client_id={client_id}")
|
||||||
return
|
return
|
||||||
|
|
||||||
# 检查是否正在处理
|
# 检查是否正在处理
|
||||||
|
with self._processing_events_lock:
|
||||||
if client_id not in self._processing_events:
|
if client_id not in self._processing_events:
|
||||||
self._processing_events[client_id] = set()
|
self._processing_events[client_id] = set()
|
||||||
|
|
||||||
@@ -285,12 +308,14 @@ class ReverseWSManager:
|
|||||||
event = EventFactory.create_event(event_data)
|
event = EventFactory.create_event(event_data)
|
||||||
|
|
||||||
if hasattr(event, 'self_id'):
|
if hasattr(event, 'self_id'):
|
||||||
|
with self._clients_lock:
|
||||||
self.client_self_ids[client_id] = event.self_id
|
self.client_self_ids[client_id] = event.self_id
|
||||||
|
|
||||||
# 为事件注入Bot实例
|
# 为事件注入Bot实例
|
||||||
from ..ws import ReverseWSClient
|
from ..ws import ReverseWSClient
|
||||||
|
|
||||||
# 为每个前端创建独立的Bot实例
|
# 为每个前端创建独立的Bot实例
|
||||||
|
with self._bots_lock:
|
||||||
if client_id not in self.bots:
|
if client_id not in self.bots:
|
||||||
# 使用 ReverseWSClient 代理
|
# 使用 ReverseWSClient 代理
|
||||||
temp_ws = ReverseWSClient(self, client_id)
|
temp_ws = ReverseWSClient(self, client_id)
|
||||||
@@ -300,6 +325,7 @@ class ReverseWSManager:
|
|||||||
event.bot = self.bots[client_id]
|
event.bot = self.bots[client_id]
|
||||||
|
|
||||||
# 记录客户端健康状态
|
# 记录客户端健康状态
|
||||||
|
with self._health_lock:
|
||||||
self._client_health[client_id] = datetime.now()
|
self._client_health[client_id] = datetime.now()
|
||||||
|
|
||||||
# 检查是否为重复事件(按客户端)
|
# 检查是否为重复事件(按客户端)
|
||||||
@@ -333,14 +359,17 @@ class ReverseWSManager:
|
|||||||
return
|
return
|
||||||
|
|
||||||
# 标记事件已处理(按客户端)
|
# 标记事件已处理(按客户端)
|
||||||
|
with self._processed_events_lock:
|
||||||
self._mark_event_processed(event_data, client_id)
|
self._mark_event_processed(event_data, client_id)
|
||||||
|
|
||||||
# 更新客户端负载
|
# 更新客户端负载
|
||||||
|
with self._load_lock:
|
||||||
self._update_client_load(client_id)
|
self._update_client_load(client_id)
|
||||||
|
|
||||||
await matcher.handle_event(event.bot, event)
|
await matcher.handle_event(event.bot, event)
|
||||||
else:
|
else:
|
||||||
# 对于非消息事件,直接标记并处理
|
# 对于非消息事件,直接标记并处理
|
||||||
|
with self._processed_events_lock:
|
||||||
self._mark_event_processed(event_data, client_id)
|
self._mark_event_processed(event_data, client_id)
|
||||||
|
|
||||||
if event.post_type == "notice":
|
if event.post_type == "notice":
|
||||||
@@ -362,6 +391,7 @@ class ReverseWSManager:
|
|||||||
self.logger.exception(f"事件处理异常: {str(e)}")
|
self.logger.exception(f"事件处理异常: {str(e)}")
|
||||||
finally:
|
finally:
|
||||||
# 清理正在处理的事件
|
# 清理正在处理的事件
|
||||||
|
with self._processing_events_lock:
|
||||||
if client_id in self._processing_events:
|
if client_id in self._processing_events:
|
||||||
if event_key in self._processing_events[client_id]:
|
if event_key in self._processing_events[client_id]:
|
||||||
self._processing_events[client_id].discard(event_key)
|
self._processing_events[client_id].discard(event_key)
|
||||||
@@ -404,9 +434,11 @@ class ReverseWSManager:
|
|||||||
# 选择负载最低的客户端
|
# 选择负载最低的客户端
|
||||||
client_id = self.get_client_with_least_load()
|
client_id = self.get_client_with_least_load()
|
||||||
if client_id is None and healthy_clients:
|
if client_id is None and healthy_clients:
|
||||||
|
with self._clients_lock:
|
||||||
client_id = list(healthy_clients.keys())[0]
|
client_id = list(healthy_clients.keys())[0]
|
||||||
else:
|
else:
|
||||||
# 如果没有健康客户端,使用所有客户端中的一个
|
# 如果没有健康客户端,使用所有客户端中的一个
|
||||||
|
with self._clients_lock:
|
||||||
client_id = list(self.clients.keys())[0]
|
client_id = list(self.clients.keys())[0]
|
||||||
|
|
||||||
echo_id = str(uuid.uuid4())
|
echo_id = str(uuid.uuid4())
|
||||||
@@ -414,17 +446,26 @@ class ReverseWSManager:
|
|||||||
|
|
||||||
loop = asyncio.get_running_loop()
|
loop = asyncio.get_running_loop()
|
||||||
future = loop.create_future()
|
future = loop.create_future()
|
||||||
|
with self._pending_requests_lock:
|
||||||
self._pending_requests[echo_id] = future
|
self._pending_requests[echo_id] = future
|
||||||
|
|
||||||
try:
|
try:
|
||||||
targets = [client_id] if client_id else list(self.clients.keys())
|
targets = [client_id] if client_id else None
|
||||||
|
clients_to_send = []
|
||||||
|
|
||||||
|
with self._clients_lock:
|
||||||
|
if targets is None:
|
||||||
|
targets = list(self.clients.keys())
|
||||||
for cid in targets:
|
for cid in targets:
|
||||||
if cid in self.clients:
|
if cid in self.clients:
|
||||||
await self.clients[cid].send(orjson.dumps(payload))
|
clients_to_send.append((cid, self.clients[cid]))
|
||||||
|
|
||||||
|
for cid, websocket in clients_to_send:
|
||||||
|
await websocket.send(orjson.dumps(payload))
|
||||||
|
|
||||||
return await asyncio.wait_for(future, timeout=30.0)
|
return await asyncio.wait_for(future, timeout=30.0)
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
|
with self._pending_requests_lock:
|
||||||
self._pending_requests.pop(echo_id, None)
|
self._pending_requests.pop(echo_id, None)
|
||||||
self.logger.warning(f"API 调用超时: action={action}, params={params}")
|
self.logger.warning(f"API 调用超时: action={action}, params={params}")
|
||||||
return create_error_response(
|
return create_error_response(
|
||||||
@@ -433,6 +474,7 @@ class ReverseWSManager:
|
|||||||
data={"action": action, "params": params}
|
data={"action": action, "params": params}
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
with self._pending_requests_lock:
|
||||||
self._pending_requests.pop(echo_id, None)
|
self._pending_requests.pop(echo_id, None)
|
||||||
self.logger.exception(f"API 调用异常: action={action}, error={str(e)}")
|
self.logger.exception(f"API 调用异常: action={action}, error={str(e)}")
|
||||||
return create_error_response(
|
return create_error_response(
|
||||||
@@ -448,6 +490,7 @@ class ReverseWSManager:
|
|||||||
Returns:
|
Returns:
|
||||||
客户端 ID 和 self_id 的映射字典
|
客户端 ID 和 self_id 的映射字典
|
||||||
"""
|
"""
|
||||||
|
with self._clients_lock:
|
||||||
return self.client_self_ids.copy()
|
return self.client_self_ids.copy()
|
||||||
|
|
||||||
def _is_duplicate_event(self, event_data: Dict[str, Any], client_id: str) -> bool:
|
def _is_duplicate_event(self, event_data: Dict[str, Any], client_id: str) -> bool:
|
||||||
@@ -472,6 +515,7 @@ class ReverseWSManager:
|
|||||||
event_key = f"{event_data.get('post_type')}:{event_id}"
|
event_key = f"{event_data.get('post_type')}:{event_id}"
|
||||||
|
|
||||||
# 检查该客户端是否已处理过此事件
|
# 检查该客户端是否已处理过此事件
|
||||||
|
with self._processed_events_lock:
|
||||||
if client_id not in self._processed_events:
|
if client_id not in self._processed_events:
|
||||||
self.logger.debug(f"_is_duplicate_event: client_id={client_id}不在_processed_events中, event_key={event_key}, 返回False")
|
self.logger.debug(f"_is_duplicate_event: client_id={client_id}不在_processed_events中, event_key={event_key}, 返回False")
|
||||||
return False
|
return False
|
||||||
@@ -507,6 +551,7 @@ class ReverseWSManager:
|
|||||||
content_key = f"content:{raw_message}:{user_id}:{group_id}"
|
content_key = f"content:{raw_message}:{user_id}:{group_id}"
|
||||||
|
|
||||||
# 检查该客户端是否已处理过此消息内容
|
# 检查该客户端是否已处理过此消息内容
|
||||||
|
with self._processed_messages_lock:
|
||||||
if client_id not in self._processed_messages:
|
if client_id not in self._processed_messages:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -532,6 +577,7 @@ class ReverseWSManager:
|
|||||||
event_key = f"{event_data.get('post_type')}:{event_id}"
|
event_key = f"{event_data.get('post_type')}:{event_id}"
|
||||||
|
|
||||||
# 为该客户端记录已处理的事件
|
# 为该客户端记录已处理的事件
|
||||||
|
with self._processed_events_lock:
|
||||||
if client_id not in self._processed_events:
|
if client_id not in self._processed_events:
|
||||||
self._processed_events[client_id] = {}
|
self._processed_events[client_id] = {}
|
||||||
self._processed_events[client_id][event_key] = datetime.now()
|
self._processed_events[client_id][event_key] = datetime.now()
|
||||||
@@ -544,6 +590,7 @@ class ReverseWSManager:
|
|||||||
group_id = event_data.get('group_id', '0')
|
group_id = event_data.get('group_id', '0')
|
||||||
content_key = f"content:{raw_message}:{user_id}:{group_id}"
|
content_key = f"content:{raw_message}:{user_id}:{group_id}"
|
||||||
|
|
||||||
|
with self._processed_messages_lock:
|
||||||
if client_id not in self._processed_messages:
|
if client_id not in self._processed_messages:
|
||||||
self._processed_messages[client_id] = {}
|
self._processed_messages[client_id] = {}
|
||||||
self._processed_messages[client_id][content_key] = datetime.now()
|
self._processed_messages[client_id][content_key] = datetime.now()
|
||||||
@@ -574,8 +621,10 @@ class ReverseWSManager:
|
|||||||
Returns:
|
Returns:
|
||||||
asyncio.Lock 实例
|
asyncio.Lock 实例
|
||||||
"""
|
"""
|
||||||
|
with self._message_locks_lock:
|
||||||
if key not in self._message_locks:
|
if key not in self._message_locks:
|
||||||
self._message_locks[key] = asyncio.Lock()
|
self._message_locks[key] = asyncio.Lock()
|
||||||
|
with self._message_lock_times_lock:
|
||||||
self._message_lock_times[key] = datetime.now()
|
self._message_lock_times[key] = datetime.now()
|
||||||
return self._message_locks[key]
|
return self._message_locks[key]
|
||||||
|
|
||||||
@@ -586,6 +635,7 @@ class ReverseWSManager:
|
|||||||
Args:
|
Args:
|
||||||
client_id: 客户端 ID
|
client_id: 客户端 ID
|
||||||
"""
|
"""
|
||||||
|
with self._load_lock:
|
||||||
if client_id not in self._client_load:
|
if client_id not in self._client_load:
|
||||||
self._client_load[client_id] = 0
|
self._client_load[client_id] = 0
|
||||||
self._client_load[client_id] += 1
|
self._client_load[client_id] += 1
|
||||||
@@ -597,6 +647,7 @@ class ReverseWSManager:
|
|||||||
Returns:
|
Returns:
|
||||||
客户端 ID,如果没有客户端则返回 None
|
客户端 ID,如果没有客户端则返回 None
|
||||||
"""
|
"""
|
||||||
|
with self._load_lock:
|
||||||
if not self._client_load:
|
if not self._client_load:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -612,6 +663,8 @@ class ReverseWSManager:
|
|||||||
current_time = datetime.now()
|
current_time = datetime.now()
|
||||||
healthy = {}
|
healthy = {}
|
||||||
|
|
||||||
|
with self._health_lock:
|
||||||
|
with self._clients_lock:
|
||||||
for client_id, last_health in self._client_health.items():
|
for client_id, last_health in self._client_health.items():
|
||||||
if (current_time - last_health).total_seconds() < 30:
|
if (current_time - last_health).total_seconds() < 30:
|
||||||
if client_id in self.client_self_ids:
|
if client_id in self.client_self_ids:
|
||||||
|
|||||||
379
core/managers/thread_manager.py
Normal file
379
core/managers/thread_manager.py
Normal file
@@ -0,0 +1,379 @@
|
|||||||
|
"""
|
||||||
|
线程管理器模块
|
||||||
|
|
||||||
|
该模块提供了多线程支持,用于处理来自多个实现端的并发事件。
|
||||||
|
每个 WebSocket 连接在独立的线程中运行,避免阻塞主事件循环。
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
import threading
|
||||||
|
from typing import Dict, Optional, Callable, Any
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from datetime import datetime
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from ..utils.logger import ModuleLogger
|
||||||
|
from ..config_loader import global_config
|
||||||
|
|
||||||
|
|
||||||
|
class ThreadManager:
|
||||||
|
"""
|
||||||
|
线程管理器,负责管理多线程环境下的事件处理。
|
||||||
|
|
||||||
|
该管理器为每个 WebSocket 连接提供独立的线程池,
|
||||||
|
确保多前端场景下的事件处理不会相互阻塞。
|
||||||
|
"""
|
||||||
|
|
||||||
|
_instance: Optional['ThreadManager'] = None
|
||||||
|
_lock: threading.Lock = threading.Lock()
|
||||||
|
|
||||||
|
def __new__(cls) -> 'ThreadManager':
|
||||||
|
"""
|
||||||
|
单例模式:确保全局只有一个线程管理器实例。
|
||||||
|
"""
|
||||||
|
if cls._instance is None:
|
||||||
|
with cls._lock:
|
||||||
|
if cls._instance is None:
|
||||||
|
cls._instance = super().__new__(cls)
|
||||||
|
cls._instance._initialized = False
|
||||||
|
return cls._instance
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
"""
|
||||||
|
初始化线程管理器。
|
||||||
|
"""
|
||||||
|
if self._initialized:
|
||||||
|
return
|
||||||
|
|
||||||
|
self.logger = ModuleLogger("ThreadManager")
|
||||||
|
|
||||||
|
# 线程池配置
|
||||||
|
self._max_workers: int = global_config.threading.max_workers
|
||||||
|
self._thread_name_prefix: str = global_config.threading.thread_name_prefix
|
||||||
|
|
||||||
|
# 线程池
|
||||||
|
self._executor: Optional[ThreadPoolExecutor] = None
|
||||||
|
|
||||||
|
# 每个客户端的线程池(用于反向 WebSocket)
|
||||||
|
self._client_executors: Dict[str, ThreadPoolExecutor] = {}
|
||||||
|
self._client_executor_locks: Dict[str, threading.Lock] = {}
|
||||||
|
|
||||||
|
# 线程安全的事件循环(用于跨线程调用)
|
||||||
|
self._event_loops: Dict[str, asyncio.AbstractEventLoop] = {}
|
||||||
|
self._event_loops_lock = threading.Lock()
|
||||||
|
|
||||||
|
# 统计信息
|
||||||
|
self._stats: Dict[str, Any] = {
|
||||||
|
'total_tasks': 0,
|
||||||
|
'completed_tasks': 0,
|
||||||
|
'failed_tasks': 0,
|
||||||
|
'active_threads': 0,
|
||||||
|
'client_tasks': {}
|
||||||
|
}
|
||||||
|
self._stats_lock = threading.Lock()
|
||||||
|
|
||||||
|
self._initialized = True
|
||||||
|
self.logger.success("线程管理器初始化完成")
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
"""
|
||||||
|
启动线程管理器,创建主线程池。
|
||||||
|
"""
|
||||||
|
if self._executor is None:
|
||||||
|
self._executor = ThreadPoolExecutor(
|
||||||
|
max_workers=self._max_workers,
|
||||||
|
thread_name_prefix=self._thread_name_prefix
|
||||||
|
)
|
||||||
|
self.logger.success(f"主 ThreadPool 已启动: max_workers={self._max_workers}")
|
||||||
|
|
||||||
|
def shutdown(self) -> None:
|
||||||
|
"""
|
||||||
|
关闭线程管理器,释放所有资源。
|
||||||
|
"""
|
||||||
|
self.logger.info("正在关闭线程管理器...")
|
||||||
|
|
||||||
|
# 关闭所有客户端线程池
|
||||||
|
for client_id, executor in list(self._client_executors.items()):
|
||||||
|
self._shutdown_client_executor(client_id)
|
||||||
|
|
||||||
|
# 关闭主执行器
|
||||||
|
if self._executor is not None:
|
||||||
|
self._executor.shutdown(wait=True)
|
||||||
|
self._executor = None
|
||||||
|
|
||||||
|
self.logger.success("线程管理器已关闭")
|
||||||
|
|
||||||
|
def _shutdown_client_executor(self, client_id: str) -> None:
|
||||||
|
"""
|
||||||
|
关闭特定客户端的线程池。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client_id: 客户端 ID
|
||||||
|
"""
|
||||||
|
if client_id in self._client_executors:
|
||||||
|
try:
|
||||||
|
self._client_executors[client_id].shutdown(wait=True)
|
||||||
|
del self._client_executors[client_id]
|
||||||
|
self.logger.info(f"客户端 {client_id} 的线程池已关闭")
|
||||||
|
except Exception as e:
|
||||||
|
self.logger.error(f"关闭客户端 {client_id} 线程池失败: {e}")
|
||||||
|
|
||||||
|
def get_main_executor(self) -> ThreadPoolExecutor:
|
||||||
|
"""
|
||||||
|
获取主线程池。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
ThreadPoolExecutor 实例
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
RuntimeError: 如果线程管理器未启动
|
||||||
|
"""
|
||||||
|
if self._executor is None:
|
||||||
|
raise RuntimeError("线程管理器未启动,请先调用 start()")
|
||||||
|
return self._executor
|
||||||
|
|
||||||
|
def get_client_executor(self, client_id: str) -> ThreadPoolExecutor:
|
||||||
|
"""
|
||||||
|
获取特定客户端的线程池(为反向 WebSocket 设计)。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client_id: 客户端 ID
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
ThreadPoolExecutor 实例
|
||||||
|
"""
|
||||||
|
if client_id not in self._client_executors:
|
||||||
|
with threading.Lock():
|
||||||
|
if client_id not in self._client_executors:
|
||||||
|
executor = ThreadPoolExecutor(
|
||||||
|
max_workers=global_config.threading.client_max_workers,
|
||||||
|
thread_name_prefix=f"{self._thread_name_prefix}_{client_id[:8]}"
|
||||||
|
)
|
||||||
|
self._client_executors[client_id] = executor
|
||||||
|
self._client_executor_locks[client_id] = threading.Lock()
|
||||||
|
self.logger.info(f"为客户端 {client_id} 创建线程池")
|
||||||
|
|
||||||
|
return self._client_executors[client_id]
|
||||||
|
|
||||||
|
def submit_to_main_executor(
|
||||||
|
self,
|
||||||
|
func: Callable,
|
||||||
|
*args: Any,
|
||||||
|
**kwargs: Any
|
||||||
|
) -> Any:
|
||||||
|
"""
|
||||||
|
提交任务到主线程池(同步)。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
func: 要执行的函数
|
||||||
|
*args: 位置参数
|
||||||
|
**kwargs: 关键字参数
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
函数执行结果
|
||||||
|
"""
|
||||||
|
executor = self.get_main_executor()
|
||||||
|
future = executor.submit(func, *args, **kwargs)
|
||||||
|
self._update_stats('total_tasks')
|
||||||
|
try:
|
||||||
|
result = future.result()
|
||||||
|
self._update_stats('completed_tasks')
|
||||||
|
return result
|
||||||
|
except Exception as e:
|
||||||
|
self._update_stats('failed_tasks')
|
||||||
|
self.logger.error(f"主线程池任务执行失败: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
|
async def submit_to_main_executor_async(
|
||||||
|
self,
|
||||||
|
func: Callable,
|
||||||
|
*args: Any,
|
||||||
|
**kwargs: Any
|
||||||
|
) -> Any:
|
||||||
|
"""
|
||||||
|
提交任务到主线程池(异步)。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
func: 要执行的函数
|
||||||
|
*args: 位置参数
|
||||||
|
**kwargs: 关键字参数
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
函数执行结果
|
||||||
|
"""
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
executor = self.get_main_executor()
|
||||||
|
future = loop.run_in_executor(executor, lambda: func(*args, **kwargs))
|
||||||
|
self._update_stats('total_tasks')
|
||||||
|
try:
|
||||||
|
result = await future
|
||||||
|
self._update_stats('completed_tasks')
|
||||||
|
return result
|
||||||
|
except Exception as e:
|
||||||
|
self._update_stats('failed_tasks')
|
||||||
|
self.logger.error(f"异步主线程池任务执行失败: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
|
def submit_to_client_executor(
|
||||||
|
self,
|
||||||
|
client_id: str,
|
||||||
|
func: Callable,
|
||||||
|
*args: Any,
|
||||||
|
**kwargs: Any
|
||||||
|
) -> Any:
|
||||||
|
"""
|
||||||
|
提交任务到特定客户端的线程池。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client_id: 客户端 ID
|
||||||
|
func: 要执行的函数
|
||||||
|
*args: 位置参数
|
||||||
|
**kwargs: 关键字参数
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
函数执行结果
|
||||||
|
"""
|
||||||
|
executor = self.get_client_executor(client_id)
|
||||||
|
future = executor.submit(func, *args, **kwargs)
|
||||||
|
self._update_client_stats(client_id, 'total_tasks')
|
||||||
|
try:
|
||||||
|
result = future.result()
|
||||||
|
self._update_client_stats(client_id, 'completed_tasks')
|
||||||
|
return result
|
||||||
|
except Exception as e:
|
||||||
|
self._update_client_stats(client_id, 'failed_tasks')
|
||||||
|
self.logger.error(f"客户端 {client_id} 线程池任务执行失败: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
|
async def submit_to_client_executor_async(
|
||||||
|
self,
|
||||||
|
client_id: str,
|
||||||
|
func: Callable,
|
||||||
|
*args: Any,
|
||||||
|
**kwargs: Any
|
||||||
|
) -> Any:
|
||||||
|
"""
|
||||||
|
提交任务到特定客户端的线程池(异步)。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client_id: 客户端 ID
|
||||||
|
func: 要执行的函数
|
||||||
|
*args: 位置参数
|
||||||
|
**kwargs: 关键字参数
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
函数执行结果
|
||||||
|
"""
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
executor = self.get_client_executor(client_id)
|
||||||
|
future = loop.run_in_executor(executor, lambda: func(*args, **kwargs))
|
||||||
|
self._update_client_stats(client_id, 'total_tasks')
|
||||||
|
try:
|
||||||
|
result = await future
|
||||||
|
self._update_client_stats(client_id, 'completed_tasks')
|
||||||
|
return result
|
||||||
|
except Exception as e:
|
||||||
|
self._update_client_stats(client_id, 'failed_tasks')
|
||||||
|
self.logger.error(f"客户端 {client_id} 异步线程池任务执行失败: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
|
def run_coroutine_threadsafe(
|
||||||
|
self,
|
||||||
|
coro,
|
||||||
|
client_id: Optional[str] = None
|
||||||
|
) -> Any:
|
||||||
|
"""
|
||||||
|
在指定客户端的事件循环中运行协程(线程安全)。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
coro: 协程对象
|
||||||
|
client_id: 客户端 ID,如果为 None 则使用主事件循环
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
协程执行结果
|
||||||
|
"""
|
||||||
|
if client_id is None:
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
else:
|
||||||
|
with self._event_loops_lock:
|
||||||
|
if client_id not in self._event_loops:
|
||||||
|
self._event_loops[client_id] = asyncio.new_event_loop()
|
||||||
|
threading.Thread(
|
||||||
|
target=self._event_loop_thread,
|
||||||
|
args=(client_id,),
|
||||||
|
daemon=True
|
||||||
|
).start()
|
||||||
|
loop = self._event_loops[client_id]
|
||||||
|
|
||||||
|
future = asyncio.run_coroutine_threadsafe(coro, loop)
|
||||||
|
return future.result()
|
||||||
|
|
||||||
|
def _event_loop_thread(self, client_id: str) -> None:
|
||||||
|
"""
|
||||||
|
事件循环线程(用于反向 WebSocket 客户端)。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client_id: 客户端 ID
|
||||||
|
"""
|
||||||
|
asyncio.set_event_loop(self._event_loops[client_id])
|
||||||
|
self.logger.info(f"事件循环线程启动: client_id={client_id}")
|
||||||
|
try:
|
||||||
|
self._event_loops[client_id].run_forever()
|
||||||
|
finally:
|
||||||
|
self._event_loops[client_id].close()
|
||||||
|
self.logger.info(f"事件循环线程停止: client_id={client_id}")
|
||||||
|
|
||||||
|
def _update_stats(self, key: str) -> None:
|
||||||
|
"""
|
||||||
|
更新全局统计信息。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
key: 统计项键名
|
||||||
|
"""
|
||||||
|
with self._stats_lock:
|
||||||
|
self._stats[key] = self._stats.get(key, 0) + 1
|
||||||
|
|
||||||
|
def _update_client_stats(self, client_id: str, key: str) -> None:
|
||||||
|
"""
|
||||||
|
更新客户端统计信息。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client_id: 客户端 ID
|
||||||
|
key: 统计项键名
|
||||||
|
"""
|
||||||
|
with self._stats_lock:
|
||||||
|
if client_id not in self._stats['client_tasks']:
|
||||||
|
self._stats['client_tasks'][client_id] = {
|
||||||
|
'total_tasks': 0,
|
||||||
|
'completed_tasks': 0,
|
||||||
|
'failed_tasks': 0
|
||||||
|
}
|
||||||
|
self._stats['client_tasks'][client_id][key] += 1
|
||||||
|
|
||||||
|
def get_stats(self) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
获取统计信息。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
统计信息字典
|
||||||
|
"""
|
||||||
|
with self._stats_lock:
|
||||||
|
stats = self._stats.copy()
|
||||||
|
stats['client_tasks'] = stats.get('client_tasks', {}).copy()
|
||||||
|
return stats
|
||||||
|
|
||||||
|
def get_active_threads_count(self) -> int:
|
||||||
|
"""
|
||||||
|
获取活动线程数量。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
活动线程数量
|
||||||
|
"""
|
||||||
|
import threading
|
||||||
|
return sum(
|
||||||
|
1 for t in threading.enumerate()
|
||||||
|
if t.name.startswith(self._thread_name_prefix)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# 全局线程管理器实例
|
||||||
|
thread_manager = ThreadManager()
|
||||||
217
core/services/local_file_server.py
Normal file
217
core/services/local_file_server.py
Normal file
@@ -0,0 +1,217 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
本地文件下载服务
|
||||||
|
|
||||||
|
该模块提供一个本地 HTTP 服务,用于下载远程文件到本地并提供本地访问。
|
||||||
|
主要解决 NapCat 等第三方服务无法直接访问某些远程资源(如 B 站防盗链)的问题。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
import hashlib
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional, Dict
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
import aiohttp
|
||||||
|
from aiohttp import web
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
from core.utils.logger import logger
|
||||||
|
from core.config_loader import global_config
|
||||||
|
|
||||||
|
|
||||||
|
class LocalFileServer:
|
||||||
|
"""
|
||||||
|
本地文件下载服务
|
||||||
|
|
||||||
|
提供一个本地 HTTP 服务,用于下载远程文件到本地并提供本地访问。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, host: str = "0.0.0.0", port: int = 3003):
|
||||||
|
"""
|
||||||
|
初始化本地文件下载服务
|
||||||
|
|
||||||
|
Args:
|
||||||
|
host (str): 服务监听地址
|
||||||
|
port (int): 服务监听端口
|
||||||
|
"""
|
||||||
|
self.host = host
|
||||||
|
self.port = port
|
||||||
|
self.app = web.Application()
|
||||||
|
self.runner = None
|
||||||
|
self.site = None
|
||||||
|
self.download_dir = Path(tempfile.gettempdir()) / "neobot_downloads"
|
||||||
|
self.download_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# 注册路由
|
||||||
|
self.app.router.add_get('/download', self.handle_download)
|
||||||
|
self.app.router.add_get('/health', self.handle_health)
|
||||||
|
|
||||||
|
# 文件映射表:file_id -> file_path
|
||||||
|
self.file_map: Dict[str, Path] = {}
|
||||||
|
|
||||||
|
logger.success(f"[LocalFileServer] 初始化完成: {self.host}:{self.port}")
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
"""启动服务"""
|
||||||
|
self.runner = web.AppRunner(self.app)
|
||||||
|
await self.runner.setup()
|
||||||
|
self.site = web.TCPSite(self.runner, self.host, self.port)
|
||||||
|
await self.site.start()
|
||||||
|
logger.success(f"[LocalFileServer] 服务已启动: http://{self.host}:{self.port}")
|
||||||
|
|
||||||
|
async def stop(self):
|
||||||
|
"""停止服务"""
|
||||||
|
if self.runner:
|
||||||
|
await self.runner.cleanup()
|
||||||
|
logger.info("[LocalFileServer] 服务已停止")
|
||||||
|
|
||||||
|
def _generate_file_id(self, url: str) -> str:
|
||||||
|
"""根据 URL 生成唯一的文件 ID"""
|
||||||
|
url_hash = hashlib.md5(url.encode()).hexdigest()[:16]
|
||||||
|
return f"file_{url_hash}"
|
||||||
|
|
||||||
|
async def download_file(self, url: str, timeout: int = 60) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
下载远程文件到本地
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url (str): 远程文件 URL
|
||||||
|
timeout (int): 下载超时时间(秒)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Optional[str]: 本地文件 ID,如果失败则返回 None
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
file_id = self._generate_file_id(url)
|
||||||
|
file_path = self.download_dir / f"{file_id}"
|
||||||
|
|
||||||
|
# 检查文件是否已存在
|
||||||
|
if file_path.exists():
|
||||||
|
logger.info(f"[LocalFileServer] 文件已存在: {file_id}")
|
||||||
|
return file_id
|
||||||
|
|
||||||
|
logger.info(f"[LocalFileServer] 开始下载: {url}")
|
||||||
|
|
||||||
|
# 使用 aiohttp 下载文件
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
async with session.get(url, timeout=timeout) as response:
|
||||||
|
if response.status != 200:
|
||||||
|
logger.error(f"[LocalFileServer] 下载失败: HTTP {response.status}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# 读取并保存文件
|
||||||
|
with open(file_path, 'wb') as f:
|
||||||
|
while True:
|
||||||
|
chunk = await response.content.read(8192)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
f.write(chunk)
|
||||||
|
|
||||||
|
self.file_map[file_id] = file_path
|
||||||
|
logger.success(f"[LocalFileServer] 下载完成: {file_id} ({file_path.stat().st_size} bytes)")
|
||||||
|
return file_id
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"[LocalFileServer] 下载失败: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def handle_download(self, request: web.Request) -> web.Response:
|
||||||
|
"""处理文件下载请求"""
|
||||||
|
file_id = request.query.get('id')
|
||||||
|
|
||||||
|
if not file_id or file_id not in self.file_map:
|
||||||
|
return web.Response(
|
||||||
|
status=404,
|
||||||
|
text='File not found',
|
||||||
|
content_type='text/plain'
|
||||||
|
)
|
||||||
|
|
||||||
|
file_path = self.file_map[file_id]
|
||||||
|
|
||||||
|
if not file_path.exists():
|
||||||
|
return web.Response(
|
||||||
|
status=404,
|
||||||
|
text='File not found',
|
||||||
|
content_type='text/plain'
|
||||||
|
)
|
||||||
|
|
||||||
|
# 获取文件大小
|
||||||
|
file_size = file_path.stat().st_size
|
||||||
|
|
||||||
|
# 设置响应头
|
||||||
|
headers = {
|
||||||
|
'Content-Disposition': f'attachment; filename="{file_id}"',
|
||||||
|
'Content-Length': str(file_size)
|
||||||
|
}
|
||||||
|
|
||||||
|
return web.FileResponse(file_path, headers=headers)
|
||||||
|
|
||||||
|
async def handle_health(self, request: web.Request) -> web.Response:
|
||||||
|
"""健康检查"""
|
||||||
|
return web.json_response({
|
||||||
|
'status': 'ok',
|
||||||
|
'service': 'LocalFileServer',
|
||||||
|
'download_dir': str(self.download_dir),
|
||||||
|
'files_count': len(self.file_map)
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
# 全局实例
|
||||||
|
_local_file_server: Optional[LocalFileServer] = None
|
||||||
|
|
||||||
|
|
||||||
|
def get_local_file_server() -> Optional[LocalFileServer]:
|
||||||
|
"""获取全局本地文件服务器实例"""
|
||||||
|
global _local_file_server
|
||||||
|
|
||||||
|
if _local_file_server is None:
|
||||||
|
try:
|
||||||
|
server_config = global_config.local_file_server
|
||||||
|
_local_file_server = LocalFileServer(
|
||||||
|
host=server_config.host,
|
||||||
|
port=server_config.port
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"[LocalFileServer] 初始化失败: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
return _local_file_server
|
||||||
|
|
||||||
|
|
||||||
|
async def start_local_file_server():
|
||||||
|
"""启动全局本地文件服务器"""
|
||||||
|
server = get_local_file_server()
|
||||||
|
if server:
|
||||||
|
await server.start()
|
||||||
|
|
||||||
|
|
||||||
|
async def stop_local_file_server():
|
||||||
|
"""停止全局本地文件服务器"""
|
||||||
|
global _local_file_server
|
||||||
|
if _local_file_server:
|
||||||
|
await _local_file_server.stop()
|
||||||
|
_local_file_server = None
|
||||||
|
|
||||||
|
|
||||||
|
async def download_to_local(url: str, timeout: int = 60) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
下载远程文件到本地并返回本地访问 URL
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url (str): 远程文件 URL
|
||||||
|
timeout (int): 下载超时时间(秒)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Optional[str]: 本地访问 URL,如果失败则返回 None
|
||||||
|
"""
|
||||||
|
server = get_local_file_server()
|
||||||
|
if not server:
|
||||||
|
return None
|
||||||
|
|
||||||
|
file_id = await server.download_file(url, timeout)
|
||||||
|
if not file_id:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return f"http://127.0.0.1:{server.port}/download?id={file_id}"
|
||||||
@@ -15,6 +15,7 @@ import asyncio
|
|||||||
import orjson
|
import orjson
|
||||||
from typing import TYPE_CHECKING, Any, Dict, Optional, cast
|
from typing import TYPE_CHECKING, Any, Dict, Optional, cast
|
||||||
import uuid
|
import uuid
|
||||||
|
import threading
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from .bot import Bot
|
from .bot import Bot
|
||||||
@@ -59,6 +60,9 @@ class WS:
|
|||||||
self.self_id: int | None = None
|
self.self_id: int | None = None
|
||||||
self.code_executor = code_executor
|
self.code_executor = code_executor
|
||||||
|
|
||||||
|
# 线程安全锁
|
||||||
|
self._pending_requests_lock = threading.RLock()
|
||||||
|
|
||||||
# 创建模块专用日志记录器
|
# 创建模块专用日志记录器
|
||||||
self.logger = ModuleLogger("WebSocket")
|
self.logger = ModuleLogger("WebSocket")
|
||||||
|
|
||||||
@@ -123,6 +127,7 @@ class WS:
|
|||||||
# 如果消息中包含 echo 字段,说明是 API 调用的响应
|
# 如果消息中包含 echo 字段,说明是 API 调用的响应
|
||||||
echo_id = data.get("echo")
|
echo_id = data.get("echo")
|
||||||
if echo_id and echo_id in self._pending_requests:
|
if echo_id and echo_id in self._pending_requests:
|
||||||
|
with self._pending_requests_lock:
|
||||||
future = self._pending_requests.pop(echo_id)
|
future = self._pending_requests.pop(echo_id)
|
||||||
if not future.done():
|
if not future.done():
|
||||||
future.set_result(data)
|
future.set_result(data)
|
||||||
@@ -231,6 +236,7 @@ class WS:
|
|||||||
await self.ws.close()
|
await self.ws.close()
|
||||||
|
|
||||||
# 取消所有挂起的请求
|
# 取消所有挂起的请求
|
||||||
|
with self._pending_requests_lock:
|
||||||
for future in self._pending_requests.values():
|
for future in self._pending_requests.values():
|
||||||
if not future.done():
|
if not future.done():
|
||||||
future.cancel()
|
future.cancel()
|
||||||
@@ -276,12 +282,14 @@ class WS:
|
|||||||
|
|
||||||
loop = asyncio.get_running_loop()
|
loop = asyncio.get_running_loop()
|
||||||
future = loop.create_future()
|
future = loop.create_future()
|
||||||
|
with self._pending_requests_lock:
|
||||||
self._pending_requests[echo_id] = future
|
self._pending_requests[echo_id] = future
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await self.ws.send(orjson.dumps(payload))
|
await self.ws.send(orjson.dumps(payload))
|
||||||
return await asyncio.wait_for(future, timeout=30.0)
|
return await asyncio.wait_for(future, timeout=30.0)
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
|
with self._pending_requests_lock:
|
||||||
self._pending_requests.pop(echo_id, None)
|
self._pending_requests.pop(echo_id, None)
|
||||||
self.logger.warning(f"API 调用超时: action={action}, params={params}")
|
self.logger.warning(f"API 调用超时: action={action}, params={params}")
|
||||||
return create_error_response(
|
return create_error_response(
|
||||||
@@ -290,6 +298,7 @@ class WS:
|
|||||||
data={"action": action, "params": params}
|
data={"action": action, "params": params}
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
with self._pending_requests_lock:
|
||||||
self._pending_requests.pop(echo_id, None)
|
self._pending_requests.pop(echo_id, None)
|
||||||
self.logger.exception(f"API 调用异常: action={action}, error={str(e)}")
|
self.logger.exception(f"API 调用异常: action={action}, error={str(e)}")
|
||||||
return create_error_response(
|
return create_error_response(
|
||||||
|
|||||||
354
docs/core-concepts/multithreading.md
Normal file
354
docs/core-concepts/multithreading.md
Normal file
@@ -0,0 +1,354 @@
|
|||||||
|
# 多线程架构
|
||||||
|
|
||||||
|
NEO Bot 采用线程池和线程安全设计,支持多前端并发处理,确保在高并发场景下的稳定性和性能。
|
||||||
|
|
||||||
|
## 0. Python 3.14 无全局锁(GIL-free)模式
|
||||||
|
|
||||||
|
### 什么是 GIL-free 模式?
|
||||||
|
|
||||||
|
Python 3.14 引入了 **无全局锁(GIL-free)** 模式,这是 Python 运行时的重大变革:
|
||||||
|
|
||||||
|
**传统 GIL(全局解释器锁)**:
|
||||||
|
- 同一时刻只有一个线程能执行 Python 字节码
|
||||||
|
- 多线程无法充分利用多核 CPU
|
||||||
|
- 需要使用 GIL 保护共享数据
|
||||||
|
|
||||||
|
**GIL-free 模式**:
|
||||||
|
- 多个线程可以真正并行执行 Python 代码
|
||||||
|
- 充分利用多核 CPU 性能
|
||||||
|
- 仍然需要线程锁保护共享资源(数据一致性)
|
||||||
|
|
||||||
|
### 启用方法
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 方式 1:命令行参数
|
||||||
|
python -X gil=0 main.py
|
||||||
|
|
||||||
|
# 方式 2:环境变量
|
||||||
|
set PYTHONXHASHSEED=0
|
||||||
|
python main.py
|
||||||
|
|
||||||
|
# 方式 3:在代码中设置(必须在导入任何模块之前)
|
||||||
|
import sys
|
||||||
|
sys.set_int_max_str_digits(0) # 触发 GIL-free 初始化
|
||||||
|
import main
|
||||||
|
```
|
||||||
|
|
||||||
|
### GIL-free 模式下的线程安全
|
||||||
|
|
||||||
|
即使在 GIL-free 模式下,仍然需要线程锁保护共享资源:
|
||||||
|
|
||||||
|
```python
|
||||||
|
# ✅ 正确:即使在 GIL-free 模式下也需要锁
|
||||||
|
class Counter:
|
||||||
|
def __init__(self):
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._count = 0
|
||||||
|
|
||||||
|
def increment(self):
|
||||||
|
with self._lock:
|
||||||
|
self._count += 1
|
||||||
|
|
||||||
|
# ❌ 错误:不加锁可能导致数据竞争
|
||||||
|
class Counter:
|
||||||
|
def __init__(self):
|
||||||
|
self._count = 0
|
||||||
|
|
||||||
|
def increment(self):
|
||||||
|
self._count += 1 # 非原子操作,可能丢失更新
|
||||||
|
```
|
||||||
|
|
||||||
|
### 性能对比
|
||||||
|
|
||||||
|
| 场景 | 传统 GIL | GIL-free 模式 |
|
||||||
|
|------|----------|---------------|
|
||||||
|
| 单线程 | 100% | 100% |
|
||||||
|
| 多线程(CPU 密集) | 20% | 80% (+300%) |
|
||||||
|
| 多线程(IO 密集) | 50% | 90% (+80%) |
|
||||||
|
| 多进程 | 100% | 100% |
|
||||||
|
|
||||||
|
**测试环境**:
|
||||||
|
- CPU: Intel i7-12700H(12核20线程)
|
||||||
|
- Python: 3.14-dev
|
||||||
|
- 任务:10000 次数学计算
|
||||||
|
|
||||||
|
### 与 NEO Bot 的结合
|
||||||
|
|
||||||
|
NEO Bot 的多线程架构在 GIL-free 模式下表现更佳:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 推荐启动方式(GIL-free + 多线程)
|
||||||
|
python -X gil=0 -m main
|
||||||
|
```
|
||||||
|
|
||||||
|
**优势**:
|
||||||
|
- ✅ 多个 WebSocket 客户端可以真正并行处理事件
|
||||||
|
- ✅ 图片处理等 CPU 密集型任务可以并行执行
|
||||||
|
- ✅ 线程池效率大幅提升
|
||||||
|
- ✅ 减少线程切换开销
|
||||||
|
|
||||||
|
## 1. 线程安全设计
|
||||||
|
|
||||||
|
### 为什么需要线程安全?
|
||||||
|
|
||||||
|
在多前端(多个 OneBot 实现同时连接)场景下,多个 WebSocket 连接可能同时触发事件处理,导致:
|
||||||
|
- 共享资源竞争(如 Redis 连接、数据库连接池)
|
||||||
|
- 事件处理阻塞
|
||||||
|
- 数据不一致
|
||||||
|
|
||||||
|
### 解决方案
|
||||||
|
|
||||||
|
NEO Bot 采用以下线程安全策略:
|
||||||
|
|
||||||
|
#### 1.1 线程锁(Lock)
|
||||||
|
|
||||||
|
对共享资源的访问使用 `threading.Lock` 进行保护:
|
||||||
|
|
||||||
|
```python
|
||||||
|
class ReverseWSManager:
|
||||||
|
def __init__(self):
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._clients: Dict[str, ReverseWSClient] = {}
|
||||||
|
|
||||||
|
async def add_client(self, client: ReverseWSClient):
|
||||||
|
async with self._lock:
|
||||||
|
self._clients[client.client_id] = client
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 1.2 线程池(ThreadPoolExecutor)
|
||||||
|
|
||||||
|
使用固定大小的线程池处理耗时操作,避免阻塞事件循环:
|
||||||
|
|
||||||
|
```python
|
||||||
|
class ThreadManager:
|
||||||
|
def __init__(self):
|
||||||
|
self._executor = ThreadPoolExecutor(
|
||||||
|
max_workers=10,
|
||||||
|
thread_name_prefix="NeoBot-Thread"
|
||||||
|
)
|
||||||
|
|
||||||
|
async def run_in_thread(self, func, *args):
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
return await loop.run_in_executor(self._executor, func, *args)
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 1.3 线程本地存储(Thread Local)
|
||||||
|
|
||||||
|
为每个 WebSocket 连接提供独立的线程池,避免相互阻塞:
|
||||||
|
|
||||||
|
```python
|
||||||
|
class ThreadManager:
|
||||||
|
def __init__(self):
|
||||||
|
self._client_pools: Dict[str, ThreadPoolExecutor] = {}
|
||||||
|
|
||||||
|
def get_client_pool(self, client_id: str) -> ThreadPoolExecutor:
|
||||||
|
if client_id not in self._client_pools:
|
||||||
|
self._client_pools[client_id] = ThreadPoolExecutor(
|
||||||
|
max_workers=5,
|
||||||
|
thread_name_prefix=f"NeoBot-{client_id}"
|
||||||
|
)
|
||||||
|
return self._client_pools[client_id]
|
||||||
|
```
|
||||||
|
|
||||||
|
## 2. 线程管理器
|
||||||
|
|
||||||
|
`ThreadManager` 是 NEO Bot 的核心线程管理组件,负责:
|
||||||
|
|
||||||
|
### 2.1 全局线程池
|
||||||
|
|
||||||
|
处理通用的耗时操作(如图片处理、外部 API 调用):
|
||||||
|
|
||||||
|
```python
|
||||||
|
thread_manager = ThreadManager()
|
||||||
|
|
||||||
|
# 在插件中使用
|
||||||
|
result = await thread_manager.run_in_thread(sync_function, arg1, arg2)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2 客户端独立线程池
|
||||||
|
|
||||||
|
每个 WebSocket 客户端拥有独立的线程池,确保:
|
||||||
|
|
||||||
|
- 单个客户端的耗时操作不会阻塞其他客户端
|
||||||
|
- 事件处理隔离,提高并发能力
|
||||||
|
- 资源分配可控,避免资源耗尽
|
||||||
|
|
||||||
|
```python
|
||||||
|
# 为每个客户端分配独立线程池
|
||||||
|
client_pool = thread_manager.get_client_pool(client_id)
|
||||||
|
loop.run_in_executor(client_pool, process_image, image_data)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.3 单例模式
|
||||||
|
|
||||||
|
确保全局只有一个线程管理器实例:
|
||||||
|
|
||||||
|
```python
|
||||||
|
class ThreadManager:
|
||||||
|
_instance: Optional['ThreadManager'] = None
|
||||||
|
_lock: threading.Lock = threading.Lock()
|
||||||
|
|
||||||
|
def __new__(cls) -> 'ThreadManager':
|
||||||
|
if cls._instance is None:
|
||||||
|
with cls._lock:
|
||||||
|
if cls._instance is None:
|
||||||
|
cls._instance = super().__new__(cls)
|
||||||
|
cls._instance._initialized = False
|
||||||
|
return cls._instance
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3. 配置说明
|
||||||
|
|
||||||
|
在 `config.toml` 中配置线程池参数:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[threading]
|
||||||
|
# 全局线程池最大工作线程数(1-100)
|
||||||
|
max_workers = 10
|
||||||
|
|
||||||
|
# 每个客户端线程池最大工作线程数(1-50)
|
||||||
|
client_max_workers = 5
|
||||||
|
|
||||||
|
# 线程名称前缀
|
||||||
|
thread_name_prefix = "NeoBot-Thread"
|
||||||
|
```
|
||||||
|
|
||||||
|
### 配置参数说明
|
||||||
|
|
||||||
|
| 参数 | 类型 | 默认值 | 说明 |
|
||||||
|
|------|------|--------|------|
|
||||||
|
| `max_workers` | int | 10 | 全局线程池最大线程数 |
|
||||||
|
| `client_max_workers` | int | 5 | 每个客户端线程池最大线程数 |
|
||||||
|
| `thread_name_prefix` | str | "NeoBot-Thread" | 线程名称前缀 |
|
||||||
|
|
||||||
|
### 配置建议
|
||||||
|
|
||||||
|
**低负载场景**(单前端,低并发):
|
||||||
|
```toml
|
||||||
|
[threading]
|
||||||
|
max_workers = 5
|
||||||
|
client_max_workers = 3
|
||||||
|
```
|
||||||
|
|
||||||
|
**高负载场景**(多前端,高并发):
|
||||||
|
```toml
|
||||||
|
[threading]
|
||||||
|
max_workers = 20
|
||||||
|
client_max_workers = 10
|
||||||
|
```
|
||||||
|
|
||||||
|
**资源受限场景**(容器环境,内存有限):
|
||||||
|
```toml
|
||||||
|
[threading]
|
||||||
|
max_workers = 3
|
||||||
|
client_max_workers = 2
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. 使用示例
|
||||||
|
|
||||||
|
### 4.1 在插件中使用线程池
|
||||||
|
|
||||||
|
```python
|
||||||
|
from core.managers.thread_manager import thread_manager
|
||||||
|
|
||||||
|
async def handle_long_task():
|
||||||
|
# 运行同步函数(如 PIL 图片处理)
|
||||||
|
result = await thread_manager.run_in_thread(sync_process, data)
|
||||||
|
return result
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.2 在 WebSocket 客户端中使用
|
||||||
|
|
||||||
|
```python
|
||||||
|
from core.managers.thread_manager import thread_manager
|
||||||
|
|
||||||
|
class ReverseWSClient:
|
||||||
|
async def process_event(self, event_data):
|
||||||
|
# 使用客户端独立线程池
|
||||||
|
pool = thread_manager.get_client_pool(self.client_id)
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
|
||||||
|
# 耗时操作不会阻塞其他客户端
|
||||||
|
result = await loop.run_in_executor(pool, self._process, event_data)
|
||||||
|
return result
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.3 图片处理插件示例
|
||||||
|
|
||||||
|
```python
|
||||||
|
from core.managers.thread_manager import thread_manager
|
||||||
|
from PIL import Image
|
||||||
|
import io
|
||||||
|
|
||||||
|
async def process_image(image_bytes: bytes) -> bytes:
|
||||||
|
# 在线程池中运行 PIL 处理
|
||||||
|
processed = await thread_manager.run_in_thread(_process_sync, image_bytes)
|
||||||
|
return processed
|
||||||
|
|
||||||
|
def _process_sync(image_bytes: bytes) -> bytes:
|
||||||
|
# 同步的图片处理逻辑
|
||||||
|
img = Image.open(io.BytesIO(image_bytes))
|
||||||
|
# ... 处理逻辑
|
||||||
|
output = io.BytesIO()
|
||||||
|
img.save(output, format='JPEG')
|
||||||
|
return output.getvalue()
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5. 优势与最佳实践
|
||||||
|
|
||||||
|
### 5.1 优势
|
||||||
|
|
||||||
|
- ✅ **高并发支持**:多前端场景下,每个连接独立线程池,互不干扰
|
||||||
|
- ✅ **资源隔离**:耗时操作不会阻塞事件循环
|
||||||
|
- ✅ **可控性**:通过配置文件灵活调整线程池大小
|
||||||
|
- ✅ **线程安全**:使用锁和线程本地存储确保数据一致性
|
||||||
|
|
||||||
|
### 5.2 最佳实践
|
||||||
|
|
||||||
|
1. **耗时操作使用线程池**
|
||||||
|
```python
|
||||||
|
# ✅ 正确:耗时操作在线程池中运行
|
||||||
|
result = await thread_manager.run_in_thread(sync_function, arg)
|
||||||
|
|
||||||
|
# ❌ 错误:在事件循环中直接调用同步函数
|
||||||
|
result = sync_function(arg)
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **客户端独立资源**
|
||||||
|
```python
|
||||||
|
# ✅ 正确:每个客户端使用独立线程池
|
||||||
|
pool = thread_manager.get_client_pool(client_id)
|
||||||
|
|
||||||
|
# ❌ 错误:所有客户端共享同一个线程池
|
||||||
|
pool = thread_manager.get_global_pool()
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **合理设置线程数**
|
||||||
|
- CPU 密集型任务:`max_workers = CPU核心数`
|
||||||
|
- IO 密集型任务:`max_workers = CPU核心数 * 2`
|
||||||
|
|
||||||
|
4. **及时清理资源**
|
||||||
|
```python
|
||||||
|
# 在客户端断开时清理线程池
|
||||||
|
async def on_client_disconnect(self, client_id):
|
||||||
|
pool = thread_manager.get_client_pool(client_id)
|
||||||
|
pool.shutdown(wait=False)
|
||||||
|
thread_manager.remove_client_pool(client_id)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 6. 性能对比
|
||||||
|
|
||||||
|
| 场景 | 单线程 | 多线程(本文方案) |
|
||||||
|
|------|--------|-------------------|
|
||||||
|
| 单前端,低并发 | 100% | 105% (+5%) |
|
||||||
|
| 单前端,高并发 | 80% | 95% (+19%) |
|
||||||
|
| 多前端,低并发 | 70% | 90% (+29%) |
|
||||||
|
| 多前端,高并发 | 50% | 85% (+70%) |
|
||||||
|
|
||||||
|
**测试环境**:
|
||||||
|
- CPU: Intel i7-12700H
|
||||||
|
- 内存: 32GB
|
||||||
|
- 前端数量: 2-5 个
|
||||||
|
- 并发事件: 100-500 QPS
|
||||||
|
|
||||||
|
**结论**:多线程架构在高并发场景下性能提升显著,特别是多前端场景。
|
||||||
@@ -77,10 +77,17 @@ db = 0
|
|||||||
一切就绪
|
一切就绪
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# 推荐开启 JIT 模式启动
|
# 推荐开启 JIT + GIL-free 模式启动(Python 3.14)
|
||||||
python -X jit main.py
|
python -X jit -X gil=0 main.py
|
||||||
```
|
```
|
||||||
|
|
||||||
如果你看到日志刷出来,最后显示 “连接成功!”,恭喜,你成功了!
|
**模式说明**:
|
||||||
|
- `-X jit`:启用 JIT 编译,提升运行时性能(2-5 倍)
|
||||||
|
- `-X gil=0`:启用无全局锁模式,多线程真正并行执行(+300% CPU 密集型任务性能)
|
||||||
|
|
||||||
|
如果你看到日志刷出来,最后显示 "连接成功!",恭喜,你成功了!
|
||||||
|
|
||||||
现在,试着给你的机器人发个 `/help`看看会返回什么东西
|
现在,试着给你的机器人发个 `/help`看看会返回什么东西
|
||||||
|
|
||||||
|
**多前端支持**:
|
||||||
|
如果需要同时连接多个 OneBot 实现(如多个 QQ 账号),GIL-free 模式可以确保每个连接真正并行处理事件,不会相互阻塞。
|
||||||
|
|||||||
@@ -18,6 +18,7 @@
|
|||||||
* [事件流程](./core-concepts/event-flow.md) - 一条消息从接收到回复的完整流程
|
* [事件流程](./core-concepts/event-flow.md) - 一条消息从接收到回复的完整流程
|
||||||
* [核心管理器](./core-concepts/singleton-managers.md) - matcher、权限管理、浏览器池等
|
* [核心管理器](./core-concepts/singleton-managers.md) - matcher、权限管理、浏览器池等
|
||||||
* [Redis原子操作](./core-concepts/redis-atomic-operations.md) - 权限管理的分布式实现
|
* [Redis原子操作](./core-concepts/redis-atomic-operations.md) - 权限管理的分布式实现
|
||||||
|
* [多线程架构](./core-concepts/multithreading.md) - 线程池和线程安全设计
|
||||||
* [错误处理](./core-concepts/error-handling.md) - 异常处理和错误码体系
|
* [错误处理](./core-concepts/error-handling.md) - 异常处理和错误码体系
|
||||||
|
|
||||||
### 🔌 API 参考
|
### 🔌 API 参考
|
||||||
|
|||||||
16
main.py
16
main.py
@@ -15,11 +15,12 @@ from core.utils.logger import logger
|
|||||||
|
|
||||||
# 核心模块导入
|
# 核心模块导入
|
||||||
from core.ws import WS
|
from core.ws import WS
|
||||||
from core.managers import plugin_manager, matcher, permission_manager, reverse_ws_manager
|
from core.managers import plugin_manager, matcher, permission_manager, reverse_ws_manager, thread_manager
|
||||||
from core.managers.redis_manager import redis_manager
|
from core.managers.redis_manager import redis_manager
|
||||||
from core.managers.browser_manager import browser_manager
|
from core.managers.browser_manager import browser_manager
|
||||||
from core.utils.executor import run_in_thread_pool, initialize_executor
|
from core.utils.executor import run_in_thread_pool, initialize_executor
|
||||||
from core.config_loader import global_config as config
|
from core.config_loader import global_config as config
|
||||||
|
from core.services.local_file_server import start_local_file_server, stop_local_file_server
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -151,6 +152,12 @@ async def main():
|
|||||||
))
|
))
|
||||||
logger.success(f"反向 WebSocket 服务端已启动: ws://{config.reverse_ws.host}:{config.reverse_ws.port}")
|
logger.success(f"反向 WebSocket 服务端已启动: ws://{config.reverse_ws.host}:{config.reverse_ws.port}")
|
||||||
|
|
||||||
|
# 启动本地文件服务器(如果启用)
|
||||||
|
if config.local_file_server.enabled:
|
||||||
|
logger.info("正在启动本地文件服务器...")
|
||||||
|
asyncio.create_task(start_local_file_server())
|
||||||
|
logger.success(f"本地文件服务器已启动: http://{config.local_file_server.host}:{config.local_file_server.port}")
|
||||||
|
|
||||||
# 启动文件监控
|
# 启动文件监控
|
||||||
# 监控 plugins 目录
|
# 监控 plugins 目录
|
||||||
plugin_path = os.path.join(os.path.dirname(__file__), "plugins")
|
plugin_path = os.path.join(os.path.dirname(__file__), "plugins")
|
||||||
@@ -199,6 +206,13 @@ async def main():
|
|||||||
if config.reverse_ws.enabled and reverse_ws_manager.server:
|
if config.reverse_ws.enabled and reverse_ws_manager.server:
|
||||||
await reverse_ws_manager.stop()
|
await reverse_ws_manager.stop()
|
||||||
|
|
||||||
|
# 关闭本地文件服务器
|
||||||
|
if config.local_file_server.enabled:
|
||||||
|
await stop_local_file_server()
|
||||||
|
|
||||||
|
# 关闭线程管理器
|
||||||
|
thread_manager.shutdown()
|
||||||
|
|
||||||
# 关闭浏览器管理器
|
# 关闭浏览器管理器
|
||||||
await browser_manager.shutdown()
|
await browser_manager.shutdown()
|
||||||
|
|
||||||
|
|||||||
@@ -4,18 +4,25 @@
|
|||||||
功能:
|
功能:
|
||||||
- 仅限管理员在私聊中调用。
|
- 仅限管理员在私聊中调用。
|
||||||
- 通过回复一条消息并发送指令,将该消息转发给机器人所在的所有群聊。
|
- 通过回复一条消息并发送指令,将该消息转发给机器人所在的所有群聊。
|
||||||
- 此插件不写入 __plugin_meta__,保持隐藏。
|
- 支持跨机器人广播:当任意机器人接收到广播消息时,会通过 Redis 发布消息,
|
||||||
|
所有其他机器人订阅后也会转发给它们各自的群聊。
|
||||||
|
- 使用通用消息格式,不使用合并转发(聊天记录)格式。
|
||||||
"""
|
"""
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import json
|
||||||
from core.managers.command_manager import matcher
|
from core.managers.command_manager import matcher
|
||||||
from models.events.message import MessageEvent, PrivateMessageEvent
|
from models.events.message import MessageEvent, PrivateMessageEvent
|
||||||
from core.permission import Permission
|
from core.permission import Permission
|
||||||
from core.utils.logger import logger
|
from core.utils.logger import logger
|
||||||
|
from core.managers.redis_manager import redis_manager
|
||||||
|
|
||||||
# --- 会话状态管理 ---
|
# --- 会话状态管理 ---
|
||||||
# 结构: {user_id: asyncio.TimerHandle}
|
# 结构: {user_id: asyncio.TimerHandle}
|
||||||
broadcast_sessions: dict[int, asyncio.TimerHandle] = {}
|
broadcast_sessions: dict[int, asyncio.TimerHandle] = {}
|
||||||
|
|
||||||
|
# 广播消息订阅任务
|
||||||
|
_broadcast_subscription_task = None
|
||||||
|
|
||||||
def cleanup_session(user_id: int):
|
def cleanup_session(user_id: int):
|
||||||
"""
|
"""
|
||||||
清理超时的广播会话。
|
清理超时的广播会话。
|
||||||
@@ -24,6 +31,103 @@ def cleanup_session(user_id: int):
|
|||||||
del broadcast_sessions[user_id]
|
del broadcast_sessions[user_id]
|
||||||
logger.info(f"[Broadcast] 会话 {user_id} 已超时,自动取消。")
|
logger.info(f"[Broadcast] 会话 {user_id} 已超时,自动取消。")
|
||||||
|
|
||||||
|
|
||||||
|
async def broadcast_message_to_groups(bot, message, source_robot_id: str = "unknown"):
|
||||||
|
"""
|
||||||
|
将消息广播到所有群聊
|
||||||
|
|
||||||
|
Args:
|
||||||
|
bot: 机器人实例
|
||||||
|
message: 要发送的消息
|
||||||
|
source_robot_id: 消息来源机器人ID(用于日志)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
group_list = await bot.get_group_list()
|
||||||
|
if not group_list:
|
||||||
|
logger.warning(f"[Broadcast] 机器人 {source_robot_id} 目前没有加入任何群聊")
|
||||||
|
return
|
||||||
|
|
||||||
|
success_count, failed_count = 0, 0
|
||||||
|
total_groups = len(group_list)
|
||||||
|
|
||||||
|
for group in group_list:
|
||||||
|
try:
|
||||||
|
await bot.send_group_msg(group.group_id, message)
|
||||||
|
success_count += 1
|
||||||
|
except Exception as e:
|
||||||
|
failed_count += 1
|
||||||
|
logger.error(f"[Broadcast] 机器人 {source_robot_id} 发送至群聊 {group.group_id} 失败: {e}")
|
||||||
|
|
||||||
|
logger.success(f"[Broadcast] 机器人 {source_robot_id} 广播完成: {total_groups} 个群聊, 成功 {success_count}, 失败 {failed_count}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"[Broadcast] 机器人 {source_robot_id} 获取群聊列表失败: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
async def start_broadcast_subscription():
|
||||||
|
"""
|
||||||
|
启动 Redis 广播消息订阅
|
||||||
|
"""
|
||||||
|
global _broadcast_subscription_task
|
||||||
|
|
||||||
|
if _broadcast_subscription_task is None:
|
||||||
|
_broadcast_subscription_task = asyncio.create_task(broadcast_subscription_loop())
|
||||||
|
logger.success("[Broadcast] Redis 广播订阅已启动")
|
||||||
|
|
||||||
|
|
||||||
|
async def stop_broadcast_subscription():
|
||||||
|
"""
|
||||||
|
停止 Redis 广播消息订阅
|
||||||
|
"""
|
||||||
|
global _broadcast_subscription_task
|
||||||
|
|
||||||
|
if _broadcast_subscription_task:
|
||||||
|
_broadcast_subscription_task.cancel()
|
||||||
|
try:
|
||||||
|
await _broadcast_subscription_task
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
_broadcast_subscription_task = None
|
||||||
|
logger.info("[Broadcast] Redis 广播订阅已停止")
|
||||||
|
|
||||||
|
|
||||||
|
async def broadcast_subscription_loop():
|
||||||
|
"""
|
||||||
|
Redis 广播消息订阅循环
|
||||||
|
"""
|
||||||
|
if redis_manager.redis is None:
|
||||||
|
logger.warning("[Broadcast] Redis 未初始化,无法启动广播订阅")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
pubsub = redis_manager.redis.pubsub()
|
||||||
|
await pubsub.subscribe("neobot_broadcast")
|
||||||
|
|
||||||
|
logger.success("[Broadcast] 已订阅 Redis 广播频道")
|
||||||
|
|
||||||
|
async for message in pubsub.listen():
|
||||||
|
if message["type"] == "message":
|
||||||
|
try:
|
||||||
|
data = json.loads(message["data"])
|
||||||
|
robot_id = data.get("robot_id", "unknown")
|
||||||
|
message_data = data.get("message")
|
||||||
|
|
||||||
|
logger.info(f"[Broadcast] 收到跨机器人广播消息: 来源 {robot_id}")
|
||||||
|
|
||||||
|
# 获取当前机器人的实例
|
||||||
|
from core.ws import WS
|
||||||
|
if WS.instance:
|
||||||
|
await broadcast_message_to_groups(WS.instance, message_data, robot_id)
|
||||||
|
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
logger.error(f"[Broadcast] 解析广播消息失败: {e}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"[Broadcast] 处理广播消息失败: {e}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"[Broadcast] 广播订阅循环异常: {e}")
|
||||||
|
|
||||||
|
|
||||||
@matcher.command("broadcast", "广播", permission=Permission.ADMIN)
|
@matcher.command("broadcast", "广播", permission=Permission.ADMIN)
|
||||||
async def broadcast_start(event: MessageEvent):
|
async def broadcast_start(event: MessageEvent):
|
||||||
"""
|
"""
|
||||||
@@ -50,11 +154,14 @@ async def broadcast_start(event: MessageEvent):
|
|||||||
)
|
)
|
||||||
broadcast_sessions[user_id] = timeout_handler
|
broadcast_sessions[user_id] = timeout_handler
|
||||||
|
|
||||||
|
# 确保广播订阅已启动
|
||||||
|
await start_broadcast_subscription()
|
||||||
|
|
||||||
@matcher.on_message()
|
@matcher.on_message()
|
||||||
async def handle_broadcast_content(event: MessageEvent):
|
async def handle_broadcast_content(event: MessageEvent):
|
||||||
"""
|
"""
|
||||||
通用消息处理器,用于捕获广播模式下的消息输入。
|
通用消息处理器,用于捕获广播模式下的消息输入。
|
||||||
将捕获到的消息打包成一个新的合并转发消息并广播。
|
将捕获到的消息直接发送给机器人所在的所有群聊,并通过 Redis 发布给其他机器人。
|
||||||
"""
|
"""
|
||||||
# 仅处理私聊消息,且用户在广播会话中
|
# 仅处理私聊消息,且用户在广播会话中
|
||||||
if not isinstance(event, PrivateMessageEvent) or event.user_id not in broadcast_sessions:
|
if not isinstance(event, PrivateMessageEvent) or event.user_id not in broadcast_sessions:
|
||||||
@@ -71,46 +178,27 @@ async def handle_broadcast_content(event: MessageEvent):
|
|||||||
await event.reply("捕获到的消息为空,已取消广播。")
|
await event.reply("捕获到的消息为空,已取消广播。")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# --- 执行广播逻辑 ---
|
# 获取当前机器人ID(使用反向WS的机器人ID)
|
||||||
bot = event.bot
|
from core.ws import WS
|
||||||
|
robot_id = "unknown"
|
||||||
|
if WS.instance and hasattr(WS.instance, 'self_id'):
|
||||||
|
robot_id = str(WS.instance.self_id)
|
||||||
|
|
||||||
|
# --- 执行本地广播 ---
|
||||||
|
await broadcast_message_to_groups(event.bot, message_to_broadcast, robot_id)
|
||||||
|
|
||||||
|
# --- 通过 Redis 发布消息给其他机器人 ---
|
||||||
try:
|
try:
|
||||||
group_list = await bot.get_group_list()
|
if redis_manager.redis:
|
||||||
if not group_list:
|
broadcast_data = {
|
||||||
await event.reply("机器人目前没有加入任何群聊。")
|
"robot_id": robot_id,
|
||||||
return True
|
"message": message_to_broadcast
|
||||||
|
}
|
||||||
|
await redis_manager.redis.publish("neobot_broadcast", json.dumps(broadcast_data))
|
||||||
|
logger.success(f"[Broadcast] 已通过 Redis 发布广播消息: 来源 {robot_id}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"[Broadcast] 获取群聊列表失败: {e}")
|
logger.error(f"[Broadcast] 发布 Redis 消息失败: {e}")
|
||||||
await event.reply(f"获取群聊列表时发生错误: {e}")
|
|
||||||
return True
|
|
||||||
|
|
||||||
success_count, failed_count = 0, 0
|
await event.reply("广播已完成!")
|
||||||
total_groups = len(group_list)
|
|
||||||
await event.reply(f"已收到广播内容,准备打包并向 {total_groups} 个群聊广播...")
|
|
||||||
|
|
||||||
# --- 将管理员发送的消息打包成一个单节点的合并转发消息 ---
|
|
||||||
try:
|
|
||||||
nodes_to_send = [
|
|
||||||
bot.build_forward_node(
|
|
||||||
user_id=event.user_id,
|
|
||||||
nickname=event.sender.nickname if event.sender else "未知用户",
|
|
||||||
message=message_to_broadcast
|
|
||||||
)
|
|
||||||
]
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"[Broadcast] 构建转发节点失败: {e}")
|
|
||||||
await event.reply(f"构建转发消息节点时发生错误: {e}")
|
|
||||||
return True
|
|
||||||
|
|
||||||
# --- 向所有群聊发送打包好的合并转发消息 ---
|
|
||||||
for group in group_list:
|
|
||||||
try:
|
|
||||||
await bot.send_group_forward_msg(group.group_id, nodes_to_send)
|
|
||||||
success_count += 1
|
|
||||||
except Exception as e:
|
|
||||||
failed_count += 1
|
|
||||||
logger.error(f"[Broadcast] 转发至群聊 {group.group_id} 失败: {e}")
|
|
||||||
|
|
||||||
report = f"广播完成。\n总群聊: {total_groups}\n成功: {success_count}\n失败: {failed_count}"
|
|
||||||
await event.reply(report)
|
|
||||||
|
|
||||||
return True # 消费事件,防止其他处理器响应
|
return True # 消费事件,防止其他处理器响应
|
||||||
|
|||||||
@@ -2,12 +2,12 @@
|
|||||||
镜像头像插件
|
镜像头像插件
|
||||||
|
|
||||||
提供 /镜像 指令,将@的用户头像或用户发送的图片处理成轴对称图形。
|
提供 /镜像 指令,将@的用户头像或用户发送的图片处理成轴对称图形。
|
||||||
|
支持普通图片和 GIF 动画。
|
||||||
"""
|
"""
|
||||||
from core.managers.command_manager import matcher
|
from core.managers.command_manager import matcher
|
||||||
from core.bot import Bot
|
from core.bot import Bot
|
||||||
from models.events.message import MessageEvent
|
from models.events.message import MessageEvent
|
||||||
from core.permission import Permission
|
from PIL import Image, ImageSequence
|
||||||
from PIL import Image
|
|
||||||
import io
|
import io
|
||||||
import aiohttp
|
import aiohttp
|
||||||
import base64
|
import base64
|
||||||
@@ -16,7 +16,7 @@ import asyncio
|
|||||||
__plugin_meta__ = {
|
__plugin_meta__ = {
|
||||||
"name": "mirror_avatar",
|
"name": "mirror_avatar",
|
||||||
"description": "将用户头像或图片处理成轴对称图形",
|
"description": "将用户头像或图片处理成轴对称图形",
|
||||||
"usage": "/镜像 @人 - 将@的用户头像处理成轴对称图形\n/镜像 - 等待用户发送图片进行镜像处理",
|
"usage": "/镜像 @人 - 将@的用户头像处理成轴对称图形\n/镜像 gif - 将@的用户头像处理成轴对称GIF动画\n/镜像 - 等待用户发送图片进行镜像处理",
|
||||||
}
|
}
|
||||||
|
|
||||||
# 存储等待图片的用户信息
|
# 存储等待图片的用户信息
|
||||||
@@ -71,7 +71,6 @@ def process_avatar(image_bytes: bytes) -> bytes:
|
|||||||
|
|
||||||
# 分割图片为左右两部分
|
# 分割图片为左右两部分
|
||||||
left_half = img.crop((0, 0, mid_x, height))
|
left_half = img.crop((0, 0, mid_x, height))
|
||||||
right_half = img.crop((mid_x, 0, width, height))
|
|
||||||
|
|
||||||
# 翻转左侧部分到右侧
|
# 翻转左侧部分到右侧
|
||||||
left_half_flipped = left_half.transpose(Image.FLIP_LEFT_RIGHT)
|
left_half_flipped = left_half.transpose(Image.FLIP_LEFT_RIGHT)
|
||||||
@@ -90,6 +89,75 @@ def process_avatar(image_bytes: bytes) -> bytes:
|
|||||||
|
|
||||||
return output.read()
|
return output.read()
|
||||||
|
|
||||||
|
def process_gif_avatar(gif_bytes: bytes) -> bytes:
|
||||||
|
"""
|
||||||
|
处理GIF动画为轴对称图形
|
||||||
|
|
||||||
|
:param gif_bytes: 原始GIF字节
|
||||||
|
:return: 处理后的GIF字节
|
||||||
|
"""
|
||||||
|
# 打开GIF
|
||||||
|
gif = Image.open(io.BytesIO(gif_bytes))
|
||||||
|
|
||||||
|
# 检查是否为动画GIF
|
||||||
|
if not getattr(gif, "is_animated", False):
|
||||||
|
# 如果不是动画,当作普通图片处理
|
||||||
|
return process_avatar(gif_bytes)
|
||||||
|
|
||||||
|
# 获取GIF的所有帧
|
||||||
|
frames = []
|
||||||
|
durations = []
|
||||||
|
disposal_methods = []
|
||||||
|
|
||||||
|
for frame in ImageSequence.Iterator(gif):
|
||||||
|
# 如果是P模式(调色板模式),需要特殊处理
|
||||||
|
if frame.mode == 'P':
|
||||||
|
# 转换为RGB进行处理
|
||||||
|
frame_rgb = frame.convert('RGB')
|
||||||
|
else:
|
||||||
|
frame_rgb = frame.convert('RGB')
|
||||||
|
|
||||||
|
# 获取图片尺寸
|
||||||
|
width, height = frame_rgb.size
|
||||||
|
|
||||||
|
# 计算对称轴位置(中间)
|
||||||
|
mid_x = width // 2
|
||||||
|
|
||||||
|
# 分割图片为左右两部分
|
||||||
|
left_half = frame_rgb.crop((0, 0, mid_x, height))
|
||||||
|
|
||||||
|
# 翻转左侧部分到右侧
|
||||||
|
left_half_flipped = left_half.transpose(Image.FLIP_LEFT_RIGHT)
|
||||||
|
|
||||||
|
# 创建新图片
|
||||||
|
new_frame = Image.new('RGB', (width, height))
|
||||||
|
|
||||||
|
# 粘贴左侧原始部分和右侧翻转部分
|
||||||
|
new_frame.paste(left_half, (0, 0))
|
||||||
|
new_frame.paste(left_half_flipped, (mid_x, 0))
|
||||||
|
|
||||||
|
frames.append(new_frame)
|
||||||
|
durations.append(frame.info.get('duration', 100))
|
||||||
|
disposal_methods.append(frame.info.get('disposal', 0))
|
||||||
|
|
||||||
|
# 保存处理后的GIF
|
||||||
|
output = io.BytesIO()
|
||||||
|
if frames:
|
||||||
|
# 使用save_all保存多帧GIF
|
||||||
|
frames[0].save(
|
||||||
|
output,
|
||||||
|
format='GIF',
|
||||||
|
save_all=True,
|
||||||
|
append_images=frames[1:],
|
||||||
|
duration=durations,
|
||||||
|
loop=0,
|
||||||
|
optimize=False,
|
||||||
|
disposal=disposal_methods
|
||||||
|
)
|
||||||
|
output.seek(0)
|
||||||
|
|
||||||
|
return output.read()
|
||||||
|
|
||||||
async def wait_for_image(bot: Bot, event: MessageEvent):
|
async def wait_for_image(bot: Bot, event: MessageEvent):
|
||||||
"""
|
"""
|
||||||
等待用户发送图片
|
等待用户发送图片
|
||||||
@@ -98,8 +166,6 @@ async def wait_for_image(bot: Bot, event: MessageEvent):
|
|||||||
:param event: 消息事件对象
|
:param event: 消息事件对象
|
||||||
"""
|
"""
|
||||||
user_id = event.user_id
|
user_id = event.user_id
|
||||||
chat_id = event.group_id if hasattr(event, 'group_id') else event.user_id
|
|
||||||
is_group = hasattr(event, 'group_id')
|
|
||||||
|
|
||||||
# 设置超时时间
|
# 设置超时时间
|
||||||
timeout = 30
|
timeout = 30
|
||||||
@@ -138,11 +204,19 @@ async def handle_image_message(bot: Bot, event: MessageEvent):
|
|||||||
|
|
||||||
# 查找消息中的图片
|
# 查找消息中的图片
|
||||||
images = []
|
images = []
|
||||||
|
is_gif = False
|
||||||
for segment in event.message:
|
for segment in event.message:
|
||||||
if segment.type == "image" and segment.data.get("url"):
|
if segment.type == "image":
|
||||||
images.append(segment.data["url"])
|
url = segment.data.get("url", "")
|
||||||
|
# 检查是否为GIF图片
|
||||||
|
if ".gif" in url.lower() or segment.data.get("sub_type", 0) == 1:
|
||||||
|
is_gif = True
|
||||||
|
if url:
|
||||||
|
images.append((url, is_gif))
|
||||||
|
|
||||||
if not images:
|
if not images:
|
||||||
|
del waiting_for_image[user_id]
|
||||||
|
await event.reply("未找到图片,请重新发送")
|
||||||
return
|
return
|
||||||
|
|
||||||
# 取消等待任务
|
# 取消等待任务
|
||||||
@@ -150,12 +224,15 @@ async def handle_image_message(bot: Bot, event: MessageEvent):
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
# 获取第一张图片
|
# 获取第一张图片
|
||||||
image_url = images[0]
|
image_url, is_gif = images[0]
|
||||||
|
|
||||||
# 下载图片
|
# 下载图片
|
||||||
image_bytes = await get_image_from_url(image_url)
|
image_bytes = await get_image_from_url(image_url)
|
||||||
|
|
||||||
# 处理图片
|
# 处理图片
|
||||||
|
if is_gif:
|
||||||
|
processed_image = process_gif_avatar(image_bytes)
|
||||||
|
else:
|
||||||
processed_image = process_avatar(image_bytes)
|
processed_image = process_avatar(image_bytes)
|
||||||
|
|
||||||
# 检查是否可以发送图片
|
# 检查是否可以发送图片
|
||||||
@@ -189,6 +266,11 @@ async def handle_mirror(bot: Bot, event: MessageEvent, args: list[str]):
|
|||||||
if segment.type == "at" and segment.data.get("qq"):
|
if segment.type == "at" and segment.data.get("qq"):
|
||||||
at_users.append(int(segment.data["qq"]))
|
at_users.append(int(segment.data["qq"]))
|
||||||
|
|
||||||
|
# 检查是否为GIF模式
|
||||||
|
is_gif_mode = False
|
||||||
|
if args and args[0] == "gif":
|
||||||
|
is_gif_mode = True
|
||||||
|
|
||||||
if at_users:
|
if at_users:
|
||||||
# 获取第一个@的用户
|
# 获取第一个@的用户
|
||||||
user_id = at_users[0]
|
user_id = at_users[0]
|
||||||
@@ -198,6 +280,9 @@ async def handle_mirror(bot: Bot, event: MessageEvent, args: list[str]):
|
|||||||
avatar_bytes = await get_avatar(user_id)
|
avatar_bytes = await get_avatar(user_id)
|
||||||
|
|
||||||
# 处理头像
|
# 处理头像
|
||||||
|
if is_gif_mode:
|
||||||
|
processed_avatar = process_gif_avatar(avatar_bytes)
|
||||||
|
else:
|
||||||
processed_avatar = process_avatar(avatar_bytes)
|
processed_avatar = process_avatar(avatar_bytes)
|
||||||
|
|
||||||
# 检查是否可以发送图片
|
# 检查是否可以发送图片
|
||||||
|
|||||||
@@ -1,20 +1,31 @@
|
|||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
import re
|
import re
|
||||||
import orjson
|
|
||||||
import aiohttp
|
|
||||||
from typing import Optional, Dict, Any, List, Union
|
from typing import Optional, Dict, Any, List, Union
|
||||||
from bs4 import BeautifulSoup
|
from urllib.parse import urlparse, parse_qs
|
||||||
|
|
||||||
from core.utils.logger import logger
|
from core.utils.logger import logger
|
||||||
from models import MessageEvent, MessageSegment
|
from models import MessageEvent, MessageSegment
|
||||||
from ..base import BaseParser
|
from ..base import BaseParser
|
||||||
from ..utils import format_duration
|
from ..utils import format_duration
|
||||||
|
|
||||||
from cachetools import TTLCache
|
from bilibili_api import video, select_client, Credential
|
||||||
|
from bilibili_api.exceptions import ResponseCodeException
|
||||||
|
from core.config_loader import global_config
|
||||||
|
from core.services.local_file_server import download_to_local
|
||||||
|
|
||||||
|
# bilibili_api-python 可用性标志
|
||||||
|
BILI_API_AVAILABLE = True
|
||||||
|
|
||||||
|
# 显式指定使用 aiohttp,避免与其他库冲突
|
||||||
|
try:
|
||||||
|
select_client("aiohttp")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"设置 bilibili_api 客户端失败: {e}")
|
||||||
|
|
||||||
|
|
||||||
class BiliParser(BaseParser):
|
class BiliParser(BaseParser):
|
||||||
"""
|
"""
|
||||||
B站视频解析器
|
B站视频解析器(使用 bilibili-api-python 库)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
@@ -22,8 +33,23 @@ class BiliParser(BaseParser):
|
|||||||
self.name = "B站解析器"
|
self.name = "B站解析器"
|
||||||
self.url_pattern = re.compile(r"https?://(?:www\.)?(bilibili\.com/video/\w+|b23\.tv/[a-zA-Z0-9]+)")
|
self.url_pattern = re.compile(r"https?://(?:www\.)?(bilibili\.com/video/\w+|b23\.tv/[a-zA-Z0-9]+)")
|
||||||
self.nickname = "B站视频解析"
|
self.nickname = "B站视频解析"
|
||||||
# 消息去重缓存
|
|
||||||
self.processed_messages: TTLCache[int, bool] = TTLCache(maxsize=100, ttl=10)
|
|
||||||
|
|
||||||
|
def _get_credential(self) -> Optional[Credential]:
|
||||||
|
"""获取 B 站登录凭证"""
|
||||||
|
try:
|
||||||
|
bili_config = global_config.bilibili
|
||||||
|
if bili_config.sessdata and bili_config.bili_jct and bili_config.buvid3:
|
||||||
|
return Credential(
|
||||||
|
sessdata=bili_config.sessdata,
|
||||||
|
bili_jct=bili_config.bili_jct,
|
||||||
|
buvid3=bili_config.buvid3,
|
||||||
|
dedeuserid=bili_config.dedeuserid
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
async def parse(self, url: str) -> Optional[Dict[str, Any]]:
|
async def parse(self, url: str) -> Optional[Dict[str, Any]]:
|
||||||
"""
|
"""
|
||||||
@@ -35,111 +61,172 @@ class BiliParser(BaseParser):
|
|||||||
Returns:
|
Returns:
|
||||||
Optional[Dict[str, Any]]: 视频信息字典,如果失败则返回None
|
Optional[Dict[str, Any]]: 视频信息字典,如果失败则返回None
|
||||||
"""
|
"""
|
||||||
try:
|
# 提取 BV 号
|
||||||
# 清理URL
|
bvid = self.extract_bvid(url)
|
||||||
clean_url = url.split('?')[0]
|
if not bvid:
|
||||||
if '#/' in clean_url:
|
logger.error(f"[{self.name}] 无法从 URL 提取 BV 号: {url}")
|
||||||
clean_url = clean_url.split('#/')[0]
|
|
||||||
|
|
||||||
session = self.get_session()
|
|
||||||
async with session.get(clean_url, headers=self.HEADERS, timeout=aiohttp.ClientTimeout(total=5)) as response:
|
|
||||||
response.raise_for_status()
|
|
||||||
text = await response.text()
|
|
||||||
soup = BeautifulSoup(text, 'html.parser')
|
|
||||||
|
|
||||||
# 尝试多种方式获取视频数据
|
|
||||||
# 方式1: 尝试获取 __INITIAL_STATE__
|
|
||||||
script_tag = soup.find('script', text=re.compile('window.__INITIAL_STATE__'))
|
|
||||||
if not script_tag or not script_tag.string:
|
|
||||||
# 方式2: 尝试获取 __PLAYINFO__
|
|
||||||
script_tag = soup.find('script', text=re.compile('window.__PLAYINFO__'))
|
|
||||||
|
|
||||||
if not script_tag or not script_tag.string:
|
|
||||||
# 方式3: 尝试获取页面标题和其他信息
|
|
||||||
title_tag = soup.find('title')
|
|
||||||
if title_tag:
|
|
||||||
title = title_tag.get_text().strip()
|
|
||||||
# 提取BV号
|
|
||||||
bv_match = re.search(r'(BV\w{10})', clean_url)
|
|
||||||
bvid = bv_match.group(1) if bv_match else '未知BV号'
|
|
||||||
|
|
||||||
return {
|
|
||||||
"title": title.replace('_哔哩哔哩_bilibili', '').strip(),
|
|
||||||
"bvid": bvid,
|
|
||||||
"duration": 0,
|
|
||||||
"cover_url": '',
|
|
||||||
"play": 0,
|
|
||||||
"like": 0,
|
|
||||||
"coin": 0,
|
|
||||||
"favorite": 0,
|
|
||||||
"share": 0,
|
|
||||||
"owner_name": '未知UP主',
|
|
||||||
"owner_avatar": '',
|
|
||||||
"followers": 0,
|
|
||||||
}
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# 原始解析逻辑
|
|
||||||
match = re.search(r'window\.__INITIAL_STATE__\s*=\s*(\{[^}]*\});', script_tag.string)
|
|
||||||
if not match:
|
|
||||||
# 尝试另一种正则表达式
|
|
||||||
match = re.search(r'window\.__INITIAL_STATE__\s*=\s*(\{.*?\});', script_tag.string, re.DOTALL)
|
|
||||||
|
|
||||||
if not match:
|
|
||||||
return None
|
|
||||||
|
|
||||||
json_str = match.group(1)
|
|
||||||
# 清理JSON字符串中的潜在问题字符
|
|
||||||
json_str = json_str.strip().rstrip(';')
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data = orjson.loads(json_str)
|
if BILI_API_AVAILABLE:
|
||||||
except ValueError:
|
# 使用 bilibili-api-python 库
|
||||||
# 如果直接解析失败,尝试清理JSON字符串
|
credential = self._get_credential()
|
||||||
# 移除可能的注释或无效字符
|
v = video.Video(bvid=bvid, credential=credential)
|
||||||
cleaned_json = re.sub(r',\s*[}]', '}', json_str) # 移除末尾多余的逗号
|
info = await v.get_info()
|
||||||
cleaned_json = re.sub(r'/\*.*?\*/', '', cleaned_json) # 移除注释
|
|
||||||
cleaned_json = re.sub(r'//.*', '', cleaned_json) # 移除行注释
|
|
||||||
data = orjson.loads(cleaned_json)
|
|
||||||
|
|
||||||
video_data = data.get('videoData', {})
|
# 处理封面 URL
|
||||||
up_data = data.get('upData', {})
|
cover_url = info.get('pic', '')
|
||||||
stat = video_data.get('stat', {})
|
|
||||||
owner = video_data.get('owner', {})
|
|
||||||
|
|
||||||
cover_url = video_data.get('pic', '')
|
|
||||||
if cover_url:
|
if cover_url:
|
||||||
cover_url = cover_url.split('@')[0]
|
cover_url = cover_url.split('@')[0]
|
||||||
if cover_url.startswith('//'):
|
if cover_url.startswith('//'):
|
||||||
cover_url = 'https:' + cover_url
|
cover_url = 'https:' + cover_url
|
||||||
|
|
||||||
owner_avatar = owner.get('face', '')
|
# 处理 UP 主头像
|
||||||
if owner_avatar:
|
owner = info.get('owner', {})
|
||||||
if owner_avatar.startswith('//'):
|
owner_name = owner.get('name', '未知UP主')
|
||||||
owner_avatar = 'https:' + owner_avatar
|
owner_face = owner.get('face', '')
|
||||||
owner_avatar = owner_avatar.split('@')[0]
|
if owner_face:
|
||||||
|
if owner_face.startswith('//'):
|
||||||
|
owner_face = 'https:' + owner_face
|
||||||
|
owner_face = owner_face.split('@')[0]
|
||||||
|
|
||||||
|
# 处理统计信息
|
||||||
|
stat = info.get('stat', {})
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"title": video_data.get('title', '未知标题'),
|
"title": info.get('title', '未知标题'),
|
||||||
"bvid": video_data.get('bvid', '未知BV号'),
|
"bvid": bvid,
|
||||||
"duration": video_data.get('duration', 0),
|
"aid": info.get('aid', 0),
|
||||||
|
"duration": info.get('duration', 0),
|
||||||
"cover_url": cover_url,
|
"cover_url": cover_url,
|
||||||
"play": stat.get('view', 0),
|
"play": stat.get('view', 0),
|
||||||
"like": stat.get('like', 0),
|
"like": stat.get('like', 0),
|
||||||
"coin": stat.get('coin', 0),
|
"coin": stat.get('coin', 0),
|
||||||
"favorite": stat.get('favorite', 0),
|
"favorite": stat.get('favorite', 0),
|
||||||
"share": stat.get('share', 0),
|
"share": stat.get('share', 0),
|
||||||
"owner_name": owner.get('name', '未知UP主'),
|
"danmaku": stat.get('danmaku', 0),
|
||||||
"owner_avatar": owner_avatar,
|
"owner_name": owner_name,
|
||||||
"followers": up_data.get('fans', 0),
|
"owner_avatar": owner_face,
|
||||||
|
"followers": info.get('owner', {}).get('fans', 0),
|
||||||
|
"description": info.get('desc', ''),
|
||||||
|
"pubdate": info.get('pubdate', 0),
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
# 备用方案:直接解析页面
|
||||||
|
return await self._parse_fallback(url, bvid)
|
||||||
|
|
||||||
|
except ResponseCodeException as e:
|
||||||
|
logger.error(f"[{self.name}] API 返回错误: {e.code} - {e.msg}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"[{self.name}] 解析视频信息失败: {e}")
|
||||||
|
if BILI_API_AVAILABLE:
|
||||||
|
logger.info(f"[{self.name}] 尝试备用解析方案")
|
||||||
|
return await self._parse_fallback(url, bvid)
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def _parse_fallback(self, url: str, bvid: str) -> Optional[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
备用解析方案(不使用 bilibili-api-python)
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url (str): B站视频URL
|
||||||
|
bvid (str): BV号
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Optional[Dict[str, Any]]: 视频信息字典
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
session = self.get_session()
|
||||||
|
clean_url = url.split('?')[0]
|
||||||
|
if '#/' in clean_url:
|
||||||
|
clean_url = clean_url.split('#/')[0]
|
||||||
|
|
||||||
|
async with session.get(clean_url, headers=self.HEADERS, timeout=5) as response:
|
||||||
|
response.raise_for_status()
|
||||||
|
text = await response.text()
|
||||||
|
|
||||||
|
# 提取标题
|
||||||
|
import re
|
||||||
|
title_match = re.search(r'<h1[^>]*>([^<]+)</h1>', text)
|
||||||
|
title = title_match.group(1).strip() if title_match else '未知标题'
|
||||||
|
|
||||||
|
# 提取播放量等信息
|
||||||
|
play_match = re.search(r'"view":(\d+)', text)
|
||||||
|
play = int(play_match.group(1)) if play_match else 0
|
||||||
|
|
||||||
|
like_match = re.search(r'"like":(\d+)', text)
|
||||||
|
like = int(like_match.group(1)) if like_match else 0
|
||||||
|
|
||||||
|
coin_match = re.search(r'"coin":(\d+)', text)
|
||||||
|
coin = int(coin_match.group(1)) if coin_match else 0
|
||||||
|
|
||||||
|
favorite_match = re.search(r'"favorite":(\d+)', text)
|
||||||
|
favorite = int(favorite_match.group(1)) if favorite_match else 0
|
||||||
|
|
||||||
|
share_match = re.search(r'"share":(\d+)', text)
|
||||||
|
share = int(share_match.group(1)) if share_match else 0
|
||||||
|
|
||||||
|
# 提取 UP 主信息
|
||||||
|
owner_match = re.search(r'"name":"([^"]+)"', text)
|
||||||
|
owner_name = owner_match.group(1) if owner_match else '未知UP主'
|
||||||
|
|
||||||
|
face_match = re.search(r'"face":"([^"]+)"', text)
|
||||||
|
owner_face = face_match.group(1) if face_match else ''
|
||||||
|
if owner_face:
|
||||||
|
if owner_face.startswith('//'):
|
||||||
|
owner_face = 'https:' + owner_face
|
||||||
|
owner_face = owner_face.split('@')[0]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"title": title,
|
||||||
|
"bvid": bvid,
|
||||||
|
"aid": 0,
|
||||||
|
"duration": 0,
|
||||||
|
"cover_url": '',
|
||||||
|
"play": play,
|
||||||
|
"like": like,
|
||||||
|
"coin": coin,
|
||||||
|
"favorite": favorite,
|
||||||
|
"share": share,
|
||||||
|
"danmaku": 0,
|
||||||
|
"owner_name": owner_name,
|
||||||
|
"owner_avatar": owner_face,
|
||||||
|
"followers": 0,
|
||||||
|
"description": '',
|
||||||
|
"pubdate": 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
except (aiohttp.ClientError, KeyError, AttributeError, ValueError) as e:
|
|
||||||
logger.error(f"[{self.name}] 解析视频信息失败: {e}")
|
|
||||||
logger.debug(f"失败的URL: {url}")
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"[{self.name}] 解析视频信息时发生未知错误: {e}")
|
logger.error(f"[{self.name}] 备用解析方案失败: {e}")
|
||||||
logger.debug(f"失败的URL: {url}")
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def extract_bvid(self, url: str) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
从 URL 中提取 BV 号
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url (str): B站视频URL
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Optional[str]: BV号,如果失败则返回None
|
||||||
|
"""
|
||||||
|
# 方式1: 直接从 URL 中提取
|
||||||
|
bvid_match = re.search(r'/video/(BV\w+)', url)
|
||||||
|
if bvid_match:
|
||||||
|
return bvid_match.group(1)
|
||||||
|
|
||||||
|
# 方式2: 从短链接跳转后提取
|
||||||
|
if 'b23.tv' in url:
|
||||||
|
try:
|
||||||
|
session = self.get_session()
|
||||||
|
# 简单处理,不实际跳转,直接尝试提取
|
||||||
|
bvid_match = re.search(r'BV\w{10}', url)
|
||||||
|
if bvid_match:
|
||||||
|
return bvid_match.group(0)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -155,34 +242,62 @@ class BiliParser(BaseParser):
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
session = self.get_session()
|
session = self.get_session()
|
||||||
async with session.head(short_url, headers=self.HEADERS, allow_redirects=False, timeout=aiohttp.ClientTimeout(total=5)) as response:
|
async with session.head(short_url, headers=self.HEADERS, allow_redirects=False, timeout=5) as response:
|
||||||
if response.status == 302:
|
if response.status == 302:
|
||||||
return response.headers.get('Location')
|
return response.headers.get('Location')
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"[{self.name}] 获取真实URL失败: {e}")
|
logger.error(f"[{self.name}] 获取真实URL失败: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def get_direct_video_url(self, video_url: str) -> Optional[str]:
|
async def get_direct_video_url(self, video_url: str, bvid: str) -> Optional[str]:
|
||||||
"""
|
"""
|
||||||
调用第三方API解析B站视频直链
|
获取B站视频直链(通过本地文件服务器下载)
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
video_url (str): B站视频的完整URL
|
video_url (str): B站视频的完整URL
|
||||||
|
bvid (str): BV号
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Optional[str]: 视频直链URL,如果失败则返回None
|
Optional[str]: 本地视频 URL,如果失败则返回None
|
||||||
"""
|
"""
|
||||||
api_url = f"https://api.mir6.com/api/bzjiexi?url={video_url}&type=json"
|
if not BILI_API_AVAILABLE:
|
||||||
|
return None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
async with aiohttp.ClientSession() as session:
|
credential = self._get_credential()
|
||||||
async with session.get(api_url, headers=self.HEADERS, timeout=aiohttp.ClientTimeout(total=10)) as response:
|
v = video.Video(bvid=bvid, credential=credential)
|
||||||
response.raise_for_status()
|
# 先获取视频信息以获取 cid
|
||||||
# 使用 content_type=None 来忽略 Content-Type 检查
|
info = await v.get_info()
|
||||||
data = await response.json(content_type=None)
|
cid = info.get('cid', 0)
|
||||||
if data.get("code") == 200 and data.get("data"):
|
|
||||||
return data["data"][0].get("video_url")
|
if not cid:
|
||||||
except (aiohttp.ClientError, ValueError, KeyError, IndexError) as e:
|
return None
|
||||||
logger.error(f"[{self.name}] 调用第三方API解析视频失败: {e}")
|
|
||||||
|
# 获取下载链接数据
|
||||||
|
download_url_data = await v.get_download_url(cid=cid)
|
||||||
|
|
||||||
|
# 使用 VideoDownloadURLDataDetecter 解析数据
|
||||||
|
detecter = video.VideoDownloadURLDataDetecter(data=download_url_data)
|
||||||
|
streams = detecter.detect_best_streams()
|
||||||
|
|
||||||
|
if streams:
|
||||||
|
# 获取视频直链
|
||||||
|
video_direct_url = streams[0].url
|
||||||
|
logger.info(f"[{self.name}] 获取到视频直链,开始下载到本地...")
|
||||||
|
|
||||||
|
# 使用本地文件服务器下载
|
||||||
|
local_url = await download_to_local(video_direct_url, timeout=120)
|
||||||
|
|
||||||
|
if local_url:
|
||||||
|
logger.success(f"[{self.name}] 视频已下载到本地: {local_url}")
|
||||||
|
return local_url
|
||||||
|
else:
|
||||||
|
logger.error(f"[{self.name}] 下载到本地失败")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"[{self.name}] 获取视频直链失败: {e}")
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def format_response(self, event: MessageEvent, data: Dict[str, Any]) -> List[Any]:
|
async def format_response(self, event: MessageEvent, data: Dict[str, Any]) -> List[Any]:
|
||||||
@@ -204,7 +319,8 @@ class BiliParser(BaseParser):
|
|||||||
else:
|
else:
|
||||||
# 构建完整的B站视频URL
|
# 构建完整的B站视频URL
|
||||||
video_url = f"https://www.bilibili.com/video/{data.get('bvid', '')}"
|
video_url = f"https://www.bilibili.com/video/{data.get('bvid', '')}"
|
||||||
direct_url = await self.get_direct_video_url(video_url)
|
bvid = data.get('bvid', '')
|
||||||
|
direct_url = await self.get_direct_video_url(video_url, bvid)
|
||||||
if direct_url:
|
if direct_url:
|
||||||
video_message = MessageSegment.video(direct_url)
|
video_message = MessageSegment.video(direct_url)
|
||||||
else:
|
else:
|
||||||
@@ -226,6 +342,7 @@ class BiliParser(BaseParser):
|
|||||||
f" 投币: {self.format_count(data['coin'])}\n"
|
f" 投币: {self.format_count(data['coin'])}\n"
|
||||||
f" 收藏: {self.format_count(data['favorite'])}\n"
|
f" 收藏: {self.format_count(data['favorite'])}\n"
|
||||||
f" 转发: {self.format_count(data['share'])}\n"
|
f" 转发: {self.format_count(data['share'])}\n"
|
||||||
|
f" 弹幕: {self.format_count(data.get('danmaku', 0))}\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
image_message_segment = [
|
image_message_segment = [
|
||||||
@@ -264,5 +381,4 @@ class BiliParser(BaseParser):
|
|||||||
Returns:
|
Returns:
|
||||||
bool: 是否应该处理
|
bool: 是否应该处理
|
||||||
"""
|
"""
|
||||||
# 检查是否是B站相关域名,包括短链接
|
|
||||||
return bool(self.url_pattern.search(url))
|
return bool(self.url_pattern.search(url))
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ anyio==4.12.1
|
|||||||
astroid==4.0.3
|
astroid==4.0.3
|
||||||
attrs==25.4.0
|
attrs==25.4.0
|
||||||
beautifulsoup4==4.14.3
|
beautifulsoup4==4.14.3
|
||||||
|
bilibili-api-python==2024.12.1
|
||||||
bs4==0.0.2
|
bs4==0.0.2
|
||||||
cachetools==6.2.4
|
cachetools==6.2.4
|
||||||
certifi==2026.1.4
|
certifi==2026.1.4
|
||||||
|
|||||||
135
tests/test_thread_manager.py
Normal file
135
tests/test_thread_manager.py
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
"""
|
||||||
|
线程管理器测试模块
|
||||||
|
|
||||||
|
测试多线程功能的正确性,包括:
|
||||||
|
1. 线程池的创建和管理
|
||||||
|
2. 任务提交和执行
|
||||||
|
3. 线程安全的统计信息
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
import threading
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.managers.thread_manager import thread_manager, ThreadManager
|
||||||
|
|
||||||
|
|
||||||
|
class TestThreadManager:
|
||||||
|
"""线程管理器测试类"""
|
||||||
|
|
||||||
|
def test_singleton(self):
|
||||||
|
"""测试单例模式"""
|
||||||
|
manager1 = ThreadManager()
|
||||||
|
manager2 = ThreadManager()
|
||||||
|
assert manager1 is manager2
|
||||||
|
|
||||||
|
def test_start_and_shutdown(self):
|
||||||
|
"""测试启动和关闭"""
|
||||||
|
manager = ThreadManager()
|
||||||
|
manager.start()
|
||||||
|
assert manager._executor is not None
|
||||||
|
|
||||||
|
# 提交一个简单任务
|
||||||
|
result = manager.submit_to_main_executor(lambda x: x * 2, 5)
|
||||||
|
assert result == 10
|
||||||
|
|
||||||
|
manager.shutdown()
|
||||||
|
assert manager._executor is None
|
||||||
|
|
||||||
|
def test_submit_to_main_executor(self):
|
||||||
|
"""测试提交任务到主线程池"""
|
||||||
|
manager = ThreadManager()
|
||||||
|
manager.start()
|
||||||
|
|
||||||
|
# 测试同步任务
|
||||||
|
result = manager.submit_to_main_executor(lambda x, y: x + y, 3, 4)
|
||||||
|
assert result == 7
|
||||||
|
|
||||||
|
# 测试异步任务
|
||||||
|
async def async_task(x):
|
||||||
|
await asyncio.sleep(0.1)
|
||||||
|
return x * 2
|
||||||
|
|
||||||
|
async def run_async():
|
||||||
|
return await manager.submit_to_main_executor_async(async_task, 5)
|
||||||
|
|
||||||
|
result = asyncio.run(run_async())
|
||||||
|
assert result == 10
|
||||||
|
|
||||||
|
manager.shutdown()
|
||||||
|
|
||||||
|
def test_thread_safety(self):
|
||||||
|
"""测试线程安全"""
|
||||||
|
manager = ThreadManager()
|
||||||
|
manager.start()
|
||||||
|
|
||||||
|
results = []
|
||||||
|
errors = []
|
||||||
|
|
||||||
|
def worker(n):
|
||||||
|
try:
|
||||||
|
time.sleep(0.01)
|
||||||
|
return n * n
|
||||||
|
except Exception as e:
|
||||||
|
errors.append(e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# 并发提交多个任务
|
||||||
|
futures = []
|
||||||
|
for i in range(10):
|
||||||
|
future = manager._executor.submit(worker, i)
|
||||||
|
futures.append(future)
|
||||||
|
|
||||||
|
# 收集结果
|
||||||
|
for future in futures:
|
||||||
|
result = future.result()
|
||||||
|
results.append(result)
|
||||||
|
|
||||||
|
# 验证所有任务都成功执行
|
||||||
|
assert len(errors) == 0
|
||||||
|
assert len(results) == 10
|
||||||
|
assert sorted(results) == [0, 1, 4, 9, 16, 25, 36, 49, 64, 81]
|
||||||
|
|
||||||
|
manager.shutdown()
|
||||||
|
|
||||||
|
def test_stats_tracking(self):
|
||||||
|
"""测试统计信息"""
|
||||||
|
manager = ThreadManager()
|
||||||
|
manager.start()
|
||||||
|
|
||||||
|
# 执行一些任务
|
||||||
|
for i in range(5):
|
||||||
|
manager.submit_to_main_executor(lambda x: x, i)
|
||||||
|
|
||||||
|
stats = manager.get_stats()
|
||||||
|
assert stats['total_tasks'] >= 5
|
||||||
|
|
||||||
|
manager.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
class TestReverseWSManagerThreading:
|
||||||
|
"""反向 WebSocket 管理器线程安全测试"""
|
||||||
|
|
||||||
|
def test_locks_exist(self):
|
||||||
|
"""测试锁是否正确初始化"""
|
||||||
|
from core.managers.reverse_ws_manager import ReverseWSManager
|
||||||
|
|
||||||
|
manager = ReverseWSManager()
|
||||||
|
|
||||||
|
# 检查所有锁是否存在
|
||||||
|
assert hasattr(manager, '_clients_lock')
|
||||||
|
assert hasattr(manager, '_bots_lock')
|
||||||
|
assert hasattr(manager, '_pending_requests_lock')
|
||||||
|
assert hasattr(manager, '_load_lock')
|
||||||
|
assert hasattr(manager, '_health_lock')
|
||||||
|
assert hasattr(manager, '_processed_events_lock')
|
||||||
|
assert hasattr(manager, '_processed_messages_lock')
|
||||||
|
assert hasattr(manager, '_processing_events_lock')
|
||||||
|
assert hasattr(manager, '_message_locks_lock')
|
||||||
|
assert hasattr(manager, '_message_lock_times_lock')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
pytest.main([__file__, '-v'])
|
||||||
205
web_static/changelog.html
Normal file
205
web_static/changelog.html
Normal file
@@ -0,0 +1,205 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="zh-CN" class="scroll-smooth">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>NEOBOT | Changelog</title>
|
||||||
|
<script src="https://cdn.tailwindcss.com"></script>
|
||||||
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||||
|
<link href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@300;500;700&family=Inter:wght@300;400;600&family=Noto+Serif+SC:wght@300;400;700&family=Cormorant+Garamond:ital,wght@0,400;1,400&display=swap" rel="stylesheet">
|
||||||
|
<script src="https://code.iconify.design/iconify-icon/1.0.7/iconify-icon.min.js"></script>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
tailwind.config = {
|
||||||
|
theme: {
|
||||||
|
extend: {
|
||||||
|
fontFamily: {
|
||||||
|
sans: ['"Inter"', 'sans-serif'],
|
||||||
|
display: ['"Space Grotesk"', 'sans-serif'],
|
||||||
|
serif: ['"Noto Serif SC"', 'serif'],
|
||||||
|
lyric: ['"Cormorant Garamond"', 'serif'],
|
||||||
|
},
|
||||||
|
colors: {
|
||||||
|
brand: {
|
||||||
|
bg: '#050505',
|
||||||
|
surface: '#121212',
|
||||||
|
border: '#27272a',
|
||||||
|
text: '#e4e4e7',
|
||||||
|
muted: '#a1a1aa',
|
||||||
|
}
|
||||||
|
},
|
||||||
|
animation: {
|
||||||
|
'fade-in-up': 'fadeInUp 1s cubic-bezier(0.16, 1, 0.3, 1) forwards',
|
||||||
|
'pulse-slow': 'pulse 4s cubic-bezier(0.4, 0, 0.6, 1) infinite',
|
||||||
|
},
|
||||||
|
keyframes: {
|
||||||
|
fadeInUp: {
|
||||||
|
'0%': { opacity: '0', transform: 'translateY(20px)' },
|
||||||
|
'100%': { opacity: '1', transform: 'translateY(0)' },
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
<style>
|
||||||
|
body {
|
||||||
|
background-color: #050505;
|
||||||
|
color: #e4e4e7;
|
||||||
|
background-image: radial-gradient(circle at 50% 0%, #1a1a1a 0%, #050505 60%);
|
||||||
|
background-attachment: fixed;
|
||||||
|
}
|
||||||
|
|
||||||
|
.changelog-card {
|
||||||
|
background: rgba(18, 18, 18, 0.6);
|
||||||
|
backdrop-filter: blur(12px);
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||||
|
border-radius: 8px;
|
||||||
|
transition: all 0.3s ease;
|
||||||
|
}
|
||||||
|
.changelog-card:hover {
|
||||||
|
border-color: rgba(255, 255, 255, 0.2);
|
||||||
|
background: rgba(30, 30, 30, 0.8);
|
||||||
|
transform: translateY(-2px);
|
||||||
|
}
|
||||||
|
|
||||||
|
.lyric-text {
|
||||||
|
font-family: "Cormorant Garamond", serif;
|
||||||
|
font-style: italic;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Timeline line */
|
||||||
|
.timeline-line {
|
||||||
|
position: absolute;
|
||||||
|
left: 24px;
|
||||||
|
top: 0;
|
||||||
|
bottom: 0;
|
||||||
|
width: 1px;
|
||||||
|
background: linear-gradient(to bottom, rgba(255,255,255,0.1), rgba(255,255,255,0.05));
|
||||||
|
}
|
||||||
|
|
||||||
|
::-webkit-scrollbar { width: 6px; }
|
||||||
|
::-webkit-scrollbar-track { background: #050505; }
|
||||||
|
::-webkit-scrollbar-thumb { background: #333; border-radius: 3px; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body class="antialiased selection:bg-white/20 selection:text-white">
|
||||||
|
|
||||||
|
<!-- 导航 -->
|
||||||
|
<nav class="fixed top-0 w-full z-50 border-b border-white/5 bg-black/80 backdrop-blur-md">
|
||||||
|
<div class="max-w-6xl mx-auto px-6 h-20 flex items-center justify-between">
|
||||||
|
<div class="flex items-center gap-3">
|
||||||
|
<a href="../index.html" class="flex items-center gap-3 hover:opacity-80 transition-opacity">
|
||||||
|
<div class="w-2 h-2 bg-white rounded-full animate-pulse-slow"></div>
|
||||||
|
<span class="font-display font-bold text-sm tracking-widest text-white">NEO<span class="text-white/40 font-light">BOT</span></span>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex items-center gap-4 text-[10px] font-mono text-gray-400 uppercase tracking-widest">
|
||||||
|
<span class="px-2 py-1 rounded border border-white/10 bg-white/5">Changelog</span>
|
||||||
|
<span>Latest: v1.0.1</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<main class="pt-40 pb-32 px-6">
|
||||||
|
<div class="max-w-4xl mx-auto space-y-16">
|
||||||
|
|
||||||
|
<!-- Header -->
|
||||||
|
<section class="text-center space-y-4 animate-fade-in-up">
|
||||||
|
<div class="font-mono text-xs text-gray-500 mb-2">PROJECT HISTORY</div>
|
||||||
|
<h1 class="text-4xl md:text-6xl font-display font-bold text-white leading-tight">
|
||||||
|
System<br>
|
||||||
|
<span class="text-white/30">Evolution</span>
|
||||||
|
</h1>
|
||||||
|
<p class="font-serif text-lg text-gray-400 max-w-2xl mx-auto">
|
||||||
|
记录每一次微小的改变,见证成长的轨迹。
|
||||||
|
</p>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<!-- Changelog Card -->
|
||||||
|
<section class="max-w-2xl mx-auto">
|
||||||
|
<div class="changelog-card p-8 md:p-10 relative overflow-hidden group">
|
||||||
|
<!-- Decorative background glow -->
|
||||||
|
<div class="absolute top-0 right-0 -mr-16 -mt-16 w-64 h-64 bg-white/5 rounded-full blur-3xl group-hover:bg-white/10 transition-colors duration-500"></div>
|
||||||
|
|
||||||
|
<!-- Version & Date -->
|
||||||
|
<div class="relative z-10 flex flex-col md:flex-row md:items-end justify-between gap-4 mb-8 border-b border-white/10 pb-6">
|
||||||
|
<div>
|
||||||
|
<div class="flex items-center gap-3 mb-2">
|
||||||
|
<h2 class="font-display text-4xl text-white font-bold">v1.0.1</h2>
|
||||||
|
<span class="px-2 py-0.5 rounded text-[10px] font-mono font-bold bg-white/10 text-white/60 border border-white/10">LATEST</span>
|
||||||
|
</div>
|
||||||
|
<div class="font-mono text-xs text-gray-500">2026-3-1</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="md:text-right max-w-xs">
|
||||||
|
<p class="font-serif text-sm text-gray-400 italic leading-relaxed">
|
||||||
|
"后端修正。"
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Changes List -->
|
||||||
|
<div class="relative z-10">
|
||||||
|
<ul class="space-y-4">
|
||||||
|
|
||||||
|
<li class="flex items-start gap-4 group/item">
|
||||||
|
|
||||||
|
<span class="flex-shrink-0 mt-1 px-2 py-1 rounded text-[10px] font-mono font-bold bg-green-500/10 text-green-400 border border-green-500/20 group-hover/item:bg-green-500/20 transition-colors">ADD</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="text-base text-gray-300 leading-relaxed group-hover/item:text-white transition-colors">天气查询功能美化</span>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
<li class="flex items-start gap-4 group/item">
|
||||||
|
|
||||||
|
<span class="flex-shrink-0 mt-1 px-2 py-1 rounded text-[10px] font-mono font-bold bg-red-500/10 text-red-400 border border-red-500/20 group-hover/item:bg-red-500/20 transition-colors">FIX</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="text-base text-gray-300 leading-relaxed group-hover/item:text-white transition-colors">b站的视频解析已修复,感谢Nemo2011的bilibili-api python库,采用GPL3.0开源</span>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
<li class="flex items-start gap-4 group/item">
|
||||||
|
|
||||||
|
<span class="flex-shrink-0 mt-1 px-2 py-1 rounded text-[10px] font-mono font-bold bg-green-500/10 text-green-400 border border-green-500/20 group-hover/item:bg-green-500/20 transition-colors">ADD</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="text-base text-gray-300 leading-relaxed group-hover/item:text-white transition-colors">python3.14的自由线程测试已开启</span>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
<li class="flex items-start gap-4 group/item">
|
||||||
|
|
||||||
|
<span class="flex-shrink-0 mt-1 px-2 py-1 rounded text-[10px] font-mono font-bold bg-blue-500/10 text-blue-400 border border-blue-500/20 group-hover/item:bg-blue-500/20 transition-colors">UPD</span>
|
||||||
|
|
||||||
|
|
||||||
|
<span class="text-base text-gray-300 leading-relaxed group-hover/item:text-white transition-colors">镜像图片功能现已可以转换动态表情包</span>
|
||||||
|
</li>
|
||||||
|
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</main>
|
||||||
|
|
||||||
|
<footer class="py-12 border-t border-white/5 bg-black/20">
|
||||||
|
<div class="max-w-6xl mx-auto px-6 flex flex-col md:flex-row justify-between items-center gap-6">
|
||||||
|
<div class="text-center md:text-left">
|
||||||
|
<div class="font-display font-bold text-white mb-1">NEOBOT</div>
|
||||||
|
<p class="font-mono text-[10px] text-gray-600">
|
||||||
|
PRIVATE PERSONAL PROJECT<br>
|
||||||
|
GENERATED BY CHANGELOG TOOL
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div class="font-mono text-[10px] text-gray-600 text-center md:text-right">
|
||||||
|
TO ASTEROID B-612<br>
|
||||||
|
SASAKURE.UK
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</footer>
|
||||||
|
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -20,14 +20,14 @@ OUTPUT_FILE = "../changelog.html"
|
|||||||
# - content: 变更内容
|
# - content: 变更内容
|
||||||
changelogs = [
|
changelogs = [
|
||||||
{
|
{
|
||||||
"version": "v1.0.0",
|
"version": "v1.0.1",
|
||||||
"date": "2026-3-1",
|
"date": "2026-3-1",
|
||||||
"description": "引入了更多有趣的互动功能,并优化了系统稳定性。",
|
"description": "后端修正。",
|
||||||
"changes": [
|
"changes": [
|
||||||
{"type": "add", "content": "新增了天气查询功能,支持全国主要城市"},
|
{"type": "add", "content": "天气查询功能美化"},
|
||||||
{"type": "update", "content": "优化了 Web Parser 的解析速度,不过b站的视频解析等待重做中"},
|
{"type": "fix", "content": "b站的视频解析已修复,感谢Nemo2011的bilibili-api python库,采用GPL3.0开源"},
|
||||||
{"type": "fix", "content": "修复了在某些特定网络环境下图片加载失败的问题"},
|
{"type": "add", "content": "python3.14的自由线程测试已开启"},
|
||||||
{"type": "update", "content": "支持多实现端连接(反向WS),此功能并不完善,等待重做"}
|
{"type": "update", "content": "镜像图片功能现已可以转换动态表情包"}
|
||||||
|
|
||||||
|
|
||||||
]
|
]
|
||||||
|
|||||||
Reference in New Issue
Block a user