feat: 添加性能优化和架构文档,更新依赖和核心模块

refactor(browser_manager): 实现页面池机制以提升性能
refactor(image_manager): 添加模板缓存并集成页面池
refactor(bili_parser): 迁移到异步HTTP请求并实现会话复用
docs: 新增性能优化、架构设计和最佳实践文档
chore: 更新requirements.txt添加新依赖
This commit is contained in:
2026-01-13 03:56:31 +08:00
parent 5996f6eeaf
commit 24af862924
18 changed files with 589 additions and 489 deletions

View File

@@ -15,6 +15,8 @@ class BrowserManager:
_instance = None
_playwright: Optional[Playwright] = None
_browser: Optional[Browser] = None
_page_pool: Optional[asyncio.Queue] = None
_pool_size: int = 3
def __new__(cls):
if cls._instance is None:
@@ -36,6 +38,73 @@ class BrowserManager:
logger.exception(f"无头浏览器启动失败: {e}")
self._browser = None
async def init_pool(self, size: int = 3):
"""
初始化页面池
"""
if not self._browser:
await self.initialize()
if not self._browser:
logger.error("浏览器初始化失败,无法创建页面池")
return
self._pool_size = size
self._page_pool = asyncio.Queue(maxsize=size)
logger.info(f"正在初始化页面池 (大小: {size})...")
for i in range(size):
try:
page = await self._browser.new_page()
await self._page_pool.put(page)
except Exception as e:
logger.error(f"创建页面池页面 {i+1} 失败: {e}")
logger.success(f"页面池初始化完成,当前可用页面: {self._page_pool.qsize()}")
async def get_page(self) -> Optional[Page]:
"""
从池中获取一个页面。如果池未初始化或为空,则尝试创建一个新页面(不入池)。
"""
if self._page_pool and not self._page_pool.empty():
try:
page = self._page_pool.get_nowait()
# 简单的健康检查
if page.is_closed():
logger.warning("检测到池中页面已关闭,重新创建一个...")
if self._browser:
page = await self._browser.new_page()
else:
return None
return page
except asyncio.QueueEmpty:
pass
# 如果池空了或者没初始化,回退到临时创建
logger.debug("页面池为空或未初始化,创建临时页面")
return await self.get_new_page()
async def release_page(self, page: Page):
"""
归还页面到池中。如果池已满或未初始化,则关闭页面。
"""
if not page or page.is_closed():
return
if self._page_pool:
try:
# 重置页面状态 (例如清空内容),防止数据污染
# 注意: goto('about:blank') 比 close() 快得多
await page.goto("about:blank")
self._page_pool.put_nowait(page)
return
except asyncio.QueueFull:
pass
# 池满或未启用池,直接关闭
await page.close()
async def get_new_page(self) -> Optional[Page]:
"""
获取一个新的页面 (Page)
@@ -58,6 +127,16 @@ class BrowserManager:
"""
关闭浏览器和 Playwright
"""
# 清空页面池
if self._page_pool:
while not self._page_pool.empty():
try:
page = self._page_pool.get_nowait()
await page.close()
except Exception:
pass
self._page_pool = None
if self._browser:
await self._browser.close()
self._browser = None