|
|
@@ -954,7 +954,103 @@ async def browser_find_text(text: str) -> ToolResult:
|
|
|
long_term_memory=f"查找文本 '{text}' 失败"
|
|
|
)
|
|
|
|
|
|
+@tool()
|
|
|
+async def browser_get_visual_selector_map() -> ToolResult:
|
|
|
+ """
|
|
|
+ 获取当前页面的视觉快照和交互元素索引映射。
|
|
|
+ Get visual snapshot and selector map of interactive elements.
|
|
|
+
|
|
|
+ 该工具会同时执行两个操作：
|
|
|
+ 1. 捕捉当前页面的截图，并用 browser-use 内置方法在截图上标注元素索引号。
|
|
|
+ 2. 生成页面所有可交互元素的索引字典（含 href、type 等属性信息）。
|
|
|
+
|
|
|
+ Returns:
|
|
|
+ ToolResult: 包含高亮截图（在 images 中）和元素列表的工具返回对象。
|
|
|
+ """
|
|
|
+ try:
|
|
|
+ browser, _ = await get_browser_session()
|
|
|
+
|
|
|
+ # 1. 构造同时包含 DOM 和 截图 的请求
|
|
|
+ from browser_use.browser.events import BrowserStateRequestEvent
|
|
|
+ from browser_use.browser.python_highlights import create_highlighted_screenshot_async
|
|
|
+
|
|
|
+ event = browser.event_bus.dispatch(
|
|
|
+ BrowserStateRequestEvent(
|
|
|
+ include_dom=True,
|
|
|
+ include_screenshot=True,
|
|
|
+ include_recent_events=False
|
|
|
+ )
|
|
|
+ )
|
|
|
+
|
|
|
+ # 2. 等待浏览器返回完整状态
|
|
|
+ browser_state = await event.event_result(raise_if_none=True, raise_if_any=True)
|
|
|
+
|
|
|
+ # 3. 提取 Selector Map
|
|
|
+ selector_map = browser_state.dom_state.selector_map if browser_state.dom_state else {}
|
|
|
+
|
|
|
+ # 4. 提取截图并生成带索引标注的高亮截图（通过 CDP 获取精确 DPI 和滚动偏移）
|
|
|
+ screenshot_b64 = browser_state.screenshot or ""
|
|
|
+ highlighted_b64 = ""
|
|
|
+ if screenshot_b64 and selector_map:
|
|
|
+ try:
|
|
|
+ cdp_session = await browser.get_or_create_cdp_session()
|
|
|
+ highlighted_b64 = await create_highlighted_screenshot_async(
|
|
|
+ screenshot_b64, selector_map,
|
|
|
+ cdp_session=cdp_session,
|
|
|
+ filter_highlight_ids=False
|
|
|
+ )
|
|
|
+ except Exception:
|
|
|
+ highlighted_b64 = screenshot_b64 # fallback to raw screenshot
|
|
|
+ else:
|
|
|
+ highlighted_b64 = screenshot_b64
|
|
|
|
|
|
+ # 5. 构建供 Agent 阅读的完整元素列表，包含丰富的属性信息
|
|
|
+ elements_info = []
|
|
|
+ for index, node in selector_map.items():
|
|
|
+ tag = node.tag_name
|
|
|
+ attrs = node.attributes or {}
|
|
|
+ desc = attrs.get('aria-label') or attrs.get('placeholder') or attrs.get('title') or node.get_all_children_text(max_depth=1) or ""
|
|
|
+ # 收集有用的属性片段
|
|
|
+ extra_parts = []
|
|
|
+ if attrs.get('href'):
|
|
|
+ extra_parts.append(f"href={attrs['href'][:60]}")
|
|
|
+ if attrs.get('type'):
|
|
|
+ extra_parts.append(f"type={attrs['type']}")
|
|
|
+ if attrs.get('role'):
|
|
|
+ extra_parts.append(f"role={attrs['role']}")
|
|
|
+ if attrs.get('name'):
|
|
|
+ extra_parts.append(f"name={attrs['name']}")
|
|
|
+ extra = f" ({', '.join(extra_parts)})" if extra_parts else ""
|
|
|
+ elements_info.append(f"Index {index}: <{tag}> \"{desc[:50]}\"{extra}")
|
|
|
+
|
|
|
+ output = f"页面截图已捕获（含元素索引标注）\n找到 {len(selector_map)} 个交互元素\n\n"
|
|
|
+ output += "元素列表:\n" + "\n".join(elements_info)
|
|
|
+
|
|
|
+ # 6. 将高亮截图存入 images 字段，metadata 保留结构化数据
|
|
|
+ images = []
|
|
|
+ if highlighted_b64:
|
|
|
+ images.append({"type": "base64", "media_type": "image/png", "data": highlighted_b64})
|
|
|
+
|
|
|
+ return ToolResult(
|
|
|
+ title="视觉元素观察",
|
|
|
+ output=output,
|
|
|
+ long_term_memory=f"在页面观察到 {len(selector_map)} 个元素并保存了截图",
|
|
|
+ images=images,
|
|
|
+ metadata={
|
|
|
+ "selector_map": {k: str(v) for k, v in list(selector_map.items())[:100]},
|
|
|
+ "url": browser_state.url,
|
|
|
+ "title": browser_state.title
|
|
|
+ }
|
|
|
+ )
|
|
|
+
|
|
|
+ except Exception as e:
|
|
|
+ return ToolResult(
|
|
|
+ title="视觉观察失败",
|
|
|
+ output="",
|
|
|
+ error=f"Failed to get visual selector map: {str(e)}",
|
|
|
+ long_term_memory="获取视觉元素映射失败"
|
|
|
+ )
|
|
|
+
|
|
|
@tool()
|
|
|
async def browser_screenshot() -> ToolResult:
|
|
|
"""
|
|
|
@@ -1787,6 +1883,116 @@ async def browser_done(text: str, success: bool = True,
|
|
|
)
|
|
|
|
|
|
|
|
|
+# ============================================================
|
|
|
+# Cookie 持久化工具
|
|
|
+# ============================================================
|
|
|
+
|
|
|
+_COOKIES_DIR = Path(__file__).parent.parent.parent.parent / ".cookies"
|
|
|
+
|
|
|
+@tool()
|
|
|
+async def browser_export_cookies(name: str = "") -> ToolResult:
|
|
|
+ """
|
|
|
+ 导出当前浏览器的所有 Cookie 到本地 JSON 文件。
|
|
|
+ Export all browser cookies to a local JSON file.
|
|
|
+
|
|
|
+ 登录成功后调用此工具，下次启动时可通过 browser_load_cookies 恢复登录态。
|
|
|
+
|
|
|
+ Args:
|
|
|
+ name: 保存名称（可选，默认用当前域名）
|
|
|
+
|
|
|
+ Returns:
|
|
|
+ ToolResult: 导出结果
|
|
|
+ """
|
|
|
+ try:
|
|
|
+ browser, _ = await get_browser_session()
|
|
|
+ cookies = await browser._cdp_get_cookies()
|
|
|
+
|
|
|
+ if not cookies:
|
|
|
+ return ToolResult(
|
|
|
+ title="Cookie 导出",
|
|
|
+ output="当前浏览器没有 Cookie",
|
|
|
+ long_term_memory="导出 Cookie 失败：无 Cookie"
|
|
|
+ )
|
|
|
+
|
|
|
+ # 确定文件名
|
|
|
+ if not name:
|
|
|
+ url = getattr(browser, 'current_url', '') or ''
|
|
|
+ from urllib.parse import urlparse
|
|
|
+ parsed = urlparse(url)
|
|
|
+ name = parsed.netloc.replace("www.", "") or "default"
|
|
|
+
|
|
|
+ _COOKIES_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
+ cookie_file = _COOKIES_DIR / f"{name}.json"
|
|
|
+
|
|
|
+ cookie_file.write_text(
|
|
|
+ json.dumps(cookies, ensure_ascii=False, indent=2),
|
|
|
+ encoding="utf-8"
|
|
|
+ )
|
|
|
+
|
|
|
+ return ToolResult(
|
|
|
+ title="Cookie 已导出",
|
|
|
+ output=f"已保存 {len(cookies)} 条 Cookie 到 {cookie_file.name}",
|
|
|
+ long_term_memory=f"导出 {len(cookies)} 条 Cookie 到 .cookies/{cookie_file.name}"
|
|
|
+ )
|
|
|
+
|
|
|
+ except Exception as e:
|
|
|
+ return ToolResult(
|
|
|
+ title="Cookie 导出失败",
|
|
|
+ output="",
|
|
|
+ error=f"Failed to export cookies: {str(e)}",
|
|
|
+ long_term_memory="导出 Cookie 失败"
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+@tool()
|
|
|
+async def browser_load_cookies(name: str, url: str = "") -> ToolResult:
|
|
|
+ """
|
|
|
+ 从本地 JSON 文件加载 Cookie 到浏览器，恢复登录态。
|
|
|
+ Load cookies from a local JSON file into the browser.
|
|
|
+
|
|
|
+ Args:
|
|
|
+ name: Cookie 文件名（不含 .json 后缀）
|
|
|
+ url: 加载后导航到的 URL（可选）
|
|
|
+
|
|
|
+ Returns:
|
|
|
+ ToolResult: 加载结果
|
|
|
+ """
|
|
|
+ try:
|
|
|
+ browser, tools = await get_browser_session()
|
|
|
+
|
|
|
+ cookie_file = _COOKIES_DIR / f"{name}.json"
|
|
|
+ if not cookie_file.exists():
|
|
|
+ # 列出可用文件
|
|
|
+ available = [f.stem for f in _COOKIES_DIR.glob("*.json")] if _COOKIES_DIR.exists() else []
|
|
|
+ return ToolResult(
|
|
|
+ title="Cookie 文件不存在",
|
|
|
+ output=f"未找到 .cookies/{name}.json，可用: {available}",
|
|
|
+ error=f"Cookie file not found: {name}.json",
|
|
|
+ long_term_memory=f"Cookie 文件 {name}.json 不存在"
|
|
|
+ )
|
|
|
+
|
|
|
+ cookies = json.loads(cookie_file.read_text(encoding="utf-8"))
|
|
|
+ await browser._cdp_set_cookies(cookies)
|
|
|
+
|
|
|
+ if url:
|
|
|
+ await tools.navigate(url=url, browser_session=browser)
|
|
|
+ await tools.wait(seconds=2, browser_session=browser)
|
|
|
+
|
|
|
+ return ToolResult(
|
|
|
+ title="Cookie 已加载",
|
|
|
+ output=f"已注入 {len(cookies)} 条 Cookie（来自 {name}.json）",
|
|
|
+ long_term_memory=f"从 .cookies/{name}.json 加载了 {len(cookies)} 条 Cookie"
|
|
|
+ )
|
|
|
+
|
|
|
+ except Exception as e:
|
|
|
+ return ToolResult(
|
|
|
+ title="Cookie 加载失败",
|
|
|
+ output="",
|
|
|
+ error=f"Failed to load cookies: {str(e)}",
|
|
|
+ long_term_memory="加载 Cookie 失败"
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
# ============================================================
|
|
|
# 导出所有工具函数（供外部使用）
|
|
|
# ============================================================
|
|
|
@@ -1828,6 +2034,7 @@ __all__ = [
|
|
|
'browser_get_page_html',
|
|
|
'browser_read_long_content',
|
|
|
'browser_get_selector_map',
|
|
|
+ 'browser_get_visual_selector_map',
|
|
|
|
|
|
# JavaScript 执行工具
|
|
|
'browser_evaluate',
|
|
|
@@ -1838,4 +2045,8 @@ __all__ = [
|
|
|
|
|
|
# 任务完成
|
|
|
'browser_done',
|
|
|
+
|
|
|
+ # Cookie 持久化
|
|
|
+ 'browser_export_cookies',
|
|
|
+ 'browser_load_cookies',
|
|
|
]
|
guantao