ソースを参照

补齐五层分层验收主路径

SamLee 1 日 前
コミット
b546b98533
1 ファイル変更184 行追加18 行削除
  1. 184 18
      tests/test_recursive_context_integration.py

+ 184 - 18
tests/test_recursive_context_integration.py

@@ -19,8 +19,11 @@ from cyber_agent.trace.store import FileSystemTraceStore
 
 
 ROOT_ANCHOR = {
-    "objective": "通过五层局部分析得出可追溯的根结论",
-    "completion_criteria": ["五层任务都经过直接父级审核", "根结果通过独立验收"],
+    "objective": "发布只使用官方数字的建议",
+    "completion_criteria": [
+        "最终建议采用官方12%",
+        "明确30%仅见于转载且无官方支持",
+    ],
     "constraints": ["不得编造证据"],
 }
 
@@ -34,7 +37,8 @@ def recursive_env():
         "AGENT_MAX_TOTAL_TOKENS": "100000",
         "AGENT_MAX_TOTAL_COST_USD": "10",
         "AGENT_MAX_DURATION_SECONDS": "600",
-        "AGENT_RESERVED_FINAL_CALLS": "1",
+        "AGENT_RESERVED_FINAL_CALLS": "8",
+        "AGENT_VALIDATOR_SEARCH_PROVIDER": "serper",
     }, clear=False)
 
 
@@ -136,6 +140,39 @@ class FiveLevelRecursiveContextTest(unittest.IsolatedAsyncioTestCase):
             read_depths = []
             observed_tools = {}
             validator_packets = []
+            validator_packet_keys = set()
+            search_queries = []
+
+            class SearchProvider:
+                async def search(self, query, max_results):
+                    search_queries.append((query, max_results))
+                    return [
+                        {
+                            "title": "转载页面",
+                            "link": "https://repost.example/30-percent",
+                            "snippet": "未经核实的30%",
+                        },
+                        {
+                            "title": "官方页面",
+                            "link": "https://official.example/12-percent",
+                            "snippet": "官方数字12%",
+                        },
+                    ][:max_results]
+
+            async def page_fetcher(url, resolver=None):
+                del resolver
+                return {
+                    "source_id": "src-official-12",
+                    "url": url,
+                    "final_url": url,
+                    "title": "官方页面",
+                    "content_type": "text/plain",
+                    "retrieved_at": "2026-07-17T00:00:00+00:00",
+                    "content_sha256": "c" * 64,
+                    "text": "官方数字是12%,不支持30%。",
+                    "truncated": False,
+                    "untrusted_material": True,
+                }
 
             async def fake_llm(**kwargs):
                 nonlocal call_count, token_total
@@ -147,17 +184,89 @@ class FiveLevelRecursiveContextTest(unittest.IsolatedAsyncioTestCase):
 
                 if any(
                     message.get("role") == "system"
-                    and "independent validator" in str(message.get("content", ""))
+                    and "recursive_validation_protocol" in str(message.get("content", ""))
                     for message in messages
                 ):
-                    packet = json.loads(messages[-1]["content"])
-                    validator_packets.append(packet)
+                    packet = next(
+                        json.loads(message["content"])
+                        for message in messages
+                        if message.get("role") == "user"
+                        and "recursive_validation_protocol" in str(message.get("content", ""))
+                    )
+                    scope = packet["validation_scope"]
+                    packet_key = (packet["validation_plan"]["plan_hash"], scope)
+                    if packet_key not in validator_packet_keys:
+                        validator_packet_keys.add(packet_key)
+                        validator_packets.append(packet)
+                    if scope == "root":
+                        self.assertIn("官方12%", packet["candidate_output"])
+                        self.assertIn("30%仅见于转载", packet["candidate_output"])
+                    else:
+                        brief = packet["task_brief"]
+                        local_depth = brief["context"]["local_depth"]
+                        report_text = json.dumps(
+                            packet["task_report"], ensure_ascii=False,
+                        )
+                        self.assertIn(
+                            self._semantic_result(local_depth),
+                            report_text,
+                        )
+                        self.assertIn(
+                            scope,
+                            [*brief["validation_scopes"], "task"],
+                        )
+                        task_criteria = [
+                            item["criterion"]
+                            for item in packet["validation_plan"]["checks"]
+                            if item["check_id"].startswith("task.criterion.")
+                        ]
+                        self.assertEqual(
+                            brief["completion_criteria"],
+                            task_criteria,
+                        )
+                        self.assertTrue(
+                            set(ROOT_ANCHOR["completion_criteria"]).isdisjoint(
+                                task_criteria
+                            )
+                        )
+                    validator_last_tool = last_tool_name(messages)
+                    if scope == "evidence" and validator_last_tool is None:
+                        response = tool_call(
+                            "validator_web_search",
+                            {"query": "官方数字 12% 30%", "max_results": 5},
+                            call_count,
+                        )
+                        token_total += 5
+                        return response
+                    if (
+                        scope == "evidence"
+                        and validator_last_tool == "validator_web_search"
+                    ):
+                        response = tool_call(
+                            "validator_open_url",
+                            {"url": "https://official.example/12-percent"},
+                            call_count,
+                        )
+                        token_total += 5
+                        return response
+                    evidence_refs = (
+                        ["src-official-12"] if scope == "evidence" else []
+                    )
                     response = {
                         "content": json.dumps({
                             "outcome": "passed",
-                            "scope": packet["validation_scope"],
+                            "scope": scope,
+                            "checks": [
+                                {
+                                    "check_id": item["check_id"],
+                                    "status": "passed",
+                                    "evidence_refs": evidence_refs,
+                                    "issue": None,
+                                }
+                                for item in packet["validation_plan"]["checks"]
+                                if item["scope"] == scope
+                            ],
                             "reason": "已核对持久化轨迹、标准和输出",
-                            "issues": [],
                             "retry_from": None,
                         }, ensure_ascii=False),
                         "tool_calls": [],
@@ -192,7 +301,10 @@ class FiveLevelRecursiveContextTest(unittest.IsolatedAsyncioTestCase):
                 elif last_tool == "review_task_result":
                     if depth == 0:
                         response = {
-                            "content": "根任务的五层结果已逐层审核完成",
+                            "content": (
+                                "最终建议采用官方12%;30%仅见于转载,"
+                                "没有官方支持,不能作为发布依据。"
+                            ),
                             "tool_calls": [],
                             "finish_reason": "stop",
                             "prompt_tokens": 3,
@@ -233,6 +345,8 @@ class FiveLevelRecursiveContextTest(unittest.IsolatedAsyncioTestCase):
                 trace_store=store,
                 tool_registry=get_tool_registry(),
                 llm_call=fake_llm,
+                validator_search_provider=SearchProvider(),
+                validator_page_fetcher=page_fetcher,
             )
             config = RunConfig(
                 tools=[
@@ -330,9 +444,14 @@ class FiveLevelRecursiveContextTest(unittest.IsolatedAsyncioTestCase):
                 trace for trace in traces
                 if trace.context.get("created_by_tool") == "validator"
             ]
-            self.assertEqual(6, len(validators))
-            self.assertEqual(6, len(validator_packets))
+            self.assertEqual(11, len(validators))
+            self.assertEqual(11, len(validator_packets))
             self.assertTrue(all(packet["root_task_anchor"] == ROOT_ANCHOR for packet in validator_packets))
+            self.assertEqual(2, len(search_queries))
+            self.assertEqual(
+                ["evidence", "hypothesis", "output", "root", "task"],
+                sorted({packet["validation_scope"] for packet in validator_packets}),
+            )
             packets_with_real_ref_reads = [
                 packet
                 for packet in validator_packets
@@ -342,40 +461,87 @@ class FiveLevelRecursiveContextTest(unittest.IsolatedAsyncioTestCase):
                     for item in packet["trajectory"]
                 )
             ]
-            self.assertEqual(2, len(packets_with_real_ref_reads))
+            self.assertEqual(4, len(packets_with_real_ref_reads))
             self.assertTrue(ensure_task_protocol(business[0].context)["root_validation_passed"])
             usage = await ResourceBudgetController(store).get_usage(result["trace_id"])
             self.assertEqual(6, usage.total_agents)
             self.assertEqual(call_count, usage.llm_calls)
             self.assertEqual(token_total, usage.total_tokens)
+            self.assertEqual(4, usage.validation_tool_calls)
+            self.assertGreater(usage.validation_material_chars, 0)
 
     @staticmethod
     def _brief(depth):
+        definitions = {
+            1: (
+                "汇总最终发布建议,只采用通过验证的官方数字",
+                "建议采用官方12%并说明30%没有官方支持",
+                "可发布的最终建议",
+                ["output"],
+            ),
+            2: (
+                "判断提升30%的假设是否获得证据支持",
+                "区分30%转载说法与12%官方数字",
+                "假设取舍结论",
+                ["hypothesis"],
+            ),
+            3: (
+                "生成可用和禁用的数字表达",
+                "可用表达只采用官方12%,禁用官方30%表达",
+                "可用与禁用表达清单",
+                ["output"],
+            ),
+            4: (
+                "建立数字与来源的对应关系",
+                "明确官方来源对应12%、转载对应30%",
+                "数字来源对应表",
+                ["evidence"],
+            ),
+            5: (
+                "核实官方页面真实支持的数字",
+                "确认官方只支持12%且不支持30%",
+                "官方数字核查结论",
+                ["evidence"],
+            ),
+        }
+        objective, criterion, expected_output, validation_scopes = definitions[depth]
         return {
-            "objective": f"depth-{depth}",
+            "objective": f"depth-{depth} {objective}",
             "reason": f"depth-{depth - 1} 需要直属子任务结果",
-            "completion_criteria": [f"depth-{depth} 结果通过验收"],
-            "expected_outputs": [f"depth-{depth} 可追溯结论"],
+            "completion_criteria": [criterion],
+            "expected_outputs": [expected_output],
             "parent_findings": [f"直接父级结论 depth-{depth - 1}"],
             "context": {"local_depth": depth},
             "constraints": [f"约束-{depth}"],
+            "validation_scopes": validation_scopes,
         }
 
     @staticmethod
     def _report(depth):
+        result = FiveLevelRecursiveContextTest._semantic_result(depth)
         return {
-            "summary": f"depth-{depth} 局部任务完成",
+            "summary": result,
             "outcome": "satisfied",
             "validation": {"hard_passed": True, "open_issues": []},
             "next_step_suggestion": {
                 "direction": "ASCEND",
                 "reason": "当前层已满足完成标准",
             },
-            "outputs": [{"depth": depth, "result": f"结论-{depth}"}],
-            "evidence": [{"depth": depth, "source": "fake-persisted-trace"}],
+            "outputs": [{"depth": depth, "result": result}],
+            "evidence": [{"depth": depth, "source": "官方页面与转载页面"}],
             "remaining_issues": [],
         }
 
+    @staticmethod
+    def _semantic_result(depth):
+        return {
+            1: "最终建议采用官方12%,并明确30%仅见于转载、无官方支持",
+            2: "30%假设无官方证据,采用12%有官方来源",
+            3: "可用表达:官方12%;禁用表达:官方30%",
+            4: "官方来源→12%;二次转载→30%,二者不可混用",
+            5: "官方页面只支持12%,不支持30%",
+        }[depth]
+
 
 if __name__ == "__main__":
     unittest.main()