|
@@ -19,8 +19,11 @@ from cyber_agent.trace.store import FileSystemTraceStore
|
|
|
|
|
|
|
|
|
|
|
|
|
ROOT_ANCHOR = {
|
|
ROOT_ANCHOR = {
|
|
|
- "objective": "通过五层局部分析得出可追溯的根结论",
|
|
|
|
|
- "completion_criteria": ["五层任务都经过直接父级审核", "根结果通过独立验收"],
|
|
|
|
|
|
|
+ "objective": "发布只使用官方数字的建议",
|
|
|
|
|
+ "completion_criteria": [
|
|
|
|
|
+ "最终建议采用官方12%",
|
|
|
|
|
+ "明确30%仅见于转载且无官方支持",
|
|
|
|
|
+ ],
|
|
|
"constraints": ["不得编造证据"],
|
|
"constraints": ["不得编造证据"],
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -34,7 +37,8 @@ def recursive_env():
|
|
|
"AGENT_MAX_TOTAL_TOKENS": "100000",
|
|
"AGENT_MAX_TOTAL_TOKENS": "100000",
|
|
|
"AGENT_MAX_TOTAL_COST_USD": "10",
|
|
"AGENT_MAX_TOTAL_COST_USD": "10",
|
|
|
"AGENT_MAX_DURATION_SECONDS": "600",
|
|
"AGENT_MAX_DURATION_SECONDS": "600",
|
|
|
- "AGENT_RESERVED_FINAL_CALLS": "1",
|
|
|
|
|
|
|
+ "AGENT_RESERVED_FINAL_CALLS": "8",
|
|
|
|
|
+ "AGENT_VALIDATOR_SEARCH_PROVIDER": "serper",
|
|
|
}, clear=False)
|
|
}, clear=False)
|
|
|
|
|
|
|
|
|
|
|
|
@@ -136,6 +140,39 @@ class FiveLevelRecursiveContextTest(unittest.IsolatedAsyncioTestCase):
|
|
|
read_depths = []
|
|
read_depths = []
|
|
|
observed_tools = {}
|
|
observed_tools = {}
|
|
|
validator_packets = []
|
|
validator_packets = []
|
|
|
|
|
+ validator_packet_keys = set()
|
|
|
|
|
+ search_queries = []
|
|
|
|
|
+
|
|
|
|
|
+ class SearchProvider:
|
|
|
|
|
+ async def search(self, query, max_results):
|
|
|
|
|
+ search_queries.append((query, max_results))
|
|
|
|
|
+ return [
|
|
|
|
|
+ {
|
|
|
|
|
+ "title": "转载页面",
|
|
|
|
|
+ "link": "https://repost.example/30-percent",
|
|
|
|
|
+ "snippet": "未经核实的30%",
|
|
|
|
|
+ },
|
|
|
|
|
+ {
|
|
|
|
|
+ "title": "官方页面",
|
|
|
|
|
+ "link": "https://official.example/12-percent",
|
|
|
|
|
+ "snippet": "官方数字12%",
|
|
|
|
|
+ },
|
|
|
|
|
+ ][:max_results]
|
|
|
|
|
+
|
|
|
|
|
+ async def page_fetcher(url, resolver=None):
|
|
|
|
|
+ del resolver
|
|
|
|
|
+ return {
|
|
|
|
|
+ "source_id": "src-official-12",
|
|
|
|
|
+ "url": url,
|
|
|
|
|
+ "final_url": url,
|
|
|
|
|
+ "title": "官方页面",
|
|
|
|
|
+ "content_type": "text/plain",
|
|
|
|
|
+ "retrieved_at": "2026-07-17T00:00:00+00:00",
|
|
|
|
|
+ "content_sha256": "c" * 64,
|
|
|
|
|
+ "text": "官方数字是12%,不支持30%。",
|
|
|
|
|
+ "truncated": False,
|
|
|
|
|
+ "untrusted_material": True,
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
async def fake_llm(**kwargs):
|
|
async def fake_llm(**kwargs):
|
|
|
nonlocal call_count, token_total
|
|
nonlocal call_count, token_total
|
|
@@ -147,17 +184,89 @@ class FiveLevelRecursiveContextTest(unittest.IsolatedAsyncioTestCase):
|
|
|
|
|
|
|
|
if any(
|
|
if any(
|
|
|
message.get("role") == "system"
|
|
message.get("role") == "system"
|
|
|
- and "independent validator" in str(message.get("content", ""))
|
|
|
|
|
|
|
+ and "recursive_validation_protocol" in str(message.get("content", ""))
|
|
|
for message in messages
|
|
for message in messages
|
|
|
):
|
|
):
|
|
|
- packet = json.loads(messages[-1]["content"])
|
|
|
|
|
- validator_packets.append(packet)
|
|
|
|
|
|
|
+ packet = next(
|
|
|
|
|
+ json.loads(message["content"])
|
|
|
|
|
+ for message in messages
|
|
|
|
|
+ if message.get("role") == "user"
|
|
|
|
|
+ and "recursive_validation_protocol" in str(message.get("content", ""))
|
|
|
|
|
+ )
|
|
|
|
|
+ scope = packet["validation_scope"]
|
|
|
|
|
+ packet_key = (packet["validation_plan"]["plan_hash"], scope)
|
|
|
|
|
+ if packet_key not in validator_packet_keys:
|
|
|
|
|
+ validator_packet_keys.add(packet_key)
|
|
|
|
|
+ validator_packets.append(packet)
|
|
|
|
|
+ if scope == "root":
|
|
|
|
|
+ self.assertIn("官方12%", packet["candidate_output"])
|
|
|
|
|
+ self.assertIn("30%仅见于转载", packet["candidate_output"])
|
|
|
|
|
+ else:
|
|
|
|
|
+ brief = packet["task_brief"]
|
|
|
|
|
+ local_depth = brief["context"]["local_depth"]
|
|
|
|
|
+ report_text = json.dumps(
|
|
|
|
|
+ packet["task_report"], ensure_ascii=False,
|
|
|
|
|
+ )
|
|
|
|
|
+ self.assertIn(
|
|
|
|
|
+ self._semantic_result(local_depth),
|
|
|
|
|
+ report_text,
|
|
|
|
|
+ )
|
|
|
|
|
+ self.assertIn(
|
|
|
|
|
+ scope,
|
|
|
|
|
+ [*brief["validation_scopes"], "task"],
|
|
|
|
|
+ )
|
|
|
|
|
+ task_criteria = [
|
|
|
|
|
+ item["criterion"]
|
|
|
|
|
+ for item in packet["validation_plan"]["checks"]
|
|
|
|
|
+ if item["check_id"].startswith("task.criterion.")
|
|
|
|
|
+ ]
|
|
|
|
|
+ self.assertEqual(
|
|
|
|
|
+ brief["completion_criteria"],
|
|
|
|
|
+ task_criteria,
|
|
|
|
|
+ )
|
|
|
|
|
+ self.assertTrue(
|
|
|
|
|
+ set(ROOT_ANCHOR["completion_criteria"]).isdisjoint(
|
|
|
|
|
+ task_criteria
|
|
|
|
|
+ )
|
|
|
|
|
+ )
|
|
|
|
|
+ validator_last_tool = last_tool_name(messages)
|
|
|
|
|
+ if scope == "evidence" and validator_last_tool is None:
|
|
|
|
|
+ response = tool_call(
|
|
|
|
|
+ "validator_web_search",
|
|
|
|
|
+ {"query": "官方数字 12% 30%", "max_results": 5},
|
|
|
|
|
+ call_count,
|
|
|
|
|
+ )
|
|
|
|
|
+ token_total += 5
|
|
|
|
|
+ return response
|
|
|
|
|
+ if (
|
|
|
|
|
+ scope == "evidence"
|
|
|
|
|
+ and validator_last_tool == "validator_web_search"
|
|
|
|
|
+ ):
|
|
|
|
|
+ response = tool_call(
|
|
|
|
|
+ "validator_open_url",
|
|
|
|
|
+ {"url": "https://official.example/12-percent"},
|
|
|
|
|
+ call_count,
|
|
|
|
|
+ )
|
|
|
|
|
+ token_total += 5
|
|
|
|
|
+ return response
|
|
|
|
|
+ evidence_refs = (
|
|
|
|
|
+ ["src-official-12"] if scope == "evidence" else []
|
|
|
|
|
+ )
|
|
|
response = {
|
|
response = {
|
|
|
"content": json.dumps({
|
|
"content": json.dumps({
|
|
|
"outcome": "passed",
|
|
"outcome": "passed",
|
|
|
- "scope": packet["validation_scope"],
|
|
|
|
|
|
|
+ "scope": scope,
|
|
|
|
|
+ "checks": [
|
|
|
|
|
+ {
|
|
|
|
|
+ "check_id": item["check_id"],
|
|
|
|
|
+ "status": "passed",
|
|
|
|
|
+ "evidence_refs": evidence_refs,
|
|
|
|
|
+ "issue": None,
|
|
|
|
|
+ }
|
|
|
|
|
+ for item in packet["validation_plan"]["checks"]
|
|
|
|
|
+ if item["scope"] == scope
|
|
|
|
|
+ ],
|
|
|
"reason": "已核对持久化轨迹、标准和输出",
|
|
"reason": "已核对持久化轨迹、标准和输出",
|
|
|
- "issues": [],
|
|
|
|
|
"retry_from": None,
|
|
"retry_from": None,
|
|
|
}, ensure_ascii=False),
|
|
}, ensure_ascii=False),
|
|
|
"tool_calls": [],
|
|
"tool_calls": [],
|
|
@@ -192,7 +301,10 @@ class FiveLevelRecursiveContextTest(unittest.IsolatedAsyncioTestCase):
|
|
|
elif last_tool == "review_task_result":
|
|
elif last_tool == "review_task_result":
|
|
|
if depth == 0:
|
|
if depth == 0:
|
|
|
response = {
|
|
response = {
|
|
|
- "content": "根任务的五层结果已逐层审核完成",
|
|
|
|
|
|
|
+ "content": (
|
|
|
|
|
+ "最终建议采用官方12%;30%仅见于转载,"
|
|
|
|
|
+ "没有官方支持,不能作为发布依据。"
|
|
|
|
|
+ ),
|
|
|
"tool_calls": [],
|
|
"tool_calls": [],
|
|
|
"finish_reason": "stop",
|
|
"finish_reason": "stop",
|
|
|
"prompt_tokens": 3,
|
|
"prompt_tokens": 3,
|
|
@@ -233,6 +345,8 @@ class FiveLevelRecursiveContextTest(unittest.IsolatedAsyncioTestCase):
|
|
|
trace_store=store,
|
|
trace_store=store,
|
|
|
tool_registry=get_tool_registry(),
|
|
tool_registry=get_tool_registry(),
|
|
|
llm_call=fake_llm,
|
|
llm_call=fake_llm,
|
|
|
|
|
+ validator_search_provider=SearchProvider(),
|
|
|
|
|
+ validator_page_fetcher=page_fetcher,
|
|
|
)
|
|
)
|
|
|
config = RunConfig(
|
|
config = RunConfig(
|
|
|
tools=[
|
|
tools=[
|
|
@@ -330,9 +444,14 @@ class FiveLevelRecursiveContextTest(unittest.IsolatedAsyncioTestCase):
|
|
|
trace for trace in traces
|
|
trace for trace in traces
|
|
|
if trace.context.get("created_by_tool") == "validator"
|
|
if trace.context.get("created_by_tool") == "validator"
|
|
|
]
|
|
]
|
|
|
- self.assertEqual(6, len(validators))
|
|
|
|
|
- self.assertEqual(6, len(validator_packets))
|
|
|
|
|
|
|
+ self.assertEqual(11, len(validators))
|
|
|
|
|
+ self.assertEqual(11, len(validator_packets))
|
|
|
self.assertTrue(all(packet["root_task_anchor"] == ROOT_ANCHOR for packet in validator_packets))
|
|
self.assertTrue(all(packet["root_task_anchor"] == ROOT_ANCHOR for packet in validator_packets))
|
|
|
|
|
+ self.assertEqual(2, len(search_queries))
|
|
|
|
|
+ self.assertEqual(
|
|
|
|
|
+ ["evidence", "hypothesis", "output", "root", "task"],
|
|
|
|
|
+ sorted({packet["validation_scope"] for packet in validator_packets}),
|
|
|
|
|
+ )
|
|
|
packets_with_real_ref_reads = [
|
|
packets_with_real_ref_reads = [
|
|
|
packet
|
|
packet
|
|
|
for packet in validator_packets
|
|
for packet in validator_packets
|
|
@@ -342,40 +461,87 @@ class FiveLevelRecursiveContextTest(unittest.IsolatedAsyncioTestCase):
|
|
|
for item in packet["trajectory"]
|
|
for item in packet["trajectory"]
|
|
|
)
|
|
)
|
|
|
]
|
|
]
|
|
|
- self.assertEqual(2, len(packets_with_real_ref_reads))
|
|
|
|
|
|
|
+ self.assertEqual(4, len(packets_with_real_ref_reads))
|
|
|
self.assertTrue(ensure_task_protocol(business[0].context)["root_validation_passed"])
|
|
self.assertTrue(ensure_task_protocol(business[0].context)["root_validation_passed"])
|
|
|
usage = await ResourceBudgetController(store).get_usage(result["trace_id"])
|
|
usage = await ResourceBudgetController(store).get_usage(result["trace_id"])
|
|
|
self.assertEqual(6, usage.total_agents)
|
|
self.assertEqual(6, usage.total_agents)
|
|
|
self.assertEqual(call_count, usage.llm_calls)
|
|
self.assertEqual(call_count, usage.llm_calls)
|
|
|
self.assertEqual(token_total, usage.total_tokens)
|
|
self.assertEqual(token_total, usage.total_tokens)
|
|
|
|
|
+ self.assertEqual(4, usage.validation_tool_calls)
|
|
|
|
|
+ self.assertGreater(usage.validation_material_chars, 0)
|
|
|
|
|
|
|
|
@staticmethod
|
|
@staticmethod
|
|
|
def _brief(depth):
|
|
def _brief(depth):
|
|
|
|
|
+ definitions = {
|
|
|
|
|
+ 1: (
|
|
|
|
|
+ "汇总最终发布建议,只采用通过验证的官方数字",
|
|
|
|
|
+ "建议采用官方12%并说明30%没有官方支持",
|
|
|
|
|
+ "可发布的最终建议",
|
|
|
|
|
+ ["output"],
|
|
|
|
|
+ ),
|
|
|
|
|
+ 2: (
|
|
|
|
|
+ "判断提升30%的假设是否获得证据支持",
|
|
|
|
|
+ "区分30%转载说法与12%官方数字",
|
|
|
|
|
+ "假设取舍结论",
|
|
|
|
|
+ ["hypothesis"],
|
|
|
|
|
+ ),
|
|
|
|
|
+ 3: (
|
|
|
|
|
+ "生成可用和禁用的数字表达",
|
|
|
|
|
+ "可用表达只采用官方12%,禁用官方30%表达",
|
|
|
|
|
+ "可用与禁用表达清单",
|
|
|
|
|
+ ["output"],
|
|
|
|
|
+ ),
|
|
|
|
|
+ 4: (
|
|
|
|
|
+ "建立数字与来源的对应关系",
|
|
|
|
|
+ "明确官方来源对应12%、转载对应30%",
|
|
|
|
|
+ "数字来源对应表",
|
|
|
|
|
+ ["evidence"],
|
|
|
|
|
+ ),
|
|
|
|
|
+ 5: (
|
|
|
|
|
+ "核实官方页面真实支持的数字",
|
|
|
|
|
+ "确认官方只支持12%且不支持30%",
|
|
|
|
|
+ "官方数字核查结论",
|
|
|
|
|
+ ["evidence"],
|
|
|
|
|
+ ),
|
|
|
|
|
+ }
|
|
|
|
|
+ objective, criterion, expected_output, validation_scopes = definitions[depth]
|
|
|
return {
|
|
return {
|
|
|
- "objective": f"depth-{depth}",
|
|
|
|
|
|
|
+ "objective": f"depth-{depth} {objective}",
|
|
|
"reason": f"depth-{depth - 1} 需要直属子任务结果",
|
|
"reason": f"depth-{depth - 1} 需要直属子任务结果",
|
|
|
- "completion_criteria": [f"depth-{depth} 结果通过验收"],
|
|
|
|
|
- "expected_outputs": [f"depth-{depth} 可追溯结论"],
|
|
|
|
|
|
|
+ "completion_criteria": [criterion],
|
|
|
|
|
+ "expected_outputs": [expected_output],
|
|
|
"parent_findings": [f"直接父级结论 depth-{depth - 1}"],
|
|
"parent_findings": [f"直接父级结论 depth-{depth - 1}"],
|
|
|
"context": {"local_depth": depth},
|
|
"context": {"local_depth": depth},
|
|
|
"constraints": [f"约束-{depth}"],
|
|
"constraints": [f"约束-{depth}"],
|
|
|
|
|
+ "validation_scopes": validation_scopes,
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
@staticmethod
|
|
@staticmethod
|
|
|
def _report(depth):
|
|
def _report(depth):
|
|
|
|
|
+ result = FiveLevelRecursiveContextTest._semantic_result(depth)
|
|
|
return {
|
|
return {
|
|
|
- "summary": f"depth-{depth} 局部任务完成",
|
|
|
|
|
|
|
+ "summary": result,
|
|
|
"outcome": "satisfied",
|
|
"outcome": "satisfied",
|
|
|
"validation": {"hard_passed": True, "open_issues": []},
|
|
"validation": {"hard_passed": True, "open_issues": []},
|
|
|
"next_step_suggestion": {
|
|
"next_step_suggestion": {
|
|
|
"direction": "ASCEND",
|
|
"direction": "ASCEND",
|
|
|
"reason": "当前层已满足完成标准",
|
|
"reason": "当前层已满足完成标准",
|
|
|
},
|
|
},
|
|
|
- "outputs": [{"depth": depth, "result": f"结论-{depth}"}],
|
|
|
|
|
- "evidence": [{"depth": depth, "source": "fake-persisted-trace"}],
|
|
|
|
|
|
|
+ "outputs": [{"depth": depth, "result": result}],
|
|
|
|
|
+ "evidence": [{"depth": depth, "source": "官方页面与转载页面"}],
|
|
|
"remaining_issues": [],
|
|
"remaining_issues": [],
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+ @staticmethod
|
|
|
|
|
+ def _semantic_result(depth):
|
|
|
|
|
+ return {
|
|
|
|
|
+ 1: "最终建议采用官方12%,并明确30%仅见于转载、无官方支持",
|
|
|
|
|
+ 2: "30%假设无官方证据,采用12%有官方来源",
|
|
|
|
|
+ 3: "可用表达:官方12%;禁用表达:官方30%",
|
|
|
|
|
+ 4: "官方来源→12%;二次转载→30%,二者不可混用",
|
|
|
|
|
+ 5: "官方页面只支持12%,不支持30%",
|
|
|
|
|
+ }[depth]
|
|
|
|
|
+
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if __name__ == "__main__":
|
|
|
unittest.main()
|
|
unittest.main()
|