|
|
@@ -0,0 +1,326 @@
|
|
|
+from __future__ import annotations
|
|
|
+
|
|
|
+import json
|
|
|
+from collections import Counter
|
|
|
+from typing import Any
|
|
|
+
|
|
|
+from content_agent.integrations.runtime_files import RUNTIME_FILENAMES
|
|
|
+from content_agent.interfaces import RuntimeFileStore
|
|
|
+
|
|
|
+
|
|
|
+JSON_FILES = {"source_context.json", "pattern_seed_pack.json", "final_output.json"}
|
|
|
+JSONL_FILES = set(RUNTIME_FILENAMES) - JSON_FILES
|
|
|
+FINAL_ACTIONS = {"POOL", "CANDIDATE", "PENDING", "REJECT"}
|
|
|
+SOURCE_EVIDENCE_FIELDS = {
|
|
|
+ "source_kind",
|
|
|
+ "pattern_source_system",
|
|
|
+ "case_id_type",
|
|
|
+ "source_post_id",
|
|
|
+ "pattern_execution_id",
|
|
|
+ "mining_config_id",
|
|
|
+ "itemset_ids",
|
|
|
+ "itemset_items",
|
|
|
+ "category_bindings",
|
|
|
+ "element_bindings",
|
|
|
+ "matched_post_ids",
|
|
|
+ "seed_terms",
|
|
|
+ "origin_source",
|
|
|
+ "immediate_source",
|
|
|
+ "origin_edge_id",
|
|
|
+ "trace_id",
|
|
|
+ "candidate_aweme_id",
|
|
|
+ "source_certainty",
|
|
|
+ "validation_status",
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+def validate_run(trace_id: str, runtime: RuntimeFileStore) -> dict[str, Any]:
|
|
|
+ findings: list[dict[str, Any]] = []
|
|
|
+ data = _load_files(trace_id, runtime, findings)
|
|
|
+ if any(finding["level"] == "fail" for finding in findings):
|
|
|
+ return _result(trace_id, findings)
|
|
|
+
|
|
|
+ _check_trace_ids(trace_id, data, findings)
|
|
|
+ _check_unique_ids(data, findings)
|
|
|
+ _check_references(data, findings)
|
|
|
+ _check_source_evidence(data, findings)
|
|
|
+ _check_source_paths(data, findings)
|
|
|
+ _check_summary(data, findings)
|
|
|
+ return _result(trace_id, findings)
|
|
|
+
|
|
|
+
|
|
|
+def _load_files(
|
|
|
+ trace_id: str,
|
|
|
+ runtime: RuntimeFileStore,
|
|
|
+ findings: list[dict[str, Any]],
|
|
|
+) -> dict[str, Any]:
|
|
|
+ run_dir = runtime.run_dir(trace_id)
|
|
|
+ data: dict[str, Any] = {}
|
|
|
+ for filename in RUNTIME_FILENAMES:
|
|
|
+ path = run_dir / filename
|
|
|
+ if not path.exists():
|
|
|
+ _fail(findings, "file_missing", f"missing runtime file: {filename}")
|
|
|
+ continue
|
|
|
+ try:
|
|
|
+ if filename in JSON_FILES:
|
|
|
+ data[filename] = json.loads(path.read_text(encoding="utf-8"))
|
|
|
+ else:
|
|
|
+ rows = []
|
|
|
+ for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
|
|
+ if line.strip():
|
|
|
+ rows.append(json.loads(line))
|
|
|
+ data[filename] = rows
|
|
|
+ except json.JSONDecodeError as exc:
|
|
|
+ _fail(findings, "json_parse_failed", f"{filename} cannot parse: {exc}")
|
|
|
+ return data
|
|
|
+
|
|
|
+
|
|
|
+def _check_trace_ids(trace_id: str, data: dict[str, Any], findings: list[dict[str, Any]]) -> None:
|
|
|
+ for filename, value in data.items():
|
|
|
+ rows = value if isinstance(value, list) else [value]
|
|
|
+ for row in rows:
|
|
|
+ if isinstance(row, dict) and row.get("trace_id") != trace_id:
|
|
|
+ _fail(findings, "trace_id_mismatch", f"{filename} has mismatched trace_id")
|
|
|
+
|
|
|
+
|
|
|
+def _check_unique_ids(data: dict[str, Any], findings: list[dict[str, Any]]) -> None:
|
|
|
+ checks = [
|
|
|
+ ("queries.jsonl", "query_id"),
|
|
|
+ ("candidate_pool.jsonl", "candidate_id"),
|
|
|
+ ("candidate_pool.jsonl", "aweme_id"),
|
|
|
+ ("rule_decisions.jsonl", "decision_id"),
|
|
|
+ ("rule_decisions.jsonl", "entity_id"),
|
|
|
+ ("source_edges.jsonl", "edge_id"),
|
|
|
+ ]
|
|
|
+ for filename, field in checks:
|
|
|
+ rows = data.get(filename, [])
|
|
|
+ values = [row.get(field) for row in rows]
|
|
|
+ duplicates = [value for value, count in Counter(values).items() if value and count > 1]
|
|
|
+ if duplicates:
|
|
|
+ _fail(findings, "duplicate_id", f"{filename}.{field} duplicates: {duplicates}")
|
|
|
+
|
|
|
+
|
|
|
+def _check_references(data: dict[str, Any], findings: list[dict[str, Any]]) -> None:
|
|
|
+ queries = {row["query_id"] for row in data.get("queries.jsonl", [])}
|
|
|
+ candidates = data.get("candidate_pool.jsonl", [])
|
|
|
+ candidate_aweme_ids = {row["aweme_id"] for row in candidates}
|
|
|
+ decisions = data.get("rule_decisions.jsonl", [])
|
|
|
+ decision_ids = {row["decision_id"] for row in decisions}
|
|
|
+ edge_ids = {row["edge_id"] for row in data.get("source_edges.jsonl", [])}
|
|
|
+
|
|
|
+ for candidate in candidates:
|
|
|
+ if candidate.get("query_id") not in queries:
|
|
|
+ _fail(findings, "missing_query_ref", f"candidate has unknown query_id: {candidate.get('query_id')}")
|
|
|
+
|
|
|
+ for media in data.get("media_assets.jsonl", []):
|
|
|
+ if media.get("aweme_id") not in candidate_aweme_ids:
|
|
|
+ _fail(findings, "missing_candidate_ref", f"media has unknown aweme_id: {media.get('aweme_id')}")
|
|
|
+
|
|
|
+ for decision in decisions:
|
|
|
+ if decision.get("entity_id") not in candidate_aweme_ids:
|
|
|
+ _fail(findings, "missing_candidate_ref", f"decision has unknown entity_id: {decision.get('entity_id')}")
|
|
|
+ if decision.get("final_action") not in FINAL_ACTIONS:
|
|
|
+ _fail(findings, "bad_final_action", f"unsupported final_action: {decision.get('final_action')}")
|
|
|
+
|
|
|
+ for edge in data.get("source_edges.jsonl", []):
|
|
|
+ decision_id = edge.get("decision_id")
|
|
|
+ if decision_id and decision_id not in decision_ids:
|
|
|
+ _fail(findings, "missing_decision_ref", f"edge has unknown decision_id: {decision_id}")
|
|
|
+
|
|
|
+ final_output = data.get("final_output.json", {})
|
|
|
+ for asset in final_output.get("content_assets", []):
|
|
|
+ if asset.get("decision_id") not in decision_ids:
|
|
|
+ _fail(findings, "missing_decision_ref", f"asset has unknown decision_id: {asset.get('decision_id')}")
|
|
|
+ for edge_id in asset.get("source_edge_ids", []):
|
|
|
+ if edge_id not in edge_ids:
|
|
|
+ _fail(findings, "missing_edge_ref", f"asset has unknown source_edge_id: {edge_id}")
|
|
|
+ for record in final_output.get("reject_records", []):
|
|
|
+ if record.get("decision_id") not in decision_ids:
|
|
|
+ _fail(findings, "missing_decision_ref", f"reject has unknown decision_id: {record.get('decision_id')}")
|
|
|
+ for author_asset in final_output.get("author_assets", []):
|
|
|
+ if author_asset.get("decision_id") not in decision_ids:
|
|
|
+ _fail(findings, "missing_decision_ref", f"author asset has unknown decision_id: {author_asset.get('decision_id')}")
|
|
|
+ for record in final_output.get("decision_records", []):
|
|
|
+ if record.get("decision_id") not in decision_ids:
|
|
|
+ _fail(findings, "missing_decision_ref", f"decision record has unknown decision_id: {record.get('decision_id')}")
|
|
|
+
|
|
|
+
|
|
|
+def _check_source_evidence(data: dict[str, Any], findings: list[dict[str, Any]]) -> None:
|
|
|
+ source_context = data.get("source_context.json", {})
|
|
|
+ evidence_pack = source_context.get("ext_data", {}).get("evidence_pack", {})
|
|
|
+ for decision in data.get("rule_decisions.jsonl", []):
|
|
|
+ source_evidence = decision.get("source_evidence") or {}
|
|
|
+ _check_one_source_evidence(
|
|
|
+ findings,
|
|
|
+ source_evidence,
|
|
|
+ evidence_pack,
|
|
|
+ f"decision {decision.get('decision_id')}",
|
|
|
+ )
|
|
|
+
|
|
|
+ for asset in data.get("final_output.json", {}).get("content_assets", []):
|
|
|
+ _check_one_source_evidence(
|
|
|
+ findings,
|
|
|
+ asset.get("source_evidence") or {},
|
|
|
+ evidence_pack,
|
|
|
+ f"asset {asset.get('aweme_id')}",
|
|
|
+ )
|
|
|
+ for record in data.get("final_output.json", {}).get("reject_records", []):
|
|
|
+ _check_one_source_evidence(
|
|
|
+ findings,
|
|
|
+ record.get("source_evidence") or {},
|
|
|
+ evidence_pack,
|
|
|
+ f"reject {record.get('entity_id')}",
|
|
|
+ )
|
|
|
+ for record in data.get("final_output.json", {}).get("decision_records", []):
|
|
|
+ _check_one_source_evidence(
|
|
|
+ findings,
|
|
|
+ record.get("source_evidence") or {},
|
|
|
+ evidence_pack,
|
|
|
+ f"decision record {record.get('decision_id')}",
|
|
|
+ )
|
|
|
+ _check_final_decision_coverage(data, findings)
|
|
|
+
|
|
|
+
|
|
|
+def _check_one_source_evidence(
|
|
|
+ findings: list[dict[str, Any]],
|
|
|
+ source_evidence: dict[str, Any],
|
|
|
+ evidence_pack: dict[str, Any],
|
|
|
+ label: str,
|
|
|
+) -> None:
|
|
|
+ missing_fields = sorted(field for field in SOURCE_EVIDENCE_FIELDS if field not in source_evidence)
|
|
|
+ if missing_fields:
|
|
|
+ _fail(findings, "source_evidence_missing_fields", f"{label} missing {missing_fields}")
|
|
|
+ if not source_evidence.get("category_bindings") and not source_evidence.get("element_bindings"):
|
|
|
+ _fail(findings, "source_evidence_missing_binding", f"{label} lacks category or element binding")
|
|
|
+ for field in [
|
|
|
+ "pattern_execution_id",
|
|
|
+ "source_post_id",
|
|
|
+ "pattern_source_system",
|
|
|
+ "case_id_type",
|
|
|
+ "mining_config_id",
|
|
|
+ "source_certainty",
|
|
|
+ "validation_status",
|
|
|
+ ]:
|
|
|
+ if source_evidence.get(field) != evidence_pack.get(field):
|
|
|
+ _fail(findings, "source_evidence_mismatch", f"{label} mismatched {field}")
|
|
|
+ for field in ["itemset_ids", "itemset_items", "matched_post_ids", "seed_terms"]:
|
|
|
+ if source_evidence.get(field) != evidence_pack.get(field):
|
|
|
+ _fail(findings, "source_evidence_mismatch", f"{label} mismatched {field}")
|
|
|
+ candidate_aweme_id = source_evidence.get("candidate_aweme_id")
|
|
|
+ if candidate_aweme_id:
|
|
|
+ if candidate_aweme_id == source_evidence.get("source_post_id"):
|
|
|
+ _fail(findings, "source_evidence_aweme_pollution", f"{label} rewrites source_post_id")
|
|
|
+ if candidate_aweme_id in (source_evidence.get("matched_post_ids") or []):
|
|
|
+ _fail(findings, "source_evidence_aweme_pollution", f"{label} rewrites matched_post_ids")
|
|
|
+
|
|
|
+
|
|
|
+def _check_final_decision_coverage(data: dict[str, Any], findings: list[dict[str, Any]]) -> None:
|
|
|
+ decision_ids = {decision["decision_id"] for decision in data.get("rule_decisions.jsonl", [])}
|
|
|
+ final_decision_ids = {
|
|
|
+ record.get("decision_id")
|
|
|
+ for record in data.get("final_output.json", {}).get("decision_records", [])
|
|
|
+ }
|
|
|
+ missing = sorted(decision_ids - final_decision_ids)
|
|
|
+ if missing:
|
|
|
+ _fail(findings, "final_decision_missing", f"final_output decision_records missing {missing}")
|
|
|
+
|
|
|
+
|
|
|
+def _check_source_paths(data: dict[str, Any], findings: list[dict[str, Any]]) -> None:
|
|
|
+ source_context = data.get("source_context.json", {})
|
|
|
+ pattern_execution_id = source_context.get("ext_data", {}).get("evidence_pack", {}).get("pattern_execution_id")
|
|
|
+ edges = data.get("source_edges.jsonl", [])
|
|
|
+ edge_by_id = {edge["edge_id"]: edge for edge in edges}
|
|
|
+ pattern_query_edges = {
|
|
|
+ edge["to_node_id"]: edge
|
|
|
+ for edge in edges
|
|
|
+ if edge.get("edge_type") == "pattern_to_query"
|
|
|
+ }
|
|
|
+ query_video_edges = {
|
|
|
+ edge["to_node_id"]: edge
|
|
|
+ for edge in edges
|
|
|
+ if edge.get("edge_type") == "query_to_video"
|
|
|
+ }
|
|
|
+
|
|
|
+ for decision in data.get("rule_decisions.jsonl", []):
|
|
|
+ _check_entity_source_path(
|
|
|
+ findings,
|
|
|
+ label=f"decision {decision.get('decision_id')}",
|
|
|
+ entity_id=decision.get("entity_id"),
|
|
|
+ pattern_execution_id=pattern_execution_id,
|
|
|
+ pattern_query_edges=pattern_query_edges,
|
|
|
+ query_video_edges=query_video_edges,
|
|
|
+ )
|
|
|
+
|
|
|
+ for asset in data.get("final_output.json", {}).get("content_assets", []):
|
|
|
+ aweme_id = asset.get("aweme_id")
|
|
|
+ edge_ids = set(asset.get("source_edge_ids", []))
|
|
|
+ query_video = query_video_edges.get(aweme_id)
|
|
|
+ if not query_video or query_video.get("edge_id") not in edge_ids:
|
|
|
+ _fail(findings, "source_path_broken", f"asset lacks query_to_video edge: {aweme_id}")
|
|
|
+ continue
|
|
|
+ pattern_query = pattern_query_edges.get(query_video.get("from_node_id"))
|
|
|
+ if not pattern_query or pattern_query.get("edge_id") not in edge_ids:
|
|
|
+ _fail(findings, "source_path_broken", f"asset lacks pattern_to_query edge: {aweme_id}")
|
|
|
+ continue
|
|
|
+ if pattern_query.get("from_node_id") != pattern_execution_id:
|
|
|
+ _fail(findings, "source_path_broken", f"asset path starts from wrong pattern: {aweme_id}")
|
|
|
+ for edge_id in edge_ids:
|
|
|
+ if edge_id not in edge_by_id:
|
|
|
+ _fail(findings, "source_path_broken", f"asset edge missing: {edge_id}")
|
|
|
+
|
|
|
+
|
|
|
+def _check_entity_source_path(
|
|
|
+ findings: list[dict[str, Any]],
|
|
|
+ label: str,
|
|
|
+ entity_id: Any,
|
|
|
+ pattern_execution_id: Any,
|
|
|
+ pattern_query_edges: dict[str, dict[str, Any]],
|
|
|
+ query_video_edges: dict[str, dict[str, Any]],
|
|
|
+) -> None:
|
|
|
+ query_video = query_video_edges.get(entity_id)
|
|
|
+ if not query_video:
|
|
|
+ _fail(findings, "source_path_broken", f"{label} lacks query_to_video edge: {entity_id}")
|
|
|
+ return
|
|
|
+ pattern_query = pattern_query_edges.get(query_video.get("from_node_id"))
|
|
|
+ if not pattern_query:
|
|
|
+ _fail(findings, "source_path_broken", f"{label} lacks pattern_to_query edge: {entity_id}")
|
|
|
+ return
|
|
|
+ if pattern_query.get("from_node_id") != pattern_execution_id:
|
|
|
+ _fail(findings, "source_path_broken", f"{label} path starts from wrong pattern: {entity_id}")
|
|
|
+
|
|
|
+
|
|
|
+def _check_summary(data: dict[str, Any], findings: list[dict[str, Any]]) -> None:
|
|
|
+ decisions = data.get("rule_decisions.jsonl", [])
|
|
|
+ action_counts = Counter(decision.get("final_action") for decision in decisions)
|
|
|
+ summary = data.get("final_output.json", {}).get("summary", {})
|
|
|
+ expected = {
|
|
|
+ "pool_count": action_counts["POOL"],
|
|
|
+ "candidate_count": action_counts["CANDIDATE"],
|
|
|
+ "pending_count": action_counts["PENDING"],
|
|
|
+ "reject_count": action_counts["REJECT"],
|
|
|
+ }
|
|
|
+ for field, value in expected.items():
|
|
|
+ if summary.get(field) != value:
|
|
|
+ _fail(findings, "summary_mismatch", f"summary.{field} expected {value}, got {summary.get(field)}")
|
|
|
+
|
|
|
+ clue_counts = Counter()
|
|
|
+ for clue in data.get("search_clues.jsonl", []):
|
|
|
+ clue_counts["POOL"] += clue.get("pool_count", 0)
|
|
|
+ clue_counts["CANDIDATE"] += clue.get("candidate_count", 0)
|
|
|
+ clue_counts["PENDING"] += clue.get("pending_count", 0)
|
|
|
+ clue_counts["REJECT"] += clue.get("reject_count", 0)
|
|
|
+ for action, count in action_counts.items():
|
|
|
+ if clue_counts[action] != count:
|
|
|
+ _fail(findings, "search_clue_mismatch", f"search_clues {action} expected {count}, got {clue_counts[action]}")
|
|
|
+
|
|
|
+
|
|
|
+def _result(trace_id: str, findings: list[dict[str, Any]]) -> dict[str, Any]:
|
|
|
+ return {
|
|
|
+ "trace_id": trace_id,
|
|
|
+ "status": "fail" if any(finding["level"] == "fail" for finding in findings) else "pass",
|
|
|
+ "findings": findings,
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def _fail(findings: list[dict[str, Any]], check_id: str, message: str) -> None:
|
|
|
+ findings.append({"level": "fail", "check_id": check_id, "message": message})
|