|
|
@@ -0,0 +1,322 @@
|
|
|
+from __future__ import annotations
|
|
|
+
|
|
|
+from datetime import datetime, timezone
|
|
|
+from typing import Any, Callable
|
|
|
+
|
|
|
+from content_agent.business_modules.content_discovery.pattern_recall.category_match import (
|
|
|
+ match_decode_terms,
|
|
|
+)
|
|
|
+from content_agent.business_modules.content_discovery.pattern_recall.decode import decode_content
|
|
|
+from content_agent.constants import RUNTIME_RECORD_SCHEMA_VERSION
|
|
|
+from content_agent.integrations.decode_api import redact_sensitive_payload
|
|
|
+from content_agent.interfaces import CategoryMatchClient, DecodeClient, RuntimeFileStore
|
|
|
+
|
|
|
+
|
|
|
+def run(
|
|
|
+ run_id: str,
|
|
|
+ policy_run_id: str,
|
|
|
+ discovered_content_items: list[dict[str, Any]],
|
|
|
+ content_media_records: list[dict[str, Any]],
|
|
|
+ evidence_bundles: list[dict[str, Any]],
|
|
|
+ source_context: dict[str, Any],
|
|
|
+ pattern_seed_pack: dict[str, Any],
|
|
|
+ runtime: RuntimeFileStore,
|
|
|
+ decode_client: DecodeClient,
|
|
|
+ category_match_client: CategoryMatchClient,
|
|
|
+ max_wait_seconds: float = 1200.0,
|
|
|
+ poll_interval_seconds: float = 5.0,
|
|
|
+ now_fn: Callable[[], float] | None = None,
|
|
|
+ sleep_fn: Callable[[float], None] | None = None,
|
|
|
+) -> dict[str, Any]:
|
|
|
+ created_at = datetime.now(timezone.utc).isoformat()
|
|
|
+ evidence_rows: list[dict[str, Any]] = []
|
|
|
+ updated_items: list[dict[str, Any]] = []
|
|
|
+ updated_bundles: list[dict[str, Any]] = []
|
|
|
+
|
|
|
+ media_by_content_id = {
|
|
|
+ row["platform_content_id"]: row
|
|
|
+ for row in content_media_records
|
|
|
+ }
|
|
|
+ for index, item in enumerate(discovered_content_items, start=1):
|
|
|
+ media = media_by_content_id.get(item["platform_content_id"], {})
|
|
|
+ bundle = evidence_bundles[index - 1]
|
|
|
+ decision = _recall_one(
|
|
|
+ index=index,
|
|
|
+ content=item,
|
|
|
+ media=media,
|
|
|
+ source_context=source_context,
|
|
|
+ pattern_seed_pack=pattern_seed_pack,
|
|
|
+ decode_client=decode_client,
|
|
|
+ category_match_client=category_match_client,
|
|
|
+ max_wait_seconds=max_wait_seconds,
|
|
|
+ poll_interval_seconds=poll_interval_seconds,
|
|
|
+ now_fn=now_fn,
|
|
|
+ sleep_fn=sleep_fn,
|
|
|
+ )
|
|
|
+ pattern_match_result = decision["pattern_match_result"]
|
|
|
+ evidence_row = _build_evidence_row(
|
|
|
+ run_id=run_id,
|
|
|
+ policy_run_id=policy_run_id,
|
|
|
+ content=item,
|
|
|
+ decision=decision,
|
|
|
+ created_at=created_at,
|
|
|
+ )
|
|
|
+ updated_item = _update_discovered_item(item, pattern_match_result)
|
|
|
+ updated_bundle = _update_evidence_bundle(bundle, pattern_match_result)
|
|
|
+ evidence_rows.append(evidence_row)
|
|
|
+ updated_items.append(updated_item)
|
|
|
+ updated_bundles.append(updated_bundle)
|
|
|
+
|
|
|
+ runtime.append_jsonl(run_id, "pattern_recall_evidence.jsonl", evidence_rows)
|
|
|
+ runtime.append_jsonl(run_id, "discovered_content_items.jsonl", updated_items)
|
|
|
+ return {
|
|
|
+ "pattern_recall_evidence": evidence_rows,
|
|
|
+ "discovered_content_items": updated_items,
|
|
|
+ "evidence_bundles": updated_bundles,
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def _recall_one(
|
|
|
+ *,
|
|
|
+ index: int,
|
|
|
+ content: dict[str, Any],
|
|
|
+ media: dict[str, Any],
|
|
|
+ source_context: dict[str, Any],
|
|
|
+ pattern_seed_pack: dict[str, Any],
|
|
|
+ decode_client: DecodeClient,
|
|
|
+ category_match_client: CategoryMatchClient,
|
|
|
+ max_wait_seconds: float,
|
|
|
+ poll_interval_seconds: float,
|
|
|
+ now_fn: Callable[[], float] | None,
|
|
|
+ sleep_fn: Callable[[float], None] | None,
|
|
|
+) -> dict[str, Any]:
|
|
|
+ recall_evidence_id = f"recall_{index:03d}"
|
|
|
+ decode_result = decode_content(
|
|
|
+ content=content,
|
|
|
+ media=media,
|
|
|
+ source_context=source_context,
|
|
|
+ decode_client=decode_client,
|
|
|
+ max_wait_seconds=max_wait_seconds,
|
|
|
+ poll_interval_seconds=poll_interval_seconds,
|
|
|
+ now_fn=now_fn,
|
|
|
+ sleep_fn=sleep_fn,
|
|
|
+ )
|
|
|
+ decode_status = decode_result["decode_status"]
|
|
|
+ if decode_status in {"pending", "running"}:
|
|
|
+ return _decision(
|
|
|
+ recall_evidence_id=recall_evidence_id,
|
|
|
+ decode_result=decode_result,
|
|
|
+ recall_status="pending",
|
|
|
+ pattern_recall="pattern_recall_pending",
|
|
|
+ category_or_element_binding="pattern_recall_pending",
|
|
|
+ match_result={},
|
|
|
+ )
|
|
|
+ if decode_status == "failed":
|
|
|
+ is_client_failure = decode_result.get("failure_reason") == "decode_client_error"
|
|
|
+ return _decision(
|
|
|
+ recall_evidence_id=recall_evidence_id,
|
|
|
+ decode_result=decode_result,
|
|
|
+ recall_status="failed" if is_client_failure else "rejected",
|
|
|
+ pattern_recall="pattern_recall_failed" if is_client_failure else "pattern_recall_rejected",
|
|
|
+ category_or_element_binding=(
|
|
|
+ "pattern_recall_failed" if is_client_failure else "pattern_recall_rejected"
|
|
|
+ ),
|
|
|
+ match_result={},
|
|
|
+ )
|
|
|
+
|
|
|
+ try:
|
|
|
+ match_result = match_decode_terms(
|
|
|
+ decode_elements=decode_result.get("decode_elements") or {},
|
|
|
+ category_match_client=category_match_client,
|
|
|
+ )
|
|
|
+ except Exception as exc:
|
|
|
+ match_result = _match_client_failure(exc)
|
|
|
+ return _decision(
|
|
|
+ recall_evidence_id=recall_evidence_id,
|
|
|
+ decode_result=decode_result,
|
|
|
+ recall_status="failed",
|
|
|
+ pattern_recall="pattern_recall_failed",
|
|
|
+ category_or_element_binding="pattern_recall_failed",
|
|
|
+ match_result=match_result,
|
|
|
+ )
|
|
|
+ matched_terms = match_result.get("matched_terms") or []
|
|
|
+ matched_paths = match_result.get("matched_category_paths") or []
|
|
|
+ if not matched_terms or not matched_paths:
|
|
|
+ return _decision(
|
|
|
+ recall_evidence_id=recall_evidence_id,
|
|
|
+ decode_result=decode_result,
|
|
|
+ recall_status="no_match",
|
|
|
+ pattern_recall="pattern_recall_no_match",
|
|
|
+ category_or_element_binding="pattern_recall_no_match",
|
|
|
+ match_result=match_result,
|
|
|
+ )
|
|
|
+ if not _can_explain_pattern(matched_terms, matched_paths, source_context, pattern_seed_pack):
|
|
|
+ return _decision(
|
|
|
+ recall_evidence_id=recall_evidence_id,
|
|
|
+ decode_result=decode_result,
|
|
|
+ recall_status="no_match",
|
|
|
+ pattern_recall="pattern_recall_no_match",
|
|
|
+ category_or_element_binding="pattern_recall_no_match",
|
|
|
+ match_result=match_result,
|
|
|
+ )
|
|
|
+ return _decision(
|
|
|
+ recall_evidence_id=recall_evidence_id,
|
|
|
+ decode_result=decode_result,
|
|
|
+ recall_status="matched",
|
|
|
+ pattern_recall="matched",
|
|
|
+ category_or_element_binding="tree_walk_match",
|
|
|
+ match_result=match_result,
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+def _decision(
|
|
|
+ *,
|
|
|
+ recall_evidence_id: str,
|
|
|
+ decode_result: dict[str, Any],
|
|
|
+ recall_status: str,
|
|
|
+ pattern_recall: str,
|
|
|
+ category_or_element_binding: str,
|
|
|
+ match_result: dict[str, Any],
|
|
|
+) -> dict[str, Any]:
|
|
|
+ matched_paths = match_result.get("matched_category_paths") or []
|
|
|
+ matched_terms = match_result.get("matched_terms") or []
|
|
|
+ primary_path = matched_paths[0] if matched_paths else None
|
|
|
+ pattern_match_result = {
|
|
|
+ "pattern_recall": pattern_recall,
|
|
|
+ "category_or_element_binding": category_or_element_binding,
|
|
|
+ "decode_status": decode_result.get("decode_status"),
|
|
|
+ "match_status": "matched" if recall_status == "matched" else recall_status,
|
|
|
+ "recall_status": recall_status,
|
|
|
+ "matched_terms": matched_terms,
|
|
|
+ "matched_category_paths": matched_paths,
|
|
|
+ "primary_matched_category_path": primary_path,
|
|
|
+ "decode_elements": decode_result.get("decode_elements") or {},
|
|
|
+ "pattern_recall_evidence_id": recall_evidence_id,
|
|
|
+ }
|
|
|
+ return {
|
|
|
+ "recall_evidence_id": recall_evidence_id,
|
|
|
+ "decode_result": decode_result,
|
|
|
+ "match_result": match_result,
|
|
|
+ "pattern_match_result": pattern_match_result,
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def _build_evidence_row(
|
|
|
+ *,
|
|
|
+ run_id: str,
|
|
|
+ policy_run_id: str,
|
|
|
+ content: dict[str, Any],
|
|
|
+ decision: dict[str, Any],
|
|
|
+ created_at: str,
|
|
|
+) -> dict[str, Any]:
|
|
|
+ decode_result = decision["decode_result"]
|
|
|
+ match_result = decision["match_result"]
|
|
|
+ pattern_match = decision["pattern_match_result"]
|
|
|
+ raw_payload = redact_sensitive_payload(
|
|
|
+ {
|
|
|
+ "run_id": run_id,
|
|
|
+ "policy_run_id": policy_run_id,
|
|
|
+ "recall_evidence_id": decision["recall_evidence_id"],
|
|
|
+ "platform": content.get("platform"),
|
|
|
+ "primary_matched_category_path": pattern_match.get("primary_matched_category_path"),
|
|
|
+ "pending_reason": decode_result.get("pending_reason"),
|
|
|
+ "failure_reason": decode_result.get("failure_reason") or match_result.get("failure_reason"),
|
|
|
+ "decode_request": decode_result.get("raw_request"),
|
|
|
+ "decode_response": decode_result.get("raw_response"),
|
|
|
+ "match_paths_request": match_result.get("request"),
|
|
|
+ "match_paths_response": match_result.get("response"),
|
|
|
+ }
|
|
|
+ )
|
|
|
+ return {
|
|
|
+ "record_schema_version": RUNTIME_RECORD_SCHEMA_VERSION,
|
|
|
+ "run_id": run_id,
|
|
|
+ "policy_run_id": policy_run_id,
|
|
|
+ "recall_evidence_id": decision["recall_evidence_id"],
|
|
|
+ "content_discovery_id": content.get("content_discovery_id"),
|
|
|
+ "platform": content.get("platform"),
|
|
|
+ "platform_content_id": content.get("platform_content_id"),
|
|
|
+ "decode_status": pattern_match.get("decode_status"),
|
|
|
+ "decode_task_id": decode_result.get("decode_task_id"),
|
|
|
+ "recall_status": pattern_match.get("recall_status"),
|
|
|
+ "matched_terms": pattern_match.get("matched_terms"),
|
|
|
+ "matched_category_paths": pattern_match.get("matched_category_paths"),
|
|
|
+ "decode_elements": pattern_match.get("decode_elements"),
|
|
|
+ "match_paths_request": match_result.get("request"),
|
|
|
+ "match_paths_response": match_result.get("response"),
|
|
|
+ "evidence_summary": {
|
|
|
+ "pattern_recall": pattern_match.get("pattern_recall"),
|
|
|
+ "category_or_element_binding": pattern_match.get("category_or_element_binding"),
|
|
|
+ "primary_matched_category_path": pattern_match.get("primary_matched_category_path"),
|
|
|
+ "pending_reason": decode_result.get("pending_reason"),
|
|
|
+ "failure_reason": decode_result.get("failure_reason") or match_result.get("failure_reason"),
|
|
|
+ },
|
|
|
+ "raw_payload": raw_payload,
|
|
|
+ "created_at": created_at,
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def _update_discovered_item(
|
|
|
+ item: dict[str, Any],
|
|
|
+ pattern_match_result: dict[str, Any],
|
|
|
+) -> dict[str, Any]:
|
|
|
+ updated = {**item, "pattern_match_result": pattern_match_result}
|
|
|
+ raw_payload = dict(updated.get("raw_payload") or {})
|
|
|
+ raw_payload["pattern_match_result"] = pattern_match_result
|
|
|
+ updated["raw_payload"] = raw_payload
|
|
|
+ return updated
|
|
|
+
|
|
|
+
|
|
|
+def _update_evidence_bundle(
|
|
|
+ bundle: dict[str, Any],
|
|
|
+ pattern_match_result: dict[str, Any],
|
|
|
+) -> dict[str, Any]:
|
|
|
+ existing = bundle.get("pattern_match_result") or {}
|
|
|
+ return {**bundle, "pattern_match_result": {**existing, **pattern_match_result}}
|
|
|
+
|
|
|
+
|
|
|
+def _can_explain_pattern(
|
|
|
+ matched_terms: list[str],
|
|
|
+ matched_paths: list[str],
|
|
|
+ source_context: dict[str, Any],
|
|
|
+ pattern_seed_pack: dict[str, Any],
|
|
|
+) -> bool:
|
|
|
+ evidence_pack = source_context.get("ext_data", {}).get("evidence_pack", {})
|
|
|
+ seed_terms = set(pattern_seed_pack.get("seed_terms") or evidence_pack.get("seed_terms") or [])
|
|
|
+ if seed_terms.intersection(matched_terms):
|
|
|
+ return True
|
|
|
+ binding_paths = [
|
|
|
+ binding.get("category_path")
|
|
|
+ for binding in [
|
|
|
+ *(pattern_seed_pack.get("category_bindings") or []),
|
|
|
+ *(evidence_pack.get("category_bindings") or []),
|
|
|
+ *(pattern_seed_pack.get("itemsets") or []),
|
|
|
+ *(pattern_seed_pack.get("itemset_items") or []),
|
|
|
+ *(evidence_pack.get("itemset_items") or []),
|
|
|
+ ]
|
|
|
+ if isinstance(binding, dict) and binding.get("category_path")
|
|
|
+ ]
|
|
|
+ for matched_path in matched_paths:
|
|
|
+ for binding_path in binding_paths:
|
|
|
+ if matched_path in binding_path or binding_path in matched_path:
|
|
|
+ return True
|
|
|
+ if _path_leaf(matched_path) == _path_leaf(binding_path):
|
|
|
+ return True
|
|
|
+ return False
|
|
|
+
|
|
|
+
|
|
|
+def _match_client_failure(exc: Exception) -> dict[str, Any]:
|
|
|
+ return {
|
|
|
+ "request": {},
|
|
|
+ "response": {
|
|
|
+ "operation": "match_paths",
|
|
|
+ "error_type": type(exc).__name__,
|
|
|
+ },
|
|
|
+ "matched_terms": [],
|
|
|
+ "matched_category_paths": [],
|
|
|
+ "path_matches": [],
|
|
|
+ "failure_reason": "category_match_client_error",
|
|
|
+ }
|
|
|
+
|
|
|
+
|
|
|
+def _path_leaf(path: str) -> str:
|
|
|
+ return str(path).rstrip("/").split("/")[-1]
|