|
@@ -2,216 +2,166 @@ from __future__ import annotations
|
|
|
|
|
|
|
|
from copy import deepcopy
|
|
from copy import deepcopy
|
|
|
|
|
|
|
|
-import pytest
|
|
|
|
|
-
|
|
|
|
|
from content_agent.business_modules.rule_judgment.evaluator import decide
|
|
from content_agent.business_modules.rule_judgment.evaluator import decide
|
|
|
-from content_agent.run_service import RunService
|
|
|
|
|
-from content_agent.schemas import RunStartRequest
|
|
|
|
|
-from tests.p1_helpers import FakeQueryVariantClient, REAL_SOURCE_FIXTURE
|
|
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
-def test_scorecard_uses_active_dimensions_and_thresholds(tmp_path):
|
|
|
|
|
- state = _state(tmp_path)
|
|
|
|
|
- bundle = deepcopy(state["evidence_bundles"][0])
|
|
|
|
|
- # M3 2-dim scorecard: relevance gte0.8 -> 60, platform_heat gte0.4 -> 20 => 80 (pool).
|
|
|
|
|
- bundle["pattern_match_result"]["relevance_score"] = 0.8
|
|
|
|
|
- bundle["content_engagement_metrics"]["platform_heat"] = 0.4
|
|
|
|
|
-
|
|
|
|
|
- decision = decide(
|
|
|
|
|
- state["run_id"],
|
|
|
|
|
- state["policy_run_id"],
|
|
|
|
|
- 1,
|
|
|
|
|
- bundle,
|
|
|
|
|
- state["policy_bundle"],
|
|
|
|
|
- )
|
|
|
|
|
|
|
|
|
|
|
|
+
|
|
|
|
|
+def test_v4_scorecard_uses_query_and_platform_50_50():
|
|
|
|
|
+ decision = decide("run_1", "policy_1", 1, _bundle(query=80, platform=70), _policy())
|
|
|
|
|
+
|
|
|
|
|
+ assert decision["score"] == 75
|
|
|
assert decision["decision_action"] == "ADD_TO_CONTENT_POOL"
|
|
assert decision["decision_action"] == "ADD_TO_CONTENT_POOL"
|
|
|
- assert decision["score"] == 80
|
|
|
|
|
- dimensions = {row["key"]: row for row in decision["scorecard"]["dimensions"]}
|
|
|
|
|
- assert dimensions["relevance"]["score"] == 60
|
|
|
|
|
- assert dimensions["platform_heat"]["score"] == 20
|
|
|
|
|
- # 2026-06-12 清理: 5 个 deprecated 维度已从规则包物理删除,scorecard 只剩 2 个 active 维度。
|
|
|
|
|
- assert set(dimensions) == {"relevance", "platform_heat"}
|
|
|
|
|
|
|
+ assert decision["decision_reason_code"] == "v4_query_and_platform_pass"
|
|
|
|
|
+ assert decision["search_query_effect_status"] == "success"
|
|
|
|
|
+ assert decision["scorecard"]["schema_version"] == "v4_scorecard.v1"
|
|
|
|
|
+ assert decision["scorecard"]["query_relevance_score"] == 80
|
|
|
|
|
+ assert decision["scorecard"]["platform_performance_score"] == 70
|
|
|
|
|
+ assert decision["decision_replay_data"]["allow_walk"] is True
|
|
|
|
|
+ assert decision["decision_replay_data"]["walk_gate_snapshot"] == {
|
|
|
|
|
+ "query_relevance_score": 80,
|
|
|
|
|
+ "platform_performance_score": 70,
|
|
|
|
|
+ "score": 75,
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_v4_threshold_boundaries():
|
|
|
|
|
+ cases = [
|
|
|
|
|
+ (54, 100, "REJECT_CONTENT", "v4_query_or_score_below_threshold"),
|
|
|
|
|
+ (55, 55, "KEEP_CONTENT_FOR_REVIEW", "v4_score_review_needed"),
|
|
|
|
|
+ (69, 69, "KEEP_CONTENT_FOR_REVIEW", "v4_score_review_needed"),
|
|
|
|
|
+ (70, 70, "ADD_TO_CONTENT_POOL", "v4_query_and_platform_pass"),
|
|
|
|
|
+ ]
|
|
|
|
|
+ for query, platform, action, reason in cases:
|
|
|
|
|
+ decision = decide("run_1", "policy_1", 1, _bundle(query=query, platform=platform), _policy())
|
|
|
|
|
+ assert decision["decision_action"] == action
|
|
|
|
|
+ assert decision["decision_reason_code"] == reason
|
|
|
|
|
|
|
|
|
|
|
|
|
-def test_rule_pack_scorecard_has_only_two_active_dimensions():
|
|
|
|
|
- # 配置层钉死: 规则包里只剩 relevance + platform_heat,5 个废弃维度定义已删干净。
|
|
|
|
|
- import json
|
|
|
|
|
- from pathlib import Path
|
|
|
|
|
|
|
+def test_v4_allow_walk_requires_platform_65():
|
|
|
|
|
+ decision = decide("run_1", "policy_1", 1, _bundle(query=80, platform=64), _policy())
|
|
|
|
|
|
|
|
- rule_pack = json.loads(
|
|
|
|
|
- Path("product_documents/规则包/douyin_rule_packs.v1.json").read_text(encoding="utf-8")
|
|
|
|
|
- )
|
|
|
|
|
- for pack in rule_pack["rule_packs"]:
|
|
|
|
|
- keys = [dim["key"] for dim in pack["scorecard"]["dimensions"]]
|
|
|
|
|
- assert keys == ["relevance", "platform_heat"], keys
|
|
|
|
|
- assert all(dim["runtime_status"] == "active" for dim in pack["scorecard"]["dimensions"])
|
|
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
-def test_missing_scoring_rules_fail_fast(tmp_path):
|
|
|
|
|
- state = _state(tmp_path)
|
|
|
|
|
- policy_bundle = deepcopy(state["policy_bundle"])
|
|
|
|
|
- policy_bundle["rule_pack"]["scorecard"]["scoring_rules"] = []
|
|
|
|
|
-
|
|
|
|
|
- with pytest.raises(ValueError, match="active scorecard dimensions require"):
|
|
|
|
|
- decide(
|
|
|
|
|
- state["run_id"],
|
|
|
|
|
- state["policy_run_id"],
|
|
|
|
|
- 1,
|
|
|
|
|
- state["evidence_bundles"][0],
|
|
|
|
|
- policy_bundle,
|
|
|
|
|
- )
|
|
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
-def test_no_scoring_evidence_uses_missing_score_policy(tmp_path):
|
|
|
|
|
- state = _state(tmp_path)
|
|
|
|
|
- bundle = deepcopy(state["evidence_bundles"][0])
|
|
|
|
|
- # Drop evidence for both active dims (relevance + platform_heat) so no scoring rule matches.
|
|
|
|
|
- bundle["pattern_match_result"].pop("relevance_score", None)
|
|
|
|
|
- bundle["content_engagement_metrics"].pop("platform_heat", None)
|
|
|
|
|
-
|
|
|
|
|
- decision = decide(state["run_id"], state["policy_run_id"], 1, bundle, state["policy_bundle"])
|
|
|
|
|
-
|
|
|
|
|
- assert decision["decision_action"] == "REJECT_CONTENT"
|
|
|
|
|
- assert decision["decision_reason_code"] == "missing_score"
|
|
|
|
|
- assert decision["search_query_effect_status"] == "failed"
|
|
|
|
|
- assert decision["score"] is None
|
|
|
|
|
|
|
+ assert decision["decision_action"] == "ADD_TO_CONTENT_POOL"
|
|
|
|
|
+ assert decision["score"] == 72
|
|
|
|
|
+ assert decision["decision_replay_data"]["allow_walk"] is False
|
|
|
|
|
|
|
|
|
|
|
|
|
-@pytest.mark.parametrize(
|
|
|
|
|
- ("total_score", "expected_action", "expected_status"),
|
|
|
|
|
- [
|
|
|
|
|
- (59, "REJECT_CONTENT", "failed"),
|
|
|
|
|
- (60, "KEEP_CONTENT_FOR_REVIEW", "pending"),
|
|
|
|
|
- (69, "KEEP_CONTENT_FOR_REVIEW", "pending"),
|
|
|
|
|
- (70, "ADD_TO_CONTENT_POOL", "success"),
|
|
|
|
|
- ],
|
|
|
|
|
-)
|
|
|
|
|
-def test_score_threshold_boundaries(tmp_path, total_score, expected_action, expected_status):
|
|
|
|
|
- state = _state(tmp_path)
|
|
|
|
|
- policy_bundle = _policy_with_total_score(state["policy_bundle"], total_score)
|
|
|
|
|
-
|
|
|
|
|
- decision = decide(
|
|
|
|
|
- state["run_id"],
|
|
|
|
|
- state["policy_run_id"],
|
|
|
|
|
- 1,
|
|
|
|
|
- state["evidence_bundles"][0],
|
|
|
|
|
- policy_bundle,
|
|
|
|
|
- )
|
|
|
|
|
|
|
+def test_v4_missing_score_routes_to_technical_retry_review():
|
|
|
|
|
+ decision = decide("run_1", "policy_1", 1, _bundle(query=None, platform=70), _policy())
|
|
|
|
|
|
|
|
- assert decision["score"] == total_score
|
|
|
|
|
- assert decision["decision_action"] == expected_action
|
|
|
|
|
- assert decision["search_query_effect_status"] == expected_status
|
|
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
-def test_scoring_rule_unknown_operator_fails_fast(tmp_path):
|
|
|
|
|
- state = _state(tmp_path)
|
|
|
|
|
- policy_bundle = deepcopy(state["policy_bundle"])
|
|
|
|
|
- scoring_rules = policy_bundle["rule_pack"]["scorecard"]["scoring_rules"]
|
|
|
|
|
- for rule in scoring_rules:
|
|
|
|
|
- if rule["scoring_rule_id"] == "score_relevance_high":
|
|
|
|
|
- rule["operator"] = "contains"
|
|
|
|
|
-
|
|
|
|
|
- with pytest.raises(ValueError, match="unsupported rule operator"):
|
|
|
|
|
- decide(
|
|
|
|
|
- state["run_id"],
|
|
|
|
|
- state["policy_run_id"],
|
|
|
|
|
- 1,
|
|
|
|
|
- state["evidence_bundles"][0],
|
|
|
|
|
- policy_bundle,
|
|
|
|
|
- )
|
|
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
-def test_single_missing_dimension_scores_zero_and_keeps_threshold_flow(tmp_path):
|
|
|
|
|
- state = _state(tmp_path)
|
|
|
|
|
- bundle = deepcopy(state["evidence_bundles"][0])
|
|
|
|
|
- # relevance evidence present (0.8 -> 60); platform_heat evidence absent -> scores 0, not missing_score.
|
|
|
|
|
- bundle["pattern_match_result"]["relevance_score"] = 0.8
|
|
|
|
|
- bundle["content_engagement_metrics"].pop("platform_heat", None)
|
|
|
|
|
-
|
|
|
|
|
- decision = decide(state["run_id"], state["policy_run_id"], 1, bundle, state["policy_bundle"])
|
|
|
|
|
-
|
|
|
|
|
- dimensions = {row["key"]: row for row in decision["scorecard"]["dimensions"]}
|
|
|
|
|
- assert dimensions["platform_heat"]["score_missing"] is True
|
|
|
|
|
- assert dimensions["platform_heat"]["score"] == 0
|
|
|
|
|
- assert dimensions["relevance"]["score_missing"] is False
|
|
|
|
|
- assert dimensions["relevance"]["score"] == 60
|
|
|
|
|
- assert decision["score"] == 60
|
|
|
|
|
- assert decision["decision_reason_code"] != "missing_score"
|
|
|
|
|
- assert decision["scorecard"]["score_missing"] is False
|
|
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
-def test_all_dimensions_missing_uses_score_missing_policy(tmp_path):
|
|
|
|
|
- state = _state(tmp_path)
|
|
|
|
|
- bundle = deepcopy(state["evidence_bundles"][0])
|
|
|
|
|
- # Both active dims (relevance + platform_heat) lack evidence -> score_missing policy.
|
|
|
|
|
- bundle["pattern_match_result"].pop("relevance_score", None)
|
|
|
|
|
- bundle["content_engagement_metrics"].pop("platform_heat", None)
|
|
|
|
|
-
|
|
|
|
|
- decision = decide(state["run_id"], state["policy_run_id"], 1, bundle, state["policy_bundle"])
|
|
|
|
|
-
|
|
|
|
|
- assert decision["decision_action"] == "REJECT_CONTENT"
|
|
|
|
|
- assert decision["decision_reason_code"] == "missing_score"
|
|
|
|
|
assert decision["score"] is None
|
|
assert decision["score"] is None
|
|
|
assert decision["scorecard"]["score_missing"] is True
|
|
assert decision["scorecard"]["score_missing"] is True
|
|
|
- assert all(row["score_missing"] for row in decision["scorecard"]["dimensions"])
|
|
|
|
|
|
|
+ assert decision["decision_action"] == "KEEP_CONTENT_FOR_REVIEW"
|
|
|
|
|
+ assert decision["decision_reason_code"] == "v4_technical_retry_needed"
|
|
|
|
|
+ assert decision["search_query_effect_status"] == "pending"
|
|
|
|
|
+ assert decision["decision_replay_data"]["allow_walk"] is False
|
|
|
|
|
|
|
|
|
|
|
|
|
-def test_dimension_missing_metadata_is_recorded(tmp_path):
|
|
|
|
|
- state = _state(tmp_path)
|
|
|
|
|
- bundle = deepcopy(state["evidence_bundles"][0])
|
|
|
|
|
- bundle["content_engagement_metrics"].pop("platform_heat", None)
|
|
|
|
|
|
|
+def test_v4_scorecard_and_replay_data_do_not_contain_legacy_fields():
|
|
|
|
|
+ decision = decide("run_1", "policy_1", 1, _bundle(query=80, platform=70), _policy())
|
|
|
|
|
+ keys = _keys({"scorecard": decision["scorecard"], "replay": decision["decision_replay_data"]})
|
|
|
|
|
|
|
|
- decision = decide(state["run_id"], state["policy_run_id"], 1, bundle, state["policy_bundle"])
|
|
|
|
|
- assert decision["decision_replay_data"]["missing_dimensions"] == ["platform_heat"]
|
|
|
|
|
|
|
+ assert not (keys & {"fit_senior_50plus", "fit_confidence", "platform_heat", "relevance_score"})
|
|
|
|
|
|
|
|
- full = deepcopy(state["evidence_bundles"][0])
|
|
|
|
|
- full["content_engagement_metrics"]["platform_heat"] = 0.8
|
|
|
|
|
- full_decision = decide(state["run_id"], state["policy_run_id"], 2, full, state["policy_bundle"])
|
|
|
|
|
- assert full_decision["decision_replay_data"]["missing_dimensions"] == []
|
|
|
|
|
|
|
|
|
|
|
|
+def _bundle(query, platform):
|
|
|
|
|
+ return {
|
|
|
|
|
+ "source_evidence": {"source_kind": "pattern_itemset"},
|
|
|
|
|
+ "content": {
|
|
|
|
|
+ "decision_target_type": "content",
|
|
|
|
|
+ "decision_target_id": "content_1",
|
|
|
|
|
+ "platform_content_id": "content_1",
|
|
|
|
|
+ },
|
|
|
|
|
+ "pattern_match_result": {
|
|
|
|
|
+ "query_relevance_score": query,
|
|
|
|
|
+ "judge_status": "ok",
|
|
|
|
|
+ },
|
|
|
|
|
+ "content_engagement_metrics": {
|
|
|
|
|
+ "platform_performance": {
|
|
|
|
|
+ "schema_version": "v4_platform_performance.v1",
|
|
|
|
|
+ "platform": "douyin",
|
|
|
|
|
+ "platform_performance_score": platform,
|
|
|
|
|
+ "platform_performance_components": [
|
|
|
|
|
+ {
|
|
|
|
|
+ "field": "statistics.digg_count",
|
|
|
|
|
+ "value": 100,
|
|
|
|
|
+ "weight": 1,
|
|
|
|
|
+ "normalized_score": platform or 0,
|
|
|
|
|
+ }
|
|
|
|
|
+ ],
|
|
|
|
|
+ "missing_observable_fields": [],
|
|
|
|
|
+ }
|
|
|
|
|
+ },
|
|
|
|
|
+ "run_context": {"decision_input_snapshot_id": "evidence_bundle:run_1:content_1"},
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
-def _state(tmp_path):
|
|
|
|
|
- service = RunService(
|
|
|
|
|
- runtime_root=tmp_path / "runtime" / "v1",
|
|
|
|
|
- query_variant_client=FakeQueryVariantClient(),
|
|
|
|
|
- )
|
|
|
|
|
- return service.start_run(
|
|
|
|
|
- RunStartRequest(platform_mode="mock", source=str(REAL_SOURCE_FIXTURE))
|
|
|
|
|
- )
|
|
|
|
|
|
|
|
|
|
|
|
+def _policy():
|
|
|
|
|
+ return deepcopy(
|
|
|
|
|
+ {
|
|
|
|
|
+ "policy_bundle_id": "policy_bundle_v4",
|
|
|
|
|
+ "rule_pack_id": "douyin_content_discovery_rule_pack_v1",
|
|
|
|
|
+ "rule_pack_version": "4.0.0",
|
|
|
|
|
+ "strategy_id": "douyin_content_find_v4",
|
|
|
|
|
+ "strategy_version": "V4",
|
|
|
|
|
+ "policy_bundle_hash": "hash",
|
|
|
|
|
+ "dispatch_id": "dispatch_content",
|
|
|
|
|
+ "runtime_stage": "V1.0",
|
|
|
|
|
+ "rule_package_id": "douyin_rule_packs_v1",
|
|
|
|
|
+ "rule_pack": {
|
|
|
|
|
+ "input_contract": {
|
|
|
|
|
+ "required_fields": [
|
|
|
|
|
+ "source_evidence",
|
|
|
|
|
+ "pattern_match_result.query_relevance_score",
|
|
|
|
|
+ "content_engagement_metrics.platform_performance.platform_performance_score",
|
|
|
|
|
+ ]
|
|
|
|
|
+ },
|
|
|
|
|
+ "hard_gates": [],
|
|
|
|
|
+ "scorecard": {"schema_version": "v4_scorecard.v1"},
|
|
|
|
|
+ },
|
|
|
|
|
+ "effect_status_mapping": [
|
|
|
|
|
+ {
|
|
|
|
|
+ "mapping_id": "map_add_to_pool_success",
|
|
|
|
|
+ "target_level": "content",
|
|
|
|
|
+ "decision_action": "ADD_TO_CONTENT_POOL",
|
|
|
|
|
+ "reason_category": "score_pass",
|
|
|
|
|
+ "is_hard_gate": False,
|
|
|
|
|
+ "content_effect_status": "success",
|
|
|
|
|
+ "priority": 10,
|
|
|
|
|
+ "enabled": True,
|
|
|
|
|
+ },
|
|
|
|
|
+ {
|
|
|
|
|
+ "mapping_id": "map_keep_for_review_pending",
|
|
|
|
|
+ "target_level": "content",
|
|
|
|
|
+ "decision_action": "KEEP_CONTENT_FOR_REVIEW",
|
|
|
|
|
+ "reason_category": "review_needed",
|
|
|
|
|
+ "is_hard_gate": False,
|
|
|
|
|
+ "content_effect_status": "pending",
|
|
|
|
|
+ "priority": 20,
|
|
|
|
|
+ "enabled": True,
|
|
|
|
|
+ },
|
|
|
|
|
+ {
|
|
|
|
|
+ "mapping_id": "map_reject_failed",
|
|
|
|
|
+ "target_level": "content",
|
|
|
|
|
+ "decision_action": "REJECT_CONTENT",
|
|
|
|
|
+ "reason_category": "score_or_data_failed",
|
|
|
|
|
+ "is_hard_gate": False,
|
|
|
|
|
+ "content_effect_status": "failed",
|
|
|
|
|
+ "priority": 40,
|
|
|
|
|
+ "enabled": True,
|
|
|
|
|
+ },
|
|
|
|
|
+ ],
|
|
|
|
|
+ }
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
-def _policy_with_total_score(policy_bundle, total_score):
|
|
|
|
|
- """Build an exact total score from the two M3 active dims (relevance max60, platform_heat max40).
|
|
|
|
|
|
|
|
|
|
- Replaces every scoring rule with one always-matching rule per active dimension whose
|
|
|
|
|
- score_value sums to ``total_score`` (relevance carries up to 60, heat the remainder).
|
|
|
|
|
- """
|
|
|
|
|
- policy_bundle = deepcopy(policy_bundle)
|
|
|
|
|
- scorecard = policy_bundle["rule_pack"]["scorecard"]
|
|
|
|
|
- relevance_score = min(total_score, 60)
|
|
|
|
|
- heat_score = total_score - relevance_score
|
|
|
|
|
- assert heat_score <= 40, "total_score exceeds combined active-dimension caps"
|
|
|
|
|
- scorecard["scoring_rules"] = [
|
|
|
|
|
- {
|
|
|
|
|
- "scoring_rule_id": "test_relevance_score",
|
|
|
|
|
- "dimension_key": "relevance",
|
|
|
|
|
- "field_path": "content.decision_target_type",
|
|
|
|
|
- "operator": "eq",
|
|
|
|
|
- "expected_value": "content",
|
|
|
|
|
- "score_value": relevance_score,
|
|
|
|
|
- "priority": 1,
|
|
|
|
|
- "enabled": True,
|
|
|
|
|
- },
|
|
|
|
|
- {
|
|
|
|
|
- "scoring_rule_id": "test_heat_score",
|
|
|
|
|
- "dimension_key": "platform_heat",
|
|
|
|
|
- "field_path": "content.decision_target_type",
|
|
|
|
|
- "operator": "eq",
|
|
|
|
|
- "expected_value": "content",
|
|
|
|
|
- "score_value": heat_score,
|
|
|
|
|
- "priority": 1,
|
|
|
|
|
- "enabled": True,
|
|
|
|
|
- },
|
|
|
|
|
- ]
|
|
|
|
|
- return policy_bundle
|
|
|
|
|
|
|
+def _keys(value):
|
|
|
|
|
+ if isinstance(value, dict):
|
|
|
|
|
+ result = set(value)
|
|
|
|
|
+ for child in value.values():
|
|
|
|
|
+ result |= _keys(child)
|
|
|
|
|
+ return result
|
|
|
|
|
+ if isinstance(value, list):
|
|
|
|
|
+ result = set()
|
|
|
|
|
+ for child in value:
|
|
|
|
|
+ result |= _keys(child)
|
|
|
|
|
+ return result
|
|
|
|
|
+ return set()
|