|
|
@@ -27,8 +27,15 @@ from ...tools.brief_reader import create_brief_reader_tool
|
|
|
from ...run.langgraph_checkpointer import create_sqlite_checkpointer
|
|
|
from ...preprocess.brief_access import (
|
|
|
build_allowed_source_paths,
|
|
|
+ build_global_data_audit_paths,
|
|
|
load_production_brief,
|
|
|
)
|
|
|
+from ...contracts.evaluation import (
|
|
|
+ evaluate_global_data_stage_report,
|
|
|
+ evaluate_stage_validator_candidate,
|
|
|
+ stage_validation_verdict,
|
|
|
+)
|
|
|
+from ...contracts.identifiers import stage_validator_run_id_for
|
|
|
from ...tools.registry import create_default_tool_registry
|
|
|
from .task_agent import (
|
|
|
ValidatorOutputError,
|
|
|
@@ -39,134 +46,20 @@ from .task_context import build_validator_system_prompt
|
|
|
from .skills.registry import load_global_data_stage_validator_skill
|
|
|
from .stage_context import (
|
|
|
build_stage_validator_user_message,
|
|
|
- create_stage_validator_run_id,
|
|
|
- required_stage_audit_paths,
|
|
|
validate_stage_candidate_identity,
|
|
|
)
|
|
|
|
|
|
|
|
|
-def _validate_candidate(
|
|
|
- output: GlobalDataStageValidatorCandidate,
|
|
|
- *,
|
|
|
- plan: GlobalDataPlan,
|
|
|
- candidate: GlobalDataStageCandidate,
|
|
|
- actually_inspected_paths: set[str] | None = None,
|
|
|
-) -> None:
|
|
|
- mismatches: list[str] = []
|
|
|
- for field in ("run_id", "plan_id", "plan_version"):
|
|
|
- if getattr(output, field) != getattr(candidate, field):
|
|
|
- mismatches.append(field)
|
|
|
- expected_ids = [
|
|
|
- item.requirement_id for item in plan.stage_requirements
|
|
|
- ]
|
|
|
- if [
|
|
|
- item.requirement_id for item in output.requirement_results
|
|
|
- ] != expected_ids:
|
|
|
- mismatches.append("requirement_results")
|
|
|
- if any(not item.evidence for item in output.requirement_results):
|
|
|
- mismatches.append("requirement_results.evidence")
|
|
|
-
|
|
|
- artifact_ids = {
|
|
|
- artifact.artifact_id for artifact in candidate.artifacts
|
|
|
- }
|
|
|
- required_paths = set(
|
|
|
- required_stage_audit_paths(candidate.production_brief_uri)
|
|
|
- )
|
|
|
- declared_paths = set(output.inspected_source_paths)
|
|
|
- if not required_paths.issubset(declared_paths):
|
|
|
- mismatches.append("inspected_source_paths")
|
|
|
- if (
|
|
|
- actually_inspected_paths is not None
|
|
|
- and not declared_paths.issubset(actually_inspected_paths)
|
|
|
- ):
|
|
|
- mismatches.append("inspected_source_paths.tool_evidence")
|
|
|
- evidence_ids = {
|
|
|
- artifact_id
|
|
|
- for result in output.requirement_results
|
|
|
- for artifact_id in result.evidence_artifact_ids
|
|
|
- }
|
|
|
- if evidence_ids.difference(artifact_ids):
|
|
|
- mismatches.append("requirement_results.evidence_artifact_ids")
|
|
|
- if {
|
|
|
- item.artifact_id for item in output.artifact_rejections
|
|
|
- }.difference(artifact_ids):
|
|
|
- mismatches.append("artifact_rejections")
|
|
|
-
|
|
|
- artifact_ids_by_requirement = {
|
|
|
- item.requirement_id: set(item.actual_artifact_ids)
|
|
|
- for item in candidate.requirement_evaluations
|
|
|
- }
|
|
|
- if any(
|
|
|
- not set(result.evidence_artifact_ids).issubset(
|
|
|
- artifact_ids_by_requirement.get(result.requirement_id, set())
|
|
|
- )
|
|
|
- for result in output.requirement_results
|
|
|
- ):
|
|
|
- mismatches.append("requirement_results.cross_requirement_evidence")
|
|
|
- gap_requirement_ids = {
|
|
|
- gap.requirement_id for gap in candidate.expectation_gaps
|
|
|
- }
|
|
|
- if any(
|
|
|
- result.verdict == "PASS"
|
|
|
- and result.requirement_id in gap_requirement_ids
|
|
|
- for result in output.requirement_results
|
|
|
- ):
|
|
|
- mismatches.append("requirement_results.expectation_gaps")
|
|
|
-
|
|
|
- _, brief = load_production_brief(candidate.production_brief_uri)
|
|
|
- allowed_paths = set(build_allowed_source_paths(brief))
|
|
|
- if any(
|
|
|
- set(item.source_paths).difference(allowed_paths)
|
|
|
- for item in output.missing_requirements
|
|
|
- ):
|
|
|
- mismatches.append("missing_requirements.source_paths")
|
|
|
- if mismatches:
|
|
|
- raise ValidatorOutputError(
|
|
|
- "Stage ValidatorCandidate 身份、Requirement 或 Artifact 不匹配:"
|
|
|
- + ", ".join(mismatches)
|
|
|
- )
|
|
|
-
|
|
|
-
|
|
|
-def _has_blocking_failure(
|
|
|
- plan: GlobalDataPlan,
|
|
|
- candidate: GlobalDataStageCandidate,
|
|
|
- output: GlobalDataStageValidatorCandidate,
|
|
|
-) -> bool:
|
|
|
- importance = {
|
|
|
- item.requirement_id: item.importance
|
|
|
- for item in plan.stage_requirements
|
|
|
- }
|
|
|
- if any(
|
|
|
- importance[result.requirement_id] == "critical"
|
|
|
- and result.verdict == "FAIL"
|
|
|
- for result in output.requirement_results
|
|
|
- ):
|
|
|
- return True
|
|
|
- if any(
|
|
|
- importance[gap.requirement_id] == "critical"
|
|
|
- for gap in candidate.expectation_gaps
|
|
|
- ):
|
|
|
- return True
|
|
|
- return (
|
|
|
- any(
|
|
|
- item.importance == "critical"
|
|
|
- for item in output.missing_requirements
|
|
|
- )
|
|
|
- or bool(output.artifact_rejections)
|
|
|
- )
|
|
|
-
|
|
|
-
|
|
|
def _build_report(
|
|
|
plan: GlobalDataPlan,
|
|
|
candidate: GlobalDataStageCandidate,
|
|
|
output: GlobalDataStageValidatorCandidate,
|
|
|
validator_run_id: str,
|
|
|
) -> GlobalDataStageValidationReport:
|
|
|
- failed = _has_blocking_failure(plan, candidate, output)
|
|
|
return GlobalDataStageValidationReport(
|
|
|
**output.model_dump(),
|
|
|
validator_run_id=validator_run_id,
|
|
|
- verdict="FAIL" if failed else "PASS",
|
|
|
+ verdict=stage_validation_verdict(plan, candidate, output),
|
|
|
)
|
|
|
|
|
|
|
|
|
@@ -175,23 +68,18 @@ def validate_global_data_stage_report(
|
|
|
candidate: GlobalDataStageCandidate,
|
|
|
report: GlobalDataStageValidationReport,
|
|
|
) -> None:
|
|
|
- validate_stage_candidate_identity(plan, candidate)
|
|
|
- output = GlobalDataStageValidatorCandidate.model_validate(
|
|
|
- report.model_dump(exclude={"validator_run_id", "verdict"})
|
|
|
- )
|
|
|
- _validate_candidate(output, plan=plan, candidate=candidate)
|
|
|
- if report.validator_run_id != create_stage_validator_run_id(candidate):
|
|
|
- raise ValidatorOutputError(
|
|
|
- "Stage Report.validator_run_id 与当前 Candidate 不一致"
|
|
|
- )
|
|
|
- verdict = (
|
|
|
- "FAIL"
|
|
|
- if _has_blocking_failure(plan, candidate, output)
|
|
|
- else "PASS"
|
|
|
+ _, brief = load_production_brief(candidate.production_brief_uri)
|
|
|
+ issues = evaluate_global_data_stage_report(
|
|
|
+ plan,
|
|
|
+ candidate,
|
|
|
+ report,
|
|
|
+ required_source_paths=set(build_global_data_audit_paths(brief)),
|
|
|
+ allowed_source_paths=set(build_allowed_source_paths(brief)),
|
|
|
)
|
|
|
- if report.verdict != verdict:
|
|
|
+ if issues:
|
|
|
raise ValidatorOutputError(
|
|
|
- "Stage Report 的逐项结果与 verdict 不一致"
|
|
|
+ "Stage ValidationReport 合同无效:"
|
|
|
+ + ";".join(f"{item.code}: {item.message}" for item in issues)
|
|
|
)
|
|
|
|
|
|
|
|
|
@@ -243,7 +131,7 @@ def run_global_data_stage_validator(
|
|
|
validate_stage_candidate_identity(plan, candidate)
|
|
|
_, brief = load_production_brief(candidate.production_brief_uri)
|
|
|
skill = load_global_data_stage_validator_skill()
|
|
|
- validator_run_id = create_stage_validator_run_id(candidate)
|
|
|
+ validator_run_id = stage_validator_run_id_for(candidate)
|
|
|
config = {
|
|
|
"configurable": {"thread_id": validator_run_id},
|
|
|
"recursion_limit": 16,
|
|
|
@@ -287,14 +175,28 @@ def run_global_data_stage_validator(
|
|
|
error_type=ValidatorOutputError,
|
|
|
)
|
|
|
)
|
|
|
- _validate_candidate(
|
|
|
+ issues = evaluate_stage_validator_candidate(
|
|
|
+ plan,
|
|
|
+ candidate,
|
|
|
output,
|
|
|
- plan=plan,
|
|
|
- candidate=candidate,
|
|
|
+ required_source_paths=set(
|
|
|
+ build_global_data_audit_paths(brief)
|
|
|
+ ),
|
|
|
+ allowed_source_paths=set(
|
|
|
+ build_allowed_source_paths(brief)
|
|
|
+ ),
|
|
|
actually_inspected_paths=_inspected_paths(
|
|
|
result["messages"]
|
|
|
),
|
|
|
)
|
|
|
+ if issues:
|
|
|
+ raise ValidatorOutputError(
|
|
|
+ "Stage ValidatorCandidate 合同无效:"
|
|
|
+ + ";".join(
|
|
|
+ f"{item.code}: {item.message}"
|
|
|
+ for item in issues
|
|
|
+ )
|
|
|
+ )
|
|
|
break
|
|
|
except (ValidationError, ValidatorOutputError, ValueError) as exc:
|
|
|
if attempt >= max_attempts:
|