|
|
@@ -8,12 +8,12 @@ from typing import Iterable, Mapping, Sequence
|
|
|
from ..agents.executor.skills.registry import SkillCapability
|
|
|
from .models import (
|
|
|
ContractIssue,
|
|
|
- CoverageGap,
|
|
|
+ ExpectationGap,
|
|
|
ExecutorDelivery,
|
|
|
- ExpectationCoverage,
|
|
|
+ ExpectationEvaluation,
|
|
|
GlobalDataPlan,
|
|
|
ProductionBrief,
|
|
|
- RequirementCoverage,
|
|
|
+ RequirementEvaluation,
|
|
|
StageEvaluation,
|
|
|
TaskPackage,
|
|
|
ValidationReport,
|
|
|
@@ -204,7 +204,7 @@ def evaluate_task_delivery(
|
|
|
delivery: ExecutorDelivery,
|
|
|
capability_catalog: Mapping[str, SkillCapability],
|
|
|
) -> list[ContractIssue]:
|
|
|
- """在模型 Validator 前验证显式 Fulfillment 的完整性。"""
|
|
|
+ """在模型 Validator 前验证显式 Binding 的完整性。"""
|
|
|
|
|
|
issues: list[ContractIssue] = []
|
|
|
for field in ("run_id", "plan_id", "task_id", "plan_version"):
|
|
|
@@ -283,46 +283,46 @@ def evaluate_task_delivery(
|
|
|
asset.source_asset_id: asset for asset in task.source_assets
|
|
|
}
|
|
|
tool_ids = {item.tool_call_id for item in delivery.tool_calls}
|
|
|
- fulfillment_keys: set[tuple[str, str, str | None]] = set()
|
|
|
+ binding_keys: set[tuple[str, str, str | None]] = set()
|
|
|
by_expectation: dict[str, list] = defaultdict(list)
|
|
|
- for fulfillment in delivery.fulfillments:
|
|
|
+ for binding in delivery.artifact_expectation_bindings:
|
|
|
key = (
|
|
|
- fulfillment.expectation_id,
|
|
|
- fulfillment.artifact_id,
|
|
|
- fulfillment.source_asset_id,
|
|
|
+ binding.expectation_id,
|
|
|
+ binding.artifact_id,
|
|
|
+ binding.source_asset_id,
|
|
|
)
|
|
|
- if key in fulfillment_keys:
|
|
|
+ if key in binding_keys:
|
|
|
issues.append(
|
|
|
_issue(
|
|
|
- "duplicate_fulfillment",
|
|
|
- "同一 ArtifactFulfillment 重复",
|
|
|
+ "duplicate_artifact_binding",
|
|
|
+ "同一 ArtifactExpectationBinding 重复",
|
|
|
task_id=task.task_id,
|
|
|
- expectation_id=fulfillment.expectation_id,
|
|
|
- artifact_id=fulfillment.artifact_id,
|
|
|
+ expectation_id=binding.expectation_id,
|
|
|
+ artifact_id=binding.artifact_id,
|
|
|
)
|
|
|
)
|
|
|
continue
|
|
|
- fulfillment_keys.add(key)
|
|
|
- expectation = expectations.get(fulfillment.expectation_id)
|
|
|
- artifact = artifacts.get(fulfillment.artifact_id)
|
|
|
+ binding_keys.add(key)
|
|
|
+ expectation = expectations.get(binding.expectation_id)
|
|
|
+ artifact = artifacts.get(binding.artifact_id)
|
|
|
if expectation is None:
|
|
|
issues.append(
|
|
|
_issue(
|
|
|
- "unknown_fulfillment_expectation",
|
|
|
- "Fulfillment 引用了 Task 未授权的 Expectation",
|
|
|
+ "unknown_binding_expectation",
|
|
|
+ "Binding 引用了 Task 未授权的 Expectation",
|
|
|
task_id=task.task_id,
|
|
|
- expectation_id=fulfillment.expectation_id,
|
|
|
+ expectation_id=binding.expectation_id,
|
|
|
)
|
|
|
)
|
|
|
continue
|
|
|
if artifact is None:
|
|
|
issues.append(
|
|
|
_issue(
|
|
|
- "unknown_fulfillment_artifact",
|
|
|
- "Fulfillment 引用了不存在的 Artifact",
|
|
|
+ "unknown_binding_artifact",
|
|
|
+ "Binding 引用了不存在的 Artifact",
|
|
|
task_id=task.task_id,
|
|
|
- expectation_id=fulfillment.expectation_id,
|
|
|
- artifact_id=fulfillment.artifact_id,
|
|
|
+ expectation_id=binding.expectation_id,
|
|
|
+ artifact_id=binding.artifact_id,
|
|
|
)
|
|
|
)
|
|
|
continue
|
|
|
@@ -334,43 +334,43 @@ def evaluate_task_delivery(
|
|
|
"unsupported_delivery_artifact",
|
|
|
"Skill 不支持 Delivery 中的 Artifact 类型",
|
|
|
task_id=task.task_id,
|
|
|
- expectation_id=fulfillment.expectation_id,
|
|
|
- artifact_id=fulfillment.artifact_id,
|
|
|
+ expectation_id=binding.expectation_id,
|
|
|
+ artifact_id=binding.artifact_id,
|
|
|
)
|
|
|
)
|
|
|
if artifact.artifact_type != expectation.artifact_type:
|
|
|
issues.append(
|
|
|
_issue(
|
|
|
- "fulfillment_artifact_type_mismatch",
|
|
|
+ "binding_artifact_type_mismatch",
|
|
|
"Artifact 类型与 Expectation 不一致",
|
|
|
task_id=task.task_id,
|
|
|
- expectation_id=fulfillment.expectation_id,
|
|
|
- artifact_id=fulfillment.artifact_id,
|
|
|
+ expectation_id=binding.expectation_id,
|
|
|
+ artifact_id=binding.artifact_id,
|
|
|
)
|
|
|
)
|
|
|
missing_tools = set(
|
|
|
- fulfillment.evidence_tool_call_ids
|
|
|
+ binding.evidence_tool_call_ids
|
|
|
).difference(tool_ids)
|
|
|
if missing_tools:
|
|
|
issues.append(
|
|
|
_issue(
|
|
|
- "unknown_fulfillment_evidence",
|
|
|
- "Fulfillment 引用了不存在的 ToolCallRecord",
|
|
|
+ "unknown_binding_evidence",
|
|
|
+ "Binding 引用了不存在的 ToolCallRecord",
|
|
|
task_id=task.task_id,
|
|
|
- expectation_id=fulfillment.expectation_id,
|
|
|
- artifact_id=fulfillment.artifact_id,
|
|
|
+ expectation_id=binding.expectation_id,
|
|
|
+ artifact_id=binding.artifact_id,
|
|
|
)
|
|
|
)
|
|
|
if expectation.source_asset_ids:
|
|
|
- if fulfillment.source_asset_id not in expectation.source_asset_ids:
|
|
|
+ if binding.source_asset_id not in expectation.source_asset_ids:
|
|
|
issues.append(
|
|
|
_issue(
|
|
|
"invalid_source_asset_binding",
|
|
|
- "Fulfillment 未绑定该 Expectation 授权的 SourceAsset",
|
|
|
+ "Binding 未绑定该 Expectation 授权的 SourceAsset",
|
|
|
task_id=task.task_id,
|
|
|
- expectation_id=fulfillment.expectation_id,
|
|
|
- artifact_id=fulfillment.artifact_id,
|
|
|
- source_asset_id=fulfillment.source_asset_id,
|
|
|
+ expectation_id=binding.expectation_id,
|
|
|
+ artifact_id=binding.artifact_id,
|
|
|
+ source_asset_id=binding.source_asset_id,
|
|
|
)
|
|
|
)
|
|
|
if delivery.skill_id != "reference-inspection":
|
|
|
@@ -379,12 +379,12 @@ def evaluate_task_delivery(
|
|
|
"generated_artifact_cannot_adopt_source",
|
|
|
"只有 reference-inspection 可满足 source_identity",
|
|
|
task_id=task.task_id,
|
|
|
- expectation_id=fulfillment.expectation_id,
|
|
|
- artifact_id=fulfillment.artifact_id,
|
|
|
+ expectation_id=binding.expectation_id,
|
|
|
+ artifact_id=binding.artifact_id,
|
|
|
)
|
|
|
)
|
|
|
source_asset = source_assets.get(
|
|
|
- fulfillment.source_asset_id or ""
|
|
|
+ binding.source_asset_id or ""
|
|
|
)
|
|
|
if (
|
|
|
source_asset is not None
|
|
|
@@ -398,33 +398,33 @@ def evaluate_task_delivery(
|
|
|
"source_artifact_identity_mismatch",
|
|
|
"Artifact 的类型或 source_uri 与 SourceAsset 不一致",
|
|
|
task_id=task.task_id,
|
|
|
- expectation_id=fulfillment.expectation_id,
|
|
|
- artifact_id=fulfillment.artifact_id,
|
|
|
- source_asset_id=fulfillment.source_asset_id,
|
|
|
+ expectation_id=binding.expectation_id,
|
|
|
+ artifact_id=binding.artifact_id,
|
|
|
+ source_asset_id=binding.source_asset_id,
|
|
|
)
|
|
|
)
|
|
|
- elif fulfillment.source_asset_id is not None:
|
|
|
+ elif binding.source_asset_id is not None:
|
|
|
issues.append(
|
|
|
_issue(
|
|
|
"unexpected_source_asset_binding",
|
|
|
"非 source-bound Expectation 不得绑定 SourceAsset",
|
|
|
task_id=task.task_id,
|
|
|
- expectation_id=fulfillment.expectation_id,
|
|
|
- artifact_id=fulfillment.artifact_id,
|
|
|
+ expectation_id=binding.expectation_id,
|
|
|
+ artifact_id=binding.artifact_id,
|
|
|
)
|
|
|
)
|
|
|
- by_expectation[fulfillment.expectation_id].append(fulfillment)
|
|
|
+ by_expectation[binding.expectation_id].append(binding)
|
|
|
|
|
|
for expectation_id, expectation in expectations.items():
|
|
|
- fulfillments = by_expectation[expectation_id]
|
|
|
+ artifact_expectation_bindings = by_expectation[expectation_id]
|
|
|
artifact_count = len(
|
|
|
- {item.artifact_id for item in fulfillments}
|
|
|
+ {item.artifact_id for item in artifact_expectation_bindings}
|
|
|
)
|
|
|
if artifact_count < expectation.minimum_count:
|
|
|
issues.append(
|
|
|
_issue(
|
|
|
- "insufficient_fulfillments",
|
|
|
- "显式 ArtifactFulfillment 数量不足",
|
|
|
+ "insufficient_artifact_bindings",
|
|
|
+ "显式 ArtifactExpectationBinding 数量不足",
|
|
|
task_id=task.task_id,
|
|
|
expectation_id=expectation_id,
|
|
|
)
|
|
|
@@ -432,7 +432,7 @@ def evaluate_task_delivery(
|
|
|
if expectation.source_asset_ids:
|
|
|
delivered_sources = {
|
|
|
item.source_asset_id
|
|
|
- for item in fulfillments
|
|
|
+ for item in artifact_expectation_bindings
|
|
|
if item.source_asset_id is not None
|
|
|
}
|
|
|
for source_asset_id in set(
|
|
|
@@ -440,8 +440,8 @@ def evaluate_task_delivery(
|
|
|
).difference(delivered_sources):
|
|
|
issues.append(
|
|
|
_issue(
|
|
|
- "missing_source_asset_fulfillment",
|
|
|
- "SourceAsset 没有独立的 ArtifactFulfillment",
|
|
|
+ "missing_source_asset_binding",
|
|
|
+ "SourceAsset 没有独立的 ArtifactExpectationBinding",
|
|
|
task_id=task.task_id,
|
|
|
expectation_id=expectation_id,
|
|
|
source_asset_id=source_asset_id,
|
|
|
@@ -455,7 +455,7 @@ def evaluate_stage(
|
|
|
passed_deliveries: Sequence[ExecutorDelivery],
|
|
|
task_reports: Sequence[ValidationReport],
|
|
|
) -> StageEvaluation:
|
|
|
- """只依据 PASS Delivery 中的显式 Fulfillment 计算阶段覆盖。"""
|
|
|
+ """只依据 PASS Delivery 中的正式绑定计算阶段满足情况。"""
|
|
|
|
|
|
reports = {report.task_id: report for report in task_reports}
|
|
|
deliveries = {
|
|
|
@@ -485,33 +485,33 @@ def evaluate_stage(
|
|
|
if artifact.artifact_id in replacements
|
|
|
]
|
|
|
active_ids = {artifact.artifact_id for artifact in active}
|
|
|
- fulfillment_rows = [
|
|
|
- (task_id, fulfillment)
|
|
|
+ binding_rows = [
|
|
|
+ (task_id, binding)
|
|
|
for task_id, delivery in deliveries.items()
|
|
|
- for fulfillment in delivery.fulfillments
|
|
|
- if fulfillment.artifact_id in active_ids
|
|
|
+ for binding in delivery.artifact_expectation_bindings
|
|
|
+ if binding.artifact_id in active_ids
|
|
|
]
|
|
|
|
|
|
- coverages: list[RequirementCoverage] = []
|
|
|
- gaps: list[CoverageGap] = []
|
|
|
+ requirement_evaluations: list[RequirementEvaluation] = []
|
|
|
+ gaps: list[ExpectationGap] = []
|
|
|
for requirement in plan.stage_requirements:
|
|
|
- expectation_coverages: list[ExpectationCoverage] = []
|
|
|
+ expectation_evaluations: list[ExpectationEvaluation] = []
|
|
|
requirement_artifact_ids: set[str] = set()
|
|
|
requirement_task_ids: set[str] = set()
|
|
|
for expectation in requirement.artifact_expectations:
|
|
|
rows = [
|
|
|
- (task_id, fulfillment)
|
|
|
- for task_id, fulfillment in fulfillment_rows
|
|
|
- if fulfillment.expectation_id == expectation.expectation_id
|
|
|
+ (task_id, binding)
|
|
|
+ for task_id, binding in binding_rows
|
|
|
+ if binding.expectation_id == expectation.expectation_id
|
|
|
]
|
|
|
artifact_ids = sorted(
|
|
|
- {fulfillment.artifact_id for _, fulfillment in rows}
|
|
|
+ {binding.artifact_id for _, binding in rows}
|
|
|
)
|
|
|
source_asset_ids = sorted(
|
|
|
{
|
|
|
- fulfillment.source_asset_id
|
|
|
- for _, fulfillment in rows
|
|
|
- if fulfillment.source_asset_id is not None
|
|
|
+ binding.source_asset_id
|
|
|
+ for _, binding in rows
|
|
|
+ if binding.source_asset_id is not None
|
|
|
}
|
|
|
)
|
|
|
missing_sources = sorted(
|
|
|
@@ -523,8 +523,8 @@ def evaluate_stage(
|
|
|
len(artifact_ids) >= expectation.minimum_count
|
|
|
and not missing_sources
|
|
|
)
|
|
|
- expectation_coverages.append(
|
|
|
- ExpectationCoverage(
|
|
|
+ expectation_evaluations.append(
|
|
|
+ ExpectationEvaluation(
|
|
|
expectation_id=expectation.expectation_id,
|
|
|
expected_count=expectation.minimum_count,
|
|
|
actual_count=len(artifact_ids),
|
|
|
@@ -537,7 +537,7 @@ def evaluate_stage(
|
|
|
requirement_task_ids.update(task_id for task_id, _ in rows)
|
|
|
if not satisfied:
|
|
|
gaps.append(
|
|
|
- CoverageGap(
|
|
|
+ ExpectationGap(
|
|
|
requirement_id=requirement.requirement_id,
|
|
|
expectation_id=expectation.expectation_id,
|
|
|
expected_count=expectation.minimum_count,
|
|
|
@@ -546,21 +546,21 @@ def evaluate_stage(
|
|
|
reason_code=(
|
|
|
"missing_source_assets"
|
|
|
if missing_sources
|
|
|
- else "insufficient_fulfillments"
|
|
|
+ else "insufficient_artifact_bindings"
|
|
|
),
|
|
|
)
|
|
|
)
|
|
|
- coverages.append(
|
|
|
- RequirementCoverage(
|
|
|
+ requirement_evaluations.append(
|
|
|
+ RequirementEvaluation(
|
|
|
requirement_id=requirement.requirement_id,
|
|
|
- covering_task_ids=sorted(requirement_task_ids),
|
|
|
- expectation_coverages=expectation_coverages,
|
|
|
+ supporting_task_ids=sorted(requirement_task_ids),
|
|
|
+ expectation_evaluations=expectation_evaluations,
|
|
|
actual_artifact_ids=sorted(requirement_artifact_ids),
|
|
|
)
|
|
|
)
|
|
|
return StageEvaluation(
|
|
|
- requirement_coverages=coverages,
|
|
|
- coverage_gaps=gaps,
|
|
|
+ requirement_evaluations=requirement_evaluations,
|
|
|
+ expectation_gaps=gaps,
|
|
|
active_artifacts=active,
|
|
|
superseded_artifacts=superseded,
|
|
|
)
|
|
|
@@ -571,7 +571,7 @@ def validate_replan(
|
|
|
next_plan: GlobalDataPlan,
|
|
|
passed_task_ids: Iterable[str],
|
|
|
rejected_artifact_ids: Iterable[str],
|
|
|
- stage_gaps: Sequence[CoverageGap],
|
|
|
+ stage_gaps: Sequence[ExpectationGap],
|
|
|
) -> list[ContractIssue]:
|
|
|
"""验证 Replan 版本和不可变历史,不依赖 Validator 文案。"""
|
|
|
|