|
|
@@ -113,6 +113,7 @@ from cyber_agent.core.artifacts import (
|
|
|
ValidationMaterial,
|
|
|
extract_artifact_refs,
|
|
|
material_chars,
|
|
|
+ material_content_hash,
|
|
|
resolve_artifact_refs,
|
|
|
)
|
|
|
from cyber_agent.core.validation import (
|
|
|
@@ -505,6 +506,7 @@ class AgentRunner:
|
|
|
tool_calls: int = 0,
|
|
|
material_chars_count: int = 0,
|
|
|
provider_cost_usd: float = 0.0,
|
|
|
+ operation_id: str | None = None,
|
|
|
) -> None:
|
|
|
"""把 Validator网页工具和真实材料字符计入同一棵树的预算。"""
|
|
|
resolved = await self._resource_budget_for_trace(trace_id)
|
|
|
@@ -519,6 +521,7 @@ class AgentRunner:
|
|
|
tool_calls=tool_calls,
|
|
|
material_chars=material_chars_count,
|
|
|
provider_cost_usd=provider_cost_usd,
|
|
|
+ operation_id=operation_id,
|
|
|
)
|
|
|
|
|
|
async def validate_recursive_trace(
|
|
|
@@ -693,17 +696,51 @@ class AgentRunner:
|
|
|
fixed_checks: dict[str, ValidationCheck] = {}
|
|
|
fixed_scope_errors: dict[ValidationScope, str] = {}
|
|
|
quality_rules: tuple[Any, ...] = ()
|
|
|
- if candidate_ref is not None:
|
|
|
+ quality_materials: tuple[ValidationMaterial, ...] = ()
|
|
|
+ if self.application_binding is not None:
|
|
|
from cyber_agent.application.quality import ValidationSubject
|
|
|
|
|
|
+ brief_payload = (
|
|
|
+ task_brief.model_dump(mode="json")
|
|
|
+ if hasattr(task_brief, "model_dump")
|
|
|
+ else (task_brief or {})
|
|
|
+ )
|
|
|
+ requested_quality_scopes = (
|
|
|
+ {"output"}
|
|
|
+ if candidate_ref is not None
|
|
|
+ else set(policy.effective_scopes(
|
|
|
+ brief_payload.get("validation_scopes", []),
|
|
|
+ root=root_validator,
|
|
|
+ ))
|
|
|
+ )
|
|
|
validation_subject = ValidationSubject(
|
|
|
- subject_type="candidate",
|
|
|
+ subject_type=("candidate" if candidate_ref is not None else "trace"),
|
|
|
trace_id=evaluated_trace_id,
|
|
|
candidate_ref=candidate_ref,
|
|
|
)
|
|
|
quality_rules = tuple(
|
|
|
- self.application_binding.application.quality_rules
|
|
|
+ rule
|
|
|
+ for rule in self.application_binding.application.quality_rules
|
|
|
+ if validation_subject.subject_type in rule.subject_types
|
|
|
+ and rule.scope in requested_quality_scopes
|
|
|
)
|
|
|
+ if candidate_ref is not None and candidate_material is not None:
|
|
|
+ quality_materials = (candidate_material,)
|
|
|
+ elif candidate_ref is None:
|
|
|
+ quality_materials = tuple(materials)
|
|
|
+ if isinstance(task_report, dict):
|
|
|
+ report_material = ValidationMaterial(
|
|
|
+ artifact_id=f"trace:{evaluated_trace_id}:task_report",
|
|
|
+ version=str(progress_revision or 0),
|
|
|
+ content_hash=material_content_hash(task_report),
|
|
|
+ kind="framework.task_report",
|
|
|
+ mime_type="application/json",
|
|
|
+ root_trace_id=root_trace_id,
|
|
|
+ uid=evaluated.uid,
|
|
|
+ content=task_report,
|
|
|
+ )
|
|
|
+ quality_materials = (*quality_materials, report_material)
|
|
|
+ total_material_chars += material_chars(report_material)
|
|
|
quality_specs = [
|
|
|
ValidationCheckSpec(
|
|
|
check_id=f"quality.{rule.rule_id}",
|
|
|
@@ -753,12 +790,6 @@ class AgentRunner:
|
|
|
cached=True,
|
|
|
)
|
|
|
cached = None
|
|
|
- if quality_rules and candidate_material is not None:
|
|
|
- fixed_checks, fixed_scope_errors = await self._run_quality_checks(
|
|
|
- validation_subject,
|
|
|
- candidate_material,
|
|
|
- quality_rules,
|
|
|
- )
|
|
|
validation_cache: dict[str, Any]
|
|
|
resume_scope_results: list[ScopeValidationResult] = []
|
|
|
if isinstance(cached, dict) and cached.get("plan_hash") == plan.plan_hash:
|
|
|
@@ -801,6 +832,77 @@ class AgentRunner:
|
|
|
context=evaluated.context,
|
|
|
)
|
|
|
|
|
|
+ candidate_checkpoint = None
|
|
|
+ if candidate_ref is not None:
|
|
|
+ candidate_checkpoint = await self.candidate_service.begin_validation_checkpoint(
|
|
|
+ evaluated_trace_id,
|
|
|
+ candidate_ref,
|
|
|
+ plan=plan.model_dump(mode="json"),
|
|
|
+ plan_hash=plan.plan_hash,
|
|
|
+ validated_at_sequence=plan.evaluated_head_sequence,
|
|
|
+ material_usage_recorded=total_material_chars == 0,
|
|
|
+ )
|
|
|
+ validation_cache = {
|
|
|
+ "validation_plan": candidate_checkpoint.validation_plan,
|
|
|
+ "plan_hash": candidate_checkpoint.plan_hash,
|
|
|
+ "scope_results": list(candidate_checkpoint.scope_results),
|
|
|
+ "aggregate_result": candidate_checkpoint.aggregate_result,
|
|
|
+ "validated_at_sequence": candidate_checkpoint.validated_at_sequence,
|
|
|
+ "material_usage_recorded": (
|
|
|
+ candidate_checkpoint.material_usage_recorded
|
|
|
+ ),
|
|
|
+ }
|
|
|
+ resume_scope_results = [
|
|
|
+ ScopeValidationResult.model_validate(item)
|
|
|
+ for item in candidate_checkpoint.scope_results
|
|
|
+ ]
|
|
|
+ if quality_rules:
|
|
|
+ if candidate_checkpoint.quality_completed:
|
|
|
+ fixed_checks = {
|
|
|
+ item["check_id"]: ValidationCheck.model_validate(item)
|
|
|
+ for item in candidate_checkpoint.fixed_checks
|
|
|
+ }
|
|
|
+ fixed_scope_errors = {
|
|
|
+ key: value
|
|
|
+ for key, value in candidate_checkpoint.fixed_scope_errors.items()
|
|
|
+ }
|
|
|
+ else:
|
|
|
+ fixed_checks, fixed_scope_errors = await self._run_quality_checks(
|
|
|
+ validation_subject,
|
|
|
+ quality_materials,
|
|
|
+ quality_rules,
|
|
|
+ usage_operation_prefix=(
|
|
|
+ f"candidate-validation:{candidate_ref.candidate_id}:"
|
|
|
+ f"{candidate_ref.revision}:{plan.plan_hash}"
|
|
|
+ ),
|
|
|
+ )
|
|
|
+ candidate_checkpoint = (
|
|
|
+ await self.candidate_service.update_validation_checkpoint(
|
|
|
+ evaluated_trace_id,
|
|
|
+ candidate_ref,
|
|
|
+ plan.plan_hash,
|
|
|
+ fixed_checks=tuple(
|
|
|
+ item.model_dump(mode="json")
|
|
|
+ for item in fixed_checks.values()
|
|
|
+ ),
|
|
|
+ fixed_scope_errors={
|
|
|
+ key: value
|
|
|
+ for key, value in fixed_scope_errors.items()
|
|
|
+ },
|
|
|
+ quality_completed=True,
|
|
|
+ )
|
|
|
+ )
|
|
|
+
|
|
|
+ if candidate_ref is None and quality_rules:
|
|
|
+ fixed_checks, fixed_scope_errors = await self._run_quality_checks(
|
|
|
+ validation_subject,
|
|
|
+ quality_materials,
|
|
|
+ quality_rules,
|
|
|
+ usage_operation_prefix=(
|
|
|
+ f"trace-validation:{evaluated_trace_id}:{plan.plan_hash}"
|
|
|
+ ),
|
|
|
+ )
|
|
|
+
|
|
|
if (
|
|
|
total_material_chars
|
|
|
and not validation_cache.get("material_usage_recorded", False)
|
|
|
@@ -808,9 +910,20 @@ class AgentRunner:
|
|
|
await self.record_recursive_validation_usage(
|
|
|
evaluated_trace_id,
|
|
|
material_chars_count=total_material_chars,
|
|
|
+ operation_id=(
|
|
|
+ f"validation-materials:{evaluated_trace_id}:{plan.plan_hash}"
|
|
|
+ ),
|
|
|
)
|
|
|
if candidate_ref is not None:
|
|
|
validation_cache["material_usage_recorded"] = True
|
|
|
+ candidate_checkpoint = (
|
|
|
+ await self.candidate_service.update_validation_checkpoint(
|
|
|
+ evaluated_trace_id,
|
|
|
+ candidate_ref,
|
|
|
+ plan.plan_hash,
|
|
|
+ material_usage_recorded=True,
|
|
|
+ )
|
|
|
+ )
|
|
|
else:
|
|
|
fresh = await self.trace_store.get_trace(evaluated_trace_id)
|
|
|
if not fresh:
|
|
|
@@ -917,6 +1030,12 @@ class AgentRunner:
|
|
|
for item in plan.effective_scopes
|
|
|
if item in by_scope
|
|
|
]
|
|
|
+ await self.candidate_service.update_validation_checkpoint(
|
|
|
+ evaluated_trace_id,
|
|
|
+ candidate_ref,
|
|
|
+ plan.plan_hash,
|
|
|
+ scope_results=tuple(validation_cache["scope_results"]),
|
|
|
+ )
|
|
|
return
|
|
|
fresh = await self.trace_store.get_trace(evaluated_trace_id)
|
|
|
if not fresh:
|
|
|
@@ -1009,8 +1128,30 @@ class AgentRunner:
|
|
|
async def _run_quality_checks(
|
|
|
self,
|
|
|
subject: Any,
|
|
|
- material: ValidationMaterial,
|
|
|
+ materials: tuple[ValidationMaterial, ...],
|
|
|
rules: tuple[Any, ...],
|
|
|
+ usage_operation_prefix: str,
|
|
|
+ ) -> tuple[dict[str, ValidationCheck], dict[ValidationScope, str]]:
|
|
|
+ checks: dict[str, ValidationCheck] = {}
|
|
|
+ errors: dict[ValidationScope, str] = {}
|
|
|
+ for scope in dict.fromkeys(rule.scope for rule in rules):
|
|
|
+ scoped_rules = tuple(rule for rule in rules if rule.scope == scope)
|
|
|
+ scoped_checks, scoped_errors = await self._run_quality_check_batch(
|
|
|
+ subject,
|
|
|
+ materials,
|
|
|
+ scoped_rules,
|
|
|
+ usage_operation_id=f"{usage_operation_prefix}:quality:{scope}",
|
|
|
+ )
|
|
|
+ checks.update(scoped_checks)
|
|
|
+ errors.update(scoped_errors)
|
|
|
+ return checks, errors
|
|
|
+
|
|
|
+ async def _run_quality_check_batch(
|
|
|
+ self,
|
|
|
+ subject: Any,
|
|
|
+ materials: tuple[ValidationMaterial, ...],
|
|
|
+ rules: tuple[Any, ...],
|
|
|
+ usage_operation_id: str,
|
|
|
) -> tuple[dict[str, ValidationCheck], dict[ValidationScope, str]]:
|
|
|
"""Execute one frozen quality batch and reject provider-shaped plans."""
|
|
|
from cyber_agent.application.quality import (
|
|
|
@@ -1031,15 +1172,17 @@ class AgentRunner:
|
|
|
)
|
|
|
for check_id in check_ids.values()
|
|
|
},
|
|
|
- {"output": reason},
|
|
|
+ {rule.scope: reason for rule in rules},
|
|
|
)
|
|
|
|
|
|
if provider is None:
|
|
|
return provider_error("QualityCheckProvider is unavailable")
|
|
|
+ if not materials:
|
|
|
+ return provider_error("QualityCheckProvider has no authorized material")
|
|
|
request = QualityCheckInput(
|
|
|
subject=subject,
|
|
|
rules=rules,
|
|
|
- material=material,
|
|
|
+ materials=materials,
|
|
|
)
|
|
|
try:
|
|
|
raw = await asyncio.wait_for(
|
|
|
@@ -1076,6 +1219,7 @@ class AgentRunner:
|
|
|
await self.record_recursive_validation_usage(
|
|
|
subject.trace_id,
|
|
|
tool_calls=len(rules),
|
|
|
+ operation_id=usage_operation_id,
|
|
|
)
|
|
|
except Exception as exc:
|
|
|
return provider_error(f"Quality validation budget failed: {exc}")
|