test_phase_two_boundary.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465
  1. from __future__ import annotations
  2. import hashlib
  3. import json
  4. from dataclasses import replace
  5. from datetime import UTC, datetime
  6. import pytest
  7. from agent.orchestration import (
  8. AcceptanceCriterion,
  9. ArtifactRef,
  10. ArtifactSnapshot,
  11. AttemptStatus,
  12. AttemptSubmission,
  13. DecisionAction,
  14. PlannerDecision,
  15. TaskAttempt,
  16. TaskLedger,
  17. TaskRecord,
  18. TaskSpec,
  19. TaskStatus,
  20. ValidationReport,
  21. ValidationRunStatus,
  22. ValidationVerdict,
  23. )
  24. from script_build_host.application.phase_two_boundary import ScriptPhaseTwoBoundaryVerifier
  25. from script_build_host.application.phase_two_inputs import AcceptedInputResolver
  26. from script_build_host.domain.artifacts import (
  27. ArtifactKind,
  28. ArtifactState,
  29. ArtifactVersion,
  30. DirectionArtifact,
  31. DirectionGoal,
  32. )
  33. from script_build_host.domain.errors import PhaseTwoBoundaryNotReady
  34. from script_build_host.domain.phase_two_artifacts import (
  35. CandidatePortfolioArtifactV1,
  36. GoalCoverage,
  37. ScriptParagraphV1,
  38. StructuredScriptArtifactV1,
  39. )
  40. from script_build_host.domain.records import MissionBinding
  41. from script_build_host.domain.task_contracts import (
  42. AcceptedDecisionRef,
  43. ScriptCriterion,
  44. ScriptIntentClass,
  45. ScriptTaskBudget,
  46. ScriptTaskContractV1,
  47. ScriptTaskKind,
  48. )
  49. SNAPSHOT_ID = "11"
  50. INPUT_DIGEST = "sha256:" + "c" * 64
  51. SCRIPT_DIGEST = "sha256:" + "a" * 64
  52. PORTFOLIO_DIGEST = "sha256:" + "b" * 64
  53. SCOPE = "script-build://scopes/full"
  54. WRITE = "script-build://writes/full"
  55. def _task(task_id: str, parent: str | None, kind: str, status: TaskStatus) -> TaskRecord:
  56. return TaskRecord(
  57. task_id=task_id,
  58. parent_task_id=parent,
  59. display_path=task_id,
  60. specs=[
  61. TaskSpec(
  62. version=1,
  63. objective="bounded",
  64. acceptance_criteria=(AcceptanceCriterion("closed", "closed"),),
  65. context_refs=(
  66. f"script-build://task-kinds/{kind}",
  67. f"script-build://inputs/{SNAPSHOT_ID}",
  68. ),
  69. )
  70. ],
  71. status=status,
  72. )
  73. def _compose_contract() -> ScriptTaskContractV1:
  74. return ScriptTaskContractV1(
  75. ScriptTaskKind.COMPOSE,
  76. SCOPE,
  77. ScriptIntentClass.COMPOSE,
  78. "compose one complete candidate",
  79. (),
  80. None,
  81. (WRITE,),
  82. None,
  83. "structured-script/v1",
  84. (ScriptCriterion("closed", "closed"),),
  85. ScriptTaskBudget(),
  86. ("goal-1",),
  87. )
  88. def _portfolio_contract(ref: ArtifactRef) -> ScriptTaskContractV1:
  89. accepted = AcceptedDecisionRef("decision-compose", ref, SCOPE, ScriptTaskKind.COMPOSE)
  90. return ScriptTaskContractV1(
  91. ScriptTaskKind.CANDIDATE_PORTFOLIO,
  92. SCOPE,
  93. ScriptIntentClass.PORTFOLIO,
  94. "govern the closed candidate set",
  95. (accepted,),
  96. None,
  97. (WRITE,),
  98. None,
  99. "candidate-portfolio/v1",
  100. (ScriptCriterion("closed", "closed"),),
  101. ScriptTaskBudget(),
  102. ("goal-1",),
  103. candidate_closure_decision_refs=(accepted,),
  104. adopted_decision_ids=("decision-compose",),
  105. compose_order=("decision-compose",),
  106. )
  107. class _Store:
  108. def __init__(self, value: object) -> None:
  109. self.value = value
  110. async def load(self, root: str) -> TaskLedger:
  111. assert root == "root"
  112. assert isinstance(self.value, TaskLedger)
  113. return self.value
  114. async def get(self, root: str, snapshot: str) -> ArtifactSnapshot:
  115. assert root == "root"
  116. return self.value[snapshot] # type: ignore[index]
  117. class _Contracts:
  118. def __init__(self, values: dict[str, ScriptTaskContractV1]) -> None:
  119. self.values = values
  120. async def read_for_task(
  121. self, *, root_trace_id: str, task: TaskRecord, spec_version: int | None = None
  122. ) -> ScriptTaskContractV1:
  123. assert root_trace_id == "root" and spec_version == 1
  124. return self.values[task.task_id]
  125. class _Artifacts:
  126. def __init__(self, versions: dict[int, ArtifactVersion]) -> None:
  127. self.versions = versions
  128. async def read_by_ref(
  129. self,
  130. ref: ArtifactRef,
  131. *,
  132. script_build_id: int,
  133. task_id: str | None = None,
  134. attempt_id: str | None = None,
  135. ) -> ArtifactVersion:
  136. assert script_build_id == 7 and ref.version is not None
  137. value = self.versions[int(ref.version)]
  138. assert value.task_id == task_id and value.attempt_id == attempt_id
  139. assert value.canonical_sha256 == ref.digest
  140. return value
  141. async def get_by_id(self, identifier: int, *, script_build_id: int) -> ArtifactVersion:
  142. assert script_build_id == 7
  143. return self.versions[identifier]
  144. class _Bindings:
  145. async def get_by_build(self, script_build_id: int) -> MissionBinding:
  146. assert script_build_id == 7
  147. now = datetime.now(UTC)
  148. return MissionBinding(1, 7, "root", 11, 9, None, "test", "v1", now, now)
  149. def _version(
  150. identifier: int,
  151. task_id: str,
  152. attempt_id: str,
  153. kind: ArtifactKind,
  154. digest: str,
  155. artifact: object,
  156. ) -> ArtifactVersion:
  157. now = datetime.now(UTC)
  158. return ArtifactVersion(
  159. identifier,
  160. 7,
  161. task_id,
  162. attempt_id,
  163. 1,
  164. kind,
  165. digest,
  166. ArtifactState.FROZEN,
  167. artifact, # type: ignore[arg-type]
  168. now,
  169. now,
  170. )
  171. async def _fixture() -> tuple[ScriptPhaseTwoBoundaryVerifier, TaskLedger]:
  172. script_ref = ArtifactRef(
  173. "script-build://artifact-versions/1",
  174. ArtifactKind.STRUCTURED_SCRIPT.value,
  175. "1",
  176. SCRIPT_DIGEST,
  177. )
  178. root = _task("root-task", None, "root", TaskStatus.BLOCKED)
  179. portfolio_task = _task("portfolio", "root-task", "candidate-portfolio", TaskStatus.RUNNING)
  180. compose_task = _task("compose", "portfolio", "compose", TaskStatus.COMPLETED)
  181. root.child_task_ids.append("portfolio")
  182. portfolio_task.child_task_ids.append("compose")
  183. compose_attempt = TaskAttempt(
  184. "attempt-compose",
  185. "compose",
  186. 1,
  187. "worker-compose",
  188. "script_compose_worker",
  189. "new",
  190. (),
  191. AttemptStatus.SUBMITTED,
  192. snapshot_id="snapshot-compose",
  193. submission=AttemptSubmission("closed", artifact_refs=[script_ref]),
  194. )
  195. compose_validation = ValidationReport(
  196. "validation-compose",
  197. "compose",
  198. "attempt-compose",
  199. 1,
  200. "snapshot-compose",
  201. "validator-compose",
  202. status=ValidationRunStatus.COMPLETED,
  203. verdict=ValidationVerdict.PASSED,
  204. )
  205. compose_decision = PlannerDecision(
  206. "decision-compose",
  207. "compose",
  208. DecisionAction.ACCEPT,
  209. "passed",
  210. TaskStatus.AWAITING_DECISION,
  211. TaskStatus.COMPLETED,
  212. "attempt-compose",
  213. "validation-compose",
  214. )
  215. compose_task.attempt_ids.append("attempt-compose")
  216. compose_task.validation_ids.append("validation-compose")
  217. compose_task.decision_ids.append("decision-compose")
  218. portfolio_attempt = TaskAttempt(
  219. "attempt-portfolio",
  220. "portfolio",
  221. 1,
  222. "worker-portfolio",
  223. "script_candidate_portfolio_worker",
  224. "new",
  225. ("decision-compose",),
  226. )
  227. ledger = TaskLedger(
  228. "root",
  229. "mission",
  230. "root-task",
  231. tasks={item.task_id: item for item in (root, portfolio_task, compose_task)},
  232. attempts={
  233. compose_attempt.attempt_id: compose_attempt,
  234. portfolio_attempt.attempt_id: portfolio_attempt,
  235. },
  236. validations={compose_validation.validation_id: compose_validation},
  237. decisions={compose_decision.decision_id: compose_decision},
  238. )
  239. compose_normalized = _normalized("closed", script_ref)
  240. snapshots = {
  241. "snapshot-compose": ArtifactSnapshot(
  242. "snapshot-compose",
  243. "attempt-compose",
  244. compose_normalized,
  245. _snapshot_sha(compose_normalized),
  246. [script_ref],
  247. [],
  248. )
  249. }
  250. paragraph = ScriptParagraphV1(1, 1, 1, None, "opening", {}, description="done")
  251. structured = StructuredScriptArtifactV1(
  252. "script-build://artifact-versions/9",
  253. INPUT_DIGEST,
  254. (paragraph,),
  255. (),
  256. (),
  257. ("script-build://artifact-versions/8",),
  258. (GoalCoverage("goal-1", ("script-build://artifact-versions/8",)),),
  259. (),
  260. (),
  261. SCRIPT_DIGEST,
  262. )
  263. artifacts = _Artifacts(
  264. {
  265. 1: _version(
  266. 1,
  267. "compose",
  268. "attempt-compose",
  269. ArtifactKind.STRUCTURED_SCRIPT,
  270. SCRIPT_DIGEST,
  271. structured,
  272. ),
  273. 9: _version(
  274. 9,
  275. "direction",
  276. "attempt-direction",
  277. ArtifactKind.DIRECTION,
  278. INPUT_DIGEST,
  279. DirectionArtifact(
  280. goals=(
  281. DirectionGoal(
  282. "goal-1",
  283. "complete the script",
  284. "the delivery needs one complete script",
  285. success_criteria=("the script is complete",),
  286. ),
  287. ),
  288. evidence_refs=("script-build://artifact-versions/99",),
  289. ),
  290. ),
  291. }
  292. )
  293. contracts = _Contracts(
  294. {"compose": _compose_contract(), "portfolio": _portfolio_contract(script_ref)}
  295. )
  296. accepted = AcceptedInputResolver(
  297. task_store=_Store(ledger),
  298. artifact_store=_Store(snapshots),
  299. artifacts=artifacts,
  300. contracts=contracts,
  301. )
  302. bundle = await accepted.resolve(
  303. root_trace_id="root",
  304. script_build_id=7,
  305. task_id="portfolio",
  306. attempt_id="attempt-portfolio",
  307. contract=contracts.values["portfolio"],
  308. input_snapshot_id=SNAPSHOT_ID,
  309. )
  310. portfolio = CandidatePortfolioArtifactV1(
  311. script_ref.uri,
  312. (script_ref.uri,),
  313. ("decision-compose",),
  314. (),
  315. (),
  316. bundle.input_closure_digest,
  317. (),
  318. ("decision-compose",),
  319. PORTFOLIO_DIGEST,
  320. )
  321. portfolio_ref = ArtifactRef(
  322. "script-build://artifact-versions/2",
  323. ArtifactKind.CANDIDATE_PORTFOLIO.value,
  324. "2",
  325. PORTFOLIO_DIGEST,
  326. )
  327. portfolio_attempt.status = AttemptStatus.SUBMITTED
  328. portfolio_attempt.snapshot_id = "snapshot-portfolio"
  329. portfolio_attempt.submission = AttemptSubmission("closed", artifact_refs=[portfolio_ref])
  330. portfolio_validation = ValidationReport(
  331. "validation-portfolio",
  332. "portfolio",
  333. "attempt-portfolio",
  334. 1,
  335. "snapshot-portfolio",
  336. "validator-portfolio",
  337. status=ValidationRunStatus.COMPLETED,
  338. verdict=ValidationVerdict.PASSED,
  339. )
  340. portfolio_decision = PlannerDecision(
  341. "decision-portfolio",
  342. "portfolio",
  343. DecisionAction.ACCEPT,
  344. "passed",
  345. TaskStatus.AWAITING_DECISION,
  346. TaskStatus.COMPLETED,
  347. "attempt-portfolio",
  348. "validation-portfolio",
  349. )
  350. portfolio_task.status = TaskStatus.COMPLETED
  351. portfolio_task.attempt_ids.append("attempt-portfolio")
  352. portfolio_task.validation_ids.append("validation-portfolio")
  353. portfolio_task.decision_ids.append("decision-portfolio")
  354. ledger.validations[portfolio_validation.validation_id] = portfolio_validation
  355. ledger.decisions[portfolio_decision.decision_id] = portfolio_decision
  356. portfolio_normalized = _normalized("closed", portfolio_ref)
  357. snapshots["snapshot-portfolio"] = ArtifactSnapshot(
  358. "snapshot-portfolio",
  359. "attempt-portfolio",
  360. portfolio_normalized,
  361. _snapshot_sha(portfolio_normalized),
  362. [portfolio_ref],
  363. [],
  364. )
  365. artifacts.versions[2] = _version(
  366. 2,
  367. "portfolio",
  368. "attempt-portfolio",
  369. ArtifactKind.CANDIDATE_PORTFOLIO,
  370. PORTFOLIO_DIGEST,
  371. portfolio,
  372. )
  373. verifier = ScriptPhaseTwoBoundaryVerifier(
  374. task_store=_Store(ledger),
  375. artifact_store=_Store(snapshots),
  376. artifacts=artifacts,
  377. bindings=_Bindings(),
  378. accepted_inputs=accepted,
  379. )
  380. return verifier, ledger
  381. @pytest.mark.asyncio
  382. async def test_phase_two_boundary_requires_one_closed_adopted_portfolio() -> None:
  383. verifier, _ = await _fixture()
  384. await verifier.verify_checkpoint(
  385. script_build_id=7, root_trace_id="root", input_snapshot_id=SNAPSHOT_ID
  386. )
  387. @pytest.mark.asyncio
  388. async def test_phase_two_boundary_rejects_nonterminal_subtree_and_early_advance() -> None:
  389. verifier, ledger = await _fixture()
  390. ledger.tasks["compose"].status = TaskStatus.RUNNING
  391. with pytest.raises(PhaseTwoBoundaryNotReady, match="subtree is not terminal"):
  392. await verifier.verify_checkpoint(
  393. script_build_id=7, root_trace_id="root", input_snapshot_id=SNAPSHOT_ID
  394. )
  395. with pytest.raises(PhaseTwoBoundaryNotReady, match="portfolio already exists"):
  396. await verifier.verify_before_advance(
  397. script_build_id=7, root_trace_id="root", input_snapshot_id=SNAPSHOT_ID
  398. )
  399. @pytest.mark.asyncio
  400. async def test_phase_two_boundary_rejects_closure_digest_drift() -> None:
  401. verifier, _ = await _fixture()
  402. version = verifier._artifacts.versions[2]
  403. artifact = replace(version.artifact, input_closure_digest="sha256:" + "f" * 64)
  404. verifier._artifacts.versions[2] = replace(version, artifact=artifact)
  405. with pytest.raises(PhaseTwoBoundaryNotReady, match="closure digest"):
  406. await verifier.verify_checkpoint(
  407. script_build_id=7, root_trace_id="root", input_snapshot_id=SNAPSHOT_ID
  408. )
  409. def _normalized(summary: str, ref: ArtifactRef) -> dict[str, object]:
  410. return {
  411. "summary": summary,
  412. "artifact_refs": [
  413. {
  414. "uri": ref.uri,
  415. "kind": ref.kind,
  416. "version": ref.version,
  417. "digest": ref.digest,
  418. "summary": ref.summary,
  419. "metadata": ref.metadata,
  420. }
  421. ],
  422. "evidence_refs": [],
  423. }
  424. def _snapshot_sha(value: dict[str, object]) -> str:
  425. canonical = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
  426. return hashlib.sha256(canonical.encode()).hexdigest()