test_phase_three_ownership.py 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196
  1. from __future__ import annotations
  2. from pathlib import Path
  3. import pytest
  4. from sqlalchemy import select
  5. from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
  6. from script_build_host.domain.errors import (
  7. MissionAlreadyOwned,
  8. MissionFencingTokenStale,
  9. MissionStopRequested,
  10. )
  11. from script_build_host.domain.records import BuildStatus
  12. from script_build_host.infrastructure.legacy_tables import script_build_record
  13. from script_build_host.infrastructure.ownership import FencedCommandGate, OwnerLease
  14. from script_build_host.infrastructure.tables import mission_binding_table
  15. from script_build_host.repositories.legacy_state import SqlAlchemyLegacyBuildStateRepository
  16. from script_build_host.repositories.sqlalchemy import SqlAlchemyMissionBindingRepository
  17. async def _bound_build(sessions: async_sessionmaker[AsyncSession]) -> int:
  18. state = SqlAlchemyLegacyBuildStateRepository(sessions)
  19. build_id = await state.create(
  20. execution_id=1,
  21. topic_build_id=2,
  22. topic_id=3,
  23. agent_type="AigcAgent",
  24. agent_config={},
  25. data_source_url=None,
  26. strategies_config={},
  27. root_trace_id="root-owner-test",
  28. )
  29. await SqlAlchemyMissionBindingRepository(sessions).create(
  30. script_build_id=build_id,
  31. root_trace_id="root-owner-test",
  32. input_snapshot_id=1,
  33. engine_version="test",
  34. schema_version="test/v1",
  35. )
  36. return build_id
  37. @pytest.mark.asyncio
  38. async def test_owner_lease_excludes_second_owner_and_grows_epoch(
  39. database: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], tmp_path: Path
  40. ) -> None:
  41. _, sessions = database
  42. build_id = await _bound_build(sessions)
  43. first = OwnerLease(sessions, tmp_path / "leases", owner_instance_id="owner-a")
  44. second = OwnerLease(sessions, tmp_path / "leases", owner_instance_id="owner-b")
  45. first_token = await first.acquire(build_id)
  46. with pytest.raises(MissionAlreadyOwned):
  47. await second.acquire(build_id)
  48. await first.release()
  49. second_token = await second.acquire(build_id)
  50. assert second_token.owner_epoch == first_token.owner_epoch + 1
  51. with pytest.raises(MissionFencingTokenStale):
  52. await FencedCommandGate(sessions).verify(first_token)
  53. await second.release()
  54. @pytest.mark.asyncio
  55. async def test_stop_epoch_fences_current_owner_without_taking_process_lease(
  56. database: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], tmp_path: Path
  57. ) -> None:
  58. _, sessions = database
  59. build_id = await _bound_build(sessions)
  60. lease = OwnerLease(sessions, tmp_path / "leases", owner_instance_id="owner")
  61. token = await lease.acquire(build_id)
  62. gate = FencedCommandGate(sessions)
  63. await gate.verify(token)
  64. assert await gate.request_stop(build_id) == token.stop_epoch + 1
  65. with pytest.raises(MissionStopRequested):
  66. await gate.verify(token)
  67. assert (
  68. await SqlAlchemyLegacyBuildStateRepository(sessions).get_status(build_id)
  69. is BuildStatus.STOPPING
  70. )
  71. await lease.release()
  72. @pytest.mark.asyncio
  73. async def test_new_owner_can_cas_stopping_build_to_stopped(
  74. database: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], tmp_path: Path
  75. ) -> None:
  76. _, sessions = database
  77. build_id = await _bound_build(sessions)
  78. old_lease = OwnerLease(sessions, tmp_path / "leases", owner_instance_id="old")
  79. old_token = await old_lease.acquire(build_id)
  80. gate = FencedCommandGate(sessions)
  81. stop_epoch = await gate.request_stop(build_id)
  82. assert await gate.request_stop(build_id) == stop_epoch
  83. await old_lease.release()
  84. takeover = OwnerLease(sessions, tmp_path / "leases", owner_instance_id="takeover")
  85. token = await takeover.acquire(build_id)
  86. try:
  87. assert token.owner_epoch == old_token.owner_epoch + 1
  88. assert token.stop_epoch == stop_epoch
  89. await gate.verify_stop_owner(token)
  90. await gate.complete_stop(token)
  91. assert (
  92. await SqlAlchemyLegacyBuildStateRepository(sessions).get_status(build_id)
  93. is BuildStatus.STOPPED
  94. )
  95. with pytest.raises(MissionFencingTokenStale):
  96. await gate.complete_stop(old_token)
  97. finally:
  98. await takeover.release()
  99. @pytest.mark.asyncio
  100. async def test_phase_three_state_and_snapshot_mutations_share_the_owner_fence(
  101. database: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], tmp_path: Path
  102. ) -> None:
  103. _, sessions = database
  104. build_id = await _bound_build(sessions)
  105. lease = OwnerLease(sessions, tmp_path / "leases", owner_instance_id="phase-three-owner")
  106. token = await lease.acquire(build_id)
  107. gate = FencedCommandGate(sessions)
  108. try:
  109. await gate.compare_and_set_input_snapshot(token, expected_snapshot_id=1, new_snapshot_id=2)
  110. await gate.begin_phase(token)
  111. await gate.set_checkpoint(
  112. token,
  113. checkpoint_code="PHASE_THREE_ROOT_ACCEPTED",
  114. summary="accepted",
  115. )
  116. await gate.set_checkpoint(
  117. token,
  118. checkpoint_code="PHASE_THREE_ROOT_ACCEPTED",
  119. summary="accepted",
  120. )
  121. async with sessions() as session:
  122. assert (
  123. await session.scalar(
  124. select(mission_binding_table.c.input_snapshot_id).where(
  125. mission_binding_table.c.script_build_id == build_id
  126. )
  127. )
  128. == 2
  129. )
  130. row = (
  131. (
  132. await session.execute(
  133. select(script_build_record).where(script_build_record.c.id == build_id)
  134. )
  135. )
  136. .mappings()
  137. .one()
  138. )
  139. assert row["status"] == BuildStatus.PARTIAL.value
  140. assert row["error_message"] == "PHASE_THREE_ROOT_ACCEPTED"
  141. await gate.request_stop(build_id)
  142. with pytest.raises(MissionStopRequested):
  143. await gate.begin_phase(token)
  144. with pytest.raises(MissionStopRequested):
  145. await gate.compare_and_set_input_snapshot(
  146. token, expected_snapshot_id=2, new_snapshot_id=3
  147. )
  148. finally:
  149. await lease.release()
  150. @pytest.mark.asyncio
  151. async def test_failed_build_cannot_be_resurrected_by_an_owner_token(
  152. database: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], tmp_path: Path
  153. ) -> None:
  154. _, sessions = database
  155. build_id = await _bound_build(sessions)
  156. lease = OwnerLease(sessions, tmp_path / "leases", owner_instance_id="failed-owner")
  157. token = await lease.acquire(build_id)
  158. state = SqlAlchemyLegacyBuildStateRepository(sessions)
  159. await state.set_status(build_id, BuildStatus.FAILED, error_summary="failed")
  160. gate = FencedCommandGate(sessions)
  161. try:
  162. with pytest.raises(MissionFencingTokenStale):
  163. await gate.begin_phase(token)
  164. with pytest.raises(MissionFencingTokenStale):
  165. await gate.set_checkpoint(
  166. token,
  167. checkpoint_code="PHASE_THREE_ROOT_ACCEPTED",
  168. summary="must not restore",
  169. )
  170. with pytest.raises(MissionFencingTokenStale):
  171. await gate.compare_and_set_input_snapshot(
  172. token, expected_snapshot_id=1, new_snapshot_id=2
  173. )
  174. assert await state.get_status(build_id) is BuildStatus.FAILED
  175. finally:
  176. await lease.release()