test_resume_cancelled_run.py 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153
  1. from __future__ import annotations
  2. from contextlib import contextmanager
  3. from sqlalchemy import create_engine
  4. from sqlalchemy.orm import Session, sessionmaker
  5. from supply_infra.db.models.pipeline_run import PipelineRun
  6. from supply_infra.db.models.pipeline_step_run import PipelineStepRun
  7. from supply_infra.pipeline import run_service
  8. def _session_factory() -> sessionmaker[Session]:
  9. engine = create_engine("sqlite+pysqlite:///:memory:")
  10. PipelineRun.__table__.create(engine)
  11. PipelineStepRun.__table__.create(engine)
  12. return sessionmaker(bind=engine, expire_on_commit=False)
  13. def _run(status: str = "cancelled") -> PipelineRun:
  14. return PipelineRun(
  15. run_id="run-cancelled",
  16. dedupe_key="supply_pipeline:20260727:full",
  17. pipeline_key="supply_pipeline",
  18. biz_dt="20260727",
  19. trigger_type="api",
  20. run_mode="full",
  21. dry_run=True,
  22. status=status,
  23. current_step=None,
  24. deadline_at=None,
  25. config_snapshot_json={},
  26. date_snapshot_json={},
  27. summary_json={"old": "summary"},
  28. error_code="cancellation_requested",
  29. error_message="cancelled",
  30. )
  31. def _step(
  32. *,
  33. step_run_id: str,
  34. key: str,
  35. order: int,
  36. status: str,
  37. ) -> PipelineStepRun:
  38. return PipelineStepRun(
  39. step_run_id=step_run_id,
  40. run_id="run-cancelled",
  41. step_key=key,
  42. step_order=order,
  43. attempt=1,
  44. status=status,
  45. critical=True,
  46. dependency_snapshot_json=[],
  47. input_snapshot_json={},
  48. timeout_seconds=60,
  49. max_attempts=1,
  50. retryable=False,
  51. error_code="cancelled" if status == "cancelled" else None,
  52. error_message="Cancellation requested" if status == "cancelled" else None,
  53. )
  54. def _patch_session(monkeypatch, factory: sessionmaker[Session]) -> None:
  55. @contextmanager
  56. def get_test_session():
  57. session = factory()
  58. try:
  59. yield session
  60. session.commit()
  61. except Exception:
  62. session.rollback()
  63. raise
  64. finally:
  65. session.close()
  66. monkeypatch.setattr(run_service, "get_session", get_test_session)
  67. def test_cancelled_run_resumes_from_first_cancelled_step(monkeypatch) -> None:
  68. factory = _session_factory()
  69. _patch_session(monkeypatch, factory)
  70. with factory() as session:
  71. session.add_all(
  72. [
  73. _run(),
  74. _step(
  75. step_run_id="step-1",
  76. key="global_tree_sync",
  77. order=1,
  78. status="succeeded",
  79. ),
  80. _step(
  81. step_run_id="step-2",
  82. key="source_video_sync",
  83. order=2,
  84. status="cancelled",
  85. ),
  86. _step(
  87. step_run_id="step-3",
  88. key="demand_grade",
  89. order=3,
  90. status="cancelled",
  91. ),
  92. ]
  93. )
  94. session.commit()
  95. assert run_service.resume_pipeline_run("run-cancelled") is True
  96. with factory() as session:
  97. run = session.get(PipelineRun, "run-cancelled")
  98. assert run is not None
  99. assert run.status == "queued"
  100. assert run.current_step == "source_video_sync"
  101. assert run.finished_at is None
  102. assert run.summary_json is None
  103. assert run.error_code is None
  104. assert run.error_message is None
  105. steps = (
  106. session.query(PipelineStepRun)
  107. .filter(PipelineStepRun.run_id == run.run_id)
  108. .order_by(PipelineStepRun.step_order, PipelineStepRun.attempt)
  109. .all()
  110. )
  111. assert [(step.step_key, step.attempt, step.status) for step in steps] == [
  112. ("global_tree_sync", 1, "succeeded"),
  113. ("source_video_sync", 1, "cancelled"),
  114. ("source_video_sync", 2, "ready"),
  115. ("demand_grade", 1, "pending"),
  116. ]
  117. def test_cancelling_run_cannot_resume_before_cancellation_finishes(monkeypatch) -> None:
  118. factory = _session_factory()
  119. _patch_session(monkeypatch, factory)
  120. with factory() as session:
  121. session.add_all(
  122. [
  123. _run(status="cancelling"),
  124. _step(
  125. step_run_id="step-1",
  126. key="source_video_sync",
  127. order=1,
  128. status="running",
  129. ),
  130. ]
  131. )
  132. session.commit()
  133. assert run_service.resume_pipeline_run("run-cancelled") is False