test_health.py 3.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130
  1. from __future__ import annotations
  2. from datetime import timedelta
  3. from supply_infra.config import InfraSettings
  4. from supply_infra.pipeline.dates import china_now
  5. from supply_infra.pipeline.health import (
  6. run_alert_level,
  7. runtime_components_ready,
  8. )
  9. def _settings() -> InfraSettings:
  10. return InfraSettings(_env_file=None)
  11. def test_alert_level_escalates_against_next_schedule_deadline() -> None:
  12. settings = _settings()
  13. now = china_now()
  14. assert (
  15. run_alert_level(
  16. status="running",
  17. started_at=now - timedelta(hours=13),
  18. created_at=now - timedelta(hours=13),
  19. deadline_at=now + timedelta(hours=5),
  20. now=now,
  21. settings=settings,
  22. )
  23. == "warning"
  24. )
  25. assert (
  26. run_alert_level(
  27. status="running",
  28. started_at=now - timedelta(hours=1),
  29. created_at=now - timedelta(hours=1),
  30. deadline_at=now + timedelta(minutes=90),
  31. now=now,
  32. settings=settings,
  33. )
  34. == "critical"
  35. )
  36. assert (
  37. run_alert_level(
  38. status="blocked_previous_run",
  39. started_at=None,
  40. created_at=now - timedelta(hours=1),
  41. deadline_at=now + timedelta(minutes=20),
  42. now=now,
  43. settings=settings,
  44. )
  45. == "final_warning"
  46. )
  47. def test_terminal_run_has_no_duration_alert() -> None:
  48. now = china_now()
  49. assert (
  50. run_alert_level(
  51. status="failed",
  52. started_at=now - timedelta(days=2),
  53. created_at=now - timedelta(days=2),
  54. deadline_at=now - timedelta(days=1),
  55. now=now,
  56. settings=_settings(),
  57. )
  58. is None
  59. )
  60. def test_manual_run_without_deadline_only_uses_duration_warning() -> None:
  61. now = china_now()
  62. settings = _settings()
  63. assert (
  64. run_alert_level(
  65. status="running",
  66. started_at=now - timedelta(hours=1),
  67. created_at=now - timedelta(hours=1),
  68. deadline_at=None,
  69. now=now,
  70. settings=settings,
  71. )
  72. is None
  73. )
  74. assert (
  75. run_alert_level(
  76. status="running",
  77. started_at=now - timedelta(hours=13),
  78. created_at=now - timedelta(hours=13),
  79. deadline_at=None,
  80. now=now,
  81. settings=settings,
  82. )
  83. == "warning"
  84. )
  85. def test_runtime_readiness_requires_all_workers_and_reconciler() -> None:
  86. assert runtime_components_ready(
  87. scheduler_enabled=True,
  88. scheduler_running=True,
  89. active_worker_count=4,
  90. expected_worker_count=4,
  91. reconciler_running=True,
  92. )
  93. assert not runtime_components_ready(
  94. scheduler_enabled=True,
  95. scheduler_running=True,
  96. active_worker_count=3,
  97. expected_worker_count=4,
  98. reconciler_running=True,
  99. )
  100. assert not runtime_components_ready(
  101. scheduler_enabled=True,
  102. scheduler_running=True,
  103. active_worker_count=4,
  104. expected_worker_count=4,
  105. reconciler_running=False,
  106. )
  107. def test_runtime_readiness_allows_intentionally_disabled_scheduler() -> None:
  108. assert runtime_components_ready(
  109. scheduler_enabled=False,
  110. scheduler_running=False,
  111. active_worker_count=1,
  112. expected_worker_count=1,
  113. reconciler_running=True,
  114. )