feat: 重构定时任务为持久化流水线

.dockerignore

@@ -1,5 +1,6 @@
 .git
 .gitignore
+.DS_Store
 .venv
 __pycache__
 *.py[cod]
@@ -12,6 +13,7 @@ __pycache__
 .pytest_cache
 .ruff_cache
 *.log
+*.result.json
 logs/
 tests/
 examples/

.env.example

@@ -26,7 +26,14 @@ MYSQL_PORT=3306
 MYSQL_USER=root
 MYSQL_PASSWORD=
 MYSQL_DATABASE=supply_agent
-MYSQL_POOL_SIZE=5
+MYSQL_CONNECTION_BUDGET=40
+MYSQL_POOL_SIZE=1
+MYSQL_POOL_SIZE_API=6
+MYSQL_POOL_SIZE_CONTROL=1
+MYSQL_MAX_OVERFLOW=0
+MYSQL_POOL_TIMEOUT_SECONDS=10
+MYSQL_POOL_RECYCLE_SECONDS=1800
+MYSQL_OPERATIONAL_RESERVE=4
 MYSQL_ECHO=false
 
 # ODPS (MaxCompute)
@@ -35,9 +42,28 @@ ODPS_ACCESS_KEY=
 ODPS_PROJECT=
 ODPS_ENDPOINT=https://service.cn.maxcompute.aliyun.com/api
 
-# Scheduler（供给流水线每天 14:30 Asia/Shanghai 触发）
-SCHEDULER_ENABLED=true
+# Scheduler（独立进程；供给流水线每天 15:00 Asia/Shanghai 触发）
+SCHEDULER_ENABLED=false
 SCHEDULER_TIMEZONE=Asia/Shanghai
+SCHEDULER_CRON_HOUR=15
+SCHEDULER_CRON_MINUTE=0
+
+# Pipeline control plane
+PROCESS_ROLE=api
+DATABASE_AUTO_CREATE=false
+PIPELINE_WORKER_PROCESSES=4
+PIPELINE_MAX_ACTIVE_STEPS=4
+PIPELINE_WORKER_POLL_SECONDS=2
+PIPELINE_LEASE_SECONDS=120
+PIPELINE_HEARTBEAT_SECONDS=30
+PIPELINE_RECONCILE_SECONDS=60
+PIPELINE_MISSED_RUN_GRACE_SECONDS=600
+PIPELINE_SHUTDOWN_GRACE_SECONDS=300
+PIPELINE_WARN_AFTER_HOURS=12
+PIPELINE_CRITICAL_BEFORE_NEXT_MINUTES=120
+PIPELINE_FINAL_WARN_BEFORE_NEXT_MINUTES=30
+PIPELINE_LOG_DIR=logs/pipeline
+PIPELINE_EXTERNAL_EFFECTS_ENABLED=false
 # Aliyun OSS (agent 运行日志可视化上传；qwen 视频截断后片段也上传到此 bucket)
 ALIYUN_OSS_ACCESS_KEY_ID=
 ALIYUN_OSS_ACCESS_KEY_SECRET=
@@ -51,4 +77,4 @@ LOG_OSS_UPLOAD_ENABLED=true
 
 # AIGC platform (视频爬取/发布计划)
 AIGC_API_TOKEN=
-AIGC_DRY_RUN=false
+AIGC_DRY_RUN=true

.gitignore

@@ -1,7 +1,13 @@
 __pycache__/
+.DS_Store
 *.py[cod]
 *$py.class
 *.egg-info/
+.coverage
+coverage.xml
+htmlcov/
+test-results/
+*.result.json
 dist/
 build/
 .venv/
@@ -11,8 +17,14 @@ build/
 .ruff_cache/
 *.log
 logs/
-tests/
+tests/*
+!tests/__init__.py
+!tests/supply_infra/
+tests/supply_infra/*
+!tests/supply_infra/__init__.py
+!tests/supply_infra/pipeline/
+!tests/supply_infra/scheduler/
 node_modules/
 web/dist/
 examples/
-skills/
+skills/

ARCHITECTURE.md

@@ -68,7 +68,7 @@ SupplyAgent/
                     ┌─────────────┐
                     │   ODPS      │
                     └──────┬──────┘
-                           │ 定时任务 (每天 14:30)
+                           │ 定时任务 (每天 15:00)
                            ▼
 ┌──────────┐    ┌─────────────────────┐    ┌──────────┐
 │  Agent   │───▶│  Repository (ORM)   │◀───│  Agent   │

Dockerfile

@@ -25,13 +25,18 @@ ENV PYTHONUNBUFFERED=1 \
     PIP_NO_CACHE_DIR=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     TZ=Asia/Shanghai \
-    SCHEDULER_TIMEZONE=Asia/Shanghai
+    SCHEDULER_TIMEZONE=Asia/Shanghai \
+    SCHEDULER_CRON_HOUR=15 \
+    SCHEDULER_CRON_MINUTE=0 \
+    PIPELINE_WORKER_PROCESSES=4 \
+    PIPELINE_MAX_ACTIVE_STEPS=4 \
+    MYSQL_CONNECTION_BUDGET=40
 
 RUN sed -i 's|deb.debian.org|mirrors.aliyun.com|g; s|security.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
     sed -i 's|deb.debian.org|mirrors.aliyun.com|g; s|security.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || true
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends gcc ffmpeg \
+    && apt-get install -y --no-install-recommends gcc ffmpeg supervisor \
     && ffmpeg -version >/dev/null \
     && ffprobe -version >/dev/null \
     && ffmpeg -hide_banner -loglevel error \
@@ -48,13 +53,20 @@ COPY supply_agent/ supply_agent/
 COPY supply_infra/ supply_infra/
 COPY agents/ agents/
 COPY api/ api/
+COPY alembic.ini ./
+COPY alembic/ alembic/
+COPY deploy/ deploy/
 COPY scripts/verify_video_truncate_runtime.py scripts/
+COPY scripts/container-entrypoint.sh scripts/
 
 RUN pip install -i https://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com ".[odps]" \
-    && python scripts/verify_video_truncate_runtime.py
+    && python scripts/verify_video_truncate_runtime.py \
+    && chmod +x scripts/container-entrypoint.sh
 
 COPY --from=web-builder /app/web/dist ./web/dist
 
 EXPOSE 8080
 
-CMD ["python", "-m", "api"]
+STOPSIGNAL SIGTERM
+
+ENTRYPOINT ["/app/scripts/container-entrypoint.sh"]

PRD.md

@@ -106,7 +106,7 @@ SupplyAgent 是一套面向内容供给的每日需求处理系统。它将多
 
 ```mermaid
 flowchart LR
-    START["API/CLI 启动 Scheduler"] --> CRON["每日 14:30<br/>Asia/Shanghai"]
+    START["API/CLI 启动 Scheduler"] --> CRON["每日 15:00<br/>Asia/Shanghai"]
     CRON --> T1["① 同步 T-1 全局分类树"]
     T1 --> POOL["② 同步当日策略需求池"]
     POOL --> CLASSIFY["归类新词并建立匹配边"]
@@ -141,7 +141,7 @@ flowchart LR
 
 | 参数 | 当前值 |
 |---|---|
-| 触发时间 | 每日 14:30 |
+| 触发时间 | 每日 15:00 |
 | 时区 | 默认 `Asia/Shanghai` |
 | Job ID | `run_supply_pipeline` |
 | 同一 Scheduler 最大实例 | 1 |
@@ -811,7 +811,7 @@ flowchart TB
 
 ## 23. 待业务确认
 
-1. 每日 `biz_dt` 应使用当天还是 T-1，14:30 时上游当天分区是否已稳定；
+1. 每日 `biz_dt` 应使用当天还是 T-1，15:00 时上游当天分区是否已稳定；
 2. S 与 A 的资源排序是否必须严格 S 优先；
 3. `backup` 是否允许无人工确认直接进入 AIGC；
 4. 每天 Top 200 是固定预算，还是应按分类、等级、探索比例动态分配；

alembic.ini

@@ -0,0 +1,39 @@
+[alembic]
+script_location = alembic
+prepend_sys_path = .
+path_separator = os
+sqlalchemy.url = driver://unused
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

alembic/env.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+from supply_infra.config import get_infra_settings
+from supply_infra.db.base import Base
+import supply_infra.db.models  # noqa: F401
+
+config = context.config
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# Alembic uses ConfigParser interpolation; escaped credentials may contain "%".
+config.set_main_option(
+    "sqlalchemy.url",
+    get_infra_settings().mysql_url.replace("%", "%%"),
+)
+target_metadata = Base.metadata
+
+
+def run_migrations_offline() -> None:
+    context.configure(
+        url=config.get_main_option("sqlalchemy.url"),
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+        compare_type=True,
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+    with connectable.connect() as connection:
+        if connection.dialect.name == "mysql":
+            connection.exec_driver_sql("SET time_zone = '+00:00'")
+            connection.commit()
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            compare_type=True,
+        )
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/20260727_01_pipeline_control_plane.py

@@ -0,0 +1,219 @@
+"""create durable pipeline control plane
+
+Revision ID: 20260727_01
+Revises:
+Create Date: 2026-07-27
+"""
+from __future__ import annotations
+
+from collections.abc import Sequence
+from datetime import datetime
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "20260727_01"
+down_revision: str | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "pipeline_run",
+        sa.Column("run_id", sa.String(36), primary_key=True),
+        sa.Column("dedupe_key", sa.String(191), nullable=False),
+        sa.Column("pipeline_key", sa.String(64), nullable=False),
+        sa.Column("biz_dt", sa.String(8), nullable=False),
+        sa.Column("trigger_type", sa.String(24), nullable=False),
+        sa.Column("trigger_source", sa.String(64), nullable=True),
+        sa.Column("triggered_by", sa.String(128), nullable=True),
+        sa.Column("trigger_reason", sa.Text(), nullable=True),
+        sa.Column("parent_run_id", sa.String(36), nullable=True),
+        sa.Column("run_mode", sa.String(24), nullable=False),
+        sa.Column("dry_run", sa.Boolean(), nullable=False),
+        sa.Column("status", sa.String(32), nullable=False),
+        sa.Column("current_step", sa.String(64), nullable=True),
+        sa.Column("scheduled_for", sa.DateTime(), nullable=True),
+        sa.Column("deadline_at", sa.DateTime(), nullable=False),
+        sa.Column("started_at", sa.DateTime(), nullable=True),
+        sa.Column("finished_at", sa.DateTime(), nullable=True),
+        sa.Column("heartbeat_at", sa.DateTime(), nullable=True),
+        sa.Column("lease_owner", sa.String(191), nullable=True),
+        sa.Column("lease_until", sa.DateTime(), nullable=True),
+        sa.Column("code_version", sa.String(128), nullable=True),
+        sa.Column("config_snapshot_json", sa.JSON(), nullable=False),
+        sa.Column("date_snapshot_json", sa.JSON(), nullable=False),
+        sa.Column("summary_json", sa.JSON(), nullable=True),
+        sa.Column("error_code", sa.String(64), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.func.now(), nullable=False),
+        sa.UniqueConstraint("dedupe_key", name="uk_pipeline_run_dedupe_key"),
+        mysql_charset="utf8mb4",
+    )
+    op.create_index(
+        "idx_pipeline_run_biz",
+        "pipeline_run",
+        ["pipeline_key", "biz_dt", "created_at"],
+    )
+    op.create_index(
+        "idx_pipeline_run_status",
+        "pipeline_run",
+        ["status", "lease_until"],
+    )
+    op.create_index(
+        "idx_pipeline_run_deadline",
+        "pipeline_run",
+        ["status", "deadline_at"],
+    )
+
+    op.create_table(
+        "pipeline_step_run",
+        sa.Column("step_run_id", sa.String(36), primary_key=True),
+        sa.Column(
+            "run_id",
+            sa.String(36),
+            sa.ForeignKey("pipeline_run.run_id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("step_key", sa.String(64), nullable=False),
+        sa.Column("step_order", sa.Integer(), nullable=False),
+        sa.Column("attempt", sa.Integer(), nullable=False),
+        sa.Column("status", sa.String(32), nullable=False),
+        sa.Column("critical", sa.Boolean(), nullable=False),
+        sa.Column("dependency_snapshot_json", sa.JSON(), nullable=False),
+        sa.Column("input_snapshot_json", sa.JSON(), nullable=False),
+        sa.Column("timeout_seconds", sa.Integer(), nullable=False),
+        sa.Column("max_attempts", sa.Integer(), nullable=False),
+        sa.Column("retryable", sa.Boolean(), nullable=False),
+        sa.Column("next_retry_at", sa.DateTime(), nullable=True),
+        sa.Column("started_at", sa.DateTime(), nullable=True),
+        sa.Column("finished_at", sa.DateTime(), nullable=True),
+        sa.Column("heartbeat_at", sa.DateTime(), nullable=True),
+        sa.Column("lease_owner", sa.String(191), nullable=True),
+        sa.Column("lease_until", sa.DateTime(), nullable=True),
+        sa.Column("exit_code", sa.Integer(), nullable=True),
+        sa.Column("result_summary_json", sa.JSON(), nullable=True),
+        sa.Column("metrics_json", sa.JSON(), nullable=True),
+        sa.Column("error_code", sa.String(64), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+        sa.Column("log_uri", sa.String(1024), nullable=True),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.func.now(), nullable=False),
+        sa.UniqueConstraint(
+            "run_id",
+            "step_key",
+            "attempt",
+            name="uk_pipeline_step_run_attempt",
+        ),
+        mysql_charset="utf8mb4",
+    )
+    op.create_index(
+        "idx_pipeline_step_claim",
+        "pipeline_step_run",
+        ["status", "next_retry_at", "step_order"],
+    )
+    op.create_index(
+        "idx_pipeline_step_run",
+        "pipeline_step_run",
+        ["run_id", "step_order", "attempt"],
+    )
+    op.create_index(
+        "idx_pipeline_step_lease",
+        "pipeline_step_run",
+        ["status", "lease_until"],
+    )
+
+    op.create_table(
+        "pipeline_lock",
+        sa.Column("lock_key", sa.String(191), primary_key=True),
+        sa.Column("owner_run_id", sa.String(36), nullable=False),
+        sa.Column("owner_instance", sa.String(191), nullable=False),
+        sa.Column("lease_until", sa.DateTime(), nullable=False),
+        sa.Column("heartbeat_at", sa.DateTime(), nullable=False),
+        sa.Column("version", sa.BigInteger(), nullable=False),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.func.now(), nullable=False),
+        mysql_charset="utf8mb4",
+    )
+    pipeline_lock = sa.table(
+        "pipeline_lock",
+        sa.column("lock_key", sa.String()),
+        sa.column("owner_run_id", sa.String()),
+        sa.column("owner_instance", sa.String()),
+        sa.column("lease_until", sa.DateTime()),
+        sa.column("heartbeat_at", sa.DateTime()),
+        sa.column("version", sa.BigInteger()),
+    )
+    epoch = datetime(1970, 1, 1)
+    op.bulk_insert(
+        pipeline_lock,
+        [
+            {
+                "lock_key": "pipeline:claim_guard",
+                "owner_run_id": "control-plane",
+                "owner_instance": "claim-coordinator",
+                "lease_until": epoch,
+                "heartbeat_at": epoch,
+                "version": 1,
+            }
+        ],
+    )
+
+    op.create_table(
+        "pipeline_outbox",
+        sa.Column("outbox_id", sa.String(36), primary_key=True),
+        sa.Column(
+            "run_id",
+            sa.String(36),
+            sa.ForeignKey("pipeline_run.run_id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column(
+            "step_run_id",
+            sa.String(36),
+            sa.ForeignKey("pipeline_step_run.step_run_id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("effect_type", sa.String(64), nullable=False),
+        sa.Column("idempotency_key", sa.String(191), nullable=False),
+        sa.Column("payload_hash", sa.String(64), nullable=False),
+        sa.Column("payload_json", sa.JSON(), nullable=True),
+        sa.Column("payload_uri", sa.String(1024), nullable=True),
+        sa.Column("status", sa.String(32), nullable=False),
+        sa.Column("dry_run", sa.Boolean(), nullable=False),
+        sa.Column("record_error", sa.Text(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), server_default=sa.func.now(), nullable=False),
+        sa.UniqueConstraint(
+            "idempotency_key",
+            name="uk_pipeline_outbox_idempotency",
+        ),
+        mysql_charset="utf8mb4",
+    )
+    op.create_index(
+        "idx_pipeline_outbox_run",
+        "pipeline_outbox",
+        ["run_id", "step_run_id"],
+    )
+    op.create_index(
+        "idx_pipeline_outbox_status",
+        "pipeline_outbox",
+        ["status", "created_at"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("idx_pipeline_outbox_status", table_name="pipeline_outbox")
+    op.drop_index("idx_pipeline_outbox_run", table_name="pipeline_outbox")
+    op.drop_table("pipeline_outbox")
+    op.drop_table("pipeline_lock")
+    op.drop_index("idx_pipeline_step_lease", table_name="pipeline_step_run")
+    op.drop_index("idx_pipeline_step_run", table_name="pipeline_step_run")
+    op.drop_index("idx_pipeline_step_claim", table_name="pipeline_step_run")
+    op.drop_table("pipeline_step_run")
+    op.drop_index("idx_pipeline_run_deadline", table_name="pipeline_run")
+    op.drop_index("idx_pipeline_run_status", table_name="pipeline_run")
+    op.drop_index("idx_pipeline_run_biz", table_name="pipeline_run")
+    op.drop_table("pipeline_run")

alembic/versions/20260727_02_nullable_manual_deadline.py

@@ -0,0 +1,49 @@
+"""allow manual pipeline runs without a deadline
+
+Revision ID: 20260727_02
+Revises: 20260727_01
+Create Date: 2026-07-27
+"""
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "20260727_02"
+down_revision: str | None = "20260727_01"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "pipeline_run",
+        "deadline_at",
+        existing_type=sa.DateTime(),
+        nullable=True,
+    )
+    op.execute(
+        sa.text(
+            "UPDATE pipeline_run "
+            "SET deadline_at = NULL "
+            "WHERE trigger_type NOT IN ('cron', 'reconcile')"
+        )
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        sa.text(
+            "UPDATE pipeline_run "
+            "SET deadline_at = UTC_TIMESTAMP() "
+            "WHERE deadline_at IS NULL"
+        )
+    )
+    op.alter_column(
+        "pipeline_run",
+        "deadline_at",
+        existing_type=sa.DateTime(),
+        nullable=False,
+    )

api/app.py

@@ -8,7 +8,9 @@ from fastapi import FastAPI, HTTPException, Query
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel, Field
+from sqlalchemy import text
 
+from api.routers.pipeline import router as pipeline_router
 from api.services.agent_catalog import (
     get_agent_detail,
     update_agent_document_injection,
@@ -24,24 +26,26 @@ from api.services.scheduler import (
     list_triggerable_jobs,
     run_scheduler_job,
     run_supply_pipeline,
+    scheduler_status as get_persistent_scheduler_status,
 )
 from api.services.video_discovery import (
     get_video_discovery_demand,
     list_video_discovery_demands,
 )
-from supply_infra.db import init_db
-from supply_infra.scheduler.app import get_scheduler_status, start_scheduler, stop_scheduler
+from supply_infra.config import get_infra_settings
+from supply_infra.db import dispose_engine, get_session, init_db
 
 
 @asynccontextmanager
 async def lifespan(_app: FastAPI):
-    init_db()
-    start_scheduler()
+    if get_infra_settings().database_auto_create:
+        init_db()
     yield
-    stop_scheduler()
+    dispose_engine()
 
 
 app = FastAPI(title="SupplyAgent API", version="0.1.0", lifespan=lifespan)
+app.include_router(pipeline_router)
 
 app.add_middleware(
     CORSMiddleware,
@@ -62,10 +66,25 @@ def health() -> dict[str, str]:
     return {"status": "ok"}
 
 
+@app.get("/health/live")
+def health_live() -> dict[str, str]:
+    return {"status": "ok"}
+
+
+@app.get("/health/ready")
+def health_ready() -> dict[str, str]:
+    try:
+        with get_session() as session:
+            session.execute(text("SELECT 1"))
+    except Exception as exc:
+        raise HTTPException(status_code=503, detail="database unavailable") from exc
+    return {"status": "ready"}
+
+
 @app.get("/api/scheduler/status")
 def scheduler_status() -> dict:
-    """Return scheduler enabled/running state and next run times."""
-    return get_scheduler_status()
+    """Deprecated compatibility view backed by MySQL pipeline state."""
+    return get_persistent_scheduler_status()
 
 
 class RunPipelineBody(BaseModel):
@@ -164,11 +183,13 @@ def trigger_scheduler_job(
         )
     except KeyError:
         raise HTTPException(status_code=404, detail=f"scheduler job not found: {job_id}") from None
+    except ValueError as exc:
+        raise HTTPException(status_code=409, detail=str(exc)) from None
 
 
 @app.get("/api/scheduler/runs/{run_id}")
 def scheduler_job_run(run_id: str) -> dict:
-    """Return manual job run status (in-memory; cleared on process restart)."""
+    """Return durable pipeline run status."""
     result = get_scheduler_job_run(run_id)
     if result is None:
         raise HTTPException(status_code=404, detail="scheduler run not found")

api/routers/__init__.py

@@ -0,0 +1 @@
+"""FastAPI routers."""

api/routers/pipeline.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException, Query
+
+from api.schemas.pipeline import CreatePipelineRunBody
+from api.services.pipeline import (
+    cancel_run,
+    create_run,
+    get_run,
+    list_runs,
+    pipeline_health,
+    resume_run,
+    retry_step,
+)
+
+router = APIRouter(prefix="/api/pipeline", tags=["pipeline"])
+
+
+@router.get("/runs")
+def pipeline_runs(
+    limit: int = Query(default=50, ge=1, le=200),
+    status: str | None = None,
+    biz_dt: str | None = Query(default=None, pattern=r"^\d{8}$"),
+) -> dict:
+    return {"items": list_runs(limit=limit, status=status, biz_dt=biz_dt)}
+
+
+@router.post("/runs", status_code=202)
+def create_pipeline_run(body: CreatePipelineRunBody | None = None) -> dict:
+    payload = body or CreatePipelineRunBody()
+    return create_run(
+        biz_dt=payload.biz_dt,
+        source="api",
+        reason=payload.reason,
+    )
+
+
+@router.get("/runs/{run_id}")
+def pipeline_run_detail(run_id: str) -> dict:
+    result = get_run(run_id)
+    if result is None:
+        raise HTTPException(status_code=404, detail="pipeline run not found")
+    return result
+
+
+@router.get("/runs/{run_id}/steps")
+def pipeline_run_steps(run_id: str) -> dict:
+    result = get_run(run_id)
+    if result is None:
+        raise HTTPException(status_code=404, detail="pipeline run not found")
+    return {"items": result["steps"]}
+
+
+@router.post("/runs/{run_id}/resume", status_code=202)
+def resume_pipeline_run(run_id: str) -> dict:
+    if not resume_run(run_id):
+        raise HTTPException(status_code=404, detail="pipeline run not found or not resumable")
+    result = get_run(run_id)
+    assert result is not None
+    return {"accepted": True, "run_id": run_id, "status": result["status"]}
+
+
+@router.post("/runs/{run_id}/cancel", status_code=202)
+def cancel_pipeline_run(run_id: str) -> dict:
+    if not cancel_run(run_id):
+        raise HTTPException(status_code=404, detail="pipeline run not found")
+    result = get_run(run_id)
+    assert result is not None
+    return {"accepted": True, "run_id": run_id, "status": result["status"]}
+
+
+@router.post("/runs/{run_id}/steps/{step_key}/retry", status_code=202)
+def retry_pipeline_run_step(run_id: str, step_key: str) -> dict:
+    if not retry_step(run_id, step_key):
+        raise HTTPException(
+            status_code=409,
+            detail="step is not the first failed step or run is not resumable",
+        )
+    return {
+        "accepted": True,
+        "run_id": run_id,
+        "step_key": step_key,
+        "status": "queued",
+    }
+
+
+@router.get("/health")
+def health() -> dict:
+    return pipeline_health()

api/schemas/__init__.py

@@ -0,0 +1 @@
+"""API request/response schemas."""

api/schemas/pipeline.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+
+class CreatePipelineRunBody(BaseModel):
+    biz_dt: str | None = Field(default=None, pattern=r"^\d{8}$")
+    reason: str | None = Field(default=None, max_length=2000)
+
+
+class PipelineRunActionResponse(BaseModel):
+    accepted: bool
+    run_id: str
+    status: str

api/services/pipeline.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from typing import Any
+
+from supply_infra.pipeline.dag import PIPELINE_STEPS
+from supply_infra.pipeline.health import pipeline_health_snapshot
+from supply_infra.pipeline.run_service import (
+    cancel_pipeline_run,
+    get_pipeline_run,
+    list_pipeline_runs,
+    resume_pipeline_run,
+    retry_pipeline_step,
+    submit_pipeline_run,
+)
+
+
+def create_run(
+    *,
+    biz_dt: str | None,
+    source: str,
+    reason: str | None = None,
+) -> dict[str, Any]:
+    return submit_pipeline_run(
+        biz_dt=biz_dt,
+        trigger_type="api",
+        trigger_source=source,
+        trigger_reason=reason,
+    ).to_dict()
+
+
+def list_runs(
+    *,
+    limit: int,
+    status: str | None,
+    biz_dt: str | None,
+) -> list[dict[str, Any]]:
+    return list_pipeline_runs(limit=limit, status=status, biz_dt=biz_dt)
+
+
+def get_run(run_id: str) -> dict[str, Any] | None:
+    return get_pipeline_run(run_id)
+
+
+def resume_run(run_id: str) -> bool:
+    return resume_pipeline_run(run_id)
+
+
+def cancel_run(run_id: str) -> bool:
+    return cancel_pipeline_run(run_id)
+
+
+def retry_step(run_id: str, step_key: str) -> bool:
+    return retry_pipeline_step(run_id, step_key)
+
+
+def pipeline_health() -> dict[str, Any]:
+    recent = list_pipeline_runs(limit=1)
+    latest = recent[0] if recent else None
+    return {
+        "scheduler_embedded_in_api": False,
+        "pipeline_key": "supply_pipeline",
+        "dry_run": True,
+        "steps": len(PIPELINE_STEPS),
+        "latest_run": latest,
+        **pipeline_health_snapshot(),
+    }

api/services/scheduler.py

@@ -1,18 +1,28 @@
-"""Scheduler API helpers."""
+"""Compatibility facade over the durable pipeline control plane."""
 from __future__ import annotations
 
 from typing import Any
 
-from supply_infra.scheduler.constants import SUPPLY_PIPELINE_JOB_ID
-from supply_infra.scheduler.manual_jobs import (
-    get_manual_run,
-    list_manual_jobs,
-    trigger_manual_job,
-)
+from api.services.pipeline import get_run, pipeline_health
+from supply_infra.config import get_infra_settings
+from supply_infra.pipeline.dag import PIPELINE_STEPS
+from supply_infra.pipeline.dates import next_schedule_utc
+from supply_infra.pipeline.run_service import submit_pipeline_run
+from supply_infra.scheduler.constants import SUPPLY_PIPELINE_JOB_ID, SUPPLY_PIPELINE_JOB_NAME
 
 
 def list_triggerable_jobs() -> list[dict[str, Any]]:
-    return list_manual_jobs()
+    return [
+        {
+            "id": SUPPLY_PIPELINE_JOB_ID,
+            "name": SUPPLY_PIPELINE_JOB_NAME,
+            "description": "提交严格门禁的 12 步持久化供给流水线",
+            "accepts_biz_dt": True,
+            "accepts_partition_date": False,
+            "deprecated": False,
+            "steps": [step.key for step in PIPELINE_STEPS],
+        }
+    ]
 
 
 def run_scheduler_job(
@@ -22,25 +32,41 @@ def run_scheduler_job(
     partition_date: str | None = None,
     wait: bool = False,
 ) -> dict[str, Any]:
-    return trigger_manual_job(
-        job_id,
+    del partition_date
+    if job_id != SUPPLY_PIPELINE_JOB_ID:
+        raise KeyError(job_id)
+    if wait:
+        raise ValueError("wait=true is disabled for the durable API; poll by run_id")
+    return submit_pipeline_run(
         biz_dt=biz_dt,
-        partition_date=partition_date,
-        wait=wait,
-    )
+        trigger_type="api",
+        trigger_source="legacy_scheduler_api",
+    ).to_dict()
 
 
 def get_scheduler_job_run(run_id: str) -> dict[str, Any] | None:
-    return get_manual_run(run_id)
+    return get_run(run_id)
 
 
-def run_supply_pipeline(
-    *,
-    biz_dt: str | None = None,
-) -> dict[str, Any]:
-    """异步执行供给数据全流程，立即返回 run_id。"""
-    return run_scheduler_job(
-        SUPPLY_PIPELINE_JOB_ID,
-        biz_dt=biz_dt,
-        wait=False,
-    )
+def run_supply_pipeline(*, biz_dt: str | None = None) -> dict[str, Any]:
+    return run_scheduler_job(SUPPLY_PIPELINE_JOB_ID, biz_dt=biz_dt)
+
+
+def scheduler_status() -> dict[str, Any]:
+    settings = get_infra_settings()
+    health = pipeline_health()
+    return {
+        "enabled": settings.scheduler_enabled,
+        "running": health["scheduler_running"],
+        "embedded": False,
+        "deprecated": True,
+        "timezone": settings.scheduler_timezone,
+        "jobs": [
+            {
+                "id": SUPPLY_PIPELINE_JOB_ID,
+                "name": SUPPLY_PIPELINE_JOB_NAME,
+                "next_run_time": next_schedule_utc(settings=settings).isoformat(),
+            }
+        ],
+        **health,
+    }

deploy/README.md

@@ -0,0 +1,36 @@
+# Pipeline 单 Docker 部署
+
+当前生产保持一个镜像、一个容器。Supervisor 在容器内分别托管 API、独立
+Scheduler、多个 Pipeline Worker 和 Reconciler。
+
+## 启动前
+
+1. MySQL 必须是 8.0.36，并为本应用保留 40 个连接；
+2. 挂载 `/app/logs/pipeline` 到持久化目录；
+3. `SCHEDULER_ENABLED=true` 只影响独立 Scheduler 进程，API 不再内嵌调度器；
+   默认调度时间为 `Asia/Shanghai` 每日 15:00；
+4. `AIGC_DRY_RUN=true` 且 `PIPELINE_EXTERNAL_EFFECTS_ENABLED=false`；
+5. Docker 停止窗口至少 300 秒。
+6. 容器和 Alembic 连接 MySQL 后会将会话时区固定为 UTC；定时日批有次日
+   调度 deadline，手工/API/CLI 补数不设置 deadline。
+
+示例：
+
+```bash
+docker run \
+  --stop-timeout 300 \
+  --env-file /path/to/supply-agent.env \
+  -v /host/supply-agent-pipeline-logs:/app/logs/pipeline \
+  -p 8080:8080 \
+  registry.example/supply-agent:tag
+```
+
+标准档使用 40 个连接、4 个 Worker、4 个最大活跃步骤。高并发补跑档使用
+50 个连接、20 个 Worker，但必须将最大活跃步骤限制为 10。
+
+容器启动时默认运行 `alembic upgrade head`。紧急回滚代码时不自动 downgrade
+数据库；四张控制面表保留审计数据。
+
+Reconciler 会把失败、失联、漏跑恢复以及 12 小时/2 小时/30 分钟分级预警以
+`pipeline_alert` 结构化日志写到容器 stdout。第一阶段不上传到现有公共 CDN OSS；
+生产应由 Docker 日志采集器接入现有告警平台，或至少保留容器日志。

deploy/supervisord.pipeline.conf

@@ -0,0 +1,67 @@
+[supervisord]
+nodaemon=true
+logfile=/dev/null
+logfile_maxbytes=0
+pidfile=/tmp/supervisord.pid
+
+[program:api]
+command=python -m api
+directory=/app
+environment=PROCESS_ROLE="api"
+autostart=true
+autorestart=unexpected
+startsecs=3
+stopasgroup=true
+killasgroup=true
+stopwaitsecs=300
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:scheduler]
+command=python -m supply_infra.scheduler
+directory=/app
+environment=PROCESS_ROLE="scheduler"
+autostart=true
+autorestart=unexpected
+startsecs=3
+stopasgroup=true
+killasgroup=true
+stopwaitsecs=300
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:pipeline-worker]
+command=python -m supply_infra.pipeline.cli worker
+directory=/app
+environment=PROCESS_ROLE="worker"
+numprocs=%(ENV_PIPELINE_WORKER_PROCESSES)s
+process_name=%(program_name)s-%(process_num)02d
+autostart=true
+autorestart=unexpected
+startsecs=3
+stopasgroup=true
+killasgroup=true
+stopwaitsecs=300
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[program:pipeline-reconciler]
+command=python -m supply_infra.pipeline.cli reconciler
+directory=/app
+environment=PROCESS_ROLE="reconciler"
+autostart=true
+autorestart=unexpected
+startsecs=3
+stopasgroup=true
+killasgroup=true
+stopwaitsecs=300
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0

pyproject.toml

@@ -16,6 +16,8 @@ dependencies = [
     "rich>=13.0",
     "sqlalchemy>=2.0",
     "pymysql>=1.1",
+    "alembic>=1.13",
+    "requests>=2.32",
     "apscheduler>=3.10",
     "python-dotenv>=1.0",
     "oss2>=2.18.0",
@@ -39,6 +41,8 @@ packages = ["supply_agent", "supply_infra", "agents", "api"]
 [project.scripts]
 supply-visualize = "supply_agent.logging.cli:main"
 supply-api = "api.run:main"
+supply-scheduler = "supply_infra.scheduler.__main__:main"
+supply-pipeline = "supply_infra.pipeline.cli:main"
 
 [tool.ruff]
 line-length = 100

requirements.txt

@@ -13,6 +13,8 @@ markdown>=3.6
 # Database
 sqlalchemy>=2.0
 pymysql>=1.1
+alembic>=1.13
+requests>=2.32
 
 # Scheduler
 apscheduler>=3.10

scripts/container-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "${RUN_DB_MIGRATIONS:-true}" == "true" ]]; then
+  alembic upgrade head
+fi
+
+exec /usr/bin/supervisord -c /app/deploy/supervisord.pipeline.conf

scripts/docker-deploy.sh

@@ -68,7 +68,11 @@ cd "$BUILD_DIR"
 docker build -t "$IMAGE_TAG" -t "$LATEST_TAG" .
 
 echo "==> Verifying video truncate runtime in image"
-docker run --rm "$IMAGE_TAG" python scripts/verify_video_truncate_runtime.py
+docker run --rm --entrypoint python "$IMAGE_TAG" scripts/verify_video_truncate_runtime.py
+
+echo "==> Verifying pipeline imports and migration head"
+docker run --rm --entrypoint python "$IMAGE_TAG" -m compileall -q supply_infra api alembic
+docker run --rm --entrypoint alembic "$IMAGE_TAG" heads
 
 if [[ "$PUSH" == "true" ]]; then
   echo "==> Pushing: ${IMAGE_TAG}"

supply_infra/aigc/client.py

@@ -40,7 +40,13 @@ class AigcClient:
                 dry_run = env_dry_run in {"1", "true", "yes", "on"}
             else:
                 dry_run = settings.aigc_dry_run
-        self.dry_run = bool(dry_run)
+        # Phase one safety barrier: callers cannot enable real side effects unless
+        # the global control-plane switch is also explicitly enabled.
+        self.dry_run = (
+            bool(dry_run)
+            or settings.aigc_dry_run
+            or not settings.pipeline_external_effects_enabled
+        )
 
     def create_video_crawler_plan(
         self,
@@ -194,9 +200,9 @@ class AigcClient:
             return response.json()
         except Exception as exc:
             logger.error(
-                "invoke aigc platform error. url=%s request=%s error=%s",
+                "invoke aigc platform error. url=%s params=%s error=%s",
                 url,
-                json.dumps(request, ensure_ascii=False),
+                json.dumps(params, ensure_ascii=False),
                 exc,
             )
             return {}

supply_infra/config.py

@@ -4,7 +4,7 @@ from functools import lru_cache
 from pathlib import Path
 from urllib.parse import quote_plus
 
-from pydantic import Field
+from pydantic import Field, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 # supply_infra/config.py -> project root; avoid depending on process cwd
@@ -27,7 +27,35 @@ class InfraSettings(BaseSettings):
     mysql_user: str = Field(default="root", alias="MYSQL_USER")
     mysql_password: str = Field(default="", alias="MYSQL_PASSWORD")
     mysql_database: str = Field(default="supply_agent", alias="MYSQL_DATABASE")
-    mysql_pool_size: int = Field(default=5, alias="MYSQL_POOL_SIZE")
+    mysql_pool_size: int = Field(default=1, ge=1, alias="MYSQL_POOL_SIZE")
+    mysql_connection_budget: int = Field(
+        default=40,
+        ge=30,
+        le=50,
+        alias="MYSQL_CONNECTION_BUDGET",
+    )
+    mysql_pool_size_api: int = Field(default=6, ge=1, alias="MYSQL_POOL_SIZE_API")
+    mysql_pool_size_control: int = Field(
+        default=1,
+        ge=1,
+        alias="MYSQL_POOL_SIZE_CONTROL",
+    )
+    mysql_max_overflow: int = Field(default=0, ge=0, alias="MYSQL_MAX_OVERFLOW")
+    mysql_pool_timeout_seconds: int = Field(
+        default=10,
+        ge=1,
+        alias="MYSQL_POOL_TIMEOUT_SECONDS",
+    )
+    mysql_pool_recycle_seconds: int = Field(
+        default=1800,
+        ge=60,
+        alias="MYSQL_POOL_RECYCLE_SECONDS",
+    )
+    mysql_operational_reserve: int = Field(
+        default=4,
+        ge=2,
+        alias="MYSQL_OPERATIONAL_RESERVE",
+    )
     mysql_echo: bool = Field(default=False, alias="MYSQL_ECHO")
 
     # ODPS (MaxCompute)
@@ -41,7 +69,80 @@ class InfraSettings(BaseSettings):
 
     # Scheduler
     scheduler_timezone: str = Field(default="Asia/Shanghai", alias="SCHEDULER_TIMEZONE")
-    scheduler_enabled: bool = Field(default=True, alias="SCHEDULER_ENABLED")
+    scheduler_enabled: bool = Field(default=False, alias="SCHEDULER_ENABLED")
+    scheduler_cron_hour: int = Field(default=15, ge=0, le=23, alias="SCHEDULER_CRON_HOUR")
+    scheduler_cron_minute: int = Field(
+        default=0,
+        ge=0,
+        le=59,
+        alias="SCHEDULER_CRON_MINUTE",
+    )
+
+    # Pipeline process/control plane
+    process_role: str = Field(default="api", alias="PROCESS_ROLE")
+    database_auto_create: bool = Field(default=False, alias="DATABASE_AUTO_CREATE")
+    pipeline_worker_processes: int = Field(
+        default=4,
+        ge=1,
+        le=20,
+        alias="PIPELINE_WORKER_PROCESSES",
+    )
+    pipeline_max_active_steps: int = Field(
+        default=4,
+        ge=1,
+        le=20,
+        alias="PIPELINE_MAX_ACTIVE_STEPS",
+    )
+    pipeline_worker_poll_seconds: float = Field(
+        default=2.0,
+        gt=0,
+        alias="PIPELINE_WORKER_POLL_SECONDS",
+    )
+    pipeline_lease_seconds: int = Field(
+        default=120,
+        ge=30,
+        alias="PIPELINE_LEASE_SECONDS",
+    )
+    pipeline_heartbeat_seconds: int = Field(
+        default=30,
+        ge=5,
+        alias="PIPELINE_HEARTBEAT_SECONDS",
+    )
+    pipeline_reconcile_seconds: int = Field(
+        default=60,
+        ge=10,
+        alias="PIPELINE_RECONCILE_SECONDS",
+    )
+    pipeline_missed_run_grace_seconds: int = Field(
+        default=600,
+        ge=60,
+        alias="PIPELINE_MISSED_RUN_GRACE_SECONDS",
+    )
+    pipeline_shutdown_grace_seconds: int = Field(
+        default=300,
+        ge=10,
+        alias="PIPELINE_SHUTDOWN_GRACE_SECONDS",
+    )
+    pipeline_warn_after_hours: int = Field(
+        default=12,
+        ge=1,
+        alias="PIPELINE_WARN_AFTER_HOURS",
+    )
+    pipeline_critical_before_next_minutes: int = Field(
+        default=120,
+        ge=1,
+        alias="PIPELINE_CRITICAL_BEFORE_NEXT_MINUTES",
+    )
+    pipeline_final_warn_before_next_minutes: int = Field(
+        default=30,
+        ge=1,
+        alias="PIPELINE_FINAL_WARN_BEFORE_NEXT_MINUTES",
+    )
+    pipeline_log_dir: str = Field(default="logs/pipeline", alias="PIPELINE_LOG_DIR")
+    pipeline_external_effects_enabled: bool = Field(
+        default=False,
+        alias="PIPELINE_EXTERNAL_EFFECTS_ENABLED",
+    )
 
     # Aliyun OSS (agent run log publishing)
     aliyun_oss_access_key_id: str = Field(default="", alias="ALIYUN_OSS_ACCESS_KEY_ID")
@@ -61,7 +162,40 @@ class InfraSettings(BaseSettings):
 
     # AIGC platform
     aigc_api_token: str = Field(default="", alias="AIGC_API_TOKEN")
-    aigc_dry_run: bool = Field(default=False, alias="AIGC_DRY_RUN")
+    aigc_dry_run: bool = Field(default=True, alias="AIGC_DRY_RUN")
+
+    @model_validator(mode="after")
+    def validate_pipeline_runtime(self) -> "InfraSettings":
+        roles = {"api", "scheduler", "worker", "reconciler", "step", "cli"}
+        if self.process_role not in roles:
+            raise ValueError(
+                f"PROCESS_ROLE must be one of {sorted(roles)}, got {self.process_role!r}"
+            )
+        if self.pipeline_heartbeat_seconds >= self.pipeline_lease_seconds:
+            raise ValueError(
+                "PIPELINE_HEARTBEAT_SECONDS must be smaller than PIPELINE_LEASE_SECONDS"
+            )
+        if self.pipeline_max_active_steps > self.pipeline_worker_processes:
+            raise ValueError(
+                "PIPELINE_MAX_ACTIVE_STEPS cannot exceed PIPELINE_WORKER_PROCESSES"
+            )
+        if self.mysql_max_overflow != 0:
+            raise ValueError("MYSQL_MAX_OVERFLOW must remain 0 for connection budgeting")
+        required = self.pipeline_required_connections
+        if required > self.mysql_connection_budget:
+            raise ValueError(
+                "Pipeline connection budget exceeded: "
+                f"required={required} budget={self.mysql_connection_budget}"
+            )
+        if (
+            self.pipeline_external_effects_enabled
+            and not self.aigc_dry_run
+        ):
+            raise ValueError(
+                "Real AIGC effects are disabled in the current migration phase; "
+                "set AIGC_DRY_RUN=true"
+            )
+        return self
 
     @property
     def mysql_url(self) -> str:
@@ -73,6 +207,25 @@ class InfraSettings(BaseSettings):
             f"?charset=utf8mb4"
         )
 
+    @property
+    def selected_mysql_pool_size(self) -> int:
+        if self.process_role == "api":
+            return self.mysql_pool_size_api
+        if self.process_role in {"scheduler", "worker", "reconciler", "step"}:
+            return self.mysql_pool_size_control
+        return self.mysql_pool_size
+
+    @property
+    def pipeline_required_connections(self) -> int:
+        control_processes = 2  # scheduler + reconciler
+        return (
+            self.mysql_pool_size_api
+            + control_processes * self.mysql_pool_size_control
+            + self.pipeline_worker_processes * self.mysql_pool_size_control
+            + self.pipeline_max_active_steps * self.mysql_pool_size_control
+            + self.mysql_operational_reserve
+        )
+
 
 @lru_cache
 def get_infra_settings() -> InfraSettings:

supply_infra/db/__init__.py

@@ -1,6 +1,6 @@
 """Database layer."""
 
 from supply_infra.db.base import Base
-from supply_infra.db.session import get_session, init_db
+from supply_infra.db.session import dispose_engine, get_session, init_db
 
-__all__ = ["Base", "get_session", "init_db"]
+__all__ = ["Base", "dispose_engine", "get_session", "init_db"]

supply_infra/db/models/__init__.py

@@ -23,6 +23,10 @@ from supply_infra.db.models.multi_demand_pool_di import MultiDemandPoolDi
 from supply_infra.db.models.multi_demand_video_detail import MultiDemandVideoDetail
 from supply_infra.db.models.multi_demand_video_point import MultiDemandVideoPoint
 from supply_infra.db.models.oss_log import OssLog
+from supply_infra.db.models.pipeline_lock import PipelineLock
+from supply_infra.db.models.pipeline_outbox import PipelineOutbox
+from supply_infra.db.models.pipeline_run import PipelineRun
+from supply_infra.db.models.pipeline_step_run import PipelineStepRun
 from supply_infra.db.models.scheduler_job_execution import SchedulerJobExecution
 from supply_infra.db.models.video_discovery import (
     VideoDiscoveryCandidate,
@@ -50,6 +54,10 @@ __all__ = [
     "MultiDemandVideoDetail",
     "MultiDemandVideoPoint",
     "OssLog",
+    "PipelineLock",
+    "PipelineOutbox",
+    "PipelineRun",
+    "PipelineStepRun",
     "SchedulerJobExecution",
     "VideoDiscoveryCandidate",
     "VideoDiscoveryRun",

supply_infra/db/models/pipeline_lock.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import BigInteger, DateTime, String, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from supply_infra.db.base import Base
+
+
+class PipelineLock(Base):
+    """Auditable lease used for cross-process pipeline coordination."""
+
+    __tablename__ = "pipeline_lock"
+
+    lock_key: Mapped[str] = mapped_column(String(191), primary_key=True)
+    owner_run_id: Mapped[str] = mapped_column(String(36), nullable=False)
+    owner_instance: Mapped[str] = mapped_column(String(191), nullable=False)
+    lease_until: Mapped[datetime] = mapped_column(DateTime, nullable=False)
+    heartbeat_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
+    version: Mapped[int] = mapped_column(BigInteger, nullable=False, default=1)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime,
+        nullable=False,
+        server_default=func.now(),
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime,
+        nullable=False,
+        server_default=func.now(),
+        onupdate=func.now(),
+    )

supply_infra/db/models/pipeline_outbox.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import (
+    Boolean,
+    DateTime,
+    ForeignKey,
+    Index,
+    JSON,
+    String,
+    Text,
+    UniqueConstraint,
+    func,
+)
+from sqlalchemy.orm import Mapped, mapped_column
+
+from supply_infra.db.base import Base
+
+
+class PipelineOutbox(Base):
+    """Dry-run ledger of intended AIGC writes; it is not dispatched in phase one."""
+
+    __tablename__ = "pipeline_outbox"
+    __table_args__ = (
+        UniqueConstraint("idempotency_key", name="uk_pipeline_outbox_idempotency"),
+        Index("idx_pipeline_outbox_run", "run_id", "step_run_id"),
+        Index("idx_pipeline_outbox_status", "status", "created_at"),
+    )
+
+    outbox_id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    run_id: Mapped[str] = mapped_column(
+        String(36),
+        ForeignKey("pipeline_run.run_id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    step_run_id: Mapped[str] = mapped_column(
+        String(36),
+        ForeignKey("pipeline_step_run.step_run_id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    effect_type: Mapped[str] = mapped_column(String(64), nullable=False)
+    idempotency_key: Mapped[str] = mapped_column(String(191), nullable=False)
+    payload_hash: Mapped[str] = mapped_column(String(64), nullable=False)
+    payload_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
+    payload_uri: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    status: Mapped[str] = mapped_column(
+        String(32),
+        nullable=False,
+        default="dry_run_recorded",
+    )
+    dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    record_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime,
+        nullable=False,
+        server_default=func.now(),
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime,
+        nullable=False,
+        server_default=func.now(),
+        onupdate=func.now(),
+    )

supply_infra/db/models/pipeline_run.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import Boolean, DateTime, Index, JSON, String, Text, UniqueConstraint, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from supply_infra.db.base import Base
+
+
+class PipelineRun(Base):
+    """One durable execution request for the complete supply pipeline."""
+
+    __tablename__ = "pipeline_run"
+    __table_args__ = (
+        UniqueConstraint("dedupe_key", name="uk_pipeline_run_dedupe_key"),
+        Index("idx_pipeline_run_biz", "pipeline_key", "biz_dt", "created_at"),
+        Index("idx_pipeline_run_status", "status", "lease_until"),
+        Index("idx_pipeline_run_deadline", "status", "deadline_at"),
+    )
+
+    run_id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    dedupe_key: Mapped[str] = mapped_column(String(191), nullable=False)
+    pipeline_key: Mapped[str] = mapped_column(
+        String(64),
+        nullable=False,
+        default="supply_pipeline",
+    )
+    biz_dt: Mapped[str] = mapped_column(String(8), nullable=False)
+    trigger_type: Mapped[str] = mapped_column(String(24), nullable=False)
+    trigger_source: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    triggered_by: Mapped[str | None] = mapped_column(String(128), nullable=True)
+    trigger_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
+    parent_run_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+    run_mode: Mapped[str] = mapped_column(String(24), nullable=False, default="full")
+    dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
+    current_step: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    scheduled_for: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    deadline_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    lease_owner: Mapped[str | None] = mapped_column(String(191), nullable=True)
+    lease_until: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    code_version: Mapped[str | None] = mapped_column(String(128), nullable=True)
+    config_snapshot_json: Mapped[dict[str, Any]] = mapped_column(
+        JSON,
+        nullable=False,
+        default=dict,
+    )
+    date_snapshot_json: Mapped[dict[str, Any]] = mapped_column(
+        JSON,
+        nullable=False,
+        default=dict,
+    )
+    summary_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
+    error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime,
+        nullable=False,
+        server_default=func.now(),
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime,
+        nullable=False,
+        server_default=func.now(),
+        onupdate=func.now(),
+    )

supply_infra/db/models/pipeline_step_run.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import (
+    Boolean,
+    DateTime,
+    ForeignKey,
+    Index,
+    Integer,
+    JSON,
+    String,
+    Text,
+    UniqueConstraint,
+    func,
+)
+from sqlalchemy.orm import Mapped, mapped_column
+
+from supply_infra.db.base import Base
+
+
+class PipelineStepRun(Base):
+    """One durable attempt of a pipeline step."""
+
+    __tablename__ = "pipeline_step_run"
+    __table_args__ = (
+        UniqueConstraint(
+            "run_id",
+            "step_key",
+            "attempt",
+            name="uk_pipeline_step_run_attempt",
+        ),
+        Index("idx_pipeline_step_claim", "status", "next_retry_at", "step_order"),
+        Index("idx_pipeline_step_run", "run_id", "step_order", "attempt"),
+        Index("idx_pipeline_step_lease", "status", "lease_until"),
+    )
+
+    step_run_id: Mapped[str] = mapped_column(String(36), primary_key=True)
+    run_id: Mapped[str] = mapped_column(
+        String(36),
+        ForeignKey("pipeline_run.run_id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    step_key: Mapped[str] = mapped_column(String(64), nullable=False)
+    step_order: Mapped[int] = mapped_column(Integer, nullable=False)
+    attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    status: Mapped[str] = mapped_column(String(32), nullable=False, default="pending")
+    critical: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    dependency_snapshot_json: Mapped[list[str]] = mapped_column(
+        JSON,
+        nullable=False,
+        default=list,
+    )
+    input_snapshot_json: Mapped[dict[str, Any]] = mapped_column(
+        JSON,
+        nullable=False,
+        default=dict,
+    )
+    timeout_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
+    max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    retryable: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    next_retry_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    lease_owner: Mapped[str | None] = mapped_column(String(191), nullable=True)
+    lease_until: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    exit_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    result_summary_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
+    metrics_json: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
+    error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+    log_uri: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime,
+        nullable=False,
+        server_default=func.now(),
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime,
+        nullable=False,
+        server_default=func.now(),
+        onupdate=func.now(),
+    )

supply_infra/db/repositories/__init__.py

@@ -36,6 +36,10 @@ from supply_infra.db.repositories.multi_demand_video_point_repo import (
     MultiDemandVideoPointRepository,
 )
 from supply_infra.db.repositories.oss_log_repo import OssLogRepository
+from supply_infra.db.repositories.pipeline_lock_repo import PipelineLockRepository
+from supply_infra.db.repositories.pipeline_outbox_repo import PipelineOutboxRepository
+from supply_infra.db.repositories.pipeline_run_repo import PipelineRunRepository
+from supply_infra.db.repositories.pipeline_step_run_repo import PipelineStepRunRepository
 from supply_infra.db.repositories.scheduler_job_execution_repo import (
     SchedulerJobExecutionRepository,
 )
@@ -60,6 +64,10 @@ __all__ = [
     "MultiDemandVideoDetailRepository",
     "MultiDemandVideoPointRepository",
     "OssLogRepository",
+    "PipelineLockRepository",
+    "PipelineOutboxRepository",
+    "PipelineRunRepository",
+    "PipelineStepRunRepository",
     "SchedulerJobExecutionRepository",
     "VideoDiscoveryRepository",
 ]

supply_infra/db/repositories/pipeline_lock_repo.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+from sqlalchemy import select
+
+from supply_infra.db.models.pipeline_lock import PipelineLock
+from supply_infra.db.repositories.base import BaseRepository
+
+CONTROL_PLANE_GUARD_KEY = "pipeline:claim_guard"
+
+
+class PipelineLockRepository(BaseRepository[PipelineLock]):
+    model = PipelineLock
+
+    def get(self, lock_key: str) -> PipelineLock | None:
+        return self.session.get(PipelineLock, lock_key)
+
+    def lock_control_plane(self, *, now: datetime) -> PipelineLock:
+        """Lock the singleton row used for atomic scheduling decisions."""
+        lock = self.session.scalar(
+            select(PipelineLock)
+            .where(PipelineLock.lock_key == CONTROL_PLANE_GUARD_KEY)
+            .with_for_update()
+        )
+        if lock is None:
+            # Production gets this row from Alembic. This fallback supports
+            # local databases created from SQLAlchemy metadata.
+            lock = PipelineLock(
+                lock_key=CONTROL_PLANE_GUARD_KEY,
+                owner_run_id="control-plane",
+                owner_instance="claim-coordinator",
+                lease_until=now,
+                heartbeat_at=now,
+                version=1,
+            )
+            self.session.add(lock)
+            self.session.flush()
+        return lock
+
+    def touch(
+        self,
+        *,
+        lock_key: str,
+        owner: str,
+        lease_seconds: int,
+        now: datetime,
+    ) -> PipelineLock:
+        lock = self.session.scalar(
+            select(PipelineLock)
+            .where(PipelineLock.lock_key == lock_key)
+            .with_for_update()
+        )
+        lease_until = now + timedelta(seconds=lease_seconds)
+        if lock is None:
+            lock = PipelineLock(
+                lock_key=lock_key,
+                owner_run_id="control-plane",
+                owner_instance=owner,
+                lease_until=lease_until,
+                heartbeat_at=now,
+                version=1,
+            )
+            self.session.add(lock)
+        else:
+            lock.owner_instance = owner
+            lock.lease_until = lease_until
+            lock.heartbeat_at = now
+            lock.version += 1
+        self.session.flush()
+        return lock
+
+    def acquire(
+        self,
+        *,
+        lock_key: str,
+        run_id: str,
+        owner: str,
+        lease_seconds: int,
+        now: datetime,
+    ) -> bool:
+        stmt = (
+            select(PipelineLock)
+            .where(PipelineLock.lock_key == lock_key)
+            .with_for_update()
+        )
+        lock = self.session.scalar(stmt)
+        lease_until = now + timedelta(seconds=lease_seconds)
+        if lock is None:
+            self.add(
+                PipelineLock(
+                    lock_key=lock_key,
+                    owner_run_id=run_id,
+                    owner_instance=owner,
+                    lease_until=lease_until,
+                    heartbeat_at=now,
+                    version=1,
+                )
+            )
+            return True
+        if lock.lease_until >= now and lock.owner_run_id != run_id:
+            return False
+        lock.owner_run_id = run_id
+        lock.owner_instance = owner
+        lock.lease_until = lease_until
+        lock.heartbeat_at = now
+        lock.version += 1
+        self.session.flush()
+        return True
+
+    def release(self, lock_key: str, *, run_id: str) -> bool:
+        stmt = (
+            select(PipelineLock)
+            .where(PipelineLock.lock_key == lock_key)
+            .with_for_update()
+        )
+        lock = self.session.scalar(stmt)
+        if lock is None or lock.owner_run_id != run_id:
+            return False
+        self.session.delete(lock)
+        self.session.flush()
+        return True

supply_infra/db/repositories/pipeline_outbox_repo.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from sqlalchemy import select
+
+from supply_infra.db.models.pipeline_outbox import PipelineOutbox
+from supply_infra.db.repositories.base import BaseRepository
+
+
+class PipelineOutboxRepository(BaseRepository[PipelineOutbox]):
+    model = PipelineOutbox
+
+    def get_by_idempotency_key(self, key: str) -> PipelineOutbox | None:
+        stmt = select(PipelineOutbox).where(PipelineOutbox.idempotency_key == key)
+        return self.session.scalar(stmt)
+
+    def record_dry_run(
+        self,
+        *,
+        run_id: str,
+        step_run_id: str,
+        effect_type: str,
+        idempotency_key: str,
+        payload_hash: str,
+        payload: dict[str, Any] | None,
+        payload_uri: str | None = None,
+    ) -> PipelineOutbox:
+        existing = self.get_by_idempotency_key(idempotency_key)
+        if existing is not None:
+            return existing
+        return self.add(
+            PipelineOutbox(
+                outbox_id=str(uuid.uuid4()),
+                run_id=run_id,
+                step_run_id=step_run_id,
+                effect_type=effect_type,
+                idempotency_key=idempotency_key,
+                payload_hash=payload_hash,
+                payload_json=payload,
+                payload_uri=payload_uri,
+                status="dry_run_recorded",
+                dry_run=True,
+            )
+        )
+
+    def list_for_run(self, run_id: str) -> list[PipelineOutbox]:
+        stmt = (
+            select(PipelineOutbox)
+            .where(PipelineOutbox.run_id == run_id)
+            .order_by(PipelineOutbox.created_at)
+        )
+        return list(self.session.scalars(stmt).all())

supply_infra/db/repositories/pipeline_run_repo.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import exists, select
+
+from supply_infra.db.models.pipeline_run import PipelineRun
+from supply_infra.db.models.pipeline_step_run import PipelineStepRun
+from supply_infra.db.repositories.base import BaseRepository
+
+
+class PipelineRunRepository(BaseRepository[PipelineRun]):
+    model = PipelineRun
+
+    def get(self, run_id: str, *, for_update: bool = False) -> PipelineRun | None:
+        stmt = select(PipelineRun).where(PipelineRun.run_id == run_id)
+        if for_update:
+            stmt = stmt.with_for_update()
+        return self.session.scalar(stmt)
+
+    def get_by_dedupe_key(self, dedupe_key: str) -> PipelineRun | None:
+        stmt = select(PipelineRun).where(PipelineRun.dedupe_key == dedupe_key)
+        return self.session.scalar(stmt)
+
+    def get_for_business_day(
+        self,
+        *,
+        pipeline_key: str,
+        biz_dt: str,
+    ) -> PipelineRun | None:
+        stmt = (
+            select(PipelineRun)
+            .where(PipelineRun.pipeline_key == pipeline_key)
+            .where(PipelineRun.biz_dt == biz_dt)
+            .order_by(PipelineRun.created_at.desc())
+            .limit(1)
+        )
+        return self.session.scalar(stmt)
+
+    def create(self, values: dict[str, Any]) -> PipelineRun:
+        return self.add(PipelineRun(**values))
+
+    def list_recent(
+        self,
+        *,
+        limit: int = 50,
+        status: str | None = None,
+        biz_dt: str | None = None,
+    ) -> list[PipelineRun]:
+        stmt = select(PipelineRun)
+        if status:
+            stmt = stmt.where(PipelineRun.status == status)
+        if biz_dt:
+            stmt = stmt.where(PipelineRun.biz_dt == biz_dt)
+        stmt = stmt.order_by(PipelineRun.created_at.desc()).limit(limit)
+        return list(self.session.scalars(stmt).all())
+
+    def get_latest_nonterminal(self) -> PipelineRun | None:
+        has_running_step = exists(
+            select(PipelineStepRun.step_run_id)
+            .where(PipelineStepRun.run_id == PipelineRun.run_id)
+            .where(PipelineStepRun.status == "running")
+        )
+        stmt = (
+            select(PipelineRun)
+            .where(
+                PipelineRun.status.in_(
+                    ("queued", "running", "cancelling", "blocked_previous_run")
+                )
+                | has_running_step
+            )
+            .order_by(PipelineRun.created_at.desc())
+            .limit(1)
+        )
+        return self.session.scalar(stmt)
+
+    def list_nonterminal(self, *, limit: int = 100) -> list[PipelineRun]:
+        stmt = (
+            select(PipelineRun)
+            .where(
+                PipelineRun.status.in_(
+                    ("queued", "running", "cancelling", "blocked_previous_run")
+                )
+            )
+            .order_by(PipelineRun.created_at)
+            .limit(limit)
+        )
+        return list(self.session.scalars(stmt).all())
+
+    def list_past_deadline(
+        self,
+        now: datetime,
+        *,
+        limit: int = 100,
+    ) -> list[PipelineRun]:
+        stmt = (
+            select(PipelineRun)
+            .where(PipelineRun.status.in_(("queued", "running")))
+            .where(PipelineRun.deadline_at <= now)
+            .order_by(PipelineRun.deadline_at)
+            .with_for_update(skip_locked=True)
+            .limit(limit)
+        )
+        return list(self.session.scalars(stmt).all())
+
+    def list_blocked_previous(self, *, limit: int = 100) -> list[PipelineRun]:
+        stmt = (
+            select(PipelineRun)
+            .where(PipelineRun.status == "blocked_previous_run")
+            .order_by(PipelineRun.biz_dt, PipelineRun.created_at)
+            .with_for_update(skip_locked=True)
+            .limit(limit)
+        )
+        return list(self.session.scalars(stmt).all())
+
+    def mark_running(
+        self,
+        run: PipelineRun,
+        *,
+        step_key: str,
+        owner: str,
+        now: datetime,
+        lease_until: datetime,
+    ) -> None:
+        if run.started_at is None:
+            run.started_at = now
+        run.status = "running"
+        run.current_step = step_key
+        run.heartbeat_at = now
+        run.lease_owner = owner
+        run.lease_until = lease_until
+        self.session.flush()
+
+    def finish(
+        self,
+        run: PipelineRun,
+        *,
+        status: str,
+        now: datetime,
+        summary: dict[str, Any] | None = None,
+        error_code: str | None = None,
+        error_message: str | None = None,
+    ) -> None:
+        run.status = status
+        run.finished_at = now
+        run.heartbeat_at = now
+        run.lease_owner = None
+        run.lease_until = None
+        run.current_step = None
+        run.summary_json = summary
+        run.error_code = error_code
+        run.error_message = error_message
+        self.session.flush()
+
+    def heartbeat(
+        self,
+        run_id: str,
+        *,
+        owner: str,
+        now: datetime,
+        lease_until: datetime,
+    ) -> bool:
+        run = self.get(run_id, for_update=True)
+        if run is None or run.lease_owner != owner or run.status != "running":
+            return False
+        run.heartbeat_at = now
+        run.lease_until = lease_until
+        self.session.flush()
+        return True

supply_infra/db/repositories/pipeline_step_run_repo.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timedelta
+from typing import Any
+
+from sqlalchemy import func, select
+
+from supply_infra.db.models.pipeline_step_run import PipelineStepRun
+from supply_infra.db.repositories.base import BaseRepository
+from supply_infra.db.repositories.pipeline_lock_repo import PipelineLockRepository
+
+
+class PipelineStepRunRepository(BaseRepository[PipelineStepRun]):
+    model = PipelineStepRun
+
+    def get(
+        self,
+        step_run_id: str,
+        *,
+        for_update: bool = False,
+    ) -> PipelineStepRun | None:
+        stmt = select(PipelineStepRun).where(
+            PipelineStepRun.step_run_id == step_run_id
+        )
+        if for_update:
+            stmt = stmt.with_for_update()
+        return self.session.scalar(stmt)
+
+    def create_steps(self, rows: list[dict[str, Any]]) -> list[PipelineStepRun]:
+        entities = [PipelineStepRun(**row) for row in rows]
+        self.session.add_all(entities)
+        self.session.flush()
+        return entities
+
+    def list_for_run(self, run_id: str) -> list[PipelineStepRun]:
+        stmt = (
+            select(PipelineStepRun)
+            .where(PipelineStepRun.run_id == run_id)
+            .order_by(PipelineStepRun.step_order, PipelineStepRun.attempt)
+        )
+        return list(self.session.scalars(stmt).all())
+
+    def latest_attempts(self, run_id: str) -> dict[str, PipelineStepRun]:
+        latest: dict[str, PipelineStepRun] = {}
+        for item in self.list_for_run(run_id):
+            latest[item.step_key] = item
+        return latest
+
+    def claim_next_ready(
+        self,
+        *,
+        owner: str,
+        lease_seconds: int,
+        max_active_steps: int,
+        now: datetime,
+    ) -> PipelineStepRun | None:
+        # Serialize count-and-claim so concurrent workers cannot oversubscribe
+        # the configured global active-step cap.
+        PipelineLockRepository(self.session).lock_control_plane(now=now)
+
+        active_steps = self.session.scalar(
+            select(func.count())
+            .select_from(PipelineStepRun)
+            .where(PipelineStepRun.status == "running")
+        )
+        if int(active_steps or 0) >= max_active_steps:
+            return None
+        running_run_ids = list(
+            self.session.scalars(
+                select(PipelineStepRun.run_id)
+                .where(PipelineStepRun.status == "running")
+                .distinct()
+            ).all()
+        )
+
+        stmt = (
+            select(PipelineStepRun)
+            .where(PipelineStepRun.status == "ready")
+            .where(
+                (PipelineStepRun.next_retry_at.is_(None))
+                | (PipelineStepRun.next_retry_at <= now)
+            )
+            .order_by(PipelineStepRun.created_at, PipelineStepRun.step_order)
+            .with_for_update(skip_locked=True)
+            .limit(1)
+        )
+        if running_run_ids:
+            # A daily run may contain parallel steps in the future, but a
+            # different run must never overlap while an old process is alive.
+            stmt = stmt.where(PipelineStepRun.run_id.in_(running_run_ids))
+        step = self.session.scalar(stmt)
+        if step is None:
+            return None
+        step.status = "running"
+        step.started_at = step.started_at or now
+        step.heartbeat_at = now
+        step.lease_owner = owner
+        step.lease_until = now + timedelta(seconds=lease_seconds)
+        self.session.flush()
+        return step
+
+    def has_running_for_run(self, run_id: str) -> bool:
+        count = self.session.scalar(
+            select(func.count())
+            .select_from(PipelineStepRun)
+            .where(PipelineStepRun.run_id == run_id)
+            .where(PipelineStepRun.status == "running")
+        )
+        return bool(count)
+
+    def heartbeat(
+        self,
+        step_run_id: str,
+        *,
+        owner: str,
+        now: datetime,
+        lease_until: datetime,
+    ) -> bool:
+        step = self.get(step_run_id, for_update=True)
+        if step is None or step.status != "running" or step.lease_owner != owner:
+            return False
+        step.heartbeat_at = now
+        step.lease_until = lease_until
+        self.session.flush()
+        return True
+
+    def finish(
+        self,
+        step: PipelineStepRun,
+        *,
+        status: str,
+        now: datetime,
+        exit_code: int | None,
+        result: dict[str, Any] | None,
+        error_code: str | None,
+        error_message: str | None,
+        log_uri: str | None,
+    ) -> None:
+        step.status = status
+        step.finished_at = now
+        step.heartbeat_at = now
+        step.lease_owner = None
+        step.lease_until = None
+        step.exit_code = exit_code
+        step.result_summary_json = result
+        step.error_code = error_code
+        step.error_message = error_message
+        step.log_uri = log_uri
+        self.session.flush()
+
+    def create_retry_attempt(
+        self,
+        previous: PipelineStepRun,
+        *,
+        status: str = "ready",
+        next_retry_at: datetime | None = None,
+    ) -> PipelineStepRun:
+        entity = PipelineStepRun(
+            step_run_id=str(uuid.uuid4()),
+            run_id=previous.run_id,
+            step_key=previous.step_key,
+            step_order=previous.step_order,
+            attempt=previous.attempt + 1,
+            status=status,
+            critical=previous.critical,
+            dependency_snapshot_json=list(previous.dependency_snapshot_json or []),
+            input_snapshot_json=dict(previous.input_snapshot_json or {}),
+            timeout_seconds=previous.timeout_seconds,
+            max_attempts=previous.max_attempts,
+            retryable=previous.retryable,
+            next_retry_at=next_retry_at,
+        )
+        return self.add(entity)
+
+    def list_expired_running(self, now: datetime, *, limit: int = 100) -> list[PipelineStepRun]:
+        stmt = (
+            select(PipelineStepRun)
+            .where(PipelineStepRun.status == "running")
+            .where(PipelineStepRun.lease_until.is_not(None))
+            .where(PipelineStepRun.lease_until < now)
+            .order_by(PipelineStepRun.lease_until)
+            .with_for_update(skip_locked=True)
+            .limit(limit)
+        )
+        return list(self.session.scalars(stmt).all())
+
+    def promote_due_retries(self, now: datetime, *, limit: int = 100) -> int:
+        stmt = (
+            select(PipelineStepRun)
+            .where(PipelineStepRun.status == "retry_wait")
+            .where(PipelineStepRun.next_retry_at <= now)
+            .order_by(PipelineStepRun.next_retry_at)
+            .with_for_update(skip_locked=True)
+            .limit(limit)
+        )
+        count = 0
+        for step in self.session.scalars(stmt).all():
+            step.status = "ready"
+            count += 1
+        self.session.flush()
+        return count
+
+    def block_unfinished_downstream(
+        self,
+        *,
+        run_id: str,
+        after_order: int,
+        reason: str,
+    ) -> int:
+        count = 0
+        for step in self.list_for_run(run_id):
+            if step.step_order <= after_order or step.status in {
+                "succeeded",
+                "failed",
+                "timed_out",
+                "cancelled",
+            }:
+                continue
+            step.status = "blocked"
+            step.error_code = "upstream_failed"
+            step.error_message = reason
+            count += 1
+        self.session.flush()
+        return count

supply_infra/db/session.py

@@ -4,7 +4,7 @@ from collections.abc import Generator
 from contextlib import contextmanager
 from typing import Any
 
-from sqlalchemy import create_engine, inspect
+from sqlalchemy import create_engine, event, inspect
 from sqlalchemy.orm import Session, sessionmaker
 
 from supply_infra.config import get_infra_settings
@@ -14,6 +14,15 @@ _engine: Any = None
 _SessionLocal: sessionmaker[Session] | None = None
 
 
+def _set_mysql_session_utc(dbapi_connection: Any, _connection_record: Any) -> None:
+    """Keep MySQL-generated timestamps aligned with application UTC values."""
+    cursor = dbapi_connection.cursor()
+    try:
+        cursor.execute("SET time_zone = '+00:00'")
+    finally:
+        cursor.close()
+
+
 def get_engine():
     """Lazy-init SQLAlchemy engine (singleton)."""
     global _engine, _SessionLocal
@@ -21,14 +30,28 @@ def get_engine():
         settings = get_infra_settings()
         _engine = create_engine(
             settings.mysql_url,
-            pool_size=settings.mysql_pool_size,
+            pool_size=settings.selected_mysql_pool_size,
+            max_overflow=settings.mysql_max_overflow,
+            pool_timeout=settings.mysql_pool_timeout_seconds,
+            pool_recycle=settings.mysql_pool_recycle_seconds,
             pool_pre_ping=True,
             echo=settings.mysql_echo,
         )
+        if _engine.dialect.name == "mysql":
+            event.listen(_engine, "connect", _set_mysql_session_utc)
         _SessionLocal = sessionmaker(bind=_engine, autoflush=False, autocommit=False)
     return _engine
 
 
+def dispose_engine() -> None:
+    """Dispose process-local connections, mainly for graceful shutdown and tests."""
+    global _engine, _SessionLocal
+    if _engine is not None:
+        _engine.dispose()
+    _engine = None
+    _SessionLocal = None
+
+
 def init_db() -> dict[str, list[str]]:
     """Create all tables (dev / first-run). Import models before calling."""
     import supply_infra.db.models  # noqa: F401 — register all models

supply_infra/pipeline/__init__.py

@@ -0,0 +1,5 @@
+"""Durable supply pipeline control plane."""
+
+from supply_infra.pipeline.run_service import RunSubmission, submit_pipeline_run
+
+__all__ = ["RunSubmission", "submit_pipeline_run"]

supply_infra/pipeline/cli.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+
+from supply_infra.pipeline.reconciler import PipelineReconciler, reconcile_once
+from supply_infra.pipeline.run_service import (
+    get_pipeline_run,
+    submit_pipeline_run,
+)
+from supply_infra.pipeline.worker import PipelineWorker
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="SupplyAgent durable pipeline control")
+    sub = parser.add_subparsers(dest="command", required=True)
+    submit = sub.add_parser("submit")
+    submit.add_argument("--biz-dt")
+    submit.add_argument("--reason")
+    inspect = sub.add_parser("inspect")
+    inspect.add_argument("--run-id", required=True)
+    sub.add_parser("worker")
+    sub.add_parser("reconciler")
+    sub.add_parser("reconcile-once")
+    return parser
+
+
+def main() -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    )
+    args = _parser().parse_args()
+    if args.command == "submit":
+        result = submit_pipeline_run(
+            biz_dt=args.biz_dt,
+            trigger_type="cli",
+            trigger_source="cli",
+            trigger_reason=args.reason,
+        ).to_dict()
+        print(json.dumps(result, ensure_ascii=False))
+        return
+    if args.command == "inspect":
+        result = get_pipeline_run(args.run_id)
+        print(json.dumps(result, ensure_ascii=False, default=str))
+        raise SystemExit(0 if result is not None else 1)
+    if args.command == "worker":
+        PipelineWorker().run_forever()
+        return
+    if args.command == "reconciler":
+        PipelineReconciler().run_forever()
+        return
+    print(json.dumps(reconcile_once(), ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

supply_infra/pipeline/contracts.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class StepContext:
+    run_id: str
+    step_run_id: str
+    step_key: str
+    biz_dt: str
+    date_snapshot: dict[str, Any]
+    config_snapshot: dict[str, Any]
+    input_snapshot: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class StepResult:
+    success: bool
+    payload: dict[str, Any]
+    error_code: str | None = None
+    error_message: str | None = None
+    exit_code: int | None = None
+    log_uri: str | None = None
+    timed_out: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "success": self.success,
+            "payload": self.payload,
+            "error_code": self.error_code,
+            "error_message": self.error_message,
+            "exit_code": self.exit_code,
+            "log_uri": self.log_uri,
+            "timed_out": self.timed_out,
+        }

supply_infra/pipeline/dag.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class StepDefinition:
+    key: str
+    order: int
+    dependencies: tuple[str, ...]
+    timeout_seconds: int
+    max_attempts: int = 1
+    retryable: bool = False
+    critical: bool = True
+
+
+_RAW_STEPS: tuple[tuple[str, int, bool], ...] = (
+    ("global_tree_sync", 3600, True),
+    ("demand_pool_source_sync", 3600, True),
+    ("demand_classify", 21600, False),
+    ("demand_belong_rel_sync", 3600, True),
+    ("real_metrics_sync", 3600, True),
+    ("popularity_stats", 3600, True),
+    ("category_tree_weight", 3600, True),
+    ("source_video_sync", 7200, True),
+    ("demand_grade", 21600, False),
+    ("demand_expand", 21600, False),
+    ("video_discovery", 21600, False),
+    ("aigc_write_record", 1800, True),
+)
+
+
+def _build_steps() -> tuple[StepDefinition, ...]:
+    result: list[StepDefinition] = []
+    previous: str | None = None
+    for order, (key, timeout_seconds, retryable) in enumerate(_RAW_STEPS, start=1):
+        result.append(
+            StepDefinition(
+                key=key,
+                order=order,
+                dependencies=(previous,) if previous else (),
+                timeout_seconds=timeout_seconds,
+                max_attempts=2 if retryable else 1,
+                retryable=retryable,
+            )
+        )
+        previous = key
+    return tuple(result)
+
+
+PIPELINE_STEPS = _build_steps()
+STEP_BY_KEY = {step.key: step for step in PIPELINE_STEPS}
+
+
+def validate_dag() -> None:
+    seen: set[str] = set()
+    for step in PIPELINE_STEPS:
+        if step.key in seen:
+            raise ValueError(f"Duplicate step key: {step.key}")
+        missing = set(step.dependencies) - seen
+        if missing:
+            raise ValueError(f"Step {step.key} has unresolved dependencies: {sorted(missing)}")
+        seen.add(step.key)
+
+
+validate_dag()

supply_infra/pipeline/dates.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from zoneinfo import ZoneInfo
+
+from supply_infra.config import InfraSettings, get_infra_settings
+
+
+def validate_biz_dt(value: str) -> str:
+    text = str(value).strip()
+    parsed = datetime.strptime(text, "%Y%m%d")
+    if parsed.strftime("%Y%m%d") != text:
+        raise ValueError(f"Invalid biz_dt: {value!r}")
+    return text
+
+
+def resolve_biz_dt(
+    biz_dt: str | None,
+    *,
+    settings: InfraSettings | None = None,
+    now: datetime | None = None,
+) -> str:
+    if biz_dt:
+        return validate_biz_dt(biz_dt)
+    active = settings or get_infra_settings()
+    local_now = now or datetime.now(ZoneInfo(active.scheduler_timezone))
+    if local_now.tzinfo is None:
+        local_now = local_now.replace(tzinfo=ZoneInfo(active.scheduler_timezone))
+    return local_now.astimezone(ZoneInfo(active.scheduler_timezone)).strftime("%Y%m%d")
+
+
+def build_date_snapshot(biz_dt: str) -> dict[str, str]:
+    resolved = validate_biz_dt(biz_dt)
+    day = datetime.strptime(resolved, "%Y%m%d")
+    previous = (day - timedelta(days=1)).strftime("%Y%m%d")
+    return {
+        "biz_dt": resolved,
+        "global_tree_partition": previous,
+        "demand_pool_partition": resolved,
+        "video_decode_partition": previous,
+    }
+
+
+def utc_now() -> datetime:
+    return datetime.now(timezone.utc).replace(tzinfo=None)
+
+
+def next_schedule_utc(
+    *,
+    settings: InfraSettings | None = None,
+    now: datetime | None = None,
+) -> datetime:
+    active = settings or get_infra_settings()
+    zone = ZoneInfo(active.scheduler_timezone)
+    local_now = now or datetime.now(zone)
+    if local_now.tzinfo is None:
+        local_now = local_now.replace(tzinfo=zone)
+    else:
+        local_now = local_now.astimezone(zone)
+    candidate = local_now.replace(
+        hour=active.scheduler_cron_hour,
+        minute=active.scheduler_cron_minute,
+        second=0,
+        microsecond=0,
+    )
+    if candidate <= local_now:
+        candidate += timedelta(days=1)
+    return candidate.astimezone(timezone.utc).replace(tzinfo=None)
+
+
+def next_daily_deadline_utc(
+    *,
+    settings: InfraSettings | None = None,
+    now: datetime | None = None,
+) -> datetime:
+    """Return the next local calendar day's scheduler start as UTC-naive."""
+    active = settings or get_infra_settings()
+    zone = ZoneInfo(active.scheduler_timezone)
+    local_now = now or datetime.now(zone)
+    if local_now.tzinfo is None:
+        local_now = local_now.replace(tzinfo=zone)
+    else:
+        local_now = local_now.astimezone(zone)
+    candidate = (local_now + timedelta(days=1)).replace(
+        hour=active.scheduler_cron_hour,
+        minute=active.scheduler_cron_minute,
+        second=0,
+        microsecond=0,
+    )
+    return candidate.astimezone(timezone.utc).replace(tzinfo=None)
+
+
+def deadline_for_trigger(
+    trigger_type: str,
+    *,
+    settings: InfraSettings | None = None,
+    now: datetime | None = None,
+) -> datetime | None:
+    """Scheduled daily runs have a next-day deadline; manual runs do not."""
+    if trigger_type not in {"cron", "reconcile"}:
+        return None
+    return next_daily_deadline_utc(settings=settings, now=now)

supply_infra/pipeline/enums.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from enum import StrEnum
+
+
+class RunStatus(StrEnum):
+    QUEUED = "queued"
+    RUNNING = "running"
+    SUCCEEDED = "succeeded"
+    FAILED = "failed"
+    BLOCKED = "blocked"
+    CANCELLING = "cancelling"
+    CANCELLED = "cancelled"
+    DEADLINE_EXCEEDED = "deadline_exceeded"
+    BLOCKED_PREVIOUS_RUN = "blocked_previous_run"
+
+
+class StepStatus(StrEnum):
+    PENDING = "pending"
+    READY = "ready"
+    RUNNING = "running"
+    RETRY_WAIT = "retry_wait"
+    SUCCEEDED = "succeeded"
+    FAILED = "failed"
+    BLOCKED = "blocked"
+    TIMED_OUT = "timed_out"
+    CANCELLED = "cancelled"
+
+
+class OutboxStatus(StrEnum):
+    DRY_RUN_RECORDED = "dry_run_recorded"
+    RECORD_FAILED = "record_failed"

supply_infra/pipeline/gates.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class GateDecision:
+    passed: bool
+    error_code: str | None = None
+    error_message: str | None = None
+
+
+def evaluate_step_gate(step_key: str, payload: dict[str, Any]) -> GateDecision:
+    if payload.get("success") is False:
+        return GateDecision(
+            passed=False,
+            error_code=str(payload.get("error_code") or "step_reported_failure"),
+            error_message=str(payload.get("error") or "Step returned success=false"),
+        )
+
+    if step_key == "demand_classify" and payload.get("failed_batches"):
+        return GateDecision(False, "classification_failed_batches", "Classification has failed batches")
+
+    if step_key in {"demand_grade", "demand_expand", "video_discovery"}:
+        failed = int(payload.get("failed", payload.get("failed_count", 0)) or 0)
+        if failed > 0:
+            return GateDecision(
+                False,
+                "business_failures",
+                f"{step_key} reported {failed} failed item(s)",
+            )
+
+    if step_key == "aigc_write_record":
+        if payload.get("dry_run_recorded") is not True or not payload.get("payload_hash"):
+            return GateDecision(
+                False,
+                "aigc_record_incomplete",
+                "AIGC dry-run payload/hash was not durably recorded",
+            )
+        if payload.get("external_request_made") is True:
+            return GateDecision(
+                False,
+                "external_effect_detected",
+                "AIGC external request is forbidden in phase one",
+            )
+
+    return GateDecision(True)

supply_infra/pipeline/health.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+from typing import Any
+
+from supply_infra.config import InfraSettings, get_infra_settings
+from supply_infra.db.repositories.pipeline_lock_repo import PipelineLockRepository
+from supply_infra.db.repositories.pipeline_run_repo import PipelineRunRepository
+from supply_infra.db.session import get_session
+from supply_infra.pipeline.dates import utc_now
+
+SCHEDULER_HEARTBEAT_KEY = "pipeline:scheduler"
+_ACTIVE_STATUSES = {
+    "queued",
+    "running",
+    "cancelling",
+    "blocked_previous_run",
+}
+
+
+def touch_scheduler_heartbeat(
+    *,
+    owner: str,
+    settings: InfraSettings | None = None,
+) -> None:
+    active = settings or get_infra_settings()
+    with get_session() as session:
+        PipelineLockRepository(session).touch(
+            lock_key=SCHEDULER_HEARTBEAT_KEY,
+            owner=owner,
+            lease_seconds=max(
+                active.pipeline_lease_seconds,
+                active.pipeline_heartbeat_seconds * 3,
+            ),
+            now=utc_now(),
+        )
+
+
+def run_alert_level(
+    *,
+    status: str,
+    started_at: datetime | None,
+    created_at: datetime,
+    deadline_at: datetime | None,
+    now: datetime,
+    settings: InfraSettings,
+) -> str | None:
+    if status not in _ACTIVE_STATUSES:
+        return None
+    if deadline_at is not None:
+        remaining = deadline_at - now
+        if remaining <= timedelta(minutes=settings.pipeline_final_warn_before_next_minutes):
+            return "final_warning"
+        if remaining <= timedelta(minutes=settings.pipeline_critical_before_next_minutes):
+            return "critical"
+    baseline = started_at or created_at
+    if now - baseline >= timedelta(hours=settings.pipeline_warn_after_hours):
+        return "warning"
+    return None
+
+
+def pipeline_health_snapshot(
+    settings: InfraSettings | None = None,
+) -> dict[str, Any]:
+    active = settings or get_infra_settings()
+    now = utc_now()
+    with get_session() as session:
+        scheduler_lock = PipelineLockRepository(session).get(SCHEDULER_HEARTBEAT_KEY)
+        latest = PipelineRunRepository(session).list_recent(limit=1)
+        latest_run = latest[0] if latest else None
+        scheduler_heartbeat_at = (
+            scheduler_lock.heartbeat_at if scheduler_lock is not None else None
+        )
+        scheduler_lease_until = (
+            scheduler_lock.lease_until if scheduler_lock is not None else None
+        )
+        latest_snapshot = (
+            {
+                "run_id": latest_run.run_id,
+                "status": latest_run.status,
+                "started_at": latest_run.started_at,
+                "created_at": latest_run.created_at,
+                "deadline_at": latest_run.deadline_at,
+            }
+            if latest_run is not None
+            else None
+        )
+
+    scheduler_heartbeat = (
+        scheduler_heartbeat_at.isoformat()
+        if scheduler_heartbeat_at is not None
+        else None
+    )
+    scheduler_running = bool(
+        active.scheduler_enabled
+        and scheduler_lease_until is not None
+        and scheduler_lease_until > now
+    )
+    alert_level = None
+    if latest_snapshot is not None:
+        alert_level = run_alert_level(
+            status=latest_snapshot["status"],
+            started_at=latest_snapshot["started_at"],
+            created_at=latest_snapshot["created_at"],
+            deadline_at=latest_snapshot["deadline_at"],
+            now=now,
+            settings=active,
+        )
+    return {
+        "scheduler_enabled": active.scheduler_enabled,
+        "scheduler_running": scheduler_running,
+        "scheduler_heartbeat_at": scheduler_heartbeat,
+        "latest_run_id": latest_snapshot["run_id"] if latest_snapshot else None,
+        "latest_run_status": latest_snapshot["status"] if latest_snapshot else None,
+        "latest_run_alert_level": alert_level,
+        "checked_at": now.isoformat(),
+    }

supply_infra/pipeline/orchestrator.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+from datetime import timedelta
+
+from sqlalchemy.orm import Session
+
+from supply_infra.db.repositories.pipeline_run_repo import PipelineRunRepository
+from supply_infra.db.repositories.pipeline_step_run_repo import PipelineStepRunRepository
+from supply_infra.pipeline.contracts import StepResult
+from supply_infra.pipeline.dates import utc_now
+from supply_infra.pipeline.enums import RunStatus, StepStatus
+from supply_infra.pipeline.gates import evaluate_step_gate
+
+_TRANSIENT_ERRORS = {
+    "step_timed_out",
+    "step_launch_failed",
+    "db_pool_exhausted",
+}
+
+
+def complete_step(
+    session: Session,
+    *,
+    step_run_id: str,
+    owner: str,
+    result: StepResult,
+) -> str:
+    now = utc_now()
+    step_repo = PipelineStepRunRepository(session)
+    run_repo = PipelineRunRepository(session)
+    step = step_repo.get(step_run_id, for_update=True)
+    if step is None:
+        raise ValueError(f"step_run_id not found: {step_run_id}")
+    if step.status != StepStatus.RUNNING.value or step.lease_owner != owner:
+        raise RuntimeError(f"Step lease lost: {step_run_id}")
+    run = run_repo.get(step.run_id, for_update=True)
+    if run is None:
+        raise ValueError(f"pipeline run not found: {step.run_id}")
+    if run.status in {
+        RunStatus.CANCELLING.value,
+        RunStatus.CANCELLED.value,
+    }:
+        step_repo.finish(
+            step,
+            status=StepStatus.CANCELLED.value,
+            now=now,
+            exit_code=result.exit_code,
+            result=result.payload,
+            error_code="cancelled",
+            error_message="Step result discarded after pipeline cancellation",
+            log_uri=result.log_uri,
+        )
+        if run.status == RunStatus.CANCELLING.value:
+            run_repo.finish(run, status=RunStatus.CANCELLED.value, now=now)
+        return RunStatus.CANCELLED.value
+    if run.status == RunStatus.DEADLINE_EXCEEDED.value:
+        step_repo.finish(
+            step,
+            status=StepStatus.TIMED_OUT.value,
+            now=now,
+            exit_code=result.exit_code,
+            result=result.payload,
+            error_code="deadline_exceeded",
+            error_message="Step result discarded after pipeline deadline",
+            log_uri=result.log_uri,
+        )
+        return RunStatus.DEADLINE_EXCEEDED.value
+
+    gate = evaluate_step_gate(step.step_key, result.payload) if result.success else None
+    passed = bool(result.success and gate and gate.passed)
+    error_code = result.error_code
+    error_message = result.error_message
+    if result.success and gate is not None and not gate.passed:
+        error_code = gate.error_code
+        error_message = gate.error_message
+
+    if passed:
+        step_repo.finish(
+            step,
+            status=StepStatus.SUCCEEDED.value,
+            now=now,
+            exit_code=result.exit_code,
+            result=result.payload,
+            error_code=None,
+            error_message=None,
+            log_uri=result.log_uri,
+        )
+        latest = step_repo.latest_attempts(run.run_id)
+        incomplete = [
+            item
+            for item in sorted(latest.values(), key=lambda value: value.step_order)
+            if item.status != StepStatus.SUCCEEDED.value
+        ]
+        if not incomplete:
+            run_repo.finish(
+                run,
+                status=RunStatus.SUCCEEDED.value,
+                now=now,
+                summary={"steps": len(latest), "dry_run": run.dry_run},
+            )
+            return RunStatus.SUCCEEDED.value
+        next_step = incomplete[0]
+        if next_step.status == StepStatus.PENDING.value:
+            next_step.status = StepStatus.READY.value
+            next_step.error_code = None
+            next_step.error_message = None
+        run.status = RunStatus.QUEUED.value
+        run.current_step = next_step.step_key
+        run.heartbeat_at = now
+        run.lease_owner = None
+        run.lease_until = None
+        session.flush()
+        return RunStatus.QUEUED.value
+
+    failed_status = (
+        StepStatus.TIMED_OUT.value if result.timed_out else StepStatus.FAILED.value
+    )
+    step_repo.finish(
+        step,
+        status=failed_status,
+        now=now,
+        exit_code=result.exit_code,
+        result=result.payload,
+        error_code=error_code or "step_failed",
+        error_message=error_message,
+        log_uri=result.log_uri,
+    )
+    should_retry = (
+        step.retryable
+        and step.attempt < step.max_attempts
+        and (error_code or "") in _TRANSIENT_ERRORS
+    )
+    if should_retry:
+        retry_at = now + timedelta(seconds=min(300, 30 * (2 ** (step.attempt - 1))))
+        step_repo.create_retry_attempt(
+            step,
+            status=StepStatus.RETRY_WAIT.value,
+            next_retry_at=retry_at,
+        )
+        run.status = RunStatus.QUEUED.value
+        run.current_step = step.step_key
+        run.heartbeat_at = now
+        run.lease_owner = None
+        run.lease_until = None
+        session.flush()
+        return StepStatus.RETRY_WAIT.value
+
+    reason = error_message or error_code or "critical step failed"
+    step_repo.block_unfinished_downstream(
+        run_id=run.run_id,
+        after_order=step.step_order,
+        reason=reason,
+    )
+    run_repo.finish(
+        run,
+        status=RunStatus.FAILED.value,
+        now=now,
+        error_code=error_code or "critical_step_failed",
+        error_message=reason,
+        summary={"failed_step": step.step_key, "attempt": step.attempt},
+    )
+    return RunStatus.FAILED.value

supply_infra/pipeline/reconciler.py

@@ -0,0 +1,226 @@
+from __future__ import annotations
+
+import logging
+import signal
+import threading
+from datetime import timedelta, timezone
+from zoneinfo import ZoneInfo
+
+from supply_infra.config import InfraSettings, get_infra_settings
+from supply_infra.db.repositories.pipeline_run_repo import PipelineRunRepository
+from supply_infra.db.repositories.pipeline_step_run_repo import PipelineStepRunRepository
+from supply_infra.db.session import dispose_engine, get_session
+from supply_infra.pipeline.dates import utc_now
+from supply_infra.pipeline.enums import RunStatus, StepStatus
+from supply_infra.pipeline.health import run_alert_level
+from supply_infra.pipeline.run_service import PIPELINE_KEY, submit_pipeline_run
+
+logger = logging.getLogger(__name__)
+
+
+def reconcile_once() -> dict[str, int]:
+    settings = get_infra_settings()
+    now = utc_now()
+    stats = {
+        "retries_promoted": 0,
+        "expired_steps": 0,
+        "deadline_exceeded": 0,
+        "previous_run_unblocked": 0,
+        "alerts_emitted": 0,
+        "missed_runs_submitted": 0,
+    }
+    with get_session() as session:
+        run_repo = PipelineRunRepository(session)
+        step_repo = PipelineStepRunRepository(session)
+        stats["retries_promoted"] = step_repo.promote_due_retries(now)
+
+        for step in step_repo.list_expired_running(now):
+            run = run_repo.get(step.run_id, for_update=True)
+            if run is not None and run.status in {
+                RunStatus.CANCELLING.value,
+                RunStatus.CANCELLED.value,
+            }:
+                step.status = StepStatus.CANCELLED.value
+                step.finished_at = now
+                step.lease_owner = None
+                step.lease_until = None
+                step.error_code = "cancelled"
+                step.error_message = "Cancellation completed after worker lease expired"
+                if run.status == RunStatus.CANCELLING.value:
+                    run_repo.finish(
+                        run,
+                        status=RunStatus.CANCELLED.value,
+                        now=now,
+                    )
+                stats["expired_steps"] += 1
+                continue
+            if run is not None and run.status == RunStatus.DEADLINE_EXCEEDED.value:
+                step.status = StepStatus.TIMED_OUT.value
+                step.finished_at = now
+                step.lease_owner = None
+                step.lease_until = None
+                step.error_code = "deadline_exceeded"
+                step.error_message = "Worker lease expired after pipeline deadline"
+                stats["expired_steps"] += 1
+                continue
+            step.status = StepStatus.TIMED_OUT.value
+            step.finished_at = now
+            step.lease_owner = None
+            step.lease_until = None
+            step.error_code = "lease_expired"
+            step.error_message = "Worker heartbeat lease expired"
+            if step.retryable and step.attempt < step.max_attempts:
+                step_repo.create_retry_attempt(step, status=StepStatus.READY.value)
+                if run is not None:
+                    run.status = RunStatus.QUEUED.value
+                    run.current_step = step.step_key
+                    run.lease_owner = None
+                    run.lease_until = None
+            else:
+                step_repo.block_unfinished_downstream(
+                    run_id=step.run_id,
+                    after_order=step.step_order,
+                    reason="Worker heartbeat lease expired",
+                )
+                if run is not None:
+                    run_repo.finish(
+                        run,
+                        status=RunStatus.FAILED.value,
+                        now=now,
+                        error_code="lease_expired",
+                        error_message=f"Lease expired at step {step.step_key}",
+                    )
+            stats["expired_steps"] += 1
+
+        for run in run_repo.list_past_deadline(now):
+            for step in step_repo.list_for_run(run.run_id):
+                if step.status == StepStatus.RUNNING.value:
+                    step.error_code = "deadline_exceeded"
+                    step.error_message = (
+                        "Pipeline exceeded next scheduled start; waiting for "
+                        "the worker or lease expiry"
+                    )
+                    continue
+                if step.status in {
+                    StepStatus.PENDING.value,
+                    StepStatus.READY.value,
+                    StepStatus.RETRY_WAIT.value,
+                    StepStatus.BLOCKED.value,
+                }:
+                    step.status = StepStatus.BLOCKED.value
+                    step.error_code = "deadline_exceeded"
+                    step.error_message = "Pipeline exceeded next scheduled start"
+            run_repo.finish(
+                run,
+                status=RunStatus.DEADLINE_EXCEEDED.value,
+                now=now,
+                error_code="deadline_exceeded",
+                error_message="Pipeline exceeded next scheduled start",
+            )
+            stats["deadline_exceeded"] += 1
+
+        for run in run_repo.list_blocked_previous():
+            blocked_by = (run.summary_json or {}).get("blocked_by_run_id")
+            previous = run_repo.get(str(blocked_by)) if blocked_by else None
+            if previous is not None and previous.status in {
+                RunStatus.QUEUED.value,
+                RunStatus.RUNNING.value,
+                RunStatus.CANCELLING.value,
+                RunStatus.BLOCKED_PREVIOUS_RUN.value,
+            }:
+                continue
+            if previous is not None and step_repo.has_running_for_run(previous.run_id):
+                continue
+            latest = step_repo.latest_attempts(run.run_id)
+            first = min(latest.values(), key=lambda item: item.step_order, default=None)
+            if first is None:
+                continue
+            first.status = StepStatus.READY.value
+            first.error_code = None
+            first.error_message = None
+            for step in latest.values():
+                if step.step_run_id != first.step_run_id:
+                    step.status = StepStatus.PENDING.value
+                    step.error_code = None
+                    step.error_message = None
+            run.status = RunStatus.QUEUED.value
+            run.summary_json = None
+            stats["previous_run_unblocked"] += 1
+
+        for run in run_repo.list_nonterminal():
+            alert_level = run_alert_level(
+                status=run.status,
+                started_at=run.started_at,
+                created_at=run.created_at,
+                deadline_at=run.deadline_at,
+                now=now,
+                settings=settings,
+            )
+            if alert_level is not None:
+                logger.warning(
+                    "pipeline_alert level=%s run_id=%s biz_dt=%s status=%s deadline=%s",
+                    alert_level,
+                    run.run_id,
+                    run.biz_dt,
+                    run.status,
+                    run.deadline_at.isoformat() if run.deadline_at else None,
+                )
+                stats["alerts_emitted"] += 1
+
+    local_now = now.replace(tzinfo=timezone.utc).astimezone(
+        ZoneInfo(settings.scheduler_timezone)
+    )
+    missed_cutoff = local_now.replace(
+        hour=settings.scheduler_cron_hour,
+        minute=settings.scheduler_cron_minute,
+        second=0,
+        microsecond=0,
+    ) + timedelta(seconds=settings.pipeline_missed_run_grace_seconds)
+    if settings.scheduler_enabled and local_now >= missed_cutoff:
+        biz_dt = local_now.strftime("%Y%m%d")
+        with get_session() as session:
+            existing = PipelineRunRepository(session).get_for_business_day(
+                pipeline_key=PIPELINE_KEY,
+                biz_dt=biz_dt,
+            )
+        if existing is None:
+            submission = submit_pipeline_run(
+                biz_dt=biz_dt,
+                trigger_type="reconcile",
+                trigger_source="missed_run_reconciler",
+                trigger_reason="Scheduled batch was absent after the grace window",
+                settings=settings,
+            )
+            if submission.created:
+                logger.error(
+                    "pipeline_alert level=critical type=missed_run_recovered "
+                    "run_id=%s biz_dt=%s",
+                    submission.run_id,
+                    biz_dt,
+                )
+                stats["missed_runs_submitted"] = 1
+    return stats
+
+
+class PipelineReconciler:
+    def __init__(self, settings: InfraSettings | None = None) -> None:
+        self.settings = settings or get_infra_settings()
+        self._stop = threading.Event()
+
+    def stop(self, *_args: object) -> None:
+        self._stop.set()
+
+    def run_forever(self) -> None:
+        signal.signal(signal.SIGTERM, self.stop)
+        signal.signal(signal.SIGINT, self.stop)
+        logger.info("Pipeline reconciler started")
+        try:
+            while not self._stop.is_set():
+                try:
+                    logger.info("Pipeline reconcile result: %s", reconcile_once())
+                except Exception:
+                    logger.exception("Pipeline reconcile failed")
+                self._stop.wait(self.settings.pipeline_reconcile_seconds)
+        finally:
+            dispose_engine()
+            logger.info("Pipeline reconciler stopped")

supply_infra/pipeline/registry.py

@@ -0,0 +1,194 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from supply_infra.config import get_infra_settings
+from supply_infra.db.repositories.pipeline_outbox_repo import PipelineOutboxRepository
+from supply_infra.db.session import get_session
+from supply_infra.pipeline.contracts import StepContext
+
+StepHandler = Callable[[StepContext], dict[str, Any]]
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+_MAX_INLINE_EFFECT_BYTES = 512_000
+
+
+def _global_tree(context: StepContext) -> dict[str, Any]:
+    from supply_infra.scheduler.jobs.sync_global_tree_odps_to_mysql import (
+        sync_global_tree_odps_to_mysql,
+    )
+
+    return sync_global_tree_odps_to_mysql(
+        partition_date=str(context.date_snapshot["global_tree_partition"])
+    )
+
+
+def _demand_source(context: StepContext) -> dict[str, Any]:
+    from supply_infra.scheduler.jobs.demand_pool.sync import _sync_pool_rows
+
+    return _sync_pool_rows(str(context.date_snapshot["demand_pool_partition"]))
+
+
+def _demand_classify(context: StepContext) -> dict[str, Any]:
+    from supply_infra.scheduler.jobs.demand_pool.sync import _classify_words
+
+    return _classify_words(context.biz_dt)
+
+
+def _demand_rel(_context: StepContext) -> dict[str, Any]:
+    from supply_infra.scheduler.jobs.demand_pool.belong_rel import (
+        sync_demand_belong_pool_rel,
+    )
+
+    return sync_demand_belong_pool_rel()
+
+
+def _real_metrics(context: StepContext) -> dict[str, Any]:
+    from supply_infra.scheduler.jobs.demand_pool.sync import enrich_real_rov_vov_7d
+
+    return enrich_real_rov_vov_7d(context.biz_dt)
+
+
+def _popularity(context: StepContext) -> dict[str, Any]:
+    from supply_infra.scheduler.jobs.demand_pool.sync import compute_popularity_stats
+
+    return compute_popularity_stats(context.biz_dt)
+
+
+def _tree_weight(context: StepContext) -> dict[str, Any]:
+    from supply_infra.scheduler.jobs.demand_pool.tree_weight import (
+        compute_category_tree_weight,
+    )
+
+    return compute_category_tree_weight(context.biz_dt)
+
+
+def _source_videos(context: StepContext) -> dict[str, Any]:
+    from supply_infra.scheduler.jobs.demand_pool.videos import sync_multi_demand_videos
+
+    return sync_multi_demand_videos(
+        limit=None,
+        decode_dt=str(context.date_snapshot["video_decode_partition"]),
+    )
+
+
+def _grade(context: StepContext) -> dict[str, Any]:
+    from supply_infra.scheduler.jobs.grade_demand_pool import grade_demand_pool
+
+    return grade_demand_pool(context.biz_dt)
+
+
+def _expand(context: StepContext) -> dict[str, Any]:
+    from supply_infra.scheduler.jobs.expand_demand_from_video_points import (
+        expand_demand_from_video_points,
+    )
+
+    return expand_demand_from_video_points(
+        context.biz_dt,
+        workers=5,
+    )
+
+
+def _discover(context: StepContext) -> dict[str, Any]:
+    from supply_infra.scheduler.constants import (
+        PIPELINE_FIND_AGENT_TOP_DEMANDS,
+        PIPELINE_FIND_AGENT_WORKERS,
+    )
+    from supply_infra.scheduler.jobs.discover_videos_from_demands import (
+        discover_videos_from_demands,
+    )
+
+    return discover_videos_from_demands(
+        context.biz_dt,
+        workers=PIPELINE_FIND_AGENT_WORKERS,
+        top_limit=PIPELINE_FIND_AGENT_TOP_DEMANDS,
+    )
+
+
+def _aigc_write_record(context: StepContext) -> dict[str, Any]:
+    from supply_infra.scheduler.jobs.publish_videos_from_discovery import (
+        publish_videos_from_discovery,
+    )
+
+    payload = publish_videos_from_discovery(
+        biz_dt=context.biz_dt,
+        dry_run=True,
+    )
+    canonical = json.dumps(
+        payload,
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(",", ":"),
+        default=str,
+    )
+    encoded = canonical.encode("utf-8")
+    payload_hash = hashlib.sha256(encoded).hexdigest()
+    idempotency_key = f"aigc_write_record:{context.biz_dt}:{payload_hash}"
+    payload_uri: str | None = None
+    stored_payload: dict[str, Any] | None = payload
+    if len(encoded) > _MAX_INLINE_EFFECT_BYTES:
+        log_dir = Path(get_infra_settings().pipeline_log_dir)
+        if not log_dir.is_absolute():
+            log_dir = _REPO_ROOT / log_dir
+        effect_dir = log_dir / context.run_id / "effects"
+        effect_dir.mkdir(parents=True, exist_ok=True)
+        effect_path = effect_dir / f"{context.step_run_id}-{payload_hash}.json"
+        temporary_path = effect_path.with_suffix(".json.tmp")
+        temporary_path.write_bytes(encoded)
+        temporary_path.replace(effect_path)
+        payload_uri = str(effect_path)
+        stored_payload = {
+            "externalized": True,
+            "payload_bytes": len(encoded),
+            "payload_hash": payload_hash,
+        }
+    with get_session() as session:
+        record = PipelineOutboxRepository(session).record_dry_run(
+            run_id=context.run_id,
+            step_run_id=context.step_run_id,
+            effect_type="aigc_write_record",
+            idempotency_key=idempotency_key,
+            payload_hash=payload_hash,
+            payload=stored_payload,
+            payload_uri=payload_uri,
+        )
+        outbox_id = record.outbox_id
+    return {
+        "success": True,
+        "dry_run_recorded": True,
+        "external_request_made": False,
+        "outbox_id": outbox_id,
+        "payload_hash": payload_hash,
+        "payload_uri": payload_uri,
+        "candidate_count": payload.get("candidate_count", 0),
+        "batch_count": payload.get("batch_count", 0),
+    }
+
+
+STEP_REGISTRY: dict[str, StepHandler] = {
+    "global_tree_sync": _global_tree,
+    "demand_pool_source_sync": _demand_source,
+    "demand_classify": _demand_classify,
+    "demand_belong_rel_sync": _demand_rel,
+    "real_metrics_sync": _real_metrics,
+    "popularity_stats": _popularity,
+    "category_tree_weight": _tree_weight,
+    "source_video_sync": _source_videos,
+    "demand_grade": _grade,
+    "demand_expand": _expand,
+    "video_discovery": _discover,
+    "aigc_write_record": _aigc_write_record,
+}
+
+
+def execute_registered_step(context: StepContext) -> dict[str, Any]:
+    handler = STEP_REGISTRY.get(context.step_key)
+    if handler is None:
+        raise KeyError(f"Unknown pipeline step: {context.step_key}")
+    payload = handler(context)
+    if not isinstance(payload, dict):
+        return {"success": True, "result": payload}
+    return payload

supply_infra/pipeline/run_service.py

@@ -0,0 +1,369 @@
+from __future__ import annotations
+
+import os
+import uuid
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+from sqlalchemy.exc import IntegrityError
+
+from supply_infra.config import InfraSettings, get_infra_settings
+from supply_infra.db.models.pipeline_run import PipelineRun
+from supply_infra.db.models.pipeline_step_run import PipelineStepRun
+from supply_infra.db.repositories.pipeline_lock_repo import PipelineLockRepository
+from supply_infra.db.repositories.pipeline_outbox_repo import PipelineOutboxRepository
+from supply_infra.db.repositories.pipeline_run_repo import PipelineRunRepository
+from supply_infra.db.repositories.pipeline_step_run_repo import PipelineStepRunRepository
+from supply_infra.db.session import get_session
+from supply_infra.pipeline.dag import PIPELINE_STEPS
+from supply_infra.pipeline.dates import (
+    build_date_snapshot,
+    deadline_for_trigger,
+    next_schedule_utc,
+    resolve_biz_dt,
+    utc_now,
+)
+from supply_infra.pipeline.enums import RunStatus, StepStatus
+
+PIPELINE_KEY = "supply_pipeline"
+
+
+@dataclass(frozen=True)
+class RunSubmission:
+    run_id: str
+    created: bool
+    status: str
+    biz_dt: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "accepted": True,
+            "run_id": self.run_id,
+            "created": self.created,
+            "status": self.status,
+            "biz_dt": self.biz_dt,
+            "dry_run": True,
+        }
+
+
+def _config_snapshot(settings: InfraSettings) -> dict[str, Any]:
+    return {
+        "scheduler_timezone": settings.scheduler_timezone,
+        "scheduler_cron_hour": settings.scheduler_cron_hour,
+        "scheduler_cron_minute": settings.scheduler_cron_minute,
+        "lease_seconds": settings.pipeline_lease_seconds,
+        "heartbeat_seconds": settings.pipeline_heartbeat_seconds,
+        "worker_processes": settings.pipeline_worker_processes,
+        "max_active_steps": settings.pipeline_max_active_steps,
+        "mysql_connection_budget": settings.mysql_connection_budget,
+        "aigc_dry_run": True,
+        "external_effects_enabled": False,
+    }
+
+
+def _code_version() -> str | None:
+    return os.getenv("BUILD_TIMESTAMP") or os.getenv("GIT_COMMIT")
+
+
+def submit_pipeline_run(
+    *,
+    biz_dt: str | None = None,
+    trigger_type: str = "api",
+    trigger_source: str | None = None,
+    trigger_reason: str | None = None,
+    scheduled_for: datetime | None = None,
+    settings: InfraSettings | None = None,
+) -> RunSubmission:
+    active = settings or get_infra_settings()
+    resolved_biz_dt = resolve_biz_dt(biz_dt, settings=active)
+    dedupe_key = f"{PIPELINE_KEY}:{resolved_biz_dt}:full"
+    run_id = str(uuid.uuid4())
+    date_snapshot = build_date_snapshot(resolved_biz_dt)
+
+    try:
+        with get_session() as session:
+            run_repo = PipelineRunRepository(session)
+            PipelineLockRepository(session).lock_control_plane(now=utc_now())
+            existing = run_repo.get_by_dedupe_key(dedupe_key)
+            if existing is not None:
+                return RunSubmission(
+                    run_id=existing.run_id,
+                    created=False,
+                    status=existing.status,
+                    biz_dt=existing.biz_dt,
+                )
+
+            # All entry points share the same no-overlap rule. Linking each new
+            # run to the latest nonterminal run also creates a safe FIFO chain
+            # when several future batches are submitted in advance.
+            previous = run_repo.get_latest_nonterminal()
+            initial_status = (
+                RunStatus.BLOCKED_PREVIOUS_RUN.value
+                if previous is not None
+                else RunStatus.QUEUED.value
+            )
+            run = run_repo.create(
+                {
+                    "run_id": run_id,
+                    "dedupe_key": dedupe_key,
+                    "pipeline_key": PIPELINE_KEY,
+                    "biz_dt": resolved_biz_dt,
+                    "trigger_type": trigger_type,
+                    "trigger_source": trigger_source or trigger_type,
+                    "triggered_by": None,
+                    "trigger_reason": trigger_reason,
+                    "run_mode": "full",
+                    "dry_run": True,
+                    "status": initial_status,
+                    "scheduled_for": scheduled_for,
+                    "deadline_at": deadline_for_trigger(
+                        trigger_type,
+                        settings=active,
+                    ),
+                    "code_version": _code_version(),
+                    "config_snapshot_json": _config_snapshot(active),
+                    "date_snapshot_json": date_snapshot,
+                    "summary_json": (
+                        {"blocked_by_run_id": previous.run_id}
+                        if previous is not None
+                        else None
+                    ),
+                }
+            )
+            rows: list[dict[str, Any]] = []
+            for index, step in enumerate(PIPELINE_STEPS):
+                if previous is not None:
+                    status = StepStatus.BLOCKED.value
+                else:
+                    status = (
+                        StepStatus.READY.value if index == 0 else StepStatus.PENDING.value
+                    )
+                rows.append(
+                    {
+                        "step_run_id": str(uuid.uuid4()),
+                        "run_id": run.run_id,
+                        "step_key": step.key,
+                        "step_order": step.order,
+                        "attempt": 1,
+                        "status": status,
+                        "critical": step.critical,
+                        "dependency_snapshot_json": list(step.dependencies),
+                        "input_snapshot_json": {},
+                        "timeout_seconds": step.timeout_seconds,
+                        "max_attempts": step.max_attempts,
+                        "retryable": step.retryable,
+                        "error_code": (
+                            "previous_run_active" if previous is not None else None
+                        ),
+                        "error_message": (
+                            f"Blocked by previous run {previous.run_id}"
+                            if previous is not None
+                            else None
+                        ),
+                    }
+                )
+            PipelineStepRunRepository(session).create_steps(rows)
+            return RunSubmission(
+                run_id=run.run_id,
+                created=True,
+                status=run.status,
+                biz_dt=run.biz_dt,
+            )
+    except IntegrityError:
+        with get_session() as session:
+            existing = PipelineRunRepository(session).get_by_dedupe_key(dedupe_key)
+            if existing is None:
+                raise
+            return RunSubmission(
+                run_id=existing.run_id,
+                created=False,
+                status=existing.status,
+                biz_dt=existing.biz_dt,
+            )
+
+
+def get_pipeline_run(run_id: str) -> dict[str, Any] | None:
+    with get_session() as session:
+        run = PipelineRunRepository(session).get(run_id)
+        if run is None:
+            return None
+        steps = PipelineStepRunRepository(session).list_for_run(run_id)
+        outbox = PipelineOutboxRepository(session).list_for_run(run_id)
+        return {
+            **serialize_run(run),
+            "steps": [serialize_step(item) for item in steps],
+            "effects": [
+                {
+                    "outbox_id": item.outbox_id,
+                    "step_run_id": item.step_run_id,
+                    "effect_type": item.effect_type,
+                    "payload_hash": item.payload_hash,
+                    "payload_uri": item.payload_uri,
+                    "status": item.status,
+                    "dry_run": item.dry_run,
+                    "created_at": _iso(item.created_at),
+                }
+                for item in outbox
+            ],
+        }
+
+
+def list_pipeline_runs(
+    *,
+    limit: int = 50,
+    status: str | None = None,
+    biz_dt: str | None = None,
+) -> list[dict[str, Any]]:
+    with get_session() as session:
+        rows = PipelineRunRepository(session).list_recent(
+            limit=min(max(limit, 1), 200),
+            status=status,
+            biz_dt=biz_dt,
+        )
+        return [serialize_run(item) for item in rows]
+
+
+def cancel_pipeline_run(run_id: str) -> bool:
+    now = utc_now()
+    with get_session() as session:
+        run_repo = PipelineRunRepository(session)
+        step_repo = PipelineStepRunRepository(session)
+        run = run_repo.get(run_id, for_update=True)
+        if run is None:
+            return False
+        if run.status in {
+            RunStatus.SUCCEEDED.value,
+            RunStatus.FAILED.value,
+            RunStatus.CANCELLED.value,
+            RunStatus.DEADLINE_EXCEEDED.value,
+        }:
+            return True
+        has_running_step = False
+        for step in step_repo.list_for_run(run_id):
+            if step.status == StepStatus.RUNNING.value:
+                has_running_step = True
+                continue
+            if step.status in {
+                StepStatus.PENDING.value,
+                StepStatus.READY.value,
+                StepStatus.RETRY_WAIT.value,
+                StepStatus.BLOCKED.value,
+            }:
+                step.status = StepStatus.CANCELLED.value
+                step.finished_at = now
+        if has_running_step:
+            run.status = RunStatus.CANCELLING.value
+            run.error_code = "cancellation_requested"
+            run.error_message = "Waiting for the running step to stop"
+            run.heartbeat_at = now
+            run.lease_owner = None
+            run.lease_until = None
+        else:
+            run_repo.finish(run, status=RunStatus.CANCELLED.value, now=now)
+        return True
+
+
+def resume_pipeline_run(run_id: str, *, step_key: str | None = None) -> bool:
+    with get_session() as session:
+        run_repo = PipelineRunRepository(session)
+        step_repo = PipelineStepRunRepository(session)
+        run = run_repo.get(run_id, for_update=True)
+        if run is None:
+            return False
+        if run.status not in {
+            RunStatus.FAILED.value,
+            RunStatus.DEADLINE_EXCEEDED.value,
+        }:
+            return False
+        latest = step_repo.latest_attempts(run_id)
+        failed = next(
+            (
+                item
+                for item in sorted(latest.values(), key=lambda row: row.step_order)
+                if item.status
+                in {
+                    StepStatus.FAILED.value,
+                    StepStatus.TIMED_OUT.value,
+                    StepStatus.BLOCKED.value,
+                }
+            ),
+            None,
+        )
+        if failed is None:
+            return False
+        if step_key is not None and failed.step_key != step_key:
+            return False
+        retry = step_repo.create_retry_attempt(failed, status=StepStatus.READY.value)
+        for item in latest.values():
+            if item.step_order > failed.step_order and item.status == StepStatus.BLOCKED.value:
+                item.status = StepStatus.PENDING.value
+                item.error_code = None
+                item.error_message = None
+        run.status = RunStatus.QUEUED.value
+        run.current_step = retry.step_key
+        if run.deadline_at is not None and run.deadline_at <= utc_now():
+            run.deadline_at = next_schedule_utc()
+        run.finished_at = None
+        run.error_code = None
+        run.error_message = None
+        return True
+
+
+def retry_pipeline_step(run_id: str, step_key: str) -> bool:
+    return resume_pipeline_run(run_id, step_key=step_key)
+
+
+def serialize_run(run: PipelineRun) -> dict[str, Any]:
+    return {
+        "run_id": run.run_id,
+        "pipeline_key": run.pipeline_key,
+        "biz_dt": run.biz_dt,
+        "trigger_type": run.trigger_type,
+        "trigger_source": run.trigger_source,
+        "run_mode": run.run_mode,
+        "dry_run": run.dry_run,
+        "status": run.status,
+        "current_step": run.current_step,
+        "scheduled_for": _iso(run.scheduled_for),
+        "deadline_at": _iso(run.deadline_at),
+        "started_at": _iso(run.started_at),
+        "finished_at": _iso(run.finished_at),
+        "heartbeat_at": _iso(run.heartbeat_at),
+        "summary": run.summary_json,
+        "error_code": run.error_code,
+        "error_message": run.error_message,
+        "created_at": _iso(run.created_at),
+        "updated_at": _iso(run.updated_at),
+    }
+
+
+def serialize_step(step: PipelineStepRun) -> dict[str, Any]:
+    return {
+        "step_run_id": step.step_run_id,
+        "run_id": step.run_id,
+        "step_key": step.step_key,
+        "step_order": step.step_order,
+        "attempt": step.attempt,
+        "status": step.status,
+        "critical": step.critical,
+        "timeout_seconds": step.timeout_seconds,
+        "started_at": _iso(step.started_at),
+        "finished_at": _iso(step.finished_at),
+        "heartbeat_at": _iso(step.heartbeat_at),
+        "result": step.result_summary_json,
+        "error_code": step.error_code,
+        "error_message": step.error_message,
+        "log_uri": step.log_uri,
+    }
+
+
+def _iso(value: datetime | None) -> str | None:
+    if value is None:
+        return None
+    aware = (
+        value.replace(tzinfo=timezone.utc)
+        if value.tzinfo is None
+        else value.astimezone(timezone.utc)
+    )
+    return aware.isoformat().replace("+00:00", "Z")

supply_infra/pipeline/step_cli.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from supply_infra.db.repositories.pipeline_run_repo import PipelineRunRepository
+from supply_infra.db.repositories.pipeline_step_run_repo import PipelineStepRunRepository
+from supply_infra.db.session import dispose_engine, get_session
+from supply_infra.pipeline.contracts import StepContext
+from supply_infra.pipeline.registry import execute_registered_step
+
+logger = logging.getLogger(__name__)
+
+
+def execute_step_run(step_run_id: str) -> dict[str, Any]:
+    with get_session() as session:
+        step = PipelineStepRunRepository(session).get(step_run_id)
+        if step is None:
+            raise ValueError(f"step_run_id not found: {step_run_id}")
+        run = PipelineRunRepository(session).get(step.run_id)
+        if run is None:
+            raise ValueError(f"pipeline run not found: {step.run_id}")
+        context = StepContext(
+            run_id=run.run_id,
+            step_run_id=step.step_run_id,
+            step_key=step.step_key,
+            biz_dt=run.biz_dt,
+            date_snapshot=dict(run.date_snapshot_json or {}),
+            config_snapshot=dict(run.config_snapshot_json or {}),
+            input_snapshot=dict(step.input_snapshot_json or {}),
+        )
+    return execute_registered_step(context)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Execute one durable pipeline step")
+    parser.add_argument("step_run_id")
+    parser.add_argument("--result-file", required=True)
+    args = parser.parse_args()
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    )
+    result: dict[str, Any]
+    exit_code = 0
+    try:
+        result = execute_step_run(args.step_run_id)
+        if result.get("success") is False:
+            exit_code = 1
+    except Exception as exc:
+        logger.exception("Pipeline step failed: step_run_id=%s", args.step_run_id)
+        result = {"success": False, "error_code": "step_exception", "error": str(exc)}
+        exit_code = 1
+    finally:
+        dispose_engine()
+    Path(args.result_file).write_text(
+        json.dumps(result, ensure_ascii=False, default=str),
+        encoding="utf-8",
+    )
+    raise SystemExit(exit_code)
+
+
+if __name__ == "__main__":
+    main()

supply_infra/pipeline/step_runner.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+import signal
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from supply_infra.config import InfraSettings, get_infra_settings
+from supply_infra.pipeline.contracts import StepResult
+
+logger = logging.getLogger(__name__)
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+_MAX_RESULT_BYTES = 100_000
+
+
+@dataclass(frozen=True)
+class StepExecutionRequest:
+    run_id: str
+    step_run_id: str
+    step_key: str
+    timeout_seconds: int
+
+
+def _log_directory(settings: InfraSettings, run_id: str) -> Path:
+    base = Path(settings.pipeline_log_dir)
+    if not base.is_absolute():
+        base = _REPO_ROOT / base
+    path = base / run_id
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def _read_result(path: Path) -> dict[str, Any]:
+    if not path.is_file():
+        return {"success": False, "error_code": "missing_result", "error": "Result file missing"}
+    raw = path.read_bytes()
+    if len(raw) > _MAX_RESULT_BYTES:
+        return {
+            "success": False,
+            "error_code": "result_too_large",
+            "error": f"Step result exceeds {_MAX_RESULT_BYTES} bytes",
+            "result_bytes": len(raw),
+        }
+    try:
+        payload = json.loads(raw.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+        return {
+            "success": False,
+            "error_code": "invalid_result_json",
+            "error": str(exc),
+        }
+    if not isinstance(payload, dict):
+        return {
+            "success": False,
+            "error_code": "invalid_result_type",
+            "error": "Step result must be a JSON object",
+        }
+    return payload
+
+
+def run_step_subprocess(
+    request: StepExecutionRequest,
+    *,
+    settings: InfraSettings | None = None,
+) -> StepResult:
+    active = settings or get_infra_settings()
+    directory = _log_directory(active, request.run_id)
+    stem = f"{request.step_key}-attempt-{request.step_run_id}"
+    log_path = directory / f"{stem}.log"
+    result_path = directory / f"{stem}.result.json"
+    result_path.unlink(missing_ok=True)
+    command = [
+        sys.executable,
+        "-m",
+        "supply_infra.pipeline.step_cli",
+        request.step_run_id,
+        "--result-file",
+        str(result_path),
+    ]
+    env = os.environ.copy()
+    env["PROCESS_ROLE"] = "step"
+    logger.info("Starting pipeline step: step_run_id=%s", request.step_run_id)
+
+    timed_out = False
+    exit_code: int | None = None
+    with log_path.open("a", encoding="utf-8") as log_file:
+        try:
+            process = subprocess.Popen(
+                command,
+                cwd=str(_REPO_ROOT),
+                env=env,
+                stdout=log_file,
+                stderr=subprocess.STDOUT,
+                text=True,
+                start_new_session=True,
+            )
+        except Exception as exc:
+            logger.exception("Failed to launch pipeline step: %s", request.step_run_id)
+            return StepResult(
+                success=False,
+                payload={},
+                error_code="step_launch_failed",
+                error_message=str(exc),
+                log_uri=str(log_path),
+            )
+        try:
+            exit_code = process.wait(timeout=request.timeout_seconds)
+        except subprocess.TimeoutExpired:
+            timed_out = True
+            os.killpg(process.pid, signal.SIGTERM)
+            try:
+                exit_code = process.wait(timeout=min(active.pipeline_shutdown_grace_seconds, 30))
+            except subprocess.TimeoutExpired:
+                os.killpg(process.pid, signal.SIGKILL)
+                exit_code = process.wait()
+
+    if timed_out:
+        return StepResult(
+            success=False,
+            payload={},
+            error_code="step_timed_out",
+            error_message=f"Step timed out after {request.timeout_seconds}s",
+            exit_code=exit_code,
+            log_uri=str(log_path),
+            timed_out=True,
+        )
+
+    payload = _read_result(result_path)
+    result_path.unlink(missing_ok=True)
+    success = exit_code == 0 and payload.get("success") is not False
+    return StepResult(
+        success=success,
+        payload=payload,
+        error_code=(
+            None
+            if success
+            else str(payload.get("error_code") or "step_failed")
+        ),
+        error_message=(
+            None
+            if success
+            else str(payload.get("error") or f"Step exited with code {exit_code}")
+        ),
+        exit_code=exit_code,
+        log_uri=str(log_path),
+    )

supply_infra/pipeline/worker.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import logging
+import os
+import signal
+import socket
+import threading
+import uuid
+from dataclasses import dataclass
+from datetime import timedelta
+
+from supply_infra.config import InfraSettings, get_infra_settings
+from supply_infra.db.repositories.pipeline_run_repo import PipelineRunRepository
+from supply_infra.db.repositories.pipeline_step_run_repo import PipelineStepRunRepository
+from supply_infra.db.session import dispose_engine, get_session
+from supply_infra.pipeline.dates import utc_now
+from supply_infra.pipeline.orchestrator import complete_step
+from supply_infra.pipeline.contracts import StepResult
+from supply_infra.pipeline.step_runner import StepExecutionRequest, run_step_subprocess
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class ClaimedStep:
+    run_id: str
+    step_run_id: str
+    step_key: str
+    timeout_seconds: int
+
+
+class PipelineWorker:
+    def __init__(
+        self,
+        *,
+        settings: InfraSettings | None = None,
+        worker_id: str | None = None,
+    ) -> None:
+        self.settings = settings or get_infra_settings()
+        self.worker_id = worker_id or (
+            f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:8]}"
+        )
+        self._stop = threading.Event()
+
+    def stop(self, *_args: object) -> None:
+        logger.info("Worker stop requested: worker_id=%s", self.worker_id)
+        self._stop.set()
+
+    def claim(self) -> ClaimedStep | None:
+        now = utc_now()
+        with get_session() as session:
+            step = PipelineStepRunRepository(session).claim_next_ready(
+                owner=self.worker_id,
+                lease_seconds=self.settings.pipeline_lease_seconds,
+                max_active_steps=self.settings.pipeline_max_active_steps,
+                now=now,
+            )
+            if step is None:
+                return None
+            run = PipelineRunRepository(session).get(step.run_id, for_update=True)
+            if run is None:
+                raise ValueError(f"pipeline run not found: {step.run_id}")
+            PipelineRunRepository(session).mark_running(
+                run,
+                step_key=step.step_key,
+                owner=self.worker_id,
+                now=now,
+                lease_until=now + timedelta(seconds=self.settings.pipeline_lease_seconds),
+            )
+            return ClaimedStep(
+                run_id=step.run_id,
+                step_run_id=step.step_run_id,
+                step_key=step.step_key,
+                timeout_seconds=step.timeout_seconds,
+            )
+
+    def _heartbeat_loop(self, claimed: ClaimedStep, stopped: threading.Event) -> None:
+        while not stopped.wait(self.settings.pipeline_heartbeat_seconds):
+            now = utc_now()
+            lease_until = now + timedelta(seconds=self.settings.pipeline_lease_seconds)
+            try:
+                with get_session() as session:
+                    step_ok = PipelineStepRunRepository(session).heartbeat(
+                        claimed.step_run_id,
+                        owner=self.worker_id,
+                        now=now,
+                        lease_until=lease_until,
+                    )
+                    run_ok = PipelineRunRepository(session).heartbeat(
+                        claimed.run_id,
+                        owner=self.worker_id,
+                        now=now,
+                        lease_until=lease_until,
+                    )
+                    if not step_ok or not run_ok:
+                        logger.error(
+                            "Pipeline lease lost: worker=%s step=%s",
+                            self.worker_id,
+                            claimed.step_run_id,
+                        )
+                        stopped.set()
+            except Exception:
+                logger.exception("Pipeline heartbeat failed: step=%s", claimed.step_run_id)
+
+    def run_once(self) -> bool:
+        claimed = self.claim()
+        if claimed is None:
+            return False
+        heartbeat_stop = threading.Event()
+        heartbeat = threading.Thread(
+            target=self._heartbeat_loop,
+            args=(claimed, heartbeat_stop),
+            name=f"pipeline-heartbeat-{claimed.step_run_id[:8]}",
+            daemon=True,
+        )
+        heartbeat.start()
+        try:
+            try:
+                result = run_step_subprocess(
+                    StepExecutionRequest(
+                        run_id=claimed.run_id,
+                        step_run_id=claimed.step_run_id,
+                        step_key=claimed.step_key,
+                        timeout_seconds=claimed.timeout_seconds,
+                    ),
+                    settings=self.settings,
+                )
+            except Exception as exc:
+                logger.exception("Unexpected step runner failure: %s", claimed.step_run_id)
+                result = StepResult(
+                    success=False,
+                    payload={},
+                    error_code="step_runner_exception",
+                    error_message=str(exc),
+                )
+        finally:
+            heartbeat_stop.set()
+            heartbeat.join(timeout=2)
+        try:
+            with get_session() as session:
+                state = complete_step(
+                    session,
+                    step_run_id=claimed.step_run_id,
+                    owner=self.worker_id,
+                    result=result,
+                )
+        except RuntimeError:
+            logger.exception(
+                "Discarding step result after lease loss: step=%s",
+                claimed.step_run_id,
+            )
+            return True
+        logger.info(
+            "Pipeline step completed: step=%s state=%s",
+            claimed.step_run_id,
+            state,
+        )
+        return True
+
+    def run_forever(self) -> None:
+        signal.signal(signal.SIGTERM, self.stop)
+        signal.signal(signal.SIGINT, self.stop)
+        logger.info("Pipeline worker started: worker_id=%s", self.worker_id)
+        try:
+            while not self._stop.is_set():
+                if not self.run_once():
+                    self._stop.wait(self.settings.pipeline_worker_poll_seconds)
+        finally:
+            dispose_engine()
+            logger.info("Pipeline worker stopped: worker_id=%s", self.worker_id)

supply_infra/scheduler/__main__.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import logging
+import os
+import signal
+import socket
+import threading
+
+from supply_infra.config import get_infra_settings
+from supply_infra.db import dispose_engine
+from supply_infra.pipeline.health import touch_scheduler_heartbeat
+from supply_infra.scheduler.app import start_scheduler, stop_scheduler
+
+
+def main() -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    )
+    stopped = threading.Event()
+
+    def _stop(*_args: object) -> None:
+        stopped.set()
+
+    signal.signal(signal.SIGTERM, _stop)
+    signal.signal(signal.SIGINT, _stop)
+    scheduler = start_scheduler()
+    if scheduler is None:
+        logging.getLogger(__name__).warning("Scheduler process exits because it is disabled")
+        return
+    settings = get_infra_settings()
+    owner = f"{socket.gethostname()}:{os.getpid()}"
+    try:
+        while not stopped.is_set():
+            try:
+                touch_scheduler_heartbeat(owner=owner, settings=settings)
+            except Exception:
+                logging.getLogger(__name__).exception("Scheduler heartbeat failed")
+            stopped.wait(settings.pipeline_heartbeat_seconds)
+    finally:
+        stop_scheduler()
+        dispose_engine()
+
+
+if __name__ == "__main__":
+    main()

supply_infra/scheduler/app.py

@@ -7,13 +7,8 @@ from apscheduler.schedulers.background import BackgroundScheduler
 from apscheduler.triggers.cron import CronTrigger
 
 from supply_infra.config import get_infra_settings
-from supply_infra.scheduler.constants import (
-    PIPELINE_CRON_HOUR,
-    PIPELINE_CRON_MINUTE,
-    SUPPLY_PIPELINE_JOB_ID,
-    SUPPLY_PIPELINE_JOB_NAME,
-)
-from supply_infra.scheduler.jobs.run_supply_pipeline import run_supply_pipeline
+from supply_infra.pipeline.run_service import submit_pipeline_run
+from supply_infra.scheduler.constants import SUPPLY_PIPELINE_JOB_ID, SUPPLY_PIPELINE_JOB_NAME
 
 if TYPE_CHECKING:
     from apscheduler.schedulers.base import BaseScheduler
@@ -23,18 +18,25 @@ logger = logging.getLogger(__name__)
 _scheduler: BackgroundScheduler | None = None
 
 
+def submit_scheduled_pipeline() -> dict:
+    """Cron callback: create a durable batch and return immediately."""
+    return submit_pipeline_run(
+        trigger_type="cron",
+        trigger_source="scheduler",
+    ).to_dict()
+
+
 def create_scheduler() -> BackgroundScheduler:
-    """Create and configure the scheduler with the chained supply pipeline job."""
+    """Create a Scheduler that only submits durable pipeline batches."""
     settings = get_infra_settings()
     tz = settings.scheduler_timezone
     scheduler = BackgroundScheduler(timezone=tz)
 
-    # 每天 14:30（上海时区）串行执行：全局树 → 需求池 → 分级 → 拓展 → find_agent → 发布
     scheduler.add_job(
-        run_supply_pipeline,
+        submit_scheduled_pipeline,
         trigger=CronTrigger(
-            hour=PIPELINE_CRON_HOUR,
-            minute=PIPELINE_CRON_MINUTE,
+            hour=settings.scheduler_cron_hour,
+            minute=settings.scheduler_cron_minute,
             timezone=tz,
         ),
         id=SUPPLY_PIPELINE_JOB_ID,
@@ -49,8 +51,8 @@ def create_scheduler() -> BackgroundScheduler:
         "Scheduler configured with %d job(s) | timezone=%s | pipeline_cron=%02d:%02d",
         len(scheduler.get_jobs()),
         tz,
-        PIPELINE_CRON_HOUR,
-        PIPELINE_CRON_MINUTE,
+        settings.scheduler_cron_hour,
+        settings.scheduler_cron_minute,
     )
     return scheduler
 
@@ -64,7 +66,7 @@ def _log_next_runs(scheduler: BaseScheduler) -> None:
 
 
 def start_scheduler() -> BackgroundScheduler | None:
-    """Start the background scheduler (idempotent). Used by API lifespan."""
+    """Start the independent scheduler process (idempotent)."""
     global _scheduler
 
     settings = get_infra_settings()

supply_infra/scheduler/jobs/demand_pool/videos.py

@@ -178,23 +178,27 @@ def sync_multi_demand_videos(
     limit: int | None = None,
     offset: int = 0,
     batch_size: int = VIDEO_SYNC_BATCH_SIZE,
+    *,
+    decode_dt: str | None = None,
 ) -> dict[str, Any]:
     """
     增量同步需求池视频与最终选题。
 
     - 视频来源：multi_demand_pool_di 全表 video_list
     - 已有 vid 跳过
-    - ODPS decode 分区永远用「今天的昨天」
+    - ODPS decode 分区由批次冻结参数传入；兼容调用未传时使用「今天的昨天」
     - 按 batch_size 分批：每批查询后立刻写入
     - limit: 本轮最多处理的 pending 数；None 表示从 offset 起全部
     - offset: pending 列表起始偏移（手动分批用）
     """
-    decode_dt = (datetime.now() - timedelta(days=1)).strftime("%Y%m%d")
+    resolved_decode_dt = decode_dt or (
+        datetime.now() - timedelta(days=1)
+    ).strftime("%Y%m%d")
     start = max(0, int(offset))
     chunk = max(1, int(batch_size))
     logger.info(
         "Starting multi demand video sync: decode_dt=%s limit=%s offset=%d batch_size=%d",
-        decode_dt,
+        resolved_decode_dt,
         limit,
         start,
         chunk,
@@ -224,7 +228,7 @@ def sync_multi_demand_videos(
     )
 
     empty = {
-        "decode_dt": decode_dt,
+        "decode_dt": resolved_decode_dt,
         "pool_video_lists": len(video_lists),
         "unique_vids": len(all_vids),
         "pending_total": len(pending_all),
@@ -251,7 +255,7 @@ def sync_multi_demand_videos(
     total_missing = 0
 
     for idx, batch_vids in enumerate(batches, start=1):
-        stats = _sync_one_batch(odps, decode_dt, batch_vids, idx, len(batches))
+        stats = _sync_one_batch(odps, resolved_decode_dt, batch_vids, idx, len(batches))
         total_odps_rows += stats["odps_rows"]
         total_inserted += stats["inserted"]
         total_skipped += stats["skipped_no_topic"]
@@ -259,7 +263,7 @@ def sync_multi_demand_videos(
 
     next_offset = start + len(pending)
     result = {
-        "decode_dt": decode_dt,
+        "decode_dt": resolved_decode_dt,
         "pool_video_lists": len(video_lists),
         "unique_vids": len(all_vids),
         "pending_total": len(pending_all),

supply_infra/scheduler/jobs/publish_videos_from_discovery.py

@@ -15,7 +15,6 @@ from supply_infra.aigc.plan_map import (
     list_unique_plan_pairs,
 )
 from supply_infra.config import get_infra_settings
-from supply_infra.db.models.video_discovery import VideoDiscoveryCandidate
 from supply_infra.db.repositories.demand_grade_repo import DemandGradeRepository
 from supply_infra.db.repositories.video_discovery_repo import VideoDiscoveryRepository
 from supply_infra.db.session import get_session
@@ -102,6 +101,12 @@ def publish_videos_from_discovery(
 
     仅处理 decision_bucket 为 primary / backup 且 aweme_id 非空的候选，不区分品类。
     """
+    settings = get_infra_settings()
+    effective_dry_run = (
+        dry_run
+        or settings.aigc_dry_run
+        or not settings.pipeline_external_effects_enabled
+    )
     resolved_biz_dt = _resolve_biz_dt(biz_dt)
     plan_pairs = list_unique_plan_pairs()
     if not plan_pairs:
@@ -127,10 +132,11 @@ def publish_videos_from_discovery(
             "biz_dt": resolved_biz_dt,
             "run_id": run_id,
             "plan_count": len(plan_pairs),
-        "candidate_count": 0,
-        "decision_buckets": list(_PUBLISHABLE_BUCKETS),
-        "batches": [],
-    }
+            "candidate_count": 0,
+            "decision_buckets": list(_PUBLISHABLE_BUCKETS),
+            "dry_run": effective_dry_run,
+            "batches": [],
+        }
 
     grouped = _group_candidates_by_plan(candidates, plan_pairs)
     distribution_preview = assignment_summary(
@@ -138,8 +144,8 @@ def publish_videos_from_discovery(
         [bucket for _, bucket in grouped],
     )
 
-    client = AigcClient(dry_run=dry_run)
-    timezone = ZoneInfo(get_infra_settings().scheduler_timezone)
+    client = AigcClient(dry_run=effective_dry_run)
+    timezone = ZoneInfo(settings.scheduler_timezone)
     timestamp = datetime.now(timezone).strftime("%Y%m%d%H%M%S")
     batch_results: list[PublishBatchResult] = []
 
@@ -162,7 +168,7 @@ def publish_videos_from_discovery(
                 publish_plan_id=plan_pair.publish_plan_id,
                 aweme_ids=aweme_batch,
                 candidate_ids=id_batch,
-                dry_run=dry_run,
+                dry_run=effective_dry_run,
             )
 
             if not create_result.get("success"):
@@ -184,7 +190,7 @@ def publish_videos_from_discovery(
             if not batch.bind_success:
                 batch.bind_error = str(bind_result.get("error") or "绑定生成计划失败")
 
-            if not dry_run and crawler_plan_id and batch.bind_success:
+            if not effective_dry_run and crawler_plan_id and batch.bind_success:
                 with get_session() as session:
                     updated = VideoDiscoveryRepository(session).mark_candidates_aigc_plans(
                         id_batch,
@@ -214,6 +220,7 @@ def publish_videos_from_discovery(
         "distribution": distribution_preview,
         "batch_count": len(batch_results),
         "failed_batch_count": len(failed_batches),
+        "dry_run": effective_dry_run,
         "batches": [item.to_dict() for item in batch_results],
     }
 

supply_infra/scheduler/jobs/run_supply_pipeline.py

@@ -1,223 +1,72 @@
-"""
-串联执行的供给数据流水线（可重复执行、幂等）：
-
-1. ODPS → MySQL 全局树同步（T-1 分区）
-2. ODPS → MySQL 策略需求池同步（当天 biz_dt）
-3. 需求池分级评估（同上 biz_dt）
-4. S/A 需求视频点位拓展判断
-5. find_agent 视频发现（top 200 需求，2 线程并行）
-6. AIGC 发布（find_agent 完成后发布全部符合条件的视频）
-
-各子步骤以独立 CLI 子进程执行（``python -m supply_infra.scheduler.jobs.*``）；
-内部已做去重（INSERT IGNORE、diff 同步、跳过已分级词等）。
-本文件额外用进程内锁防止同一轮次并发重入，并隔离各步骤异常：前一步失败时记录
-error 后继续后续步骤，最终返回失败结果而不向 APScheduler 抛异常。
-"""
+"""Compatibility entry point for the durable supply pipeline."""
 from __future__ import annotations
 
-import logging
-import sys
-import threading
-from datetime import datetime, timedelta
+import argparse
+import time
 from typing import Any
-from zoneinfo import ZoneInfo
 
-from supply_infra.config import get_infra_settings
+from supply_infra.pipeline.run_service import get_pipeline_run, submit_pipeline_run
 from supply_infra.scheduler.cli_result import run_cli
-from supply_infra.scheduler.constants import (
-    PIPELINE_FIND_AGENT_TOP_DEMANDS,
-    PIPELINE_FIND_AGENT_WORKERS,
-    SUPPLY_PIPELINE_JOB_ID,
-    SUPPLY_PIPELINE_JOB_NAME,
-)
-from supply_infra.scheduler.job_execution import JobExecutionRecorder, record_skipped
-from supply_infra.scheduler.step_runner import run_step_module
-
-logger = logging.getLogger(__name__)
-
-_pipeline_lock = threading.Lock()
-
-
-def _resolve_dates(biz_dt: str | None) -> tuple[str, str]:
-    """返回 (biz_dt, global_tree_partition_date)，global_tree 使用 biz_dt 前一日。"""
-    if biz_dt:
-        resolved_biz_dt = biz_dt
-    else:
-        timezone = ZoneInfo(get_infra_settings().scheduler_timezone)
-        resolved_biz_dt = datetime.now(timezone).strftime("%Y%m%d")
-    tree_partition = (
-        datetime.strptime(resolved_biz_dt, "%Y%m%d") - timedelta(days=1)
-    ).strftime("%Y%m%d")
-    return resolved_biz_dt, tree_partition
-
 
-def _preflight_failure_result(biz_dt: str | None, exc: Exception) -> dict[str, Any]:
-    """日期/配置解析失败时也记录结果并正常返回，避免调度线程出现未捕获异常。"""
-    started_at = datetime.now()
-    recorder = JobExecutionRecorder(
-        job_name=SUPPLY_PIPELINE_JOB_NAME,
-        job_id=SUPPLY_PIPELINE_JOB_ID,
-        biz_dt=str(biz_dt) if biz_dt is not None else None,
-    )
-    recorder.record_started()
-    result: dict[str, Any] = {
-        "run_id": recorder.run_id,
-        "biz_dt": str(biz_dt) if biz_dt is not None else None,
-        "success": False,
-        "error": str(exc),
-        "started_at": started_at.isoformat(),
-    }
-    finished_at = datetime.now()
-    result["finished_at"] = finished_at.isoformat()
-    result["duration_seconds"] = round((finished_at - started_at).total_seconds(), 2)
-    recorder.record_finished(success=False, result=result, error_message=str(exc))
-    return result
-
-
-def run_supply_pipeline(biz_dt: str | None = None) -> dict[str, Any]:
+_TERMINAL = {
+    "succeeded",
+    "failed",
+    "cancelled",
+    "deadline_exceeded",
+}
+
+
+def run_supply_pipeline(
+    biz_dt: str | None = None,
+    *,
+    wait: bool = False,
+    wait_timeout_seconds: int = 86400,
+    poll_seconds: float = 2.0,
+) -> dict[str, Any]:
     """
-    按顺序以子进程执行：全局树同步 → 需求池同步 → 需求分级 → 视频点位拓展
-    → find_agent 找片 → AIGC 发布。
-
-    Args:
-        biz_dt: 业务日 YYYYMMDD；省略则取当天。
+    Submit a durable batch.
 
-    Returns:
-        各步骤统计；若上一轮仍在执行则返回 skipped。
+    This function no longer executes business steps directly. ``wait=True`` is
+    retained for diagnostics and waits on MySQL state rather than an in-memory
+    thread.
     """
-    try:
-        resolved_biz_dt, tree_partition = _resolve_dates(biz_dt)
-    except Exception as exc:
-        logger.exception("Supply pipeline preflight failed: biz_dt=%s", biz_dt)
-        return _preflight_failure_result(biz_dt, exc)
-
-    if not _pipeline_lock.acquire(blocking=False):
-        logger.warning("Supply pipeline already running, skip this round")
-        record_skipped(
-            job_name=SUPPLY_PIPELINE_JOB_NAME,
-            job_id=SUPPLY_PIPELINE_JOB_ID,
-            biz_dt=resolved_biz_dt,
-            reason="already_running",
-        )
-        return {
-            "skipped": True,
-            "reason": "already_running",
-            "run_at": datetime.now().isoformat(),
-        }
-
-    started_at = datetime.now()
-    recorder = JobExecutionRecorder(
-        job_name=SUPPLY_PIPELINE_JOB_NAME,
-        job_id=SUPPLY_PIPELINE_JOB_ID,
-        biz_dt=resolved_biz_dt,
-    )
-    recorder.record_started()
-
-    logger.info(
-        "Supply pipeline start: biz_dt=%s global_tree_partition=%s run_id=%s",
-        resolved_biz_dt,
-        tree_partition,
-        recorder.run_id,
+    submission = submit_pipeline_run(
+        biz_dt=biz_dt,
+        trigger_type="cli",
+        trigger_source="legacy_run_supply_pipeline",
     )
-
-    result: dict[str, Any] = {
-        "run_id": recorder.run_id,
-        "biz_dt": resolved_biz_dt,
-        "global_tree_partition": tree_partition,
-        "started_at": started_at.isoformat(),
-    }
-    errors: list[str] = []
-    step_status: dict[str, dict[str, Any]] = {}
-    success = False
-
-    try:
-        steps: list[tuple[str, str, list[str]]] = [
-            (
-                "global_tree",
-                "supply_infra.scheduler.jobs.sync_global_tree_odps_to_mysql",
-                [tree_partition],
-            ),
-            (
-                "demand_pool",
-                "supply_infra.scheduler.jobs.demand_pool",
-                [resolved_biz_dt],
-            ),
-            (
-                "grade",
-                "supply_infra.scheduler.jobs.grade_demand_pool",
-                [resolved_biz_dt],
-            ),
-            (
-                "expand_video_points",
-                "supply_infra.scheduler.jobs.expand_demand_from_video_points",
-                [resolved_biz_dt, "5"],
-            ),
-            (
-                "discover_videos",
-                "supply_infra.scheduler.jobs.discover_videos_from_demands",
-                [
-                    resolved_biz_dt,
-                    str(PIPELINE_FIND_AGENT_WORKERS),
-                    "--top-limit",
-                    str(PIPELINE_FIND_AGENT_TOP_DEMANDS),
-                ],
-            ),
-            (
-                "publish_videos",
-                "supply_infra.scheduler.jobs.publish_videos_from_discovery",
-                [resolved_biz_dt],
-            ),
-        ]
-        for step_name, module, module_args in steps:
-            payload, step_success, step_error = run_step_module(
-                module,
-                module_args,
-            )
-            result[step_name] = payload
-            step_status[step_name] = {
-                "success": step_success,
-                "error": step_error,
+    accepted = submission.to_dict()
+    if not wait:
+        return accepted
+
+    deadline = time.monotonic() + wait_timeout_seconds
+    while time.monotonic() < deadline:
+        payload = get_pipeline_run(submission.run_id)
+        if payload is None:
+            return {
+                **accepted,
+                "success": False,
+                "error": "Persistent run disappeared",
             }
-            if step_error:
-                errors.append(f"{step_name}: {step_error}")
-
-        success = all(item["success"] for item in step_status.values())
-        result["success"] = success
-    except Exception as exc:
-        # 保护流水线编排本身；正常子步骤异常应已由 run_step_module 消化。
-        result["success"] = False
-        errors.append(f"pipeline: {exc}")
-        logger.exception(
-            "Supply pipeline orchestration failed but will not escape scheduler: "
-            "biz_dt=%s global_tree_partition=%s",
-            resolved_biz_dt,
-            tree_partition,
-        )
-    finally:
-        finished_at = datetime.now()
-        result["steps"] = step_status
-        if errors:
-            result["errors"] = errors
-        result["finished_at"] = finished_at.isoformat()
-        result["duration_seconds"] = round((finished_at - started_at).total_seconds(), 2)
-        try:
-            recorder.record_finished(
-                success=success,
-                result=result,
-                error_message=" | ".join(errors) if errors else None,
-            )
-        except Exception:
-            # recorder 当前已自行兜底；这里防止未来实现变化造成锁无法释放。
-            logger.exception("Unexpected failure while recording supply pipeline finish")
-        finally:
-            _pipeline_lock.release()
-        logger.info("Supply pipeline finished: %s", result)
-
-    return result
+        if payload["status"] in _TERMINAL:
+            return {
+                **payload,
+                "success": payload["status"] == "succeeded",
+            }
+        time.sleep(max(0.1, poll_seconds))
+    return {
+        **accepted,
+        "success": False,
+        "error": f"Timed out waiting for run after {wait_timeout_seconds}s",
+    }
 
 
 if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Submit the durable supply pipeline")
+    parser.add_argument("biz_dt", nargs="?")
+    parser.add_argument("--wait", action="store_true")
+    args = parser.parse_args()
     run_cli(
-        lambda: run_supply_pipeline(sys.argv[1] if len(sys.argv) > 1 else None),
+        lambda: run_supply_pipeline(args.biz_dt, wait=args.wait),
         label="run_supply_pipeline",
     )

supply_infra/scheduler/manual_jobs.py

@@ -1,25 +1,11 @@
-"""手动触发定时任务：任务注册表、后台执行与运行状态查询。"""
+"""Deprecated compatibility facade for durable pipeline submissions."""
 from __future__ import annotations
 
-import logging
-import threading
-import uuid
-from collections.abc import Callable
 from dataclasses import dataclass
-from datetime import datetime, timedelta
 from typing import Any
-from zoneinfo import ZoneInfo
 
-from supply_infra.config import get_infra_settings
-from supply_infra.scheduler.constants import (
-    SUPPLY_PIPELINE_JOB_ID,
-    SUPPLY_PIPELINE_JOB_NAME,
-)
-
-logger = logging.getLogger(__name__)
-
-_RUNS: dict[str, dict[str, Any]] = {}
-_RUNS_LOCK = threading.Lock()
+from supply_infra.pipeline.run_service import get_pipeline_run, submit_pipeline_run
+from supply_infra.scheduler.constants import SUPPLY_PIPELINE_JOB_ID, SUPPLY_PIPELINE_JOB_NAME
 
 
 @dataclass(frozen=True)
@@ -27,161 +13,29 @@ class ManualJobSpec:
     job_id: str
     name: str
     description: str
-    runner: Callable[..., dict[str, Any]]
-    accepts_biz_dt: bool = False
+    accepts_biz_dt: bool = True
     accepts_partition_date: bool = False
 
 
-def _resolve_biz_dt(biz_dt: str | None) -> str:
-    if biz_dt:
-        return biz_dt
-    timezone = ZoneInfo(get_infra_settings().scheduler_timezone)
-    return datetime.now(timezone).strftime("%Y%m%d")
-
-
-def _resolve_tree_partition(biz_dt: str | None, partition_date: str | None) -> str:
-    if partition_date:
-        return partition_date
-    resolved_biz_dt = _resolve_biz_dt(biz_dt)
-    return (
-        datetime.strptime(resolved_biz_dt, "%Y%m%d") - timedelta(days=1)
-    ).strftime("%Y%m%d")
-
-
-def _run_supply_pipeline_job(
-    *,
-    biz_dt: str | None,
-    partition_date: str | None,
-) -> dict[str, Any]:
-    from supply_infra.scheduler.jobs.run_supply_pipeline import run_supply_pipeline
-
-    del partition_date
-    return run_supply_pipeline(biz_dt=biz_dt)
-
-
-def _run_global_tree_job(
-    *,
-    biz_dt: str | None,
-    partition_date: str | None,
-) -> dict[str, Any]:
-    from supply_infra.scheduler.jobs.sync_global_tree_odps_to_mysql import (
-        sync_global_tree_odps_to_mysql,
-    )
-
-    return sync_global_tree_odps_to_mysql(
-        partition_date=_resolve_tree_partition(biz_dt, partition_date),
-    )
-
-
-def _run_demand_pool_job(
-    *,
-    biz_dt: str | None,
-    partition_date: str | None,
-) -> dict[str, Any]:
-    from supply_infra.scheduler.jobs.demand_pool import (
-        sync_multi_demand_pool_odps_to_mysql,
-    )
-
-    return sync_multi_demand_pool_odps_to_mysql(
-        partition_date=partition_date or _resolve_biz_dt(biz_dt),
-    )
-
-
-def _run_biz_dt_job(
-    import_path: str,
-    fn_name: str,
-    *,
-    biz_dt: str | None,
-    partition_date: str | None,
-) -> dict[str, Any]:
-    del partition_date
-    module = __import__(import_path, fromlist=[fn_name])
-    fn = getattr(module, fn_name)
-    return fn(biz_dt)
-
-
-MANUAL_JOBS: dict[str, ManualJobSpec] = {
+MANUAL_JOBS = {
     SUPPLY_PIPELINE_JOB_ID: ManualJobSpec(
         job_id=SUPPLY_PIPELINE_JOB_ID,
         name=SUPPLY_PIPELINE_JOB_NAME,
-        description="串行执行：全局树 → 需求池 → 分级 → 拓展 → find_agent → AIGC 发布",
-        runner=_run_supply_pipeline_job,
-        accepts_biz_dt=True,
-    ),
-    "sync_global_tree_odps_to_mysql": ManualJobSpec(
-        job_id="sync_global_tree_odps_to_mysql",
-        name="全局树 ODPS 同步",
-        description="从 ODPS 同步 global_tree_category / global_tree_element 到 MySQL",
-        runner=_run_global_tree_job,
-        accepts_biz_dt=True,
-        accepts_partition_date=True,
-    ),
-    "sync_multi_demand_pool_odps_to_mysql": ManualJobSpec(
-        job_id="sync_multi_demand_pool_odps_to_mysql",
-        name="策略需求池 ODPS 同步",
-        description="同步 multi_demand_pool_di，并执行归属分类、热度统计、树权重与视频同步",
-        runner=_run_demand_pool_job,
-        accepts_biz_dt=True,
-        accepts_partition_date=True,
-    ),
-    "grade_demand_pool": ManualJobSpec(
-        job_id="grade_demand_pool",
-        name="需求池分级",
-        description="对指定业务日的需求词执行分级评估",
-        runner=lambda **kwargs: _run_biz_dt_job(
-            "supply_infra.scheduler.jobs.grade_demand_pool",
-            "grade_demand_pool",
-            **kwargs,
-        ),
-        accepts_biz_dt=True,
-    ),
-    "expand_demand_from_video_points": ManualJobSpec(
-        job_id="expand_demand_from_video_points",
-        name="视频点位拓展",
-        description="对 S/A 需求执行视频点位拓展判断",
-        runner=lambda **kwargs: _run_biz_dt_job(
-            "supply_infra.scheduler.jobs.expand_demand_from_video_points",
-            "expand_demand_from_video_points",
-            **kwargs,
-        ),
-        accepts_biz_dt=True,
-    ),
-    "discover_videos_from_demands": ManualJobSpec(
-        job_id="discover_videos_from_demands",
-        name="find_agent 视频发现",
-        description="对 S/A 需求拓展点位调用 find_agent 找片",
-        runner=lambda **kwargs: _run_biz_dt_job(
-            "supply_infra.scheduler.jobs.discover_videos_from_demands",
-            "discover_videos_from_demands",
-            **kwargs,
-        ),
-        accepts_biz_dt=True,
-    ),
-    "publish_videos_from_discovery": ManualJobSpec(
-        job_id="publish_videos_from_discovery",
-        name="AIGC 视频发布",
-        description="将 find_agent 发现结果发布到 AIGC 平台",
-        runner=lambda **kwargs: _run_biz_dt_job(
-            "supply_infra.scheduler.jobs.publish_videos_from_discovery",
-            "publish_videos_from_discovery",
-            **kwargs,
-        ),
-        accepts_biz_dt=True,
-    ),
+        description="提交严格门禁的持久化供给流水线",
+    )
 }
 
 
 def list_manual_jobs() -> list[dict[str, Any]]:
-    """返回可手动触发的任务列表。"""
     return [
         {
-            "id": spec.job_id,
-            "name": spec.name,
-            "description": spec.description,
-            "accepts_biz_dt": spec.accepts_biz_dt,
-            "accepts_partition_date": spec.accepts_partition_date,
+            "id": item.job_id,
+            "name": item.name,
+            "description": item.description,
+            "accepts_biz_dt": item.accepts_biz_dt,
+            "accepts_partition_date": item.accepts_partition_date,
         }
-        for spec in MANUAL_JOBS.values()
+        for item in MANUAL_JOBS.values()
     ]
 
 
@@ -190,77 +44,7 @@ def get_manual_job(job_id: str) -> ManualJobSpec | None:
 
 
 def get_manual_run(run_id: str) -> dict[str, Any] | None:
-    with _RUNS_LOCK:
-        payload = _RUNS.get(run_id)
-        return dict(payload) if payload else None
-
-
-def _set_run_state(run_id: str, payload: dict[str, Any]) -> None:
-    with _RUNS_LOCK:
-        _RUNS[run_id] = payload
-
-
-def _execute_job(
-    run_id: str,
-    spec: ManualJobSpec,
-    *,
-    biz_dt: str | None,
-    partition_date: str | None,
-) -> None:
-    started_at = datetime.now().isoformat()
-    _set_run_state(
-        run_id,
-        {
-            "run_id": run_id,
-            "job_id": spec.job_id,
-            "job_name": spec.name,
-            "status": "running",
-            "biz_dt": biz_dt,
-            "partition_date": partition_date,
-            "started_at": started_at,
-        },
-    )
-    try:
-        result = spec.runner(biz_dt=biz_dt, partition_date=partition_date)
-        success = not (isinstance(result, dict) and result.get("success") is False)
-        finished_at = datetime.now().isoformat()
-        _set_run_state(
-            run_id,
-            {
-                "run_id": run_id,
-                "job_id": spec.job_id,
-                "job_name": spec.name,
-                "status": "finished" if success else "failed",
-                "biz_dt": biz_dt,
-                "partition_date": partition_date,
-                "started_at": started_at,
-                "finished_at": finished_at,
-                "result": result,
-            },
-        )
-        logger.info(
-            "Manual job finished: job_id=%s run_id=%s status=%s",
-            spec.job_id,
-            run_id,
-            "finished" if success else "failed",
-        )
-    except Exception as exc:
-        finished_at = datetime.now().isoformat()
-        logger.exception("Manual job failed: job_id=%s run_id=%s", spec.job_id, run_id)
-        _set_run_state(
-            run_id,
-            {
-                "run_id": run_id,
-                "job_id": spec.job_id,
-                "job_name": spec.name,
-                "status": "failed",
-                "biz_dt": biz_dt,
-                "partition_date": partition_date,
-                "started_at": started_at,
-                "finished_at": finished_at,
-                "error": str(exc),
-            },
-        )
+    return get_pipeline_run(run_id)
 
 
 def trigger_manual_job(
@@ -270,48 +54,13 @@ def trigger_manual_job(
     partition_date: str | None = None,
     wait: bool = False,
 ) -> dict[str, Any]:
-    """
-    手动触发任务。
-
-    wait=False 时在后台线程执行并立即返回 run_id；
-    wait=True 时同步执行并返回完整结果。
-    """
-    spec = get_manual_job(job_id)
-    if spec is None:
+    del partition_date
+    if job_id != SUPPLY_PIPELINE_JOB_ID:
         raise KeyError(job_id)
-
-    if spec.accepts_biz_dt and biz_dt is None:
-        biz_dt = _resolve_biz_dt(None)
-    if spec.job_id == "sync_global_tree_odps_to_mysql" and partition_date is None:
-        partition_date = _resolve_tree_partition(biz_dt, None)
-    if spec.job_id == "sync_multi_demand_pool_odps_to_mysql" and partition_date is None:
-        partition_date = biz_dt or _resolve_biz_dt(None)
-
-    run_id = str(uuid.uuid4())
     if wait:
-        _execute_job(run_id, spec, biz_dt=biz_dt, partition_date=partition_date)
-        payload = get_manual_run(run_id)
-        assert payload is not None
-        return payload
-
-    thread = threading.Thread(
-        target=_execute_job,
-        kwargs={
-            "run_id": run_id,
-            "spec": spec,
-            "biz_dt": biz_dt,
-            "partition_date": partition_date,
-        },
-        name=f"manual-job-{job_id}-{run_id[:8]}",
-        daemon=True,
-    )
-    thread.start()
-    return {
-        "accepted": True,
-        "run_id": run_id,
-        "job_id": spec.job_id,
-        "job_name": spec.name,
-        "status": "running",
-        "biz_dt": biz_dt,
-        "partition_date": partition_date,
-    }
+        raise ValueError("wait=true is disabled; poll the persistent run_id")
+    return submit_pipeline_run(
+        biz_dt=biz_dt,
+        trigger_type="api",
+        trigger_source="legacy_manual_job",
+    ).to_dict()

supply_infra/scheduler/step_runner.py

@@ -1,122 +1,8 @@
-"""Run pipeline step modules as isolated subprocesses via ``python -m``."""
-from __future__ import annotations
+"""Deprecated import compatibility for the durable pipeline step runner."""
 
-import json
-import logging
-import subprocess
-import sys
-from pathlib import Path
-from typing import Any
+from supply_infra.pipeline.step_runner import (
+    StepExecutionRequest,
+    run_step_subprocess,
+)
 
-logger = logging.getLogger(__name__)
-
-_REPO_ROOT = Path(__file__).resolve().parents[2]
-
-
-def run_step_module(
-    module: str,
-    args: list[str] | None = None,
-    *,
-    timeout: float | None = None,
-) -> tuple[Any, bool, str | None]:
-    """
-    Execute ``python -m <module>`` with the current interpreter.
-
-    Returns:
-        (payload, success, error_message)
-    """
-    cmd = [sys.executable, "-m", module, *(args or [])]
-    logger.info("Pipeline step subprocess start: cmd=%s", cmd)
-    try:
-        completed = subprocess.run(
-            cmd,
-            cwd=str(_REPO_ROOT),
-            capture_output=True,
-            text=True,
-            timeout=timeout,
-            check=False,
-        )
-    except subprocess.TimeoutExpired as exc:
-        error = f"step timed out after {timeout}s: {module}"
-        logger.exception(error)
-        stderr_tail = (exc.stderr or "")[-2000:] if isinstance(exc.stderr, str) else ""
-        return (
-            {
-                "success": False,
-                "error": error,
-                "stderr": stderr_tail,
-            },
-            False,
-            error,
-        )
-    except Exception as exc:
-        error = f"failed to launch step module {module}: {exc}"
-        logger.exception(error)
-        return {"success": False, "error": error}, False, error
-
-    stdout = (completed.stdout or "").strip()
-    stderr = (completed.stderr or "").strip()
-    if stderr:
-        logger.info(
-            "Pipeline step stderr: module=%s exit=%s stderr_tail=%s",
-            module,
-            completed.returncode,
-            stderr[-2000:],
-        )
-
-    payload: Any
-    parse_error: str | None = None
-    if not stdout:
-        payload = {
-            "success": False,
-            "error": "empty stdout from step module",
-            "exit_code": completed.returncode,
-            "stderr": stderr[-2000:] if stderr else None,
-        }
-        parse_error = "empty stdout from step module"
-    else:
-        # Prefer the last non-empty line in case logging leaked to stdout.
-        last_line = stdout.splitlines()[-1]
-        try:
-            payload = json.loads(last_line)
-        except json.JSONDecodeError:
-            try:
-                payload = json.loads(stdout)
-            except json.JSONDecodeError as exc:
-                parse_error = f"invalid JSON from step module: {exc}"
-                payload = {
-                    "success": False,
-                    "error": parse_error,
-                    "exit_code": completed.returncode,
-                    "stdout_tail": stdout[-2000:],
-                    "stderr": stderr[-2000:] if stderr else None,
-                }
-
-    success = completed.returncode == 0
-    if isinstance(payload, dict) and payload.get("success") is False:
-        success = False
-
-    error: str | None = None
-    if not success:
-        if isinstance(payload, dict):
-            error = str(
-                payload.get("error")
-                or parse_error
-                or f"{module} failed with exit_code={completed.returncode}"
-            )
-        else:
-            error = parse_error or f"{module} failed with exit_code={completed.returncode}"
-        logger.error(
-            "Pipeline step failed: module=%s exit=%s error=%s",
-            module,
-            completed.returncode,
-            error,
-        )
-    else:
-        logger.info(
-            "Pipeline step ok: module=%s exit=%s",
-            module,
-            completed.returncode,
-        )
-
-    return payload, success, error
+__all__ = ["StepExecutionRequest", "run_step_subprocess"]

tests/supply_infra/__init__.py

@@ -0,0 +1 @@
+"""supply_infra tests."""

tests/supply_infra/pipeline/__init__.py

@@ -0,0 +1 @@
+"""Pipeline control-plane tests."""

tests/supply_infra/pipeline/test_aigc_safety.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from supply_infra.aigc.client import AigcClient
+from supply_infra.config import InfraSettings
+
+
+def test_aigc_client_forces_dry_run_when_external_effects_are_disabled(
+    monkeypatch,
+) -> None:
+    called = False
+
+    def _unexpected_post(*_args, **_kwargs):
+        nonlocal called
+        called = True
+        raise AssertionError("network must not be called")
+
+    monkeypatch.setattr("supply_infra.aigc.client.requests.post", _unexpected_post)
+    client = AigcClient(token="secret", dry_run=False)
+    result = client.create_video_crawler_plan(["123"])
+    assert result["dry_run"] is True
+    assert called is False
+
+
+def test_global_aigc_dry_run_cannot_be_disabled_by_caller(monkeypatch) -> None:
+    settings = InfraSettings(
+        PIPELINE_EXTERNAL_EFFECTS_ENABLED=True,
+        AIGC_DRY_RUN=True,
+    )
+    monkeypatch.setattr(
+        "supply_infra.aigc.client.get_infra_settings",
+        lambda: settings,
+    )
+    client = AigcClient(token="secret", dry_run=False)
+    assert client.dry_run is True

tests/supply_infra/pipeline/test_config_budget.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from supply_infra.config import InfraSettings
+
+
+def _settings(**overrides: object) -> InfraSettings:
+    values: dict[str, object] = {
+        "PROCESS_ROLE": "api",
+        "MYSQL_CONNECTION_BUDGET": 40,
+        "MYSQL_POOL_SIZE_API": 6,
+        "MYSQL_POOL_SIZE_CONTROL": 1,
+        "MYSQL_OPERATIONAL_RESERVE": 4,
+        "MYSQL_MAX_OVERFLOW": 0,
+        "PIPELINE_WORKER_PROCESSES": 4,
+        "PIPELINE_MAX_ACTIVE_STEPS": 4,
+        "AIGC_DRY_RUN": True,
+        "PIPELINE_EXTERNAL_EFFECTS_ENABLED": False,
+    }
+    values.update(overrides)
+    return InfraSettings(_env_file=None, **values)
+
+
+def test_standard_connection_profile_uses_twenty_of_forty() -> None:
+    settings = _settings()
+    assert settings.pipeline_required_connections == 20
+    assert settings.mysql_connection_budget == 40
+
+
+def test_high_concurrency_profile_stays_within_fifty() -> None:
+    settings = _settings(
+        MYSQL_CONNECTION_BUDGET=50,
+        PIPELINE_WORKER_PROCESSES=20,
+        PIPELINE_MAX_ACTIVE_STEPS=10,
+    )
+    assert settings.pipeline_required_connections == 42
+
+
+def test_connection_budget_rejects_unbounded_twenty_by_twenty() -> None:
+    with pytest.raises(ValidationError, match="connection budget exceeded"):
+        _settings(
+            MYSQL_CONNECTION_BUDGET=50,
+            PIPELINE_WORKER_PROCESSES=20,
+            PIPELINE_MAX_ACTIVE_STEPS=20,
+        )
+
+
+def test_pool_overflow_must_remain_zero() -> None:
+    with pytest.raises(ValidationError, match="MYSQL_MAX_OVERFLOW"):
+        _settings(MYSQL_MAX_OVERFLOW=1)

tests/supply_infra/pipeline/test_dag_and_gates.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from supply_infra.pipeline.dag import PIPELINE_STEPS
+from supply_infra.pipeline.gates import evaluate_step_gate
+
+
+def test_pipeline_has_twelve_strictly_ordered_steps() -> None:
+    assert len(PIPELINE_STEPS) == 12
+    assert PIPELINE_STEPS[0].key == "global_tree_sync"
+    assert PIPELINE_STEPS[-1].key == "aigc_write_record"
+    for previous, current in zip(PIPELINE_STEPS, PIPELINE_STEPS[1:]):
+        assert current.dependencies == (previous.key,)
+        assert current.critical is True
+
+
+def test_aigc_gate_requires_durable_dry_run_record() -> None:
+    failed = evaluate_step_gate("aigc_write_record", {"success": True})
+    assert failed.passed is False
+    passed = evaluate_step_gate(
+        "aigc_write_record",
+        {
+            "success": True,
+            "dry_run_recorded": True,
+            "payload_hash": "abc",
+            "external_request_made": False,
+        },
+    )
+    assert passed.passed is True
+
+
+def test_explicit_step_failure_fails_closed() -> None:
+    decision = evaluate_step_gate(
+        "global_tree_sync",
+        {"success": False, "error": "source missing"},
+    )
+    assert decision.passed is False
+    assert decision.error_code == "step_reported_failure"

tests/supply_infra/pipeline/test_dates.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from datetime import datetime
+from zoneinfo import ZoneInfo
+
+from supply_infra.config import InfraSettings
+from supply_infra.pipeline.dates import (
+    build_date_snapshot,
+    deadline_for_trigger,
+    next_daily_deadline_utc,
+    next_schedule_utc,
+    resolve_biz_dt,
+)
+
+
+def _settings() -> InfraSettings:
+    return InfraSettings(
+        _env_file=None,
+        SCHEDULER_TIMEZONE="Asia/Shanghai",
+        SCHEDULER_CRON_HOUR=15,
+        SCHEDULER_CRON_MINUTE=0,
+        AIGC_DRY_RUN=True,
+    )
+
+
+def test_date_snapshot_is_frozen_from_biz_dt() -> None:
+    assert build_date_snapshot("20260727") == {
+        "biz_dt": "20260727",
+        "global_tree_partition": "20260726",
+        "demand_pool_partition": "20260727",
+        "video_decode_partition": "20260726",
+    }
+
+
+def test_resolve_biz_dt_uses_scheduler_timezone() -> None:
+    now = datetime(2026, 7, 27, 1, 2, tzinfo=ZoneInfo("Asia/Shanghai"))
+    assert resolve_biz_dt(None, settings=_settings(), now=now) == "20260727"
+
+
+def test_deadline_is_next_scheduler_start() -> None:
+    now = datetime(2026, 7, 27, 15, 1, tzinfo=ZoneInfo("Asia/Shanghai"))
+    deadline = next_schedule_utc(settings=_settings(), now=now)
+    assert deadline.isoformat() == "2026-07-28T07:00:00"
+
+
+def test_scheduled_daily_deadline_is_always_next_local_day() -> None:
+    before_cron = datetime(2026, 7, 27, 14, 0, tzinfo=ZoneInfo("Asia/Shanghai"))
+    after_cron = datetime(2026, 7, 27, 16, 0, tzinfo=ZoneInfo("Asia/Shanghai"))
+
+    assert next_daily_deadline_utc(
+        settings=_settings(),
+        now=before_cron,
+    ).isoformat() == "2026-07-28T07:00:00"
+    assert next_daily_deadline_utc(
+        settings=_settings(),
+        now=after_cron,
+    ).isoformat() == "2026-07-28T07:00:00"
+
+
+def test_only_scheduled_triggers_receive_a_deadline() -> None:
+    now = datetime(2026, 7, 27, 15, 0, tzinfo=ZoneInfo("Asia/Shanghai"))
+
+    assert deadline_for_trigger(
+        "cron",
+        settings=_settings(),
+        now=now,
+    ).isoformat() == "2026-07-28T07:00:00"
+    assert deadline_for_trigger(
+        "reconcile",
+        settings=_settings(),
+        now=now,
+    ).isoformat() == "2026-07-28T07:00:00"
+    assert deadline_for_trigger("api", settings=_settings(), now=now) is None
+    assert deadline_for_trigger("cli", settings=_settings(), now=now) is None

tests/supply_infra/pipeline/test_health.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from datetime import timedelta
+
+from supply_infra.config import InfraSettings
+from supply_infra.pipeline.dates import utc_now
+from supply_infra.pipeline.health import run_alert_level
+
+
+def _settings() -> InfraSettings:
+    return InfraSettings(
+        _env_file=None,
+        AIGC_DRY_RUN=True,
+        PIPELINE_EXTERNAL_EFFECTS_ENABLED=False,
+    )
+
+
+def test_alert_level_escalates_against_next_schedule_deadline() -> None:
+    settings = _settings()
+    now = utc_now()
+
+    assert (
+        run_alert_level(
+            status="running",
+            started_at=now - timedelta(hours=13),
+            created_at=now - timedelta(hours=13),
+            deadline_at=now + timedelta(hours=5),
+            now=now,
+            settings=settings,
+        )
+        == "warning"
+    )
+    assert (
+        run_alert_level(
+            status="running",
+            started_at=now - timedelta(hours=1),
+            created_at=now - timedelta(hours=1),
+            deadline_at=now + timedelta(minutes=90),
+            now=now,
+            settings=settings,
+        )
+        == "critical"
+    )
+    assert (
+        run_alert_level(
+            status="blocked_previous_run",
+            started_at=None,
+            created_at=now - timedelta(hours=1),
+            deadline_at=now + timedelta(minutes=20),
+            now=now,
+            settings=settings,
+        )
+        == "final_warning"
+    )
+
+
+def test_terminal_run_has_no_duration_alert() -> None:
+    now = utc_now()
+    assert (
+        run_alert_level(
+            status="failed",
+            started_at=now - timedelta(days=2),
+            created_at=now - timedelta(days=2),
+            deadline_at=now - timedelta(days=1),
+            now=now,
+            settings=_settings(),
+        )
+        is None
+    )
+
+
+def test_manual_run_without_deadline_only_uses_duration_warning() -> None:
+    now = utc_now()
+    settings = _settings()
+
+    assert (
+        run_alert_level(
+            status="running",
+            started_at=now - timedelta(hours=1),
+            created_at=now - timedelta(hours=1),
+            deadline_at=None,
+            now=now,
+            settings=settings,
+        )
+        is None
+    )
+    assert (
+        run_alert_level(
+            status="running",
+            started_at=now - timedelta(hours=13),
+            created_at=now - timedelta(hours=13),
+            deadline_at=None,
+            now=now,
+            settings=settings,
+        )
+        == "warning"
+    )

tests/supply_infra/pipeline/test_orchestrator.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from supply_infra.db.models.pipeline_run import PipelineRun
+from supply_infra.db.models.pipeline_step_run import PipelineStepRun
+from supply_infra.pipeline.contracts import StepResult
+from supply_infra.pipeline.orchestrator import complete_step
+
+
+def _session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    PipelineRun.__table__.create(engine)
+    PipelineStepRun.__table__.create(engine)
+    return Session(engine, expire_on_commit=False)
+
+
+def _seed(session: Session) -> tuple[PipelineRun, PipelineStepRun, PipelineStepRun]:
+    run = PipelineRun(
+        run_id="run-1",
+        dedupe_key="supply_pipeline:20260727:full",
+        pipeline_key="supply_pipeline",
+        biz_dt="20260727",
+        trigger_type="test",
+        run_mode="full",
+        dry_run=True,
+        status="running",
+        deadline_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=1),
+        config_snapshot_json={},
+        date_snapshot_json={},
+        lease_owner="worker-1",
+    )
+    first = PipelineStepRun(
+        step_run_id="step-1",
+        run_id=run.run_id,
+        step_key="global_tree_sync",
+        step_order=1,
+        attempt=1,
+        status="running",
+        critical=True,
+        dependency_snapshot_json=[],
+        input_snapshot_json={},
+        timeout_seconds=10,
+        max_attempts=1,
+        retryable=False,
+        lease_owner="worker-1",
+    )
+    second = PipelineStepRun(
+        step_run_id="step-2",
+        run_id=run.run_id,
+        step_key="demand_pool_source_sync",
+        step_order=2,
+        attempt=1,
+        status="pending",
+        critical=True,
+        dependency_snapshot_json=["global_tree_sync"],
+        input_snapshot_json={},
+        timeout_seconds=10,
+        max_attempts=1,
+        retryable=False,
+    )
+    session.add_all([run, first, second])
+    session.flush()
+    return run, first, second
+
+
+def test_failure_blocks_all_downstream_steps() -> None:
+    with _session() as session:
+        run, first, second = _seed(session)
+        state = complete_step(
+            session,
+            step_run_id=first.step_run_id,
+            owner="worker-1",
+            result=StepResult(
+                success=False,
+                payload={"success": False},
+                error_code="test_failure",
+                error_message="boom",
+                exit_code=1,
+            ),
+        )
+        assert state == "failed"
+        assert run.status == "failed"
+        assert first.status == "failed"
+        assert second.status == "blocked"
+
+
+def test_success_only_releases_immediate_next_step() -> None:
+    with _session() as session:
+        run, first, second = _seed(session)
+        state = complete_step(
+            session,
+            step_run_id=first.step_run_id,
+            owner="worker-1",
+            result=StepResult(
+                success=True,
+                payload={"success": True},
+                exit_code=0,
+            ),
+        )
+        assert state == "queued"
+        assert run.status == "queued"
+        assert first.status == "succeeded"
+        assert second.status == "ready"
+
+
+def test_completion_cannot_revive_a_cancelled_run() -> None:
+    with _session() as session:
+        run, first, second = _seed(session)
+        run.status = "cancelling"
+
+        state = complete_step(
+            session,
+            step_run_id=first.step_run_id,
+            owner="worker-1",
+            result=StepResult(
+                success=True,
+                payload={"success": True},
+                exit_code=0,
+            ),
+        )
+
+        assert state == "cancelled"
+        assert run.status == "cancelled"
+        assert first.status == "cancelled"
+        assert second.status == "pending"
+
+
+def test_completion_cannot_revive_a_deadline_exceeded_run() -> None:
+    with _session() as session:
+        run, first, second = _seed(session)
+        run.status = "deadline_exceeded"
+
+        state = complete_step(
+            session,
+            step_run_id=first.step_run_id,
+            owner="worker-1",
+            result=StepResult(
+                success=True,
+                payload={"success": True},
+                exit_code=0,
+            ),
+        )
+
+        assert state == "deadline_exceeded"
+        assert run.status == "deadline_exceeded"
+        assert first.status == "timed_out"
+        assert second.status == "pending"

tests/supply_infra/pipeline/test_run_serialization.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from supply_infra.db.models.pipeline_run import PipelineRun
+from supply_infra.pipeline.run_service import serialize_run
+
+
+def test_manual_run_serializes_null_deadline_and_utc_timestamps() -> None:
+    run = PipelineRun(
+        run_id="manual-run",
+        dedupe_key="supply_pipeline:20260727:full",
+        pipeline_key="supply_pipeline",
+        biz_dt="20260727",
+        trigger_type="api",
+        run_mode="full",
+        dry_run=True,
+        status="queued",
+        deadline_at=None,
+        config_snapshot_json={},
+        date_snapshot_json={},
+        created_at=datetime(2026, 7, 27, 6, 30),
+        updated_at=datetime(2026, 7, 27, 6, 31),
+    )
+
+    payload = serialize_run(run)
+
+    assert payload["deadline_at"] is None
+    assert payload["created_at"] == "2026-07-27T06:30:00Z"
+    assert payload["updated_at"] == "2026-07-27T06:31:00Z"

tests/supply_infra/pipeline/test_step_claim.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+from datetime import timedelta
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from supply_infra.db.models.pipeline_lock import PipelineLock
+from supply_infra.db.models.pipeline_run import PipelineRun
+from supply_infra.db.models.pipeline_step_run import PipelineStepRun
+from supply_infra.db.repositories.pipeline_step_run_repo import (
+    PipelineStepRunRepository,
+)
+from supply_infra.pipeline.dates import utc_now
+
+
+def _session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:")
+    PipelineRun.__table__.create(engine)
+    PipelineStepRun.__table__.create(engine)
+    PipelineLock.__table__.create(engine)
+    return Session(engine, expire_on_commit=False)
+
+
+def _run(run_id: str) -> PipelineRun:
+    now = utc_now()
+    return PipelineRun(
+        run_id=run_id,
+        dedupe_key=f"supply_pipeline:20260727:{run_id}",
+        pipeline_key="supply_pipeline",
+        biz_dt="20260727",
+        trigger_type="test",
+        run_mode="full",
+        dry_run=True,
+        status="queued",
+        deadline_at=now + timedelta(days=1),
+        config_snapshot_json={},
+        date_snapshot_json={},
+    )
+
+
+def _step(
+    *,
+    step_run_id: str,
+    run_id: str,
+    status: str,
+    order: int,
+) -> PipelineStepRun:
+    return PipelineStepRun(
+        step_run_id=step_run_id,
+        run_id=run_id,
+        step_key=f"step-{step_run_id}",
+        step_order=order,
+        attempt=1,
+        status=status,
+        critical=True,
+        dependency_snapshot_json=[],
+        input_snapshot_json={},
+        timeout_seconds=10,
+        max_attempts=1,
+        retryable=False,
+    )
+
+
+def test_claim_respects_global_active_step_cap() -> None:
+    with _session() as session:
+        session.add_all(
+            [
+                _run("run-1"),
+                _run("run-2"),
+                _step(
+                    step_run_id="running",
+                    run_id="run-1",
+                    status="running",
+                    order=1,
+                ),
+                _step(
+                    step_run_id="ready",
+                    run_id="run-2",
+                    status="ready",
+                    order=1,
+                ),
+            ]
+        )
+        session.flush()
+
+        claimed = PipelineStepRunRepository(session).claim_next_ready(
+            owner="worker-2",
+            lease_seconds=120,
+            max_active_steps=1,
+            now=utc_now(),
+        )
+
+        assert claimed is None
+        ready = session.get(PipelineStepRun, "ready")
+        assert ready is not None
+        assert ready.status == "ready"
+
+
+def test_claim_does_not_overlap_a_different_pipeline_run() -> None:
+    with _session() as session:
+        session.add_all(
+            [
+                _run("run-1"),
+                _run("run-2"),
+                _step(
+                    step_run_id="running",
+                    run_id="run-1",
+                    status="running",
+                    order=1,
+                ),
+                _step(
+                    step_run_id="ready",
+                    run_id="run-2",
+                    status="ready",
+                    order=1,
+                ),
+            ]
+        )
+        session.flush()
+
+        claimed = PipelineStepRunRepository(session).claim_next_ready(
+            owner="worker-2",
+            lease_seconds=120,
+            max_active_steps=2,
+            now=utc_now(),
+        )
+
+        assert claimed is None

tests/supply_infra/scheduler/__init__.py

@@ -0,0 +1 @@
+"""scheduler tests."""

tests/supply_infra/scheduler/test_grade_demand_pool.py

@@ -0,0 +1,48 @@
+"""Current demand-grade scheduler wrapper tests."""
+from __future__ import annotations
+
+from unittest.mock import patch
+
+from supply_infra.scheduler.jobs.grade_demand_pool import (
+    execute_plan_tasks_until_complete,
+    grade_demand_pool,
+)
+
+
+@patch("supply_infra.scheduler.jobs.grade_demand_pool._execute_plan_tasks")
+def test_plan_execution_stops_when_no_groups_are_claimable(mock_execute) -> None:
+    snapshot = {
+        "execution_complete": True,
+        "assigned_category_ids": [],
+        "planned_groups": 0,
+        "group_status": {},
+    }
+    mock_execute.return_value = {
+        "attempted_groups": 0,
+        "groups_run": 0,
+        "workers": 0,
+        "final_snapshot": snapshot,
+        "execution_complete": True,
+    }
+
+    result = execute_plan_tasks_until_complete(
+        "20260721",
+        workers=5,
+        max_demands_per_batch=20,
+    )
+
+    assert result["rounds"] == 1
+    assert result["execution_complete"] is True
+    mock_execute.assert_called_once()
+
+
+@patch("supply_infra.scheduler.jobs.grade_demand_pool._grade_demand_pool_impl")
+@patch("supply_infra.scheduler.jobs.grade_demand_pool._resolve_biz_dt", return_value="20260721")
+def test_grade_job_never_raises_to_pipeline(_mock_resolve, mock_impl) -> None:
+    mock_impl.side_effect = RuntimeError("boom")
+
+    result = grade_demand_pool("20260721")
+
+    assert result["success"] is False
+    assert result["biz_dt"] == "20260721"
+    assert result["error"] == "boom"

tests/supply_infra/scheduler/test_run_supply_pipeline.py

@@ -0,0 +1,57 @@
+"""Compatibility wrapper tests for the durable pipeline."""
+from __future__ import annotations
+
+from unittest.mock import patch
+
+from supply_infra.pipeline.run_service import RunSubmission
+from supply_infra.scheduler.jobs.run_supply_pipeline import run_supply_pipeline
+
+
+@patch("supply_infra.scheduler.jobs.run_supply_pipeline.submit_pipeline_run")
+def test_pipeline_wrapper_only_submits_durable_run(mock_submit) -> None:
+    mock_submit.return_value = RunSubmission(
+        run_id="run-1",
+        created=True,
+        status="queued",
+        biz_dt="20260721",
+    )
+
+    result = run_supply_pipeline("20260721")
+
+    assert result["run_id"] == "run-1"
+    assert result["status"] == "queued"
+    assert result["dry_run"] is True
+    mock_submit.assert_called_once()
+
+
+@patch("supply_infra.scheduler.jobs.run_supply_pipeline.get_pipeline_run")
+@patch("supply_infra.scheduler.jobs.run_supply_pipeline.submit_pipeline_run")
+def test_wait_reads_terminal_state_from_mysql(mock_submit, mock_get) -> None:
+    mock_submit.return_value = RunSubmission(
+        run_id="run-2",
+        created=True,
+        status="queued",
+        biz_dt="20260721",
+    )
+    mock_get.return_value = {
+        "run_id": "run-2",
+        "status": "failed",
+        "error_message": "strict gate failed",
+    }
+
+    result = run_supply_pipeline("20260721", wait=True, poll_seconds=0.01)
+
+    assert result["success"] is False
+    assert result["status"] == "failed"
+
+
+@patch("supply_infra.scheduler.jobs.run_supply_pipeline.submit_pipeline_run")
+def test_submit_validation_error_is_not_hidden(mock_submit) -> None:
+    mock_submit.side_effect = ValueError("Invalid biz_dt")
+
+    try:
+        run_supply_pipeline("invalid-date")
+    except ValueError as exc:
+        assert "Invalid biz_dt" in str(exc)
+    else:
+        raise AssertionError("validation error must propagate to the CLI/API boundary")

tests/supply_infra/scheduler/test_sync_multi_demand_pool.py

@@ -0,0 +1,88 @@
+"""需求池同步内部子阶段的异常隔离测试。"""
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+from supply_infra.scheduler.jobs.demand_pool.sync import (
+    _classify_words,
+    sync_multi_demand_pool_odps_to_mysql,
+)
+
+
+@patch(
+    "supply_infra.scheduler.jobs.demand_pool.sync.sync_multi_demand_videos"
+)
+@patch(
+    "supply_infra.scheduler.jobs.demand_pool.sync.compute_category_tree_weight"
+)
+@patch(
+    "supply_infra.scheduler.jobs.demand_pool.sync.compute_popularity_stats"
+)
+@patch(
+    "supply_infra.scheduler.jobs.demand_pool.sync.enrich_real_rov_vov_7d"
+)
+@patch(
+    "supply_infra.scheduler.jobs.demand_pool.sync.sync_demand_belong_pool_rel"
+)
+@patch("supply_infra.scheduler.jobs.demand_pool.sync._classify_words")
+@patch("supply_infra.scheduler.jobs.demand_pool.sync._sync_pool_rows")
+def test_substep_failure_does_not_skip_later_substeps(
+    mock_source_sync,
+    mock_classify,
+    mock_rel,
+    mock_real_metrics,
+    mock_popularity,
+    mock_tree_weight,
+    mock_videos,
+) -> None:
+    mock_source_sync.return_value = {"inserted": 10}
+    mock_classify.side_effect = RuntimeError("agent unavailable")
+    mock_rel.return_value = {"inserted": 2}
+    mock_real_metrics.return_value = {"updated_rows": 3}
+    mock_popularity.return_value = {"upserted": 4}
+    mock_tree_weight.return_value = {"upserted": 5}
+    mock_videos.return_value = {"inserted": 6}
+
+    result = sync_multi_demand_pool_odps_to_mysql("20260721")
+
+    assert result["success"] is False
+    assert result["classify"] == {"success": False, "error": "agent unavailable"}
+    assert result["belong_pool_rel"] == {"inserted": 2}
+    assert result["real_metrics"] == {"updated_rows": 3}
+    assert result["popularity"] == {"upserted": 4}
+    assert result["tree_weight"] == {"upserted": 5}
+    assert result["videos"] == {"inserted": 6}
+    assert [item["stage"] for item in result["errors"]] == ["classify"]
+
+
+@patch(
+    "supply_infra.scheduler.jobs.demand_pool.sync.classify_demand_words"
+)
+@patch(
+    "supply_infra.scheduler.jobs.demand_pool.sync.DemandBelongCategoryRepository"
+)
+@patch(
+    "supply_infra.scheduler.jobs.demand_pool.sync.MultiDemandPoolDiRepository"
+)
+@patch("supply_infra.scheduler.jobs.demand_pool.sync.get_session")
+def test_classification_batch_failure_continues_remaining_batches(
+    mock_get_session,
+    mock_pool_repo_cls,
+    mock_belong_repo_cls,
+    mock_classify,
+) -> None:
+    mock_get_session.return_value.__enter__.return_value = MagicMock()
+    mock_pool_repo_cls.return_value.list_demand_names_by_biz_dt.return_value = [
+        f"需求{i:03d}" for i in range(120)
+    ]
+    mock_belong_repo_cls.return_value.get_existing_names.return_value = set()
+    mock_classify.side_effect = [RuntimeError("first batch failed"), None, None]
+
+    result = _classify_words("20260721")
+
+    assert mock_classify.call_count == 3
+    assert result["success"] is False
+    assert result["batches"] == 3
+    assert result["failed_batches"] == [
+        {"batch": 1, "size": 50, "error": "first batch failed"}
+    ]

web/src/App.vue

@@ -12,6 +12,7 @@ const navItems = [
   { to: '/', label: '供给总览', icon: '◫' },
   { to: '/demand-map', label: '全局需求地图', icon: '⌁' },
   { to: '/video-discovery', label: '需求汇总', icon: '▷' },
+  { to: '/pipeline-runs', label: '定时任务', icon: '◷' },
   { to: '/demand-process', label: 'Agent 审计', icon: '◎' },
 ]
 

web/src/api/dashboard.ts

@@ -6,7 +6,7 @@ export interface SchedulerJobStatus {
 
 export interface SchedulerStatus {
   enabled: boolean
-  running: boolean
+  running: boolean | null
   timezone: string
   jobs: SchedulerJobStatus[]
 }
@@ -22,14 +22,15 @@ export interface ManualJob {
 export interface ManualRun {
   accepted?: boolean
   run_id: string
-  job_id: string
-  job_name: string
-  status: 'running' | 'finished' | 'failed' | string
-  biz_dt: string | null
-  partition_date: string | null
+  job_id?: string
+  job_name?: string
+  status: 'queued' | 'running' | 'succeeded' | 'failed' | 'blocked_previous_run' | string
+  biz_dt: string
+  partition_date?: string | null
   started_at?: string
   finished_at?: string
   error?: string
+  error_message?: string
 }
 
 async function readJson<T>(response: Response, message: string): Promise<T> {

web/src/api/pipeline.ts

@@ -0,0 +1,53 @@
+import type { PipelineRun } from '../types/pipeline'
+
+async function readJson<T>(response: Response, message: string): Promise<T> {
+  if (!response.ok) {
+    const detail = await response.text()
+    throw new Error(`${message}: ${response.status} ${detail || response.statusText}`)
+  }
+  return response.json() as Promise<T>
+}
+
+export async function fetchPipelineRuns(): Promise<{ items: PipelineRun[] }> {
+  return readJson(await fetch('/api/pipeline/runs'), '加载流水线批次失败')
+}
+
+export async function fetchPipelineRun(runId: string): Promise<PipelineRun> {
+  return readJson(
+    await fetch(`/api/pipeline/runs/${encodeURIComponent(runId)}`),
+    '加载流水线详情失败',
+  )
+}
+
+export async function createPipelineRun(bizDt: string): Promise<{
+  accepted: boolean
+  run_id: string
+  status: string
+}> {
+  return readJson(
+    await fetch('/api/pipeline/runs', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ biz_dt: bizDt || null }),
+    }),
+    '创建流水线批次失败',
+  )
+}
+
+export async function resumePipelineRun(runId: string): Promise<void> {
+  await readJson(
+    await fetch(`/api/pipeline/runs/${encodeURIComponent(runId)}/resume`, {
+      method: 'POST',
+    }),
+    '恢复流水线失败',
+  )
+}
+
+export async function cancelPipelineRun(runId: string): Promise<void> {
+  await readJson(
+    await fetch(`/api/pipeline/runs/${encodeURIComponent(runId)}/cancel`, {
+      method: 'POST',
+    }),
+    '取消流水线失败',
+  )
+}

web/src/components/pipeline/PipelineRunActions.vue

@@ -0,0 +1,29 @@
+<script setup lang="ts">
+import type { PipelineRun } from '../../types/pipeline'
+
+defineProps<{ run: PipelineRun; busy?: boolean }>()
+defineEmits<{ resume: []; cancel: [] }>()
+</script>
+
+<template>
+  <div class="pipeline-actions">
+    <button
+      :disabled="busy || !['failed', 'deadline_exceeded'].includes(run.status)"
+      @click="$emit('resume')"
+    >
+      从失败点恢复
+    </button>
+    <button
+      :disabled="busy || ['succeeded', 'failed', 'deadline_exceeded', 'cancelling', 'cancelled'].includes(run.status)"
+      @click="$emit('cancel')"
+    >
+      取消批次
+    </button>
+  </div>
+</template>
+
+<style scoped>
+.pipeline-actions { display: flex; gap: 8px; }
+button { border: 1px solid #cbd2df; border-radius: 8px; background: white; padding: 8px 12px; cursor: pointer; }
+button:disabled { opacity: .45; cursor: default; }
+</style>

web/src/components/pipeline/PipelineStatusCard.vue

@@ -0,0 +1,29 @@
+<script setup lang="ts">
+import type { PipelineRun } from '../../types/pipeline'
+
+defineProps<{ run: PipelineRun }>()
+</script>
+
+<template>
+  <article class="pipeline-status-card">
+    <div>
+      <strong>{{ run.biz_dt }}</strong>
+      <span :class="['pipeline-status', `is-${run.status}`]">{{ run.status }}</span>
+    </div>
+    <p>{{ run.current_step || '无执行中步骤' }}</p>
+    <small v-if="run.deadline_at">
+      截止 {{ new Date(run.deadline_at).toLocaleString('zh-CN') }}
+    </small>
+    <small v-else>手工任务无截止时间</small>
+    <b v-if="run.dry_run">AIGC DRY-RUN</b>
+  </article>
+</template>
+
+<style scoped>
+.pipeline-status-card { border: 1px solid #d9dee8; border-radius: 12px; padding: 14px; }
+.pipeline-status-card > div { display: flex; justify-content: space-between; gap: 12px; }
+.pipeline-status { font-size: 12px; text-transform: uppercase; }
+.is-succeeded { color: #16835d; }
+.is-failed, .is-deadline_exceeded { color: #c23d4b; }
+b { display: inline-block; margin-top: 8px; color: #8a5b00; font-size: 11px; }
+</style>

web/src/components/pipeline/PipelineStepTimeline.vue

@@ -0,0 +1,26 @@
+<script setup lang="ts">
+import type { PipelineStepRun } from '../../types/pipeline'
+
+defineProps<{ steps: PipelineStepRun[] }>()
+</script>
+
+<template>
+  <ol class="step-timeline">
+    <li v-for="step in steps" :key="step.step_run_id">
+      <span>{{ String(step.step_order).padStart(2, '0') }}</span>
+      <div>
+        <strong>{{ step.step_key }}</strong>
+        <small>attempt {{ step.attempt }} · {{ step.status }}</small>
+        <p v-if="step.error_message">{{ step.error_message }}</p>
+      </div>
+    </li>
+  </ol>
+</template>
+
+<style scoped>
+.step-timeline { list-style: none; padding: 0; display: grid; gap: 8px; }
+li { display: grid; grid-template-columns: 34px 1fr; gap: 10px; padding: 10px; border: 1px solid #e4e7ee; border-radius: 10px; }
+li > span { color: #6b7280; font-variant-numeric: tabular-nums; }
+small { display: block; color: #6b7280; }
+p { margin: 6px 0 0; color: #b42335; }
+</style>

web/src/router.ts

@@ -3,6 +3,7 @@ import CategoryTreeView from './views/CategoryTreeView.vue'
 import DemandProcessView from './views/DemandProcessView.vue'
 import GlobalDemandMapView from './views/GlobalDemandMapView.vue'
 import OverviewView from './views/OverviewView.vue'
+import PipelineRunsView from './views/PipelineRunsView.vue'
 import VideoDiscoveryView from './views/VideoDiscoveryView.vue'
 
 export const router = createRouter({
@@ -38,5 +39,11 @@ export const router = createRouter({
       component: VideoDiscoveryView,
       meta: { title: '需求汇总' },
     },
+    {
+      path: '/pipeline-runs',
+      name: 'pipeline-runs',
+      component: PipelineRunsView,
+      meta: { title: '定时任务运行中心' },
+    },
   ],
 })

web/src/types/pipeline.ts

@@ -0,0 +1,74 @@
+export type PipelineRunStatus =
+  | 'queued'
+  | 'running'
+  | 'succeeded'
+  | 'failed'
+  | 'blocked'
+  | 'cancelling'
+  | 'cancelled'
+  | 'deadline_exceeded'
+  | 'blocked_previous_run'
+
+export type PipelineStepStatus =
+  | 'pending'
+  | 'ready'
+  | 'running'
+  | 'retry_wait'
+  | 'succeeded'
+  | 'failed'
+  | 'blocked'
+  | 'timed_out'
+  | 'cancelled'
+
+export interface PipelineStepRun {
+  step_run_id: string
+  run_id: string
+  step_key: string
+  step_order: number
+  attempt: number
+  status: PipelineStepStatus
+  critical: boolean
+  timeout_seconds: number
+  started_at: string | null
+  finished_at: string | null
+  heartbeat_at: string | null
+  result: Record<string, unknown> | null
+  error_code: string | null
+  error_message: string | null
+  log_uri: string | null
+}
+
+export interface PipelineEffect {
+  outbox_id: string
+  step_run_id: string
+  effect_type: string
+  payload_hash: string
+  payload_uri: string | null
+  status: 'dry_run_recorded' | 'record_failed'
+  dry_run: boolean
+  created_at: string | null
+}
+
+export interface PipelineRun {
+  run_id: string
+  pipeline_key: string
+  biz_dt: string
+  trigger_type: string
+  trigger_source: string | null
+  run_mode: string
+  dry_run: boolean
+  status: PipelineRunStatus
+  current_step: string | null
+  scheduled_for: string | null
+  deadline_at: string | null
+  started_at: string | null
+  finished_at: string | null
+  heartbeat_at: string | null
+  summary: Record<string, unknown> | null
+  error_code: string | null
+  error_message: string | null
+  created_at: string
+  updated_at: string
+  steps?: PipelineStepRun[]
+  effects?: PipelineEffect[]
+}

web/src/views/OverviewView.vue

@@ -72,8 +72,8 @@ const pipeline = [
   },
   {
     index: '06',
-    title: 'AIGC 分发',
-    description: '把合格候选幂等交付给视频爬取与生产计划。',
+    title: 'AIGC 写入记录',
+    description: '以 dry-run 记录准备写入的数据与哈希，不调用真实下游。',
     color: 'green',
   },
 ]
@@ -429,16 +429,17 @@ function pollRun(runId: string) {
     try {
       const run = await fetchManualRun(runId)
       activeRun.value = run
-      if (run.status === 'running') {
+      if (['queued', 'running', 'retry_wait', 'cancelling', 'blocked_previous_run'].includes(run.status)) {
         pollRun(runId)
         return
       }
       runningJobId.value = null
-      noticeType.value = run.status === 'finished' ? 'success' : 'error'
+      noticeType.value = run.status === 'succeeded' ? 'success' : 'error'
+      const runName = run.job_name || '供给流水线'
       notice.value =
-        run.status === 'finished'
-          ? `${run.job_name}执行完成`
-          : `${run.job_name}执行失败：${run.error || '请查看运行日志'}`
+        run.status === 'succeeded'
+          ? `${runName}执行完成`
+          : `${runName}执行失败：${run.error_message || run.error || '请查看运行中心'}`
       await loadDashboard()
     } catch {
       runningJobId.value = null
@@ -458,7 +459,7 @@ function pollRun(runId: string) {
         </div>
         <h1>从需求信号到<br /><em>可执行内容供给</em></h1>
         <p>
-          SupplyAgent 把分类、热度、分级、视频拓展、内容发现与 AIGC 分发连接成一条
+          SupplyAgent 把分类、热度、分级、视频拓展、内容发现与 AIGC dry-run 记录连接成一条
           可追溯的智能流水线。
         </p>
         <div class="hero-actions">
@@ -668,7 +669,7 @@ function pollRun(runId: string) {
           <h2>每日自动供给流水线</h2>
         </div>
         <div class="pipeline-meta">
-          <span class="live-pill"><i /> 每日 14:30 自动执行</span>
+          <span class="live-pill"><i /> 每日 15:00 自动执行</span>
           <span>6 个阶段串行推进</span>
         </div>
       </div>

web/src/views/PipelineRunsView.vue

@@ -0,0 +1,130 @@
+<script setup lang="ts">
+import { onMounted, ref } from 'vue'
+import {
+  cancelPipelineRun,
+  createPipelineRun,
+  fetchPipelineRun,
+  fetchPipelineRuns,
+  resumePipelineRun,
+} from '../api/pipeline'
+import PipelineRunActions from '../components/pipeline/PipelineRunActions.vue'
+import PipelineStatusCard from '../components/pipeline/PipelineStatusCard.vue'
+import PipelineStepTimeline from '../components/pipeline/PipelineStepTimeline.vue'
+import type { PipelineRun } from '../types/pipeline'
+
+const runs = ref<PipelineRun[]>([])
+const selected = ref<PipelineRun | null>(null)
+const loading = ref(false)
+const busy = ref(false)
+const error = ref('')
+const bizDt = ref(new Intl.DateTimeFormat('sv-SE', {
+  timeZone: 'Asia/Shanghai',
+  year: 'numeric',
+  month: '2-digit',
+  day: '2-digit',
+}).format(new Date()).replaceAll('-', ''))
+
+async function loadRuns() {
+  loading.value = true
+  error.value = ''
+  try {
+    runs.value = (await fetchPipelineRuns()).items
+    if (!selected.value && runs.value[0]) await selectRun(runs.value[0].run_id)
+  } catch (cause) {
+    error.value = cause instanceof Error ? cause.message : String(cause)
+  } finally {
+    loading.value = false
+  }
+}
+
+async function selectRun(runId: string) {
+  selected.value = await fetchPipelineRun(runId)
+}
+
+async function submit() {
+  busy.value = true
+  try {
+    const result = await createPipelineRun(bizDt.value)
+    await loadRuns()
+    await selectRun(result.run_id)
+  } finally {
+    busy.value = false
+  }
+}
+
+async function resume() {
+  if (!selected.value) return
+  busy.value = true
+  try {
+    await resumePipelineRun(selected.value.run_id)
+    await selectRun(selected.value.run_id)
+  } finally {
+    busy.value = false
+  }
+}
+
+async function cancel() {
+  if (!selected.value) return
+  busy.value = true
+  try {
+    await cancelPipelineRun(selected.value.run_id)
+    await selectRun(selected.value.run_id)
+  } finally {
+    busy.value = false
+  }
+}
+
+onMounted(loadRuns)
+</script>
+
+<template>
+  <main class="pipeline-runs">
+    <header>
+      <div>
+        <span>PIPELINE CONTROL CENTER</span>
+        <h1>定时任务运行中心</h1>
+        <p>MySQL 持久化状态 · 严格门禁 · AIGC dry-run</p>
+      </div>
+      <form @submit.prevent="submit">
+        <input v-model="bizDt" pattern="\d{8}" aria-label="业务日" />
+        <button :disabled="busy">创建批次</button>
+      </form>
+    </header>
+
+    <p v-if="error" class="error">{{ error }}</p>
+    <div class="layout">
+      <aside>
+        <p v-if="loading">加载中…</p>
+        <button v-for="run in runs" :key="run.run_id" class="run-item" @click="selectRun(run.run_id)">
+          <PipelineStatusCard :run="run" />
+        </button>
+      </aside>
+      <section v-if="selected" class="detail">
+        <div class="detail-head">
+          <div>
+            <h2>{{ selected.biz_dt }} · {{ selected.status }}</h2>
+            <code>{{ selected.run_id }}</code>
+          </div>
+          <PipelineRunActions :run="selected" :busy="busy" @resume="resume" @cancel="cancel" />
+        </div>
+        <PipelineStepTimeline :steps="selected.steps || []" />
+      </section>
+    </div>
+  </main>
+</template>
+
+<style scoped>
+.pipeline-runs { max-width: 1240px; margin: 0 auto; padding: 32px 24px 72px; }
+header, .detail-head { display: flex; align-items: flex-start; justify-content: space-between; gap: 24px; }
+header span { font-size: 12px; letter-spacing: .12em; color: #667085; }
+h1 { margin: 6px 0; }
+form { display: flex; gap: 8px; }
+input, form button { border: 1px solid #cbd2df; border-radius: 8px; padding: 9px 12px; }
+.layout { display: grid; grid-template-columns: 320px 1fr; gap: 24px; margin-top: 28px; }
+aside { display: grid; gap: 10px; align-content: start; }
+.run-item { text-align: left; border: 0; background: transparent; padding: 0; cursor: pointer; }
+.detail { border: 1px solid #d9dee8; border-radius: 14px; padding: 20px; }
+.detail-head { margin-bottom: 20px; }
+.error { color: #b42335; }
+@media (max-width: 800px) { .layout { grid-template-columns: 1fr; } header { flex-direction: column; } }
+</style>

zhangbo.md

@@ -503,7 +503,7 @@ multi_demand_pool_di.id
 
 `supply_infra/scheduler/app.py` 当前只注册一条任务：
 
-1. 每天 `14:30`（`SCHEDULER_TIMEZONE`）：串行执行全链路  
+1. 每天 `15:00`（`SCHEDULER_TIMEZONE`）：串行执行全链路
    `run_supply_pipeline`（全局树 → 需求池 → 分级 → 拓展 → 找片 → AIGC 发布）。
 
 ### 9.2 全局树同步