|
@@ -14,10 +14,7 @@ from supply_infra.db.repositories.demand_grade_plan_repo import DemandGradePlanR
|
|
|
from supply_infra.db.repositories.demand_grade_repo import DemandGradeRepository
|
|
from supply_infra.db.repositories.demand_grade_repo import DemandGradeRepository
|
|
|
from supply_infra.db.repositories.multi_demand_pool_di_repo import MultiDemandPoolDiRepository
|
|
from supply_infra.db.repositories.multi_demand_pool_di_repo import MultiDemandPoolDiRepository
|
|
|
from supply_infra.db.session import get_session
|
|
from supply_infra.db.session import get_session
|
|
|
-from supply_infra.scheduler.plan_group_batch import (
|
|
|
|
|
- MAX_DEMANDS_PER_BATCH,
|
|
|
|
|
- list_pending_demands_by_category_ids,
|
|
|
|
|
-)
|
|
|
|
|
|
|
+from supply_infra.scheduler.plan_group_batch import MAX_DEMANDS_PER_BATCH, split_even_batches
|
|
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
@@ -30,36 +27,57 @@ def _resolve_biz_dt(biz_dt: str | None) -> str:
|
|
|
return datetime.now(ZoneInfo(get_infra_settings().scheduler_timezone)).strftime("%Y%m%d")
|
|
return datetime.now(ZoneInfo(get_infra_settings().scheduler_timezone)).strftime("%Y%m%d")
|
|
|
|
|
|
|
|
|
|
|
|
|
-def _graded_names(biz_dt: str) -> set[str]:
|
|
|
|
|
|
|
+def _materialize_pending_group_items(biz_dt: str) -> int:
|
|
|
|
|
+ """执行前为 pending 计划组物化待分级需求明细。"""
|
|
|
with get_session() as session:
|
|
with get_session() as session:
|
|
|
- return DemandGradeRepository(session).get_existing_demand_names(biz_dt)
|
|
|
|
|
|
|
+ graded_names = DemandGradeRepository(session).get_existing_demand_names(biz_dt)
|
|
|
|
|
+ return DemandGradePlanRepository(session).materialize_pending_groups(
|
|
|
|
|
+ biz_dt,
|
|
|
|
|
+ graded_names=graded_names,
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def _group_success(item_summary: dict[str, int], processed_batches: int) -> bool:
|
|
|
|
|
+ if item_summary.get("pending", 0) > 0:
|
|
|
|
|
+ return False
|
|
|
|
|
+ total = sum(item_summary.values())
|
|
|
|
|
+ if total == 0:
|
|
|
|
|
+ return True
|
|
|
|
|
+ if processed_batches > 0:
|
|
|
|
|
+ return True
|
|
|
|
|
+ return item_summary.get("failed", 0) == 0 and (
|
|
|
|
|
+ item_summary.get("finished", 0) > 0 or item_summary.get("skipped", 0) > 0
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
|
|
|
|
|
def _run_group(biz_dt: str, group_id: int, *, max_demands_per_batch: int) -> int:
|
|
def _run_group(biz_dt: str, group_id: int, *, max_demands_per_batch: int) -> int:
|
|
|
- """领取并执行一个固定任务,组内按批处理,失败子批跳过并继续。"""
|
|
|
|
|
|
|
+ """领取并执行一个固定任务,从组内需求明细表按批读取。"""
|
|
|
with get_session() as session:
|
|
with get_session() as session:
|
|
|
group = DemandGradePlanRepository(session).claim_group(biz_dt, group_id)
|
|
group = DemandGradePlanRepository(session).claim_group(biz_dt, group_id)
|
|
|
if group is None:
|
|
if group is None:
|
|
|
logger.warning("分级任务已被其他 worker 领取或状态已变化: biz_dt=%s group_id=%s", biz_dt, group_id)
|
|
logger.warning("分级任务已被其他 worker 领取或状态已变化: biz_dt=%s group_id=%s", biz_dt, group_id)
|
|
|
return 0
|
|
return 0
|
|
|
|
|
|
|
|
|
|
+ gid = int(group["id"])
|
|
|
processed_batches = 0
|
|
processed_batches = 0
|
|
|
batch_errors: list[str] = []
|
|
batch_errors: list[str] = []
|
|
|
- skipped_names: set[str] = set()
|
|
|
|
|
try:
|
|
try:
|
|
|
- while True:
|
|
|
|
|
- graded_before = _graded_names(biz_dt) | skipped_names
|
|
|
|
|
- demands = list_pending_demands_by_category_ids(
|
|
|
|
|
- biz_dt,
|
|
|
|
|
- group["category_ids"],
|
|
|
|
|
- max_demands=max_demands_per_batch,
|
|
|
|
|
- excluded_demand_names=sorted(graded_before),
|
|
|
|
|
- )
|
|
|
|
|
- if not demands:
|
|
|
|
|
- break
|
|
|
|
|
|
|
+ with get_session() as session:
|
|
|
|
|
+ items = DemandGradePlanRepository(session).list_pending_group_items(gid)
|
|
|
|
|
|
|
|
|
|
+ for batch_items in split_even_batches(items, max_per_batch=max_demands_per_batch):
|
|
|
|
|
+ demands = [
|
|
|
|
|
+ {"pool_id": item["pool_id"], "demand_name": item["demand_name"]}
|
|
|
|
|
+ for item in batch_items
|
|
|
|
|
+ ]
|
|
|
|
|
+ item_ids = [int(item["item_id"]) for item in batch_items]
|
|
|
try:
|
|
try:
|
|
|
grade_demand_words(demands, biz_dt=biz_dt)
|
|
grade_demand_words(demands, biz_dt=biz_dt)
|
|
|
|
|
+ with get_session() as session:
|
|
|
|
|
+ DemandGradePlanRepository(session).mark_group_items_status(
|
|
|
|
|
+ item_ids,
|
|
|
|
|
+ status="finished",
|
|
|
|
|
+ )
|
|
|
processed_batches += 1
|
|
processed_batches += 1
|
|
|
except Exception as exc:
|
|
except Exception as exc:
|
|
|
logger.exception(
|
|
logger.exception(
|
|
@@ -69,12 +87,19 @@ def _run_group(biz_dt: str, group_id: int, *, max_demands_per_batch: int) -> int
|
|
|
len(demands),
|
|
len(demands),
|
|
|
)
|
|
)
|
|
|
batch_errors.append(str(exc))
|
|
batch_errors.append(str(exc))
|
|
|
- skipped_names.update(item["demand_name"] for item in demands)
|
|
|
|
|
|
|
+ with get_session() as session:
|
|
|
|
|
+ DemandGradePlanRepository(session).mark_group_items_status(
|
|
|
|
|
+ item_ids,
|
|
|
|
|
+ status="failed",
|
|
|
|
|
+ error_message=str(exc),
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
with get_session() as session:
|
|
with get_session() as session:
|
|
|
- DemandGradePlanRepository(session).finish_group(
|
|
|
|
|
- group["id"],
|
|
|
|
|
- success=processed_batches > 0,
|
|
|
|
|
|
|
+ repo = DemandGradePlanRepository(session)
|
|
|
|
|
+ item_summary = repo.summarize_group_items(gid)
|
|
|
|
|
+ repo.finish_group(
|
|
|
|
|
+ gid,
|
|
|
|
|
+ success=_group_success(item_summary, processed_batches),
|
|
|
error_message="; ".join(batch_errors) if batch_errors else None,
|
|
error_message="; ".join(batch_errors) if batch_errors else None,
|
|
|
)
|
|
)
|
|
|
except Exception as exc:
|
|
except Exception as exc:
|
|
@@ -86,7 +111,7 @@ def _run_group(biz_dt: str, group_id: int, *, max_demands_per_batch: int) -> int
|
|
|
try:
|
|
try:
|
|
|
with get_session() as session:
|
|
with get_session() as session:
|
|
|
DemandGradePlanRepository(session).finish_group(
|
|
DemandGradePlanRepository(session).finish_group(
|
|
|
- group["id"],
|
|
|
|
|
|
|
+ gid,
|
|
|
success=False,
|
|
success=False,
|
|
|
error_message=str(exc),
|
|
error_message=str(exc),
|
|
|
)
|
|
)
|
|
@@ -202,6 +227,9 @@ def _grade_demand_pool_impl(
|
|
|
except Exception:
|
|
except Exception:
|
|
|
logger.exception("统筹 Agent 执行失败,继续处理数据库中已有任务: biz_dt=%s", resolved_biz_dt)
|
|
logger.exception("统筹 Agent 执行失败,继续处理数据库中已有任务: biz_dt=%s", resolved_biz_dt)
|
|
|
|
|
|
|
|
|
|
+ materialized = _materialize_pending_group_items(resolved_biz_dt)
|
|
|
|
|
+ logger.info("物化计划组需求明细: biz_dt=%s items=%s", resolved_biz_dt, materialized)
|
|
|
|
|
+
|
|
|
plan_execution = execute_plan_tasks_until_complete(
|
|
plan_execution = execute_plan_tasks_until_complete(
|
|
|
resolved_biz_dt,
|
|
resolved_biz_dt,
|
|
|
workers=max(1, int(workers)),
|
|
workers=max(1, int(workers)),
|
|
@@ -218,6 +246,7 @@ def _grade_demand_pool_impl(
|
|
|
"total": total,
|
|
"total": total,
|
|
|
"graded_before": graded_before,
|
|
"graded_before": graded_before,
|
|
|
"graded_after": graded_after,
|
|
"graded_after": graded_after,
|
|
|
|
|
+ "materialized_items": materialized,
|
|
|
"planned_category_count": len(final_snapshot["assigned_category_ids"]),
|
|
"planned_category_count": len(final_snapshot["assigned_category_ids"]),
|
|
|
"planned_groups": final_snapshot["planned_groups"],
|
|
"planned_groups": final_snapshot["planned_groups"],
|
|
|
"group_status": final_snapshot["group_status"],
|
|
"group_status": final_snapshot["group_status"],
|