execute_creation_once.py 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104
  1. """模块 B 创意搭建子系统 — 主循环入口(P0-A / P0-C / P0-E,2026-06-09)。
  2. 数据流(三阶段,先审后挂):
  3. Phase 1 — 准备:
  4. 扫账户 → find_ads_needing_creatives
  5. → 对每条广告 prepare_one_creative_for_ad(召回 + 上传图 + xcx/save + build body)
  6. → pending_records (含完整 Phase 3 用的 _request_body)
  7. → 写 outputs/data/creation_pending_{date}.json(追溯 + 独立 apply 用)
  8. Phase 2 — 审批(若 CREATION_APPROVAL_REQUIRED=True):
  9. run_approval_workflow 生成 20 列 xlsx → 上传飞书 sheet → 发链接 → 轮询读决策列
  10. actions {row_idx: approve/reject/hold}
  11. → 写回 records["action"]
  12. Phase 3 — 执行:
  13. 对 action=approve 的 → POST /dynamic_creatives/add
  14. → 写 outputs/data/creation_run_{date}.json
  15. → 发飞书"执行汇报"消息
  16. 开关 CREATION_APPROVAL_REQUIRED=False 时跳过 Phase 2,全 records 直接 approve。
  17. """
  18. import argparse
  19. import json
  20. import logging
  21. import os
  22. import sys
  23. import time
  24. from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
  25. from datetime import date
  26. from pathlib import Path
  27. _HERE = Path(__file__).parent
  28. sys.path.insert(0, str(_HERE.parent.parent))
  29. sys.path.insert(0, str(_HERE))
  30. from dotenv import load_dotenv # noqa: E402
  31. load_dotenv(_HERE / ".env")
  32. from config import ( # noqa: E402
  33. ADS_PER_ACCOUNT,
  34. CREATION_APPROVAL_REQUIRED,
  35. CREATION_APPROVAL_TIMEOUT_MINUTES,
  36. CREATIVE_PREPARE_MAX_WORKERS,
  37. CREATIVE_PREPARE_TASK_BUFFER,
  38. MAX_SAME_LANDING_PER_AD_IN_RUN,
  39. TARGET_CREATIVES_PER_AD,
  40. WHITELIST_ACCOUNTS,
  41. get_creation_account_ids,
  42. now_in_timezone,
  43. )
  44. # config import 副作用会改 sys.path(把 im-client / agent/tools/builtin 顶到前面),
  45. # 强占 sys.path[0],只影响本入口自己的 import resolution。
  46. while str(_HERE) in sys.path:
  47. sys.path.remove(str(_HERE))
  48. sys.path.insert(0, str(_HERE))
  49. from tools.ad_creation import ( # noqa: E402
  50. build_ad_request_body,
  51. compute_fingerprint,
  52. enumerate_new_ad_candidates,
  53. post_ad_with_prepared_body,
  54. )
  55. from tools.audience_grant import ensure_account_audience_grant # noqa: E402
  56. from tools.creative_creation import ( # noqa: E402
  57. build_landing_candidate_pool,
  58. find_ads_needing_creatives,
  59. prepare_one_creative_for_ad,
  60. )
  61. from tools.creative_material_usage import ( # noqa: E402
  62. load_recent_landing_usage_counts,
  63. load_recoverable_prepared_records,
  64. record_prepared_material_usage,
  65. )
  66. from tools.account_material_strategy import ( # noqa: E402
  67. MATERIAL_SOURCE_AI_GENERATED,
  68. MATERIAL_SOURCE_EXTERNAL_RECALL,
  69. load_account_material_strategy,
  70. )
  71. from tools.video_recall import get_account_crowd_package # noqa: E402
  72. from execute_creation_apply import ( # noqa: E402
  73. apply_pending_records,
  74. write_summary as write_apply_summary,
  75. _send_apply_summary_to_feishu,
  76. )
  77. logger = logging.getLogger("execute_creation_once")
  78. MATERIAL_SOURCE_HISTORY = "history"
  79. def _env_flag(name: str, default: bool = False) -> bool:
  80. raw = os.getenv(name)
  81. if raw is None:
  82. return default
  83. return raw.strip().lower() in {"1", "true", "yes", "y", "on"}
  84. def _env_csv_set(name: str) -> set[str]:
  85. return {
  86. item.strip()
  87. for item in os.getenv(name, "").split(",")
  88. if item.strip()
  89. }
  90. def _parse_account_id_arg(raw: str) -> int:
  91. """Parse one explicit CLI account id and fail closed on malformed input."""
  92. try:
  93. account_id = int(raw)
  94. except (TypeError, ValueError) as exc:
  95. raise argparse.ArgumentTypeError(f"账户 ID 必须是正整数: {raw!r}") from exc
  96. if account_id <= 0:
  97. raise argparse.ArgumentTypeError(f"账户 ID 必须是正整数: {raw!r}")
  98. return account_id
  99. def _normalize_requested_account_ids(
  100. account_ids: list[int] | None,
  101. ) -> set[int] | None:
  102. """Normalize CLI scope; reject the removed fail-open environment variable."""
  103. legacy_scope = os.getenv("CREATION_ONLY_ACCOUNT_IDS", "").strip()
  104. if legacy_scope:
  105. raise ValueError(
  106. "CREATION_ONLY_ACCOUNT_IDS 已停用;请使用可重复的 --account-id 参数"
  107. )
  108. if account_ids is None:
  109. return None
  110. requested = {int(account_id) for account_id in account_ids}
  111. if not requested:
  112. raise ValueError("指定账户执行时至少需要一个 --account-id")
  113. return requested
  114. def _validate_requested_accounts(
  115. requested: set[int] | None,
  116. eligible_accounts: list[int],
  117. config_date: date | None,
  118. ) -> None:
  119. """Ensure every explicitly requested account is enabled for this config date."""
  120. if requested is None:
  121. return
  122. rejected = sorted(requested - set(eligible_accounts))
  123. if rejected:
  124. scope = config_date.isoformat() if config_date is not None else "各账户最新飞书配置"
  125. raise ValueError(
  126. f"账户 {rejected} 不在 {scope} 的启用创建配置中,拒绝执行"
  127. )
  128. def _filter_creation_accounts(
  129. accounts: list[int],
  130. phase: str,
  131. only_account_ids: set[int] | None = None,
  132. ) -> list[int]:
  133. """Apply explicit CLI account scope and optional crowd-package filter."""
  134. account_filter = only_account_ids
  135. crowd_filter = _env_csv_set("CREATION_ONLY_CROWD_PACKAGES")
  136. if account_filter is None and not crowd_filter:
  137. return accounts
  138. selected: list[int] = []
  139. for account_id in accounts:
  140. if account_filter is not None and account_id not in account_filter:
  141. continue
  142. if crowd_filter:
  143. try:
  144. crowd_package = get_account_crowd_package(account_id)
  145. except Exception as e:
  146. logger.warning(
  147. "[%s] account=%d 临时过滤读取 crowd_package 失败,跳过:%s",
  148. phase, account_id, e,
  149. )
  150. continue
  151. if crowd_package not in crowd_filter:
  152. continue
  153. selected.append(account_id)
  154. logger.info(
  155. "[%s] 临时运行过滤 account_ids=%s crowd_packages=%s: %d -> %d",
  156. phase,
  157. sorted(account_filter) if account_filter is not None else "ALL",
  158. sorted(crowd_filter) if crowd_filter else "ALL",
  159. len(accounts),
  160. len(selected),
  161. )
  162. return selected
  163. def _setup_logging() -> None:
  164. """配置日志格式与级别,挂载 SLS 上报(配置缺失时自动降级)。"""
  165. from logging_setup import setup_logging
  166. setup_logging(level=os.getenv("LOG_LEVEL", "INFO"))
  167. def _fetch_existing_fingerprints_for_account(account_id: int) -> set[str]:
  168. """Task 27:拉账户下所有非 DELETED 广告 → 反推 fingerprint 集合(预校验唯一性)。
  169. 腾讯文档:NORMAL + SUSPEND 状态广告均占用唯一性槽位,DENIED 也算占用。
  170. list 接口默认返回所有非 DELETED 状态 ad,故无需显式 filter status。
  171. 若腾讯 list 接口某些字段不返回(field 实测差异),降级:跳过该条 + log 警告。
  172. 返回空集 → 上层退化为"不预校验",enumerate 仍能跑(原有行为)。
  173. """
  174. from tools.ad_api import _get
  175. fingerprints: set[str] = set()
  176. skipped = 0
  177. page = 1
  178. try:
  179. while True:
  180. r = _get("/adgroups/get", {
  181. "account_id": account_id, "page": page, "page_size": 100,
  182. "fields": [
  183. "adgroup_id", "configured_status", "site_set",
  184. "targeting", "scene_spec",
  185. ],
  186. })
  187. ads = (r.get("data") or {}).get("list") or []
  188. if not ads:
  189. break
  190. for ad in ads:
  191. tgt = ad.get("targeting") or {}
  192. sc = ad.get("scene_spec") or {}
  193. wp = sc.get("wechat_position") if sc else None
  194. site_set = ad.get("site_set") or []
  195. age = tgt.get("age") or []
  196. geo_regions = (tgt.get("geo_location") or {}).get("regions") or []
  197. custom_audience = tgt.get("custom_audience")
  198. if not site_set or not age:
  199. skipped += 1
  200. continue
  201. try:
  202. fp = compute_fingerprint(
  203. account_id=account_id,
  204. site_set=site_set,
  205. custom_audience=custom_audience,
  206. age=age,
  207. geo_regions=geo_regions,
  208. wechat_position=wp,
  209. )
  210. fingerprints.add(fp)
  211. except Exception as e:
  212. skipped += 1
  213. logger.debug(
  214. "[phase0] fingerprint 算失败 ad=%s: %s",
  215. ad.get("adgroup_id"), e,
  216. )
  217. if len(ads) < 100:
  218. break
  219. page += 1
  220. except Exception as e:
  221. logger.warning(
  222. "[phase0] account=%d 拉 fingerprint 集失败(降级为不预校验): %s",
  223. account_id, e,
  224. )
  225. return set()
  226. logger.info(
  227. "[phase0] 已存在 fingerprint=%d 个(skip 缺字段广告 %d 条)",
  228. len(fingerprints), skipped,
  229. )
  230. return fingerprints
  231. def phase0_create_ads(
  232. target_ads: int = ADS_PER_ACCOUNT,
  233. config_date: date | None = None,
  234. only_account_ids: set[int] | None = None,
  235. ) -> list[dict]:
  236. """Phase 0:对每账户检查广告数,不足则建到 target_ads 条(模块 A,2026-06-09 P1-G)。
  237. 流程:
  238. account → 查当前广告数(NORMAL+SUSPEND 算占用)→ 不足 target_ads →
  239. → 反查 fingerprint 集 → enumerate candidates(排除已有 fp)→
  240. → 飞书审批 → approve 项 → 真 POST /adgroups/add → 返回新建的 adgroup_id 列表
  241. Returns: 本次新建成功的广告 list[dict],每项含 {account_id, adgroup_id, adgroup_name, wechat_position}
  242. """
  243. from tools.ad_api import _get
  244. from tools.im_approval_ad_creation import run_ad_approval_workflow
  245. creation_accounts = get_creation_account_ids(config_date=config_date)
  246. if not creation_accounts:
  247. logger.error("[phase0] 待投放账户配置为空,退出")
  248. return []
  249. creation_accounts = _filter_creation_accounts(
  250. creation_accounts, "phase0", only_account_ids
  251. )
  252. if not creation_accounts:
  253. logger.info("[phase0] 临时过滤后无待处理账户")
  254. return []
  255. # Task 26:NORMAL + SUSPEND 都算占用唯一性槽位(腾讯文档:删除前历史广告占槽位)
  256. OCCUPIED_STATUSES = {"AD_STATUS_NORMAL", "AD_STATUS_SUSPEND"}
  257. pending_ad_records: list[dict] = []
  258. for account_id in creation_accounts:
  259. logger.info("=" * 60)
  260. logger.info("[phase0] 账户 %d 处理开始", account_id)
  261. try:
  262. grant_result = ensure_account_audience_grant(account_id)
  263. logger.info(
  264. "[phase0] 人群包校验完成: status=%s audience_id=%s audience=%r",
  265. grant_result.get("status"),
  266. grant_result.get("audience_id"),
  267. grant_result.get("audience_name"),
  268. )
  269. except Exception as e:
  270. logger.exception("[phase0] account=%d 人群包授权/验证失败,跳过:%s", account_id, e)
  271. continue
  272. # Task 26:查所有非 DELETED 广告,NORMAL+SUSPEND 都算占用槽位
  273. try:
  274. r = _get("/adgroups/get", {
  275. "account_id": account_id, "page": 1, "page_size": 100,
  276. "fields": ["adgroup_id", "configured_status"],
  277. })
  278. all_ads = (r.get("data") or {}).get("list") or []
  279. normal_n = sum(1 for a in all_ads if a.get("configured_status") == "AD_STATUS_NORMAL")
  280. suspend_n = sum(1 for a in all_ads if a.get("configured_status") == "AD_STATUS_SUSPEND")
  281. occupied = [a for a in all_ads if a.get("configured_status") in OCCUPIED_STATUSES]
  282. logger.info(
  283. "[phase0] 广告数: 总 %d,NORMAL %d + SUSPEND %d = 占用 %d / 目标 %d",
  284. len(all_ads), normal_n, suspend_n, len(occupied), target_ads,
  285. )
  286. except Exception as e:
  287. logger.exception("[phase0] account=%d 查广告数失败: %s", account_id, e)
  288. continue
  289. to_create = max(0, target_ads - len(occupied))
  290. if to_create == 0:
  291. logger.info("[phase0] 占用槽位已满,跳过")
  292. continue
  293. # Task 27:fingerprint 预校验 — 反查现存广告的 fp 集合,enumerate 时排除
  294. existing_fps = _fetch_existing_fingerprints_for_account(account_id)
  295. # enumerate 候选(模块 A,差异化 wechat_position 有/无)
  296. try:
  297. candidates = enumerate_new_ad_candidates(
  298. account_id, count=to_create,
  299. existing_fingerprints=existing_fps,
  300. )
  301. except Exception as e:
  302. logger.exception("[phase0] account=%d enumerate 失败: %s", account_id, e)
  303. continue
  304. if not candidates:
  305. logger.warning("[phase0] enumerate 返回 0 条候选,跳过")
  306. continue
  307. today = now_in_timezone().strftime("%Y-%m-%d")
  308. for c in candidates:
  309. age_range = ",".join(
  310. f"{a.get('min')}-{a.get('max')}" for a in c.age
  311. )
  312. # 新建广告默认开启;实际投放仍受创意审核/账户/预算控制。
  313. # 这里显式传 NORMAL,避免未来默认值变化影响端到端创建语义。
  314. body = build_ad_request_body(c, configured_status="AD_STATUS_NORMAL")
  315. rec = {
  316. "approval_date": today,
  317. "account_id": c.account_id,
  318. "audience_tier_label": c.audience_tier_label,
  319. "adgroup_name": c.adgroup_name,
  320. "site_set": c.site_set,
  321. "delivery_version": c.delivery_version,
  322. "wechat_position": c.wechat_position,
  323. "bid_amount_fen": c.bid_amount_fen,
  324. "bid_scene": c.bid_scene,
  325. "custom_cost_cap_fen": c.custom_cost_cap_fen,
  326. "automatic_site_enabled": c.automatic_site_enabled,
  327. "age_range": age_range,
  328. "fingerprint": c.fingerprint,
  329. "_request_body": body,
  330. }
  331. pending_ad_records.append(rec)
  332. if not pending_ad_records:
  333. logger.info("[phase0] 无广告需新建,Phase 0 结束")
  334. return []
  335. # 飞书审批(如需要)
  336. data_dir = _HERE / "outputs" / "data"
  337. if CREATION_APPROVAL_REQUIRED:
  338. logger.info(
  339. "[phase0] CREATION_APPROVAL_REQUIRED=True → 飞书审批 %d 条广告候选",
  340. len(pending_ad_records),
  341. )
  342. sheet_meta, actions = run_ad_approval_workflow(
  343. records=pending_ad_records,
  344. xlsx_output_dir=data_dir,
  345. timeout_minutes=CREATION_APPROVAL_TIMEOUT_MINUTES,
  346. )
  347. for i, rec in enumerate(pending_ad_records, start=1):
  348. rec["action"] = actions.get(i, "skip")
  349. else:
  350. for rec in pending_ad_records:
  351. rec["action"] = "approve"
  352. # POST 真建 approve 项
  353. created: list[dict] = []
  354. for rec in pending_ad_records:
  355. if rec.get("action") != "approve":
  356. continue
  357. adgroup_id = post_ad_with_prepared_body(
  358. account_id=int(rec["account_id"]),
  359. body=rec["_request_body"],
  360. )
  361. if adgroup_id:
  362. created.append({
  363. "account_id": rec["account_id"],
  364. "adgroup_id": adgroup_id,
  365. "adgroup_name": rec["adgroup_name"],
  366. "wechat_position": rec.get("wechat_position"),
  367. })
  368. logger.info(
  369. "[phase0] 完成:候选 %d 条,approve %d 条,实际建 %d 条",
  370. len(pending_ad_records),
  371. sum(1 for r in pending_ad_records if r.get("action") == "approve"),
  372. len(created),
  373. )
  374. return created
  375. def _wait_created_ads_visible(created_ads: list[dict], timeout_seconds: int = 60) -> None:
  376. """等待刚创建的广告在 /adgroups/get 中可见。
  377. 腾讯 adgroups/add 成功后,list 接口存在短暂一致性延迟。若立刻进入 Phase 1,
  378. 可能只扫到部分新广告,导致同一轮只给一条广告补创意。
  379. """
  380. if not created_ads:
  381. return
  382. from tools.ad_api import _get
  383. by_account: dict[int, set[int]] = {}
  384. for ad in created_ads:
  385. by_account.setdefault(int(ad["account_id"]), set()).add(int(ad["adgroup_id"]))
  386. deadline = time.time() + timeout_seconds
  387. remaining = {acct: set(ids) for acct, ids in by_account.items()}
  388. while time.time() < deadline and any(remaining.values()):
  389. for account_id, ids in list(remaining.items()):
  390. if not ids:
  391. continue
  392. try:
  393. resp = _get("/adgroups/get", {
  394. "account_id": account_id,
  395. "page": 1,
  396. "page_size": 100,
  397. "fields": ["adgroup_id", "configured_status", "system_status"],
  398. })
  399. visible = {
  400. int(item["adgroup_id"])
  401. for item in ((resp.get("data") or {}).get("list") or [])
  402. if item.get("adgroup_id") is not None
  403. }
  404. ids.difference_update(visible)
  405. except Exception as e:
  406. logger.warning(
  407. "[phase0] 等待新广告可见失败 account=%d: %s",
  408. account_id, e,
  409. )
  410. if any(remaining.values()):
  411. logger.info("[phase0] 等待新广告在列表接口可见: %s", remaining)
  412. time.sleep(5)
  413. not_visible = {acct: sorted(ids) for acct, ids in remaining.items() if ids}
  414. if not_visible:
  415. logger.warning("[phase0] 部分新广告仍未在 list 可见,继续 Phase 1: %s", not_visible)
  416. else:
  417. logger.info("[phase0] 新建广告均已在 list 接口可见")
  418. def phase1_prepare(
  419. target_creatives: int = TARGET_CREATIVES_PER_AD,
  420. config_date: date | None = None,
  421. only_account_ids: set[int] | None = None,
  422. ) -> list[dict]:
  423. """Phase 1:扫账户 → 准备 pending records(不 POST 腾讯)。
  424. Returns:
  425. pending_records — 每个元素是 prepare_one_creative_for_ad 返回的 dict
  426. """
  427. creation_accounts = get_creation_account_ids(config_date=config_date)
  428. if not creation_accounts:
  429. logger.error("[phase1] 待投放账户配置为空,Phase 1 退出")
  430. return []
  431. creation_accounts = _filter_creation_accounts(
  432. creation_accounts, "phase1", only_account_ids
  433. )
  434. if not creation_accounts:
  435. logger.info("[phase1] 临时过滤后无待处理账户")
  436. return []
  437. pending_records: list[dict] = []
  438. # 素材排重只读取已有明确结果的 DB 记录。
  439. # pending/prepared 没有审批或投放结果,不进入排重。
  440. # 但同一轮审批候选需要做展示去重,避免表格里同 crowd_package 反复出现同一素材。
  441. # 这个 set 只存在内存里,不写历史排重库;进程结束即失效。
  442. display_used_material_ids_by_crowd: dict[str, set[str]] = {}
  443. # landing_video 按素材来源拆池。历史素材和 AI 生成素材互不占用,但池内都严格去重。
  444. landing_usage_counts_by_crowd: dict[str, dict[str, dict[int, int]]] = {}
  445. def accept_pending_record(
  446. rec: dict,
  447. *,
  448. crowd_package: str,
  449. display_excluded_material_ids: set[str],
  450. crowd_landing_counts: dict[str, dict[int, int]],
  451. landing_counts_for_ad: dict[int, int],
  452. landing_max_uses: int,
  453. recovered: bool = False,
  454. ) -> bool:
  455. material_id = rec.get("_material_id")
  456. if material_id and str(material_id) in display_excluded_material_ids:
  457. logger.info(
  458. "[phase1] pending 接收排重丢弃 crowd=%r material=%s%s",
  459. crowd_package, material_id,
  460. " recovered" if recovered else "",
  461. )
  462. return False
  463. landing_video_id = rec.get("landing_video_id")
  464. record_material_source = str(rec.get("material_source") or "")
  465. record_material_id = str(rec.get("_material_id") or "")
  466. if (
  467. record_material_source == MATERIAL_SOURCE_EXTERNAL_RECALL
  468. or record_material_id.startswith("external:")
  469. ):
  470. actual_material_source = MATERIAL_SOURCE_EXTERNAL_RECALL
  471. elif (
  472. record_material_source == MATERIAL_SOURCE_AI_GENERATED
  473. or record_material_id.startswith("ai:")
  474. ):
  475. actual_material_source = MATERIAL_SOURCE_AI_GENERATED
  476. else:
  477. actual_material_source = MATERIAL_SOURCE_HISTORY
  478. if landing_video_id is not None:
  479. landing_video_id = int(landing_video_id)
  480. if landing_counts_for_ad.get(landing_video_id, 0) >= MAX_SAME_LANDING_PER_AD_IN_RUN:
  481. logger.info(
  482. "[phase1] pending 接收排重丢弃 adgroup=%s landing=%d 同广告已达上限%s",
  483. rec.get("adgroup_id"), landing_video_id,
  484. " recovered" if recovered else "",
  485. )
  486. return False
  487. source_counts = crowd_landing_counts.setdefault(actual_material_source, {})
  488. if source_counts.get(landing_video_id, 0) >= landing_max_uses:
  489. logger.info(
  490. "[phase1] pending 接收排重丢弃 crowd=%r material_source=%s landing=%d 同人群包已达上限%s",
  491. crowd_package, actual_material_source, landing_video_id,
  492. " recovered" if recovered else "",
  493. )
  494. return False
  495. pending_records.append(rec)
  496. if material_id:
  497. display_excluded_material_ids.add(str(material_id))
  498. logger.info(
  499. "[phase1] 本轮展示去重登记 crowd=%r material=%s size=%d%s",
  500. crowd_package, material_id,
  501. len(display_excluded_material_ids),
  502. " recovered" if recovered else "",
  503. )
  504. if landing_video_id is not None:
  505. landing_counts_for_ad[landing_video_id] = (
  506. landing_counts_for_ad.get(landing_video_id, 0) + 1
  507. )
  508. source_counts = crowd_landing_counts.setdefault(actual_material_source, {})
  509. source_counts[landing_video_id] = source_counts.get(landing_video_id, 0) + 1
  510. logger.info(
  511. "[phase1] 同人群包 landing 使用登记 crowd=%r material_source=%s landing=%d count=%d max=%d%s",
  512. crowd_package, actual_material_source, landing_video_id,
  513. source_counts[landing_video_id],
  514. landing_max_uses,
  515. " recovered" if recovered else "",
  516. )
  517. logger.info(
  518. "[phase1] 本轮同广告 landing 计数 adgroup=%s landing=%d count=%d limit=%d%s",
  519. rec.get("adgroup_id"), landing_video_id,
  520. landing_counts_for_ad[landing_video_id],
  521. MAX_SAME_LANDING_PER_AD_IN_RUN,
  522. " recovered" if recovered else "",
  523. )
  524. if not recovered:
  525. try:
  526. record_prepared_material_usage(rec)
  527. except Exception as e:
  528. logger.warning(
  529. "[phase1] material usage 记录失败 account=%s adgroup=%s material=%s:%s",
  530. rec.get("account_id"), rec.get("adgroup_id"),
  531. rec.get("_material_id"), e,
  532. )
  533. return True
  534. for account_id in creation_accounts:
  535. logger.info("=" * 60)
  536. logger.info("[phase1] 账户 %d 处理开始", account_id)
  537. try:
  538. ads = find_ads_needing_creatives(account_id, min_creatives=target_creatives)
  539. except Exception as e:
  540. logger.exception("[phase1] account=%d find_ads 失败,跳过:%s", account_id, e)
  541. continue
  542. if not ads:
  543. logger.info("[phase1] account=%d 无广告需补创意", account_id)
  544. continue
  545. try:
  546. crowd_package = get_account_crowd_package(account_id)
  547. except Exception as e:
  548. logger.exception(
  549. "[phase1] account=%d 读取 crowd_package 失败,跳过:%s",
  550. account_id, e,
  551. )
  552. continue
  553. display_excluded_material_ids = display_used_material_ids_by_crowd.setdefault(
  554. crowd_package, set(),
  555. )
  556. if crowd_package not in landing_usage_counts_by_crowd:
  557. try:
  558. landing_usage_counts_by_crowd[crowd_package] = load_recent_landing_usage_counts(
  559. crowd_package,
  560. )
  561. except Exception as e:
  562. logger.warning(
  563. "[phase1] crowd=%r 读取 landing 使用历史失败,仅使用本轮排重:%s",
  564. crowd_package, e,
  565. )
  566. landing_usage_counts_by_crowd[crowd_package] = {
  567. MATERIAL_SOURCE_HISTORY: {},
  568. MATERIAL_SOURCE_AI_GENERATED: {},
  569. MATERIAL_SOURCE_EXTERNAL_RECALL: {},
  570. }
  571. logger.info(
  572. "[phase1] crowd=%r 近期 landing 使用 history=%d ai_generated=%d external_recall=%d",
  573. crowd_package,
  574. len(landing_usage_counts_by_crowd[crowd_package].get(MATERIAL_SOURCE_HISTORY, {})),
  575. len(landing_usage_counts_by_crowd[crowd_package].get(MATERIAL_SOURCE_AI_GENERATED, {})),
  576. len(landing_usage_counts_by_crowd[crowd_package].get(MATERIAL_SOURCE_EXTERNAL_RECALL, {})),
  577. )
  578. crowd_landing_counts = landing_usage_counts_by_crowd[crowd_package]
  579. crowd_landing_counts.setdefault(MATERIAL_SOURCE_HISTORY, {})
  580. crowd_landing_counts.setdefault(MATERIAL_SOURCE_AI_GENERATED, {})
  581. crowd_landing_counts.setdefault(MATERIAL_SOURCE_EXTERNAL_RECALL, {})
  582. try:
  583. material_strategy = load_account_material_strategy(account_id)
  584. except Exception as e:
  585. logger.warning(
  586. "[phase1] account=%d 读取素材策略失败,按 history 排重:%s",
  587. account_id, e,
  588. )
  589. material_strategy = None
  590. requested_material_source = (
  591. material_strategy.material_source
  592. if material_strategy is not None
  593. else MATERIAL_SOURCE_HISTORY
  594. )
  595. landing_max_uses = MAX_SAME_LANDING_PER_AD_IN_RUN
  596. landing_candidate_pool = None
  597. logger.info(
  598. "[phase1] account=%d landing 排重池=%s max_uses=%d",
  599. account_id, requested_material_source, landing_max_uses,
  600. )
  601. for ad in ads:
  602. adgroup_id = ad["adgroup_id"]
  603. already_have = ad["creative_count"]
  604. to_add = max(0, target_creatives - already_have)
  605. prepared_for_ad = 0
  606. failed_prepare_for_ad = 0
  607. failed_landing_ids_for_ad: set[int] = set()
  608. landing_counts_for_ad: dict[int, int] = {}
  609. logger.info(
  610. "[phase1] adgroup=%d(have=%d need=%d)",
  611. adgroup_id, already_have, to_add,
  612. )
  613. recovered_records = load_recoverable_prepared_records(
  614. account_id=account_id,
  615. adgroup_id=adgroup_id,
  616. crowd_package=crowd_package,
  617. material_source=requested_material_source,
  618. limit=to_add,
  619. )
  620. recovered_accepted = 0
  621. for rec in recovered_records:
  622. if accept_pending_record(
  623. rec,
  624. crowd_package=crowd_package,
  625. display_excluded_material_ids=display_excluded_material_ids,
  626. crowd_landing_counts=crowd_landing_counts,
  627. landing_counts_for_ad=landing_counts_for_ad,
  628. landing_max_uses=landing_max_uses,
  629. recovered=True,
  630. ):
  631. recovered_accepted += 1
  632. if recovered_records:
  633. prepared_for_ad += recovered_accepted
  634. logger.info(
  635. "[phase1] adgroup=%d 恢复未提交 prepared records=%d accepted=%d remaining=%d",
  636. adgroup_id, len(recovered_records), recovered_accepted,
  637. max(0, to_add - prepared_for_ad),
  638. )
  639. def prepare_attempt(
  640. *,
  641. attempt_no: int,
  642. excluded_material_ids_snapshot: set[str],
  643. excluded_landing_ids_snapshot: set[int],
  644. ) -> dict | None:
  645. nonlocal landing_candidate_pool
  646. try:
  647. if landing_candidate_pool is None:
  648. landing_candidate_pool = build_landing_candidate_pool(account_id)
  649. return prepare_one_creative_for_ad(
  650. account_id, adgroup_id,
  651. excluded_material_ids=excluded_material_ids_snapshot,
  652. excluded_landing_ids=excluded_landing_ids_snapshot,
  653. landing_candidates=landing_candidate_pool,
  654. failed_landing_ids=failed_landing_ids_for_ad,
  655. )
  656. except Exception as e:
  657. logger.exception(
  658. "[phase1] adgroup=%d attempt=%d 准备失败:%s",
  659. adgroup_id, attempt_no, e,
  660. )
  661. return None
  662. def current_excluded_landing_ids() -> set[int]:
  663. landing_excluded_for_ad = {
  664. vid
  665. for vid, count in landing_counts_for_ad.items()
  666. if count >= MAX_SAME_LANDING_PER_AD_IN_RUN
  667. }
  668. landing_excluded_for_source = {
  669. vid
  670. for vid, count in crowd_landing_counts
  671. .get(requested_material_source, {})
  672. .items()
  673. if count >= landing_max_uses
  674. }
  675. return landing_excluded_for_source | landing_excluded_for_ad
  676. remaining_to_add = max(0, to_add - prepared_for_ad)
  677. max_workers = max(1, int(CREATIVE_PREPARE_MAX_WORKERS))
  678. if max_workers > 1:
  679. logger.warning(
  680. "[phase1] CREATIVE_PREPARE_MAX_WORKERS=%d 已配置但当前 prepare_one_creative_for_ad "
  681. "包含图片上传/xcx-save 等外部副作用,为避免重复创建落地计划,本轮强制串行执行",
  682. max_workers,
  683. )
  684. max_workers = 1
  685. task_buffer = max(remaining_to_add, int(CREATIVE_PREPARE_TASK_BUFFER))
  686. task_limit = min(max(0, remaining_to_add * 2), task_buffer)
  687. if remaining_to_add > 0 and max_workers > 1:
  688. logger.info(
  689. "[phase1] adgroup=%d 并行准备 start remaining=%d workers=%d task_limit=%d",
  690. adgroup_id, remaining_to_add, max_workers, task_limit,
  691. )
  692. submitted = 0
  693. accepted_parallel = 0
  694. dropped_parallel = 0
  695. failed_parallel = 0
  696. started_at = time.monotonic()
  697. with ThreadPoolExecutor(max_workers=max_workers) as executor:
  698. futures = {}
  699. def submit_next() -> None:
  700. nonlocal submitted
  701. # 已接收 + 运行中任务不超过缺口,避免并发准备产出无法追踪的多余副作用。
  702. if (
  703. submitted >= task_limit
  704. or prepared_for_ad + len(futures) >= to_add
  705. ):
  706. return
  707. submitted += 1
  708. attempt_no = submitted
  709. future = executor.submit(
  710. prepare_attempt,
  711. attempt_no=attempt_no,
  712. excluded_material_ids_snapshot=set(display_excluded_material_ids),
  713. excluded_landing_ids_snapshot=current_excluded_landing_ids(),
  714. )
  715. futures[future] = attempt_no
  716. for _ in range(min(max_workers, task_limit)):
  717. submit_next()
  718. while futures and prepared_for_ad < to_add:
  719. done, _ = wait(futures, return_when=FIRST_COMPLETED)
  720. for future in done:
  721. futures.pop(future, None)
  722. rec = future.result()
  723. if not rec:
  724. failed_prepare_for_ad += 1
  725. failed_parallel += 1
  726. elif accept_pending_record(
  727. rec,
  728. crowd_package=crowd_package,
  729. display_excluded_material_ids=display_excluded_material_ids,
  730. crowd_landing_counts=crowd_landing_counts,
  731. landing_counts_for_ad=landing_counts_for_ad,
  732. landing_max_uses=landing_max_uses,
  733. ):
  734. prepared_for_ad += 1
  735. accepted_parallel += 1
  736. else:
  737. dropped_parallel += 1
  738. if prepared_for_ad < to_add:
  739. submit_next()
  740. else:
  741. break
  742. if prepared_for_ad >= to_add:
  743. logger.info(
  744. "[phase1] adgroup=%d 并行准备 target reached accepted=%d",
  745. adgroup_id, accepted_parallel,
  746. )
  747. cancelled = 0
  748. for pending_future in futures:
  749. if pending_future.cancel():
  750. cancelled += 1
  751. if cancelled:
  752. logger.info(
  753. "[phase1] adgroup=%d 并行准备 cancelled_pending=%d",
  754. adgroup_id, cancelled,
  755. )
  756. for future in futures:
  757. if future.cancelled():
  758. continue
  759. rec = future.result()
  760. if rec:
  761. dropped_parallel += 1
  762. logger.info(
  763. "[phase1] adgroup=%d 并行结果丢弃:target reached material=%s landing=%s",
  764. adgroup_id, rec.get("_material_id"), rec.get("landing_video_id"),
  765. )
  766. logger.info(
  767. "[phase1] adgroup=%d 并行准备 done submitted=%d accepted=%d dropped=%d failed=%d elapsed=%.1fs",
  768. adgroup_id, submitted, accepted_parallel, dropped_parallel,
  769. failed_parallel, time.monotonic() - started_at,
  770. )
  771. else:
  772. for attempt_no in range(1, remaining_to_add + 1):
  773. rec = prepare_attempt(
  774. attempt_no=attempt_no,
  775. excluded_material_ids_snapshot=set(display_excluded_material_ids),
  776. excluded_landing_ids_snapshot=current_excluded_landing_ids(),
  777. )
  778. if rec:
  779. if accept_pending_record(
  780. rec,
  781. crowd_package=crowd_package,
  782. display_excluded_material_ids=display_excluded_material_ids,
  783. crowd_landing_counts=crowd_landing_counts,
  784. landing_counts_for_ad=landing_counts_for_ad,
  785. landing_max_uses=landing_max_uses,
  786. ):
  787. prepared_for_ad += 1
  788. else:
  789. failed_prepare_for_ad += 1
  790. # 2026-06-10 用户要求:单条 prepare 失败 → continue 不 break
  791. # 同广告剩余 to_add 创意还能继续试,不被一次失败拖累
  792. logger.info(
  793. "[phase1] adgroup=%d 本条创意 prepare 失败,试下一条",
  794. adgroup_id,
  795. )
  796. logger.info(
  797. "[phase1] adgroup=%d 补创意完成 target=%d have_before=%d "
  798. "planned=%d prepared=%d failed_prepare=%d",
  799. adgroup_id, target_creatives, already_have, to_add,
  800. prepared_for_ad, failed_prepare_for_ad,
  801. )
  802. if prepared_for_ad < to_add:
  803. logger.warning(
  804. "[phase1] adgroup=%d 未补满:缺口=%d。详细原因见上方 "
  805. "prepare_one_creative source_summary/失败日志",
  806. adgroup_id, to_add - prepared_for_ad,
  807. )
  808. logger.info("=" * 60)
  809. logger.info("[phase1] 准备完成,共 %d 条 pending records", len(pending_records))
  810. return pending_records
  811. def _write_pending_records(records: list[dict], output_dir: Path) -> Path:
  812. """落 Phase 1 产物 JSON(供 Phase 3 独立 apply 用 / 追溯)。"""
  813. output_dir.mkdir(parents=True, exist_ok=True)
  814. now = now_in_timezone()
  815. date_str = now.strftime("%Y%m%d")
  816. ts = now.strftime("%H%M%S")
  817. out_path = output_dir / f"creation_pending_{date_str}_{ts}.json"
  818. with open(out_path, "w", encoding="utf-8") as f:
  819. json.dump(records, f, ensure_ascii=False, indent=2)
  820. return out_path
  821. def phase2_approval(records: list[dict], xlsx_output_dir: Path) -> dict[int, str]:
  822. """Phase 2:飞书审批(若需要)。
  823. Returns: actions {row_idx: action}, 1-based row_idx
  824. """
  825. from tools.im_approval_creation import run_approval_workflow
  826. sheet_meta, actions = run_approval_workflow(
  827. records=records,
  828. xlsx_output_dir=xlsx_output_dir,
  829. timeout_minutes=CREATION_APPROVAL_TIMEOUT_MINUTES,
  830. )
  831. logger.info(
  832. "[phase2] 审批完成 sheet_url=%s actions=%d/%d",
  833. sheet_meta.get("url"), len(actions), len(records),
  834. )
  835. return actions
  836. def run_once(
  837. config_date: str | None = None,
  838. account_ids: list[int] | None = None,
  839. ) -> dict:
  840. """完整主循环:Phase 0 → Phase 1 → Phase 2(若开关 True)→ Phase 3。"""
  841. run_started = now_in_timezone().isoformat()
  842. sync_stats: dict = {}
  843. requested_accounts = _normalize_requested_account_ids(account_ids)
  844. # Phase -1:每日主流程启动时先同步飞书「自动化账户」配置表。
  845. logger.info("=" * 60)
  846. logger.info("[main] Phase -1 启动 — 同步飞书自动化账户配置")
  847. try:
  848. from sync_feishu_account_config import sync_from_feishu
  849. sync_stats = sync_from_feishu(
  850. config_date=config_date,
  851. account_ids=requested_accounts,
  852. )
  853. logger.info("[main] Phase -1 完成: %s", sync_stats)
  854. except Exception as e:
  855. logger.exception("[main] Phase -1 同步飞书配置失败,本轮停止:%s", e)
  856. return {
  857. "run_started": run_started,
  858. "run_finished": now_in_timezone().isoformat(),
  859. "approval_required": CREATION_APPROVAL_REQUIRED,
  860. "sync_stats": {"error": str(e)},
  861. "phase0_created_ads": 0,
  862. "total": {"phase1_prepared": 0},
  863. }
  864. effective_config_date = (
  865. date.fromisoformat(sync_stats["config_date"])
  866. if sync_stats.get("config_date") else None
  867. )
  868. eligible_accounts = get_creation_account_ids(config_date=effective_config_date)
  869. _validate_requested_accounts(
  870. requested_accounts,
  871. eligible_accounts,
  872. effective_config_date,
  873. )
  874. # Phase 0:模块 A 建广告(满足每账户 ADS_PER_ACCOUNT 条)
  875. logger.info("=" * 60)
  876. if _env_flag("CREATION_SKIP_PHASE0"):
  877. logger.info("[main] CREATION_SKIP_PHASE0=True → 跳过模块 A 建广告")
  878. created_ads = []
  879. else:
  880. logger.info("[main] Phase 0 启动 — 模块 A 检查 + 建广告")
  881. created_ads = phase0_create_ads(
  882. config_date=effective_config_date,
  883. only_account_ids=requested_accounts,
  884. )
  885. logger.info("[main] Phase 0 完成:本轮新建广告 %d 条", len(created_ads))
  886. _wait_created_ads_visible(created_ads)
  887. # Phase 1:模块 B 给所有广告(新+旧)补创意
  888. logger.info("=" * 60)
  889. logger.info("[main] Phase 1 启动 — 模块 B 给广告补创意")
  890. pending_records = phase1_prepare(
  891. config_date=effective_config_date,
  892. only_account_ids=requested_accounts,
  893. )
  894. if not pending_records:
  895. logger.info("[main] Phase 1 无 pending records,主循环退出")
  896. return {
  897. "run_started": run_started,
  898. "run_finished": now_in_timezone().isoformat(),
  899. "approval_required": CREATION_APPROVAL_REQUIRED,
  900. "sync_stats": sync_stats,
  901. "phase0_created_ads": len(created_ads),
  902. "total": {"phase1_prepared": 0},
  903. }
  904. data_dir = _HERE / "outputs" / "data"
  905. pending_path = _write_pending_records(pending_records, data_dir)
  906. logger.info("[main] pending records 已写入: %s", pending_path)
  907. # Phase 2:审批(或 skip)
  908. if CREATION_APPROVAL_REQUIRED:
  909. logger.info("[main] CREATION_APPROVAL_REQUIRED=True → 进 Phase 2 飞书审批")
  910. actions = phase2_approval(pending_records, data_dir)
  911. for i, rec in enumerate(pending_records, start=1):
  912. rec["action"] = actions.get(i, "skip")
  913. else:
  914. logger.info("[main] CREATION_APPROVAL_REQUIRED=False → 全 records approve")
  915. for rec in pending_records:
  916. rec["action"] = "approve"
  917. # Phase 3:执行
  918. logger.info("=" * 60)
  919. logger.info("[main] Phase 3 执行启动")
  920. summary = apply_pending_records(pending_records)
  921. summary["approval_required"] = CREATION_APPROVAL_REQUIRED
  922. summary["pending_records_path"] = str(pending_path)
  923. summary_path = write_apply_summary(summary, data_dir)
  924. logger.info("[main] summary 已写入: %s", summary_path)
  925. # 发执行汇报
  926. _send_apply_summary_to_feishu(summary)
  927. summary["run_started"] = run_started
  928. summary["run_finished"] = now_in_timezone().isoformat()
  929. summary["sync_stats"] = sync_stats
  930. summary["phase0_created_ads"] = len(created_ads)
  931. summary["phase0_ads"] = created_ads
  932. return summary
  933. def main() -> int:
  934. parser = argparse.ArgumentParser(description="执行广告与创意创建主流程")
  935. parser.add_argument(
  936. "--config-date",
  937. help="历史复现时限定飞书配置日期;默认每账户使用日期最新的一行",
  938. )
  939. parser.add_argument(
  940. "--account-id",
  941. action="append",
  942. type=_parse_account_id_arg,
  943. dest="account_ids",
  944. help="只处理指定账户;可重复传入。账户必须在该配置日期启用",
  945. )
  946. args = parser.parse_args()
  947. """模块 B 创意搭建子系统主入口:初始化日志 → 运行主循环 → 输出汇总。"""
  948. _setup_logging()
  949. logger.info("=" * 60)
  950. logger.info("模块 B 创意搭建子系统 — 主循环启动")
  951. logger.info("TARGET_CREATIVES_PER_AD = %d", TARGET_CREATIVES_PER_AD)
  952. logger.info("CREATION_APPROVAL_REQUIRED = %s", CREATION_APPROVAL_REQUIRED)
  953. logger.info("WHITELIST_ACCOUNTS = %s", WHITELIST_ACCOUNTS)
  954. logger.info("=" * 60)
  955. started = time.monotonic()
  956. logger.info(
  957. "event=creation_run stage=run_once result=started config_date=%s",
  958. args.config_date or "today",
  959. )
  960. try:
  961. summary = run_once(
  962. config_date=args.config_date,
  963. account_ids=args.account_ids,
  964. )
  965. except ValueError as exc:
  966. parser.error(str(exc))
  967. except Exception:
  968. logger.exception(
  969. "event=creation_run stage=run_once result=failed config_date=%s "
  970. "duration_ms=%d",
  971. args.config_date or "today",
  972. int((time.monotonic() - started) * 1000),
  973. )
  974. raise
  975. if (summary.get("sync_stats") or {}).get("error"):
  976. return 1
  977. t = summary.get("total") or {}
  978. logger.info("=" * 60)
  979. logger.info("[main] 主循环结束")
  980. if "phase1_prepared" in t:
  981. logger.info(" phase1_prepared = %d", t["phase1_prepared"])
  982. else:
  983. logger.info(" records = %d", t.get("records", 0))
  984. logger.info(" approved = %d", t.get("approved", 0))
  985. logger.info(" posted_ok = %d", t.get("posted_ok", 0))
  986. logger.info(" posted_failed = %d", t.get("posted_failed", 0))
  987. logger.info("=" * 60)
  988. logger.info(
  989. "event=creation_run stage=run_once result=%s duration_ms=%d "
  990. "prepared=%d posted_ok=%d posted_failed=%d",
  991. "succeeded" if t.get("posted_failed", 0) == 0 else "failed",
  992. int((time.monotonic() - started) * 1000),
  993. t.get("phase1_prepared", t.get("records", 0)),
  994. t.get("posted_ok", 0),
  995. t.get("posted_failed", 0),
  996. )
  997. return 0 if t.get("posted_failed", 0) == 0 else 1
  998. if __name__ == "__main__":
  999. sys.exit(main())