execute_creation_once.py 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981
  1. """模块 B 创意搭建子系统 — 主循环入口(P0-A / P0-C / P0-E,2026-06-09)。
  2. 数据流(三阶段,先审后挂):
  3. Phase 1 — 准备:
  4. 扫账户 → 关联点过滤(读调控当日 latest_decisions 排除 pause)
  5. → find_ads_needing_creatives
  6. → 对每条广告 prepare_one_creative_for_ad(召回 + 上传图 + xcx/save + build body)
  7. → pending_records (含完整 Phase 3 用的 _request_body)
  8. → 写 outputs/data/creation_pending_{date}.json(追溯 + 独立 apply 用)
  9. Phase 2 — 审批(若 CREATION_APPROVAL_REQUIRED=True):
  10. run_approval_workflow 生成 20 列 xlsx → 上传飞书 sheet → 发链接 → 轮询读决策列
  11. actions {row_idx: approve/reject/hold}
  12. → 写回 records["action"]
  13. Phase 3 — 执行:
  14. 对 action=approve 的 → POST /dynamic_creatives/add
  15. → 写 outputs/data/creation_run_{date}.json
  16. → 发飞书"执行汇报"消息
  17. 开关 CREATION_APPROVAL_REQUIRED=False 时跳过 Phase 2,全 records 直接 approve。
  18. """
  19. import json
  20. import logging
  21. import os
  22. import sys
  23. import time
  24. from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
  25. from pathlib import Path
  26. _HERE = Path(__file__).parent
  27. sys.path.insert(0, str(_HERE.parent.parent))
  28. sys.path.insert(0, str(_HERE))
  29. from dotenv import load_dotenv # noqa: E402
  30. load_dotenv(_HERE / ".env")
  31. from config import ( # noqa: E402
  32. ADS_PER_ACCOUNT,
  33. CREATION_APPROVAL_REQUIRED,
  34. CREATION_APPROVAL_TIMEOUT_MINUTES,
  35. CREATIVE_PREPARE_MAX_WORKERS,
  36. CREATIVE_PREPARE_TASK_BUFFER,
  37. MAX_SAME_LANDING_PER_AD_IN_RUN,
  38. TARGET_CREATIVES_PER_AD,
  39. WHITELIST_ACCOUNTS,
  40. get_creation_account_ids,
  41. now_in_timezone,
  42. )
  43. # config import 副作用会改 sys.path(把 im-client / agent/tools/builtin 顶到前面),
  44. # 强占 sys.path[0],只影响本入口自己的 import resolution。
  45. while str(_HERE) in sys.path:
  46. sys.path.remove(str(_HERE))
  47. sys.path.insert(0, str(_HERE))
  48. from tools.ad_creation import ( # noqa: E402
  49. build_ad_request_body,
  50. compute_fingerprint,
  51. enumerate_new_ad_candidates,
  52. post_ad_with_prepared_body,
  53. )
  54. from tools.audience_grant import ensure_account_audience_grant # noqa: E402
  55. from tools.creative_creation import ( # noqa: E402
  56. find_ads_needing_creatives,
  57. load_excluded_ad_ids_from_adjustment,
  58. prepare_one_creative_for_ad,
  59. )
  60. from tools.creative_material_usage import ( # noqa: E402
  61. load_recent_landing_usage_counts,
  62. load_recoverable_prepared_records,
  63. record_prepared_material_usage,
  64. )
  65. from tools.account_material_strategy import ( # noqa: E402
  66. MATERIAL_SOURCE_AI_GENERATED,
  67. load_account_material_strategy,
  68. )
  69. from tools.video_recall import get_account_crowd_package # noqa: E402
  70. from execute_creation_apply import ( # noqa: E402
  71. apply_pending_records,
  72. write_summary as write_apply_summary,
  73. _send_apply_summary_to_feishu,
  74. )
  75. logger = logging.getLogger("execute_creation_once")
  76. MATERIAL_SOURCE_HISTORY = "history"
  77. def _env_flag(name: str, default: bool = False) -> bool:
  78. raw = os.getenv(name)
  79. if raw is None:
  80. return default
  81. return raw.strip().lower() in {"1", "true", "yes", "y", "on"}
  82. def _env_csv_set(name: str) -> set[str]:
  83. return {
  84. item.strip()
  85. for item in os.getenv(name, "").split(",")
  86. if item.strip()
  87. }
  88. def _filter_creation_accounts(accounts: list[int], phase: str) -> list[int]:
  89. """Apply one-off run filters. Empty env means production default: no filter."""
  90. account_filter = {
  91. int(v)
  92. for v in _env_csv_set("CREATION_ONLY_ACCOUNT_IDS")
  93. if v.isdigit()
  94. }
  95. crowd_filter = _env_csv_set("CREATION_ONLY_CROWD_PACKAGES")
  96. if not account_filter and not crowd_filter:
  97. return accounts
  98. selected: list[int] = []
  99. for account_id in accounts:
  100. if account_filter and account_id not in account_filter:
  101. continue
  102. if crowd_filter:
  103. try:
  104. crowd_package = get_account_crowd_package(account_id)
  105. except Exception as e:
  106. logger.warning(
  107. "[%s] account=%d 临时过滤读取 crowd_package 失败,跳过:%s",
  108. phase, account_id, e,
  109. )
  110. continue
  111. if crowd_package not in crowd_filter:
  112. continue
  113. selected.append(account_id)
  114. logger.info(
  115. "[%s] 临时运行过滤 account_ids=%s crowd_packages=%s: %d -> %d",
  116. phase,
  117. sorted(account_filter) if account_filter else "ALL",
  118. sorted(crowd_filter) if crowd_filter else "ALL",
  119. len(accounts),
  120. len(selected),
  121. )
  122. return selected
  123. def _setup_logging() -> None:
  124. logging.basicConfig(
  125. level=logging.INFO,
  126. format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
  127. datefmt="%H:%M:%S",
  128. )
  129. for noisy in ("httpx", "httpcore", "config", "db.config"):
  130. logging.getLogger(noisy).setLevel(logging.WARNING)
  131. # SLS 上报(2026-06-11 接入,K8s pod 直发)— 配置缺失自动降级,不阻塞主链路
  132. try:
  133. from tools.sls_setup import attach_sls_handler
  134. attach_sls_handler()
  135. except Exception as e:
  136. logger.warning("[sls] 挂载异常(降级为本地 only):%s", e)
  137. def _fetch_existing_fingerprints_for_account(account_id: int) -> set[str]:
  138. """Task 27:拉账户下所有非 DELETED 广告 → 反推 fingerprint 集合(预校验唯一性)。
  139. 腾讯文档:NORMAL + SUSPEND 状态广告均占用唯一性槽位,DENIED 也算占用。
  140. list 接口默认返回所有非 DELETED 状态 ad,故无需显式 filter status。
  141. 若腾讯 list 接口某些字段不返回(field 实测差异),降级:跳过该条 + log 警告。
  142. 返回空集 → 上层退化为"不预校验",enumerate 仍能跑(原有行为)。
  143. """
  144. from tools.ad_api import _get
  145. fingerprints: set[str] = set()
  146. skipped = 0
  147. page = 1
  148. try:
  149. while True:
  150. r = _get("/adgroups/get", {
  151. "account_id": account_id, "page": page, "page_size": 100,
  152. "fields": [
  153. "adgroup_id", "configured_status", "site_set",
  154. "targeting", "scene_spec",
  155. ],
  156. })
  157. ads = (r.get("data") or {}).get("list") or []
  158. if not ads:
  159. break
  160. for ad in ads:
  161. tgt = ad.get("targeting") or {}
  162. sc = ad.get("scene_spec") or {}
  163. wp = sc.get("wechat_position") if sc else None
  164. site_set = ad.get("site_set") or []
  165. age = tgt.get("age") or []
  166. geo_regions = (tgt.get("geo_location") or {}).get("regions") or []
  167. custom_audience = tgt.get("custom_audience")
  168. if not site_set or not age:
  169. skipped += 1
  170. continue
  171. try:
  172. fp = compute_fingerprint(
  173. account_id=account_id,
  174. site_set=site_set,
  175. custom_audience=custom_audience,
  176. age=age,
  177. geo_regions=geo_regions,
  178. wechat_position=wp,
  179. )
  180. fingerprints.add(fp)
  181. except Exception as e:
  182. skipped += 1
  183. logger.debug(
  184. "[phase0] fingerprint 算失败 ad=%s: %s",
  185. ad.get("adgroup_id"), e,
  186. )
  187. if len(ads) < 100:
  188. break
  189. page += 1
  190. except Exception as e:
  191. logger.warning(
  192. "[phase0] account=%d 拉 fingerprint 集失败(降级为不预校验): %s",
  193. account_id, e,
  194. )
  195. return set()
  196. logger.info(
  197. "[phase0] 已存在 fingerprint=%d 个(skip 缺字段广告 %d 条)",
  198. len(fingerprints), skipped,
  199. )
  200. return fingerprints
  201. def phase0_create_ads(target_ads: int = ADS_PER_ACCOUNT) -> list[dict]:
  202. """Phase 0:对每账户检查广告数,不足则建到 target_ads 条(模块 A,2026-06-09 P1-G)。
  203. 流程:
  204. account → 查当前广告数(NORMAL+SUSPEND 算占用)→ 不足 target_ads →
  205. → 反查 fingerprint 集 → enumerate candidates(排除已有 fp)→
  206. → 飞书审批 → approve 项 → 真 POST /adgroups/add → 返回新建的 adgroup_id 列表
  207. Returns: 本次新建成功的广告 list[dict],每项含 {account_id, adgroup_id, adgroup_name, wechat_position}
  208. """
  209. from tools.ad_api import _get
  210. from tools.im_approval_ad_creation import run_ad_approval_workflow
  211. creation_accounts = get_creation_account_ids()
  212. if not creation_accounts:
  213. logger.error("[phase0] 待投放账户配置为空,退出")
  214. return []
  215. creation_accounts = _filter_creation_accounts(creation_accounts, "phase0")
  216. if not creation_accounts:
  217. logger.info("[phase0] 临时过滤后无待处理账户")
  218. return []
  219. # Task 26:NORMAL + SUSPEND 都算占用唯一性槽位(腾讯文档:删除前历史广告占槽位)
  220. OCCUPIED_STATUSES = {"AD_STATUS_NORMAL", "AD_STATUS_SUSPEND"}
  221. pending_ad_records: list[dict] = []
  222. for account_id in creation_accounts:
  223. logger.info("=" * 60)
  224. logger.info("[phase0] 账户 %d 处理开始", account_id)
  225. try:
  226. grant_result = ensure_account_audience_grant(account_id)
  227. logger.info(
  228. "[phase0] 人群包校验完成: status=%s audience_id=%s audience=%r",
  229. grant_result.get("status"),
  230. grant_result.get("audience_id"),
  231. grant_result.get("audience_name"),
  232. )
  233. except Exception as e:
  234. logger.exception("[phase0] account=%d 人群包授权/验证失败,跳过:%s", account_id, e)
  235. continue
  236. # Task 26:查所有非 DELETED 广告,NORMAL+SUSPEND 都算占用槽位
  237. try:
  238. r = _get("/adgroups/get", {
  239. "account_id": account_id, "page": 1, "page_size": 100,
  240. "fields": ["adgroup_id", "configured_status"],
  241. })
  242. all_ads = (r.get("data") or {}).get("list") or []
  243. normal_n = sum(1 for a in all_ads if a.get("configured_status") == "AD_STATUS_NORMAL")
  244. suspend_n = sum(1 for a in all_ads if a.get("configured_status") == "AD_STATUS_SUSPEND")
  245. occupied = [a for a in all_ads if a.get("configured_status") in OCCUPIED_STATUSES]
  246. logger.info(
  247. "[phase0] 广告数: 总 %d,NORMAL %d + SUSPEND %d = 占用 %d / 目标 %d",
  248. len(all_ads), normal_n, suspend_n, len(occupied), target_ads,
  249. )
  250. except Exception as e:
  251. logger.exception("[phase0] account=%d 查广告数失败: %s", account_id, e)
  252. continue
  253. to_create = max(0, target_ads - len(occupied))
  254. if to_create == 0:
  255. logger.info("[phase0] 占用槽位已满,跳过")
  256. continue
  257. # Task 27:fingerprint 预校验 — 反查现存广告的 fp 集合,enumerate 时排除
  258. existing_fps = _fetch_existing_fingerprints_for_account(account_id)
  259. # enumerate 候选(模块 A,差异化 wechat_position 有/无)
  260. try:
  261. candidates = enumerate_new_ad_candidates(
  262. account_id, count=to_create,
  263. existing_fingerprints=existing_fps,
  264. )
  265. except Exception as e:
  266. logger.exception("[phase0] account=%d enumerate 失败: %s", account_id, e)
  267. continue
  268. if not candidates:
  269. logger.warning("[phase0] enumerate 返回 0 条候选,跳过")
  270. continue
  271. today = now_in_timezone().strftime("%Y-%m-%d")
  272. for c in candidates:
  273. age_range = ",".join(
  274. f"{a.get('min')}-{a.get('max')}" for a in c.age
  275. )
  276. # 新建广告默认开启;实际投放仍受创意审核/账户/预算控制。
  277. # 这里显式传 NORMAL,避免未来默认值变化影响端到端创建语义。
  278. body = build_ad_request_body(c, configured_status="AD_STATUS_NORMAL")
  279. rec = {
  280. "approval_date": today,
  281. "account_id": c.account_id,
  282. "audience_tier_label": c.audience_tier_label,
  283. "adgroup_name": c.adgroup_name,
  284. "site_set": c.site_set,
  285. "delivery_version": c.delivery_version,
  286. "wechat_position": c.wechat_position,
  287. "bid_amount_fen": c.bid_amount_fen,
  288. "bid_scene": c.bid_scene,
  289. "custom_cost_cap_fen": c.custom_cost_cap_fen,
  290. "automatic_site_enabled": c.automatic_site_enabled,
  291. "age_range": age_range,
  292. "fingerprint": c.fingerprint,
  293. "_request_body": body,
  294. }
  295. pending_ad_records.append(rec)
  296. if not pending_ad_records:
  297. logger.info("[phase0] 无广告需新建,Phase 0 结束")
  298. return []
  299. # 飞书审批(如需要)
  300. data_dir = _HERE / "outputs" / "data"
  301. if CREATION_APPROVAL_REQUIRED:
  302. logger.info(
  303. "[phase0] CREATION_APPROVAL_REQUIRED=True → 飞书审批 %d 条广告候选",
  304. len(pending_ad_records),
  305. )
  306. sheet_meta, actions = run_ad_approval_workflow(
  307. records=pending_ad_records,
  308. xlsx_output_dir=data_dir,
  309. timeout_minutes=CREATION_APPROVAL_TIMEOUT_MINUTES,
  310. )
  311. for i, rec in enumerate(pending_ad_records, start=1):
  312. rec["action"] = actions.get(i, "skip")
  313. else:
  314. for rec in pending_ad_records:
  315. rec["action"] = "approve"
  316. # POST 真建 approve 项
  317. created: list[dict] = []
  318. for rec in pending_ad_records:
  319. if rec.get("action") != "approve":
  320. continue
  321. adgroup_id = post_ad_with_prepared_body(
  322. account_id=int(rec["account_id"]),
  323. body=rec["_request_body"],
  324. )
  325. if adgroup_id:
  326. created.append({
  327. "account_id": rec["account_id"],
  328. "adgroup_id": adgroup_id,
  329. "adgroup_name": rec["adgroup_name"],
  330. "wechat_position": rec.get("wechat_position"),
  331. })
  332. logger.info(
  333. "[phase0] 完成:候选 %d 条,approve %d 条,实际建 %d 条",
  334. len(pending_ad_records),
  335. sum(1 for r in pending_ad_records if r.get("action") == "approve"),
  336. len(created),
  337. )
  338. return created
  339. def _wait_created_ads_visible(created_ads: list[dict], timeout_seconds: int = 60) -> None:
  340. """等待刚创建的广告在 /adgroups/get 中可见。
  341. 腾讯 adgroups/add 成功后,list 接口存在短暂一致性延迟。若立刻进入 Phase 1,
  342. 可能只扫到部分新广告,导致同一轮只给一条广告补创意。
  343. """
  344. if not created_ads:
  345. return
  346. from tools.ad_api import _get
  347. by_account: dict[int, set[int]] = {}
  348. for ad in created_ads:
  349. by_account.setdefault(int(ad["account_id"]), set()).add(int(ad["adgroup_id"]))
  350. deadline = time.time() + timeout_seconds
  351. remaining = {acct: set(ids) for acct, ids in by_account.items()}
  352. while time.time() < deadline and any(remaining.values()):
  353. for account_id, ids in list(remaining.items()):
  354. if not ids:
  355. continue
  356. try:
  357. resp = _get("/adgroups/get", {
  358. "account_id": account_id,
  359. "page": 1,
  360. "page_size": 100,
  361. "fields": ["adgroup_id", "configured_status", "system_status"],
  362. })
  363. visible = {
  364. int(item["adgroup_id"])
  365. for item in ((resp.get("data") or {}).get("list") or [])
  366. if item.get("adgroup_id") is not None
  367. }
  368. ids.difference_update(visible)
  369. except Exception as e:
  370. logger.warning(
  371. "[phase0] 等待新广告可见失败 account=%d: %s",
  372. account_id, e,
  373. )
  374. if any(remaining.values()):
  375. logger.info("[phase0] 等待新广告在列表接口可见: %s", remaining)
  376. time.sleep(5)
  377. not_visible = {acct: sorted(ids) for acct, ids in remaining.items() if ids}
  378. if not_visible:
  379. logger.warning("[phase0] 部分新广告仍未在 list 可见,继续 Phase 1: %s", not_visible)
  380. else:
  381. logger.info("[phase0] 新建广告均已在 list 接口可见")
  382. def phase1_prepare(target_creatives: int = TARGET_CREATIVES_PER_AD) -> list[dict]:
  383. """Phase 1:扫账户 → 关联点过滤 → 准备 pending records(不 POST 腾讯)。
  384. Returns:
  385. pending_records — 每个元素是 prepare_one_creative_for_ad 返回的 dict
  386. """
  387. creation_accounts = get_creation_account_ids()
  388. if not creation_accounts:
  389. logger.error("[phase1] 待投放账户配置为空,Phase 1 退出")
  390. return []
  391. creation_accounts = _filter_creation_accounts(creation_accounts, "phase1")
  392. if not creation_accounts:
  393. logger.info("[phase1] 临时过滤后无待处理账户")
  394. return []
  395. excluded_ad_ids = load_excluded_ad_ids_from_adjustment()
  396. logger.info("[phase1] 关联点过滤集合 size=%d", len(excluded_ad_ids))
  397. pending_records: list[dict] = []
  398. # 素材排重只读取已有明确结果的 DB 记录。
  399. # pending/prepared 没有审批或投放结果,不进入排重。
  400. # 但同一轮审批候选需要做展示去重,避免表格里同 crowd_package 反复出现同一素材。
  401. # 这个 set 只存在内存里,不写历史排重库;进程结束即失效。
  402. display_used_material_ids_by_crowd: dict[str, set[str]] = {}
  403. # landing_video 按素材来源拆池。历史素材和 AI 生成素材互不占用,但池内都严格去重。
  404. landing_usage_counts_by_crowd: dict[str, dict[str, dict[int, int]]] = {}
  405. def accept_pending_record(
  406. rec: dict,
  407. *,
  408. crowd_package: str,
  409. display_excluded_material_ids: set[str],
  410. crowd_landing_counts: dict[str, dict[int, int]],
  411. landing_counts_for_ad: dict[int, int],
  412. landing_max_uses: int,
  413. recovered: bool = False,
  414. ) -> bool:
  415. material_id = rec.get("_material_id")
  416. if material_id and str(material_id) in display_excluded_material_ids:
  417. logger.info(
  418. "[phase1] pending 接收排重丢弃 crowd=%r material=%s%s",
  419. crowd_package, material_id,
  420. " recovered" if recovered else "",
  421. )
  422. return False
  423. landing_video_id = rec.get("landing_video_id")
  424. actual_material_source = (
  425. MATERIAL_SOURCE_AI_GENERATED
  426. if str(rec.get("material_source") or "") == MATERIAL_SOURCE_AI_GENERATED
  427. or str(rec.get("_material_id") or "").startswith("ai:")
  428. else MATERIAL_SOURCE_HISTORY
  429. )
  430. if landing_video_id is not None:
  431. landing_video_id = int(landing_video_id)
  432. if landing_counts_for_ad.get(landing_video_id, 0) >= MAX_SAME_LANDING_PER_AD_IN_RUN:
  433. logger.info(
  434. "[phase1] pending 接收排重丢弃 adgroup=%s landing=%d 同广告已达上限%s",
  435. rec.get("adgroup_id"), landing_video_id,
  436. " recovered" if recovered else "",
  437. )
  438. return False
  439. source_counts = crowd_landing_counts.setdefault(actual_material_source, {})
  440. if source_counts.get(landing_video_id, 0) >= landing_max_uses:
  441. logger.info(
  442. "[phase1] pending 接收排重丢弃 crowd=%r material_source=%s landing=%d 同人群包已达上限%s",
  443. crowd_package, actual_material_source, landing_video_id,
  444. " recovered" if recovered else "",
  445. )
  446. return False
  447. pending_records.append(rec)
  448. if material_id:
  449. display_excluded_material_ids.add(str(material_id))
  450. logger.info(
  451. "[phase1] 本轮展示去重登记 crowd=%r material=%s size=%d%s",
  452. crowd_package, material_id,
  453. len(display_excluded_material_ids),
  454. " recovered" if recovered else "",
  455. )
  456. if landing_video_id is not None:
  457. landing_counts_for_ad[landing_video_id] = (
  458. landing_counts_for_ad.get(landing_video_id, 0) + 1
  459. )
  460. source_counts = crowd_landing_counts.setdefault(actual_material_source, {})
  461. source_counts[landing_video_id] = source_counts.get(landing_video_id, 0) + 1
  462. logger.info(
  463. "[phase1] 同人群包 landing 使用登记 crowd=%r material_source=%s landing=%d count=%d max=%d%s",
  464. crowd_package, actual_material_source, landing_video_id,
  465. source_counts[landing_video_id],
  466. landing_max_uses,
  467. " recovered" if recovered else "",
  468. )
  469. logger.info(
  470. "[phase1] 本轮同广告 landing 计数 adgroup=%s landing=%d count=%d limit=%d%s",
  471. rec.get("adgroup_id"), landing_video_id,
  472. landing_counts_for_ad[landing_video_id],
  473. MAX_SAME_LANDING_PER_AD_IN_RUN,
  474. " recovered" if recovered else "",
  475. )
  476. if not recovered:
  477. try:
  478. record_prepared_material_usage(rec)
  479. except Exception as e:
  480. logger.warning(
  481. "[phase1] material usage 记录失败 account=%s adgroup=%s material=%s:%s",
  482. rec.get("account_id"), rec.get("adgroup_id"),
  483. rec.get("_material_id"), e,
  484. )
  485. return True
  486. for account_id in creation_accounts:
  487. logger.info("=" * 60)
  488. logger.info("[phase1] 账户 %d 处理开始", account_id)
  489. try:
  490. ads = find_ads_needing_creatives(account_id, min_creatives=target_creatives)
  491. except Exception as e:
  492. logger.exception("[phase1] account=%d find_ads 失败,跳过:%s", account_id, e)
  493. continue
  494. ads_after_filter = [a for a in ads if a["adgroup_id"] not in excluded_ad_ids]
  495. excluded_count = len(ads) - len(ads_after_filter)
  496. if excluded_count:
  497. logger.info(
  498. "[phase1] account=%d 关联点过滤排除 %d 条",
  499. account_id, excluded_count,
  500. )
  501. if not ads_after_filter:
  502. logger.info("[phase1] account=%d 无广告需补创意", account_id)
  503. continue
  504. try:
  505. crowd_package = get_account_crowd_package(account_id)
  506. except Exception as e:
  507. logger.exception(
  508. "[phase1] account=%d 读取 crowd_package 失败,跳过:%s",
  509. account_id, e,
  510. )
  511. continue
  512. display_excluded_material_ids = display_used_material_ids_by_crowd.setdefault(
  513. crowd_package, set(),
  514. )
  515. if crowd_package not in landing_usage_counts_by_crowd:
  516. try:
  517. landing_usage_counts_by_crowd[crowd_package] = load_recent_landing_usage_counts(
  518. crowd_package,
  519. )
  520. except Exception as e:
  521. logger.warning(
  522. "[phase1] crowd=%r 读取 landing 使用历史失败,仅使用本轮排重:%s",
  523. crowd_package, e,
  524. )
  525. landing_usage_counts_by_crowd[crowd_package] = {
  526. MATERIAL_SOURCE_HISTORY: {},
  527. MATERIAL_SOURCE_AI_GENERATED: {},
  528. }
  529. logger.info(
  530. "[phase1] crowd=%r 近期 landing 使用 history=%d ai_generated=%d",
  531. crowd_package,
  532. len(landing_usage_counts_by_crowd[crowd_package].get(MATERIAL_SOURCE_HISTORY, {})),
  533. len(landing_usage_counts_by_crowd[crowd_package].get(MATERIAL_SOURCE_AI_GENERATED, {})),
  534. )
  535. crowd_landing_counts = landing_usage_counts_by_crowd[crowd_package]
  536. crowd_landing_counts.setdefault(MATERIAL_SOURCE_HISTORY, {})
  537. crowd_landing_counts.setdefault(MATERIAL_SOURCE_AI_GENERATED, {})
  538. try:
  539. material_strategy = load_account_material_strategy(account_id)
  540. except Exception as e:
  541. logger.warning(
  542. "[phase1] account=%d 读取素材策略失败,按 history 排重:%s",
  543. account_id, e,
  544. )
  545. material_strategy = None
  546. requested_material_source = (
  547. MATERIAL_SOURCE_AI_GENERATED
  548. if material_strategy is not None and material_strategy.use_ai_generated
  549. else MATERIAL_SOURCE_HISTORY
  550. )
  551. landing_max_uses = MAX_SAME_LANDING_PER_AD_IN_RUN
  552. logger.info(
  553. "[phase1] account=%d landing 排重池=%s max_uses=%d",
  554. account_id, requested_material_source, landing_max_uses,
  555. )
  556. for ad in ads_after_filter:
  557. adgroup_id = ad["adgroup_id"]
  558. already_have = ad["creative_count"]
  559. to_add = max(0, target_creatives - already_have)
  560. prepared_for_ad = 0
  561. failed_prepare_for_ad = 0
  562. landing_counts_for_ad: dict[int, int] = {}
  563. logger.info(
  564. "[phase1] adgroup=%d(have=%d need=%d)",
  565. adgroup_id, already_have, to_add,
  566. )
  567. recovered_records = load_recoverable_prepared_records(
  568. account_id=account_id,
  569. adgroup_id=adgroup_id,
  570. crowd_package=crowd_package,
  571. material_source=requested_material_source,
  572. limit=to_add,
  573. )
  574. recovered_accepted = 0
  575. for rec in recovered_records:
  576. if accept_pending_record(
  577. rec,
  578. crowd_package=crowd_package,
  579. display_excluded_material_ids=display_excluded_material_ids,
  580. crowd_landing_counts=crowd_landing_counts,
  581. landing_counts_for_ad=landing_counts_for_ad,
  582. landing_max_uses=landing_max_uses,
  583. recovered=True,
  584. ):
  585. recovered_accepted += 1
  586. if recovered_records:
  587. prepared_for_ad += recovered_accepted
  588. logger.info(
  589. "[phase1] adgroup=%d 恢复未提交 prepared records=%d accepted=%d remaining=%d",
  590. adgroup_id, len(recovered_records), recovered_accepted,
  591. max(0, to_add - prepared_for_ad),
  592. )
  593. def prepare_attempt(
  594. *,
  595. attempt_no: int,
  596. excluded_material_ids_snapshot: set[str],
  597. excluded_landing_ids_snapshot: set[int],
  598. ) -> dict | None:
  599. try:
  600. return prepare_one_creative_for_ad(
  601. account_id, adgroup_id,
  602. excluded_material_ids=excluded_material_ids_snapshot,
  603. excluded_landing_ids=excluded_landing_ids_snapshot,
  604. )
  605. except Exception as e:
  606. logger.exception(
  607. "[phase1] adgroup=%d attempt=%d 准备失败:%s",
  608. adgroup_id, attempt_no, e,
  609. )
  610. return None
  611. def current_excluded_landing_ids() -> set[int]:
  612. landing_excluded_for_ad = {
  613. vid
  614. for vid, count in landing_counts_for_ad.items()
  615. if count >= MAX_SAME_LANDING_PER_AD_IN_RUN
  616. }
  617. landing_excluded_for_source = {
  618. vid
  619. for vid, count in crowd_landing_counts
  620. .get(requested_material_source, {})
  621. .items()
  622. if count >= landing_max_uses
  623. }
  624. return landing_excluded_for_source | landing_excluded_for_ad
  625. remaining_to_add = max(0, to_add - prepared_for_ad)
  626. max_workers = max(1, int(CREATIVE_PREPARE_MAX_WORKERS))
  627. if max_workers > 1:
  628. logger.warning(
  629. "[phase1] CREATIVE_PREPARE_MAX_WORKERS=%d 已配置但当前 prepare_one_creative_for_ad "
  630. "包含图片上传/xcx-save 等外部副作用,为避免重复创建落地计划,本轮强制串行执行",
  631. max_workers,
  632. )
  633. max_workers = 1
  634. task_buffer = max(remaining_to_add, int(CREATIVE_PREPARE_TASK_BUFFER))
  635. task_limit = min(max(0, remaining_to_add * 2), task_buffer)
  636. if remaining_to_add > 0 and max_workers > 1:
  637. logger.info(
  638. "[phase1] adgroup=%d 并行准备 start remaining=%d workers=%d task_limit=%d",
  639. adgroup_id, remaining_to_add, max_workers, task_limit,
  640. )
  641. submitted = 0
  642. accepted_parallel = 0
  643. dropped_parallel = 0
  644. failed_parallel = 0
  645. started_at = time.monotonic()
  646. with ThreadPoolExecutor(max_workers=max_workers) as executor:
  647. futures = {}
  648. def submit_next() -> None:
  649. nonlocal submitted
  650. # 已接收 + 运行中任务不超过缺口,避免并发准备产出无法追踪的多余副作用。
  651. if (
  652. submitted >= task_limit
  653. or prepared_for_ad + len(futures) >= to_add
  654. ):
  655. return
  656. submitted += 1
  657. attempt_no = submitted
  658. future = executor.submit(
  659. prepare_attempt,
  660. attempt_no=attempt_no,
  661. excluded_material_ids_snapshot=set(display_excluded_material_ids),
  662. excluded_landing_ids_snapshot=current_excluded_landing_ids(),
  663. )
  664. futures[future] = attempt_no
  665. for _ in range(min(max_workers, task_limit)):
  666. submit_next()
  667. while futures and prepared_for_ad < to_add:
  668. done, _ = wait(futures, return_when=FIRST_COMPLETED)
  669. for future in done:
  670. futures.pop(future, None)
  671. rec = future.result()
  672. if not rec:
  673. failed_prepare_for_ad += 1
  674. failed_parallel += 1
  675. elif accept_pending_record(
  676. rec,
  677. crowd_package=crowd_package,
  678. display_excluded_material_ids=display_excluded_material_ids,
  679. crowd_landing_counts=crowd_landing_counts,
  680. landing_counts_for_ad=landing_counts_for_ad,
  681. landing_max_uses=landing_max_uses,
  682. ):
  683. prepared_for_ad += 1
  684. accepted_parallel += 1
  685. else:
  686. dropped_parallel += 1
  687. if prepared_for_ad < to_add:
  688. submit_next()
  689. else:
  690. break
  691. if prepared_for_ad >= to_add:
  692. logger.info(
  693. "[phase1] adgroup=%d 并行准备 target reached accepted=%d",
  694. adgroup_id, accepted_parallel,
  695. )
  696. cancelled = 0
  697. for pending_future in futures:
  698. if pending_future.cancel():
  699. cancelled += 1
  700. if cancelled:
  701. logger.info(
  702. "[phase1] adgroup=%d 并行准备 cancelled_pending=%d",
  703. adgroup_id, cancelled,
  704. )
  705. for future in futures:
  706. if future.cancelled():
  707. continue
  708. rec = future.result()
  709. if rec:
  710. dropped_parallel += 1
  711. logger.info(
  712. "[phase1] adgroup=%d 并行结果丢弃:target reached material=%s landing=%s",
  713. adgroup_id, rec.get("_material_id"), rec.get("landing_video_id"),
  714. )
  715. logger.info(
  716. "[phase1] adgroup=%d 并行准备 done submitted=%d accepted=%d dropped=%d failed=%d elapsed=%.1fs",
  717. adgroup_id, submitted, accepted_parallel, dropped_parallel,
  718. failed_parallel, time.monotonic() - started_at,
  719. )
  720. else:
  721. for attempt_no in range(1, remaining_to_add + 1):
  722. rec = prepare_attempt(
  723. attempt_no=attempt_no,
  724. excluded_material_ids_snapshot=set(display_excluded_material_ids),
  725. excluded_landing_ids_snapshot=current_excluded_landing_ids(),
  726. )
  727. if rec:
  728. if accept_pending_record(
  729. rec,
  730. crowd_package=crowd_package,
  731. display_excluded_material_ids=display_excluded_material_ids,
  732. crowd_landing_counts=crowd_landing_counts,
  733. landing_counts_for_ad=landing_counts_for_ad,
  734. landing_max_uses=landing_max_uses,
  735. ):
  736. prepared_for_ad += 1
  737. else:
  738. failed_prepare_for_ad += 1
  739. # 2026-06-10 用户要求:单条 prepare 失败 → continue 不 break
  740. # 同广告剩余 to_add 创意还能继续试,不被一次失败拖累
  741. logger.info(
  742. "[phase1] adgroup=%d 本条创意 prepare 失败,试下一条",
  743. adgroup_id,
  744. )
  745. logger.info(
  746. "[phase1] adgroup=%d 补创意完成 target=%d have_before=%d "
  747. "planned=%d prepared=%d failed_prepare=%d",
  748. adgroup_id, target_creatives, already_have, to_add,
  749. prepared_for_ad, failed_prepare_for_ad,
  750. )
  751. if prepared_for_ad < to_add:
  752. logger.warning(
  753. "[phase1] adgroup=%d 未补满:缺口=%d。详细原因见上方 "
  754. "prepare_one_creative source_summary/失败日志",
  755. adgroup_id, to_add - prepared_for_ad,
  756. )
  757. logger.info("=" * 60)
  758. logger.info("[phase1] 准备完成,共 %d 条 pending records", len(pending_records))
  759. return pending_records
  760. def _write_pending_records(records: list[dict], output_dir: Path) -> Path:
  761. """落 Phase 1 产物 JSON(供 Phase 3 独立 apply 用 / 追溯)。"""
  762. output_dir.mkdir(parents=True, exist_ok=True)
  763. now = now_in_timezone()
  764. date_str = now.strftime("%Y%m%d")
  765. ts = now.strftime("%H%M%S")
  766. out_path = output_dir / f"creation_pending_{date_str}_{ts}.json"
  767. with open(out_path, "w", encoding="utf-8") as f:
  768. json.dump(records, f, ensure_ascii=False, indent=2)
  769. return out_path
  770. def phase2_approval(records: list[dict], xlsx_output_dir: Path) -> dict[int, str]:
  771. """Phase 2:飞书审批(若需要)。
  772. Returns: actions {row_idx: action}, 1-based row_idx
  773. """
  774. from tools.im_approval_creation import run_approval_workflow
  775. sheet_meta, actions = run_approval_workflow(
  776. records=records,
  777. xlsx_output_dir=xlsx_output_dir,
  778. timeout_minutes=CREATION_APPROVAL_TIMEOUT_MINUTES,
  779. )
  780. logger.info(
  781. "[phase2] 审批完成 sheet_url=%s actions=%d/%d",
  782. sheet_meta.get("url"), len(actions), len(records),
  783. )
  784. return actions
  785. def run_once() -> dict:
  786. """完整主循环:Phase 0 → Phase 1 → Phase 2(若开关 True)→ Phase 3。"""
  787. run_started = now_in_timezone().isoformat()
  788. sync_stats: dict = {}
  789. # Phase -1:每日主流程启动时先同步飞书「自动化账户」配置表。
  790. logger.info("=" * 60)
  791. logger.info("[main] Phase -1 启动 — 同步飞书自动化账户配置")
  792. try:
  793. from sync_feishu_account_config import sync_from_feishu
  794. sync_stats = sync_from_feishu()
  795. logger.info("[main] Phase -1 完成: %s", sync_stats)
  796. except Exception as e:
  797. logger.exception("[main] Phase -1 同步飞书配置失败,本轮停止:%s", e)
  798. return {
  799. "run_started": run_started,
  800. "run_finished": now_in_timezone().isoformat(),
  801. "approval_required": CREATION_APPROVAL_REQUIRED,
  802. "sync_stats": {"error": str(e)},
  803. "phase0_created_ads": 0,
  804. "total": {"phase1_prepared": 0},
  805. }
  806. # Phase 0:模块 A 建广告(满足每账户 ADS_PER_ACCOUNT 条)
  807. logger.info("=" * 60)
  808. if _env_flag("CREATION_SKIP_PHASE0"):
  809. logger.info("[main] CREATION_SKIP_PHASE0=True → 跳过模块 A 建广告")
  810. created_ads = []
  811. else:
  812. logger.info("[main] Phase 0 启动 — 模块 A 检查 + 建广告")
  813. created_ads = phase0_create_ads()
  814. logger.info("[main] Phase 0 完成:本轮新建广告 %d 条", len(created_ads))
  815. _wait_created_ads_visible(created_ads)
  816. # Phase 1:模块 B 给所有广告(新+旧)补创意
  817. logger.info("=" * 60)
  818. logger.info("[main] Phase 1 启动 — 模块 B 给广告补创意")
  819. pending_records = phase1_prepare()
  820. if not pending_records:
  821. logger.info("[main] Phase 1 无 pending records,主循环退出")
  822. return {
  823. "run_started": run_started,
  824. "run_finished": now_in_timezone().isoformat(),
  825. "approval_required": CREATION_APPROVAL_REQUIRED,
  826. "sync_stats": sync_stats,
  827. "phase0_created_ads": len(created_ads),
  828. "total": {"phase1_prepared": 0},
  829. }
  830. data_dir = _HERE / "outputs" / "data"
  831. pending_path = _write_pending_records(pending_records, data_dir)
  832. logger.info("[main] pending records 已写入: %s", pending_path)
  833. # Phase 2:审批(或 skip)
  834. if CREATION_APPROVAL_REQUIRED:
  835. logger.info("[main] CREATION_APPROVAL_REQUIRED=True → 进 Phase 2 飞书审批")
  836. actions = phase2_approval(pending_records, data_dir)
  837. for i, rec in enumerate(pending_records, start=1):
  838. rec["action"] = actions.get(i, "skip")
  839. else:
  840. logger.info("[main] CREATION_APPROVAL_REQUIRED=False → 全 records approve")
  841. for rec in pending_records:
  842. rec["action"] = "approve"
  843. # Phase 3:执行
  844. logger.info("=" * 60)
  845. logger.info("[main] Phase 3 执行启动")
  846. summary = apply_pending_records(pending_records)
  847. summary["approval_required"] = CREATION_APPROVAL_REQUIRED
  848. summary["pending_records_path"] = str(pending_path)
  849. summary_path = write_apply_summary(summary, data_dir)
  850. logger.info("[main] summary 已写入: %s", summary_path)
  851. # 发执行汇报
  852. _send_apply_summary_to_feishu(summary)
  853. summary["run_started"] = run_started
  854. summary["run_finished"] = now_in_timezone().isoformat()
  855. summary["sync_stats"] = sync_stats
  856. summary["phase0_created_ads"] = len(created_ads)
  857. summary["phase0_ads"] = created_ads
  858. return summary
  859. def main() -> int:
  860. _setup_logging()
  861. logger.info("=" * 60)
  862. logger.info("模块 B 创意搭建子系统 — 主循环启动")
  863. logger.info("TARGET_CREATIVES_PER_AD = %d", TARGET_CREATIVES_PER_AD)
  864. logger.info("CREATION_APPROVAL_REQUIRED = %s", CREATION_APPROVAL_REQUIRED)
  865. logger.info("WHITELIST_ACCOUNTS = %s", WHITELIST_ACCOUNTS)
  866. logger.info("=" * 60)
  867. summary = run_once()
  868. t = summary.get("total") or {}
  869. logger.info("=" * 60)
  870. logger.info("[main] 主循环结束")
  871. if "phase1_prepared" in t:
  872. logger.info(" phase1_prepared = %d", t["phase1_prepared"])
  873. else:
  874. logger.info(" records = %d", t.get("records", 0))
  875. logger.info(" approved = %d", t.get("approved", 0))
  876. logger.info(" posted_ok = %d", t.get("posted_ok", 0))
  877. logger.info(" posted_failed = %d", t.get("posted_failed", 0))
  878. logger.info("=" * 60)
  879. return 0 if t.get("posted_failed", 0) == 0 else 1
  880. if __name__ == "__main__":
  881. sys.exit(main())