execute_creation_once.py 41 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006
  1. """模块 B 创意搭建子系统 — 主循环入口(P0-A / P0-C / P0-E,2026-06-09)。
  2. 数据流(三阶段,先审后挂):
  3. Phase 1 — 准备:
  4. 扫账户 → 关联点过滤(读调控当日 latest_decisions 排除 pause)
  5. → find_ads_needing_creatives
  6. → 对每条广告 prepare_one_creative_for_ad(召回 + 上传图 + xcx/save + build body)
  7. → pending_records (含完整 Phase 3 用的 _request_body)
  8. → 写 outputs/data/creation_pending_{date}.json(追溯 + 独立 apply 用)
  9. Phase 2 — 审批(若 CREATION_APPROVAL_REQUIRED=True):
  10. run_approval_workflow 生成 20 列 xlsx → 上传飞书 sheet → 发链接 → 轮询读决策列
  11. actions {row_idx: approve/reject/hold}
  12. → 写回 records["action"]
  13. Phase 3 — 执行:
  14. 对 action=approve 的 → POST /dynamic_creatives/add
  15. → 写 outputs/data/creation_run_{date}.json
  16. → 发飞书"执行汇报"消息
  17. 开关 CREATION_APPROVAL_REQUIRED=False 时跳过 Phase 2,全 records 直接 approve。
  18. """
  19. import argparse
  20. import json
  21. import logging
  22. import os
  23. import sys
  24. import time
  25. from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
  26. from datetime import date
  27. from pathlib import Path
  28. _HERE = Path(__file__).parent
  29. sys.path.insert(0, str(_HERE.parent.parent))
  30. sys.path.insert(0, str(_HERE))
  31. from dotenv import load_dotenv # noqa: E402
  32. load_dotenv(_HERE / ".env")
  33. from config import ( # noqa: E402
  34. ADS_PER_ACCOUNT,
  35. CREATION_APPROVAL_REQUIRED,
  36. CREATION_APPROVAL_TIMEOUT_MINUTES,
  37. CREATIVE_PREPARE_MAX_WORKERS,
  38. CREATIVE_PREPARE_TASK_BUFFER,
  39. MAX_SAME_LANDING_PER_AD_IN_RUN,
  40. TARGET_CREATIVES_PER_AD,
  41. WHITELIST_ACCOUNTS,
  42. get_creation_account_ids,
  43. now_in_timezone,
  44. )
  45. # config import 副作用会改 sys.path(把 im-client / agent/tools/builtin 顶到前面),
  46. # 强占 sys.path[0],只影响本入口自己的 import resolution。
  47. while str(_HERE) in sys.path:
  48. sys.path.remove(str(_HERE))
  49. sys.path.insert(0, str(_HERE))
  50. from tools.ad_creation import ( # noqa: E402
  51. build_ad_request_body,
  52. compute_fingerprint,
  53. enumerate_new_ad_candidates,
  54. post_ad_with_prepared_body,
  55. )
  56. from tools.audience_grant import ensure_account_audience_grant # noqa: E402
  57. from tools.creative_creation import ( # noqa: E402
  58. build_landing_candidate_pool,
  59. find_ads_needing_creatives,
  60. load_excluded_ad_ids_from_adjustment,
  61. prepare_one_creative_for_ad,
  62. )
  63. from tools.creative_material_usage import ( # noqa: E402
  64. load_recent_landing_usage_counts,
  65. load_recoverable_prepared_records,
  66. record_prepared_material_usage,
  67. )
  68. from tools.account_material_strategy import ( # noqa: E402
  69. MATERIAL_SOURCE_AI_GENERATED,
  70. load_account_material_strategy,
  71. )
  72. from tools.video_recall import get_account_crowd_package # noqa: E402
  73. from execute_creation_apply import ( # noqa: E402
  74. apply_pending_records,
  75. write_summary as write_apply_summary,
  76. _send_apply_summary_to_feishu,
  77. )
  78. logger = logging.getLogger("execute_creation_once")
  79. MATERIAL_SOURCE_HISTORY = "history"
  80. def _env_flag(name: str, default: bool = False) -> bool:
  81. raw = os.getenv(name)
  82. if raw is None:
  83. return default
  84. return raw.strip().lower() in {"1", "true", "yes", "y", "on"}
  85. def _env_csv_set(name: str) -> set[str]:
  86. return {
  87. item.strip()
  88. for item in os.getenv(name, "").split(",")
  89. if item.strip()
  90. }
  91. def _filter_creation_accounts(accounts: list[int], phase: str) -> list[int]:
  92. """Apply one-off run filters. Empty env means production default: no filter."""
  93. account_filter = {
  94. int(v)
  95. for v in _env_csv_set("CREATION_ONLY_ACCOUNT_IDS")
  96. if v.isdigit()
  97. }
  98. crowd_filter = _env_csv_set("CREATION_ONLY_CROWD_PACKAGES")
  99. if not account_filter and not crowd_filter:
  100. return accounts
  101. selected: list[int] = []
  102. for account_id in accounts:
  103. if account_filter and account_id not in account_filter:
  104. continue
  105. if crowd_filter:
  106. try:
  107. crowd_package = get_account_crowd_package(account_id)
  108. except Exception as e:
  109. logger.warning(
  110. "[%s] account=%d 临时过滤读取 crowd_package 失败,跳过:%s",
  111. phase, account_id, e,
  112. )
  113. continue
  114. if crowd_package not in crowd_filter:
  115. continue
  116. selected.append(account_id)
  117. logger.info(
  118. "[%s] 临时运行过滤 account_ids=%s crowd_packages=%s: %d -> %d",
  119. phase,
  120. sorted(account_filter) if account_filter else "ALL",
  121. sorted(crowd_filter) if crowd_filter else "ALL",
  122. len(accounts),
  123. len(selected),
  124. )
  125. return selected
  126. def _setup_logging() -> None:
  127. logging.basicConfig(
  128. level=logging.INFO,
  129. format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
  130. datefmt="%H:%M:%S",
  131. )
  132. for noisy in ("httpx", "httpcore", "config", "db.config"):
  133. logging.getLogger(noisy).setLevel(logging.WARNING)
  134. # SLS 上报(2026-06-11 接入,K8s pod 直发)— 配置缺失自动降级,不阻塞主链路
  135. try:
  136. from tools.sls_setup import attach_sls_handler
  137. attach_sls_handler()
  138. except Exception as e:
  139. logger.warning("[sls] 挂载异常(降级为本地 only):%s", e)
  140. def _fetch_existing_fingerprints_for_account(account_id: int) -> set[str]:
  141. """Task 27:拉账户下所有非 DELETED 广告 → 反推 fingerprint 集合(预校验唯一性)。
  142. 腾讯文档:NORMAL + SUSPEND 状态广告均占用唯一性槽位,DENIED 也算占用。
  143. list 接口默认返回所有非 DELETED 状态 ad,故无需显式 filter status。
  144. 若腾讯 list 接口某些字段不返回(field 实测差异),降级:跳过该条 + log 警告。
  145. 返回空集 → 上层退化为"不预校验",enumerate 仍能跑(原有行为)。
  146. """
  147. from tools.ad_api import _get
  148. fingerprints: set[str] = set()
  149. skipped = 0
  150. page = 1
  151. try:
  152. while True:
  153. r = _get("/adgroups/get", {
  154. "account_id": account_id, "page": page, "page_size": 100,
  155. "fields": [
  156. "adgroup_id", "configured_status", "site_set",
  157. "targeting", "scene_spec",
  158. ],
  159. })
  160. ads = (r.get("data") or {}).get("list") or []
  161. if not ads:
  162. break
  163. for ad in ads:
  164. tgt = ad.get("targeting") or {}
  165. sc = ad.get("scene_spec") or {}
  166. wp = sc.get("wechat_position") if sc else None
  167. site_set = ad.get("site_set") or []
  168. age = tgt.get("age") or []
  169. geo_regions = (tgt.get("geo_location") or {}).get("regions") or []
  170. custom_audience = tgt.get("custom_audience")
  171. if not site_set or not age:
  172. skipped += 1
  173. continue
  174. try:
  175. fp = compute_fingerprint(
  176. account_id=account_id,
  177. site_set=site_set,
  178. custom_audience=custom_audience,
  179. age=age,
  180. geo_regions=geo_regions,
  181. wechat_position=wp,
  182. )
  183. fingerprints.add(fp)
  184. except Exception as e:
  185. skipped += 1
  186. logger.debug(
  187. "[phase0] fingerprint 算失败 ad=%s: %s",
  188. ad.get("adgroup_id"), e,
  189. )
  190. if len(ads) < 100:
  191. break
  192. page += 1
  193. except Exception as e:
  194. logger.warning(
  195. "[phase0] account=%d 拉 fingerprint 集失败(降级为不预校验): %s",
  196. account_id, e,
  197. )
  198. return set()
  199. logger.info(
  200. "[phase0] 已存在 fingerprint=%d 个(skip 缺字段广告 %d 条)",
  201. len(fingerprints), skipped,
  202. )
  203. return fingerprints
  204. def phase0_create_ads(
  205. target_ads: int = ADS_PER_ACCOUNT,
  206. config_date: date | None = None,
  207. ) -> list[dict]:
  208. """Phase 0:对每账户检查广告数,不足则建到 target_ads 条(模块 A,2026-06-09 P1-G)。
  209. 流程:
  210. account → 查当前广告数(NORMAL+SUSPEND 算占用)→ 不足 target_ads →
  211. → 反查 fingerprint 集 → enumerate candidates(排除已有 fp)→
  212. → 飞书审批 → approve 项 → 真 POST /adgroups/add → 返回新建的 adgroup_id 列表
  213. Returns: 本次新建成功的广告 list[dict],每项含 {account_id, adgroup_id, adgroup_name, wechat_position}
  214. """
  215. from tools.ad_api import _get
  216. from tools.im_approval_ad_creation import run_ad_approval_workflow
  217. creation_accounts = get_creation_account_ids(config_date=config_date)
  218. if not creation_accounts:
  219. logger.error("[phase0] 待投放账户配置为空,退出")
  220. return []
  221. creation_accounts = _filter_creation_accounts(creation_accounts, "phase0")
  222. if not creation_accounts:
  223. logger.info("[phase0] 临时过滤后无待处理账户")
  224. return []
  225. # Task 26:NORMAL + SUSPEND 都算占用唯一性槽位(腾讯文档:删除前历史广告占槽位)
  226. OCCUPIED_STATUSES = {"AD_STATUS_NORMAL", "AD_STATUS_SUSPEND"}
  227. pending_ad_records: list[dict] = []
  228. for account_id in creation_accounts:
  229. logger.info("=" * 60)
  230. logger.info("[phase0] 账户 %d 处理开始", account_id)
  231. try:
  232. grant_result = ensure_account_audience_grant(account_id)
  233. logger.info(
  234. "[phase0] 人群包校验完成: status=%s audience_id=%s audience=%r",
  235. grant_result.get("status"),
  236. grant_result.get("audience_id"),
  237. grant_result.get("audience_name"),
  238. )
  239. except Exception as e:
  240. logger.exception("[phase0] account=%d 人群包授权/验证失败,跳过:%s", account_id, e)
  241. continue
  242. # Task 26:查所有非 DELETED 广告,NORMAL+SUSPEND 都算占用槽位
  243. try:
  244. r = _get("/adgroups/get", {
  245. "account_id": account_id, "page": 1, "page_size": 100,
  246. "fields": ["adgroup_id", "configured_status"],
  247. })
  248. all_ads = (r.get("data") or {}).get("list") or []
  249. normal_n = sum(1 for a in all_ads if a.get("configured_status") == "AD_STATUS_NORMAL")
  250. suspend_n = sum(1 for a in all_ads if a.get("configured_status") == "AD_STATUS_SUSPEND")
  251. occupied = [a for a in all_ads if a.get("configured_status") in OCCUPIED_STATUSES]
  252. logger.info(
  253. "[phase0] 广告数: 总 %d,NORMAL %d + SUSPEND %d = 占用 %d / 目标 %d",
  254. len(all_ads), normal_n, suspend_n, len(occupied), target_ads,
  255. )
  256. except Exception as e:
  257. logger.exception("[phase0] account=%d 查广告数失败: %s", account_id, e)
  258. continue
  259. to_create = max(0, target_ads - len(occupied))
  260. if to_create == 0:
  261. logger.info("[phase0] 占用槽位已满,跳过")
  262. continue
  263. # Task 27:fingerprint 预校验 — 反查现存广告的 fp 集合,enumerate 时排除
  264. existing_fps = _fetch_existing_fingerprints_for_account(account_id)
  265. # enumerate 候选(模块 A,差异化 wechat_position 有/无)
  266. try:
  267. candidates = enumerate_new_ad_candidates(
  268. account_id, count=to_create,
  269. existing_fingerprints=existing_fps,
  270. )
  271. except Exception as e:
  272. logger.exception("[phase0] account=%d enumerate 失败: %s", account_id, e)
  273. continue
  274. if not candidates:
  275. logger.warning("[phase0] enumerate 返回 0 条候选,跳过")
  276. continue
  277. today = now_in_timezone().strftime("%Y-%m-%d")
  278. for c in candidates:
  279. age_range = ",".join(
  280. f"{a.get('min')}-{a.get('max')}" for a in c.age
  281. )
  282. # 新建广告默认开启;实际投放仍受创意审核/账户/预算控制。
  283. # 这里显式传 NORMAL,避免未来默认值变化影响端到端创建语义。
  284. body = build_ad_request_body(c, configured_status="AD_STATUS_NORMAL")
  285. rec = {
  286. "approval_date": today,
  287. "account_id": c.account_id,
  288. "audience_tier_label": c.audience_tier_label,
  289. "adgroup_name": c.adgroup_name,
  290. "site_set": c.site_set,
  291. "delivery_version": c.delivery_version,
  292. "wechat_position": c.wechat_position,
  293. "bid_amount_fen": c.bid_amount_fen,
  294. "bid_scene": c.bid_scene,
  295. "custom_cost_cap_fen": c.custom_cost_cap_fen,
  296. "automatic_site_enabled": c.automatic_site_enabled,
  297. "age_range": age_range,
  298. "fingerprint": c.fingerprint,
  299. "_request_body": body,
  300. }
  301. pending_ad_records.append(rec)
  302. if not pending_ad_records:
  303. logger.info("[phase0] 无广告需新建,Phase 0 结束")
  304. return []
  305. # 飞书审批(如需要)
  306. data_dir = _HERE / "outputs" / "data"
  307. if CREATION_APPROVAL_REQUIRED:
  308. logger.info(
  309. "[phase0] CREATION_APPROVAL_REQUIRED=True → 飞书审批 %d 条广告候选",
  310. len(pending_ad_records),
  311. )
  312. sheet_meta, actions = run_ad_approval_workflow(
  313. records=pending_ad_records,
  314. xlsx_output_dir=data_dir,
  315. timeout_minutes=CREATION_APPROVAL_TIMEOUT_MINUTES,
  316. )
  317. for i, rec in enumerate(pending_ad_records, start=1):
  318. rec["action"] = actions.get(i, "skip")
  319. else:
  320. for rec in pending_ad_records:
  321. rec["action"] = "approve"
  322. # POST 真建 approve 项
  323. created: list[dict] = []
  324. for rec in pending_ad_records:
  325. if rec.get("action") != "approve":
  326. continue
  327. adgroup_id = post_ad_with_prepared_body(
  328. account_id=int(rec["account_id"]),
  329. body=rec["_request_body"],
  330. )
  331. if adgroup_id:
  332. created.append({
  333. "account_id": rec["account_id"],
  334. "adgroup_id": adgroup_id,
  335. "adgroup_name": rec["adgroup_name"],
  336. "wechat_position": rec.get("wechat_position"),
  337. })
  338. logger.info(
  339. "[phase0] 完成:候选 %d 条,approve %d 条,实际建 %d 条",
  340. len(pending_ad_records),
  341. sum(1 for r in pending_ad_records if r.get("action") == "approve"),
  342. len(created),
  343. )
  344. return created
  345. def _wait_created_ads_visible(created_ads: list[dict], timeout_seconds: int = 60) -> None:
  346. """等待刚创建的广告在 /adgroups/get 中可见。
  347. 腾讯 adgroups/add 成功后,list 接口存在短暂一致性延迟。若立刻进入 Phase 1,
  348. 可能只扫到部分新广告,导致同一轮只给一条广告补创意。
  349. """
  350. if not created_ads:
  351. return
  352. from tools.ad_api import _get
  353. by_account: dict[int, set[int]] = {}
  354. for ad in created_ads:
  355. by_account.setdefault(int(ad["account_id"]), set()).add(int(ad["adgroup_id"]))
  356. deadline = time.time() + timeout_seconds
  357. remaining = {acct: set(ids) for acct, ids in by_account.items()}
  358. while time.time() < deadline and any(remaining.values()):
  359. for account_id, ids in list(remaining.items()):
  360. if not ids:
  361. continue
  362. try:
  363. resp = _get("/adgroups/get", {
  364. "account_id": account_id,
  365. "page": 1,
  366. "page_size": 100,
  367. "fields": ["adgroup_id", "configured_status", "system_status"],
  368. })
  369. visible = {
  370. int(item["adgroup_id"])
  371. for item in ((resp.get("data") or {}).get("list") or [])
  372. if item.get("adgroup_id") is not None
  373. }
  374. ids.difference_update(visible)
  375. except Exception as e:
  376. logger.warning(
  377. "[phase0] 等待新广告可见失败 account=%d: %s",
  378. account_id, e,
  379. )
  380. if any(remaining.values()):
  381. logger.info("[phase0] 等待新广告在列表接口可见: %s", remaining)
  382. time.sleep(5)
  383. not_visible = {acct: sorted(ids) for acct, ids in remaining.items() if ids}
  384. if not_visible:
  385. logger.warning("[phase0] 部分新广告仍未在 list 可见,继续 Phase 1: %s", not_visible)
  386. else:
  387. logger.info("[phase0] 新建广告均已在 list 接口可见")
  388. def phase1_prepare(
  389. target_creatives: int = TARGET_CREATIVES_PER_AD,
  390. config_date: date | None = None,
  391. ) -> list[dict]:
  392. """Phase 1:扫账户 → 关联点过滤 → 准备 pending records(不 POST 腾讯)。
  393. Returns:
  394. pending_records — 每个元素是 prepare_one_creative_for_ad 返回的 dict
  395. """
  396. creation_accounts = get_creation_account_ids(config_date=config_date)
  397. if not creation_accounts:
  398. logger.error("[phase1] 待投放账户配置为空,Phase 1 退出")
  399. return []
  400. creation_accounts = _filter_creation_accounts(creation_accounts, "phase1")
  401. if not creation_accounts:
  402. logger.info("[phase1] 临时过滤后无待处理账户")
  403. return []
  404. excluded_ad_ids = load_excluded_ad_ids_from_adjustment()
  405. logger.info("[phase1] 关联点过滤集合 size=%d", len(excluded_ad_ids))
  406. pending_records: list[dict] = []
  407. # 素材排重只读取已有明确结果的 DB 记录。
  408. # pending/prepared 没有审批或投放结果,不进入排重。
  409. # 但同一轮审批候选需要做展示去重,避免表格里同 crowd_package 反复出现同一素材。
  410. # 这个 set 只存在内存里,不写历史排重库;进程结束即失效。
  411. display_used_material_ids_by_crowd: dict[str, set[str]] = {}
  412. # landing_video 按素材来源拆池。历史素材和 AI 生成素材互不占用,但池内都严格去重。
  413. landing_usage_counts_by_crowd: dict[str, dict[str, dict[int, int]]] = {}
  414. def accept_pending_record(
  415. rec: dict,
  416. *,
  417. crowd_package: str,
  418. display_excluded_material_ids: set[str],
  419. crowd_landing_counts: dict[str, dict[int, int]],
  420. landing_counts_for_ad: dict[int, int],
  421. landing_max_uses: int,
  422. recovered: bool = False,
  423. ) -> bool:
  424. material_id = rec.get("_material_id")
  425. if material_id and str(material_id) in display_excluded_material_ids:
  426. logger.info(
  427. "[phase1] pending 接收排重丢弃 crowd=%r material=%s%s",
  428. crowd_package, material_id,
  429. " recovered" if recovered else "",
  430. )
  431. return False
  432. landing_video_id = rec.get("landing_video_id")
  433. actual_material_source = (
  434. MATERIAL_SOURCE_AI_GENERATED
  435. if str(rec.get("material_source") or "") == MATERIAL_SOURCE_AI_GENERATED
  436. or str(rec.get("_material_id") or "").startswith("ai:")
  437. else MATERIAL_SOURCE_HISTORY
  438. )
  439. if landing_video_id is not None:
  440. landing_video_id = int(landing_video_id)
  441. if landing_counts_for_ad.get(landing_video_id, 0) >= MAX_SAME_LANDING_PER_AD_IN_RUN:
  442. logger.info(
  443. "[phase1] pending 接收排重丢弃 adgroup=%s landing=%d 同广告已达上限%s",
  444. rec.get("adgroup_id"), landing_video_id,
  445. " recovered" if recovered else "",
  446. )
  447. return False
  448. source_counts = crowd_landing_counts.setdefault(actual_material_source, {})
  449. if source_counts.get(landing_video_id, 0) >= landing_max_uses:
  450. logger.info(
  451. "[phase1] pending 接收排重丢弃 crowd=%r material_source=%s landing=%d 同人群包已达上限%s",
  452. crowd_package, actual_material_source, landing_video_id,
  453. " recovered" if recovered else "",
  454. )
  455. return False
  456. pending_records.append(rec)
  457. if material_id:
  458. display_excluded_material_ids.add(str(material_id))
  459. logger.info(
  460. "[phase1] 本轮展示去重登记 crowd=%r material=%s size=%d%s",
  461. crowd_package, material_id,
  462. len(display_excluded_material_ids),
  463. " recovered" if recovered else "",
  464. )
  465. if landing_video_id is not None:
  466. landing_counts_for_ad[landing_video_id] = (
  467. landing_counts_for_ad.get(landing_video_id, 0) + 1
  468. )
  469. source_counts = crowd_landing_counts.setdefault(actual_material_source, {})
  470. source_counts[landing_video_id] = source_counts.get(landing_video_id, 0) + 1
  471. logger.info(
  472. "[phase1] 同人群包 landing 使用登记 crowd=%r material_source=%s landing=%d count=%d max=%d%s",
  473. crowd_package, actual_material_source, landing_video_id,
  474. source_counts[landing_video_id],
  475. landing_max_uses,
  476. " recovered" if recovered else "",
  477. )
  478. logger.info(
  479. "[phase1] 本轮同广告 landing 计数 adgroup=%s landing=%d count=%d limit=%d%s",
  480. rec.get("adgroup_id"), landing_video_id,
  481. landing_counts_for_ad[landing_video_id],
  482. MAX_SAME_LANDING_PER_AD_IN_RUN,
  483. " recovered" if recovered else "",
  484. )
  485. if not recovered:
  486. try:
  487. record_prepared_material_usage(rec)
  488. except Exception as e:
  489. logger.warning(
  490. "[phase1] material usage 记录失败 account=%s adgroup=%s material=%s:%s",
  491. rec.get("account_id"), rec.get("adgroup_id"),
  492. rec.get("_material_id"), e,
  493. )
  494. return True
  495. for account_id in creation_accounts:
  496. logger.info("=" * 60)
  497. logger.info("[phase1] 账户 %d 处理开始", account_id)
  498. try:
  499. ads = find_ads_needing_creatives(account_id, min_creatives=target_creatives)
  500. except Exception as e:
  501. logger.exception("[phase1] account=%d find_ads 失败,跳过:%s", account_id, e)
  502. continue
  503. ads_after_filter = [a for a in ads if a["adgroup_id"] not in excluded_ad_ids]
  504. excluded_count = len(ads) - len(ads_after_filter)
  505. if excluded_count:
  506. logger.info(
  507. "[phase1] account=%d 关联点过滤排除 %d 条",
  508. account_id, excluded_count,
  509. )
  510. if not ads_after_filter:
  511. logger.info("[phase1] account=%d 无广告需补创意", account_id)
  512. continue
  513. try:
  514. crowd_package = get_account_crowd_package(account_id)
  515. except Exception as e:
  516. logger.exception(
  517. "[phase1] account=%d 读取 crowd_package 失败,跳过:%s",
  518. account_id, e,
  519. )
  520. continue
  521. display_excluded_material_ids = display_used_material_ids_by_crowd.setdefault(
  522. crowd_package, set(),
  523. )
  524. if crowd_package not in landing_usage_counts_by_crowd:
  525. try:
  526. landing_usage_counts_by_crowd[crowd_package] = load_recent_landing_usage_counts(
  527. crowd_package,
  528. )
  529. except Exception as e:
  530. logger.warning(
  531. "[phase1] crowd=%r 读取 landing 使用历史失败,仅使用本轮排重:%s",
  532. crowd_package, e,
  533. )
  534. landing_usage_counts_by_crowd[crowd_package] = {
  535. MATERIAL_SOURCE_HISTORY: {},
  536. MATERIAL_SOURCE_AI_GENERATED: {},
  537. }
  538. logger.info(
  539. "[phase1] crowd=%r 近期 landing 使用 history=%d ai_generated=%d",
  540. crowd_package,
  541. len(landing_usage_counts_by_crowd[crowd_package].get(MATERIAL_SOURCE_HISTORY, {})),
  542. len(landing_usage_counts_by_crowd[crowd_package].get(MATERIAL_SOURCE_AI_GENERATED, {})),
  543. )
  544. crowd_landing_counts = landing_usage_counts_by_crowd[crowd_package]
  545. crowd_landing_counts.setdefault(MATERIAL_SOURCE_HISTORY, {})
  546. crowd_landing_counts.setdefault(MATERIAL_SOURCE_AI_GENERATED, {})
  547. try:
  548. material_strategy = load_account_material_strategy(account_id)
  549. except Exception as e:
  550. logger.warning(
  551. "[phase1] account=%d 读取素材策略失败,按 history 排重:%s",
  552. account_id, e,
  553. )
  554. material_strategy = None
  555. requested_material_source = (
  556. MATERIAL_SOURCE_AI_GENERATED
  557. if material_strategy is not None and material_strategy.use_ai_generated
  558. else MATERIAL_SOURCE_HISTORY
  559. )
  560. landing_max_uses = MAX_SAME_LANDING_PER_AD_IN_RUN
  561. landing_candidate_pool = None
  562. logger.info(
  563. "[phase1] account=%d landing 排重池=%s max_uses=%d",
  564. account_id, requested_material_source, landing_max_uses,
  565. )
  566. for ad in ads_after_filter:
  567. adgroup_id = ad["adgroup_id"]
  568. already_have = ad["creative_count"]
  569. to_add = max(0, target_creatives - already_have)
  570. prepared_for_ad = 0
  571. failed_prepare_for_ad = 0
  572. failed_landing_ids_for_ad: set[int] = set()
  573. landing_counts_for_ad: dict[int, int] = {}
  574. logger.info(
  575. "[phase1] adgroup=%d(have=%d need=%d)",
  576. adgroup_id, already_have, to_add,
  577. )
  578. recovered_records = load_recoverable_prepared_records(
  579. account_id=account_id,
  580. adgroup_id=adgroup_id,
  581. crowd_package=crowd_package,
  582. material_source=requested_material_source,
  583. limit=to_add,
  584. )
  585. recovered_accepted = 0
  586. for rec in recovered_records:
  587. if accept_pending_record(
  588. rec,
  589. crowd_package=crowd_package,
  590. display_excluded_material_ids=display_excluded_material_ids,
  591. crowd_landing_counts=crowd_landing_counts,
  592. landing_counts_for_ad=landing_counts_for_ad,
  593. landing_max_uses=landing_max_uses,
  594. recovered=True,
  595. ):
  596. recovered_accepted += 1
  597. if recovered_records:
  598. prepared_for_ad += recovered_accepted
  599. logger.info(
  600. "[phase1] adgroup=%d 恢复未提交 prepared records=%d accepted=%d remaining=%d",
  601. adgroup_id, len(recovered_records), recovered_accepted,
  602. max(0, to_add - prepared_for_ad),
  603. )
  604. def prepare_attempt(
  605. *,
  606. attempt_no: int,
  607. excluded_material_ids_snapshot: set[str],
  608. excluded_landing_ids_snapshot: set[int],
  609. ) -> dict | None:
  610. nonlocal landing_candidate_pool
  611. try:
  612. if landing_candidate_pool is None:
  613. landing_candidate_pool = build_landing_candidate_pool(account_id)
  614. return prepare_one_creative_for_ad(
  615. account_id, adgroup_id,
  616. excluded_material_ids=excluded_material_ids_snapshot,
  617. excluded_landing_ids=excluded_landing_ids_snapshot,
  618. landing_candidates=landing_candidate_pool,
  619. failed_landing_ids=failed_landing_ids_for_ad,
  620. )
  621. except Exception as e:
  622. logger.exception(
  623. "[phase1] adgroup=%d attempt=%d 准备失败:%s",
  624. adgroup_id, attempt_no, e,
  625. )
  626. return None
  627. def current_excluded_landing_ids() -> set[int]:
  628. landing_excluded_for_ad = {
  629. vid
  630. for vid, count in landing_counts_for_ad.items()
  631. if count >= MAX_SAME_LANDING_PER_AD_IN_RUN
  632. }
  633. landing_excluded_for_source = {
  634. vid
  635. for vid, count in crowd_landing_counts
  636. .get(requested_material_source, {})
  637. .items()
  638. if count >= landing_max_uses
  639. }
  640. return landing_excluded_for_source | landing_excluded_for_ad
  641. remaining_to_add = max(0, to_add - prepared_for_ad)
  642. max_workers = max(1, int(CREATIVE_PREPARE_MAX_WORKERS))
  643. if max_workers > 1:
  644. logger.warning(
  645. "[phase1] CREATIVE_PREPARE_MAX_WORKERS=%d 已配置但当前 prepare_one_creative_for_ad "
  646. "包含图片上传/xcx-save 等外部副作用,为避免重复创建落地计划,本轮强制串行执行",
  647. max_workers,
  648. )
  649. max_workers = 1
  650. task_buffer = max(remaining_to_add, int(CREATIVE_PREPARE_TASK_BUFFER))
  651. task_limit = min(max(0, remaining_to_add * 2), task_buffer)
  652. if remaining_to_add > 0 and max_workers > 1:
  653. logger.info(
  654. "[phase1] adgroup=%d 并行准备 start remaining=%d workers=%d task_limit=%d",
  655. adgroup_id, remaining_to_add, max_workers, task_limit,
  656. )
  657. submitted = 0
  658. accepted_parallel = 0
  659. dropped_parallel = 0
  660. failed_parallel = 0
  661. started_at = time.monotonic()
  662. with ThreadPoolExecutor(max_workers=max_workers) as executor:
  663. futures = {}
  664. def submit_next() -> None:
  665. nonlocal submitted
  666. # 已接收 + 运行中任务不超过缺口,避免并发准备产出无法追踪的多余副作用。
  667. if (
  668. submitted >= task_limit
  669. or prepared_for_ad + len(futures) >= to_add
  670. ):
  671. return
  672. submitted += 1
  673. attempt_no = submitted
  674. future = executor.submit(
  675. prepare_attempt,
  676. attempt_no=attempt_no,
  677. excluded_material_ids_snapshot=set(display_excluded_material_ids),
  678. excluded_landing_ids_snapshot=current_excluded_landing_ids(),
  679. )
  680. futures[future] = attempt_no
  681. for _ in range(min(max_workers, task_limit)):
  682. submit_next()
  683. while futures and prepared_for_ad < to_add:
  684. done, _ = wait(futures, return_when=FIRST_COMPLETED)
  685. for future in done:
  686. futures.pop(future, None)
  687. rec = future.result()
  688. if not rec:
  689. failed_prepare_for_ad += 1
  690. failed_parallel += 1
  691. elif accept_pending_record(
  692. rec,
  693. crowd_package=crowd_package,
  694. display_excluded_material_ids=display_excluded_material_ids,
  695. crowd_landing_counts=crowd_landing_counts,
  696. landing_counts_for_ad=landing_counts_for_ad,
  697. landing_max_uses=landing_max_uses,
  698. ):
  699. prepared_for_ad += 1
  700. accepted_parallel += 1
  701. else:
  702. dropped_parallel += 1
  703. if prepared_for_ad < to_add:
  704. submit_next()
  705. else:
  706. break
  707. if prepared_for_ad >= to_add:
  708. logger.info(
  709. "[phase1] adgroup=%d 并行准备 target reached accepted=%d",
  710. adgroup_id, accepted_parallel,
  711. )
  712. cancelled = 0
  713. for pending_future in futures:
  714. if pending_future.cancel():
  715. cancelled += 1
  716. if cancelled:
  717. logger.info(
  718. "[phase1] adgroup=%d 并行准备 cancelled_pending=%d",
  719. adgroup_id, cancelled,
  720. )
  721. for future in futures:
  722. if future.cancelled():
  723. continue
  724. rec = future.result()
  725. if rec:
  726. dropped_parallel += 1
  727. logger.info(
  728. "[phase1] adgroup=%d 并行结果丢弃:target reached material=%s landing=%s",
  729. adgroup_id, rec.get("_material_id"), rec.get("landing_video_id"),
  730. )
  731. logger.info(
  732. "[phase1] adgroup=%d 并行准备 done submitted=%d accepted=%d dropped=%d failed=%d elapsed=%.1fs",
  733. adgroup_id, submitted, accepted_parallel, dropped_parallel,
  734. failed_parallel, time.monotonic() - started_at,
  735. )
  736. else:
  737. for attempt_no in range(1, remaining_to_add + 1):
  738. rec = prepare_attempt(
  739. attempt_no=attempt_no,
  740. excluded_material_ids_snapshot=set(display_excluded_material_ids),
  741. excluded_landing_ids_snapshot=current_excluded_landing_ids(),
  742. )
  743. if rec:
  744. if accept_pending_record(
  745. rec,
  746. crowd_package=crowd_package,
  747. display_excluded_material_ids=display_excluded_material_ids,
  748. crowd_landing_counts=crowd_landing_counts,
  749. landing_counts_for_ad=landing_counts_for_ad,
  750. landing_max_uses=landing_max_uses,
  751. ):
  752. prepared_for_ad += 1
  753. else:
  754. failed_prepare_for_ad += 1
  755. # 2026-06-10 用户要求:单条 prepare 失败 → continue 不 break
  756. # 同广告剩余 to_add 创意还能继续试,不被一次失败拖累
  757. logger.info(
  758. "[phase1] adgroup=%d 本条创意 prepare 失败,试下一条",
  759. adgroup_id,
  760. )
  761. logger.info(
  762. "[phase1] adgroup=%d 补创意完成 target=%d have_before=%d "
  763. "planned=%d prepared=%d failed_prepare=%d",
  764. adgroup_id, target_creatives, already_have, to_add,
  765. prepared_for_ad, failed_prepare_for_ad,
  766. )
  767. if prepared_for_ad < to_add:
  768. logger.warning(
  769. "[phase1] adgroup=%d 未补满:缺口=%d。详细原因见上方 "
  770. "prepare_one_creative source_summary/失败日志",
  771. adgroup_id, to_add - prepared_for_ad,
  772. )
  773. logger.info("=" * 60)
  774. logger.info("[phase1] 准备完成,共 %d 条 pending records", len(pending_records))
  775. return pending_records
  776. def _write_pending_records(records: list[dict], output_dir: Path) -> Path:
  777. """落 Phase 1 产物 JSON(供 Phase 3 独立 apply 用 / 追溯)。"""
  778. output_dir.mkdir(parents=True, exist_ok=True)
  779. now = now_in_timezone()
  780. date_str = now.strftime("%Y%m%d")
  781. ts = now.strftime("%H%M%S")
  782. out_path = output_dir / f"creation_pending_{date_str}_{ts}.json"
  783. with open(out_path, "w", encoding="utf-8") as f:
  784. json.dump(records, f, ensure_ascii=False, indent=2)
  785. return out_path
  786. def phase2_approval(records: list[dict], xlsx_output_dir: Path) -> dict[int, str]:
  787. """Phase 2:飞书审批(若需要)。
  788. Returns: actions {row_idx: action}, 1-based row_idx
  789. """
  790. from tools.im_approval_creation import run_approval_workflow
  791. sheet_meta, actions = run_approval_workflow(
  792. records=records,
  793. xlsx_output_dir=xlsx_output_dir,
  794. timeout_minutes=CREATION_APPROVAL_TIMEOUT_MINUTES,
  795. )
  796. logger.info(
  797. "[phase2] 审批完成 sheet_url=%s actions=%d/%d",
  798. sheet_meta.get("url"), len(actions), len(records),
  799. )
  800. return actions
  801. def run_once(config_date: str | None = None) -> dict:
  802. """完整主循环:Phase 0 → Phase 1 → Phase 2(若开关 True)→ Phase 3。"""
  803. run_started = now_in_timezone().isoformat()
  804. sync_stats: dict = {}
  805. # Phase -1:每日主流程启动时先同步飞书「自动化账户」配置表。
  806. logger.info("=" * 60)
  807. logger.info("[main] Phase -1 启动 — 同步飞书自动化账户配置")
  808. try:
  809. from sync_feishu_account_config import sync_from_feishu
  810. sync_stats = sync_from_feishu(config_date=config_date)
  811. logger.info("[main] Phase -1 完成: %s", sync_stats)
  812. except Exception as e:
  813. logger.exception("[main] Phase -1 同步飞书配置失败,本轮停止:%s", e)
  814. return {
  815. "run_started": run_started,
  816. "run_finished": now_in_timezone().isoformat(),
  817. "approval_required": CREATION_APPROVAL_REQUIRED,
  818. "sync_stats": {"error": str(e)},
  819. "phase0_created_ads": 0,
  820. "total": {"phase1_prepared": 0},
  821. }
  822. effective_config_date = date.fromisoformat(sync_stats["config_date"])
  823. # Phase 0:模块 A 建广告(满足每账户 ADS_PER_ACCOUNT 条)
  824. logger.info("=" * 60)
  825. if _env_flag("CREATION_SKIP_PHASE0"):
  826. logger.info("[main] CREATION_SKIP_PHASE0=True → 跳过模块 A 建广告")
  827. created_ads = []
  828. else:
  829. logger.info("[main] Phase 0 启动 — 模块 A 检查 + 建广告")
  830. created_ads = phase0_create_ads(config_date=effective_config_date)
  831. logger.info("[main] Phase 0 完成:本轮新建广告 %d 条", len(created_ads))
  832. _wait_created_ads_visible(created_ads)
  833. # Phase 1:模块 B 给所有广告(新+旧)补创意
  834. logger.info("=" * 60)
  835. logger.info("[main] Phase 1 启动 — 模块 B 给广告补创意")
  836. pending_records = phase1_prepare(config_date=effective_config_date)
  837. if not pending_records:
  838. logger.info("[main] Phase 1 无 pending records,主循环退出")
  839. return {
  840. "run_started": run_started,
  841. "run_finished": now_in_timezone().isoformat(),
  842. "approval_required": CREATION_APPROVAL_REQUIRED,
  843. "sync_stats": sync_stats,
  844. "phase0_created_ads": len(created_ads),
  845. "total": {"phase1_prepared": 0},
  846. }
  847. data_dir = _HERE / "outputs" / "data"
  848. pending_path = _write_pending_records(pending_records, data_dir)
  849. logger.info("[main] pending records 已写入: %s", pending_path)
  850. # Phase 2:审批(或 skip)
  851. if CREATION_APPROVAL_REQUIRED:
  852. logger.info("[main] CREATION_APPROVAL_REQUIRED=True → 进 Phase 2 飞书审批")
  853. actions = phase2_approval(pending_records, data_dir)
  854. for i, rec in enumerate(pending_records, start=1):
  855. rec["action"] = actions.get(i, "skip")
  856. else:
  857. logger.info("[main] CREATION_APPROVAL_REQUIRED=False → 全 records approve")
  858. for rec in pending_records:
  859. rec["action"] = "approve"
  860. # Phase 3:执行
  861. logger.info("=" * 60)
  862. logger.info("[main] Phase 3 执行启动")
  863. summary = apply_pending_records(pending_records)
  864. summary["approval_required"] = CREATION_APPROVAL_REQUIRED
  865. summary["pending_records_path"] = str(pending_path)
  866. summary_path = write_apply_summary(summary, data_dir)
  867. logger.info("[main] summary 已写入: %s", summary_path)
  868. # 发执行汇报
  869. _send_apply_summary_to_feishu(summary)
  870. summary["run_started"] = run_started
  871. summary["run_finished"] = now_in_timezone().isoformat()
  872. summary["sync_stats"] = sync_stats
  873. summary["phase0_created_ads"] = len(created_ads)
  874. summary["phase0_ads"] = created_ads
  875. return summary
  876. def main() -> int:
  877. parser = argparse.ArgumentParser(description="执行广告与创意创建主流程")
  878. parser.add_argument(
  879. "--config-date",
  880. help="执行指定飞书配置日期,格式 YYYYMMDD 或 YYYY-MM-DD;默认当天",
  881. )
  882. args = parser.parse_args()
  883. _setup_logging()
  884. logger.info("=" * 60)
  885. logger.info("模块 B 创意搭建子系统 — 主循环启动")
  886. logger.info("TARGET_CREATIVES_PER_AD = %d", TARGET_CREATIVES_PER_AD)
  887. logger.info("CREATION_APPROVAL_REQUIRED = %s", CREATION_APPROVAL_REQUIRED)
  888. logger.info("WHITELIST_ACCOUNTS = %s", WHITELIST_ACCOUNTS)
  889. logger.info("=" * 60)
  890. summary = run_once(config_date=args.config_date)
  891. t = summary.get("total") or {}
  892. logger.info("=" * 60)
  893. logger.info("[main] 主循环结束")
  894. if "phase1_prepared" in t:
  895. logger.info(" phase1_prepared = %d", t["phase1_prepared"])
  896. else:
  897. logger.info(" records = %d", t.get("records", 0))
  898. logger.info(" approved = %d", t.get("approved", 0))
  899. logger.info(" posted_ok = %d", t.get("posted_ok", 0))
  900. logger.info(" posted_failed = %d", t.get("posted_failed", 0))
  901. logger.info("=" * 60)
  902. return 0 if t.get("posted_failed", 0) == 0 else 1
  903. if __name__ == "__main__":
  904. sys.exit(main())