execute_creation_once.py 42 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018
  1. """模块 B 创意搭建子系统 — 主循环入口(P0-A / P0-C / P0-E,2026-06-09)。
  2. 数据流(三阶段,先审后挂):
  3. Phase 1 — 准备:
  4. 扫账户 → 关联点过滤(读调控当日 latest_decisions 排除 pause)
  5. → find_ads_needing_creatives
  6. → 对每条广告 prepare_one_creative_for_ad(召回 + 上传图 + xcx/save + build body)
  7. → pending_records (含完整 Phase 3 用的 _request_body)
  8. → 写 outputs/data/creation_pending_{date}.json(追溯 + 独立 apply 用)
  9. Phase 2 — 审批(若 CREATION_APPROVAL_REQUIRED=True):
  10. run_approval_workflow 生成 20 列 xlsx → 上传飞书 sheet → 发链接 → 轮询读决策列
  11. actions {row_idx: approve/reject/hold}
  12. → 写回 records["action"]
  13. Phase 3 — 执行:
  14. 对 action=approve 的 → POST /dynamic_creatives/add
  15. → 写 outputs/data/creation_run_{date}.json
  16. → 发飞书"执行汇报"消息
  17. 开关 CREATION_APPROVAL_REQUIRED=False 时跳过 Phase 2,全 records 直接 approve。
  18. """
  19. import argparse
  20. import json
  21. import logging
  22. import os
  23. import sys
  24. import time
  25. from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
  26. from datetime import date
  27. from pathlib import Path
  28. _HERE = Path(__file__).parent
  29. sys.path.insert(0, str(_HERE.parent.parent))
  30. sys.path.insert(0, str(_HERE))
  31. from dotenv import load_dotenv # noqa: E402
  32. load_dotenv(_HERE / ".env")
  33. from config import ( # noqa: E402
  34. ADS_PER_ACCOUNT,
  35. CREATION_APPROVAL_REQUIRED,
  36. CREATION_APPROVAL_TIMEOUT_MINUTES,
  37. CREATIVE_PREPARE_MAX_WORKERS,
  38. CREATIVE_PREPARE_TASK_BUFFER,
  39. MAX_SAME_LANDING_PER_AD_IN_RUN,
  40. TARGET_CREATIVES_PER_AD,
  41. WHITELIST_ACCOUNTS,
  42. get_creation_account_ids,
  43. now_in_timezone,
  44. )
  45. # config import 副作用会改 sys.path(把 im-client / agent/tools/builtin 顶到前面),
  46. # 强占 sys.path[0],只影响本入口自己的 import resolution。
  47. while str(_HERE) in sys.path:
  48. sys.path.remove(str(_HERE))
  49. sys.path.insert(0, str(_HERE))
  50. from tools.ad_creation import ( # noqa: E402
  51. build_ad_request_body,
  52. compute_fingerprint,
  53. enumerate_new_ad_candidates,
  54. post_ad_with_prepared_body,
  55. )
  56. from tools.audience_grant import ensure_account_audience_grant # noqa: E402
  57. from tools.creative_creation import ( # noqa: E402
  58. build_landing_candidate_pool,
  59. find_ads_needing_creatives,
  60. load_excluded_ad_ids_from_adjustment,
  61. prepare_one_creative_for_ad,
  62. )
  63. from tools.creative_material_usage import ( # noqa: E402
  64. load_recent_landing_usage_counts,
  65. load_recoverable_prepared_records,
  66. record_prepared_material_usage,
  67. )
  68. from tools.account_material_strategy import ( # noqa: E402
  69. MATERIAL_SOURCE_AI_GENERATED,
  70. MATERIAL_SOURCE_EXTERNAL_RECALL,
  71. load_account_material_strategy,
  72. )
  73. from tools.video_recall import get_account_crowd_package # noqa: E402
  74. from execute_creation_apply import ( # noqa: E402
  75. apply_pending_records,
  76. write_summary as write_apply_summary,
  77. _send_apply_summary_to_feishu,
  78. )
  79. logger = logging.getLogger("execute_creation_once")
  80. MATERIAL_SOURCE_HISTORY = "history"
  81. def _env_flag(name: str, default: bool = False) -> bool:
  82. raw = os.getenv(name)
  83. if raw is None:
  84. return default
  85. return raw.strip().lower() in {"1", "true", "yes", "y", "on"}
  86. def _env_csv_set(name: str) -> set[str]:
  87. return {
  88. item.strip()
  89. for item in os.getenv(name, "").split(",")
  90. if item.strip()
  91. }
  92. def _filter_creation_accounts(accounts: list[int], phase: str) -> list[int]:
  93. """Apply one-off run filters. Empty env means production default: no filter."""
  94. account_filter = {
  95. int(v)
  96. for v in _env_csv_set("CREATION_ONLY_ACCOUNT_IDS")
  97. if v.isdigit()
  98. }
  99. crowd_filter = _env_csv_set("CREATION_ONLY_CROWD_PACKAGES")
  100. if not account_filter and not crowd_filter:
  101. return accounts
  102. selected: list[int] = []
  103. for account_id in accounts:
  104. if account_filter and account_id not in account_filter:
  105. continue
  106. if crowd_filter:
  107. try:
  108. crowd_package = get_account_crowd_package(account_id)
  109. except Exception as e:
  110. logger.warning(
  111. "[%s] account=%d 临时过滤读取 crowd_package 失败,跳过:%s",
  112. phase, account_id, e,
  113. )
  114. continue
  115. if crowd_package not in crowd_filter:
  116. continue
  117. selected.append(account_id)
  118. logger.info(
  119. "[%s] 临时运行过滤 account_ids=%s crowd_packages=%s: %d -> %d",
  120. phase,
  121. sorted(account_filter) if account_filter else "ALL",
  122. sorted(crowd_filter) if crowd_filter else "ALL",
  123. len(accounts),
  124. len(selected),
  125. )
  126. return selected
  127. def _setup_logging() -> None:
  128. logging.basicConfig(
  129. level=logging.INFO,
  130. format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
  131. datefmt="%H:%M:%S",
  132. )
  133. for noisy in ("httpx", "httpcore", "config", "db.config"):
  134. logging.getLogger(noisy).setLevel(logging.WARNING)
  135. # SLS 上报(2026-06-11 接入,K8s pod 直发)— 配置缺失自动降级,不阻塞主链路
  136. try:
  137. from tools.sls_setup import attach_sls_handler
  138. attach_sls_handler()
  139. except Exception as e:
  140. logger.warning("[sls] 挂载异常(降级为本地 only):%s", e)
  141. def _fetch_existing_fingerprints_for_account(account_id: int) -> set[str]:
  142. """Task 27:拉账户下所有非 DELETED 广告 → 反推 fingerprint 集合(预校验唯一性)。
  143. 腾讯文档:NORMAL + SUSPEND 状态广告均占用唯一性槽位,DENIED 也算占用。
  144. list 接口默认返回所有非 DELETED 状态 ad,故无需显式 filter status。
  145. 若腾讯 list 接口某些字段不返回(field 实测差异),降级:跳过该条 + log 警告。
  146. 返回空集 → 上层退化为"不预校验",enumerate 仍能跑(原有行为)。
  147. """
  148. from tools.ad_api import _get
  149. fingerprints: set[str] = set()
  150. skipped = 0
  151. page = 1
  152. try:
  153. while True:
  154. r = _get("/adgroups/get", {
  155. "account_id": account_id, "page": page, "page_size": 100,
  156. "fields": [
  157. "adgroup_id", "configured_status", "site_set",
  158. "targeting", "scene_spec",
  159. ],
  160. })
  161. ads = (r.get("data") or {}).get("list") or []
  162. if not ads:
  163. break
  164. for ad in ads:
  165. tgt = ad.get("targeting") or {}
  166. sc = ad.get("scene_spec") or {}
  167. wp = sc.get("wechat_position") if sc else None
  168. site_set = ad.get("site_set") or []
  169. age = tgt.get("age") or []
  170. geo_regions = (tgt.get("geo_location") or {}).get("regions") or []
  171. custom_audience = tgt.get("custom_audience")
  172. if not site_set or not age:
  173. skipped += 1
  174. continue
  175. try:
  176. fp = compute_fingerprint(
  177. account_id=account_id,
  178. site_set=site_set,
  179. custom_audience=custom_audience,
  180. age=age,
  181. geo_regions=geo_regions,
  182. wechat_position=wp,
  183. )
  184. fingerprints.add(fp)
  185. except Exception as e:
  186. skipped += 1
  187. logger.debug(
  188. "[phase0] fingerprint 算失败 ad=%s: %s",
  189. ad.get("adgroup_id"), e,
  190. )
  191. if len(ads) < 100:
  192. break
  193. page += 1
  194. except Exception as e:
  195. logger.warning(
  196. "[phase0] account=%d 拉 fingerprint 集失败(降级为不预校验): %s",
  197. account_id, e,
  198. )
  199. return set()
  200. logger.info(
  201. "[phase0] 已存在 fingerprint=%d 个(skip 缺字段广告 %d 条)",
  202. len(fingerprints), skipped,
  203. )
  204. return fingerprints
  205. def phase0_create_ads(
  206. target_ads: int = ADS_PER_ACCOUNT,
  207. config_date: date | None = None,
  208. ) -> list[dict]:
  209. """Phase 0:对每账户检查广告数,不足则建到 target_ads 条(模块 A,2026-06-09 P1-G)。
  210. 流程:
  211. account → 查当前广告数(NORMAL+SUSPEND 算占用)→ 不足 target_ads →
  212. → 反查 fingerprint 集 → enumerate candidates(排除已有 fp)→
  213. → 飞书审批 → approve 项 → 真 POST /adgroups/add → 返回新建的 adgroup_id 列表
  214. Returns: 本次新建成功的广告 list[dict],每项含 {account_id, adgroup_id, adgroup_name, wechat_position}
  215. """
  216. from tools.ad_api import _get
  217. from tools.im_approval_ad_creation import run_ad_approval_workflow
  218. creation_accounts = get_creation_account_ids(config_date=config_date)
  219. if not creation_accounts:
  220. logger.error("[phase0] 待投放账户配置为空,退出")
  221. return []
  222. creation_accounts = _filter_creation_accounts(creation_accounts, "phase0")
  223. if not creation_accounts:
  224. logger.info("[phase0] 临时过滤后无待处理账户")
  225. return []
  226. # Task 26:NORMAL + SUSPEND 都算占用唯一性槽位(腾讯文档:删除前历史广告占槽位)
  227. OCCUPIED_STATUSES = {"AD_STATUS_NORMAL", "AD_STATUS_SUSPEND"}
  228. pending_ad_records: list[dict] = []
  229. for account_id in creation_accounts:
  230. logger.info("=" * 60)
  231. logger.info("[phase0] 账户 %d 处理开始", account_id)
  232. try:
  233. grant_result = ensure_account_audience_grant(account_id)
  234. logger.info(
  235. "[phase0] 人群包校验完成: status=%s audience_id=%s audience=%r",
  236. grant_result.get("status"),
  237. grant_result.get("audience_id"),
  238. grant_result.get("audience_name"),
  239. )
  240. except Exception as e:
  241. logger.exception("[phase0] account=%d 人群包授权/验证失败,跳过:%s", account_id, e)
  242. continue
  243. # Task 26:查所有非 DELETED 广告,NORMAL+SUSPEND 都算占用槽位
  244. try:
  245. r = _get("/adgroups/get", {
  246. "account_id": account_id, "page": 1, "page_size": 100,
  247. "fields": ["adgroup_id", "configured_status"],
  248. })
  249. all_ads = (r.get("data") or {}).get("list") or []
  250. normal_n = sum(1 for a in all_ads if a.get("configured_status") == "AD_STATUS_NORMAL")
  251. suspend_n = sum(1 for a in all_ads if a.get("configured_status") == "AD_STATUS_SUSPEND")
  252. occupied = [a for a in all_ads if a.get("configured_status") in OCCUPIED_STATUSES]
  253. logger.info(
  254. "[phase0] 广告数: 总 %d,NORMAL %d + SUSPEND %d = 占用 %d / 目标 %d",
  255. len(all_ads), normal_n, suspend_n, len(occupied), target_ads,
  256. )
  257. except Exception as e:
  258. logger.exception("[phase0] account=%d 查广告数失败: %s", account_id, e)
  259. continue
  260. to_create = max(0, target_ads - len(occupied))
  261. if to_create == 0:
  262. logger.info("[phase0] 占用槽位已满,跳过")
  263. continue
  264. # Task 27:fingerprint 预校验 — 反查现存广告的 fp 集合,enumerate 时排除
  265. existing_fps = _fetch_existing_fingerprints_for_account(account_id)
  266. # enumerate 候选(模块 A,差异化 wechat_position 有/无)
  267. try:
  268. candidates = enumerate_new_ad_candidates(
  269. account_id, count=to_create,
  270. existing_fingerprints=existing_fps,
  271. )
  272. except Exception as e:
  273. logger.exception("[phase0] account=%d enumerate 失败: %s", account_id, e)
  274. continue
  275. if not candidates:
  276. logger.warning("[phase0] enumerate 返回 0 条候选,跳过")
  277. continue
  278. today = now_in_timezone().strftime("%Y-%m-%d")
  279. for c in candidates:
  280. age_range = ",".join(
  281. f"{a.get('min')}-{a.get('max')}" for a in c.age
  282. )
  283. # 新建广告默认开启;实际投放仍受创意审核/账户/预算控制。
  284. # 这里显式传 NORMAL,避免未来默认值变化影响端到端创建语义。
  285. body = build_ad_request_body(c, configured_status="AD_STATUS_NORMAL")
  286. rec = {
  287. "approval_date": today,
  288. "account_id": c.account_id,
  289. "audience_tier_label": c.audience_tier_label,
  290. "adgroup_name": c.adgroup_name,
  291. "site_set": c.site_set,
  292. "delivery_version": c.delivery_version,
  293. "wechat_position": c.wechat_position,
  294. "bid_amount_fen": c.bid_amount_fen,
  295. "bid_scene": c.bid_scene,
  296. "custom_cost_cap_fen": c.custom_cost_cap_fen,
  297. "automatic_site_enabled": c.automatic_site_enabled,
  298. "age_range": age_range,
  299. "fingerprint": c.fingerprint,
  300. "_request_body": body,
  301. }
  302. pending_ad_records.append(rec)
  303. if not pending_ad_records:
  304. logger.info("[phase0] 无广告需新建,Phase 0 结束")
  305. return []
  306. # 飞书审批(如需要)
  307. data_dir = _HERE / "outputs" / "data"
  308. if CREATION_APPROVAL_REQUIRED:
  309. logger.info(
  310. "[phase0] CREATION_APPROVAL_REQUIRED=True → 飞书审批 %d 条广告候选",
  311. len(pending_ad_records),
  312. )
  313. sheet_meta, actions = run_ad_approval_workflow(
  314. records=pending_ad_records,
  315. xlsx_output_dir=data_dir,
  316. timeout_minutes=CREATION_APPROVAL_TIMEOUT_MINUTES,
  317. )
  318. for i, rec in enumerate(pending_ad_records, start=1):
  319. rec["action"] = actions.get(i, "skip")
  320. else:
  321. for rec in pending_ad_records:
  322. rec["action"] = "approve"
  323. # POST 真建 approve 项
  324. created: list[dict] = []
  325. for rec in pending_ad_records:
  326. if rec.get("action") != "approve":
  327. continue
  328. adgroup_id = post_ad_with_prepared_body(
  329. account_id=int(rec["account_id"]),
  330. body=rec["_request_body"],
  331. )
  332. if adgroup_id:
  333. created.append({
  334. "account_id": rec["account_id"],
  335. "adgroup_id": adgroup_id,
  336. "adgroup_name": rec["adgroup_name"],
  337. "wechat_position": rec.get("wechat_position"),
  338. })
  339. logger.info(
  340. "[phase0] 完成:候选 %d 条,approve %d 条,实际建 %d 条",
  341. len(pending_ad_records),
  342. sum(1 for r in pending_ad_records if r.get("action") == "approve"),
  343. len(created),
  344. )
  345. return created
  346. def _wait_created_ads_visible(created_ads: list[dict], timeout_seconds: int = 60) -> None:
  347. """等待刚创建的广告在 /adgroups/get 中可见。
  348. 腾讯 adgroups/add 成功后,list 接口存在短暂一致性延迟。若立刻进入 Phase 1,
  349. 可能只扫到部分新广告,导致同一轮只给一条广告补创意。
  350. """
  351. if not created_ads:
  352. return
  353. from tools.ad_api import _get
  354. by_account: dict[int, set[int]] = {}
  355. for ad in created_ads:
  356. by_account.setdefault(int(ad["account_id"]), set()).add(int(ad["adgroup_id"]))
  357. deadline = time.time() + timeout_seconds
  358. remaining = {acct: set(ids) for acct, ids in by_account.items()}
  359. while time.time() < deadline and any(remaining.values()):
  360. for account_id, ids in list(remaining.items()):
  361. if not ids:
  362. continue
  363. try:
  364. resp = _get("/adgroups/get", {
  365. "account_id": account_id,
  366. "page": 1,
  367. "page_size": 100,
  368. "fields": ["adgroup_id", "configured_status", "system_status"],
  369. })
  370. visible = {
  371. int(item["adgroup_id"])
  372. for item in ((resp.get("data") or {}).get("list") or [])
  373. if item.get("adgroup_id") is not None
  374. }
  375. ids.difference_update(visible)
  376. except Exception as e:
  377. logger.warning(
  378. "[phase0] 等待新广告可见失败 account=%d: %s",
  379. account_id, e,
  380. )
  381. if any(remaining.values()):
  382. logger.info("[phase0] 等待新广告在列表接口可见: %s", remaining)
  383. time.sleep(5)
  384. not_visible = {acct: sorted(ids) for acct, ids in remaining.items() if ids}
  385. if not_visible:
  386. logger.warning("[phase0] 部分新广告仍未在 list 可见,继续 Phase 1: %s", not_visible)
  387. else:
  388. logger.info("[phase0] 新建广告均已在 list 接口可见")
  389. def phase1_prepare(
  390. target_creatives: int = TARGET_CREATIVES_PER_AD,
  391. config_date: date | None = None,
  392. ) -> list[dict]:
  393. """Phase 1:扫账户 → 关联点过滤 → 准备 pending records(不 POST 腾讯)。
  394. Returns:
  395. pending_records — 每个元素是 prepare_one_creative_for_ad 返回的 dict
  396. """
  397. creation_accounts = get_creation_account_ids(config_date=config_date)
  398. if not creation_accounts:
  399. logger.error("[phase1] 待投放账户配置为空,Phase 1 退出")
  400. return []
  401. creation_accounts = _filter_creation_accounts(creation_accounts, "phase1")
  402. if not creation_accounts:
  403. logger.info("[phase1] 临时过滤后无待处理账户")
  404. return []
  405. excluded_ad_ids = load_excluded_ad_ids_from_adjustment()
  406. logger.info("[phase1] 关联点过滤集合 size=%d", len(excluded_ad_ids))
  407. pending_records: list[dict] = []
  408. # 素材排重只读取已有明确结果的 DB 记录。
  409. # pending/prepared 没有审批或投放结果,不进入排重。
  410. # 但同一轮审批候选需要做展示去重,避免表格里同 crowd_package 反复出现同一素材。
  411. # 这个 set 只存在内存里,不写历史排重库;进程结束即失效。
  412. display_used_material_ids_by_crowd: dict[str, set[str]] = {}
  413. # landing_video 按素材来源拆池。历史素材和 AI 生成素材互不占用,但池内都严格去重。
  414. landing_usage_counts_by_crowd: dict[str, dict[str, dict[int, int]]] = {}
  415. def accept_pending_record(
  416. rec: dict,
  417. *,
  418. crowd_package: str,
  419. display_excluded_material_ids: set[str],
  420. crowd_landing_counts: dict[str, dict[int, int]],
  421. landing_counts_for_ad: dict[int, int],
  422. landing_max_uses: int,
  423. recovered: bool = False,
  424. ) -> bool:
  425. material_id = rec.get("_material_id")
  426. if material_id and str(material_id) in display_excluded_material_ids:
  427. logger.info(
  428. "[phase1] pending 接收排重丢弃 crowd=%r material=%s%s",
  429. crowd_package, material_id,
  430. " recovered" if recovered else "",
  431. )
  432. return False
  433. landing_video_id = rec.get("landing_video_id")
  434. record_material_source = str(rec.get("material_source") or "")
  435. record_material_id = str(rec.get("_material_id") or "")
  436. if (
  437. record_material_source == MATERIAL_SOURCE_EXTERNAL_RECALL
  438. or record_material_id.startswith("external:")
  439. ):
  440. actual_material_source = MATERIAL_SOURCE_EXTERNAL_RECALL
  441. elif (
  442. record_material_source == MATERIAL_SOURCE_AI_GENERATED
  443. or record_material_id.startswith("ai:")
  444. ):
  445. actual_material_source = MATERIAL_SOURCE_AI_GENERATED
  446. else:
  447. actual_material_source = MATERIAL_SOURCE_HISTORY
  448. if landing_video_id is not None:
  449. landing_video_id = int(landing_video_id)
  450. if landing_counts_for_ad.get(landing_video_id, 0) >= MAX_SAME_LANDING_PER_AD_IN_RUN:
  451. logger.info(
  452. "[phase1] pending 接收排重丢弃 adgroup=%s landing=%d 同广告已达上限%s",
  453. rec.get("adgroup_id"), landing_video_id,
  454. " recovered" if recovered else "",
  455. )
  456. return False
  457. source_counts = crowd_landing_counts.setdefault(actual_material_source, {})
  458. if source_counts.get(landing_video_id, 0) >= landing_max_uses:
  459. logger.info(
  460. "[phase1] pending 接收排重丢弃 crowd=%r material_source=%s landing=%d 同人群包已达上限%s",
  461. crowd_package, actual_material_source, landing_video_id,
  462. " recovered" if recovered else "",
  463. )
  464. return False
  465. pending_records.append(rec)
  466. if material_id:
  467. display_excluded_material_ids.add(str(material_id))
  468. logger.info(
  469. "[phase1] 本轮展示去重登记 crowd=%r material=%s size=%d%s",
  470. crowd_package, material_id,
  471. len(display_excluded_material_ids),
  472. " recovered" if recovered else "",
  473. )
  474. if landing_video_id is not None:
  475. landing_counts_for_ad[landing_video_id] = (
  476. landing_counts_for_ad.get(landing_video_id, 0) + 1
  477. )
  478. source_counts = crowd_landing_counts.setdefault(actual_material_source, {})
  479. source_counts[landing_video_id] = source_counts.get(landing_video_id, 0) + 1
  480. logger.info(
  481. "[phase1] 同人群包 landing 使用登记 crowd=%r material_source=%s landing=%d count=%d max=%d%s",
  482. crowd_package, actual_material_source, landing_video_id,
  483. source_counts[landing_video_id],
  484. landing_max_uses,
  485. " recovered" if recovered else "",
  486. )
  487. logger.info(
  488. "[phase1] 本轮同广告 landing 计数 adgroup=%s landing=%d count=%d limit=%d%s",
  489. rec.get("adgroup_id"), landing_video_id,
  490. landing_counts_for_ad[landing_video_id],
  491. MAX_SAME_LANDING_PER_AD_IN_RUN,
  492. " recovered" if recovered else "",
  493. )
  494. if not recovered:
  495. try:
  496. record_prepared_material_usage(rec)
  497. except Exception as e:
  498. logger.warning(
  499. "[phase1] material usage 记录失败 account=%s adgroup=%s material=%s:%s",
  500. rec.get("account_id"), rec.get("adgroup_id"),
  501. rec.get("_material_id"), e,
  502. )
  503. return True
  504. for account_id in creation_accounts:
  505. logger.info("=" * 60)
  506. logger.info("[phase1] 账户 %d 处理开始", account_id)
  507. try:
  508. ads = find_ads_needing_creatives(account_id, min_creatives=target_creatives)
  509. except Exception as e:
  510. logger.exception("[phase1] account=%d find_ads 失败,跳过:%s", account_id, e)
  511. continue
  512. ads_after_filter = [a for a in ads if a["adgroup_id"] not in excluded_ad_ids]
  513. excluded_count = len(ads) - len(ads_after_filter)
  514. if excluded_count:
  515. logger.info(
  516. "[phase1] account=%d 关联点过滤排除 %d 条",
  517. account_id, excluded_count,
  518. )
  519. if not ads_after_filter:
  520. logger.info("[phase1] account=%d 无广告需补创意", account_id)
  521. continue
  522. try:
  523. crowd_package = get_account_crowd_package(account_id)
  524. except Exception as e:
  525. logger.exception(
  526. "[phase1] account=%d 读取 crowd_package 失败,跳过:%s",
  527. account_id, e,
  528. )
  529. continue
  530. display_excluded_material_ids = display_used_material_ids_by_crowd.setdefault(
  531. crowd_package, set(),
  532. )
  533. if crowd_package not in landing_usage_counts_by_crowd:
  534. try:
  535. landing_usage_counts_by_crowd[crowd_package] = load_recent_landing_usage_counts(
  536. crowd_package,
  537. )
  538. except Exception as e:
  539. logger.warning(
  540. "[phase1] crowd=%r 读取 landing 使用历史失败,仅使用本轮排重:%s",
  541. crowd_package, e,
  542. )
  543. landing_usage_counts_by_crowd[crowd_package] = {
  544. MATERIAL_SOURCE_HISTORY: {},
  545. MATERIAL_SOURCE_AI_GENERATED: {},
  546. MATERIAL_SOURCE_EXTERNAL_RECALL: {},
  547. }
  548. logger.info(
  549. "[phase1] crowd=%r 近期 landing 使用 history=%d ai_generated=%d external_recall=%d",
  550. crowd_package,
  551. len(landing_usage_counts_by_crowd[crowd_package].get(MATERIAL_SOURCE_HISTORY, {})),
  552. len(landing_usage_counts_by_crowd[crowd_package].get(MATERIAL_SOURCE_AI_GENERATED, {})),
  553. len(landing_usage_counts_by_crowd[crowd_package].get(MATERIAL_SOURCE_EXTERNAL_RECALL, {})),
  554. )
  555. crowd_landing_counts = landing_usage_counts_by_crowd[crowd_package]
  556. crowd_landing_counts.setdefault(MATERIAL_SOURCE_HISTORY, {})
  557. crowd_landing_counts.setdefault(MATERIAL_SOURCE_AI_GENERATED, {})
  558. crowd_landing_counts.setdefault(MATERIAL_SOURCE_EXTERNAL_RECALL, {})
  559. try:
  560. material_strategy = load_account_material_strategy(account_id)
  561. except Exception as e:
  562. logger.warning(
  563. "[phase1] account=%d 读取素材策略失败,按 history 排重:%s",
  564. account_id, e,
  565. )
  566. material_strategy = None
  567. requested_material_source = (
  568. material_strategy.material_source
  569. if material_strategy is not None
  570. else MATERIAL_SOURCE_HISTORY
  571. )
  572. landing_max_uses = MAX_SAME_LANDING_PER_AD_IN_RUN
  573. landing_candidate_pool = None
  574. logger.info(
  575. "[phase1] account=%d landing 排重池=%s max_uses=%d",
  576. account_id, requested_material_source, landing_max_uses,
  577. )
  578. for ad in ads_after_filter:
  579. adgroup_id = ad["adgroup_id"]
  580. already_have = ad["creative_count"]
  581. to_add = max(0, target_creatives - already_have)
  582. prepared_for_ad = 0
  583. failed_prepare_for_ad = 0
  584. failed_landing_ids_for_ad: set[int] = set()
  585. landing_counts_for_ad: dict[int, int] = {}
  586. logger.info(
  587. "[phase1] adgroup=%d(have=%d need=%d)",
  588. adgroup_id, already_have, to_add,
  589. )
  590. recovered_records = load_recoverable_prepared_records(
  591. account_id=account_id,
  592. adgroup_id=adgroup_id,
  593. crowd_package=crowd_package,
  594. material_source=requested_material_source,
  595. limit=to_add,
  596. )
  597. recovered_accepted = 0
  598. for rec in recovered_records:
  599. if accept_pending_record(
  600. rec,
  601. crowd_package=crowd_package,
  602. display_excluded_material_ids=display_excluded_material_ids,
  603. crowd_landing_counts=crowd_landing_counts,
  604. landing_counts_for_ad=landing_counts_for_ad,
  605. landing_max_uses=landing_max_uses,
  606. recovered=True,
  607. ):
  608. recovered_accepted += 1
  609. if recovered_records:
  610. prepared_for_ad += recovered_accepted
  611. logger.info(
  612. "[phase1] adgroup=%d 恢复未提交 prepared records=%d accepted=%d remaining=%d",
  613. adgroup_id, len(recovered_records), recovered_accepted,
  614. max(0, to_add - prepared_for_ad),
  615. )
  616. def prepare_attempt(
  617. *,
  618. attempt_no: int,
  619. excluded_material_ids_snapshot: set[str],
  620. excluded_landing_ids_snapshot: set[int],
  621. ) -> dict | None:
  622. nonlocal landing_candidate_pool
  623. try:
  624. if landing_candidate_pool is None:
  625. landing_candidate_pool = build_landing_candidate_pool(account_id)
  626. return prepare_one_creative_for_ad(
  627. account_id, adgroup_id,
  628. excluded_material_ids=excluded_material_ids_snapshot,
  629. excluded_landing_ids=excluded_landing_ids_snapshot,
  630. landing_candidates=landing_candidate_pool,
  631. failed_landing_ids=failed_landing_ids_for_ad,
  632. )
  633. except Exception as e:
  634. logger.exception(
  635. "[phase1] adgroup=%d attempt=%d 准备失败:%s",
  636. adgroup_id, attempt_no, e,
  637. )
  638. return None
  639. def current_excluded_landing_ids() -> set[int]:
  640. landing_excluded_for_ad = {
  641. vid
  642. for vid, count in landing_counts_for_ad.items()
  643. if count >= MAX_SAME_LANDING_PER_AD_IN_RUN
  644. }
  645. landing_excluded_for_source = {
  646. vid
  647. for vid, count in crowd_landing_counts
  648. .get(requested_material_source, {})
  649. .items()
  650. if count >= landing_max_uses
  651. }
  652. return landing_excluded_for_source | landing_excluded_for_ad
  653. remaining_to_add = max(0, to_add - prepared_for_ad)
  654. max_workers = max(1, int(CREATIVE_PREPARE_MAX_WORKERS))
  655. if max_workers > 1:
  656. logger.warning(
  657. "[phase1] CREATIVE_PREPARE_MAX_WORKERS=%d 已配置但当前 prepare_one_creative_for_ad "
  658. "包含图片上传/xcx-save 等外部副作用,为避免重复创建落地计划,本轮强制串行执行",
  659. max_workers,
  660. )
  661. max_workers = 1
  662. task_buffer = max(remaining_to_add, int(CREATIVE_PREPARE_TASK_BUFFER))
  663. task_limit = min(max(0, remaining_to_add * 2), task_buffer)
  664. if remaining_to_add > 0 and max_workers > 1:
  665. logger.info(
  666. "[phase1] adgroup=%d 并行准备 start remaining=%d workers=%d task_limit=%d",
  667. adgroup_id, remaining_to_add, max_workers, task_limit,
  668. )
  669. submitted = 0
  670. accepted_parallel = 0
  671. dropped_parallel = 0
  672. failed_parallel = 0
  673. started_at = time.monotonic()
  674. with ThreadPoolExecutor(max_workers=max_workers) as executor:
  675. futures = {}
  676. def submit_next() -> None:
  677. nonlocal submitted
  678. # 已接收 + 运行中任务不超过缺口,避免并发准备产出无法追踪的多余副作用。
  679. if (
  680. submitted >= task_limit
  681. or prepared_for_ad + len(futures) >= to_add
  682. ):
  683. return
  684. submitted += 1
  685. attempt_no = submitted
  686. future = executor.submit(
  687. prepare_attempt,
  688. attempt_no=attempt_no,
  689. excluded_material_ids_snapshot=set(display_excluded_material_ids),
  690. excluded_landing_ids_snapshot=current_excluded_landing_ids(),
  691. )
  692. futures[future] = attempt_no
  693. for _ in range(min(max_workers, task_limit)):
  694. submit_next()
  695. while futures and prepared_for_ad < to_add:
  696. done, _ = wait(futures, return_when=FIRST_COMPLETED)
  697. for future in done:
  698. futures.pop(future, None)
  699. rec = future.result()
  700. if not rec:
  701. failed_prepare_for_ad += 1
  702. failed_parallel += 1
  703. elif accept_pending_record(
  704. rec,
  705. crowd_package=crowd_package,
  706. display_excluded_material_ids=display_excluded_material_ids,
  707. crowd_landing_counts=crowd_landing_counts,
  708. landing_counts_for_ad=landing_counts_for_ad,
  709. landing_max_uses=landing_max_uses,
  710. ):
  711. prepared_for_ad += 1
  712. accepted_parallel += 1
  713. else:
  714. dropped_parallel += 1
  715. if prepared_for_ad < to_add:
  716. submit_next()
  717. else:
  718. break
  719. if prepared_for_ad >= to_add:
  720. logger.info(
  721. "[phase1] adgroup=%d 并行准备 target reached accepted=%d",
  722. adgroup_id, accepted_parallel,
  723. )
  724. cancelled = 0
  725. for pending_future in futures:
  726. if pending_future.cancel():
  727. cancelled += 1
  728. if cancelled:
  729. logger.info(
  730. "[phase1] adgroup=%d 并行准备 cancelled_pending=%d",
  731. adgroup_id, cancelled,
  732. )
  733. for future in futures:
  734. if future.cancelled():
  735. continue
  736. rec = future.result()
  737. if rec:
  738. dropped_parallel += 1
  739. logger.info(
  740. "[phase1] adgroup=%d 并行结果丢弃:target reached material=%s landing=%s",
  741. adgroup_id, rec.get("_material_id"), rec.get("landing_video_id"),
  742. )
  743. logger.info(
  744. "[phase1] adgroup=%d 并行准备 done submitted=%d accepted=%d dropped=%d failed=%d elapsed=%.1fs",
  745. adgroup_id, submitted, accepted_parallel, dropped_parallel,
  746. failed_parallel, time.monotonic() - started_at,
  747. )
  748. else:
  749. for attempt_no in range(1, remaining_to_add + 1):
  750. rec = prepare_attempt(
  751. attempt_no=attempt_no,
  752. excluded_material_ids_snapshot=set(display_excluded_material_ids),
  753. excluded_landing_ids_snapshot=current_excluded_landing_ids(),
  754. )
  755. if rec:
  756. if accept_pending_record(
  757. rec,
  758. crowd_package=crowd_package,
  759. display_excluded_material_ids=display_excluded_material_ids,
  760. crowd_landing_counts=crowd_landing_counts,
  761. landing_counts_for_ad=landing_counts_for_ad,
  762. landing_max_uses=landing_max_uses,
  763. ):
  764. prepared_for_ad += 1
  765. else:
  766. failed_prepare_for_ad += 1
  767. # 2026-06-10 用户要求:单条 prepare 失败 → continue 不 break
  768. # 同广告剩余 to_add 创意还能继续试,不被一次失败拖累
  769. logger.info(
  770. "[phase1] adgroup=%d 本条创意 prepare 失败,试下一条",
  771. adgroup_id,
  772. )
  773. logger.info(
  774. "[phase1] adgroup=%d 补创意完成 target=%d have_before=%d "
  775. "planned=%d prepared=%d failed_prepare=%d",
  776. adgroup_id, target_creatives, already_have, to_add,
  777. prepared_for_ad, failed_prepare_for_ad,
  778. )
  779. if prepared_for_ad < to_add:
  780. logger.warning(
  781. "[phase1] adgroup=%d 未补满:缺口=%d。详细原因见上方 "
  782. "prepare_one_creative source_summary/失败日志",
  783. adgroup_id, to_add - prepared_for_ad,
  784. )
  785. logger.info("=" * 60)
  786. logger.info("[phase1] 准备完成,共 %d 条 pending records", len(pending_records))
  787. return pending_records
  788. def _write_pending_records(records: list[dict], output_dir: Path) -> Path:
  789. """落 Phase 1 产物 JSON(供 Phase 3 独立 apply 用 / 追溯)。"""
  790. output_dir.mkdir(parents=True, exist_ok=True)
  791. now = now_in_timezone()
  792. date_str = now.strftime("%Y%m%d")
  793. ts = now.strftime("%H%M%S")
  794. out_path = output_dir / f"creation_pending_{date_str}_{ts}.json"
  795. with open(out_path, "w", encoding="utf-8") as f:
  796. json.dump(records, f, ensure_ascii=False, indent=2)
  797. return out_path
  798. def phase2_approval(records: list[dict], xlsx_output_dir: Path) -> dict[int, str]:
  799. """Phase 2:飞书审批(若需要)。
  800. Returns: actions {row_idx: action}, 1-based row_idx
  801. """
  802. from tools.im_approval_creation import run_approval_workflow
  803. sheet_meta, actions = run_approval_workflow(
  804. records=records,
  805. xlsx_output_dir=xlsx_output_dir,
  806. timeout_minutes=CREATION_APPROVAL_TIMEOUT_MINUTES,
  807. )
  808. logger.info(
  809. "[phase2] 审批完成 sheet_url=%s actions=%d/%d",
  810. sheet_meta.get("url"), len(actions), len(records),
  811. )
  812. return actions
  813. def run_once(config_date: str | None = None) -> dict:
  814. """完整主循环:Phase 0 → Phase 1 → Phase 2(若开关 True)→ Phase 3。"""
  815. run_started = now_in_timezone().isoformat()
  816. sync_stats: dict = {}
  817. # Phase -1:每日主流程启动时先同步飞书「自动化账户」配置表。
  818. logger.info("=" * 60)
  819. logger.info("[main] Phase -1 启动 — 同步飞书自动化账户配置")
  820. try:
  821. from sync_feishu_account_config import sync_from_feishu
  822. sync_stats = sync_from_feishu(config_date=config_date)
  823. logger.info("[main] Phase -1 完成: %s", sync_stats)
  824. except Exception as e:
  825. logger.exception("[main] Phase -1 同步飞书配置失败,本轮停止:%s", e)
  826. return {
  827. "run_started": run_started,
  828. "run_finished": now_in_timezone().isoformat(),
  829. "approval_required": CREATION_APPROVAL_REQUIRED,
  830. "sync_stats": {"error": str(e)},
  831. "phase0_created_ads": 0,
  832. "total": {"phase1_prepared": 0},
  833. }
  834. effective_config_date = date.fromisoformat(sync_stats["config_date"])
  835. # Phase 0:模块 A 建广告(满足每账户 ADS_PER_ACCOUNT 条)
  836. logger.info("=" * 60)
  837. if _env_flag("CREATION_SKIP_PHASE0"):
  838. logger.info("[main] CREATION_SKIP_PHASE0=True → 跳过模块 A 建广告")
  839. created_ads = []
  840. else:
  841. logger.info("[main] Phase 0 启动 — 模块 A 检查 + 建广告")
  842. created_ads = phase0_create_ads(config_date=effective_config_date)
  843. logger.info("[main] Phase 0 完成:本轮新建广告 %d 条", len(created_ads))
  844. _wait_created_ads_visible(created_ads)
  845. # Phase 1:模块 B 给所有广告(新+旧)补创意
  846. logger.info("=" * 60)
  847. logger.info("[main] Phase 1 启动 — 模块 B 给广告补创意")
  848. pending_records = phase1_prepare(config_date=effective_config_date)
  849. if not pending_records:
  850. logger.info("[main] Phase 1 无 pending records,主循环退出")
  851. return {
  852. "run_started": run_started,
  853. "run_finished": now_in_timezone().isoformat(),
  854. "approval_required": CREATION_APPROVAL_REQUIRED,
  855. "sync_stats": sync_stats,
  856. "phase0_created_ads": len(created_ads),
  857. "total": {"phase1_prepared": 0},
  858. }
  859. data_dir = _HERE / "outputs" / "data"
  860. pending_path = _write_pending_records(pending_records, data_dir)
  861. logger.info("[main] pending records 已写入: %s", pending_path)
  862. # Phase 2:审批(或 skip)
  863. if CREATION_APPROVAL_REQUIRED:
  864. logger.info("[main] CREATION_APPROVAL_REQUIRED=True → 进 Phase 2 飞书审批")
  865. actions = phase2_approval(pending_records, data_dir)
  866. for i, rec in enumerate(pending_records, start=1):
  867. rec["action"] = actions.get(i, "skip")
  868. else:
  869. logger.info("[main] CREATION_APPROVAL_REQUIRED=False → 全 records approve")
  870. for rec in pending_records:
  871. rec["action"] = "approve"
  872. # Phase 3:执行
  873. logger.info("=" * 60)
  874. logger.info("[main] Phase 3 执行启动")
  875. summary = apply_pending_records(pending_records)
  876. summary["approval_required"] = CREATION_APPROVAL_REQUIRED
  877. summary["pending_records_path"] = str(pending_path)
  878. summary_path = write_apply_summary(summary, data_dir)
  879. logger.info("[main] summary 已写入: %s", summary_path)
  880. # 发执行汇报
  881. _send_apply_summary_to_feishu(summary)
  882. summary["run_started"] = run_started
  883. summary["run_finished"] = now_in_timezone().isoformat()
  884. summary["sync_stats"] = sync_stats
  885. summary["phase0_created_ads"] = len(created_ads)
  886. summary["phase0_ads"] = created_ads
  887. return summary
  888. def main() -> int:
  889. parser = argparse.ArgumentParser(description="执行广告与创意创建主流程")
  890. parser.add_argument(
  891. "--config-date",
  892. help="执行指定飞书配置日期,格式 YYYYMMDD 或 YYYY-MM-DD;默认当天",
  893. )
  894. args = parser.parse_args()
  895. _setup_logging()
  896. logger.info("=" * 60)
  897. logger.info("模块 B 创意搭建子系统 — 主循环启动")
  898. logger.info("TARGET_CREATIVES_PER_AD = %d", TARGET_CREATIVES_PER_AD)
  899. logger.info("CREATION_APPROVAL_REQUIRED = %s", CREATION_APPROVAL_REQUIRED)
  900. logger.info("WHITELIST_ACCOUNTS = %s", WHITELIST_ACCOUNTS)
  901. logger.info("=" * 60)
  902. summary = run_once(config_date=args.config_date)
  903. t = summary.get("total") or {}
  904. logger.info("=" * 60)
  905. logger.info("[main] 主循环结束")
  906. if "phase1_prepared" in t:
  907. logger.info(" phase1_prepared = %d", t["phase1_prepared"])
  908. else:
  909. logger.info(" records = %d", t.get("records", 0))
  910. logger.info(" approved = %d", t.get("approved", 0))
  911. logger.info(" posted_ok = %d", t.get("posted_ok", 0))
  912. logger.info(" posted_failed = %d", t.get("posted_failed", 0))
  913. logger.info("=" * 60)
  914. return 0 if t.get("posted_failed", 0) == 0 else 1
  915. if __name__ == "__main__":
  916. sys.exit(main())