|
|
@@ -6,6 +6,7 @@ import hashlib
|
|
|
import json
|
|
|
import logging
|
|
|
import os
|
|
|
+import threading
|
|
|
from collections import defaultdict
|
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
from datetime import date, datetime, timedelta
|
|
|
@@ -43,6 +44,7 @@ DELETE_CREATIVE = "DELETE_CREATIVE"
|
|
|
ALERT_ONLY = "ALERT_ONLY"
|
|
|
DEFAULT_PARTIAL_CREATIVE_COST_THRESHOLD_YUAN = 50.0
|
|
|
DEFAULT_WECHAT_MINI_PROGRAM_COST_THRESHOLD_YUAN = 100.0
|
|
|
+DEFAULT_DELETE_CLAIM_STALE_MINUTES = 30
|
|
|
AGENCY_REPORT_COLUMNS = (
|
|
|
"代理名称",
|
|
|
"账户ID",
|
|
|
@@ -370,6 +372,36 @@ def _resolve_agency(
|
|
|
|
|
|
|
|
|
def upsert_cleanup_candidate(record: dict[str, Any]) -> dict[str, Any]:
|
|
|
+ component_ids_json = _json(record.get("component_ids") or [])
|
|
|
+ element_ids_json = _json(record.get("element_ids") or [])
|
|
|
+ review_result_json = _json(record.get("review_result") or {})
|
|
|
+ pre_state_json = _json(record.get("pre_state") or {})
|
|
|
+ cleanup_status = (
|
|
|
+ "ALERT_PENDING"
|
|
|
+ if record["cleanup_action"] == ALERT_ONLY
|
|
|
+ else "DISCOVERED"
|
|
|
+ )
|
|
|
+ insert_values = (
|
|
|
+ record["account_id"],
|
|
|
+ record.get("account_name"),
|
|
|
+ record.get("agency_name"),
|
|
|
+ record["adgroup_id"],
|
|
|
+ record.get("adgroup_name"),
|
|
|
+ record["dynamic_creative_id"],
|
|
|
+ record.get("dynamic_creative_name"),
|
|
|
+ record["check_date"],
|
|
|
+ record["cleanup_action"],
|
|
|
+ component_ids_json,
|
|
|
+ element_ids_json,
|
|
|
+ record.get("recent_cost_fen"),
|
|
|
+ record.get("cost_start_date"),
|
|
|
+ record.get("cost_end_date"),
|
|
|
+ record.get("action_reason") or record["reject_reason"],
|
|
|
+ record["reject_reason"],
|
|
|
+ review_result_json,
|
|
|
+ pre_state_json,
|
|
|
+ cleanup_status,
|
|
|
+ )
|
|
|
connection = get_connection()
|
|
|
try:
|
|
|
with connection.cursor() as cursor:
|
|
|
@@ -384,101 +416,108 @@ def upsert_cleanup_candidate(record: dict[str, Any]) -> dict[str, Any]:
|
|
|
cost_start_date, cost_end_date, action_reason, reject_reason,
|
|
|
review_result_json, pre_state_json, cleanup_status)
|
|
|
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
|
|
|
- ON DUPLICATE KEY UPDATE
|
|
|
- account_name=COALESCE(NULLIF(VALUES(account_name),''), account_name),
|
|
|
- agency_name=COALESCE(NULLIF(VALUES(agency_name),''), agency_name),
|
|
|
- adgroup_id=VALUES(adgroup_id),
|
|
|
- adgroup_name=COALESCE(NULLIF(VALUES(adgroup_name),''), adgroup_name),
|
|
|
- dynamic_creative_name=COALESCE(
|
|
|
- NULLIF(VALUES(dynamic_creative_name),''), dynamic_creative_name
|
|
|
+ ON DUPLICATE KEY UPDATE id=LAST_INSERT_ID(id)
|
|
|
+ """,
|
|
|
+ insert_values,
|
|
|
+ )
|
|
|
+ cursor.execute(
|
|
|
+ """
|
|
|
+ UPDATE creative_rejection_cleanup_item AS item
|
|
|
+ JOIN (
|
|
|
+ SELECT
|
|
|
+ %s AS account_id, %s AS account_name, %s AS agency_name,
|
|
|
+ %s AS adgroup_id, %s AS adgroup_name,
|
|
|
+ %s AS dynamic_creative_id, %s AS dynamic_creative_name,
|
|
|
+ %s AS check_date, %s AS cleanup_action,
|
|
|
+ %s AS target_component_ids_json,
|
|
|
+ %s AS target_element_ids_json, %s AS recent_cost_fen,
|
|
|
+ %s AS cost_start_date, %s AS cost_end_date,
|
|
|
+ %s AS action_reason, %s AS reject_reason,
|
|
|
+ %s AS review_result_json, %s AS pre_state_json,
|
|
|
+ %s AS cleanup_status
|
|
|
+ ) AS incoming
|
|
|
+ ON incoming.account_id=item.account_id
|
|
|
+ AND incoming.dynamic_creative_id=item.dynamic_creative_id
|
|
|
+ AND incoming.check_date=item.check_date
|
|
|
+ SET
|
|
|
+ item.account_name=COALESCE(
|
|
|
+ NULLIF(incoming.account_name,''), item.account_name
|
|
|
+ ),
|
|
|
+ item.agency_name=COALESCE(
|
|
|
+ NULLIF(incoming.agency_name,''), item.agency_name
|
|
|
+ ),
|
|
|
+ item.adgroup_id=incoming.adgroup_id,
|
|
|
+ item.adgroup_name=COALESCE(
|
|
|
+ NULLIF(incoming.adgroup_name,''), item.adgroup_name
|
|
|
),
|
|
|
- action_reason=VALUES(action_reason),
|
|
|
- reject_reason=VALUES(reject_reason),
|
|
|
- review_result_json=VALUES(review_result_json),
|
|
|
- pre_state_json=VALUES(pre_state_json),
|
|
|
+ item.dynamic_creative_name=COALESCE(
|
|
|
+ NULLIF(incoming.dynamic_creative_name,''),
|
|
|
+ item.dynamic_creative_name
|
|
|
+ ),
|
|
|
+ item.action_reason=incoming.action_reason,
|
|
|
+ item.reject_reason=incoming.reject_reason,
|
|
|
+ item.review_result_json=incoming.review_result_json,
|
|
|
+ item.pre_state_json=incoming.pre_state_json,
|
|
|
notified_at=CASE
|
|
|
- WHEN NOT (cleanup_action <=> VALUES(cleanup_action))
|
|
|
- OR NOT (target_component_ids_json <=> VALUES(target_component_ids_json))
|
|
|
- OR NOT (target_element_ids_json <=> VALUES(target_element_ids_json))
|
|
|
- OR NOT (recent_cost_fen <=> VALUES(recent_cost_fen))
|
|
|
- OR NOT (cost_end_date <=> VALUES(cost_end_date))
|
|
|
- THEN NULL ELSE notified_at
|
|
|
+ WHEN NOT (item.cleanup_action <=> incoming.cleanup_action)
|
|
|
+ OR NOT (item.target_component_ids_json <=> incoming.target_component_ids_json)
|
|
|
+ OR NOT (item.target_element_ids_json <=> incoming.target_element_ids_json)
|
|
|
+ OR NOT (item.recent_cost_fen <=> incoming.recent_cost_fen)
|
|
|
+ OR NOT (item.cost_end_date <=> incoming.cost_end_date)
|
|
|
+ THEN NULL ELSE item.notified_at
|
|
|
END,
|
|
|
- agency_notified_at=CASE
|
|
|
- WHEN NOT (cleanup_action <=> VALUES(cleanup_action))
|
|
|
- OR NOT (target_component_ids_json <=> VALUES(target_component_ids_json))
|
|
|
- OR NOT (target_element_ids_json <=> VALUES(target_element_ids_json))
|
|
|
- OR NOT (recent_cost_fen <=> VALUES(recent_cost_fen))
|
|
|
- OR NOT (cost_end_date <=> VALUES(cost_end_date))
|
|
|
- THEN NULL ELSE agency_notified_at
|
|
|
+ item.agency_notified_at=CASE
|
|
|
+ WHEN NOT (item.cleanup_action <=> incoming.cleanup_action)
|
|
|
+ OR NOT (item.target_component_ids_json <=> incoming.target_component_ids_json)
|
|
|
+ OR NOT (item.target_element_ids_json <=> incoming.target_element_ids_json)
|
|
|
+ OR NOT (item.recent_cost_fen <=> incoming.recent_cost_fen)
|
|
|
+ OR NOT (item.cost_end_date <=> incoming.cost_end_date)
|
|
|
+ THEN NULL ELSE item.agency_notified_at
|
|
|
END,
|
|
|
- operator_notified_at=CASE
|
|
|
- WHEN NOT (cleanup_action <=> VALUES(cleanup_action))
|
|
|
- OR NOT (target_component_ids_json <=> VALUES(target_component_ids_json))
|
|
|
- OR NOT (target_element_ids_json <=> VALUES(target_element_ids_json))
|
|
|
- OR NOT (recent_cost_fen <=> VALUES(recent_cost_fen))
|
|
|
- OR NOT (cost_end_date <=> VALUES(cost_end_date))
|
|
|
- THEN NULL ELSE operator_notified_at
|
|
|
+ item.operator_notified_at=CASE
|
|
|
+ WHEN NOT (item.cleanup_action <=> incoming.cleanup_action)
|
|
|
+ OR NOT (item.target_component_ids_json <=> incoming.target_component_ids_json)
|
|
|
+ OR NOT (item.target_element_ids_json <=> incoming.target_element_ids_json)
|
|
|
+ OR NOT (item.recent_cost_fen <=> incoming.recent_cost_fen)
|
|
|
+ OR NOT (item.cost_end_date <=> incoming.cost_end_date)
|
|
|
+ THEN NULL ELSE item.operator_notified_at
|
|
|
END,
|
|
|
- deleted_at=CASE
|
|
|
- WHEN NOT (cleanup_action <=> VALUES(cleanup_action))
|
|
|
- OR NOT (target_component_ids_json <=> VALUES(target_component_ids_json))
|
|
|
- OR NOT (target_element_ids_json <=> VALUES(target_element_ids_json))
|
|
|
- OR NOT (recent_cost_fen <=> VALUES(recent_cost_fen))
|
|
|
- OR NOT (cost_end_date <=> VALUES(cost_end_date))
|
|
|
- THEN NULL ELSE deleted_at
|
|
|
+ item.deleted_at=CASE
|
|
|
+ WHEN NOT (item.cleanup_action <=> incoming.cleanup_action)
|
|
|
+ OR NOT (item.target_component_ids_json <=> incoming.target_component_ids_json)
|
|
|
+ OR NOT (item.target_element_ids_json <=> incoming.target_element_ids_json)
|
|
|
+ OR NOT (item.recent_cost_fen <=> incoming.recent_cost_fen)
|
|
|
+ OR NOT (item.cost_end_date <=> incoming.cost_end_date)
|
|
|
+ THEN NULL ELSE item.deleted_at
|
|
|
END,
|
|
|
- readback_json=CASE
|
|
|
- WHEN NOT (cleanup_action <=> VALUES(cleanup_action))
|
|
|
- OR NOT (target_component_ids_json <=> VALUES(target_component_ids_json))
|
|
|
- OR NOT (target_element_ids_json <=> VALUES(target_element_ids_json))
|
|
|
- OR NOT (recent_cost_fen <=> VALUES(recent_cost_fen))
|
|
|
- OR NOT (cost_end_date <=> VALUES(cost_end_date))
|
|
|
- THEN NULL ELSE readback_json
|
|
|
+ item.readback_json=CASE
|
|
|
+ WHEN NOT (item.cleanup_action <=> incoming.cleanup_action)
|
|
|
+ OR NOT (item.target_component_ids_json <=> incoming.target_component_ids_json)
|
|
|
+ OR NOT (item.target_element_ids_json <=> incoming.target_element_ids_json)
|
|
|
+ OR NOT (item.recent_cost_fen <=> incoming.recent_cost_fen)
|
|
|
+ OR NOT (item.cost_end_date <=> incoming.cost_end_date)
|
|
|
+ THEN NULL ELSE item.readback_json
|
|
|
END,
|
|
|
- cleanup_status=CASE
|
|
|
- WHEN NOT (cleanup_action <=> VALUES(cleanup_action))
|
|
|
- OR NOT (target_component_ids_json <=> VALUES(target_component_ids_json))
|
|
|
- OR NOT (target_element_ids_json <=> VALUES(target_element_ids_json))
|
|
|
- OR NOT (recent_cost_fen <=> VALUES(recent_cost_fen))
|
|
|
- OR NOT (cost_end_date <=> VALUES(cost_end_date))
|
|
|
- THEN VALUES(cleanup_status)
|
|
|
- WHEN cleanup_status='SKIPPED_REVIEW_NOT_RECONFIRMED'
|
|
|
+ item.cleanup_status=CASE
|
|
|
+ WHEN NOT (item.cleanup_action <=> incoming.cleanup_action)
|
|
|
+ OR NOT (item.target_component_ids_json <=> incoming.target_component_ids_json)
|
|
|
+ OR NOT (item.target_element_ids_json <=> incoming.target_element_ids_json)
|
|
|
+ OR NOT (item.recent_cost_fen <=> incoming.recent_cost_fen)
|
|
|
+ OR NOT (item.cost_end_date <=> incoming.cost_end_date)
|
|
|
+ THEN incoming.cleanup_status
|
|
|
+ WHEN item.cleanup_status='SKIPPED_REVIEW_NOT_RECONFIRMED'
|
|
|
THEN 'DISCOVERED'
|
|
|
- ELSE cleanup_status
|
|
|
+ ELSE item.cleanup_status
|
|
|
END,
|
|
|
- target_component_ids_json=VALUES(target_component_ids_json),
|
|
|
- target_element_ids_json=VALUES(target_element_ids_json),
|
|
|
- recent_cost_fen=VALUES(recent_cost_fen),
|
|
|
- cost_start_date=VALUES(cost_start_date),
|
|
|
- cost_end_date=VALUES(cost_end_date),
|
|
|
- cleanup_action=VALUES(cleanup_action)
|
|
|
+ item.target_component_ids_json=incoming.target_component_ids_json,
|
|
|
+ item.target_element_ids_json=incoming.target_element_ids_json,
|
|
|
+ item.recent_cost_fen=incoming.recent_cost_fen,
|
|
|
+ item.cost_start_date=incoming.cost_start_date,
|
|
|
+ item.cost_end_date=incoming.cost_end_date,
|
|
|
+ item.cleanup_action=incoming.cleanup_action
|
|
|
+ WHERE item.cleanup_status NOT IN ('DELETING','CREATIVE_DELETED')
|
|
|
""",
|
|
|
- (
|
|
|
- record["account_id"],
|
|
|
- record.get("account_name"),
|
|
|
- record.get("agency_name"),
|
|
|
- record["adgroup_id"],
|
|
|
- record.get("adgroup_name"),
|
|
|
- record["dynamic_creative_id"],
|
|
|
- record.get("dynamic_creative_name"),
|
|
|
- record["check_date"],
|
|
|
- record["cleanup_action"],
|
|
|
- _json(record.get("component_ids") or []),
|
|
|
- _json(record.get("element_ids") or []),
|
|
|
- record.get("recent_cost_fen"),
|
|
|
- record.get("cost_start_date"),
|
|
|
- record.get("cost_end_date"),
|
|
|
- record.get("action_reason") or record["reject_reason"],
|
|
|
- record["reject_reason"],
|
|
|
- _json(record.get("review_result") or {}),
|
|
|
- _json(record.get("pre_state") or {}),
|
|
|
- (
|
|
|
- "ALERT_PENDING"
|
|
|
- if record["cleanup_action"] == ALERT_ONLY
|
|
|
- else "DISCOVERED"
|
|
|
- ),
|
|
|
- ),
|
|
|
+ insert_values,
|
|
|
)
|
|
|
cursor.execute(
|
|
|
"""
|
|
|
@@ -497,6 +536,16 @@ def upsert_cleanup_candidate(record: dict[str, Any]) -> dict[str, Any]:
|
|
|
|
|
|
|
|
|
def load_retryable_cleanup_items() -> list[dict[str, Any]]:
|
|
|
+ stale_minutes = int(
|
|
|
+ os.getenv(
|
|
|
+ "DAILY_REJECTED_CREATIVE_DELETE_CLAIM_STALE_MINUTES",
|
|
|
+ str(DEFAULT_DELETE_CLAIM_STALE_MINUTES),
|
|
|
+ )
|
|
|
+ )
|
|
|
+ if stale_minutes <= 0:
|
|
|
+ raise ValueError(
|
|
|
+ "DAILY_REJECTED_CREATIVE_DELETE_CLAIM_STALE_MINUTES must be positive"
|
|
|
+ )
|
|
|
connection = get_connection()
|
|
|
try:
|
|
|
with connection.cursor() as cursor:
|
|
|
@@ -512,18 +561,64 @@ def load_retryable_cleanup_items() -> list[dict[str, Any]]:
|
|
|
ON latest.account_id=item.account_id
|
|
|
AND latest.dynamic_creative_id=item.dynamic_creative_id
|
|
|
AND latest.check_date=item.check_date
|
|
|
- WHERE item.cleanup_status IN
|
|
|
- ('DISCOVERED','DEFERRED','FAILED','WRITE_OUTCOME_UNKNOWN')
|
|
|
+ WHERE (
|
|
|
+ item.cleanup_status IN
|
|
|
+ ('DISCOVERED','DEFERRED','FAILED','WRITE_OUTCOME_UNKNOWN')
|
|
|
+ OR (
|
|
|
+ item.cleanup_status='DELETING'
|
|
|
+ AND item.updated_at < DATE_SUB(NOW(), INTERVAL %s MINUTE)
|
|
|
+ )
|
|
|
+ )
|
|
|
AND item.cleanup_action='DELETE_CREATIVE'
|
|
|
ORDER BY item.id
|
|
|
- """
|
|
|
+ """,
|
|
|
+ (stale_minutes,),
|
|
|
)
|
|
|
return list(cursor.fetchall())
|
|
|
finally:
|
|
|
connection.close()
|
|
|
|
|
|
|
|
|
-def update_cleanup_item(item_id: int, **values: Any) -> None:
|
|
|
+def claim_cleanup_item(item_id: int) -> bool:
|
|
|
+ """Atomically claim a retryable deletion row for this run."""
|
|
|
+
|
|
|
+ stale_minutes = int(
|
|
|
+ os.getenv(
|
|
|
+ "DAILY_REJECTED_CREATIVE_DELETE_CLAIM_STALE_MINUTES",
|
|
|
+ str(DEFAULT_DELETE_CLAIM_STALE_MINUTES),
|
|
|
+ )
|
|
|
+ )
|
|
|
+ if stale_minutes <= 0:
|
|
|
+ raise ValueError(
|
|
|
+ "DAILY_REJECTED_CREATIVE_DELETE_CLAIM_STALE_MINUTES must be positive"
|
|
|
+ )
|
|
|
+ connection = get_connection()
|
|
|
+ try:
|
|
|
+ with connection.cursor() as cursor:
|
|
|
+ cursor.execute(
|
|
|
+ """
|
|
|
+ UPDATE creative_rejection_cleanup_item
|
|
|
+ SET cleanup_status='DELETING', error_message=NULL, updated_at=NOW()
|
|
|
+ WHERE id=%s
|
|
|
+ AND cleanup_action='DELETE_CREATIVE'
|
|
|
+ AND (
|
|
|
+ cleanup_status IN
|
|
|
+ ('DISCOVERED','DEFERRED','FAILED','WRITE_OUTCOME_UNKNOWN')
|
|
|
+ OR (
|
|
|
+ cleanup_status='DELETING'
|
|
|
+ AND updated_at < DATE_SUB(NOW(), INTERVAL %s MINUTE)
|
|
|
+ )
|
|
|
+ )
|
|
|
+ """,
|
|
|
+ (item_id, stale_minutes),
|
|
|
+ )
|
|
|
+ return cursor.rowcount == 1
|
|
|
+ finally:
|
|
|
+ connection.close()
|
|
|
+
|
|
|
+
|
|
|
+def update_cleanup_item(item_id: int, **values: Any) -> bool:
|
|
|
+ expected_cleanup_status = values.pop("_expected_cleanup_status", None)
|
|
|
allowed = {
|
|
|
"agency_name",
|
|
|
"cleanup_action",
|
|
|
@@ -548,15 +643,29 @@ def update_cleanup_item(item_id: int, **values: Any) -> None:
|
|
|
if unknown:
|
|
|
raise ValueError(f"Unsupported cleanup fields: {sorted(unknown)}")
|
|
|
if not values:
|
|
|
- return
|
|
|
+ return False
|
|
|
assignments = ", ".join(f"{name}=%s" for name in values)
|
|
|
connection = get_connection()
|
|
|
try:
|
|
|
with connection.cursor() as cursor:
|
|
|
+ where = "WHERE id=%s"
|
|
|
+ params = [*values.values(), item_id]
|
|
|
+ if expected_cleanup_status is not None:
|
|
|
+ if isinstance(expected_cleanup_status, (tuple, list, set, frozenset)):
|
|
|
+ statuses = list(expected_cleanup_status)
|
|
|
+ if not statuses:
|
|
|
+ return False
|
|
|
+ placeholders = ",".join(["%s"] * len(statuses))
|
|
|
+ where += f" AND cleanup_status IN ({placeholders})"
|
|
|
+ params.extend(statuses)
|
|
|
+ else:
|
|
|
+ where += " AND cleanup_status=%s"
|
|
|
+ params.append(expected_cleanup_status)
|
|
|
cursor.execute(
|
|
|
- f"UPDATE creative_rejection_cleanup_item SET {assignments} WHERE id=%s",
|
|
|
- [*values.values(), item_id],
|
|
|
+ f"UPDATE creative_rejection_cleanup_item SET {assignments} {where}",
|
|
|
+ params,
|
|
|
)
|
|
|
+ return cursor.rowcount > 0
|
|
|
finally:
|
|
|
connection.close()
|
|
|
|
|
|
@@ -651,6 +760,16 @@ def mark_cleanup_items_notified(
|
|
|
mark_cleanup_items_agency_notified(item_ids, notified_at)
|
|
|
|
|
|
|
|
|
+def _update_owned_cleanup_item(item_id: int, **values: Any) -> bool:
|
|
|
+ """Update a row only while this run owns its DELETING claim."""
|
|
|
+
|
|
|
+ return update_cleanup_item(
|
|
|
+ item_id,
|
|
|
+ _expected_cleanup_status="DELETING",
|
|
|
+ **values,
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
def upsert_cleanup_delivery(record: dict[str, Any]) -> dict[str, Any]:
|
|
|
connection = get_connection()
|
|
|
try:
|
|
|
@@ -1456,8 +1575,23 @@ def run_rejected_creative_cleanup(
|
|
|
max_workers=process_workers,
|
|
|
thread_name_prefix="creative-process",
|
|
|
) as executor:
|
|
|
- results = executor.map(process_creative, tasks)
|
|
|
- for completed, result in enumerate(results, start=1):
|
|
|
+ futures = {
|
|
|
+ executor.submit(process_creative, task): task for task in tasks
|
|
|
+ }
|
|
|
+ for completed, future in enumerate(as_completed(futures), start=1):
|
|
|
+ task = futures[future]
|
|
|
+ account_id = task[1]
|
|
|
+ creative_id = _as_int(task[2].get("dynamic_creative_id"))
|
|
|
+ try:
|
|
|
+ result = future.result()
|
|
|
+ except Exception as exc:
|
|
|
+ error = (
|
|
|
+ "creative processing failed "
|
|
|
+ f"account={account_id} creative={creative_id}: {exc}"
|
|
|
+ )
|
|
|
+ scan_errors.append(error)
|
|
|
+ logger.exception(error)
|
|
|
+ continue
|
|
|
if result is None:
|
|
|
continue
|
|
|
account_id, creative_id, action = result
|
|
|
@@ -1484,9 +1618,16 @@ def run_rejected_creative_cleanup(
|
|
|
item_id = int(item["id"])
|
|
|
account_id = int(item["account_id"])
|
|
|
creative_id = int(item["dynamic_creative_id"])
|
|
|
+ snapshot_status = str(item.get("cleanup_status") or "")
|
|
|
+ if snapshot_status == "DELETING":
|
|
|
+ # A stale claim is recovered only after obtaining the global
|
|
|
+ # Tencent write lock; do not mutate a potentially live owner.
|
|
|
+ deletable_items.append(item)
|
|
|
+ continue
|
|
|
if item.get("cleanup_action") != DELETE_CREATIVE:
|
|
|
update_cleanup_item(
|
|
|
item_id,
|
|
|
+ _expected_cleanup_status=snapshot_status,
|
|
|
cleanup_status="SKIPPED_REVIEW_NOT_RECONFIRMED",
|
|
|
error_message="当前规则仅允许删除整条创意",
|
|
|
)
|
|
|
@@ -1495,7 +1636,12 @@ def run_rejected_creative_cleanup(
|
|
|
context, account_id, creative_id
|
|
|
)
|
|
|
if agency and agency != item.get("agency_name"):
|
|
|
- update_cleanup_item(item_id, agency_name=agency)
|
|
|
+ update_cleanup_item(
|
|
|
+ item_id,
|
|
|
+ _expected_cleanup_status=snapshot_status,
|
|
|
+ agency_name=agency,
|
|
|
+ )
|
|
|
+ item["agency_name"] = agency
|
|
|
webhook_url = resolve_agency_webhook(
|
|
|
agency, webhook_config.webhooks or {}
|
|
|
)
|
|
|
@@ -1505,14 +1651,19 @@ def run_rejected_creative_cleanup(
|
|
|
if not agency
|
|
|
else f"代理商 {agency} 未配置通知群,禁止自动删除"
|
|
|
)
|
|
|
- update_cleanup_item(
|
|
|
+ updated = update_cleanup_item(
|
|
|
item_id,
|
|
|
+ _expected_cleanup_status=snapshot_status,
|
|
|
cleanup_status="DEFERRED",
|
|
|
error_message=reason,
|
|
|
)
|
|
|
- deferred += 1
|
|
|
+ if updated is not False:
|
|
|
+ deferred += 1
|
|
|
continue
|
|
|
- if item.get("cleanup_status") != "WRITE_OUTCOME_UNKNOWN":
|
|
|
+ if item.get("cleanup_status") not in {
|
|
|
+ "WRITE_OUTCOME_UNKNOWN",
|
|
|
+ "DELETING",
|
|
|
+ }:
|
|
|
precondition_failure = cleanup_precondition_failure(
|
|
|
item,
|
|
|
scanned_accounts,
|
|
|
@@ -1520,12 +1671,13 @@ def run_rejected_creative_cleanup(
|
|
|
)
|
|
|
if precondition_failure:
|
|
|
status, reason = precondition_failure
|
|
|
- update_cleanup_item(
|
|
|
+ updated = update_cleanup_item(
|
|
|
item_id,
|
|
|
+ _expected_cleanup_status=snapshot_status,
|
|
|
cleanup_status=status,
|
|
|
error_message=reason,
|
|
|
)
|
|
|
- if status == "DEFERRED":
|
|
|
+ if status == "DEFERRED" and updated is not False:
|
|
|
deferred += 1
|
|
|
continue
|
|
|
deletable_items.append(item)
|
|
|
@@ -1534,11 +1686,23 @@ def run_rejected_creative_cleanup(
|
|
|
# 每 worker 用独立 TencentClient(requests.Session 非线程安全),
|
|
|
# 避免共享 session 并发导致不可预测行为(与扫描阶段一致)。
|
|
|
if deletable_items:
|
|
|
+ configured_delete_workers = int(
|
|
|
+ os.getenv("TENCENT_AD_DELETE_WORKERS", "4")
|
|
|
+ )
|
|
|
+ if configured_delete_workers < 1:
|
|
|
+ raise ValueError("TENCENT_AD_DELETE_WORKERS must be at least 1")
|
|
|
delete_workers = min(
|
|
|
- int(os.getenv("TENCENT_AD_DELETE_WORKERS", "4")),
|
|
|
+ configured_delete_workers,
|
|
|
len(deletable_items),
|
|
|
16,
|
|
|
)
|
|
|
+ if not owned_tencent:
|
|
|
+ # A caller-supplied client may wrap a requests.Session and is
|
|
|
+ # not assumed to be thread-safe.
|
|
|
+ delete_workers = 1
|
|
|
+ delete_worker_local = threading.local()
|
|
|
+ delete_worker_clients = []
|
|
|
+ delete_worker_clients_lock = threading.Lock()
|
|
|
|
|
|
def delete_one(item, delete_client):
|
|
|
"""锁内单条创意:回读 → 复审 → 删除;返回 (deleted, deferred, error)。"""
|
|
|
@@ -1553,22 +1717,28 @@ def run_rejected_creative_cleanup(
|
|
|
except Exception as read_exc:
|
|
|
if (
|
|
|
item.get("cleanup_action") == DELETE_CREATIVE
|
|
|
- and item.get("cleanup_status")
|
|
|
- == "WRITE_OUTCOME_UNKNOWN"
|
|
|
and str(read_exc).startswith(
|
|
|
"Dynamic creative not found:"
|
|
|
)
|
|
|
):
|
|
|
- update_cleanup_item(
|
|
|
+ updated = _update_owned_cleanup_item(
|
|
|
item_id,
|
|
|
cleanup_status="CREATIVE_DELETED",
|
|
|
error_message=None,
|
|
|
readback_json=_json({"deleted_from_listing": True}),
|
|
|
deleted_at=effective_now,
|
|
|
)
|
|
|
- return 1, 0, None
|
|
|
+ if updated is not False:
|
|
|
+ return 1, 0, None
|
|
|
+ return 0, 0, (
|
|
|
+ f"account={account_id} creative={creative_id}: "
|
|
|
+ "delete result ignored because claim ownership was lost"
|
|
|
+ )
|
|
|
raise
|
|
|
- if item.get("cleanup_status") == "WRITE_OUTCOME_UNKNOWN":
|
|
|
+ if item.get("cleanup_status") in {
|
|
|
+ "WRITE_OUTCOME_UNKNOWN",
|
|
|
+ "DELETING",
|
|
|
+ }:
|
|
|
precondition_failure = cleanup_precondition_failure(
|
|
|
item,
|
|
|
scanned_accounts,
|
|
|
@@ -1576,7 +1746,7 @@ def run_rejected_creative_cleanup(
|
|
|
)
|
|
|
if precondition_failure:
|
|
|
status, reason = precondition_failure
|
|
|
- update_cleanup_item(
|
|
|
+ _update_owned_cleanup_item(
|
|
|
item_id,
|
|
|
cleanup_status=status,
|
|
|
error_message=reason,
|
|
|
@@ -1629,7 +1799,7 @@ def run_rejected_creative_cleanup(
|
|
|
fresh_action
|
|
|
and fresh_action.get("cleanup_action") == ALERT_ONLY
|
|
|
):
|
|
|
- update_cleanup_item(
|
|
|
+ _update_owned_cleanup_item(
|
|
|
item_id,
|
|
|
cleanup_action=ALERT_ONLY,
|
|
|
target_component_ids_json="[]",
|
|
|
@@ -1651,7 +1821,7 @@ def run_rejected_creative_cleanup(
|
|
|
notified_at=None,
|
|
|
)
|
|
|
return 0, 0, None
|
|
|
- update_cleanup_item(
|
|
|
+ updated = _update_owned_cleanup_item(
|
|
|
item_id,
|
|
|
cleanup_status="SKIPPED_REVIEW_NOT_RECONFIRMED",
|
|
|
error_message="写锁内回读发现整创意删除条件已变化",
|
|
|
@@ -1662,7 +1832,7 @@ def run_rejected_creative_cleanup(
|
|
|
readback = delete_client.delete_dynamic_creative(
|
|
|
account_id, creative_id
|
|
|
)
|
|
|
- update_cleanup_item(
|
|
|
+ updated = _update_owned_cleanup_item(
|
|
|
item_id,
|
|
|
cleanup_status="CREATIVE_DELETED",
|
|
|
error_message=None,
|
|
|
@@ -1670,12 +1840,18 @@ def run_rejected_creative_cleanup(
|
|
|
readback_json=_json(readback),
|
|
|
deleted_at=effective_now,
|
|
|
)
|
|
|
- return 1, 0, None
|
|
|
+ if updated is not False:
|
|
|
+ return 1, 0, None
|
|
|
+ return 0, 0, (
|
|
|
+ f"account={account_id} creative={creative_id}: "
|
|
|
+ "delete result ignored because claim ownership was lost"
|
|
|
+ )
|
|
|
return 0, 0, None
|
|
|
except Exception as exc:
|
|
|
from tencent_client import (
|
|
|
PostWriteVerificationError,
|
|
|
TencentWriteOutcomeUnknownError,
|
|
|
+ TencentWriteRateLimitedError,
|
|
|
)
|
|
|
|
|
|
outcome_unknown = isinstance(
|
|
|
@@ -1685,63 +1861,166 @@ def run_rejected_creative_cleanup(
|
|
|
PostWriteVerificationError,
|
|
|
),
|
|
|
)
|
|
|
- update_cleanup_item(
|
|
|
- item_id,
|
|
|
- cleanup_status=(
|
|
|
- "WRITE_OUTCOME_UNKNOWN" if outcome_unknown else "FAILED"
|
|
|
- ),
|
|
|
- error_message=str(exc)[:4000],
|
|
|
- )
|
|
|
- return 0, 0, f"account={account_id} creative={creative_id}: {exc}"
|
|
|
+ rate_limited = isinstance(exc, TencentWriteRateLimitedError)
|
|
|
+ error = f"account={account_id} creative={creative_id}: {exc}"
|
|
|
+ try:
|
|
|
+ _update_owned_cleanup_item(
|
|
|
+ item_id,
|
|
|
+ cleanup_status=(
|
|
|
+ "DEFERRED"
|
|
|
+ if rate_limited
|
|
|
+ else (
|
|
|
+ "WRITE_OUTCOME_UNKNOWN"
|
|
|
+ if outcome_unknown
|
|
|
+ else "FAILED"
|
|
|
+ )
|
|
|
+ ),
|
|
|
+ error_message=str(exc)[:4000],
|
|
|
+ )
|
|
|
+ except Exception as update_exc:
|
|
|
+ logger.exception(
|
|
|
+ "creative delete failure status update failed "
|
|
|
+ "account=%d creative=%d",
|
|
|
+ account_id,
|
|
|
+ creative_id,
|
|
|
+ )
|
|
|
+ error += f"; status_update_failed={update_exc}"
|
|
|
+ return 0, int(rate_limited), error
|
|
|
|
|
|
def run_delete(item):
|
|
|
if owned_tencent:
|
|
|
- from tencent_client import TencentClient
|
|
|
-
|
|
|
- delete_client = TencentClient()
|
|
|
- delete_client.seed_access_tokens(prefetched_tokens)
|
|
|
+ delete_client = getattr(delete_worker_local, "client", None)
|
|
|
+ if delete_client is None:
|
|
|
+ from tencent_client import TencentClient
|
|
|
+
|
|
|
+ delete_client = TencentClient()
|
|
|
+ delete_client.seed_access_tokens(prefetched_tokens)
|
|
|
+ delete_worker_local.client = delete_client
|
|
|
+ with delete_worker_clients_lock:
|
|
|
+ delete_worker_clients.append(delete_client)
|
|
|
else:
|
|
|
delete_client = client
|
|
|
- try:
|
|
|
- return delete_one(item, delete_client)
|
|
|
- finally:
|
|
|
- if delete_client is not client:
|
|
|
- delete_client.session.close()
|
|
|
+ return delete_one(item, delete_client)
|
|
|
|
|
|
with advisory_lock(write_lock_name) as acquired:
|
|
|
if not acquired:
|
|
|
for item in deletable_items:
|
|
|
- update_cleanup_item(
|
|
|
+ snapshot_status = str(item.get("cleanup_status") or "")
|
|
|
+ if snapshot_status == "DELETING":
|
|
|
+ continue
|
|
|
+ updated = update_cleanup_item(
|
|
|
int(item["id"]),
|
|
|
+ _expected_cleanup_status=snapshot_status,
|
|
|
cleanup_status="DEFERRED",
|
|
|
error_message="腾讯写锁被实时调控占用",
|
|
|
)
|
|
|
- deferred += 1
|
|
|
+ if updated is not False:
|
|
|
+ deferred += 1
|
|
|
else:
|
|
|
+ claimed_items = []
|
|
|
+ for item in deletable_items:
|
|
|
+ try:
|
|
|
+ if claim_cleanup_item(int(item["id"])):
|
|
|
+ claimed_items.append(item)
|
|
|
+ except Exception as exc:
|
|
|
+ error = (
|
|
|
+ "creative delete claim failed "
|
|
|
+ f"account={item['account_id']} "
|
|
|
+ f"creative={item['dynamic_creative_id']}: {exc}"
|
|
|
+ )
|
|
|
+ delete_errors.append(error)
|
|
|
+ logger.exception(error)
|
|
|
+ executable_items = []
|
|
|
+ for item in claimed_items:
|
|
|
+ agency = str(item.get("agency_name") or "") or _resolve_agency(
|
|
|
+ context,
|
|
|
+ int(item["account_id"]),
|
|
|
+ int(item["dynamic_creative_id"]),
|
|
|
+ )
|
|
|
+ webhook_url = resolve_agency_webhook(
|
|
|
+ agency, webhook_config.webhooks or {}
|
|
|
+ )
|
|
|
+ if agency and webhook_url:
|
|
|
+ if agency != item.get("agency_name"):
|
|
|
+ _update_owned_cleanup_item(
|
|
|
+ int(item["id"]),
|
|
|
+ agency_name=agency,
|
|
|
+ )
|
|
|
+ item["agency_name"] = agency
|
|
|
+ executable_items.append(item)
|
|
|
+ continue
|
|
|
+ reason = (
|
|
|
+ "代理商归属为空,禁止自动删除"
|
|
|
+ if not agency
|
|
|
+ else f"代理商 {agency} 未配置通知群,禁止自动删除"
|
|
|
+ )
|
|
|
+ updated = _update_owned_cleanup_item(
|
|
|
+ int(item["id"]),
|
|
|
+ cleanup_status="DEFERRED",
|
|
|
+ error_message=reason,
|
|
|
+ )
|
|
|
+ if updated is not False:
|
|
|
+ deferred += 1
|
|
|
logger.info(
|
|
|
- "creative delete started items=%d workers=%d",
|
|
|
+ "creative delete started candidates=%d claimed=%d workers=%d",
|
|
|
len(deletable_items),
|
|
|
+ len(executable_items),
|
|
|
delete_workers,
|
|
|
)
|
|
|
- with ThreadPoolExecutor(
|
|
|
- max_workers=delete_workers,
|
|
|
- thread_name_prefix="creative-delete",
|
|
|
- ) as executor:
|
|
|
- for d_deleted, d_deferred, d_error in executor.map(
|
|
|
- run_delete, deletable_items
|
|
|
- ):
|
|
|
- deleted += d_deleted
|
|
|
- deferred += d_deferred
|
|
|
- if d_error:
|
|
|
- delete_errors.append(d_error)
|
|
|
+ if executable_items:
|
|
|
+ try:
|
|
|
+ with ThreadPoolExecutor(
|
|
|
+ max_workers=min(delete_workers, len(executable_items)),
|
|
|
+ thread_name_prefix="creative-delete",
|
|
|
+ ) as executor:
|
|
|
+ futures = {
|
|
|
+ executor.submit(run_delete, item): item
|
|
|
+ for item in executable_items
|
|
|
+ }
|
|
|
+ for future in as_completed(futures):
|
|
|
+ item = futures[future]
|
|
|
+ try:
|
|
|
+ d_deleted, d_deferred, d_error = future.result()
|
|
|
+ except Exception as exc:
|
|
|
+ d_deleted = 0
|
|
|
+ d_deferred = 0
|
|
|
+ d_error = (
|
|
|
+ "creative delete worker failed "
|
|
|
+ f"account={item['account_id']} "
|
|
|
+ f"creative={item['dynamic_creative_id']}: {exc}"
|
|
|
+ )
|
|
|
+ logger.exception(d_error)
|
|
|
+ try:
|
|
|
+ _update_owned_cleanup_item(
|
|
|
+ int(item["id"]),
|
|
|
+ cleanup_status="FAILED",
|
|
|
+ error_message=str(exc)[:4000],
|
|
|
+ )
|
|
|
+ except Exception as update_exc:
|
|
|
+ logger.exception(
|
|
|
+ "creative delete worker failure status "
|
|
|
+ "update failed account=%s creative=%s",
|
|
|
+ item["account_id"],
|
|
|
+ item["dynamic_creative_id"],
|
|
|
+ )
|
|
|
+ d_error += (
|
|
|
+ f"; status_update_failed={update_exc}"
|
|
|
+ )
|
|
|
+ deleted += d_deleted
|
|
|
+ deferred += d_deferred
|
|
|
+ if d_error:
|
|
|
+ delete_errors.append(d_error)
|
|
|
+ finally:
|
|
|
+ for delete_worker_client in delete_worker_clients:
|
|
|
+ delete_worker_client.session.close()
|
|
|
|
|
|
- pending_notifications = load_unnotified_deleted_items(
|
|
|
- include_discovered=not apply_enabled,
|
|
|
- )
|
|
|
deliveries: list[dict[str, object]] = []
|
|
|
operator_deliveries: list[dict[str, object]] = []
|
|
|
notification_errors: list[str] = []
|
|
|
- if pending_notifications and webhook_config.enabled:
|
|
|
+
|
|
|
+ def publish_pending_notifications(
|
|
|
+ pending_notifications: list[dict[str, Any]],
|
|
|
+ ) -> None:
|
|
|
owned_publisher = publisher is None
|
|
|
sheet_publisher = publisher or RoiFeishuPublisher(require_chat_ids=False)
|
|
|
try:
|
|
|
@@ -1822,6 +2101,27 @@ def run_rejected_creative_cleanup(
|
|
|
if owned_publisher:
|
|
|
sheet_publisher.close()
|
|
|
|
|
|
+ pending_notification_probe = load_unnotified_deleted_items(
|
|
|
+ include_discovered=not apply_enabled,
|
|
|
+ )
|
|
|
+ if pending_notification_probe and webhook_config.enabled:
|
|
|
+ notification_lock_name = os.getenv(
|
|
|
+ "DAILY_REJECTED_CREATIVE_NOTIFICATION_LOCK_NAME",
|
|
|
+ "ad_rejected_creative_notification",
|
|
|
+ )
|
|
|
+ with advisory_lock(notification_lock_name) as acquired:
|
|
|
+ if not acquired:
|
|
|
+ logger.info(
|
|
|
+ "creative cleanup notification skipped: lock busy name=%s",
|
|
|
+ notification_lock_name,
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ pending_notifications = load_unnotified_deleted_items(
|
|
|
+ include_discovered=not apply_enabled,
|
|
|
+ )
|
|
|
+ if pending_notifications:
|
|
|
+ publish_pending_notifications(pending_notifications)
|
|
|
+
|
|
|
return {
|
|
|
"apply_enabled": apply_enabled,
|
|
|
"account_scope": "opengid_recent_3d_spend",
|