|
@@ -35,12 +35,13 @@ async def demand_enqueue(param: str) -> dict:
|
|
|
pg = get_db().pg_manager("pg")
|
|
pg = get_db().pg_manager("pg")
|
|
|
fetcher = FetchDemands(OdpsConfig(), pg, task_logger=tlog)
|
|
fetcher = FetchDemands(OdpsConfig(), pg, task_logger=tlog)
|
|
|
items = await fetcher.deal(dt)
|
|
items = await fetcher.deal(dt)
|
|
|
- await tlog.phase_end("fetch", item_count=len(items), dt=dt)
|
|
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
logger.exception("ODPS 拉取失败, dt=%s", dt)
|
|
logger.exception("ODPS 拉取失败, dt=%s", dt)
|
|
|
await tlog.task_failed(e, dt=dt)
|
|
await tlog.task_failed(e, dt=dt)
|
|
|
return {"code": 500, "msg": "ODPS 拉取异常"}
|
|
return {"code": 500, "msg": "ODPS 拉取异常"}
|
|
|
|
|
|
|
|
|
|
+ await tlog.phase_end("fetch", item_count=len(items), dt=dt)
|
|
|
|
|
+
|
|
|
if not items:
|
|
if not items:
|
|
|
logger.info("无队列项生成, dt=%s", dt)
|
|
logger.info("无队列项生成, dt=%s", dt)
|
|
|
await tlog.task_completed(message="无队列项", dt=dt, items=0)
|
|
await tlog.task_completed(message="无队列项", dt=dt, items=0)
|
|
@@ -50,15 +51,16 @@ async def demand_enqueue(param: str) -> dict:
|
|
|
try:
|
|
try:
|
|
|
await tlog.phase_start("enqueue", dt=dt)
|
|
await tlog.phase_start("enqueue", dt=dt)
|
|
|
mysql = get_db().mysql_manager("long_articles")
|
|
mysql = get_db().mysql_manager("long_articles")
|
|
|
- enqueuer = EnqueueDemands(mysql, task_logger=tlog)
|
|
|
|
|
|
|
+ enqueuer = EnqueueDemands(mysql)
|
|
|
n = await enqueuer.deal(items)
|
|
n = await enqueuer.deal(items)
|
|
|
accounts = len(set(it["account"] for it in items))
|
|
accounts = len(set(it["account"] for it in items))
|
|
|
- await tlog.phase_end("enqueue", enqueued=n, accounts=accounts, dt=dt)
|
|
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
logger.exception("MySQL 入队失败, dt=%s", dt)
|
|
logger.exception("MySQL 入队失败, dt=%s", dt)
|
|
|
await tlog.task_failed(e, dt=dt)
|
|
await tlog.task_failed(e, dt=dt)
|
|
|
return {"code": 500, "msg": "入队异常"}
|
|
return {"code": 500, "msg": "入队异常"}
|
|
|
|
|
|
|
|
|
|
+ await tlog.phase_end("enqueue", enqueued=n, accounts=accounts, dt=dt)
|
|
|
|
|
+
|
|
|
logger.info(
|
|
logger.info(
|
|
|
"demandEnqueue done: enqueued %d items (%d accounts) for dt=%s",
|
|
"demandEnqueue done: enqueued %d items (%d accounts) for dt=%s",
|
|
|
n,
|
|
n,
|