|
|
@@ -58,7 +58,7 @@ class CreateAdPlatformArticlesDecodeTask(DecodeArticleConst):
|
|
|
|
|
|
if not result:
|
|
|
await self.mapper.update_article_decode_status(
|
|
|
- article_id, self.TaskStatus.PROCESSING, self.TaskStatus.FAILED
|
|
|
+ article_id, self.TaskStatus.PROCESSING, self.TaskStatus.INIT
|
|
|
)
|
|
|
await self.log_service.log(
|
|
|
contents={
|
|
|
@@ -66,7 +66,7 @@ class CreateAdPlatformArticlesDecodeTask(DecodeArticleConst):
|
|
|
"wx_sn": wx_sn,
|
|
|
"task": "create_decode_task_v2",
|
|
|
"status": "fail",
|
|
|
- "message": "no response for channel_content_id",
|
|
|
+ "message": "no response for channel_content_id, rolled back to INIT",
|
|
|
}
|
|
|
)
|
|
|
continue
|
|
|
@@ -74,7 +74,7 @@ class CreateAdPlatformArticlesDecodeTask(DecodeArticleConst):
|
|
|
status = result.get("status")
|
|
|
if status == self.SubmitStatus.FAILED:
|
|
|
await self.mapper.update_article_decode_status(
|
|
|
- article_id, self.TaskStatus.PROCESSING, self.TaskStatus.FAILED
|
|
|
+ article_id, self.TaskStatus.PROCESSING, self.TaskStatus.INIT
|
|
|
)
|
|
|
await self.log_service.log(
|
|
|
contents={
|
|
|
@@ -174,7 +174,7 @@ class CreateAdPlatformArticlesDecodeTask(DecodeArticleConst):
|
|
|
)
|
|
|
else:
|
|
|
await self.mapper.update_article_decode_status(
|
|
|
- article_id, self.TaskStatus.PROCESSING, self.TaskStatus.FAILED
|
|
|
+ article_id, self.TaskStatus.PROCESSING, self.TaskStatus.INIT
|
|
|
)
|
|
|
await self.log_service.log(
|
|
|
contents={
|
|
|
@@ -182,7 +182,7 @@ class CreateAdPlatformArticlesDecodeTask(DecodeArticleConst):
|
|
|
"wx_sn": wx_sn,
|
|
|
"task": "create_decode_task_v2",
|
|
|
"status": "fail",
|
|
|
- "message": f"unexpected submit status: {status}",
|
|
|
+ "message": f"unexpected submit status: {status}, rolled back to INIT",
|
|
|
"data": result,
|
|
|
}
|
|
|
)
|
|
|
@@ -208,42 +208,148 @@ class CreateAdPlatformArticlesDecodeTask(DecodeArticleConst):
|
|
|
|
|
|
|
|
|
class CreateInnerArticlesDecodeTask(DecodeArticleConst):
|
|
|
+ _TEST_MODE = False
|
|
|
+
|
|
|
def __init__(self, pool: DatabaseManager, log_service: LogService):
|
|
|
self.pool = pool
|
|
|
self.log_service = log_service
|
|
|
self.mapper = InnerArticlesDecodeTaskMapper(self.pool)
|
|
|
self.tool = InnerArticlesDecodeUtils()
|
|
|
|
|
|
- async def deal(self):
|
|
|
+ async def _acquire_articles(self) -> List[Dict]:
|
|
|
+ """获取待解构文章,并加锁(status INIT → PROCESSING)"""
|
|
|
article_list = await self.mapper.fetch_inner_articles()
|
|
|
- if not article_list:
|
|
|
+ if self._TEST_MODE:
|
|
|
+ return article_list
|
|
|
+
|
|
|
+ locked = []
|
|
|
+ for article in article_list:
|
|
|
+ article_id = article["id"]
|
|
|
+ acquired = await self.mapper.update_inner_article_status(
|
|
|
+ article_id, self.TaskStatus.INIT, self.TaskStatus.PROCESSING
|
|
|
+ )
|
|
|
+ if acquired:
|
|
|
+ locked.append(article)
|
|
|
+ else:
|
|
|
+ await self.log_service.log(
|
|
|
+ contents={
|
|
|
+ "article_id": article_id,
|
|
|
+ "task": "create_inner_decode_task",
|
|
|
+ "status": "skip",
|
|
|
+ "message": "acquire lock failed",
|
|
|
+ }
|
|
|
+ )
|
|
|
+ return locked
|
|
|
+
|
|
|
+ async def _handle_result(
|
|
|
+ self,
|
|
|
+ article: Dict,
|
|
|
+ channel_content_id: str,
|
|
|
+ result: Dict,
|
|
|
+ posts_by_cid: Dict,
|
|
|
+ config_id: int,
|
|
|
+ ):
|
|
|
+ wx_sn = article["wx_sn"]
|
|
|
+
|
|
|
+ if not result:
|
|
|
await self.log_service.log(
|
|
|
contents={
|
|
|
- "task": "create_inner_decode_task_v2",
|
|
|
- "message": "No more articles to decode",
|
|
|
+ "wx_sn": wx_sn,
|
|
|
+ "task": "create_inner_decode_task",
|
|
|
+ "status": "fail",
|
|
|
+ "message": "no response for channel_content_id",
|
|
|
}
|
|
|
)
|
|
|
return
|
|
|
|
|
|
- # 过滤已有任务记录的文章
|
|
|
- all_wx_sns = [a["wx_sn"] for a in article_list]
|
|
|
- existing = await self.mapper.fetch_existing_channel_content_ids(all_wx_sns)
|
|
|
- new_articles = [a for a in article_list if a["wx_sn"] not in existing]
|
|
|
- skipped = len(article_list) - len(new_articles)
|
|
|
- if skipped > 0:
|
|
|
+ status = result.get("status")
|
|
|
+ if status == self.SubmitStatus.FAILED:
|
|
|
await self.log_service.log(
|
|
|
contents={
|
|
|
- "task": "create_inner_decode_task_v2",
|
|
|
- "message": f"Skipped {skipped} already-submitted articles",
|
|
|
+ "wx_sn": wx_sn,
|
|
|
+ "task": "create_inner_decode_task",
|
|
|
+ "status": "fail",
|
|
|
+ "data": result,
|
|
|
}
|
|
|
)
|
|
|
- if not new_articles:
|
|
|
+ elif status == self.SubmitStatus.PENDING:
|
|
|
+ await self.mapper.insert_decode_task(
|
|
|
+ channel_content_id=channel_content_id,
|
|
|
+ content_id=str(article.get("source_id", "")),
|
|
|
+ source=self.SourceType.INNER,
|
|
|
+ payload=json.dumps(
|
|
|
+ posts_by_cid.get(channel_content_id, {}), ensure_ascii=False
|
|
|
+ ),
|
|
|
+ remark="内部文章解构任务已提交",
|
|
|
+ )
|
|
|
+ elif status == self.SubmitStatus.SUCCESS:
|
|
|
+ query_results = await self.tool.query_decode_results_batch(
|
|
|
+ [channel_content_id], config_id=config_id
|
|
|
+ )
|
|
|
+ result_data = query_results.get(channel_content_id)
|
|
|
+ data_content = result_data.get("dataContent") if result_data else None
|
|
|
+ if data_content:
|
|
|
+ await self.mapper.insert_decode_task(
|
|
|
+ channel_content_id=channel_content_id,
|
|
|
+ content_id=str(article.get("source_id", "")),
|
|
|
+ source=self.SourceType.INNER,
|
|
|
+ payload=json.dumps(
|
|
|
+ posts_by_cid.get(channel_content_id, {}), ensure_ascii=False
|
|
|
+ ),
|
|
|
+ remark="内部文章解构结果已获取",
|
|
|
+ )
|
|
|
+ await self.mapper.set_decode_result(
|
|
|
+ channel_content_id=channel_content_id,
|
|
|
+ result=json.dumps(
|
|
|
+ {"dataContent": data_content}, ensure_ascii=False
|
|
|
+ ),
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ await self.mapper.insert_decode_task(
|
|
|
+ channel_content_id=channel_content_id,
|
|
|
+ content_id=str(article.get("source_id", "")),
|
|
|
+ source=self.SourceType.INNER,
|
|
|
+ payload=json.dumps(result, ensure_ascii=False),
|
|
|
+ remark="提交返回SUCCESS,查询未果,等待轮询",
|
|
|
+ )
|
|
|
+ else:
|
|
|
await self.log_service.log(
|
|
|
contents={
|
|
|
- "task": "create_inner_decode_task_v2",
|
|
|
- "message": "All articles already submitted",
|
|
|
+ "wx_sn": wx_sn,
|
|
|
+ "task": "create_inner_decode_task",
|
|
|
+ "status": "fail",
|
|
|
+ "message": f"unexpected submit status: {status}",
|
|
|
+ "data": result,
|
|
|
}
|
|
|
)
|
|
|
+
|
|
|
+ async def _submit_and_record(self, articles: List[Dict]):
|
|
|
+ if not articles:
|
|
|
+ return
|
|
|
+
|
|
|
+ # 过滤已有任务记录的文章(测试模式跳过)
|
|
|
+ if not self._TEST_MODE:
|
|
|
+ all_wx_sns = [a["wx_sn"] for a in articles]
|
|
|
+ existing = await self.mapper.fetch_existing_channel_content_ids(all_wx_sns)
|
|
|
+ new_articles = [a for a in articles if a["wx_sn"] not in existing]
|
|
|
+ skipped = len(articles) - len(new_articles)
|
|
|
+ if skipped > 0:
|
|
|
+ await self.log_service.log(
|
|
|
+ contents={
|
|
|
+ "task": "create_inner_decode_task",
|
|
|
+ "message": f"Skipped {skipped} already-submitted articles",
|
|
|
+ }
|
|
|
+ )
|
|
|
+
|
|
|
+ for article in articles:
|
|
|
+ if article not in new_articles:
|
|
|
+ await self.mapper.update_inner_article_status(
|
|
|
+ article["id"], self.TaskStatus.PROCESSING, self.TaskStatus.SUCCESS
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ new_articles = articles
|
|
|
+
|
|
|
+ if not new_articles:
|
|
|
return
|
|
|
|
|
|
# 批量获取 produce 信息
|
|
|
@@ -256,86 +362,49 @@ class CreateInnerArticlesDecodeTask(DecodeArticleConst):
|
|
|
produce_info_map[article["wx_sn"]] = produce_info
|
|
|
|
|
|
posts = self.tool.prepare_posts(new_articles, produce_info_map)
|
|
|
- submit_results = await self.tool.submit_decode_batch(posts)
|
|
|
- posts_by_wx = {p["channelContentId"]: p for p in posts}
|
|
|
+
|
|
|
+ submit_results = await self.tool.submit_decode_batch(
|
|
|
+ posts, config_id=self.CONFIG_ID, skip_completed=True
|
|
|
+ )
|
|
|
+ posts_by_cid = {p["channelContentId"]: p for p in posts}
|
|
|
|
|
|
for article in tqdm(new_articles):
|
|
|
wx_sn = article["wx_sn"]
|
|
|
+ article_id = article["id"]
|
|
|
+
|
|
|
result = submit_results.get(wx_sn)
|
|
|
- if not result:
|
|
|
- await self.log_service.log(
|
|
|
- contents={
|
|
|
- "wx_sn": wx_sn,
|
|
|
- "task": "create_inner_decode_task_v2",
|
|
|
- "status": "fail",
|
|
|
- "message": "no response for channel_content_id",
|
|
|
- }
|
|
|
- )
|
|
|
- continue
|
|
|
+ await self._handle_result(
|
|
|
+ article, wx_sn, result, posts_by_cid, self.CONFIG_ID
|
|
|
+ )
|
|
|
|
|
|
- status = result.get("status")
|
|
|
- if status == self.SubmitStatus.FAILED:
|
|
|
- await self.log_service.log(
|
|
|
- contents={
|
|
|
- "wx_sn": wx_sn,
|
|
|
- "task": "create_inner_decode_task_v2",
|
|
|
- "status": "fail",
|
|
|
- "data": result,
|
|
|
- }
|
|
|
- )
|
|
|
- elif status == self.SubmitStatus.PENDING:
|
|
|
- await self.mapper.insert_decode_task(
|
|
|
- channel_content_id=wx_sn,
|
|
|
- content_id=str(article.get("source_id", "")),
|
|
|
- source=self.SourceType.INNER,
|
|
|
- payload=json.dumps(
|
|
|
- posts_by_wx.get(wx_sn, {}), ensure_ascii=False
|
|
|
- ),
|
|
|
- remark="内部文章解构任务已提交",
|
|
|
- )
|
|
|
- elif status == self.SubmitStatus.SUCCESS:
|
|
|
- query_results = await self.tool.query_decode_results_batch([wx_sn])
|
|
|
- result_data = query_results.get(wx_sn)
|
|
|
- data_content = result_data.get("dataContent") if result_data else None
|
|
|
- if data_content:
|
|
|
- await self.mapper.insert_decode_task(
|
|
|
- channel_content_id=wx_sn,
|
|
|
- content_id=str(article.get("source_id", "")),
|
|
|
- source=self.SourceType.INNER,
|
|
|
- payload=json.dumps(
|
|
|
- posts_by_wx.get(wx_sn, {}), ensure_ascii=False
|
|
|
- ),
|
|
|
- remark="内部文章解构结果已获取",
|
|
|
- )
|
|
|
- await self.mapper.set_decode_result(
|
|
|
- channel_content_id=wx_sn,
|
|
|
- result=json.dumps(
|
|
|
- {"dataContent": data_content}, ensure_ascii=False
|
|
|
- ),
|
|
|
+ if not self._TEST_MODE:
|
|
|
+ ok = result and result.get("status") != self.SubmitStatus.FAILED
|
|
|
+ if ok:
|
|
|
+ await self.mapper.update_inner_article_status(
|
|
|
+ article_id, self.TaskStatus.PROCESSING, self.TaskStatus.SUCCESS
|
|
|
)
|
|
|
else:
|
|
|
- await self.mapper.insert_decode_task(
|
|
|
- channel_content_id=wx_sn,
|
|
|
- content_id=str(article.get("source_id", "")),
|
|
|
- source=self.SourceType.INNER,
|
|
|
- payload=json.dumps(result, ensure_ascii=False),
|
|
|
- remark="提交返回SUCCESS,查询未果,等待轮询",
|
|
|
+ # 提交失败或无响应,回锁为 INIT 等待下次重试
|
|
|
+ await self.mapper.update_inner_article_status(
|
|
|
+ article_id, self.TaskStatus.PROCESSING, self.TaskStatus.INIT
|
|
|
)
|
|
|
- else:
|
|
|
- await self.log_service.log(
|
|
|
- contents={
|
|
|
- "wx_sn": wx_sn,
|
|
|
- "task": "create_inner_decode_task_v2",
|
|
|
- "status": "fail",
|
|
|
- "message": f"unexpected submit status: {status}",
|
|
|
- "data": result,
|
|
|
- }
|
|
|
- )
|
|
|
|
|
|
+ async def deal(self):
|
|
|
+ article_list = await self._acquire_articles()
|
|
|
+ if not article_list:
|
|
|
+ await self.log_service.log(
|
|
|
+ contents={
|
|
|
+ "task": "create_inner_decode_task",
|
|
|
+ "message": "No more articles to decode",
|
|
|
+ }
|
|
|
+ )
|
|
|
+ return
|
|
|
+
|
|
|
+ await self._submit_and_record(article_list)
|
|
|
await self.log_service.log(
|
|
|
contents={
|
|
|
- "task": "create_inner_decode_task_v2",
|
|
|
- "message": f"Processed {len(new_articles)} articles",
|
|
|
+ "task": "create_inner_decode_task",
|
|
|
+ "message": f"Processed {len(article_list)} articles",
|
|
|
}
|
|
|
)
|
|
|
|