|
|
@@ -1,5 +1,6 @@
|
|
|
-"""账号域 DB 读写 —— demand_search_accounts + demand_search_article_relation.save_account_status"""
|
|
|
+"""账号域 DB 读写 —— demand_search_accounts + demand_search_article_relation + demand_account_articles"""
|
|
|
|
|
|
+import json
|
|
|
from typing import Dict, List, Optional
|
|
|
|
|
|
from src.infra.database.mysql.manager import MysqlManager
|
|
|
@@ -28,38 +29,22 @@ class AccountMapper(DemandSearchAccountConst):
|
|
|
LIMIT %s
|
|
|
"""
|
|
|
return await self.pool.fetch(
|
|
|
- query=query,
|
|
|
- params=(self.AccountSaveStatus.INIT, limit),
|
|
|
+ query=query, params=(self.AccountSaveStatus.INIT, limit),
|
|
|
)
|
|
|
|
|
|
async def fetch_relation_ids_by_biz(self, biz: str) -> List[int]:
|
|
|
"""取某个 biz 下所有 save_account_status=0 的 relation id"""
|
|
|
- query = f"""
|
|
|
- SELECT id
|
|
|
- FROM {self.RELATION_TABLE}
|
|
|
- WHERE biz = %s AND save_account_status = %s
|
|
|
- """
|
|
|
rows = await self.pool.fetch(
|
|
|
- query=query,
|
|
|
+ query=f"SELECT id FROM {self.RELATION_TABLE} WHERE biz = %s AND save_account_status = %s",
|
|
|
params=(biz, self.AccountSaveStatus.INIT),
|
|
|
)
|
|
|
return [r["id"] for r in rows]
|
|
|
|
|
|
async def fetch_relation_ids_by_biz_any_status(self, biz: str) -> List[int]:
|
|
|
- """取某个 biz 下未完成(INIT 或 PROCESSING)的 relation id,用于异常恢复"""
|
|
|
- query = f"""
|
|
|
- SELECT id
|
|
|
- FROM {self.RELATION_TABLE}
|
|
|
- WHERE biz = %s
|
|
|
- AND save_account_status IN (%s, %s)
|
|
|
- """
|
|
|
+ """取某个 biz 下未完成(INIT/PROCESSING)的 relation id,异常恢复用"""
|
|
|
rows = await self.pool.fetch(
|
|
|
- query=query,
|
|
|
- params=(
|
|
|
- biz,
|
|
|
- self.AccountSaveStatus.INIT,
|
|
|
- self.AccountSaveStatus.PROCESSING,
|
|
|
- ),
|
|
|
+ query=f"SELECT id FROM {self.RELATION_TABLE} WHERE biz = %s AND save_account_status IN (%s, %s)",
|
|
|
+ params=(biz, self.AccountSaveStatus.INIT, self.AccountSaveStatus.PROCESSING),
|
|
|
)
|
|
|
return [r["id"] for r in rows]
|
|
|
|
|
|
@@ -67,78 +52,38 @@ class AccountMapper(DemandSearchAccountConst):
|
|
|
# relation 表 — save_account_status 状态更新
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
|
|
|
- async def relation_mark_processing(self, ids: List[int]) -> int:
|
|
|
- """INIT → PROCESSING,CAS 防重复"""
|
|
|
+ async def _mark_batch(self, target_status: int, ids: List[int], from_statuses: List[int]) -> int:
|
|
|
+ """通用:将 ids 从 from_statuses 任一改为 target_status"""
|
|
|
if not ids:
|
|
|
return 0
|
|
|
placeholders = ",".join(["%s"] * len(ids))
|
|
|
+ status_placeholders = ",".join(["%s"] * len(from_statuses))
|
|
|
query = f"""
|
|
|
UPDATE {self.RELATION_TABLE}
|
|
|
SET save_account_status = %s
|
|
|
- WHERE id IN ({placeholders}) AND save_account_status = %s
|
|
|
+ WHERE id IN ({placeholders}) AND save_account_status IN ({status_placeholders})
|
|
|
"""
|
|
|
return await self.pool.save(
|
|
|
query=query,
|
|
|
- params=(
|
|
|
- self.AccountSaveStatus.PROCESSING,
|
|
|
- *ids,
|
|
|
- self.AccountSaveStatus.INIT,
|
|
|
- ),
|
|
|
+ params=(target_status, *ids, *from_statuses),
|
|
|
)
|
|
|
|
|
|
+ async def relation_mark_processing(self, ids: List[int]) -> int:
|
|
|
+ """INIT → PROCESSING,CAS 防重复"""
|
|
|
+ return await self._mark_batch(self.AccountSaveStatus.PROCESSING, ids, [self.AccountSaveStatus.INIT])
|
|
|
+
|
|
|
async def relation_mark_success(self, ids: List[int]) -> int:
|
|
|
"""PROCESSING → SUCCESS"""
|
|
|
- if not ids:
|
|
|
- return 0
|
|
|
- placeholders = ",".join(["%s"] * len(ids))
|
|
|
- query = f"""
|
|
|
- UPDATE {self.RELATION_TABLE}
|
|
|
- SET save_account_status = %s
|
|
|
- WHERE id IN ({placeholders}) AND save_account_status = %s
|
|
|
- """
|
|
|
- return await self.pool.save(
|
|
|
- query=query,
|
|
|
- params=(
|
|
|
- self.AccountSaveStatus.SUCCESS,
|
|
|
- *ids,
|
|
|
- self.AccountSaveStatus.PROCESSING,
|
|
|
- ),
|
|
|
- )
|
|
|
+ return await self._mark_batch(self.AccountSaveStatus.SUCCESS, ids, [self.AccountSaveStatus.PROCESSING])
|
|
|
|
|
|
async def relation_mark_success_direct(self, ids: List[int]) -> int:
|
|
|
"""INIT → SUCCESS,跳过 PROCESSING(查重命中时用)"""
|
|
|
- if not ids:
|
|
|
- return 0
|
|
|
- placeholders = ",".join(["%s"] * len(ids))
|
|
|
- query = f"""
|
|
|
- UPDATE {self.RELATION_TABLE}
|
|
|
- SET save_account_status = %s
|
|
|
- WHERE id IN ({placeholders}) AND save_account_status = %s
|
|
|
- """
|
|
|
- return await self.pool.save(
|
|
|
- query=query,
|
|
|
- params=(self.AccountSaveStatus.SUCCESS, *ids, self.AccountSaveStatus.INIT),
|
|
|
- )
|
|
|
+ return await self._mark_batch(self.AccountSaveStatus.SUCCESS, ids, [self.AccountSaveStatus.INIT])
|
|
|
|
|
|
async def relation_mark_failed(self, ids: List[int]) -> int:
|
|
|
- """INIT 或 PROCESSING → FAIL,异常恢复用,不挑前置状态"""
|
|
|
- if not ids:
|
|
|
- return 0
|
|
|
- placeholders = ",".join(["%s"] * len(ids))
|
|
|
- query = f"""
|
|
|
- UPDATE {self.RELATION_TABLE}
|
|
|
- SET save_account_status = %s
|
|
|
- WHERE id IN ({placeholders})
|
|
|
- AND save_account_status IN (%s, %s)
|
|
|
- """
|
|
|
- return await self.pool.save(
|
|
|
- query=query,
|
|
|
- params=(
|
|
|
- self.AccountSaveStatus.FAIL,
|
|
|
- *ids,
|
|
|
- self.AccountSaveStatus.INIT,
|
|
|
- self.AccountSaveStatus.PROCESSING,
|
|
|
- ),
|
|
|
+ """INIT/PROCESSING → FAIL,异常恢复用"""
|
|
|
+ return await self._mark_batch(
|
|
|
+ self.AccountSaveStatus.FAIL, ids, [self.AccountSaveStatus.INIT, self.AccountSaveStatus.PROCESSING],
|
|
|
)
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════
|
|
|
@@ -147,32 +92,86 @@ class AccountMapper(DemandSearchAccountConst):
|
|
|
|
|
|
async def find_by_biz(self, biz: str) -> Optional[Dict]:
|
|
|
"""按 biz 查账号是否已存在"""
|
|
|
- query = f"""
|
|
|
- SELECT * FROM {self.ACCOUNT_TABLE}
|
|
|
- WHERE biz = %s
|
|
|
- LIMIT 1
|
|
|
- """
|
|
|
- return await self.pool.fetch_one(query=query, params=(biz,))
|
|
|
+ return await self.pool.fetch_one(
|
|
|
+ query=f"SELECT * FROM {self.ACCOUNT_TABLE} WHERE biz = %s LIMIT 1",
|
|
|
+ params=(biz,),
|
|
|
+ )
|
|
|
|
|
|
async def insert_account(self, account: Dict) -> int:
|
|
|
"""写入账号,channel_account_id 冲突则跳过"""
|
|
|
query = f"""
|
|
|
INSERT IGNORE INTO {self.ACCOUNT_TABLE}
|
|
|
(channel_account_id, gh_id, account_name, biz, biz_info, description)
|
|
|
- VALUES
|
|
|
- (%s, %s, %s, %s, %s, %s)
|
|
|
+ VALUES (%s, %s, %s, %s, %s, %s)
|
|
|
"""
|
|
|
return await self.pool.save(
|
|
|
query=query,
|
|
|
- params=(
|
|
|
- account["channel_account_id"],
|
|
|
- account["gh_id"],
|
|
|
- account["account_name"],
|
|
|
- account["biz"],
|
|
|
- account["biz_info"],
|
|
|
- account["description"],
|
|
|
- ),
|
|
|
+ params=(account["channel_account_id"], account["gh_id"], account["account_name"],
|
|
|
+ account["biz"], account["biz_info"], account["description"]),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def list_accounts_for_article_crawl(self) -> List[Dict]:
|
|
|
+ """列出所有有 gh_id 的账号,用于文章抓取"""
|
|
|
+ query = f"""
|
|
|
+ SELECT channel_account_id, gh_id, crawl_cursor, crawl_deep_done
|
|
|
+ FROM {self.ACCOUNT_TABLE}
|
|
|
+ WHERE gh_id IS NOT NULL AND gh_id != ''
|
|
|
+ ORDER BY crawl_deep_done ASC, gh_id ASC
|
|
|
+ """
|
|
|
+ return await self.pool.fetch(query=query)
|
|
|
+
|
|
|
+ async def update_crawl_status(
|
|
|
+ self, gh_id: str, *, cursor: str = "", deep_done: int = 0,
|
|
|
+ ) -> int:
|
|
|
+ """更新账号的爬取游标和深翻状态"""
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.ACCOUNT_TABLE}
|
|
|
+ SET crawl_cursor = %s, crawl_deep_done = %s, last_crawl_at = NOW()
|
|
|
+ WHERE gh_id = %s
|
|
|
+ """
|
|
|
+ return await self.pool.save(query=query, params=(cursor, deep_done, gh_id))
|
|
|
+
|
|
|
+ # ═══════════════════════════════════════════════════════════════
|
|
|
+ # demand_account_articles 表
|
|
|
+ # ═══════════════════════════════════════════════════════════════
|
|
|
+
|
|
|
+ async def insert_articles_batch(self, articles: List[Dict]) -> int:
|
|
|
+ """批量写入文章,wx_sn 冲突则跳过"""
|
|
|
+ if not articles:
|
|
|
+ return 0
|
|
|
+ query = f"""
|
|
|
+ INSERT IGNORE INTO {self.ARTICLES_TABLE}
|
|
|
+ (gh_id, app_msg_id, item_index, title, content_url, wx_sn,
|
|
|
+ digest, is_original, item_show_type, source_url, cover_img_url, show_desc,
|
|
|
+ show_view_count, show_like_count, show_pay_count, show_zs_count,
|
|
|
+ type, send_time, create_time, update_time, date_time, base_info)
|
|
|
+ VALUES
|
|
|
+ (%s, %s, %s, %s, %s, %s,
|
|
|
+ %s, %s, %s, %s, %s, %s,
|
|
|
+ %s, %s, %s, %s,
|
|
|
+ %s, %s, %s, %s, %s, %s)
|
|
|
+ """
|
|
|
+ params = [
|
|
|
+ (
|
|
|
+ a["gh_id"], a["app_msg_id"], a["item_index"], a["title"],
|
|
|
+ a["content_url"], a["wx_sn"],
|
|
|
+ a["digest"], a["is_original"], a["item_show_type"],
|
|
|
+ a["source_url"], a["cover_img_url"], a["show_desc"],
|
|
|
+ a["show_view_count"], a["show_like_count"], a["show_pay_count"], a["show_zs_count"],
|
|
|
+ a["type"], a["send_time"], a["create_time"], a["update_time"],
|
|
|
+ a["date_time"], json.dumps(a["base_info"], ensure_ascii=False),
|
|
|
+ )
|
|
|
+ for a in articles
|
|
|
+ ]
|
|
|
+ return await self.pool.save(query=query, params=params, batch=True)
|
|
|
+
|
|
|
+ async def count_articles_by_gh_id(self, gh_id: str) -> int:
|
|
|
+ """统计某账号已抓文章数"""
|
|
|
+ row = await self.pool.fetch_one(
|
|
|
+ query=f"SELECT COUNT(*) AS cnt FROM {self.ARTICLES_TABLE} WHERE gh_id = %s",
|
|
|
+ params=(gh_id,),
|
|
|
)
|
|
|
+ return row["cnt"] if row else 0
|
|
|
|
|
|
|
|
|
__all__ = ["AccountMapper"]
|