|
|
@@ -0,0 +1,388 @@
|
|
|
+"""内容质量评估域 DB 读写 —— quantile / title / category 三表 + detail / articles 查询"""
|
|
|
+
|
|
|
+from typing import Dict, List, Optional
|
|
|
+
|
|
|
+from src.infra.database.mysql.manager import MysqlManager
|
|
|
+
|
|
|
+from ._const import ContentQualityConst
|
|
|
+
|
|
|
+
|
|
|
+class QualityMapper(ContentQualityConst):
|
|
|
+ """三张质量评估表 + detail.stat_status 的读写"""
|
|
|
+
|
|
|
+ def __init__(self, pool: MysqlManager):
|
|
|
+ self.pool = pool
|
|
|
+
|
|
|
+ # ═══════════════════════════════════════════════════════════════
|
|
|
+ # detail 表 — 前置校验
|
|
|
+ # ═══════════════════════════════════════════════════════════════
|
|
|
+
|
|
|
+ async def fetch_crawled_account_ids(self) -> List[str]:
|
|
|
+ """取已完成深翻的账号 channel_account_id 列表"""
|
|
|
+ query = f"""
|
|
|
+ SELECT channel_account_id
|
|
|
+ FROM {self.ACCOUNT_TABLE}
|
|
|
+ WHERE crawl_deep_done = 1 AND channel_account_id != ''
|
|
|
+ """
|
|
|
+ rows = await self.pool.fetch(query=query)
|
|
|
+ return [r["channel_account_id"] for r in rows]
|
|
|
+
|
|
|
+ async def fetch_pending_detail_by_account(
|
|
|
+ self, account_id: str, *, limit: int = 500
|
|
|
+ ) -> List[Dict]:
|
|
|
+ """取 detail 中 stat_status=INIT 且属于指定账号的文章"""
|
|
|
+ query = f"""
|
|
|
+ SELECT id, wx_sn, title, view_count, like_count
|
|
|
+ FROM {self.DETAIL_TABLE}
|
|
|
+ WHERE stat_status = %s AND channel_account_id = %s
|
|
|
+ ORDER BY id ASC
|
|
|
+ LIMIT %s
|
|
|
+ """
|
|
|
+ return await self.pool.fetch(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.INIT, account_id, limit),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def mark_detail_processing(self, ids: List[int]) -> int:
|
|
|
+ if not ids:
|
|
|
+ return 0
|
|
|
+ placeholders = ",".join(["%s"] * len(ids))
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.DETAIL_TABLE}
|
|
|
+ SET stat_status = %s
|
|
|
+ WHERE id IN ({placeholders}) AND stat_status = %s
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.PROCESSING, *ids, self.Status.INIT),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def mark_detail_done(self, item_id: int) -> int:
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.DETAIL_TABLE}
|
|
|
+ SET stat_status = %s
|
|
|
+ WHERE id = %s AND stat_status = %s
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.DONE, item_id, self.Status.PROCESSING),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def mark_detail_failed(self, item_id: int, remark: str = "") -> int:
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.DETAIL_TABLE}
|
|
|
+ SET stat_status = %s, remark = %s
|
|
|
+ WHERE id = %s AND stat_status = %s
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.FAIL, remark[:512], item_id, self.Status.PROCESSING),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def revert_detail_init(self, item_id: int) -> int:
|
|
|
+ """PROCESSING → INIT(数据未就绪,回退等下次重试)"""
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.DETAIL_TABLE}
|
|
|
+ SET stat_status = %s
|
|
|
+ WHERE id = %s AND stat_status = %s
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.INIT, item_id, self.Status.PROCESSING),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def fetch_relation_titles_not_in_table(
|
|
|
+ self, table: str, *, limit: int = 500
|
|
|
+ ) -> List[Dict]:
|
|
|
+ """取 relation 中 title 不在 title/category 表的去重标题"""
|
|
|
+ query = f"""
|
|
|
+ SELECT DISTINCT r.title_md5, r.title
|
|
|
+ FROM {self.RELATION_TABLE} r
|
|
|
+ WHERE r.title != '' AND r.title_md5 != ''
|
|
|
+ AND NOT EXISTS (
|
|
|
+ SELECT 1 FROM {table} t WHERE t.title_md5 = r.title_md5
|
|
|
+ )
|
|
|
+ LIMIT %s
|
|
|
+ """
|
|
|
+ return await self.pool.fetch(query=query, params=(limit,))
|
|
|
+
|
|
|
+ async def backfill_relation_quality_score(self) -> int:
|
|
|
+ """将 title 表中已评分的 score 回写到 relation.quality_score"""
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.RELATION_TABLE} r
|
|
|
+ JOIN {self.TITLE_TABLE} t ON r.title_md5 = t.title_md5
|
|
|
+ SET r.quality_score = t.score
|
|
|
+ WHERE t.status = %s AND r.quality_score != t.score
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.DONE,),
|
|
|
+ )
|
|
|
+
|
|
|
+ # ═══════════════════════════════════════════════════════════════
|
|
|
+ # articles 表 — 查群发位置
|
|
|
+ # ═══════════════════════════════════════════════════════════════
|
|
|
+
|
|
|
+ async def get_article_position(self, wx_sn: str) -> Optional[Dict]:
|
|
|
+ query = f"""
|
|
|
+ SELECT gh_id, item_index, type
|
|
|
+ FROM {self.ARTICLES_TABLE}
|
|
|
+ WHERE wx_sn = %s
|
|
|
+ LIMIT 1
|
|
|
+ """
|
|
|
+ return await self.pool.fetch_one(query=query, params=(wx_sn,))
|
|
|
+
|
|
|
+ # ═══════════════════════════════════════════════════════════════
|
|
|
+ # quantile 表
|
|
|
+ # ═══════════════════════════════════════════════════════════════
|
|
|
+
|
|
|
+ async def insert_quantile(self, row: Dict) -> int:
|
|
|
+ query = f"""
|
|
|
+ INSERT IGNORE INTO {self.QUANTILE_TABLE}
|
|
|
+ (wx_sn, view_count, like_count)
|
|
|
+ VALUES (%s, %s, %s)
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(row["wx_sn"], row.get("view_count", 0), row.get("like_count", 0)),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def fetch_pending_quantile(self, *, limit: int = 500) -> List[Dict]:
|
|
|
+ query = f"""
|
|
|
+ SELECT q.id, q.wx_sn, q.view_count, q.like_count, d.publish_at
|
|
|
+ FROM {self.QUANTILE_TABLE} q
|
|
|
+ JOIN {self.DETAIL_TABLE} d ON q.wx_sn = d.wx_sn
|
|
|
+ WHERE q.status = %s
|
|
|
+ ORDER BY q.id ASC
|
|
|
+ LIMIT %s
|
|
|
+ """
|
|
|
+ return await self.pool.fetch(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.INIT, limit),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def mark_quantile_processing(self, ids: List[int]) -> int:
|
|
|
+ if not ids:
|
|
|
+ return 0
|
|
|
+ placeholders = ",".join(["%s"] * len(ids))
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.QUANTILE_TABLE}
|
|
|
+ SET status = %s
|
|
|
+ WHERE id IN ({placeholders}) AND status = %s
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.PROCESSING, *ids, self.Status.INIT),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def revert_quantile_init(self, item_id: int) -> int:
|
|
|
+ """PROCESSING → INIT(数据未就绪,回退等下次重试)"""
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.QUANTILE_TABLE}
|
|
|
+ SET status = %s
|
|
|
+ WHERE id = %s AND status = %s
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.INIT, item_id, self.Status.PROCESSING),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def get_avg_view_count_60d(
|
|
|
+ self, gh_id: str, item_index: int, publish_time,
|
|
|
+ ) -> Optional[Dict]:
|
|
|
+ query = f"""
|
|
|
+ SELECT AVG(show_view_count) AS avg_count, COUNT(*) AS sample_count
|
|
|
+ FROM {self.ARTICLES_TABLE}
|
|
|
+ WHERE gh_id = %s
|
|
|
+ AND item_index = %s
|
|
|
+ AND send_time >= UNIX_TIMESTAMP(DATE_SUB(%s, INTERVAL %s DAY))
|
|
|
+ AND send_time < UNIX_TIMESTAMP(%s)
|
|
|
+ AND type = %s
|
|
|
+ AND show_view_count > 0
|
|
|
+ """
|
|
|
+ return await self.pool.fetch_one(
|
|
|
+ query=query,
|
|
|
+ params=(
|
|
|
+ gh_id, item_index, publish_time,
|
|
|
+ self.AVG_WINDOW_DAYS, publish_time, self.MASS_SEND_TYPE,
|
|
|
+ ),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def update_quantile_result(
|
|
|
+ self,
|
|
|
+ item_id: int,
|
|
|
+ *,
|
|
|
+ avg_view_count_60d: float | None = None,
|
|
|
+ view_ratio: float | None = None,
|
|
|
+ like_ratio: float | None = None,
|
|
|
+ sample_count_60d: int = 0,
|
|
|
+ status: int,
|
|
|
+ ) -> int:
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.QUANTILE_TABLE}
|
|
|
+ SET avg_view_count_60d = %s,
|
|
|
+ view_ratio = %s,
|
|
|
+ like_ratio = %s,
|
|
|
+ sample_count_60d = %s,
|
|
|
+ status = %s
|
|
|
+ WHERE id = %s AND status = %s
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(
|
|
|
+ avg_view_count_60d, view_ratio, like_ratio,
|
|
|
+ sample_count_60d, status, item_id, self.Status.PROCESSING,
|
|
|
+ ),
|
|
|
+ )
|
|
|
+
|
|
|
+ # ═══════════════════════════════════════════════════════════════
|
|
|
+ # title 表(标题粒度,按 title_md5 去重)
|
|
|
+ # ═══════════════════════════════════════════════════════════════
|
|
|
+
|
|
|
+ async def seed_titles(self, *, limit: int = 500) -> int:
|
|
|
+ """relation → 新标题 INSERT title 表"""
|
|
|
+ rows = await self.fetch_relation_titles_not_in_table(self.TITLE_TABLE, limit=limit)
|
|
|
+ if not rows:
|
|
|
+ return 0
|
|
|
+ query = f"""
|
|
|
+ INSERT IGNORE INTO {self.TITLE_TABLE} (title_md5, title)
|
|
|
+ VALUES (%s, %s)
|
|
|
+ """
|
|
|
+ params = [(r["title_md5"], r["title"]) for r in rows]
|
|
|
+ return await self.pool.save(query=query, params=params, batch=True)
|
|
|
+
|
|
|
+ async def fetch_pending_titles(self, *, limit: int = 100) -> List[Dict]:
|
|
|
+ query = f"""
|
|
|
+ SELECT id, title_md5, title
|
|
|
+ FROM {self.TITLE_TABLE}
|
|
|
+ WHERE status = %s
|
|
|
+ ORDER BY id ASC
|
|
|
+ LIMIT %s
|
|
|
+ """
|
|
|
+ return await self.pool.fetch(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.INIT, limit),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def mark_title_processing(self, ids: List[int]) -> int:
|
|
|
+ if not ids:
|
|
|
+ return 0
|
|
|
+ placeholders = ",".join(["%s"] * len(ids))
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.TITLE_TABLE}
|
|
|
+ SET status = %s
|
|
|
+ WHERE id IN ({placeholders}) AND status = %s
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.PROCESSING, *ids, self.Status.INIT),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def batch_update_title_score(self, updates: List[Dict]) -> int:
|
|
|
+ if not updates:
|
|
|
+ return 0
|
|
|
+ total = 0
|
|
|
+ for u in updates:
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.TITLE_TABLE}
|
|
|
+ SET score = %s, status = %s
|
|
|
+ WHERE id = %s AND status = %s
|
|
|
+ """
|
|
|
+ n = await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(u["score"], self.Status.DONE, u["id"], self.Status.PROCESSING),
|
|
|
+ )
|
|
|
+ total += n
|
|
|
+ return total
|
|
|
+
|
|
|
+ async def mark_title_batch_failed(self, ids: List[int]) -> int:
|
|
|
+ if not ids:
|
|
|
+ return 0
|
|
|
+ placeholders = ",".join(["%s"] * len(ids))
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.TITLE_TABLE}
|
|
|
+ SET status = %s
|
|
|
+ WHERE id IN ({placeholders}) AND status = %s
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.FAIL, *ids, self.Status.PROCESSING),
|
|
|
+ )
|
|
|
+
|
|
|
+ # ═══════════════════════════════════════════════════════════════
|
|
|
+ # category 表(标题粒度,按 title_md5 去重)
|
|
|
+ # ═══════════════════════════════════════════════════════════════
|
|
|
+
|
|
|
+ async def seed_categories(self, *, limit: int = 500) -> int:
|
|
|
+ rows = await self.fetch_relation_titles_not_in_table(
|
|
|
+ self.CATEGORY_TABLE, limit=limit,
|
|
|
+ )
|
|
|
+ if not rows:
|
|
|
+ return 0
|
|
|
+ query = f"""
|
|
|
+ INSERT IGNORE INTO {self.CATEGORY_TABLE} (title_md5, title)
|
|
|
+ VALUES (%s, %s)
|
|
|
+ """
|
|
|
+ params = [(r["title_md5"], r["title"]) for r in rows]
|
|
|
+ return await self.pool.save(query=query, params=params, batch=True)
|
|
|
+
|
|
|
+ async def fetch_pending_categories(self, *, limit: int = 100) -> List[Dict]:
|
|
|
+ query = f"""
|
|
|
+ SELECT id, title_md5, title
|
|
|
+ FROM {self.CATEGORY_TABLE}
|
|
|
+ WHERE status = %s
|
|
|
+ ORDER BY id ASC
|
|
|
+ LIMIT %s
|
|
|
+ """
|
|
|
+ return await self.pool.fetch(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.INIT, limit),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def mark_category_processing(self, ids: List[int]) -> int:
|
|
|
+ if not ids:
|
|
|
+ return 0
|
|
|
+ placeholders = ",".join(["%s"] * len(ids))
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.CATEGORY_TABLE}
|
|
|
+ SET status = %s
|
|
|
+ WHERE id IN ({placeholders}) AND status = %s
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.PROCESSING, *ids, self.Status.INIT),
|
|
|
+ )
|
|
|
+
|
|
|
+ async def batch_update_category(self, updates: List[Dict]) -> int:
|
|
|
+ if not updates:
|
|
|
+ return 0
|
|
|
+ total = 0
|
|
|
+ for u in updates:
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.CATEGORY_TABLE}
|
|
|
+ SET category = %s, status = %s
|
|
|
+ WHERE id = %s AND status = %s
|
|
|
+ """
|
|
|
+ n = await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(u["category"], self.Status.DONE, u["id"], self.Status.PROCESSING),
|
|
|
+ )
|
|
|
+ total += n
|
|
|
+ return total
|
|
|
+
|
|
|
+ async def mark_category_batch_failed(self, ids: List[int]) -> int:
|
|
|
+ if not ids:
|
|
|
+ return 0
|
|
|
+ placeholders = ",".join(["%s"] * len(ids))
|
|
|
+ query = f"""
|
|
|
+ UPDATE {self.CATEGORY_TABLE}
|
|
|
+ SET status = %s
|
|
|
+ WHERE id IN ({placeholders}) AND status = %s
|
|
|
+ """
|
|
|
+ return await self.pool.save(
|
|
|
+ query=query,
|
|
|
+ params=(self.Status.FAIL, *ids, self.Status.PROCESSING),
|
|
|
+ )
|
|
|
+
|
|
|
+
|
|
|
+__all__ = ["QualityMapper"]
|