|
|
@@ -1,256 +0,0 @@
|
|
|
-"""从 ODPS 拉取需求 → 提取搜索词 → 按账号聚合"""
|
|
|
-
|
|
|
-import asyncio
|
|
|
-import logging
|
|
|
-from collections import defaultdict
|
|
|
-from typing import Dict, List
|
|
|
-
|
|
|
-from src.config.odps import OdpsConfig
|
|
|
-from src.infra.external.odps import fetch_from_odps
|
|
|
-
|
|
|
-from ._const import DemandConst
|
|
|
-
|
|
|
-logger = logging.getLogger(__name__)
|
|
|
-
|
|
|
-# ODPS SELECT 列清单
|
|
|
-_SELECT_COLUMNS = [
|
|
|
- "id", # 表自增主键
|
|
|
- "dt", # 数据日期(yyyyMMdd)
|
|
|
- "channel_name", # 渠道类型
|
|
|
- "channel_level3", # 账号名称(聚合维度)
|
|
|
- "match_config_code", # 匹配配置编码
|
|
|
- "standard_element", # 标准化元素(搜索词,优先级最高)
|
|
|
- "match_text", # 匹配文本(搜索词,优先级次之)
|
|
|
- "match_generalized_element", # 泛化元素(搜索词,优先级最低)
|
|
|
- "point_type", # 点类型
|
|
|
- "element_dimension", # 元素维度
|
|
|
- "category_name", # 分类名称
|
|
|
- "demand_type", # 需求类型
|
|
|
- "demand_content_topic", # 需求内容选题
|
|
|
- "demand_id", # 需求ID
|
|
|
- "match_video_id", # 匹配视频ID
|
|
|
- "match_score", # 匹配得分
|
|
|
- "demand_strategy", # 需求策略
|
|
|
- "experiment_id", # 实验批次ID
|
|
|
-]
|
|
|
-_SELECT_CLAUSE = ", ".join(_SELECT_COLUMNS)
|
|
|
-
|
|
|
-
|
|
|
-class FetchDemands(DemandConst):
|
|
|
- """从 ODPS 拉取需求并聚合
|
|
|
-
|
|
|
- 使用方式:
|
|
|
- config = OdpsConfig()
|
|
|
- fetcher = FetchDemands(config)
|
|
|
- demands = await fetcher.deal("20260701")
|
|
|
- """
|
|
|
-
|
|
|
- def __init__(self, config: OdpsConfig):
|
|
|
- self._odps_config = config
|
|
|
-
|
|
|
- # ═══════════════════════════════════════════════════════════════
|
|
|
- # 入口
|
|
|
- # ═══════════════════════════════════════════════════════════════
|
|
|
-
|
|
|
- async def deal(self, dt: str) -> List[Dict]:
|
|
|
- """查询指定日期的需求,提取搜索词,按账号聚合 top N"""
|
|
|
- rows = await self._query_odps(dt)
|
|
|
- demands = [self._row_to_demand(r) for r in rows]
|
|
|
- # 两步过滤: 空账号 → 非白名单账号
|
|
|
- valid, dropped_empty = self._filter_empty_account(demands)
|
|
|
- valid_accounts = await self._load_valid_accounts()
|
|
|
- filtered, dropped_unknown = self._filter_unknown_accounts(valid, valid_accounts)
|
|
|
- total_dropped = dropped_empty + dropped_unknown
|
|
|
- if total_dropped:
|
|
|
- logger.warning(
|
|
|
- "丢弃 %d 行 (空账号 %d + 非白名单账号 %d), dt=%s",
|
|
|
- total_dropped, dropped_empty, dropped_unknown, dt,
|
|
|
- )
|
|
|
- logger.info(
|
|
|
- "ODPS 查询完成: dt=%s, 原始 %d 行 → 有效 %d 行, 丢弃 %d 行",
|
|
|
- dt, len(demands), len(filtered), total_dropped,
|
|
|
- )
|
|
|
- aggregated = self._aggregate(filtered)
|
|
|
- return aggregated
|
|
|
-
|
|
|
- # ═══════════════════════════════════════════════════════════════
|
|
|
- # ODPS 查询
|
|
|
- # ═══════════════════════════════════════════════════════════════
|
|
|
-
|
|
|
- async def _query_odps(self, dt: str) -> List[Dict]:
|
|
|
- """执行 ODPS SQL,返回原始 dict 列表"""
|
|
|
- sql = (
|
|
|
- f"SELECT {_SELECT_CLAUSE} "
|
|
|
- f"FROM public_channel_demand_match_result "
|
|
|
- f"WHERE dt = '{dt}' "
|
|
|
- f" AND channel_name IN {self._format_tuple(self.CHANNEL_NAMES)} "
|
|
|
- f" AND match_status = {self.MATCH_STATUS} "
|
|
|
- f" AND match_config_code IN {self._format_tuple(self.MATCH_CONFIG_CODES)} "
|
|
|
- f" AND element_dimension = '{self.ELEMENT_DIMENSION}' "
|
|
|
- f" AND match_score > 0.1 "
|
|
|
- f"ORDER BY match_score DESC"
|
|
|
- )
|
|
|
- logger.info("ODPS 查询需求, dt=%s", dt)
|
|
|
- return await asyncio.to_thread(fetch_from_odps, sql, self._odps_config)
|
|
|
-
|
|
|
- async def _load_valid_accounts(self) -> set[str]:
|
|
|
- """从两张账号基础表加载白名单,仅取 account_status='开' 的账号
|
|
|
-
|
|
|
- 两张表:
|
|
|
- - feishu_wechat_mp_account_base (公众号投流)
|
|
|
- - feishu_wechat_fwh_account_base (服务号投流)
|
|
|
- """
|
|
|
- valid: set[str] = set()
|
|
|
- for table in self.VALID_ACCOUNT_TABLES.values():
|
|
|
- sql = (
|
|
|
- f"SELECT account_name "
|
|
|
- f"FROM loghubods.{table} "
|
|
|
- f"WHERE account_status = '开'"
|
|
|
- )
|
|
|
- rows = await asyncio.to_thread(fetch_from_odps, sql, self._odps_config)
|
|
|
- names = {r["account_name"] for r in rows if r.get("account_name")}
|
|
|
- valid.update(names)
|
|
|
- logger.info("白名单加载: %s → %d 个有效账号", table, len(names))
|
|
|
- logger.info("白名单合计: %d 个有效账号", len(valid))
|
|
|
- return valid
|
|
|
-
|
|
|
- # ═══════════════════════════════════════════════════════════════
|
|
|
- # 行映射
|
|
|
- # ═══════════════════════════════════════════════════════════════
|
|
|
-
|
|
|
- @staticmethod
|
|
|
- def _row_to_demand(row: Dict) -> Dict:
|
|
|
- """ODPS row → 需求 dict
|
|
|
-
|
|
|
- search_terms: 去重后的搜索词列表,供下游 ArticleService 消费
|
|
|
- standard_element / match_text / match_generalized_element:
|
|
|
- 原始列值,供 mapper 写入 demand_search_queue 对应列
|
|
|
- """
|
|
|
- return {
|
|
|
- # 搜索词(两套表示,用途不同)
|
|
|
- "search_terms": FetchDemands._extract_search_terms(row),
|
|
|
- "standard_element": row.get("standard_element") or "",
|
|
|
- "match_text": row.get("match_text") or "",
|
|
|
- "match_generalized_element": row.get("match_generalized_element") or "",
|
|
|
- # 业务维度
|
|
|
- "dt": row.get("dt") or "",
|
|
|
- "channel_name": row.get("channel_name") or "",
|
|
|
- "channel_level3": row.get("channel_level3") or "",
|
|
|
- "config_code": row.get("match_config_code") or "",
|
|
|
- "point_type": row.get("point_type") or "",
|
|
|
- "category_name": row.get("category_name") or "",
|
|
|
- "demand_type": row.get("demand_type") or "",
|
|
|
- "demand_topic": row.get("demand_content_topic") or "",
|
|
|
- "demand_id": row.get("demand_id") or "",
|
|
|
- # 匹配信息
|
|
|
- "match_video_id": int(row.get("match_video_id") or 0),
|
|
|
- "experiment_id": row.get("experiment_id") or "",
|
|
|
- "match_score": float(row.get("match_score") or 0),
|
|
|
- # 元数据
|
|
|
- "row_id": row.get("id") or 0,
|
|
|
- }
|
|
|
-
|
|
|
- @staticmethod
|
|
|
- def _filter_empty_account(demands: List[Dict]) -> tuple[List[Dict], int]:
|
|
|
- """过滤 channel_level3 为空的无效行
|
|
|
-
|
|
|
- Returns:
|
|
|
- (valid_demands, dropped_count)
|
|
|
- """
|
|
|
- valid = [d for d in demands if d["channel_level3"]]
|
|
|
- return valid, len(demands) - len(valid)
|
|
|
-
|
|
|
- @staticmethod
|
|
|
- def _filter_unknown_accounts(
|
|
|
- demands: List[Dict],
|
|
|
- valid_accounts: set[str],
|
|
|
- ) -> tuple[List[Dict], int]:
|
|
|
- """过滤不在白名单中的账号
|
|
|
-
|
|
|
- Returns:
|
|
|
- (valid_demands, dropped_count)
|
|
|
- """
|
|
|
- if not valid_accounts:
|
|
|
- return demands, 0
|
|
|
- filtered: list[dict] = []
|
|
|
- dropped: list[str] = []
|
|
|
- for d in demands:
|
|
|
- if d["channel_level3"] in valid_accounts:
|
|
|
- filtered.append(d)
|
|
|
- else:
|
|
|
- dropped.append(d["channel_level3"])
|
|
|
- if dropped:
|
|
|
- unique_dropped = set(dropped)
|
|
|
- logger.warning(
|
|
|
- "非白名单账号 (共 %d 个, %d 行): %s",
|
|
|
- len(unique_dropped), len(dropped), sorted(unique_dropped),
|
|
|
- )
|
|
|
- return filtered, len(dropped)
|
|
|
-
|
|
|
- @staticmethod
|
|
|
- def _extract_search_terms(row: Dict) -> List[str]:
|
|
|
- """从三列搜索词中提取、去重(case-insensitive),保持优先级顺序"""
|
|
|
- seen: set[str] = set()
|
|
|
- terms: list[str] = []
|
|
|
- for col in DemandConst.SearchTermColumn.all_columns():
|
|
|
- val = row.get(col)
|
|
|
- if not val:
|
|
|
- continue
|
|
|
- text = str(val).strip()
|
|
|
- if not text:
|
|
|
- continue
|
|
|
- key = text.lower()
|
|
|
- if key not in seen:
|
|
|
- seen.add(key)
|
|
|
- terms.append(text)
|
|
|
- return terms
|
|
|
-
|
|
|
- # ═══════════════════════════════════════════════════════════════
|
|
|
- # 聚合
|
|
|
- # ═══════════════════════════════════════════════════════════════
|
|
|
-
|
|
|
- def _aggregate(
|
|
|
- self,
|
|
|
- demands: List[Dict],
|
|
|
- max_per_account: int | None = None,
|
|
|
- ) -> List[Dict]:
|
|
|
- """按 channel_level3 分组,每组取 match_score 最高的 top N
|
|
|
-
|
|
|
- 入参已按 match_score DESC 排序,直接取每组头 N 个即可。
|
|
|
- """
|
|
|
- limit = max_per_account or self.MAX_PER_ACCOUNT_PER_DAY
|
|
|
- buckets: dict[str, list[dict]] = defaultdict(list)
|
|
|
- for d in demands:
|
|
|
- key = d["channel_level3"]
|
|
|
- if len(buckets[key]) < limit:
|
|
|
- buckets[key].append(d)
|
|
|
-
|
|
|
- result: list[dict] = []
|
|
|
- for account, items in sorted(buckets.items()):
|
|
|
- result.extend(items)
|
|
|
- logger.info(
|
|
|
- "聚合账号 %s: 保留 %d/%d 条 (top score=%.4f)",
|
|
|
- account,
|
|
|
- len(items),
|
|
|
- sum(1 for d in demands if d["channel_level3"] == account),
|
|
|
- items[0]["match_score"] if items else 0,
|
|
|
- )
|
|
|
-
|
|
|
- logger.info(
|
|
|
- "聚合完成: %d 条 → %d 条 (%d 个账号, max=%d/账号)",
|
|
|
- len(demands), len(result), len(buckets), limit,
|
|
|
- )
|
|
|
- return result
|
|
|
-
|
|
|
- # ═══════════════════════════════════════════════════════════════
|
|
|
- # 工具
|
|
|
- # ═══════════════════════════════════════════════════════════════
|
|
|
-
|
|
|
- @staticmethod
|
|
|
- def _format_tuple(values: tuple) -> str:
|
|
|
- """('a', 'b') → "('a', 'b')" """
|
|
|
- quoted = ", ".join(f"'{v}'" for v in values)
|
|
|
- return f"({quoted})"
|
|
|
-
|
|
|
-
|
|
|
-__all__ = ["FetchDemands"]
|