Merge branch 'cooperation_video_candidate_pool_improved_lld_0509' into test

api-module/src/main/java/com/tzld/piaoquan/api/dao/mapper/contentplatform/ext/ContentPlatformDemandVideoMapperExt.java

@@ -45,6 +45,7 @@ public interface ContentPlatformDemandVideoMapperExt {
                                                        @Param("demandFilterSortStrategyLike") String demandFilterSortStrategyLike,
                                                        @Param("channelLevel3") String channelLevel3,
                                                        @Param("driveDimensionTime") String driveDimensionTime,
+                                                       @Param("category") String category,
                                                        @Param("limit") int limit,
                                                        @Param("excludeSelfTitle") boolean excludeSelfTitle);
 

api-module/src/main/java/com/tzld/piaoquan/api/job/contentplatform/ContentPlatformDemandVideoJob.java

@@ -3,9 +3,11 @@ package com.tzld.piaoquan.api.job.contentplatform;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
+import com.aliyun.odps.data.Record;
 import com.tzld.piaoquan.api.dao.mapper.contentplatform.ext.ContentPlatformDemandVideoMapperExt;
 import com.tzld.piaoquan.api.model.po.contentplatform.ContentPlatformDemandVideo;
 import com.tzld.piaoquan.growth.common.component.HttpPoolClient;
+import com.tzld.piaoquan.growth.common.component.ODPSManager;
 import com.tzld.piaoquan.growth.common.model.bo.VideoDetail;
 import com.tzld.piaoquan.growth.common.service.MessageAttachmentService;
 import com.tzld.piaoquan.growth.common.utils.DateUtil;
@@ -35,6 +37,11 @@ public class ContentPlatformDemandVideoJob {
     @Autowired
     private MessageAttachmentService messageAttachmentService;
 
+    @Autowired
+    private ODPSManager odpsManager;
+
+    private static final int ODPS_CATEGORY_CHUNK = 5000;
+
     private static final List<String> SYNC_CHANNEL_NAMES = Arrays.asList(
             "公众号合作-即转-稳定",
             "群/企微合作-稳定"
@@ -223,6 +230,16 @@ public class ContentPlatformDemandVideoJob {
                 .filter(v -> StringUtils.hasText(v.getTitle()))
                 .collect(Collectors.toList());
 
+        // 从 ODPS video_merge_tag 拉取 merge_leve2(二级品类),填到 category 列,供下发侧黑名单过滤使用
+        Map<Long, String> videoIdToCategory = fetchVideoCategoryMap(
+                saveList.stream().map(ContentPlatformDemandVideo::getVideoId).distinct().collect(Collectors.toList()));
+        for (ContentPlatformDemandVideo demandVideo : saveList) {
+            String cat = videoIdToCategory.get(demandVideo.getVideoId());
+            if (StringUtils.hasText(cat)) {
+                demandVideo.setCategory(cat);
+            }
+        }
+
 //        // 按crowd_segment粒度去重，相同videoId保留分数最高的一条
 //        saveList = saveList.stream()
 //                .collect(Collectors.groupingBy(v -> v.getCrowdSegment() + "_"
@@ -243,6 +260,40 @@ public class ContentPlatformDemandVideoJob {
         log.info("syncByChannel success, dt={}, channelName={}, count={}", dt, syncChannelName, saveList.size());
     }
 
+    /**
+     * 批查 loghubods.video_merge_tag 拿 videoid → merge_leve2(二级品类)。
+     * 该表无分区,按 videoid IN (...) 分批拉取,缺失/异常一律返回不含该 id 的 entry,调用方按 null 处理。
+     */
+    private Map<Long, String> fetchVideoCategoryMap(List<Long> videoIds) {
+        Map<Long, String> result = new HashMap<>();
+        if (CollectionUtils.isEmpty(videoIds)) {
+            return result;
+        }
+        for (List<Long> partition : Lists.partition(videoIds, ODPS_CATEGORY_CHUNK)) {
+            String inClause = partition.stream().map(String::valueOf).collect(Collectors.joining(","));
+            String sql = "SELECT videoid, merge_leve2 FROM loghubods.video_merge_tag WHERE videoid IN (" + inClause + ")";
+            try {
+                List<Record> records = odpsManager.query(sql);
+                if (CollectionUtils.isEmpty(records)) {
+                    continue;
+                }
+                for (Record r : records) {
+                    Object vid = r.get("videoid");
+                    Object cat = r.get("merge_leve2");
+                    if (vid == null || cat == null) continue;
+                    try {
+                        result.put(Long.parseLong(vid.toString()), cat.toString());
+                    } catch (NumberFormatException ignore) {
+                    }
+                }
+            } catch (Exception e) {
+                log.error("fetchVideoCategoryMap error, chunkSize={}", partition.size(), e);
+            }
+        }
+        log.info("fetchVideoCategoryMap done, input={}, mapped={}", videoIds.size(), result.size());
+        return result;
+    }
+
     @XxlJob("checkContentPlatformDemandVideoStatusJob")
     public ReturnT<String> checkContentPlatformDemandVideoStatusJob(String param) {
         String dt = demandVideoMapperExt.getMaxDt();

api-module/src/main/java/com/tzld/piaoquan/api/model/po/contentplatform/ContentPlatformDemandVideo.java

@@ -49,6 +49,8 @@ public class ContentPlatformDemandVideo {
 
     private String categoryName;
 
+    private String category;
+
     private Integer crowdCount;
 
     private Integer videoCount;
@@ -279,6 +281,14 @@ public class ContentPlatformDemandVideo {
         this.categoryName = categoryName;
     }
 
+    public String getCategory() {
+        return category;
+    }
+
+    public void setCategory(String category) {
+        this.category = category;
+    }
+
     public Integer getCrowdCount() {
         return crowdCount;
     }
@@ -461,6 +471,7 @@ public class ContentPlatformDemandVideo {
         sb.append(", standardElement=").append(standardElement);
         sb.append(", elementDimension=").append(elementDimension);
         sb.append(", categoryName=").append(categoryName);
+        sb.append(", category=").append(category);
         sb.append(", crowdCount=").append(crowdCount);
         sb.append(", videoCount=").append(videoCount);
         sb.append(", visitUv=").append(visitUv);

api-module/src/main/java/com/tzld/piaoquan/api/service/contentplatform/impl/ContentPlatformPlanServiceImpl.java

@@ -867,16 +867,17 @@ public class ContentPlatformPlanServiceImpl implements ContentPlatformPlanServic
         String ghName = StringUtils.hasText(param.getGhName()) ? param.getGhName() : null;
         String channelName = resolveChannelName(param);
 
+        String category = StringUtils.hasText(param.getCategory()) ? param.getCategory() : null;
         List<ContentPlatformDemandVideo> rows = demandVideoMapperExt.selectForRecommend(
-                dt, channelName, crowdSegment, DEMAND_STRATEGY_PRIOR_SCENE, null, null, null, ghName, null, limit, false);
+                dt, channelName, crowdSegment, DEMAND_STRATEGY_PRIOR_SCENE, null, null, null, ghName, null, category, limit, false);
         if (ghName != null && rows.isEmpty()) {
             rows = demandVideoMapperExt.selectForRecommend(
-                    dt, channelName, crowdSegment, DEMAND_STRATEGY_PRIOR_SCENE, null, null, null, null, null, limit, false);
+                    dt, channelName, crowdSegment, DEMAND_STRATEGY_PRIOR_SCENE, null, null, null, null, null, category, limit, false);
         }
         // 跨渠道退化:channel_name 命中但 crowd_segment 在对侧渠道下 0 行(如公众号账号切到企微入口)→ 去 crowd_segment,只按 channel_name 拉通用数据
         if (channelName != null && rows.isEmpty()) {
             rows = demandVideoMapperExt.selectForRecommend(
-                    dt, channelName, null, DEMAND_STRATEGY_PRIOR_SCENE, null, null, null, null, null, limit, false);
+                    dt, channelName, null, DEMAND_STRATEGY_PRIOR_SCENE, null, null, null, null, null, category, limit, false);
         }
         // 1. 同 video_id 取 total_rov 最大的代表行(SQL 已排序,putIfAbsent 保留首次)
         LinkedHashMap<Long, ContentPlatformDemandVideo> bestPerVideo = new LinkedHashMap<>();
@@ -921,18 +922,19 @@ public class ContentPlatformPlanServiceImpl implements ContentPlatformPlanServic
         String channelName = resolveChannelName(param);
         int fetchLimit = Math.max(limit * 3, DEMAND_CANDIDATE_LIMIT);
 
+        String category = StringUtils.hasText(param.getCategory()) ? param.getCategory() : null;
         List<ContentPlatformDemandVideo> rows = demandVideoMapperExt.selectForRecommend(
-                dt, channelName, crowdSegment, DEMAND_STRATEGY_PRIOR, PRIOR_PREMIUM_DIMENSION, null, null, ghName, null, fetchLimit, false);
+                dt, channelName, crowdSegment, DEMAND_STRATEGY_PRIOR, PRIOR_PREMIUM_DIMENSION, null, null, ghName, null, category, fetchLimit, false);
 
         // 退化:该 ghName 无数据 → 退回渠道粒度
         if (ghName != null && rows.isEmpty()) {
             rows = demandVideoMapperExt.selectForRecommend(
-                    dt, channelName, crowdSegment, DEMAND_STRATEGY_PRIOR, PRIOR_PREMIUM_DIMENSION, null, null, null, null, fetchLimit, false);
+                    dt, channelName, crowdSegment, DEMAND_STRATEGY_PRIOR, PRIOR_PREMIUM_DIMENSION, null, null, null, null, category, fetchLimit, false);
         }
         // 跨渠道退化:channel_name 命中但 crowd_segment 在对侧 0 行 → 去 crowd_segment 拉通用数据
         if (channelName != null && rows.isEmpty()) {
             rows = demandVideoMapperExt.selectForRecommend(
-                    dt, channelName, null, DEMAND_STRATEGY_PRIOR, PRIOR_PREMIUM_DIMENSION, null, null, null, null, fetchLimit, false);
+                    dt, channelName, null, DEMAND_STRATEGY_PRIOR, PRIOR_PREMIUM_DIMENSION, null, null, null, null, category, fetchLimit, false);
         }
 
         // prior 池近 7 日 rov 下限,过滤掉低质量近期表现的视频(0513 验证 ≥0.02 保留 ~41%)
@@ -1000,24 +1002,25 @@ public class ContentPlatformPlanServiceImpl implements ContentPlatformPlanServic
         String channelName = resolveChannelName(param);
         int fetchLimit = Math.max(limit * 3, DEMAND_CANDIDATE_LIMIT);
 
+        String category = StringUtils.hasText(param.getCategory()) ? param.getCategory() : null;
         List<ContentPlatformDemandVideo> stageAbs = demandVideoMapperExt.selectForRecommend(
-                dt, channelName, crowdSegment, DEMAND_STRATEGY_POSTERIOR, null, null, POSTERIOR_FILTER_ABS_LIKE, ghName, POSTERIOR_DRIVE_DIMENSION_TIME, fetchLimit, true);
+                dt, channelName, crowdSegment, DEMAND_STRATEGY_POSTERIOR, null, null, POSTERIOR_FILTER_ABS_LIKE, ghName, POSTERIOR_DRIVE_DIMENSION_TIME, category, fetchLimit, true);
         List<ContentPlatformDemandVideo> stageRel = demandVideoMapperExt.selectForRecommend(
-                dt, channelName, crowdSegment, DEMAND_STRATEGY_POSTERIOR, null, null, POSTERIOR_FILTER_REL_LIKE, ghName, POSTERIOR_DRIVE_DIMENSION_TIME, fetchLimit, true);
+                dt, channelName, crowdSegment, DEMAND_STRATEGY_POSTERIOR, null, null, POSTERIOR_FILTER_REL_LIKE, ghName, POSTERIOR_DRIVE_DIMENSION_TIME, category, fetchLimit, true);
 
         // 退化:该 ghName 在两阶段都无数据 → 退回渠道粒度(drive_dimension_time 仍严格为"昨日")
         if (ghName != null && stageAbs.isEmpty() && stageRel.isEmpty()) {
             stageAbs = demandVideoMapperExt.selectForRecommend(
-                    dt, channelName, crowdSegment, DEMAND_STRATEGY_POSTERIOR, null, null, POSTERIOR_FILTER_ABS_LIKE, null, POSTERIOR_DRIVE_DIMENSION_TIME, fetchLimit, true);
+                    dt, channelName, crowdSegment, DEMAND_STRATEGY_POSTERIOR, null, null, POSTERIOR_FILTER_ABS_LIKE, null, POSTERIOR_DRIVE_DIMENSION_TIME, category, fetchLimit, true);
             stageRel = demandVideoMapperExt.selectForRecommend(
-                    dt, channelName, crowdSegment, DEMAND_STRATEGY_POSTERIOR, null, null, POSTERIOR_FILTER_REL_LIKE, null, POSTERIOR_DRIVE_DIMENSION_TIME, fetchLimit, true);
+                    dt, channelName, crowdSegment, DEMAND_STRATEGY_POSTERIOR, null, null, POSTERIOR_FILTER_REL_LIKE, null, POSTERIOR_DRIVE_DIMENSION_TIME, category, fetchLimit, true);
         }
         // 跨渠道退化:channel_name 命中但 crowd_segment 在对侧 0 行 → 去 crowd_segment 拉通用数据
         if (channelName != null && stageAbs.isEmpty() && stageRel.isEmpty()) {
             stageAbs = demandVideoMapperExt.selectForRecommend(
-                    dt, channelName, null, DEMAND_STRATEGY_POSTERIOR, null, null, POSTERIOR_FILTER_ABS_LIKE, null, POSTERIOR_DRIVE_DIMENSION_TIME, fetchLimit, true);
+                    dt, channelName, null, DEMAND_STRATEGY_POSTERIOR, null, null, POSTERIOR_FILTER_ABS_LIKE, null, POSTERIOR_DRIVE_DIMENSION_TIME, category, fetchLimit, true);
             stageRel = demandVideoMapperExt.selectForRecommend(
-                    dt, channelName, null, DEMAND_STRATEGY_POSTERIOR, null, null, POSTERIOR_FILTER_REL_LIKE, null, POSTERIOR_DRIVE_DIMENSION_TIME, fetchLimit, true);
+                    dt, channelName, null, DEMAND_STRATEGY_POSTERIOR, null, null, POSTERIOR_FILTER_REL_LIKE, null, POSTERIOR_DRIVE_DIMENSION_TIME, category, fetchLimit, true);
         }
 
         Function<ContentPlatformDemandVideo, String> keyFn = r ->

api-module/src/main/resources/mapper/contentplatform/ContentPlatformDemandVideoMapper.xml

@@ -26,6 +26,7 @@
     <result column="standard_element" jdbcType="VARCHAR" property="standardElement" />
     <result column="element_dimension" jdbcType="VARCHAR" property="elementDimension" />
     <result column="category_name" jdbcType="VARCHAR" property="categoryName" />
+    <result column="category" jdbcType="VARCHAR" property="category" />
     <result column="crowd_count" jdbcType="INTEGER" property="crowdCount" />
     <result column="video_count" jdbcType="INTEGER" property="videoCount" />
     <result column="visit_uv" jdbcType="BIGINT" property="visitUv" />

api-module/src/main/resources/mapper/contentplatform/ext/ContentPlatformDemandVideoMapperExt.xml

@@ -5,7 +5,7 @@
     <insert id="batchInsert" parameterType="java.util.List">
         INSERT INTO content_platform_demand_video
         (dt, channel_name, crowd_segment, dimension, point_type, standard_element, element_dimension,
-         category_name, demand_id, crowd_package, conversion_target, partner, account, scene_value,
+         category_name, category, demand_id, crowd_package, conversion_target, partner, account, scene_value,
          demand_strategy, drive_dimension_time, demand_filter_sort_strategy, demand_type,
          demand_content_id, demand_content_title, demand_content_topic,
          crowd_count, video_count, visit_uv, uv_ratio, total_rov, online_action, match_experiment_id,
@@ -14,7 +14,7 @@
         VALUES
         <foreach collection="list" item="item" separator=",">
             (#{item.dt}, #{item.channelName}, #{item.crowdSegment}, #{item.dimension}, #{item.pointType}, #{item.standardElement}, #{item.elementDimension},
-             #{item.categoryName}, #{item.demandId}, #{item.crowdPackage}, #{item.conversionTarget}, #{item.partner}, #{item.account}, #{item.sceneValue},
+             #{item.categoryName}, #{item.category}, #{item.demandId}, #{item.crowdPackage}, #{item.conversionTarget}, #{item.partner}, #{item.account}, #{item.sceneValue},
              #{item.demandStrategy}, #{item.driveDimensionTime}, #{item.demandFilterSortStrategy}, #{item.demandType},
              #{item.demandContentId}, #{item.demandContentTitle}, #{item.demandContentTopic},
              #{item.crowdCount}, #{item.videoCount}, #{item.visitUv}, #{item.uvRatio}, #{item.totalRov}, #{item.onlineAction}, #{item.matchExperimentId},
@@ -33,7 +33,7 @@
 
     <select id="selectByCondition" resultType="com.tzld.piaoquan.api.model.po.contentplatform.ContentPlatformDemandVideo">
         SELECT id, dt, channel_name, channel_level3, crowd_segment, dimension, point_type, standard_element, element_dimension,
-               category_name, demand_id, crowd_package, conversion_target, partner, account, scene_value,
+               category_name, category, demand_id, crowd_package, conversion_target, partner, account, scene_value,
                demand_strategy, drive_dimension_time, demand_filter_sort_strategy, demand_type,
                demand_content_id, demand_content_title, demand_content_topic,
                crowd_count, video_count, visit_uv, uv_ratio, total_rov, online_action, match_experiment_id,
@@ -80,7 +80,7 @@
 
     <select id="selectForRecommend" resultType="com.tzld.piaoquan.api.model.po.contentplatform.ContentPlatformDemandVideo">
         SELECT id, dt, channel_name, channel_level3, crowd_segment, dimension, point_type, standard_element,
-               category_name, demand_id, crowd_package, conversion_target, partner, account, scene_value,
+               category_name, category, demand_id, crowd_package, conversion_target, partner, account, scene_value,
                demand_strategy, drive_dimension_time, demand_filter_sort_strategy, demand_type,
                demand_content_id, demand_content_title, demand_content_topic,
                crowd_count, video_count, visit_uv, uv_ratio, total_rov, online_action, match_experiment_id,
@@ -88,6 +88,7 @@
                match_text, title, cover, video, experiment_id, status, create_timestamp, update_timestamp
         FROM content_platform_demand_video
         WHERE dt = #{dt} AND status = 1
+        AND (category IS NULL OR category NOT IN ('早中晚好','节日祝福'))
         <if test="channelName != null and channelName != ''">
             AND channel_name = #{channelName}
         </if>
@@ -112,6 +113,9 @@
         <if test="driveDimensionTime != null and driveDimensionTime != ''">
             AND drive_dimension_time = #{driveDimensionTime}
         </if>
+        <if test="category != null and category != ''">
+            AND category = #{category}
+        </if>
         <if test="excludeSelfTitle">
             AND (title IS NULL OR demand_content_title IS NULL OR title &lt;&gt; demand_content_title)
         </if>