增加视频替换

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/job/MatchVideoJob.java

@@ -68,6 +68,17 @@ public class MatchVideoJob {
         return ReturnT.SUCCESS;
     }
 
+    @XxlJob("updateRovnJob")
+    public ReturnT<String> updateRovnJob(String param) {
+        try {
+            matchVideoService.updateRovn();
+        } catch (Exception e) {
+            LarkRobotUtil.sendMessage("updateRovnJob异常，请及时查看，@薛一鸣");
+            log.error("updateRovnJob error", e);
+        }
+        return ReturnT.SUCCESS;
+    }
+
 
     @XxlJob("testJob")
     public ReturnT<String> testJob(String param) {

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/model/po/longarticle/PiaoquanSourceVideoPool.java

@@ -17,6 +17,8 @@ public class PiaoquanSourceVideoPool {
 
     private Integer status;
 
+    private Double rovn;
+
     private Long statusUpdateTimestamp;
 
     public Integer getId() {
@@ -75,6 +77,14 @@ public class PiaoquanSourceVideoPool {
         this.status = status;
     }
 
+    public Double getRovn() {
+        return rovn;
+    }
+
+    public void setRovn(Double rovn) {
+        this.rovn = rovn;
+    }
+
     public Long getStatusUpdateTimestamp() {
         return statusUpdateTimestamp;
     }
@@ -96,6 +106,7 @@ public class PiaoquanSourceVideoPool {
         sb.append(", videoRecallStrategy=").append(videoRecallStrategy);
         sb.append(", recallDate=").append(recallDate);
         sb.append(", status=").append(status);
+        sb.append(", rovn=").append(rovn);
         sb.append(", statusUpdateTimestamp=").append(statusUpdateTimestamp);
         sb.append("]");
         return sb.toString();

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/model/po/longarticle/PiaoquanSourceVideoPoolExample.java

@@ -593,6 +593,66 @@ public class PiaoquanSourceVideoPoolExample {
             return (Criteria) this;
         }
 
+        public Criteria andRovnIsNull() {
+            addCriterion("rovn is null");
+            return (Criteria) this;
+        }
+
+        public Criteria andRovnIsNotNull() {
+            addCriterion("rovn is not null");
+            return (Criteria) this;
+        }
+
+        public Criteria andRovnEqualTo(Double value) {
+            addCriterion("rovn =", value, "rovn");
+            return (Criteria) this;
+        }
+
+        public Criteria andRovnNotEqualTo(Double value) {
+            addCriterion("rovn <>", value, "rovn");
+            return (Criteria) this;
+        }
+
+        public Criteria andRovnGreaterThan(Double value) {
+            addCriterion("rovn >", value, "rovn");
+            return (Criteria) this;
+        }
+
+        public Criteria andRovnGreaterThanOrEqualTo(Double value) {
+            addCriterion("rovn >=", value, "rovn");
+            return (Criteria) this;
+        }
+
+        public Criteria andRovnLessThan(Double value) {
+            addCriterion("rovn <", value, "rovn");
+            return (Criteria) this;
+        }
+
+        public Criteria andRovnLessThanOrEqualTo(Double value) {
+            addCriterion("rovn <=", value, "rovn");
+            return (Criteria) this;
+        }
+
+        public Criteria andRovnIn(List<Double> values) {
+            addCriterion("rovn in", values, "rovn");
+            return (Criteria) this;
+        }
+
+        public Criteria andRovnNotIn(List<Double> values) {
+            addCriterion("rovn not in", values, "rovn");
+            return (Criteria) this;
+        }
+
+        public Criteria andRovnBetween(Double value1, Double value2) {
+            addCriterion("rovn between", value1, value2, "rovn");
+            return (Criteria) this;
+        }
+
+        public Criteria andRovnNotBetween(Double value1, Double value2) {
+            addCriterion("rovn not between", value1, value2, "rovn");
+            return (Criteria) this;
+        }
+
         public Criteria andStatusUpdateTimestampIsNull() {
             addCriterion("status_update_timestamp is null");
             return (Criteria) this;

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/service/local/impl/ContentServiceImpl.java

@@ -370,21 +370,20 @@ public class ContentServiceImpl implements ContentService {
             if (accountNameSet.contains(matchVideo.getAccountName()) && i == 2) {
                 String articleTitle = kimiText.getArticleTitle();
                 PiaoquanSourceVideoPoolExample example = new PiaoquanSourceVideoPoolExample();
-                example.createCriteria().andStatusEqualTo(2);
+                example.createCriteria().andStatusEqualTo(2).andRovnGreaterThan(0.0);
                 List<PiaoquanSourceVideoPool> piaoquanSourceVideoPools = piaoquanSourceVideoPoolMapper.selectByExample(example);
                 if (!CollectionUtils.isEmpty(piaoquanSourceVideoPools)) {
                     List<String> titles = piaoquanSourceVideoPools.stream().map(PiaoquanSourceVideoPool::getTitle).collect(Collectors.toList());
-                    List<Float> floats = NlpUtils.getTitleSimilarityWithNlp(articleTitle, titles);
-                    if (floats != null) {
-                        int maxIndex = 0;
-                        float maxValue = floats.get(0);
-                        for (int j = 1; j < floats.size(); j++) {
-                            if (floats.get(j) > maxValue) {
-                                maxValue = floats.get(j);
-                                maxIndex = j;
+                    List<Float> scores = NlpUtils.getTitleSimilarityWithNlp(articleTitle, titles);
+                    if (scores != null) {
+                        List<PiaoquanSourceVideoPool> filteredResults = new ArrayList<>();
+                        final float SCORE_THRESHOLD = 0.5f;
+                        for (int j = 0; j < scores.size(); j++) {
+                            if (scores.get(j) > SCORE_THRESHOLD) {
+                                filteredResults.add(piaoquanSourceVideoPools.get(j));
                             }
                         }
-                        PiaoquanSourceVideoPool piaoquanSourceVideoPool = piaoquanSourceVideoPools.get(maxIndex);
+                        PiaoquanSourceVideoPool piaoquanSourceVideoPool = filteredResults.stream().max(Comparator.comparingDouble(PiaoquanSourceVideoPool::getRovn)).get();
                         JSONObject videoDetail = VideoUtils.getVideoDetail((long) piaoquanSourceVideoPool.getVideoId());
                         if (videoDetail != null) {
                             crawlerVideoId = piaoquanSourceVideoPool.getVideoId();

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/service/local/impl/MatchVideoServiceImpl.java

@@ -2,6 +2,7 @@ package com.tzld.piaoquan.longarticle.service.local.impl;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.aliyun.odps.data.Record;
 import com.tzld.piaoquan.longarticle.common.enums.ContentResultStatusEnum;
 import com.tzld.piaoquan.longarticle.common.enums.ContentStatusEnum;
 import com.tzld.piaoquan.longarticle.common.enums.PublicFlagEnum;
@@ -16,7 +17,9 @@ import com.tzld.piaoquan.longarticle.model.po.aigc.ProducePlanExeRecordExample;
 import com.tzld.piaoquan.longarticle.model.po.longarticle.*;
 import com.tzld.piaoquan.longarticle.service.local.KimiService;
 import com.tzld.piaoquan.longarticle.service.remote.MatchService;
+import com.tzld.piaoquan.longarticle.utils.DateUtil;
 import com.tzld.piaoquan.longarticle.utils.LarkRobotUtil;
+import com.tzld.piaoquan.longarticle.utils.OdpsUtil;
 import com.tzld.piaoquan.longarticle.utils.page.Page;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -91,6 +94,9 @@ public class MatchVideoServiceImpl {
     @Autowired
     private MatchService matchService;
 
+    @Autowired
+    private PiaoquanSourceVideoPoolMapper piaoquanSourceVideoPoolMapper;
+
 
     // 定义一个阻塞队列
     private static final ArrayBlockingQueue<MatchVideo> matchKimiVideoQueue = new ArrayBlockingQueue<>(1000000);
@@ -706,6 +712,60 @@ public class MatchVideoServiceImpl {
         updateStatus(matchVideo.getId(), ContentStatusEnum.SUCCESS_3.getStatusCode());
         updateResultStatus(matchVideo.getContentId(), ContentResultStatusEnum.SUCCESS.getStatus());
     }
+
+    public void updateRovn() {
+        PiaoquanSourceVideoPoolExample example = new PiaoquanSourceVideoPoolExample();
+        example.createCriteria().andStatusEqualTo(2);
+        List<PiaoquanSourceVideoPool> piaoquanSourceVideoPools = piaoquanSourceVideoPoolMapper.selectByExample(example);
+        final int BATCH_SIZE = 100;
+        for (int i = 0; i < piaoquanSourceVideoPools.size(); i += BATCH_SIZE) {
+            List<PiaoquanSourceVideoPool> batch = piaoquanSourceVideoPools.subList(
+                    i,
+                    Math.min(i + BATCH_SIZE, piaoquanSourceVideoPools.size())
+            );
+            if (!CollectionUtils.isEmpty(batch)) {
+                StringBuilder inClause = new StringBuilder();
+                inClause.append("(");
+                for (int j = 0; j < batch.size(); j++) {
+                    inClause.append("'").append(batch.get(j).getVideoId()).append("'");
+                    if (j < batch.size() - 1) {
+                        inClause.append(",");
+                    }
+                }
+                inClause.append(")");
+                String sqlStr = "SELECT * " +
+                        "FROM ( " +
+                        "    SELECT *, " +
+                        "           ROW_NUMBER() OVER (PARTITION BY 视频id ORDER BY dt DESC) AS rn " +
+                        "    FROM loghubods.wecom_cooperation_video_candidate_pool " +
+                        "    WHERE dt <= %s " +
+                        "      AND 视频id IN %s " +
+                        ") t " +
+                        "WHERE rn = 1;";
+                String sql = String.format(sqlStr, DateUtil.getBeforeDayDateString("yyyyMMdd"), inClause);
+                List<Record> recordList = OdpsUtil.getOdpsData(sql);
+                if (CollectionUtils.isEmpty(recordList)) {
+                    continue;
+                }
+                Map<Integer, Double> scoreMap = new HashMap<>();
+                for (Record record : recordList) {
+                    Integer videoId = Integer.parseInt((String) record.get(0));
+                    Double score = Double.parseDouble((String) record.get(4));
+                    scoreMap.put(videoId, score);
+                }
+                for (PiaoquanSourceVideoPool pool : batch) {
+                    Double score = scoreMap.get(pool.getVideoId());
+                    if (score != null) {
+                        Integer id = pool.getId();
+                        PiaoquanSourceVideoPool updatePiaoquanSourceVideoPool = new PiaoquanSourceVideoPool();
+                        updatePiaoquanSourceVideoPool.setId(id);
+                        updatePiaoquanSourceVideoPool.setRovn(score);
+                        piaoquanSourceVideoPoolMapper.updateByPrimaryKeySelective(updatePiaoquanSourceVideoPool);
+                    }
+                }
+            }
+        }
+    }
 }
 
 

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/utils/DateUtil.java

@@ -40,5 +40,12 @@ public class DateUtil {
         return dateFormat.format(tomorrow);
     }
 
+    public static String getBeforeDayDateString(String format) {
+        DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern(format);
+        LocalDate today = LocalDate.now();
+        LocalDate yesterday = today.minusDays(1);
+        return dateFormat.format(yesterday);
+    }
+
 
 }

long-article-server/src/main/java/com/tzld/piaoquan/longarticle/utils/OdpsUtil.java

@@ -0,0 +1,33 @@
+package com.tzld.piaoquan.longarticle.utils;
+
+import com.aliyun.odps.Instance;
+import com.aliyun.odps.Odps;
+import com.aliyun.odps.OdpsException;
+import com.aliyun.odps.account.Account;
+import com.aliyun.odps.account.AliyunAccount;
+import com.aliyun.odps.data.Record;
+import com.aliyun.odps.task.SQLTask;
+
+import java.util.List;
+
+public class OdpsUtil {
+
+    public static List<Record> getOdpsData(String sql) {
+        String accessId = "LTAI9EBa0bd5PrDa";
+        String accessKey = "vAalxds7YxhfOA2yVv8GziCg3Y87v5";
+        String endpoint = "http://service.odps.aliyun.com/api";
+        Account account = new AliyunAccount(accessId, accessKey);
+        Odps odps = new Odps(account);
+        odps.setEndpoint(endpoint);
+        odps.setDefaultProject("loghubods");
+        Instance i;
+        try {
+            i = SQLTask.run(odps, sql);
+            i.waitForSuccess();
+            return SQLTask.getResult(i);
+        } catch (OdpsException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+}

long-article-server/src/main/resources/mapper/longarticle/PiaoquanSourceVideoPoolMapper.xml

@@ -9,6 +9,7 @@
     <result column="video_recall_strategy" jdbcType="VARCHAR" property="videoRecallStrategy" />
     <result column="recall_date" jdbcType="DATE" property="recallDate" />
     <result column="status" jdbcType="INTEGER" property="status" />
+    <result column="rovn" jdbcType="DOUBLE" property="rovn" />
     <result column="status_update_timestamp" jdbcType="BIGINT" property="statusUpdateTimestamp" />
   </resultMap>
   <sql id="Example_Where_Clause">
@@ -70,7 +71,8 @@
     </where>
   </sql>
   <sql id="Base_Column_List">
-    id, video_id, title, category, video_recall_strategy, recall_date, `status`, status_update_timestamp
+    id, video_id, title, category, video_recall_strategy, recall_date, `status`, rovn, 
+    status_update_timestamp
   </sql>
   <select id="selectByExample" parameterType="com.tzld.piaoquan.longarticle.model.po.longarticle.PiaoquanSourceVideoPoolExample" resultMap="BaseResultMap">
     select
@@ -108,10 +110,12 @@
   <insert id="insert" parameterType="com.tzld.piaoquan.longarticle.model.po.longarticle.PiaoquanSourceVideoPool">
     insert into piaoquan_source_video_pool (id, video_id, title, 
       category, video_recall_strategy, recall_date, 
-      `status`, status_update_timestamp)
+      `status`, rovn, status_update_timestamp
+      )
     values (#{id,jdbcType=INTEGER}, #{videoId,jdbcType=INTEGER}, #{title,jdbcType=VARCHAR}, 
       #{category,jdbcType=VARCHAR}, #{videoRecallStrategy,jdbcType=VARCHAR}, #{recallDate,jdbcType=DATE}, 
-      #{status,jdbcType=INTEGER}, #{statusUpdateTimestamp,jdbcType=BIGINT})
+      #{status,jdbcType=INTEGER}, #{rovn,jdbcType=DOUBLE}, #{statusUpdateTimestamp,jdbcType=BIGINT}
+      )
   </insert>
   <insert id="insertSelective" parameterType="com.tzld.piaoquan.longarticle.model.po.longarticle.PiaoquanSourceVideoPool">
     insert into piaoquan_source_video_pool
@@ -137,6 +141,9 @@
       <if test="status != null">
         `status`,
       </if>
+      <if test="rovn != null">
+        rovn,
+      </if>
       <if test="statusUpdateTimestamp != null">
         status_update_timestamp,
       </if>
@@ -163,6 +170,9 @@
       <if test="status != null">
         #{status,jdbcType=INTEGER},
       </if>
+      <if test="rovn != null">
+        #{rovn,jdbcType=DOUBLE},
+      </if>
       <if test="statusUpdateTimestamp != null">
         #{statusUpdateTimestamp,jdbcType=BIGINT},
       </if>
@@ -198,6 +208,9 @@
       <if test="record.status != null">
         `status` = #{record.status,jdbcType=INTEGER},
       </if>
+      <if test="record.rovn != null">
+        rovn = #{record.rovn,jdbcType=DOUBLE},
+      </if>
       <if test="record.statusUpdateTimestamp != null">
         status_update_timestamp = #{record.statusUpdateTimestamp,jdbcType=BIGINT},
       </if>
@@ -215,6 +228,7 @@
       video_recall_strategy = #{record.videoRecallStrategy,jdbcType=VARCHAR},
       recall_date = #{record.recallDate,jdbcType=DATE},
       `status` = #{record.status,jdbcType=INTEGER},
+      rovn = #{record.rovn,jdbcType=DOUBLE},
       status_update_timestamp = #{record.statusUpdateTimestamp,jdbcType=BIGINT}
     <if test="_parameter != null">
       <include refid="Update_By_Example_Where_Clause" />
@@ -241,6 +255,9 @@
       <if test="status != null">
         `status` = #{status,jdbcType=INTEGER},
       </if>
+      <if test="rovn != null">
+        rovn = #{rovn,jdbcType=DOUBLE},
+      </if>
       <if test="statusUpdateTimestamp != null">
         status_update_timestamp = #{statusUpdateTimestamp,jdbcType=BIGINT},
       </if>
@@ -255,6 +272,7 @@
       video_recall_strategy = #{videoRecallStrategy,jdbcType=VARCHAR},
       recall_date = #{recallDate,jdbcType=DATE},
       `status` = #{status,jdbcType=INTEGER},
+      rovn = #{rovn,jdbcType=DOUBLE},
       status_update_timestamp = #{statusUpdateTimestamp,jdbcType=BIGINT}
     where id = #{id,jdbcType=INTEGER}
   </update>