视频数据查询缓存

core/src/main/java/com/tzld/videoVector/job/VideoDetailSyncJob.java

@@ -41,7 +41,7 @@ public class VideoDetailSyncJob {
     private RedisUtils redisUtils;
 
     /** ODPS SQL IN 子句批次大小 */
-    private static final int ODPS_BATCH_SIZE = 200;
+    private static final int ODPS_BATCH_SIZE = 1000;
 
     /** 查询近一年数据的 dt 偏移天数 */
     private static final int DT_RANGE_DAYS = 365;
@@ -192,7 +192,7 @@ public class VideoDetailSyncJob {
                         "  WHERE dt >= '%s' AND dt <= '%s' AND 视频id IN (%s) " +
                         "  GROUP BY 视频id" +
                         ") m ON t.视频id = m.vid AND t.dt = m.max_dt " +
-                        "WHERE t.dt >= '%s' AND t.dt <= '%s' AND t.视频id IN (%s)",
+                        "WHERE t.dt >= '%s' AND t.dt <= '%s' AND t.视频id IN (%s);",
                 DIMENSION_SELECT,
                 dtMin, dtMax, idsStr,
                 dtMin, dtMax, idsStr);
@@ -207,7 +207,7 @@ public class VideoDetailSyncJob {
                 "SELECT 视频id, %s " +
                         "FROM loghubods.video_dimension_detail_add_column " +
                         "WHERE dt >= '%s' AND dt <= '%s' AND 视频id IN (%s) " +
-                        "GROUP BY 视频id",
+                        "GROUP BY 视频id;",
                 METRICS_SQL,
                 dtMin, dtMax, idsStr);
     }
@@ -228,7 +228,17 @@ public class VideoDetailSyncJob {
                     String colName = columns[i].getName();
                     // 排除视频id本身，其余字段全部写入
                     if (!"视频id".equals(colName)) {
-                        detail.put(colName, record.getString(i));
+                        String strValue;
+                        try {
+                            strValue = record.getString(i);
+                        } catch (Exception ex) {
+                            Object val = record.get(i);
+                            strValue = val == null ? null : String.valueOf(val);
+                        }
+                        if ("\\N".equals(strValue)) {
+                            strValue = null;
+                        }
+                        detail.put(colName, strValue);
                     }
                 }
             }

pom.xml

@@ -146,7 +146,7 @@
         <dependency>
             <groupId>com.aliyun.odps</groupId>
             <artifactId>odps-sdk-core</artifactId>
-            <version>0.27.2-public</version>
+            <version>0.48.6-public</version>
         </dependency>
 
         <dependency>

server/src/main/java/com/tzld/videoVector/controller/XxlJobController.java

@@ -23,6 +23,9 @@ public class XxlJobController {
     @Autowired
     private VideoVectorTextBackfillJob videoVectorTextBackfillJob;
 
+    @Autowired
+    private VideoDetailSyncJob videoDetailSyncJob;
+
     // ==================== 视频向量化任务 ====================
 
     @GetMapping("/vectorVideoJob")
@@ -75,5 +78,12 @@ public class XxlJobController {
         return CommonResponse.success();
     }
 
+    // ==================== 视频详情同步任务 ====================
+
+    @GetMapping("/syncVideoDetailJob")
+    public CommonResponse<Void> syncVideoDetailJob() {
+        videoDetailSyncJob.syncVideoDetailJob(null);
+        return CommonResponse.success();
+    }
 
 }