log

etl-core/src/main/java/com/tzld/crawler/etl/service/impl/EtlServiceImpl.java

@@ -37,6 +37,7 @@ import com.tzld.crawler.etl.model.dto.StrategyDataDto;
 import com.tzld.crawler.etl.model.dto.VideoInfoDto;
 import com.tzld.crawler.etl.model.param.CrawlerVideoSendParam;
 import com.tzld.crawler.etl.model.po.CrawlerVideo;
+import com.tzld.crawler.etl.model.po.CrawlerVideoExample;
 import com.tzld.crawler.etl.model.vo.CrawlerVideoVO;
 import com.tzld.crawler.etl.model.vo.WxVideoVO;
 import com.tzld.crawler.etl.service.EtlService;
@@ -119,6 +120,15 @@ public class EtlServiceImpl implements EtlService {
         String platform = param.getPlatform();
         String strategy = param.getStrategy();
         try {
+            CrawlerVideoExample example = new CrawlerVideoExample();
+            example.createCriteria().andOutVideoIdEqualTo(param.getOutVideoId());
+            long count = crawlerVideoMapper.countByExample(example);
+            if (count > 1) {
+                slsService.log("message", "video" + param.getOutVideoId() + " has exist", "crawler", platform, "mode", strategy);
+                log.info("video {} platform {} strategy {}  has exist.", param.getOutVideoId(), param.getPlatform(), param.getStrategy());
+                return;
+            }
+
             // 1.策略应用
             // 获取用户选择策略
             List<String> stategies = Lists.newArrayList("titleScore");
@@ -194,6 +204,9 @@ public class EtlServiceImpl implements EtlService {
                         log.error("feishu.sheet.id.map config of {} is null", platform);
                         return;
                     }
+                    String range = feishuRangeMap.get(platform + strategyType);
+
+                    log.info("sheetToken {} sheetId {} range {}", sheetToken, sheetId, range);
                     String fsResp = FeishuUtils.insertRows(feishuAppid, feishuAppsecret, sheetToken,
                             sheetId, 1, 2);
                     log.debug("insert columns to feishu sheet response is {}", fsResp);
@@ -222,8 +235,7 @@ public class EtlServiceImpl implements EtlService {
                     value.add(data.getVideoUrl());
                     values.add(value);
 
-                    fsResp = FeishuUtils.updateValues(feishuAppid, feishuAppsecret, sheetToken,
-                            feishuRangeMap.get(platform + strategyType), values);
+                    fsResp = FeishuUtils.updateValues(feishuAppid, feishuAppsecret, sheetToken, range, values);
                     log.debug("update feishu sheet value response is {}", fsResp);
 
                     slsService.log("message", "视频已保存至云文档", "crawler", platform, "mode", strategy);