log

etl-core/src/main/java/com/tzld/crawler/etl/service/impl/EtlServiceImpl.java

@@ -120,8 +120,9 @@ public class EtlServiceImpl implements EtlService {
         String platform = param.getPlatform();
         String strategy = param.getStrategy();
         try {
+            // 视频去重
             CrawlerVideoExample example = new CrawlerVideoExample();
-            example.createCriteria().andOutVideoIdEqualTo(param.getOutVideoId());
+            example.createCriteria().andOutVideoIdEqualTo(param.getOutVideoId()).andPlatformEqualTo(platform);
             long count = crawlerVideoMapper.countByExample(example);
             if (count > 1) {
                 slsService.log("message", "video" + param.getOutVideoId() + " has exist", "crawler", platform, "mode", strategy);