|
@@ -53,6 +53,7 @@ import com.tzld.crawler.etl.service.SlsService;
|
|
|
import com.tzld.crawler.etl.service.feign.LongVideoFeign;
|
|
|
import com.tzld.crawler.etl.service.strategy.StrategyHandlerService;
|
|
|
import com.tzld.crawler.etl.util.*;
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
import org.apache.commons.lang3.tuple.Pair;
|
|
|
import org.slf4j.Logger;
|
|
|
import org.slf4j.LoggerFactory;
|
|
@@ -162,10 +163,10 @@ public class EtlServiceImpl implements EtlService {
|
|
|
if (result.notSuccess()) {
|
|
|
throw new CommonException(ExceptionEnum.PARAM_ERROR, "param validate failed." + result.getMessage());
|
|
|
}
|
|
|
-
|
|
|
- // 保存数据库(去重校验)
|
|
|
+ // 保存数据库(去重校验)
|
|
|
+ // 需要加一个判断,如果是看一看的视频,则不走去重策略
|
|
|
id = save2db(crawlerVideo);
|
|
|
- if (id <= 0) {
|
|
|
+ if (id <= 0 && !StringUtils.equals(crawlerVideo.getPlatform(), "kanyikan")) {
|
|
|
throw new CommonException(ExceptionEnum.DATA_ERROR, "save2db failed, duplicated out video id.");
|
|
|
}
|
|
|
|