dedup message

etl-core/src/main/java/com/tzld/crawler/etl/mq/EtlMQConsumer.java

@@ -114,7 +114,8 @@ public class EtlMQConsumer {
                     String messageId = message.getMessageId();
                     String key = String.format(DEDUP_KEY, messageId);
                     if (!redisUtil.setNx(key, "1", CacheConstant.ONE_DAY)) {
-
+                        log.info("dedup message: {} from topic: {}, group: {} messageId: {}", message, topic, groupId);
+                        continue;
                     }
                     CrawlerEtlParam param = JSONObject.parseObject(message.getMessageBodyString(), CrawlerEtlParam.class);
                     param.setMessageId(message.getMessageId());