feat:抽取公共模块

pom.xml

@@ -15,10 +15,15 @@
     <name>recommend-feature</name>
     <description>recommend-feature</description>
 
+    <properties>
+        <common.version>1.0.0</common.version>
+    </properties>
+
     <modules>
         <module>recommend-feature-service</module>
         <module>recommend-feature-client</module>
         <module>recommend-feature-produce</module>
+        <module>recommend-feature-common</module>
     </modules>
 
     <dependencies>

recommend-feature-client/pom.xml

@@ -59,9 +59,9 @@
                 <artifactId>protobuf-maven-plugin</artifactId>
                 <version>0.6.1</version>
                 <configuration>
-                    <protocArtifact>com.google.protobuf:protoc:3.12.0:exe:${os.detected.classifier}</protocArtifact>
+                    <protocArtifact>com.google.protobuf:protoc:3.12.0:exe:osx-x86_64</protocArtifact>
                     <pluginId>grpc-java</pluginId>
-                    <pluginArtifact>io.grpc:protoc-gen-grpc-java:1.34.1:exe:${os.detected.classifier}</pluginArtifact>
+                    <pluginArtifact>io.grpc:protoc-gen-grpc-java:1.34.1:exe:osx-x86_64</pluginArtifact>
                     <!--设置grpc生成代码到指定路径-->
                     <outputDirectory>${project.basedir}/src/main/java</outputDirectory>
                     <!--生成代码前是否清空目录-->

recommend-feature-common/pom.xml

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>recommend-feature</artifactId>
+        <groupId>com.tzld.piaoquan</groupId>
+        <version>1.0.0</version>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>recommend-feature-common</artifactId>
+
+    <dependencies>
+        <!-- 工具类依赖 -->
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.16</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-collections4</artifactId>
+            <version>4.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>fastjson</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-pool2</artifactId>
+        </dependency>
+        <!-- Snappy compression library -->
+        <dependency>
+            <groupId>org.xerial.snappy</groupId>
+            <artifactId>snappy-java</artifactId>
+            <version>1.1.8.4</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+            <version>1.7.28</version>
+        </dependency>
+    </dependencies>
+
+
+</project>

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/model/DTSConfig.java → recommend-feature-common/src/main/java/com/tzld/piaoquan/recommend/feature/common/model/DTSConfig.java

@@ -1,7 +1,6 @@
-package com.tzld.piaoquan.recommend.feature.produce.model;
+package com.tzld.piaoquan.recommend.feature.common.model;
 
 import lombok.Data;
-import lombok.extern.slf4j.Slf4j;
 
 import java.io.Serializable;
 import java.util.List;
@@ -10,7 +9,6 @@ import java.util.List;
  * @author dyp
  */
 @Data
-@Slf4j
 public class DTSConfig implements Serializable {
     private ODPS odps;
     private Redis redis;
@@ -33,11 +31,9 @@ public class DTSConfig implements Serializable {
 
     public boolean selfCheck() {
         if (odps == null) {
-            log.error("odps not config");
             return false;
         }
         if (redis == null) {
-            log.error("redis not config");
             return false;
         }
         return true;

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/util/CommonCollectionUtils.java → recommend-feature-common/src/main/java/com/tzld/piaoquan/recommend/feature/common/util/CommonCollectionUtils.java

@@ -1,7 +1,8 @@
-package com.tzld.piaoquan.recommend.feature.produce.util;
+package com.tzld.piaoquan.recommend.feature.common.util;
 
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.collections.MapUtils;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.MapUtils;
 
 import java.util.*;
 import java.util.function.Function;

recommend-feature-common/src/main/java/com/tzld/piaoquan/recommend/feature/common/util/CompressionUtil.java

@@ -0,0 +1,57 @@
+package com.tzld.piaoquan.recommend.feature.common.util;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.xerial.snappy.Snappy;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Objects;
+
+
+/**
+ * @author dyp
+ */
+@Slf4j
+public class CompressionUtil {
+
+    public static String snappyCompress(String input) throws IOException {
+        byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8);
+        byte[] compressedBytes = Snappy.compress(inputBytes);
+        return Base64.getEncoder().encodeToString(compressedBytes);
+    }
+
+    public static byte[] snappyCompressV2(String input) throws IOException {
+        byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8);
+        return Snappy.compress(inputBytes);
+    }
+
+    // 将Snappy压缩后的String解压缩回String
+    public static String snappyDecompress(String compressedInput) {
+        if (StringUtils.isBlank(compressedInput)) {
+            return "";
+        }
+        try {
+            byte[] compressedBytes = Base64.getDecoder().decode(compressedInput);
+            byte[] decompressedBytes = Snappy.uncompress(compressedBytes);
+            return new String(decompressedBytes, StandardCharsets.UTF_8);
+        } catch (Exception e) {
+            log.error("snappyDecompress error compressedInput {}", compressedInput, e);
+            return "";
+        }
+    }
+
+    public static String snappyDecompressV2(byte[] compressedInput) {
+        if (Objects.isNull(compressedInput)) {
+            return "";
+        }
+        try {
+            byte[] decompressedBytes = Snappy.uncompress(compressedInput);
+            return new String(decompressedBytes, StandardCharsets.UTF_8);
+        } catch (Exception e) {
+            log.error("snappyDecompressV2 error compressedInput {}", compressedInput, e);
+            return "";
+        }
+    }
+}

recommend-feature-service/src/main/java/com/tzld/piaoquan/recommend/feature/util/DateUtils.java → recommend-feature-common/src/main/java/com/tzld/piaoquan/recommend/feature/common/util/DateUtils.java

@@ -1,4 +1,4 @@
-package com.tzld.piaoquan.recommend.feature.util;
+package com.tzld.piaoquan.recommend.feature.common.util;
 
 import java.text.SimpleDateFormat;
 import java.util.Calendar;

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/util/JSONUtils.java → recommend-feature-common/src/main/java/com/tzld/piaoquan/recommend/feature/common/util/JSONUtils.java

@@ -1,4 +1,4 @@
-package com.tzld.piaoquan.recommend.feature.produce.util;
+package com.tzld.piaoquan.recommend.feature.common.util;
 
 import com.google.common.reflect.TypeToken;
 import com.google.gson.Gson;
@@ -18,7 +18,6 @@ public class JSONUtils implements Serializable {
         try {
             return new Gson().toJson(obj);
         } catch (Exception e) {
-            log.error("toJson exception", e);
             return "";
         }
     }
@@ -31,7 +30,6 @@ public class JSONUtils implements Serializable {
         try {
             return new Gson().fromJson(value, typeToken.getType());
         } catch (Exception e) {
-            log.error("fromJson error! value=[{}]", value, e);
         }
         return defaultValue;
     }

recommend-feature-common/src/main/java/com/tzld/piaoquan/recommend/feature/common/util/RedisUtil.java

@@ -0,0 +1,48 @@
+package com.tzld.piaoquan.recommend.feature.common.util;
+
+import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.common.model.DTSConfig;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RedisUtil {
+
+
+    public static String buildRedisKey(Map<String, String> fieldValueMap, DTSConfig config) {
+        // Note：写入和读取的key生成规则应保持一致
+        List<String> fields = config.getRedis().getKey();
+        if (CollectionUtils.isEmpty(fields)) {
+            return config.getRedis().getPrefix();
+        }
+        StringBuilder sb = new StringBuilder();
+        if (StringUtils.isNotBlank(config.getRedis().getPrefix())) {
+            sb.append(config.getRedis().getPrefix());
+        }
+        for (String field : fields) {
+            sb.append(":");
+            sb.append(fieldValueMap.get(field));
+        }
+        return sb.toString();
+    }
+
+    public static Map<String, String> buildValueMap(Map<String, String> record, DTSConfig config) {
+        Map<String, String> valueMap = new HashMap<>();
+        if (StringUtils.isNotBlank(config.getRedis().getFeatureField())) {
+            valueMap = JSONUtils.fromJson(record.get(config.getRedis().getFeatureField()), new TypeToken<Map<String, String>>() {
+            }, Collections.emptyMap());
+        } else if (CollectionUtils.isNotEmpty(config.getRedis().getValue())) {
+            for (String valueKey : config.getRedis().getValue()) {
+                valueMap.put(valueKey, record.get(valueKey));
+            }
+        } else {
+            valueMap = record;
+        }
+        return valueMap;
+    }
+
+}

recommend-feature-produce/pom.xml

@@ -97,11 +97,6 @@
             <artifactId>apollo-client</artifactId>
             <version>1.8.0</version>
         </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-simple</artifactId>
-            <version>1.7.28</version>
-        </dependency>
         <dependency>
             <groupId>com.aliyun.oss</groupId>
             <artifactId>aliyun-sdk-oss</artifactId>
@@ -113,16 +108,16 @@
             <artifactId>snappy-java</artifactId>
             <version>1.1.8.4</version>
         </dependency>
-<!--        <dependency>-->
-<!--            <groupId>com.squareup.okhttp3</groupId>-->
-<!--            <artifactId>okhttp</artifactId>-->
-<!--            <version>3.14.9</version>-->
-<!--        </dependency>-->
         <dependency>
             <groupId>com.squareup.okhttp</groupId>
             <artifactId>okhttp</artifactId>
             <version>2.7.5</version>
         </dependency>
+        <dependency>
+            <groupId>com.tzld.piaoquan</groupId>
+            <artifactId>recommend-feature-common</artifactId>
+            <version>${common.version}</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/FeatureDiff.java

@@ -2,7 +2,7 @@ package com.tzld.piaoquan.recommend.feature.produce;
 
 import com.google.common.reflect.TypeToken;
 import com.tzld.piaoquan.recommend.feature.produce.service.ODPSService;
-import com.tzld.piaoquan.recommend.feature.produce.util.JSONUtils;
+import com.tzld.piaoquan.recommend.feature.common.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang.StringUtils;
 import org.apache.spark.SparkConf;

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/FeatureRedisClean.java

@@ -1,11 +1,11 @@
 package com.tzld.piaoquan.recommend.feature.produce;
 
-import com.tzld.piaoquan.recommend.feature.produce.model.DTSConfig;
+import com.tzld.piaoquan.recommend.feature.common.model.DTSConfig;
 import com.tzld.piaoquan.recommend.feature.produce.service.CMDService;
 import com.tzld.piaoquan.recommend.feature.produce.service.DTSConfigService;
 import com.tzld.piaoquan.recommend.feature.produce.service.ODPSService;
 import com.tzld.piaoquan.recommend.feature.produce.service.RedisService;
-import com.tzld.piaoquan.recommend.feature.produce.util.JSONUtils;
+import com.tzld.piaoquan.recommend.feature.common.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.math.NumberUtils;

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/ODPSToRedis.java

@@ -5,12 +5,12 @@ import com.squareup.okhttp.MediaType;
 import com.squareup.okhttp.OkHttpClient;
 import com.squareup.okhttp.Request;
 import com.squareup.okhttp.RequestBody;
-import com.tzld.piaoquan.recommend.feature.produce.model.DTSConfig;
+import com.tzld.piaoquan.recommend.feature.common.model.DTSConfig;
 import com.tzld.piaoquan.recommend.feature.produce.service.CMDService;
 import com.tzld.piaoquan.recommend.feature.produce.service.DTSConfigService;
 import com.tzld.piaoquan.recommend.feature.produce.service.ODPSService;
 import com.tzld.piaoquan.recommend.feature.produce.service.RedisService;
-import com.tzld.piaoquan.recommend.feature.produce.util.JSONUtils;
+import com.tzld.piaoquan.recommend.feature.common.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.math.NumberUtils;

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/service/DTSConfigService.java

@@ -2,8 +2,8 @@ package com.tzld.piaoquan.recommend.feature.produce.service;
 
 import com.ctrip.framework.apollo.ConfigService;
 import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.feature.produce.model.DTSConfig;
-import com.tzld.piaoquan.recommend.feature.produce.util.JSONUtils;
+import com.tzld.piaoquan.recommend.feature.common.model.DTSConfig;
+import com.tzld.piaoquan.recommend.feature.common.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/service/ODPSService.java

@@ -11,8 +11,8 @@ import com.aliyun.odps.data.SimpleJsonValue;
 import com.aliyun.odps.task.SQLTask;
 import com.aliyun.odps.utils.StringUtils;
 import com.google.common.base.Joiner;
-import com.tzld.piaoquan.recommend.feature.produce.model.DTSConfig;
-import com.tzld.piaoquan.recommend.feature.produce.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.feature.common.model.DTSConfig;
+import com.tzld.piaoquan.recommend.feature.common.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.math.NumberUtils;
 import org.apache.spark.aliyun.odps.OdpsOps;

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/service/RedisService.java

@@ -1,9 +1,9 @@
 package com.tzld.piaoquan.recommend.feature.produce.service;
 
-import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.feature.produce.model.DTSConfig;
-import com.tzld.piaoquan.recommend.feature.produce.util.CompressionUtil;
-import com.tzld.piaoquan.recommend.feature.produce.util.JSONUtils;
+import com.tzld.piaoquan.recommend.feature.common.model.DTSConfig;
+import com.tzld.piaoquan.recommend.feature.common.util.RedisUtil;
+import com.tzld.piaoquan.recommend.feature.common.util.CompressionUtil;
+import com.tzld.piaoquan.recommend.feature.common.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
@@ -46,8 +46,8 @@ public class RedisService implements Serializable {
         while (dataIte.hasNext()) {
             Map<String, String> record = dataIte.next();
 
-            String redisKey = buildRedisKey(record, config);
-            Map<String, String> valueMap = this.buildValueMap(record, config);
+            String redisKey = RedisUtil.buildRedisKey(record, config);
+            Map<String, String> valueMap = RedisUtil.buildValueMap(record, config);
 
             String value = JSONUtils.toJson(valueMap);
             batch.put(redisKey, value);
@@ -84,7 +84,7 @@ public class RedisService implements Serializable {
         jedis.auth(password);
         while (dataIte.hasNext()) {
             Map<String, String> record = dataIte.next();
-            String redisKey = buildRedisKey(record, config);
+            String redisKey = RedisUtil.buildRedisKey(record, config);
             batch.add(redisKey);
             if (batch.size() >= 1000) {
                 delete(jedis, batch);
@@ -124,35 +124,4 @@ public class RedisService implements Serializable {
         pipeline.sync();
     }
 
-    private String buildRedisKey(Map<String, String> fieldValueMap, DTSConfig config) {
-        // Note：写入和读取的key生成规则应保持一致
-        List<String> fields = config.getRedis().getKey();
-        if (CollectionUtils.isEmpty(fields)) {
-            return config.getRedis().getPrefix();
-        }
-        StringBuilder sb = new StringBuilder();
-        if (StringUtils.isNotBlank(config.getRedis().getPrefix())) {
-            sb.append(config.getRedis().getPrefix());
-        }
-        for (String field : fields) {
-            sb.append(":");
-            sb.append(fieldValueMap.get(field));
-        }
-        return sb.toString();
-    }
-
-    private Map<String, String> buildValueMap(Map<String, String> record, DTSConfig config) {
-        Map<String, String> valueMap = new HashMap<>();
-        if (StringUtils.isNotBlank(config.getRedis().getFeatureField())) {
-            valueMap = JSONUtils.fromJson(record.get(config.getRedis().getFeatureField()), new TypeToken<Map<String, String>>() {
-            }, Collections.emptyMap());
-        } else if (CollectionUtils.isNotEmpty(config.getRedis().getValue())) {
-            for (String valueKey : config.getRedis().getValue()) {
-                valueMap.put(valueKey, record.get(valueKey));
-            }
-        } else {
-            valueMap = record;
-        }
-        return valueMap;
-    }
 }

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/util/CompressionUtil.java

@@ -1,115 +0,0 @@
-package com.tzld.piaoquan.recommend.feature.produce.util;
-
-import net.jpountz.lz4.LZ4Compressor;
-import net.jpountz.lz4.LZ4Factory;
-import net.jpountz.lz4.LZ4SafeDecompressor;
-import org.xerial.snappy.Snappy;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-import java.util.List;
-
-
-/**
- * @author dyp
- */
-public class CompressionUtil {
-    public static String lz4Compress(String input) {
-        byte[] data = input.getBytes(StandardCharsets.UTF_8);
-        LZ4Factory factory = LZ4Factory.fastestInstance();
-        LZ4Compressor compressor = factory.fastCompressor();
-        int maxCompressedLength = compressor.maxCompressedLength(data.length);
-        byte[] compressed = new byte[maxCompressedLength];
-        int compressedLength = compressor.compress(data, 0, data.length, compressed, 0, maxCompressedLength);
-        byte[] result = new byte[compressedLength];
-        System.arraycopy(compressed, 0, result, 0, compressedLength);
-        return Base64.getEncoder().encodeToString(result);
-    }
-
-    public static String lz4Decompress(String input) {
-        // 将Base64编码的字符串解码为字节数组
-        byte[] data = Base64.getDecoder().decode(input);
-        LZ4Factory factory = LZ4Factory.fastestInstance();
-        LZ4SafeDecompressor decompressor = factory.safeDecompressor();
-
-        // 创建一个缓冲区来存储解压缩后的数据
-        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-
-        // 使用ByteBuffer来处理压缩数据
-        ByteBuffer buffer = ByteBuffer.wrap(data);
-
-        // 假设每次读取的块大小为4KB（可以根据实际情况调整）
-        byte[] chunk = new byte[4096];
-        byte[] decompressedChunk = new byte[4096 * 2]; // 解压缩后的块可能会更大
-
-        while (buffer.hasRemaining()) {
-            int remaining = Math.min(buffer.remaining(), chunk.length);
-            buffer.get(chunk, 0, remaining);
-
-            // 解压缩当前块
-            int decompressedLength = decompressor.decompress(chunk, 0, remaining, decompressedChunk, 0);
-
-            // 将解压缩后的数据写入输出流
-            outputStream.write(decompressedChunk, 0, decompressedLength);
-        }
-
-        // 返回解压后的数据
-        return new String(outputStream.toByteArray(), StandardCharsets.UTF_8);
-    }
-
-    public static String snappyCompress(String input) throws IOException {
-        byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8);
-        byte[] compressedBytes = Snappy.compress(inputBytes);
-        return Base64.getEncoder().encodeToString(compressedBytes);
-    }
-
-    public static byte[] snappyCompressV2(String input) throws IOException{
-        byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8);
-        return Snappy.compress(inputBytes);
-    }
-
-    // 将Snappy压缩后的String解压缩回String
-    public static String snappyDecompress(String compressedInput) throws IOException {
-        byte[] compressedBytes = Base64.getDecoder().decode(compressedInput);
-        byte[] decompressedBytes = Snappy.uncompress(compressedBytes);
-        return new String(decompressedBytes, StandardCharsets.UTF_8);
-    }
-
-    public static void main(String[] args) throws IOException {
-        String str = "gBR8eyJmZWF0dXJlIjoie1wicm9zXzFoXCI6XCIwLjBcIiwJEwAyRhMAADNGEwAANEYTAAA2RhMABDdkOl8AAHZGcgAAdkZyAAB2RnIAAHZGcgAAdkZyAAB2PnIACHN0cj5yAAETOuQAARM65AABEzrkAAETOuQAARM65AAEdm9CcgABEzpyAAETOnIAARM6cgABEzpyAAETQnIAQlYBAHZGVgEBE0JyAEJWAQEmQnIAQlYBRTo60gABFAAyOsAAAHIBmkIoAAB2QigAIUc+KAABFD5QAAx2aWV3Ps8BBRQ6ZAAFFD7tAAE8OqAABRQ+7wABKEJiAQByQqAAIYlCoABK8AABFD4oABByX2NudD7JAAkVOsoACRU6ywAJFTrMAAkVOs0ACRU6zgAQc2hhcmU+fgAJFTp+AAkVOn4ACRU6fgAJFTp+AAkVOn4AAHYhYD7tAQUVWiYBOpQACdNGLAAIcmF0QtUAAHIJFkJCAAUsOtcADSwANDrFBAksOtkADRY62gAUcmV0dXJuPlkBDRY+hAAJLD6EAAkWADQ6hAAJFj6EAAkWOoQAKXM+NAEJFkY0AQxjbnQ0RgQGDRc+sgANFz6zAA0XPrQADRc+tQANFzq2AC1QPrcADRdCuAAJ+kIuAAkXQi4ADXNCLwANGEIwAAlHAHZFp1KJAQUbPh4BHTY+IgEdGwA0ZlEAPioBHTY+LgEdG0LWAB0cPtoAAHYB0whoYXMNJDrdADoeADrgADoeADrjAAB2NloAOuYAOh4AOukAOh4AOuwASbpitQA+HwA6tgA+HwA6twA+HwA6uAA+HwA6uQA+HwA6ugAAdjYUAT6pATofAD6sAT5dAD4/AD4gADJAAFx9IiwidmlkZW9pZCI6IjEwMDAwMTYwIn0=";
-        System.out.println(CompressionUtil.snappyDecompress(str));
-    }
-
-    private static void testSnappyDecompress(List<String> strs, int num) {
-        try {
-            long startTime = System.currentTimeMillis();
-            for (int i = num; i > 0; i--) {
-                for (String str : strs) {
-                    System.out.println(CompressionUtil.snappyDecompress(str));
-                }
-            }
-            System.out.println(num + " : snappy decompress cost: " + (System.currentTimeMillis() - startTime) + "ms");
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
-    private static void testLz4Decompress(List<String> strs, int num) {
-        try {
-            long startTime = System.currentTimeMillis();
-            for (int i = num; i > 0; i--) {
-                for (String str : strs) {
-                    CompressionUtil.lz4Decompress(str);
-                }
-            }
-            System.out.println(num + " : lz4 decompress cost: " + (System.currentTimeMillis() - startTime) + "ms");
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
-
-}

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/util/DateUtils.java

@@ -1,30 +0,0 @@
-package com.tzld.piaoquan.recommend.feature.produce.util;
-
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Date;
-
-/**
- * @author dyp
- */
-public final class DateUtils {
-    public static String getCurrentDateStr(String format) {
-        SimpleDateFormat dateFormat = new SimpleDateFormat(format);
-        Date currentDate = new Date();
-        return dateFormat.format(currentDate);
-    }
-
-    public static int getCurrentHour() {
-        Calendar calendar = Calendar.getInstance();
-        return calendar.get(Calendar.HOUR_OF_DAY);
-    }
-
-    public static String getBeforeDaysDateStr(String format, int d) {
-        SimpleDateFormat dateFormat = new SimpleDateFormat(format);
-        Calendar calendar = Calendar.getInstance();
-        calendar.setTime(new Date());
-        calendar.add(Calendar.DAY_OF_MONTH, -d);
-        Date previousDate = calendar.getTime();
-        return dateFormat.format(previousDate);
-    }
-}

recommend-feature-service/pom.xml

@@ -24,10 +24,6 @@
             <groupId>org.springframework.cloud</groupId>
             <artifactId>spring-cloud-starter-netflix-eureka-server</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-pool2</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>
@@ -39,30 +35,10 @@
                 </exclusion>
             </exclusions>
         </dependency>
-        <!--        <dependency>-->
-        <!--            <groupId>mysql</groupId>-->
-        <!--            <artifactId>mysql-connector-java</artifactId>-->
-        <!--        </dependency>-->
-        <!--        <dependency>-->
-        <!--            <groupId>org.mybatis.spring.boot</groupId>-->
-        <!--            <artifactId>mybatis-spring-boot-starter</artifactId>-->
-        <!--        </dependency>-->
-        <!--        <dependency>-->
-        <!--            <groupId>org.mybatis.generator</groupId>-->
-        <!--            <artifactId>mybatis-generator-core</artifactId>-->
-        <!--        </dependency>-->
-        <!--        <dependency>-->
-        <!--            <groupId>com.github.pagehelper</groupId>-->
-        <!--            <artifactId>pagehelper-spring-boot-starter</artifactId>-->
-        <!--        </dependency>-->
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-data-redis</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.alibaba</groupId>
-            <artifactId>fastjson</artifactId>
-        </dependency>
         <dependency>
             <groupId>com.ctrip.framework.apollo</groupId>
             <artifactId>apollo-client</artifactId>
@@ -79,10 +55,7 @@
             <groupId>com.aliyun.openservices</groupId>
             <artifactId>aliyun-log-logback-appender</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-lang3</artifactId>
-        </dependency>
+
         <dependency>
             <groupId>net.devh</groupId>
             <artifactId>grpc-server-spring-boot-starter</artifactId>
@@ -98,11 +71,6 @@
             <artifactId>protobuf-java</artifactId>
             <version>3.12.0</version>
         </dependency>
-        <dependency>
-            <groupId>org.projectlombok</groupId>
-            <artifactId>lombok</artifactId>
-            <version>1.18.16</version>
-        </dependency>
 
         <dependency>
             <groupId>com.sun.jersey</groupId>
@@ -119,16 +87,11 @@
             <artifactId>jersey-apache-client4</artifactId>
             <version>1.9.1</version>
         </dependency>
+
         <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-collections4</artifactId>
-            <version>4.1</version>
-        </dependency>
-        <!-- Snappy compression library -->
-        <dependency>
-            <groupId>org.xerial.snappy</groupId>
-            <artifactId>snappy-java</artifactId>
-            <version>1.1.8.4</version>
+            <groupId>com.tzld.piaoquan</groupId>
+            <artifactId>recommend-feature-common</artifactId>
+            <version>${common.version}</version>
         </dependency>
     </dependencies>
     <build>

recommend-feature-service/src/main/java/com/tzld/piaoquan/recommend/feature/service/AbstractFeatureService.java

@@ -4,8 +4,8 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.reflect.TypeToken;
-import com.tzld.piaoquan.recommend.feature.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.feature.util.JSONUtils;
+import com.tzld.piaoquan.recommend.feature.common.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.feature.common.util.JSONUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.springframework.beans.factory.annotation.Autowired;

recommend-feature-service/src/main/java/com/tzld/piaoquan/recommend/feature/service/AdItemFeatureService.java

@@ -1,10 +1,10 @@
 package com.tzld.piaoquan.recommend.feature.service;
 
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.common.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.feature.domain.ad.base.AdItemFeature;
 import com.tzld.piaoquan.recommend.feature.model.common.Result;
 import com.tzld.piaoquan.recommend.feature.model.feature.*;
-import com.tzld.piaoquan.recommend.feature.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang.StringUtils;

recommend-feature-service/src/main/java/com/tzld/piaoquan/recommend/feature/service/DTSConfig.java

@@ -1,43 +0,0 @@
-package com.tzld.piaoquan.recommend.feature.service;
-
-import lombok.Data;
-import lombok.extern.slf4j.Slf4j;
-
-import java.io.Serializable;
-import java.util.List;
-
-/**
- * @author dyp
- */
-@Data
-@Slf4j
-public class DTSConfig implements Serializable {
-    private ODPS odps;
-    private Redis redis;
-
-    @Data
-    public static class ODPS implements Serializable {
-        private String table;
-        private List<String> cols;
-        private List<String> partition;
-    }
-
-    @Data
-    public static class Redis implements Serializable {
-        private String prefix;
-        private List<String> key;
-        private long expire;
-    }
-
-    public boolean selfCheck() {
-        if (odps == null) {
-            log.error("odps not config");
-            return false;
-        }
-        if (redis == null) {
-            log.error("redis not config");
-            return false;
-        }
-        return true;
-    }
-}

recommend-feature-service/src/main/java/com/tzld/piaoquan/recommend/feature/service/FeatureV2Service.java

@@ -3,12 +3,13 @@ package com.tzld.piaoquan.recommend.feature.service;
 import com.ctrip.framework.apollo.spring.annotation.ApolloJsonValue;
 import com.google.common.base.Strings;
 import com.tzld.piaoquan.recommend.feature.common.ThreadPoolFactory;
+import com.tzld.piaoquan.recommend.feature.common.model.DTSConfig;
+import com.tzld.piaoquan.recommend.feature.common.util.CommonCollectionUtils;
+import com.tzld.piaoquan.recommend.feature.common.util.CompressionUtil;
 import com.tzld.piaoquan.recommend.feature.model.common.Result;
 import com.tzld.piaoquan.recommend.feature.model.feature.FeatureKeyProto;
 import com.tzld.piaoquan.recommend.feature.model.feature.MultiGetFeatureRequest;
 import com.tzld.piaoquan.recommend.feature.model.feature.MultiGetFeatureResponse;
-import com.tzld.piaoquan.recommend.feature.util.CommonCollectionUtils;
-import com.tzld.piaoquan.recommend.feature.util.CompressionUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;

recommend-feature-service/src/main/java/com/tzld/piaoquan/recommend/feature/service/VideoFeatureService.java

@@ -1,13 +1,13 @@
 package com.tzld.piaoquan.recommend.feature.service;
 
 import com.google.common.reflect.TypeToken;
+import com.tzld.piaoquan.recommend.feature.common.util.CommonCollectionUtils;
 import com.tzld.piaoquan.recommend.feature.domain.video.base.ItemFeature;
 import com.tzld.piaoquan.recommend.feature.model.common.Result;
 import com.tzld.piaoquan.recommend.feature.model.feature.GetAllVideoFeatureRequest;
 import com.tzld.piaoquan.recommend.feature.model.feature.GetAllVideoFeatureResponse;
 import com.tzld.piaoquan.recommend.feature.model.feature.GetVideoFeatureRequest;
 import com.tzld.piaoquan.recommend.feature.model.feature.GetVideoFeatureResponse;
-import com.tzld.piaoquan.recommend.feature.util.CommonCollectionUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang.StringUtils;

recommend-feature-service/src/main/java/com/tzld/piaoquan/recommend/feature/util/CommonCollectionUtils.java

@@ -1,41 +0,0 @@
-package com.tzld.piaoquan.recommend.feature.util;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.collections4.MapUtils;
-
-import java.util.*;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-/**
- * @author dyp
- */
-public class CommonCollectionUtils {
-    public static <T, R> List<R> toList(Collection<T> list, Function<T, R> map) {
-        if (CollectionUtils.isEmpty(list)) {
-            return Collections.emptyList();
-        }
-        return list.stream().map(map).collect(Collectors.toList());
-    }
-
-    public static <T, K, V> Map<K, V> toMap(List<T> list, Function<T, K> keyFunc, Function<T, V> valueFunc) {
-        if (CollectionUtils.isEmpty(list)) {
-            return Collections.emptyMap();
-        }
-        return list.stream().collect(Collectors.toMap(keyFunc::apply, valueFunc::apply));
-    }
-
-    public static <K, V> Map<K, V> merge(Map<K, V> map1, Map<K, V> map2) {
-        if (MapUtils.isEmpty(map1)) {
-            return map2;
-        }
-        if (MapUtils.isEmpty(map2)) {
-            return map1;
-        }
-
-        Map<K, V> map = new HashMap<>();
-        map.putAll(map1);
-        map.putAll(map2);
-        return map;
-    }
-}

recommend-feature-service/src/main/java/com/tzld/piaoquan/recommend/feature/util/CompressionUtil.java

@@ -1,99 +0,0 @@
-package com.tzld.piaoquan.recommend.feature.util;
-
-import lombok.extern.slf4j.Slf4j;
-import net.jpountz.lz4.LZ4Compressor;
-import net.jpountz.lz4.LZ4Factory;
-import net.jpountz.lz4.LZ4SafeDecompressor;
-import org.apache.commons.lang3.StringUtils;
-import org.xerial.snappy.Snappy;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.Base64;
-import java.util.Objects;
-
-
-/**
- * @author dyp
- */
-@Slf4j
-public class CompressionUtil {
-    public static String lz4Compress(String input) {
-        byte[] data = input.getBytes(StandardCharsets.UTF_8);
-        LZ4Factory factory = LZ4Factory.fastestInstance();
-        LZ4Compressor compressor = factory.fastCompressor();
-        int maxCompressedLength = compressor.maxCompressedLength(data.length);
-        byte[] compressed = new byte[maxCompressedLength];
-        int compressedLength = compressor.compress(data, 0, data.length, compressed, 0, maxCompressedLength);
-        byte[] result = new byte[compressedLength];
-        System.arraycopy(compressed, 0, result, 0, compressedLength);
-        return Base64.getEncoder().encodeToString(result);
-    }
-
-    public static String lz4Decompress(String input) {
-        // 将Base64编码的字符串解码为字节数组
-        byte[] data = Base64.getDecoder().decode(input);
-        LZ4Factory factory = LZ4Factory.fastestInstance();
-        LZ4SafeDecompressor decompressor = factory.safeDecompressor();
-
-        // 创建一个缓冲区来存储解压缩后的数据
-        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-
-        // 使用ByteBuffer来处理压缩数据
-        ByteBuffer buffer = ByteBuffer.wrap(data);
-
-        // 假设每次读取的块大小为4KB（可以根据实际情况调整）
-        byte[] chunk = new byte[4096];
-        byte[] decompressedChunk = new byte[4096 * 2]; // 解压缩后的块可能会更大
-
-        while (buffer.hasRemaining()) {
-            int remaining = Math.min(buffer.remaining(), chunk.length);
-            buffer.get(chunk, 0, remaining);
-
-            // 解压缩当前块
-            int decompressedLength = decompressor.decompress(chunk, 0, remaining, decompressedChunk, 0);
-
-            // 将解压缩后的数据写入输出流
-            outputStream.write(decompressedChunk, 0, decompressedLength);
-        }
-
-        // 返回解压后的数据
-        return new String(outputStream.toByteArray(), StandardCharsets.UTF_8);
-    }
-
-    public static String snappyCompress(String input) throws IOException {
-        byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8);
-        byte[] compressedBytes = Snappy.compress(inputBytes);
-        return Base64.getEncoder().encodeToString(compressedBytes);
-    }
-
-    // 将Snappy压缩后的String解压缩回String
-    public static String snappyDecompress(String compressedInput) {
-        if (StringUtils.isBlank(compressedInput)) {
-            return "";
-        }
-        try {
-            byte[] compressedBytes = Base64.getDecoder().decode(compressedInput);
-            byte[] decompressedBytes = Snappy.uncompress(compressedBytes);
-            return new String(decompressedBytes, StandardCharsets.UTF_8);
-        } catch (Exception e) {
-            log.error("snappyDecompress error compressedInput {}", compressedInput, e);
-            return "";
-        }
-    }
-
-    public static String snappyDecompressV2(byte[] compressedInput) {
-        if (Objects.isNull(compressedInput)) {
-            return "";
-        }
-        try {
-            byte[] decompressedBytes = Snappy.uncompress(compressedInput);
-            return new String(decompressedBytes, StandardCharsets.UTF_8);
-        } catch (Exception e) {
-            log.error("snappyDecompressV2 error compressedInput {}", compressedInput, e);
-            return "";
-        }
-    }
-}

recommend-feature-service/src/main/java/com/tzld/piaoquan/recommend/feature/util/JSONUtils.java

@@ -1,42 +0,0 @@
-package com.tzld.piaoquan.recommend.feature.util;
-
-import com.alibaba.fastjson.JSONObject;
-import com.google.common.reflect.TypeToken;
-import com.google.gson.Gson;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-
-@Slf4j
-public class JSONUtils {
-
-    /**
-     * Gson 单例，线程安全，避免重复创建
-     */
-    private static final Gson GSON = new Gson();
-
-    public static String toJson(Object obj) {
-        if (obj == null) {
-            return "";
-        }
-        try {
-            return GSON.toJson(obj);
-        } catch (Exception e) {
-            log.error("toJson exception", e);
-            return "";
-        }
-    }
-
-    public static <T> T fromJson(String value, TypeToken<T> typeToken, T defaultValue) {
-
-        if (StringUtils.isBlank(value)) {
-            return defaultValue;
-        }
-        try {
-            return JSONObject.parseObject(value, typeToken.getType());
-        } catch (Exception e) {
-            log.error("parseObject error! value=[{}]", value, e);
-        }
-        return defaultValue;
-    }
-
-}

recommend-feature-service/src/test/java/com/tzld/piaoquan/recommend/feature/service/FeatureV2ServiceTest.java

@@ -1,6 +1,6 @@
 package com.tzld.piaoquan.recommend.feature.service;
 
-import com.tzld.piaoquan.recommend.feature.util.CompressionUtil;
+import com.tzld.piaoquan.recommend.feature.common.util.CompressionUtil;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;