特征生产

recommend-feature-produce/pom.xml

@@ -12,18 +12,20 @@
     <artifactId>recommend-feature-produce</artifactId>
 
     <properties>
-        <spark.version>2.3.1</spark.version>
+        <!--        <spark.version>3.3.1</spark.version>-->
+        <!--        <scala.version>2.12.15</scala.version>-->
+        <spark.version>2.3.0</spark.version>
+        <scala.version>2.11.8</scala.version>
         <emr.version>2.0.0</emr.version>
         <java.version>1.8</java.version>
         <odps.version>0.48.4-public</odps.version>
         <fastjson.version>1.2.45</fastjson.version>
-        <scala.version>2.12.15</scala.version>
+
         <maven.compiler.source>8</maven.compiler.source>
         <maven.compiler.target>8</maven.compiler.target>
     </properties>
 
     <dependencies>
-
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_2.11</artifactId>
@@ -33,58 +35,29 @@
                     <groupId>io.netty</groupId>
                     <artifactId>netty-all</artifactId>
                 </exclusion>
-                <exclusion>
-                    <artifactId>scala-library</artifactId>
-                    <groupId>org.scala-lang</groupId>
-                </exclusion>
-                <exclusion>
-                    <artifactId>scala-parser-combinators_2.11</artifactId>
-                    <groupId>org.scala-lang.modules</groupId>
-                </exclusion>
-                <exclusion>
-                    <artifactId>scala-reflect</artifactId>
-                    <groupId>org.scala-lang</groupId>
-                </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-all</artifactId>
+            <version>4.1.17.Final</version>
+        </dependency>
+        <dependency>
+            <groupId>com.aliyun.emr</groupId>
+            <artifactId>emr-maxcompute_2.11</artifactId>
+            <version>${emr.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-library</artifactId>
             <version>${scala.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>io.netty</groupId>
-            <artifactId>netty-all</artifactId>
-            <version>4.1.17.Final</version>
-        </dependency>
         <dependency>
             <groupId>com.alibaba</groupId>
             <artifactId>fastjson</artifactId>
             <version>${fastjson.version}</version>
         </dependency>
-        <!--        <dependency>-->
-        <!--            <groupId>org.apache.spark</groupId>-->
-        <!--            <artifactId>spark-mllib_2.11</artifactId>-->
-        <!--            <version>${spark.version}</version>-->
-        <!--        </dependency>-->
-
-        <!--        <dependency>-->
-        <!--            <groupId>org.apache.spark</groupId>-->
-        <!--            <artifactId>spark-sql_2.11</artifactId>-->
-        <!--            <version>${spark.version}</version>-->
-        <!--        </dependency>-->
-
-        <!--        <dependency>-->
-        <!--            <groupId>org.apache.spark</groupId>-->
-        <!--            <artifactId>spark-streaming_2.11</artifactId>-->
-        <!--            <version>${spark.version}</version>-->
-        <!--        </dependency>-->
-        <dependency>
-            <groupId>com.aliyun.emr</groupId>
-            <artifactId>emr-maxcompute_2.11</artifactId>
-            <version>${emr.version}</version>
-        </dependency>
         <dependency>
             <groupId>redis.clients</groupId>
             <artifactId>jedis</artifactId>
@@ -105,7 +78,6 @@
                 </exclusion>
             </exclusions>
         </dependency>
-
         <dependency>
             <groupId>com.aliyun.odps</groupId>
             <artifactId>odps-sdk-commons</artifactId>

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/ODPSToRedis.java

@@ -34,7 +34,7 @@ public class ODPSToRedis {
         }
 
         SparkConf sparkConf = new SparkConf()
-                //.setMaster("local")
+                .setMaster("local")
                 .setAppName("odps sync to redis");
         for (Map.Entry<String, String> e : argMap.entrySet()) {
             sparkConf.set(e.getKey(), e.getValue());

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/Test.java

@@ -12,11 +12,13 @@ public class Test {
         args[2] = "-table";
         args[3] = "alg_vid_feature_all_exp";
         args[4] = "-dt";
-        args[5] = "20240611";
+        args[5] = "20240612";
         args[6] = "-hh";
-        args[7] = "01";
+        args[7] = "15";
         args[8] = "-env";
         args[9] = "test";
+        args[10] = "-partitionNum";
+        args[11] = "50";
         ODPSToRedis.main(args);
     }
 

recommend-feature-produce/src/main/java/com/tzld/piaoquan/recommend/feature/produce/service/ODPSService.java

@@ -93,7 +93,6 @@ public class ODPSService {
         }
     }
 
-
     public long count(DTSConfig config, Map<String, String> argMap) {
         String project = argMap.get("project");
         String table = argMap.get("table");