all

src/main/scala/com/aliyun/odps/spark/examples/makedata/makedata_09_user2redis_freq.scala

@@ -86,7 +86,7 @@ object makedata_09_user2redis_freq {
     println("------------mid处理完毕,近期保留的用户有：" + midRdd.count() + "------------------")
 
     //5 用户区分
-    val savePathPart = savePathUser + "/" + partition
+    val savePathPart = savePathUser + "/all/" + partition
     val userDataRead = sc.textFile(savePathPart).filter(_.split("\t").length >= 2)
       .map(r => {
         val rList = r.split("\t")

src/main/scala/com/aliyun/odps/spark/examples/临时记录的脚本

@@ -29,4 +29,16 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 --master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 64 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 savePath:/dw/recommend/model/11_str_data_v3/ beginStr:20240222 endStr:20240225 ifRepart:100 \
-> p11.log 2>&1 &
+> p11.log 2>&1 &
+
+
+
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata.makedata_09_user2redis_freq \
+--name makedata_09_user2redis_freq \
+--master yarn --driver-memory 1G --executor-memory 3G --executor-cores 1 --num-executors 32 \
+--conf spark.yarn.executor.memoryoverhead=1024 \
+/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+date:20230227 tablePart:32 expireDay:3  \
+ifWriteRedisUser:True midDays:7 \
+savePathUser:/dw/recommend/model/09_feature/user/