feat:解决冲突

src/main/scala/com/aliyun/odps/spark/examples/临时记录的脚本-推荐

@@ -168,7 +168,8 @@ readDate:20240626 \
 
 /dw/recommend/model/17_for_check/
 
-
+------------------------------------------------------------------------------------------------------------------------
+------------------------------------------------------------------------------------------------------------------------
 ------------------------------------------------------------------------------------------------------------------------
 
 nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
@@ -176,10 +177,10 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 tablePart:64 repartition:32 \
-beginStr:2024070508 endStr:2024070508 \
+beginStr:2024070913 endStr:2024070923 \
 savePath:/dw/recommend/model/41_recsys_sample_data/ \
 table:alg_recsys_sample_all \
-> p41_2024070508.log 2>&1 &
+> p41_2024070913.log 2>&1 &
 
 nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_42_bucket_20240709 \
@@ -198,11 +199,11 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 readPath:/dw/recommend/model/41_recsys_sample_data/ \
 savePath:/dw/recommend/model/43_recsys_train_data/ \
-beginStr:20240705 endStr:20240705 repartition:100 \
+beginStr:20240709 endStr:20240709 repartition:500 \
 filterNames:XXXXXXXXX \
 fileName:20240609_bucket_314.txt \
 whatLabel:is_return whatApps:0,4,21,3,6,17,23 \
-> p43_20240705.log 2>&1 &
+> p43_20240706.log 2>&1 &
 
 ------------- 20240709_recsys_bucket_314.txt ------------ 20240609_bucket_274.txt -------------
 ------------- filterNames:b123_1h_ROS,b123_2h_ROS,b123_3h_ROS,b123_4h_ROS,b123_12h_ROS,b123_1d_ROS,b123_3d_ROS,b123_7d_ROS,b167_1h_ROS,b167_2h_ROS,b167_3h_ROS,b167_4h_ROS,b167_12h_ROS,b167_1d_ROS,b167_3d_ROS,b167_7d_ROS,b8910_1h_ROS,b8910_2h_ROS,b8910_3h_ROS,b8910_4h_ROS,b8910_12h_ROS,b8910_1d_ROS,b8910_3d_ROS,b8910_7d_ROS,b111213_1h_ROS,b111213_2h_ROS,b111213_3h_ROS,b111213_4h_ROS,b111213_12h_ROS,b111213_1d_ROS,b111213_3d_ROS,b111213_7d_ROS,b171819_1h_ROS,b171819_2h_ROS,b171819_3h_ROS,b171819_4h_ROS,b171819_12h_ROS,b171819_1d_ROS,b171819_3d_ROS,b171819_7d_ROS \

zhangbo/01_train.sh

@@ -23,3 +23,5 @@ $HADOOP fs -text ${train_path}/${day}/* | /root/sunmingze/alphaFM/bin/fm_train -
 # nohup sh 01_train.sh 20240620 /dw/recommend/model/33_ad_train_data/ model_bkb16 1,1,16 >p1_model_bkb16.log 2>&1 &
 
 
+# nohup sh 01_train.sh 20240705 /dw/recommend/model/43_recsys_train_data/ model_nba8 1,1,8 >p1_model_nba8.log 2>&1 &
+# nohup sh 01_train.sh 20240705 /dw/recommend/model/43_recsys_train_data/ model_nba12 1,1,12 >p1_model_nba12.log 2>&1 &

zhangbo/02_train_go.sh

@@ -22,7 +22,7 @@ while [[ "$current_date" != "$end_date" ]]; do
     current_date=$(date -d "$current_date + 1 day" +%Y%m%d)
 done
 
-# nohup sh 02_train_go.sh 20240615 20240616 model_aka8 /dw/recommend/model/16_train_data/ 1,1,8 >p2_model_aka8.log 2>&1 &
+# nohup sh 02_train_go.sh 20240709 20240710 model_nba8 /dw/recommend/model/43_recsys_train_data/ 1,1,8 >p2_model_nba8.log 2>&1 &
 
 
 # nohup sh 02_train_go.sh 20240623 20240624 model_bkb8 /dw/recommend/model/33_ad_train_data/ 1,1,8 >p2_model_bkb8.log 2>&1 &

zhangbo/03_predict.sh

@@ -12,8 +12,8 @@ $HADOOP fs -text ${train_path}/${day}/* | /root/sunmingze/alphaFM/bin/fm_predict
 cat predict/${output_file}_$day.txt | /root/sunmingze/AUC/AUC
 
 
-# nohup sh zhangbo/03_predict.sh 20240805 /dw/recommend/model/33_ad_train_data_v4/ ad/model_bkb8_v55_20240804.txt ad/model_bkb8_v55_20240804 8 >logs/predict_model_bkb8_v55_20240804_05.log 2>&1 &
-# nohup sh 03_predict.sh 20240611 /dw/recommend/model/16_train_data/ model_aka4_20240610.txt model_aka4_20240610 4 >p3_model_aka4.log 2>&1 &
+# nohup sh 03_predict.sh 20240707 /dw/recommend/model/43_recsys_train_data/ model_nba8_20240706.txt model_nba8_20240706 8 >p3_model_nba8_20240706.log 2>&1 &
+# nohup sh 03_predict.sh 20240707 /dw/recommend/model/43_recsys_train_data/ model_nba12_20240706.txt model_nba12_20240706 12 >p3_model_nba12_20240706.log 2>&1 &
 # nohup sh 03_predict.sh 20240613 /dw/recommend/model/16_train_data/ model_aka8_20240612.txt model_aka8_20240612 8 >p3_model_aka8_12.log 2>&1 &
 
 

zhangbo/04_upload.sh

@@ -7,7 +7,7 @@ dfs -put /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_aka0_202406
 
 
 
-cat /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_aka8_20240608.txt |
+cat /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_nba8_20240708.txt |
 awk -F " " '{
     if (NR == 1) {
         print $1"\t"$2
@@ -20,9 +20,9 @@ awk -F " " '{
         }
         print line
     }
-}' > /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_aka8_20240608_change.txt
+}' > /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_nba8_20240708_change.txt
 
-dfs -put /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_aka8_20240608_change.txt oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/model_aka8.txt
+dfs -put /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_nba8_20240708_change.txt oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/model_nba8.txt