循环训练

src/main/scala/com/aliyun/odps/spark/examples/临时记录的脚本

@@ -69,19 +69,19 @@ savePath:/dw/recommend/model/04_str_data/ beginStr:20240311 endStr:20240312 feat
 
 nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata.makedata_13_originData_20240529 \
---master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 32 \
+--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 tablePart:64 repartition:32 \
-beginStr:2024060712 endStr:2024060716 \
+beginStr:2024060816 endStr:2024060823 \
 table:alg_recsys_sample_all \
-> p13_data060712.log 2>&1 &
+> p13_2024060816.log 2>&1 &
 
 
 nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata.makedata_14_valueData_20240608 \
 --master yarn --driver-memory 1G --executor-memory 3G --executor-cores 1 --num-executors 32 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-beginStr:20240607 endStr:20240607 repartition:200 \
+beginStr:20240607 endStr:20240607 repartition:1000 \
 > p14_data.log 2>&1 &
 
 
@@ -97,9 +97,9 @@ bucketNum:200 sampleRate:0.1 \
 
 nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata.makedata_16_bucketData_20240609 \
---master yarn --driver-memory 1G --executor-memory 4G --executor-cores 1 --num-executors 16 \
+--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-beginStr:20240607 endStr:20240607 repartition:400 \
+beginStr:20240607 endStr:20240607 repartition:1000 \
 > p16_data.log 2>&1 &
 
 

zhangbo/01_train.sh

@@ -13,3 +13,4 @@ $HADOOP fs -text ${train_path}/${day}/* | /root/sunmingze/alphaFM/bin/fm_train -
 
 # nohup sh 01_train.sh 20240606 /dw/recommend/model/16_train_data/ model_aka8 1,1,8 >p1_model_aka8.log 2>&1 &
 # nohup sh 01_train.sh 20240606 /dw/recommend/model/16_train_data/ model_aka0 1,1,0 >p1_model_aka0.log 2>&1 &
+# nohup sh 01_train.sh 20240606 /dw/recommend/model/16_train_data/ model_aka4 1,1,4 >p1_model_aka4.log 2>&1 &

zhangbo/02_train_go.sh

@@ -18,12 +18,8 @@ while [[ "$current_date" != "$end_date" ]]; do
     yesterday=$(date -d "$current_date - 1 day" +%Y%m%d)
     echo model-day-$yesterday
     echo data-day-$current_date
-    $HADOOP fs -text ${SAMPLE_PATH}/dt=$current_date/* | ${FM_TRAIN} -m $MODEL_PATH/${model_name}_$current_date.txt -dim ${bias} -core 8 -im $MODEL_PATH/${model_name}_$yesterday.txt
+    $HADOOP fs -text ${SAMPLE_PATH}/$current_date/* | ${FM_TRAIN} -m $MODEL_PATH/${model_name}_$current_date.txt -dim ${bias} -core 8 -im $MODEL_PATH/${model_name}_$yesterday.txt
     current_date=$(date -d "$current_date + 1 day" +%Y%m%d)
 done
 
-# nohup sh 02_train_go.sh 20240226 20240228 model_tom /dw/recommend/model/11_str_data_v3/ 0,1,0 >p2_model_tom.log 2>&1 &
-# nohup sh 02_train_go.sh 20240226 20240228 model_jerry /dw/recommend/model/12_ros_data_v3/ 0,1,0 >p2_model_jerry.log 2>&1 &
-
-
-# nohup sh 02_train_go.sh 20240311 20240314 model_str_mid /dw/recommend/model/04_str_data/ 1,1,0 >p2_model_str_mid.log 2>&1 &
+# nohup sh 02_train_go.sh 20240607 20240608 model_aka0 /dw/recommend/model/16_train_data/ 1,1,0 >p2_model_aka0.log 2>&1 &

zhangbo/03_predict.sh

@@ -12,3 +12,7 @@ $HADOOP fs -text ${train_path}/${day}/* | /root/sunmingze/alphaFM/bin/fm_predict
 cat predict/${output_file}_$day.txt | /root/sunmingze/AUC/AUC
 
 # nohup sh 03_predict.sh 20240607 /dw/recommend/model/16_train_data/ model_fuck_20240606.txt model_fuck_20240606 8 >p3_model_fuck.log 2>&1 &
+
+# nohup sh 03_predict.sh 20240607 /dw/recommend/model/16_train_data/ model_aka0_20240606.txt model_aka0_20240606 0 >p3_model_aka0.log 2>&1 &
+# nohup sh 03_predict.sh 20240607 /dw/recommend/model/16_train_data/ model_aka4_20240606.txt model_aka4_20240606 4 >p3_model_aka4.log 2>&1 &
+# nohup sh 03_predict.sh 20240607 /dw/recommend/model/16_train_data/ model_aka8_20240606.txt model_aka8_20240606 8 >p3_model_aka8.log 2>&1 &