新样本数据生产

src/main/scala/com/aliyun/odps/spark/examples/临时记录的脚本

@@ -2,7 +2,7 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 --class com.aliyun.odps.spark.examples.makedata.makedata_10_originData_v3 \
 --master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 64 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-tablePart:64 savePath:/dw/recommend/model/10_sample_data_v3/ beginStr:20240222 endStr:20240223 > p10_.log 2>&1 &
+tablePart:64 savePath:/dw/recommend/model/10_sample_data_v3/ beginStr:20240227 endStr:20240227 > p10_.log 2>&1 &
 
 
 
@@ -12,14 +12,14 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 --class com.aliyun.odps.spark.examples.makedata.makedata_12_rosData_v3 \
 --master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 32 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-savePath:/dw/recommend/model/12_ros_data_v3/ beginStr:20240226 endStr:20240226 ifRepart:10 \
+savePath:/dw/recommend/model/12_ros_data_v3/ beginStr:20240222 endStr:20240226 ifRepart:10 \
 > p12_1.log 2>&1 &
 
 nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata.makedata_12_rosData_v3_noweight \
 --master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 32 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-savePath:/dw/recommend/model/12_ros_data_v3_noweight/ beginStr:20240222 endStr:20240225 ifRepart:10 \
+savePath:/dw/recommend/model/12_ros_data_v3_noweight/ beginStr:20240222 endStr:20240226 ifRepart:10 \
 > p12_2.log 2>&1 &
 
 

zhangbo/01_train.sh

@@ -9,12 +9,12 @@ model_name=$3
 #v_l2=$5
 
 HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
-$HADOOP fs -text ${train_path}/dt=$day/* | /root/sunmingze/alphaFM/bin/fm_train -m model/${model_name}_${day}.txt -dim 1,1,0 -core 8
+$HADOOP fs -text ${train_path}/dt=$day/* | /root/sunmingze/alphaFM/bin/fm_train -m model/${model_name}_${day}.txt -dim 0,1,0 -core 8
 # -v_l1 ${v_l1} -v_l2 ${v_l2}
 
 
-# nohup sh 01_train.sh 20240222 /dw/recommend/model/12_ros_data_v3 model_ros_tom >p1_train.log 2>&1 &
-# nohup sh 01_train.sh 20240222 /dw/recommend/model/11_str_data_v3 model_str_tom >p1_train.log 2>&1 &
+# nohup sh 01_train.sh 20240222 /dw/recommend/model/12_ros_data_v3 model_jerry >p1_train.log 2>&1 &
+# nohup sh 01_train.sh 20240222 /dw/recommend/model/11_str_data_v3 model_tom >p1_train.log 2>&1 &
 # nohup sh 01_train.sh 20240222 /dw/recommend/model/12_ros_data_v3_noweight model_str_tom_noweight >p1_train.log 2>&1 &
 
 

zhangbo/02_train_go.sh

@@ -17,10 +17,9 @@ while [[ "$current_date" != "$end_date" ]]; do
     yesterday=$(date -d "$current_date - 1 day" +%Y%m%d)
     echo model-day-$yesterday
     echo data-day-$current_date
-    $HADOOP fs -text ${SAMPLE_PATH}/dt=$current_date/* | ${FM_TRAIN} -m $MODEL_PATH/${model_name}_$current_date.txt -dim 1,1,0 -core 8 -im $MODEL_PATH/${model_name}_$yesterday.txt
+    $HADOOP fs -text ${SAMPLE_PATH}/dt=$current_date/* | ${FM_TRAIN} -m $MODEL_PATH/${model_name}_$current_date.txt -dim 0,1,0 -core 8 -im $MODEL_PATH/${model_name}_$yesterday.txt
     current_date=$(date -d "$current_date + 1 day" +%Y%m%d)
 done
 
-# nohup sh 02_train_go.sh 20240225 20240226 model_ros_tom /dw/recommend/model/12_ros_data_v3/ >p2.log 2>&1 &
-# nohup sh 02_train_go.sh 20240223 20240226 model_str_tom /dw/recommend/model/11_str_data_v3/ >p2.log 2>&1 &
+# nohup sh 02_train_go.sh 20240223 20240226 model_jerry /dw/recommend/model/12_ros_data_v3/ >p2_go.log 2>&1 &
 # nohup sh 02_train_go.sh 20240223 20240226 model_str_tom_noweight /dw/recommend/model/12_ros_data_v3_noweight/ >p2.log 2>&1 &

zhangbo/03_predict.sh

@@ -10,26 +10,8 @@ HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
 $HADOOP fs -text ${train_path}/dt=$day/* | /root/sunmingze/alphaFM/bin/fm_predict -m model/$model_name -dim 0 -core 8 -out predict/${output_file}_$day.txt
 cat predict/${output_file}_$day.txt | /root/sunmingze/AUC/AUC
 
-# nohup sh 03_predict.sh 20240226 /dw/recommend/model/12_ros_data_v3/ model_ros_tom_20240225.txt model_ros_tom >p3_pred.log 2>&1 &
+# nohup sh 03_predict.sh 20240226 /dw/recommend/model/12_ros_data_v3/ model_jerry_20240225.txt model_jerry >p3_pred.log 2>&1 &
 # nohup sh 03_predict.sh 20240226 /dw/recommend/model/12_ros_data_v3_noweight/ model_ros_tom_noweight_20240225.txt model_ros_tom_noweight2 >p3_pred.log 2>&1 &
 # nohup sh 03_predict.sh 20240226 /dw/recommend/model/11_str_data_v3/ model_str_tom_20240222.txt model_str_tom >p3.log 2>&1 &
 
 # nohup sh 03_predict.sh 20240226 /dw/recommend/model/11_str_data_v3/ model_str_tom_20240225.txt model_str_tom >p3.123.log 2>&1 &
-
-# str:
-# nohup sh 03_predict.sh 20240115 /dw/recommend/model/01_str_data/ model_str_big_20240114.txt model_str_big >p1_pred.log 2>&1 &
-# nohup sh 03_predict.sh 20240115 /dw/recommend/model/01_str_data/ model_str_big1_20240114.txt model_str_big1 >p1_pred.log 2>&1 &
-# nohup sh 03_predict.sh 20240115 /dw/recommend/model/02_str_data/ model_str_small_20240114.txt model_str_small >p2_pred.log 2>&1 &
-# nohup sh 03_predict.sh 20240115 /dw/recommend/model/02_str_data/ model_str_small1_20240114.txt model_str_small1 >p2_pred.log 2>&1 &
-# nohup sh 03_predict.sh 20240117 /dw/recommend/model/04_str_data/ model_str_mid_20240116.txt model_str_mid >p3_pred.log 2>&1 &
-# nohup sh 03_predict.sh 20240115 /dw/recommend/model/04_str_data/ model_str_mid2_20240114.txt model_str_mid2 >p4_pred.log 2>&1 &
-# nohup sh 03_predict.sh 20240115 /dw/recommend/model/04_str_data/ model_str_mid4_20240114.txt model_str_mid4 >p4_pred.log 2>&1 &
-# nohup sh 03_predict.sh 20240115 /dw/recommend/model/05_str_data/ model_str_mid3_20240114.txt model_str_mid3 >p5_pred.log 2>&1 &
-
-
-
-# ros:
-# nohup sh 03_predict.sh 20231221 /dw/recommend/model/ros_sample/ model_ros_v1_20231220.txt model_ros_v1 >p_pred.log 2>&1 &
-# nohup sh 03_predict.sh 20231221 /dw/recommend/model/ros_sample_v2/ model_ros_v2_20231220.txt model_ros_v2 >p_pred.log 2>&1 &
-# nohup sh 03_predict.sh 20231221 /dw/recommend/model/ros_sample/ model_ros_v2_20231220.txt model_ros_v2 >p_pred.log 2>&1 &
-