训练脚本更新

src/main/scala/com/aliyun/odps/spark/examples/临时记录的脚本

@@ -72,7 +72,7 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 --master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 32 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 tablePart:32 \
-beginStr:2024060701 endStr:2024060701 \
+beginStr:2024060600 endStr:2024060601 \
 table:alg_recsys_sample_all \
 > p13_data.log 2>&1 &
 
@@ -81,7 +81,7 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 --class com.aliyun.odps.spark.examples.makedata.makedata_14_valueData_20240608 \
 --master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 32 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-partitionPrefix:dt=20240607 date:20240607 \
+partitionPrefix:dt=20240606 date:20240606 \
 > p14_data.log 2>&1 &
 
 
@@ -90,4 +90,17 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 --master yarn --driver-memory 16G --executor-memory 1G --executor-cores 1 --num-executors 32 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 partitionPrefix:20240607 date:20240607_200 bucketNum:200 \
-> p15_data.log 2>&1 &
+> p15_data.log 2>&1 &
+
+
+nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata.makedata_16_bucketData_20240609 \
+--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 32 \
+./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+beginStr:20240606 endStr:20240606 \
+> p16_data.log 2>&1 &
+
+
+/dw/recommend/model/13_sample_data/
+/dw/recommend/model/14_feature_data/
+/dw/recommend/model/16_train_data/

zhangbo/01_train.sh

@@ -8,9 +8,8 @@ model_name=$3
 bias=$4 # 0,1,0 1,1,0
 
 HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
-$HADOOP fs -text ${train_path}/dt=$day/* | /root/sunmingze/alphaFM/bin/fm_train -m model/${model_name}_${day}.txt -dim ${bias} -core 8
+$HADOOP fs -text ${train_path}/${day}/* | /root/sunmingze/alphaFM/bin/fm_train -m model/${model_name}_${day}.txt -dim ${bias} -core 8
 # -v_l1 ${v_l1} -v_l2 ${v_l2}
 
-# nohup sh 01_train.sh 20240222 /dw/recommend/model/11_str_data_v3 model_tom112 1,1,2 >p1_model_tom112.log 2>&1 &
-# nohup sh 01_train.sh 20240222 /dw/recommend/model/12_ros_data_v3_noweight model_jerry_noweight 0,1,0 >p1_model_jerry_noweight.log 2>&1 &
+# nohup sh 01_train.sh 20240606 /dw/recommend/model/16_train_data/ model_fuck 1,1,0 >p1_model_fuck.log 2>&1 &