推荐模型-并行

qiaojialiang/handle_rov.sh

@@ -68,14 +68,49 @@ done
 
 # 1 生产原始数据
 echo "$(date +%Y-%m-%d_%H-%M-%S)----------step1------------开始根据${table}生产原始数据"
+#/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+#--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_41_originData_20240709 \
+#--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
+#../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+#tablePart:64 repartition:32 \
+#beginStr:${begin_early_2_Str}${beginHhStr} endStr:${end_early_2_Str}${endHhStr} \
+#savePath:${originDataPath} \
+#table:${table}
+#if [ $? -ne 0 ]; then
+#   echo "Spark原始样本生产任务执行失败"
+#   /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step1根据${table}生产原始数据\n【是否成功】:error\n【信息】:Spark原始样本生产任务执行失败"
+#   exit 1
+#else
+#   echo "spark原始样本生产执行成功"
+#fi
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_41_originData_20240709 \
+--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
+../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+tablePart:64 repartition:32 \
+beginStr:${begin_early_2_Str}00 endStr:${end_early_2_Str}08 \
+savePath:${originDataPath} \
+table:${table} &
+
 /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_41_originData_20240709 \
 --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
 ../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 tablePart:64 repartition:32 \
-beginStr:${begin_early_2_Str}${beginHhStr} endStr:${end_early_2_Str}${endHhStr} \
+beginStr:${begin_early_2_Str}09 endStr:${end_early_2_Str}18 \
 savePath:${originDataPath} \
-table:${table}
+table:${table} &
+
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_41_originData_20240709 \
+--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
+../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+tablePart:64 repartition:32 \
+beginStr:${begin_early_2_Str}19 endStr:${end_early_2_Str}23 \
+savePath:${originDataPath} \
+table:${table} &
+
+wait
 if [ $? -ne 0 ]; then
    echo "Spark原始样本生产任务执行失败"
    /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step1根据${table}生产原始数据\n【是否成功】:error\n【信息】:Spark原始样本生产任务执行失败"
@@ -84,7 +119,7 @@ else
    echo "spark原始样本生产执行成功"
 fi
 
-# 3 特征分桶
+# 2 特征分桶
 echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据"
 /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
@@ -104,61 +139,8 @@ else
    echo "spark特征分桶处理执行成功"
 fi
 
-# 1 生产原始数据
-#echo "$(date +%Y-%m-%d_%H-%M-%S)----------step1------------开始根据${table}生产原始数据"
-#/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
-#--class com.aliyun.odps.spark.examples.makedata_qiao.makedata_13_originData_20240705 \
-#--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
-#../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-#tablePart:64 repartition:32 \
-#beginStr:${begin_early_2_Str}${beginHhStr} endStr:${end_early_2_Str}${endHhStr} \
-#savePath:${originDataPath} \
-#table:${table}
-#if [ $? -ne 0 ]; then
-#   echo "Spark原始样本生产任务执行失败"
-#   /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step1根据${table}生产原始数据\n【是否成功】:error\n【信息】:Spark原始样本生产任务执行失败"
-#   exit 1
-#else
-#   echo "spark原始样本生产执行成功"
-#fi
-
-
-# 2 特征值拼接
-#echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------开始特征值拼接"
-#/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
-#--class com.aliyun.odps.spark.examples.makedata_qiao.makedata_14_valueData_20240705 \
-#--master yarn --driver-memory 1G --executor-memory 3G --executor-cores 1 --num-executors 32 \
-#../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-#readPath:${originDataPath} \
-#savePath:${valueDataPath} \
-#beginStr:${begin_early_2_Str} endStr:${end_early_2_Str} repartition:1000
-#if [ $? -ne 0 ]; then
-#   echo "Spark特征值拼接处理任务执行失败"
-#   /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step2特征值拼接\n【是否成功】:error\n【信息】:Spark特征值拼接处理任务执行失败"
-#   exit 1
-#else
-#   echo "spark特征值拼接处理执行成功"
-#fi
-
-# 3 特征分桶
-#echo "$(date +%Y-%m-%d_%H-%M-%S)----------step3------------根据特征分桶生产重打分特征数据"
-#/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
-#--class com.aliyun.odps.spark.examples.makedata_qiao.makedata_16_bucketData_20240705 \
-#--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
-#../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-#readPath:${valueDataPath} \
-#savePath:${bucketDataPath} \
-#beginStr:${begin_early_2_Str} endStr:${end_early_2_Str} repartition:1000
-#if [ $? -ne 0 ]; then
-#   echo "Spark特征分桶处理任务执行失败"
-#   /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step3训练数据产出\n【是否成功】:error\n【信息】:Spark特征分桶处理任务执行失败"
-#   exit 1
-#else
-#   echo "spark特征分桶处理执行成功"
-#fi
-
 
-# 4 对比AUC 前置对比3日模型数据 与 线上模型数据效果对比，如果3日模型优于线上，更新线上模型
+# 3 对比AUC 前置对比3日模型数据 与 线上模型数据效果对比，如果3日模型优于线上，更新线上模型
 echo "$(date +%Y-%m-%d_%H-%M-%S)----------step3------------开始对比，新：${MODEL_PATH}/${model_name}_${today_early_3}.txt，与线上online模型数据auc效果"
 $HADOOP fs -text ${bucketDataPath}/${begin_early_2_Str}/* | ${FM_HOME}/fm_predict -m ${LAST_MODEL_HOME}/model_online.txt -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_${today}_online.txt
 if [ $? -ne 0 ]; then
@@ -235,7 +217,7 @@ else
   fi
 fi
 
-# 5 模型训练
+# 4 模型训练
 echo "$(date +%Y-%m-%d_%H-%M-%S)----------step4------------开始模型训练，增量训练：${MODEL_PATH}/${model_name}_${today_early_3}.txt"
 #$HADOOP fs -text ${bucketDataPath}/${begin_early_2_Str}/* | ${FM_HOME}/fm_train -m ${MODEL_PATH}/${model_name}_${begin_early_2_Str}.txt -dim 1,1,8 -im ${LAST_MODEL_HOME}/model_online.txt -core 8
 $HADOOP fs -text ${bucketDataPath}/${begin_early_2_Str}/* | ${FM_HOME}/fm_train -m ${MODEL_PATH}/${model_name}_${begin_early_2_Str}.txt -dim 1,1,8 -im ${MODEL_PATH}/${model_name}_${today_early_3}.txt -core 8