推荐模型自动化更新-脚本

qiaojialiang/handle_rov.sh

@@ -36,7 +36,7 @@ PREDICT_PATH=/root/joe/recommend-emr-dataprocess/predict
 # 历史线上正在使用的模型数据路径
 LAST_MODEL_HOME=/root/joe/model_online
 # 模型数据文件前缀
-model_name=aka8
+model_name=model_nba8
 # fm模型
 FM_HOME=/root/sunmingze/alphaFM/bin
 # hadoop
@@ -68,9 +68,9 @@ done
 # 1 生产原始数据
 echo "$(date +%Y-%m-%d_%H-%M-%S)----------step1------------开始根据${table}生产原始数据"
 /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
---class com.aliyun.odps.spark.examples.makedata_qiao.makedata_13_originData_20240705 \
+--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_41_originData_20240709 \
 --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
-../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 tablePart:64 repartition:32 \
 beginStr:${begin_early_2_Str}${beginHhStr} endStr:${end_early_2_Str}${endHhStr} \
 savePath:${originDataPath} \
@@ -83,33 +83,18 @@ else
    echo "spark原始样本生产执行成功"
 fi
 
-
-# 2 特征值拼接
-echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------开始特征值拼接"
-/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
---class com.aliyun.odps.spark.examples.makedata_qiao.makedata_14_valueData_20240705 \
---master yarn --driver-memory 1G --executor-memory 3G --executor-cores 1 --num-executors 32 \
-../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-readPath:${originDataPath} \
-savePath:${valueDataPath} \
-beginStr:${begin_early_2_Str} endStr:${end_early_2_Str} repartition:1000
-if [ $? -ne 0 ]; then
-   echo "Spark特征值拼接处理任务执行失败"
-   /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step2特征值拼接\n【是否成功】:error\n【信息】:Spark特征值拼接处理任务执行失败"
-   exit 1
-else
-   echo "spark特征值拼接处理执行成功"
-fi
-
 # 3 特征分桶
-echo "$(date +%Y-%m-%d_%H-%M-%S)----------step3------------根据特征分桶生产重打分特征数据"
+echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据"
 /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
---class com.aliyun.odps.spark.examples.makedata_qiao.makedata_16_bucketData_20240705 \
+--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
 --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
-../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-readPath:${valueDataPath} \
+./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+readPath:${originDataPath} \
 savePath:${bucketDataPath} \
-beginStr:${begin_early_2_Str} endStr:${end_early_2_Str} repartition:1000
+beginStr:${begin_early_2_Str} endStr:${end_early_2_Str} repartition:500 \
+filterNames:XXXXXXXXX \
+fileName:20240609_bucket_314.txt \
+whatLabel:is_return whatApps:0,4,21,3,6,17,23
 if [ $? -ne 0 ]; then
    echo "Spark特征分桶处理任务执行失败"
    /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step3训练数据产出\n【是否成功】:error\n【信息】:Spark特征分桶处理任务执行失败"
@@ -118,9 +103,62 @@ else
    echo "spark特征分桶处理执行成功"
 fi
 
+# 1 生产原始数据
+#echo "$(date +%Y-%m-%d_%H-%M-%S)----------step1------------开始根据${table}生产原始数据"
+#/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+#--class com.aliyun.odps.spark.examples.makedata_qiao.makedata_13_originData_20240705 \
+#--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
+#../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+#tablePart:64 repartition:32 \
+#beginStr:${begin_early_2_Str}${beginHhStr} endStr:${end_early_2_Str}${endHhStr} \
+#savePath:${originDataPath} \
+#table:${table}
+#if [ $? -ne 0 ]; then
+#   echo "Spark原始样本生产任务执行失败"
+#   /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step1根据${table}生产原始数据\n【是否成功】:error\n【信息】:Spark原始样本生产任务执行失败"
+#   exit 1
+#else
+#   echo "spark原始样本生产执行成功"
+#fi
+
+
+# 2 特征值拼接
+#echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------开始特征值拼接"
+#/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+#--class com.aliyun.odps.spark.examples.makedata_qiao.makedata_14_valueData_20240705 \
+#--master yarn --driver-memory 1G --executor-memory 3G --executor-cores 1 --num-executors 32 \
+#../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+#readPath:${originDataPath} \
+#savePath:${valueDataPath} \
+#beginStr:${begin_early_2_Str} endStr:${end_early_2_Str} repartition:1000
+#if [ $? -ne 0 ]; then
+#   echo "Spark特征值拼接处理任务执行失败"
+#   /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step2特征值拼接\n【是否成功】:error\n【信息】:Spark特征值拼接处理任务执行失败"
+#   exit 1
+#else
+#   echo "spark特征值拼接处理执行成功"
+#fi
+
+# 3 特征分桶
+#echo "$(date +%Y-%m-%d_%H-%M-%S)----------step3------------根据特征分桶生产重打分特征数据"
+#/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+#--class com.aliyun.odps.spark.examples.makedata_qiao.makedata_16_bucketData_20240705 \
+#--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
+#../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+#readPath:${valueDataPath} \
+#savePath:${bucketDataPath} \
+#beginStr:${begin_early_2_Str} endStr:${end_early_2_Str} repartition:1000
+#if [ $? -ne 0 ]; then
+#   echo "Spark特征分桶处理任务执行失败"
+#   /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step3训练数据产出\n【是否成功】:error\n【信息】:Spark特征分桶处理任务执行失败"
+#   exit 1
+#else
+#   echo "spark特征分桶处理执行成功"
+#fi
+
 
 # 4 对比AUC 前置对比3日模型数据 与 线上模型数据效果对比，如果3日模型优于线上，更新线上模型
-echo "$(date +%Y-%m-%d_%H-%M-%S)----------step4------------开始对比，新：${MODEL_PATH}/${model_name}_${today_early_3}.txt，与线上online模型数据auc效果"
+echo "$(date +%Y-%m-%d_%H-%M-%S)----------step3------------开始对比，新：${MODEL_PATH}/${model_name}_${today_early_3}.txt，与线上online模型数据auc效果"
 $HADOOP fs -text ${bucketDataPath}/${begin_early_2_Str}/* | ${FM_HOME}/fm_predict -m ${LAST_MODEL_HOME}/model_online.txt -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_${today}_online.txt
 if [ $? -ne 0 ]; then
   echo "推荐线上模型AUC计算失败"
@@ -164,7 +202,7 @@ else
                /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step4模型格式转换\n【是否成功】:error\n【信息】:新模型文件格式转换失败${MODEL_PATH}/${model_name}_${today_early_3}.txt"
             else
               # 4.1.2 模型文件上传OSS
-              online_model_path=${OSS_PATH}/${model_name}.txt
+              online_model_path=${OSS_PATH}/${model_name}_qjl_test.txt
               $HADOOP fs -test -e ${online_model_path}
               if [ $? -eq 0 ]; then
                   echo "数据存在, 先删除。"
@@ -197,7 +235,7 @@ else
 fi
 
 # 5 模型训练
-echo "$(date +%Y-%m-%d_%H-%M-%S)----------step5------------开始模型训练"
+echo "$(date +%Y-%m-%d_%H-%M-%S)----------step4------------开始模型训练"
 $HADOOP fs -text ${bucketDataPath}/${begin_early_2_Str}/* | ${FM_HOME}/fm_train -m ${MODEL_PATH}/${model_name}_${begin_early_2_Str}.txt -dim 1,1,8 -im ${LAST_MODEL_HOME}/model_online.txt -core 8
 if [ $? -ne 0 ]; then
    echo "模型训练失败"