feat:修改定时任务脚本

ad/01_ad_model_update_everyday.sh

@@ -84,46 +84,46 @@ export JAVA_HOME=/usr/lib/jvm/java-1.8.0
 
 
 # 4 模型训练
-#$HADOOP fs -text ${bucketFeatureSavePath}/${today_early_1}/* | ${FM_HOME}/bin/fm_train -m ${MODEL_PATH}/${model_name}_${today_early_1}.txt -dim 1,1,8  -im ${LAST_MODEL_HOME}/model_online.txt -core 8
-#if [ $? -ne 0 ]; then
-#   echo "模型训练失败"
-#   /root/anaconda3/bin/python ad/ad_monitor_util.py "广告模型训练失败"
-#   exit 1
-#fi
+$HADOOP fs -text ${bucketFeatureSavePath}/${today_early_1}/* | ${FM_HOME}/bin/fm_train -m ${MODEL_PATH}/${model_name}_${today_early_1}.txt -dim 1,1,8  -im ${LAST_MODEL_HOME}/model_online.txt -core 8
+if [ $? -ne 0 ]; then
+   echo "模型训练失败"
+   /root/anaconda3/bin/python ad/ad_monitor_util.py "广告模型训练失败"
+   exit 1
+fi
 
 
-## 5 对比AUC
-## 5.1 校验今天10分区的数据是否生产完成
-#$HADOOP fs -text ${bucketFeatureSavePath}/${today}/* | ${FM_HOME}/bin/fm_predict -m ${LAST_MODEL_HOME}/model_online.txt -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_${today}_online.txt
-#$HADOOP fs -text ${bucketFeatureSavePath}/${today}/* | ${FM_HOME}/bin/fm_predict -m ${MODEL_PATH}/${model_name}_${today_early_1}.txt -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_${today}_new.txt
-#
-## shellcheck disable=SC2006
-#online_auc=`cat ${PREDICT_PATH}/${model_name}_${today}_online.txt | /root/sunmingze/AUC/AUC`
-#if [ $? -ne 0 ]; then
-#   echo "线上模型AUC计算失败"
-#   /root/anaconda3/bin/python ad/ad_monitor_util.py "线上模型AUC计算失败"
-#   exit 1
-#fi
-#
-#new_auc=`cat ${PREDICT_PATH}/${model_name}_${today}_new.txt | /root/sunmingze/AUC/AUC`
-#if [ $? -ne 0 ]; then
-#   echo "新模型AUC计算失败"
-#   /root/anaconda3/bin/python ad/ad_monitor_util.py "新模型AUC计算失败"
-#   exit 1
-#fi
-#
-#if [ "$online_auc" \< "$new_auc" ]; then
-#    echo "新模型优于线上模型: 线上模型AUC: ${online_auc}, 新模型AUC: ${new_auc}"
-#    /root/anaconda3/bin/python ad/ad_monitor_util.py "新模型优于线上模型: 线上模型AUC: ${online_auc}, 新模型AUC: ${new_auc}"
-#else
-#    echo "新模型不如线上模型: 线上模型AUC: ${online_auc}, 新模型AUC: ${new_auc}"
-#    /root/anaconda3/bin/python ad/ad_monitor_util.py "新模型不如线上模型: 线上模型AUC: ${online_auc}, 新模型AUC: ${new_auc}"
-#    exit 1
-#fi
+# 5 对比AUC
+# 5.1 校验今天10分区的数据是否生产完成
+$HADOOP fs -text ${bucketFeatureSavePath}/${today}/* | ${FM_HOME}/bin/fm_predict -m ${LAST_MODEL_HOME}/model_online.txt -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_${today}_online.txt
+$HADOOP fs -text ${bucketFeatureSavePath}/${today}/* | ${FM_HOME}/bin/fm_predict -m ${MODEL_PATH}/${model_name}_${today_early_1}.txt -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_${today}_new.txt
+
+# shellcheck disable=SC2006
+online_auc=`cat ${PREDICT_PATH}/${model_name}_${today}_online.txt | /root/sunmingze/AUC/AUC`
+if [ $? -ne 0 ]; then
+   echo "线上模型AUC计算失败"
+   /root/anaconda3/bin/python ad/ad_monitor_util.py "线上模型AUC计算失败"
+   exit 1
+fi
+
+new_auc=`cat ${PREDICT_PATH}/${model_name}_${today}_new.txt | /root/sunmingze/AUC/AUC`
+if [ $? -ne 0 ]; then
+   echo "新模型AUC计算失败"
+   /root/anaconda3/bin/python ad/ad_monitor_util.py "新模型AUC计算失败"
+   exit 1
+fi
+
+if [ "$online_auc" \< "$new_auc" ]; then
+    echo "新模型优于线上模型: 线上模型AUC: ${online_auc}, 新模型AUC: ${new_auc}"
+    /root/anaconda3/bin/python ad/ad_monitor_util.py "新模型优于线上模型: 线上模型AUC: ${online_auc}, 新模型AUC: ${new_auc}"
+else
+    echo "新模型不如线上模型: 线上模型AUC: ${online_auc}, 新模型AUC: ${new_auc}"
+    /root/anaconda3/bin/python ad/ad_monitor_util.py "新模型不如线上模型: 线上模型AUC: ${online_auc}, 新模型AUC: ${new_auc}"
+    exit 1
+fi
 
 
 # 6 模型格式转换
-cat cat ${MODEL_PATH}/${model_name}_${today_early_1}.txt |
+cat ${MODEL_PATH}/${model_name}_${today_early_1}.txt |
 awk -F " " '{
     if (NR == 1) {
         print $1"\t"$2
@@ -138,10 +138,6 @@ awk -F " " '{
     }
 }' > ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt
 
-#cat ${MODEL_PATH}/${model_name}_${today_early_1}.txt \
-#| sed '1d' | awk -F " " '{if($2!="0") print $1"\t"$2}' \
-#> ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt
-
 
 # 7 模型文件上传OSS
 #online_model_path=${OSS_PATH}/${model_name}.txt
@@ -163,12 +159,12 @@ awk -F " " '{
 #fi
 
 # 7.3 本地保存最新的线上使用的模型，用于下一次的AUC验证
-#cp -f ${LAST_MODEL_HOME}/model_online.txt ${LAST_MODEL_HOME}/model_online_$(date +\%Y\%m\%d).txt
-#cp -f ${MODEL_PATH}/${model_name}_${today_early_1}.txt ${LAST_MODEL_HOME}/model_online.txt
-#if [ $? -ne 0 ]; then
-#   echo "模型备份失败"
-#   /root/anaconda3/bin/python ad/ad_monitor_util.py "模型备份失败 - 最新模型地址: ${MODEL_PATH}/${model_name}_${today_early_1}.txt"
-#   exit 1
-#fi
+cp -f ${LAST_MODEL_HOME}/model_online.txt ${LAST_MODEL_HOME}/model_online_$(date +\%Y\%m\%d).txt
+cp -f ${MODEL_PATH}/${model_name}_${today_early_1}.txt ${LAST_MODEL_HOME}/model_online.txt
+if [ $? -ne 0 ]; then
+   echo "模型备份失败"
+   /root/anaconda3/bin/python ad/ad_monitor_util.py "模型备份失败 - 最新模型地址: ${MODEL_PATH}/${model_name}_${today_early_1}.txt"
+   exit 1
+fi
 
 # 32 16 * * * cd /root/zhangbo/recommend-emr-dataprocess && /bin/sh ./ad/01_ad_model_update_everyday.sh > logs/01_update_eventday$(date +\%Y-\%m-\%d_\%H).log 2>&1