feat:添加告警脚本

ad/01_ad_model_update_everyday.sh

@@ -1,125 +1,125 @@
 #!/bin/sh
 set -ex
-
-# 0 全局变量/参数
-originDataSavePath=/dw/recommend/model/31_ad_sample_data_auto/
-bucketFeatureSavePath=/dw/recommend/model/33_ad_train_data_nosparse_auto/
-model_name=model_lr0
-today="$(date +%Y%m%d)"
-today_early_1="$(date -d '1 days ago' +%Y%m%d)"
-beginTime=08
-endTime=23
-beginStr=${today_early_1}${beginTime}
-endStr=${today_early_1}${endTime}
-
-MODEL_PATH=/root/zhaohp/recommend-emr-dataprocess/model
-HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
-FM_HOME=/root/sunmingze/alphaFM
-OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/ad_model/
-
-
-# 1 判断依赖的数据表是否生产完成
-source /root/anaconda3/bin/activate py37
-max_hour=15
-max_minute=00
-while true; do
-  python_return_code=$(python ad/ad_utils.py --excute_program check_ad_origin_hive --partition ${today_early_1} --hh ${endTime})
-  if [ $python_return_code -eq 0 ]; then
-    echo "Python程序返回0，退出循环。"
-    break
-  fi
-  echo "Python程序返回非0值，等待五分钟后再次调用。"
-  sleep 300
-  current_hour=$(date +%H)
-  current_minute=$(date +%M)
-  if (( current_hour > max_hour || (current_hour == max_hour && current_minute >= max_minute) )); then
-    echo "最长等待时间已到，失败:${current_hour}-${current_minute}"
-    msg="广告特征数据校验失败，大数据分区没有数据: ${today_early_1}${endTime}"
-    /root/anaconda3/bin/python ad/utils_monitor.py ${msg}
-    exit 1
-  fi
-done
-
-
-# 2 原始特征生成
-/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
---class com.aliyun.odps.spark.zhp.makedata_ad.makedata_ad_31_originData_20240620 \
---master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
-./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-tablePart:64 repartition:32 \
-beginStr:${beginStr} endStr:${endStr} \
-savePath:${originDataSavePath} \
-table:alg_recsys_ad_sample_all_new
-if [ $? -ne 0 ]; then
-   echo "Spark原始样本生产任务执行失败"
-   msg="广告特征数据生成失败，Spark原始样本生产任务执行失败"
-   /root/anaconda3/bin/python ad/utils_monitor.py ${msg}
-   exit 1
-else
-    echo "spark原始样本生产执行成功"
-fi
-
-
-# 3 特征分桶
-/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
---class com.aliyun.odps.spark.zhp.makedata_ad.makedata_ad_33_bucketData_20240622 \
---master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
-./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-beginStr:${today_early_1} endStr:${today_early_1} repartition:400 \
-filterNames:XXXXX \
-bucketFileName:20240620_ad_bucket_249_fix.txt \
-readPath:${originDataSavePath} \
-savePath:${bucketFeatureSavePath}
-if [ $? -ne 0 ]; then
-   echo "Spark特征分桶处理任务执行失败"
-   msg="广告特征分桶失败，Spark特征分桶处理任务执行失败"
-   /root/anaconda3/bin/python ad/utils_monitor.py ${msg}
-   exit 1
-else
-   echo "spark特征分桶处理执行成功"
-fi
-
-
-# 4 模型训练
-$HADOOP fs -text ${bucketFeatureSavePath}/${today_early_1}/* | /root/sunmingze/alphaFM/bin/fm_train -m model/${model_name}_${today_early_1}.txt -dim 1,1,0 -core 8
-if [ $? -ne 0 ]; then
-   echo "模型训练失败"
-   /root/anaconda3/bin/python ad/utils_monitor.py "广告模型训练失败"
-   exit 1
-fi
-
-
-# 5 对比AUC
-# 5.1 生成今天08-10的原始特征数据
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-# 6 模型格式转换
-cat ${MODEL_PATH}/${model_name}_${today_early_1}.txt \
-| sed '1d' | awk -F " " '{if($2!="0") print $1"\t"$2}' \
-> ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt
+#
+## 0 全局变量/参数
+#originDataSavePath=/dw/recommend/model/31_ad_sample_data_auto/
+#bucketFeatureSavePath=/dw/recommend/model/33_ad_train_data_nosparse_auto/
+#model_name=model_lr0
+#today="$(date +%Y%m%d)"
+#today_early_1="$(date -d '1 days ago' +%Y%m%d)"
+#beginTime=08
+#endTime=23
+#beginStr=${today_early_1}${beginTime}
+#endStr=${today_early_1}${endTime}
+#
+#MODEL_PATH=/root/zhaohp/recommend-emr-dataprocess/model
+#HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
+#FM_HOME=/root/sunmingze/alphaFM
+#OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/ad_model/
+#
+#
+## 1 判断依赖的数据表是否生产完成
+#source /root/anaconda3/bin/activate py37
+#max_hour=15
+#max_minute=00
+#while true; do
+#  python_return_code=$(python ad/ad_utils.py --excute_program check_ad_origin_hive --partition ${today_early_1} --hh ${endTime})
+#  if [ $python_return_code -eq 0 ]; then
+#    echo "Python程序返回0，退出循环。"
+#    break
+#  fi
+#  echo "Python程序返回非0值，等待五分钟后再次调用。"
+#  sleep 300
+#  current_hour=$(date +%H)
+#  current_minute=$(date +%M)
+#  if (( current_hour > max_hour || (current_hour == max_hour && current_minute >= max_minute) )); then
+#    echo "最长等待时间已到，失败:${current_hour}-${current_minute}"
+#    msg="广告特征数据校验失败，大数据分区没有数据: ${today_early_1}${endTime}"
+#    /root/anaconda3/bin/python ad/utils_monitor.py ${msg}
+#    exit 1
+#  fi
+#done
+#
+#
+## 2 原始特征生成
+#/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+#--class com.aliyun.odps.spark.zhp.makedata_ad.makedata_ad_31_originData_20240620 \
+#--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
+#./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+#tablePart:64 repartition:32 \
+#beginStr:${beginStr} endStr:${endStr} \
+#savePath:${originDataSavePath} \
+#table:alg_recsys_ad_sample_all_new
+#if [ $? -ne 0 ]; then
+#   echo "Spark原始样本生产任务执行失败"
+#   msg="广告特征数据生成失败，Spark原始样本生产任务执行失败"
+#   /root/anaconda3/bin/python ad/utils_monitor.py ${msg}
+#   exit 1
+#else
+#    echo "spark原始样本生产执行成功"
+#fi
+#
+#
+## 3 特征分桶
+#/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+#--class com.aliyun.odps.spark.zhp.makedata_ad.makedata_ad_33_bucketData_20240622 \
+#--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
+#./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+#beginStr:${today_early_1} endStr:${today_early_1} repartition:400 \
+#filterNames:XXXXX \
+#bucketFileName:20240620_ad_bucket_249_fix.txt \
+#readPath:${originDataSavePath} \
+#savePath:${bucketFeatureSavePath}
+#if [ $? -ne 0 ]; then
+#   echo "Spark特征分桶处理任务执行失败"
+#   msg="广告特征分桶失败，Spark特征分桶处理任务执行失败"
+#   /root/anaconda3/bin/python ad/utils_monitor.py ${msg}
+#   exit 1
+#else
+#   echo "spark特征分桶处理执行成功"
+#fi
+#
+#
+## 4 模型训练
+#$HADOOP fs -text ${bucketFeatureSavePath}/${today_early_1}/* | /root/sunmingze/alphaFM/bin/fm_train -m model/${model_name}_${today_early_1}.txt -dim 1,1,0 -core 8
+#if [ $? -ne 0 ]; then
+#   echo "模型训练失败"
+#   /root/anaconda3/bin/python ad/utils_monitor.py "广告模型训练失败"
+#   exit 1
+#fi
+#
+#
+## 5 对比AUC
+## 5.1 生成今天08-10的原始特征数据
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+#
+## 6 模型格式转换
+#cat ${MODEL_PATH}/${model_name}_${today_early_1}.txt \
+#| sed '1d' | awk -F " " '{if($2!="0") print $1"\t"$2}' \
+#> ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt
 
 
 # 7 模型文件上传OSS
 online_model_path=${OSS_PATH}/${model_name}.txt
-#$HADOOP fs -test -e ${online_model_path}
-#if [ $? -eq 0 ]; then
-#    echo "数据存在, 先删除。"
-#    $HADOOP fs -rm -r ${online_model_path}
-#else
-#    echo "数据不存在"
-#fi
+$HADOOP fs -test -e ${online_model_path}
+if [ $? -eq 0 ]; then
+    echo "数据存在, 先删除。"
+    $HADOOP fs -rm -r ${online_model_path}
+else
+    echo "数据不存在"
+fi
 
 $HADOOP fs -put ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt ${online_model_path}
 if [ $? -eq 0 ]; then