feat:添加评估结果分析脚本

ad/01_ad_model_update.sh

@@ -13,10 +13,10 @@ export JAVA_HOME=/usr/lib/jvm/java-1.8.0
 # 全局常量
 HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
 
-TRAIN_PATH=/dw/recommend/model/31_ad_sample_data_v4/
-BUCKET_FEATURE_PATH=/dw/recommend/model/33_ad_train_data_v4/
-MODEL_PATH=/dw/recommend/model/35_ad_model_test/
-PREDICT_RESULT_SAVE_PATH=/dw/recommend/model/34_ad_predict_data_test/
+TRAIN_PATH=/dw/recommend/model/31_ad_sample_data_v4
+BUCKET_FEATURE_PATH=/dw/recommend/model/33_ad_train_data_v4
+MODEL_PATH=/dw/recommend/model/35_ad_model_test
+PREDICT_RESULT_SAVE_PATH=/dw/recommend/model/34_ad_predict_data_test
 TABLE=alg_recsys_ad_sample_all
 
 # 特征文件名
@@ -27,6 +27,9 @@ MODEL_OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/model/
 # 线上模型名，测试时修改为其他模型名，避免影响线上
 model_name=model_xgb_351_1000_v2_test
 
+# 本地保存HDFS模型路径文件
+model_path_file=/root/zhaohp/XGB/online_model_path.txt
+
 today_early_1="$(date -d '1 days ago' +%Y%m%d)"
 # 训练用的数据路径
 train_data_path=""
@@ -41,7 +44,7 @@ model_local_path=/root/zhaohp/XGB
 # 任务开始时间
 start_time=$(date +%s)
 # 线上模型在HDFS中的路径
-online_model_path=`cat /root/zhaohp/XGB/online_model_path.txt`
+online_model_path=`cat ${model_path_file}`
 
 # 校验命令的退出码
 check_run_status() {
@@ -165,31 +168,33 @@ xgb_train() {
 model_predict() {
 
   # 线上模型评估最新的数据
-  # local step_start_time=$(date +%s)
-  # /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
-  # --class com.tzld.piaoquan.recommend.model.pred_01_xgb_ad_hdfsfile_20240813 \
-  # --master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 30 \
-  # --conf spark.yarn.executor.memoryoverhead=1024 \
-  # --conf spark.shuffle.service.enabled=true \
-  # --conf spark.shuffle.service.port=7337 \
-  # --conf spark.shuffle.consolidateFiles=true \
-  # --conf spark.shuffle.manager=sort \
-  # --conf spark.storage.memoryFraction=0.4 \
-  # --conf spark.shuffle.memoryFraction=0.5 \
-  # --conf spark.default.parallelism=200 \
-  # /root/zhangbo/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar  \
-  # featureFile:20240703_ad_feature_name.txt \
-  # testPath:${predict_date_path} \
-  # savePath:${online_model_predict_result_path} \
-  # modelPath:${online_model_path}
-
-  # local return_code=$?
-  # check_run_status $return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据"
-
-  # local mean_abs_diff=$(python ${sh_path}/model_predict_analyse.py -p ${online_model_predict_result_path} ${new_model_predict_result_path})
-  local p1="/dw/recommend/model/34_ad_predict_data/20241007_351_0927_1003_1000/"
-  local p2="/dw/recommend/model/34_ad_predict_data/20241007_351_0930_1006_1000/"
-  local mean_abs_diff=$(python ${sh_path}/model_predict_analyse.py -p ${p1} ${p2})
+  local step_start_time=$(date +%s)
+  /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
+  --class com.tzld.piaoquan.recommend.model.pred_01_xgb_ad_hdfsfile_20240813 \
+  --master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 30 \
+  --conf spark.yarn.executor.memoryoverhead=1024 \
+  --conf spark.shuffle.service.enabled=true \
+  --conf spark.shuffle.service.port=7337 \
+  --conf spark.shuffle.consolidateFiles=true \
+  --conf spark.shuffle.manager=sort \
+  --conf spark.storage.memoryFraction=0.4 \
+  --conf spark.shuffle.memoryFraction=0.5 \
+  --conf spark.default.parallelism=200 \
+  /root/zhangbo/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar  \
+  featureFile:20240703_ad_feature_name.txt \
+  testPath:${predict_date_path} \
+  savePath:${online_model_predict_result_path} \
+  modelPath:${online_model_path}
+
+  local return_code=$?
+  check_run_status $return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据"
+
+  local mean_abs_diff=$(python ${sh_path}/model_predict_analyse.py -p ${online_model_predict_result_path} ${new_model_predict_result_path})
+  if (( $(echo "${mean_abs_diff} > 0.000400" | bl -l ) ));then
+    check_run_status 1 $step_start_time "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.000400，请检查"
+    echo "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.000400，请检查"
+    exit 1
+  fi 
 }
 
 model_upload_oss() {
@@ -211,15 +216,16 @@ model_upload_oss() {
   $hadoop fs -put ${model_name}.tar.gz ${MODEL_OSS_PATH}
   check_run_status $return_code $step_start_time "模型上传OSS"
 
+  echo ${model_save_path} > ${model_path_file}
 }
 
 # 主方法
 main() {
   init
 
-  # check_ad_hive
+  check_ad_hive
 
-  # xgb_train
+  xgb_train
 
   model_predict