feat:添加评估结果分析脚本

ad/01_ad_model_update.sh

@@ -168,40 +168,30 @@ xgb_train() {
 model_predict() {
 
   # 线上模型评估最新的数据
-  # local step_start_time=$(date +%s)
-  # /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
-  # --class com.tzld.piaoquan.recommend.model.pred_01_xgb_ad_hdfsfile_20240813 \
-  # --master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 30 \
-  # --conf spark.yarn.executor.memoryoverhead=1024 \
-  # --conf spark.shuffle.service.enabled=true \
-  # --conf spark.shuffle.service.port=7337 \
-  # --conf spark.shuffle.consolidateFiles=true \
-  # --conf spark.shuffle.manager=sort \
-  # --conf spark.storage.memoryFraction=0.4 \
-  # --conf spark.shuffle.memoryFraction=0.5 \
-  # --conf spark.default.parallelism=200 \
-  # /root/zhangbo/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar \
-  # featureFile:20240703_ad_feature_name.txt \
-  # testPath:${predict_date_path} \
-  # savePath:${online_model_predict_result_path} \
-  # modelPath:${online_model_path}
-
-  # local return_code=$?
-  # check_run_status $return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据"
-
-  # local mean_abs_diff=$(python ${sh_path}/model_predict_analyse.py -p ${online_model_predict_result_path} ${new_model_predict_result_path})
-  # if (( $(echo "${mean_abs_diff} > 0.000400" | bc -l ) ));then
-  #   check_run_status 1 $step_start_time "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.000400，请检查"
-  #   echo "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.000400，请检查"
-  #   exit 1
-  # fi 
+  local step_start_time=$(date +%s)
+  /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
+  --class com.tzld.piaoquan.recommend.model.pred_01_xgb_ad_hdfsfile_20240813 \
+  --master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 30 \
+  --conf spark.yarn.executor.memoryoverhead=1024 \
+  --conf spark.shuffle.service.enabled=true \
+  --conf spark.shuffle.service.port=7337 \
+  --conf spark.shuffle.consolidateFiles=true \
+  --conf spark.shuffle.manager=sort \
+  --conf spark.storage.memoryFraction=0.4 \
+  --conf spark.shuffle.memoryFraction=0.5 \
+  --conf spark.default.parallelism=200 \
+  /root/zhangbo/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar \
+  featureFile:20240703_ad_feature_name.txt \
+  testPath:${predict_date_path} \
+  savePath:${online_model_predict_result_path} \
+  modelPath:${online_model_path}
 
-  
-  local p1="/dw/recommend/model/34_ad_predict_data_test/20241008_351_1000_1002_1007"
-  local p2="/dw/recommend/model/34_ad_predict_data_test/20241008_351_1000_0930_1006"
-  local mean_abs_diff=$(/root/anaconda3/bin/python ${sh_path}/model_predict_analyse.py -p ${p1} ${p2})
+  local return_code=$?
+  check_run_status $return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据"
+
+  local mean_abs_diff=$(/root/anaconda3/bin/python ${sh_path}/model_predict_analyse.py -p ${online_model_predict_result_path} ${new_model_predict_result_path})
   if (( $(echo "${mean_abs_diff} > 0.000400" | bc -l ) ));then
-    # check_run_status 1 $step_start_time "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.000400，请检查"
+    check_run_status 1 $step_start_time "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.000400，请检查"
     echo "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.000400，请检查"
     exit 1
   fi 
@@ -235,7 +225,7 @@ main() {
 
   # check_ad_hive
 
-  # xgb_train
+  xgb_train
 
   model_predict