feat:添加评估结果分析脚本

ad/01_ad_model_update.sh

@@ -228,7 +228,7 @@ calc_model_predict() {
   check_run_status $return_code $step_start_time "计算Top10差异"
 
 
-  new_incr_rate_list=$( echo "scale=6; ${new_total_diff} / ${count}" | bc l )
+  new_incr_rate_avg=$( echo "scale=6; ${new_total_diff} / ${count}" | bc l )
   return_code=$?
   check_run_status $return_code $step_start_time "计算Top10差异"
 }
@@ -237,25 +237,25 @@ model_predict() {
 
   # 线上模型评估最新的数据
   local step_start_time=$(date +%s)
-  /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
-  --class com.tzld.piaoquan.recommend.model.pred_01_xgb_ad_hdfsfile_20240813 \
-  --master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 30 \
-  --conf spark.yarn.executor.memoryoverhead=1024 \
-  --conf spark.shuffle.service.enabled=true \
-  --conf spark.shuffle.service.port=7337 \
-  --conf spark.shuffle.consolidateFiles=true \
-  --conf spark.shuffle.manager=sort \
-  --conf spark.storage.memoryFraction=0.4 \
-  --conf spark.shuffle.memoryFraction=0.5 \
-  --conf spark.default.parallelism=200 \
-  /root/zhangbo/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar \
-  featureFile:20240703_ad_feature_name.txt \
-  testPath:${predict_date_path} \
-  savePath:${online_model_predict_result_path} \
-  modelPath:${online_model_path}
-
-  local return_code=$?
-  check_run_status $return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据"
+  # /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
+  # --class com.tzld.piaoquan.recommend.model.pred_01_xgb_ad_hdfsfile_20240813 \
+  # --master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 30 \
+  # --conf spark.yarn.executor.memoryoverhead=1024 \
+  # --conf spark.shuffle.service.enabled=true \
+  # --conf spark.shuffle.service.port=7337 \
+  # --conf spark.shuffle.consolidateFiles=true \
+  # --conf spark.shuffle.manager=sort \
+  # --conf spark.storage.memoryFraction=0.4 \
+  # --conf spark.shuffle.memoryFraction=0.5 \
+  # --conf spark.default.parallelism=200 \
+  # /root/zhangbo/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar \
+  # featureFile:20240703_ad_feature_name.txt \
+  # testPath:${predict_date_path} \
+  # savePath:${online_model_predict_result_path} \
+  # modelPath:${online_model_path}
+
+  # local return_code=$?
+  # check_run_status $return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据"
 
   # 结果分析
   local python_return_code=$(python ${sh_path}/model_predict_analyse.py -p ${online_model_predict_result_path} ${new_model_predict_result_path})
@@ -263,7 +263,7 @@ model_predict() {
 
   calc_model_predict
 
-  if (( $(echo "${new_incr_rate_list} > 0.100000" | bc -l ) ));then 
+  if (( $(echo "${new_incr_rate_avg} > 0.100000" | bc -l ) ));then 
     check_run_status 1 $step_start_time "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.1，请检查"
     echo "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.1，请检查"
     exit 1
@@ -302,17 +302,17 @@ model_upload_oss() {
 main() {
   init
 
-  check_ad_hive
+  # check_ad_hive
 
-  origin_data
+  # origin_data
 
-  bucket_feature
+  # bucket_feature
 
-  xgb_train
+  # xgb_train
 
   model_predict
 
-  model_upload_oss
+  # model_upload_oss
 
 }