feat:添加评估结果分析脚本

ad/01_ad_model_update.sh

@@ -18,6 +18,8 @@ BUCKET_FEATURE_PATH=/dw/recommend/model/33_ad_train_data_v4
 TABLE=alg_recsys_ad_sample_all
 # 特征文件名
 feature_file=20240703_ad_feature_name.txt
+# 模型本地临时保存路径
+model_local_home=/root/zhaohp/XGB/
 
 # 模型HDFS保存路径，测试时修改为其他路径，避免影响线上
 MODEL_PATH=/dw/recommend/model/35_ad_model_test
@@ -28,7 +30,7 @@ MODEL_OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/model/
 # 线上模型名，测试时修改为其他模型名，避免影响线上
 model_name=model_xgb_351_1000_v2_test
 # 本地保存HDFS模型路径文件，测试时修改为其他模型名，避免影响线上
-model_path_file=/root/zhaohp/XGB/online_model_path_test.txt
+model_path_file=${model_local_home}/online_model_path_test.txt
 
 today_early_1="$(date -d '1 days ago' +%Y%m%d)"
 # 训练用的数据路径
@@ -39,8 +41,7 @@ predict_date_path=""
 new_model_predict_result_path=""
 # 模型保存路径
 model_save_path=""
-# 模型本地临时保存路径
-model_local_path=/root/zhaohp/XGB
+
 # 任务开始时间
 start_time=$(date +%s)
 # 线上模型在HDFS中的路径
@@ -65,7 +66,7 @@ check_run_status() {
     if [ $status -ne 0 ]; then
         echo "$LOG_PREFIX -- ${step_name}失败: 耗时 $step_elapsed"
         local elapsed=$(($step_end_time - $start_time))
-        /root/anaconda3/bin/python ${sh_path}/ad_monitor_util.py --level error --msg "$msg" --start "$start_time" --elapsed "$elapsed"
+        # /root/anaconda3/bin/python ${sh_path}/ad_monitor_util.py --level error --msg "$msg" --start "$start_time" --elapsed "$elapsed"
         exit 1
     else
         echo "$LOG_PREFIX -- ${step_name}成功: 耗时 $step_elapsed"
@@ -106,7 +107,7 @@ init() {
   predict_date_path=${BUCKET_FEATURE_PATH}/${today_early_1}
   new_model_predict_result_path=${PREDICT_RESULT_SAVE_PATH}/${today_early_1}_351_1000_${train_first_day: -4}_${train_last_day: -4}
   online_model_predict_result_path=${PREDICT_RESULT_SAVE_PATH}/${today_early_1}_351_1000_${online_model_path: -9}
-  predict_analyse_file_path=${model_local_path}/predict_analyse_file/${today_early_1}_351_1000_analyse.txt
+  predict_analyse_file_path=${model_local_home}/predict_analyse_file/${today_early_1}_351_1000_analyse.txt
 
   echo "init param train_data_path: ${train_data_path}"
   echo "init param predict_date_path: ${predict_date_path}"
@@ -116,7 +117,7 @@ init() {
   echo "init param online_model_path: ${online_model_path}"
   echo "init param feature_file: ${feature_file}"
   echo "init param model_name: ${model_name}"
-  echo "init param model_local_path: ${model_local_path}"
+  echo "init param model_local_home: ${model_local_home}"
   echo "init param model_oss_path: ${MODEL_OSS_PATH}"
   echo "init param predict_analyse_file_path: ${predict_analyse_file_path}"
 
@@ -263,8 +264,8 @@ model_predict() {
   # check_run_status $return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据"
 
   # 结果分析
-  local python_return_code=$(python ${sh_path}/model_predict_analyse.py -p ${online_model_predict_result_path} ${new_model_predict_result_path} -f ${predict_analyse_file_path})
-  check_run_status $python_return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据"
+  # local python_return_code=$(python ${sh_path}/model_predict_analyse.py -p ${online_model_predict_result_path} ${new_model_predict_result_path} -f ${predict_analyse_file_path})
+  # check_run_status $python_return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据"
 
   calc_model_predict
 
@@ -276,22 +277,25 @@ model_predict() {
 }
 
 model_upload_oss() {
-  cd ${model_local_path}
-  $hadoop fs -get ${model_save_path} ./${model_name}
 
-  if [ ! -d ./${model_name} ]; then
+  local step_start_time=$(date +%s)
+
+  cd ${model_local_home}
+
+  ${HADOOP} fs -get ${model_save_path} ${model_name}
+  if [ ! -d ${model_name} ]; then
     echo "从HDFS下载模型失败"
-    check_run_status 1 $step_start_time "XGB模型训练任务" 
+    check_run_status 1 $step_start_time "HDFS下载模型任务" 
     exit 1 
   fi
 
   tar -czvf ${model_name}.tar.gz -C ${model_name} .
 
-  rm -rf .${model_name}.tar.gz.crc
+  rm -rf ${model_name}.tar.gz.crc
 
-  $hadoop fs -rm -r -skipTrash ${MODEL_OSS_PATH}/${model_name}.tar.gz
+  ${HADOOP} fs -rm -r -skipTrash ${MODEL_OSS_PATH}/${model_name}.tar.gz
   
-  $hadoop fs -put ${model_name}.tar.gz ${MODEL_OSS_PATH}
+  ${HADOOP} fs -put ${model_name}.tar.gz ${MODEL_OSS_PATH}
   check_run_status $return_code $step_start_time "模型上传OSS"
 
   echo ${model_save_path} > ${model_path_file}
@@ -299,9 +303,12 @@ model_upload_oss() {
   local step_end_time=$(date +%s)
   local elapsed=$(($step_end_time - $start_time))
 
+  rm -f ./${model_name}.tar.gz
+  rm -rf ./${model_name}
+
   echo -e "$LOG_PREFIX -- 模型更新完成 -- 模型更新成功: 耗时 $elapsed"
   local msg="\n\t - 广告模型文件更新完成 \n\t - 前一天线上模型全天Top差异平均值: ${old_incr_rate_avg} \n\t - 前一天线上模型全天Top差异: ${old_incr_rate_list} \n\t - 前一天新模型全天Top10差异平均值: ${new_incr_rate_avg} \n\t - 前一天新模型全天Top差异: ${new_incr_rate_list} \n\t - 模型上传路径: ${MODEL_OSS_PATH}/${model_name}.tar.gz"
-  /root/anaconda3/bin/python ${sh_path}/ad_monitor_util.py --level info --msg ${msg} --start "${start_time}" --elapsed "${elapsed}"
+  # /root/anaconda3/bin/python ${sh_path}/ad_monitor_util.py --level info --msg ${msg} --start "${start_time}" --elapsed "${elapsed}"
 }
 
 # 主方法