feat:放开飞书通知

ad/02_ad_model_update_test.sh

@@ -53,7 +53,7 @@ new_model_predict_result_path=""
 # 模型保存路径
 model_save_path=""
 # 评测结果保存路径，后续需要根据此文件评估是否要更新模型
-predict_analyse_file_path=""
+predict_analyse_file_path=/root/zhaohp/XGB/predict_analyse_file/20241105_351_1000_analyse.txt
 # 校准文件保存路径
 calibration_file_path=""
 
@@ -63,105 +63,12 @@ new_incr_rate_avg=0
 
 top10_msg=""
 
-# 校验命令的退出码
-check_run_status() {
-    local status=$1
-    local step_start_time=$2
-    local step_name=$3
-    local msg=$4
-
-    local step_end_time=$(date +%s)
-    local step_elapsed=$(($step_end_time - $step_start_time))
-
-    if [ $status -ne 0 ]; then
-        echo "$LOG_PREFIX -- ${step_name}失败: 耗时 $step_elapsed"
-        local elapsed=$(($step_end_time - $start_time))
-        /root/anaconda3/bin/python ${sh_path}/ad_monitor_util.py --level error --msg "$msg" --start "$start_time" --elapsed "$elapsed" --top10 "${top10_msg}"
-        exit 1
-    else
-        echo "$LOG_PREFIX -- ${step_name}成功: 耗时 $step_elapsed"
-    fi
-}
-
-send_success_upload_msg(){ 
-  # 发送更新成功通知
-  local msg=" 广告模型文件更新完成"
-  msg+="\n\t - 老模型Top10差异平均值: ${old_incr_rate_avg}"
-  msg+="\n\t - 新模型Top10差异平均值: ${new_incr_rate_avg}"
-  msg+="\n\t - 模型在HDFS中的路径: ${model_save_path}"
-  msg+="\n\t - 模型上传OSS中的路径: ${MODEL_OSS_PATH}/${model_name}.tar.gz"
-
-  local step_end_time=$(date +%s)
-  local elapsed=$(($step_end_time - $start_time))
-
-  /root/anaconda3/bin/python ${sh_path}/ad_monitor_util.py --level info --msg "${msg}" --start "${start_time}" --elapsed "${elapsed}" --top10 "${top10_msg}"
-}
-
-init() {
-  
-  declare -a date_keys=()
-  local count=1
-  local current_data="$(date -d '2 days ago' +%Y%m%d)"
-  # 循环获取前 n 天的非节日日期
-  while [[ $count -lt 7 ]]; do
-    date_key=$(date -d "$current_data" +%Y%m%d)
-    # 判断是否是节日，并拼接训练数据路径
-    if [ $(is_not_holidays $date_key) -eq 1 ]; then
-
-      # 将 date_key 放入数组
-      date_keys+=("$date_key")
-
-      if [[ -z ${train_data_path} ]]; then
-        train_data_path="${BUCKET_FEATURE_PATH}/${date_key}"
-      else
-        train_data_path="${BUCKET_FEATURE_PATH}/${date_key},${train_data_path}"
-      fi 
-      count=$((count + 1))
-    else
-      echo "日期: ${date_key}是节日，跳过"
-    fi
-    current_data=$(date -d "$current_data -1 day" +%Y%m%d)
-  done
-
-  last_index=$((${#date_keys[@]} - 1))
-  train_first_day=${date_keys[$last_index]}
-  train_last_day=${date_keys[0]}
-
-  model_save_path=${MODEL_PATH}/${model_name}_${train_first_day: -4}_${train_last_day: -4}
-  predict_date_path=${BUCKET_FEATURE_PATH}/${today_early_1}
-  online_model_predict_result_path=/dw/recommend/model/34_ad_predict_data/20241103_351_1000_1025_1030
-  new_model_predict_result_path=/dw/recommend/model/34_ad_predict_data/20241104_351_1000_1028_1102
-  predict_analyse_file_path=${model_local_home}/predict_analyse_file/${today_early_1}_351_1000_analyse.txt
-  calibration_file_path=${model_local_home}/${OSS_CALIBRATION_FILE_NAME}.txt
-
-  echo "init param train_data_path: ${train_data_path}"
-  echo "init param predict_date_path: ${predict_date_path}"
-  echo "init param new_model_predict_result_path: ${new_model_predict_result_path}"
-  echo "init param online_model_predict_result_path: ${online_model_predict_result_path}"
-  echo "init param model_save_path: ${model_save_path}"
-  echo "init param online_model_path: ${online_model_path}"
-  echo "init param feature_file: ${feature_file}"
-  echo "init param model_name: ${model_name}"
-  echo "init param model_local_home: ${model_local_home}"
-  echo "init param model_oss_path: ${MODEL_OSS_PATH}"
-  echo "init param predict_analyse_file_path: ${predict_analyse_file_path}"
-  echo "init param calibration_file_path: ${calibration_file_path}"
-  echo "init param current_day_of_week: ${current_day_of_week}"
-
-  echo "当前Python环境安装的Python版本: $(python --version)"
-  echo "当前Python环境安装的三方包:  $(python -m pip list)"
-}
-
 
 calc_model_predict() {
   local count=0
   local max_line=10
   local old_total_diff=0
   local new_total_diff=0
-
-  local declare -A real_score_map
-  local declare -A old_score_map
-  local declare -A new_score_map
   top10_msg="| CID  | 老模型相对真实CTCVR的变化 | 新模型相对真实CTCVR的变化 |"
   top10_msg+=" \n| ---- | --------- | -------- |"
   while read -r line && [ ${count} -lt ${max_line} ]; do
@@ -180,8 +87,10 @@ calc_model_predict() {
 
       # 拼接Top10详情的飞书消息
       top10_msg="${top10_msg} \n| ${numbers[0]} | ${numbers[6]} | ${numbers[7]} | "
-      old_abs_score=$( echo "if(${numbers[6]} < 0) -${numbers[6]} else ${numbers[6]}" | bc -l)
-      new_abs_score=$( echo "if(${numbers[7]} < 0) -${numbers[7]} else ${numbers[7]}" | bc -l)
+
+      # 计算top10相对误差绝对值的均值
+      old_abs_score=$( echo "if(${numbers[6]} < 0) -${numbers[6]} else ${numbers[6]}" | bc -l )
+      new_abs_score=$( echo "if(${numbers[7]} < 0) -${numbers[7]} else ${numbers[7]}" | bc -l )
 
       old_total_diff=$( echo "${old_total_diff} + ${old_abs_score}" | bc -l )
       new_total_diff=$( echo "${new_total_diff} + ${new_abs_score}" | bc -l )
@@ -208,34 +117,5 @@ calc_model_predict() {
   done
 }
 
-model_predict() {
-
-  calc_model_predict
-
-  if (( $(echo "${new_incr_rate_avg} > 0.100000" | bc -l ) ));then 
-    echo "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.1，请检查"
-    check_run_status 1 $step_start_time "${predict_date_path: -8}的数据，绝对误差大于0.1" "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.1，请检查"
-    exit 1
-  fi 
-
-
-  # 对比两个模型的差异
-  score_diff=$( echo "${new_incr_rate_avg} - ${old_incr_rate_avg}" | bc -l )
-  if (( $(echo "${score_diff} > 0.050000" | bc -l ) ));then 
-    echo "两个模型评估${predict_date_path: -8}的数据，两个模型分数差异为: ${score_diff}, 大于0.05, 请检查"
-    check_run_status 1 $step_start_time "两个模型评估${predict_date_path: -8}的数据" "两个模型评估${predict_date_path: -8}的数据，两个模型分数差异为: ${score_diff}, 大于0.05"
-    exit 1
-  fi 
-
-}
-
-# 主方法
-main() {
-  init
-
-  model_predict
-
-}
-
 
-main
+calc_model_predict