recommend-emr-dataprocess/ad/02_ad_model_update_test.sh

#!/bin/sh
set -x

export PATH=$SPARK_HOME/bin:$PATH
export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
export JAVA_HOME=/usr/lib/jvm/java-1.8.0

sh_path=$(cd $(dirname $0); pwd)
source ${sh_path}/00_common.sh

source /root/anaconda3/bin/activate py37


# 全局常量
HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
TRAIN_PATH=/dw/recommend/model/31_ad_sample_data_v4
BUCKET_FEATURE_PATH=/dw/recommend/model/33_ad_train_data_v4
TABLE=alg_recsys_ad_sample_all
# 特征文件名
feature_file=20240703_ad_feature_name.txt
# 模型本地临时保存路径
model_local_home=/root/zhaohp/XGB/

# 模型HDFS保存路径，测试时修改为其他路径，避免影响线上
MODEL_PATH=/dw/recommend/model/35_ad_model
# 预测结果保存路径，测试时修改为其他路径，避免影响线上
PREDICT_RESULT_SAVE_PATH=/dw/recommend/model/34_ad_predict_data
# 模型OSS保存路径，测试时修改为其他路径，避免影响线上
MODEL_OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/
# 线上模型名，测试时修改为其他模型名，避免影响线上
model_name=model_xgb_351_1000_v2
# 线上校准文件名
OSS_CALIBRATION_FILE_NAME=model_xgb_351_1000_v2_calibration


# 本地保存HDFS模型路径文件，测试时修改为其他模型名，避免影响线上
model_path_file=${model_local_home}/online_model_path.txt
# 获取当前是星期几，1表示星期一
current_day_of_week="$(date +"%u")"

# 任务开始时间
start_time=$(date +%s)
# 前一天
today_early_1="$(date -d '1 days ago' +%Y%m%d)"
# 线上模型在HDFS中的路径
online_model_path=`cat ${model_path_file}`
# 训练用的数据路径
train_data_path=""
# 评估用的数据路径
predict_date_path=""
#评估结果保存路径
new_model_predict_result_path=""
# 模型保存路径
model_save_path=""
# 评测结果保存路径，后续需要根据此文件评估是否要更新模型
predict_analyse_file_path=""
# 校准文件保存路径
calibration_file_path=""

# 保存模型评估的分析结果
old_incr_rate_avg=0
new_incr_rate_avg=0

top10_msg=""

# 校验命令的退出码
check_run_status() {
    local status=$1
    local step_start_time=$2
    local step_name=$3
    local msg=$4

    local step_end_time=$(date +%s)
    local step_elapsed=$(($step_end_time - $step_start_time))

    if [ $status -ne 0 ]; then
        echo "$LOG_PREFIX -- ${step_name}失败: 耗时 $step_elapsed"
        local elapsed=$(($step_end_time - $start_time))
        /root/anaconda3/bin/python ${sh_path}/ad_monitor_util.py --level error --msg "$msg" --start "$start_time" --elapsed "$elapsed" --top10 "${top10_msg}"
        exit 1
    else
        echo "$LOG_PREFIX -- ${step_name}成功: 耗时 $step_elapsed"
    fi
}

send_success_upload_msg(){ 
  # 发送更新成功通知
  local msg=" 广告模型文件更新完成"
  msg+="\n\t - 老模型Top10差异平均值: ${old_incr_rate_avg}"
  msg+="\n\t - 新模型Top10差异平均值: ${new_incr_rate_avg}"
  msg+="\n\t - 模型在HDFS中的路径: ${model_save_path}"
  msg+="\n\t - 模型上传OSS中的路径: ${MODEL_OSS_PATH}/${model_name}.tar.gz"

  local step_end_time=$(date +%s)
  local elapsed=$(($step_end_time - $start_time))

  /root/anaconda3/bin/python ${sh_path}/ad_monitor_util.py --level info --msg "${msg}" --start "${start_time}" --elapsed "${elapsed}" --top10 "${top10_msg}"
}

init() {
  
  declare -a date_keys=()
  local count=1
  local current_data="$(date -d '2 days ago' +%Y%m%d)"
  # 循环获取前 n 天的非节日日期
  while [[ $count -lt 7 ]]; do
    date_key=$(date -d "$current_data" +%Y%m%d)
    # 判断是否是节日，并拼接训练数据路径
    if [ $(is_not_holidays $date_key) -eq 1 ]; then

      # 将 date_key 放入数组
      date_keys+=("$date_key")

      if [[ -z ${train_data_path} ]]; then
        train_data_path="${BUCKET_FEATURE_PATH}/${date_key}"
      else
        train_data_path="${BUCKET_FEATURE_PATH}/${date_key},${train_data_path}"
      fi 
      count=$((count + 1))
    else
      echo "日期: ${date_key}是节日，跳过"
    fi
    current_data=$(date -d "$current_data -1 day" +%Y%m%d)
  done

  last_index=$((${#date_keys[@]} - 1))
  train_first_day=${date_keys[$last_index]}
  train_last_day=${date_keys[0]}

  model_save_path=${MODEL_PATH}/${model_name}_${train_first_day: -4}_${train_last_day: -4}
  predict_date_path=${BUCKET_FEATURE_PATH}/${today_early_1}
  new_model_predict_result_path=/dw/recommend/model/34_ad_predict_data/20241104_351_1000_1028_1102
  online_model_predict_result_path=/dw/recommend/model/34_ad_predict_data/20241104_351_1000_1028_1102
  predict_analyse_file_path=${model_local_home}/predict_analyse_file/${today_early_1}_351_1000_analyse.txt
  calibration_file_path=${model_local_home}/${OSS_CALIBRATION_FILE_NAME}.txt

  echo "init param train_data_path: ${train_data_path}"
  echo "init param predict_date_path: ${predict_date_path}"
  echo "init param new_model_predict_result_path: ${new_model_predict_result_path}"
  echo "init param online_model_predict_result_path: ${online_model_predict_result_path}"
  echo "init param model_save_path: ${model_save_path}"
  echo "init param online_model_path: ${online_model_path}"
  echo "init param feature_file: ${feature_file}"
  echo "init param model_name: ${model_name}"
  echo "init param model_local_home: ${model_local_home}"
  echo "init param model_oss_path: ${MODEL_OSS_PATH}"
  echo "init param predict_analyse_file_path: ${predict_analyse_file_path}"
  echo "init param calibration_file_path: ${calibration_file_path}"
  echo "init param current_day_of_week: ${current_day_of_week}"

  echo "当前Python环境安装的Python版本: $(python --version)"
  echo "当前Python环境安装的三方包: $(python -m pip list)"
}


calc_model_predict() {
  local count=0
  local max_line=10
  local old_total_diff=0
  local new_total_diff=0

  local declare -A real_score_map
  local declare -A old_score_map
  local declare -A new_score_map
  top10_msg="| CID  | 老模型相对真实CTCVR的变化 | 新模型相对真实CTCVR的变化 |"
  top10_msg+=" \n| ---- | --------- | -------- |"
  while read -r line && [ ${count} -lt ${max_line} ]; do

      # 使用 ! 取反判断，只有当行中不包含 "cid" 时才执行继续的逻辑
      if [[ "${line}" == *"cid"* ]]; then
          continue
      fi

      read -a numbers <<< "${line}"

      # 分数分别保存
      real_score_map[${numbers[0]}]=${numbers[3]}
      old_score_map[${numbers[0]}]=${numbers[6]}
      new_score_map[${numbers[0]}]=${numbers[7]}

      # 拼接Top10详情的飞书消息
      top10_msg="${top10_msg} \n| ${numbers[0]} | ${numbers[6]} | ${numbers[7]} | "
      old_abs_score=$( echo "if(${numbers[6]} < 0) -${numbers[6]} else ${numbers[6]}" | bc -l)
      new_abs_score=$( echo "if(${numbers[7]} < 0) -${numbers[7]} else ${numbers[7]}" | bc -l)

      old_total_diff=$( echo "${old_total_diff} + ${old_abs_score}" | bc -l )
      new_total_diff=$( echo "${new_total_diff} + ${new_abs_score}" | bc -l )

      count=$((${count} + 1))

  done < "${predict_analyse_file_path}"

  local return_code=$?
  check_run_status $return_code $step_start_time "计算Top10差异" "计算Top10差异异常"

  old_incr_rate_avg=$( echo "scale=6; ${old_total_diff} / ${count}" | bc -l )
  check_run_status $? $step_start_time "计算老模型Top10差异" "计算老模型Top10差异异常"


  new_incr_rate_avg=$( echo "scale=6; ${new_total_diff} / ${count}" | bc -l )
  check_run_status $? $step_start_time "计算新模型Top10差异" "计算新模型Top10差异异常"

  echo "老模型Top10差异平均值: ${old_incr_rate_avg}"
  echo "新模型Top10差异平均值: ${new_incr_rate_avg}"
  echo "新老模型分数对比: "
  for cid in "${!new_score_map[@]}"; do
    echo "\t CID: $cid, 老模型分数: ${old_score_map[$cid]}, 新模型分数: ${new_score_map[$cid]}"
  done
}

model_predict() {

  local return_code=$?
  check_run_status $return_code $step_start_time "线上模型评估${predict_date_path: -8}的数据" "线上模型评估${predict_date_path: -8}的数据失败"

  # 结果分析
  local python_return_code=$(python ${sh_path}/model_predict_analyse.py -op ${online_model_predict_result_path} -np ${new_model_predict_result_path} -af ${predict_analyse_file_path} -cf ${calibration_file_path})
  check_run_status $python_return_code $step_start_time "分析线上模型评估${predict_date_path: -8}的数据" "分析线上模型评估${predict_date_path: -8}的数据失败"

  calc_model_predict

  if (( $(echo "${new_incr_rate_avg} > 0.100000" | bc -l ) ));then 
    echo "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.1，请检查"
    check_run_status 1 $step_start_time "${predict_date_path: -8}的数据，绝对误差大于0.1" "线上模型评估${predict_date_path: -8}的数据，绝对误差大于0.1，请检查"
    exit 1
  fi 


  # 对比两个模型的差异
  score_diff=$( echo "${new_incr_rate_avg} - ${old_incr_rate_avg}" | bc -l )
  if (( $(echo "${score_diff} > 0.050000" | bc -l ) ));then 
    echo "两个模型评估${predict_date_path: -8}的数据，两个模型分数差异为: ${score_diff}, 大于0.05, 请检查"
    check_run_status 1 $step_start_time "两个模型评估${predict_date_path: -8}的数据" "两个模型评估${predict_date_path: -8}的数据，两个模型分数差异为: ${score_diff}, 大于0.05"
    exit 1
  fi 

}

# 主方法
main() {
  init

  model_predict

}


main