|
|
@@ -8,54 +8,8 @@ export JAVA_HOME=/usr/lib/jvm/java-1.8.0
|
|
|
sh_path=$(cd $(dirname $0); pwd)
|
|
|
source ${sh_path}/00_common.sh
|
|
|
|
|
|
-source /root/anaconda3/bin/activate py37
|
|
|
-
|
|
|
-
|
|
|
-# 全局常量
|
|
|
-HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
|
|
|
-TRAIN_PATH=/dw/recommend/model/31_ad_sample_data_v4
|
|
|
-BUCKET_FEATURE_PATH=/dw/recommend/model/33_ad_train_data_v4
|
|
|
-TABLE=alg_recsys_ad_sample_all
|
|
|
-# 特征文件名
|
|
|
-feature_file=20240703_ad_feature_name.txt
|
|
|
-# 模型本地临时保存路径
|
|
|
-model_local_home=/root/zhaohp/XGB/
|
|
|
-
|
|
|
-# 模型HDFS保存路径，测试时修改为其他路径，避免影响线上
|
|
|
-MODEL_PATH=/dw/recommend/model/35_ad_model
|
|
|
-# 预测结果保存路径，测试时修改为其他路径，避免影响线上
|
|
|
-PREDICT_RESULT_SAVE_PATH=/dw/recommend/model/34_ad_predict_data
|
|
|
-# 模型OSS保存路径，测试时修改为其他路径，避免影响线上
|
|
|
-MODEL_OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/
|
|
|
-# 线上模型名，测试时修改为其他模型名，避免影响线上
|
|
|
-model_name=model_xgb_351_1000_v2
|
|
|
-# 线上校准文件名
|
|
|
-OSS_CALIBRATION_FILE_NAME=model_xgb_351_1000_v2_calibration
|
|
|
-
|
|
|
-
|
|
|
-# 本地保存HDFS模型路径文件，测试时修改为其他模型名，避免影响线上
|
|
|
-model_path_file=${model_local_home}/online_model_path.txt
|
|
|
-# 获取当前是星期几，1表示星期一
|
|
|
-current_day_of_week="$(date +"%u")"
|
|
|
-
|
|
|
-# 任务开始时间
|
|
|
-start_time=$(date +%s)
|
|
|
-# 前一天
|
|
|
-today_early_1="$(date -d '1 days ago' +%Y%m%d)"
|
|
|
-# 线上模型在HDFS中的路径
|
|
|
-online_model_path=`cat ${model_path_file}`
|
|
|
-# 训练用的数据路径
|
|
|
-train_data_path=""
|
|
|
-# 评估用的数据路径
|
|
|
-predict_date_path=""
|
|
|
-#评估结果保存路径
|
|
|
-new_model_predict_result_path=""
|
|
|
-# 模型保存路径
|
|
|
-model_save_path=""
|
|
|
# 评测结果保存路径，后续需要根据此文件评估是否要更新模型
|
|
|
predict_analyse_file_path=/root/zhaohp/XGB/predict_analyse_file/20241105_351_1000_analyse.txt
|
|
|
-# 校准文件保存路径
|
|
|
-calibration_file_path=""
|
|
|
|
|
|
# 保存模型评估的分析结果
|
|
|
old_incr_rate_avg=0
|
|
|
@@ -99,15 +53,9 @@ calc_model_predict() {
|
|
|
|
|
|
done < "${predict_analyse_file_path}"
|
|
|
|
|
|
- local return_code=$?
|
|
|
- check_run_status $return_code $step_start_time "计算Top10差异" "计算Top10差异异常"
|
|
|
-
|
|
|
old_incr_rate_avg=$( echo "scale=6; ${old_total_diff} / ${count}" | bc -l )
|
|
|
- check_run_status $? $step_start_time "计算老模型Top10差异" "计算老模型Top10差异异常"
|
|
|
-
|
|
|
|
|
|
new_incr_rate_avg=$( echo "scale=6; ${new_total_diff} / ${count}" | bc -l )
|
|
|
- check_run_status $? $step_start_time "计算新模型Top10差异" "计算新模型Top10差异异常"
|
|
|
|
|
|
echo "老模型Top10差异平均值: ${old_incr_rate_avg}"
|
|
|
echo "新模型Top10差异平均值: ${new_incr_rate_avg}"
|
