recommend-emr-dataprocess/ad/21_ad_model_add_dt_train_predict_auc.sh

#!/bin/sh

# 指定基础模型，模型增量训练，预测，计算AUC脚本

set -x

begin_date=$1
end_date=$2
model_name=$3
train_dim=$4
predict_dim=$5

PROJECT_HOME=/root/zhaohp/recommend-emr-dataprocess
HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
HDFS_TRAIN_DATE_PATH=/dw/recommend/model/33_ad_train_data_v4
MODEL_PATH=${PROJECT_HOME}/model
PREDICT_PATH=${PROJECT_HOME}/predict

FM_TRAIN=/root/sunmingze/alphaFM/bin/fm_train
FM_PREDICT=/root/sunmingze/alphaFM/bin/fm_predict

train_date=$begin_date

# 计算模型的AUC，从训练日期的后一天到参数的end_date
predict_auc() {
    echo -e "\t==================== 开始预测 $train_date 模型 ===================="

    predict_date=$(date -d "$train_date +1 day" +%Y%m%d)
    predict_end_date=$(date -d "$end_date +1 day" +%Y%m%d)
    while [ "$predict_date" != "$predict_end_date" ]; do

        $HADOOP fs -text ${HDFS_TRAIN_DATE_PATH}/${predict_date}/* | ${FM_PREDICT} -m ${MODEL_PATH}/${model_name}_${train_date}.txt -dim ${predict_dim} -core 8 -out ${PREDICT_PATH}/${model_name}_${train_date}_${predict_date}.txt
        auc=`cat ${PREDICT_PATH}/${model_name}_${train_date}_${predict_date}.txt | /root/sunmingze/AUC/AUC`

        echo "模型训练日期: ${train_date}, 模型预测日期: ${predict_date}, AUC: ${auc}, 模型路径: ${MODEL_PATH}/${model_name}_${train_date}.txt"

        predict_date=$(date -d "$predict_date +1 day" +%Y%m%d)

    done

    echo -e "\n\t==================== 预测 $train_date 模型结束 ===================="

}
main() {

    # 增量训练模型
    while [ "$train_date" != "$end_date" ]; do
        echo "==================== 开始训练 $train_date 模型 ===================="

        # 模型训练
        yesterday=$(date -d "$train_date -1 day" +%Y%m%d)

        input_model=${MODEL_PATH}/${model_name}_${yesterday}.txt
        if [ ! -e "${input_model}" ]; then
            echo "输入模型: ${input_model} 不存在，退出"
            exit 1
        fi

        $HADOOP fs -text ${HDFS_TRAIN_DATE_PATH}/${train_date}/* | ${FM_TRAIN} -m ${MODEL_PATH}/${model_name}_${train_date}.txt -dim ${train_dim} -core 8 -im ${input_model}

        predict_auc

        train_date=$(date -d "$train_date +1 day" +%Y%m%d)

        echo "==================== 训练 $train_date 模型结束 ===================="
        echo -e "\n\n\n\n\n\n"
    done

}

main