recommend-emr-dataprocess/zhangbo/05_update_everyday_ros.sh

#!/bin/sh
set -e
set -x
# 0 全局变量/参数
savePath=/dw/recommend/model/ros_sample_v2/
model_name=model_ros_v2
today="$(date +%Y%m%d)"
today_early_1="$(date -d '1 days ago' +%Y%m%d)"
#yesterday="$(date -d '1 days ago' +%Y%m%d)"
yesterday=20231227
jar_main=$makedata_04_rosHdfsFromTablev2

HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
FM_TRAIN="/root/sunmingze/alphaFM/bin/fm_train"
MODEL_PATH="/root/zhangbo/recommend-emr-dataprocess/zhangbo/model/"
OSS_PATH="oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model"

# 1 生产数据
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.${jar_main} \
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
tablePart:32 savePath:${savePath} beginStr:${today_early_1} endStr:${today_early_1}

if [ $? -eq 1 ]; then
    echo "Spark任务执行失败"
    exit 1
fi

# 2 加载上次模型 训练本轮数据 保存本轮模型
end_date=${today}
loop_date=${yesterday}
while [[ "$loop_date" != "$end_date" ]]; do
    echo -------train ${loop_date}----------
    loop_date_model=$(date -d "$loop_date - 1 day" +%Y%m%d)
    $HADOOP fs -text ${savePath}/dt=${loop_date}/* | ${FM_TRAIN} -m ${MODEL_PATH}/${model_name}_${loop_date}.txt \
-dim 1,1,0 -core 8 -im ${MODEL_PATH}/${model_name}_${loop_date_model}.txt
    if [ $? -eq 1 ]; then
        echo "训练失败"
        exit 1
    fi
    echo -------save ${MODEL_PATH}/${model_name}_${loop_date}.txt----------
    loop_date=$(date -d "$loop_date + 1 day" +%Y%m%d)
done

# 3 本轮模型格式转换
cat ${MODEL_PATH}/${model_name}_${today_early_1}.txt \
| sed '1d' | awk -F " " '{if($2!="0") print $1"\t"$2}' \
> ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt

# 4 转换后模型上传oss
$HADOOP fs -rm -r oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model/${model_name}_change.txt
$HADOOP fs -put ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt ${OSS_PATH}/${model_name}_change.txt


#nohup sh 05_update_everyday_ros.sh > p.log 2>&1 &