| 1234567891011121314151617181920212223242526272829303132333435363738394041 |
- #!/bin/sh
- # 0 全局变量/参数
- savePath=/dw/recommend/model/ros_sample_v2/
- model_name=model_ros_v2
- today="$(date -d +%Y%m%d)"
- #yesterday="$(date -d '1 days ago' +%Y%m%d)"
- yesterday=20231221
- HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
- FM_TRAIN="/root/sunmingze/alphaFM/bin/fm_train"
- MODEL_PATH="/root/zhangbo/recommend-emr-dataprocess/zhangbo/model/"
- OSS_PATH="oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model"
- # 1 生产数据
- /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
- --class com.aliyun.odps.spark.examples.makedata.makedata_01_readtable2hdfs \
- --master yarn --driver-memory 1G --executor-memory 4G --executor-cores 2 --num-executors 16 \
- ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
- tablePart:32 savePath:${savePath} beginStr:${yesterday} endStr:${yesterday}
- # 2 加载上次模型 训练本轮数据 保存本轮模型
- end_date=${today}
- loop_date=${yesterday}
- while [[ "$loop_date" != "$end_date" ]]; do
- echo -------train ${loop_date}----------
- loop_date_model=$(date -d "$loop_date - 1 day" +%Y%m%d)
- $HADOOP fs -text ${savePath}/dt=${loop_date}/* | ${FM_TRAIN} -m ${MODEL_PATH}/${model_name}_${loop_date}.txt \
- -dim 1,1,0 -core 8 -im ${MODEL_PATH}/${model_name}_${loop_date_model}.txt
- echo -------save ${MODEL_PATH}/${model_name}_${loop_date}.txt----------
- loop_date=$(date -d "$loop_date + 1 day" +%Y%m%d)
- done
- # 3 本轮模型格式转换
- cat ${MODEL_PATH}/${model_name}_${yesterday}.txt \
- | sed '1d' | awk -F " " '{if($2!="0") print $1"\t"$2}' \
- > ${MODEL_PATH}/${model_name}_${yesterday}_change.txt
- # 4 转换后模型上传oss
- $HADOOP fs -rm -r oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model/${model_name}_change.txt
- $HADOOP fs -put ${MODEL_PATH}/${model_name}_${yesterday}_change.txt ${OSS_PATH}/${model_name}_change.txt
|
