05_update_everyday_ros.sh 2.0 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950
  1. #!/bin/sh
  2. set -e
  3. set -x
  4. # 0 全局变量/参数
  5. savePath=/dw/recommend/model/ros_sample_v2/
  6. model_name=model_ros_v2
  7. today="$(date +%Y%m%d)"
  8. #yesterday="$(date -d '1 days ago' +%Y%m%d)"
  9. yesterday=20231221
  10. HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
  11. FM_TRAIN="/root/sunmingze/alphaFM/bin/fm_train"
  12. MODEL_PATH="/root/zhangbo/recommend-emr-dataprocess/zhangbo/model/"
  13. OSS_PATH="oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model"
  14. # 1 生产数据
  15. #/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
  16. #--class com.aliyun.odps.spark.examples.makedata.makedata_01_readtable2hdfs \
  17. #--master yarn --driver-memory 1G --executor-memory 4G --executor-cores 2 --num-executors 16 \
  18. #/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
  19. #tablePart:32 savePath:${savePath} beginStr:${yesterday} endStr:${yesterday}
  20. #
  21. #if [ $? -eq 1 ]; then
  22. # echo "Spark任务执行失败"
  23. # exit 1
  24. # 2 加载上次模型 训练本轮数据 保存本轮模型
  25. end_date=${today}
  26. loop_date=${yesterday}
  27. while [[ "$loop_date" != "$end_date" ]]; do
  28. echo -------train ${loop_date}----------
  29. loop_date_model=$(date -d "$loop_date - 1 day" +%Y%m%d)
  30. $HADOOP fs -text ${savePath}/dt=${loop_date}/* | ${FM_TRAIN} -m ${MODEL_PATH}/${model_name}_${loop_date}.txt \
  31. -dim 1,1,0 -core 8 -im ${MODEL_PATH}/${model_name}_${loop_date_model}.txt
  32. echo -------save ${MODEL_PATH}/${model_name}_${loop_date}.txt----------
  33. loop_date=$(date -d "$loop_date + 1 day" +%Y%m%d)
  34. done
  35. # 3 本轮模型格式转换
  36. cat ${MODEL_PATH}/${model_name}_${yesterday}.txt \
  37. | sed '1d' | awk -F " " '{if($2!="0") print $1"\t"$2}' \
  38. > ${MODEL_PATH}/${model_name}_${yesterday}_change.txt
  39. # 4 转换后模型上传oss
  40. $HADOOP fs -rm -r oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model/${model_name}_change.txt
  41. $HADOOP fs -put ${MODEL_PATH}/${model_name}_${yesterday}_change.txt ${OSS_PATH}/${model_name}_change.txt
  42. #nohup sh 05_update_everyday_ros.sh > p.log 2>&1 &