Home Explore Help
Sign In
algorithm
/
recommend-emr-dataprocess
12
0
Fork 0
Files Issues 0 Pull Requests 1 Wiki
Tree: 41c27c57d7
Branches Tags
recommend-em...
/
zhangbo
/
05_update_everyday_str.sh

05_update_everyday_str.sh 2.0 KB
History Raw

123456789101112131415161718192021222324252627282930313233343536373839404142 
  1. ##!/bin/sh
    2. 

  2. ## 0 全局变量/参数
    3. 

  3. #savePath=/dw/recommend/model/share_ratio_samples_v2/
    4. 

  4. #model_name=model_sharev2
    5. 

  5. #today="$(date -d '1 days ago' +%Y%m%d)"
    6. 

  6. #yesterday="$(date -d '2 days ago' +%Y%m%d)"
    7. 

  7. #yesterday_early=20231222
    8. 

  8. ##yesterday_early="$(date -d '3 days ago' +%Y%m%d)"
    9. 

  9. #
    10. 

  10. #
    11. 

  11. #HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
    12. 

  12. #FM_TRAIN="/root/sunmingze/alphaFM/bin/fm_train"
    13. 

  13. #MODEL_PATH="/root/zhangbo/recommend-emr-dataprocess/zhangbo/model/"
    14. 

  14. #OSS_PATH="oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model"
    15. 

  15. #
    16. 

  16. ## 1 生产数据
    17. 

  17. #/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
    18. 

  18. #--class com.aliyun.odps.spark.examples.makedata.makedata_01_readtable2hdfs \
    19. 

  19. #--master yarn --driver-memory 1G --executor-memory 4G --executor-cores 2 --num-executors 16 \
    20. 

  20. #./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
    21. 

  21. #tablePart:32 savePath:${savePath} beginStr:${yesterday} endStr:${yesterday}
    22. 

  22. #
    23. 

  23. ## 2 加载上次模型 训练本轮数据 保存本轮模型
    24. 

  24. #end_date=${today}
    25. 

  25. #loop_date=${yesterday_early}
    26. 

  26. #while [[ "$loop_date" != "$end_date" ]]; do
    27. 

  27. #    echo -------train ${loop_date}----------
    28. 

  28. #    loop_date_model=$(date -d "$loop_date - 1 day" +%Y%m%d)
    29. 

  29. #    $HADOOP fs -text ${savePath}/dt=${loop_date}/* | ${FM_TRAIN} -m ${MODEL_PATH}/${model_name}_${loop_date}.txt \
    30. 

  30. #-dim 1,1,0 -core 8 -im ${MODEL_PATH}/${model_name}_${loop_date_model}.txt
    31. 

  31. #    echo -------save ${MODEL_PATH}/${model_name}_${loop_date}.txt----------
    32. 

  32. #    loop_date=$(date -d "$loop_date + 1 day" +%Y%m%d)
    33. 

  33. #done
    34. 

  34. #
    35. 

  35. ## 3 本轮模型格式转换
    36. 

  36. #cat ${MODEL_PATH}/${model_name}_${yesterday}.txt \
    37. 

  37. #| sed '1d' | awk -F " " '{if($2!="0") print $1"\t"$2}' \
    38. 

  38. #> ${MODEL_PATH}/${model_name}_${yesterday}_change.txt
    39. 

  39. #
    40. 

  40. ## 4 转换后模型上传oss
    41. 

  41. #$HADOOP fs -rm -r oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model/${model_name}_change.txt
    42. 

  42. #$HADOOP fs -put ${MODEL_PATH}/${model_name}_${yesterday}_change.txt ${OSS_PATH}/${model_name}_change.txt
    43.