recommend-emr-dataprocess/qiaojialiang/change_oss.sh

#!/bin/sh
set -x

source /root/anaconda3/bin/activate py37

export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
export PATH=$SPARK_HOME/bin:$PATH
export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
export JAVA_HOME=/usr/lib/jvm/java-1.8.0

#  nohup sh handle_rov.sh > "$(date +%Y%m%d_%H%M%S)_handle_rov.log" 2>&1 &

# 原始数据table name
table='alg_recsys_sample_all'
today="$(date +%Y%m%d)"
today_early_3="$(date -d '3 days ago' +%Y%m%d)"
#table='alg_recsys_sample_all_test'
# 处理分区配置 推荐数据间隔一天生产，所以5日0点使用3日0-23点数据生产new模型数据
begin_early_2_Str="$(date -d '2 days ago' +%Y%m%d)"
end_early_2_Str="$(date -d '2 days ago' +%Y%m%d)"
beginHhStr=00
endHhStr=23
max_hour=05
max_minute=00
# 各节点产出hdfs文件绝对路径
# 源数据文件
originDataPath=/dw/recommend/model/41_recsys_sample_data/
# 特征值
valueDataPath=/dw/recommend/model/14_feature_data/
# 特征分桶
bucketDataPath=/dw/recommend/model/43_recsys_train_data/
# 模型数据路径
MODEL_PATH=/root/joe/recommend-emr-dataprocess/model
# 预测路径
PREDICT_PATH=/root/joe/recommend-emr-dataprocess/predict
# 历史线上正在使用的模型数据路径
LAST_MODEL_HOME=/root/joe/model_online
# 模型数据文件前缀
model_name=model_nba8
# fm模型
FM_HOME=/root/sunmingze/alphaFM/bin
# hadoop
HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/


cat /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_22.txt |
  awk -F " " '{
      if (NR == 1) {
          print $1"\t"$2
      } else {
          split($0, fields, " ");
          OFS="\t";
          line=""
          for (i = 1; i <= 10 && i <= length(fields); i++) {
              line = (line ? line "\t" : "") fields[i];
          }
              print line
          }
      }' > /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_22_change.txt
if [ $? -ne 0 ]; then
   echo "新模型文件格式转换失败"
fi
# 4.1.2 模型文件上传OSS
online_model_path=${OSS_PATH}/${model_name}.txt
$HADOOP fs -test -e ${online_model_path}
if [ $? -eq 0 ]; then
    echo "数据存在, 先删除。"
    $HADOOP fs -rm -r -skipTrash ${online_model_path}
else
    echo "数据不存在"
fi
$HADOOP fs -put /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_22_change.txt ${online_model_path}
if [ $? -eq 0 ]; then
    echo "推荐模型文件至OSS成功"
    # 4.1.3 本地保存最新的线上使用的模型，用于下一次的AUC验证
    cp -f ${LAST_MODEL_HOME}/model_online.txt ${LAST_MODEL_HOME}/model_online_$(date +\%Y\%m\%d).txt
    cp -f /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_22.txt ${LAST_MODEL_HOME}/model_online.txt
    if [ $? -ne 0 ]; then
        echo "模型备份失败"
    fi
    /root/anaconda3/bin/python monitor_util.py --level info --msg "荐模型数据更新 \n【任务名称】:step模型更新\n【是否成功】:success\n【信息】:已更新/root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_22.txt模型}"
else
    echo "推荐模型文件至OSS失败"
    /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step模型推送oss\n【是否成功】:error\n【信息】:推荐模型文件至OSS失败/root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_22.txt --- ${online_model_path}"
fi