#!/bin/sh set -x source /root/anaconda3/bin/activate py37 export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8 export PATH=$SPARK_HOME/bin:$PATH export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf export JAVA_HOME=/usr/lib/jvm/java-1.8.0 # nohup sh handle_rov.sh > "$(date +%Y%m%d_%H%M%S)_handle_rov.log" 2>&1 & # 原始数据table name table='alg_recsys_sample_all' today="$(date +%Y%m%d)" today_early_3="$(date -d '3 days ago' +%Y%m%d)" #table='alg_recsys_sample_all_test' # 处理分区配置 推荐数据间隔一天生产,所以5日0点使用3日0-23点数据生产new模型数据 begin_early_2_Str="$(date -d '2 days ago' +%Y%m%d)" end_early_2_Str="$(date -d '2 days ago' +%Y%m%d)" beginHhStr=00 endHhStr=23 max_hour=05 max_minute=00 # 各节点产出hdfs文件绝对路径 # 源数据文件 originDataPath=/dw/recommend/model/41_recsys_sample_data/ # 特征值 valueDataPath=/dw/recommend/model/14_feature_data/ # 特征分桶 bucketDataPath=/dw/recommend/model/43_recsys_train_data/ # 模型数据路径 MODEL_PATH=/root/joe/recommend-emr-dataprocess/model # 预测路径 PREDICT_PATH=/root/joe/recommend-emr-dataprocess/predict # 历史线上正在使用的模型数据路径 LAST_MODEL_HOME=/root/joe/model_online # 模型数据文件前缀 model_name=model_nba8 # fm模型 FM_HOME=/root/sunmingze/alphaFM/bin # hadoop HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/ $HADOOP fs -text ${bucketDataPath}/20240713/* | ${FM_HOME}/fm_train -m ${MODEL_PATH}/${model_name}_0709_0713.txt -dim 1,1,8 -im /root/joe/recommend-emr-dataprocess/model/model_nba8_0709_0712.txt -core 8 $HADOOP fs -text ${bucketDataPath}/20240714/* | ${FM_HOME}/fm_predict -m ${MODEL_PATH}/${model_name}_0709_0713.txt -dim 8 -core 8 -out ${PREDICT_PATH}/${model_name}_0709_0713_new.txt new_auc=`cat ${PREDICT_PATH}/${model_name}_0709_0713_new.txt | /root/sunmingze/AUC/AUC` echo "0709-0713 auc:${new_auc}"