推荐模型自动化更新-增量训练、清理脚本

qiaojialiang/delPredictFile.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+set -x
+
+source /root/anaconda3/bin/activate py37
+
+export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
+export PATH=$SPARK_HOME/bin:$PATH
+export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
+export JAVA_HOME=/usr/lib/jvm/java-1.8.0
+
+today="$(date -d '3 days ago' +%Y%m%d)"
+model_name=model_nba8
+PREDICT_PATH=/root/joe/recommend-emr-dataprocess/predict
+fileName1=${PREDICT_PATH}/${model_name}_${today}_online.txt
+fileName2=${PREDICT_PATH}/${model_name}_${today}_new.txt
+
+
+if [ -f "${fileName1}" ]; then
+    echo "文件 ${fileName1} 存在，正在删除..."
+    # 使用rm命令删除文件
+#    rm "${fileName1}"
+    mv "${fileName1}"  ${PREDICT_PATH}/xxx_online.txt
+    if [ $? -eq 0 ]; then
+        echo "文件 ${fileName1} 已成功删除。"
+    else
+        echo "删除文件 ${fileName1} 时出错。"
+    fi
+else
+    echo "文件 ${fileName1} 不存在。"
+fi
+
+
+if [ -f "${fileName2}" ]; then
+    echo "文件 ${fileName2} 存在，正在删除..."
+    # 使用rm命令删除文件
+#    rm "${fileName2}"
+    mv "${fileName2}"  ${PREDICT_PATH}/xxx_new.txt
+    if [ $? -eq 0 ]; then
+        echo "文件 ${fileName2} 已成功删除。"
+    else
+        echo "删除文件 ${fileName2} 时出错。"
+    fi
+else
+    echo "文件 ${fileName2} 不存在。"
+fi

qiaojialiang/handle_rov.sh

@@ -236,8 +236,9 @@ else
 fi
 
 # 5 模型训练
-echo "$(date +%Y-%m-%d_%H-%M-%S)----------step4------------开始模型训练"
-$HADOOP fs -text ${bucketDataPath}/${begin_early_2_Str}/* | ${FM_HOME}/fm_train -m ${MODEL_PATH}/${model_name}_${begin_early_2_Str}.txt -dim 1,1,8 -im ${LAST_MODEL_HOME}/model_online.txt -core 8
+echo "$(date +%Y-%m-%d_%H-%M-%S)----------step4------------开始模型训练，增量训练：${MODEL_PATH}/${model_name}_${today_early_3}.txt"
+#$HADOOP fs -text ${bucketDataPath}/${begin_early_2_Str}/* | ${FM_HOME}/fm_train -m ${MODEL_PATH}/${model_name}_${begin_early_2_Str}.txt -dim 1,1,8 -im ${LAST_MODEL_HOME}/model_online.txt -core 8
+$HADOOP fs -text ${bucketDataPath}/${begin_early_2_Str}/* | ${FM_HOME}/fm_train -m ${MODEL_PATH}/${model_name}_${begin_early_2_Str}.txt -dim 1,1,8 -im ${MODEL_PATH}/${model_name}_${today_early_3}.txt -core 8
 if [ $? -ne 0 ]; then
    echo "模型训练失败"
    /root/anaconda3/bin/python monitor_util.py --level error --msg "荐模型数据更新 \n【任务名称】:step5模型训练\n【是否成功】:error\n【信息】:${bucketDataPath}/${begin_early_2_Str}训练失败"