all

zhangbo/05_update_everyday_ros.sh

@@ -15,15 +15,15 @@ MODEL_PATH="/root/zhangbo/recommend-emr-dataprocess/zhangbo/model/"
 OSS_PATH="oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model"
 
 # 1 生产数据
-#/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
-#--class com.aliyun.odps.spark.examples.makedata.makedata_01_readtable2hdfs \
-#--master yarn --driver-memory 1G --executor-memory 4G --executor-cores 2 --num-executors 16 \
-#/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-#tablePart:32 savePath:${savePath} beginStr:${yesterday} endStr:${yesterday}
-#
-#if [ $? -eq 1 ]; then
-#    echo "Spark任务执行失败"
-#    exit  1
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata.makedata_01_readtable2hdfs \
+--master yarn --driver-memory 1G --executor-memory 4G --executor-cores 2 --num-executors 16 \
+/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+tablePart:32 savePath:${savePath} beginStr:${yesterday} endStr:${yesterday}
+
+if [ $? -eq 1 ]; then
+    echo "Spark任务执行失败"
+    exit
 
 # 2 加载上次模型 训练本轮数据 保存本轮模型
 end_date=${today}
@@ -33,6 +33,9 @@ while [[ "$loop_date" != "$end_date" ]]; do
     loop_date_model=$(date -d "$loop_date - 1 day" +%Y%m%d)
     $HADOOP fs -text ${savePath}/dt=${loop_date}/* | ${FM_TRAIN} -m ${MODEL_PATH}/${model_name}_${loop_date}.txt \
 -dim 1,1,0 -core 8 -im ${MODEL_PATH}/${model_name}_${loop_date_model}.txt
+    if [ $? -eq 1 ]; then
+        echo "训练失败"
+        exit
     echo -------save ${MODEL_PATH}/${model_name}_${loop_date}.txt----------
     loop_date=$(date -d "$loop_date + 1 day" +%Y%m%d)
 done