|
|
@@ -13,6 +13,25 @@ FM_TRAIN="/root/sunmingze/alphaFM/bin/fm_train"
|
|
|
MODEL_PATH="/root/zhangbo/recommend-emr-dataprocess/zhangbo/model/"
|
|
|
OSS_PATH="oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model/"
|
|
|
|
|
|
+# 0 判断上游表是否生产完成，最长等待到12点
|
|
|
+max_hour=10
|
|
|
+max_minute=50
|
|
|
+while true; do
|
|
|
+ current_hour=$(date +%H)
|
|
|
+ current_minute=$(date +%M)
|
|
|
+ if (( current_hour > max_hour || (current_hour == max_hour && current_minute >= max_minute) )); then
|
|
|
+ echo "最长等待时间已到，失败。"
|
|
|
+ exit 1
|
|
|
+ fi
|
|
|
+ python_return_code=$(python utils.py --excute_program check_origin_hive --partition ${today_early_1})
|
|
|
+ if [ $python_return_code -eq 0 ]; then
|
|
|
+ echo "Python程序返回0，退出循环。"
|
|
|
+ break
|
|
|
+ fi
|
|
|
+ echo "Python程序返回非0值，等待五分钟后再次调用。"
|
|
|
+ sleep 300
|
|
|
+done
|
|
|
+
|
|
|
# 1 生产数据
|
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
|
--class com.aliyun.odps.spark.examples.makedata.makedata_06_originData \
|
zhangbo