| 123456789101112131415161718192021222324252627282930313233343536373839404142 |
- #!/bin/sh
- set -ex
- #partitionDt="$(date -d '1 days ago' +%Y%m%d)"
- #partitionHh="14"
- table='alg_recsys_sample_all_test'
- beginStr=2024061600
- endStr=2024061623
- # 0 判断上游表是否生产完成，最长等待到12点
- source /root/anaconda3/bin/activate py37
- max_hour=11
- max_minute=00
- echo "开始校验是否生产完数据，分区信息：beginStr:${beginStr},endStr:${endStr}"
- while true; do
- python_return_code=$(python checkHiveDataUtil.py --table ${table} --beginStr ${beginStr} --endStr ${endStr})
- echo "python 返回值：${python_return_code}"
- if [ $python_return_code -eq 0 ]; then
- echo "Python程序返回0，校验存在数据，退出循环。"
- break
- fi
- echo "Python程序返回非0值，不存在数据，等待五分钟后再次调用。"
- sleep 300
- current_hour=$(date +%H)
- current_minute=$(date +%M)
- if (( current_hour > max_hour || (current_hour == max_hour && current_minute >= max_minute) )); then
- echo "最长等待时间已到，失败:${current_hour}-${current_minute}"
- exit 1
- fi
- done
- # 1 生产原始数据
- #nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
- #--class com.aliyun.odps.spark.examples.makedata.makedata_13_originData_20240529 \
- #--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
- #./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
- #tablePart:64 repartition:32 \
- #beginStr:${beginStr} endStr:${endStr} \
- #savePath:/dw/recommend/model/13_sample_data/ \
- #table:${beginStr} \
- #> p13_2024061600.log 2>&1 &
|
