recommend-emr-dataprocess/zhangbo/06_update_everyday_feature.sh

#!/bin/sh
set -ex
# 0 全局变量/参数
today="$(date +%Y%m%d)"
today_early_1="$(date -d '1 days ago' +%Y%m%d)"

HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
export PATH=$SPARK_HOME/bin:$PATH
export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
export JAVA_HOME=/usr/lib/jvm/java-1.8.0

max_hour=11
max_minute=00

# 0 判断上游表是否生产完成，最长等待到12点
source /root/anaconda3/bin/activate py37
while true; do
  python_return_code=$(python utils.py --excute_program check_item_hive --partition ${today_early_1})
  if [ $python_return_code -eq 0 ]; then
    echo "Python程序返回0，退出循环。"
    break
  fi
  echo "Python程序返回非0值，等待五分钟后再次调用。"
  sleep 300
  current_hour=$(date +%H)
  current_minute=$(date +%M)
  if (( current_hour > max_hour || (current_hour == max_hour && current_minute >= max_minute) )); then
    echo "最长等待时间已到，失败:${current_hour}-${current_minute}"
    exit 1
  fi
done
conda deactivate
# 1 item 生产数据
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_08_item2redis \
--name makedata_08_item2redis_${today} \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 16 \
/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
date:${today_early_1} tablePart:32 expireDay:4 ifDebug:False \
ifVideo:True ifWriteRedis:True savePathVideo:/dw/recommend/model/09_feature/video

if [ $? -eq 1 ]; then
    echo "---------item写入redis执行失败---------"
    exit 1
else
    echo "---------item写入redis执行成功---------"
fi

# 2 检查user上游表
source /root/anaconda3/bin/activate py37
while true; do
  python_return_code=$(python utils.py --excute_program check_user_hive --partition ${today_early_1})
  if [ $python_return_code -eq 0 ]; then
    echo "Python程序返回0，退出循环。"
    break
  fi
  echo "Python程序返回非0值，等待五分钟后再次调用。"
  sleep 300
  current_hour=$(date +%H)
  current_minute=$(date +%M)
  if (( current_hour > max_hour || (current_hour == max_hour && current_minute >= max_minute) )); then
    echo "最长等待时间已到，失败:${current_hour}-${current_minute}"
    exit 1
  fi
done

# 3 检查mid 时间，上游表
while true; do
  python_return_code=$(python utils.py --excute_program check_hive --partition ${today_early_1} --project loghubods --table mid_uid)
  if [ $python_return_code -eq 0 ]; then
    echo "Python程序返回0，退出循环。上游表loghubods.mid_uid=${today_early_1} 已生产完毕"
    break
  fi
  echo "Python程序返回非0值，等待五分钟后再次调用。上游表loghubods.mid_uid=${today_early_1} 未完成"
  sleep 300
  current_hour=$(date +%H)
  current_minute=$(date +%M)
  if (( current_hour > max_hour || (current_hour == max_hour && current_minute >= max_minute) )); then
    echo "最长等待时间已到，失败:${current_hour}-${current_minute}"
    exit 1
  fi
done

conda deactivate
# 4 user 生产数据
#/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
#--class com.aliyun.odps.spark.examples.makedata.makedata_09_user2redis \
#--name makedata_09_user2redis_${today} \
#--master yarn --driver-memory 1G --executor-memory 4G --executor-cores 1 --num-executors 32 \
#--conf spark.yarn.executor.memoryoverhead=1024 \
#/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
#date:${today_early_1} tablePart:32 expireDay:3 ifDebug:False \
#ifUser:True ifDeleteRedisUser:False ifWriteRedisUser:True sampleRate:1.0 midDays:7 \
#savePathUser:/dw/recommend/model/feature/user/

/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_09_user2redis_freq \
--name makedata_09_user2redis_freq \
--master yarn --driver-memory 1G --executor-memory 4G --executor-cores 1 --num-executors 32 \
--conf spark.yarn.executor.memoryoverhead=2024 \
--conf spark.shuffle.service.enabled=true \
--conf spark.shuffle.service.port=7337 \
--conf spark.shuffle.consolidateFiles=true \
--conf spark.shuffle.manager=sort \
--conf spark.storage.memoryFraction=0.4 \
--conf spark.shuffle.memoryFraction=0.5 \
--conf spark.default.parallelism=400	\
--conf spark.speculation=true \
--conf spark.speculation.multiplier=10 \
--conf spark.speculation.quantile=0.75	\
--conf spark.network.timeout=120 \
/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
date:${today_early_1} tablePart:64 expireDay:3 ifWriteRedisUser:True ifUser:True midDays:14 redisLimit:100000000 \
savePathUser:/dw/recommend/model/09_feature/user/

if [ $? -eq 1 ]; then
    echo "---------user写入redis执行失败---------"
    exit 1
else
    echo "---------user写入redis执行成功---------"
fi

#nohup sh 06_update_everyday_feature.sh > p6.log 2>&1 &