feat:广告算法抛弃8点之前的数据

ad/01_ad_model_update_everyday.sh

@@ -9,21 +9,19 @@ today="$(date +%Y%m%d)"
 today_early_1="$(date -d '1 days ago' +%Y%m%d)"
 beginTime=08
 endTime=23
-beginStr=${today_early_1}${beginTime}
-endStr=${today_early_1}${endTime}
 
 MODEL_PATH=/root/zhaohp/recommend-emr-dataprocess/model
 HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
 FM_HOME=/root/sunmingze/alphaFM
 OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/ad_model/
+max_hour=16
+max_minute=00
 
 
 # 1 判断依赖的数据表是否生产完成
 source /root/anaconda3/bin/activate py37
-max_hour=15
-max_minute=00
 while true; do
-  python_return_code=$(python ad/ad_utils.py --excute_program check_ad_origin_hive --partition ${today_early_1} --hh ${endTime})
+  python_return_code=$(python ad/ad_utils.py --excute_program check_ad_origin_hive --partition ${today} --hh 10)
   if [ $python_return_code -eq 0 ]; then
     echo "Python程序返回0，退出循环。"
     break
@@ -34,7 +32,7 @@ while true; do
   current_minute=$(date +%M)
   if (( current_hour > max_hour || (current_hour == max_hour && current_minute >= max_minute) )); then
     echo "最长等待时间已到，失败:${current_hour}-${current_minute}"
-    msg="广告特征数据校验失败，大数据分区没有数据: ${today_early_1}${endTime}"
+    msg="广告特征数据校验失败，大数据分区没有数据: ${today}10"
     /root/anaconda3/bin/python ad/ad_monitor_util.py ${msg}
     exit 1
   fi
@@ -47,7 +45,7 @@ done
 --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 tablePart:64 repartition:32 \
-beginStr:20240624 endStr:20240624 \
+beginStr:${today_early_1}08 endStr:${today}10 \
 savePath:${originDataSavePath} \
 table:alg_recsys_ad_sample_all_new
 if [ $? -ne 0 ]; then
@@ -65,7 +63,7 @@ fi
 --class com.aliyun.odps.spark.zhp.makedata_ad.makedata_ad_33_bucketData_20240622 \
 --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-beginStr:${today_early_1} endStr:${today_early_1} repartition:400 \
+beginStr:${today_early_1} endStr:${today} repartition:400 \
 filterNames:XXXXX \
 bucketFileName:20240620_ad_bucket_249_fix.txt \
 readPath:${originDataSavePath} \
@@ -90,10 +88,7 @@ fi
 
 
 # 5 对比AUC
-# 5.1 生成今天08-10的原始特征数据
-
-
-
+# 5.1 校验今天10分区的数据是否生产完成
 
 
 

src/main/scala/com/aliyun/odps/spark/zhp/makedata_ad/makedata_ad_31_originData_20240620.scala

@@ -36,8 +36,14 @@ object makedata_ad_31_originData_20240620 {
     // 2 读取odps+表信息
     val odpsOps = env.getODPS(sc)
 
-    // 3 循环执行数据生产
+    val noDateSet = Set("01", "02", "03", "04", "05", "06", "07")
+
+    // 3 循环执行数据生产，8点之前的分区抛弃
     val timeRange = MyDateUtils.getDateHourRange(beginStr, endStr)
+      .filter(item => {
+        val hh =item.substring(8, 10)
+        !noDateSet.contains(hh)
+      })
     for (dt_hh <- timeRange) {
       val dt = dt_hh.substring(0, 8)
       val hh = dt_hh.substring(8, 10)