feat:修改定时脚本

zhangbo/07_ad_model_update_everyday.sh

@@ -2,8 +2,8 @@
 set -ex
 
 # 0 全局变量/参数
-originDataSavePath=/dw/recommend/model/31_ad_sample_data_fix/
-bucketFeatureSavePath=/dw/recommend/model/12_ros_data_v3/
+originDataSavePath=/dw/recommend/model/31_ad_sample_data_auto/
+bucketFeatureSavePath=/dw/recommend/model/33_ad_train_data_nosparse_auto/
 model_name=ad_model_lr
 today="$(date +%Y%m%d)"
 today_early_1="$(date -d '1 days ago' +%Y%m%d)"
@@ -18,7 +18,7 @@ source /root/anaconda3/bin/activate py37
 max_hour=15
 max_minute=00
 while true; do
-  python_return_code=$(python utils.py --excute_program check_ad_origin_hive --partition ${today_early_1} --hh ${endTime})
+  python_return_code=$(python zhangbo/utils.py --excute_program check_ad_origin_hive --partition ${today_early_1} --hh ${endTime})
   if [ $python_return_code -eq 0 ]; then
     echo "Python程序返回0，退出循环。"
     break
@@ -36,34 +36,38 @@ done
 
 
 # 2 原始特征生成
-# /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
-# --class com.aliyun.odps.spark.examples.makedata_ad.makedata_ad_31_originData_20240620 \
-# --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
-# ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-# tablePart:64 repartition:32 \
-# beginStr:${beginStr} endStr:${endStr} \
-# savePath:/dw/recommend/model/31_ad_sample_data_auto/ \
-# table:alg_recsys_ad_sample_all_new
-# if [ $? -eq 1 ]; then
-#    echo "Spark原始样本生产任务执行失败"
-#    exit 1
-# else
-#     echo "spark原始样本生产执行成功"
-# fi
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.zhp.makedata_ad.makedata_ad_31_originData_20240620 \
+--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
+./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+tablePart:64 repartition:32 \
+beginStr:${beginStr} endStr:${endStr} \
+savePath:${originDataSavePath} \
+table:alg_recsys_ad_sample_all_new
+if [ $? -ne 0 ]; then
+   echo "Spark原始样本生产任务执行失败"
+   exit 1
+else
+    echo "spark原始样本生产执行成功"
+fi
 
 
 # 3 特征分桶
-# /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
-# --class com.aliyun.odps.spark.examples.makedata_ad.makedata_ad_33_bucketData_20240622 \
-# --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
-# ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
-# beginStr:${today_early_1} endStr:${today_early_1} repartition:400
-# if [ $? -eq 1 ]; then
-#     echo "Spark特征分桶任务执行失败"
-#    exit 1
-# else
-#     echo "spark特征分桶执行失败"
-# fi
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.zhp.makedata_ad.makedata_ad_33_bucketData_20240622 \
+--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
+./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+beginStr:${today_early_1} endStr:${today_early_1} repartition:400 \
+filterNames:XXXXX \
+bucketFileName:20240620_ad_bucket_249_fix.txt \
+readPath:${originDataSavePath} \
+savePath:${bucketFeatureSavePath}
+if [ $? -ne 0 ]: then
+   echo "Spark特征分桶处理任务执行失败"
+   exit 1
+else
+   echo "spark特征分桶处理执行成功"
+fi
 
 
 # 4 模型训练