feat:添加新的特征表监控

recommend/00_train_data_make.sh

@@ -1,13 +1,22 @@
 #!/bin/sh
 set -x
 
-dts=('20250307' '20250308' '20250309' '20250310')
+export PATH=$SPARK_HOME/bin:$PATH
+export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
+export JAVA_HOME=/usr/lib/jvm/java-1.8.0
+export PREDICT_CACHE_PATH=/root/zhaohp/XGB/predict_cache/
+export SEGMENT_BASE_PATH=/dw/recommend/model/36_model_attachment/score_calibration_file
+
+HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
+
+
+dts=('20250309' '20250310')
 for dt in "${dts[@]}"; do
     echo "开始处理: ${dt}"
 
     /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
     --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
-    --master yarn --driver-memory 2G --executor-memory 5G --executor-cores 1 --num-executors 10 \
+    --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 10 \
     --conf spark.yarn.executor.memoryOverhead=2G \
     /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
     tablePart:64 beginStr:${dt}00 endStr:${dt}11 repartition:32 \
@@ -16,7 +25,7 @@ for dt in "${dts[@]}"; do
 
     /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
     --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
-    --master yarn --driver-memory 2G --executor-memory 5G --executor-cores 1 --num-executors 10 \
+    --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 10 \
     --conf spark.yarn.executor.memoryOverhead=2G \
     /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
     tablePart:64 beginStr:${dt}12 endStr:${dt}17 repartition:32 \
@@ -25,7 +34,7 @@ for dt in "${dts[@]}"; do
 
     /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
     --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
-    --master yarn --driver-memory 2G --executor-memory 5G --executor-cores 1 --num-executors 10 \
+    --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 10 \
     --conf spark.yarn.executor.memoryOverhead=2G \
     /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
     tablePart:64 beginStr:${dt}18 endStr:${dt}23 repartition:32 \
@@ -48,7 +57,7 @@ for dt in "${dts[@]}"; do
     # STR负样本采样
     /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
     --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_str_train_data_20250218 \
-    --master yarn --driver-memory 4G --executor-memory 8G --executor-cores 1 --num-executors 20 \
+    --master yarn --driver-memory 4G --executor-memory 5G --executor-cores 1 --num-executors 12 \
     --conf spark.driver.maxResultSize=2g \
     ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
     readPath:/dw/recommend/model/41_recsys_origin_date/${dt}*/* \
@@ -58,4 +67,8 @@ for dt in "${dts[@]}"; do
     wait
     echo "${dt} 负样本采样完成"
 
+    # 删除原始特征，释放空间
+    ${HADOOP} fs -rm -r /dw/recommend/model/41_recsys_origin_date/${dt}*
+    echo "删除 ${dt} 的原始特征数据"
+
 done