feat:添加分桶脚本

recommend/00_train_data_make.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+set -x
+
+dts=('20250222' '20250223' '20250224')
+for dt in "${dts[@]}"; do
+    echo "开始处理: ${dt}"
+
+    /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+    --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
+    --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
+    /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+    tablePart:64 beginStr:${dt}00 endStr:${dt}11 repartition:32 \
+    savePath:/dw/recommend/model/41_recsys_origin_date \
+    table:dwd_recsys_alg_sample_all_20250212 &
+
+    /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+    --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
+    --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
+    /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+    tablePart:64 beginStr:${dt}12 endStr:${dt}17 repartition:32 \
+    savePath:/dw/recommend/model/41_recsys_origin_date \
+    table:dwd_recsys_alg_sample_all_20250212 &
+
+    /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+    --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
+    --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
+    /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+    tablePart:64 beginStr:${dt}18 endStr:${dt}23 repartition:32 \
+    savePath:/dw/recommend/model/41_recsys_origin_date \
+    table:dwd_recsys_alg_sample_all_20250212 &
+
+    wait 
+    echo "${dt} 原始特征生产完成"
+
+    /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+    --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_data_fu_sample_20250218 \
+    --master yarn --driver-memory 6G --executor-memory 8G --executor-cores 1 --num-executors 30 \
+    --conf spark.driver.maxResultSize=2g \
+    /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+    readPath:/dw/recommend/model/41_recsys_origin_date/${dt}*/* \
+    savePath:/dw/recommend/model/41_recsys_sample_data/${dt} \
+    fuSampleRate:0.05 whatLabel:is_share repartition:64
+
+    echo "${dt} 负样本采样完成"
+
+    /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+    --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_43_bucketData_20250218 \
+    --master yarn --driver-memory 4G --executor-memory 8G --executor-cores 1 --num-executors 16 \
+    /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+    readPath:/dw/recommend/model/41_recsys_sample_data/ \
+    savePath:/dw/recommend/model/43_recsys_train_data_20250218/ \
+    beginStr:${dt} endStr:${dt} whatLabel:is_share fileName:20250218_bucket_322.txt &
+done