recommend-emr-dataprocess/recommend/00_train_data_make.sh

#!/bin/sh
set -x

dts=('20250222' '20250223' '20250224')
for dt in "${dts[@]}"; do
    echo "开始处理: ${dt}"

    /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
    --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
    --master yarn --driver-memory 2G --executor-memory 5G --executor-cores 1 --num-executors 12 \
    --conf spark.yarn.executor.memoryOverhead=2G \
    /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
    tablePart:64 beginStr:${dt}00 endStr:${dt}11 repartition:32 \
    savePath:/dw/recommend/model/41_recsys_origin_date \
    table:dwd_recsys_alg_sample_all_20250212 &

    /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
    --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
    --master yarn --driver-memory 2G --executor-memory 5G --executor-cores 1 --num-executors 12 \
    --conf spark.yarn.executor.memoryOverhead=2G \
    /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
    tablePart:64 beginStr:${dt}12 endStr:${dt}17 repartition:32 \
    savePath:/dw/recommend/model/41_recsys_origin_date \
    table:dwd_recsys_alg_sample_all_20250212 &

    /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
    --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
    --master yarn --driver-memory 2G --executor-memory 5G --executor-cores 1 --num-executors 12 \
    --conf spark.yarn.executor.memoryOverhead=2G \
    /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
    tablePart:64 beginStr:${dt}18 endStr:${dt}23 repartition:32 \
    savePath:/dw/recommend/model/41_recsys_origin_date \
    table:dwd_recsys_alg_sample_all_20250212 &

    wait 
    echo "${dt} 原始特征生产完成"

    /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
    --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_data_fu_sample_20250218 \
    --master yarn --driver-memory 6G --executor-memory 8G --executor-cores 1 --num-executors 30 \
    --conf spark.driver.maxResultSize=2g \
    /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
    readPath:/dw/recommend/model/41_recsys_origin_date/${dt}*/* \
    savePath:/dw/recommend/model/41_recsys_sample_data/${dt} \
    fuSampleRate:0.05 whatLabel:is_share repartition:64

    echo "${dt} 负样本采样完成"

    /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
    --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_43_bucketData_20250218 \
    --master yarn --driver-memory 4G --executor-memory 8G --executor-cores 1 --num-executors 16 \
    /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
    readPath:/dw/recommend/model/41_recsys_sample_data/ \
    savePath:/dw/recommend/model/43_recsys_train_data_20250218/ \
    beginStr:${dt} endStr:${dt} whatLabel:is_share fileName:20250218_bucket_322.txt &
done