recommend-emr-dataprocess/scripts/nor/nor_sample.sh

#!/bin/sh
set -x

export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
export PATH=$SPARK_HOME/bin:$PATH
export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
export JAVA_HOME=/usr/lib/jvm/java-1.8.0

# params
data_date=$(date +%Y%m%d -d "-2 $days day")
start_date=${data_date}
end_date=${data_date}
sampleRate=-1
readPath=/dw/recommend/model/82_origin_data/
savePath=/dw/recommend/model/82_recsys_nor_train_data/

echo `date` "nor sample"

/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys_r_rate.makedata_recsys_82_nor_sample_20250221 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
/mnt/disk1/jch/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-jar-with-dependencies.jar \
readPath:${readPath} \
beginStr:${start_date} endStr:${end_date} \
whatApps:0,3,4,21,17 \
whatLabel:return_n_uv_noself \
fuSampleRate:${sampleRate} \
notUseBucket:1 \
featureName:20250303_recsys_nor_name.txt \
featureBucket:20250303_recsys_nor_bucket.txt \
repartition:8 \
savePath:${savePath} \