recommend-emr-dataprocess/recommend/22_supplementary_data_new_table.sh

#!/bin/sh
set -x


export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
export PATH=$SPARK_HOME/bin:$PATH
export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
export JAVA_HOME=/usr/lib/jvm/java-1.8.0

source /root/anaconda3/bin/activate py37

# 原始数据table name
table='alg_recsys_sample_all_v2'
# 各节点产出hdfs文件绝对路径
# 源数据文件
originDataPath=/dw/recommend/model/41_recsys_sample_data_new_table/
# 特征分桶
bucketDataPath=/dw/recommend/model/43_recsys_train_data_new_table/
# hadoop
HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop

# 1 生产原始数据
echo "$(date +%Y-%m-%d_%H-%M-%S)----------step1------------开始根据${table}生产原始数据"

/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_41_originData_20240709 \
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
tablePart:64 repartition:32 \
beginStr:2024103114 endStr:2024110119 \
savePath:${originDataPath} \
table:${table} &

/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_41_originData_20240709 \
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
tablePart:64 repartition:32 \
beginStr:2024110120 endStr:2024110223 \
savePath:${originDataPath} \
table:${table} &


wait
if [ $? -ne 0 ]; then
   echo "Spark原始样本生产任务执行失败"
   exit 1
else
   echo "spark原始样本生产执行成功"
fi

# 特征采样分桶
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_fu_sample_20240709 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
readPath:/dw/recommend/model/41_recsys_sample_data_new_table \
savePath:/dw/recommend/model/43_recsys_train_data_new_table_274_sample_01 \
beginStr:20241031 endStr:20241102 repartition:500 \
filterNames:ROS fuSampleRate:0.1 \
fileName:20240609_bucket_314.txt \
whatLabel:is_return whatApps:0,3,4,21,17

if [ $? -ne 0 ]; then
   echo "Spark特征分桶任务执行失败"
   exit 1
else
   echo "spark特征分桶任务执行成功"
fi


# 定时任务配置
# 0 11 * * * cd /root/zhaohp/recommend-emr-dataprocess && /bin/sh ./recommend/data_new_table.sh > logs/recommend/data_new_table/$(date +\%Y\%m\%d\%H\%M).log 2>&1