recommend-emr-dataprocess/ad_display/pipline/eval_qq.sh

#!/bin/sh

# env
export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
export JAVA_HOME=/usr/lib/jvm/java-1.8.0

# params
predictPath=/dw/recommend/model/ad_display/eval
#bucketNum=200
bucketNum=2000
savePath=/dw/recommend/model/ad_display/qq

echo `date` "stat qq"
set -x
/opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
--class com.tzld.piaoquan.recommend.model.stat_qq \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 4 \
--conf spark.yarn.executor.memoryoverhead=1024 \
--conf spark.shuffle.service.enabled=true \
--conf spark.shuffle.service.port=7337 \
--conf spark.shuffle.consolidateFiles=true \
--conf spark.shuffle.manager=sort \
--conf spark.storage.memoryFraction=0.4 \
--conf spark.shuffle.memoryFraction=0.5 \
--conf spark.default.parallelism=200 \
--conf spark.debug.maxToStringFields=100 \
/mnt/disk1/jch/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar \
predictPath:${predictPath} \
bucketNum:${bucketNum} \
savePath:${savePath} \