all

zhangbo/06_update_everyday_feature.sh

@@ -97,8 +97,19 @@ conda deactivate
 /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata.makedata_09_user2redis_freq \
 --name makedata_09_user2redis_freq \
---master yarn --driver-memory 1G --executor-memory 5G --executor-cores 1 --num-executors 32 \
---conf spark.yarn.executor.memoryoverhead=1024 \
+--master yarn --driver-memory 1G --executor-memory 4G --executor-cores 1 --num-executors 32 \
+--conf spark.yarn.executor.memoryoverhead=2024 \
+--conf spark.shuffle.service.enabled=true \
+--conf spark.shuffle.service.port=7337 \
+--conf spark.shuffle.consolidateFiles=true \
+--conf spark.shuffle.manager=sort \
+--conf spark.storage.memoryFraction=0.4 \
+--conf spark.shuffle.memoryFraction=0.5 \
+--conf spark.default.parallelism=600	\
+--conf spark.speculation=true \
+--conf spark.speculation.multiplier=10 \
+--conf spark.speculation.quantile=0.75	\
+--conf spark.network.timeout=120 \
 /root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 date:${today_early_1} tablePart:64 expireDay:3 ifWriteRedisUser:True ifUser:True midDays:14 redisLimit:100000000 \
 savePathUser:/dw/recommend/model/09_feature/user/