|
|
@@ -97,8 +97,19 @@ conda deactivate
|
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
|
--class com.aliyun.odps.spark.examples.makedata.makedata_09_user2redis_freq \
|
|
|
--name makedata_09_user2redis_freq \
|
|
|
---master yarn --driver-memory 1G --executor-memory 5G --executor-cores 1 --num-executors 32 \
|
|
|
---conf spark.yarn.executor.memoryoverhead=1024 \
|
|
|
+--master yarn --driver-memory 1G --executor-memory 4G --executor-cores 1 --num-executors 32 \
|
|
|
+--conf spark.yarn.executor.memoryoverhead=2024 \
|
|
|
+--conf spark.shuffle.service.enabled=true \
|
|
|
+--conf spark.shuffle.service.port=7337 \
|
|
|
+--conf spark.shuffle.consolidateFiles=true \
|
|
|
+--conf spark.shuffle.manager=sort \
|
|
|
+--conf spark.storage.memoryFraction=0.4 \
|
|
|
+--conf spark.shuffle.memoryFraction=0.5 \
|
|
|
+--conf spark.default.parallelism=600 \
|
|
|
+--conf spark.speculation=true \
|
|
|
+--conf spark.speculation.multiplier=10 \
|
|
|
+--conf spark.speculation.quantile=0.75 \
|
|
|
+--conf spark.network.timeout=120 \
|
|
|
/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
date:${today_early_1} tablePart:64 expireDay:3 ifWriteRedisUser:True ifUser:True midDays:14 redisLimit:100000000 \
|
|
|
savePathUser:/dw/recommend/model/09_feature/user/
|
zhangbo