|
@@ -7,19 +7,28 @@ for dt in "${dts[@]}"; do
|
|
|
|
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
--class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
|
|
--class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
|
|
- --master yarn --driver-memory 1G --executor-memory 4G --executor-cores 1 --num-executors 10 \
|
|
|
|
|
|
+ --master yarn --driver-memory 2G --executor-memory 5G --executor-cores 1 --num-executors 12 \
|
|
--conf spark.yarn.executor.memoryOverhead=2G \
|
|
--conf spark.yarn.executor.memoryOverhead=2G \
|
|
/root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
/root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
- tablePart:64 beginStr:${dt}00 endStr:${dt}15 repartition:32 \
|
|
|
|
|
|
+ tablePart:64 beginStr:${dt}00 endStr:${dt}11 repartition:32 \
|
|
savePath:/dw/recommend/model/41_recsys_origin_date \
|
|
savePath:/dw/recommend/model/41_recsys_origin_date \
|
|
table:dwd_recsys_alg_sample_all_20250212 &
|
|
table:dwd_recsys_alg_sample_all_20250212 &
|
|
|
|
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
--class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
|
|
--class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
|
|
- --master yarn --driver-memory 1G --executor-memory 4G --executor-cores 1 --num-executors 10 \
|
|
|
|
|
|
+ --master yarn --driver-memory 2G --executor-memory 5G --executor-cores 1 --num-executors 12 \
|
|
--conf spark.yarn.executor.memoryOverhead=2G \
|
|
--conf spark.yarn.executor.memoryOverhead=2G \
|
|
/root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
/root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
- tablePart:64 beginStr:${dt}16 endStr:${dt}23 repartition:32 \
|
|
|
|
|
|
+ tablePart:64 beginStr:${dt}12 endStr:${dt}17 repartition:32 \
|
|
|
|
+ savePath:/dw/recommend/model/41_recsys_origin_date \
|
|
|
|
+ table:dwd_recsys_alg_sample_all_20250212 &
|
|
|
|
+
|
|
|
|
+ /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
|
|
+ --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
|
|
|
|
+ --master yarn --driver-memory 2G --executor-memory 5G --executor-cores 1 --num-executors 12 \
|
|
|
|
+ --conf spark.yarn.executor.memoryOverhead=2G \
|
|
|
|
+ /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
|
+ tablePart:64 beginStr:${dt}18 endStr:${dt}23 repartition:32 \
|
|
savePath:/dw/recommend/model/41_recsys_origin_date \
|
|
savePath:/dw/recommend/model/41_recsys_origin_date \
|
|
table:dwd_recsys_alg_sample_all_20250212 &
|
|
table:dwd_recsys_alg_sample_all_20250212 &
|
|
|
|
|