Переглянути джерело

feat:修改补数据脚本

zhaohaipeng 1 місяць тому
батько
коміт
90974ae890
1 змінених файлів з 3 додано та 3 видалено
  1. 3 3
      recommend/00_train_data_make.sh

+ 3 - 3
recommend/00_train_data_make.sh

@@ -7,7 +7,7 @@ for dt in "${dts[@]}"; do
 
     /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
     --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
-    --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
+    --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 10 \
     /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
     tablePart:64 beginStr:${dt}00 endStr:${dt}11 repartition:32 \
     savePath:/dw/recommend/model/41_recsys_origin_date \
@@ -15,7 +15,7 @@ for dt in "${dts[@]}"; do
 
     /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
     --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
-    --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
+    --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 10 \
     /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
     tablePart:64 beginStr:${dt}12 endStr:${dt}17 repartition:32 \
     savePath:/dw/recommend/model/41_recsys_origin_date \
@@ -23,7 +23,7 @@ for dt in "${dts[@]}"; do
 
     /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
     --class com.aliyun.odps.spark.examples.makedata_recsys.v20250218.makedata_recsys_41_originData_20250218 \
-    --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
+    --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 10 \
     /root/zhaohp/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
     tablePart:64 beginStr:${dt}18 endStr:${dt}23 repartition:32 \
     savePath:/dw/recommend/model/41_recsys_origin_date \