|
|
@@ -56,7 +56,7 @@ echo "$(date +%Y-%m-%d_%H-%M-%S)----------step1------------开始根据${table}
|
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
|
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_41_originData_20240709 \
|
|
|
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
|
|
|
-../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
+./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
tablePart:64 repartition:32 \
|
|
|
beginStr:${begin_early_2_Str}00 endStr:${end_early_2_Str}09 \
|
|
|
savePath:${originDataPath} \
|
|
|
@@ -65,7 +65,7 @@ table:${table} &
|
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
|
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_41_originData_20240709 \
|
|
|
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
|
|
|
-../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
+./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
tablePart:64 repartition:32 \
|
|
|
beginStr:${begin_early_2_Str}10 endStr:${end_early_2_Str}15 \
|
|
|
savePath:${originDataPath} \
|
|
|
@@ -74,7 +74,7 @@ table:${table} &
|
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
|
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_41_originData_20240709 \
|
|
|
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
|
|
|
-../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
+./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
tablePart:64 repartition:32 \
|
|
|
beginStr:${begin_early_2_Str}16 endStr:${end_early_2_Str}23 \
|
|
|
savePath:${originDataPath} \
|
|
|
@@ -93,7 +93,7 @@ echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生
|
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
|
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
|
|
|
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
|
|
|
-../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
+./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
readPath:${originDataPath} \
|
|
|
savePath:${bucketDataPath} \
|
|
|
beginStr:${begin_early_2_Str} endStr:${end_early_2_Str} repartition:500 \
|
