|
|
@@ -13,8 +13,8 @@ export JAVA_HOME=/usr/lib/jvm/java-1.8.0
|
|
|
# 原始数据table name
|
|
|
table='alg_recsys_sample_all_v2'
|
|
|
# 处理分区配置 推荐数据间隔一天生产，所以5日0点使用3日0-23点数据生产new模型数据
|
|
|
-begin_early_2_Str=20240728
|
|
|
-end_early_2_Str=20240728
|
|
|
+begin_early_2_Str=20240803
|
|
|
+end_early_2_Str=20240803
|
|
|
beginHhStr=00
|
|
|
endHhStr=23
|
|
|
max_hour=05
|
|
|
@@ -37,7 +37,7 @@ echo "$(date +%Y-%m-%d_%H-%M-%S)----------step1------------开始根据${table}
|
|
|
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
|
|
|
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
tablePart:64 repartition:32 \
|
|
|
-beginStr:2024072700 endStr:2024072723 \
|
|
|
+beginStr:${begin_early_2_Str}00 endStr:${end_early_2_Str}09 \
|
|
|
savePath:${originDataPath} \
|
|
|
table:${table} &
|
|
|
|
|
|
@@ -46,7 +46,7 @@ table:${table} &
|
|
|
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
|
|
|
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
tablePart:64 repartition:32 \
|
|
|
-beginStr:2024073000 endStr:2024073023 \
|
|
|
+beginStr:${begin_early_2_Str}10 endStr:${end_early_2_Str}15 \
|
|
|
savePath:${originDataPath} \
|
|
|
table:${table} &
|
|
|
|
|
|
@@ -55,7 +55,7 @@ table:${table} &
|
|
|
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
|
|
|
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
tablePart:64 repartition:32 \
|
|
|
-beginStr:2024073100 endStr:2024073123 \
|
|
|
+beginStr:${begin_early_2_Str}16 endStr:${end_early_2_Str}23 \
|
|
|
savePath:${originDataPath} \
|
|
|
table:${table} &
|
|
|
|
