|
|
@@ -142,23 +142,24 @@ check_ad_hive() {
|
|
|
|
|
|
xgb_train() {
|
|
|
local step_start_time=$(date +%s)
|
|
|
- /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
|
|
|
- --class com.tzld.piaoquan.recommend.model.train_01_xgb_ad_20240808 \
|
|
|
- --master yarn --driver-memory 6G --executor-memory 9G --executor-cores 1 --num-executors 31 \
|
|
|
- --conf spark.yarn.executor.memoryoverhead=1000 \
|
|
|
- --conf spark.shuffle.service.enabled=true \
|
|
|
- --conf spark.shuffle.service.port=7337 \
|
|
|
- --conf spark.shuffle.consolidateFiles=true \
|
|
|
- --conf spark.shuffle.manager=sort \
|
|
|
- --conf spark.storage.memoryFraction=0.4 \
|
|
|
- --conf spark.shuffle.memoryFraction=0.5 \
|
|
|
- --conf spark.default.parallelism=200 \
|
|
|
- /root/zhangbo/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar \
|
|
|
- featureFile:20240703_ad_feature_name.txt \
|
|
|
- trainPath:${train_data_path} \
|
|
|
- testPath:${predict_date_path} \
|
|
|
- savePath:${new_model_predict_result_path} \
|
|
|
- modelPath:${model_save_path} \
|
|
|
+
|
|
|
+ /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
|
|
|
+ --class com.tzld.piaoquan.recommend.model.train_01_xgb_ad_20240808 \
|
|
|
+ --master yarn --driver-memory 6G --executor-memory 9G --executor-cores 1 --num-executors 31 \
|
|
|
+ --conf spark.yarn.executor.memoryoverhead=1000 \
|
|
|
+ --conf spark.shuffle.service.enabled=true \
|
|
|
+ --conf spark.shuffle.service.port=7337 \
|
|
|
+ --conf spark.shuffle.consolidateFiles=true \
|
|
|
+ --conf spark.shuffle.manager=sort \
|
|
|
+ --conf spark.storage.memoryFraction=0.4 \
|
|
|
+ --conf spark.shuffle.memoryFraction=0.5 \
|
|
|
+ --conf spark.default.parallelism=200 \
|
|
|
+ /root/zhangbo/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar \
|
|
|
+ featureFile:20240703_ad_feature_name.txt \
|
|
|
+ trainPath:${train_data_path} \
|
|
|
+ testPath:${predict_date_path} \
|
|
|
+ savePath:${new_model_predict_result_path} \
|
|
|
+ modelPath:${model_save_path} \
|
|
|
eta:0.01 gamma:0.0 max_depth:5 num_round:1000 num_worker:30 repartition:20
|
|
|
|
|
|
local return_code=$?
|
|
|
@@ -180,7 +181,7 @@ model_predict() {
|
|
|
--conf spark.storage.memoryFraction=0.4 \
|
|
|
--conf spark.shuffle.memoryFraction=0.5 \
|
|
|
--conf spark.default.parallelism=200 \
|
|
|
- /root/zhangbo/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar \
|
|
|
+ /root/zhangbo/recommend-model/recommend-model-produce/target/recommend-model-produce-jar-with-dependencies.jar \
|
|
|
featureFile:20240703_ad_feature_name.txt \
|
|
|
testPath:${predict_date_path} \
|
|
|
savePath:${online_model_predict_result_path} \
|
|
|
@@ -223,7 +224,7 @@ model_upload_oss() {
|
|
|
main() {
|
|
|
init
|
|
|
|
|
|
- check_ad_hive
|
|
|
+ # check_ad_hive
|
|
|
|
|
|
xgb_train
|
|
|
|
