scala pred

01-脚本记录

@@ -86,9 +86,11 @@ savePath:/dw/recommend/model/34_ad_predict_data/case_tmp/ \
 modelPath:/dw/recommend/model/35_ad_model/model_xgb_1000 \
 > p5.log 2>&1 &
 
+-------------------------------预测-----------------------------------------
+
 nohup /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
 --class com.tzld.piaoquan.recommend.model.pred_01_xgb_ad_hdfsfile_20240813 \
---master yarn --driver-memory 5G --executor-memory 5G --executor-cores 1 --num-executors 30 \
+--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 30 \
 --conf spark.yarn.executor.memoryoverhead=1024 \
 --conf spark.shuffle.service.enabled=true \
 --conf spark.shuffle.service.port=7337 \
@@ -99,11 +101,14 @@ nohup /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.sp
 --conf spark.default.parallelism=200 \
 ./target/recommend-model-produce-jar-with-dependencies.jar \
 featureFile:20240809_ad_feature_name_517.txt \
-testPath:/dw/recommend/model/33_ad_train_data_v4/20240814/ \
-savePath:/dw/recommend/model/34_ad_predict_data/20240814_v1/ \
-modelPath:/dw/recommend/model/35_ad_model/model_xgb_1000 \
-> p1_pred_20240814.log 2>&1 &
+testPath:/dw/recommend/model/33_ad_train_data_v4/20240815/ \
+savePath:/dw/recommend/model/34_ad_predict_data/20240815_new2/ \
+modelPath:/dw/recommend/model/35_ad_model/model_xgb_7day_2000 \
+> p1_pred_20240815_new2.log 2>&1 &
 
+cat p1_pred_20240815_new2.log | grep -E "^3353|^3606|^2670|^3585|^2912|^3869|^3857" | grep "实验组"
+
+-------------------------------训练-----------------------------------------
 
 nohup /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
 --class com.tzld.piaoquan.recommend.model.train_01_xgb_ad_20240808 \
@@ -120,11 +125,11 @@ nohup /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.sp
 featureFile:20240809_ad_feature_name_517.txt \
 trainPath:/dw/recommend/model/33_ad_train_data_v4/2024080[7-9],/dw/recommend/model/33_ad_train_data_v4/2024081[0-3] \
 testPath:/dw/recommend/model/33_ad_train_data_v4/20240814/ \
-savePath:/dw/recommend/model/34_ad_predict_data/20240814_1000/ \
-modelPath:/dw/recommend/model/35_ad_model/model_xgb_7day \
-eta:0.01 gamma:0.0 max_depth:5 num_round:1000 num_worker:30 \
+savePath:/dw/recommend/model/34_ad_predict_data/20240814_2000/ \
+modelPath:/dw/recommend/model/35_ad_model/model_xgb_7day_2000 \
+eta:0.01 gamma:0.0 max_depth:5 num_round:2000 num_worker:30 \
 repartition:20 \
-> p1_train_0814.log 2>&1 &
+> p2_train_0814.log 2>&1 &
 
 dfs -get /dw/recommend/model/35_ad_model/model_xgb_7day ./
 tar -czvf model_xgb_1000.tar.gz -C model_xgb_7day .