recommend-model/01-脚本记录

spark-submit --class com.tzld.piaoquan.recommend.model.produce.xgboost.XGBoostTrain --master yarn --driver-memory 512M --executor-memory 512M --executor-cores 1 --num-executors 4 /root/recommend-model/recommend-model-produce-new.jar > ~/recommend-model/log 2>&1 &





recommend-model-produce-jar-with-dependencies.jar







nohup /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
--class com.tzld.piaoquan.recommend.model.produce.xgboost.XGBoostTrain \
--master yarn --driver-memory 512M --executor-memory 512M --executor-cores 1 --num-executors 2 \
./target/recommend-model-produce-jar-with-dependencies.jar \
> p.log 2>&1 &

nohup /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit --class com.tzld.piaoquan.recommend.model.train_01_xgb_ad_20240808 --master yarn --driver-memory 6G --executor-memory 6G --executor-cores 1 --num-executors 32 --conf spark.yarn.executor.memoryoverhead=1024 --conf spark.shuffle.service.enabled=true --conf spark.shuffle.service.port=7337 --conf spark.shuffle.consolidateFiles=true --conf spark.shuffle.manager=sort --conf spark.storage.memoryFraction=0.4 --conf spark.shuffle.memoryFraction=0.5 --conf spark.default.parallelism=200 ./target/recommend-model-produce-jar-with-dependencies.jar > p.log 2>&1 &


nohup /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
--class com.tzld.piaoquan.recommend.model.train_01_xgb_ad_20240808 \
--master yarn --driver-memory 6G --executor-memory 3G --executor-cores 1 --num-executors 160 \
--conf spark.yarn.executor.memoryoverhead=1000 \
--conf spark.shuffle.service.enabled=true \
--conf spark.shuffle.service.port=7337 \
--conf spark.shuffle.consolidateFiles=true \
--conf spark.shuffle.manager=sort \
--conf spark.storage.memoryFraction=0.4 \
--conf spark.shuffle.memoryFraction=0.5 \
--conf spark.default.parallelism=200 \
./target/recommend-model-produce-jar-with-dependencies.jar \
featureFile:20240809_ad_feature_name_517.txt \
trainPath:/dw/recommend/model/33_ad_train_data_v4/2024080[6-9],/dw/recommend/model/33_ad_train_data_v4/2024081[0-2] \
testPath:/dw/recommend/model/33_ad_train_data_v4/20240813/ \
savePath:/dw/recommend/model/34_ad_predict_data/20240813_1000/ \
modelPath:/dw/recommend/model/35_ad_model/model_xgb_7day \
eta:0.01 gamma:0.0 max_depth:5 num_round:1000 num_worker:63 \
repartition:20 \
> p5.log 2>&1 &

0.7316512679739304 1000
2024072[5-9],2024073[0-1],2024080[1-4]
/dw/recommend/model/33_ad_train_data_v4/(20240725|20240726|20240727|20240728|20240729|20240730|20240731|20240801|20240802|20240803|20240804)


nohup /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
--class com.tzld.piaoquan.recommend.model.ana_01_xgb_ad_20240809 \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 32 \
--conf spark.yarn.executor.memoryoverhead=1024 \
--conf spark.shuffle.service.enabled=true \
--conf spark.shuffle.service.port=7337 \
--conf spark.shuffle.consolidateFiles=true \
--conf spark.shuffle.manager=sort \
--conf spark.storage.memoryFraction=0.4 \
--conf spark.shuffle.memoryFraction=0.5 \
--conf spark.default.parallelism=200 \
./target/recommend-model-produce-jar-with-dependencies.jar \
savePath:/dw/recommend/model/34_ad_predict_data/20240805_1000/ \
> p1.log 2>&1 &

dfs -get /dw/recommend/model/35_ad_model/model_xgb_1000 ./
tar -czvf model_xgb_1000.tar.gz -C model_xgb_1000 .
dfs -put model_xgb_1000.tar.gz oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/

oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/model_xgb_1000.tar.gz


nohup /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
--class com.tzld.piaoquan.recommend.model.pred_01_xgb_ad_jsonfile_20240813 \
--master yarn --driver-memory 6G --executor-memory 6G --executor-cores 1 --num-executors 32 \
--conf spark.yarn.executor.memoryoverhead=1024 \
--conf spark.shuffle.service.enabled=true \
--conf spark.shuffle.service.port=7337 \
--conf spark.shuffle.consolidateFiles=true \
--conf spark.shuffle.manager=sort \
--conf spark.storage.memoryFraction=0.4 \
--conf spark.shuffle.memoryFraction=0.5 \
--conf spark.default.parallelism=200 \
./target/recommend-model-produce-jar-with-dependencies.jar \
featureFile:20240809_ad_feature_name_517.txt \
savePath:/dw/recommend/model/34_ad_predict_data/case_tmp/ \
modelPath:/dw/recommend/model/35_ad_model/model_xgb_1000 \
> p5.log 2>&1 &

-------------------------------预测-----------------------------------------

nohup /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
--class com.tzld.piaoquan.recommend.model.pred_01_xgb_ad_hdfsfile_20240813 \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 30 \
--conf spark.yarn.executor.memoryoverhead=1024 \
--conf spark.shuffle.service.enabled=true \
--conf spark.shuffle.service.port=7337 \
--conf spark.shuffle.consolidateFiles=true \
--conf spark.shuffle.manager=sort \
--conf spark.storage.memoryFraction=0.4 \
--conf spark.shuffle.memoryFraction=0.5 \
--conf spark.default.parallelism=200 \
./target/recommend-model-produce-jar-with-dependencies.jar \
featureFile:20240809_ad_feature_name_517.txt \
testPath:/dw/recommend/model/33_ad_train_data_v4/20240815/ \
savePath:/dw/recommend/model/34_ad_predict_data/20240815_new2/ \
modelPath:/dw/recommend/model/35_ad_model/model_xgb_7day_2000 \
> p1_pred_20240815_new2.log 2>&1 &

cat p1_pred_20240815_new2.log | grep -E "^3353|^3606|^2670|^3585|^2912|^3869|^3857" | grep "实验组"

-------------------------------训练-----------------------------------------

nohup /opt/apps/SPARK3/spark-3.3.1-hadoop3.2-1.0.5/bin/spark-class org.apache.spark.deploy.SparkSubmit \
--class com.tzld.piaoquan.recommend.model.train_01_xgb_ad_20240808 \
--master yarn --driver-memory 6G --executor-memory 9G --executor-cores 1 --num-executors 31 \
--conf spark.yarn.executor.memoryoverhead=1000 \
--conf spark.shuffle.service.enabled=true \
--conf spark.shuffle.service.port=7337 \
--conf spark.shuffle.consolidateFiles=true \
--conf spark.shuffle.manager=sort \
--conf spark.storage.memoryFraction=0.4 \
--conf spark.shuffle.memoryFraction=0.5 \
--conf spark.default.parallelism=200 \
./target/recommend-model-produce-jar-with-dependencies.jar \
featureFile:20240809_ad_feature_name_517.txt \
trainPath:/dw/recommend/model/33_ad_train_data_v4/2024080[7-9],/dw/recommend/model/33_ad_train_data_v4/2024081[0-3] \
testPath:/dw/recommend/model/33_ad_train_data_v4/20240814/ \
savePath:/dw/recommend/model/34_ad_predict_data/20240814_2000/ \
modelPath:/dw/recommend/model/35_ad_model/model_xgb_7day_2000 \
eta:0.01 gamma:0.0 max_depth:5 num_round:2000 num_worker:30 \
repartition:20 \
> p2_train_0814.log 2>&1 &

dfs -get /dw/recommend/model/35_ad_model/model_xgb_7day ./
tar -czvf model_xgb_1000.tar.gz -C model_xgb_7day .
rm -rf .model.tar.gz.crc
dfs -rm -r -skipTrash oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/model.tar.gz
dfs -put model_xgb_1000.tar.gz oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/