|
|
@@ -2,9 +2,9 @@
|
|
|
set -x
|
|
|
|
|
|
# 0 全局变量/参数
|
|
|
-originDataSavePath=/dw/recommend/model/31_ad_sample_data_auto/
|
|
|
-bucketFeatureSavePath=/dw/recommend/model/33_ad_train_data_nosparse_auto/
|
|
|
-model_name=model_lr0
|
|
|
+originDataSavePath=/dw/recommend/model/31_ad_sample_data_v3_auto/
|
|
|
+bucketFeatureSavePath=/dw/recommend/model/33_ad_train_data_v3_auto/
|
|
|
+model_name=model_bkb8_v3
|
|
|
today="$(date +%Y%m%d)"
|
|
|
today_early_1="$(date -d '1 days ago' +%Y%m%d)"
|
|
|
|
|
|
@@ -12,8 +12,8 @@ MODEL_PATH=/root/zhaohp/recommend-emr-dataprocess/model
|
|
|
PREDICT_PATH=/root/zhaohp/recommend-emr-dataprocess/predict
|
|
|
HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
|
|
|
FM_HOME=/root/sunmingze/alphaFM
|
|
|
-OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/ad_model/
|
|
|
-max_hour=16
|
|
|
+OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/
|
|
|
+max_hour=17
|
|
|
max_minute=00
|
|
|
|
|
|
|
|
|
@@ -43,8 +43,8 @@ done
|
|
|
--class com.aliyun.odps.spark.zhp.makedata_ad.makedata_ad_31_originData_20240620 \
|
|
|
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
|
|
|
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
-tablePart:64 repartition:32 \
|
|
|
-beginStr:${today_early_1}08 endStr:${today}10 \
|
|
|
+tablePart:64 repartition:16 \
|
|
|
+beginStr:${today_early_1}00 endStr:${today}10 \
|
|
|
savePath:${originDataSavePath} \
|
|
|
table:alg_recsys_ad_sample_all filterHours:00,01,02,03,04,05,06,07 \
|
|
|
idDefaultValue:0.01
|
|
|
@@ -63,9 +63,8 @@ fi
|
|
|
--class com.aliyun.odps.spark.zhp.makedata_ad.makedata_ad_33_bucketData_20240622 \
|
|
|
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
|
|
|
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
-beginStr:${today_early_1} endStr:${today} repartition:400 \
|
|
|
+beginStr:${today_early_1} endStr:${today} repartition:100 \
|
|
|
filterNames:adid_,targeting_conversion_ \
|
|
|
-bucketFileName:20240620_ad_bucket_249_fix.txt \
|
|
|
readPath:${originDataSavePath} \
|
|
|
savePath:${bucketFeatureSavePath}
|
|
|
if [ $? -ne 0 ]; then
|
|
|
@@ -79,7 +78,7 @@ fi
|
|
|
|
|
|
|
|
|
# 4 模型训练
|
|
|
-$HADOOP fs -text ${bucketFeatureSavePath}/${today_early_1}/* | ${FM_HOME}/fm_train -m ${MODEL_PATH}/${model_name}_${today_early_1}.txt -dim 1,1,8 -core 8
|
|
|
+$HADOOP fs -text ${bucketFeatureSavePath}/${today_early_1}/* | ${FM_HOME}/fm_train -m ${MODEL_PATH}/${model_name}_${today_early_1}.txt -dim 1,1,8 -im ${MODEL_PATH}/model_online.txt -core 8
|
|
|
if [ $? -ne 0 ]; then
|
|
|
echo "模型训练失败"
|
|
|
/root/anaconda3/bin/python ad/ad_monitor_util.py "广告模型训练失败"
|
