|
|
@@ -1,92 +0,0 @@
|
|
|
-#!/bin/sh
|
|
|
-set -ex
|
|
|
-
|
|
|
-# 0 全局变量/参数
|
|
|
-originDataSavePath=/dw/recommend/model/31_ad_sample_data_auto/
|
|
|
-bucketFeatureSavePath=/dw/recommend/model/33_ad_train_data_nosparse_auto/
|
|
|
-model_name=model_lr0
|
|
|
-today="$(date +%Y%m%d)"
|
|
|
-today_early_1="$(date -d '1 days ago' +%Y%m%d)"
|
|
|
-beginTime=08
|
|
|
-endTime=23
|
|
|
-beginStr=${today_early_1}${beginTime}
|
|
|
-endStr=${today_early_1}${endTime}
|
|
|
-
|
|
|
-MODEL_PATH=/root/zhaohp/recommend-emr-dataprocess/model
|
|
|
-HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
|
|
|
-FM_HOME=/root/sunmingze/alphaFM
|
|
|
-OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/ad_model/
|
|
|
-
|
|
|
-# 1 判断依赖的数据表是否生产完成
|
|
|
-source /root/anaconda3/bin/activate py37
|
|
|
-max_hour=15
|
|
|
-max_minute=00
|
|
|
-while true; do
|
|
|
- python_return_code=$(python zhangbo/utils.py --excute_program check_ad_origin_hive --partition ${today_early_1} --hh ${endTime})
|
|
|
- if [ $python_return_code -eq 0 ]; then
|
|
|
- echo "Python程序返回0，退出循环。"
|
|
|
- break
|
|
|
- fi
|
|
|
- echo "Python程序返回非0值，等待五分钟后再次调用。"
|
|
|
- sleep 300
|
|
|
- current_hour=$(date +%H)
|
|
|
- current_minute=$(date +%M)
|
|
|
- if (( current_hour > max_hour || (current_hour == max_hour && current_minute >= max_minute) )); then
|
|
|
- echo "最长等待时间已到，失败:${current_hour}-${current_minute}"
|
|
|
- exit 1
|
|
|
- fi
|
|
|
-done
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-# 2 原始特征生成
|
|
|
-/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
|
---class com.aliyun.odps.spark.zhp.makedata_ad.makedata_ad_31_originData_20240620 \
|
|
|
---master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
|
|
|
-./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
-tablePart:64 repartition:32 \
|
|
|
-beginStr:${beginStr} endStr:${endStr} \
|
|
|
-savePath:${originDataSavePath} \
|
|
|
-table:alg_recsys_ad_sample_all_new
|
|
|
-if [ $? -ne 0 ]; then
|
|
|
- echo "Spark原始样本生产任务执行失败"
|
|
|
- exit 1
|
|
|
-else
|
|
|
- echo "spark原始样本生产执行成功"
|
|
|
-fi
|
|
|
-
|
|
|
-
|
|
|
-# 3 特征分桶
|
|
|
-/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
|
---class com.aliyun.odps.spark.zhp.makedata_ad.makedata_ad_33_bucketData_20240622 \
|
|
|
---master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
|
|
|
-./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
-beginStr:${today_early_1} endStr:${today_early_1} repartition:400 \
|
|
|
-filterNames:XXXXX \
|
|
|
-bucketFileName:20240620_ad_bucket_249_fix.txt \
|
|
|
-readPath:${originDataSavePath} \
|
|
|
-savePath:${bucketFeatureSavePath}
|
|
|
-if [ $? -ne 0 ]; then
|
|
|
- echo "Spark特征分桶处理任务执行失败"
|
|
|
- exit 1
|
|
|
-else
|
|
|
- echo "spark特征分桶处理执行成功"
|
|
|
-fi
|
|
|
-
|
|
|
-
|
|
|
-# 4 模型训练
|
|
|
-$HADOOP fs -text ${bucketFeatureSavePath}/${today_early_1}/* | /root/sunmingze/alphaFM/bin/fm_train -m model/${model_name}_${today_early_1}.txt -dim 1,1,0 -core 8
|
|
|
-if [ $? -ne 0 ]; then
|
|
|
- echo "模型训练失败"
|
|
|
- exit 1
|
|
|
-fi
|
|
|
-
|
|
|
-
|
|
|
-# 5 对比AUC
|
|
|
-
|
|
|
-# 6 模型格式转换
|
|
|
-cat ${MODEL_PATH}/${model_name}_${today_early_1}.txt \
|
|
|
-| sed '1d' | awk -F " " '{if($2!="0") print $1"\t"$2}' \
|
|
|
-> ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt
|
|
|
-
|
|
|
-# 7 模型文件上传OSS
|
