|
@@ -1,8 +1,39 @@
|
|
|
#!/bin/sh
|
|
#!/bin/sh
|
|
|
set -ex
|
|
set -ex
|
|
|
|
|
|
|
|
-HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
|
|
|
|
|
-FM_TRAIN="/root/sunmingze/alphaFM/bin/fm_train"
|
|
|
|
|
|
|
+
|
|
|
|
|
+# 原始数据table name
|
|
|
|
|
+table='alg_recsys_sample_all'
|
|
|
|
|
+today="$(date +%Y%m%d)"
|
|
|
|
|
+today_early_3="$(date -d '3 days ago' +%Y%m%d)"
|
|
|
|
|
+#table='alg_recsys_sample_all_test'
|
|
|
|
|
+# 处理分区配置 推荐数据间隔一天生产，所以5日0点使用3日0-23点数据生产new模型数据
|
|
|
|
|
+begin_early_2_Str="$(date -d '2 days ago' +%Y%m%d)"
|
|
|
|
|
+end_early_2_Str="$(date -d '2 days ago' +%Y%m%d)"
|
|
|
|
|
+beginHhStr=00
|
|
|
|
|
+endHhStr=23
|
|
|
|
|
+max_hour=05
|
|
|
|
|
+max_minute=00
|
|
|
|
|
+# 各节点产出hdfs文件绝对路径
|
|
|
|
|
+# 源数据文件
|
|
|
|
|
+originDataPath=/dw/recommend/model/41_recsys_sample_data/
|
|
|
|
|
+# 特征值
|
|
|
|
|
+valueDataPath=/dw/recommend/model/14_feature_data/
|
|
|
|
|
+# 特征分桶
|
|
|
|
|
+bucketDataPath=/dw/recommend/model/43_recsys_train_data/
|
|
|
|
|
+# 模型数据路径
|
|
|
|
|
+MODEL_PATH=/root/joe/recommend-emr-dataprocess/model
|
|
|
|
|
+# 预测路径
|
|
|
|
|
+PREDICT_PATH=/root/joe/recommend-emr-dataprocess/predict
|
|
|
|
|
+# 历史线上正在使用的模型数据路径
|
|
|
|
|
+LAST_MODEL_HOME=/root/joe/model_online
|
|
|
|
|
+# 模型数据文件前缀
|
|
|
|
|
+model_name=model_nba8
|
|
|
|
|
+# fm模型
|
|
|
|
|
+FM_HOME=/root/sunmingze/alphaFM/bin
|
|
|
|
|
+# hadoop
|
|
|
|
|
+HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
|
|
|
|
|
+OSS_PATH=oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/
|
|
|
|
|
|
|
|
echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240717"
|
|
echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240717"
|
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
Joe