|
|
@@ -1,32 +1,46 @@
|
|
|
#!/bin/sh
|
|
|
-set -e
|
|
|
-set -x
|
|
|
+set -ex
|
|
|
# 0 全局变量/参数
|
|
|
-savePath=/dw/recommend/model/share_ratio_samples_v2/
|
|
|
-model_name=model_sharev2
|
|
|
+savePath=/dw/recommend/model/04_str_data/
|
|
|
+model_name=model_str_mid
|
|
|
today="$(date +%Y%m%d)"
|
|
|
today_early_1="$(date -d '1 days ago' +%Y%m%d)"
|
|
|
-#yesterday="$(date -d '1 days ago' +%Y%m%d)"
|
|
|
-yesterday=20231221
|
|
|
-jar_main=makedata_01_readtable2hdfs
|
|
|
+yesterday="$(date -d '1 days ago' +%Y%m%d)"
|
|
|
|
|
|
HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
|
|
|
FM_TRAIN="/root/sunmingze/alphaFM/bin/fm_train"
|
|
|
MODEL_PATH="/root/zhangbo/recommend-emr-dataprocess/zhangbo/model/"
|
|
|
-OSS_PATH="oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model"
|
|
|
+OSS_PATH="oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model/"
|
|
|
|
|
|
# 1 生产数据
|
|
|
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
|
---class com.aliyun.odps.spark.examples.makedata.${jar_main} \
|
|
|
---master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
|
|
|
+--class com.aliyun.odps.spark.examples.makedata.makedata_06_originData \
|
|
|
+--name every_day_origindata_${model_name}_${today} \
|
|
|
+--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 16 \
|
|
|
/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
tablePart:32 savePath:${savePath} beginStr:${today_early_1} endStr:${today_early_1}
|
|
|
+if [ $? -eq 1 ]; then
|
|
|
+ echo "Spark原始样本生产任务执行失败"
|
|
|
+ exit 1
|
|
|
+else
|
|
|
+ echo "spark原始样本生产执行成功"
|
|
|
+fi
|
|
|
|
|
|
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
|
+--class com.aliyun.odps.spark.examples.makedata.makedata_07_strData \
|
|
|
+--name every_day_strdata_${model_name}_${today} \
|
|
|
+--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 16 \
|
|
|
+./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
+savePath:${savePath} beginStr:${today_early_1} endStr:${today_early_1} featureVersion:v4 ifRepart:100
|
|
|
if [ $? -eq 1 ]; then
|
|
|
- echo "Spark任务执行失败"
|
|
|
+ echo "Spark训练样本生产任务执行失败"
|
|
|
exit 1
|
|
|
+else
|
|
|
+ echo "spark训练样本生产执行成功"
|
|
|
fi
|
|
|
|
|
|
+
|
|
|
+
|
|
|
# 2 加载上次模型 训练本轮数据 保存本轮模型
|
|
|
end_date=${today}
|
|
|
loop_date=${yesterday}
|
|
|
@@ -49,8 +63,15 @@ cat ${MODEL_PATH}/${model_name}_${today_early_1}.txt \
|
|
|
> ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt
|
|
|
|
|
|
# 4 转换后模型上传oss
|
|
|
-$HADOOP fs -rm -r oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model/${model_name}_change.txt
|
|
|
-$HADOOP fs -put ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt ${OSS_PATH}/${model_name}_change.txt
|
|
|
+online_model_path=${OSS_PATH}/${model_name}.txt
|
|
|
+$HADOOP fs -test -e ${online_model_path}
|
|
|
+if [ $? -eq 0 ]; then
|
|
|
+ echo "数据存在, 先删除。"
|
|
|
+ $HADOOP fs -rm -r ${online_model_path}
|
|
|
+else
|
|
|
+ echo "数据不存在"
|
|
|
+fi
|
|
|
+$HADOOP fs -put ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt ${online_model_path}
|
|
|
|
|
|
|
|
|
#nohup sh 05_update_everyday_str.sh > p.log 2>&1 &
|
zhangbo