|
@@ -1,42 +1,56 @@
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
+
|
|
|
+set -e
|
|
|
+set -x
|
|
|
+
|
|
|
+savePath=/dw/recommend/model/share_ratio_samples_v2/
|
|
|
+model_name=model_sharev2
|
|
|
+today="$(date +%Y%m%d)"
|
|
|
+today_early_1="$(date -d '1 days ago' +%Y%m%d)"
|
|
|
+
|
|
|
+yesterday=20231221
|
|
|
+jar_main=makedata_01_readtable2hdfs
|
|
|
+
|
|
|
+HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
|
|
|
+FM_TRAIN="/root/sunmingze/alphaFM/bin/fm_train"
|
|
|
+MODEL_PATH="/root/zhangbo/recommend-emr-dataprocess/zhangbo/model/"
|
|
|
+OSS_PATH="oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model"
|
|
|
+
|
|
|
+
|
|
|
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
|
|
|
+--class com.aliyun.odps.spark.examples.makedata.${jar_main} \
|
|
|
+--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
|
|
|
+/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
|
|
|
+tablePart:32 savePath:${savePath} beginStr:${today_early_1} endStr:${today_early_1}
|
|
|
+
|
|
|
+if [ $? -eq 1 ]; then
|
|
|
+ echo "Spark任务执行失败"
|
|
|
+ exit 1
|
|
|
+fi
|
|
|
+
|
|
|
+
|
|
|
+end_date=${today}
|
|
|
+loop_date=${yesterday}
|
|
|
+while [[ "$loop_date" != "$end_date" ]]; do
|
|
|
+ echo -------train ${loop_date}----------
|
|
|
+ loop_date_model=$(date -d "$loop_date - 1 day" +%Y%m%d)
|
|
|
+ $HADOOP fs -text ${savePath}/dt=${loop_date}/* | ${FM_TRAIN} -m ${MODEL_PATH}/${model_name}_${loop_date}.txt \
|
|
|
+-dim 1,1,0 -core 8 -im ${MODEL_PATH}/${model_name}_${loop_date_model}.txt
|
|
|
+ if [ $? -eq 1 ]; then
|
|
|
+ echo "训练失败"
|
|
|
+ exit 1
|
|
|
+ fi
|
|
|
+ echo -------save ${MODEL_PATH}/${model_name}_${loop_date}.txt----------
|
|
|
+ loop_date=$(date -d "$loop_date + 1 day" +%Y%m%d)
|
|
|
+done
|
|
|
+
|
|
|
+
|
|
|
+cat ${MODEL_PATH}/${model_name}_${today_early_1}.txt \
|
|
|
+| sed '1d' | awk -F " " '{if($2!="0") print $1"\t"$2}' \
|
|
|
+> ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt
|
|
|
+
|
|
|
+
|
|
|
+$HADOOP fs -rm -r oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/video_str_model/${model_name}_change.txt
|
|
|
+$HADOOP fs -put ${MODEL_PATH}/${model_name}_${today_early_1}_change.txt ${OSS_PATH}/${model_name}_change.txt
|
|
|
+
|
|
|
+
|
|
|
+
|