recommend-emr-dataprocess/qiaojialiang/xunlian_0724.sh

#!/bin/sh
set -ex

HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
FM_TRAIN="/root/sunmingze/alphaFM/bin/fm_train"

echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240717"
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
readPath:${originDataPath} \
savePath:${bucketDataPath} \
beginStr:20240717 endStr:20240717 repartition:500 \
filterNames:XXXXXXXXX \
fileName:20240609_bucket_314.txt \
whatLabel:is_return whatApps:0,4,21,17

echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240718"
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
readPath:${originDataPath} \
savePath:${bucketDataPath} \
beginStr:20240718 endStr:20240718 repartition:500 \
filterNames:XXXXXXXXX \
fileName:20240609_bucket_314.txt \
whatLabel:is_return whatApps:0,4,21,17

echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240719"
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
readPath:${originDataPath} \
savePath:${bucketDataPath} \
beginStr:20240719 endStr:20240719 repartition:500 \
filterNames:XXXXXXXXX \
fileName:20240609_bucket_314.txt \
whatLabel:is_return whatApps:0,4,21,17

echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240720"
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
readPath:${originDataPath} \
savePath:${bucketDataPath} \
beginStr:20240720 endStr:20240720 repartition:500 \
filterNames:XXXXXXXXX \
fileName:20240609_bucket_314.txt \
whatLabel:is_return whatApps:0,4,21,17

echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240721"
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
readPath:${originDataPath} \
savePath:${bucketDataPath} \
beginStr:20240721 endStr:20240721 repartition:500 \
filterNames:XXXXXXXXX \
fileName:20240609_bucket_314.txt \
whatLabel:is_return whatApps:0,4,21,17

echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240722"
/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
readPath:${originDataPath} \
savePath:${bucketDataPath} \
beginStr:20240722 endStr:20240722 repartition:500 \
filterNames:XXXXXXXXX \
fileName:20240609_bucket_314.txt \
whatLabel:is_return whatApps:0,4,21,17



$HADOOP fs -text /dw/recommend/model/43_recsys_train_data/20240717/* | ${FM_TRAIN} -m /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_17.txt -dim ${bias} -core 8 -im /root/joe/recommend-emr-dataprocess/model/model_nba8_20240716.txt
echo '$(date +%Y-%m-%d_%H-%M-%S)----step------out model 0709~0722: /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_17.txt'

$HADOOP fs -text /dw/recommend/model/43_recsys_train_data/20240718/* | ${FM_TRAIN} -m /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_18.txt -dim ${bias} -core 8 -im /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_17.txt
echo '$(date +%Y-%m-%d_%H-%M-%S)----step------out model 0709~0722: /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_18.txt'

$HADOOP fs -text /dw/recommend/model/43_recsys_train_data/20240719/* | ${FM_TRAIN} -m /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_19.txt -dim ${bias} -core 8 -im /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_18.txt
echo '$(date +%Y-%m-%d_%H-%M-%S)----step------out model 0709~0722: /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_19.txt'

$HADOOP fs -text /dw/recommend/model/43_recsys_train_data/20240720/* | ${FM_TRAIN} -m /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_20.txt -dim ${bias} -core 8 -im /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_19.txt
echo '$(date +%Y-%m-%d_%H-%M-%S)----step------out model 0709~0722: /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_20.txt'

$HADOOP fs -text /dw/recommend/model/43_recsys_train_data/20240721/* | ${FM_TRAIN} -m /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_21.txt -dim ${bias} -core 8 -im /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_20.txt
echo '$(date +%Y-%m-%d_%H-%M-%S)----step------out model 0709~0722: /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_21.txt'

$HADOOP fs -text /dw/recommend/model/43_recsys_train_data/20240722/* | ${FM_TRAIN} -m /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_22.txt -dim ${bias} -core 8 -im /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_21.txt
echo '$(date +%Y-%m-%d_%H-%M-%S)----step------out model 0709~0722: /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_22.txt'