add

qiaojialiang/xunlian_0724.sh

@@ -4,6 +4,80 @@ set -ex
 HADOOP="/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop"
 FM_TRAIN="/root/sunmingze/alphaFM/bin/fm_train"
 
+echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240717"
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
+--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
+../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+readPath:${originDataPath} \
+savePath:${bucketDataPath} \
+beginStr:20240717 endStr:20240717 repartition:500 \
+filterNames:XXXXXXXXX \
+fileName:20240609_bucket_314.txt \
+whatLabel:is_return whatApps:0,4,21,17
+
+echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240718"
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
+--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
+../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+readPath:${originDataPath} \
+savePath:${bucketDataPath} \
+beginStr:20240718 endStr:20240718 repartition:500 \
+filterNames:XXXXXXXXX \
+fileName:20240609_bucket_314.txt \
+whatLabel:is_return whatApps:0,4,21,17
+
+echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240719"
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
+--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
+../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+readPath:${originDataPath} \
+savePath:${bucketDataPath} \
+beginStr:20240719 endStr:20240719 repartition:500 \
+filterNames:XXXXXXXXX \
+fileName:20240609_bucket_314.txt \
+whatLabel:is_return whatApps:0,4,21,17
+
+echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240720"
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
+--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
+../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+readPath:${originDataPath} \
+savePath:${bucketDataPath} \
+beginStr:20240720 endStr:20240720 repartition:500 \
+filterNames:XXXXXXXXX \
+fileName:20240609_bucket_314.txt \
+whatLabel:is_return whatApps:0,4,21,17
+
+echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240721"
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
+--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
+../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+readPath:${originDataPath} \
+savePath:${bucketDataPath} \
+beginStr:20240721 endStr:20240721 repartition:500 \
+filterNames:XXXXXXXXX \
+fileName:20240609_bucket_314.txt \
+whatLabel:is_return whatApps:0,4,21,17
+
+echo "$(date +%Y-%m-%d_%H-%M-%S)----------step2------------根据特征分桶生产重打分特征数据20240722"
+/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
+--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
+--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
+../target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
+readPath:${originDataPath} \
+savePath:${bucketDataPath} \
+beginStr:20240722 endStr:20240722 repartition:500 \
+filterNames:XXXXXXXXX \
+fileName:20240609_bucket_314.txt \
+whatLabel:is_return whatApps:0,4,21,17
+
+
+
 $HADOOP fs -text /dw/recommend/model/43_recsys_train_data/20240717/* | ${FM_TRAIN} -m /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_17.txt -dim ${bias} -core 8 -im /root/joe/recommend-emr-dataprocess/model/model_nba8_20240716.txt
 echo '$(date +%Y-%m-%d_%H-%M-%S)----step------out model 0709~0722: /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_17.txt'
 
@@ -21,3 +95,4 @@ echo '$(date +%Y-%m-%d_%H-%M-%S)----step------out model 0709~0722: /root/joe/rec
 
 $HADOOP fs -text /dw/recommend/model/43_recsys_train_data/20240722/* | ${FM_TRAIN} -m /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_22.txt -dim ${bias} -core 8 -im /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_21.txt
 echo '$(date +%Y-%m-%d_%H-%M-%S)----step------out model 0709~0722: /root/joe/recommend-emr-dataprocess/model/model_nba8_all_9_22.txt'
+