recommend-emr-dataprocess/src/main/scala/com/aliyun/odps/spark/examples/临时记录的脚本-推荐

【新 上游样本】
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_10_originData_v3 \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 64 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
tablePart:64 savePath:/dw/recommend/model/10_sample_data_v3/ beginStr:20240227 endStr:20240227 > p10_.log 2>&1 &

[ros样本生产]
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_12_rosData_v3 \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 32 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
savePath:/dw/recommend/model/12_ros_data_v3/ beginStr:20240228 endStr:20240228 ifRepart:10 \
> p12_1.log 2>&1 &

nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_12_rosData_v3_noweight \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 32 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
savePath:/dw/recommend/model/12_ros_data_v3_noweight/ beginStr:20240222 endStr:20240226 ifRepart:10 \
> p12_2.log 2>&1 &

[str样本生产]
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_11_strData_v3 \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 64 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
savePath:/dw/recommend/model/11_str_data_v3/ beginStr:20240227 endStr:20240227 ifRepart:100 \
> p11.log 2>&1 &


[user写redis]
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_09_user2redis_freq \
--name makedata_09_user2redis_freq \
--master yarn --driver-memory 1G --executor-memory 4G --executor-cores 1 --num-executors 32 \
--conf spark.yarn.executor.memoryoverhead=1024 \
/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
date:20240302 tablePart:96 expireDay:3 ifWriteRedisUser:True ifUser:True midDays:14 redisLimit:80000000 \
savePathUser:/dw/recommend/model/09_feature/user/ > p09.log 2>&1 &



--------------
【旧STR 上游样本】
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_06_originData \
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 32 \
--conf spark.yarn.executor.memoryoverhead=1024 \
--conf spark.shuffle.service.enabled=true \
--conf spark.shuffle.service.port=7337 \
--conf spark.shuffle.consolidateFiles=true \
--conf spark.shuffle.manager=sort \
--conf spark.storage.memoryFraction=0.4 \
--conf spark.shuffle.memoryFraction=0.5 \
--conf spark.default.parallelism=200 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
tablePart:64 savePath:/dw/recommend/model/00_sample_data/ beginStr:20240311 endStr:20240312 > p6.log 2>&1 &
【旧STR 训练数据】
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_07_strData \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 32 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
savePath:/dw/recommend/model/04_str_data/ beginStr:20240311 endStr:20240312 featureVersion:v4 ifRepart:100 \
> p7.log 2>&1 &
---------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------------------
--------------------------------下-----------------------------------------------------------
--------------------------------面-----------------------------------------------------------
--------------------------------为-----------------------------------------------------------
--------------------------------准-----------------------------------------------------------
---------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------------------

nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_13_originData_20240529 \
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
tablePart:64 repartition:32 \
beginStr:2024061600 endStr:2024061623 \
savePath:/dw/recommend/model/13_sample_data/ \
table:alg_recsys_sample_all \
> p13_2024061600.log 2>&1 &


nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_14_valueData_20240608 \
--master yarn --driver-memory 1G --executor-memory 3G --executor-cores 1 --num-executors 32 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
readPath:/dw/recommend/model/13_sample_data/ \
savePath:/dw/recommend/model/14_feature_data/ \
beginStr:20240615 endStr:20240615 repartition:1000 \
> p14_data_check.log 2>&1 &


nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_15_bucket_20240608 \
--master yarn --driver-memory 16G --executor-memory 1G --executor-cores 1 --num-executors 16 \
--conf spark.driver.maxResultSize=16G \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
readPath:/dw/recommend/model/14_feature_data/20240606/ fileName:20240606_200_v3 \
bucketNum:200 sampleRate:0.1 \
> p15_data2.log 2>&1 &


nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_16_bucketData_20240609 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
beginStr:20240615 endStr:20240615 repartition:1000 \
> p16_data.log 2>&1 &


/dw/recommend/model/13_sample_data/
/dw/recommend/model/14_feature_data/
/dw/recommend/model/16_train_data/

-----
一个执行：只有用线上打印特征的才执行
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_13_originData_20240529_check \
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
tablePart:64 repartition:32 \
beginStr:2024061500 endStr:2024061523 \
savePath:/dw/recommend/model/13_sample_data_check_print/ \
table:alg_recsys_sample_all_new \
> p13_2024061500_check.log 2>&1 &

两个都要执行：过滤不需要的样本
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_16_bucketData_20240609_check \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
readPath:/dw/recommend/model/14_feature_data_check_print/ \
savePath:/dw/recommend/model/16_train_data_check_print/ \
beginStr:20240615 endStr:20240615 repartition:1000 \
> p16_data_check.log 2>&1 &

/dw/recommend/model/13_sample_data_check/
/dw/recommend/model/13_sample_data_check_print/
/dw/recommend/model/14_feature_data_check/
/dw/recommend/model/14_feature_data_check_print/
/dw/recommend/model/16_train_data_check/
/dw/recommend/model/16_train_data_check_print/


nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_17_bucketDataPrint_20240617 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
> p17_data_check.log 2>&1 &

nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_18_mergehour2day_20240617 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
> p18_data_check.log 2>&1 &

nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_17_bucketDataPrint_20240617 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
beginStr:2024062600 endStr:2024062623 \
readDate:20240626 \
> p17_20240626.log 2>&1 &

/dw/recommend/model/17_for_check/


------------------------------------------------------------------------------------------------------------------------

nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_41_originData_20240709 \
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
tablePart:64 repartition:32 \
beginStr:2024070508 endStr:2024070508 \
savePath:/dw/recommend/model/41_recsys_sample_data/ \
table:alg_recsys_sample_all \
> p41_2024070508.log 2>&1 &

nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_42_bucket_20240709 \
--master yarn --driver-memory 16G --executor-memory 1G --executor-cores 1 --num-executors 16 \
--conf spark.driver.maxResultSize=16G \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
readPath:/dw/recommend/model/41_recsys_sample_data_v1/20240705* \
savePath:/dw/recommend/model/42_recsys_bucket/ \
fileName:20240705_314_200 \
bucketNum:200 sampleRate:1.0 \
> p42.log 2>&1 &

nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata_recsys.makedata_recsys_43_bucketData_20240709 \
--master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
readPath:/dw/recommend/model/41_recsys_sample_data/ \
savePath:/dw/recommend/model/43_recsys_train_data/ \
beginStr:20240705 endStr:20240705 repartition:100 \
filterNames:XXXXXXXXX \
fileName:20240609_bucket_314.txt \
whatLabel:is_return whatApps:0,4,21,3,6,17,23 \
> p43_20240705.log 2>&1 &

------------- 20240709_recsys_bucket_314.txt ------------ 20240609_bucket_274.txt -------------
------------- filterNames:b123_1h_ROS,b123_2h_ROS,b123_3h_ROS,b123_4h_ROS,b123_12h_ROS,b123_1d_ROS,b123_3d_ROS,b123_7d_ROS,b167_1h_ROS,b167_2h_ROS,b167_3h_ROS,b167_4h_ROS,b167_12h_ROS,b167_1d_ROS,b167_3d_ROS,b167_7d_ROS,b8910_1h_ROS,b8910_2h_ROS,b8910_3h_ROS,b8910_4h_ROS,b8910_12h_ROS,b8910_1d_ROS,b8910_3d_ROS,b8910_7d_ROS,b111213_1h_ROS,b111213_2h_ROS,b111213_3h_ROS,b111213_4h_ROS,b111213_12h_ROS,b111213_1d_ROS,b111213_3d_ROS,b111213_7d_ROS,b171819_1h_ROS,b171819_2h_ROS,b171819_3h_ROS,b171819_4h_ROS,b171819_12h_ROS,b171819_1d_ROS,b171819_3d_ROS,b171819_7d_ROS \
------------- filterNames:XXXXXXXXX \