recommend-emr-dataprocess/src/main/scala/com/aliyun/odps/spark/examples/临时记录的脚本

【新 上游样本】
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_10_originData_v3 \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 64 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
tablePart:64 savePath:/dw/recommend/model/10_sample_data_v3/ beginStr:20240227 endStr:20240227 > p10_.log 2>&1 &

[ros样本生产]
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_12_rosData_v3 \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 32 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
savePath:/dw/recommend/model/12_ros_data_v3/ beginStr:20240228 endStr:20240228 ifRepart:10 \
> p12_1.log 2>&1 &

nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_12_rosData_v3_noweight \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 32 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
savePath:/dw/recommend/model/12_ros_data_v3_noweight/ beginStr:20240222 endStr:20240226 ifRepart:10 \
> p12_2.log 2>&1 &

[str样本生产]
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_11_strData_v3 \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 64 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
savePath:/dw/recommend/model/11_str_data_v3/ beginStr:20240227 endStr:20240227 ifRepart:100 \
> p11.log 2>&1 &


[user写redis]
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_09_user2redis_freq \
--name makedata_09_user2redis_freq \
--master yarn --driver-memory 1G --executor-memory 4G --executor-cores 1 --num-executors 32 \
--conf spark.yarn.executor.memoryoverhead=1024 \
/root/zhangbo/recommend-emr-dataprocess/target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
date:20240302 tablePart:96 expireDay:3 ifWriteRedisUser:True ifUser:True midDays:14 redisLimit:80000000 \
savePathUser:/dw/recommend/model/09_feature/user/ > p09.log 2>&1 &



--------------
【旧STR 上游样本】
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_06_originData \
--master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 32 \
--conf spark.yarn.executor.memoryoverhead=1024 \
--conf spark.shuffle.service.enabled=true \
--conf spark.shuffle.service.port=7337 \
--conf spark.shuffle.consolidateFiles=true \
--conf spark.shuffle.manager=sort \
--conf spark.storage.memoryFraction=0.4 \
--conf spark.shuffle.memoryFraction=0.5 \
--conf spark.default.parallelism=200 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
tablePart:64 savePath:/dw/recommend/model/00_sample_data/ beginStr:20240311 endStr:20240312 > p6.log 2>&1 &
【旧STR 训练数据】
nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
--class com.aliyun.odps.spark.examples.makedata.makedata_07_strData \
--master yarn --driver-memory 1G --executor-memory 1G --executor-cores 1 --num-executors 32 \
./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
savePath:/dw/recommend/model/04_str_data/ beginStr:20240311 endStr:20240312 featureVersion:v4 ifRepart:100 \
> p7.log 2>&1 &