| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273 |
- nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
- --class com.aliyun.odps.spark.examples.makedata_dssm.makedata_i2i_01_originData_20241127 \
- --master yarn --driver-memory 2G --executor-memory 2G --executor-cores 1 --num-executors 16 \
- ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
- beginStr:2024113000 endStr:2024113023 negCnt:20 \
- tablePart:64 savePath:/dw/recommend/model/51_dssm_i2i_sample/ > p51_2.log 2>&1 &
- nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
- --class com.aliyun.odps.spark.examples.makedata_dssm.makedata_i2i_02_joinFeatureData_20241128 \
- --master yarn --driver-memory 2G --executor-memory 2G --executor-cores 1 --num-executors 32 \
- ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
- beginStr:2024113000 endStr:2024113023 \
- tablePart:64 \
- readPath:/dw/recommend/model/51_dssm_i2i_sample/ \
- savePath:/dw/recommend/model/52_dssm_i2i_joinfeature/ > p52_2.log 2>&1 &
- nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
- --class com.aliyun.odps.spark.examples.makedata_dssm.makedata_i2i_03_onehotFile_20241128 \
- --master yarn --driver-memory 16G --executor-memory 1G --executor-cores 1 --num-executors 32 \
- ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
- dt:20241201 \
- tablePart:64 \
- readPath:/dw/recommend/model/53_dssm_i2i_onehot/after_20241201_file \
- savePath:/dw/recommend/model/53_dssm_i2i_onehot/after_20241201_file > p53.log 2>&1 &
- 数据量:3544265打印各个特征多少枚举值：vid 3534920
- video_style 7382
- captions_color 782
- valid_time 754
- audience_age_group 80
- cate2 67
- audience_value_type 65
- font_size 49
- audience_gender 47
- cover_persons_num 45
- cate1 42
- sentiment_tendency 11
- video_type 8
- background_music_type 6
- captions 3
- has_end_credit_guide 2
- timeliness 2
- nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
- --class com.aliyun.odps.spark.examples.makedata_dssm.makedata_i2i_04_bucketFile_20241128 \
- --master yarn --driver-memory 16G --executor-memory 2G --executor-cores 1 --num-executors 32 \
- ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
- readPath:/dw/recommend/model/52_dssm_i2i_joinfeature/20241126* \
- savePath:/dw/recommend/model/54_dssm_i2i_bucketfile/ \
- fileName:47_rate_v2 bucketNum:100 > p54.log 2>&1 &
- nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
- --class com.aliyun.odps.spark.examples.makedata_dssm.makedata_i2i_05_trainData_20241129 \
- --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 32 \
- ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
- beginStr:2024113000 endStr:2024113023 \
- readPath:/dw/recommend/model/52_dssm_i2i_joinfeature/ \
- savePath:/dw/recommend/model/55_dssm_i2i_traindata/ \
- onehotPath:/dw/recommend/model/53_dssm_i2i_onehot/after_20241201_file \
- bucketFile:20241128_recsys_i2i_bucket_47_v2.txt > p55_2.log 2>&1 &
- nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
- --class com.aliyun.odps.spark.examples.makedata_dssm.makedata_i2i_06_itemPred_20241206 \
- --master yarn --driver-memory 2G --executor-memory 2G --executor-cores 1 --num-executors 16 \
- ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
- onehotPath:/dw/recommend/model/53_dssm_i2i_onehot/after_20241201_file \
- bucketFile:20241128_recsys_i2i_bucket_47_v2.txt repartition:10 ifFilterCate:true \
- dt:20241225 \
- savePath:/dw/recommend/model/56_dssm_i2i_itempredData/ \
- > p56.log 2>&1 &
|
