Home Explore Help
Sign In
algorithm
/
recommend-emr-dataprocess
12
0
Fork 0
Files Issues 0 Pull Requests 2 Wiki
Tree: f072c442b8
Branches Tags
recommend-em...
/
src
/
main
/
scala
/
com
/
aliyun
/
odps
/
spark
/
examples
/
临时记录的脚本-广告

临时记录的脚本-广告 2.5 KB
History Raw

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263 
  1. 

  2. 

  3. 

  4. nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
    5. 

  5. --class com.aliyun.odps.spark.examples.makedata_ad.makedata_ad_31_originData_20240620 \
    6. 

  6. --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
    7. 

  7. ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
    8. 

  8. tablePart:64 repartition:16 \
    9. 

  9. beginStr:2024070108 endStr:2024070323 \
    10. 

  10. savePath:/dw/recommend/model/31_ad_sample_data_v3/ \
    11. 

  11. table:alg_recsys_ad_sample_all filterHours:00,01,02,03,04,05,06,07 \
    12. 

  12. idDefaultValue:0.01 \
    13. 

  13. > p31_2024070108.log 2>&1 &
    14. 

  14. 

  15. 

  16. 

  17. 

  18. nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
    19. 

  19. --class com.aliyun.odps.spark.examples.makedata_ad.makedata_ad_32_bucket_20240622 \
    20. 

  20. --master yarn --driver-memory 16G --executor-memory 1G --executor-cores 1 --num-executors 16 \
    21. 

  21. --conf spark.driver.maxResultSize=16G \
    22. 

  22. ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
    23. 

  23. readPath:/dw/recommend/model/31_ad_sample_data_v3/2024062[01]* \
    24. 

  24. savePath:/dw/recommend/model/32_bucket_file/ \
    25. 

  25. fileName:20240703_100_fix sampleRate:1.0 bucketNum:100 \
    26. 

  26. > p32_data.log 2>&1 &
    27. 

  27. 

  28. 

  29. nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
    30. 

  30. --class com.aliyun.odps.spark.examples.makedata_ad.makedata_ad_33_bucketData_20240622 \
    31. 

  31. --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
    32. 

  32. ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
    33. 

  33. readPath:/dw/recommend/model/31_ad_sample_data_v3/ \
    34. 

  34. savePath:/dw/recommend/model/33_ad_train_data_v3/ \
    35. 

  35. beginStr:20240620 endStr:20240621 repartition:100 \
    36. 

  36. filterNames:adid_,targeting_conversion_ \
    37. 

  37. > p33_data.log 2>&1 &
    38. 

  38. 

  39. filterNames:cid_,adid_,adverid_,targeting_conversion_ \
    40. 

  40. savePath:/dw/recommend/model/33_ad_train_data_nosparse/ \
    41. 

  41. 

  42. 

  43. /dw/recommend/model/31_ad_sample_data/
    44. 

  44. /dw/recommend/model/33_ad_train_data/
    45. 

  45. 

  46. /dw/recommend/model/31_ad_sample_data_fix/
    47. 

  47. 

  48. 

  49. 

  50. 

  51. 

  52. nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
    53. 

  53. --class com.aliyun.odps.spark.examples.makedata_ad.makedata_ad_33_bucketDataPrint_20240628 \
    54. 

  54. --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
    55. 

  55. ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
    56. 

  56. beginStr:2024062908 endStr:2024062923 \
    57. 

  57. readDate:20240629 \
    58. 

  58. table:alg_recsys_ad_sample_all_new \
    59. 

  59. savePath:/dw/recommend/model/33_for_check/ \
    60. 

  60. > p33_data_check.log 2>&1 &
    61. 

  61. 

  62. 

  63. /dw/recommend/model/33_for_check_v1/
    64.