24_ad_data_make.sh 1.8 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455
  1. #!/bin/sh
  2. # 广告数据生产
  3. set -x
  4. source /root/anaconda3/bin/activate py37
  5. export SPARK_HOME=/opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8
  6. export PATH=$SPARK_HOME/bin:$PATH
  7. export HADOOP_CONF_DIR=/etc/taihao-apps/hadoop-conf
  8. export JAVA_HOME=/usr/lib/jvm/java-1.8.0
  9. # 全局常量
  10. HADOOP=/opt/apps/HADOOP-COMMON/hadoop-common-current/bin/hadoop
  11. FM_HOME=/root/sunmingze/alphaFM
  12. TRAIN_PATH=/dw/recommend/model/31_ad_sample_data_v4
  13. BUCKET_FEATURE_PATH=/dw/recommend/model/33_ad_train_data_v4
  14. TABLE=alg_recsys_ad_sample_all
  15. today="$(date +%Y%m%d)"
  16. today_early_1="$(date -d '1 days ago' +%Y%m%d)"
  17. make_origin_data() {
  18. /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
  19. --class com.aliyun.odps.spark.examples.makedata_ad.v20240718.makedata_ad_31_originData_20240718 \
  20. --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
  21. ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
  22. tablePart:64 repartition:32 \
  23. beginStr:${today_early_1}00 endStr:${today_early_1}23 \
  24. savePath: ${TRAIN_PATH} \
  25. table: ${TABLE} \
  26. filterHours:00,01,02,03,04,05,06,07 \
  27. idDefaultValue:0.1
  28. }
  29. make_bucket_feature() {
  30. /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
  31. --class com.aliyun.odps.spark.examples.makedata_ad.v20240718.makedata_ad_33_bucketData_20240718 \
  32. --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
  33. ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
  34. beginStr:${today_early_1} endStr:${today_early_1} repartition:100 \
  35. filterNames:_4h_,_5h_,_ecpm,ecpm_,adid_,targeting_conversion_ \
  36. readPath:${TRAIN_PATH} \
  37. savePath:${BUCKET_FEATURE_PATH} \
  38. }
  39. main() {
  40. make_origin_data
  41. make_bucket_feature
  42. }
  43. main