Merge branch 'feature/zhangbo_makedata_v2' into feature_zhaohaipeng

src/main/scala/com/aliyun/odps/spark/examples/临时记录的脚本-广告

@@ -1,6 +1,4 @@
 
-
-
 nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata_ad.makedata_ad_31_originData_20240620 \
 --master yarn --driver-memory 1G --executor-memory 2G --executor-cores 1 --num-executors 16 \
@@ -13,8 +11,6 @@ idDefaultValue:0.01 \
 > p31_2024070108.log 2>&1 &
 
 
-
-
 nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.spark.deploy.SparkSubmit \
 --class com.aliyun.odps.spark.examples.makedata_ad.makedata_ad_32_bucket_20240622 \
 --master yarn --driver-memory 16G --executor-memory 1G --executor-cores 1 --num-executors 16 \
@@ -32,12 +28,14 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 readPath:/dw/recommend/model/31_ad_sample_data_v3/ \
 savePath:/dw/recommend/model/33_ad_train_data_v3/ \
-beginStr:20240620 endStr:20240621 repartition:100 \
+beginStr:20240703 endStr:20240703 repartition:100 \
 filterNames:adid_,targeting_conversion_ \
-> p33_data.log 2>&1 &
+> p33_20240703_.log 2>&1 &
 
+filterNames:adid_,targeting_conversion_ \
 filterNames:cid_,adid_,adverid_,targeting_conversion_ \
-savePath:/dw/recommend/model/33_ad_train_data_nosparse/ \
+filterNames:"XXXXXX,adid_,targeting_conversion_,b2_3h_click,b2_3h_conver*log(view),b2_3h_conver*ctcvr,b2_6h_click,b2_6h_conver*log(view),b2_6h_conver*ctcvr,b2_12h_click,b2_12h_conver*log(view),b2_12h_conver*ctcvr,b2_1d_click,b2_1d_conver*log(view),b2_1d_conver*ctcvr,b2_3d_click,b2_3d_conver*log(view),b2_3d_conver*ctcvr,b2_7d_click,b2_7d_conver*log(view),b2_7d_conver*ctcvr,b3_3h_click,b3_3h_conver*log(view),b3_3h_conver*ctcvr,b3_6h_click,b3_6h_conver*log(view),b3_6h_conver*ctcvr,b3_12h_click,b3_12h_conver*log(view),b3_12h_conver*ctcvr,b3_1d_click,b3_1d_conver*log(view),b3_1d_conver*ctcvr,b3_3d_click,b3_3d_conver*log(view),b3_3d_conver*ctcvr,b3_7d_click,b3_7d_conver*log(view),b3_7d_conver*ctcvr,b4_3h_click,b4_3h_conver*log(view),b4_3h_conver*ctcvr,b4_6h_click,b4_6h_conver*log(view),b4_6h_conver*ctcvr,b4_12h_click,b4_12h_conver*log(view),b4_12h_conver*ctcvr,b4_1d_click,b4_1d_conver*log(view),b4_1d_conver*ctcvr,b4_3d_click,b4_3d_conver*log(view),b4_3d_conver*ctcvr,b4_7d_click,b4_7d_conver*log(view),b4_7d_conver*ctcvr,b5_3h_click,b5_3h_conver*log(view),b5_3h_conver*ctcvr,b5_6h_click,b5_6h_conver*log(view),b5_6h_conver*ctcvr,b5_12h_click,b5_12h_conver*log(view),b5_12h_conver*ctcvr,b5_1d_click,b5_1d_conver*log(view),b5_1d_conver*ctcvr,b5_3d_click,b5_3d_conver*log(view),b5_3d_conver*ctcvr,b5_7d_click,b5_7d_conver*log(view),b5_7d_conver*ctcvr,b8_3h_click,b8_3h_conver*log(view),b8_3h_conver*ctcvr,b8_6h_click,b8_6h_conver*log(view),b8_6h_conver*ctcvr,b8_12h_click,b8_12h_conver*log(view),b8_12h_conver*ctcvr,b8_1d_click,b8_1d_conver*log(view),b8_1d_conver*ctcvr,b8_3d_click,b8_3d_conver*log(view),b8_3d_conver*ctcvr,b8_7d_click,b8_7d_conver*log(view),b8_7d_conver*ctcvr,b6_7d_click,b6_7d_conver*log(view),b6_7d_conver*ctcvr,b6_14d_click,b6_14d_conver*log(view),b6_14d_conver*ctcvr,b7_7d_click,b7_7d_conver*log(view),b7_7d_conver*ctcvr,b7_14d_click,b7_14d_conver*log(view),b7_14d_conver*ctcvr,XXXXXX" \
+
 
 
 /dw/recommend/model/31_ad_sample_data/

zhangbo/01_train.sh

@@ -16,8 +16,8 @@ $HADOOP fs -text ${train_path}/${day}/* | /root/sunmingze/alphaFM/bin/fm_train -
 # nohup sh 01_train.sh 20240606 /dw/recommend/model/16_train_data/ model_aka4 1,1,4 >p1_model_aka4.log 2>&1 &
 
 
-# nohup sh 01_train.sh 20240623 /dw/recommend/model/33_ad_train_data_nosparse/ model_test 1,1,0 >p1_model_bkb0.log 2>&1 &
-# nohup sh 01_train.sh 20240620 /dw/recommend/model/33_ad_train_data/ model_bkb8_2 1,1,8 >p1_model_bkb8_2.log 2>&1 &
+# nohup sh 01_train.sh 20240623 /dw/recommend/model/33_ad_train_data_v3/ model_bkb0_v3_23 1,1,0 >p1_model_bkb0_v3_23.log 2>&1 &
+# nohup sh 01_train.sh 20240623 /dw/recommend/model/33_ad_train_data_v3/ model_bkb8_v3_23 1,1,8 >p1_model_bkb8_v3_23.log 2>&1 &
 # nohup sh 01_train.sh 20240620 /dw/recommend/model/33_ad_train_data/ model_bkb4 1,1,4 >p1_model_bkb4.log 2>&1 &
 # nohup sh 01_train.sh 20240620 /dw/recommend/model/33_ad_train_data/ model_bkb12 1,1,12 >p1_model_bkb12.log 2>&1 &
 # nohup sh 01_train.sh 20240620 /dw/recommend/model/33_ad_train_data/ model_bkb16 1,1,16 >p1_model_bkb16.log 2>&1 &

zhangbo/02_train_go.sh

@@ -27,4 +27,4 @@ done
 
 # nohup sh 02_train_go.sh 20240623 20240624 model_bkb8 /dw/recommend/model/33_ad_train_data/ 1,1,8 >p2_model_bkb8.log 2>&1 &
 
-# nohup sh 02_train_go.sh 20240621 20240623 model_bkb0_3 /dw/recommend/model/33_ad_train_data_nosparse/ 1,1,0 >p2_model_bkb0.log 2>&1 &
+# nohup sh 02_train_go.sh 20240703 20240704 model_bkb8_v3 /dw/recommend/model/33_ad_train_data_v3/ 1,1,8 >p2_model_bkb8.log 2>&1 &

zhangbo/03_predict.sh

@@ -17,7 +17,7 @@ cat predict/${output_file}_$day.txt | /root/sunmingze/AUC/AUC
 # nohup sh 03_predict.sh 20240613 /dw/recommend/model/16_train_data/ model_aka8_20240612.txt model_aka8_20240612 8 >p3_model_aka8_12.log 2>&1 &
 
 
-# nohup sh 03_predict.sh 20240624 /dw/recommend/model/33_ad_train_data_nosparse/ model_test_20240623.txt model_test_20240623 0 >p3_model_aka8_on.log 2>&1 &
+# nohup sh 03_predict.sh 20240703 /dw/recommend/model/33_ad_train_data_v3/ model_bkb0_v3_20240702.txt model_bkb0_v3_20240702 0 >p3_model_bkb0_v3.log 2>&1 &
 
 
 

zhangbo/04_upload.sh

@@ -31,3 +31,20 @@ dfs -put /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_aka8_202406
 
 cat /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_bkb0_20240622.txt | awk -F " " '{print $1"\t"$2}' > /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_bkb0_20240622_change.txt
 dfs -put /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_bkb0_20240622_change.txt oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/model_bkb0.txt
+
+cat /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_bkb8_v3_20240702.txt |
+awk -F " " '{
+    if (NR == 1) {
+        print $1"\t"$2
+    } else {
+        split($0, fields, " ");
+        OFS="\t";
+        line=""
+        for (i = 1; i <= 10 && i <= length(fields); i++) {
+            line = (line ? line "\t" : "") fields[i];
+        }
+        print line
+    }
+}' > /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_bkb8_v3_20240702_change.txt
+
+dfs -put /root/zhangbo/recommend-emr-dataprocess/zhangbo/model/model_bkb8_v3_20240702_change.txt oss://art-recommend.oss-cn-hangzhou.aliyuncs.com/zhangbo/model_bkb8_v3.txt