个人记录

src/main/scala/com/aliyun/odps/spark/examples/makedata/makedata_17_bucketDataPrint_20240617.scala

@@ -282,13 +282,13 @@ object makedata_17_bucketDataPrint_20240617 {
           val useOfflineNames = Set("c3_feature_tags_1d_matchnum","c3_feature_tags_1d_maxscore","c3_feature_tags_1d_avgscore","c3_feature_tags_3d_matchnum","c3_feature_tags_3d_maxscore","c3_feature_tags_3d_avgscore","c3_feature_tags_7d_matchnum","c3_feature_tags_7d_maxscore","c3_feature_tags_7d_avgscore","c4_feature_tags_1d_matchnum","c4_feature_tags_1d_maxscore","c4_feature_tags_1d_avgscore","c4_feature_tags_3d_matchnum","c4_feature_tags_3d_maxscore","c4_feature_tags_3d_avgscore","c4_feature_tags_7d_matchnum","c4_feature_tags_7d_maxscore","c4_feature_tags_7d_avgscore","c5_feature_tags_1d_matchnum","c5_feature_tags_1d_maxscore","c5_feature_tags_1d_avgscore","c5_feature_tags_3d_matchnum","c5_feature_tags_3d_maxscore","c5_feature_tags_3d_avgscore","c5_feature_tags_7d_matchnum","c5_feature_tags_7d_maxscore","c5_feature_tags_7d_avgscore","c6_feature_tags_1d_matchnum","c6_feature_tags_1d_maxscore","c6_feature_tags_1d_avgscore","c6_feature_tags_3d_matchnum","c6_feature_tags_3d_maxscore","c6_feature_tags_3d_avgscore","c6_feature_tags_7d_matchnum","c6_feature_tags_7d_maxscore","c6_feature_tags_7d_avgscore","c7_feature_tags_1d_matchnum","c7_feature_tags_1d_maxscore","c7_feature_tags_1d_avgscore","c7_feature_tags_3d_matchnum","c7_feature_tags_3d_maxscore","c7_feature_tags_3d_avgscore","c7_feature_tags_7d_matchnum","c7_feature_tags_7d_maxscore","c7_feature_tags_7d_avgscore","c8_feature_share_score","c8_feature_share_num","c8_feature_share_rank","c8_feature_return_score","c8_feature_return_num","c8_feature_return_rank","c9_feature_share_score","c9_feature_share_num","c9_feature_share_rank","c9_feature_return_score","c9_feature_return_num","c9_feature_return_rank")
           if (useOfflineNames.contains(name)){
             if (offlineFeatrueMap.contains(name)){
-              name + ":" + offlineFeatrueMap.get(name)
+              name + ":" + offlineFeatrueMap(name)
             }else{
               ""
             }
           }else{
             if (allfeaturemap.contains(name)) {
-              name + ":" + offlineFeatrueMap.get(name)
+              name + ":" + offlineFeatrueMap(name)
             } else {
               ""
             }
@@ -319,23 +319,23 @@ object makedata_17_bucketDataPrint_20240617 {
       result.iterator
     })
 
-    val saveV1 = "/dw/recommend/model/17_for_check_v1/" + readDate
-    if (saveV1.nonEmpty && saveV1.startsWith("/dw/recommend/model/")) {
-      println("删除路径并开始数据写入:" + saveV1)
-      MyHdfsUtils.delete_hdfs_path(saveV1)
-      data2.map(r => r._1 + "\t" + r._2.map(r=> r._1 + ":" + r._2).mkString("\t")).saveAsTextFile(saveV1, classOf[GzipCodec])
-    } else {
-      println("路径不合法，无法写入:" + saveV1)
-    }
-
-    val saveV2 = "/dw/recommend/model/17_for_check_v2/" + readDate
-    if (saveV2.nonEmpty && saveV2.startsWith("/dw/recommend/model/")) {
-      println("删除路径并开始数据写入:" + saveV2)
-      MyHdfsUtils.delete_hdfs_path(saveV2)
-      data2.map(r => r._1 + "\t" + r._3.map(r=> r._1 + ":" + r._2).mkString("\t")).saveAsTextFile(saveV2, classOf[GzipCodec])
-    } else {
-      println("路径不合法，无法写入:" + saveV2)
-    }
+//    val saveV1 = "/dw/recommend/model/17_for_check_v1/" + readDate
+//    if (saveV1.nonEmpty && saveV1.startsWith("/dw/recommend/model/")) {
+//      println("删除路径并开始数据写入:" + saveV1)
+//      MyHdfsUtils.delete_hdfs_path(saveV1)
+//      data2.map(r => r._1 + "\t" + r._2.map(r=> r._1 + ":" + r._2).mkString("\t")).saveAsTextFile(saveV1, classOf[GzipCodec])
+//    } else {
+//      println("路径不合法，无法写入:" + saveV1)
+//    }
+//
+//    val saveV2 = "/dw/recommend/model/17_for_check_v2/" + readDate
+//    if (saveV2.nonEmpty && saveV2.startsWith("/dw/recommend/model/")) {
+//      println("删除路径并开始数据写入:" + saveV2)
+//      MyHdfsUtils.delete_hdfs_path(saveV2)
+//      data2.map(r => r._1 + "\t" + r._3.map(r=> r._1 + ":" + r._2).mkString("\t")).saveAsTextFile(saveV2, classOf[GzipCodec])
+//    } else {
+//      println("路径不合法，无法写入:" + saveV2)
+//    }
 
     val saveV3 = "/dw/recommend/model/17_for_check_v3/" + readDate
     if (saveV3.nonEmpty && saveV3.startsWith("/dw/recommend/model/")) {

src/main/scala/com/aliyun/odps/spark/examples/临时记录的脚本

@@ -157,4 +157,5 @@ nohup /opt/apps/SPARK2/spark-2.4.8-hadoop3.2-1.0.8/bin/spark-class2 org.apache.s
 --master yarn --driver-memory 2G --executor-memory 4G --executor-cores 1 --num-executors 16 \
 ./target/spark-examples-1.0.0-SNAPSHOT-shaded.jar \
 beginStr:2024061800 endStr:2024061814 \
+readDate:20240618 \
 > p17_data_check.log 2>&1 &

zhangbo/03_predict.sh

@@ -22,4 +22,10 @@ cat predict/${output_file}_$day.txt | /root/sunmingze/AUC/AUC
 
 
 
-# cat tmpfile | /root/sunmingze/alphaFM/bin/fm_predict -m model/model_aka8_20240608.txt -dim 8 -core 1 -out tmpfile_out.txt
+# cat tmpfile | /root/sunmingze/alphaFM/bin/fm_predict -m model/model_aka8_20240608.txt -dim 8 -core 1 -out tmpfile_out.txt
+
+
+# nohup sh 03_predict.sh 20240618 /dw/recommend/model/17_for_check_v1/ model_aka8_20240608.txt v1 8 >v1.log 2>&1 &
+# nohup sh 03_predict.sh 20240618 /dw/recommend/model/17_for_check_v2/ model_aka8_20240608.txt v2 8 >v2.log 2>&1 &
+# nohup sh 03_predict.sh 20240618 /dw/recommend/model/17_for_check_v3/ model_aka8_20240608.txt v3 8 >v3.log 2>&1 &
+# nohup sh 03_predict.sh 20240618 /dw/recommend/model/17_for_check_v4/ model_aka8_20240608.txt v4 8 >v4.log 2>&1 &