Merge branch 'dev-xym-similarity' of algorithm/recommend-emr-dataprocess into feature/20250104-zt-update

src/main/java/examples/utils/SimilarityUtils.java

@@ -14,7 +14,7 @@ import java.util.List;
 @Slf4j
 public final class SimilarityUtils {
 
-    private static Word2Vec vec = new Word2Vec();
+    private static Word2Vec vec;
 
     public static void init() {
         Segment.getWords("1");
@@ -36,6 +36,9 @@ public final class SimilarityUtils {
     }
 
     public static float word2VecSimilarity(String str1, String str2) {
+        if (vec == null) {
+            init();
+        }
         List<String> words1 = Segment.getWords(str1);
         List<String> words2 = Segment.getWords(str2);
         return vec.sentenceSimilarity(words1, words2);

src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/makedata_ad_33_bucketDataFromOriginToHive_20250228.scala

@@ -5,13 +5,13 @@ import com.aliyun.odps.TableSchema
 import com.aliyun.odps.data.Record
 import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, ParamUtils, env}
 import examples.extractor.{ExtractorUtils, RankExtractorFeature_20240530}
-import examples.utils.{AdUtil, DateTimeUtil}
+import examples.utils.{AdUtil, DateTimeUtil, SimilarityUtils}
 import org.apache.spark.sql.SparkSession
 import org.xm.Similarity
+
 import java.time.Instant
 import java.time.ZoneId
 import java.time.ZonedDateTime
-
 import scala.collection.JavaConversions._
 import scala.collection.mutable.ArrayBuffer
 import scala.io.Source
@@ -399,7 +399,7 @@ object makedata_ad_33_bucketDataFromOriginToHive_20250228 {
                 for ((en, prefix1) <- List((e1, "e1"), (e2, "e2"))) {
                   for (prefix2 <- List("tags_3d", "tags_7d", "tags_14d")) {
                     if (en.nonEmpty && en.containsKey(prefix2) && en.getString(prefix2).nonEmpty) {
-                      val (f1, f2, f3, f4) = funcC34567ForTags(en.getString(prefix2), title)
+                      val (f1, f2, f3, f4) = funcC34567ForTagsNew(en.getString(prefix2), title)
                       featureMap.put(prefix1 + "_" + prefix2 + "_matchnum", f1)
                       featureMap.put(prefix1 + "_" + prefix2 + "_maxscore", f3)
                       featureMap.put(prefix1 + "_" + prefix2 + "_avgscore", f4)
@@ -626,7 +626,7 @@ object makedata_ad_33_bucketDataFromOriginToHive_20250228 {
     record
   }
 
-  def funcC34567ForTags(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
+  def funcC34567ForTagsNew(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
     // 匹配数量 匹配词 语义最高相似度分 语义平均相似度分
     val tagsList = tags.split(",")
     var d1 = 0.0
@@ -638,7 +638,7 @@ object makedata_ad_33_bucketDataFromOriginToHive_20250228 {
         d1 = d1 + 1.0
         d2.add(tag)
       }
-      val score = Similarity.conceptSimilarity(tag, title)
+      val score = SimilarityUtils.word2VecSimilarity(tag, title)
       d3 = if (score > d3) score else d3
       d4 = d4 + score
     }