修改相似度计算方法

src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/makedata_ad_33_bucketDataFromOriginToHive_20250228.scala

@@ -5,13 +5,13 @@ import com.aliyun.odps.TableSchema
 import com.aliyun.odps.data.Record
 import com.aliyun.odps.spark.examples.myUtils.{MyDateUtils, ParamUtils, env}
 import examples.extractor.{ExtractorUtils, RankExtractorFeature_20240530}
-import examples.utils.{AdUtil, DateTimeUtil}
+import examples.utils.{AdUtil, DateTimeUtil, SimilarityUtils}
 import org.apache.spark.sql.SparkSession
 import org.xm.Similarity
+
 import java.time.Instant
 import java.time.ZoneId
 import java.time.ZonedDateTime
-
 import scala.collection.JavaConversions._
 import scala.collection.mutable.ArrayBuffer
 import scala.io.Source
@@ -399,7 +399,7 @@ object makedata_ad_33_bucketDataFromOriginToHive_20250228 {
                 for ((en, prefix1) <- List((e1, "e1"), (e2, "e2"))) {
                   for (prefix2 <- List("tags_3d", "tags_7d", "tags_14d")) {
                     if (en.nonEmpty && en.containsKey(prefix2) && en.getString(prefix2).nonEmpty) {
-                      val (f1, f2, f3, f4) = funcC34567ForTags(en.getString(prefix2), title)
+                      val (f1, f2, f3, f4) = funcC34567ForTagsNew(en.getString(prefix2), title)
                       featureMap.put(prefix1 + "_" + prefix2 + "_matchnum", f1)
                       featureMap.put(prefix1 + "_" + prefix2 + "_maxscore", f3)
                       featureMap.put(prefix1 + "_" + prefix2 + "_avgscore", f4)
@@ -626,7 +626,7 @@ object makedata_ad_33_bucketDataFromOriginToHive_20250228 {
     record
   }
 
-  def funcC34567ForTags(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
+  def funcC34567ForTagsNew(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
     // 匹配数量 匹配词 语义最高相似度分 语义平均相似度分
     val tagsList = tags.split(",")
     var d1 = 0.0
@@ -638,7 +638,7 @@ object makedata_ad_33_bucketDataFromOriginToHive_20250228 {
         d1 = d1 + 1.0
         d2.add(tag)
       }
-      val score = Similarity.conceptSimilarity(tag, title)
+      val score = SimilarityUtils.word2VecSimilarity(tag, title)
       d3 = if (score > d3) score else d3
       d4 = d4 + score
     }

src/main/scala/com/aliyun/odps/spark/examples/makedata_ad/v20240718/test.scala

@@ -1,35 +0,0 @@
-package com.aliyun.odps.spark.examples.makedata_ad.v20240718
-
-import examples.utils.SimilarityUtils
-import org.xm.Similarity
-
-import scala.collection.JavaConversions.asJavaCollection
-import scala.collection.mutable.ArrayBuffer
-
-object test {
-
-  def main(args: Array[String]): Unit = {
-    println(funcC34567ForTagsNew("高兴", "快乐"))
-  }
-
-  def funcC34567ForTagsNew(tags: String, title: String): Tuple4[Double, String, Double, Double] = {
-    // 匹配数量 匹配词 语义最高相似度分 语义平均相似度分
-    val tagsList = tags.split(",")
-    var d1 = 0.0
-    val d2 = new ArrayBuffer[String]()
-    var d3 = 0.0
-    var d4 = 0.0
-    for (tag <- tagsList) {
-      if (title.contains(tag)) {
-        d1 = d1 + 1.0
-        d2.add(tag)
-      }
-      val score = SimilarityUtils.word2VecSimilarity(tag, title)
-      d3 = if (score > d3) score else d3
-      d4 = d4 + score
-    }
-    d4 = if (tagsList.nonEmpty) d4 / tagsList.size else d4
-    (d1, d2.mkString(","), d3, d4)
-  }
-
-}