优化爬虫搜索

applications/const/server_const.py

@@ -62,5 +62,12 @@ class ServerConst:
     TASK_FAIL_CODE = 99
     TASK_PROCESSING_CODE = 101
 
+    # 相关性过滤阈值
+    NLP_SIMILARITY_THRESHOLD = 0.45
+
+    JCD_SIMILARITY_THRESHOLD = 0
+
+
+
 
 

applications/match_algorithm/__init__.py

@@ -1,4 +1,5 @@
 """
 @author: luojunhui
 匹配算法
-"""
+"""
+from .rank import title_similarity_with_nlp

applications/match_algorithm/rank.py

@@ -1,7 +1,10 @@
 """
 @author: luojunhui
 """
+from typing import Dict
+
 from applications.match_algorithm.title_similarity import jcd_title_similarity
+from applications.match_algorithm.title_similarity import nlp_title_similarity
 
 
 def jac_score(d1, d2):
@@ -51,3 +54,37 @@ def title_similarity_rank(content_title, recall_list):
         include_title_list.append(item)
     sorted_list = sorted(include_title_list, key=lambda x: x['score'], reverse=True)
     return sorted_list
+
+
+async def title_similarity_with_nlp(content_title, recall_list) -> Dict:
+    """
+    通过相关性模型来计算文章标题和搜索标题之间的相关性
+    """
+    title_list = [i['title'] for i in recall_list]
+    score_list = await nlp_title_similarity(
+        ori_title=content_title,
+        search_title_list=title_list
+    )
+    if score_list:
+        sorted_list = sorted(
+            (
+                {**item, 'score': score}
+                for item, score in zip(recall_list, score_list)
+            ),
+            key=lambda x: x['score'],
+            reverse=True
+        )
+        response = {
+            "alg": "nlp",
+            "result": sorted_list
+        }
+    else:
+        # if nlp server is down, use jcd similarity instead
+        response = {
+            "alg": "jcd",
+            "result": title_similarity_rank(content_title, recall_list)
+        }
+    return response
+
+
+

applications/match_algorithm/title_similarity.py

@@ -1,6 +1,9 @@
 """
 @author: luojunhui
 """
+import aiohttp
+
+empty_list = []
 
 
 def jcd_title_similarity(ori_title, search_title):
@@ -15,3 +18,29 @@ def jcd_title_similarity(ori_title, search_title):
     intersection = len(set1 & set2)
     union = len(set1 | set2)
     return intersection / union
+
+
+async def nlp_title_similarity(ori_title, search_title_list):
+    """
+    nlp title similarity
+    """
+    headers = {"Content-Type": "application/json"}
+    body = {
+        "data": {
+            "text_list_a": [ori_title],
+            "text_list_b": search_title_list,
+        },
+        "function": "similarities_cross",
+        "use_cache": "False"
+    }
+    url = 'http://61.48.133.26:6060/nlp'
+
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, headers=headers, json=body) as response:
+            response_text = await response.text()
+            if response_text:
+                res = await response.json()
+                score_list = res['score_list_list'][0]
+                return score_list
+            else:
+                return empty_list

applications/search/hksp_search.py

@@ -117,10 +117,9 @@ def hksp_search(key, sensitive_words, trace_id):
                 video_id = data['vid']
                 title = data['title']
                 duration = int(data['duration'].split(":")[0]) * 60 + int(data['duration'].split(":")[1])
-                if sensitive_flag(sensitive_words, title) and int(duration) <= 300:
+                if sensitive_flag(sensitive_words, title) and int(duration) <= 900:
                     res = get_video_detail(video_id)
                     L.append(res)
-                    return L
                 else:
                     continue
             except Exception as e:

applications/spider/__init__.py

@@ -3,9 +3,10 @@
 """
 from datetime import datetime
 
+from applications.const import server_const
 from applications.functions.video_item import VideoProducer
 from applications.log import logging
-from applications.match_algorithm.rank import title_similarity_rank
+from applications.match_algorithm import title_similarity_with_nlp
 from .spiderAB import SearchABTest
 from .spiderSchedule import SearchMethod
 
@@ -94,7 +95,7 @@ async def search_videos_from_web(info, gh_id_map, db_client):
     trace_id = info['trace_id']
     gh_id = info['gh_id']
     content_id = info['content_id']
-    recall_list = await search_AB.ab_5()
+    recall_list = await search_AB.ab_6()
     logging(
         code="1006",
         info="搜索到{}条视频".format(len(recall_list)),
@@ -102,12 +103,22 @@ async def search_videos_from_web(info, gh_id_map, db_client):
         trace_id=info['trace_id']
     )
     # 按照标题相似度排序
-    ranked_list = title_similarity_rank(content_title=info['ori_title'].split("@@")[-1], recall_list=recall_list)
+    ranked_result = await title_similarity_with_nlp(content_title=info['ori_title'].split("@@")[-1], recall_list=recall_list)
+    rank_alg = ranked_result['alg']
+    ranked_list = ranked_result['result']
     for recall_obj in ranked_list:
         if recall_obj:
             platform = recall_obj['platform']
             recall_video = recall_obj['result']
             score = recall_obj['score']
+            # 过滤掉jcd分数为0的
+            if rank_alg == 'jcd' and score == server_const.JCD_SIMILARITY_THRESHOLD:
+                continue
+
+            # 过滤掉nlp分低于0.3的
+            if rank_alg == 'nlp' and score < server_const.NLP_SIMILARITY_THRESHOLD:
+                continue
+
             if recall_video:
                 await save_video_to_mysql(
                     video_obj=recall_video,

applications/spider/spiderAB.py

@@ -122,7 +122,7 @@ class SearchABTest(object):
             text=self.article_summary[:15],
             trace_id=self.trace_id
         )
-        if len(result_list) > 3:
+        if len(result_list) > 5:
             return result_list
         else:
             result_list += await self.search_method.search_v2(
@@ -151,3 +151,26 @@ class SearchABTest(object):
                             trace_id=self.trace_id
                         )
                         return result_list
+
+    async def ab_6(self):
+        """
+        依次搜索
+        """
+        search_result_by_summary = await self.search_method.search_v2(
+            text=self.article_summary[:15],
+            trace_id=self.trace_id
+        )
+
+        search_result_by_ori_title = await self.search_method.search_v2(
+            text=self.ori_title[:15],
+            trace_id=self.trace_id
+        )
+
+        search_result_by_article_keys = await self.search_method.search_v2(
+            text=",".join(self.article_keys),
+            trace_id=self.trace_id
+        )
+
+        return search_result_by_summary + search_result_by_ori_title + search_result_by_article_keys
+
+

applications/spider/spiderSchedule.py

@@ -46,13 +46,12 @@ class SearchMethod(object):
         douyin_result = douyin_search(keyword=text, sensitive_words=cls.s_words, trace_id=trace_id)
         for vid_obj in douyin_result:
             L.append({"platform": "dy_search", "result": vid_obj})
-        if len(L) >= 3:
-            return L
         else:
             baidu_result = hksp_search(key=text, sensitive_words=cls.s_words, trace_id=trace_id)
             if baidu_result:
-                L.append({"platform": "baidu_search", "result": baidu_result[0]})
-            xigua_result = xigua_search_v2(keyword=text, sensitive_words=cls.s_words)
-            if xigua_result:
-                L.append({"platform": "xg_search", "result": xigua_result[0]})
+                for baidu_obj in baidu_result:
+                    L.append({"platform": "baidu_search", "result": baidu_obj})
+            # xigua_result = xigua_search_v2(keyword=text, sensitive_words=cls.s_words)
+            # if xigua_result:
+            #     L.append({"platform": "xg_search", "result": xigua_result[0]})
             return L