LongArticlesJob/applications/api/nlp_api.py

"""
@author: luojunhui
"""
import requests
import traceback
from requests.exceptions import RequestException, JSONDecodeError

from applications.aliyunLogApi import log


def similarity_between_title_list(target_title_list: list[str], base_title_list: list[str]) -> list[list[float]]:
    """
    cal the similarity between two list of title
    :param target_title_list: target title_list
    :param base_title_list: base title_list
    :return: list of similarity
    """

    url = 'http://61.48.133.26:6060/nlp'
    url_backup = 'http://192.168.203.4:6060/nlp'
    body = {
        "data": {
            "text_list_a": target_title_list,
            "text_list_b": base_title_list
        },
        "function": "similarities_cross",
        "use_cache": False
    }

    try:
        response = requests.post(url, json=body, timeout=120)
        if response.status_code != 200:
            response = requests.post(url_backup, json=body, timeout=120)
    except RequestException as e:
        log(
            task="nlp",
            function="similarity_between_title_list",
            status="fail",
            message="nlp server web error",
            data={
                "e": str(e),
                "error_msg": traceback.format_exc()
            }
        )
        # use back up
        response = requests.post(url_backup, json=body, timeout=120)

    if response.status_code != 200:
        log(
            task="nlp",
            function="similarity_between_title_list",
            status="fail",
            message='nlp server request error',
            data={
                "status_code": response.status_code,
                "response_text": response.text[:200]  # 截取部分内容避免过大
            }
        )
        return []

    try:
        response_json = response.json()
        score_array = response_json['score_list_list']
    except (JSONDecodeError, KeyError) as e:
        log(
            task="nlp",
            function="similarity_between_title_list",
            status="fail",
            message='nlp server response error',
            data={
                "error_type": type(e).__name__,
                "raw_response": response.text[:200]
            }
        )
        return []

    return score_array