LongArticleAlgServer/applications/wxSpider.py

"""
@author: luojunhui
"""
import json
import time
import asyncio
import aiohttp
import requests


def retry_on_none_data():
    """
    基于None类型数据的重试装饰器
    :return:
    """

    def decorator(func):
        """
        :param func:
        :return:
        """
        max_retries = 5
        wait_seconds = 1

        def wrapper(*args, **kwargs):
            """

            :param args:
            :param kwargs:
            :return:
            """
            for attempt in range(max_retries):
                response = func(*args, **kwargs)
                if response['data'] is not None:
                    return response
                time.sleep(wait_seconds)
            return None

        return wrapper

    return decorator


def retryAsyncOnNoneData():
    """
    异步装饰器
    :return:
    """

    def decorator(func):
        """
        :param func:
        :return:
        """
        max_retries = 5
        wait_seconds = 1

        async def wrapper(*args, **kwargs):
            """

            :param args:
            :param kwargs:
            :return:
            """
            for attempt in range(max_retries):
                response = await func(*args, **kwargs)
                if response.get('data') is not None:
                    return response
                await asyncio.sleep(wait_seconds)
            return None

        return wrapper

    return decorator


class ArticleManager(object):
    """
    Update account articles
    """

    @classmethod
    @retry_on_none_data()
    def search_articles(cls, title):
        """
        search articles in wx
        :return:
        """
        url = "http://47.98.154.124:8888/crawler/wei_xin/keyword"
        payload = json.dumps({
            "keyword": title,
            "cursor": "1"
        })
        headers = {
            'Content-Type': 'application/json'
        }

        response = requests.request("POST", url, headers=headers, data=payload)
        return response.json()

    @classmethod
    @retry_on_none_data()
    def get_article_text(cls, content_link):
        """
        获取文章
        :param content_link:
        :return:
        """
        url = "http://47.98.154.124:8888/crawler/wei_xin/detail"
        payload = json.dumps({
            "content_link": content_link,
            "is_count": False,
            "is_ad": False
        })
        headers = {
            'Content-Type': 'application/json'
        }
        response = requests.request("POST", url, headers=headers, data=payload)
        return response.json()

    @classmethod
    @retry_on_none_data()
    def update_msg_list(cls, ghId, index):
        """
        :return:
        "http://47.98.154.124:8888/crawler/wei_xin/detail"
        """
        url = 'http://47.98.154.124:8888/crawler/wei_xin/blogger'
        payload = {
            'account_id': ghId,
            'cursor': index,
        }
        headers = {
            'Content-Type': 'application/json'
        }
        response = requests.post(url, headers=headers, data=json.dumps(payload))
        return response.json()

    @classmethod
    @retryAsyncOnNoneData()
    async def get_account_by_url(cls, content_url):
        """
        通过文章获取账号信息
        :param content_url:
        :return:
        """
        async with aiohttp.ClientSession() as session:
            async with session.post(
                    url='http://47.98.154.124:8888/crawler/wei_xin/account_info',
                    headers={'Content-Type': 'application/json'},
                    json={"content_link": content_url}
            ) as response:
                return await response.json()
        # response = requests.request(
        #     "POST",
        #     url='http://8.217.190.241:8888/crawler/wei_xin/account_info',
        #     headers={'Content-Type': 'application/json'},
        #     json={"content_link": content_url}
        # )
        # return response.json()