piaoquan_crawler/common/limit.py

import os
import sys
from datetime import datetime, timedelta

sys.path.append(os.getcwd())

from common.aliyun_log import AliyunLogger
from common.scheduling_db import MysqlHelper
from common.db import RedisClient


def generate_expire_time():
    """
    计算出过期时间
    """
    now = datetime.now()

    # 当天晚上12点的时间
    midnight = datetime(now.year, now.month, now.day) + timedelta(days=1)

    # 计算当前时间到当天晚上12点的秒数
    seconds_until_midnight = int((midnight - now).total_seconds())
    return seconds_until_midnight


class AuthorLimit(object):
    """
    账号爬虫限量
    """

    def __init__(self, mode, platform):
        self.mode = mode
        self.platform = platform
        self.limit_tag_dict = {"352": "余海涛", "353": "罗情", "53": "范军", "51": "鲁涛", "131": False}

    def find_tag(self, uid):
        """
        通过 uid 去找符合标准的 tag
        """
        sql = f"""select tag from crawler_user_v3 where uid={uid};"""
        result = MysqlHelper.get_values(
            log_type=self.mode, crawler=self.platform, env="prod", sql=sql
        )
        tags = result[0]["tag"]
        if tags:
            tags = tags.split(",")
            if "131" in tags:
                return None
            else:
                for tag in tags:
                    if self.limit_tag_dict.get(tag):
                        return tag
        return None

    def author_limitation(self, user_id):
        """
        限制账号， 服务长沙四名同学
        """
        return True
        # if self.mode == "author":
        #     tag = self.find_tag(user_id)
        #     if tag:
        #         AliyunLogger.logging(
        #             code="8807",
        #             platform=self.platform,
        #             mode=self.mode,
        #             env="prod",
        #             message="找到个人账号，{}".format(tag)
        #         )
        #         R = RedisClient()
        #         if R.connect():
        #             tag_count = R.select(tag)
        #             if tag_count:
        #                 tag_count = int(tag_count.decode("utf-8"))
        #                 if tag_count <= 300:
        #                     tag_count += 1
        #                     expire_seconds = generate_expire_time()
        #                     R.insert(
        #                         key=tag, value=tag_count, expire_time=expire_seconds
        #                     )
        #                     return True
        #                 else:
        #                     # 报警
        #                     return False
        #             else:
        #                 tag_count = 1
        #                 expire_seconds = generate_expire_time()
        #                 R.insert(
        #                     key=tag, value=tag_count, expire_time=expire_seconds
        #                 )
        #                 return True
        #         else:
        #             return True
        #     else:
        #         return True
        # else:
        #     return True