import os import sys from datetime import datetime, timedelta sys.path.append(os.getcwd()) from common.aliyun_log import AliyunLogger from common.scheduling_db import MysqlHelper from common.db import RedisClient def generate_expire_time(): """ 计算出过期时间 """ now = datetime.now() # 当天晚上12点的时间 midnight = datetime(now.year, now.month, now.day) + timedelta(days=1) # 计算当前时间到当天晚上12点的秒数 seconds_until_midnight = int((midnight - now).total_seconds()) return seconds_until_midnight class AuthorLimit(object): """ 账号爬虫限量 """ def __init__(self, mode, platform): self.mode = mode self.platform = platform self.limit_tag_dict = {"352": "余海涛", "353": "罗情", "53": "范军", "51": "鲁涛", "131": False} def find_tag(self, uid): """ 通过 uid 去找符合标准的 tag """ sql = f"""select tag from crawler_user_v3 where uid={uid};""" result = MysqlHelper.get_values( log_type=self.mode, crawler=self.platform, env="prod", sql=sql ) tags = result[0]["tag"] if tags: tags = tags.split(",") if "131" in tags: return None else: for tag in tags: if self.limit_tag_dict.get(tag): return tag return None def author_limitation(self, user_id): """ 限制账号, 服务长沙四名同学 """ if self.mode == "author": tag = self.find_tag(user_id) if tag: AliyunLogger.logging( code="8807", platform=self.platform, mode=self.mode, env="prod", message="找到个人账号,{}".format(tag) ) R = RedisClient() if R.connect(): tag_count = R.select(tag) if tag_count: tag_count = int(tag_count.decode("utf-8")) if tag_count <= 300: tag_count += 1 expire_seconds = generate_expire_time() R.insert( key=tag, value=tag_count, expire_time=expire_seconds ) return True else: # 报警 return False else: tag_count = 1 expire_seconds = generate_expire_time() R.insert( key=tag, value=tag_count, expire_time=expire_seconds ) return True else: return True else: return True else: return True