import os
import sys
import json
import requests
from datetime import datetime, timedelta
sys.path.append(os.getcwd())
from common.aliyun_log import AliyunLogger
from common.scheduling_db import MysqlHelper
from common.db import RedisClient
def generate_expire_time():
"""
计算出过期时间
"""
now = datetime.now()
# 当天晚上12点的时间
midnight = datetime(now.year, now.month, now.day) + timedelta(days=1)
# 计算当前时间到当天晚上12点的秒数
seconds_until_midnight = int((midnight - now).total_seconds())
return seconds_until_midnight
def bot(name):
"""
报警机器人
"""
id_dict = {
"余海涛": "15869772902",
"范军": "15200827642",
"罗情": "15111037095",
"鲁涛": "18573105114",
"王雪珂": "13513479926",
"邓锋": "18175188134",
}
url = "https://open.feishu.cn/open-apis/bot/v2/hook/c273ff48-7b7e-4078-b3f7-d69a3d262df9"
headers = {"Content-Type": "application/json"}
payload = {
"msg_type": "interactive",
"card": {
"elements": [
{
"tag": "div",
"text": {
"content": "抓取数量触发限量通知, , , \n".format(
id_dict[name], id_dict["邓锋"], id_dict["王雪珂"]
),
"tag": "lark_md",
},
},
{
"tag": "div",
"text": {
"content": "当天已经入库 300 条视频",
"tag": "lark_md",
},
},
],
"header": {"title": {"content": "【 通知 】", "tag": "plain_text"}},
},
}
requests.post(url, headers=headers, data=json.dumps(payload))
class AuthorLimit(object):
"""
账号爬虫限量
"""
def __init__(self, mode, platform):
self.mode = mode
self.platform = platform
self.limit_tag_dict = {"352": "余海涛", "353": "罗情", "53": "范军", "51": "鲁涛"}
def find_tag(self, uid):
"""
通过 uid 去找符合标准的 tag
"""
sql = f"""select tag from crawler_user_v3 where uid={uid};"""
result = MysqlHelper.get_values(
log_type=self.mode, crawler=self.platform, env="prod", sql=sql
)
tags = result[0]["tag"]
if tags:
tags = tags.split(",")
for tag in tags:
if self.limit_tag_dict.get(tag):
return tag
return None
def author_limitation(self, user_id):
"""
限制账号, 服务长沙四名同学
"""
if self.mode == "author":
tag = self.find_tag(user_id)
if tag:
AliyunLogger.logging(
code="8807",
platform=self.platform,
mode=self.mode,
env="prod",
message="找到个人账号,{}".format(tag)
)
R = RedisClient()
if R.connect():
tag_count = R.select(tag)
if tag_count:
tag_count = int(tag_count.decode("utf-8"))
if tag_count <= 300:
tag_count += 1
expire_seconds = generate_expire_time()
R.insert(
key=tag, value=tag_count, expire_time=expire_seconds
)
return True
else:
# 报警
name = self.limit_tag_dict[tag]
bot(name)
return False
else:
tag_count = 1
expire_seconds = generate_expire_time()
R.insert(
key=tag, value=tag_count, expire_time=expire_seconds
)
return True
else:
return True
else:
return True
else:
return True