"""
独立执行,每个人执行一次,单日总量到达 300 即执行一次
"""
import json
import datetime
import time
import requests
import multiprocessing
from common.db import RedisClient
def protect_spider_timeout(function):
"""
守护进程,在程序启动后的某一个时段内守护爬虫进程
:param function: 被守护的函数
:param hour: 守护时长 / hour
"""
process = multiprocessing.Process(target=function)
process.start()
while True:
if not process.is_alive():
process.terminate()
time.sleep(60)
process = multiprocessing.Process(target=function)
process.start()
time.sleep(60)
def bot(name):
"""
报警机器人
"""
id_dict = {
"余海涛": "ou_b87d153e200a04de3d82b1b9276e8f90",
"范军": "ou_fce9cfef186e260e70554b47fee70a34",
"罗情": "ou_88139cd84c2d105c2e1d699c14ec3375",
"鲁涛": "ou_7986cccb78e6c981db8d0eef93443d05",
"王雪珂": "ou_2233fb8e1302314bae166fcfa144151f",
"邓锋": "ou_379d37645f929e1e6553a75aecda42a2"
}
url = "https://open.feishu.cn/open-apis/bot/v2/hook/df47bb77-ecaa-4628-b076-aae776415ae8"
headers = {"Content-Type": "application/json"}
payload = {
"msg_type": "interactive",
"card": {
"elements": [
{
"tag": "div",
"text": {
"content": "抓取数量触发限量通知, , , \n".format(
id_dict[name], id_dict["邓锋"], id_dict["王雪珂"]
),
"tag": "lark_md",
},
},
{
"tag": "div",
"text": {
"content": "当天已经入库 300 条视频",
"tag": "lark_md",
},
},
],
"header": {"title": {"content": "【 通知 】", "tag": "plain_text"}},
},
}
requests.post(url, headers=headers, data=json.dumps(payload))
def monitor():
"""
监测 redis 中数据
"""
keys = {"352": "余海涛", "353": "罗情", "53": "范军", "51": "鲁涛"}
now = datetime.datetime.now().time()
start_alert_time = datetime.time(10)
end_alert_time = datetime.time(20, 30)
while True:
if start_alert_time <= now <= end_alert_time:
try:
R = RedisClient()
if R.connect():
for key in keys:
count = R.select(key)
if count:
OO = int(count.decode("utf-8"))
name = keys[key]
redis_date_key = key + "-" + datetime.date.today().strftime("%Y%m%d")
if R.select(redis_date_key):
# 说明已经存储进去了, 不需要再报警了
continue
else:
if OO > 300:
R.insert(redis_date_key, "already bot", 86400)
# print("超过了, 报警", name)
bot(name)
else:
continue
except Exception as e:
print(e)
pass
# 查询一次之后等待 60 s
time.sleep(60)
if __name__ == '__main__':
protect_spider_timeout(monitor)
# monitor()
# R = RedisClient()
# R.connect()
# # w = R.select("53-20240327")
# # print(w)
# R.delete("51-20240327")
# R.delete("53-20240327")
# R.delete("353-20240327")
# R.delete("352-20240327")