|
@@ -6,10 +6,28 @@ import datetime
|
|
|
import time
|
|
|
|
|
|
import requests
|
|
|
+import multiprocessing
|
|
|
|
|
|
from common.db import RedisClient
|
|
|
|
|
|
|
|
|
+def protect_spider_timeout(function):
|
|
|
+ """
|
|
|
+ 守护进程,在程序启动后的某一个时段内守护爬虫进程
|
|
|
+ :param function: 被守护的函数
|
|
|
+ :param hour: 守护时长 / hour
|
|
|
+ """
|
|
|
+ process = multiprocessing.Process(target=function)
|
|
|
+ process.start()
|
|
|
+ while True:
|
|
|
+ if not process.is_alive():
|
|
|
+ process.terminate()
|
|
|
+ time.sleep(60)
|
|
|
+ process = multiprocessing.Process(target=function)
|
|
|
+ process.start()
|
|
|
+ time.sleep(60)
|
|
|
+
|
|
|
+
|
|
|
def bot(name):
|
|
|
"""
|
|
|
报警机器人
|
|
@@ -88,7 +106,8 @@ def monitor():
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
- monitor()
|
|
|
+ protect_spider_timeout(monitor)
|
|
|
+ # monitor()
|
|
|
# R = RedisClient()
|
|
|
# R.connect()
|
|
|
# # w = R.select("53-20240327")
|
|
@@ -97,4 +116,3 @@ if __name__ == '__main__':
|
|
|
# R.delete("53-20240327")
|
|
|
# R.delete("353-20240327")
|
|
|
# R.delete("352-20240327")
|
|
|
-
|