research_task上线

research_app.py

@@ -1,27 +1,52 @@
 """
 @author: luojunhui
 """
+import datetime
 import time
 import json
-import schedule
 import pymysql
 import requests
-from datetime import datetime, timedelta
 
+from functions import MatchRate
 
-def find_defeat_info():
+
+def find_fail_data(dt):
     """
-    查找失败的视频
+    查找1h失败的数据
+    :param dt:
     :return:
     """
+    M = MatchRate()
+    time_stamp_list = M.generate_stamp_list(dt, dt)
+    for item in time_stamp_list:
+        s_d = int(item)
+        e_d = int(item) + 24 * 60 * 60 * 1000
+        result = M.match_rate(s_d, e_d)
+        s = 0
+        f = 0
+        p = 0
+        w = []
+        for obj in result:
+            if obj[0] == 2:
+                s += 1
+            elif obj[0] == 3:
+                f += 1
+            elif obj[0] == 1:
+                p += 1
+                w.append(obj[1])
+        long_time_data = [i for i in w if int(time.time()) - int(i.split("-")[-1]) > 3600]
+        return tuple(long_time_data)
 
-    now_dt = datetime.utcfromtimestamp(int(time.time()) - 1 * 60 * 60)
-    beijing_time = now_dt + timedelta(hours=8)
-    today_dt = datetime.today().strftime("%Y-%m-%d")
+
+def find_defeat_info(trace_id_tuple):
+    """
+    查找失败的视频
+    :return:
+    """
     select_sql = f"""
-    select trace_id, article_title, article_text, gh_id, account_name 
-    from long_articles_video 
-    where update_time < '{beijing_time}' and update_time > '{today_dt}' and success = 0;"""
+        select trace_id, article_title, article_text, gh_id, account_name 
+        from `long_articles_video` where `trace_id` in {trace_id_tuple};
+    """
     connection = pymysql.connect(
         host="rm-bp1159bu17li9hi94.mysql.rds.aliyuncs.com",  # 数据库IP地址，内网地址
         port=3306,  # 端口号
@@ -36,40 +61,53 @@ def find_defeat_info():
     return fail_list
 
 
+def request_for_research(result):
+    """
+    research from new machine
+    :param result:
+    """
+    params = {
+        "trace_id": result[0],
+        "title": result[1],
+        "ghId": result[3],
+        "content": result[2],
+        "accountName": result[4]
+
+    }
+    url = "http://47.99.132.47:8111/re_search_videos"
+    a = time.time()
+    header = {
+        "Content-Type": "application/json",
+    }
+    response = requests.post(url, json=params, headers=header, timeout=600)
+    b = time.time()
+    print("total cost: ", b - a, " s")
+    print(json.dumps(response.json(), ensure_ascii=False, indent=4))
+
+
 def job2():
     """
     定时任务
     :return:
     """
-    fail_list = find_defeat_info()
+    date_str = datetime.datetime.today().strftime("%Y%m%d")
+    trace_id_t = find_fail_data(dt=date_str)
+    fail_list = find_defeat_info(trace_id_t)
+    now_time_str = datetime.datetime.now().__str__()
     if fail_list:
-        for result in fail_list:
-            params = {
-                "trace_id": result[0],
-                "title": result[1],
-                "ghId": result[3],
-                "content": result[2],
-                "accountName": result[4]
-
-            }
-            url = "http://61.48.133.26:8111/re_search_videos"
-            a = time.time()
-            header = {
-                "Content-Type": "application/json",
-            }
-
-            response = requests.post(url, json=params, headers=header, timeout=600)
-            b = time.time()
-            print(response.text)
-            print(b - a)
-            print(json.dumps(response.json(), ensure_ascii=False, indent=4))
-            time.sleep(20)
+        print("{} find {} defeat requests".format(now_time_str, len(fail_list)))
+        count = 1
+        for obj in fail_list:
+            request_for_research(obj)
+            count += 1
+        print("{} success re_search {} defeat requests".format(now_time_str, count))
     else:
-        print("No videos")
+        print("{} No videos Find".format(now_time_str))
 
 
 if __name__ == '__main__':
-    schedule.every().hour.do(job2)
     while True:
-        schedule.run_pending()
-        time.sleep(1)
+        now_time_str = datetime.datetime.now().__str__()
+        job2()
+        print("{}: 执行程序完成, 等待一小时".format(now_time_str))
+        time.sleep(60 * 60)

test.py

@@ -1,48 +1,6 @@
-"""
-@author: luojunhui
-CREATE TABLE `publish_content_miniprogram` (
-  `publish_content_id` varchar(64) NOT NULL COMMENT '发布内容ID',
-  `root_share_id` varchar(64) DEFAULT NULL,
-  `source` varchar(64) DEFAULT NULL COMMENT '来源',
-  `program_id` varchar(64) DEFAULT NULL COMMENT '小程序ID',
-  `program_name` varchar(128) DEFAULT NULL COMMENT '小程序名称',
-  `program_avatar` varchar(512) DEFAULT NULL COMMENT '小程序头像',
-  `production_cover` varchar(1024) DEFAULT NULL COMMENT '卡片封面',
-  `production_name` varchar(255) DEFAULT NULL COMMENT '卡片标题',
-  `production_path` varchar(1024) DEFAULT NULL COMMENT '卡片路径',
-  `video_url` varchar(1024) DEFAULT NULL COMMENT '视频播放地址',
-  `hide_flag` int(11) DEFAULT NULL COMMENT '是否隐藏（0-否，1-是）',
-  `status` int(11) NOT NULL DEFAULT '2' COMMENT '状态（0-待处理，1-处理中，2-成功，3-失败）',
-  `trace_id` varchar(128) DEFAULT NULL COMMENT '请求ID',
-  `error_msg` varchar(2048) DEFAULT NULL COMMENT '错误信息',
-  `create_timestamp` bigint(20) DEFAULT NULL COMMENT '创建时间戳',
-  `update_timestamp` bigint(20) DEFAULT NULL COMMENT '更新时间戳',
-  PRIMARY KEY (`publish_content_id`),
-  KEY `idx_rootShareId` (`root_share_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='发布内容-插入小程序'
-"""
-import json
+import datetime
 
-import pandas as pd
-from datetime import datetime
-from functions import RateDetail, MatchRate
+date_str = datetime.datetime.today().strftime("%Y%m%d")
 
-M = MatchRate()
-R = RateDetail()
-time_stamp_list = M.generate_stamp_list("20240529", "20240530")
-df = []
-for item in time_stamp_list:
-    s_d = int(item)
-    e_d = int(item) + 24 * 60 * 60 * 1000
-    result = M.match_rate(s_d, e_d)
-    s = 0
-    f = 0
-    p = 0
-    for obj in result:
-        if obj[0] == 2:
-            s += 1
-        elif obj[0] == 3:
-            f += 1
-        elif obj[0] == 1:
-            p += 1
-    print(s, f, p)
+
+print(date_str)