增加合成效率分布数据

app.py

@@ -23,7 +23,7 @@ registry = CollectorRegistry(auto_describe=False)
 
 # 定义后台执行调度器
 scheduler = BackgroundScheduler()
-scheduler.add_job(func=update_request_url_list(), trigger="interval", seconds=300)
+scheduler.add_job(func=update_request_url_list, trigger="interval", seconds=300)
 scheduler.start()
 atexit.register(lambda: scheduler.shutdown())
 
@@ -43,8 +43,9 @@ probe_http_pretransfer_time = Gauge("http_pretransfer_time", 'h5',['server_name'
 probe_http_first_byte_time = Gauge("http_first_byte_time", 'h5',['server_name'], registry=registry)
 slb_http_status_code = Gauge("slb_http_status_code", 'slb', ['server_name', 'status'], registry=registry)
 #视频合成
-produce_video_task_count = Gauge("produce_video_task_count", 'task_count', ['task_count'], registry=registry)
+produce_video_task_count = Gauge("produce_video_task_count", 'status', ['status'], registry=registry)
 produce_video_task_rate = Gauge("produce_video_task_rate", 'produce_video_rate', ['produce_video_rate'], registry=registry)
+produce_video_task_ratio = Gauge("produce_video_task_ratio", 'produce_video_ratio', ['ratio'], registry=registry)
 
 produce_video_tts_count = Gauge("tts_aliyun_azure", 'success', ['tts_channel'], registry=registry)
 logs_app_recommend_log_cnt_300 = Gauge("logs_app_recommend_log_null_cnt_300", "null cnt", ['cnt'], registry=registry)
@@ -178,6 +179,7 @@ def all_metric():
     """视频合成成功率"""
     start_time = int(time.strftime("%Y%m%d%H%M", time.localtime())) * 100000000000
     end_time = (int(time.strftime("%Y%m%d%H%M", time.localtime())) + 5) * 100000000000
+    print(start_time, end_time)
     query_sql = ("select round((successCount + processingCount1)/totalCount * 100,2) from  "
                 "(select count(*) as totalCount,"
                  "sum(case when produce_status in (5,6,7,8) then 1 else 0 end) as successCount,"
@@ -195,7 +197,10 @@ def all_metric():
     if res[0] is not None:
         produce_video_task_rate.labels("produce_video_task_rate").set(res[0])
     else:
-        produce_video_task_rate.labels("produce_video_task_rate").set(0)
+        produce_video_task_rate.labels("produce_video_task_rate").set(-1)
+
+
+
 
     """今日视频合成任务数"""
 
@@ -206,7 +211,45 @@ def all_metric():
     """今日视频合成失败数量"""
 
     """视频合成中任务数量"""
-#
+
+    res = produce_video_task_cnt()
+    if res:
+        if res[0]:
+            total_cnt = res[0]
+            produce_video_task_count.labels("total").set(total_cnt)
+        if res[1]:
+            success_cnt = res[1]
+            produce_video_task_count.labels("success").set(success_cnt)
+        if res[2]:
+            fail_cnt = res[2]
+            produce_video_task_count.labels("fail").set(fail_cnt)
+        if res[3]:
+            process_cnt = res[3]
+            produce_video_task_count.labels("process").set(process_cnt)
+        if res[4]:
+            r1 = res[4]
+            produce_video_task_ratio.labels("<0.5").set(r1)
+        if res[5]:
+            r2 = res[5]
+            produce_video_task_ratio.labels("0.5-0.7").set(r2)
+        if res[5]:
+            r3 = res[6]
+            produce_video_task_ratio.labels("0.7-1.0").set(r3)
+
+        if res[5]:
+            r4 = res[7]
+            produce_video_task_ratio.labels("1.0-1.5").set(r4)
+
+        if res[5]:
+            r5 = res[8]
+            produce_video_task_ratio.labels("1.5-2.0").set(r5)
+
+        if res[5]:
+            r6 = res[9]
+            produce_video_task_ratio.labels(">2.0").set(r6)
+
+
+
 #     sql = "select v1 as 平均合成耗时,v2 as 平均视频时长, round(v2/v1,1) as 时长耗时比 from  (select avg(produce_done_timestamp - submit_timestamp) as v1, avg(video_duration/1000) as v2 from produce_video_project where project_id > {} and project_id < {} and app_type not in (13,15) and produce_status in (5,6,7,8)) as t1".format(start_time, end_time)
 # #tts
 #     res = db_query(sql)
@@ -244,7 +287,6 @@ def all_metric():
             for i in range(len(res)):
                 status = res[i]['status']
                 cnt = float(res[i]['cnt'])
-                print(cnt)
                 slb_http_status_code.labels(name, status).set(cnt)
 
     #h5 healthcheck
@@ -281,18 +323,19 @@ def all_metric():
     logs_app_recommend_log_cnt_300.labels("recommend").set(res_null_cnt)
     # # logs_app_recommend_log_cnt_all.labels("recommend").inc(1)
     # logs_app_recommend_log_cnt_all.inc(res_null_cnt)
-
-
     return Response(generate_latest(registry), mimetype="text/plain")
+
 @app.route('/qps_avgtime/metrics')
 def qps_avgtime_metrics():
+
+    """统计接口QPM， RT时间，RT时间分布 """
     res, url = app_openapi_qps_avgtime_count()
     qps =  int(res.body[0]["cnt"])
     avgtime = int(float(res.body[0]["avg_time"]))
     app_type_temp = 1
     url_http_qps.labels(app_type_temp, url).set(qps)
     url_http_avgtime.labels(app_type_temp, url).set(avgtime)
-    app_type = ['1', '0', '4', '5', '6', '12', '13']
+    app_type = ['1', '0', '4', '5', '6', '12', '13', '15']
     for i in range(len(app_type)):
         appType = app_type[i]
         res = count_qps_avgtime(appType)
@@ -329,12 +372,12 @@ def qps_avgtime_metrics():
 
         """接口error统计"""
         res = error_cnt(appType)
-        for i in range(len(res)):
+        for i in range(len(res.body)):
             cnt = res.body[i]["cnt"]
             error_code = res.body[i]["resultCode"]
             url_http_error_code_cnt.labels(appType,error_code).set(cnt)
 
-            
+
     return Response(generate_latest(registry), mimetype="text/plain")
 
 
@@ -351,5 +394,5 @@ def produce_video_data():
 
 
 if __name__ == '__main__':
-    # app.run()
-    app.run(host='192.168.201.1', port=9091)
+    app.run()
+    # app.run(host='192.168.201.1', port=9091)

test.py

@@ -43,12 +43,30 @@ def  requesl_url_list(apptype):
 
 
     return  res
+
+
+def produce_video_data():
+    start_time = (int(time.strftime("%Y%m%d%H%M", time.localtime())) - 30) * 100000000000
+    end_time = (int(time.strftime("%Y%m%d%H%M", time.localtime()))) * 100000000000
+    sum_sql = ("select count(*) as totalCount from"              
+              "(select t1.project_id, t1.produce_status "
+              "from produce_video_project t1 "
+              "left join produce_video_project_connect_time t2 on t1.project_id = t2.project_id "
+              " where t1.project_id > %s and t1.project_id < %s and t1.app_type not in (1,13,15)) s1) ss1"%(start_time,end_time)
+               )
+
 if __name__ == '__main__':
     # with session_maker() as session:
     #     instance_info = session.query(InstanceList).filter(InstanceList.status==0).all()
     #     for index in range(len(instance_info)):
     #         print( instance_info[index].ipadd)
-
+    HOST = 'rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com'
+    PORT = '3306'
+    DATABASE = 'devops'
+    USERNAME = 'devops'
+    PASSWORD = 'devops@123456'
+    conn = pymysql.connect(host=HOST, user=USERNAME, password=PASSWORD, database=DATABASE, charset="utf8")
+    cursor = conn.cursor()
     requestUri_list = [
         '/longvideoapi/sys/config',
         '/longvideoapi/video/played',
@@ -166,12 +184,9 @@ if __name__ == '__main__':
     #     if status == "running":
     # #         status = 1
     # for url in requestUri_list:
-    #     IntfaceInfo = None
-    #     IntfaceInfo = IntfaceList()
-    #     with session_maker() as session:
-    #             IntfaceInfo.app_type = "1"
-    #             IntfaceInfo.interface_url = url
-    #             session.add(IntfaceInfo)
+    #     sql = "UPDATE `reuqest_url_list`  set  `level`=0 WHERE `request_url` ='%s' and `apptype` in (1,13)"%(url)
+    #     cursor.execute(sql)
+    #     conn.commit()
     # #
     # c = pycurl.Curl()
     # c.setopt(pycurl.URL, "https://longvideoh5.piaoquantv.com/core/share?shareSource=customerMessage&fromAppType=0&qrAppType=0&versionCode=321&shareUid=12463024&shareMachineCode=weixin_openid_o0w175fPwp8yrtOGihYJhvnT9Ag4&h5WxrootPageSource=vlog-pages___category&videoId=2689415&isRecommendShare=1&h5ShareId=backend493cd67dd28f4ee395781d59881567211625976055926&shareDepth=0&state=/")
@@ -196,57 +211,108 @@ if __name__ == '__main__':
     # end_time = (int(time.strftime("%Y%m%d%H%M", time.localtime())) + 5) * 100000000000
     # print(start_time , end_time)
 
-    HOST = 'rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com'
-    PORT = '3306'
-    DATABASE = 'devops'
-    USERNAME = 'devops'
-    PASSWORD = 'devops@123456'
-    conn = pymysql.connect(host=HOST, user=USERNAME, password=PASSWORD, database=DATABASE, charset="utf8")
-    cursor = conn.cursor()
 
-    appType = ['0', '4', '5', '6', '12' ,'15']
-    for index in range(len(appType)):
-        type = appType[index]
-        res = count_avg_max(type)
-        for i in range(len(res.body)):
-            url = res.body[i]["requestUri"]
-            cnt = res.body[i]["cnt"]
-            max_time = res.body[i]["max_time"]
-            avg_time = res.body[i]["avg_time"]
-            sql = "UPDATE `reuqest_url_list`  set `qps`=%s, `max_time`=%s, `avg_time`=%s WHERE `request_url` ='%s' and `apptype`=%s"%(cnt, max_time, avg_time, url, type)
-            cursor.execute(sql)
-            conn.commit()
-        for i in range(len(res.body)):
-            url = res.body[i]["requestUri"]
-            cnt = res.body[i]["cnt"]
-            sql = "UPDATE `reuqest_url_list`  set `rt_1`=%s WHERE `request_url` ='%s' and `apptype`=%s"%(cnt, url, type)
-            cursor.execute(sql)
-            conn.commit()
-
-        res = count_rt_less_time_count(type, 200, 500)
-        for i in range(len(res.body)):
-            url = res.body[i]["requestUri"]
-            cnt = res.body[i]["cnt"]
-            sql = "UPDATE `reuqest_url_list`  set `rt_2`=%s WHERE `request_url` ='%s' and `apptype`=%s" % (cnt, url, type)
-            cursor.execute(sql)
-            conn.commit()
-
-        res = count_rt_less_time_count(type, 500, 1000)
-        for i in range(len(res.body)):
-            url = res.body[i]["requestUri"]
-            cnt = res.body[i]["cnt"]
-            sql = "UPDATE `reuqest_url_list`  set `rt_3`=%s WHERE `request_url` ='%s' and `apptype`=%s" % (cnt, url, type)
-            cursor.execute(sql)
-            conn.commit()
-
-        res = count_rt_less_time_count(type, 1000, 10000)
-        for i in range(len(res.body)):
-            url = res.body[i]["requestUri"]
-            cnt = res.body[i]["cnt"]
-            sql = "UPDATE `reuqest_url_list`  set `rt_4`=%s WHERE `request_url` ='%s' and `apptype`=%s" % (cnt, url, type)
-            cursor.execute(sql)
-            conn.commit()
 
+    #
+    # appType = ['0', '4', '5', '6', '12' ,'15','1','13']
+    #
+    # #
+    # for index in range(len(appType)):
+    #     type = appType[index]
+    # #     res = requesl_url_list(type)
+    # #     for i in range(len(res.body)):
+    # #         url = res.body[i]["requestUri"]
+    # #         sql = "insert into  `reuqest_url_list`  (`apptype`, `request_url`) VALUES (%s, '%s')"%(type, url)
+    # #         print(sql)
+    # #         cursor.execute(sql)
+    # #         conn.commit()
+    #
+    #
+    #
+    #     res = count_avg_max(type)
+    #     for i in range(len(res.body)):
+    #         url = res.body[i]["requestUri"]
+    #         cnt = res.body[i]["cnt"]
+    #         max_time = res.body[i]["max_time"]
+    #         avg_time = res.body[i]["avg_time"]
+    #         sql = "UPDATE `reuqest_url_list`  set `qps`=%s, `max_time`=%s, `avg_time`=%s WHERE `request_url` ='%s' and `apptype`=%s"%(cnt, max_time, avg_time, url, type)
+    #         cursor.execute(sql)
+    #         conn.commit()
+    #     for i in range(len(res.body)):
+    #         url = res.body[i]["requestUri"]
+    #         cnt = res.body[i]["cnt"]
+    #         sql = "UPDATE `reuqest_url_list`  set `rt_1`=%s WHERE `request_url` ='%s' and `apptype`=%s"%(cnt, url, type)
+    #         cursor.execute(sql)
+    #         conn.commit()
+    #
+    #     res = count_rt_less_time_count(type, 200, 500)
+    #     for i in range(len(res.body)):
+    #         url = res.body[i]["requestUri"]
+    #         cnt = res.body[i]["cnt"]
+    #         sql = "UPDATE `reuqest_url_list`  set `rt_2`=%s WHERE `request_url` ='%s' and `apptype`=%s" % (cnt, url, type)
+    #         cursor.execute(sql)
+    #         conn.commit()
+    #
+    #     res = count_rt_less_time_count(type, 500, 1000)
+    #     for i in range(len(res.body)):
+    #         url = res.body[i]["requestUri"]
+    #         cnt = res.body[i]["cnt"]
+    #         sql = "UPDATE `reuqest_url_list`  set `rt_3`=%s WHERE `request_url` ='%s' and `apptype`=%s" % (cnt, url, type)
+    #         cursor.execute(sql)
+    #         conn.commit()
+    #
+    #     res = count_rt_less_time_count(type, 1000, 10000)
+    #     for i in range(len(res.body)):
+    #         url = res.body[i]["requestUri"]
+    #         cnt = res.body[i]["cnt"]
+    #         sql = "UPDATE `reuqest_url_list`  set `rt_4`=%s WHERE `request_url` ='%s' and `apptype`=%s" % (cnt, url, type)
+    #         cursor.execute(sql)
+    #         conn.commit()
+    #
+    #
+
+    # start_time = int(time.strftime("%Y%m%d%H%M", time.localtime())) * 100000000000
+    # end_time = (int(time.strftime("%Y%m%d%H%M", time.localtime())) + 5) * 100000000000
+    # query_sql = ("select round((successCount + processingCount1)/totalCount * 100,2) from  "
+    #             "(select count(*) as totalCount,"
+    #              "sum(case when produce_status in (5,6,7,8) then 1 else 0 end) as successCount,"
+    #              "sum(case when produce_status = 99 then 1 else 0 end) as failCount , "
+    #              "sum(case when produce_status in(0,1,2,3,4) then 1 else 0 end) as processingCount,"
+    #              "sum(case when produce_status in(0,1,2,3,4) and (rate < 1 or rate is null) then 1 else 0 end) processingCount1,"
+    #              "sum(case when produce_status in(0,1,2,3,4) and rate >= 1 then 1 else 0 end) processingCount2  from  "
+    #              "(select t1.project_id, t1.produce_status, round((t2.last_connect_timestamp - t1.submit_timestamp) / (t1.video_duration/1000), 1) as rate  from produce_video_project t1  "
+    #              "left join produce_video_project_connect_time t2 on t1.project_id = t2.project_id "
+    #              "where t1.project_id > %s and t1.project_id < %s and t1.app_type not in (1,13,15)) s1) ss1" %(start_time ,end_time)
+    #
+    #             )
+    # res = db_query(query_sql)
+    #
+    # if res[0] is not None:
+    #     print(res[0])
+
+
+    start_time = int(time.strftime("%Y%m%d", time.localtime())) * 1000000000000000
+    end_time = int(time.strftime("%Y%m%d%H%M", time.localtime())) * 100000000000
+    query_sql = ("select count(*) as totalCount,"
+                 "sum(case when produce_status in (5,6,7,8) then 1 else 0 end) as successCount,"
+                 "sum(case when produce_status = 99 then 1 else 0 end) as failCount , "
+                 "sum(case when produce_status in(5,6,7,8) then 1 else 0 end) as processingCount,"
+                 "sum(case when produce_status in(5,6,7,8) and (rate < 0.5 or rate is null) then 1 else 0 end) processingCount1,"
+                 "sum(case when produce_status in(5,6,7,8) and rate >= 0.5 and  rate < 0.7 then 1 else 0 end) processingCount2  ,"
+                 "sum(case when produce_status in(5,6,7,8) and rate >= 0.7 and  rate < 1 then 1 else 0 end) processingCount3 ,"
+                 "sum(case when produce_status in(5,6,7,8) and rate >= 1 and  rate < 1.5 then 1 else 0 end) processingCount4  ,"
+                 "sum(case when produce_status in(5,6,7,8) and rate >= 1.5 and  rate < 2 then 2 else 0 end) processingCount5 ,"
+                 "sum(case when produce_status in(5,6,7,8) and  rate >2  then 1 else 0 end) processingCount6  from  "
+                 "(select t1.project_id, t1.produce_status, round((t2.last_connect_timestamp - t1.submit_timestamp) / (t1.video_duration/1000), 1) as rate  from produce_video_project t1  "
+                 "left join produce_video_project_connect_time t2 on t1.project_id = t2.project_id "
+                 "where t1.project_id > %s and t1.project_id < %s and t1.app_type not in (1,13,15)) s1 " %(start_time ,end_time)
+
+                )
+    res = db_query(query_sql)
+    print(res)
+    if res[0] is not None:
 
+        print(res[0],res[1],res[2],res[3],res[4], res[5],res[6],res[7],res[8],res[9])
+    print(start_time, end_time)
 
 

utils.py

@@ -449,4 +449,52 @@ def  update_request_url_list():
             cnt, url, type)
             cursor.execute(sql)
             conn.commit()
-# if __name__ == '__main__':
+
+
+def  produce_video_task_cnt():
+    start_time = int(time.strftime("%Y%m%d", time.localtime())) * 1000000000000000
+    end_time = int(time.strftime("%Y%m%d%H%M", time.localtime())) * 100000000000
+    query_sql = ("select count(*) as totalCount,"
+                 "sum(case when produce_status in (5,6,7,8) then 1 else 0 end) as successCount,"
+                 "sum(case when produce_status = 99 then 1 else 0 end) as failCount , "
+                 "sum(case when produce_status in(5,6,7,8) then 1 else 0 end) as processingCount,"
+                 "sum(case when produce_status in(5,6,7,8) and (rate < 0.5 or rate is null) then 1 else 0 end) processingCount1,"
+                 "sum(case when produce_status in(5,6,7,8) and rate >= 0.5 and  rate < 0.7 then 1 else 0 end) processingCount2  ,"
+                 "sum(case when produce_status in(5,6,7,8) and rate >= 0.7 and  rate < 1 then 1 else 0 end) processingCount3 ,"
+                 "sum(case when produce_status in(5,6,7,8) and rate >= 1 and  rate < 1.5 then 1 else 0 end) processingCount4  ,"
+                 "sum(case when produce_status in(5,6,7,8) and rate >= 1.5 and  rate < 2 then 2 else 0 end) processingCount5 ,"
+                 "sum(case when produce_status in(5,6,7,8) and  rate >2  then 1 else 0 end) processingCount6  from  "
+                 "(select t1.project_id, t1.produce_status, round((t2.last_connect_timestamp - t1.submit_timestamp) / (t1.video_duration/1000), 1) as rate  from produce_video_project t1  "
+                 "left join produce_video_project_connect_time t2 on t1.project_id = t2.project_id "
+                 "where t1.project_id > %s and t1.project_id < %s and t1.app_type not in (1,13,15)) s1 " %(start_time ,end_time)
+
+                )
+    res = db_query(query_sql)
+
+    return  res
+
+if __name__ == '__main__':
+    res = produce_video_task_cnt()
+    if res:
+        if res[0]:
+            total_cnt = res[0]
+        if res[1]:
+            success_cnt = res[1]
+        if res[2]:
+            fail_cnt = res[2]
+        if res[3]:
+            process_cnt = res[3]
+        if res[4]:
+            r1 = res[4]
+        if res[5]:
+            r2 = res[5]
+        if res[5]:
+            r3 = res[6]
+        if res[5]:
+            r4 = res[7]
+        if res[5]:
+            r5 = res[8]
+        if res[5]:
+            r6 = res[9]
+
+    print(total_cnt, success_cnt,fail_cnt,process_cnt,r1)