update

app.py

@@ -25,14 +25,13 @@ registry = CollectorRegistry(auto_describe=False)
 scheduler = BackgroundScheduler()
 scheduler.add_job(func=update_request_url_list, trigger="interval", seconds=300)
 scheduler.add_job(produce_video_ratio_cnt,  'cron', hour='00', minute='03')
-
 scheduler.start()
 atexit.register(lambda: scheduler.shutdown())
 
 client = AcsClient('LTAI4GBWbFvvXoXsSVBe1o9f', 'kRAikWitb4kDxaAyBqNrmLmllMEDO3', 'cn-hangzhou')
 
 
-healthcheck_status = Gauge("healthcheck_status", "ipaddress", ['instance_id','server_name', 'ipaddress'], registry=registry)
+healthcheck_status = Gauge("healthcheck_status", "ipaddress", ['server_name', 'ipaddress'], registry=registry)
 url_http_avgtime = Gauge("url_http_times_avgs", "url  of avgs", ['appType', 'url'], registry=registry)
 url_http_qps = Gauge("url_http_qps", "url  of qps", ['appType','url'], registry=registry)
 url_http_expendtime_summary =  Gauge("url_http_expendtime_summary", "expendtime   summary", ['appType', 'url', 'duration'], registry=registry)
@@ -345,6 +344,11 @@ def all_metric():
     logs_app_recommend_log_cnt_300.labels("recommend").set(res_null_cnt)
     # # logs_app_recommend_log_cnt_all.labels("recommend").inc(1)
     # logs_app_recommend_log_cnt_all.inc(res_null_cnt)
+
+    # """    所有服务health_check  """
+
+
+
     return Response(generate_latest(registry), mimetype="text/plain")
 
 @app.route('/qps_avgtime/metrics')
@@ -402,7 +406,21 @@ def qps_avgtime_metrics():
 
     return Response(generate_latest(registry), mimetype="text/plain")
 
+@app.route('/svc/healthcheck')
+def svc_healthcheck():
+    sql = "select ipadd ,server_name from instance_list where  status=1"
+    res = db_query_devlop(sql)
+    if res is not None:
+        for i in range(len(res)):
+            ipaddr = res[i][0]
+            svc_name = res[i][1]
+            print(ipaddr, svc_name)
+            http_code = healthcheck(ipaddr, svc_name)
+            print(http_code)
+            healthcheck_status.labels(ipaddr, svc_name).set(http_code)
+
 
+    return Response(generate_latest(registry), mimetype="text/plain")
 
 def produce_video_data():
     start_time = (int(time.strftime("%Y%m%d%H%M", time.localtime())) - 30) * 100000000000
@@ -415,10 +433,7 @@ def produce_video_data():
                )
 
 
-@app.route('/healthcheck')
-def healthcheck():
 
-    return "OK"
 
 
 if __name__ == '__main__':

utils.py

@@ -16,8 +16,12 @@ import  pymysql
 from sqlalchemy.sql import func
 from model_longvideo import produce_video_project
 from datetime import date, timedelta
+from DBSession import  session_maker
+
+
 
 client = AcsClient('LTAI4GBWbFvvXoXsSVBe1o9f', 'kRAikWitb4kDxaAyBqNrmLmllMEDO3', 'cn-hangzhou')
+from prometheus_client import Gauge,Counter, generate_latest
 
 
 
@@ -43,9 +47,9 @@ def intances_list_update():
         server_name = instance_info["Instances"]["Instance"][i]["Tags"]["Tag"][0]["TagValue"]
         status = instance_info["Instances"]["Instance"][i]["Status"]
         instance_name = instance_info["Instances"]["Instance"][i]["HostName"]
-        if status == "running":
+        if status == "Running":
             status = 1
-        instance_insert(instance_id, ipaddr, instance_name, server_name, status)
+            instance_insert(instance_id, ipaddr, instance_name, server_name, status)
 
 
     return "OK"
@@ -73,38 +77,87 @@ def instance_insert(instance_id, ipadd, instance_name, server_name, status):
 def healthcheck(ipaddr,server_name):
     """返回容器upload时间，容器运行状态，健康坚持接口返回值，"""
     svc_name = server_name
-    print(server_name)
     if svc_name == "longvideoapi.prod" :
         url = "http://{}:8080/longvideoapi/test".format(ipaddr)
         res = requests.post(url, timeout=5)
         print(res.status_code)
         return res.status_code
-    if svc_name == "commonapi.prod":
+    elif svc_name == "commonapi.prod":
         url = "http://{}:8080/commonapi/test".format(ipaddr)
         res = requests.post(url, timeout=5)
         return res.status_code
-    if svc_name == "speed.prod":
+    elif svc_name == "speed.prod":
         url = "http://{}:8080/longvideoapi/test".format(ipaddr)
         res = requests.post(url, timeout=5)
         return res.status_code
-    if svc_name == "clipapi.prod":
+    elif svc_name == "clipapi.prod":
         url = "http://{}:8080/longvideoapi/test".format(ipaddr)
         res = requests.post(url, timeout=5)
         return res.status_code
-    if svc_name == "distribution.prod":
+    elif svc_name == "distribution.prod":
         url = "http://{}:8080/healthcheck".format(ipaddr)
         res = requests.post(url, timeout=5)
         return res.status_code
-    if svc_name == "manager.prod":
+    elif svc_name == "manager.prod":
         url = "http://{}:8971/dev/test/healthcheck".format(ipaddr)
         res = requests.post(url, timeout=5)
         return res.status_code
-    if svc_name == "recommend-queue.prod":
+    elif svc_name == "recommend-queue.prod":
+        url = "http://{}:8080".format(ipaddr)
+        res = requests.post(url, timeout=5)
+        return res.status_code
+
+    elif svc_name == "recommend-api.prod":
         url = "http://{}:8080/healthcheck".format(ipaddr)
         res = requests.post(url, timeout=5)
         return res.status_code
 
+    elif svc_name == "recommend-rov.prod":
+        url = "http://{}:8080/healthcheck".format(ipaddr)
+        res = requests.post(url, timeout=5)
+        return res.status_code
 
+    elif svc_name == "measure-queue.prod":
+        url = "http://{}:8080/healthcheck".format(ipaddr)
+        res = requests.post(url, timeout=5)
+        return res.status_code
+
+    elif svc_name == "message-queue.prod":
+        url = "http://{}:8080/healthcheck".format(ipaddr)
+        res = requests.post(url, timeout=5)
+        return res.status_code
+
+    elif svc_name == "search-material.prod":
+        url = "http://{}:8089/v1/search/test".format(ipaddr)
+        res = requests.post(url, timeout=5)
+        return res.status_code
+
+    elif svc_name == "spider.prod":
+        url = "http://{}:9990/version".format(ipaddr)
+        try:
+            res = requests.post(url, timeout=5)
+        except Exception as e:
+            return  -1
+        return res.status_code
+
+    elif svc_name == "video-job.prod":
+        url = "http://{}:8080/healthcheck".format(ipaddr)
+        res = requests.post(url, timeout=5)
+        return res.status_code
+
+
+    elif svc_name == "combine-job.prod":
+        url = "http://{}:8080/healthcheck".format(ipaddr)
+        res = requests.post(url, timeout=5)
+        return res.status_code
+
+    elif svc_name == "recommend-server.prod":
+        url = "http://{}:8080/healthcheck".format(ipaddr)
+        # res = requests.post(url, timeout=5)
+        return 200
+
+    else:
+        return 200
 
 # def  healthcheck_metirc(instance_id,ipaddr,server_name,http_code):
 #     healthcheck_status = Gauge("healthcheck_status", "http_code", ['instance_id', 'server_name', 'ipaddress'],
@@ -276,7 +329,18 @@ def  db_query(sql):
     cursor.execute(sql)
     res = cursor.fetchone()
 
+    return  res
 
+def db_query_devlop(sql):
+    HOST = 'rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com'
+    PORT = '3306'
+    DATABASE = 'devops'
+    USERNAME = 'devops'
+    PASSWORD = 'devops@123456'
+    conn = pymysql.connect(host=HOST, user=USERNAME, password=PASSWORD, database=DATABASE, charset="utf8")
+    cursor = conn.cursor()
+    cursor.execute(sql)
+    res = cursor.fetchall()
 
     return  res
 def  db_query_prod(sql):
@@ -499,8 +563,8 @@ def  produce_video_ratio_cnt():
     conn = pymysql.connect(host=HOST, user=USERNAME, password=PASSWORD, database=DATABASE, charset="utf8")
     cursor = conn.cursor()
     start_time = int((date.today() + timedelta(days=-1)).strftime("%Y%m%d")) * 1000000000000000
-    end_time = int(date.today().strftime("%Y%m%d")) * 1000000000000000
-    # end_time = int((date.today() + timedelta(days=0)).strftime("%Y%m%d")) * 1000000000000000
+    # end_time = int(date.today().strftime("%Y%m%d")) * 1000000000000000
+    end_time =   int((date.today() + timedelta(days=0)).strftime("%Y%m%d")) * 1000000000000000
 
     sql = ("select count(*) as totalCount, "
             "sum(case when rate < 0.5 then 1 else 0 end) as r1,"
@@ -551,5 +615,7 @@ def  produce_video_ratio_cnt():
     )
     cursor.execute(sql)
     conn.commit()
-if __name__ == '__main__':
-    produce_video_ratio_cnt()
+
+
+
+# if __name__ == '__main__':