utils.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620
  1. import os
  2. import json
  3. from aliyunsdkcore.client import AcsClient
  4. from aliyunsdkecs.request.v20140526.DescribeInstancesRequest import DescribeInstancesRequest
  5. from model import InstanceList,IntfaceList
  6. # from DBSession import session_maker
  7. from model_longvideo import produce_video_task
  8. from DBSession_longvideo import session_maker_longvideo
  9. from aliyun.log.logclient import LogClient
  10. from aliyun.log.getlogsrequest import GetLogsRequest
  11. import requests
  12. import time
  13. import datetime
  14. import pymysql
  15. from sqlalchemy.sql import func
  16. from model_longvideo import produce_video_project
  17. from datetime import date, timedelta
  18. from DBSession import session_maker
  19. client = AcsClient('LTAI4GBWbFvvXoXsSVBe1o9f', 'kRAikWitb4kDxaAyBqNrmLmllMEDO3', 'cn-hangzhou')
  20. from prometheus_client import Gauge,Counter, generate_latest
  21. def intances_list_update():
  22. request = DescribeInstancesRequest()
  23. request.set_accept_format('json')
  24. request.set_PageSize(100)
  25. request.set_InstanceNetworkType("vpc")
  26. request.set_Tags([
  27. {
  28. "Key": "ecs"
  29. }
  30. ])
  31. response = client.do_action_with_exception(request)
  32. instance_info = json.loads(response)
  33. intances_list_del()
  34. print(len(instance_info["Instances"]["Instance"]))
  35. for i in range(len(instance_info["Instances"]["Instance"])):
  36. instance_id = instance_info["Instances"]["Instance"][i]["InstanceId"]
  37. ipaddr = instance_info["Instances"]["Instance"][i]["VpcAttributes"]["PrivateIpAddress"]["IpAddress"][0]
  38. server_name = instance_info["Instances"]["Instance"][i]["Tags"]["Tag"][0]["TagValue"]
  39. status = instance_info["Instances"]["Instance"][i]["Status"]
  40. instance_name = instance_info["Instances"]["Instance"][i]["HostName"]
  41. if status == "Running":
  42. status = 1
  43. instance_insert(instance_id, ipaddr, instance_name, server_name, status)
  44. return "OK"
  45. def intances_list_del():
  46. with session_maker() as session:
  47. session.query(InstanceList).delete()
  48. def instance_insert(instance_id, ipadd, instance_name, server_name, status):
  49. InstanceInfo = None
  50. InstanceInfo = InstanceList()
  51. with session_maker() as session:
  52. InstanceInfo.instance_id = instance_id
  53. InstanceInfo.ipadd = ipadd
  54. InstanceInfo.instance_name = instance_name
  55. InstanceInfo.server_name = server_name
  56. InstanceInfo.status = status
  57. session.add(InstanceInfo)
  58. def healthcheck(ipaddr,server_name):
  59. """返回容器upload时间,容器运行状态,健康坚持接口返回值,"""
  60. svc_name = server_name
  61. if svc_name == "longvideoapi.prod" :
  62. url = "http://{}:8080/longvideoapi/test".format(ipaddr)
  63. res = requests.post(url, timeout=5)
  64. print(res.status_code)
  65. return res.status_code
  66. elif svc_name == "commonapi.prod":
  67. url = "http://{}:8080/commonapi/test".format(ipaddr)
  68. res = requests.post(url, timeout=5)
  69. return res.status_code
  70. elif svc_name == "speed.prod":
  71. url = "http://{}:8080/longvideoapi/test".format(ipaddr)
  72. res = requests.post(url, timeout=5)
  73. return res.status_code
  74. elif svc_name == "clipapi.prod":
  75. url = "http://{}:8080/longvideoapi/test".format(ipaddr)
  76. res = requests.post(url, timeout=5)
  77. return res.status_code
  78. elif svc_name == "distribution.prod":
  79. url = "http://{}:8080/healthcheck".format(ipaddr)
  80. res = requests.post(url, timeout=5)
  81. return res.status_code
  82. elif svc_name == "manager.prod":
  83. url = "http://{}:8971/dev/test/healthcheck".format(ipaddr)
  84. res = requests.post(url, timeout=5)
  85. return res.status_code
  86. elif svc_name == "recommend-queue.prod":
  87. url = "http://{}:8080".format(ipaddr)
  88. res = requests.post(url, timeout=5)
  89. return res.status_code
  90. elif svc_name == "recommend-api.prod":
  91. url = "http://{}:8080/healthcheck".format(ipaddr)
  92. res = requests.post(url, timeout=5)
  93. return res.status_code
  94. elif svc_name == "recommend-rov.prod":
  95. url = "http://{}:8080/healthcheck".format(ipaddr)
  96. res = requests.post(url, timeout=5)
  97. return res.status_code
  98. elif svc_name == "measure-queue.prod":
  99. url = "http://{}:8080/healthcheck".format(ipaddr)
  100. res = requests.post(url, timeout=5)
  101. return res.status_code
  102. elif svc_name == "message-queue.prod":
  103. url = "http://{}:8080/healthcheck".format(ipaddr)
  104. res = requests.post(url, timeout=5)
  105. return res.status_code
  106. elif svc_name == "search-material.prod":
  107. url = "http://{}:8089/v1/search/test".format(ipaddr)
  108. res = requests.post(url, timeout=5)
  109. return res.status_code
  110. elif svc_name == "spider.prod":
  111. url = "http://{}:9990/version".format(ipaddr)
  112. try:
  113. res = requests.post(url, timeout=5)
  114. except Exception as e:
  115. return -1
  116. return res.status_code
  117. elif svc_name == "video-job.prod":
  118. url = "http://{}:8080/healthcheck".format(ipaddr)
  119. res = requests.post(url, timeout=5)
  120. return res.status_code
  121. elif svc_name == "combine-job.prod":
  122. url = "http://{}:8080/healthcheck".format(ipaddr)
  123. res = requests.post(url, timeout=5)
  124. return res.status_code
  125. elif svc_name == "recommend-server.prod":
  126. url = "http://{}:8080/healthcheck".format(ipaddr)
  127. # res = requests.post(url, timeout=5)
  128. return 200
  129. else:
  130. return 200
  131. # def healthcheck_metirc(instance_id,ipaddr,server_name,http_code):
  132. # healthcheck_status = Gauge("healthcheck_status", "http_code", ['instance_id', 'server_name', 'ipaddress'],
  133. # registry=registry)
  134. #
  135. # healthcheck_status.labels(instance_id, server_name, ipaddr).set(http_code)
  136. #
  137. # return Response(generate_latest(registry),mimetype="text/plain")
  138. def count_qps(appType, url):
  139. endpoint = 'cn-hangzhou.log.aliyuncs.com'
  140. accessKeyId = 'LTAIWYUujJAm7CbH'
  141. accessKey = 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P'
  142. logstore = 'request-log'
  143. client = LogClient(endpoint, accessKeyId, accessKey)
  144. project = 'wqsd-longvideoapi'
  145. topic = ""
  146. start_time = int(time.time())
  147. end_time = start_time - 60
  148. req = GetLogsRequest(project, logstore, end_time, start_time, topic,
  149. "params.appType={} and requestUri= {} |select requestUri,count(1) as cnt, date_format(logTimestamp/1000,'%Y-%m-%d %H:%i:%S') as date group by requestUri, date order by cnt desc limit 1".format(appType, url)
  150. , 30, 0, False)
  151. try:
  152. res = client.get_logs(req)
  153. except :
  154. qps = 0
  155. else:
  156. if res.body:
  157. qps = res.body[0]["cnt"]
  158. else:
  159. qps = 0
  160. print(int(time.time())-start_time)
  161. return qps
  162. def count_avg_time(appType ,url):
  163. endpoint = 'cn-hangzhou.log.aliyuncs.com'
  164. accessKeyId = 'LTAIWYUujJAm7CbH'
  165. accessKey = 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P'
  166. logstore = 'request-log'
  167. client = LogClient(endpoint, accessKeyId, accessKey)
  168. project = 'wqsd-longvideoapi'
  169. topic = ""
  170. start_time = int(time.time())
  171. end_time = start_time - 60
  172. # url = '/longvideoapi/user/idolsUpdated'
  173. req = GetLogsRequest(project, logstore, end_time, start_time, topic,
  174. "params.appType=%s and requestUri=%s |select requestUri,round(avg(expendTime)) as avgs,count(1) as cnt group by requestUri " %(appType, url), 30, 0, False)
  175. try:
  176. res = client.get_logs(req)
  177. except Exception as e:
  178. avgs = 0
  179. else:
  180. if res.body:
  181. avgs = res.body[0]["avgs"]
  182. else:
  183. avgs = 0
  184. return avgs
  185. def slb_status_code_count(server_name):
  186. endpoint = 'cn-hangzhou.log.aliyuncs.com'
  187. accessKeyId = 'LTAIWYUujJAm7CbH'
  188. accessKey = 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P'
  189. client = LogClient(endpoint, accessKeyId, accessKey)
  190. if server_name == "longvideoapi":
  191. project = 'longvideoapi-slb-log'
  192. logstore = 'slb-access-log'
  193. if server_name == "speed":
  194. project = 'speed-slb-log'
  195. logstore = 'access-log'
  196. if server_name == "clip":
  197. project = 'clips-slb-log'
  198. logstore = 'access-log'
  199. if server_name == "commonapi":
  200. project = 'commonapi-slb-log'
  201. logstore = 'slb-access-log'
  202. topic = ""
  203. end_time = int(time.time())
  204. start_time = end_time - (end_time - time.timezone)%86400
  205. # url = '/longvideoapi/user/idolsUpdated'
  206. req = GetLogsRequest(project, logstore, start_time, end_time, topic,
  207. "* |SELECT status ,COUNT(*) as cnt group by status" , 30, 0, False)
  208. try:
  209. res = client.get_logs(req)
  210. except Exception as e:
  211. avgs = 0
  212. else:
  213. if res.body:
  214. return res.body
  215. def produce_video_task_status_count():
  216. with session_maker_longvideo() as session:
  217. video_progress_count = session.query(produce_video_task).filter(produce_video_task.task_status == 1).count()
  218. video_success_count = session.query(produce_video_task).filter(produce_video_task.task_status == 2).count()
  219. video_fail_count = session.query(produce_video_task).filter(produce_video_task.task_status == 3).count()
  220. end_time = int(time.time())
  221. start_time = end_time - (end_time - time.timezone) % 86400
  222. res = session.query(produce_video_task).filter(produce_video_task.task_status==2).order_by(produce_video_task.id.desc()).first()
  223. duration = res.duration
  224. submit_timestamp = res.submit_timestamp
  225. complete_timestamp = res.complete_timestamp
  226. print(res.id,duration,submit_timestamp,complete_timestamp)
  227. rate = (duration / (complete_timestamp - submit_timestamp) / 1000)
  228. def logs_tts_count(tts_channel, tts_status):
  229. endpoint = 'cn-hangzhou.log.aliyuncs.com'
  230. accessKeyId = 'LTAIWYUujJAm7CbH'
  231. accessKey = 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P'
  232. client = LogClient(endpoint, accessKeyId, accessKey)
  233. project = 'wqsd-monitor'
  234. logstore = 'monitor-log'
  235. topic = ""
  236. end_time = int(time.time())
  237. start_time = end_time - (end_time - time.timezone)%86400
  238. req = GetLogsRequest(project, logstore, start_time, end_time, topic,
  239. "* and __topic__: producetts and channel:{} and produceStatus:{}| select channel,produceStatus,count(*) as count group by channel,produceStatus".format(tts_channel, tts_status) , 30, 0, False)
  240. try:
  241. res = client.get_logs(req)
  242. except Exception as e:
  243. avgs = 0
  244. else:
  245. if res.body:
  246. return res.body
  247. def count_recommend_null():
  248. endpoint = 'cn-hangzhou.log.aliyuncs.com'
  249. accessKeyId = 'LTAIWYUujJAm7CbH'
  250. accessKey = 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P'
  251. logstore = 'app-recommend-log'
  252. client = LogClient(endpoint, accessKeyId, accessKey)
  253. project = 'wqsd-longvideoapi'
  254. topic = ""
  255. start_time = int(time.time())
  256. #监控获取时间间隔300s
  257. end_time = start_time - 300
  258. query_sql = "* and requestUri:'/longvideoapi/video/distribute/category/videoList/v2' | select count(DISTINCT json_extract(params, '$.machineCode')) as cnt where json_array_length(json_extract(result, '$.data')) = 0"
  259. req = GetLogsRequest(project, logstore, end_time, start_time, topic, query_sql, 30, 0, False)
  260. try:
  261. res = client.get_logs(req)
  262. except Exception as e:
  263. print(e)
  264. avgs = 0
  265. else:
  266. if res.body:
  267. cnt_300 = res.body[0]["cnt"]
  268. else:
  269. cnt_300 = 0
  270. return cnt_300
  271. def db_query(sql):
  272. HOST = 'rr-bp1x9785e8h5452bi157.mysql.rds.aliyuncs.com'
  273. PORT = '3306'
  274. DATABASE = 'longvideo'
  275. USERNAME = 'devops'
  276. PASSWORD = 'devops@123456'
  277. conn = pymysql.connect(host=HOST, user=USERNAME, password=PASSWORD, database=DATABASE, charset="utf8")
  278. cursor = conn.cursor()
  279. cursor.execute(sql)
  280. res = cursor.fetchone()
  281. return res
  282. def db_query_devlop(sql):
  283. HOST = 'rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com'
  284. PORT = '3306'
  285. DATABASE = 'devops'
  286. USERNAME = 'devops'
  287. PASSWORD = 'devops@123456'
  288. conn = pymysql.connect(host=HOST, user=USERNAME, password=PASSWORD, database=DATABASE, charset="utf8")
  289. cursor = conn.cursor()
  290. cursor.execute(sql)
  291. res = cursor.fetchall()
  292. return res
  293. def db_query_prod(sql):
  294. HOST = 'rr-bp1x9785e8h5452bi157.mysql.rds.aliyuncs.com'
  295. PORT = '3306'
  296. DATABASE = 'longvideo'
  297. USERNAME = 'devops'
  298. PASSWORD = 'devops@123456'
  299. conn = pymysql.connect(host=HOST, user=USERNAME, password=PASSWORD, database=DATABASE, charset="utf8")
  300. cursor = conn.cursor()
  301. cursor.execute(sql)
  302. res = cursor.fetchone()
  303. return res
  304. def count_qps_avgtime(appType):
  305. endpoint = 'cn-hangzhou.log.aliyuncs.com'
  306. accessKeyId = 'LTAIWYUujJAm7CbH'
  307. accessKey = 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P'
  308. logstore = 'request-log'
  309. client = LogClient(endpoint, accessKeyId, accessKey)
  310. project = 'wqsd-longvideoapi'
  311. topic = ""
  312. start_time = int(time.time())
  313. end_time = start_time - 60
  314. # url = '/longvideoapi/user/idolsUpdated'
  315. req = GetLogsRequest(project, logstore, end_time, start_time, topic,
  316. "params.appType=%s | select requestUri,count(1) as cnt , avg(expendTime) as avg_time group by requestUri order by requestUri,cnt desc limit 1000" %(appType), 30, 0, False)
  317. try:
  318. res = client.get_logs(req)
  319. except Exception as e:
  320. avgs = 0
  321. return res
  322. def count_avg_max(appType):
  323. endpoint = 'cn-hangzhou.log.aliyuncs.com'
  324. accessKeyId = 'LTAIWYUujJAm7CbH'
  325. accessKey = 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P'
  326. logstore = 'request-log'
  327. client = LogClient(endpoint, accessKeyId, accessKey)
  328. project = 'wqsd-longvideoapi'
  329. topic = ""
  330. start_time = int(time.time())
  331. end_time = start_time - (60*15)
  332. # url = '/longvideoapi/user/idolsUpdated'
  333. req = GetLogsRequest(project, logstore, end_time, start_time, topic,
  334. "params.appType=%s | select requestUri,count(1) as cnt , max(expendTime) as max_time ,round(avg(expendTime),2) as avg_time group by requestUri order by requestUri,cnt desc limit 1000" %(appType), 30, 0, False)
  335. try:
  336. res = client.get_logs(req)
  337. except Exception as e:
  338. avgs = 0
  339. return res
  340. def count_rt_less_time_count(appType, than, less):
  341. endpoint = 'cn-hangzhou.log.aliyuncs.com'
  342. accessKeyId = 'LTAIWYUujJAm7CbH'
  343. accessKey = 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P'
  344. logstore = 'request-log'
  345. client = LogClient(endpoint, accessKeyId, accessKey)
  346. project = 'wqsd-longvideoapi'
  347. topic = ""
  348. start_time = int(time.time())
  349. end_time = start_time - 60
  350. # url = '/longvideoapi/user/idolsUpdated'
  351. req = GetLogsRequest(project, logstore, end_time, start_time, topic,
  352. "params.appType=%s and expendtime >%s and expendtime <%s| select requestUri,count(1) as cnt group by requestUri order by requestUri,cnt desc limit 1000" %(appType,than,less), 30, 0, False)
  353. try:
  354. res = client.get_logs(req)
  355. except Exception as e:
  356. print(e)
  357. return res
  358. def error_cnt(appType):
  359. endpoint = 'cn-hangzhou.log.aliyuncs.com'
  360. accessKeyId = 'LTAIWYUujJAm7CbH'
  361. accessKey = 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P'
  362. logstore = 'request-log'
  363. client = LogClient(endpoint, accessKeyId, accessKey)
  364. project = 'wqsd-longvideoapi'
  365. topic = ""
  366. start_time = int(time.time())
  367. end_time = start_time - 60
  368. req = GetLogsRequest(project, logstore, end_time, start_time, topic,
  369. "params.appType=%s and (resultCode:-111 or resultCode:-990 or resultCode:-994 or resultCode:-666 or resultCode:-222)| SELECT COUNT(*) as cnt, resultCode group by resultCode" %(appType), 30, 0, False)
  370. try:
  371. res = client.get_logs(req)
  372. except Exception as e:
  373. print(e)
  374. return res
  375. def app_openapi_qps_avgtime_count():
  376. endpoint = 'cn-hangzhou.log.aliyuncs.com'
  377. accessKeyId = 'LTAIWYUujJAm7CbH'
  378. accessKey = 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P'
  379. logstore = 'request-log'
  380. client = LogClient(endpoint, accessKeyId, accessKey)
  381. project = 'wqsd-longvideoapi'
  382. topic = ""
  383. start_time = int(time.time())
  384. end_time = start_time - 60
  385. url = '/longvideoapi/openapi/video/distribute/category/videoList/v2'
  386. req = GetLogsRequest(project, logstore, end_time, start_time, topic,
  387. "requestUri=%s| select requestUri,count(1) as cnt , avg(expendTime) as avg_time group by requestUri order by requestUri,cnt desc limit 1000"%(url) , 30, 0, False)
  388. try:
  389. res = client.get_logs(req)
  390. except Exception as e:
  391. avgs = 0
  392. return res,url
  393. def produce_video_data():
  394. start_time = (int(time.strftime("%Y%m%d", time.localtime()))) * 1000000000000000
  395. end_time = (int(time.strftime("%Y%m%d", time.localtime())) + 1) * 1000000000000000
  396. # start_time = (int(datetime.strftime("%Y%m%d",datetime.date.today()))
  397. print(start_time, end_time)
  398. sum_sql = ("select count(*) as totalCount "
  399. "from produce_video_project t1 "
  400. " where t1.project_id > %s and t1.project_id < %s and t1.app_type not in (1,13,15) "% (start_time,end_time)
  401. )
  402. res = db_query(sum_sql)
  403. return
  404. def update_request_url_list():
  405. HOST = 'rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com'
  406. PORT = '3306'
  407. DATABASE = 'devops'
  408. USERNAME = 'devops'
  409. PASSWORD = 'devops@123456'
  410. conn = pymysql.connect(host=HOST, user=USERNAME, password=PASSWORD, database=DATABASE, charset="utf8")
  411. cursor = conn.cursor()
  412. print("5m定时任务开始")
  413. appType = ['0', '4', '5', '6', '12', '15']
  414. for index in range(len(appType)):
  415. type = appType[index]
  416. res = count_avg_max(type)
  417. for i in range(len(res.body)):
  418. url = res.body[i]["requestUri"]
  419. cnt = res.body[i]["cnt"]
  420. max_time = res.body[i]["max_time"]
  421. avg_time = res.body[i]["avg_time"]
  422. sql = "UPDATE `reuqest_url_list` set `qps`=%s, `max_time`=%s, `avg_time`=%s WHERE `request_url` ='%s' and `apptype`=%s" % (
  423. cnt, max_time, avg_time, url, type)
  424. cursor.execute(sql)
  425. conn.commit()
  426. for i in range(len(res.body)):
  427. url = res.body[i]["requestUri"]
  428. cnt = res.body[i]["cnt"]
  429. sql = "UPDATE `reuqest_url_list` set `rt_1`=%s WHERE `request_url` ='%s' and `apptype`=%s" % (
  430. cnt, url, type)
  431. cursor.execute(sql)
  432. conn.commit()
  433. res = count_rt_less_time_count(type, 200, 500)
  434. for i in range(len(res.body)):
  435. url = res.body[i]["requestUri"]
  436. cnt = res.body[i]["cnt"]
  437. sql = "UPDATE `reuqest_url_list` set `rt_2`=%s WHERE `request_url` ='%s' and `apptype`=%s" % (
  438. cnt, url, type)
  439. cursor.execute(sql)
  440. conn.commit()
  441. res = count_rt_less_time_count(type, 500, 1000)
  442. for i in range(len(res.body)):
  443. url = res.body[i]["requestUri"]
  444. cnt = res.body[i]["cnt"]
  445. sql = "UPDATE `reuqest_url_list` set `rt_3`=%s WHERE `request_url` ='%s' and `apptype`=%s" % (
  446. cnt, url, type)
  447. cursor.execute(sql)
  448. conn.commit()
  449. res = count_rt_less_time_count(type, 1000, 10000)
  450. for i in range(len(res.body)):
  451. url = res.body[i]["requestUri"]
  452. cnt = res.body[i]["cnt"]
  453. sql = "UPDATE `reuqest_url_list` set `rt_4`=%s WHERE `request_url` ='%s' and `apptype`=%s" % (
  454. cnt, url, type)
  455. cursor.execute(sql)
  456. conn.commit()
  457. def produce_video_task_cnt():
  458. start_time = int(time.strftime("%Y%m%d", time.localtime())) * 1000000000000000
  459. end_time = int(time.strftime("%Y%m%d%H%M", time.localtime())) * 100000000000
  460. query_sql = ("select count(*) as totalCount,"
  461. "sum(case when produce_status in (5,6,7,8) then 1 else 0 end) as successCount,"
  462. "sum(case when produce_status = 99 then 1 else 0 end) as failCount , "
  463. "sum(case when produce_status in(0,1,2,3,4) then 1 else 0 end) as processingCount,"
  464. "sum(case when produce_status in(5,6,7,8) and (rate < 0.5 or rate is null) then 1 else 0 end) processingCount1,"
  465. "sum(case when produce_status in(5,6,7,8) and rate >= 0.5 and rate < 0.7 then 1 else 0 end) processingCount2 ,"
  466. "sum(case when produce_status in(5,6,7,8) and rate >= 0.7 and rate < 1 then 1 else 0 end) processingCount3 ,"
  467. "sum(case when produce_status in(5,6,7,8) and rate >= 1 and rate < 1.5 then 1 else 0 end) processingCount4 ,"
  468. "sum(case when produce_status in(5,6,7,8) and rate >= 1.5 and rate < 2 then 2 else 0 end) processingCount5 ,"
  469. "sum(case when produce_status in(5,6,7,8) and rate >2 then 1 else 0 end) processingCount6 from "
  470. "(select t1.project_id, t1.produce_status, round((t2.last_connect_timestamp - t1.submit_timestamp) / (t1.video_duration/1000), 1) as rate from produce_video_project t1 "
  471. "left join produce_video_project_connect_time t2 on t1.project_id = t2.project_id "
  472. "where t1.project_id > %s and t1.project_id < %s and t1.app_type not in (1,13,15)) s1 " %(start_time ,end_time)
  473. )
  474. res = db_query(query_sql)
  475. return res
  476. def produce_video_ratio_cnt():
  477. HOST = 'rm-bp1k5853td1r25g3n690.mysql.rds.aliyuncs.com'
  478. PORT = '3306'
  479. DATABASE = 'devops'
  480. USERNAME = 'devops'
  481. PASSWORD = 'devops@123456'
  482. conn = pymysql.connect(host=HOST, user=USERNAME, password=PASSWORD, database=DATABASE, charset="utf8")
  483. cursor = conn.cursor()
  484. start_time = int((date.today() + timedelta(days=-1)).strftime("%Y%m%d")) * 1000000000000000
  485. # end_time = int(date.today().strftime("%Y%m%d")) * 1000000000000000
  486. end_time = int((date.today() + timedelta(days=0)).strftime("%Y%m%d")) * 1000000000000000
  487. sql = ("select count(*) as totalCount, "
  488. "sum(case when rate < 0.5 then 1 else 0 end) as r1,"
  489. "sum(case when (rate >= 0.5 and rate < 0.7) then 1 else 0 end) as r2,"
  490. "sum(case when (rate >= 0.7 and rate < 1) then 1 else 0 end) as r3,"
  491. "sum(case when (rate >= 1 and rate < 1.5) then 1 else 0 end) as r4,"
  492. "sum(case when (rate >= 1.5 and rate < 2) then 1 else 0 end) as r5,"
  493. "sum(case when rate >= 2 then 1 else 0 end) as r6 from "
  494. "(select project_id, round( (produce_done_timestamp - submit_timestamp) / (video_duration/1000),1) as rate from "
  495. "(select project_id, draftbox_id,video_duration,submit_timestamp,produce_done_timestamp from "
  496. "(select project_id, draftbox_id,video_duration,submit_timestamp,produce_done_timestamp from "
  497. "produce_video_project where project_id > %s and project_id < %s and app_type not in (1,13,15) and produce_status in (5,6,7,8) order by video_duration desc limit 100000) t1 group by draftbox_id) t2) t3"%(start_time, end_time)
  498. )
  499. y_date = (date.today() + timedelta(days=-1)).strftime("%Y%m%d")
  500. res = db_query(sql)
  501. if res[0]:
  502. cnt = res[0]
  503. else:
  504. cnt = 0
  505. if res[1]:
  506. r1 = res[1]
  507. else:
  508. r1 = 0
  509. if res[2]:
  510. r2 = res[2]
  511. else:
  512. r2 = 0
  513. if res[3]:
  514. r3 = res[3]
  515. else:
  516. r3 = 0
  517. if res[4]:
  518. r4 = res[4]
  519. else:
  520. r4 = 0
  521. if res[5]:
  522. r5 = res[5]
  523. else:
  524. r5 = 0
  525. if res[6]:
  526. r6 = res[6]
  527. else:
  528. r6 = 0
  529. sql = ("insert into produce_video_ratio (`cnt`,`r1`,`r2`,`r3`,`r4`,`r5`,`r6`,`date`) VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"%(cnt, r1, r2, r3, r4, r5, r6, y_date)
  530. )
  531. cursor.execute(sql)
  532. conn.commit()
  533. # if __name__ == '__main__':