main.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422
  1. package main
  2. import (
  3. "log"
  4. "net/http"
  5. "os"
  6. "strconv"
  7. "time"
  8. "github.com/prometheus/client_golang/prometheus"
  9. "github.com/prometheus/client_golang/prometheus/promhttp"
  10. sls "github.com/aliyun/aliyun-log-go-sdk"
  11. )
  12. var (
  13. AccessKeyID = "LTAIWYUujJAm7CbH"
  14. AccessKeySecret = "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P"
  15. Endpoint = "cn-hangzhou.log.aliyuncs.com"
  16. adActionLogStore *sls.LogStore
  17. longvideoReqeustLogStore *sls.LogStore
  18. videoPlayStore *sls.LogStore
  19. videoActionStore *sls.LogStore
  20. simpleeventStore *sls.LogStore
  21. userShareStore *sls.LogStore
  22. adRequestStore *sls.LogStore
  23. adReportQuerySQL = `* | SELECT appType, json_extract_scalar(extParams, '$.eventInfos.ab_test002') AS adGroup, CASE WHEN adType = 1 THEN 'BANNER广告' WHEN adType = 2 THEN '视频广告' WHEN adType = 3 THEN '贴片广告' WHEN adType = 4 THEN '插屏广告' WHEN adType = 5 THEN '开屏广告' WHEN adType = 6 THEN '激励广告' ELSE ' ' END AS adType, COUNT( DISTINCT CASE WHEN businessType = 'adLoaded' THEN machineCode END ) AS adLoaded, COUNT( DISTINCT CASE WHEN businessType = 'adRequest' THEN machineCode END ) AS adRequest, COUNT( DISTINCT CASE WHEN businessType = 'adView' THEN machineCode END ) AS adView, COUNT( DISTINCT CASE WHEN businessType = 'adClick' THEN machineCode END ) AS adClick, COUNT( DISTINCT CASE WHEN businessType = 'adShowError' THEN machineCode END ) AS adShowError, COUNT( DISTINCT CASE WHEN businessType = 'adLoadError' THEN machineCode END ) AS adLoadError, COUNT( DISTINCT CASE WHEN businessType = 'adConfigRequest' THEN machineCode END ) AS adConfigRequest, COUNT( DISTINCT CASE WHEN businessType = 'adConfigLoaded' THEN machineCode END ) AS adConfigLoaded GROUP BY adGroup, adType, appType HAVING adGroup IS NOT NULL AND adType != ' ' ORDER BY appType ASC LIMIT 1000`
  24. adReportQuerySQL4 = `* | SELECT appType, json_extract_scalar(extParams, '$.eventInfos.ab_test004') AS adGroup, CASE WHEN adType = 1 THEN 'BANNER广告' WHEN adType = 2 THEN '视频广告' WHEN adType = 3 THEN '贴片广告' WHEN adType = 4 THEN '插屏广告' WHEN adType = 5 THEN '开屏广告' WHEN adType = 6 THEN '激励广告' ELSE ' ' END AS adType, COUNT( DISTINCT CASE WHEN businessType = 'adLoaded' THEN machineCode END ) AS adLoaded, COUNT( DISTINCT CASE WHEN businessType = 'adRequest' THEN machineCode END ) AS adRequest, COUNT( DISTINCT CASE WHEN businessType = 'adView' THEN machineCode END ) AS adView, COUNT( DISTINCT CASE WHEN businessType = 'adClick' THEN machineCode END ) AS adClick, COUNT( DISTINCT CASE WHEN businessType = 'adShowError' THEN machineCode END ) AS adShowError, COUNT( DISTINCT CASE WHEN businessType = 'adLoadError' THEN machineCode END ) AS adLoadError, COUNT( DISTINCT CASE WHEN businessType = 'adConfigRequest' THEN machineCode END ) AS adConfigRequest, COUNT( DISTINCT CASE WHEN businessType = 'adConfigLoaded' THEN machineCode END ) AS adConfigLoaded GROUP BY adGroup, adType, appType HAVING adGroup IS NOT NULL AND adType != ' ' ORDER BY appType ASC LIMIT 1000`
  25. apiQuerySQL = `* and requestUri: "/longvideoapi/video/distribute/category/videoList/v2" or requestUri: "/longvideoapi/video/v2/detail" or requestUri: "/longvideoapi/video/v2/sharePageDetail" or requestUri: "/longvideoapi/video/recommend/sharePage/list" | SELECT requestUri, json_extract_scalar(params, '$.appType') as appType, COUNT(*) as total, count( case when resultCode = 0 THEN 1 else null end ) as success, count( case when resultCode != 0 THEN 1 else null end ) as error GROUP by appType, requestUri order by requestUri asc limit 1000`
  26. videoPlayQuerySQL = `* | select appType, COUNT(*) as cnt, businessType, pageSource GROUP by appType, businessType, pageSource ORDER BY appType ASC, cnt DESC LIMIT 1000`
  27. // videoPlayCountQuerySQL = `* AND businessType : videoPlay | select appType, COUNT(*) as plyaCount, pageSource GROUP BY appType, pageSource ORDER BY appType ASC`
  28. // adConfigQuerySQL = `* and requestUri: /longvideoapi/ad/position/info | select json_extract_scalar(params, '$.appType') as appType,count(*) as total, count(case when resultCode = 0 then 1 else null end) as success,count(case when resultCode != 0 then 1 else null end) as error,count(case when resultSize > 0 then 1 else null end) as hasAds,count(case when resultSize <= 0 then 1 else null end) as noAds group by appType`
  29. adConfigQuerySQL = `* and url : "/ad/position/info" | SELECT count(*) as total, count( case when json_extract_scalar(responseBody, '$.code') = '0' then 1 else null end ) as success, count( case when json_extract_scalar(responseBody, '$.code') != '0' then 1 else null end ) as error, count( case when json_array_length(json_extract(responseBody, '$.data')) <= 0 then 1 else null end ) as noAds, count( case when json_array_length(json_extract(responseBody, '$.data')) > 0 then 1 else null end ) as hasAds, json_extract(requestBody, '$.baseInfo.appType') as appType group by appType `
  30. apiRtQuerySQL = `* and requestUri: "/longvideoapi/video/distribute/category/videoList/v2" or requestUri: "/longvideoapi/video/v2/detail" or requestUri: "/longvideoapi/video/v2/sharePageDetail" or requestUri: "/longvideoapi/video/recommend/sharePage/list" | select "requestUri", "params.appType" as "appType" , avg("expendTime") as "rt" from log group by "requestUri", "appType" order by "appType" asc limit 1000`
  31. videoDataQuerySQL = `* | select appType, COUNT(*) as cnt, businessType, pageSource GROUP by appType, businessType, pageSource ORDER BY appType ASC, businessType LIMIT 1000`
  32. distributeApiQuerySQL = ` * and (requestUri: "/longvideoapi/video/distribute/category/videoList/v2" or requestUri: "/longvideoapi/video/recommend/sharePage/list") and not resultSize: 4 | SELECT json_extract_scalar(params, '$.appType') as appType, requestUri, COUNT( DISTINCT CASE WHEN resultSize = 0 THEN json_extract_scalar(params, '$.machineCode') END ) AS "等于0", COUNT( DISTINCT CASE WHEN (resultSize < 4 and resultSize >0) THEN json_extract_scalar(params, '$.machineCode') END ) AS "小于4" GROUP BY requestUri, appType ORDER BY appType ASC LIMIT 1000`
  33. shareDataQuerySQL = `* and ( businessType = 'videoShareFriend' OR businessType = 'videoShareH5' OR businessType = 'videoPublish' ) | select appType, COUNT(DISTINCT mid) as cnt, COUNT(*) as cnt1, businessType, pageSource GROUP by appType, pageSource, businessType ORDER BY appType ASC, cnt DESC LIMIT 1000 `
  34. shareOperation1QuerySQL = `* and objectType: weapp_quitbutton OR objectType: weapp_share_navigator_moveVideo OR objectType: weapp_share_upload OR ( objectType: weapp_share_shareFriendWindow AND businessType: buttonClick ) | select objectType, appType, COUNT(*) as cnt GROUP by objectType, appType ORDER BY appType asc, objectType asc `
  35. shareOperation2QuerySQL = `* and pageSource: "user-videos-share" | select COUNT(*) as cnt, ( CASE WHEN groupShare IS NULL THEN 'shareFriend' WHEN groupShare = 1 THEN 'shareGroup' ELSE ' ' END ) as type, appType GROUP BY type, appType ORDER BY appType asc, type asc `
  36. )
  37. func init() {
  38. client := sls.CreateNormalInterface(Endpoint, AccessKeyID, AccessKeySecret, "")
  39. var err error
  40. longvideoReqeustLogStore, err = client.GetLogStore("wqsd-longvideoapi", "request-log")
  41. if err != nil {
  42. panic(err)
  43. }
  44. adActionLogStore, err = client.GetLogStore("wqsd-longvideo-frontend-log-prd", "ad-action-log")
  45. if err != nil {
  46. panic(err)
  47. }
  48. videoActionStore, err = client.GetLogStore("wqsd-video", "video-action-log")
  49. if err != nil {
  50. panic(err)
  51. }
  52. videoPlayStore, err = client.GetLogStore("wqsd-video", "video-play-log")
  53. if err != nil {
  54. panic(err)
  55. }
  56. simpleeventStore, err = client.GetLogStore("wqsd-longvideo-frontend-log-prd", "simpleevent-log")
  57. if err != nil {
  58. panic(err)
  59. }
  60. userShareStore, err = client.GetLogStore("wqsd-user", "user-share-log")
  61. if err != nil {
  62. panic(err)
  63. }
  64. adRequestStore, err = client.GetLogStore("ms-ad", "request-log")
  65. if err != nil {
  66. panic(err)
  67. }
  68. }
  69. func (c *CustomCollector) Describe(ch chan<- *prometheus.Desc) {
  70. ch <- c.AdInfoDesc
  71. ch <- c.ApiInfoDesc
  72. ch <- c.VideoPlayDesc
  73. ch <- c.AdConfigDesc
  74. ch <- c.AdInfo4Desc
  75. ch <- c.ApiRtDesc
  76. ch <- c.VideoDataDesc
  77. ch <- c.DistributeApiDesc
  78. ch <- c.ShareDataDesc
  79. ch <- c.ShareOperationDesc
  80. }
  81. func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
  82. resp, err := longvideoReqeustLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), apiQuerySQL, 0, 0, true)
  83. if err != nil || resp == nil {
  84. log.Printf("get api logs error %+v resp is %+v.", err, resp)
  85. return
  86. }
  87. for _, v := range resp.Logs {
  88. for k, v2 := range v {
  89. if k == "total" ||
  90. k == "success" ||
  91. k == "error" {
  92. v3, _ := strconv.ParseFloat(v2, 64)
  93. ch <- prometheus.MustNewConstMetric(
  94. c.ApiInfoDesc,
  95. prometheus.GaugeValue,
  96. v3,
  97. v["appType"], v["requestUri"], k,
  98. )
  99. }
  100. }
  101. }
  102. resp, err = adActionLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), adReportQuerySQL, 0, 0, true)
  103. if err != nil || resp == nil {
  104. log.Printf("get ad report logs ab_test002 error %+v resp is %+v.", err, resp)
  105. return
  106. }
  107. for _, v := range resp.Logs {
  108. for k, v2 := range v {
  109. if k == "adRequest" ||
  110. k == "adShowError" ||
  111. k == "adLoadError" ||
  112. k == "adConfigRequest" ||
  113. k == "adConfigLoaded" ||
  114. k == "adLoaded" ||
  115. k == "adView" ||
  116. k == "adClick" {
  117. v3, _ := strconv.ParseFloat(v2, 64)
  118. ch <- prometheus.MustNewConstMetric(
  119. c.AdInfoDesc,
  120. prometheus.GaugeValue,
  121. v3,
  122. v["appType"], v["adGroup"], v["adType"], k,
  123. )
  124. }
  125. }
  126. }
  127. resp, err = adActionLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), adReportQuerySQL4, 0, 0, true)
  128. if err != nil || resp == nil {
  129. log.Printf("get ad report logs ab_test004 error %+v resp is %+v.", err, resp)
  130. return
  131. }
  132. for _, v := range resp.Logs {
  133. for k, v2 := range v {
  134. if k == "adRequest" ||
  135. k == "adShowError" ||
  136. k == "adLoadError" ||
  137. k == "adConfigRequest" ||
  138. k == "adConfigLoaded" ||
  139. k == "adLoaded" ||
  140. k == "adView" ||
  141. k == "adClick" {
  142. v3, _ := strconv.ParseFloat(v2, 64)
  143. ch <- prometheus.MustNewConstMetric(
  144. c.AdInfo4Desc,
  145. prometheus.GaugeValue,
  146. v3,
  147. v["appType"], v["adGroup"], v["adType"], k,
  148. )
  149. }
  150. }
  151. }
  152. resp, err = adRequestStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), adConfigQuerySQL, 0, 0, true)
  153. if err != nil || resp == nil {
  154. log.Printf("get ad config logs error %+v resp is %+v.", err, resp)
  155. return
  156. }
  157. for _, v := range resp.Logs {
  158. for k, v2 := range v {
  159. if k == "total" ||
  160. k == "success" ||
  161. k == "error" ||
  162. k == "hasAds" ||
  163. k == "noAds" {
  164. v3, _ := strconv.ParseFloat(v2, 64)
  165. ch <- prometheus.MustNewConstMetric(
  166. c.AdConfigDesc,
  167. prometheus.GaugeValue,
  168. v3,
  169. v["appType"], k,
  170. )
  171. }
  172. }
  173. }
  174. // resp, err = videoActionStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), videoPlayCountQuerySQL, 0, 0, true)
  175. // if err != nil || resp == nil {
  176. // log.Printf("get video plyaCount logs error %+v resp is %+v.", err, resp)
  177. // return
  178. // }
  179. // for _, v := range resp.Logs {
  180. // v2, _ := strconv.ParseFloat(v["plyaCount"], 64)
  181. // ch <- prometheus.MustNewConstMetric(
  182. // c.VideoPlayDesc,
  183. // prometheus.GaugeValue,
  184. // v2,
  185. // v["appType"], "videoPlayTotal", v["pageSource"],
  186. // )
  187. // }
  188. resp, err = videoPlayStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), videoPlayQuerySQL, 0, 0, true)
  189. if err != nil || resp == nil {
  190. log.Printf("get video play logs error %+v resp is %+v.", err, resp)
  191. return
  192. }
  193. for _, v := range resp.Logs {
  194. v2, _ := strconv.ParseFloat(v["cnt"], 64)
  195. ch <- prometheus.MustNewConstMetric(
  196. c.VideoPlayDesc,
  197. prometheus.GaugeValue,
  198. v2,
  199. v["appType"], v["businessType"], v["pageSource"],
  200. )
  201. }
  202. resp, err = longvideoReqeustLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), apiRtQuerySQL, 0, 0, true)
  203. if err != nil || resp == nil {
  204. log.Printf("get api rt logs error %+v resp is %+v.", err, resp)
  205. return
  206. }
  207. for _, v := range resp.Logs {
  208. v3, _ := strconv.ParseFloat(v["rt"], 64)
  209. ch <- prometheus.MustNewConstMetric(
  210. c.ApiRtDesc,
  211. prometheus.GaugeValue,
  212. v3,
  213. v["appType"], v["requestUri"],
  214. )
  215. }
  216. resp, err = videoActionStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), videoDataQuerySQL, 0, 0, true)
  217. if err != nil || resp == nil {
  218. log.Printf("get video data logs error %+v resp is %+v.", err, resp)
  219. return
  220. }
  221. for _, v := range resp.Logs {
  222. v3, _ := strconv.ParseFloat(v["cnt"], 64)
  223. ch <- prometheus.MustNewConstMetric(
  224. c.VideoDataDesc,
  225. prometheus.GaugeValue,
  226. v3,
  227. v["appType"], v["businessType"], v["pageSource"],
  228. )
  229. }
  230. resp, err = longvideoReqeustLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), distributeApiQuerySQL, 0, 0, true)
  231. if err != nil || resp == nil {
  232. log.Printf("get distribute api logs error %+v resp is %+v.", err, resp)
  233. return
  234. }
  235. for _, v := range resp.Logs {
  236. // log.Printf("get distribute api logs %+v \n", v)
  237. for k, v2 := range v {
  238. if k == "等于0" ||
  239. k == "小于4" {
  240. v3, _ := strconv.ParseFloat(v2, 64)
  241. ch <- prometheus.MustNewConstMetric(
  242. c.DistributeApiDesc,
  243. prometheus.GaugeValue,
  244. v3,
  245. v["appType"], v["requestUri"], k,
  246. )
  247. }
  248. }
  249. }
  250. resp, err = videoActionStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), shareDataQuerySQL, 0, 0, true)
  251. if err != nil || resp == nil {
  252. log.Printf("get share data logs error %+v resp is %+v.", err, resp)
  253. return
  254. }
  255. for _, v := range resp.Logs {
  256. v3, _ := strconv.ParseFloat(v["cnt"], 64)
  257. ch <- prometheus.MustNewConstMetric(
  258. c.ShareDataDesc,
  259. prometheus.GaugeValue,
  260. v3,
  261. v["appType"], v["businessType"], v["pageSource"], "user",
  262. )
  263. v3, _ = strconv.ParseFloat(v["cnt1"], 64)
  264. ch <- prometheus.MustNewConstMetric(
  265. c.ShareDataDesc,
  266. prometheus.GaugeValue,
  267. v3,
  268. v["appType"], v["businessType"], v["pageSource"], "all",
  269. )
  270. }
  271. resp, err = simpleeventStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), shareOperation1QuerySQL, 0, 0, true)
  272. if err != nil || resp == nil {
  273. log.Printf("get share operation1 logs error %+v resp is %+v.", err, resp)
  274. return
  275. }
  276. for _, v := range resp.Logs {
  277. v3, _ := strconv.ParseFloat(v["cnt"], 64)
  278. ch <- prometheus.MustNewConstMetric(
  279. c.ShareOperationDesc,
  280. prometheus.GaugeValue,
  281. v3,
  282. v["appType"], v["objectType"],
  283. )
  284. }
  285. resp, err = userShareStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), shareOperation2QuerySQL, 0, 0, true)
  286. if err != nil || resp == nil {
  287. log.Printf("get share operation2 logs error %+v resp is %+v.", err, resp)
  288. return
  289. }
  290. for _, v := range resp.Logs {
  291. v3, _ := strconv.ParseFloat(v["cnt"], 64)
  292. ch <- prometheus.MustNewConstMetric(
  293. c.ShareOperationDesc,
  294. prometheus.GaugeValue,
  295. v3,
  296. v["appType"], v["type"],
  297. )
  298. }
  299. }
  300. type CustomCollector struct {
  301. AdInfoDesc *prometheus.Desc
  302. ApiInfoDesc *prometheus.Desc
  303. AdConfigDesc *prometheus.Desc
  304. AdInfo4Desc *prometheus.Desc
  305. // 视频播放
  306. VideoPlayDesc *prometheus.Desc
  307. // 接口响应时间
  308. ApiRtDesc *prometheus.Desc
  309. // 播放数据
  310. VideoDataDesc *prometheus.Desc
  311. // 分发服务
  312. DistributeApiDesc *prometheus.Desc
  313. // 分享卡片数据
  314. ShareDataDesc *prometheus.Desc
  315. // 分享卡片操作
  316. ShareOperationDesc *prometheus.Desc
  317. }
  318. func main() {
  319. // 注册一个采集器
  320. reg := prometheus.NewPedanticRegistry()
  321. reg.MustRegister(&CustomCollector{
  322. AdInfoDesc: prometheus.NewDesc(
  323. "ad_metrics_info",
  324. "Front report ad metrics info",
  325. []string{"app_type", "ad_group", "ad_type", "_type"},
  326. prometheus.Labels{},
  327. ),
  328. ApiInfoDesc: prometheus.NewDesc(
  329. "api_metrics_info",
  330. "Core API Metrics Info",
  331. []string{"app_type", "url", "_type"},
  332. prometheus.Labels{},
  333. ),
  334. VideoPlayDesc: prometheus.NewDesc(
  335. "video_play_metrics_info",
  336. "Video Play Metrics Info",
  337. []string{"app_type", "_type", "source"},
  338. prometheus.Labels{},
  339. ),
  340. AdConfigDesc: prometheus.NewDesc(
  341. "ad_config_metrics_info",
  342. "Ad system deal request metrics info",
  343. []string{"app_type", "_type"},
  344. prometheus.Labels{},
  345. ),
  346. AdInfo4Desc: prometheus.NewDesc(
  347. "ad_metrics_info_004",
  348. "Front report ad metrics info ab_test004",
  349. []string{"app_type", "ad_group", "ad_type", "_type"},
  350. prometheus.Labels{},
  351. ),
  352. ApiRtDesc: prometheus.NewDesc(
  353. "api_metrics_rt",
  354. "Core API Metrics Response Time",
  355. []string{"app_type", "url"},
  356. prometheus.Labels{},
  357. ),
  358. VideoDataDesc: prometheus.NewDesc(
  359. "video_data_metrics",
  360. "Video Data Metrics",
  361. []string{"app_type", "_type", "source"},
  362. prometheus.Labels{},
  363. ),
  364. DistributeApiDesc: prometheus.NewDesc(
  365. "distribute_api_metrics",
  366. "Distribute Api Metrics",
  367. []string{"app_type", "url", "_type"},
  368. prometheus.Labels{},
  369. ),
  370. ShareDataDesc: prometheus.NewDesc(
  371. "share_data_metrics",
  372. "Share Data Metrics",
  373. []string{"app_type", "_type", "source", "catg"},
  374. prometheus.Labels{},
  375. ),
  376. ShareOperationDesc: prometheus.NewDesc(
  377. "share_operation_metrics",
  378. "Share Operation Metrics",
  379. []string{"app_type", "_type"},
  380. prometheus.Labels{},
  381. ),
  382. })
  383. http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
  384. promhttp.HandlerFor(prometheus.Gatherers{reg}, promhttp.HandlerOpts{}).ServeHTTP(w, r)
  385. })
  386. log.Println("pq Collector Start server at :10000")
  387. if err := http.ListenAndServe(":10000", nil); err != nil {
  388. log.Printf("Error occur when start server %v", err)
  389. os.Exit(1)
  390. }
  391. }