main.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412
  1. package main
  2. import (
  3. "fmt"
  4. "net/http"
  5. "os"
  6. "strconv"
  7. "time"
  8. "github.com/prometheus/client_golang/prometheus"
  9. "github.com/prometheus/client_golang/prometheus/promhttp"
  10. sls "github.com/aliyun/aliyun-log-go-sdk"
  11. )
  12. var (
  13. AccessKeyID = "LTAIWYUujJAm7CbH"
  14. AccessKeySecret = "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P"
  15. Endpoint = "cn-hangzhou.log.aliyuncs.com"
  16. adActionLogStore *sls.LogStore
  17. longvideoReqeustLogStore *sls.LogStore
  18. videoPlayStore *sls.LogStore
  19. videoActionStore *sls.LogStore
  20. simpleeventStore *sls.LogStore
  21. userShareStore *sls.LogStore
  22. adReportQuerySQL = `* | SELECT appType, json_extract_scalar(extParams, '$.eventInfos.ab_test002') AS adGroup, CASE WHEN adType = 1 THEN 'BANNER广告' WHEN adType = 2 THEN '视频广告' WHEN adType = 3 THEN '贴片广告' WHEN adType = 4 THEN '插屏广告' WHEN adType = 5 THEN '开屏广告' WHEN adType = 6 THEN '激励广告' ELSE ' ' END AS adType, COUNT( DISTINCT CASE WHEN businessType = 'adLoaded' THEN machineCode END ) AS adLoaded, COUNT( DISTINCT CASE WHEN businessType = 'adRequest' THEN machineCode END ) AS adRequest, COUNT( DISTINCT CASE WHEN businessType = 'adView' THEN machineCode END ) AS adView, COUNT( DISTINCT CASE WHEN businessType = 'adClick' THEN machineCode END ) AS adClick, COUNT( DISTINCT CASE WHEN businessType = 'adShowError' THEN machineCode END ) AS adShowError, COUNT( DISTINCT CASE WHEN businessType = 'adLoadError' THEN machineCode END ) AS adLoadError, COUNT( DISTINCT CASE WHEN businessType = 'adConfigRequest' THEN machineCode END ) AS adConfigRequest, COUNT( DISTINCT CASE WHEN businessType = 'adConfigLoaded' THEN machineCode END ) AS adConfigLoaded GROUP BY adGroup, adType, appType HAVING adGroup IS NOT NULL AND adType != ' ' ORDER BY appType ASC LIMIT 1000`
  23. adReportQuerySQL4 = `* | SELECT appType, json_extract_scalar(extParams, '$.eventInfos.ab_test004') AS adGroup, CASE WHEN adType = 1 THEN 'BANNER广告' WHEN adType = 2 THEN '视频广告' WHEN adType = 3 THEN '贴片广告' WHEN adType = 4 THEN '插屏广告' WHEN adType = 5 THEN '开屏广告' WHEN adType = 6 THEN '激励广告' ELSE ' ' END AS adType, COUNT( DISTINCT CASE WHEN businessType = 'adLoaded' THEN machineCode END ) AS adLoaded, COUNT( DISTINCT CASE WHEN businessType = 'adRequest' THEN machineCode END ) AS adRequest, COUNT( DISTINCT CASE WHEN businessType = 'adView' THEN machineCode END ) AS adView, COUNT( DISTINCT CASE WHEN businessType = 'adClick' THEN machineCode END ) AS adClick, COUNT( DISTINCT CASE WHEN businessType = 'adShowError' THEN machineCode END ) AS adShowError, COUNT( DISTINCT CASE WHEN businessType = 'adLoadError' THEN machineCode END ) AS adLoadError, COUNT( DISTINCT CASE WHEN businessType = 'adConfigRequest' THEN machineCode END ) AS adConfigRequest, COUNT( DISTINCT CASE WHEN businessType = 'adConfigLoaded' THEN machineCode END ) AS adConfigLoaded GROUP BY adGroup, adType, appType HAVING adGroup IS NOT NULL AND adType != ' ' ORDER BY appType ASC LIMIT 1000`
  24. apiQuerySQL = `* and requestUri: "/longvideoapi/video/distribute/category/videoList/v2" or requestUri: "/longvideoapi/video/v2/detail" or requestUri: "/longvideoapi/video/v2/sharePageDetail" or requestUri: "/longvideoapi/video/recommend/sharePage/list" | SELECT requestUri, json_extract_scalar(params, '$.appType') as appType, COUNT(*) as total, count( case when resultCode = 0 THEN 1 else null end ) as success, count( case when resultCode != 0 THEN 1 else null end ) as error GROUP by appType, requestUri order by requestUri asc limit 1000`
  25. videoPlayQuerySQL = `* | select appType, COUNT(*) as cnt, businessType, pageSource GROUP by appType, businessType, pageSource ORDER BY appType ASC, cnt DESC LIMIT 1000`
  26. // videoPlayCountQuerySQL = `* AND businessType : videoPlay | select appType, COUNT(*) as plyaCount, pageSource GROUP BY appType, pageSource ORDER BY appType ASC`
  27. adConfigQuerySQL = `* and requestUri: /longvideoapi/ad/position/info | select json_extract_scalar(params, '$.appType') as appType,count(*) as total, count(case when resultCode = 0 then 1 else null end) as success,count(case when resultCode != 0 then 1 else null end) as error,count(case when resultSize > 0 then 1 else null end) as hasAds,count(case when resultSize <= 0 then 1 else null end) as noAds group by appType`
  28. apiRtQuerySQL = `* and requestUri: "/longvideoapi/video/distribute/category/videoList/v2" or requestUri: "/longvideoapi/video/v2/detail" or requestUri: "/longvideoapi/video/v2/sharePageDetail" or requestUri: "/longvideoapi/video/recommend/sharePage/list" | select "requestUri", "params.appType" as "appType" , avg("expendTime") as "rt" from log group by "requestUri", "appType" order by "appType" asc limit 1000`
  29. videoDataQuerySQL = `* | select appType, COUNT(*) as cnt, businessType, pageSource GROUP by appType, businessType, pageSource ORDER BY appType ASC, businessType LIMIT 1000`
  30. distributeApiQuerySQL = ` * and (requestUri: "/longvideoapi/video/distribute/category/videoList/v2" or requestUri: "/longvideoapi/video/recommend/sharePage/list") and not resultSize: 4 | SELECT json_extract_scalar(params, '$.appType') as appType, requestUri, COUNT( DISTINCT CASE WHEN resultSize = 0 THEN json_extract_scalar(params, '$.machineCode') END ) AS "等于0", COUNT( DISTINCT CASE WHEN (resultSize < 4 and resultSize >0) THEN json_extract_scalar(params, '$.machineCode') END ) AS "小于4" GROUP BY requestUri, appType ORDER BY appType ASC LIMIT 1000`
  31. shareDataQuerySQL = `* and ( businessType = 'videoShareFriend' OR businessType = 'videoShareH5' OR businessType = 'videoPublish' ) | select appType, COUNT(DISTINCT mid) as cnt, COUNT(*) as cnt1, businessType, pageSource GROUP by appType, pageSource, businessType ORDER BY appType ASC, cnt DESC LIMIT 1000 `
  32. shareOperation1QuerySQL = `* and objectType: weapp_quitbutton OR objectType: weapp_share_navigator_moveVideo OR objectType: weapp_share_upload OR ( objectType: weapp_share_shareFriendWindow AND businessType: buttonClick ) | select objectType, appType, COUNT(*) as cnt GROUP by objectType, appType ORDER BY appType asc, objectType asc `
  33. shareOperation2QuerySQL = `* and pageSource: "user-videos-share" | select COUNT(*) as cnt, ( CASE WHEN groupShare IS NULL THEN 'shareFriend' WHEN groupShare = 1 THEN 'shareGroup' ELSE ' ' END ) as type, appType GROUP BY type, appType ORDER BY appType asc, type asc `
  34. )
  35. func init() {
  36. client := sls.CreateNormalInterface(Endpoint, AccessKeyID, AccessKeySecret, "")
  37. var err error
  38. longvideoReqeustLogStore, err = client.GetLogStore("wqsd-longvideoapi", "request-log")
  39. if err != nil {
  40. panic(err)
  41. }
  42. adActionLogStore, err = client.GetLogStore("wqsd-longvideo-frontend-log-prd", "ad-action-log")
  43. if err != nil {
  44. panic(err)
  45. }
  46. videoActionStore, err = client.GetLogStore("wqsd-video", "video-action-log")
  47. if err != nil {
  48. panic(err)
  49. }
  50. videoPlayStore, err = client.GetLogStore("wqsd-video", "video-play-log")
  51. if err != nil {
  52. panic(err)
  53. }
  54. simpleeventStore, err = client.GetLogStore("wqsd-longvideo-frontend-log-prd", "simpleevent-log")
  55. if err != nil {
  56. panic(err)
  57. }
  58. userShareStore, err = client.GetLogStore("wqsd-user", "user-share-log")
  59. if err != nil {
  60. panic(err)
  61. }
  62. }
  63. func (c *CustomCollector) Describe(ch chan<- *prometheus.Desc) {
  64. ch <- c.AdInfoDesc
  65. ch <- c.ApiInfoDesc
  66. ch <- c.VideoPlayDesc
  67. ch <- c.AdConfigDesc
  68. ch <- c.AdInfo4Desc
  69. ch <- c.ApiRtDesc
  70. ch <- c.VideoDataDesc
  71. ch <- c.DistributeApiDesc
  72. ch <- c.ShareDataDesc
  73. ch <- c.ShareOperationDesc
  74. }
  75. func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
  76. resp, err := longvideoReqeustLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), apiQuerySQL, 0, 0, true)
  77. if err != nil || resp == nil {
  78. fmt.Printf("get api logs error %+v resp is %+v.", err, resp)
  79. return
  80. }
  81. for _, v := range resp.Logs {
  82. for k, v2 := range v {
  83. if k == "total" ||
  84. k == "success" ||
  85. k == "error" {
  86. v3, _ := strconv.ParseFloat(v2, 64)
  87. ch <- prometheus.MustNewConstMetric(
  88. c.ApiInfoDesc,
  89. prometheus.GaugeValue,
  90. v3,
  91. v["appType"], v["requestUri"], k,
  92. )
  93. }
  94. }
  95. }
  96. resp, err = adActionLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), adReportQuerySQL, 0, 0, true)
  97. if err != nil || resp == nil {
  98. fmt.Printf("get ad report logs ab_test002 error %+v resp is %+v.", err, resp)
  99. return
  100. }
  101. for _, v := range resp.Logs {
  102. for k, v2 := range v {
  103. if k == "adRequest" ||
  104. k == "adShowError" ||
  105. k == "adLoadError" ||
  106. k == "adConfigRequest" ||
  107. k == "adConfigLoaded" ||
  108. k == "adLoaded" ||
  109. k == "adView" ||
  110. k == "adClick" {
  111. v3, _ := strconv.ParseFloat(v2, 64)
  112. ch <- prometheus.MustNewConstMetric(
  113. c.AdInfoDesc,
  114. prometheus.GaugeValue,
  115. v3,
  116. v["appType"], v["adGroup"], v["adType"], k,
  117. )
  118. }
  119. }
  120. }
  121. resp, err = adActionLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), adReportQuerySQL4, 0, 0, true)
  122. if err != nil || resp == nil {
  123. fmt.Printf("get ad report logs ab_test004 error %+v resp is %+v.", err, resp)
  124. return
  125. }
  126. for _, v := range resp.Logs {
  127. for k, v2 := range v {
  128. if k == "adRequest" ||
  129. k == "adShowError" ||
  130. k == "adLoadError" ||
  131. k == "adConfigRequest" ||
  132. k == "adConfigLoaded" ||
  133. k == "adLoaded" ||
  134. k == "adView" ||
  135. k == "adClick" {
  136. v3, _ := strconv.ParseFloat(v2, 64)
  137. ch <- prometheus.MustNewConstMetric(
  138. c.AdInfo4Desc,
  139. prometheus.GaugeValue,
  140. v3,
  141. v["appType"], v["adGroup"], v["adType"], k,
  142. )
  143. }
  144. }
  145. }
  146. resp, err = longvideoReqeustLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), adConfigQuerySQL, 0, 0, true)
  147. if err != nil || resp == nil {
  148. fmt.Printf("get ad config logs error %+v resp is %+v.", err, resp)
  149. return
  150. }
  151. for _, v := range resp.Logs {
  152. for k, v2 := range v {
  153. if k == "total" ||
  154. k == "success" ||
  155. k == "error" ||
  156. k == "hasAds" ||
  157. k == "noAds" {
  158. v3, _ := strconv.ParseFloat(v2, 64)
  159. ch <- prometheus.MustNewConstMetric(
  160. c.AdConfigDesc,
  161. prometheus.GaugeValue,
  162. v3,
  163. v["appType"], k,
  164. )
  165. }
  166. }
  167. }
  168. // resp, err = videoActionStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), videoPlayCountQuerySQL, 0, 0, true)
  169. // if err != nil || resp == nil {
  170. // fmt.Printf("get video plyaCount logs error %+v resp is %+v.", err, resp)
  171. // return
  172. // }
  173. // for _, v := range resp.Logs {
  174. // v2, _ := strconv.ParseFloat(v["plyaCount"], 64)
  175. // ch <- prometheus.MustNewConstMetric(
  176. // c.VideoPlayDesc,
  177. // prometheus.GaugeValue,
  178. // v2,
  179. // v["appType"], "videoPlayTotal", v["pageSource"],
  180. // )
  181. // }
  182. resp, err = videoPlayStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), videoPlayQuerySQL, 0, 0, true)
  183. if err != nil || resp == nil {
  184. fmt.Printf("get video play logs error %+v resp is %+v.", err, resp)
  185. return
  186. }
  187. for _, v := range resp.Logs {
  188. v2, _ := strconv.ParseFloat(v["cnt"], 64)
  189. ch <- prometheus.MustNewConstMetric(
  190. c.VideoPlayDesc,
  191. prometheus.GaugeValue,
  192. v2,
  193. v["appType"], v["businessType"], v["pageSource"],
  194. )
  195. }
  196. resp, err = longvideoReqeustLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), apiRtQuerySQL, 0, 0, true)
  197. if err != nil || resp == nil {
  198. fmt.Printf("get api rt logs error %+v resp is %+v.", err, resp)
  199. return
  200. }
  201. for _, v := range resp.Logs {
  202. v3, _ := strconv.ParseFloat(v["rt"], 64)
  203. ch <- prometheus.MustNewConstMetric(
  204. c.ApiRtDesc,
  205. prometheus.GaugeValue,
  206. v3,
  207. v["appType"], v["requestUri"],
  208. )
  209. }
  210. resp, err = videoActionStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), videoDataQuerySQL, 0, 0, true)
  211. if err != nil || resp == nil {
  212. fmt.Printf("get video data logs error %+v resp is %+v.", err, resp)
  213. return
  214. }
  215. for _, v := range resp.Logs {
  216. v3, _ := strconv.ParseFloat(v["cnt"], 64)
  217. ch <- prometheus.MustNewConstMetric(
  218. c.VideoDataDesc,
  219. prometheus.GaugeValue,
  220. v3,
  221. v["appType"], v["businessType"], v["pageSource"],
  222. )
  223. }
  224. resp, err = longvideoReqeustLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), distributeApiQuerySQL, 0, 0, true)
  225. if err != nil || resp == nil {
  226. fmt.Printf("get distribute api logs error %+v resp is %+v.", err, resp)
  227. return
  228. }
  229. for _, v := range resp.Logs {
  230. // fmt.Printf("get distribute api logs %+v \n", v)
  231. for k, v2 := range v {
  232. if k == "等于0" ||
  233. k == "小于4" {
  234. v3, _ := strconv.ParseFloat(v2, 64)
  235. ch <- prometheus.MustNewConstMetric(
  236. c.DistributeApiDesc,
  237. prometheus.GaugeValue,
  238. v3,
  239. v["appType"], v["requestUri"], k,
  240. )
  241. }
  242. }
  243. }
  244. resp, err = videoActionStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), shareDataQuerySQL, 0, 0, true)
  245. if err != nil || resp == nil {
  246. fmt.Printf("get share data logs error %+v resp is %+v.", err, resp)
  247. return
  248. }
  249. for _, v := range resp.Logs {
  250. v3, _ := strconv.ParseFloat(v["cnt"], 64)
  251. ch <- prometheus.MustNewConstMetric(
  252. c.ShareDataDesc,
  253. prometheus.GaugeValue,
  254. v3,
  255. v["appType"], v["businessType"], v["pageSource"], "user",
  256. )
  257. v3, _ = strconv.ParseFloat(v["cnt1"], 64)
  258. ch <- prometheus.MustNewConstMetric(
  259. c.ShareDataDesc,
  260. prometheus.GaugeValue,
  261. v3,
  262. v["appType"], v["businessType"], v["pageSource"], "all",
  263. )
  264. }
  265. resp, err = simpleeventStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), shareOperation1QuerySQL, 0, 0, true)
  266. if err != nil || resp == nil {
  267. fmt.Printf("get share operation1 logs error %+v resp is %+v.", err, resp)
  268. return
  269. }
  270. for _, v := range resp.Logs {
  271. v3, _ := strconv.ParseFloat(v["cnt"], 64)
  272. ch <- prometheus.MustNewConstMetric(
  273. c.ShareOperationDesc,
  274. prometheus.GaugeValue,
  275. v3,
  276. v["appType"], v["objectType"],
  277. )
  278. }
  279. resp, err = userShareStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), shareOperation2QuerySQL, 0, 0, true)
  280. if err != nil || resp == nil {
  281. fmt.Printf("get share operation2 logs error %+v resp is %+v.", err, resp)
  282. return
  283. }
  284. for _, v := range resp.Logs {
  285. v3, _ := strconv.ParseFloat(v["cnt"], 64)
  286. ch <- prometheus.MustNewConstMetric(
  287. c.ShareOperationDesc,
  288. prometheus.GaugeValue,
  289. v3,
  290. v["appType"], v["type"],
  291. )
  292. }
  293. }
  294. type CustomCollector struct {
  295. AdInfoDesc *prometheus.Desc
  296. ApiInfoDesc *prometheus.Desc
  297. // 视频播放
  298. VideoPlayDesc *prometheus.Desc
  299. AdConfigDesc *prometheus.Desc
  300. AdInfo4Desc *prometheus.Desc
  301. // 接口响应时间
  302. ApiRtDesc *prometheus.Desc
  303. // 播放数据
  304. VideoDataDesc *prometheus.Desc
  305. // 分发服务
  306. DistributeApiDesc *prometheus.Desc
  307. // 分享卡片数据
  308. ShareDataDesc *prometheus.Desc
  309. // 分享卡片操作
  310. ShareOperationDesc *prometheus.Desc
  311. }
  312. func main() {
  313. // 注册一个采集器
  314. reg := prometheus.NewPedanticRegistry()
  315. reg.MustRegister(&CustomCollector{
  316. AdInfoDesc: prometheus.NewDesc(
  317. "ad_metrics_info",
  318. "Front report ad metrics info",
  319. []string{"app_type", "ad_group", "ad_type", "_type"},
  320. prometheus.Labels{},
  321. ),
  322. ApiInfoDesc: prometheus.NewDesc(
  323. "api_metrics_info",
  324. "Core API Metrics Info",
  325. []string{"app_type", "url", "_type"},
  326. prometheus.Labels{},
  327. ),
  328. VideoPlayDesc: prometheus.NewDesc(
  329. "video_play_metrics_info",
  330. "Video Play Metrics Info",
  331. []string{"app_type", "_type", "source"},
  332. prometheus.Labels{},
  333. ),
  334. AdConfigDesc: prometheus.NewDesc(
  335. "ad_config_metrics_info",
  336. "Ad system deal request metrics info",
  337. []string{"app_type", "_type"},
  338. prometheus.Labels{},
  339. ),
  340. AdInfo4Desc: prometheus.NewDesc(
  341. "ad_metrics_info_004",
  342. "Front report ad metrics info ab_test004",
  343. []string{"app_type", "ad_group", "ad_type", "_type"},
  344. prometheus.Labels{},
  345. ),
  346. ApiRtDesc: prometheus.NewDesc(
  347. "api_metrics_rt",
  348. "Core API Metrics Response Time",
  349. []string{"app_type", "url"},
  350. prometheus.Labels{},
  351. ),
  352. VideoDataDesc: prometheus.NewDesc(
  353. "video_data_metrics",
  354. "Video Data Metrics",
  355. []string{"app_type", "_type", "source"},
  356. prometheus.Labels{},
  357. ),
  358. DistributeApiDesc: prometheus.NewDesc(
  359. "distribute_api_metrics",
  360. "Distribute Api Metrics",
  361. []string{"app_type", "url", "_type"},
  362. prometheus.Labels{},
  363. ),
  364. ShareDataDesc: prometheus.NewDesc(
  365. "share_data_metrics",
  366. "Share Data Metrics",
  367. []string{"app_type", "_type", "source", "catg"},
  368. prometheus.Labels{},
  369. ),
  370. ShareOperationDesc: prometheus.NewDesc(
  371. "share_operation_metrics",
  372. "Share Operation Metrics",
  373. []string{"app_type", "_type"},
  374. prometheus.Labels{},
  375. ),
  376. })
  377. http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
  378. promhttp.HandlerFor(prometheus.Gatherers{reg}, promhttp.HandlerOpts{}).ServeHTTP(w, r)
  379. })
  380. fmt.Println("Start server at :10000")
  381. if err := http.ListenAndServe(":10000", nil); err != nil {
  382. fmt.Printf("Error occur when start server %v", err)
  383. os.Exit(1)
  384. }
  385. }