update Prometheus exporter

all/prometheus/main.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"fmt"
+	"log"
 	"net/http"
 	"os"
 	"strconv"
@@ -23,13 +23,18 @@ var (
 	videoActionStore         *sls.LogStore
 	simpleeventStore         *sls.LogStore
 	userShareStore           *sls.LogStore
+	adRequestStore           *sls.LogStore
 
 	adReportQuerySQL  = `* | SELECT appType, json_extract_scalar(extParams, '$.eventInfos.ab_test002') AS adGroup, CASE WHEN adType = 1 THEN 'BANNER广告' WHEN adType = 2 THEN '视频广告' WHEN adType = 3 THEN '贴片广告' WHEN adType = 4 THEN '插屏广告' WHEN adType = 5 THEN '开屏广告' WHEN adType = 6 THEN '激励广告' ELSE ' ' END AS adType, COUNT( DISTINCT CASE WHEN businessType = 'adLoaded' THEN machineCode END ) AS adLoaded, COUNT( DISTINCT CASE WHEN businessType = 'adRequest' THEN machineCode END ) AS adRequest, COUNT( DISTINCT CASE WHEN businessType = 'adView' THEN machineCode END ) AS adView, COUNT( DISTINCT CASE WHEN businessType = 'adClick' THEN machineCode END ) AS adClick, COUNT( DISTINCT CASE WHEN businessType = 'adShowError' THEN machineCode END ) AS adShowError, COUNT( DISTINCT CASE WHEN businessType = 'adLoadError' THEN machineCode END ) AS adLoadError, COUNT( DISTINCT CASE WHEN businessType = 'adConfigRequest' THEN machineCode END ) AS adConfigRequest, COUNT( DISTINCT CASE WHEN businessType = 'adConfigLoaded' THEN machineCode END ) AS adConfigLoaded GROUP BY adGroup, adType, appType HAVING adGroup IS NOT NULL AND adType != ' ' ORDER BY appType ASC LIMIT 1000`
 	adReportQuerySQL4 = `* | SELECT appType, json_extract_scalar(extParams, '$.eventInfos.ab_test004') AS adGroup, CASE WHEN adType = 1 THEN 'BANNER广告' WHEN adType = 2 THEN '视频广告' WHEN adType = 3 THEN '贴片广告' WHEN adType = 4 THEN '插屏广告' WHEN adType = 5 THEN '开屏广告' WHEN adType = 6 THEN '激励广告' ELSE ' ' END AS adType, COUNT( DISTINCT CASE WHEN businessType = 'adLoaded' THEN machineCode END ) AS adLoaded, COUNT( DISTINCT CASE WHEN businessType = 'adRequest' THEN machineCode END ) AS adRequest, COUNT( DISTINCT CASE WHEN businessType = 'adView' THEN machineCode END ) AS adView, COUNT( DISTINCT CASE WHEN businessType = 'adClick' THEN machineCode END ) AS adClick, COUNT( DISTINCT CASE WHEN businessType = 'adShowError' THEN machineCode END ) AS adShowError, COUNT( DISTINCT CASE WHEN businessType = 'adLoadError' THEN machineCode END ) AS adLoadError, COUNT( DISTINCT CASE WHEN businessType = 'adConfigRequest' THEN machineCode END ) AS adConfigRequest, COUNT( DISTINCT CASE WHEN businessType = 'adConfigLoaded' THEN machineCode END ) AS adConfigLoaded GROUP BY adGroup, adType, appType HAVING adGroup IS NOT NULL AND adType != ' ' ORDER BY appType ASC LIMIT 1000`
 	apiQuerySQL       = `* and requestUri: "/longvideoapi/video/distribute/category/videoList/v2" or requestUri: "/longvideoapi/video/v2/detail" or requestUri: "/longvideoapi/video/v2/sharePageDetail" or requestUri: "/longvideoapi/video/recommend/sharePage/list" | SELECT requestUri, json_extract_scalar(params, '$.appType') as appType, COUNT(*) as total, count( case when resultCode = 0 THEN 1 else null end ) as success, count( case when resultCode != 0 THEN 1 else null end ) as error GROUP by appType, requestUri order by requestUri asc limit 1000`
 	videoPlayQuerySQL = `* | select appType, COUNT(*) as cnt, businessType, pageSource GROUP by appType, businessType, pageSource ORDER BY appType ASC, cnt DESC LIMIT 1000`
 	// videoPlayCountQuerySQL = `* AND businessType : videoPlay | select appType, COUNT(*) as plyaCount, pageSource GROUP BY appType, pageSource ORDER BY appType ASC`
-	adConfigQuerySQL        = `* and requestUri: /longvideoapi/ad/position/info | select json_extract_scalar(params, '$.appType') as appType,count(*) as total, count(case when resultCode = 0 then 1 else null end) as success,count(case when resultCode != 0 then 1 else null end) as error,count(case when resultSize > 0 then 1 else null end) as hasAds,count(case when resultSize <= 0 then 1 else null end) as noAds group by appType`
+
+	// adConfigQuerySQL = `* and requestUri: /longvideoapi/ad/position/info | select json_extract_scalar(params, '$.appType') as appType,count(*) as total, count(case when resultCode = 0 then 1 else null end) as success,count(case when resultCode != 0 then 1 else null end) as error,count(case when resultSize > 0 then 1 else null end) as hasAds,count(case when resultSize <= 0 then 1 else null end) as noAds group by appType`
+
+	adConfigQuerySQL = `* and url : "/ad/position/info"  | SELECT count(*) as total, count( case when json_extract_scalar(responseBody, '$.code') = '0' then 1 else null end ) as success, count( case when json_extract_scalar(responseBody, '$.code') != '0' then 1 else null end ) as error, count( case when json_array_length(json_extract(responseBody, '$.data')) <= 0 then 1 else null end ) as noAds, count( case when json_array_length(json_extract(responseBody, '$.data')) > 0 then 1 else null end ) as hasAds, json_extract(requestBody, '$.baseInfo.appType') as appType group by appType `
+
 	apiRtQuerySQL           = `* and requestUri: "/longvideoapi/video/distribute/category/videoList/v2" or requestUri: "/longvideoapi/video/v2/detail" or requestUri: "/longvideoapi/video/v2/sharePageDetail" or requestUri: "/longvideoapi/video/recommend/sharePage/list" | select "requestUri", "params.appType" as "appType" , avg("expendTime") as "rt" from log group by "requestUri", "appType" order by "appType" asc limit 1000`
 	videoDataQuerySQL       = `* | select appType, COUNT(*) as cnt, businessType, pageSource GROUP by appType, businessType, pageSource ORDER BY appType ASC, businessType LIMIT 1000`
 	distributeApiQuerySQL   = ` * and (requestUri: "/longvideoapi/video/distribute/category/videoList/v2" or requestUri: "/longvideoapi/video/recommend/sharePage/list") and not resultSize: 4 | SELECT json_extract_scalar(params, '$.appType') as appType, requestUri, COUNT( DISTINCT CASE WHEN resultSize = 0 THEN json_extract_scalar(params, '$.machineCode') END ) AS "等于0", COUNT( DISTINCT CASE WHEN (resultSize < 4 and resultSize >0) THEN json_extract_scalar(params, '$.machineCode') END ) AS "小于4" GROUP BY requestUri, appType  ORDER BY appType ASC LIMIT 1000`
@@ -71,6 +76,11 @@ func init() {
 	if err != nil {
 		panic(err)
 	}
+
+	adRequestStore, err = client.GetLogStore("ms-ad", "request-log")
+	if err != nil {
+		panic(err)
+	}
 }
 
 func (c *CustomCollector) Describe(ch chan<- *prometheus.Desc) {
@@ -89,7 +99,7 @@ func (c *CustomCollector) Describe(ch chan<- *prometheus.Desc) {
 func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 	resp, err := longvideoReqeustLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), apiQuerySQL, 0, 0, true)
 	if err != nil || resp == nil {
-		fmt.Printf("get api logs error %+v resp is %+v.", err, resp)
+		log.Printf("get api logs error %+v resp is %+v.", err, resp)
 		return
 	}
 	for _, v := range resp.Logs {
@@ -110,7 +120,7 @@ func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 
 	resp, err = adActionLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), adReportQuerySQL, 0, 0, true)
 	if err != nil || resp == nil {
-		fmt.Printf("get ad report logs ab_test002 error %+v resp is %+v.", err, resp)
+		log.Printf("get ad report logs ab_test002 error %+v resp is %+v.", err, resp)
 		return
 	}
 	for _, v := range resp.Logs {
@@ -135,7 +145,7 @@ func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 	}
 	resp, err = adActionLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), adReportQuerySQL4, 0, 0, true)
 	if err != nil || resp == nil {
-		fmt.Printf("get ad report logs ab_test004 error %+v resp is %+v.", err, resp)
+		log.Printf("get ad report logs ab_test004 error %+v resp is %+v.", err, resp)
 		return
 	}
 	for _, v := range resp.Logs {
@@ -159,9 +169,9 @@ func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 		}
 	}
 
-	resp, err = longvideoReqeustLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), adConfigQuerySQL, 0, 0, true)
+	resp, err = adRequestStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), adConfigQuerySQL, 0, 0, true)
 	if err != nil || resp == nil {
-		fmt.Printf("get ad config logs error %+v resp is %+v.", err, resp)
+		log.Printf("get ad config logs error %+v resp is %+v.", err, resp)
 		return
 	}
 	for _, v := range resp.Logs {
@@ -184,7 +194,7 @@ func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 
 	// resp, err = videoActionStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), videoPlayCountQuerySQL, 0, 0, true)
 	// if err != nil || resp == nil {
-	// 	fmt.Printf("get video plyaCount logs error %+v resp is %+v.", err, resp)
+	// 	log.Printf("get video plyaCount logs error %+v resp is %+v.", err, resp)
 	// 	return
 	// }
 	// for _, v := range resp.Logs {
@@ -199,7 +209,7 @@ func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 
 	resp, err = videoPlayStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), videoPlayQuerySQL, 0, 0, true)
 	if err != nil || resp == nil {
-		fmt.Printf("get video play logs error %+v resp is %+v.", err, resp)
+		log.Printf("get video play logs error %+v resp is %+v.", err, resp)
 		return
 	}
 	for _, v := range resp.Logs {
@@ -214,7 +224,7 @@ func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 
 	resp, err = longvideoReqeustLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), apiRtQuerySQL, 0, 0, true)
 	if err != nil || resp == nil {
-		fmt.Printf("get api rt logs error %+v resp is %+v.", err, resp)
+		log.Printf("get api rt logs error %+v resp is %+v.", err, resp)
 		return
 	}
 	for _, v := range resp.Logs {
@@ -229,7 +239,7 @@ func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 
 	resp, err = videoActionStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), videoDataQuerySQL, 0, 0, true)
 	if err != nil || resp == nil {
-		fmt.Printf("get video data logs error %+v resp is %+v.", err, resp)
+		log.Printf("get video data logs error %+v resp is %+v.", err, resp)
 		return
 	}
 	for _, v := range resp.Logs {
@@ -244,11 +254,11 @@ func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 
 	resp, err = longvideoReqeustLogStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), distributeApiQuerySQL, 0, 0, true)
 	if err != nil || resp == nil {
-		fmt.Printf("get distribute api logs error %+v resp is %+v.", err, resp)
+		log.Printf("get distribute api logs error %+v resp is %+v.", err, resp)
 		return
 	}
 	for _, v := range resp.Logs {
-		// fmt.Printf("get distribute api logs  %+v \n", v)
+		// log.Printf("get distribute api logs  %+v \n", v)
 		for k, v2 := range v {
 			if k == "等于0" ||
 				k == "小于4" {
@@ -265,7 +275,7 @@ func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 
 	resp, err = videoActionStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), shareDataQuerySQL, 0, 0, true)
 	if err != nil || resp == nil {
-		fmt.Printf("get share data logs error %+v resp is %+v.", err, resp)
+		log.Printf("get share data logs error %+v resp is %+v.", err, resp)
 		return
 	}
 	for _, v := range resp.Logs {
@@ -288,7 +298,7 @@ func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 
 	resp, err = simpleeventStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), shareOperation1QuerySQL, 0, 0, true)
 	if err != nil || resp == nil {
-		fmt.Printf("get share operation1 logs error %+v resp is %+v.", err, resp)
+		log.Printf("get share operation1 logs error %+v resp is %+v.", err, resp)
 		return
 	}
 	for _, v := range resp.Logs {
@@ -302,7 +312,7 @@ func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 	}
 	resp, err = userShareStore.GetLogs("", time.Now().Unix()-60, time.Now().Unix(), shareOperation2QuerySQL, 0, 0, true)
 	if err != nil || resp == nil {
-		fmt.Printf("get share operation2 logs error %+v resp is %+v.", err, resp)
+		log.Printf("get share operation2 logs error %+v resp is %+v.", err, resp)
 		return
 	}
 	for _, v := range resp.Logs {
@@ -317,12 +327,12 @@ func (c *CustomCollector) Collect(ch chan<- prometheus.Metric) {
 }
 
 type CustomCollector struct {
-	AdInfoDesc  *prometheus.Desc
-	ApiInfoDesc *prometheus.Desc
+	AdInfoDesc   *prometheus.Desc
+	ApiInfoDesc  *prometheus.Desc
+	AdConfigDesc *prometheus.Desc
+	AdInfo4Desc  *prometheus.Desc
 	// 视频播放
 	VideoPlayDesc *prometheus.Desc
-	AdConfigDesc  *prometheus.Desc
-	AdInfo4Desc   *prometheus.Desc
 	// 接口响应时间
 	ApiRtDesc *prometheus.Desc
 	// 播放数据
@@ -404,9 +414,9 @@ func main() {
 	http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
 		promhttp.HandlerFor(prometheus.Gatherers{reg}, promhttp.HandlerOpts{}).ServeHTTP(w, r)
 	})
-	fmt.Println("Start server at :10000")
+	log.Println("pq Collector Start server at :10000")
 	if err := http.ListenAndServe(":10000", nil); err != nil {
-		fmt.Printf("Error occur when start server %v", err)
+		log.Printf("Error occur when start server %v", err)
 		os.Exit(1)
 	}
 }