exporter-ex/aliyun-log-exporter.py

import prometheus_client
from prometheus_client import Gauge,start_http_server,Counter
import time
import threading
from io import BytesIO
from aliyun.log.logclient import LogClient
from aliyun.log.getlogsrequest import GetLogsRequest

#
# url_http_code = Counter("url_http_code", "request http_code of the host",['code','url'])
# url_http_request_time = Counter("url_http_request_time", "request http_request_time of the host",['le','url'])
# http_request_total = Counter("http_request_total", "request request total of the host",['url'])
url_http_times_avgs = Gauge("url_http_times_avgs","url  of avgs",['appType', 'url'])
url_http_qps = Gauge("url_http_qps","url  of qps",['url'])

endpoint = 'cn-hangzhou.log.aliyuncs.com'
accessKeyId = 'LTAIWYUujJAm7CbH'
accessKey = 'RfSjdiWwED1sGFlsjXv0DlfTnZTG1P'
client = LogClient(endpoint, accessKeyId, accessKey)
start_time = int(time.time())
end_time = start_time - 60

def count_qps(url):

    logstore = 'request-log'
    project = 'wqsd-longvideoapi'
    topic = ""
    req = GetLogsRequest(project, logstore, end_time, start_time, topic,
                         " params.appType=15 and requestUri:'{}' |select requestUri,count(1) as cnt, date_format(logTimestamp/1000,'%Y-%m-%d %H:%i:%S') as date  group by requestUri, date order by cnt desc limit 1".format(url)
                         , 30, 0, False)
    try:
        res = client.get_logs(req)
    except Exception as e:
        print(e)
        qps = 0
    else:
        if res.body:
            qps = res.body[0]["cnt"]
        else:
            qps = 0
    return  qps


def  count_metric(url):

    qps = count_qps(url)
    # url_http_times_avgs.labels(url).set(avgs)

    url_http_qps.labels(url).set(qps)

def count_threads(url):
    while True:
        t = threading.Thread(target=count_metric,args=(url,))
        t.setDaemon(True)
        t.start()
        time.sleep(60)
if __name__ == '__main__':
    start_http_server(9092)
    requestUri_list = [
        'longvideoapi/sys/config',
        'longvideoapi/video/played',
        'longvideoapi/video/shared/weixin/friend',
        'longvideoapi/user/login/v2',
        'longvideoapi/user/info/getUserInfoExt',
        'longvideoapi/user/token/check',
        'longvideoapi/user/token/refresh',
        'longvideoapi/user/info/homepageHead',
        'longvideoapi/user/share/homepage/circle',
        'longvideoapi/user/idols',
        'longvideoapi/user/fans',
        'longvideoapi/user/recommend/list/v4',
        'longvideoapi/user/outside/subscribeBatch',
        'longvideoapi/user/info/getBaseUserInfo',
        'longvideoapi/user/info/getTargetBaseUserInfo',
        'longvideoapi/user/getWxUserInfoByCode',
        'longvideoapi/user/follow',
        'longvideoapi/user/unfollow',
        'longvideoapi/video/favorite',
        'longvideoapi/video/unfavorite',
        'longvideoapi/video/shared/weixin/circle',
        'longvideoapi/videoCollection/loadUserVideoCollections',
        'longvideoapi/video/v2/sharePage/queryVideosByCollectionId',
        'longvideoapi/video/v2/detailPage/queryVideosByCollectionId',
        'longvideoapi/videoCollection/loadUserVideoCollectionsForShareOrDetailPage',
        'longvideoapi/video/distribute/getAppInfo',
        'longvideoapi/video/distribute/category/videoList/v2',
        'longvideoapi/video/v2/followed',
        'longvideoapi/video/v2/loadHotVideosByCollectionId',
        'longvideoapi/video/v2/loadLatelyVideosByCollectionId',
        'longvideoapi/video/v2/detail',
        'longvideoapi/video/v2/sharePageDetail',
        'longvideoapi/video/send',
        'longvideoapi/video/getCoverImagePaths',
        'longvideoapi/video/updateVideo',
        'longvideoapi/video/deleteVideo',
        'longvideoapi/video/barrage/send',
        'longvideoapi/video/barrage/switch',
        'longvideoapi/video/videoActionReport',
        'longvideoapi/video/barrage/v2/list',
        'longvideoapi/video/shared/h5"',
        'longvideoapi/user/share/report',
        'longvideoapi/oss/signature',
        'longvideoapi/oss/completeUpload',
        'longvideoapi/oss/producevideo/getStsToken',
        'longvideoapi/operate/message/list',
        'longvideoapi/video/message/list',
        'longvideoapi/sys/message/list',
        'longvideoapi/user/station/message/statistics/info',
        'longvideoapi/user/station/message/statistics/read',
        'longvideoapi/user/station/message/totalCount/v2',
        'longvideoapi/user/station/message/totalRead',
        'longvideoapi/user/station/notice/list',
        'longvideoapi/user/station/message/share/list',
        'longvideoapi/user/station/message/prise/list',
        'longvideoapi/user/station/message/favorite/list',
        'longvideoapi/user/station/message/comment/list',
        'longvideoapi/user/station/message/barrage/list',
        'longvideoapi/operation/message/list',
        'longvideoapi/search/userandvideo/list',
        'longvideoapi/search/tips',
        'longvideoapi/search/user/list',
        'longvideoapi/search/video/list',
        'longvideoapi/search/hot/words',
        'longvideoapi/search/hot/videos',
        'longvideoapi/video/distribute/search/recommendVideoes/v2',
        'longvideoapi/comment/top/save',
        'longvideoapi/comment/top/page',
        'longvideoapi/comment/praise',
        'longvideoapi/comment/second/save',
        'longvideoapi/comment/second/page',
        'longvideoapi/comment/delete',
        'longvideoapi/comment/updelete',
        'longvideoapi/comment/hots',
        'longvideoapi/bank/withdraw/getBankWithdrawConfig',
        'longvideoapi/bank/withdraw/getBankWithdrawRecordList',
        'longvideoapi/bank/withdraw/wxSamllapp/requestWithdraw',
        'longvideoapi/purchase/video/list',
        'longvideoapi/profits/myself/head',
        'longvideoapi/profits/myself/list',
        'longvideoapi/profits/myself/recordList/v2',
        'longvideoapi/profits/reward/recordListAccordingToUser',
        'longvideoapi/profits/reward/recordListAccordingToVideo',
        'longvideoapi/video/reward/getVideoRewardHeadInfo',
        'longvideoapi/video/reward/getVideoRewardRecordList',
        'longvideoapi/video/reward/getVideoRewardConfig',
        'longvideoapi/video/reward/getVideoRewardConfig/v2',
        'longvideoapi/order/updateClientPayStatus',
        'longvideoapi/order/purchase',
        'longvideoapi/profits/videoPay/head',
        'longvideoapi/profits/videoPay/list',
        'longvideoapi/video/getPayedVideoPath',
        'longvideoapi/order/reward',
        'longvideoapi/video/reward/getVideoDetailRewardInfo',
        'longvideoapi/video/recommend/detailPage/list',
        'longvideoapi/video/recommend/sharePage/list',
        'longvideoapi/measure/report/view',
        'longvideoapi/video/validatePwd"',
        'longvideoapi/video/validatePwdAuth"',
        'longvideoapi/video/v2/getCutVideo',
        'longvideoapi/frontConfig/getWxFrontConfig',
        'longvideoapi/frontConfig/getVideoPlayConfig',
        'longvideoapi/frontConfig/getUserFrontConfig',
        'longvideoapi/rhythmapp/homepage/getAllVideos',
        'longvideoapi/ad/position/info'
            ]
    threads = []
    for url in requestUri_list:
        t = threading.Thread(target=count_threads,args=(url,))
        threads.append(t)
    for thread in threads:
        thread.setDaemon(True)
        thread.start()
    thread.join()