hot-words-offline/utils/utils.py

import requests
import json
import traceback
from odps import ODPS
from log import Log
from config import set_config
log_ = Log()
config_, env = set_config()


def request_post(request_url, headers, request_data):
    """
    post 请求 HTTP接口
    :param request_url: 接口URL
    :param headers: 请求头
    :param request_data: 请求参数
    :return: res_data json格式
    """
    try:
        response = requests.post(url=request_url, json=request_data, headers=headers)
        if response.status_code == 200:
            res_data = json.loads(response.text)
            return res_data
        else:
            return None
    except Exception as e:
        log_.error('url: {}, exception: {}, traceback: {}'.format(request_url, e, traceback.format_exc()))
        return None


def request_get(request_url, headers, params=None):
    """
    get 请求 HTTP接口
    :param request_url: 接口URL
    :param headers: 请求头
    :param params: 请求参数
    :return: res_data json格式
    """
    try:
        response = requests.get(url=request_url, headers=headers, params=params)
        if response.status_code == 200:
            res_data = json.loads(response.text)
            return res_data
        else:
            return None
    except Exception as e:
        log_.error('url: {}, exception: {}, traceback: {}'.format(request_url, e, traceback.format_exc()))
        return None


def get_data_from_odps(date, project, table, connect_timeout=3000, read_timeout=500000,
                       pool_maxsize=1000, pool_connections=1000):
    """
    从odps获取数据
    :param date: 日期 type-string '%Y%m%d'
    :param project: type-string
    :param table: 表名 type-string
    :param connect_timeout: 连接超时设置
    :param read_timeout: 读取超时设置
    :param pool_maxsize:
    :param pool_connections:
    :return: records
    """
    odps = ODPS(
        access_id=config_.ODPS_CONFIG['ACCESSID'],
        secret_access_key=config_.ODPS_CONFIG['ACCESSKEY'],
        project=project,
        endpoint=config_.ODPS_CONFIG['ENDPOINT'],
        connect_timeout=connect_timeout,
        read_timeout=read_timeout,
        pool_maxsize=pool_maxsize,
        pool_connections=pool_connections
    )
    records = odps.read_table(name=table, partition='dt=%s' % date)
    return records