model_script/client/ODPSClient.py

import json
import re

import requests
from odps import ODPS
from odps.tunnel import TableTunnel
from util import convert_util, date_util


class ODPSClient:
    def __init__(self):
        self.accessId = "LTAIWYUujJAm7CbH"
        self.accessSecret = "RfSjdiWwED1sGFlsjXv0DlfTnZTG1P"
        self.endpoint = "http://service.odps.aliyun.com/api"
        self.tunnelUrl = "http://dt.cn-hangzhou.maxcompute.aliyun-inc.com"
        self.odps = ODPS(
            self.accessId,
            self.accessSecret,
            "",
            self.endpoint
        )

    def get_all_record(self, project: str, table: str, dt: str) -> list:
        tunnel = TableTunnel(self.odps)
        download_session = tunnel.create_download_session(f"{project}.{table}", partition_spec=f"dt={dt}")
        with download_session.open_record_reader(0, download_session.count) as reader:
            for record in reader:
                print(record)

    @classmethod
    def get_all_partition_info(cls, table_name: str, page_num=1, page_size=30, project="loghubods") -> list[dict]:
        header = {
            "cookie": 'yunpk=1894469520484605; t=35f3266f9f98d4a391e691a92b49fae6; cna=8+1iHgu92GsCAT0whRrtEI4u; '
                      'login_aliyunid_pk=1894469520484605; sca=2176fa73; bd=s0ouCmI%3D; '
                      'copilot-session-id-dw-copilot=6e694b9e-aa2e-46fd-9ef5-77d4680755f1; '
                      'help_csrf=doehZiDyB3oB1Z%2Fn3cPDTOjfQgZgfK3SmTSveK6mkcuU30ul8euzz4E'
                      '%2BkHqlvnYgQ6A2GgsUAGTPFblrZ8s7xJmv7zwWDzSffe4ceCSGnoEo0tIfCoPvPcutqc2iScScUmvCqxLY9dxJnl9Dag0a'
                      'dw%3D%3D; cr_token=83b9653a-28d7-4981-af91-45f5828cd63b; _samesite_flag_=true; activeRegionId='
                      'cn-hangzhou; cookie2=1e2ab9af438ed18a85b4e68fbf4956d5; _tb_token_=e3f046f7b7bfe; aliyun_lang='
                      'zh; ch-m-extra-1=true; mini_login_aliyunid_ticket=3R5H3e3HY2c8gwLZuY5GmS7J.111N6WK3pzaiprUFY6'
                      'PArbC7KmiFxsfbwTDpNuDrALFYtkxoUoWD8bewoZ3BSqdnW8NRpptWx92BTNY5KfrWXrmzTBsxm5gSoiHkqzBDpxeiQ5J'
                      'tAYwpT7Y73ZXA4j2BMT4XSuGyzKEsrETyffAXgLhF4TPX7mn5fHWu7jsocNdMsS7xH6gZrvBu1tcAAbw1Pwqeg4xe1ekC'
                      'HcHog5tx1MytitCbXyyZ4QvMvxWjB47tTyFvbxLxY4VQHLjpW6XQyTGFKPNJnGWJ1Per9Pa6RNsdJwDHfjup.2mWNaj2h'
                      'xzDgGG43QBHWDbN8fJYD21nYmoFyrTT2dwkuVG9mUptkQewi3bhwPMdyBg; login_current_pk=2088112089146396'
                      '46; bs_n_lang=zh_CN; cnaui=%2522zhaohaipeng%2520%2540%25201894469520484605%2522; aui=%2522zha'
                      'ohaipeng%2520%2540%25201894469520484605%2522; login_aliyunid_csrf=_csrf_tk_1929015259855532;'
                      ' login_aliyunid="zhaohaipeng @ 1894469520484605"; login_aliyunid_ticket=3R5H3e3HY2c8gwLZuY5Gm'
                      'S7J.1118P2HCue5f1vZ1yf4bXhrmHGZLNDGj5MZW8SinPps2tPh8RCYRkxDq391yUE2enUjtzR1neLcAmV7FgXbdr7CZy'
                      'e7gX1D8viERV6QmBGoZACsLuJDnjE3useRX6vRio8yTkkEfGxrWSSmUkaCCesJMKZW3zW282np5TgxvNMBuL35MEgv.2m'
                      'WNaj2G4W2ax92ZQn9kmFDwDKXYYc9PumdViGC6kXWkLqzeJe4gGK7HEzt81jXGke; aliyun_site=CN; login_aliyu'
                      'nid_sc=3R5H3e3HY2c8gwLZuY5GmS7K.1113bVmMKv851SdF2aYNXdbgTb2UWJHsApX4rU6pbztYdef5Cc1SWjFqwueUU'
                      'NUgbfpAPb.1QDta7oX8HMFGXX7k6ic77ZGWhChZRfmsKSScgXqZFvWcwKGmzwhpHRzs9YpPy37M; atpsida=bdb8dbc3'
                      '436ae0f6eb6706f9_1724123715_1; c_token=a0f8c3bfb3c211f02349dd40eea30c46; ck2=8918a1a152bf75fa'
                      'e4570ce4ab7cb750; an=zhaohaipeng; lg=true; sg=g26; dw_bff=208811208914639646.%E8%B5%B5%E6%B5%'
                      'B7%E9%B9%8F.42901.1724120895027.1894469520484605.1185874277.26842.OFFICIAL.2.1724304596731.zh'
                      'aohaipeng.5.0dd716a2473fc98b00111c110d1a7a9d6bc696297; currentRegionId=cn-hangzhou; csrf_toke'
                      'n=7a9647ecb28a44b31724204973v33fc17f87; tfstk=f81S02cNPMCq3AQXxLU2hp81SGRB2gNa92TdSwhrJQdJRB_'
                      'GA0-yLg-dhGjITzbLEyNCzN9epuAPOisd0LIUEwoQpgjmU3EhzHsfDGKEaaKeOpscf3KzxBulnG7ta_ykTXOktBEab5PZ'
                      'raAH9wgIK-ooMFLUTbdpJ1p_LTZab5P4eqdQVlWUkTXylexvyepKer9vSFTJ9pK-MrTp80HKwMQYlexx9ehpeqdvSUH-p'
                      'UVj5ewWqaaY8dPfHym1rnGK1VYye5_SLXchGU959aKXuE5XPLt93hqs51_Oy1YCtPHBAwWR4EjQ1oOANg1JhBNIiQ71Hg'
                      '9fFkGXntQG9K60fAbGyg69B_FboMsASOARwkcyq9QA9n588XtOQNAXo6rmwH6Ap_Jkt0EWl__1O9srtf-6paMQ6IlBlhz'
                      'blvDFvz6vmVqtYHpJoU94lriRKccKT05UlYXHeEYWPraj29f..; isg=BJubgTlFeefQqoYNgvt9L9EEKvkFcK9y0QISL'
                      '43Y1hqxbLhOFkIDw4uqBsxizAdq'
        }

        url = f"https://bff-cn-hangzhou.data.aliyun.com/dma/listPartitions_2?pageSize={page_size}&pageNum={page_num}&entityGuid=odps.{project}.{table_name}&entityType=odps-table"
        print(f"请求的阿里云接口为: {url}")
        response = requests.get(url, headers=header)
        resp_json = json.loads(response.text)
        result = []
        dt_hh = re.compile(r'dt=(\d{8})/hh=(\d{2})')
        dt_hh_mm = re.compile(r'dt=(\d{8})/hh=(\d{2})/mm=(\d{2})')
        for datum in resp_json["data"]["data"]:
            s1 = dt_hh.search(datum['name'])
            # s2 = dt_hh_mm.search(datum['name'])
            partition = datum['name']
            if s1:
                partition = f"{s1.group(1)}{s1.group(2)}"
            item = {
                "表名": table_name,
                "name": datum["name"],
                "分区": partition,
                "数据量": datum["recordCount"],
                "数据大小": convert_util.byte_convert(datum['dataSize']),
                "创建时间": date_util.ts_cover_str(datum["gmtCreate"]),
                "更新时间": date_util.ts_cover_str(datum['gmtModified'])
            }
            result.append(item)
        return result