recommend-emr-dataprocess/ad/model_predict_analyse.py

import argparse
import sys
import gzip

from hdfs import InsecureClient

client = InsecureClient("http://master-1-1.c-7f31a3eea195cb73.cn-hangzhou.emr.aliyuncs.com:9870", user="spark")


def read_predict(hdfs_path):
    dir = "/dw/recommend/model/34_ad_predict_data/20241004_351_0927_1003_1000/"
    for file in client.list(dir):
        with client.read(dir + file, encoding="utf-8") as reador:
            with gzip.GzipFile(fileobj=reador, mode="rb") as gz_file:
                content = gz_file.read().decode("utf-8")
                print(f"Content of {file}:\n {content}")


def _main():
    read_predict("")


if __name__ == '__main__':
    # parser = argparse.ArgumentParser(description="model_predict_analyse.py")
    # parser.add_argument("-p", "--predict_path_list", type=list, help="config file path")
    # args = parser.parse_args()
    #
    # predict_path_list = args.predict_path_list
    # # 判断参数是否正常
    # if len(predict_path_list) != 2:
    #     sys.exit(1)
    _main()