requirement.txt

read_data_from_odps_daily.py

@@ -18,7 +18,7 @@ def read_data_from_odps_daily(dt):
     Read data from ODPS daily and save file to static folder
     :return:
     """
-    sql = f"""select * from loghubods.lastday_return where dt = '{dt}' limit 20;"""
+    sql = f"""select * from loghubods.lastday_return where dt = '{dt}' limit 2000;"""
     data_list = PyODPS().select(sql)
     obj_list = [
         {
@@ -66,10 +66,10 @@ def run():
     yesterday_str = yesterday.strftime("%Y%m%d")
     data_list = read_data_from_odps_daily(yesterday_str)
     # print(data_list)
-    for obj in data_list:
-        save_file_to_local(obj)
-    # with ThreadPoolExecutor(max_workers=10) as Pool:
-    #     Pool.map(save_file_to_local, data_list)
+    # for obj in data_list:
+    #     save_file_to_local(obj)
+    with ThreadPoolExecutor(max_workers=10) as Pool:
+        Pool.map(save_file_to_local, data_list)
 
 
 if __name__ == '__main__':