|
|
@@ -35,9 +35,11 @@ def get_partition_df(table, dt):
|
|
|
try:
|
|
|
download_session = odps_client.get_download_session(table, dt)
|
|
|
logger.info(f"表: {table} 中的分区 {dt}, 共有 {download_session.count} 条数据")
|
|
|
- with download_session.open_arrow_reader(0, download_session.count) as reader:
|
|
|
- # 将所有数据加载到 DataFrame 中
|
|
|
- df = pd.concat([batch.to_pandas() for batch in reader])
|
|
|
+ with download_session.open_record_reader(0, download_session.count) as reader:
|
|
|
+ records = []
|
|
|
+ for record in reader:
|
|
|
+ records.append(record.values) # 获取每一行的值
|
|
|
+ df = pd.DataFrame(records, columns=[col.name for col in reader.schema.columns]) # 转换为 DataFrame
|
|
|
except Exception as e:
|
|
|
logger.error(f"下载 {table} -- {dt} 的数据异常: ", e)
|
|
|
|
