|
@@ -16,6 +16,7 @@
|
|
|
python fetch_daily.py tasks/xxx/query.sql --feishu # 获取后上传到飞书表格
|
|
python fetch_daily.py tasks/xxx/query.sql --feishu # 获取后上传到飞书表格
|
|
|
python fetch_daily.py tasks/xxx/query.sql --feishu TOKEN # 指定飞书表格token
|
|
python fetch_daily.py tasks/xxx/query.sql --feishu TOKEN # 指定飞书表格token
|
|
|
python fetch_daily.py tasks/xxx/query.sql --merge --feishu # 仅合并并上传飞书
|
|
python fetch_daily.py tasks/xxx/query.sql --merge --feishu # 仅合并并上传飞书
|
|
|
|
|
+ python fetch_daily.py tasks/xxx/query.sql --config piaoquan_api # 切换 ODPS 配置
|
|
|
"""
|
|
"""
|
|
|
import argparse
|
|
import argparse
|
|
|
import sys
|
|
import sys
|
|
@@ -463,12 +464,12 @@ def get_date_range(start_str, end_str):
|
|
|
return dates
|
|
return dates
|
|
|
|
|
|
|
|
|
|
|
|
|
-def fetch_single_day(dt, sql_template, daily_dir, parallel_threads=0):
|
|
|
|
|
|
|
+def fetch_single_day(dt, sql_template, daily_dir, parallel_threads=0, config="default"):
|
|
|
"""获取单天数据"""
|
|
"""获取单天数据"""
|
|
|
global success_count, fail_count
|
|
global success_count, fail_count
|
|
|
|
|
|
|
|
try:
|
|
try:
|
|
|
- client = ODPSClient()
|
|
|
|
|
|
|
+ client = ODPSClient(config=config)
|
|
|
sql = sql_template.replace("${dt}", dt)
|
|
sql = sql_template.replace("${dt}", dt)
|
|
|
output_file = daily_dir / f"{dt}.csv"
|
|
output_file = daily_dir / f"{dt}.csv"
|
|
|
|
|
|
|
@@ -518,6 +519,7 @@ def main():
|
|
|
parser.add_argument("--sheet-id", type=str, default=None, help="飞书工作表ID")
|
|
parser.add_argument("--sheet-id", type=str, default=None, help="飞书工作表ID")
|
|
|
parser.add_argument("--sort", type=str, default=None, help="排序: 字段:asc/desc")
|
|
parser.add_argument("--sort", type=str, default=None, help="排序: 字段:asc/desc")
|
|
|
parser.add_argument("--cols", type=str, default=None, help="列映射: 原名:新名,...")
|
|
parser.add_argument("--cols", type=str, default=None, help="列映射: 原名:新名,...")
|
|
|
|
|
+ parser.add_argument("--config", type=str, default="default", help="ODPS配置: default 或 piaoquan_api")
|
|
|
args = parser.parse_args()
|
|
args = parser.parse_args()
|
|
|
|
|
|
|
|
# 解析 SQL 文件路径
|
|
# 解析 SQL 文件路径
|
|
@@ -613,7 +615,7 @@ def main():
|
|
|
output_file.parent.mkdir(parents=True, exist_ok=True)
|
|
output_file.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
|
|
try:
|
|
try:
|
|
|
- client = ODPSClient()
|
|
|
|
|
|
|
+ client = ODPSClient(config=args.config)
|
|
|
if args.parallel > 0:
|
|
if args.parallel > 0:
|
|
|
client.execute_sql_result_save_file_parallel(sql_template, str(output_file), workers=args.parallel)
|
|
client.execute_sql_result_save_file_parallel(sql_template, str(output_file), workers=args.parallel)
|
|
|
else:
|
|
else:
|
|
@@ -636,7 +638,7 @@ def main():
|
|
|
|
|
|
|
|
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
|
futures = {
|
|
futures = {
|
|
|
- executor.submit(fetch_single_day, dt, sql_template, daily_dir, args.parallel): dt
|
|
|
|
|
|
|
+ executor.submit(fetch_single_day, dt, sql_template, daily_dir, args.parallel, args.config): dt
|
|
|
for dt in missing_dates
|
|
for dt in missing_dates
|
|
|
}
|
|
}
|
|
|
|
|
|