|
|
@@ -14,8 +14,9 @@ def check_origin_hive(args):
|
|
|
project = "loghubods"
|
|
|
# table = "alg_recsys_view_sample_v2"
|
|
|
table = args.table
|
|
|
- partition = args.partition
|
|
|
- count = check_data(project, table, partition)
|
|
|
+ partitionDt = args.partitionDt
|
|
|
+ partitionHh = args.partitionHh
|
|
|
+ count = check_data(project, table, partitionDt, partitionHh)
|
|
|
if count == 0:
|
|
|
print("1")
|
|
|
exit(1)
|
|
|
@@ -24,7 +25,7 @@ def check_origin_hive(args):
|
|
|
print("0")
|
|
|
|
|
|
|
|
|
-def check_data(project, table, partition) -> int:
|
|
|
+def check_data(project, table, partitionDt, partitionDtHh) -> int:
|
|
|
"""检查数据是否准备好,输出数据条数"""
|
|
|
odps = ODPS(
|
|
|
access_id=ODPS_CONFIG['ACCESSID'],
|
|
|
@@ -39,11 +40,10 @@ def check_data(project, table, partition) -> int:
|
|
|
try:
|
|
|
t = odps.get_table(name=table)
|
|
|
# check_res = t.exist_partition(partition_spec=f'dt={partition}')
|
|
|
- check_res = t.exist_partition(partition_spec='dt=20240703')
|
|
|
- # check_res = t.exist_partition('dt=20240703')
|
|
|
+ check_res = t.exist_partition(partition_spec=f'dt={partitionDt},hh={partitionDtHh}')
|
|
|
if check_res:
|
|
|
# sql = f'select * from {project}.{table} where dt = {partition} limit 0,100'
|
|
|
- sql = f'select * from loghubods.alg_recsys_sample_all where dt = 20240703 limit 0,100;'
|
|
|
+ sql = f'select * from {project}.{table} where dt = {partitionDt} and hh={partitionDtHh}'
|
|
|
with odps.execute_sql(sql=sql).open_reader() as reader:
|
|
|
data_count = reader.count
|
|
|
else:
|
|
|
@@ -57,7 +57,8 @@ def check_data(project, table, partition) -> int:
|
|
|
if __name__ == '__main__':
|
|
|
parser = argparse.ArgumentParser(description='脚本utils')
|
|
|
# parser.add_argument('--excute_program', type=str, help='执行程序')
|
|
|
- parser.add_argument('--partition', type=str, help='表分区')
|
|
|
+ parser.add_argument('--partitionDt', type=str, help='表分区Dt')
|
|
|
+ parser.add_argument('--partitionHh', type=str, help='表分区Hh')
|
|
|
# parser.add_argument('--project', type=str, help='表空间')
|
|
|
parser.add_argument('--table', type=str, help='表名')
|
|
|
argv = parser.parse_args()
|
Joe