Joe 9 ماه پیش
والد
کامیت
1630653180
1فایلهای تغییر یافته به همراه8 افزوده شده و 7 حذف شده
  1. 8 7
      qiaojialiang/checkHiveDataUtil.py

+ 8 - 7
qiaojialiang/checkHiveDataUtil.py

@@ -14,8 +14,9 @@ def check_origin_hive(args):
     project = "loghubods"
     # table = "alg_recsys_view_sample_v2"
     table = args.table
-    partition = args.partition
-    count = check_data(project, table, partition)
+    partitionDt = args.partitionDt
+    partitionHh = args.partitionHh
+    count = check_data(project, table, partitionDt, partitionHh)
     if count == 0:
         print("1")
         exit(1)
@@ -24,7 +25,7 @@ def check_origin_hive(args):
         print("0")
 
 
-def check_data(project, table, partition) -> int:
+def check_data(project, table, partitionDt, partitionDtHh) -> int:
     """检查数据是否准备好,输出数据条数"""
     odps = ODPS(
         access_id=ODPS_CONFIG['ACCESSID'],
@@ -39,11 +40,10 @@ def check_data(project, table, partition) -> int:
     try:
         t = odps.get_table(name=table)
         # check_res = t.exist_partition(partition_spec=f'dt={partition}')
-        check_res = t.exist_partition(partition_spec='dt=20240703')
-        # check_res = t.exist_partition('dt=20240703')
+        check_res = t.exist_partition(partition_spec=f'dt={partitionDt},hh={partitionDtHh}')
         if check_res:
             # sql = f'select * from {project}.{table} where dt = {partition} limit 0,100'
-            sql = f'select * from loghubods.alg_recsys_sample_all where dt = 20240703 limit 0,100;'
+            sql = f'select * from {project}.{table} where dt = {partitionDt} and hh={partitionDtHh}'
             with odps.execute_sql(sql=sql).open_reader() as reader:
                 data_count = reader.count
         else:
@@ -57,7 +57,8 @@ def check_data(project, table, partition) -> int:
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='脚本utils')
     # parser.add_argument('--excute_program', type=str, help='执行程序')
-    parser.add_argument('--partition', type=str, help='表分区')
+    parser.add_argument('--partitionDt', type=str, help='表分区Dt')
+    parser.add_argument('--partitionHh', type=str, help='表分区Hh')
     # parser.add_argument('--project', type=str, help='表空间')
     parser.add_argument('--table', type=str, help='表名')
     argv = parser.parse_args()