|
|
@@ -1,5 +1,6 @@
|
|
|
from __future__ import annotations
|
|
|
|
|
|
+import re
|
|
|
from dataclasses import dataclass
|
|
|
|
|
|
from sqlglot import exp, parse, parse_one
|
|
|
@@ -62,11 +63,13 @@ class SQLGuard:
|
|
|
return refs
|
|
|
|
|
|
@staticmethod
|
|
|
- def validate_product_efficiency_contract(sql: str, data_mode: str | None) -> None:
|
|
|
+ def validate_product_efficiency_contract(
|
|
|
+ sql: str, data_mode: str | None, *, bucketed: bool = True
|
|
|
+ ) -> None:
|
|
|
if data_mode not in {"offline", "realtime"}:
|
|
|
return
|
|
|
statement = parse_one(sql, read="hive")
|
|
|
- if data_mode == "offline":
|
|
|
+ if data_mode == "offline" and bucketed:
|
|
|
active_selects = [
|
|
|
table.find_ancestor(exp.Select)
|
|
|
for table in statement.find_all(exp.Table)
|
|
|
@@ -88,6 +91,23 @@ class SQLGuard:
|
|
|
"禁止使用 useractive_log.rootsessionid"
|
|
|
)
|
|
|
|
|
|
+ if data_mode == "realtime":
|
|
|
+ invalid_sources: list[str] = []
|
|
|
+ for table in statement.find_all(exp.Table):
|
|
|
+ if table.name.lower() not in {
|
|
|
+ "useractive_log_per5min",
|
|
|
+ "user_share_log_per5min",
|
|
|
+ }:
|
|
|
+ continue
|
|
|
+ select = table.find_ancestor(exp.Select)
|
|
|
+ if select is None or not SQLGuard._has_day_prefix_filter(select, table):
|
|
|
+ invalid_sources.append(table.name)
|
|
|
+ if invalid_sources:
|
|
|
+ names = "、".join(sorted(set(invalid_sources)))
|
|
|
+ raise SQLValidationError(
|
|
|
+ f"实时表 {names} 必须使用 dt LIKE 'yyyyMMdd%' 查询当天累计数据"
|
|
|
+ )
|
|
|
+
|
|
|
for table in statement.find_all(exp.Table):
|
|
|
if table.name.lower() not in {"video_action_log_applet", "video_action_log_flow"}:
|
|
|
continue
|
|
|
@@ -108,37 +128,77 @@ class SQLGuard:
|
|
|
@staticmethod
|
|
|
def validate_partition_predicates(sql: str, partitions: dict[str, list[str]]) -> None:
|
|
|
statement = parse_one(sql, read="hive")
|
|
|
- predicate_roots: list[exp.Expression] = []
|
|
|
- predicate_roots.extend(where.this for where in statement.find_all(exp.Where))
|
|
|
- predicate_roots.extend(
|
|
|
- on for join in statement.find_all(exp.Join) if (on := join.args.get("on")) is not None
|
|
|
- )
|
|
|
- partitioned_count = sum(bool(columns) for columns in partitions.values())
|
|
|
missing: list[str] = []
|
|
|
for table, columns in partitions.items():
|
|
|
if not columns:
|
|
|
continue
|
|
|
table_name = table.rsplit(".", 1)[-1].lower()
|
|
|
- qualifiers = {
|
|
|
- candidate.alias_or_name.lower()
|
|
|
+ occurrences = [
|
|
|
+ candidate
|
|
|
for candidate in statement.find_all(exp.Table)
|
|
|
if candidate.name.lower() == table_name
|
|
|
- }
|
|
|
+ ]
|
|
|
partition_names = {column.lower() for column in columns}
|
|
|
if table_name == "video_action_log_applet":
|
|
|
partition_names.discard("business")
|
|
|
- found = False
|
|
|
- for predicate in predicate_roots:
|
|
|
- for candidate in predicate.find_all(exp.Column):
|
|
|
- qualifier = candidate.table.lower() if candidate.table else ""
|
|
|
- if candidate.name.lower() not in partition_names:
|
|
|
- continue
|
|
|
- if qualifier in qualifiers or (not qualifier and partitioned_count == 1):
|
|
|
- found = True
|
|
|
- break
|
|
|
- if found:
|
|
|
- break
|
|
|
- if not found:
|
|
|
+ if any(
|
|
|
+ not SQLGuard._table_has_partition_filter(occurrence, partition_names)
|
|
|
+ for occurrence in occurrences
|
|
|
+ ):
|
|
|
missing.append(f"{table}({', '.join(sorted(partition_names))})")
|
|
|
if missing:
|
|
|
raise SQLValidationError("分区表缺少明确分区条件：" + "；".join(missing))
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _table_has_partition_filter(table: exp.Table, partition_names: set[str]) -> bool:
|
|
|
+ select = table.find_ancestor(exp.Select)
|
|
|
+ if select is None:
|
|
|
+ return False
|
|
|
+ sources = [
|
|
|
+ candidate
|
|
|
+ for candidate in select.find_all(exp.Table)
|
|
|
+ if candidate.find_ancestor(exp.Select) is select
|
|
|
+ ]
|
|
|
+ allow_unqualified = len(sources) == 1
|
|
|
+ qualifier = table.alias_or_name.lower()
|
|
|
+ predicates: list[exp.Expression] = [
|
|
|
+ where.this
|
|
|
+ for where in select.find_all(exp.Where)
|
|
|
+ if where.find_ancestor(exp.Select) is select
|
|
|
+ ]
|
|
|
+ predicates.extend(
|
|
|
+ join.args["on"]
|
|
|
+ for join in select.find_all(exp.Join)
|
|
|
+ if join.find_ancestor(exp.Select) is select and join.args.get("on") is not None
|
|
|
+ )
|
|
|
+ for predicate in predicates:
|
|
|
+ for column in predicate.find_all(exp.Column):
|
|
|
+ if column.name.lower() not in partition_names:
|
|
|
+ continue
|
|
|
+ column_qualifier = column.table.lower() if column.table else ""
|
|
|
+ if column_qualifier == qualifier or (not column_qualifier and allow_unqualified):
|
|
|
+ return True
|
|
|
+ return False
|
|
|
+
|
|
|
+ @staticmethod
|
|
|
+ def _has_day_prefix_filter(select: exp.Select, table: exp.Table) -> bool:
|
|
|
+ sources = [
|
|
|
+ candidate
|
|
|
+ for candidate in select.find_all(exp.Table)
|
|
|
+ if candidate.find_ancestor(exp.Select) is select
|
|
|
+ ]
|
|
|
+ allow_unqualified = len(sources) == 1
|
|
|
+ qualifier = table.alias_or_name.lower()
|
|
|
+ for like in select.find_all(exp.Like):
|
|
|
+ if like.find_ancestor(exp.Select) is not select:
|
|
|
+ continue
|
|
|
+ column = like.this
|
|
|
+ pattern = like.expression
|
|
|
+ if not isinstance(column, exp.Column) or column.name.lower() != "dt":
|
|
|
+ continue
|
|
|
+ column_qualifier = column.table.lower() if column.table else ""
|
|
|
+ if column_qualifier != qualifier and not (not column_qualifier and allow_unqualified):
|
|
|
+ continue
|
|
|
+ if isinstance(pattern, exp.Literal) and re.fullmatch(r"\d{8}%", str(pattern.this)):
|
|
|
+ return True
|
|
|
+ return False
|
刘立冬