From 2e5fc81081fc15964ae5df064e092f699424a974 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Mon, 16 Nov 2026 20:27:28 -0800 Subject: [PATCH 1/2] Plan a prepared query again after a table in a subquery, or a view, changes A prepared query checked that the tables it reads hadn't been registered again, but missed two kinds of change, and returned stale rows after them: - A table read only in a subquery. The tables a plan reads were found by following each node's inputs, which leave out the subqueries held in its expressions. They're now read from the plan's display, which prints every subquery's plan; datafusion-python's expressions don't reach every subquery's plan, EXISTS's for one. - A view replaced or dropped. CREATE OR REPLACE VIEW and DROP VIEW now count as a table's replacement does, and a prepared query, with or without gradients, raises after a view it reads changes. Fixes #319. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NyN7idACeDMh94qrnFMu6h --- tests/test_ad.py | 13 +++++++ tests/test_prepare.py | 31 ++++++++++++++++ xarray_sql/sql.py | 85 +++++++++++++++++++++++++++++-------------- 3 files changed, 101 insertions(+), 28 deletions(-) diff --git a/tests/test_ad.py b/tests/test_ad.py index dadbe764..fb705406 100644 --- a/tests/test_ad.py +++ b/tests/test_ad.py @@ -161,6 +161,19 @@ def test_context_prepare_runs_gradient_steps_and_reads_writes(problem): ) +def test_prepared_gradient_reads_a_view_as_it_was_prepared(problem): + ctx, x, w, y = problem + ctx.sql("CREATE VIEW features AS SELECT * FROM data").collect() + step = ctx.prepare( + f"WITH {LOSS.replace('FROM data', 'FROM features')} " + "SELECT feature, grad(loss, w) AS w FROM weights ORDER BY feature" + ) + np.testing.assert_allclose(step()["w"], 2 * x.T @ (x @ w - y)) + ctx.sql("CREATE OR REPLACE VIEW features AS SELECT * FROM data").collect() + with pytest.raises(ValueError, match="replaced or dropped"): + step() + + def test_prepared_gradient_writes_only_the_tables_it_reads(problem): ctx, x, w, y = problem other = xr.Dataset( diff --git a/tests/test_prepare.py b/tests/test_prepare.py index 935c044f..cf64db60 100644 --- a/tests/test_prepare.py +++ b/tests/test_prepare.py @@ -260,6 +260,37 @@ def test_a_table_replaced_in_sql_counts_too(comment): query() +@pytest.mark.parametrize( + "query", + [ + "SELECT i FROM a WHERE i IN (SELECT i FROM b) GROUP BY i ORDER BY i", + "SELECT i, (SELECT MAX(w) FROM b) AS m FROM a GROUP BY i ORDER BY i", + ( + "SELECT i FROM a WHERE EXISTS (SELECT 1 FROM b WHERE b.i = a.i) " + "GROUP BY i ORDER BY i" + ), + ], +) +def test_a_table_read_only_in_a_subquery_counts_too(ctx, query): + prepared = ctx.prepare(query) + ctx.deregister_table("b") + ctx.from_dataset("b", _b(np.ones((4, 3)))) + with pytest.raises(ValueError, match="registered again"): + prepared() + + +@pytest.mark.parametrize( + "statement", + ["CREATE OR REPLACE VIEW v AS SELECT w AS x FROM b", "DROP VIEW v"], +) +def test_a_view_replaced_or_dropped_in_sql_counts_too(ctx, statement): + ctx.sql("CREATE VIEW v AS SELECT v AS x FROM a").collect() + query = ctx.prepare("SELECT SUM(x) AS s FROM v") + ctx.sql(statement).collect() + with pytest.raises(ValueError, match="replaced or dropped"): + query() + + def test_a_dataset_replaced_in_sql_takes_no_writes(ctx): ctx.sql("CREATE OR REPLACE TABLE a AS SELECT * FROM b").collect() query = ctx.prepare("SELECT SUM(w) AS s FROM a") diff --git a/xarray_sql/sql.py b/xarray_sql/sql.py index 8b238502..d98620d3 100644 --- a/xarray_sql/sql.py +++ b/xarray_sql/sql.py @@ -10,7 +10,6 @@ import xarray as xr from datafusion import SessionConfig, SessionContext from datafusion.catalog import Schema -from datafusion.expr import TableScan from datafusion.plan import ExecutionPlan, LogicalPlan from . import cftime as cft @@ -301,8 +300,8 @@ def sql(self, query: str, *args, **kwargs) -> XarrayDataFrame: Returns: An [XarrayDataFrame][xarray_sql.ds.XarrayDataFrame] wrapping the DataFusion DataFrame. """ - # DataFusion runs a statement that replaces or drops a table as it - # plans it, without deregistering the table. + # DataFusion runs a statement that replaces or drops a table or a + # view as it plans it, without deregistering it. replaced = _table_replaced(query) existed = replaced is not None and self.table_exist(replaced) if _ad is not None and _ad.calls_ad(query): @@ -441,16 +440,16 @@ class PreparedQuery: places, since xarray-sql reads them as NULL. To change anything else, register the new Dataset and prepare the - query again. A call raises if a table the query reads has been - registered again since it was prepared. Functions such as ``now()`` + query again. A call raises if a table the query reads, in a subquery + too, has been registered again since it was prepared, or a view it + reads has been replaced or dropped. Functions such as ``now()`` are evaluated once, when the query is prepared. A query that calls ``grad``, ``vjp`` or ``jvp`` keeps the programs ddx builds for those calls, rather than a plan. Preparing it runs it once, to build them and to find the tables it reads. A table it reads may also be registered again under the same name, with the same columns and types. - A view it reads must keep its definition; prepare the query again after - changing one. + A call raises if a view it reads has been replaced or dropped. """ def __init__(self, ctx: Context, query: str): @@ -464,6 +463,11 @@ def __init__(self, ctx: Context, query: str): self._writable = set().union(*map(_tables_read, plans)) & set( ctx._registered_datasets ) + # ddx built its programs from each view's SQL as it was then. + self._read = { + name: ctx._deregistrations.get(name, 0) + for name in set().union(*map(_views_read, plans)) + } return start = time.perf_counter() df = SessionContext.sql(ctx, query) @@ -474,15 +478,18 @@ def __init__(self, ctx: Context, query: str): # it was prepared with, and functions such as now() keep the values # they were folded into. self._optimized = df.optimized_logical_plan() - # Each table the query reads, by how often its name had been - # deregistered; and of those, the Datasets a call may write into. + # Each table and view the query reads, by how often its name had + # been deregistered; and of the tables, the Datasets a call may + # write into. + logical = df.logical_plan() + tables = _tables_read(logical) self._read = { name: ctx._deregistrations.get(name, 0) - for name in _tables_read(df.logical_plan()) + for name in tables | _views_read(logical) } self._registered = { name: ctx._registered_datasets[name] - for name in self._read + for name in tables if name in ctx._registered_datasets } # A plan keeps state from a run (a hash join keeps the table it @@ -521,14 +528,15 @@ def __call__(self, **tables: xr.Dataset) -> pa.Table: f"the query reads no Dataset registered as {name!r} " "when it was prepared" ) - if self._gradient is not None: - return self._gradient(lambda: self._ctx._write(tables)) for name, deregistrations in self._read.items(): if self._ctx._deregistrations.get(name, 0) != deregistrations: raise ValueError( - f"{name!r} has been registered again since the query was " - "prepared; prepare the query again" + f"{name!r} has been registered again, replaced or " + "dropped since the query was prepared; prepare the " + "query again" ) + if self._gradient is not None: + return self._gradient(lambda: self._ctx._write(tables)) self._ctx._write(tables) if self._plan is None: df = self._ctx.create_dataframe_from_logical_plan(self._optimized) @@ -553,31 +561,52 @@ def _copy(ctx: SessionContext, plan: ExecutionPlan) -> ExecutionPlan: return ExecutionPlan.from_bytes(ctx, plan.to_bytes()) +# A plan's display prints each of its nodes on a line of its own, with each +# subquery's plan under the node whose expression holds it. The display is +# read, rather than the nodes, since a node's inputs leave out its +# subqueries, and datafusion-python's expressions don't reach every +# subquery's plan (``EXISTS``'s, for one). DataFusion prints a scan's +# options after its table's name, and a view's plan under its name. +_SCAN = re.compile( + r"^ *TableScan: (.+?)" + r"(?: (?:projection|partial_filters|full_filters|fetch)=.*)?$", + re.MULTILINE, +) +_ALIAS = re.compile(r"^ *SubqueryAlias: (.+)$", re.MULTILINE) + + def _tables_read(plan: LogicalPlan) -> set[str]: - """The names of the tables ``plan`` scans, through any views.""" - names = set() - stack = [plan] - while stack: - node = stack.pop() - variant = node.to_variant() - if isinstance(variant, TableScan): - names.add(variant.table_name()) - stack.extend(node.inputs()) - return names + """The names of the tables ``plan`` scans, in its subqueries too, and + through any views.""" + return set(_SCAN.findall(plan.display_indent())) + + +def _views_read(plan: LogicalPlan) -> set[str]: + """The names of the views ``plan`` reads, in its subqueries too. + + DataFusion plans a view as its query, under its name, as it plans + ``FROM t AS x`` under ``x``, so these include the names a query gives + with ``AS``, less those of the tables it scans. One that's also the + name of a table the query doesn't read makes a prepared query raise + once that table is registered again, but never read anything wrongly. + """ + text = plan.display_indent() + return set(_ALIAS.findall(text)) - set(_SCAN.findall(text)) _TABLE_DDL = re.compile( # Any comments before the statement, then the statement. r"(?:\s+|--[^\n]*|/\*.*?\*/)*" - r"(?:CREATE\s+OR\s+REPLACE\s+(?:EXTERNAL\s+)?TABLE|DROP\s+TABLE)" + r"(?:CREATE\s+OR\s+REPLACE\s+(?:(?:EXTERNAL\s+)?TABLE|VIEW)" + r"|DROP\s+(?:TABLE|VIEW))" r"\s+(?:IF\s+EXISTS\s+)?(\"[^\"]+\"|[\w.]+)", re.IGNORECASE | re.DOTALL, ) def _table_replaced(query: str) -> str | None: - """The name of the table ``query`` replaces or drops, as it's written, - or ``None`` for any other statement.""" + """The name of the table or view ``query`` replaces or drops, as it's + written, or ``None`` for any other statement.""" match = _TABLE_DDL.match(query) return None if match is None else match.group(1) From 9ea55970e58d9d4b57097a710a4ec44b6ef18571 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Mon, 16 Nov 2026 20:40:54 -0800 Subject: [PATCH 2/2] Count a table or view changed under any name SQL can give it - A query may name a table with DataFusion's default catalog and schema or without them, so a name read and a name replaced are both counted without them: public.v and v are one view. - A name in DDL is read part by part, so "public"."v" is all of it, and as a plan's display prints it: quoted parts keep their case, and the rest are lowered. - Comments may come between DDL's words, as SQL allows: DROP /* x */ VIEW v drops v. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01NyN7idACeDMh94qrnFMu6h --- tests/test_prepare.py | 12 +++++++++--- xarray_sql/sql.py | 45 ++++++++++++++++++++++++++++++++----------- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/tests/test_prepare.py b/tests/test_prepare.py index cf64db60..90b5e86e 100644 --- a/tests/test_prepare.py +++ b/tests/test_prepare.py @@ -281,11 +281,17 @@ def test_a_table_read_only_in_a_subquery_counts_too(ctx, query): @pytest.mark.parametrize( "statement", - ["CREATE OR REPLACE VIEW v AS SELECT w AS x FROM b", "DROP VIEW v"], + [ + "CREATE OR REPLACE VIEW v AS SELECT w AS x FROM b", + "DROP VIEW v", + 'DROP /* refresh */ VIEW -- now\n "public"."v"', + "DROP VIEW datafusion.public.v", + ], ) -def test_a_view_replaced_or_dropped_in_sql_counts_too(ctx, statement): +@pytest.mark.parametrize("read", ["v", "public.v"]) +def test_a_view_replaced_or_dropped_in_sql_counts_too(ctx, statement, read): ctx.sql("CREATE VIEW v AS SELECT v AS x FROM a").collect() - query = ctx.prepare("SELECT SUM(x) AS s FROM v") + query = ctx.prepare(f"SELECT SUM(x) AS s FROM {read}") ctx.sql(statement).collect() with pytest.raises(ValueError, match="replaced or dropped"): query() diff --git a/xarray_sql/sql.py b/xarray_sql/sql.py index d98620d3..d3b38b4f 100644 --- a/xarray_sql/sql.py +++ b/xarray_sql/sql.py @@ -314,10 +314,7 @@ def sql(self, query: str, *args, **kwargs) -> XarrayDataFrame: else: df = super().sql(query, *args, **kwargs) if replaced is not None and existed: - # DataFusion keeps a quoted name's case, and lowers an unquoted - # one. - quoted = replaced.startswith('"') - self._forget(replaced[1:-1] if quoted else replaced.lower()) + self._forget(_resolved(replaced)) return self._wrap(df) def deregister_table(self, name: str) -> None: @@ -329,6 +326,7 @@ def deregister_table(self, name: str) -> None: def _forget(self, name: str) -> None: """Forget the Dataset registered as ``name``, whose table has been removed or replaced.""" + name = _unqualified(name) self._registered_datasets.pop(name, None) self._deregistrations[name] = self._deregistrations.get(name, 0) + 1 @@ -466,7 +464,9 @@ def __init__(self, ctx: Context, query: str): # ddx built its programs from each view's SQL as it was then. self._read = { name: ctx._deregistrations.get(name, 0) - for name in set().union(*map(_views_read, plans)) + for name in map( + _unqualified, set().union(*map(_views_read, plans)) + ) } return start = time.perf_counter() @@ -485,7 +485,7 @@ def __init__(self, ctx: Context, query: str): tables = _tables_read(logical) self._read = { name: ctx._deregistrations.get(name, 0) - for name in tables | _views_read(logical) + for name in map(_unqualified, tables | _views_read(logical)) } self._registered = { name: ctx._registered_datasets[name] @@ -594,12 +594,16 @@ def _views_read(plan: LogicalPlan) -> set[str]: return set(_ALIAS.findall(text)) - set(_SCAN.findall(text)) +# Whitespace or comments, which SQL reads between any two words. +_GAP = r"(?:\s|--[^\n]*|/\*.*?\*/)" +# A name's part: quoted, which keeps its case, or not, which DataFusion +# lowers. +_PART = r'"[^"]+"|\w+' _TABLE_DDL = re.compile( - # Any comments before the statement, then the statement. - r"(?:\s+|--[^\n]*|/\*.*?\*/)*" - r"(?:CREATE\s+OR\s+REPLACE\s+(?:(?:EXTERNAL\s+)?TABLE|VIEW)" - r"|DROP\s+(?:TABLE|VIEW))" - r"\s+(?:IF\s+EXISTS\s+)?(\"[^\"]+\"|[\w.]+)", + rf"{_GAP}*" + rf"(?:CREATE{_GAP}+OR{_GAP}+REPLACE{_GAP}+(?:(?:EXTERNAL{_GAP}+)?TABLE|VIEW)" + rf"|DROP{_GAP}+(?:TABLE|VIEW))" + rf"{_GAP}+(?:IF{_GAP}+EXISTS{_GAP}+)?((?:{_PART})(?:\.(?:{_PART}))*)", re.IGNORECASE | re.DOTALL, ) @@ -611,6 +615,25 @@ def _table_replaced(query: str) -> str | None: return None if match is None else match.group(1) +def _resolved(name: str) -> str: + """``name``, written in SQL, as a plan's display prints it: each part + unquoted, or lowered if it wasn't quoted, joined by dots.""" + return ".".join( + part[1:-1] if part.startswith('"') else part.lower() + for part in re.findall(_PART, name) + ) + + +def _unqualified(name: str) -> str: + """``name`` without DataFusion's default catalog and schema, which a + query may name or leave out: ``datafusion.public.v`` and ``public.v`` + are both ``v``.""" + for prefix in ("datafusion.public.", "public."): + if name.startswith(prefix): + return name[len(prefix) :] + return name + + def _missing(values: np.ndarray) -> np.ndarray | None: """Where ``values`` holds NaN or NaT, which xarray-sql reads as NULL, or ``None`` for a dtype that holds neither."""