From 3c108a7bbcbc109a56a62021e8af89083c2c0469 Mon Sep 17 00:00:00 2001 From: Narendra Date: Thu, 13 Aug 2026 21:09:24 +0530 Subject: [PATCH 01/50] feat: add sphinx-sitemap dependency and configuration for sitemap generation --- docs/pyproject.toml | 1 + docs/source/conf.py | 17 +++++++++++++++-- uv.lock | 27 +++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/docs/pyproject.toml b/docs/pyproject.toml index c09415e8e8c86..c4edfdcfec38e 100644 --- a/docs/pyproject.toml +++ b/docs/pyproject.toml @@ -10,4 +10,5 @@ dependencies = [ "maturin>=1.14.1,<2", "jinja2>=3.1.6,<4", "setuptools>=83.0.0,<84", + "sphinx-sitemap>=2.9.0,<3", ] diff --git a/docs/source/conf.py b/docs/source/conf.py index c8027fc71bd54..81ffce3ad1c2b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -55,6 +55,7 @@ "myst_parser", "sphinx_reredirects", "rustdoc_trim", + "sphinx_sitemap", ] source_suffix = { @@ -89,8 +90,8 @@ html_theme_options = { "logo": { - "image_light": "_static/images/original.svg", - "image_dark": "_static/images/original_dark.svg", + "image_light": "_static/images/original.svg", + "image_dark": "_static/images/original_dark.svg", }, "use_edit_page_button": True, "navbar_center": [], @@ -113,6 +114,18 @@ # resolve at the conventional URL `https://datafusion.apache.org/llms.txt`. html_extra_path = ["llms.txt"] +# Base URL of the published site. Required by sphinx-sitemap to build +# absolute entries in the generated sitemap.xml. +html_baseurl = "https://datafusion.apache.org/" + +# The site is published as a single flat, single-language tree (see +# .github/workflows/docs.yaml, which rsyncs docs/build/html/ straight to the +# site root) — there is no /en/ or version path segment. Override +# sphinx-sitemap's default "{lang}{version}{link}" scheme, which assumes a +# Read the Docs-style multi-language/multi-version layout, so entries +# match the real URLs instead of a nonexistent /en/... prefix. +sitemap_url_scheme = "{link}" + html_logo = "_static/images/2x_bgwhite_original.png" html_css_files = ["theme_overrides.css"] diff --git a/uv.lock b/uv.lock index 85fcce1e9db48..240f0e56472a7 100644 --- a/uv.lock +++ b/uv.lock @@ -341,6 +341,7 @@ dependencies = [ { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "sphinx-reredirects" }, + { name = "sphinx-sitemap" }, ] [package.metadata] @@ -352,6 +353,7 @@ requires-dist = [ { name = "setuptools", specifier = ">=83.0.0,<84" }, { name = "sphinx", specifier = ">=9,<10" }, { name = "sphinx-reredirects", specifier = ">=1.1,<2" }, + { name = "sphinx-sitemap", specifier = ">=2.9.0,<3" }, ] [[package]] @@ -1037,6 +1039,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/f7/b1884cb3188ab181fc81fa00c266699dab600f927a964df02ec3d5d1916a/sphinx-9.1.0-py3-none-any.whl", hash = "sha256:c84fdd4e782504495fe4f2c0b3413d6c2bf388589bb352d439b2a3bb99991978", size = 3921742, upload-time = "2025-12-31T15:09:25.561Z" }, ] +[[package]] +name = "sphinx-last-updated-by-git" +version = "0.3.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/03/fd/de1685b6dab173dff31da24e0d3b29f02873fc24a1cdbb7678721ddc8581/sphinx_last_updated_by_git-0.3.8.tar.gz", hash = "sha256:c145011f4609d841805b69a9300099fc02fed8f5bb9e5bcef77d97aea97b7761", size = 10785, upload-time = "2024-08-11T07:15:54.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/fb/e496f16fa11fbe2dbdd0b5e306ede153dfed050aae4766fc89d500720dc7/sphinx_last_updated_by_git-0.3.8-py3-none-any.whl", hash = "sha256:6382c8285ac1f222483a58569b78c0371af5e55f7fbf9c01e5e8a72d6fdfa499", size = 8580, upload-time = "2024-08-11T07:15:53.244Z" }, +] + [[package]] name = "sphinx-reredirects" version = "1.1.0" @@ -1050,6 +1065,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/81/b5dd07067f3daac6d23687ec737b2d593740671ebcd145830c8f92d381c5/sphinx_reredirects-1.1.0-py3-none-any.whl", hash = "sha256:4b5692273c72cd2d4d917f4c6f87d5919e4d6114a752d4be033f7f5f6310efd9", size = 6351, upload-time = "2025-12-22T08:27:59.724Z" }, ] +[[package]] +name = "sphinx-sitemap" +version = "2.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx-last-updated-by-git" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/17/56fe0f65e3567f829b2b4153a622be1d4b222b781e0d90d7db5a7738f30f/sphinx_sitemap-2.9.0.tar.gz", hash = "sha256:70f97bcdf444e3d68e118355cf82a1f54c4d3c03d651cd17fe87398b26e25e21", size = 6978, upload-time = "2025-10-06T00:24:00.036Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/94/3c57e8b1985e755c48972e2ecd59526d4bf0b52a1fe805bc52a8e98cb92d/sphinx_sitemap-2.9.0-py3-none-any.whl", hash = "sha256:f1f1d3a9ad012ba17a7ef0b560d303bff2d0db26647567d6e810bcc754466664", size = 6218, upload-time = "2025-10-06T00:23:58.778Z" }, +] + [[package]] name = "sphinxcontrib-applehelp" version = "2.0.0" From 1cae7f65137712154f02ed56679e4f07ffbd89bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 13 Aug 2026 17:27:18 +0000 Subject: [PATCH 02/50] perf(catalog): cut `datafusion-catalog` compile time ~6x (#24325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? Adresses: https://github.com/apache/datafusion/issues/13814 First of four; see also #24326 (`datafusion-session`), #24329 (`datafusion` core) and #24330 (`datafusion-catalog-listing`). All independent — different crates, so they can merge in any order. ## Rationale for this change `datafusion-catalog` is only 4.9k lines of source, but it takes **42s** of a cold `cargo build -p datafusion` (measured with `cargo build --timings`). `-Zself-profile` says ~90% of the crate's compile time is `evaluate_obligation`, and ~99% of that is proving `Send`/`Sync`: | trait | time | goals | |---|---|---| | `Send` | 4.04s | 8245 | | `Sync` | 3.97s | 8195 | | everything else | 0.03s | 7355 | 51.8s of trait solving: | impl | trait solving | |---|---| | `MemTable` | 13.7s | | `StreamTable` | 9.0s | | `CteWorkTable` | 8.9s | | `StreamWrite` | 6.9s | | `StreamTableFactory` | 4.5s | | `ViewTable` | 4.4s | | `StreamingTable` | 4.4s | ## What changes are included in this PR? For those impls, the future is now constructed in a small shim function that has **no** where-clauses, so its auto-trait obligations are proved in an empty `ParamEnv` and get cached globally. The trait method is left as a hand-written desugaring of what `#[async_trait]` would have generated, and only forwards — it never creates a coroutine of its own, so it does no auto-trait work. Isolated probe confirming the shape is what matters (5 trivial impls of a local `#[async_trait]` trait taking `&[Expr]`, added to this crate): | variant | crate build | cost of the 5 impls | |---|---|---| | no impls (baseline) | 8.31s | — | | `#[async_trait]` + `async fn` | 11.64s | +3.33s | | `async fn` delegating body to a boxed helper | 14.28s | +5.97s | | desugared signature + boxed shim | 8.01s | ~0 | Note the middle row: moving only the *body* out makes things worse. The `async fn` itself has to go, because its arguments are what the future captures. ## Are these changes tested? The change is mechanical and the compiler checks each rewritten signature against the trait declaration. Interleaved A/B of `cargo rustc -p datafusion-catalog --lib`, alternating 3 times so machine drift cancels out: ``` before: 8.08s 7.90s 7.63s after: 1.77s 1.70s 1.69s ``` `evaluate_obligation` drops from **7.31s to 70ms**, and its goal count from 25,811 to 14,934. In a full `cargo build -p datafusion` the crate's unit goes from 42.3s to ~8s; since it sits alone on the critical path, that time comes straight off the build's wall clock. ## Are there any user-facing changes? No. No public signature changes — after macro expansion the trait methods have the same signatures as before. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 --- datafusion/catalog/src/cte_worktable.rs | 85 +++++++++++---- datafusion/catalog/src/memory/table.rs | 138 ++++++++++++++++++++---- datafusion/catalog/src/stream.rs | 114 ++++++++++++++++++-- datafusion/catalog/src/streaming.rs | 33 +++++- datafusion/catalog/src/view.rs | 32 +++++- 5 files changed, 353 insertions(+), 49 deletions(-) diff --git a/datafusion/catalog/src/cte_worktable.rs b/datafusion/catalog/src/cte_worktable.rs index 5ec688526c92b..37f66b8c56a3e 100644 --- a/datafusion/catalog/src/cte_worktable.rs +++ b/datafusion/catalog/src/cte_worktable.rs @@ -18,6 +18,7 @@ //! CteWorkTable implementation used for recursive queries use std::borrow::Cow; +use std::future::ready; use std::sync::Arc; use arrow::datatypes::SchemaRef; @@ -26,6 +27,7 @@ use datafusion_common::error::Result; use datafusion_expr::{Expr, LogicalPlan, TableProviderFilterPushDown, TableType}; use datafusion_physical_plan::ExecutionPlan; use datafusion_physical_plan::work_table::WorkTableExec; +use futures::future::BoxFuture; use crate::{ScanArgs, ScanResult, Session, TableProvider}; @@ -78,30 +80,35 @@ impl TableProvider for CteWorkTable { TableType::Temporary } - async fn scan( - &self, - state: &dyn Session, - projection: Option<&Vec>, - filters: &[Expr], + fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + projection: Option<&'life2 Vec>, + filters: &'life3 [Expr], limit: Option, - ) -> Result> { - let options = ScanArgs::default() - .with_projection(projection.map(|p| p.as_slice())) - .with_filters(Some(filters)) - .with_limit(limit); - Ok(self.scan_with_args(state, options).await?.into_inner()) + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + 'life3: 'async_trait, + Self: 'async_trait, + { + self.scan_boxed(state, projection, filters, limit) } - async fn scan_with_args<'a>( - &self, - _state: &dyn Session, + fn scan_with_args<'a, 'life0, 'life1, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, args: ScanArgs<'a>, - ) -> Result { - Ok(ScanResult::new(Arc::new(WorkTableExec::new( - self.name.clone(), - Arc::clone(&self.table_schema), - args.projection().map(|p| p.to_vec()), - )?))) + ) -> BoxFuture<'async_trait, Result> + where + 'a: 'async_trait, + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + Box::pin(ready(self.scan_with_args_inner(state, &args))) } fn supports_filters_pushdown( @@ -115,3 +122,41 @@ impl TableProvider for CteWorkTable { ]) } } + +impl CteWorkTable { + fn scan_with_args_inner<'a>( + &self, + _state: &dyn Session, + args: &ScanArgs<'a>, + ) -> Result { + Ok(ScanResult::new(Arc::new(WorkTableExec::new( + self.name.clone(), + Arc::clone(&self.table_schema), + args.projection().map(|p| p.to_vec()), + )?))) + } + + fn scan_boxed<'a>( + &'a self, + state: &'a dyn Session, + projection: Option<&'a Vec>, + filters: &'a [Expr], + limit: Option, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.scan_inner(state, projection, filters, limit)) + } + + async fn scan_inner( + &self, + state: &dyn Session, + projection: Option<&Vec>, + filters: &[Expr], + limit: Option, + ) -> Result> { + let options = ScanArgs::default() + .with_projection(projection.map(|p| p.as_slice())) + .with_filters(Some(filters)) + .with_limit(limit); + Ok(self.scan_with_args(state, options).await?.into_inner()) + } +} diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rs index 4cf96cb364be8..79c6e44933110 100644 --- a/datafusion/catalog/src/memory/table.rs +++ b/datafusion/catalog/src/memory/table.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; use std::fmt::Debug; +use std::future::ready; use std::sync::Arc; use crate::TableProvider; @@ -51,6 +52,7 @@ use datafusion_physical_plan::{ use datafusion_session::Session; use async_trait::async_trait; +use futures::future::BoxFuture; use log::debug; use parking_lot::Mutex; use tokio::sync::RwLock; @@ -184,7 +186,95 @@ impl TableProvider for MemTable { TableType::Base } - async fn scan( + fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + projection: Option<&'life2 Vec>, + filters: &'life3 [Expr], + limit: Option, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + 'life3: 'async_trait, + Self: 'async_trait, + { + self.scan_boxed(state, projection, filters, limit) + } + + /// Returns an ExecutionPlan that inserts the execution results of a given [`ExecutionPlan`] into this [`MemTable`]. + /// + /// The [`ExecutionPlan`] must have the same schema as this [`MemTable`]. + /// + /// # Arguments + /// + /// * `state` - The [`SessionState`] containing the context for executing the plan. + /// * `input` - The [`ExecutionPlan`] to execute and insert. + /// + /// # Returns + /// + /// * A plan that returns the number of rows written. + /// + /// [`SessionState`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html + fn insert_into<'life0, 'life1, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + input: Arc, + insert_op: InsertOp, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.insert_into_boxed(state, input, insert_op) + } + + fn get_column_default(&self, column: &str) -> Option<&Expr> { + self.column_defaults.get(column) + } + + fn delete_from<'life0, 'life1, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + filters: Vec, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.delete_from_boxed(state, filters) + } + + fn update<'life0, 'life1, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + assignments: Vec<(String, Expr)>, + filters: Vec, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.update_boxed(state, assignments, filters) + } +} + +impl MemTable { + fn scan_boxed<'a>( + &'a self, + state: &'a dyn Session, + projection: Option<&'a Vec>, + filters: &'a [Expr], + limit: Option, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.scan_inner(state, projection, filters, limit)) + } + + async fn scan_inner( &self, state: &dyn Session, projection: Option<&Vec>, @@ -225,21 +315,16 @@ impl TableProvider for MemTable { Ok(DataSourceExec::from_data_source(source)) } - /// Returns an ExecutionPlan that inserts the execution results of a given [`ExecutionPlan`] into this [`MemTable`]. - /// - /// The [`ExecutionPlan`] must have the same schema as this [`MemTable`]. - /// - /// # Arguments - /// - /// * `state` - The [`SessionState`] containing the context for executing the plan. - /// * `input` - The [`ExecutionPlan`] to execute and insert. - /// - /// # Returns - /// - /// * A plan that returns the number of rows written. - /// - /// [`SessionState`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html - async fn insert_into( + fn insert_into_boxed<'a>( + &'a self, + state: &'a dyn Session, + input: Arc, + insert_op: InsertOp, + ) -> BoxFuture<'a, Result>> { + Box::pin(ready(self.insert_into_inner(state, input, insert_op))) + } + + fn insert_into_inner( &self, _state: &dyn Session, input: Arc, @@ -260,11 +345,15 @@ impl TableProvider for MemTable { Ok(Arc::new(DataSinkExec::new(input, Arc::new(sink), None))) } - fn get_column_default(&self, column: &str) -> Option<&Expr> { - self.column_defaults.get(column) + fn delete_from_boxed<'a>( + &'a self, + state: &'a dyn Session, + filters: Vec, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.delete_from_inner(state, filters)) } - async fn delete_from( + async fn delete_from_inner( &self, state: &dyn Session, filters: Vec, @@ -328,7 +417,16 @@ impl TableProvider for MemTable { Ok(Arc::new(DmlResultExec::new(total_deleted))) } - async fn update( + fn update_boxed<'a>( + &'a self, + state: &'a dyn Session, + assignments: Vec<(String, Expr)>, + filters: Vec, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.update_inner(state, assignments, filters)) + } + + async fn update_inner( &self, state: &dyn Session, assignments: Vec<(String, Expr)>, diff --git a/datafusion/catalog/src/stream.rs b/datafusion/catalog/src/stream.rs index c8060456dd2a7..f01874612e4d5 100644 --- a/datafusion/catalog/src/stream.rs +++ b/datafusion/catalog/src/stream.rs @@ -19,6 +19,7 @@ use std::fmt::Formatter; use std::fs::{File, OpenOptions}; +use std::future::ready; use std::io::BufReader; use std::path::PathBuf; use std::str::FromStr; @@ -40,6 +41,7 @@ use datafusion_physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan}; use async_trait::async_trait; use futures::StreamExt; +use futures::future::BoxFuture; /// A [`TableProviderFactory`] for [`StreamTable`] #[derive(Debug, Default)] @@ -47,7 +49,31 @@ pub struct StreamTableFactory {} #[async_trait] impl TableProviderFactory for StreamTableFactory { - async fn create( + fn create<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + cmd: &'life2 CreateExternalTable, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_boxed(state, cmd) + } +} + +impl StreamTableFactory { + fn create_boxed<'a>( + &'a self, + state: &'a dyn Session, + cmd: &'a CreateExternalTable, + ) -> BoxFuture<'a, Result>> { + Box::pin(ready(self.create_inner(state, cmd))) + } + + fn create_inner( &self, state: &dyn Session, cmd: &CreateExternalTable, @@ -322,7 +348,50 @@ impl TableProvider for StreamTable { TableType::Base } - async fn scan( + fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + projection: Option<&'life2 Vec>, + filters: &'life3 [Expr], + limit: Option, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + 'life3: 'async_trait, + Self: 'async_trait, + { + self.scan_boxed(state, projection, filters, limit) + } + + fn insert_into<'life0, 'life1, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + input: Arc, + insert_op: InsertOp, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.insert_into_boxed(state, input, insert_op) + } +} + +impl StreamTable { + fn scan_boxed<'a>( + &'a self, + state: &'a dyn Session, + projection: Option<&'a Vec>, + filters: &'a [Expr], + limit: Option, + ) -> BoxFuture<'a, Result>> { + Box::pin(ready(self.scan_inner(state, projection, filters, limit))) + } + + fn scan_inner( &self, state: &dyn Session, projection: Option<&Vec>, @@ -351,15 +420,23 @@ impl TableProvider for StreamTable { )?)) } - async fn insert_into( + fn insert_into_boxed<'a>( + &'a self, + state: &'a dyn Session, + input: Arc, + insert_op: InsertOp, + ) -> BoxFuture<'a, Result>> { + Box::pin(ready(self.insert_into_inner(state, input, insert_op))) + } + + fn insert_into_inner( &self, - _state: &dyn Session, + state: &dyn Session, input: Arc, _insert_op: InsertOp, ) -> Result> { let schema = self.0.source.schema(); - let orders = - create_lex_ordering(schema, &self.0.order, _state.execution_props())?; + let orders = create_lex_ordering(schema, &self.0.order, state.execution_props())?; // It is sufficient to pass only one of the equivalent orderings: let ordering = orders.into_iter().next().map(Into::into); @@ -412,7 +489,30 @@ impl DataSink for StreamWrite { self.0.source.schema() } - async fn write_all( + fn write_all<'life0, 'life1, 'async_trait>( + &'life0 self, + data: SendableRecordBatchStream, + context: &'life1 Arc, + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.write_all_boxed(data, context) + } +} + +impl StreamWrite { + fn write_all_boxed<'a>( + &'a self, + data: SendableRecordBatchStream, + context: &'a Arc, + ) -> BoxFuture<'a, Result> { + Box::pin(self.write_all_inner(data, context)) + } + + async fn write_all_inner( &self, mut data: SendableRecordBatchStream, _context: &Arc, diff --git a/datafusion/catalog/src/streaming.rs b/datafusion/catalog/src/streaming.rs index 50f05355aa75e..bb70270ce9157 100644 --- a/datafusion/catalog/src/streaming.rs +++ b/datafusion/catalog/src/streaming.rs @@ -17,6 +17,7 @@ //! A simplified [`TableProvider`] for streaming partitioned datasets +use std::future::ready; use std::sync::Arc; use arrow::datatypes::SchemaRef; @@ -31,6 +32,7 @@ use datafusion_physical_expr::{ }; use datafusion_physical_plan::ExecutionPlan; use datafusion_physical_plan::streaming::{PartitionStream, StreamingTableExec}; +use futures::future::BoxFuture; use log::debug; use crate::{Session, TableProvider}; @@ -121,7 +123,36 @@ impl TableProvider for StreamingTable { TableType::View } - async fn scan( + fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + projection: Option<&'life2 Vec>, + filters: &'life3 [Expr], + limit: Option, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + 'life3: 'async_trait, + Self: 'async_trait, + { + self.scan_boxed(state, projection, filters, limit) + } +} + +impl StreamingTable { + fn scan_boxed<'a>( + &'a self, + state: &'a dyn Session, + projection: Option<&'a Vec>, + filters: &'a [Expr], + limit: Option, + ) -> BoxFuture<'a, Result>> { + Box::pin(ready(self.scan_inner(state, projection, filters, limit))) + } + + fn scan_inner( &self, state: &dyn Session, projection: Option<&Vec>, diff --git a/datafusion/catalog/src/view.rs b/datafusion/catalog/src/view.rs index 723634c34c0f7..ac0f74c181956 100644 --- a/datafusion/catalog/src/view.rs +++ b/datafusion/catalog/src/view.rs @@ -30,6 +30,7 @@ use datafusion_expr::TableType; use datafusion_expr::{Expr, LogicalPlan}; use datafusion_expr::{LogicalPlanBuilder, TableProviderFilterPushDown}; use datafusion_physical_plan::ExecutionPlan; +use futures::future::BoxFuture; /// An implementation of `TableProvider` that uses another logical plan. #[derive(Debug)] @@ -95,7 +96,36 @@ impl TableProvider for ViewTable { Ok(vec![TableProviderFilterPushDown::Exact; filters.len()]) } - async fn scan( + fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + projection: Option<&'life2 Vec>, + filters: &'life3 [Expr], + limit: Option, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + 'life3: 'async_trait, + Self: 'async_trait, + { + self.scan_boxed(state, projection, filters, limit) + } +} + +impl ViewTable { + fn scan_boxed<'a>( + &'a self, + state: &'a dyn Session, + projection: Option<&'a Vec>, + filters: &'a [Expr], + limit: Option, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.scan_inner(state, projection, filters, limit)) + } + + async fn scan_inner( &self, state: &dyn Session, projection: Option<&Vec>, From 345eb4c22dd6d74d416bd42c12bb4dde0f5fbd74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 13 Aug 2026 17:27:28 +0000 Subject: [PATCH 03/50] perf(core): cut `datafusion` core compile time ~10x (#24329) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? Adresses: https://github.com/apache/datafusion/issues/13814 Third and largest instance of the problem from #24325 (`datafusion-catalog`), #24326 (`datafusion-session`) and #24330 (`datafusion-catalog-listing`). Independent of all of them — different crates, so they can merge in any order. ## Rationale for this change `datafusion` core is the last unit of a cold `cargo build -p datafusion` and compiles alone, so its cost lands directly on the build's wall clock. **76% of its compile time was the trait solver**: `-Zself-profile` reported 75.0s of `evaluate_obligation` out of 98.6s total, essentially all of it proving `Send`/`Sync`. `#[async_trait]` gives each `async fn` a `where 'life0: 'async_trait, ..` clause. rustc only serves auto-trait obligations from its **global** evaluation cache when the `ParamEnv` is empty, so the `Send`/`Sync` proof for everything the returned future captures is redone per method. In this crate the captured sets include `SessionState`, `&LogicalPlan` and `ListingTableConfig`, each of which reaches a large fraction of the logical-plan type graph. Grouping the goals by the `Self` type in their `ParamEnv` shows how concentrated this was — 14 impls, top 10 = 78% of the total: | impl | trait solving | | impl | trait solving | |---|---|---|---|---| | `ParquetReadOptions` | 7.73s | | `DynamicListTableFactory` | 5.18s | | `JsonReadOptions` | 7.54s | | `ListingTableFactory` | 4.56s | | `DefaultPhysicalPlanner` | 7.07s | | `TestTableFactory` | 4.47s | | `CsvReadOptions` | 6.59s | | `ListingTableConfig` | 4.30s | | `DataFrameTableProvider` | 5.68s | | `DefaultQueryPlanner` | 4.28s | | *(trait default bodies)* | 5.37s | | `DefaultTableFactory` | 4.15s | | | | | `SessionState` | 4.11s | | | | | `ArrowReadOptions` | 3.72s | For contrast, in the same compile 31,818 goals with an **empty** `ParamEnv` cost 0.19s in total — 6µs each, against ~1.3ms for the same kind of goal under `async_trait`'s bounds. ## What changes are included in this PR? **First commit.** Each of those methods becomes the hand-written desugaring of `async fn`, which only forwards; the coroutine is built in a shim with no where-clauses, so its auto-trait obligations are proved in an empty `ParamEnv` and land in the global cache. Method bodies are moved verbatim into inherent fns. The `ReadOptions` family (25.6s across four impls, plus the 5.37s default body) collapses to a **single** proof: all five impls already delegated to the `_get_resolved_schema` default body, which now hands the coroutine to a free `infer_schema_boxed`. Because that helper is a plain function with no generics and no where-clauses, its proof is cached once and shared by every impl. **Second commit**, from re-profiling after the first. 11.15s of trait solving remained, in exactly two places: 1. `ListingTableConfigExt::infer` was still an `async fn` capturing `self: ListingTableConfig` (4.37s). It now uses the same shim as `infer_options` beside it. 2. `ReadOptions::_get_resolved_schema` still carried `Self: Sync`, which `#[async_trait]` needed while its body was a coroutine capturing `&self`. After the first commit it is neither, so the bound is dead weight — and it forced every caller to prove its own type `Sync` structurally, through arrow's `DataType`/`Schema`, in a non-empty `ParamEnv` (2.1–2.4s each for Csv/Json/Parquet; `ArrowReadOptions` was already cheap, having fewer fields). Two things worth noting for review: - `DefaultPhysicalPlanner::create_initial_plan` already used exactly this shape (`-> BoxFuture<'a, _>` plus `Box::pin(async move ..)`) — there for recursion rather than for compile time. The idiom is not new to this codebase. - The second commit **relaxes a bound on a public trait method**. Nothing in tree overrides `_get_resolved_schema` (all five impls only implement `get_resolved_schema`) and the underscore prefix marks it as internal, but an external override written with `#[async_trait]` would generate `Self: Sync` and no longer match. Happy to drop that commit if you would rather not touch it. One body became eager: `TestTableFactory::create_inner` has no `.await`, so it is a plain fn wrapped in `ready(..)`. It builds a `TestTableProvider` and has no side effects. Everything that awaits stays lazy — `Box::pin(self.m_inner(..))` polls nothing. ## Are these changes tested? - `cargo test -p datafusion --lib` — 442 passed - `cargo check -p datafusion --all-targets` — clean (covers core's integration tests and benches, heavy users of these APIs) - `cargo clippy -p datafusion --lib` — clean - `cargo doc` with `-D warnings` — clean - `cargo fmt --check` — clean The compiler checks each rewritten signature against its trait declaration, and every body is moved verbatim. `cargo rustc -p datafusion --lib` with `-Ztime-passes`, alternated with the base so machine drift cancels out: | | total | `evaluate_obligation` | |---|---|---| | base | 88.9s | 75.0s | | after first commit | 18.6s | 11.15s | | after second commit | **8.2s** | **234ms** | 234ms over 34,724 goals is 6.7µs each — the same rate as goals that carry an empty `ParamEnv`, i.e. the repeated proving is gone rather than merely reduced. What remains in this crate is LLVM: 7.6s emitting objects and 4.2s in LLVM passes. An earlier interleaved wall-clock A/B of the first commit alone measured 74.4s/69.9s base against 16.6s/15.5s fixed. ## Are there any user-facing changes? No, other than the relaxed `Self: Sync` bound described above. No public signature changes — after macro expansion these methods have the same signatures as before. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 --- datafusion/core/src/dataframe/mod.rs | 32 +++++- .../core/src/datasource/dynamic_file.rs | 24 ++++- .../src/datasource/file_format/options.rs | 97 +++++++++++++------ .../core/src/datasource/listing/table.rs | 54 ++++++++--- .../src/datasource/listing_table_factory.rs | 27 +++++- datafusion/core/src/datasource/provider.rs | 27 +++++- .../core/src/execution/session_state.rs | 58 +++++++++-- datafusion/core/src/physical_planner.rs | 59 +++++++---- datafusion/core/src/test_util/mod.rs | 30 +++++- 9 files changed, 335 insertions(+), 73 deletions(-) diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs index 325ae91d27bbf..8be23f927edba 100644 --- a/datafusion/core/src/dataframe/mod.rs +++ b/datafusion/core/src/dataframe/mod.rs @@ -70,6 +70,7 @@ use datafusion_functions_aggregate::expr_fn::{ use async_trait::async_trait; use datafusion_catalog::Session; use datafusion_expr::extension_types::DFArrayFormatterFactory; +use futures::future::BoxFuture; /// Contains options that control how data is /// written out from a DataFrame @@ -2716,7 +2717,36 @@ impl TableProvider for DataFrameTableProvider { self.table_type } - async fn scan( + fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + projection: Option<&'life2 Vec>, + filters: &'life3 [Expr], + limit: Option, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + 'life3: 'async_trait, + Self: 'async_trait, + { + self.scan_boxed(state, projection, filters, limit) + } +} + +impl DataFrameTableProvider { + fn scan_boxed<'a>( + &'a self, + state: &'a dyn Session, + projection: Option<&'a Vec>, + filters: &'a [Expr], + limit: Option, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.scan_inner(state, projection, filters, limit)) + } + + async fn scan_inner( &self, state: &dyn Session, projection: Option<&Vec>, diff --git a/datafusion/core/src/datasource/dynamic_file.rs b/datafusion/core/src/datasource/dynamic_file.rs index 50ee96da3dff0..f670c9cc444aa 100644 --- a/datafusion/core/src/datasource/dynamic_file.rs +++ b/datafusion/core/src/datasource/dynamic_file.rs @@ -31,6 +31,7 @@ use datafusion_common::plan_datafusion_err; use datafusion_session::SessionStore; use async_trait::async_trait; +use futures::future::BoxFuture; /// [DynamicListTableFactory] is a factory that can create a [ListingTable] from the given url. #[derive(Default, Debug)] @@ -53,7 +54,28 @@ impl DynamicListTableFactory { #[async_trait] impl UrlTableFactory for DynamicListTableFactory { - async fn try_new(&self, url: &str) -> Result>> { + fn try_new<'life0, 'life1, 'async_trait>( + &'life0 self, + url: &'life1 str, + ) -> BoxFuture<'async_trait, Result>>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.try_new_boxed(url) + } +} + +impl DynamicListTableFactory { + fn try_new_boxed<'a>( + &'a self, + url: &'a str, + ) -> BoxFuture<'a, Result>>> { + Box::pin(self.try_new_inner(url)) + } + + async fn try_new_inner(&self, url: &str) -> Result>> { let Ok(table_url) = ListingTableUrl::parse(url) else { return Ok(None); }; diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs index 8ef780ea1e972..4babf4a421f4e 100644 --- a/datafusion/core/src/datasource/file_format/options.rs +++ b/datafusion/core/src/datasource/file_format/options.rs @@ -17,6 +17,7 @@ //! User facing options for the file formats readers +use std::future::ready; use std::sync::Arc; #[cfg(feature = "avro")] @@ -44,6 +45,7 @@ use datafusion_common::{ use async_trait::async_trait; use datafusion_datasource_json::file_format::JsonFormat; use datafusion_expr::SortExpr; +use futures::future::BoxFuture; /// Options that control the reading of CSV files. /// @@ -602,26 +604,37 @@ pub trait ReadOptions<'a> { } /// helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function. - async fn _get_resolved_schema( + fn _get_resolved_schema<'life0, 'async_trait>( &'a self, - config: &SessionConfig, + config: &'life0 SessionConfig, state: SessionState, table_path: ListingTableUrl, schema: Option<&'a Schema>, - ) -> Result + ) -> BoxFuture<'async_trait, Result> where 'a: 'async_trait, + 'life0: 'async_trait, + Self: 'async_trait, { if let Some(s) = schema { - return Ok(Arc::new(s.to_owned())); + return Box::pin(ready(Ok(Arc::new(s.to_owned())))); } - self.to_listing_options(config, state.default_table_options()) - .infer_schema(&state, &table_path) - .await + let listing_options = + self.to_listing_options(config, state.default_table_options()); + infer_schema_boxed(listing_options, state, table_path) } } +/// Infers a schema from `table_path`, boxed for [`ReadOptions::_get_resolved_schema`]. +fn infer_schema_boxed( + listing_options: ListingOptions, + state: SessionState, + table_path: ListingTableUrl, +) -> BoxFuture<'static, Result> { + Box::pin(async move { listing_options.infer_schema(&state, &table_path).await }) +} + #[async_trait] impl ReadOptions<'_> for CsvReadOptions<'_> { fn to_listing_options( @@ -649,14 +662,18 @@ impl ReadOptions<'_> for CsvReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { @@ -696,14 +713,18 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { @@ -730,14 +751,18 @@ impl ReadOptions<'_> for JsonReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { @@ -760,14 +785,18 @@ impl ReadOptions<'_> for AvroReadOptions<'_> { .with_table_partition_cols(self.table_partition_cols.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { @@ -789,14 +818,18 @@ impl ReadOptions<'_> for ArrowReadOptions<'_> { .with_table_partition_cols(self.table_partition_cols.clone()) } - async fn get_resolved_schema( - &self, - config: &SessionConfig, + fn get_resolved_schema<'life0, 'life1, 'async_trait>( + &'life0 self, + config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl, - ) -> Result { + ) -> BoxFuture<'async_trait, Result> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { self._get_resolved_schema(config, state, table_path, self.schema) - .await } fn schema_source(&self) -> SchemaSource { diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index 56a5d5779596c..7b435822f275f 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -21,6 +21,7 @@ use datafusion_catalog_listing::{ListingOptions, ListingTableConfig}; use datafusion_common::{config_datafusion_err, internal_datafusion_err}; use datafusion_session::Session; use futures::StreamExt; +use futures::future::BoxFuture; use std::collections::HashMap; /// Extension trait for [`ListingTableConfig`] that supports inferring schemas @@ -47,17 +48,50 @@ pub trait ListingTableConfigExt { #[async_trait] impl ListingTableConfigExt for ListingTableConfig { - async fn infer_options( + fn infer_options<'life0, 'async_trait>( self, - state: &dyn Session, - ) -> datafusion_common::Result { - let store = if let Some(url) = self.table_paths.first() { + state: &'life0 dyn Session, + ) -> BoxFuture<'async_trait, datafusion_common::Result> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + infer_options_boxed(self, state) + } + + fn infer<'life0, 'async_trait>( + self, + state: &'life0 dyn Session, + ) -> BoxFuture<'async_trait, datafusion_common::Result> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + infer_boxed(self, state) + } +} + +/// Body of [`ListingTableConfigExt::infer`]. +fn infer_boxed<'a>( + config: ListingTableConfig, + state: &'a dyn Session, +) -> BoxFuture<'a, datafusion_common::Result> { + Box::pin(async move { config.infer_options(state).await?.infer_schema(state).await }) +} + +/// Body of [`ListingTableConfigExt::infer_options`]. +fn infer_options_boxed<'a>( + config: ListingTableConfig, + state: &'a dyn Session, +) -> BoxFuture<'a, datafusion_common::Result> { + Box::pin(async move { + let store = if let Some(url) = config.table_paths.first() { state.runtime_env().object_store(url)? } else { - return Ok(self); + return Ok(config); }; - let file = self + let file = config .table_paths .first() .unwrap() @@ -95,12 +129,8 @@ impl ListingTableConfigExt for ListingTableConfig { let listing_options = ListingOptions::new(file_format).with_file_extension(listing_file_extension); - Ok(self.with_listing_options(listing_options)) - } - - async fn infer(self, state: &dyn Session) -> datafusion_common::Result { - self.infer_options(state).await?.infer_schema(state).await - } + Ok(config.with_listing_options(listing_options)) + }) } #[cfg(test)] diff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rs index 68f6743189447..116b8985b4115 100644 --- a/datafusion/core/src/datasource/listing_table_factory.rs +++ b/datafusion/core/src/datasource/listing_table_factory.rs @@ -36,6 +36,7 @@ use datafusion_expr::CreateExternalTable; use async_trait::async_trait; use datafusion_catalog::Session; +use futures::future::BoxFuture; /// A `TableProviderFactory` capable of creating new `ListingTable`s #[derive(Debug, Default)] @@ -50,7 +51,31 @@ impl ListingTableFactory { #[async_trait] impl TableProviderFactory for ListingTableFactory { - async fn create( + fn create<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + cmd: &'life2 CreateExternalTable, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_boxed(state, cmd) + } +} + +impl ListingTableFactory { + fn create_boxed<'a>( + &'a self, + state: &'a dyn Session, + cmd: &'a CreateExternalTable, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.create_inner(state, cmd)) + } + + async fn create_inner( &self, state: &dyn Session, cmd: &CreateExternalTable, diff --git a/datafusion/core/src/datasource/provider.rs b/datafusion/core/src/datasource/provider.rs index 9d4b67632a019..84bb338835c86 100644 --- a/datafusion/core/src/datasource/provider.rs +++ b/datafusion/core/src/datasource/provider.rs @@ -23,6 +23,7 @@ use async_trait::async_trait; use datafusion_catalog::Session; use datafusion_expr::CreateExternalTable; pub use datafusion_expr::{TableProviderFilterPushDown, TableType}; +use futures::future::BoxFuture; use crate::catalog::{TableProvider, TableProviderFactory}; use crate::datasource::listing_table_factory::ListingTableFactory; @@ -48,7 +49,31 @@ impl DefaultTableFactory { #[async_trait] impl TableProviderFactory for DefaultTableFactory { - async fn create( + fn create<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + cmd: &'life2 CreateExternalTable, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_boxed(state, cmd) + } +} + +impl DefaultTableFactory { + fn create_boxed<'a>( + &'a self, + state: &'a dyn Session, + cmd: &'a CreateExternalTable, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.create_inner(state, cmd)) + } + + async fn create_inner( &self, state: &dyn Session, cmd: &CreateExternalTable, diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index bfd38faacf816..99b16857bb1bc 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -85,6 +85,7 @@ use datafusion_sql::{ use async_trait::async_trait; use chrono::{DateTime, Utc}; +use futures::future::BoxFuture; use itertools::Itertools; use log::{debug, info}; use object_store::ObjectStore; @@ -291,11 +292,16 @@ impl Session for SessionState { SessionState::statistics_registry(self) } - async fn create_physical_plan( - &self, - logical_plan: &LogicalPlan, - ) -> datafusion_common::Result> { - self.create_physical_plan(logical_plan).await + fn create_physical_plan<'life0, 'life1, 'async_trait>( + &'life0 self, + logical_plan: &'life1 LogicalPlan, + ) -> BoxFuture<'async_trait, datafusion_common::Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.create_physical_plan_boxed(logical_plan) } fn create_physical_expr( @@ -351,6 +357,22 @@ impl Session for SessionState { } } +impl SessionState { + fn create_physical_plan_boxed<'a>( + &'a self, + logical_plan: &'a LogicalPlan, + ) -> BoxFuture<'a, datafusion_common::Result>> { + Box::pin(self.create_physical_plan_inner(logical_plan)) + } + + async fn create_physical_plan_inner( + &self, + logical_plan: &LogicalPlan, + ) -> datafusion_common::Result> { + self.create_physical_plan(logical_plan).await + } +} + impl SessionState { pub(crate) fn resolve_table_ref( &self, @@ -2336,7 +2358,31 @@ struct DefaultQueryPlanner {} #[async_trait] impl QueryPlanner for DefaultQueryPlanner { /// Given a `LogicalPlan`, create an [`ExecutionPlan`] suitable for execution - async fn create_physical_plan( + fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + logical_plan: &'life1 LogicalPlan, + session_state: &'life2 dyn Session, + ) -> BoxFuture<'async_trait, datafusion_common::Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_physical_plan_boxed(logical_plan, session_state) + } +} + +impl DefaultQueryPlanner { + fn create_physical_plan_boxed<'a>( + &'a self, + logical_plan: &'a LogicalPlan, + session_state: &'a dyn Session, + ) -> BoxFuture<'a, datafusion_common::Result>> { + Box::pin(self.create_physical_plan_inner(logical_plan, session_state)) + } + + async fn create_physical_plan_inner( &self, logical_plan: &LogicalPlan, session_state: &dyn Session, diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 3c1e7b50780a5..6e68ed9ce69b3 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -112,6 +112,7 @@ use datafusion_session::{PhysicalOptimizerContext, PhysicalOptimizerRule, Sessio use async_trait::async_trait; use datafusion_physical_plan::async_func::{AsyncFuncExec, AsyncMapper}; +use futures::future::BoxFuture; use futures::{StreamExt, TryStreamExt}; use indexmap::IndexSet; use itertools::{Itertools, multiunzip}; @@ -153,22 +154,18 @@ pub struct DefaultPhysicalPlanner { #[async_trait] impl PhysicalPlanner for DefaultPhysicalPlanner { /// Create a physical plan from a logical plan - async fn create_physical_plan( - &self, - logical_plan: &LogicalPlan, - session_state: &dyn Session, - ) -> Result> { - if let Some(plan) = self - .handle_explain_or_analyze(logical_plan, session_state) - .await? - { - return Ok(plan); - } - let plan = self - .create_initial_plan(logical_plan, session_state) - .await?; - - self.optimize_physical_plan(plan, session_state, |_, _| {}) + fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + logical_plan: &'life1 LogicalPlan, + session_state: &'life2 dyn Session, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_physical_plan_boxed(logical_plan, session_state) } /// Create a physical expression from a logical expression @@ -193,6 +190,34 @@ impl PhysicalPlanner for DefaultPhysicalPlanner { } } +impl DefaultPhysicalPlanner { + fn create_physical_plan_boxed<'a>( + &'a self, + logical_plan: &'a LogicalPlan, + session_state: &'a dyn Session, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.create_physical_plan_inner(logical_plan, session_state)) + } + + async fn create_physical_plan_inner( + &self, + logical_plan: &LogicalPlan, + session_state: &dyn Session, + ) -> Result> { + if let Some(plan) = self + .handle_explain_or_analyze(logical_plan, session_state) + .await? + { + return Ok(plan); + } + let plan = self + .create_initial_plan(logical_plan, session_state) + .await?; + + self.optimize_physical_plan(plan, session_state, |_, _| {}) + } +} + #[derive(Debug)] struct ExecutionPlanChild { /// Index needed to order children of parent to ensure consistency with original @@ -329,7 +354,7 @@ impl DefaultPhysicalPlanner { &'a self, logical_plan: &'a LogicalPlan, session_state: &'a dyn Session, - ) -> futures::future::BoxFuture<'a, Result>> { + ) -> BoxFuture<'a, Result>> { Box::pin(async move { // When `enable_physical_uncorrelated_scalar_subquery` is disabled, the // `ScalarSubqueryToJoin` optimizer rule rewrites all uncorrelated diff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rs index d70c0d186d007..036988c8fcc76 100644 --- a/datafusion/core/src/test_util/mod.rs +++ b/datafusion/core/src/test_util/mod.rs @@ -26,6 +26,7 @@ use futures::Stream; use std::collections::HashMap; use std::fmt::Formatter; use std::fs::File; +use std::future::ready; use std::io::Write; use std::path::Path; use std::sync::Arc; @@ -53,6 +54,7 @@ use datafusion_expr::{ use std::pin::Pin; use async_trait::async_trait; +use futures::future::BoxFuture; use tempfile::TempDir; // backwards compatibility @@ -182,9 +184,33 @@ pub struct TestTableFactory {} #[async_trait] impl TableProviderFactory for TestTableFactory { - async fn create( + fn create<'life0, 'life1, 'life2, 'async_trait>( + &'life0 self, + session: &'life1 dyn Session, + cmd: &'life2 CreateExternalTable, + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + Self: 'async_trait, + { + self.create_boxed(session, cmd) + } +} + +impl TestTableFactory { + fn create_boxed<'a>( + &'a self, + session: &'a dyn Session, + cmd: &'a CreateExternalTable, + ) -> BoxFuture<'a, Result>> { + Box::pin(ready(self.create_inner(session, cmd))) + } + + fn create_inner( &self, - _: &dyn Session, + _session: &dyn Session, cmd: &CreateExternalTable, ) -> Result> { let Some(location) = cmd.locations.first() else { From 91d6a23d580f4e08f5fda93334dc653fa5df8538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 13 Aug 2026 17:27:43 +0000 Subject: [PATCH 04/50] perf(catalog-listing): cut `datafusion-catalog-listing` compile time ~4.4x (#24330) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? Adresses: https://github.com/apache/datafusion/issues/13814 Fourth and last crate in the family from #24325 (`datafusion-catalog`), #24326 (`datafusion-session`) and #24329 (`datafusion` core). Independent of all three — different crates, so they can merge in any order. ## Rationale for this change `datafusion-catalog-listing` is 3,013 lines of source but spends **20.5s** in the frontend during a cold `cargo build -p datafusion` (`cargo build --timings`), and it sits on the critical path between `datafusion-catalog` and `datafusion` core. `-Zself-profile` puts 58% of the crate's compile time in `evaluate_obligation`, and grouping those goals by the `Self` type in their `ParamEnv` shows all of it in one impl: | `Self` in `ParamEnv` | time | goals | |---|---|---| | `ListingTable` | 3.16s | 6,967 | | *(empty `ParamEnv`)* | 0.04s | 8,443 | Note the second row — the same kind of goals cost ~5µs each with an empty `ParamEnv` against ~450µs here. The cause is the one from the earlier PRs: `#[async_trait]` gives each `async fn` a `where 'life0: 'async_trait, ..` clause, which makes the method's `ParamEnv` non-empty, and rustc only serves auto-trait obligations from its **global** evaluation cache when the `ParamEnv` is empty. So the `Send`/`Sync` proof for everything the future captures is redone per method. All three async methods in this impl reach `Expr`: - `scan` takes `&[Expr]` - `scan_with_args` takes `ScanArgs<'a>`, which holds `&[Expr]` - `insert_into` keeps `self.options` (`Vec>`) live across an await so each one pays for a walk of the whole `Expr`/`LogicalPlan` graph. ## What changes are included in this PR? Each method is now the hand-written desugaring of `async fn` and only forwards; the coroutine is built in a shim with no where-clauses, so its proofs land in the global cache. Bodies are moved verbatim into inherent fns and all three stay `async`, so nothing is evaluated any earlier than before — `Box::pin(self.m_inner(..))` polls nothing. Following the review on #24326, I measured each method's marginal contribution first, by reverting one at a time (together with its helpers) from the all-converted state: | state | crate build | `evaluate_obligation` | |---|---|---| | all three converted | 0.931s | 34.9ms | | revert `insert_into` | 1.866s | 1.01s | | revert `scan` | 1.943s | 1.05s | | revert `scan_with_args` | 2.118s | 1.17s | | none converted (base) | 4.201s | 3.25s | Unlike #24326 — where two of the seven bodies I first converted turned out to gain nothing — all three pull their weight here. Converting all of them leaves no coroutine in the impl at all, so the graph is never walked in a non-empty `ParamEnv`, which is why the total drops by two orders of magnitude rather than by a third. ## Are these changes tested? - `cargo test -p datafusion-catalog-listing` — 18 + 7 passed - `cargo test -p datafusion --lib` — 442 passed - `cargo check -p datafusion --all-targets` — clean (`ListingTable` is used heavily by core's integration tests and benches) - `cargo clippy -p datafusion-catalog-listing --all-targets` — clean - `cargo fmt --check` — clean The compiler checks each rewritten signature against the trait declaration, and every body is moved verbatim. Interleaved A/B of `cargo rustc -p datafusion-catalog-listing --lib`, alternating 3 times so machine drift cancels out: ``` base: 4.257s 4.189s 4.134s fix: 0.984s 0.935s 0.980s ``` `evaluate_obligation` drops from 3.25s to 34.7ms. ## Are there any user-facing changes? No. No public signature changes — after macro expansion these methods have the same signatures as before. ### Follow-up With this, the four crates that made up the serial tail of a cold build are done. The general fix remains available and would cover downstream implementors too: drop `#[async_trait]` from these traits in favour of an explicit `BoxFuture` return with a single lifetime and no where-clauses, so that *every* impl is cheap without hand-desugaring. That is a breaking change to public traits, so it is out of scope here. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5 --- datafusion/catalog-listing/src/table.rs | 155 ++++++++++++++++++------ 1 file changed, 115 insertions(+), 40 deletions(-) diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs index b3328cc06303d..46fab1f9cbff4 100644 --- a/datafusion/catalog-listing/src/table.rs +++ b/datafusion/catalog-listing/src/table.rs @@ -53,6 +53,7 @@ use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory; use datafusion_physical_expr_common::sort_expr::LexOrdering; use datafusion_physical_plan::ExecutionPlan; use datafusion_physical_plan::empty::EmptyExec; +use futures::future::BoxFuture; use futures::{Stream, StreamExt, TryStreamExt, future, stream}; use object_store::ObjectStore; use std::collections::{HashMap, HashSet}; @@ -489,21 +490,94 @@ impl TableProvider for ListingTable { TableType::Base } - async fn scan( - &self, - state: &dyn Session, - projection: Option<&Vec>, - filters: &[Expr], + fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + projection: Option<&'life2 Vec>, + filters: &'life3 [Expr], limit: Option, - ) -> datafusion_common::Result> { - let options = ScanArgs::default() - .with_projection(projection.map(|p| p.as_slice())) - .with_filters(Some(filters)) - .with_limit(limit); - Ok(self.scan_with_args(state, options).await?.into_inner()) + ) -> BoxFuture<'async_trait, datafusion_common::Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + 'life2: 'async_trait, + 'life3: 'async_trait, + Self: 'async_trait, + { + self.scan_boxed(state, projection, filters, limit) + } + + fn scan_with_args<'a, 'life0, 'life1, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + args: ScanArgs<'a>, + ) -> BoxFuture<'async_trait, datafusion_common::Result> + where + 'a: 'async_trait, + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.scan_with_args_boxed(state, args) + } + + fn supports_filters_pushdown( + &self, + filters: &[&Expr], + ) -> datafusion_common::Result> { + let partition_column_names = self + .options + .table_partition_cols + .iter() + .map(|col| col.0.as_str()) + .collect::>(); + filters + .iter() + .map(|filter| { + if can_be_evaluated_for_partition_pruning(&partition_column_names, filter) + { + // if filter can be handled by partition pruning, it is exact + return Ok(TableProviderFilterPushDown::Exact); + } + + Ok(TableProviderFilterPushDown::Inexact) + }) + .collect() + } + + fn get_table_definition(&self) -> Option<&str> { + self.definition.as_deref() + } + + fn insert_into<'life0, 'life1, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, + input: Arc, + insert_op: InsertOp, + ) -> BoxFuture<'async_trait, datafusion_common::Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + self.insert_into_boxed(state, input, insert_op) + } + + fn get_column_default(&self, column: &str) -> Option<&Expr> { + self.column_defaults.get(column) + } +} + +impl ListingTable { + fn scan_with_args_boxed<'a>( + &'a self, + state: &'a dyn Session, + args: ScanArgs<'a>, + ) -> BoxFuture<'a, datafusion_common::Result> { + Box::pin(self.scan_with_args_inner(state, args)) } - async fn scan_with_args<'a>( + async fn scan_with_args_inner<'a>( &self, state: &dyn Session, args: ScanArgs<'a>, @@ -670,35 +744,40 @@ impl TableProvider for ListingTable { Ok(ScanResult::new(plan)) } - fn supports_filters_pushdown( - &self, - filters: &[&Expr], - ) -> datafusion_common::Result> { - let partition_column_names = self - .options - .table_partition_cols - .iter() - .map(|col| col.0.as_str()) - .collect::>(); - filters - .iter() - .map(|filter| { - if can_be_evaluated_for_partition_pruning(&partition_column_names, filter) - { - // if filter can be handled by partition pruning, it is exact - return Ok(TableProviderFilterPushDown::Exact); - } + fn scan_boxed<'a>( + &'a self, + state: &'a dyn Session, + projection: Option<&'a Vec>, + filters: &'a [Expr], + limit: Option, + ) -> BoxFuture<'a, datafusion_common::Result>> { + Box::pin(self.scan_inner(state, projection, filters, limit)) + } - Ok(TableProviderFilterPushDown::Inexact) - }) - .collect() + async fn scan_inner( + &self, + state: &dyn Session, + projection: Option<&Vec>, + filters: &[Expr], + limit: Option, + ) -> datafusion_common::Result> { + let options = ScanArgs::default() + .with_projection(projection.map(|p| p.as_slice())) + .with_filters(Some(filters)) + .with_limit(limit); + Ok(self.scan_with_args(state, options).await?.into_inner()) } - fn get_table_definition(&self) -> Option<&str> { - self.definition.as_deref() + fn insert_into_boxed<'a>( + &'a self, + state: &'a dyn Session, + input: Arc, + insert_op: InsertOp, + ) -> BoxFuture<'a, datafusion_common::Result>> { + Box::pin(self.insert_into_inner(state, input, insert_op)) } - async fn insert_into( + async fn insert_into_inner( &self, state: &dyn Session, input: Arc, @@ -766,10 +845,6 @@ impl TableProvider for ListingTable { .create_writer_physical_plan(input, state, config, order_requirements) .await } - - fn get_column_default(&self, column: &str) -> Option<&Expr> { - self.column_defaults.get(column) - } } impl ListingTable { From aec8069d7b5f795eb69aeb143df0208f2e3a0b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 13 Aug 2026 17:28:00 +0000 Subject: [PATCH 05/50] perf(session): cut `datafusion-session` compile time ~1.7x (#24326) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? Adresses: https://github.com/apache/datafusion/issues/13814 ## Rationale for this change `datafusion-session` is 1,619 lines of source and spends **18.8s** in the frontend during a cold `cargo build -p datafusion` (`cargo build --timings`) — 11.6ms per line, the worst ratio in the workspace, against 0.09–0.55ms/line for crates like `datafusion-datasource` or `datafusion-physical-plan`. `-Zself-profile` puts ~72% of the crate's compile time in `evaluate_obligation`, and 98% of that in `Send`/`Sync`. ## What changes are included in this PR? `TableProvider::{delete_from, update, merge_into}` are one-line `not_impl_err!` stubs that nevertheless build a coroutine capturing `Vec` / `Expr` — proving that coroutine `Send` walks the whole `Expr`/`LogicalPlan` type graph. They are now written as the desugaring of `async fn` returning `ready(..)`, so no coroutine is created and there is nothing expensive to prove. The signatures are exactly what `#[async_trait]` generates — verified against `-Zunpretty=expanded` output of this crate — so implementors are unaffected. I originally converted seven bodies, but measuring each one's marginal contribution (reverting one at a time; `evaluate_obligation` self time, all-converted baseline 659ms) showed only the ones taking `Expr` matter: | method left as `async fn` | trait solving | marginal cost | |---|---|---| | `delete_from` | 1.29s | +631ms | | `update` | 1.30s | +641ms | | `merge_into` | 1.29s | +631ms | | `truncate` | 697ms | +38ms | | `insert_into` | 665ms | ~0 | ## Are these changes tested? Interleaved A/B of `cargo rustc -p datafusion-session --lib` ``` base: 1.582s 1.576s 1.590s fix: 0.933s 0.938s 0.932s ``` `evaluate_obligation` drops from 1.28s to 0.646s. Those are standalone-build numbers. In the feature-unified build this crate's frontend is 18.8s rather than ~1.6s (each obligation is several times dearer there), so the absolute saving on a real build should be larger — I have not measured that directly, since isolating one crate's unit in a full build is hard to do without confounding it with machine drift. ## Are there any user-facing changes? No public signature changes 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5 --- Cargo.lock | 1 + datafusion/session/Cargo.toml | 1 + datafusion/session/src/table.rs | 56 ++++++++++++++++++++++++--------- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b5d5674565ba0..4cb6117da3fd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2609,6 +2609,7 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-plan", + "futures", "parking_lot", ] diff --git a/datafusion/session/Cargo.toml b/datafusion/session/Cargo.toml index 2bbbdd20df1b8..e6c277a8b8493 100644 --- a/datafusion/session/Cargo.toml +++ b/datafusion/session/Cargo.toml @@ -37,6 +37,7 @@ datafusion-common = { workspace = true } datafusion-execution = { workspace = true } datafusion-expr = { workspace = true } datafusion-physical-plan = { workspace = true } +futures = { workspace = true } parking_lot = { workspace = true } # Note: add additional linter rules in lib.rs. diff --git a/datafusion/session/src/table.rs b/datafusion/session/src/table.rs index 69e7e731b32c7..9dbda40b9bc7a 100644 --- a/datafusion/session/src/table.rs +++ b/datafusion/session/src/table.rs @@ -18,6 +18,7 @@ use std::any::Any; use std::borrow::Cow; use std::fmt::Debug; +use std::future::ready; use std::sync::Arc; use crate::session::Session; @@ -27,6 +28,7 @@ use datafusion_common::{Constraints, Statistics, not_impl_err}; use datafusion_common::{DFSchemaRef, Result, internal_err}; use datafusion_expr::Expr; use datafusion_expr::statistics::StatisticsRequest; +use futures::future::BoxFuture; use datafusion_expr::dml::{InsertOp, MergeIntoClause}; use datafusion_expr::{ @@ -351,25 +353,41 @@ pub trait TableProvider: Any + Debug + Sync + Send { /// /// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64). /// Empty `filters` deletes all rows. - async fn delete_from( - &self, - _state: &dyn Session, + fn delete_from<'life0, 'life1, 'async_trait>( + &'life0 self, + _state: &'life1 dyn Session, _filters: Vec, - ) -> Result> { - not_impl_err!("DELETE not supported for {} table", self.table_type()) + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + Box::pin(ready(not_impl_err!( + "DELETE not supported for {} table", + self.table_type() + ))) } /// Update rows matching the filter predicates. /// /// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64). /// Empty `filters` updates all rows. - async fn update( - &self, - _state: &dyn Session, + fn update<'life0, 'life1, 'async_trait>( + &'life0 self, + _state: &'life1 dyn Session, _assignments: Vec<(String, Expr)>, _filters: Vec, - ) -> Result> { - not_impl_err!("UPDATE not supported for {} table", self.table_type()) + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + Box::pin(ready(not_impl_err!( + "UPDATE not supported for {} table", + self.table_type() + ))) } /// Remove all rows from the table. @@ -391,15 +409,23 @@ pub trait TableProvider: Any + Debug + Sync + Send { /// The `clauses` describe the WHEN MATCHED / WHEN NOT MATCHED actions. /// /// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64). - async fn merge_into( - &self, - _state: &dyn Session, + fn merge_into<'life0, 'life1, 'async_trait>( + &'life0 self, + _state: &'life1 dyn Session, _source: Arc, _merge_schema: DFSchemaRef, _on: Expr, _clauses: Vec, - ) -> Result> { - not_impl_err!("MERGE INTO not supported for {} table", self.table_type()) + ) -> BoxFuture<'async_trait, Result>> + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + Box::pin(ready(not_impl_err!( + "MERGE INTO not supported for {} table", + self.table_type() + ))) } } From 82ff110d7e76b8e92a312b6a2c4c343ac618af1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 13 Aug 2026 18:08:15 +0000 Subject: [PATCH 06/50] perf(build): don't build the doc-generator binaries by default (#24338) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? Adresses: https://github.com/apache/datafusion/issues/13814 Found while profiling compile times for #24325 / #24326 / #24329 / #24330. ## Rationale for this change `datafusion/core/src/bin/` holds three binaries that regenerate the docs under `docs/source/user-guide`: `print_config_docs`, `print_runtime_config_docs` and `print_functions_docs`. Cargo auto-discovers them and they have no `required-features`, so **every `cargo build` links all three** — each one ~174MB, since each links the whole `datafusion` rlib. Nothing in normal development uses them. They are run by `dev/update_config_docs.sh` and `dev/update_function_docs.sh`, and by the CI job that checks the committed docs are up to date. Two places where this shows up: **Cold builds.** The three binaries link *after* every other unit has finished, so they sit on the critical path with nothing to overlap with. `cargo build --timings` shows them occupying the last **3.5s** of a `cargo build -p datafusion` (~8.8s of CPU), after the last library unit completes. **The tightest inner loop** — touch a file in core, rebuild. All three are relinked every time: ``` before: 3.0s 2.4s after: 1.3s 1.1s ``` ## What changes are included in this PR? The three binaries move behind a new non-default `docs_generation` feature, and the two `dev/` scripts pass `--features docs_generation`. Using `required-features` means declaring the `[[bin]]` targets explicitly, since auto-discovered targets cannot carry it. ## Are these changes tested? - `cargo build -p datafusion` no longer produces the three binaries - `cargo build -p datafusion --features docs_generation` does - `./dev/update_config_docs.sh` still regenerates `docs/source/user-guide/configs.md` byte-identically (empty `git diff` afterwards), which is what the CI doc check compares `dev/update_function_docs.sh` uses the same invocation pattern and all three of its call sites were updated; CI exercises both scripts. ## Are there any user-facing changes? The three binaries are no longer built by a default `cargo build`. Anyone who ran them directly needs `--features docs_generation` — same as the `dev/` scripts now do. No library API changes. If you would rather these lived outside the published crate altogether, moving them to a small non-published `dev/` crate would have the same effect on build times; I went with the smaller change. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5 --- datafusion/core/Cargo.toml | 22 ++++++++++++++++++++++ dev/update_config_docs.sh | 4 ++-- dev/update_function_docs.sh | 6 +++--- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 0fe48ddf6a3e0..81e0418dd2b26 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -39,6 +39,13 @@ all-features = true workspace = true [features] +# Enables the `print_config_docs`, `print_runtime_config_docs` and +# `print_functions_docs` binaries, which regenerate the docs under +# `docs/source/user-guide`. Off by default: they are only run by +# `dev/update_config_docs.sh` and `dev/update_function_docs.sh`, and each one +# links a ~174MB binary that would otherwise be relinked on every build of this +# crate. +docs_generation = [] nested_expressions = ["datafusion-functions-nested"] # This feature is deprecated. Use the `nested_expressions` feature instead. array_expressions = ["nested_expressions"] @@ -190,6 +197,21 @@ ignored = ["datafusion-doc", "datafusion-macros", "dashmap"] [target.'cfg(not(target_os = "windows"))'.dev-dependencies] nix = { version = "0.31.3", features = ["fs"] } +[[bin]] +name = "print_config_docs" +path = "src/bin/print_config_docs.rs" +required-features = ["docs_generation"] + +[[bin]] +name = "print_runtime_config_docs" +path = "src/bin/print_runtime_config_docs.rs" +required-features = ["docs_generation"] + +[[bin]] +name = "print_functions_docs" +path = "src/bin/print_functions_docs.rs" +required-features = ["docs_generation"] + [[bench]] harness = false name = "aggregate_query_sql" diff --git a/dev/update_config_docs.sh b/dev/update_config_docs.sh index 7ab998f3dad48..df40c65210bb9 100755 --- a/dev/update_config_docs.sh +++ b/dev/update_config_docs.sh @@ -27,8 +27,8 @@ cd "${ROOT_DIR}" source "${ROOT_DIR}/ci/scripts/utils/tool_versions.sh" TARGET_FILE="docs/source/user-guide/configs.md" -PRINT_CONFIG_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --bin print_config_docs" -PRINT_RUNTIME_CONFIG_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --bin print_runtime_config_docs" +PRINT_CONFIG_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_config_docs" +PRINT_RUNTIME_CONFIG_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_runtime_config_docs" echo "Inserting header" cat <<'EOF' > "$TARGET_FILE" diff --git a/dev/update_function_docs.sh b/dev/update_function_docs.sh index 86a272ae196c8..04266b8cbc0d3 100755 --- a/dev/update_function_docs.sh +++ b/dev/update_function_docs.sh @@ -27,7 +27,7 @@ cd "${ROOT_DIR}" source "${ROOT_DIR}/ci/scripts/utils/tool_versions.sh" TARGET_FILE="docs/source/user-guide/sql/aggregate_functions.md" -PRINT_AGGREGATE_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --bin print_functions_docs -- aggregate" +PRINT_AGGREGATE_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_functions_docs -- aggregate" echo "Inserting header" cat <<'EOF' > "$TARGET_FILE" @@ -121,7 +121,7 @@ npx "prettier@${PRETTIER_VERSION}" --write "$TARGET_FILE" echo "'$TARGET_FILE' successfully updated!" TARGET_FILE="docs/source/user-guide/sql/scalar_functions.md" -PRINT_SCALAR_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --bin print_functions_docs -- scalar" +PRINT_SCALAR_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_functions_docs -- scalar" echo "Inserting header" cat <<'EOF' > "$TARGET_FILE" @@ -165,7 +165,7 @@ npx "prettier@${PRETTIER_VERSION}" --write "$TARGET_FILE" echo "'$TARGET_FILE' successfully updated!" TARGET_FILE="docs/source/user-guide/sql/window_functions.md" -PRINT_WINDOW_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --bin print_functions_docs -- window" +PRINT_WINDOW_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --features docs_generation --bin print_functions_docs -- window" echo "Inserting header" cat <<'EOF' > "$TARGET_FILE" From ff505198f8e4832e8ced7e7343f2ba82e6c0564d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 13 Aug 2026 19:10:30 +0000 Subject: [PATCH 07/50] perf(build): use line-tables-only debug info in the `dev` profile (#24339) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? Adresses: https://github.com/apache/datafusion/issues/13814 Found while profiling compile times for #24325 / #24326 / #24329 / #24330, which removed the trait-solving cost from the four crates on the critical path and left LLVM as the dominant remaining cost. ## Rationale for this change `dev` is the profile behind every `cargo build` and `cargo test`, so its debug info is generated over and over. `debug = "line-tables-only"` keeps file and line numbers — panics and `RUST_BACKTRACE` output stay just as useful — and drops the variable-level DWARF that only an interactive debugger consumes. Measured per crate, **interleaved** with the baseline so machine drift cancels out. The flag is passed to the crate under test only, so cached dependency artifacts stay valid and nothing else moves between the two measurements: | crate | `debug = 2` | `line-tables-only` | | |---|---|---|---| | `datafusion-physical-plan` | 8.89s | 7.43s | −16% | | `datafusion-functions-aggregate` | 5.86s | 4.63s | −21% | | `datafusion-physical-expr` | 4.57s | 3.59s | −21% | | `datafusion-functions` | 4.64s | 4.04s | −13% | | `datafusion-expr` | 4.54s | 3.57s | −21% | | `datafusion-functions-nested` | 4.37s | 3.06s | −30% | | `datafusion-optimizer` | 3.91s | 3.12s | −20% | | `datafusion-common` | 3.73s | 3.10s | −17% | | `datafusion-sql` | 3.57s | 2.43s | −32% | | `datafusion-datasource-parquet` | 3.27s | 2.44s | −25% | | `datafusion-datasource` | 1.90s | 1.42s | −25% | | `datafusion-physical-optimizer` | 1.22s | 0.99s | −19% | | **sum** | **50.5s** | **39.8s** | **−21%** | The saving is codegen-side, as you would expect: `datafusion-catalog`, which spends its time in the trait solver rather than in LLVM, moves only 7.4s → 7.0s. Artifacts shrink as well — `libdatafusion_physical_plan.rlib` goes from **141MB to 100MB**. ## What changes are included in this PR? One setting on `[profile.dev]`, plus an update to the profile documentation block above it, which currently advertises "full debug info" for `dev`. ## Are these changes tested? ## Are there any user-facing changes? For anyone stepping through DataFusion in a debugger, local variable inspection needs `CARGO_PROFILE_DEV_DEBUG=2 cargo build` (or a local override in `.cargo/config.toml`); the comment in `Cargo.toml` says so. Everything else — panic locations, backtraces, `#[test]` failures — is unchanged. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5 --- Cargo.toml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 440b5ae389637..65670d895e1a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -296,8 +296,11 @@ unused_qualifications = "deny" # To use a profile: `cargo [ build | run | ... ] --profile ` # # Profiles available: -# - dev: default debug build; fastest to compile, slowest to run, full debug info. -# For everyday development; default for "cargo [ build | test | run ]". +# - dev: default debug build; fastest to compile, slowest to run. Carries line +# tables only, so panics and backtraces still name a file and line but a +# debugger cannot inspect variables; build with `CARGO_PROFILE_DEV_DEBUG=2` +# when you need that. For everyday development; default for +# "cargo [ build | test | run ]". # - release: fully optimized build; slowest to compile, fastest to run, smallest # binaries. For public releases; default for "cargo [ bench | install ]". # - release-nonlto: skips LTO, so it builds much faster while staying close to @@ -312,6 +315,15 @@ unused_qualifications = "deny" # # If you want to optimize compilation, the `compile_profile` benchmark can be useful. # See `benchmarks/README.md` for more details. +# `dev` is the profile every `cargo build`/`cargo test` uses, so its debug info +# is generated over and over. `line-tables-only` keeps file and line numbers -- +# so panics and `RUST_BACKTRACE` output stay useful -- while dropping the +# variable-level DWARF that only an interactive debugger consumes. Build with +# `CARGO_PROFILE_DEV_DEBUG=2 cargo build` when you need to step through code in a +# debugger. +[profile.dev] +debug = "line-tables-only" + [profile.release] codegen-units = 1 lto = true From fce9669ce24154618abc7f9ccbe0fa8b3f45159e Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 13 Aug 2026 21:59:48 +0000 Subject: [PATCH 08/50] fix: emit unmatched NULL-key right rows in RIGHT/FULL PiecewiseMergeJoin (#24336) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #24335. ## Rationale for this change A `RIGHT`/`FULL` `PiecewiseMergeJoin` with a range predicate drops an unmatched right-side row whose join key is `NULL`. A `NULL` key never matches (`NULL < x` is UNKNOWN), so in a `RIGHT`/`FULL` join the row is unmatched and must still be emitted with NULLs on the left — but `PiecewiseMergeJoinExec` omits it, diverging from `NestedLoopJoin`. ```sql create table l(v int) as values (5); create table r(v int) as values (10), (NULL); select l.v, r.v from l right join r on l.v < r.v; -- drops (NULL, NULL) ``` Root cause: `resolve_classic_join` starts the match scan past the streamed side's `NULL`-keyed rows (they sort to the front under `nulls_first`). Those rows are never revisited, so for `Right`/`Full` they were never added to `unmatched_indices` and got dropped. ## What changes are included in this PR? - In `resolve_classic_join`, when skipping the streamed side's leading `NULL`-key rows, record them as unmatched for `Right`/`Full` joins so they are emitted (with NULLs on the buffered side). ## Are these changes tested? Yes. - Regression test in `pwmj.slt`: a `RIGHT JOIN` over the existing `null_join_*` tables now emits the `(NULL, NULL)` row. The test fails on `main` (the row is dropped) and passes with this change. - Verified more broadly with a differential fuzz against `NestedLoopJoin` (same SQL, `enable_piecewise_merge_join` on vs off): 1200 checks over random `RIGHT JOIN` inputs with `<`/`<=`/`>`/`>=` and high right-side NULL density, 0 mismatches. ## Are there any user-facing changes? `RIGHT`/`FULL` range joins via `PiecewiseMergeJoin` (behind `enable_piecewise_merge_join`, default off) now return unmatched right rows with `NULL` keys, matching `NestedLoopJoin`. No API changes. --- .../piecewise_merge_join/classic_join.rs | 10 +++++ datafusion/sqllogictest/test_files/pwmj.slt | 45 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs index 50ef78f18bf65..1c89927087fdb 100644 --- a/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs +++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs @@ -481,6 +481,16 @@ fn resolve_classic_join( buffer_idx = buffered_null_idx; stream_idx = stream_null_idx; batch_process_state.processed_null_count = true; + + // The scan below starts past the streamed side's NULL-keyed rows, which + // sit at the front (`nulls_first`). A NULL join key never matches under + // `NullEqualsNothing`, so for `Right`/`Full` those rows are unmatched and + // must still be emitted; record them here since the scan will skip them. + if matches!(join_type, JoinType::Right | JoinType::Full) { + for row_idx in 0..stream_null_idx as u32 { + batch_process_state.unmatched_indices.append_value(row_idx); + } + } } // Our buffer_idx variable allows us to start probing on the buffered side where we last matched diff --git a/datafusion/sqllogictest/test_files/pwmj.slt b/datafusion/sqllogictest/test_files/pwmj.slt index 9789c0e4e5392..25c275f4b7e93 100644 --- a/datafusion/sqllogictest/test_files/pwmj.slt +++ b/datafusion/sqllogictest/test_files/pwmj.slt @@ -342,5 +342,50 @@ ORDER BY 1,2; 1 3 2 3 +# RIGHT JOIN: every right row must be emitted, including one whose join key is +# NULL. A NULL key never matches (NullEqualsNothing), so t2.id = NULL is +# unmatched and must appear as (NULL, NULL). Regression test: the streamed-side +# NULL rows are skipped by the match scan and were previously dropped instead of +# being reported as unmatched. +query II +SELECT t1.id AS left_id, t2.id AS right_id +FROM null_join_t1 t1 +RIGHT JOIN null_join_t2 t2 + ON t1.id < t2.id +ORDER BY 1,2; +---- +1 3 +2 3 +NULL 1 +NULL NULL + +# FULL JOIN: the same fix applies to the FULL path. An unmatched right row with +# a NULL key must still be emitted as (NULL, NULL). This uses data where every +# left row matches, so only the right-side NULL emission fixed here is +# exercised (the separate left-side unmatched-row drop, tracked elsewhere, is +# not). +statement ok +CREATE TABLE full_null_t1 (id INT); + +statement ok +CREATE TABLE full_null_t2 (id INT); + +statement ok +INSERT INTO full_null_t1 VALUES (10), (20); + +statement ok +INSERT INTO full_null_t2 VALUES (100), (NULL); + +query II +SELECT t1.id AS left_id, t2.id AS right_id +FROM full_null_t1 t1 +FULL JOIN full_null_t2 t2 + ON t1.id < t2.id +ORDER BY 1,2; +---- +10 100 +20 100 +NULL NULL + statement ok set datafusion.optimizer.enable_piecewise_merge_join = false; From 0ffb22264c959dd2944e62e700effb8837e5b157 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Fri, 14 Aug 2026 05:45:16 +0000 Subject: [PATCH 09/50] bench: add parquet_row_filter_skip suite for the fully-matched RowFilter skip (#24328) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Part of #23696 — a benchmark that mechanistically exercises the per-RG fully-matched `RowFilter` skip. ## Rationale for this change #23696 adds a per-row-group fully-matched `RowFilter` skip, but **none of the existing benchmarks exercise it**: - `sort_tpch` / `tpch` don't enable `pushdown_filters` by default, so there is no `RowFilter` to skip; - ClickBench's `URL LIKE …` / equality predicates rarely make a row group's min/max fall entirely inside the satisfying range, so fully-matched RGs are rare. As raised in review (we should verify the optimization improves something mechanistically, otherwise add a benchmark first), this adds a suite that **necessarily** triggers the skip. ## What changes are included in this PR? A new `sql_benchmarks/parquet_row_filter_skip/` suite: - The **load SQL** enables `pushdown_filters` and `COPY`s a clustered Parquet file — a fixed-width, zero-padded, monotonically increasing string key (`skey`) so each row group holds a disjoint, sorted range — plus 14 payload columns. - The **query** applies a low-selectivity range filter (`skey >= '0000100000'`, `skey` not projected). The first row group straddles the threshold; every later RG is fully matched by statistics, so the per-row `RowFilter` is skipped on the fully-matched run (and `skey` isn't decoded there). - `bench.sh` integration: `./bench.sh run parquet_row_filter_skip`, data generated inline by the load SQL. Knobs: `PRED_ROWS` (row count), `RG_SIZE` (parquet row-group size). ## Are these changes tested? Smoke-tested locally via `cargo bench --bench sql -- --test`. Local A/B (main vs #23696, 10M rows / 10 RGs / `skey >= '0000100000'`): - **9 of 10 row groups fully matched** → `row_filter_skipped_fully_matched=9`; - **~18% faster** with the optimization (main ~0.143s → branch ~0.117s); an int64-key variant is ~12%. Once this lands, `run benchmark parquet_row_filter_skip` will compare any PR (e.g. #23696) against `main` in CI. ## Are there any user-facing changes? No — benchmark only. --------- Co-authored-by: Claude --- benchmarks/bench.sh | 32 ++++++++++++ benchmarks/sql_benchmarks/README.md | 4 +- .../benchmarks/control/q10.benchmark | 7 +++ .../benchmarks/skip/q01.benchmark | 7 +++ .../benchmarks/skip/q02.benchmark | 7 +++ .../benchmarks/skip/q03.benchmark | 7 +++ .../parquet_row_filter_skip/init/cleanup.sql | 1 + .../parquet_row_filter_skip/init/settings.sql | 9 ++++ .../load/clustered.sql | 44 ++++++++++++++++ .../load/scrambled.sql | 43 ++++++++++++++++ ...parquet_row_filter_skip.benchmark.template | 50 +++++++++++++++++++ .../parquet_row_filter_skip.suite | 26 ++++++++++ .../queries/control/q10.sql | 8 +++ .../queries/skip/q01.sql | 7 +++ .../queries/skip/q02.sql | 6 +++ .../queries/skip/q03.sql | 8 +++ .../scratch/.gitignore | 1 + 17 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/control/q10.benchmark create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q01.benchmark create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q02.benchmark create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q03.benchmark create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/init/cleanup.sql create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/init/settings.sql create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/load/clustered.sql create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/load/scrambled.sql create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.benchmark.template create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.suite create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/control/q10.sql create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q01.sql create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q02.sql create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q03.sql create mode 100644 benchmarks/sql_benchmarks/parquet_row_filter_skip/scratch/.gitignore diff --git a/benchmarks/bench.sh b/benchmarks/bench.sh index 52b78c844a73a..df9b7f6c94f16 100755 --- a/benchmarks/bench.sh +++ b/benchmarks/bench.sh @@ -108,6 +108,10 @@ predicate_eval: Conjunctive (AND) filter-evaluation micro-benchmarks; ea adaptive predicate-ordering system behaves across them (see https://github.com/apache/datafusion/issues/11262) (subgroups via BENCH_SUBGROUP: costsel, cost, selectivity, cardinality, width, scale, neutral, correlation, drift) (toggle a system under test with its native DATAFUSION_* env var; size data with PRED_ROWS, string width with PRED_FILL) +parquet_row_filter_skip: Per-RG fully-matched RowFilter skip on Parquet (apache/datafusion#23696); clustered string key + low-selectivity + range filter + pushdown, so most row groups are fully matched and the per-row RowFilter is skipped on them + (subgroups via BENCH_SUBGROUP: skip = clustered key so the skip fires, control = scrambled key so it never fires) + (data generated inline by the suite's load SQL; knobs: PRED_ROWS, RG_SIZE) # ClickBench Benchmarks clickbench_1: ClickBench queries against a single parquet file @@ -255,6 +259,10 @@ main() { # Data is generated inline by the suite's load SQL. echo "predicate_eval: no external data to generate" ;; + parquet_row_filter_skip) + # Data is generated inline by the suite's load SQL (COPY). + echo "parquet_row_filter_skip: no external data to generate" + ;; tpcds) data_tpcds ;; @@ -475,6 +483,9 @@ main() { predicate_eval) run_predicate_eval ;; + parquet_row_filter_skip) + run_parquet_row_filter_skip + ;; tpcds) run_tpcds ;; @@ -844,6 +855,27 @@ run_predicate_eval() { bash -c "$SQL_CARGO_COMMAND" } +# Runs the parquet_row_filter_skip suite: the load SQL COPYs a Parquet file +# inline (fixed-width string key, ordered so each row group holds a disjoint +# sorted range) and the query applies a low-selectivity range filter, so all +# but the first row group is fully matched by statistics and the per-row +# RowFilter is skipped on them (apache/datafusion#23696). The control subgroup +# scrambles the key so no row group is ever fully matched, measuring the +# overhead of the check when it cannot fire. Data is inline, so no data step. +# Knobs (string-substituted into the load SQL, not engine config): +# BENCH_SUBGROUP run one subgroup (skip, control) +# PRED_ROWS synthetic row count (default 10_000_000) +# RG_SIZE parquet row-group size (default 1_000_000) +run_parquet_row_filter_skip() { + echo "Running parquet_row_filter_skip benchmark (subgroup=${BENCH_SUBGROUP:-all}, rows=${PRED_ROWS:-10000000}, rg_size=${RG_SIZE:-1000000})..." + debug_run env BENCH_NAME=parquet_row_filter_skip \ + ${BENCH_SUBGROUP:+BENCH_SUBGROUP="${BENCH_SUBGROUP}"} \ + PRED_ROWS="${PRED_ROWS:-10000000}" \ + RG_SIZE="${RG_SIZE:-1000000}" \ + ${QUERY:+BENCH_QUERY="${QUERY}"} \ + bash -c "$SQL_CARGO_COMMAND" +} + # Runs the tpch in memory (needs tpch parquet data) run_tpch_mem() { SCALE_FACTOR=$1 diff --git a/benchmarks/sql_benchmarks/README.md b/benchmarks/sql_benchmarks/README.md index dfb09e0a3a4a2..1ce1fa488c6b8 100644 --- a/benchmarks/sql_benchmarks/README.md +++ b/benchmarks/sql_benchmarks/README.md @@ -44,6 +44,7 @@ in the community: | `tpch` | TPC‑H queries | | `wide_schema` | Small-projection queries on a wide (1024-col, 256-file) synthetic dataset; runs `wide` + `narrow` subgroups for comparison | | `predicate_eval` | Conjunctive (AND) filter-evaluation micro-benchmarks; each subgroup is a different predicate pattern, to test how an adaptive predicate-ordering system behaves across them ([#11262](https://github.com/apache/datafusion/issues/11262)). Subgroups (`--subgroup`): `costsel`, `cost`, `selectivity`, `cardinality`, `width`, `scale`, `neutral`, `correlation`, `drift`. Configure the system under test through its DataFusion settings. | +| `parquet_row_filter_skip` | Micro-benchmark for the per-row-group fully-matched RowFilter skip on Parquet scans ([#23696](https://github.com/apache/datafusion/issues/23696)). Subgroups (`--subgroup`): `skip` (clustered key, most row groups fully matched by statistics so the per-row filter is skipped), `control` (scrambled key, no row group is ever fully matched). Size the data with `PRED_ROWS` and the row-group size with `RG_SIZE`. | # Running Benchmarks @@ -168,8 +169,9 @@ Some benchmarks use custom environment variables as outlined below: | BENCH_SORTED | Used in the sort_tpch benchmark to indicate whether the lineitem table should be sorted. | false | | SORTED_BY | Used in the clickbench_sorted benchmark to indicate the column to sort by. | `EventTime` | | SORTED_ORDER | Used in the clickbench_sorted benchmark to indicate the sort order of the column. | `ASC` | -| PRED_ROWS | Used in the predicate_eval benchmark to size the synthetic table (the `scale` subgroup overrides this per query). | `1000000` | +| PRED_ROWS | Used in the predicate_eval benchmark to size the synthetic table (the `scale` subgroup overrides this per query), and in the parquet_row_filter_skip benchmark to size the generated Parquet datasets (default `10000000` there). | `1000000` | | PRED_FILL | Used in the predicate_eval benchmark as the string-column width knob (filler chars per marker). | `30` | +| RG_SIZE | Used in the parquet_row_filter_skip benchmark as the Parquet `max_row_group_size` for the generated datasets. | `1000000` | ## How it works diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/control/q10.benchmark b/benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/control/q10.benchmark new file mode 100644 index 0000000000000..4c64baa83300e --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/control/q10.benchmark @@ -0,0 +1,7 @@ +subgroup control + +template sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.benchmark.template +SUBGROUP=control +QPAD=10 +DATASET=scrambled +NAME=control_q10_scrambled_never_matched diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q01.benchmark b/benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q01.benchmark new file mode 100644 index 0000000000000..825605c6bb65e --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q01.benchmark @@ -0,0 +1,7 @@ +subgroup skip + +template sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.benchmark.template +SUBGROUP=skip +QPAD=01 +DATASET=clustered +NAME=skip_q01_string_range_low_selectivity diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q02.benchmark b/benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q02.benchmark new file mode 100644 index 0000000000000..30f1ff8b48181 --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q02.benchmark @@ -0,0 +1,7 @@ +subgroup skip + +template sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.benchmark.template +SUBGROUP=skip +QPAD=02 +DATASET=clustered +NAME=skip_q02_string_range_all_matched diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q03.benchmark b/benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q03.benchmark new file mode 100644 index 0000000000000..12015067f8b32 --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q03.benchmark @@ -0,0 +1,7 @@ +subgroup skip + +template sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.benchmark.template +SUBGROUP=skip +QPAD=03 +DATASET=clustered +NAME=skip_q03_filter_column_projected diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/init/cleanup.sql b/benchmarks/sql_benchmarks/parquet_row_filter_skip/init/cleanup.sql new file mode 100644 index 0000000000000..48f076a9fa652 --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/init/cleanup.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS t; diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/init/settings.sql b/benchmarks/sql_benchmarks/parquet_row_filter_skip/init/settings.sql new file mode 100644 index 0000000000000..0f084fae938df --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/init/settings.sql @@ -0,0 +1,9 @@ +-- Session settings for the parquet_row_filter_skip suite. `init` runs after +-- `load` and before the asserts and the benchmarked query, so these apply to +-- the measured scan (the COPY in the load script does not need them). +-- +-- information_schema is enabled so the template can assert that +-- pushdown_filters actually took effect: without pushdown there is no +-- RowFilter to skip and the suite would silently measure nothing. +set datafusion.catalog.information_schema = true; +set datafusion.execution.parquet.pushdown_filters = true; diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/load/clustered.sql b/benchmarks/sql_benchmarks/parquet_row_filter_skip/load/clustered.sql new file mode 100644 index 0000000000000..dd6bdfc42eb20 --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/load/clustered.sql @@ -0,0 +1,44 @@ +-- Clustered Parquet dataset for the fully-matched RowFilter-skip benchmark. +-- +-- `skey` is a fixed-width, zero-padded, monotonically increasing string, so +-- each row group holds a disjoint, sorted range of keys. With +-- `pushdown_filters=true` (set in init/settings.sql), a low-selectivity range +-- predicate (see the queries) leaves the first row group straddling and every +-- later row group fully matched by min/max statistics, which is exactly what +-- the per-RG RowFilter skip targets. +-- +-- The ORDER BY is load-bearing: the whole benchmark rests on the file being +-- written in key order so that row-group min/max ranges are disjoint. Without +-- it the write order is at the mercy of the physical planner (a round-robin +-- repartition + coalesce would silently interleave batches and destroy the +-- clustering, leaving nothing to measure). +-- +-- Knobs: PRED_ROWS (row count, must exceed the 100_000 predicate cutoff), +-- RG_SIZE (parquet row-group size). +COPY ( + SELECT + lpad(CAST(value AS VARCHAR), 10, '0') AS skey, + (value * 7) % 1000000 AS p0, + (value * 13) % 1000000 AS p1, + (value * 17) % 1000000 AS p2, + (value * 19) % 1000000 AS p3, + (value * 23) % 1000000 AS p4, + (value * 29) % 1000000 AS p5, + (value * 31) % 1000000 AS p6, + (value * 37) % 1000000 AS p7, + (value * 41) % 1000000 AS p8, + (value * 43) % 1000000 AS p9, + (value * 47) % 1000000 AS p10, + (value * 53) % 1000000 AS p11, + (value * 59) % 1000000 AS p12, + (value * 61) % 1000000 AS p13 + FROM generate_series(1, ${PRED_ROWS:-10000000}) + ORDER BY value +) +TO 'sql_benchmarks/parquet_row_filter_skip/scratch/clustered.parquet' +STORED AS PARQUET +OPTIONS ('format.max_row_group_size' '${RG_SIZE:-1000000}'); + +CREATE EXTERNAL TABLE t +STORED AS PARQUET +LOCATION 'sql_benchmarks/parquet_row_filter_skip/scratch/clustered.parquet'; diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/load/scrambled.sql b/benchmarks/sql_benchmarks/parquet_row_filter_skip/load/scrambled.sql new file mode 100644 index 0000000000000..c8c0e0bf97822 --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/load/scrambled.sql @@ -0,0 +1,43 @@ +-- Scrambled control dataset for the fully-matched RowFilter-skip benchmark. +-- +-- Same schema, key domain, and payload as clustered.sql, but `skey` is a +-- pseudo-random permutation of 1..PRED_ROWS (999983 is prime, so the map +-- `v -> (v * 999983) % PRED_ROWS + 1` is a bijection whenever PRED_ROWS is +-- not a multiple of 999983 -- true for the default and any power-of-ten +-- size). Every row group therefore spans nearly the whole key range and no +-- row group is ever fully matched by min/max statistics: the per-row +-- RowFilter must run everywhere. Queries over this dataset measure the +-- overhead of the fully-matched check when it can never fire. +-- +-- Keeping the domain exactly 1..PRED_ROWS lets the template's row-count +-- asserts hold for both datasets. +-- +-- Knobs: PRED_ROWS (row count, must exceed the 100_000 predicate cutoff), +-- RG_SIZE (parquet row-group size). +COPY ( + SELECT + lpad(CAST((value * 999983) % ${PRED_ROWS:-10000000} + 1 AS VARCHAR), 10, '0') AS skey, + (value * 7) % 1000000 AS p0, + (value * 13) % 1000000 AS p1, + (value * 17) % 1000000 AS p2, + (value * 19) % 1000000 AS p3, + (value * 23) % 1000000 AS p4, + (value * 29) % 1000000 AS p5, + (value * 31) % 1000000 AS p6, + (value * 37) % 1000000 AS p7, + (value * 41) % 1000000 AS p8, + (value * 43) % 1000000 AS p9, + (value * 47) % 1000000 AS p10, + (value * 53) % 1000000 AS p11, + (value * 59) % 1000000 AS p12, + (value * 61) % 1000000 AS p13 + FROM generate_series(1, ${PRED_ROWS:-10000000}) + ORDER BY value +) +TO 'sql_benchmarks/parquet_row_filter_skip/scratch/scrambled.parquet' +STORED AS PARQUET +OPTIONS ('format.max_row_group_size' '${RG_SIZE:-1000000}'); + +CREATE EXTERNAL TABLE t +STORED AS PARQUET +LOCATION 'sql_benchmarks/parquet_row_filter_skip/scratch/scrambled.parquet'; diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.benchmark.template b/benchmarks/sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.benchmark.template new file mode 100644 index 0000000000000..44c776f5f4ac2 --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.benchmark.template @@ -0,0 +1,50 @@ +# Shared template for the parquet_row_filter_skip suite. Each qNN.benchmark sets +# its `subgroup` directive and includes this template with parameters: +# SUBGROUP subgroup name, also the query sub-directory (skip, control) +# QPAD zero-padded query id / query file stem (e.g. 01) +# DATASET load script stem under load/ (clustered, scrambled) +# NAME criterion display name +# Optional (consumed by the load script via ${...:-default}): +# PRED_ROWS synthetic row count (default 10_000_000) +# RG_SIZE parquet max row-group size (default 1_000_000) +# +# The load script COPYs the Parquet dataset and init/settings.sql enables +# `pushdown_filters` so the fully-matched RowFilter-skip path is exercised. +# The table is always named `t`, so the asserts and cleanup are uniform. +# +# The queries share a fixed cutoff key of '0000100000' (row 100_000), so +# PRED_ROWS must stay above 100_000; the second assert fails loudly otherwise. +# Both datasets use the key domain 1..PRED_ROWS, so the asserts hold for both. + +load sql_benchmarks/parquet_row_filter_skip/load/${DATASET}.sql + +init sql_benchmarks/parquet_row_filter_skip/init/settings.sql + +name ${NAME} +group parquet_row_filter_skip + +# The dataset has exactly PRED_ROWS rows. +assert I +SELECT count(*) = ${PRED_ROWS:-10000000} FROM t; +---- +true + +# The cutoff predicate matches exactly PRED_ROWS - 99_999 rows. This doubles +# as a correctness canary for the optimization under test: a RowFilter skip +# that fires on a not-fully-matched row group returns extra rows and trips +# this assert. +assert I +SELECT count(*) = ${PRED_ROWS:-10000000} - 99999 FROM t WHERE skey >= '0000100000'; +---- +true + +# Guard against silent config no-ops: without pushdown_filters there is no +# RowFilter at all and the suite measures nothing. +assert I +SELECT value = 'true' FROM information_schema.df_settings WHERE name = 'datafusion.execution.parquet.pushdown_filters'; +---- +true + +run sql_benchmarks/parquet_row_filter_skip/queries/${SUBGROUP}/q${QPAD}.sql + +cleanup sql_benchmarks/parquet_row_filter_skip/init/cleanup.sql diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.suite b/benchmarks/sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.suite new file mode 100644 index 0000000000000..1cf46dd2ef188 --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.suite @@ -0,0 +1,26 @@ +description = "Micro-benchmark for the per-row-group fully-matched RowFilter skip on Parquet scans (apache/datafusion#23696). Builds a clustered Parquet file whose fixed-width string key is monotonically increasing, so each row group holds a disjoint sorted range. With filter pushdown enabled, a low-selectivity range predicate leaves the first row group straddling and the rest fully matched by statistics, so the decoder can skip the per-row RowFilter (and decoding the filter column) on the fully-matched run. The control subgroup runs the same predicate over a scrambled key, where no row group is ever fully matched, to measure the overhead of the check when it cannot fire. Size the data with PRED_ROWS and the row-group size with RG_SIZE." + +query_pattern = "q{QUERY_ID_PADDED}.benchmark" + +[[options]] +name = "rows" +short = "r" +env = "PRED_ROWS" +default = "10000000" +values = ["10000000", "..."] +help = "Sets the number of rows in the generated Parquet datasets (must be greater than 100000, the fixed predicate cutoff)." + +[[options]] +name = "rg-size" +env = "RG_SIZE" +default = "1000000" +values = ["1000000", "..."] +help = "Sets the Parquet max_row_group_size used when writing the generated datasets." + +[[examples]] +command = "cargo run --release --bin benchmark_runner -- parquet_row_filter_skip" +description = "Run every subgroup with the default 10M rows and 1M-row row groups." + +[[examples]] +command = "cargo run --release --bin benchmark_runner -- parquet_row_filter_skip --subgroup skip --rg-size 100000" +description = "Run only the skip subgroup with 100 row groups instead of 10." diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/control/q10.sql b/benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/control/q10.sql new file mode 100644 index 0000000000000..a5950e925b0ce --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/control/q10.sql @@ -0,0 +1,8 @@ +-- Control: the same cutoff predicate as skip/q01 but over the scrambled +-- dataset, where every row group spans nearly the whole key range, so no row +-- group is ever fully matched and the per-row RowFilter runs everywhere. +-- Measures the overhead of the fully-matched check when it cannot fire; the +-- skip optimization should be performance-neutral here. +SELECT sum(p0)+sum(p1)+sum(p2)+sum(p3)+sum(p4)+sum(p5)+sum(p6)+sum(p7)+sum(p8)+sum(p9)+sum(p10)+sum(p11)+sum(p12)+sum(p13) AS s +FROM t +WHERE skey >= '0000100000'; diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q01.sql b/benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q01.sql new file mode 100644 index 0000000000000..33468c19226fa --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q01.sql @@ -0,0 +1,7 @@ +-- Low-selectivity range filter on the clustered string key: every row group +-- except the first is fully matched by statistics, so the per-row RowFilter is +-- skipped on those RGs. `skey` is not projected, so the skip also avoids +-- decoding the filter column on the fully-matched run. +SELECT sum(p0)+sum(p1)+sum(p2)+sum(p3)+sum(p4)+sum(p5)+sum(p6)+sum(p7)+sum(p8)+sum(p9)+sum(p10)+sum(p11)+sum(p12)+sum(p13) AS s +FROM t +WHERE skey >= '0000100000'; diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q02.sql b/benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q02.sql new file mode 100644 index 0000000000000..a8eb2d6ca08f1 --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q02.sql @@ -0,0 +1,6 @@ +-- Matches every row: the minimum key is '0000000001', so every row group -- +-- including the first -- is fully matched by statistics and the per-row +-- RowFilter is skipped everywhere. The upper bound for the optimization. +SELECT sum(p0)+sum(p1)+sum(p2)+sum(p3)+sum(p4)+sum(p5)+sum(p6)+sum(p7)+sum(p8)+sum(p9)+sum(p10)+sum(p11)+sum(p12)+sum(p13) AS s +FROM t +WHERE skey >= '0000000000'; diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q03.sql b/benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q03.sql new file mode 100644 index 0000000000000..2879d222b240e --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q03.sql @@ -0,0 +1,8 @@ +-- Same cutoff predicate as q01, but the filter column IS projected: the +-- fully-matched skip still avoids evaluating the per-row filter, but can no +-- longer avoid decoding `skey`. Measures the common case where the win is +-- smaller than q01's best case. +SELECT min(skey) AS first_key, + sum(p0)+sum(p1)+sum(p2)+sum(p3)+sum(p4)+sum(p5)+sum(p6)+sum(p7)+sum(p8)+sum(p9)+sum(p10)+sum(p11)+sum(p12)+sum(p13) AS s +FROM t +WHERE skey >= '0000100000'; diff --git a/benchmarks/sql_benchmarks/parquet_row_filter_skip/scratch/.gitignore b/benchmarks/sql_benchmarks/parquet_row_filter_skip/scratch/.gitignore new file mode 100644 index 0000000000000..4bed5da93fb28 --- /dev/null +++ b/benchmarks/sql_benchmarks/parquet_row_filter_skip/scratch/.gitignore @@ -0,0 +1 @@ +*.parquet From e2672298d2cb3ff87190c43b713882cdefccadd7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 05:51:07 +0000 Subject: [PATCH 10/50] chore(deps): bump Swatinem/rust-cache from 2.8.1 to 2.9.2 in /.github/actions/setup-macos-aarch64-builder (#24307) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [Swatinem/rust-cache](https://github.com/swatinem/rust-cache) from 2.8.1 to 2.9.2.
Release notes

Sourced from Swatinem/rust-cache's releases.

v2.9.2

What's Changed

New Contributors

Full Changelog: https://github.com/Swatinem/rust-cache/compare/v2.9.1...v2.9.2

v2.9.1

Fix regression in hash calculation

Full Changelog: https://github.com/Swatinem/rust-cache/compare/v2.9.0...v2.9.1

v2.9.0

What's Changed

New Contributors

Full Changelog: https://github.com/Swatinem/rust-cache/compare/v2.8.2...v2.9.0

v2.8.2

What's Changed

... (truncated)

Changelog

Sourced from Swatinem/rust-cache's changelog.

Changelog

2.9.2

  • Fix credentials.toml cleanup
  • Improvements to cleanup, preserving more valid targets
  • Improvements to cargo install handling
  • Correctly sort/dedupe Rust versions

2.9.1

  • Fix regression in hash calculation

2.9.0

  • Update to node24
  • Support running from within a nix shell
  • Consider all installed toolchains for cache key
  • Use case-insensitive comparison to determine exact cache hit

2.8.2

  • Don't overwrite env for cargo-metadata call

2.8.1

  • Set empty CARGO_ENCODED_RUSTFLAGS when retrieving metadata
  • Various dependency updates

2.8.0

  • Add support for warpbuild cache provider
  • Add new cache-workspace-crates feature

2.7.8

  • Include CPU arch in the cache key

2.7.7

  • Also cache cargo install metadata

2.7.6

  • Allow opting out of caching $CARGO_HOME/bin
  • Add runner OS in cache key
  • Adds an option to do lookup-only of the cache

2.7.5

... (truncated)

Commits
  • 6323deb 2.9.2
  • b16e8d7 bump rollup and rebuild
  • 3bf42ac invert target/profile check in cleanup
  • 6e5b278 correctly sort and dedupe Rust versions
  • 5adc05f Bump the actions group across 1 directory with 3 updates (#368)
  • 66b1e95 fix: support Cargo V2 build dir layout (#371)
  • 72d126e Merge pull request #367 from Swatinem/dependabot/npm_and_yarn/dev-patch-2b495...
  • 48968d2 Bump the dev-patch group with 2 updates
  • 9f151ac update dependencies, rebuild
  • 0e24e5d Bump the actions group across 1 directory with 6 updates (#364)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Swatinem/rust-cache&package-manager=github_actions&previous-version=2.8.1&new-version=2.9.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/actions/setup-macos-aarch64-builder/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-macos-aarch64-builder/action.yaml b/.github/actions/setup-macos-aarch64-builder/action.yaml index b62370447adea..287e8275b5e15 100644 --- a/.github/actions/setup-macos-aarch64-builder/action.yaml +++ b/.github/actions/setup-macos-aarch64-builder/action.yaml @@ -44,7 +44,7 @@ runs: rustup default stable rustup component add rustfmt - name: Setup rust cache - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: save-if: ${{ github.ref_name == 'main' }} - name: Configure rust runtime env From 6a287fed416e308b8a01cc6ac84174aadc81c972 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 05:51:25 +0000 Subject: [PATCH 11/50] chore(deps): bump taiki-e/install-action from 2.85.10 to 2.85.11 (#24306) Bumps [taiki-e/install-action](https://github.com/taiki-e/install-action) from 2.85.10 to 2.85.11.
Release notes

Sourced from taiki-e/install-action's releases.

2.85.11

  • Update zola@latest to 0.23.2.

  • Update wasm-bindgen@latest to 0.2.127.

  • Update uv@latest to 0.12.3.

  • Update osv-scanner@latest to 2.5.0.

  • Update mise@latest to 2026.8.3.

  • Update kingfisher@latest to 1.112.0.

  • Update editorconfig-checker@latest to 3.10.0.

Changelog

Sourced from taiki-e/install-action's changelog.

Changelog

All notable changes to this project will be documented in this file.

This project adheres to Semantic Versioning.

[Unreleased]

[2.85.12] - 2026-08-12

  • Update zola@latest to 0.23.3.

  • Update wasm-tools@latest to 1.256.0.

  • Update tombi@latest to 1.2.10.

  • Update syft@latest to 1.51.0.

  • Update prek@latest to 0.4.13.

  • Update mise@latest to 2026.8.4.

  • Update editorconfig-checker@latest to 3.11.1.

  • Update cargo-tarpaulin@latest to 0.37.1.

  • Update cargo-rdme@latest to 2.2.1.

  • Update biome@latest to 2.5.8.

[2.85.11] - 2026-08-09

  • Update zola@latest to 0.23.2.

  • Update wasm-bindgen@latest to 0.2.127.

  • Update uv@latest to 0.12.3.

  • Update osv-scanner@latest to 2.5.0.

  • Update mise@latest to 2026.8.3.

  • Update kingfisher@latest to 1.112.0.

  • Update editorconfig-checker@latest to 3.10.0.

... (truncated)

Commits
  • 7f4eb89 Release 2.85.11
  • c17da62 Update zola@latest to 0.23.2
  • 97e8291 Update wasm-bindgen@latest to 0.2.127
  • 91f9e5c Update uv@latest to 0.12.3
  • bd0cb00 Update osv-scanner@latest to 2.5.0
  • 4def957 Update mise@latest to 2026.8.3
  • 3d149dc Update kingfisher@latest to 1.112.0
  • 20d4381 Update editorconfig-checker@latest to 3.10.0
  • 583939e codegen: Ignore clippy::assert_is_empty lint
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=taiki-e/install-action&package-manager=github_actions&previous-version=2.85.10&new-version=2.85.11)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/audit.yml | 2 +- .github/workflows/breaking_changes_detector.yml | 2 +- .github/workflows/dependencies.yml | 2 +- .github/workflows/dev.yml | 6 +++--- .github/workflows/docs.yaml | 2 +- .github/workflows/docs_pr.yaml | 2 +- .github/workflows/rust.yml | 8 ++++---- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 875eeffdf053d..c268abc9c0630 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -45,7 +45,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install cargo-audit - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: cargo-audit - name: Run audit check diff --git a/.github/workflows/breaking_changes_detector.yml b/.github/workflows/breaking_changes_detector.yml index 3a279a27f54d9..805c42dcb5c83 100644 --- a/.github/workflows/breaking_changes_detector.yml +++ b/.github/workflows/breaking_changes_detector.yml @@ -89,7 +89,7 @@ jobs: - name: Install cargo-semver-checks if: steps.changed_crates.outputs.packages != '' - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: cargo-semver-checks diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index f76ced5296871..79b5a35a6890b 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -63,7 +63,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install cargo-machete - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: cargo-machete@0.9 - name: Detect unused dependencies diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index a8c1b8c07af96..b736345ed158a 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -38,7 +38,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install HawkEye - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: hawkeye@6.2.0 - name: Run license header check @@ -66,7 +66,7 @@ jobs: source ci/scripts/utils/tool_versions.sh echo "LYCHEE_VERSION=${LYCHEE_VERSION}" >> "$GITHUB_ENV" - name: Install lychee - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: lychee@${{ env.LYCHEE_VERSION }} - name: Run markdown link check @@ -91,7 +91,7 @@ jobs: # it may cause checks to fail more often. # We can upgrade it manually once a while. - name: Install typos - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: typos@1.37.0 - name: Run typos check diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 05f48d885fb5b..21c4223dacacc 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -53,7 +53,7 @@ jobs: sudo apt-get update sudo apt-get install -y graphviz - name: Install cargo-depgraph - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: cargo-depgraph@1.6 diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml index d476d40d065b8..4362323ec97f0 100644 --- a/.github/workflows/docs_pr.yaml +++ b/.github/workflows/docs_pr.yaml @@ -59,7 +59,7 @@ jobs: sudo apt-get update sudo apt-get install -y graphviz - name: Install cargo-depgraph - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: cargo-depgraph@1.6 - name: Build docs html and check for warnings diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index eaa9b21a1b343..518e4d20eecb8 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -309,7 +309,7 @@ jobs: - name: Install llvm-tools-preview run: rustup component add llvm-tools-preview - name: Install cargo-llvm-cov - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: cargo-llvm-cov - name: Rust Dependency Cache @@ -466,7 +466,7 @@ jobs: sudo apt-get update -qq sudo apt-get install -y -qq clang - name: Setup wasm-pack - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: wasm-pack - name: Run tests with headless mode @@ -697,7 +697,7 @@ jobs: with: rust-version: stable - name: Install taplo - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: taplo-cli@0.9 # if you encounter an error, try running 'taplo format' to fix the formatting automatically. @@ -782,7 +782,7 @@ jobs: - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - name: Install cargo-msrv - uses: taiki-e/install-action@6c6fd71fe4fb72c3697d269963d0e15df8adedad # v2.85.10 + uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: cargo-msrv From 97be2664b8b90e9e3bfbd4e5ea040cb1e015e2c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 05:51:47 +0000 Subject: [PATCH 12/50] chore(deps-dev): bump webpack-cli from 5.1.4 to 7.2.2 in /datafusion/wasmtest/datafusion-wasm-app (#24302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [webpack-cli](https://github.com/webpack/webpack-cli) from 5.1.4 to 7.2.2.
Release notes

Sourced from webpack-cli's releases.

webpack-cli@7.2.2

Patch Changes

  • perf: enable the Node.js compile cache (module.enableCompileCache, available on Node.js >= 22.8.0) to speed up CLI startup (by @​bjohansebas in #4818)

webpack-cli@7.2.1

Patch Changes

webpack-cli@7.2.0

Minor Changes

  • feat: allow webpack-dev-server v6 as an optional peer dependency (^5.0.0 || ^6.0.0) (by @​bjohansebas in #4793)

  • Support tsx as a fallback loader for TypeScript and JSX configuration files (.ts, .tsx, .cts, .mts and .jsx), used when none of the loaders known to interpret (such as ts-node) are installed. (by @​alexander-akait in #4796)

webpack-cli@7.1.0

Minor Changes

  • feat(cli): refresh the --help output using commander's configureHelp API — branded headers, section dividers, colorized terms and a clearer footer. Colors and chrome collapse to plain text when output is piped or --no-color is used, so scripts keep working. (by @​alexander-akait in #4779)

  • feat: support .json5, .yaml/.yml and .toml configuration files by parsing them directly, with the parser package (json5, js-yaml, toml) installed on demand by the user and declared as optional peerDependencies so the parsers resolve correctly under Yarn PnP (by @​alexander-akait in #4777)

webpack-cli@7.0.3

Patch Changes

  • Improved CLI startup performance and reduced memory usage. (by @​alexander-akait in #4765)

  • Reduced CLI startup CPU and memory usage by caching schema-derived argument metadata, registering only the options present in the arguments, and reading config directories once during default-config discovery. (by @​alexander-akait in #4760)

  • Replace the fastest-levenshtein dependency with a small in-tree implementation used for command/option "did you mean" suggestions. (by @​alexander-akait in #4762)

webpack-cli@7.0.2

Patch Changes

webpack-cli@7.0.1

Patch Changes

webpack-cli@7.0.0

Major Changes

  • The minimum supported version of Node.js is 20.9.0. (by @​alexander-akait in #4677)

  • Use dynamic import to load webpack.config.js, fallback to interpret only when configuration can't be load by dynamic import. Using dynamic imports allows you to take advantage of Node.js's built-in TypeScript support. (by @​alexander-akait in #4677)

... (truncated)

Changelog

Sourced from webpack-cli's changelog.

7.2.2

Patch Changes

  • perf: enable the Node.js compile cache (module.enableCompileCache, available on Node.js >= 22.8.0) to speed up CLI startup (by @​bjohansebas in #4818)

7.2.1

Patch Changes

7.2.0

Minor Changes

  • feat: allow webpack-dev-server v6 as an optional peer dependency (^5.0.0 || ^6.0.0) (by @​bjohansebas in #4793)

  • Support tsx as a fallback loader for TypeScript and JSX configuration files (.ts, .tsx, .cts, .mts and .jsx), used when none of the loaders known to interpret (such as ts-node) are installed. (by @​alexander-akait in #4796)

7.1.0

Minor Changes

  • feat(cli): refresh the --help output using commander's configureHelp API — branded headers, section dividers, colorized terms and a clearer footer. Colors and chrome collapse to plain text when output is piped or --no-color is used, so scripts keep working. (by @​alexander-akait in #4779)

  • feat: support .json5, .yaml/.yml and .toml configuration files by parsing them directly, with the parser package (json5, js-yaml, toml) installed on demand by the user and declared as optional peerDependencies so the parsers resolve correctly under Yarn PnP (by @​alexander-akait in #4777)

7.0.3

Patch Changes

  • Improved CLI startup performance and reduced memory usage. (by @​alexander-akait in #4765)

  • Reduced CLI startup CPU and memory usage by caching schema-derived argument metadata, registering only the options present in the arguments, and reading config directories once during default-config discovery. (by @​alexander-akait in #4760)

  • Replace the fastest-levenshtein dependency with a small in-tree implementation used for command/option "did you mean" suggestions. (by @​alexander-akait in #4762)

7.0.2

Patch Changes

7.0.1

Patch Changes

... (truncated)

Commits
  • 191fa9f chore(release): new release (#4820)
  • 1082235 chore(deps-dev): bump js-yaml from 5.2.1 to 5.2.2 (#4821)
  • e36ae82 chore: add publish workflow and script for pkg.pr.new integration (#4819)
  • b0a2024 perf: enable the Node.js compile cache (#4818)
  • 53f6d72 chore(deps): bump actions/checkout in the dependencies group (#4816)
  • 8856346 chore: add allowscripts to work with npm 12 (#4811)
  • 0b0f1ee chore(deps): bump fast-uri from 3.1.2 to 3.1.4 (#4815)
  • d60a777 chore(deps-dev): bump postcss from 8.5.10 to 8.5.22 (#4814)
  • 8c1719a chore(deps): bump shell-quote from 1.8.4 to 1.10.0 (#4813)
  • 8c2473c chore(deps): bump immutable from 5.1.5 to 5.1.9 (#4812)
  • Additional commits viewable in compare view
Maintainer changes

This version was pushed to npm by GitHub Actions, a new releaser for webpack-cli since your current version.


[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=webpack-cli&package-manager=npm_and_yarn&previous-version=5.1.4&new-version=7.2.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../datafusion-wasm-app/package-lock.json | 310 +++++++----------- .../wasmtest/datafusion-wasm-app/package.json | 2 +- 2 files changed, 118 insertions(+), 194 deletions(-) diff --git a/datafusion/wasmtest/datafusion-wasm-app/package-lock.json b/datafusion/wasmtest/datafusion-wasm-app/package-lock.json index 34b0d22f00c4c..ad4f047bf9689 100644 --- a/datafusion/wasmtest/datafusion-wasm-app/package-lock.json +++ b/datafusion/wasmtest/datafusion-wasm-app/package-lock.json @@ -14,7 +14,7 @@ "devDependencies": { "copy-webpack-plugin": "14.0.0", "webpack": "5.105.0", - "webpack-cli": "5.1.4", + "webpack-cli": "7.2.2", "webpack-dev-server": "6.0.0" } }, @@ -23,12 +23,12 @@ "version": "0.0.1" }, "node_modules/@discoveryjs/json-ext": { - "version": "0.5.7", - "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz", - "integrity": "sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-1.1.0.tgz", + "integrity": "sha512-Xc3VhU02wqZ1HvHRJUwL09HkZSTvidqY5Ya0NXBSYOxAp+Ln9dcJr9fySI+CkONzP3PekQo9WdzCv0PGER/mOA==", "dev": true, "engines": { - "node": ">=10.0.0" + "node": ">=14.17.0" } }, "node_modules/@jridgewell/gen-mapping": { @@ -635,50 +635,6 @@ "@xtuc/long": "4.2.2" } }, - "node_modules/@webpack-cli/configtest": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@webpack-cli/configtest/-/configtest-2.1.1.tgz", - "integrity": "sha512-wy0mglZpDSiSS0XHrVR+BAdId2+yxPSoJW8fsna3ZpYSlufjvxnP4YbKTCBZnNIcGN4r6ZPXV55X4mYExOfLmw==", - "dev": true, - "engines": { - "node": ">=14.15.0" - }, - "peerDependencies": { - "webpack": "5.x.x", - "webpack-cli": "5.x.x" - } - }, - "node_modules/@webpack-cli/info": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@webpack-cli/info/-/info-2.0.2.tgz", - "integrity": "sha512-zLHQdI/Qs1UyT5UBdWNqsARasIA+AaF8t+4u2aS2nEpBQh2mWIVb8qAklq0eUENnC5mOItrIB4LiS9xMtph18A==", - "dev": true, - "engines": { - "node": ">=14.15.0" - }, - "peerDependencies": { - "webpack": "5.x.x", - "webpack-cli": "5.x.x" - } - }, - "node_modules/@webpack-cli/serve": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@webpack-cli/serve/-/serve-2.0.5.tgz", - "integrity": "sha512-lqaoKnRYBdo1UgDX8uF24AfGMifWK19TxPmM5FHc2vAGxrJ/qtyUyFBWoY1tISZdelsQ5fBcOusifo5o5wSJxQ==", - "dev": true, - "engines": { - "node": ">=14.15.0" - }, - "peerDependencies": { - "webpack": "5.x.x", - "webpack-cli": "5.x.x" - }, - "peerDependenciesMeta": { - "webpack-dev-server": { - "optional": true - } - } - }, "node_modules/@xtuc/ieee754": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", @@ -1067,12 +1023,6 @@ "node": ">=6" } }, - "node_modules/colorette": { - "version": "2.0.20", - "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", - "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", - "dev": true - }, "node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", @@ -1242,9 +1192,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "dependencies": { "path-key": "^3.1.0", @@ -1393,9 +1343,9 @@ } }, "node_modules/envinfo": { - "version": "7.10.0", - "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.10.0.tgz", - "integrity": "sha512-ZtUjZO6l5mwTHvc1L9+1q5p/R3wTopcfqMW8r5t8SJSKqeVI/LtajORwRFEKpEFuekjD0VBjwu1HMxL4UalIRw==", + "version": "7.21.0", + "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.21.0.tgz", + "integrity": "sha512-Lw7I8Zp5YKHFCXL7+Dz95g4CcbMEpgvqZNNq3AmlT5XAV6CgAAk6gyAMqn2zjw08K9BHfcNuKrMiCPLByGafow==", "dev": true, "bin": { "envinfo": "dist/cli.js" @@ -1606,15 +1556,6 @@ } ] }, - "node_modules/fastest-levenshtein": { - "version": "1.0.16", - "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", - "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", - "dev": true, - "engines": { - "node": ">= 4.9.1" - } - }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -1661,6 +1602,15 @@ "node": ">=8" } }, + "node_modules/flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "dev": true, + "bin": { + "flat": "cli.js" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -1862,9 +1812,9 @@ } }, "node_modules/import-local": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz", - "integrity": "sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", + "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", "dev": true, "dependencies": { "pkg-dir": "^4.2.0", @@ -2812,19 +2762,7 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-locate/node_modules/p-limit": { + "node_modules/p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", @@ -2839,6 +2777,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/p-retry": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-8.0.0.tgz", @@ -3867,40 +3817,46 @@ } }, "node_modules/webpack-cli": { - "version": "5.1.4", - "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-5.1.4.tgz", - "integrity": "sha512-pIDJHIEI9LR0yxHXQ+Qh95k2EvXpWzZ5l+d+jIo+RdSm9MiHfzazIxwwni/p7+x4eJZuvG1AJwgC4TNQ7NRgsg==", - "dev": true, - "dependencies": { - "@discoveryjs/json-ext": "^0.5.0", - "@webpack-cli/configtest": "^2.1.1", - "@webpack-cli/info": "^2.0.2", - "@webpack-cli/serve": "^2.0.5", - "colorette": "^2.0.14", - "commander": "^10.0.1", - "cross-spawn": "^7.0.3", - "envinfo": "^7.7.3", - "fastest-levenshtein": "^1.0.12", - "import-local": "^3.0.2", + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-7.2.2.tgz", + "integrity": "sha512-lD0pALneslq8FfV+rwvm1BMW0AFAJrHHhNupAGN4asYjMvqrtRsenU4iKpiBo09gS4ntMxKGUxl9jhTEzVt0oA==", + "dev": true, + "dependencies": { + "@discoveryjs/json-ext": "^1.1.0", + "commander": "^14.0.3", + "cross-spawn": "^7.0.6", + "envinfo": "^7.21.0", + "import-local": "^3.2.0", "interpret": "^3.1.1", "rechoir": "^0.8.0", - "webpack-merge": "^5.7.3" + "webpack-merge": "^6.0.1" }, "bin": { "webpack-cli": "bin/cli.js" }, "engines": { - "node": ">=14.15.0" + "node": ">=20.9.0" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/webpack" }, "peerDependencies": { - "webpack": "5.x.x" + "js-yaml": "^4.0.0 || ^5.0.0", + "json5": "^2.2.3", + "toml": "^3.0.0 || ^4.0.0", + "webpack": "^5.101.0", + "webpack-bundle-analyzer": "^4.0.0 || ^5.0.0", + "webpack-dev-server": "^5.0.0 || ^6.0.0" }, "peerDependenciesMeta": { - "@webpack-cli/generators": { + "js-yaml": { + "optional": true + }, + "json5": { + "optional": true + }, + "toml": { "optional": true }, "webpack-bundle-analyzer": { @@ -3912,12 +3868,12 @@ } }, "node_modules/webpack-cli/node_modules/commander": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", - "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", "dev": true, "engines": { - "node": ">=14" + "node": ">=20" } }, "node_modules/webpack-dev-middleware": { @@ -4027,16 +3983,17 @@ } }, "node_modules/webpack-merge": { - "version": "5.9.0", - "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.9.0.tgz", - "integrity": "sha512-6NbRQw4+Sy50vYNTw7EyOn41OZItPiXB8GNv3INSoe3PSFaHJEz3SHTrYVaRm2LilNGnFUzh0FAwqPEmU/CwDg==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-6.0.1.tgz", + "integrity": "sha512-hXXvrjtx2PLYx4qruKl+kyRSLc52V+cCvMxRjmKwoA+CBbbF5GfIBtR6kCvl0fYGqTUPKB+1ktVmTHqMOzgCBg==", "dev": true, "dependencies": { "clone-deep": "^4.0.1", - "wildcard": "^2.0.0" + "flat": "^5.0.2", + "wildcard": "^2.0.1" }, "engines": { - "node": ">=10.0.0" + "node": ">=18.0.0" } }, "node_modules/webpack-sources": { @@ -4115,9 +4072,9 @@ }, "dependencies": { "@discoveryjs/json-ext": { - "version": "0.5.7", - "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz", - "integrity": "sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-1.1.0.tgz", + "integrity": "sha512-Xc3VhU02wqZ1HvHRJUwL09HkZSTvidqY5Ya0NXBSYOxAp+Ln9dcJr9fySI+CkONzP3PekQo9WdzCv0PGER/mOA==", "dev": true }, "@jridgewell/gen-mapping": { @@ -4731,27 +4688,6 @@ "@xtuc/long": "4.2.2" } }, - "@webpack-cli/configtest": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@webpack-cli/configtest/-/configtest-2.1.1.tgz", - "integrity": "sha512-wy0mglZpDSiSS0XHrVR+BAdId2+yxPSoJW8fsna3ZpYSlufjvxnP4YbKTCBZnNIcGN4r6ZPXV55X4mYExOfLmw==", - "dev": true, - "requires": {} - }, - "@webpack-cli/info": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@webpack-cli/info/-/info-2.0.2.tgz", - "integrity": "sha512-zLHQdI/Qs1UyT5UBdWNqsARasIA+AaF8t+4u2aS2nEpBQh2mWIVb8qAklq0eUENnC5mOItrIB4LiS9xMtph18A==", - "dev": true, - "requires": {} - }, - "@webpack-cli/serve": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@webpack-cli/serve/-/serve-2.0.5.tgz", - "integrity": "sha512-lqaoKnRYBdo1UgDX8uF24AfGMifWK19TxPmM5FHc2vAGxrJ/qtyUyFBWoY1tISZdelsQ5fBcOusifo5o5wSJxQ==", - "dev": true, - "requires": {} - }, "@xtuc/ieee754": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", @@ -5010,12 +4946,6 @@ "shallow-clone": "^3.0.0" } }, - "colorette": { - "version": "2.0.20", - "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", - "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", - "dev": true - }, "commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", @@ -5130,9 +5060,9 @@ } }, "cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "requires": { "path-key": "^3.1.0", @@ -5237,9 +5167,9 @@ } }, "envinfo": { - "version": "7.10.0", - "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.10.0.tgz", - "integrity": "sha512-ZtUjZO6l5mwTHvc1L9+1q5p/R3wTopcfqMW8r5t8SJSKqeVI/LtajORwRFEKpEFuekjD0VBjwu1HMxL4UalIRw==", + "version": "7.21.0", + "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-7.21.0.tgz", + "integrity": "sha512-Lw7I8Zp5YKHFCXL7+Dz95g4CcbMEpgvqZNNq3AmlT5XAV6CgAAk6gyAMqn2zjw08K9BHfcNuKrMiCPLByGafow==", "dev": true }, "es-define-property": { @@ -5391,12 +5321,6 @@ "integrity": "sha512-gHwA1O9LDIcKunMKhObS/HimwtehO1nPUECKAu5TpKgaO19fcWEl4bliWe1jWxVFvIXztJjjQ4L8XQ1EU9f7Jw==", "dev": true }, - "fastest-levenshtein": { - "version": "1.0.16", - "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", - "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", - "dev": true - }, "fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -5430,6 +5354,12 @@ "path-exists": "^4.0.0" } }, + "flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "dev": true + }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -5572,9 +5502,9 @@ } }, "import-local": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz", - "integrity": "sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", + "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", "dev": true, "requires": { "pkg-dir": "^4.2.0", @@ -6150,6 +6080,15 @@ "wsl-utils": "^0.3.0" } }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + }, "p-locate": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", @@ -6157,17 +6096,6 @@ "dev": true, "requires": { "p-limit": "^2.2.0" - }, - "dependencies": { - "p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "requires": { - "p-try": "^2.0.0" - } - } } }, "p-retry": { @@ -6866,30 +6794,25 @@ } }, "webpack-cli": { - "version": "5.1.4", - "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-5.1.4.tgz", - "integrity": "sha512-pIDJHIEI9LR0yxHXQ+Qh95k2EvXpWzZ5l+d+jIo+RdSm9MiHfzazIxwwni/p7+x4eJZuvG1AJwgC4TNQ7NRgsg==", - "dev": true, - "requires": { - "@discoveryjs/json-ext": "^0.5.0", - "@webpack-cli/configtest": "^2.1.1", - "@webpack-cli/info": "^2.0.2", - "@webpack-cli/serve": "^2.0.5", - "colorette": "^2.0.14", - "commander": "^10.0.1", - "cross-spawn": "^7.0.3", - "envinfo": "^7.7.3", - "fastest-levenshtein": "^1.0.12", - "import-local": "^3.0.2", + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-7.2.2.tgz", + "integrity": "sha512-lD0pALneslq8FfV+rwvm1BMW0AFAJrHHhNupAGN4asYjMvqrtRsenU4iKpiBo09gS4ntMxKGUxl9jhTEzVt0oA==", + "dev": true, + "requires": { + "@discoveryjs/json-ext": "^1.1.0", + "commander": "^14.0.3", + "cross-spawn": "^7.0.6", + "envinfo": "^7.21.0", + "import-local": "^3.2.0", "interpret": "^3.1.1", "rechoir": "^0.8.0", - "webpack-merge": "^5.7.3" + "webpack-merge": "^6.0.1" }, "dependencies": { "commander": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", - "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", + "integrity": "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==", "dev": true } } @@ -6957,13 +6880,14 @@ } }, "webpack-merge": { - "version": "5.9.0", - "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.9.0.tgz", - "integrity": "sha512-6NbRQw4+Sy50vYNTw7EyOn41OZItPiXB8GNv3INSoe3PSFaHJEz3SHTrYVaRm2LilNGnFUzh0FAwqPEmU/CwDg==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-6.0.1.tgz", + "integrity": "sha512-hXXvrjtx2PLYx4qruKl+kyRSLc52V+cCvMxRjmKwoA+CBbbF5GfIBtR6kCvl0fYGqTUPKB+1ktVmTHqMOzgCBg==", "dev": true, "requires": { "clone-deep": "^4.0.1", - "wildcard": "^2.0.0" + "flat": "^5.0.2", + "wildcard": "^2.0.1" } }, "webpack-sources": { diff --git a/datafusion/wasmtest/datafusion-wasm-app/package.json b/datafusion/wasmtest/datafusion-wasm-app/package.json index e9e98f49495f9..f403aa5652962 100644 --- a/datafusion/wasmtest/datafusion-wasm-app/package.json +++ b/datafusion/wasmtest/datafusion-wasm-app/package.json @@ -28,7 +28,7 @@ }, "devDependencies": { "webpack": "5.105.0", - "webpack-cli": "5.1.4", + "webpack-cli": "7.2.2", "webpack-dev-server": "6.0.0", "copy-webpack-plugin": "14.0.0" } From 07d2a45a481039ee7c29830d59115c4a0d3ec592 Mon Sep 17 00:00:00 2001 From: linfeng <33561138+lyne7-sc@users.noreply.github.com> Date: Fri, 14 Aug 2026 05:52:05 +0000 Subject: [PATCH 13/50] fix: preserve NULL semantics in bitwise xor simplification (#24248) ## Which issue does this PR close? - Part of #24246 . ## Rationale for this change Bitwise XOR simplifications cancelled repeated nullable operands, which could incorrectly produce a non-NULL result when the operand was NULL. For example: ```sql SELECT i XOR i, (i XOR 7) XOR i, i XOR (7 XOR i) FROM (VALUES (NULL::INT)) AS t(i); ``` These expressions should all return NULL, but simplification could replace them with 0, 7, and 7. ## What changes are included in this PR? Only cancel repeated XOR operands when the removed operand is non-nullable. ## Are these changes tested? Yes. Added sqllogictests covering results. ## Are there any user-facing changes? Yes. Bitwise XOR expressions with repeated nullable operands now correctly preserve NULLs. --- .../simplify_expressions/expr_simplifier.rs | 88 +++++++++++++------ datafusion/sqllogictest/test_files/scalar.slt | 11 +++ 2 files changed, 73 insertions(+), 26 deletions(-) diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index 2b606687d47a3..b7e9624946e79 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -1372,7 +1372,7 @@ impl TreeNodeRewriter for Simplifier<'_> { left, op: BitwiseXor, right, - }) if expr_contains(&left, &right, BitwiseXor) => { + }) if !info.nullable(&right)? && expr_contains(&left, &right, BitwiseXor) => { let expr = delete_xor_in_complex_expr(&left, &right, false); Transformed::yes(if expr == *right { Expr::Literal( @@ -1389,7 +1389,7 @@ impl TreeNodeRewriter for Simplifier<'_> { left, op: BitwiseXor, right, - }) if expr_contains(&right, &left, BitwiseXor) => { + }) if !info.nullable(&left)? && expr_contains(&right, &left, BitwiseXor) => { let expr = delete_xor_in_complex_expr(&right, &left, true); Transformed::yes(if expr == *left { Expr::Literal( @@ -3020,16 +3020,19 @@ mod tests { // c2 ^ ((c2 ^ (c2 | c1)) ^ (c1 & c2)) --> (c2 | c1) ^ (c1 & c2) let expr = bitwise_xor( - col("c2"), + col("c2_non_null"), bitwise_xor( - bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))), - bitwise_and(col("c1"), col("c2")), + bitwise_xor( + col("c2_non_null"), + bitwise_or(col("c2_non_null"), col("c1")), + ), + bitwise_and(col("c1"), col("c2_non_null")), ), ); let expected = bitwise_xor( - bitwise_or(col("c2"), col("c1")), - bitwise_and(col("c1"), col("c2")), + bitwise_or(col("c2_non_null"), col("c1")), + bitwise_and(col("c1"), col("c2_non_null")), ); assert_eq!(simplify(expr), expected); @@ -3038,18 +3041,24 @@ mod tests { // c2 ^ (c2 ^ (c2 | c1)) ^ ((c1 & c2) ^ c2) --> c2 ^ ((c2 | c1) ^ (c1 & c2)) let expr = bitwise_xor( - col("c2"), + col("c2_non_null"), bitwise_xor( - bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))), - bitwise_xor(bitwise_and(col("c1"), col("c2")), col("c2")), + bitwise_xor( + col("c2_non_null"), + bitwise_or(col("c2_non_null"), col("c1")), + ), + bitwise_xor( + bitwise_and(col("c1"), col("c2_non_null")), + col("c2_non_null"), + ), ), ); let expected = bitwise_xor( - col("c2"), + col("c2_non_null"), bitwise_xor( - bitwise_or(col("c2"), col("c1")), - bitwise_and(col("c1"), col("c2")), + bitwise_or(col("c2_non_null"), col("c1")), + bitwise_and(col("c1"), col("c2_non_null")), ), ); @@ -3060,15 +3069,18 @@ mod tests { let expr = bitwise_xor( bitwise_xor( - bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))), - bitwise_and(col("c1"), col("c2")), + bitwise_xor( + col("c2_non_null"), + bitwise_or(col("c2_non_null"), col("c1")), + ), + bitwise_and(col("c1"), col("c2_non_null")), ), - col("c2"), + col("c2_non_null"), ); let expected = bitwise_xor( - bitwise_or(col("c2"), col("c1")), - bitwise_and(col("c1"), col("c2")), + bitwise_or(col("c2_non_null"), col("c1")), + bitwise_and(col("c1"), col("c2_non_null")), ); assert_eq!(simplify(expr), expected); @@ -3078,23 +3090,47 @@ mod tests { let expr = bitwise_xor( bitwise_xor( - bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))), - bitwise_xor(bitwise_and(col("c1"), col("c2")), col("c2")), + bitwise_xor( + col("c2_non_null"), + bitwise_or(col("c2_non_null"), col("c1")), + ), + bitwise_xor( + bitwise_and(col("c1"), col("c2_non_null")), + col("c2_non_null"), + ), ), - col("c2"), + col("c2_non_null"), ); let expected = bitwise_xor( bitwise_xor( - bitwise_or(col("c2"), col("c1")), - bitwise_and(col("c1"), col("c2")), + bitwise_or(col("c2_non_null"), col("c1")), + bitwise_and(col("c1"), col("c2_non_null")), ), - col("c2"), + col("c2_non_null"), ); assert_eq!(simplify(expr), expected); } + #[test] + fn test_does_not_cancel_nullable_bitwise_xor() { + let nullable = col("c3"); + let other = col("c3_non_null"); + + let expr = bitwise_xor(nullable.clone(), nullable.clone()); + assert_eq!(simplify(expr.clone()), expr); + + let expr = bitwise_xor( + bitwise_xor(nullable.clone(), other.clone()), + nullable.clone(), + ); + assert_eq!(simplify(expr.clone()), expr); + + let expr = bitwise_xor(nullable.clone(), bitwise_xor(other, nullable)); + assert_eq!(simplify(expr.clone()), expr); + } + #[test] fn test_simplify_negated_bitwise_and() { // !c3 & c3 --> 0 @@ -3184,13 +3220,13 @@ mod tests { #[test] fn test_simplify_simple_bitwise_xor() { // c4 ^ c4 -> 0 - let expr = (col("c4")).bitxor(col("c4")); + let expr = (col("c4_non_null")).bitxor(col("c4_non_null")); let expected = lit(0u32); assert_eq!(simplify(expr), expected); // c3 ^ c3 -> 0 - let expr = col("c3").bitxor(col("c3")); + let expr = col("c3_non_null").bitxor(col("c3_non_null")); let expected = lit(0i64); assert_eq!(simplify(expr), expected); diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.slt index 7666b680e16a8..2bedb28a85962 100644 --- a/datafusion/sqllogictest/test_files/scalar.slt +++ b/datafusion/sqllogictest/test_files/scalar.slt @@ -1444,6 +1444,17 @@ select a ^ b, c ^ d, e ^ f from signed_integers; -998 -133 -16 NULL NULL NULL +# repeated nullable operands must not be cancelled +query III rowsort +select + (a XOR b) XOR a AS left_associative, + a XOR (b XOR a) AS right_associative, + a XOR a AS self_xor +from (values (NULL::INT, 7), (3, 7)) as t(a, b); +---- +7 7 0 +NULL NULL NULL + # bitwise xor with other operators query II rowsort select 2 * c - 1 ^ 856 + d + 3, d ^ 7 >> 4 from signed_integers; From 9e9eb6611bae71c78a6ec59f06d385f957fedd90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 05:54:52 +0000 Subject: [PATCH 14/50] chore(deps): bump pygithub from 2.8.1 to 2.9.1 (#24299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [pygithub](https://github.com/pygithub/pygithub) from 2.8.1 to 2.9.1.
Release notes

Sourced from pygithub's releases.

v2.9.1

Bug Fixes

Full Changelog: https://github.com/PyGithub/PyGithub/compare/v2.9.0...v2.9.1

v2.9.0

Notable changes

Lazy PyGithub objects

The notion of lazy objects has been added to some PyGithub classes in version 2.6.0. This release now makes all CompletableGithubObjects optionally lazy (if useful). See PyGithub/PyGithub#3403 for a complete list.

In lazy mode, getting a PyGithub object does not send a request to the GitHub API. Only accessing methods and properties sends the necessary requests to the GitHub API:

# Use lazy mode
g = Github(auth=auth, lazy=True)

these method calls do not send requests to the GitHub API


user = g.get_user("PyGithub")    # get the user
repo = user.get_repo("PyGithub") # get the user's repo
pull = repo.get_pull(3403)       # get a known pull request
issue = pull.as_issue() # turn the pull request into an issue


these method and property calls send requests to Github API


issue.create_reaction("rocket")  # create a reaction
created = repo.created_at        # get property of lazy object repo


once a lazy object has been fetched, all properties are available
(no more requests)


licence = repo.license

All PyGithub classes that implement CompletableGithubObject support lazy mode (if useful). This is only useful for classes that have methods creating, changing, or getting objects.

By default, PyGithub objects are not lazy.

PyGithub objects with a paginated property

The GitHub API has the "feature" of paginated properties. Some objects returned by the API have a property that allows for pagination. Fetching subsequent pages of that property means fetching the entire object (with all other properties) and the specified page of the paginated property. Iterating over the paginated property means fetching all other properties multiple times. Fortunately, the allowed size of each page (per_page is usually 300, in contrast to the "usual" per_page maximum of 100).

Objects with paginated properties:

  • Commit.files
  • Comparison.commits
  • EnterpriseConsumedLicenses.users

This PR makes iterating those paginated properties use the configured per_page setting.

... (truncated)

Changelog

Sourced from pygithub's changelog.

Version 2.9.1 (April 14, 2026)

Bug Fixes ^^^^^^^^^

  • Fix getting release by tag in lazy mode ([#3469](https://github.com/pygithub/pygithub/issues/3469) <https://github.com/PyGithub/PyGithub/pull/3469>) (7d1ba281e <https://github.com/PyGithub/PyGithub/commit/7d1ba281e>)

Version 2.9.0 (March 22, 2026)

Notable changes ^^^^^^^^^^^^^^^

Lazy PyGithub objects """""""""""""""""""""

The notion of lazy objects has been added to some PyGithub classes in version 2.6.0. This release now makes all CompletableGithubObject\s optionally lazy (if useful). See [#3403](https://github.com/pygithub/pygithub/issues/3403) <https://github.com/PyGithub/PyGithub/pull/3403>_ for a complete list.

In lazy mode, getting a PyGithub object does not send a request to the GitHub API. Only accessing methods and properties sends the necessary requests to the GitHub API:

.. code-block:: python

# Use lazy mode
g = Github(auth=auth, lazy=True)

these method calls do not send requests to the GitHub API


user = g.get_user("PyGithub")    # get the user
repo = user.get_repo("PyGithub") # get the user's repo
pull = repo.get_pull(3403)       # get a known pull request
issue = pull.as_issue() # turn the pull request into an issue


these method and property calls send requests to Github API


issue.create_reaction("rocket")  # create a reaction
created = repo.created_at        # get property of lazy object repo


once a lazy object has been fetched, all properties are available
(no more requests)


licence = repo.license

All PyGithub classes that implement CompletableGithubObject support lazy mode (if useful). This is only useful for classes that have methods creating, changing, or getting objects.

By default, PyGithub objects are not lazy.

PyGithub objects with a paginated property """"""""""""""""""""""""""""""""""""""""""

The GitHub API has the "feature" of paginated properties.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=pygithub&package-manager=uv&previous-version=2.8.1&new-version=2.9.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 240f0e56472a7..751ed96fdb0be 100644 --- a/uv.lock +++ b/uv.lock @@ -777,7 +777,7 @@ wheels = [ [[package]] name = "pygithub" -version = "2.8.1" +version = "2.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyjwt", extra = ["crypto"] }, @@ -786,9 +786,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c1/74/e560bdeffea72ecb26cff27f0fad548bbff5ecc51d6a155311ea7f9e4c4c/pygithub-2.8.1.tar.gz", hash = "sha256:341b7c78521cb07324ff670afd1baa2bf5c286f8d9fd302c1798ba594a5400c9", size = 2246994, upload-time = "2025-09-02T17:41:54.674Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ab/c3/8465a311197e16cf5ab68789fe689535e90f6b61ab524cc32a39e67237ae/pygithub-2.9.1.tar.gz", hash = "sha256:59771d7ff63d54d427be2e7d0dad2208dfffc2b0a045fec959263787739b611c", size = 2594989, upload-time = "2026-04-14T07:26:13.622Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/07/ba/7049ce39f653f6140aac4beb53a5aaf08b4407b6a3019aae394c1c5244ff/pygithub-2.8.1-py3-none-any.whl", hash = "sha256:23a0a5bca93baef082e03411bf0ce27204c32be8bfa7abc92fe4a3e132936df0", size = 432709, upload-time = "2025-09-02T17:41:52.947Z" }, + { url = "https://files.pythonhosted.org/packages/77/aa/81a5506f089a26338bff17535e4339b3b22049ebd1bcdeff756c4d7a7559/pygithub-2.9.1-py3-none-any.whl", hash = "sha256:2ec78fca30092d51a42d76f4ddb02131b6f0c666a35dfdf364cf302cdda115b9", size = 449710, upload-time = "2026-04-14T07:26:12.382Z" }, ] [[package]] From f71c4f1e5938c496ea66fc4a1b59909fa7375e37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 06:08:46 +0000 Subject: [PATCH 15/50] chore(deps): bump setuptools from 83.0.0 to 84.0.0 (#24301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [setuptools](https://github.com/pypa/setuptools) from 83.0.0 to 84.0.0.
Changelog

Sourced from setuptools's changelog.

v84.0.0

Features

  • Newline-separated keywords and platformspypa/setuptools#4887old specification <https://peps.python.org/pep-0345/>_ separated items with spaces and the current one uses commas. (#4887)
  • Extensionpypa/distutils#373#5022)
  • The C compiler modules now emit log messages through their own compilers.C.* loggers instead of the distutils root logger, part of decoupling the compilers package from distutils. The logger names are normalized to a stable compilers.C.* prefix so they remain constant as the package migrates toward a standalone compilers.C distribution. (#5266)
  • The C compilers gained a Compiler.call method -- a thin wrapper over subprocess.check_call (with macOS deployment-target env injection) that is the modern replacement for Compiler.spawn. The compilers no longer depend on distutils.spawn, distutils.dir_util, distutils.file_util, distutils._modified, or distutils.util.execute/split_quoted: the generic newer/newer_group and split_quoted helpers are vendored into the compilers package, and Compiler.mkpath/move_file/execute are implemented directly on the standard library (os.makedirs/shutil.move). The methods are retained for backward compatibility. (#5267)
  • The compilers no longer depend on distutils.util, distutils.version, distutils.compat, or distutils._macos_compat. The platform-identification helpers (get_platform/get_host_platform/is_mingw) now live in distutils.compilers.platform.detect and the macOS deployment-target logic and compiler_fixup in distutils.compilers.platform.macos; CygwinCCompiler.gcc_version returns a packaging.version.Version. distutils.util re-exports the platform/macOS helpers from their new homes for backward compatibility rather than keeping duplicate copies. (sysconfig lookups still route through distutils pending its own decoupling.) (#5268)
  • The compilers now read their build configuration from the standard library's sysconfig instead of distutils.sysconfig. Per-compiler customization -- previously distutils.sysconfig.customize_compiler -- has moved into Compiler.configure_system(): a no-op on the base class, with UnixCCompiler applying the compiler/flag/archiver settings CPython recorded in sysconfig (and the usual CC/CFLAGS/LDSHARED/… environment overrides). distutils.sysconfig.customize_compiler is retained as a thin wrapper that calls compiler.configure_system(). (#5269)

Bugfixes

  • The MSVC linker now passes its arguments through a response file when the command line would exceed the Windows maximum length, fixing failures when linking a large number of objects. (#4177)
  • The Cygwin and MinGW compilers now pass -O1 instead of a bare -O. The two are equivalent to GCC, but cc1 rejected the bare form when building 32-bit extensions with -m32. -- by :user:dchaudhari7177 (#4873)
  • copy_filepypa/distutils#379#5079)
  • Setuptools wheels no longer bundled the project's own test modules. -- by :user:itscloud0 (#5212)
  • build_ext no longer fails when cross-compiling with a compiler other than MSVC (such as MinGW). Compiler now provides a no-op initialize()pypa/distutils#399

Improved Documentation

  • Clarified what "correspond exactly to the directory structure" means in the packages section of the Package Discovery user guide. (#4109)
  • Documented how bdist_wheel's py_limited_api option controls abi3 wheel tagging for extension modules -- by :user:Himanshuagrawal4 (#4741)

Deprecations and Removals

  • Compiler.spawn is deprecated in favor of the new Compiler.call. call raises native subprocess exceptions; spawn remains as a shim that emits a DeprecationWarning and translates them to DistutilsExecError. The MSVC spawn compatibility shim for third-party monkeypatches predating the env argument (numpy.distutils before 1.19, per pypa/distutils#15) has been removed. distutils.spawn.spawn is likewise reduced to a thin wrapper around subprocess.check_call: it no longer resolves cmd[0] via shutil.which (subprocess searches PATH itself) nor injects MACOSX_DEPLOYMENT_TARGET (that now lives with the compilers, the only callers to which it applied). (#5267)
  • Building an extension with a MACOSX_DEPLOYMENT_TARGET lower than the interpreter's configured value now raises compilers.errors.PlatformError instead of distutils.errors.DistutilsPlatformError (the macOS deployment-target check moved into the compilers package). CygwinCCompiler.gcc_version returns a packaging.version.Version rather than the removed distutils.version.LooseVersion. Completing the transition begun in pypa/distutils#246, UnixCCompiler.runtime_library_dir_option now returns the ["-Wl,--enable-new-dtags", "-Wl,-rpath,<dir>"] list directly for GNU ld rather than collapsing it into a single string, and the temporary distutils.compat.consolidate_linker_args shim has been removed. (#5268)
  • The compilers now define their own exception vocabulary instead of borrowing distutils' framework errors. Language-agnostic exceptions (Error, UnknownFileType, and a new PlatformError) live at distutils.compilers.errors, leaving room for future compilers.<language> siblings; the C/C++-specific CompileError/LinkError/LibError/PreprocessError remain in distutils.compilers.C.errors. The compilers now raise compilers.errors.PlatformError where they previously raised distutils.errors.DistutilsPlatformError/DistutilsModuleError, and compilers._modified.newer raises the stdlib FileNotFoundError. distutils.errors keeps its own framework exceptions and re-exports the compiler ones (CCompilerError, CompileError, etc.) for backward compatibility; because CCompilerError is compilers.errors.Error, code catching it (as distutils' top-level handlers do) still catches the new PlatformError. (#5270)
  • customize_compiler now asserts that the compiler-related config variables (CC, CXX, CFLAGS, etc.) resolve to strings, raising AssertionError if any are unexpectedly Nonepypa/distutils#363
Commits
  • 72e919a Merge pull request #5293 from pypa/bugfix/integration-pip-flit-backend
  • 1b29701 Select the top-level pyproject.toml when reading build requirements
  • bb1b381 Bump version: 83.0.0 → 84.0.0
  • ee6fdd7 Sync with distutils @ e8eb87855 (#5292)
  • 2a4a9e4 Merge remote-tracking branch 'origin/main' into distutils-e8eb87855
  • cbd1195 Merge https://github.com/jaraco/skeleton
  • bd3594e Merge pull request #5287 from Avasam/Configuring-lint.flake8-comprehensions.a...
  • f02e90a Configure C408 to allow dict(a=1) rather than disabling it
  • c55f52b Configuring lint.flake8-comprehensions.allow-dict-calls-with-keyword-argument...
  • e9904b0 Match the distutils sdist base type for the user_options override
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=setuptools&package-manager=uv&previous-version=83.0.0&new-version=84.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/pyproject.toml b/docs/pyproject.toml index c4edfdcfec38e..48740f2b33308 100644 --- a/docs/pyproject.toml +++ b/docs/pyproject.toml @@ -9,6 +9,6 @@ dependencies = [ "myst-parser>=5.1.0,<6", "maturin>=1.14.1,<2", "jinja2>=3.1.6,<4", - "setuptools>=83.0.0,<84", "sphinx-sitemap>=2.9.0,<3", + "setuptools>=84.0.0,<85", ] diff --git a/uv.lock b/uv.lock index 751ed96fdb0be..40c632d0fd4ef 100644 --- a/uv.lock +++ b/uv.lock @@ -350,7 +350,7 @@ requires-dist = [ { name = "maturin", specifier = ">=1.14.1,<2" }, { name = "myst-parser", specifier = ">=5.1.0,<6" }, { name = "pydata-sphinx-theme", specifier = ">=0.20.0,<1" }, - { name = "setuptools", specifier = ">=83.0.0,<84" }, + { name = "setuptools", specifier = ">=84.0.0,<85" }, { name = "sphinx", specifier = ">=9,<10" }, { name = "sphinx-reredirects", specifier = ">=1.1,<2" }, { name = "sphinx-sitemap", specifier = ">=2.9.0,<3" }, @@ -943,11 +943,11 @@ wheels = [ [[package]] name = "setuptools" -version = "83.0.0" +version = "84.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, + { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" }, ] [[package]] From f2208ff01c0809db65352f23250bea3d6dbe5fe9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 06:14:17 +0000 Subject: [PATCH 16/50] chore(deps): bump rich from 14.3.2 to 15.0.0 (#24305) Bumps [rich](https://github.com/Textualize/rich) from 14.3.2 to 15.0.0.
Release notes

Sourced from rich's releases.

The So Long 3.8 Release

A few fixes. The major version bump is to honor the passing of 3.8 support which reached its EOL in October 7, 2024

[15.0.0] - 2026-04-12

Changed

  • Breaking change: Dropped support for Python3.8

Fixed

The Faster Startup Release

No new features in this release, but there should be improved startup time for Rich apps, and potentially improved runtime if you have a lot of links.

[14.3.4] - 2026-04-11

Changed

The infinite Release

Fixed a infinite loop in split_graphemes

[14.3.3] - 2026-02-19

Fixed

Changelog

Sourced from rich's changelog.

[15.0.0] - 2026-04-12

Changed

  • Breaking change: Dropped support for Python3.8

Fixed

[14.3.4] - 2026-04-11

Changed

[14.3.3] - 2026-02-19

Fixed

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=rich&package-manager=uv&previous-version=14.3.2&new-version=15.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 40c632d0fd4ef..364e884b3fce9 100644 --- a/uv.lock +++ b/uv.lock @@ -921,15 +921,15 @@ wheels = [ [[package]] name = "rich" -version = "14.3.2" +version = "15.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "markdown-it-py" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/74/99/a4cab2acbb884f80e558b0771e97e21e939c5dfb460f488d19df485e8298/rich-14.3.2.tar.gz", hash = "sha256:e712f11c1a562a11843306f5ed999475f09ac31ffb64281f73ab29ffdda8b3b8", size = 230143, upload-time = "2026-02-01T16:20:47.908Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/45/615f5babd880b4bd7d405cc0dc348234c5ffb6ed1ea33e152ede08b2072d/rich-14.3.2-py3-none-any.whl", hash = "sha256:08e67c3e90884651da3239ea668222d19bea7b589149d8014a21c633420dbb69", size = 309963, upload-time = "2026-02-01T16:20:46.078Z" }, + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, ] [[package]] From 8ba5b9c98afe62f06cae3d98e390de50c0d82807 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 06:14:38 +0000 Subject: [PATCH 17/50] chore(deps-dev): bump webpack from 5.105.0 to 5.109.2 in /datafusion/wasmtest/datafusion-wasm-app in the all-npm-deps group across 1 directory (#24300) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the all-npm-deps group with 1 update in the /datafusion/wasmtest/datafusion-wasm-app directory: [webpack](https://github.com/webpack/webpack). Updates `webpack` from 5.105.0 to 5.109.2
Release notes

Sourced from webpack's releases.

v5.109.2

Patch Changes

  • Resolve aliases pointing at a package directory whose name ends with .js again. (by @​alexander-akait in #21542)

  • Name CSS sources in source maps by their resource path, without the css prefix. (by @​bjohansebas in #21536)

  • Delete no longer referenced files from the filesystem cache directory after storing the cache, age them by recorded time so restored caches are cleaned too, and collect every fully expired pack in one store instead of one per build. (by @​bjohansebas in #21528)

  • Report "universal" as the loader context target for the universal target. (by @​alexander-akait in #21540)

  • Skip require().prop in dead branches gated by inlined imported constants. (by @​hai-x in #21517)

  • Annotate configuration options and public hooks in the generated types with the @since JSDoc tag. (by @​bjohansebas in #21473)

v5.109.1

Patch Changes

  • Fix stray semicolon emitted before an imported call following a parenthesized sequence element. (by @​alexander-akait in #21533)

  • Make require(esm) module.exports re-export analysis independent of module processing order. (by @​alexander-akait in #21521)

  • Ignore ERR_SERVER_NOT_RUNNING on lazy-compilation backend dispose so compiler.close() succeeds on Bun. (by @​alexander-akait in #21521)

  • Name the failing key when DefinePlugin fails to evaluate a typeof value. (by @​alexander-akait in #21503)

  • Improve Deno compatibility: guard setNoDelay and force-close connections on lazy-compilation backend dispose, and return a real ArrayBuffer from the Node async/sync wasm loader so WebAssembly.instantiate accepts it. (by @​alexander-akait in #21524)

  • Speed up the HTML parser and cut its peak memory: module-scope helpers/state and tokenizer callbacks, plus exact AST column pre-sizing. (by @​alexander-akait in #21492)

  • Track CommonJS build dependencies by parsing sources when require.cache children are unavailable (e.g. Bun). (by @​alexander-akait in #21531)

  • Cook common string-literal escapes on the JS parser fast path and own the tokenizer's cold-path readers. (by @​alexander-akait in #21500)

  • Build the CSS parseA* AST on the SoA store instead of node classes, cutting parse memory and time. (by @​alexander-akait in #21498)

  • Speed up and cut memory of the experimental CSS and HTML parsers: drop two derivable AST node columns, and scan long string, url, comment, and plaintext token bodies natively. (by @​alexander-akait in #21504)

  • Speed up non-modules CSS parsing: skip redundant token re-reads, drop selector-prelude tokens without materializing nodes, allocate rule preludes lazily, and fast-path empty list seals. (by @​alexander-akait in #21511)

  • Speed up stats generation and cut its peak memory: reuse cached sort comparators instead of thrashing the comparator caches on every sort, and drop redundant module-graph lookups and allocations in the extractors. (by @​alexander-akait in #21506)

  • Speed up CSS parsing: byte-range function-name checks, indexed sibling lookahead. (by @​bjohansebas in #21520)

  • Reduce allocations and redundant work across the code-generation, module-concatenation, exports/usage-analysis, hashing, and chunk-splitting hot paths. (by @​alexander-akait in #21516)

  • Enable the Node.js compile cache in the webpack CLI entry point. (by @​bjohansebas in #21523)

  • Encode the persistent cache with V8's value serializer. (by @​avivkeller in #21514)

... (truncated)

Changelog

Sourced from webpack's changelog.

5.109.2

Patch Changes

  • Resolve aliases pointing at a package directory whose name ends with .js again. (by @​alexander-akait in #21542)

  • Name CSS sources in source maps by their resource path, without the css prefix. (by @​bjohansebas in #21536)

  • Delete no longer referenced files from the filesystem cache directory after storing the cache, age them by recorded time so restored caches are cleaned too, and collect every fully expired pack in one store instead of one per build. (by @​bjohansebas in #21528)

  • Report "universal" as the loader context target for the universal target. (by @​alexander-akait in #21540)

  • Skip require().prop in dead branches gated by inlined imported constants. (by @​hai-x in #21517)

  • Annotate configuration options and public hooks in the generated types with the @since JSDoc tag. (by @​bjohansebas in #21473)

5.109.1

Patch Changes

  • Fix stray semicolon emitted before an imported call following a parenthesized sequence element. (by @​alexander-akait in #21533)

  • Make require(esm) module.exports re-export analysis independent of module processing order. (by @​alexander-akait in #21521)

  • Ignore ERR_SERVER_NOT_RUNNING on lazy-compilation backend dispose so compiler.close() succeeds on Bun. (by @​alexander-akait in #21521)

  • Name the failing key when DefinePlugin fails to evaluate a typeof value. (by @​alexander-akait in #21503)

  • Improve Deno compatibility: guard setNoDelay and force-close connections on lazy-compilation backend dispose, and return a real ArrayBuffer from the Node async/sync wasm loader so WebAssembly.instantiate accepts it. (by @​alexander-akait in #21524)

  • Speed up the HTML parser and cut its peak memory: module-scope helpers/state and tokenizer callbacks, plus exact AST column pre-sizing. (by @​alexander-akait in #21492)

  • Track CommonJS build dependencies by parsing sources when require.cache children are unavailable (e.g. Bun). (by @​alexander-akait in #21531)

  • Cook common string-literal escapes on the JS parser fast path and own the tokenizer's cold-path readers. (by @​alexander-akait in #21500)

  • Build the CSS parseA* AST on the SoA store instead of node classes, cutting parse memory and time. (by @​alexander-akait in #21498)

  • Speed up and cut memory of the experimental CSS and HTML parsers: drop two derivable AST node columns, and scan long string, url, comment, and plaintext token bodies natively. (by @​alexander-akait in #21504)

  • Speed up non-modules CSS parsing: skip redundant token re-reads, drop selector-prelude tokens without materializing nodes, allocate rule preludes lazily, and fast-path empty list seals. (by @​alexander-akait in #21511)

  • Speed up stats generation and cut its peak memory: reuse cached sort comparators instead of thrashing the comparator caches on every sort, and drop redundant module-graph lookups and allocations in the extractors. (by @​alexander-akait in #21506)

  • Speed up CSS parsing: byte-range function-name checks, indexed sibling lookahead. (by @​bjohansebas in #21520)

  • Reduce allocations and redundant work across the code-generation, module-concatenation, exports/usage-analysis, hashing, and chunk-splitting hot paths. (by @​alexander-akait in #21516)

  • Enable the Node.js compile cache in the webpack CLI entry point. (by @​bjohansebas in #21523)

... (truncated)

Commits
  • 6a24bd6 chore(release): new release (#21534)
  • d1b5b6d fix: resolve aliases pointing at a package directory ending in .js (#21542)
  • 9cdcc57 fix(target): report "universal" as the loader target (#21540)
  • 774e7e4 feat: annotate schema options with added keywords (#21473)
  • ff07bdc fix(css): name CSS sources in source maps by their resource path (#21536)
  • d56d82d fix(cache): reclaim stale filesystem cache files reliably (#21539)
  • ae67e2a chore(readme): add codspeed (#21497)
  • 964bbfa fix: delete no longer referenced files from the filesystem cache directory (#...
  • 075d0cd feat: skip require().prop in dead branches (#21517)
  • ec39088 chore(release): new release (#21493)
  • Additional commits viewable in compare view

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .../datafusion-wasm-app/package-lock.json | 390 +++++++----------- .../wasmtest/datafusion-wasm-app/package.json | 2 +- 2 files changed, 161 insertions(+), 231 deletions(-) diff --git a/datafusion/wasmtest/datafusion-wasm-app/package-lock.json b/datafusion/wasmtest/datafusion-wasm-app/package-lock.json index ad4f047bf9689..a43843bd0fe9f 100644 --- a/datafusion/wasmtest/datafusion-wasm-app/package-lock.json +++ b/datafusion/wasmtest/datafusion-wasm-app/package-lock.json @@ -13,7 +13,7 @@ }, "devDependencies": { "copy-webpack-plugin": "14.0.0", - "webpack": "5.105.0", + "webpack": "5.109.2", "webpack-cli": "7.2.2", "webpack-dev-server": "6.0.0" } @@ -364,30 +364,10 @@ "@types/node": "*" } }, - "node_modules/@types/eslint": { - "version": "9.6.1", - "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-9.6.1.tgz", - "integrity": "sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==", - "dev": true, - "dependencies": { - "@types/estree": "*", - "@types/json-schema": "*" - } - }, - "node_modules/@types/eslint-scope": { - "version": "3.7.7", - "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.7.tgz", - "integrity": "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==", - "dev": true, - "dependencies": { - "@types/eslint": "*", - "@types/estree": "*" - } - }, "node_modules/@types/estree": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", - "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", "dev": true }, "node_modules/@types/express": { @@ -686,9 +666,9 @@ } }, "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", "dev": true, "bin": { "acorn": "bin/acorn" @@ -697,18 +677,6 @@ "node": ">=0.4.0" } }, - "node_modules/acorn-import-phases": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/acorn-import-phases/-/acorn-import-phases-1.0.4.tgz", - "integrity": "sha512-wKmbr/DDiIXzEOiWrTTUcDm24kQ2vGfZQvM2fwg2vXqR5uW6aapr7ObPtj1th32b9u90/Pf4AItvdTh42fBmVQ==", - "dev": true, - "engines": { - "node": ">=10.13.0" - }, - "peerDependencies": { - "acorn": "^8.14.0" - } - }, "node_modules/ajv": { "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", @@ -1330,13 +1298,13 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.19.0", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.19.0.tgz", - "integrity": "sha512-phv3E1Xl4tQOShqSte26C7Fl84EwUdZsyOuSSk9qtAGyyQs2s3jJzComh+Abf4g187lUUAvH+H26omrqia2aGg==", + "version": "5.24.5", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.24.5.tgz", + "integrity": "sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==", "dev": true, "dependencies": { "graceful-fs": "^4.2.4", - "tapable": "^2.3.0" + "tapable": "^2.3.3" }, "engines": { "node": ">=10.13.0" @@ -1373,9 +1341,9 @@ } }, "node_modules/es-module-lexer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.0.0.tgz", - "integrity": "sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", + "integrity": "sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==", "dev": true }, "node_modules/es-object-atoms": { @@ -1675,12 +1643,6 @@ "node": ">= 0.4" } }, - "node_modules/glob-to-regexp": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", - "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", - "dev": true - }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -2027,12 +1989,6 @@ "node": ">= 10.13.0" } }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true - }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -2058,19 +2014,6 @@ "shell-quote": "^1.8.4" } }, - "node_modules/loader-runner": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.1.tgz", - "integrity": "sha512-IWqP2SCPhyVFTBtRcgMHdzlf9ul25NwaFx4wCEH/KjAXuuHY4yNjvPXsBokp8jCB936PyWRaPKUNh8NvylLp2Q==", - "dev": true, - "engines": { - "node": ">=6.11.5" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - } - }, "node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -2650,6 +2593,66 @@ "node": ">= 0.6" } }, + "node_modules/minimizer-webpack-plugin": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/minimizer-webpack-plugin/-/minimizer-webpack-plugin-5.6.1.tgz", + "integrity": "sha512-DoeAZz8Q1C1znwsUzej1fdoi4jCf7/+Em27ouLqfK/+3m8G+D7yDhUwrc3CNhjSzGUN1kn7Iv4sWmjflQHenpw==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.25", + "jest-worker": "^27.4.5", + "schema-utils": "^4.3.0", + "terser": "^5.31.1" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@minify-html/node": { + "optional": true + }, + "@swc/core": { + "optional": true + }, + "@swc/css": { + "optional": true + }, + "@swc/html": { + "optional": true + }, + "clean-css": { + "optional": true + }, + "cssnano": { + "optional": true + }, + "csso": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "html-minifier-terser": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "postcss": { + "optional": true + }, + "uglify-js": { + "optional": true + } + } + }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -3497,9 +3500,9 @@ } }, "node_modules/tapable": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.0.tgz", - "integrity": "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", "dev": true, "engines": { "node": ">=6" @@ -3510,9 +3513,9 @@ } }, "node_modules/terser": { - "version": "5.46.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.46.0.tgz", - "integrity": "sha512-jTwoImyr/QbOWFFso3YoU3ik0jBBDJ6JTOQiy/J2YxVJdZCc+5u7skhNwiOR3FQIygFqVUPHl7qbbxtjW2K3Qg==", + "version": "5.50.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.50.0.tgz", + "integrity": "sha512-CN9BVxWhgS/hRxtUMjtC2uRWSTcSfQFHMDWma6sKKfIivCD91sM+FOPfvwoaRMqCSrUpe1nv3jDamd9eEQ4y+w==", "dev": true, "dependencies": { "@jridgewell/source-map": "^0.3.3", @@ -3527,39 +3530,6 @@ "node": ">=10" } }, - "node_modules/terser-webpack-plugin": { - "version": "5.4.0", - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.4.0.tgz", - "integrity": "sha512-Bn5vxm48flOIfkdl5CaD2+1CiUVbonWQ3KQPyP7/EuIl9Gbzq/gQFOzaMFUEgVjB1396tcK0SG8XcNJ/2kDH8g==", - "dev": true, - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.25", - "jest-worker": "^27.4.5", - "schema-utils": "^4.3.0", - "terser": "^5.31.1" - }, - "engines": { - "node": ">= 10.13.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - }, - "peerDependencies": { - "webpack": "^5.1.0" - }, - "peerDependenciesMeta": { - "@swc/core": { - "optional": true - }, - "esbuild": { - "optional": true - }, - "uglify-js": { - "optional": true - } - } - }, "node_modules/thunky": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz", @@ -3756,12 +3726,11 @@ } }, "node_modules/watchpack": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.1.tgz", - "integrity": "sha512-Zn5uXdcFNIA1+1Ei5McRd+iRzfhENPCe7LeABkJtNulSxjma+l7ltNx55BWZkRlwRnpOgHqxnjyaDgJnNXnqzg==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.2.tgz", + "integrity": "sha512-6i/00NBjP4yGPs+caKSyRfpTF/8Torsu0MOW3mMzIbhgISFder8i7xbqgHlLMwJrdiN8ndBV3UA1/AfzPSr+jg==", "dev": true, "dependencies": { - "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.1.2" }, "engines": { @@ -3769,36 +3738,31 @@ } }, "node_modules/webpack": { - "version": "5.105.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz", - "integrity": "sha512-gX/dMkRQc7QOMzgTe6KsYFM7DxeIONQSui1s0n/0xht36HvrgbxtM1xBlgx596NbpHuQU8P7QpKwrZYwUX48nw==", + "version": "5.109.2", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.109.2.tgz", + "integrity": "sha512-U9/cvLzxObKNEZ9+TtdqrHM5/9z3lgl2c+c4BzbqGxFQvQvBAq87yql5A8pQ+rrMbS496MZJeF5enVBndIy2hw==", "dev": true, "dependencies": { - "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", "@types/json-schema": "^7.0.15", "@webassemblyjs/ast": "^1.14.1", "@webassemblyjs/wasm-edit": "^1.14.1", "@webassemblyjs/wasm-parser": "^1.14.1", - "acorn": "^8.15.0", - "acorn-import-phases": "^1.0.3", + "acorn": "^8.16.0", "browserslist": "^4.28.1", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.19.0", - "es-module-lexer": "^2.0.0", + "enhanced-resolve": "^5.24.4", + "es-module-lexer": "^2.1.0", "eslint-scope": "5.1.1", "events": "^3.2.0", - "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.2.11", - "json-parse-even-better-errors": "^2.3.1", - "loader-runner": "^4.3.1", - "mime-types": "^2.1.27", + "mime-db": "^1.54.0", + "minimizer-webpack-plugin": "^5.6.1", "neo-async": "^2.6.2", "schema-utils": "^4.3.3", "tapable": "^2.3.0", - "terser-webpack-plugin": "^5.3.16", - "watchpack": "^2.5.1", - "webpack-sources": "^3.3.3" + "watchpack": "^2.5.2", + "webpack-sources": "^3.5.1" }, "bin": { "webpack": "bin/webpack.js" @@ -3997,14 +3961,23 @@ } }, "node_modules/webpack-sources": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.3.3.tgz", - "integrity": "sha512-yd1RBzSGanHkitROoPFd6qsrxt+oFhg/129YzheDGqeustzX0vTZJZsSsQjVQC4yzBQ56K55XU8gaNCtIzOnTg==", + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.5.1.tgz", + "integrity": "sha512-jyuiGJdtvY434z5bUZrjz67v76/ePNvFZTp9Mdz29IlH4+GPsgyGjiv0fKI+M7BdkU6ADjulUcKAd3tUK3WlEw==", "dev": true, "engines": { "node": ">=10.13.0" } }, + "node_modules/webpack/node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -4419,30 +4392,10 @@ "@types/node": "*" } }, - "@types/eslint": { - "version": "9.6.1", - "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-9.6.1.tgz", - "integrity": "sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==", - "dev": true, - "requires": { - "@types/estree": "*", - "@types/json-schema": "*" - } - }, - "@types/eslint-scope": { - "version": "3.7.7", - "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.7.tgz", - "integrity": "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==", - "dev": true, - "requires": { - "@types/eslint": "*", - "@types/estree": "*" - } - }, "@types/estree": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", - "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", "dev": true }, "@types/express": { @@ -4728,18 +4681,11 @@ } }, "acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", "dev": true }, - "acorn-import-phases": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/acorn-import-phases/-/acorn-import-phases-1.0.4.tgz", - "integrity": "sha512-wKmbr/DDiIXzEOiWrTTUcDm24kQ2vGfZQvM2fwg2vXqR5uW6aapr7ObPtj1th32b9u90/Pf4AItvdTh42fBmVQ==", - "dev": true, - "requires": {} - }, "ajv": { "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", @@ -5157,13 +5103,13 @@ "dev": true }, "enhanced-resolve": { - "version": "5.19.0", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.19.0.tgz", - "integrity": "sha512-phv3E1Xl4tQOShqSte26C7Fl84EwUdZsyOuSSk9qtAGyyQs2s3jJzComh+Abf4g187lUUAvH+H26omrqia2aGg==", + "version": "5.24.5", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.24.5.tgz", + "integrity": "sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==", "dev": true, "requires": { "graceful-fs": "^4.2.4", - "tapable": "^2.3.0" + "tapable": "^2.3.3" } }, "envinfo": { @@ -5185,9 +5131,9 @@ "dev": true }, "es-module-lexer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.0.0.tgz", - "integrity": "sha512-5POEcUuZybH7IdmGsD8wlf0AI55wMecM9rVBTI/qEAy2c1kTOm3DjFYjrBdI2K3BaJjJYfYFeRtM0t9ssnRuxw==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", + "integrity": "sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==", "dev": true }, "es-object-atoms": { @@ -5406,12 +5352,6 @@ "es-object-atoms": "^1.0.0" } }, - "glob-to-regexp": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", - "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", - "dev": true - }, "gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -5639,12 +5579,6 @@ "supports-color": "^8.0.0" } }, - "json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true - }, "json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -5667,12 +5601,6 @@ "shell-quote": "^1.8.4" } }, - "loader-runner": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.1.tgz", - "integrity": "sha512-IWqP2SCPhyVFTBtRcgMHdzlf9ul25NwaFx4wCEH/KjAXuuHY4yNjvPXsBokp8jCB936PyWRaPKUNh8NvylLp2Q==", - "dev": true - }, "locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -5996,6 +5924,18 @@ "mime-db": "1.52.0" } }, + "minimizer-webpack-plugin": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/minimizer-webpack-plugin/-/minimizer-webpack-plugin-5.6.1.tgz", + "integrity": "sha512-DoeAZz8Q1C1znwsUzej1fdoi4jCf7/+Em27ouLqfK/+3m8G+D7yDhUwrc3CNhjSzGUN1kn7Iv4sWmjflQHenpw==", + "dev": true, + "requires": { + "@jridgewell/trace-mapping": "^0.3.25", + "jest-worker": "^27.4.5", + "schema-utils": "^4.3.0", + "terser": "^5.31.1" + } + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -6604,15 +6544,15 @@ "dev": true }, "tapable": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.0.tgz", - "integrity": "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", "dev": true }, "terser": { - "version": "5.46.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.46.0.tgz", - "integrity": "sha512-jTwoImyr/QbOWFFso3YoU3ik0jBBDJ6JTOQiy/J2YxVJdZCc+5u7skhNwiOR3FQIygFqVUPHl7qbbxtjW2K3Qg==", + "version": "5.50.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.50.0.tgz", + "integrity": "sha512-CN9BVxWhgS/hRxtUMjtC2uRWSTcSfQFHMDWma6sKKfIivCD91sM+FOPfvwoaRMqCSrUpe1nv3jDamd9eEQ4y+w==", "dev": true, "requires": { "@jridgewell/source-map": "^0.3.3", @@ -6621,18 +6561,6 @@ "source-map-support": "~0.5.20" } }, - "terser-webpack-plugin": { - "version": "5.4.0", - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.4.0.tgz", - "integrity": "sha512-Bn5vxm48flOIfkdl5CaD2+1CiUVbonWQ3KQPyP7/EuIl9Gbzq/gQFOzaMFUEgVjB1396tcK0SG8XcNJ/2kDH8g==", - "dev": true, - "requires": { - "@jridgewell/trace-mapping": "^0.3.25", - "jest-worker": "^27.4.5", - "schema-utils": "^4.3.0", - "terser": "^5.31.1" - } - }, "thunky": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz", @@ -6751,46 +6679,48 @@ "dev": true }, "watchpack": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.1.tgz", - "integrity": "sha512-Zn5uXdcFNIA1+1Ei5McRd+iRzfhENPCe7LeABkJtNulSxjma+l7ltNx55BWZkRlwRnpOgHqxnjyaDgJnNXnqzg==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.2.tgz", + "integrity": "sha512-6i/00NBjP4yGPs+caKSyRfpTF/8Torsu0MOW3mMzIbhgISFder8i7xbqgHlLMwJrdiN8ndBV3UA1/AfzPSr+jg==", "dev": true, "requires": { - "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.1.2" } }, "webpack": { - "version": "5.105.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.0.tgz", - "integrity": "sha512-gX/dMkRQc7QOMzgTe6KsYFM7DxeIONQSui1s0n/0xht36HvrgbxtM1xBlgx596NbpHuQU8P7QpKwrZYwUX48nw==", + "version": "5.109.2", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.109.2.tgz", + "integrity": "sha512-U9/cvLzxObKNEZ9+TtdqrHM5/9z3lgl2c+c4BzbqGxFQvQvBAq87yql5A8pQ+rrMbS496MZJeF5enVBndIy2hw==", "dev": true, "requires": { - "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", "@types/json-schema": "^7.0.15", "@webassemblyjs/ast": "^1.14.1", "@webassemblyjs/wasm-edit": "^1.14.1", "@webassemblyjs/wasm-parser": "^1.14.1", - "acorn": "^8.15.0", - "acorn-import-phases": "^1.0.3", + "acorn": "^8.16.0", "browserslist": "^4.28.1", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.19.0", - "es-module-lexer": "^2.0.0", + "enhanced-resolve": "^5.24.4", + "es-module-lexer": "^2.1.0", "eslint-scope": "5.1.1", "events": "^3.2.0", - "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.2.11", - "json-parse-even-better-errors": "^2.3.1", - "loader-runner": "^4.3.1", - "mime-types": "^2.1.27", + "mime-db": "^1.54.0", + "minimizer-webpack-plugin": "^5.6.1", "neo-async": "^2.6.2", "schema-utils": "^4.3.3", "tapable": "^2.3.0", - "terser-webpack-plugin": "^5.3.16", - "watchpack": "^2.5.1", - "webpack-sources": "^3.3.3" + "watchpack": "^2.5.2", + "webpack-sources": "^3.5.1" + }, + "dependencies": { + "mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true + } } }, "webpack-cli": { @@ -6891,9 +6821,9 @@ } }, "webpack-sources": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.3.3.tgz", - "integrity": "sha512-yd1RBzSGanHkitROoPFd6qsrxt+oFhg/129YzheDGqeustzX0vTZJZsSsQjVQC4yzBQ56K55XU8gaNCtIzOnTg==", + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.5.1.tgz", + "integrity": "sha512-jyuiGJdtvY434z5bUZrjz67v76/ePNvFZTp9Mdz29IlH4+GPsgyGjiv0fKI+M7BdkU6ADjulUcKAd3tUK3WlEw==", "dev": true }, "which": { diff --git a/datafusion/wasmtest/datafusion-wasm-app/package.json b/datafusion/wasmtest/datafusion-wasm-app/package.json index f403aa5652962..c5723b079b6d0 100644 --- a/datafusion/wasmtest/datafusion-wasm-app/package.json +++ b/datafusion/wasmtest/datafusion-wasm-app/package.json @@ -27,7 +27,7 @@ "datafusion-wasmtest": "../pkg" }, "devDependencies": { - "webpack": "5.105.0", + "webpack": "5.109.2", "webpack-cli": "7.2.2", "webpack-dev-server": "6.0.0", "copy-webpack-plugin": "14.0.0" From 8b265f12c79ecee7624763afd79452d50c5311c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 06:22:33 +0000 Subject: [PATCH 18/50] chore(deps): bump typing-extensions from 4.15.0 to 4.16.0 (#24304) Bumps [typing-extensions](https://github.com/python/typing_extensions) from 4.15.0 to 4.16.0.
Release notes

Sourced from typing-extensions's releases.

4.16.0

No changes since 4.16.0rc2.

Changes since 4.15.0:

  • Make typing_extensions.TypeAliasType's __module__ attribute writable. Backport of CPython PR #149172.
  • Fix setting of __required_keys__ and __optional_keys__ when inheriting keys with the same name.
  • Add support for AsyncIterator, io.Reader, io.Writer and os.PathLike protocols as bases for other protocols.
  • Fix incorrect behaviour on Python 3.9 and Python 3.10 that meant that calling isinstance with typing_extensions.Concatenate[...] or typing_extensions.Unpack[...] as the first argument could have a different result in some situations depending on whether or not a profiling function had been set using sys.setprofile. This affected both CPython and PyPy implementations. Patch by Brian Schubert.
  • Fix __init_subclass__() behavior in the presence of multiple inheritance involving an @deprecated-decorated base class. Backport of CPython PR #138210 by Brian Schubert.
  • Raise TypeError when attempting to subclass typing_extensions.ParamSpec on Python 3.9. The typing implementation has always raised an error, and the typing_extensions implementation has raised an error on Python 3.10+ since typing_extensions v4.6.0. Patch by Brian Schubert.
  • Add the bound, covariant, contravariant, and infer_variance parameters to TypeVarTuple.
  • Officially support the bound, covariant, contravariant and infer_variance parameters to ParamSpec. Improve the validation of these parameters at runtime.
  • Rename typing_extensions.Sentinel to typing_extensions.sentinel, following the name that has been adopted for builtins.sentinel on Python 3.15. typing_extensions.Sentinel is retained as a soft-deprecated alias for backwards compatibility.
  • Add support for pickling sentinels.
  • Sentinels now preserve their identity when copied or deep-copied.
  • Deprecate passing name as a keyword argument or repr as a positional argument to the sentinel constructor.
  • The default repr of a sentinel X = sentinel("X") is now X rather than <X>.
  • Deprecate arbitrary attribute assignments to sentinels.
  • Deprecate subclassing sentinels.
  • Add support for Python 3.15.
  • Avoid a DeprecationWarning when deprecated is applied to a coroutine function on Python 3.14.0.

4.16.0rc2

Changes since 4.16.0rc1:

  • Avoid a DeprecationWarning when deprecated is applied to a coroutine function on Python 3.14.0.

Changes since 4.15.0:

  • Make typing_extensions.TypeAliasType's __module__ attribute writable. Backport of CPython PR #149172.
  • Fix setting of __required_keys__ and __optional_keys__ when inheriting keys with the same name.
  • Add support for AsyncIterator, io.Reader, io.Writer and os.PathLike protocols as bases for other protocols.
  • Fix incorrect behaviour on Python 3.9 and Python 3.10 that meant that calling isinstance with typing_extensions.Concatenate[...] or typing_extensions.Unpack[...] as the first argument could have a different result in some situations depending on whether or not a profiling function had been set using sys.setprofile. This affected both CPython and PyPy implementations. Patch by Brian Schubert.
  • Fix __init_subclass__() behavior in the presence of multiple inheritance involving an @deprecated-decorated base class. Backport of CPython PR #138210 by Brian Schubert.
  • Raise TypeError when attempting to subclass typing_extensions.ParamSpec on Python 3.9. The typing implementation has always raised an error, and the typing_extensions implementation has raised an error on Python 3.10+ since typing_extensions v4.6.0. Patch by Brian Schubert.
  • Add the bound, covariant, contravariant, and infer_variance parameters to TypeVarTuple.
  • Officially support the bound, covariant, contravariant and infer_variance parameters to ParamSpec. Improve the validation of these parameters at runtime.
  • Rename typing_extensions.Sentinel to typing_extensions.sentinel, following the name that has been adopted for builtins.sentinel on Python 3.15. typing_extensions.Sentinel is retained as a soft-deprecated alias for backwards compatibility.
  • Add support for pickling sentinels.
  • Sentinels now preserve their identity when copied or deep-copied.
  • Deprecate passing name as a keyword argument or repr as a positional argument to the sentinel constructor.
  • The default repr of a sentinel X = sentinel("X") is now X rather than <X>.
  • Deprecate arbitrary attribute assignments to sentinels.
  • Deprecate subclassing sentinels.
  • Add support for Python 3.15.

4.16.0rc1

  • Make typing_extensions.TypeAliasType's __module__ attribute writable. Backport of CPython PR #149172.
  • Fix setting of __required_keys__ and __optional_keys__ when inheriting keys with the same name.

... (truncated)

Changelog

Sourced from typing-extensions's changelog.

Release 4.16.0 (July 2, 2025)

No user-facing changes since 4.16.0rc2.

Release 4.16.0rc2 (June 25, 2026)

  • Avoid a DeprecationWarning when deprecated is applied to a coroutine function on Python 3.14.0.

Release 4.16.0rc1 (June 24, 2026)

  • Make typing_extensions.TypeAliasType's __module__ attribute writable. Backport of CPython PR #149172.
  • Fix setting of __required_keys__ and __optional_keys__ when inheriting keys with the same name.
  • Add support for AsyncIterator, io.Reader, io.Writer and os.PathLike protocols as bases for other protocols.
  • Fix incorrect behaviour on Python 3.9 and Python 3.10 that meant that calling isinstance with typing_extensions.Concatenate[...] or typing_extensions.Unpack[...] as the first argument could have a different result in some situations depending on whether or not a profiling function had been set using sys.setprofile. This affected both CPython and PyPy implementations. Patch by Brian Schubert.
  • Fix __init_subclass__() behavior in the presence of multiple inheritance involving an @deprecated-decorated base class. Backport of CPython PR #138210 by Brian Schubert.
  • Raise TypeError when attempting to subclass typing_extensions.ParamSpec on Python 3.9. The typing implementation has always raised an error, and the typing_extensions implementation has raised an error on Python 3.10+ since typing_extensions v4.6.0. Patch by Brian Schubert.
  • Add the bound, covariant, contravariant, and infer_variance parameters to TypeVarTuple.
  • Officially support the bound, covariant, contravariant and infer_variance parameters to ParamSpec. Improve the validation of these parameters at runtime.
  • Rename typing_extensions.Sentinel to typing_extensions.sentinel, following the name that has been adopted for builtins.sentinel on Python 3.15. typing_extensions.Sentinel is retained as a soft-deprecated alias for backwards compatibility.
  • Add support for pickling sentinels.
  • Sentinels now preserve their identity when copied or deep-copied.
  • Deprecate passing name as a keyword argument or repr as a positional argument to the sentinel constructor.
  • The default repr of a sentinel X = sentinel("X") is now X rather than <X>.
  • Deprecate arbitrary attribute assignments to sentinels.
  • Deprecate subclassing sentinels.
  • Add support for Python 3.15.
Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 364e884b3fce9..c93a17173f12d 100644 --- a/uv.lock +++ b/uv.lock @@ -1157,11 +1157,11 @@ wheels = [ [[package]] name = "typing-extensions" -version = "4.15.0" +version = "4.16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]] From 8076f8450194ea18480fc0afb66540dcdcbb293f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 06:22:54 +0000 Subject: [PATCH 19/50] chore(deps): bump requests from 2.33.0 to 2.34.2 (#24303) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [requests](https://github.com/psf/requests) from 2.33.0 to 2.34.2.
Release notes

Sourced from requests's releases.

v2.34.2

2.34.2 (2026-05-14)

  • Moved headers input type back to Mapping to avoid invariance issues with MutableMapping and inferred dict types. Users calling Request.headers.update() may need to narrow typing in their code. (#7441)

Full Changelog: https://github.com/psf/requests/blob/main/HISTORY.md#2342-2026-05-14

v2.34.1

2.34.1 (2026-05-13)

Bugfixes

  • Widened json input type from dict and list to Mapping and Sequence. (#7436)
  • Changed headers input type to MutableMapping and removed None from Request.headers typing to improve handling for users. (#7431)
  • Response.reason moved from str | None to str to improve handling for users. (#7437)
  • Fixed a bug where some bodies with custom __getattr__ implementations weren't being properly detected as Iterables. (#7433)

New Contributors

Full Changelog: https://github.com/psf/requests/blob/main/HISTORY.md#2341-2026-05-13

v2.34.0

2.34.0 (2026-05-11)

Announcements

  • Requests 2.34.0 introduces inline types, replacing those provided by typeshed. Public API types should be fully compatible with mypy, pyright, and ty. We believe types are comprehensive but if you find issues, please report them to the pinned tracking issue.

    Special thanks to @​bastimeyer, @​cthoyt, @​edgarrmondragon, and @​srittau for helping review and test the types ahead of the release. (#7272)

Improvements

  • Digest Auth hashing algorithms have added usedforsecurity=False to clarify security considerations. (#7310)
  • Requests added support for Python 3.15 based on beta1. Downstream projects should be able to start testing prior to its release in October. (#7422)
  • Requests added support for Python 3.14t. (#7419)

Bugfixes

  • Response.history no longer contains a reference to itself, preventing accidental looping when traversing the history list. (#7328)
  • Requests no longer performs greedy matching on no_proxy domains. The

... (truncated)

Changelog

Sourced from requests's changelog.

2.34.2 (2026-05-14)

  • Moved headers input type back to Mapping to avoid invariance issues with MutableMapping and inferred dict types. Users calling Request.headers.update() may need to narrow typing in their code. (#7441)

2.34.1 (2026-05-13)

Bugfixes

  • Widened json input type from dict and list to Mapping and Sequence. (#7436)
  • Changed headers input type to MutableMapping and removed None from Request.headers typing to improve handling for users. (#7431)
  • Response.reason moved from str | None to str to improve handling for users. (#7437)
  • Fixed a bug where some bodies with custom __getattr__ implementations weren't being properly detected as Iterables. (#7433)

2.34.0 (2026-05-11)

Announcements

  • Requests 2.34.0 introduces inline types, replacing those provided by typeshed. Public API types should be fully compatible with mypy, pyright, and ty. We believe types are comprehensive but if you find issues, please report them to the pinned tracking issue.

    Special thanks to @​bastimeyer, @​cthoyt, @​edgarrmondragon, and @​srittau for helping review and test the types ahead of the release. (#7272)

Improvements

  • Digest Auth hashing algorithms have added usedforsecurity=False to clarify security considerations. (#7310)
  • Requests added support for Python 3.15 based on beta1. Downstream projects should be able to start testing prior to its release in October. (#7422)
  • Requests added support for Python 3.14t. (#7419)

Bugfixes

  • Response.history no longer contains a reference to itself, preventing accidental looping when traversing the history list. (#7328)
  • Requests no longer performs greedy matching on no_proxy domains. The proxy_bypass implementation has been updated with CPython's fix from bpo-39057. (#7427)
  • Requests no longer incorrectly strips duplicate leading slashes in URI paths. This should address user issues with specific presigned URLs. Note the full fix requires urllib3 2.7.0+. (#7315)

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=requests&package-manager=uv&previous-version=2.33.0&new-version=2.34.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index c93a17173f12d..61dba57e516d4 100644 --- a/uv.lock +++ b/uv.lock @@ -906,7 +906,7 @@ wheels = [ [[package]] name = "requests" -version = "2.33.0" +version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -914,9 +914,9 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232, upload-time = "2026-03-25T15:10:41.586Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" }, + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] [[package]] From dcf54eae2e3fe806c24208d4dd764e63b5047d04 Mon Sep 17 00:00:00 2001 From: anchor Date: Fri, 14 Aug 2026 06:27:33 +0000 Subject: [PATCH 20/50] docs: fix signum documentation to say zero returns 0 (#24324) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? N/A — documentation-only correction; no open issue tracks it. ## Rationale for this change The user guide for `signum` says "Zero and positive numbers return `1`", but `signum(0)` returns `0` — the documented return value for zero is the opposite of what the function does. The repository's own tests already pin the real behavior: - `datafusion/sqllogictest/test_files/scalar.slt`: `select signum(-2), signum(0), signum(2);` → `-1 0 1` - The unit tests in `signum.rs` assert `signum(±0.0) == 0.0` That behavior is intentional: it matches PostgreSQL `sign()` and Spark `signum()`, and was changed on purpose in #11580 (issue #11557). Only the docs were left describing the pre-#11580 behavior, so this PR updates the docs, not the code. ## What changes are included in this PR? Documentation only: - Update the `#[user_doc]` description in `datafusion/functions/src/math/signum.rs` to state that zero returns `0`. - Regenerate `docs/source/user-guide/sql/scalar_functions.md` via `dev/update_function_docs.sh`. ## Are these changes tested? No new tests: the corrected wording describes behavior the existing tests above already assert, and those are what it was checked against. - `cargo test -p datafusion-functions --lib math::signum` → 2 passed - `cargo fmt --all -- --check` and `./ci/scripts/doc_prettier_check.sh` → clean ## Are there any user-facing changes? Yes, documentation only. No API or runtime behavior change. --- Prepared with AI assistance; every claim above was verified locally against the repository's existing tests before opening this PR. Co-authored-by: codeAnqiang-ma <273298913+codeAnqiang-ma@users.noreply.github.com> --- datafusion/functions/src/math/signum.rs | 3 ++- docs/source/user-guide/sql/scalar_functions.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/datafusion/functions/src/math/signum.rs b/datafusion/functions/src/math/signum.rs index 05b78fcffe2a7..21b38988c0a01 100644 --- a/datafusion/functions/src/math/signum.rs +++ b/datafusion/functions/src/math/signum.rs @@ -34,7 +34,8 @@ use datafusion_macros::user_doc; doc_section(label = "Math Functions"), description = r#"Returns the sign of a number. Negative numbers return `-1`. -Zero and positive numbers return `1`."#, +Zero returns `0`. +Positive numbers return `1`."#, syntax_example = "signum(numeric_expression)", standard_argument(name = "numeric_expression", prefix = "Numeric"), sql_example = r#"```sql diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index 1bfec4ce43599..644d42935ec29 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -796,7 +796,8 @@ round(numeric_expression[, decimal_places]) Returns the sign of a number. Negative numbers return `-1`. -Zero and positive numbers return `1`. +Zero returns `0`. +Positive numbers return `1`. ```sql signum(numeric_expression) From bf568be5abdcb865553c1ed54ecf8b77bbdcc545 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 14 Aug 2026 06:38:30 +0000 Subject: [PATCH 21/50] docs: update proto-models README for the datafusion-common dependency (#24279) `datafusion-proto-models` now depends on `datafusion-common` and hosts the `From` / `TryFrom` conversions between the generated proto types and their `datafusion-common` counterparts, but the crate README still claimed it had no DataFusion dependencies beyond `datafusion-proto-common` and exposed only the generated structs. Describe the conversions and why they live in this crate, and spell out the narrowness that still holds: `datafusion-common` and `datafusion-proto-common` are the only DataFusion dependencies. --------- Co-authored-by: Claude --- datafusion/proto-models/README.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/datafusion/proto-models/README.md b/datafusion/proto-models/README.md index 34adae9c1aaef..6b3b4ba9a2f49 100644 --- a/datafusion/proto-models/README.md +++ b/datafusion/proto-models/README.md @@ -22,9 +22,17 @@ [Apache DataFusion] is an extensible query execution framework, written in Rust, that uses [Apache Arrow] as its in-memory format. This crate contains the [prost]-generated Rust types for DataFusion's logical -and physical plan protobuf schemas. It is intentionally kept narrow: it has no -DataFusion dependencies beyond [`datafusion-proto-common`] and exposes only the -generated structs (and optional [pbjson]/[serde] support). +and physical plan protobuf schemas, plus the `From` / `TryFrom` conversions +between those types and the [`datafusion-common`] types they mirror. The +conversions live here because their DataFusion side sits _below_ this crate in +the dependency graph and so cannot host the impls itself — the same arrangement +[`datafusion-proto-common`] uses for [`ScalarValue`] and [`Statistics`]. + +It is otherwise intentionally kept narrow: its only DataFusion dependencies are +[`datafusion-common`] and [`datafusion-proto-common`], and apart from those +conversions it exposes only the generated structs (and optional +[pbjson]/[serde] support). In particular it does not depend on +`datafusion-expr` or on any of the execution crates. This crate is consumed by [`datafusion-proto`] and may also be depended on directly by other DataFusion crates that need to refer to the proto schema @@ -39,5 +47,8 @@ crate, there is no reason to use this crate directly in your project as well. [prost]: https://docs.rs/prost/latest/prost/ [pbjson]: https://docs.rs/pbjson/latest/pbjson/ [serde]: https://serde.rs/ +[`datafusion-common`]: https://crates.io/crates/datafusion-common [`datafusion-proto`]: https://crates.io/crates/datafusion-proto [`datafusion-proto-common`]: https://crates.io/crates/datafusion-proto-common +[`scalarvalue`]: https://docs.rs/datafusion-common/latest/datafusion_common/scalar/enum.ScalarValue.html +[`statistics`]: https://docs.rs/datafusion-common/latest/datafusion_common/stats/struct.Statistics.html From a1864a86e9dbda37aa09e1a40df80d9177f057cf Mon Sep 17 00:00:00 2001 From: dentiny Date: Fri, 14 Aug 2026 06:42:16 +0000 Subject: [PATCH 22/50] Reduce datafusion-spark dependency footprint (#24351) ## Which issue does this PR close? N/A ## Rationale for this change `datafusion-spark` pulls unnecessary crates into its production dependency graph, increasing compilation time and artifact size for downstream users. ## What changes are included in this PR? - Replace compatibility re-exports with the narrower crates that define `TableFunction` and `FunctionRegistry`. - Remove the unused `crypto_expressions` activation; Spark provides its own SHA-1, SHA-2, and CRC32 implementations. - Fix the `quote` import so builds without the optional `core` feature continue to compile. These changes reduce the production dependency graph from 279 to 257 packages. Cargo Machete does not report these dependencies because they are referenced in source through re-exports or activated through Cargo features. ## Are these changes tested? Yes ## Are there any user-facing changes? No. Public APIs and default behavior remain unchanged. --- Cargo.lock | 3 +-- datafusion/spark/Cargo.toml | 5 ++--- datafusion/spark/src/function/string/quote.rs | 6 ++++-- datafusion/spark/src/function/table/mod.rs | 2 +- datafusion/spark/src/lib.rs | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4cb6117da3fd8..5365be12a1061 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2623,14 +2623,13 @@ dependencies = [ "crc32fast", "criterion", "datafusion", - "datafusion-catalog", "datafusion-common", - "datafusion-execution", "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", "datafusion-functions-nested", + "datafusion-session", "log", "num-traits", "percent-encoding", diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml index 93987b553f2f5..01fd9ec9b3f5b 100644 --- a/datafusion/spark/Cargo.toml +++ b/datafusion/spark/Cargo.toml @@ -49,14 +49,13 @@ chrono = { workspace = true } crc32fast = "1.4" # Optional dependency for SessionStateBuilderSpark extension trait datafusion = { workspace = true, optional = true, default-features = false } -datafusion-catalog = { workspace = true } datafusion-common = { workspace = true } -datafusion-execution = { workspace = true } datafusion-expr = { workspace = true } -datafusion-functions = { workspace = true, features = ["crypto_expressions"] } +datafusion-functions = { workspace = true } datafusion-functions-aggregate = { workspace = true } datafusion-functions-aggregate-common = { workspace = true } datafusion-functions-nested = { workspace = true } +datafusion-session = { workspace = true } log = { workspace = true } num-traits = { workspace = true } percent-encoding = "2.3.2" diff --git a/datafusion/spark/src/function/string/quote.rs b/datafusion/spark/src/function/string/quote.rs index 39ad8bf841764..55d827fb7a280 100644 --- a/datafusion/spark/src/function/string/quote.rs +++ b/datafusion/spark/src/function/string/quote.rs @@ -17,12 +17,14 @@ use arrow::array::{ArrayRef, OffsetSizeTrait, StringArray}; use arrow::datatypes::DataType; -use datafusion::logical_expr::{Coercion, ColumnarValue, Signature, TypeSignatureClass}; use datafusion_common::cast::{as_generic_string_array, as_string_view_array}; use datafusion_common::types::{NativeType, logical_string}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, exec_err}; -use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Volatility}; +use datafusion_expr::{ + Coercion, ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, + TypeSignatureClass, Volatility, +}; use datafusion_functions::utils::make_scalar_function; use std::sync::Arc; diff --git a/datafusion/spark/src/function/table/mod.rs b/datafusion/spark/src/function/table/mod.rs index aba7b7ceb78ea..5ccf8d91df045 100644 --- a/datafusion/spark/src/function/table/mod.rs +++ b/datafusion/spark/src/function/table/mod.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_catalog::TableFunction; +use datafusion_session::TableFunction; use std::sync::Arc; pub mod expr_fn {} diff --git a/datafusion/spark/src/lib.rs b/datafusion/spark/src/lib.rs index 6cd4678da7560..c75f94142ff8a 100644 --- a/datafusion/spark/src/lib.rs +++ b/datafusion/spark/src/lib.rs @@ -42,7 +42,7 @@ //! Spark functions taking priority. //! //! ``` -//! # use datafusion_execution::FunctionRegistry; +//! # use datafusion_expr::registry::FunctionRegistry; //! # use datafusion_expr::{ScalarUDF, AggregateUDF, WindowUDF, HigherOrderUDF}; //! # use datafusion_expr::planner::ExprPlanner; //! # use datafusion_common::Result; @@ -137,10 +137,10 @@ mod session_state; #[cfg(feature = "core")] pub use session_state::SessionStateBuilderSpark; -use datafusion_catalog::TableFunction; use datafusion_common::Result; -use datafusion_execution::FunctionRegistry; +use datafusion_expr::registry::FunctionRegistry; use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF}; +use datafusion_session::TableFunction; use log::debug; use std::sync::Arc; From 4b620004cf00da6582c547430f636eb90b2934fc Mon Sep 17 00:00:00 2001 From: kosiew Date: Fri, 14 Aug 2026 08:39:52 +0000 Subject: [PATCH 23/50] Fix ArrowBytesViewMap retained capacity accounting (#24257) ## Which issue does this PR close? * Part of #23393 ## Rationale for this change `ArrowBytesViewMap::size()` can underreport memory retained by the map. It previously omitted the initial allocation of the hash table, counted `views` by length rather than capacity, counted completed buffers by used length rather than retained capacity, and did not include the backing allocation of the `completed` vector. Because this value is used for memory accounting, it should reflect heap allocations owned by the map while continuing to exclude `self` and external input-array buffers. ## What changes are included in this PR? * Initialize `map_size` from the hash table's initial allocated capacity. * Account for `views` and the in-progress buffer using their allocated sizes. * Include the allocation backing the `completed` vector. * Account for each completed Arrow `Buffer` by retained capacity rather than used length. * Clarify that `size()` excludes both `self` and input-array buffers. ## Are these changes tested? Yes. This PR adds: * `test_size_counts_initial_hash_table_capacity`, which verifies that a newly created map reports the initial hash table allocation. * `test_size_counts_retained_buffer_capacities`, which verifies unused `views` capacity, `completed` vector storage, retained completed-buffer capacity, and that re-inserting duplicate values does not increase the reported size. ## Are there any user-facing changes? No public API or query-result behavior changes. This fixes internal memory accounting so `ArrowBytesViewMap::size()` more accurately reports allocations retained by the map. ## LLM-generated code disclosure This PR includes LLM-generated code and comments. All LLM-generated content has been manually reviewed. --- .../src/binary_view_map.rs | 86 +++++++++++++++++-- 1 file changed, 80 insertions(+), 6 deletions(-) diff --git a/datafusion/physical-expr-common/src/binary_view_map.rs b/datafusion/physical-expr-common/src/binary_view_map.rs index 9d4b556393a24..d05a419c1c93d 100644 --- a/datafusion/physical-expr-common/src/binary_view_map.rs +++ b/datafusion/physical-expr-common/src/binary_view_map.rs @@ -155,10 +155,13 @@ where V: Debug + PartialEq + Eq + Clone + Copy + Default, { pub fn new(output_type: OutputType) -> Self { + let map = hashbrown::hash_table::HashTable::with_capacity(INITIAL_MAP_CAPACITY); + let map_size = map.capacity() * size_of::>(); + Self { output_type, - map: hashbrown::hash_table::HashTable::with_capacity(INITIAL_MAP_CAPACITY), - map_size: 0, + map, + map_size, views: Vec::new(), in_progress: Vec::new(), completed: Vec::new(), @@ -469,11 +472,14 @@ where } /// Return the total size, in bytes, of memory used to store the data in - /// this set, not including `self` + /// this set, not including `self` or input-array buffers. pub fn size(&self) -> usize { - let views_size = self.views.len() * size_of::(); - let in_progress_size = self.in_progress.capacity(); - let completed_size: usize = self.completed.iter().map(|b| b.len()).sum(); + // All fields below own their allocations. Count retained capacity rather + // than used length because this value drives memory accounting. + let views_size = self.views.allocated_size(); + let in_progress_size = self.in_progress.allocated_size(); + let completed_size = self.completed.allocated_size() + + self.completed.iter().map(Buffer::capacity).sum::(); let nulls_size = self.nulls.allocated_size(); self.map_size @@ -715,6 +721,74 @@ mod tests { assert_eq!(set.len(), 10); } + #[test] + fn test_size_counts_initial_hash_table_capacity() { + let map = ArrowBytesViewMap::<()>::new(OutputType::Utf8View); + + assert_eq!(map.size(), map.map.capacity() * size_of::>()); + } + + #[test] + fn test_size_counts_retained_buffer_capacities() { + let first = "a".repeat(BYTE_VIEW_MAX_BLOCK_SIZE / 2 + 1); + let second = "b".repeat(BYTE_VIEW_MAX_BLOCK_SIZE / 2 + 1); + let third = "c".repeat(BYTE_VIEW_MAX_BLOCK_SIZE / 2 + 1); + let values: ArrayRef = Arc::new(StringViewArray::from(vec![ + first.as_str(), + second.as_str(), + third.as_str(), + ])); + + let mut map = ArrowBytesViewMap::new(OutputType::Utf8View); + map.insert_if_new(&values, |_| (), |_| {}); + + // Make unused vector capacity explicit; the completed buffers were created + // by the map's flush path. + map.views.shrink_to_fit(); + map.views.reserve_exact(1); + map.completed.shrink_to_fit(); + map.completed.reserve_exact(1); + + // The map owns these allocations; `values` and its Arrow buffers remain external. + assert!(map.views.capacity() > map.views.len()); + assert!(map.completed.capacity() > map.completed.len()); + assert!( + map.completed + .iter() + .any(|buffer| buffer.capacity() > buffer.len()) + ); + + let expected_size = map.map_size + + map.views.allocated_size() + + map.in_progress.allocated_size() + + map.completed.allocated_size() + + map.completed.iter().map(Buffer::capacity).sum::() + + map.nulls.allocated_size() + + map.hashes_buffer.allocated_size(); + assert_eq!(map.size(), expected_size); + + // Verify the retained-capacity delta independently from the production formula. + let legacy_size = map.map_size + + map.views.len() * size_of::() + + map.in_progress.capacity() + + map.completed.iter().map(Buffer::len).sum::() + + map.nulls.allocated_size() + + map.hashes_buffer.allocated_size(); + let retained_capacity_delta = (map.views.capacity() - map.views.len()) + * size_of::() + + map.completed.capacity() * size_of::() + + map + .completed + .iter() + .map(|buffer| buffer.capacity() - buffer.len()) + .sum::(); + assert_eq!(map.size() - legacy_size, retained_capacity_delta); + + let size_after_insert = map.size(); + map.insert_if_new(&values, |_| (), |_| {}); + assert_eq!(map.size(), size_after_insert); + } + #[derive(Debug, PartialEq, Eq, Default, Clone, Copy)] struct TestPayload { // store the string value to check against input From 92f7b1f93e4ccc95f1a3e098d02c1c0fe3cba337 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Fri, 14 Aug 2026 10:43:32 +0000 Subject: [PATCH 24/50] =?UTF-8?q?fix(parquet):=20sync=20rg=5Fplan=20to=20d?= =?UTF-8?q?ecoder=20frontier=20=E2=80=94=20fix=20wrong=20TopK=20results=20?= =?UTF-8?q?from=20re-reading=20already-delivered=20row=20groups=20(#24352)?= =?UTF-8?q?=20(#24354)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #24352. ## Rationale for this change With `datafusion.execution.parquet.pushdown_filters = true` and TopK dynamic filter pushdown (both on by default), a query of the shape `SELECT b FROM t WHERE ORDER BY b LIMIT k` can silently return **wrong results** — one source row emitted several times and the true tail of the top-k missing — with no error or warning. Root cause (thanks to @hhhizzz's very detailed report + fixture in #24352): a row group whose post-predicate selection is empty is silently finished by arrow-rs **without handing back a reader**. `PushDecoderStreamState` pops its `rg_plan` **only** when a reader is returned, so after a silently-finished RG the plan trails the decoder by one. When the runtime row-group pruner then rebuilds the decoder (`into_builder().with_row_groups(...)`) from the stale `rg_plan`, it re-includes an already-delivered row group, whose rows are emitted a second time and displace the genuine top-k in the heap. ## What changes are included in this PR? - `push_decoder.rs`: before each boundary prune/rebuild, `rg_plan` is synced to the row group the decoder will actually emit next via `peek_next_row_group()` (`sync_rg_plan_to_decoder_frontier` / `advance_rg_plan_to`), dropping entries for silently-finished row groups so a rebuild can never re-include a delivered group. A rebuild frontier naming an RG not in the plan is now an internal error instead of a silent plan drain. ## Are these changes tested? - Adds @hhhizzz's fixture as an slt regression test in `dynamic_row_group_pruning.slt` (filter column `search_phrase` differs from the sort column `event_time`, one row group has an empty post-predicate selection invisible to statistics). It now returns the correct `p0 p4096 p4097 … p4104` (was the buggy `p0 p4096 p4096 …`). - clippy clean; `datasource-parquet` unit tests and the sqllogictest suite pass locally. ## Are there any user-facing changes? Fixes silently-wrong query results; no API change. ## Note This is the standalone bug fix extracted from #23696 (per review discussion in #24352): the same `rg_plan` ↔ decoder-frontier sync, on its own so it merges fast and is easy to backport. #23696 will rebase on top so it carries only the fully-matched `RowFilter` skip performance optimization. cc @alamb @adriangb @hhhizzz --- .../parquet/dynamic_row_group_pruning.rs | 115 +++++++++++++++++- .../datasource-parquet/src/push_decoder.rs | 89 +++++++++++++- .../test_files/dynamic_row_group_pruning.slt | 84 +++++++++++++ 3 files changed, 286 insertions(+), 2 deletions(-) diff --git a/datafusion/core/tests/parquet/dynamic_row_group_pruning.rs b/datafusion/core/tests/parquet/dynamic_row_group_pruning.rs index d5d648be9b7aa..917faaaa5ce64 100644 --- a/datafusion/core/tests/parquet/dynamic_row_group_pruning.rs +++ b/datafusion/core/tests/parquet/dynamic_row_group_pruning.rs @@ -32,7 +32,7 @@ use std::sync::Arc; -use arrow::array::{ArrayRef, Int64Array, RecordBatch}; +use arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; use crate::parquet::Unit::RowGroup; @@ -585,3 +585,116 @@ async fn dynamic_rg_pruning_fires_for_multi_column_sort_leading_tied() { output.description(), ); } + +/// Build the #24352 fixture: four 2048-row row groups where the filter column +/// (`search_phrase`) differs from the sort column (`event_time`), and one row +/// group (the second) has an empty post-predicate selection invisible to +/// statistics — its only small `event_time` (50) sits on the row whose +/// `search_phrase` is `''`. +/// +/// RG 0: event_time = i*1000 (i in 0..2048) +/// RG 1: i=2048 -> (50, ''), else (20000+i, 'p'||i) (i in 2048..4096) +/// RG 2: event_time = 100 + (i-4096) (i in 4096..6144) +/// RG 3: event_time = 5000 + (i-6144) (i in 6144..8192) +fn build_q26_batches(schema: &Arc) -> Vec { + (0..4i64) + .map(|rg| { + let mut event_time = Vec::with_capacity(2048); + let mut search_phrase: Vec = Vec::with_capacity(2048); + for j in 0..2048i64 { + let i = rg * 2048 + j; + let (et, sp) = if i < 2048 { + (i * 1000, format!("p{i}")) + } else if i < 4096 { + if i == 2048 { + (50, String::new()) + } else { + (20000 + i, format!("p{i}")) + } + } else if i < 6144 { + (100 + (i - 4096), format!("p{i}")) + } else { + (5000 + (i - 6144), format!("p{i}")) + }; + event_time.push(et); + search_phrase.push(sp); + } + RecordBatch::try_new( + Arc::clone(schema), + vec![ + Arc::new(Int64Array::from(event_time)) as ArrayRef, + Arc::new(StringArray::from(search_phrase)) as ArrayRef, + ], + ) + .unwrap() + }) + .collect() +} + +/// Regression for #24352: with `pushdown_filters` + TopK dynamic filter, a row +/// group whose post-predicate selection is empty is silently finished by +/// arrow-rs without handing back a reader. Before `rg_plan` was synced to the +/// decoder frontier (`peek_next_row_group`), it trailed the decoder by one, so +/// a later runtime prune rebuilt the decoder from a stale plan and re-read an +/// already-delivered row group — the duplicate rows displaced the true top-k. +#[tokio::test] +async fn topk_pushdown_does_not_reread_delivered_row_group() { + let schema = Arc::new(Schema::new(vec![ + Field::new("event_time", DataType::Int64, false), + Field::new("search_phrase", DataType::Utf8, false), + ])); + let batches = build_q26_batches(&schema); + + // `RowGroup(2048)` writes one row group per 2048-row batch (4 RGs) and + // enables `pushdown_filters`, required for the dynamic filter to reach the + // parquet scan. + let mut ctx = ContextWithParquet::with_custom_data( + Scenario::Int, + RowGroup(2048), + Arc::clone(&schema), + batches, + ) + .await; + + let output = ctx + .query( + "SELECT search_phrase FROM t \ + WHERE search_phrase <> '' ORDER BY event_time LIMIT 10", + ) + .await; + + // `search_phrase` is unique per row, so any repeated value is the same + // source row emitted twice. The correct answer is the 10 smallest- + // `event_time` non-empty phrases, matching DuckDB / pushdown-off. + assert_eq!(output.result_rows, 10, "{}", output.description()); + + // The test must actually exercise the runtime prune/rebuild path that + // caused #24352 (not just a happy-path scan), otherwise a future default or + // optimizer change could let it pass without the bug's precondition. Assert + // the dynamic filter pruned at least one row group. + let pruned = output + .row_groups_pruned_dynamic_filter() + .expect("`row_groups_pruned_dynamic_filter` metric must be registered"); + assert!( + pruned >= 1, + "test must exercise dynamic RG pruning (the #24352 path); pruned={pruned}\n{}", + output.description(), + ); + + let formatted = output.pretty_results(); + for p in [ + "p0", "p4096", "p4097", "p4098", "p4099", "p4100", "p4101", "p4102", "p4103", + "p4104", + ] { + assert!( + formatted.contains(&format!("| {p} ")), + "missing {p} from top-k; got:\n{formatted}", + ); + } + // The bug emitted p4096 twice (and dropped p4101..=p4104); assert no dup. + assert_eq!( + formatted.matches("| p4096 ").count(), + 1, + "p4096 emitted more than once — rg_plan/decoder desync; got:\n{formatted}", + ); +} diff --git a/datafusion/datasource-parquet/src/push_decoder.rs b/datafusion/datasource-parquet/src/push_decoder.rs index 14904bada2cfc..74d8997198872 100644 --- a/datafusion/datasource-parquet/src/push_decoder.rs +++ b/datafusion/datasource-parquet/src/push_decoder.rs @@ -53,7 +53,7 @@ use parquet::arrow::async_reader::AsyncFileReader; use parquet::arrow::push_decoder::{ParquetPushDecoder, ParquetPushDecoderBuilder}; use parquet::file::metadata::ParquetMetaData; -use datafusion_common::{DataFusionError, Result}; +use datafusion_common::{DataFusionError, Result, internal_err}; use datafusion_physical_expr::expressions::DynamicFilterTracking; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; use datafusion_physical_plan::metrics::{BaselineMetrics, Count, Gauge}; @@ -342,6 +342,20 @@ impl PushDecoderStreamState { .as_ref() .expect("decoder present") .is_at_row_group_boundary(); + // Only the runtime pruner rebuilds the decoder from `rg_plan`, so + // only it needs `rg_plan` kept in sync with the decoder frontier. + // arrow-rs silently finishes row groups whose post-predicate + // selection is empty without handing back a reader, so without this + // sync `rg_plan` trails the decoder by one and a rebuild re-reads an + // already-delivered row group (#24352). Gating on the pruner also + // avoids the O(remaining row groups) cost of `peek_next_row_group()` + // on ordinary scans that never rebuild. + if at_boundary + && self.row_group_pruner.is_some() + && let Err(e) = self.sync_rg_plan_to_decoder_frontier() + { + return Some((Err(e), self)); + } if at_boundary && !self.rg_plan.is_empty() { let mut pruned_count = 0usize; if let Some(pruner) = self.row_group_pruner.as_mut() { @@ -414,6 +428,51 @@ impl PushDecoderStreamState { } } + /// Keep `rg_plan.front()` aligned with the row group the decoder will emit + /// next. `try_next_reader` silently finishes row groups whose post-predicate + /// selection is empty (no reader handed back), which would otherwise leave + /// `rg_plan` trailing the decoder by one — a later prune/rebuild would then + /// re-include an already-delivered row group (#24352). + fn sync_rg_plan_to_decoder_frontier(&mut self) -> Result<()> { + match self + .decoder + .as_ref() + .expect("decoder present") + .peek_next_row_group() + .map_err(DataFusionError::from)? + { + Some(actual) => Self::advance_rg_plan_to(&mut self.rg_plan, actual)?, + // Decoder has nothing left to emit — drain our plan so the stream + // finishes cleanly. + None => self.rg_plan.clear(), + } + Ok(()) + } + + /// Pop entries off `rg_plan` until its front is `target`. + /// + /// `target` is the RG the decoder will emit next and must still be in the + /// plan. A missing `target` means the decoder's frontier and `rg_plan` have + /// diverged; we surface that as an internal error rather than silently + /// draining the plan, which would truncate the scan. Kept free-standing on + /// `rg_plan` (rather than `&mut self`) so the pop/guard logic is + /// unit-testable without constructing a full stream state. + fn advance_rg_plan_to( + rg_plan: &mut VecDeque, + target: usize, + ) -> Result<()> { + while let Some(front) = rg_plan.front() { + if front.rg_index == target { + return Ok(()); + } + rg_plan.pop_front(); + } + internal_err!( + "push decoder frontier RG {target} is not in rg_plan; \ + decoder and plan have diverged" + ) + } + /// Copies metrics from ArrowReaderMetrics (the metrics collected by the /// arrow-rs parquet reader) to the parquet file metrics for DataFusion fn copy_arrow_reader_metrics(&self) { @@ -607,4 +666,32 @@ mod tests { assert!(!pruner.should_prune(&[1])); assert!(!pruner.should_prune(&[2])); } + + #[test] + fn advance_rg_plan_to_pops_up_to_target() { + let mut plan: VecDeque = [0usize, 1, 2, 3] + .into_iter() + .map(|rg_index| RgPlanEntry { rg_index }) + .collect(); + PushDecoderStreamState::advance_rg_plan_to(&mut plan, 2).unwrap(); + assert_eq!( + plan.iter().map(|e| e.rg_index).collect::>(), + vec![2, 3], + "must pop the entries before `target` and stop at it", + ); + } + + #[test] + fn advance_rg_plan_to_errors_when_target_absent() { + let mut plan: VecDeque = [0usize, 1, 2] + .into_iter() + .map(|rg_index| RgPlanEntry { rg_index }) + .collect(); + let err = PushDecoderStreamState::advance_rg_plan_to(&mut plan, 5) + .expect_err("a target absent from the plan must be an internal error"); + assert!( + err.to_string().contains("diverged"), + "expected a divergence internal error, got: {err}", + ); + } } diff --git a/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt b/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt index 2149cacfc0a55..81d9511839725 100644 --- a/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt +++ b/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt @@ -110,3 +110,87 @@ RESET datafusion.execution.parquet.pushdown_filters; statement ok RESET datafusion.explain.analyze_level; + +# Regression test for #24352: TopK dynamic filter + `pushdown_filters` must not +# re-read an already-delivered row group. The filter column (`search_phrase`) +# differs from the sort column (`event_time`), and one row group has an empty +# post-predicate selection that row-group statistics cannot see — its only small +# `event_time` (50) sits on the row where `search_phrase = ''`. arrow-rs finishes +# that RG without handing back a reader; without syncing `rg_plan` to the decoder +# frontier via `peek_next_row_group`, `rg_plan` trailed the decoder by one, so a +# later runtime prune rebuilt the decoder from a stale plan and re-read an +# already-delivered RG — duplicating rows and dropping the true top-k tail. +statement ok +set datafusion.execution.parquet.pushdown_filters = true; + +statement ok +set datafusion.execution.target_partitions = 1; + +# Both dynamic-filter switches are on by default; set them explicitly so this +# test keeps exercising the prune/rebuild path even if the defaults change. +statement ok +set datafusion.optimizer.enable_dynamic_filter_pushdown = true; + +statement ok +set datafusion.optimizer.enable_topk_dynamic_filter_pushdown = true; + +statement ok +CREATE TABLE q26_src AS +SELECT + CAST(CASE + WHEN i < 2048 THEN i * 1000 + WHEN i < 4096 THEN (CASE WHEN i = 2048 THEN 50 ELSE 20000 + i END) + WHEN i < 6144 THEN 100 + (i - 4096) + ELSE 5000 + (i - 6144) + END AS BIGINT) AS event_time, + CASE WHEN i = 2048 THEN '' ELSE 'p' || CAST(i AS VARCHAR) END AS search_phrase +FROM generate_series(0, 8191) AS t(i); + +statement ok +COPY (SELECT * FROM q26_src) +TO 'test_files/scratch/dynamic_row_group_pruning/q26.parquet' +STORED AS PARQUET +OPTIONS ('format.max_row_group_size' '2048'); + +statement ok +drop table q26_src; + +statement ok +CREATE EXTERNAL TABLE q26 (event_time BIGINT NOT NULL, search_phrase VARCHAR NOT NULL) +STORED AS PARQUET +LOCATION 'test_files/scratch/dynamic_row_group_pruning/q26.parquet'; + +# Each search_phrase is unique, so any repeated value would be the same source +# row emitted twice. The result must be the 10 smallest-`event_time` non-empty +# phrases with no duplicates (matches DuckDB and pushdown-off DataFusion). +query T +SELECT search_phrase FROM q26 WHERE search_phrase <> '' ORDER BY event_time LIMIT 10; +---- +p0 +p4096 +p4097 +p4098 +p4099 +p4100 +p4101 +p4102 +p4103 +p4104 + +statement ok +drop table q26; + +statement ok +RESET datafusion.execution.parquet.pushdown_filters; + +# The SLT runner sets `target_partitions` to 4 instead of using the default, so +# restore it explicitly rather than RESET (which would revert to the system +# default = num_cpus and leak modified config out of this file). +statement ok +set datafusion.execution.target_partitions = 4; + +statement ok +RESET datafusion.optimizer.enable_dynamic_filter_pushdown; + +statement ok +RESET datafusion.optimizer.enable_topk_dynamic_filter_pushdown; From aacf2ce571cd10521e671a26a9a599577f05c1ed Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 14 Aug 2026 11:31:36 +0000 Subject: [PATCH 25/50] fix: correct list field inner type in array functions (#24345) ## Which issue does this PR close? - Closes #24341 ## Rationale for this change There is a regression from 54.1.0 to 55.0.0rc2 where the spark function `slice` can fail as demonstrated in the updated unit test, and as described in the connected issue. ## What changes are included in this PR? In the nested functions, get the inner field directly from the input for array slicing operations. Also, since it is possible the inner list could be non-nullable emit an empty slice instead of a null child element for the outer lists's nulls. ## Are these changes tested? Added 6 new tests in the SLT suite. ## Are there any user-facing changes? None --- datafusion/functions-nested/src/extract.rs | 36 ++++++++++++++----- .../test_files/array/array_pop.slt | 33 +++++++++++++++++ .../test_files/array/array_slice.slt | 31 ++++++++++++++++ .../test_files/spark/array/slice.slt | 6 ++++ 4 files changed, 97 insertions(+), 9 deletions(-) diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs index 9d367f0161fdd..8f2ea1f40dcb2 100644 --- a/datafusion/functions-nested/src/extract.rs +++ b/datafusion/functions-nested/src/extract.rs @@ -23,9 +23,8 @@ use arrow::array::{ }; use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow::datatypes::DataType; -use arrow::datatypes::{ - DataType::{FixedSizeList, LargeList, LargeListView, List, ListView, Null}, - Field, +use arrow::datatypes::DataType::{ + FixedSizeList, LargeList, LargeListView, List, ListView, Null, }; use datafusion_common::cast::as_large_list_array; use datafusion_common::cast::as_list_array; @@ -622,9 +621,23 @@ where let values = array.values(); let original_data = values.to_data(); let capacity = Capacities::Array(original_data.len()); + // Carry the input's list field through to the output so that the returned + // type matches the one promised by `return_type` / `return_field_from_args`, + // including the field name, nullability and metadata. + let field = match array.data_type() { + List(field) | LargeList(field) => Arc::clone(field), + other => { + return internal_err!( + "general_array_slice got unexpected data type: {other}" + ); + } + }; + // `use_nulls` is false because we never call `try_extend_nulls`: null rows are + // emitted as empty slices. Arrow still allocates a validity buffer on its own + // if the child array has nulls. let mut mutable = - MutableArrayData::with_capacities(vec![&original_data], true, capacity); + MutableArrayData::with_capacities(vec![&original_data], false, capacity); // We have the slice syntax compatible with DuckDB v0.8.1. // The rule `adjusted_from_index` and `adjusted_to_index` follows the rule of array_slice in duckdb. @@ -638,9 +651,11 @@ where let end = offset_window[1]; let len = end - start; + // The row is null, so its contents are never observed. Emit an empty + // slice rather than a null child element: the input's list field may be + // non-nullable, in which case a null child would be invalid. if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) { - mutable.try_extend_nulls(1)?; - offsets.push(offsets[row_index] + O::usize_as(1)); + offsets.push(offsets[row_index]); continue; } @@ -682,7 +697,7 @@ where let data = mutable.freeze(); Ok(Arc::new(GenericListArray::::try_new( - Arc::new(Field::new_list_field(array.value_type(), true)), + field, OffsetBuffer::::new(offsets.into()), arrow::array::make_array(data), nulls, @@ -704,12 +719,15 @@ where let field = match array.data_type() { ListView(field) | LargeListView(field) => Arc::clone(field), other => { - return internal_err!("array_slice got unexpected data type: {}", other); + return internal_err!( + "general_list_view_array_slice got unexpected data type: {other}" + ); } }; + // See the note on `use_nulls` in `general_array_slice`. let mut mutable = - MutableArrayData::with_capacities(vec![&original_data], true, capacity); + MutableArrayData::with_capacities(vec![&original_data], false, capacity); // We must build `offsets` and `sizes` buffers manually as ListView does not enforce // monotonically increasing offsets. diff --git a/datafusion/sqllogictest/test_files/array/array_pop.slt b/datafusion/sqllogictest/test_files/array/array_pop.slt index a72e566b9e7ab..0b7ebf75f4b2b 100644 --- a/datafusion/sqllogictest/test_files/array/array_pop.slt +++ b/datafusion/sqllogictest/test_files/array/array_pop.slt @@ -318,5 +318,38 @@ select array_pop_front(arrow_cast([1, 2], 'LargeListView(Int64)')); ---- [2] +# maintains inner nullability +query ??TT +select + array_pop_front(column1), + array_pop_back(column1), + arrow_typeof(array_pop_front(column1)), + arrow_typeof(array_pop_back(column1)) +from values + (arrow_cast([], 'List(non-null Int32)')), + (arrow_cast(NULL, 'List(non-null Int32)')), + (arrow_cast([1, 3, 5, -5], 'List(non-null Int32)')) +; +---- +[] [] List(non-null Int32) List(non-null Int32) +NULL NULL List(non-null Int32) List(non-null Int32) +[3, 5, -5] [1, 3, 5] List(non-null Int32) List(non-null Int32) + +query ??TT +select + array_pop_front(column1), + array_pop_back(column1), + arrow_typeof(array_pop_front(column1)), + arrow_typeof(array_pop_back(column1)) +from values + (arrow_cast([], 'LargeList(non-null Int32)')), + (arrow_cast(NULL, 'LargeList(non-null Int32)')), + (arrow_cast([1, 3, 5, -5], 'LargeList(non-null Int32)')) +; +---- +[] [] LargeList(non-null Int32) LargeList(non-null Int32) +NULL NULL LargeList(non-null Int32) LargeList(non-null Int32) +[3, 5, -5] [1, 3, 5] LargeList(non-null Int32) LargeList(non-null Int32) + include ./cleanup.slt.part diff --git a/datafusion/sqllogictest/test_files/array/array_slice.slt b/datafusion/sqllogictest/test_files/array/array_slice.slt index 14587a50b2266..76b81b28efc58 100644 --- a/datafusion/sqllogictest/test_files/array/array_slice.slt +++ b/datafusion/sqllogictest/test_files/array/array_slice.slt @@ -450,6 +450,37 @@ NULL NULL [1, 3, 5] +# maintains inner nullability +query ?T +select array_slice(column1, 2, 3), arrow_typeof(array_slice(column1, 2, 3)) +from values + (arrow_cast([], 'List(non-null Int32)')), + (arrow_cast(NULL, 'List(non-null Int32)')), + (arrow_cast([1, 3, 5, -5], 'List(non-null Int32)')) +; +---- +[] List(non-null Int32) +NULL List(non-null Int32) +[3, 5] List(non-null Int32) + +query ?T +select array_slice(column1, 2, 3), arrow_typeof(array_slice(column1, 2, 3)) +from values + (arrow_cast([], 'LargeList(non-null Int32)')), + (arrow_cast(NULL, 'LargeList(non-null Int32)')), + (arrow_cast([1, 3, 5, -5], 'LargeList(non-null Int32)')) +; +---- +[] LargeList(non-null Int32) +NULL LargeList(non-null Int32) +[3, 5] LargeList(non-null Int32) + +query ?T +select array_slice(column1, 2, 3, 2), arrow_typeof(array_slice(column1, 2, 3, 2)) +from values (arrow_cast([1, 3, 5, -5], 'List(non-null Int32)')); +---- +[3] List(non-null Int32) + # Testing with empty arguments should result in an error query error DataFusion error: Error during planning: 'array_slice' does not support zero arguments select array_slice(); diff --git a/datafusion/sqllogictest/test_files/spark/array/slice.slt b/datafusion/sqllogictest/test_files/spark/array/slice.slt index aaf4aa4909dfd..f6fb431a0769b 100644 --- a/datafusion/sqllogictest/test_files/spark/array/slice.slt +++ b/datafusion/sqllogictest/test_files/spark/array/slice.slt @@ -152,3 +152,9 @@ query ? SELECT slice(make_array(1), 3, 4) ---- [] + +# the inner field name of the input list is preserved +query ?T +SELECT slice(array(1, 2, 3, 4), 2, 2), arrow_typeof(slice(array(1, 2, 3, 4), 2, 2)); +---- +[2, 3] List(Int64, field: 'element') From b08666c4967c220f49784fb9da076e88e15c173e Mon Sep 17 00:00:00 2001 From: Subham Singhal Date: Fri, 14 Aug 2026 12:32:09 +0000 Subject: [PATCH 26/50] feat(pwmj): support LeftSemi/LeftAnti existence joins via classic scan (#23870) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this close? Part of #17427 (Make `PiecewiseMergeJoin` work in DataFusion). Adds `LeftSemi` / `LeftAnti` support, one of the epic's checklist items. Supersedes the stale #18392, taking the alternative approach that @2010YOUY01 suggested there (reuse the classic join path for generality) rather than a dedicated existence stream. ## Rationale for this change An inequality-correlated `EXISTS` / `NOT EXISTS` (e.g. `WHERE EXISTS (SELECT 1 FROM r WHERE l.x < r.y)`) has no equi-key, so it decorrelates to a `LeftSemi` / `LeftAnti` join with a single range predicate. Today `PiecewiseMergeJoinExec` rejects existence joins (`not_impl_err!`) and these queries fall back to `NestedLoopJoinExec`, which is O(n*m). Microbenchmark (20K × 20K rows, single inequality, `enable_piecewise_merge_join` on vs off), added in this PR as `piecewise_merge_join_semi_anti`: | Case | PWMJ | NestedLoopJoin | Speedup | |------------------------------|----------|----------------|---------| | LeftSemi, high selectivity | ~0.50 ms | ~75 ms | ~150× | | LeftAnti, high selectivity | ~0.44 ms | ~75 ms | ~170× | | LeftSemi, low selectivity | ~0.48 ms | ~76 ms | ~158× | | LeftAnti, low selectivity | ~0.48 ms | ~75 ms | ~155× | ## What changes are included in this PR? **Existence joins (`LeftSemi` / `LeftAnti`) for `PiecewiseMergeJoin`:** - Route `LeftSemi` / `LeftAnti` with a single range predicate to `PiecewiseMergeJoinExec` in the physical planner (still gated behind `enable_piecewise_merge_join`, default `false`). - Reuse the classic scan: on the first match, mark the matching buffered-row suffix in the visited bitmap and emit from it in the final pass (`LeftSemi` = marked rows, `LeftAnti` = unmarked; NULL join keys are never marked, so they are correctly excluded from Semi and included in Anti). Only left-side columns are produced. - `RightSemi` / `RightAnti` / `Mark` remain unsupported (they require swapping the inputs); they are still rejected in `try_new` and excluded in the planner. Left as a follow-up. - Two small optimizations so existence marking stays O(buffered), not O(n*m): stop scanning a batch after the first match (the marked suffix is maximal), and a cross-batch low-water mark so later batches only mark not-yet-marked rows. - Updated the operator docstring to describe the implemented classic-reuse path and keep the min/max fast-path as a documented follow-up. ## Are these changes tested? Yes. - 33 unit tests in `classic_join.rs` covering `LeftSemi` / `LeftAnti` across `<`, `<=`, `>`, `>=`; NULL join keys; all-null streamed side; empty inputs; `Date32` keys; multi-batch and multi-partition streamed inputs; and the low-water-mark skip branch. - End-to-end SLT coverage in `pwmj.slt` for `EXISTS` / `NOT EXISTS` (including NULLs) with `EXPLAIN` assertions confirming the plan uses `PiecewiseMergeJoin`. - The multi-partition test also guards the final-pass counter fix. ## Are there any user-facing changes? No behaviour change by default: `enable_piecewise_merge_join` remains `false`. When enabled, single-range-predicate `LeftSemi` / `LeftAnti` joins are planned as `PiecewiseMergeJoin` instead of `NestedLoopJoin`. No API changes. --- datafusion/core/src/physical_planner.rs | 8 +- datafusion/core/tests/fuzz_cases/join_fuzz.rs | 223 ++++- .../src/joins/piecewise_merge_join/exec.rs | 77 +- .../piecewise_merge_join/existence_join.rs | 819 +++++++++++++++++ .../src/joins/piecewise_merge_join/mod.rs | 1 + .../src/joins/piecewise_merge_join/utils.rs | 14 +- datafusion/sqllogictest/test_files/pwmj.slt | 822 ++++++++++++++++++ 7 files changed, 1939 insertions(+), 25 deletions(-) create mode 100644 datafusion/physical-plan/src/joins/piecewise_merge_join/existence_join.rs diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 6e68ed9ce69b3..6e50ec8aa4298 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -1602,11 +1602,13 @@ impl DefaultPhysicalPlanner { Arc::new(CrossJoinExec::new(physical_left, physical_right)) } else if num_range_filters == 1 && total_filters == 1 + // PWMJ supports classic joins and Left Semi/Anti existence joins. + // Right Semi/Anti and Mark joins are not implemented yet (they + // would require swapping the inputs so the marked side is buffered), + // so exclude them here and let them fall back to NestedLoopJoin. && !matches!( join_type, - JoinType::LeftSemi - | JoinType::RightSemi - | JoinType::LeftAnti + JoinType::RightSemi | JoinType::RightAnti | JoinType::LeftMark | JoinType::RightMark diff --git a/datafusion/core/tests/fuzz_cases/join_fuzz.rs b/datafusion/core/tests/fuzz_cases/join_fuzz.rs index 81c7c9f83928e..fce6999fd0d77 100644 --- a/datafusion/core/tests/fuzz_cases/join_fuzz.rs +++ b/datafusion/core/tests/fuzz_cases/join_fuzz.rs @@ -34,18 +34,24 @@ use datafusion::physical_plan::collect; use datafusion::physical_plan::expressions::Column; use datafusion::physical_plan::joins::utils::{ColumnIndex, JoinFilter}; use datafusion::physical_plan::joins::{ - HashJoinExec, NestedLoopJoinExec, PartitionMode, SortMergeJoinExec, + HashJoinExec, NestedLoopJoinExec, PartitionMode, PiecewiseMergeJoinExec, + SortMergeJoinExec, }; +use datafusion::physical_plan::sorts::sort::SortExec; +use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties, common}; use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_common::{NullEquality, ScalarValue}; +use datafusion_common_runtime::SpawnedTask; use datafusion_execution::TaskContext; use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; use datafusion_execution::runtime_env::RuntimeEnvBuilder; use datafusion_physical_expr::PhysicalExprRef; use datafusion_physical_expr::expressions::Literal; +use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; use itertools::Itertools; use rand::Rng; +use rand::{SeedableRng, rngs::StdRng}; use test_utils::stagger_batch_with_seed; // Determines what Fuzz tests needs to run @@ -1347,3 +1353,218 @@ fn make_staggered_batches_binary( // preserve your existing randomized partitioning stagger_batch_with_seed(batch, 42) } + +// ---- Differential fuzz: PiecewiseMergeJoin existence joins vs NestedLoopJoin ---- +// +// `PiecewiseMergeJoinExec` takes a range predicate and no equi keys, so it cannot be added to +// `JoinFuzzTestCase` above (that harness joins on `a` and `b` and folds the equality into the +// NestedLoopJoin filter). These tests use `NestedLoopJoinExec` with an equivalent filter as +// the oracle instead. +// +// What only randomization reaches: `LeftSemi`/`LeftAnti` record matches in a shared +// `AtomicUsize` watermark and emit once, from whichever streamed partition finishes last. The +// streamed side below is spread round-robin over several partitions as one-row batches, so +// batches arrive in an order no static test pins down, and the counter that gates the final +// pass is seeded from a partition count that deliberately disagrees with the `num_partitions` +// argument. + +fn pwmj_kv_schema() -> Arc { + Arc::new(Schema::new(vec![ + arrow::datatypes::Field::new("id", arrow::datatypes::DataType::Int32, false), + arrow::datatypes::Field::new("k", arrow::datatypes::DataType::Int32, true), + ])) +} + +fn pwmj_kv_batch(ids: &[i32], keys: &[Option]) -> RecordBatch { + RecordBatch::try_new( + pwmj_kv_schema(), + vec![ + Arc::new(Int32Array::from(ids.to_vec())), + Arc::new(Int32Array::from(keys.to_vec())), + ], + ) + .unwrap() +} + +/// Single-partition, single-batch input: the buffered side, and the oracle's probe side. +fn pwmj_single_exec(ids: &[i32], keys: &[Option]) -> Arc { + MemorySourceConfig::try_new_exec( + &[vec![pwmj_kv_batch(ids, keys)]], + pwmj_kv_schema(), + None, + ) + .unwrap() +} + +/// Streamed side spread round-robin across `nparts` partitions, one row per batch. +fn pwmj_parts_exec( + ids: &[i32], + keys: &[Option], + nparts: usize, +) -> Arc { + let nparts = nparts.max(1); + let mut partitions: Vec> = vec![Vec::new(); nparts]; + for (row, (&id, key)) in ids.iter().zip(keys.iter()).enumerate() { + partitions[row % nparts].push(pwmj_kv_batch(&[id], &[*key])); + } + for p in partitions.iter_mut() { + if p.is_empty() { + p.push(pwmj_kv_batch(&[], &[])); + } + } + MemorySourceConfig::try_new_exec(&partitions, pwmj_kv_schema(), None).unwrap() +} + +fn pwmj_existence_plan( + left: Arc, + right: Arc, + op: Operator, + join_type: JoinType, +) -> Arc { + // Matches `PiecewiseMergeJoinExec::required_input_ordering`: descending for `<`/`<=`, + // ascending for `>`/`>=`, NULLs first either way. + let sort_options = match op { + Operator::Lt | Operator::LtEq => SortOptions::new(true, true), + Operator::Gt | Operator::GtEq => SortOptions::new(false, true), + other => panic!("not a range operator: {other:?}"), + }; + let ordering = LexOrdering::new(vec![PhysicalSortExpr::new( + Arc::new(Column::new("k", 1)), + sort_options, + )]) + .unwrap(); + let sorted_left = Arc::new(SortExec::new(ordering, left)); + let on: (PhysicalExprRef, PhysicalExprRef) = + (Arc::new(Column::new("k", 1)), Arc::new(Column::new("k", 1))); + // `num_partitions` is 1 while the streamed side has up to 3: the final-pass counter must + // come from the streamed side's partition count, not from this argument. + Arc::new( + PiecewiseMergeJoinExec::try_new(sorted_left, right, on, op, join_type, 1) + .unwrap(), + ) +} + +fn pwmj_nlj_oracle_plan( + left: Arc, + right: Arc, + op: Operator, + join_type: JoinType, +) -> Arc { + let intermediate_schema = Schema::new(vec![ + arrow::datatypes::Field::new("k", arrow::datatypes::DataType::Int32, true), + arrow::datatypes::Field::new("k", arrow::datatypes::DataType::Int32, true), + ]); + let expr = Arc::new(BinaryExpr::new( + Arc::new(Column::new("k", 0)), + op, + Arc::new(Column::new("k", 1)), + )) as PhysicalExprRef; + let column_indices = vec![ + ColumnIndex { + index: 1, + side: JoinSide::Left, + }, + ColumnIndex { + index: 1, + side: JoinSide::Right, + }, + ]; + let filter = JoinFilter::new(expr, column_indices, Arc::new(intermediate_schema)); + Arc::new( + NestedLoopJoinExec::try_new(left, right, Some(filter), &join_type, None).unwrap(), + ) +} + +/// Executes every output partition concurrently and returns the surviving left `id`s, sorted. +/// +/// Concurrent rather than one partition at a time: the partitions share the watermark and race +/// to be the one that runs the final pass, which is the part a sequential drain cannot reach. +async fn pwmj_collect_ids( + plan: Arc, + task_ctx: Arc, +) -> Vec { + let streams = (0..plan.output_partitioning().partition_count()) + .map(|partition| plan.execute(partition, Arc::clone(&task_ctx)).unwrap()) + .collect::>(); + let per_partition = + futures::future::join_all(streams.into_iter().map(|stream| { + SpawnedTask::spawn(async move { common::collect(stream).await }) + })) + .await; + + let mut ids = Vec::new(); + for batches in per_partition { + for batch in batches.unwrap().unwrap() { + let col = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + ids.extend((0..col.len()).map(|i| col.value(i))); + } + } + ids.sort(); + ids +} + +#[tokio::test(flavor = "multi_thread")] +async fn fuzz_pwmj_existence_matches_nested_loop() { + // A small batch size splits the final-pass output across several coalesced batches even + // for these tiny inputs, covering that boundary too. + let task_ctx = Arc::new( + TaskContext::default() + .with_session_config(SessionConfig::new().with_batch_size(3)), + ); + let ops = [Operator::Lt, Operator::LtEq, Operator::Gt, Operator::GtEq]; + + for seed in 0..60u64 { + let mut rng = StdRng::seed_from_u64(seed); + let left_len = rng.random_range(0..25usize); + let right_len = rng.random_range(0..25usize); + // A narrow key range forces duplicates and equal-boundary cases. + let key_range = rng.random_range(1..6i32); + let nparts = rng.random_range(1..4usize); + + let gen_keys = |n: usize, rng: &mut StdRng| -> Vec> { + (0..n) + .map(|_| (!rng.random_bool(0.2)).then(|| rng.random_range(0..key_range))) + .collect() + }; + + let left_ids: Vec = (0..left_len as i32).collect(); + let left_keys = gen_keys(left_len, &mut rng); + let right_ids: Vec = (0..right_len as i32).collect(); + let right_keys = gen_keys(right_len, &mut rng); + + for op in ops { + for join_type in [JoinType::LeftSemi, JoinType::LeftAnti] { + let got = pwmj_collect_ids( + pwmj_existence_plan( + pwmj_single_exec(&left_ids, &left_keys), + pwmj_parts_exec(&right_ids, &right_keys, nparts), + op, + join_type, + ), + Arc::clone(&task_ctx), + ) + .await; + let want = pwmj_collect_ids( + pwmj_nlj_oracle_plan( + pwmj_single_exec(&left_ids, &left_keys), + pwmj_single_exec(&right_ids, &right_keys), + op, + join_type, + ), + Arc::clone(&task_ctx), + ) + .await; + + assert_eq!( + got, want, + "mismatch seed={seed} op={op:?} join_type={join_type:?} \ + nparts={nparts} left_keys={left_keys:?} right_keys={right_keys:?}" + ); + } + } + } +} diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs index c42ec67ef80d5..a91903934ac9c 100644 --- a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs +++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs @@ -47,8 +47,10 @@ use crate::execution_plan::{EmissionType, boundedness_from_children}; use crate::joins::piecewise_merge_join::classic_join::{ ClassicPWMJStream, PiecewiseMergeJoinStreamState, }; +use crate::joins::piecewise_merge_join::existence_join::ExistencePWMJStream; use crate::joins::piecewise_merge_join::utils::{ build_visited_indices_map, is_existence_join, is_right_existence_join, + is_supported_existence_join, }; use crate::joins::utils::asymmetric_join_output_partitioning; use crate::metrics::MetricsSet; @@ -164,17 +166,15 @@ use crate::{ /// ``` /// /// ## Existence Joins (Semi, Anti, Mark) -/// Existence joins are made magnitudes of times faster with a `PiecewiseMergeJoin` as we only need to find -/// the min/max value of the streamed side to be able to emit all matches on the buffered side. By putting -/// the side we need to mark onto the sorted buffer side, we can emit all these matches at once. +/// Currently only `LeftSemi` and `LeftAnti` are supported. For these the marked side is +/// already the left (buffered) side, so no input swap is needed. The rest are rejected in +/// [`Self::try_new`]: `RightSemi`/`RightAnti`/`RightMark` mark the right side and need an +/// input swap, and `LeftMark` needs an extra boolean column rather than a filtered slice. /// -/// For less than operations (`<`) both inputs are to be sorted in descending order and vice versa for greater -/// than (`>`) operations. `SortExec` is used to enforce sorting on the buffered side and streamed side does not -/// need to be sorted due to only needing to find the min/max. -/// -/// For Left Semi, Anti, and Mark joins we swap the inputs so that the marked side is on the buffered side. -/// -/// The pseudocode for the algorithm looks like this: +/// `LeftSemi`/`LeftAnti` are served by a dedicated stream, `ExistencePWMJStream` (see +/// `existence_join.rs`). Instead of materializing row pairs it records the matched set as a +/// single index -- the start of the matched suffix of the buffered side -- and slices the +/// buffered batch at that index once every streamed partition has been consumed. /// /// ```text /// // Using the example of a less than `<` operation @@ -296,10 +296,12 @@ impl PiecewiseMergeJoinExec { join_type: JoinType, num_partitions: usize, ) -> Result { - // TODO: Implement existence joins for PiecewiseMergeJoin - if is_existence_join(join_type) { + // Left Semi/Anti are handled by `ExistencePWMJStream` (the marked side is + // already the buffered side, so no input swap is needed). Right existence joins + // and Mark joins are not yet supported. + if is_existence_join(join_type) && !is_supported_existence_join(join_type) { return not_impl_err!( - "Existence Joins are currently not supported for PiecewiseMergeJoin" + "Existence join {join_type} is currently not supported for PiecewiseMergeJoin" ); } @@ -506,7 +508,12 @@ impl ExecutionPlan for PiecewiseMergeJoinExec { fn required_input_ordering(&self) -> Vec> { // Existence joins don't need to be sorted on one side. if is_right_existence_join(self.join_type) { - unimplemented!() + // Unreachable: `try_new` rejects right existence joins, and this signature + // cannot return `Result`. They swap the inputs, so whoever implements them + // must require the order on the streamed side instead. + unimplemented!( + "required_input_ordering for right existence joins; guarded by try_new" + ) } else { // Sort the right side in memory, so we do not need to enforce any sorting vec![ @@ -603,6 +610,14 @@ impl ExecutionPlan for PiecewiseMergeJoinExec { let on_streamed = Arc::clone(&self.on.1); let metrics = BuildProbeJoinMetrics::new(partition, &self.metrics); + // The final pass over unmatched/existence rows must run exactly once, on the + // last streamed partition to finish. That is coordinated by an atomic counter + // seeded with the number of streamed partitions that will actually call + // `execute`, which is the streamed side's output partition count — not the + // planner's `target_partitions` (they can differ, e.g. when the streamed input + // has a single partition), otherwise the counter never reaches 1 and the final + // pass is skipped. + let streamed_partitions = self.streamed.output_partitioning().partition_count(); let buffered_fut = self.buffered_fut.try_once(|| { let reservation = MemoryConsumer::new("PiecewiseMergeJoinInput") .register(context.memory_pool()); @@ -614,7 +629,7 @@ impl ExecutionPlan for PiecewiseMergeJoinExec { metrics.clone(), reservation, build_visited_indices_map(self.join_type), - self.num_partitions, + streamed_partitions, )) })?; @@ -622,9 +637,27 @@ impl ExecutionPlan for PiecewiseMergeJoinExec { let batch_size = context.session_config().batch_size(); - // TODO: Add existence joins + this is guarded at physical planner - if is_existence_join(self.join_type()) { - unreachable!() + let buffered_side = + BufferedSide::Initial(BufferedSideInitialState { buffered_fut }); + + if is_supported_existence_join(self.join_type) { + Ok(Box::pin(ExistencePWMJStream::try_new( + Arc::clone(&self.schema), + on_streamed, + self.join_type, + self.operator, + streamed, + buffered_side, + self.sort_options, + metrics, + batch_size, + ))) + } else if is_existence_join(self.join_type) { + // Right existence joins and Mark joins are rejected in `try_new`. + internal_err!( + "PiecewiseMergeJoin does not support existence join {} (should have been rejected in try_new)", + self.join_type + ) } else { Ok(Box::pin(ClassicPWMJStream::try_new( Arc::clone(&self.schema), @@ -632,7 +665,7 @@ impl ExecutionPlan for PiecewiseMergeJoinExec { self.join_type, self.operator, streamed, - BufferedSide::Initial(BufferedSideInitialState { buffered_fut }), + buffered_side, PiecewiseMergeJoinStreamState::WaitBufferedSide, self.sort_options, metrics, @@ -745,6 +778,11 @@ pub(super) struct BufferedSideData { values: ArrayRef, pub(super) visited_indices_bitmap: SharedBitmapBuilder, pub(super) remaining_partitions: AtomicUsize, + /// Existence joins only: the start of the matched suffix of the buffered side, or + /// `usize::MAX` before the first match. `[existence_min_marked, len)` *is* the matched + /// set -- no bitmap is allocated. Shared so each partition benefits from what the + /// others have marked; it only ever decreases, so a stale read is safe. + pub(super) existence_min_marked: AtomicUsize, _reservation: MemoryReservation, } @@ -761,6 +799,7 @@ impl BufferedSideData { values, visited_indices_bitmap, remaining_partitions: AtomicUsize::new(remaining_partitions), + existence_min_marked: AtomicUsize::new(usize::MAX), _reservation: reservation, } } diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/existence_join.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/existence_join.rs new file mode 100644 index 0000000000000..b2c5212999f3b --- /dev/null +++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/existence_join.rs @@ -0,0 +1,819 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! PiecewiseMergeJoin stream specialized for existence joins. +//! +//! Instantiated by [`PiecewiseMergeJoinExec`] when the join type is `LeftSemi` or `LeftAnti`. +//! The other existence joins are rejected in `PiecewiseMergeJoinExec::try_new`: +//! `RightSemi`/`RightAnti`/`RightMark` mark the right side and so need an input swap, while +//! `LeftMark` marks the left side but needs an extra boolean column rather than a slice. +//! +//! # Motivation +//! +//! `ClassicPWMJStream` (see `classic_join.rs`) materializes `(buffered, streamed)` row +//! pairs: on the first matching buffered row it emits the whole matching suffix joined +//! with the current streamed row. Existence joins only need a boolean per buffered row — +//! does any match exist? — so pair materialization is pure waste, and the resume state it +//! requires (partial output batches, per-row buffered/streamed cursors) is dead weight. +//! +//! This stream instead records matches as a single index -- the start of the matched +//! suffix -- and emits nothing at all while scanning. Output is produced once, at the end, +//! by slicing the buffered batch at that index. +//! +//! # Algorithm +//! +//! The buffered (left) side arrives globally sorted, enforced by `required_input_ordering` +//! (the streamed side carries no ordering requirement). For a streamed key, a binary search +//! finds the first matching buffered row; because the buffered side is sorted, every buffered +//! row from that position onward matches too: +//! +//! ```text +//! buffered (sorted): [1, 3, 5, 7] +//! ▲ +//! first match for this streamed row +//! → mark [1..4), i.e. buffered 3, 5, 7 +//! ``` +//! +//! Marking that suffix covers every match the batch can produce, so only one row of the +//! batch is ever compared against the buffered side: the extreme key, which reaches the +//! smallest matching `buffer_idx`, while every other row matches a subset of that suffix. +//! +//! The search stops at `min_marked`: rows from there on were already marked, by this +//! partition or another, so a match found among them would record nothing. Marking is then a +//! single `fetch_min`, and finding the first match is a binary search, so a batch costs +//! `O(log buffered)` rather than `O(buffered)`. +//! +//! `min_marked` lives in the shared buffered data, so partitions benefit from each other's +//! marking rather than each rediscovering it. It only ever decreases, so a partition that +//! reads a stale value scans a wider range than it had to -- never a narrower one. +//! +//! Once `min_marked` reaches the first non-null buffered row nothing can ever be marked +//! again, and every partition stops reading the streamed side -- checked before each poll, +//! so a partition that starts late reads nothing at all. +//! +//! Rows whose join key is NULL never satisfy a comparison predicate. Buffered NULLs sort to +//! the front, so the scan starts past them and null-keyed buffered rows are left unmarked — +//! correctly excluded from `LeftSemi` and included in `LeftAnti`. The extreme key is picked +//! from each streamed batch with NULLs ignored, so it is non-null unless the whole batch is. +//! +//! # Output +//! +//! Marking only ever covers a suffix, and each mark lowers the watermark to its own start, +//! so the matched set is always exactly `[min_marked, buffered_len)`. A bitmap would be a +//! less compact encoding of that one index, so none is allocated (see +//! `build_visited_indices_map`). +//! +//! Once every streamed partition has been consumed, the last one to finish slices the +//! buffered batch: `LeftSemi` takes `[min_marked, len)`, `LeftAnti` the complementary +//! prefix `[0, min_marked)`, which is where the null-keyed rows live. Only the buffered +//! (left) columns are produced. +//! +//! [`PiecewiseMergeJoinExec`]: super::PiecewiseMergeJoinExec + +use std::cmp::Ordering; +use std::sync::Arc; +use std::sync::atomic::Ordering as AtomicOrdering; +use std::task::{Poll, ready}; + +use arrow::array::{Array, ArrayRef, RecordBatch}; +use arrow::compute::BatchCoalescer; +use arrow_schema::{SchemaRef, SortOptions}; +use datafusion_common::{NullEquality, Result, internal_err}; +use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream}; +use datafusion_expr::{JoinType, Operator}; +use datafusion_functions_aggregate_common::min_max::{max_batch, min_batch}; +use datafusion_physical_expr::PhysicalExprRef; +use futures::{Stream, StreamExt}; + +use crate::handle_state; +use crate::joins::piecewise_merge_join::exec::{BufferedSide, BufferedSideReadyState}; +use crate::joins::utils::{ + BuildProbeJoinMetrics, JoinKeyComparator, StatefulStreamResult, +}; +use crate::stream::EmptyRecordBatchStream; + +pub(super) enum ExistencePWMJStreamState { + /// Load the buffered side into memory. + WaitBufferedSide, + /// Fetch and scan streamed batches, lowering the watermark. Emits nothing. + ScanStreamBatches, + /// Emit the result. Reached only by the last streamed partition. + EmitMatched, + Completed, +} + +pub(super) struct ExistencePWMJStream { + /// Output schema, which for `LeftSemi`/`LeftAnti` is the buffered side's schema + schema: SchemaRef, + /// Physical expression evaluated on the streamed side. The buffered side's + /// equivalent is already evaluated when the buffered side is collected. + on_streamed: PhysicalExprRef, + /// `LeftSemi` or `LeftAnti` + join_type: JoinType, + /// Comparison operator + operator: Operator, + streamed: SendableRecordBatchStream, + buffered_side: BufferedSide, + state: ExistencePWMJStreamState, + /// Whether the buffered side is sorted ascending or descending, per the operator + sort_option: SortOptions, + join_metrics: BuildProbeJoinMetrics, + /// Chunks the single final-pass batch to `batch_size` + output_batches: Box, + /// Whether the final pass has already pushed its result into `output_batches` + emitted: bool, +} + +impl ExistencePWMJStream { + #[expect(clippy::too_many_arguments)] + pub(super) fn try_new( + schema: SchemaRef, + on_streamed: PhysicalExprRef, + join_type: JoinType, + operator: Operator, + streamed: SendableRecordBatchStream, + buffered_side: BufferedSide, + sort_option: SortOptions, + join_metrics: BuildProbeJoinMetrics, + batch_size: usize, + ) -> Self { + Self { + output_batches: Box::new(BatchCoalescer::new( + Arc::clone(&schema), + batch_size, + )), + schema, + on_streamed, + join_type, + operator, + streamed, + buffered_side, + state: ExistencePWMJStreamState::WaitBufferedSide, + sort_option, + join_metrics, + emitted: false, + } + } + + fn poll_next_impl( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> Poll>> { + loop { + return match self.state { + ExistencePWMJStreamState::WaitBufferedSide => { + handle_state!(ready!(self.collect_buffered_side(cx))) + } + ExistencePWMJStreamState::ScanStreamBatches => { + handle_state!(ready!(self.scan_stream_batch(cx))) + } + ExistencePWMJStreamState::EmitMatched => { + handle_state!(self.emit_matched()) + } + ExistencePWMJStreamState::Completed => Poll::Ready(None), + }; + } + } + + /// Collects the buffered side into memory. + fn collect_buffered_side( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> Poll>>> { + let build_timer = self.join_metrics.build_time.timer(); + let buffered_data = ready!( + self.buffered_side + .try_as_initial_mut()? + .buffered_fut + .get_shared(cx) + )?; + build_timer.done(); + + self.buffered_side = + BufferedSide::Ready(BufferedSideReadyState { buffered_data }); + self.state = ExistencePWMJStreamState::ScanStreamBatches; + + Poll::Ready(Ok(StatefulStreamResult::Continue)) + } + + /// Fetches one streamed batch, reduces it to its extreme compare key, and marks the + /// buffered rows that key matches. + /// Never produces output: existence results come from the watermark in `emit_matched`. + fn scan_stream_batch( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> Poll>>> { + // Every buffered row that can ever be marked is already marked -- by this + // partition or any other, since the watermark is shared -- so no batch can lower + // the watermark further. Stop reading rather than scanning batches that provably + // cannot contribute. Checked before polling so a partition that starts after another + // has saturated the watermark reads nothing at all. + if self.nothing_left_to_mark()? { + self.finish_streamed_side()?; + return Poll::Ready(Ok(StatefulStreamResult::Continue)); + } + + match ready!(self.streamed.poll_next_unpin(cx)) { + None => self.finish_streamed_side()?, + Some(Ok(batch)) => { + let stream_values: ArrayRef = self + .on_streamed + .evaluate(&batch)? + .into_array(batch.num_rows())?; + + self.join_metrics.input_batches.add(1); + self.join_metrics.input_rows.add(batch.num_rows()); + + // Only the batch's extreme key is ever compared against the buffered side, + // so reduce the batch to that one key. + let stream_values = + extreme_key(&stream_values, self.sort_option.descending)?; + + self.mark_matched_buffered_rows(&stream_values)?; + } + Some(Err(err)) => return Poll::Ready(Err(err)), + } + + Poll::Ready(Ok(StatefulStreamResult::Continue)) + } + + /// Whether no buffered row can ever be marked again, in which case the streamed side + /// no longer needs reading. Reads the shared watermark, so one partition saturating it + /// lets every other partition stop too. + fn nothing_left_to_mark(&self) -> Result { + let buffered_data = &self.buffered_side.try_as_ready()?.buffered_data; + let min_marked = buffered_data + .existence_min_marked + .load(AtomicOrdering::SeqCst); + let buffered_values = buffered_data.values(); + + Ok(min_marked.min(buffered_values.len()) <= buffered_values.null_count()) + } + + /// Marks this partition done with the streamed side: releases the input pipeline and, + /// if this is the last streamed partition to finish, moves on to the final pass. + fn finish_streamed_side(&mut self) -> Result<()> { + // Release the streamed input pipeline's resources. + let streamed_schema = self.streamed.schema(); + self.streamed = Box::pin(EmptyRecordBatchStream::new(streamed_schema)); + + // The final pass must run exactly once, on the last streamed partition to finish. + if self + .buffered_side + .try_as_ready()? + .buffered_data + .remaining_partitions + .fetch_sub(1, AtomicOrdering::SeqCst) + == 1 + { + self.state = ExistencePWMJStreamState::EmitMatched; + } else { + self.state = ExistencePWMJStreamState::Completed; + } + + Ok(()) + } + + /// Marks every buffered row matched by `stream_values`, a one-row array holding the + /// batch's extreme compare key (null only if the whole batch was null). + fn mark_matched_buffered_rows(&mut self, stream_values: &ArrayRef) -> Result<()> { + let operator = self.operator; + let sort_option = self.sort_option; + + { + let buffered_data = &self.buffered_side.try_as_ready()?.buffered_data; + let buffered_values = buffered_data.values(); + let buffered_len = buffered_values.len(); + + // NULL keys can never match, and `sort_options` uses `nulls_first` for every + // operator (see `try_new`), so buffered nulls sit at the front -- skip past them. + let first_non_null_buffered = buffered_values.null_count(); + + // `[min_marked, buffered_len)` was already marked, by this partition or + // another, so a match found there would write nothing. Stop the scan at the + // watermark: that bounds the comparisons this batch performs, not just the + // bits it writes. + let scan_limit = buffered_data + .existence_min_marked + .load(AtomicOrdering::SeqCst) + .min(buffered_len); + + // The extreme key is the only one that can decide anything: it reaches the + // smallest matching `buffer_idx`, and every other row in the batch matches a + // subset of the same buffered suffix, so could only re-mark it. `null_count()` + // is 0 for a real key and 1 for an all-null batch, which skips the scan. + let row_idx = stream_values.null_count(); + + // `<=`/`>=` also match on equality; validated once here rather than inside + // the search below. + let match_on_equal = match operator { + Operator::Gt | Operator::Lt => false, + Operator::GtEq | Operator::LtEq => true, + _ => { + return internal_err!( + "PiecewiseMergeJoin should not contain operator, {}", + operator + ); + } + }; + + if row_idx < stream_values.len() && first_non_null_buffered < scan_limit { + let cmp = JoinKeyComparator::new( + &[Arc::clone(stream_values)], + &[Arc::clone(buffered_values)], + &[sort_option], + NullEquality::NullEqualsNothing, + )?; + let is_match = |buffer_idx: usize| { + let compare = cmp.compare(row_idx, buffer_idx); + compare == Ordering::Less + || (match_on_equal && compare == Ordering::Equal) + }; + + // Because the buffered side is sorted, `is_match` is monotone over it: + // false while the buffered key has not yet passed the streamed key, true + // from there on. So the first match is a partition point and can be found + // by binary search instead of a walk -- `O(log buffered)` per batch rather + // than `O(buffered)`. + let mut lo = first_non_null_buffered; + let mut hi = scan_limit; + while lo < hi { + let mid = lo + (hi - lo) / 2; + if is_match(mid) { + hi = mid; + } else { + lo = mid + 1; + } + } + + // `lo` is now the first matching buffered index, or `scan_limit` if this + // batch matches nothing new. + let buffer_idx = lo; + if buffer_idx < scan_limit { + // Everything from `buffer_idx` on matches, so lowering the + // watermark to it records the match: the marked set is exactly + // `[existence_min_marked, buffered_len)` and needs no bitmap. + // + // INVARIANT: sound only because the buffered side and each + // streamed batch are sorted the same way for this operator + // (`try_new` derives `sort_option`: descending for `<`/`<=`, + // ascending for `>`/`>=`). That makes this the smallest reachable + // `buffer_idx`, so the marked suffix is maximal. Only the ordering + // *within* a batch matters; batches themselves may arrive in any + // order, which is why the watermark takes a `min` rather than just + // decreasing. + buffered_data + .existence_min_marked + .fetch_min(buffer_idx, AtomicOrdering::SeqCst); + } + } + } + + Ok(()) + } + + /// Emits the existence result by slicing at the watermark: the marked buffered rows for + /// `LeftSemi`, the unmarked ones for `LeftAnti`. + fn emit_matched(&mut self) -> Result>> { + if !self.emitted { + self.emitted = true; + + let buffered_data = + Arc::clone(&self.buffered_side.try_as_ready()?.buffered_data); + let buffered_batch = buffered_data.batch(); + let buffered_len = buffered_batch.num_rows(); + + // The marked rows are always the contiguous suffix `[min_marked, len)`: each + // match covers `[k, previous min_marked)` and then lowers the watermark to + // `k`, so the union is `[k, len)`. The result is therefore a slice, with no + // index array to materialize and no `take`. + let min_marked = buffered_data + .existence_min_marked + .load(AtomicOrdering::SeqCst) + .min(buffered_len); + + let sliced = match self.join_type { + JoinType::LeftSemi => { + buffered_batch.slice(min_marked, buffered_len - min_marked) + } + // The unmarked prefix, which includes every null-keyed row: nulls sort + // first and the watermark never drops below the buffered null count. + _ => buffered_batch.slice(0, min_marked), + }; + + if sliced.num_rows() > 0 { + // Existence joins output the buffered (left) columns only; rebuild against + // the join's own schema, which keeps the slice zero-copy. + let batch = RecordBatch::try_new( + Arc::clone(&self.schema), + sliced.columns().to_vec(), + )?; + self.output_batches.push_batch(batch)?; + self.output_batches.finish_buffered_batch()?; + } + } + + // Drain one coalesced batch per poll; `emitted` keeps the block above from + // re-running, so this always terminates. + match self.output_batches.next_completed_batch() { + Some(batch) => Ok(StatefulStreamResult::Ready(Some(batch))), + None => { + self.state = ExistencePWMJStreamState::Completed; + Ok(StatefulStreamResult::Ready(None)) + } + } + } +} + +/// Reduces `values` to a one-row array holding the batch's extreme compare key: the maximum +/// when the batch's sort is `descending`, the minimum otherwise. Nulls are ignored, so that +/// row is null only when every key in the batch is (or the batch is empty), which marks +/// nothing. +/// +/// Ordered the same way as [`JoinKeyComparator`]: both use IEEE 754 totalOrder for floats, +/// and the comparator normalizes `-0.0` on either side of it. +/// +/// Numeric, temporal, string, binary and boolean keys get a typed arrow kernel -- a linear +/// scan that allocates nothing. Dictionary and nested keys fall to `min_max_batch_generic`, a +/// `ScalarValue`-per-row comparator loop; specializing those is left to a follow-up. +fn extreme_key(values: &ArrayRef, descending: bool) -> Result { + let extreme = if descending { + max_batch(values)? + } else { + min_batch(values)? + }; + extreme.to_array_of_size(1) +} + +impl RecordBatchStream for ExistencePWMJStream { + fn schema(&self) -> SchemaRef { + Arc::clone(&self.schema) + } +} + +impl Stream for ExistencePWMJStream { + type Item = Result; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll> { + // `record_poll` fills in `output_rows` and `end_time`; `elapsed_compute` is handled + // by `BuildProbeJoinMetrics::drop`. + let poll = self.poll_next_impl(cx); + self.join_metrics.baseline.record_poll(poll) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + ExecutionPlan, common, + joins::PiecewiseMergeJoinExec, + test::{TestMemoryExec, build_table_i32}, + }; + use arrow_schema::{DataType, Field, Schema}; + use datafusion_common::test_util::batches_to_string; + use datafusion_execution::TaskContext; + use datafusion_execution::config::SessionConfig; + use datafusion_physical_expr::expressions::Column; + use insta::assert_snapshot; + + // Coverage for existence joins lives in `pwmj.slt`: operators, NULL handling, empty + // sides, key types, both correlation orientations, and the streamed-side batch and + // partition layouts (which SQL pins via one `INSERT` per batch plus an `EXPLAIN` + // asserting `partition_sizes`). The two tests here are demos of the operator itself. + + fn build_table( + a: (&str, &Vec), + b: (&str, &Vec), + c: (&str, &Vec), + ) -> Arc { + let batch = build_table_i32(a, b, c); + let schema = batch.schema(); + TestMemoryExec::try_new_exec(&[vec![batch]], schema, None).unwrap() + } + + async fn join_collect(join_type: JoinType) -> Result> { + // Buffered (left) side pre-sorted ascending, as `>` requires. These tests build + // the exec directly, so there is no `SortExec` to enforce it. + let left = build_table( + ("a1", &vec![1, 2, 3]), + ("b1", &vec![1, 2, 5]), + ("c1", &vec![7, 8, 9]), + ); + let right = build_table( + ("a2", &vec![10, 20, 30]), + ("b1", &vec![2, 3, 4]), + ("c2", &vec![70, 80, 90]), + ); + let on = ( + Arc::new(Column::new_with_schema("b1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("b1", &right.schema())?) as _, + ); + let join = + PiecewiseMergeJoinExec::try_new(left, right, on, Operator::Gt, join_type, 1)?; + + let stream = join.execute(0, Arc::new(TaskContext::default()))?; + common::collect(stream).await + } + + /// `LeftSemi` keeps the buffered rows with at least one match, and outputs only the + /// buffered columns. Of b1 = {1,2,5} against streamed {2,3,4}, only 5 > some streamed + /// value. + #[tokio::test] + async fn join_left_semi() -> Result<()> { + let batches = join_collect(JoinType::LeftSemi).await?; + + assert_snapshot!(batches_to_string(&batches), @r" + +----+----+----+ + | a1 | b1 | c1 | + +----+----+----+ + | 3 | 5 | 9 | + +----+----+----+ + "); + Ok(()) + } + + /// `LeftAnti` is the complement: the buffered rows with no match. + #[tokio::test] + async fn join_left_anti() -> Result<()> { + let batches = join_collect(JoinType::LeftAnti).await?; + + assert_snapshot!(batches_to_string(&batches), @r" + +----+----+----+ + | a1 | b1 | c1 | + +----+----+----+ + | 1 | 1 | 7 | + | 2 | 2 | 8 | + +----+----+----+ + "); + Ok(()) + } + + /// Once every markable buffered row is marked, no later streamed batch can lower the + /// watermark, so the stream stops reading rather than scanning batches that provably + /// cannot contribute. The first batch below is smaller than every buffered value, so it + /// marks all four rows; the two batches after it must never be read. Asserted through + /// the `input_batches` metric, which SQL cannot observe. + #[tokio::test] + async fn early_exit_stops_reading_streamed_batches() -> Result<()> { + let left = build_table( + ("a1", &vec![1, 2, 3, 4]), + ("b1", &vec![1, 3, 5, 7]), + ("c1", &vec![10, 20, 30, 40]), + ); + + let streamed_schema = Schema::new(vec![ + Field::new("a2", DataType::Int32, false), + Field::new("b1", DataType::Int32, false), + Field::new("c2", DataType::Int32, false), + ]); + // b1=0 is below every buffered value, so batch 1 marks the whole buffered side. + let batch1 = + build_table_i32(("a2", &vec![10]), ("b1", &vec![0]), ("c2", &vec![70])); + let batch2 = + build_table_i32(("a2", &vec![20]), ("b1", &vec![2]), ("c2", &vec![80])); + let batch3 = + build_table_i32(("a2", &vec![30]), ("b1", &vec![6]), ("c2", &vec![90])); + let right = TestMemoryExec::try_new_exec( + &[vec![batch1, batch2, batch3]], + Arc::new(streamed_schema), + None, + )?; + + let on = ( + Arc::new(Column::new_with_schema("b1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("b1", &right.schema())?) as _, + ); + let join = PiecewiseMergeJoinExec::try_new( + left, + right, + on, + Operator::Gt, + JoinType::LeftSemi, + 1, + )?; + + let stream = join.execute(0, Arc::new(TaskContext::default()))?; + let batches = common::collect(stream).await?; + + // All four buffered rows still come out. + assert_snapshot!(batches_to_string(&batches), @r" + +----+----+----+ + | a1 | b1 | c1 | + +----+----+----+ + | 1 | 1 | 10 | + | 2 | 3 | 20 | + | 3 | 5 | 30 | + | 4 | 7 | 40 | + +----+----+----+ + "); + + // ...but only the first of the three streamed batches was ever read. + let consumed = join + .metrics() + .unwrap() + .sum_by_name("input_batches") + .expect("input_batches metric") + .as_usize(); + assert_eq!(consumed, 1, "expected early exit after the first batch"); + Ok(()) + } + + /// The final pass pushes one slice of the buffered batch into a `BatchCoalescer`, so a + /// result wider than `batch_size` has to be drained over several polls. Also pins the + /// `output_rows` metric, which `record_poll` in `poll_next` is what supplies. + #[tokio::test] + async fn final_pass_chunks_output_and_records_output_rows() -> Result<()> { + let left = build_table( + ("a1", &vec![1, 2, 3, 4]), + ("b1", &vec![1, 3, 5, 7]), + ("c1", &vec![10, 20, 30, 40]), + ); + + let streamed_schema = Schema::new(vec![ + Field::new("a2", DataType::Int32, false), + Field::new("b1", DataType::Int32, false), + Field::new("c2", DataType::Int32, false), + ]); + // b1=0 is below every buffered value, so every buffered row matches `>`. + let batch = + build_table_i32(("a2", &vec![10]), ("b1", &vec![0]), ("c2", &vec![70])); + let right = TestMemoryExec::try_new_exec( + &[vec![batch]], + Arc::new(streamed_schema), + None, + )?; + + let on = ( + Arc::new(Column::new_with_schema("b1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("b1", &right.schema())?) as _, + ); + let join = PiecewiseMergeJoinExec::try_new( + left, + right, + on, + Operator::Gt, + JoinType::LeftSemi, + 1, + )?; + + // batch_size 2 over a 4-row result forces the coalescer to hand back two batches. + let task_ctx = Arc::new( + TaskContext::default() + .with_session_config(SessionConfig::new().with_batch_size(2)), + ); + let stream = join.execute(0, task_ctx)?; + let batches = common::collect(stream).await?; + + assert_eq!(batches.len(), 2, "expected the final pass to be chunked"); + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 4); + + let output_rows = join + .metrics() + .unwrap() + .output_rows() + .expect("output_rows metric"); + assert_eq!(output_rows, 4); + Ok(()) + } + + /// The watermark lives in the shared buffered data, so one partition saturating it + /// lets the others stop too. Partition 0's single batch marks every buffered row; + /// partition 1 is then driven and must read none of its three batches. Only the shared + /// watermark makes that possible -- with a per-partition watermark, partition 1 would + /// rescan all three. + #[tokio::test] + async fn early_exit_is_shared_across_streamed_partitions() -> Result<()> { + let left = build_table( + ("a1", &vec![1, 2, 3, 4]), + ("b1", &vec![1, 3, 5, 7]), + ("c1", &vec![10, 20, 30, 40]), + ); + + let streamed_schema = Schema::new(vec![ + Field::new("a2", DataType::Int32, false), + Field::new("b1", DataType::Int32, false), + Field::new("c2", DataType::Int32, false), + ]); + // Partition 0: one batch below every buffered value -> marks the whole buffered side. + let p0 = build_table_i32(("a2", &vec![10]), ("b1", &vec![0]), ("c2", &vec![70])); + // Partition 1: three batches that can now contribute nothing. + let p1_b0 = + build_table_i32(("a2", &vec![20]), ("b1", &vec![2]), ("c2", &vec![80])); + let p1_b1 = + build_table_i32(("a2", &vec![30]), ("b1", &vec![4]), ("c2", &vec![90])); + let p1_b2 = + build_table_i32(("a2", &vec![40]), ("b1", &vec![6]), ("c2", &vec![100])); + let right = TestMemoryExec::try_new_exec( + &[vec![p0], vec![p1_b0, p1_b1, p1_b2]], + Arc::new(streamed_schema), + None, + )?; + + let on = ( + Arc::new(Column::new_with_schema("b1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("b1", &right.schema())?) as _, + ); + let join = PiecewiseMergeJoinExec::try_new( + left, + right, + on, + Operator::Gt, + JoinType::LeftSemi, + 2, + )?; + + // Driven in order so partition 0 saturates the watermark before partition 1 starts. + let task_ctx = Arc::new(TaskContext::default()); + let mut batches = Vec::new(); + for partition in 0..2 { + let stream = join.execute(partition, Arc::clone(&task_ctx))?; + batches.extend(common::collect(stream).await?); + } + let out = arrow::compute::concat_batches(&join.schema(), batches.iter())?; + + assert_snapshot!(batches_to_string(&[out]), @r" + +----+----+----+ + | a1 | b1 | c1 | + +----+----+----+ + | 1 | 1 | 10 | + | 2 | 3 | 20 | + | 3 | 5 | 30 | + | 4 | 7 | 40 | + +----+----+----+ + "); + + // Partition 0 read its one batch; partition 1 read none of its three. + let consumed = join + .metrics() + .unwrap() + .sum_by_name("input_batches") + .expect("input_batches metric") + .as_usize(); + assert_eq!(consumed, 1, "partition 1 should not have read any batch"); + Ok(()) + } + + /// The unsupported existence joins must be rejected at construction, not deeper in. + /// `required_input_ordering` still has an `unimplemented!()` for right existence joins + /// and cannot return an error, so this test is what keeps that panic unreachable: if + /// someone opens the gate for RightSemi/RightAnti without also supplying an ordering + /// requirement, this fails instead of panicking the optimizer at runtime. + #[test] + fn try_new_rejects_unsupported_existence_joins() -> Result<()> { + let left = build_table( + ("a1", &vec![1, 2, 3]), + ("b1", &vec![1, 2, 5]), + ("c1", &vec![7, 8, 9]), + ); + let right = build_table( + ("a2", &vec![10, 20, 30]), + ("b1", &vec![2, 3, 4]), + ("c2", &vec![70, 80, 90]), + ); + let on = ( + Arc::new(Column::new_with_schema("b1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("b1", &right.schema())?) as _, + ); + + for join_type in [ + JoinType::RightSemi, + JoinType::RightAnti, + JoinType::LeftMark, + JoinType::RightMark, + ] { + let err = PiecewiseMergeJoinExec::try_new( + Arc::clone(&left), + Arc::clone(&right), + on.clone(), + Operator::Gt, + join_type, + 1, + ) + .expect_err(&format!("{join_type} should be rejected")) + .to_string(); + assert!( + err.contains("not supported for PiecewiseMergeJoin"), + "unexpected error for {join_type}: {err}" + ); + } + Ok(()) + } +} diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/mod.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/mod.rs index c85a7cc16f657..8c6815ad6c631 100644 --- a/datafusion/physical-plan/src/joins/piecewise_merge_join/mod.rs +++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/mod.rs @@ -21,4 +21,5 @@ pub use exec::PiecewiseMergeJoinExec; mod classic_join; mod exec; +mod existence_join; mod utils; diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/utils.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/utils.rs index 5bbb496322b5f..5093be0ca19be 100644 --- a/datafusion/physical-plan/src/joins/piecewise_merge_join/utils.rs +++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/utils.rs @@ -38,6 +38,14 @@ pub(super) fn is_existence_join(join_type: JoinType) -> bool { ) } +// Returns boolean for whether the join is a left existence join that is currently +// supported by `PiecewiseMergeJoin`. These do not require swapping the inputs: the +// marked (left) side is already the buffered side, so `ExistencePWMJStream` can track the +// matched suffix and slice the buffered batch at its start. +pub(super) fn is_supported_existence_join(join_type: JoinType) -> bool { + matches!(join_type, JoinType::LeftSemi | JoinType::LeftAnti) +} + // Returns boolean to check if the join type needs to record // buffered side matches for classic joins pub(super) fn need_produce_result_in_final(join_type: JoinType) -> bool { @@ -46,14 +54,16 @@ pub(super) fn need_produce_result_in_final(join_type: JoinType) -> bool { // Returns boolean for whether or not we need to build the buffered side // bitmap for marking matched rows on the buffered side. +// +// `LeftSemi`/`LeftAnti` are absent on purpose: `ExistencePWMJStream` only ever marks a +// contiguous suffix of the buffered side, so it tracks the boundary as a single index +// (`BufferedSideData::existence_min_marked`) and needs no bitmap. pub(super) fn build_visited_indices_map(join_type: JoinType) -> bool { matches!( join_type, JoinType::Full | JoinType::Left - | JoinType::LeftAnti | JoinType::RightAnti - | JoinType::LeftSemi | JoinType::RightSemi | JoinType::LeftMark | JoinType::RightMark diff --git a/datafusion/sqllogictest/test_files/pwmj.slt b/datafusion/sqllogictest/test_files/pwmj.slt index 25c275f4b7e93..42be6c6b5cd88 100644 --- a/datafusion/sqllogictest/test_files/pwmj.slt +++ b/datafusion/sqllogictest/test_files/pwmj.slt @@ -387,5 +387,827 @@ ORDER BY 1,2; 20 100 NULL NULL +# ------------------------------------------------------------------ +# Existence joins (LeftSemi / LeftAnti) via PiecewiseMergeJoin +# ------------------------------------------------------------------ + +# EXISTS with a range correlation -> LeftSemi. Keep t1 rows that have at least one +# smaller t2_id: 22>11, 33>11, 44>{11,22}. 11 has none. +query I +SELECT t1.t1_id +FROM join_t1 t1 +WHERE EXISTS (SELECT 1 FROM join_t2 t2 WHERE t1.t1_id > t2.t2_id) +ORDER BY 1; +---- +22 +33 +44 + +query TT +EXPLAIN +SELECT t1.t1_id +FROM join_t1 t1 +WHERE EXISTS (SELECT 1 FROM join_t2 t2 WHERE t1.t1_id > t2.t2_id) +ORDER BY 1; +---- +logical_plan +01)Sort: t1.t1_id ASC NULLS LAST +02)--LeftSemi Join: Filter: t1.t1_id > __correlated_sq_1.t2_id +03)----SubqueryAlias: t1 +04)------TableScan: join_t1 projection=[t1_id] +05)----SubqueryAlias: __correlated_sq_1 +06)------SubqueryAlias: t2 +07)--------TableScan: join_t2 projection=[t2_id] +physical_plan +01)SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[false] +02)--PiecewiseMergeJoin: operator=Gt, join_type=LeftSemi, on=(t1_id > t2_id) +03)----SortExec: expr=[t1_id@0 ASC], preserve_partitioning=[false] +04)------DataSourceExec: partitions=1, partition_sizes=[1] +05)----DataSourceExec: partitions=1, partition_sizes=[1] + +# NOT EXISTS with a range correlation -> LeftAnti. Complement of the above. +query I +SELECT t1.t1_id +FROM join_t1 t1 +WHERE NOT EXISTS (SELECT 1 FROM join_t2 t2 WHERE t1.t1_id > t2.t2_id) +ORDER BY 1; +---- +11 + +query TT +EXPLAIN +SELECT t1.t1_id +FROM join_t1 t1 +WHERE NOT EXISTS (SELECT 1 FROM join_t2 t2 WHERE t1.t1_id > t2.t2_id) +ORDER BY 1; +---- +logical_plan +01)Sort: t1.t1_id ASC NULLS LAST +02)--LeftAnti Join: Filter: t1.t1_id > __correlated_sq_1.t2_id +03)----SubqueryAlias: t1 +04)------TableScan: join_t1 projection=[t1_id] +05)----SubqueryAlias: __correlated_sq_1 +06)------SubqueryAlias: t2 +07)--------TableScan: join_t2 projection=[t2_id] +physical_plan +01)SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[false] +02)--PiecewiseMergeJoin: operator=Gt, join_type=LeftAnti, on=(t1_id > t2_id) +03)----SortExec: expr=[t1_id@0 ASC], preserve_partitioning=[false] +04)------DataSourceExec: partitions=1, partition_sizes=[1] +05)----DataSourceExec: partitions=1, partition_sizes=[1] + +# NULL join key on the semi/anti (buffered) side never matches: excluded from EXISTS, +# included in NOT EXISTS. null_join_t1 = {1, 2, NULL}, null_join_t2 = {1, NULL, 3}. +# EXISTS t1.id > t2.id : 2>1 -> {2}. 1 and NULL have no smaller t2. +query I +SELECT t1.id +FROM null_join_t1 t1 +WHERE EXISTS (SELECT 1 FROM null_join_t2 t2 WHERE t1.id > t2.id) +ORDER BY 1; +---- +2 + +query I +SELECT t1.id +FROM null_join_t1 t1 +WHERE NOT EXISTS (SELECT 1 FROM null_join_t2 t2 WHERE t1.id > t2.id) +ORDER BY 1 NULLS FIRST; +---- +NULL +1 + +# ------------------------------------------------------------------ +# Existence joins: operator, NULL, empty-side and type coverage +# ------------------------------------------------------------------ + +statement ok +CREATE TABLE ex_l(id INT, v INT); + +statement ok +INSERT INTO ex_l VALUES (1, 5), (2, 4), (3, 2), (4, 1); + +statement ok +CREATE TABLE ex_r(v INT); + +statement ok +INSERT INTO ex_r VALUES (2), (3), (4); + +# `<` : left v = {5,4,2,1}, right v = {2,3,4}. 2<{3,4}; 1` : 5>{2,3,4}; 4>{2,3}. 2 and 1 have no smaller right value. +query I +SELECT l.id FROM ex_l l WHERE EXISTS (SELECT 1 FROM ex_r r WHERE l.v > r.v) ORDER BY 1; +---- +1 +2 + +query I +SELECT l.id FROM ex_l l WHERE NOT EXISTS (SELECT 1 FROM ex_r r WHERE l.v > r.v) ORDER BY 1; +---- +3 +4 + +# `>=` additionally admits v=2 via the equal right value, so it must differ from `>`. +query I +SELECT l.id FROM ex_l l WHERE EXISTS (SELECT 1 FROM ex_r r WHERE l.v >= r.v) ORDER BY 1; +---- +1 +2 +3 + +query I +SELECT l.id FROM ex_l l WHERE NOT EXISTS (SELECT 1 FROM ex_r r WHERE l.v >= r.v) ORDER BY 1; +---- +4 + +# Writing the correlation with the inner column first is the same join; the planner +# normalizes it, so the operator on the physical join flips to keep the buffered (left) +# side as the marked side. Same rows as `l.v > r.v` above. +query I +SELECT l.id FROM ex_l l WHERE EXISTS (SELECT 1 FROM ex_r r WHERE r.v < l.v) ORDER BY 1; +---- +1 +2 + +query I +SELECT l.id FROM ex_l l WHERE NOT EXISTS (SELECT 1 FROM ex_r r WHERE r.v < l.v) ORDER BY 1; +---- +3 +4 + +# The physical operator is `Gt`, not `Lt`: the predicate is flipped so the marked (left) +# side stays on the buffered side and no input swap is needed. +query TT +EXPLAIN SELECT l.id FROM ex_l l WHERE EXISTS (SELECT 1 FROM ex_r r WHERE r.v < l.v) ORDER BY 1; +---- +logical_plan +01)Sort: l.id ASC NULLS LAST +02)--Projection: l.id +03)----LeftSemi Join: Filter: __correlated_sq_1.v < l.v +04)------SubqueryAlias: l +05)--------TableScan: ex_l projection=[id, v] +06)------SubqueryAlias: __correlated_sq_1 +07)--------SubqueryAlias: r +08)----------TableScan: ex_r projection=[v] +physical_plan +01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false] +02)--ProjectionExec: expr=[id@0 as id] +03)----PiecewiseMergeJoin: operator=Gt, join_type=LeftSemi, on=(v > v) +04)------SortExec: expr=[v@1 ASC], preserve_partitioning=[false] +05)--------DataSourceExec: partitions=1, partition_sizes=[1] +06)------DataSourceExec: partitions=1, partition_sizes=[1] + +# A filter inside the subquery lets the streamed side be repartitioned, so the final pass +# has to be coordinated across more than one streamed partition. +query I +SELECT l.id FROM ex_l l WHERE EXISTS (SELECT 1 FROM ex_r r WHERE l.v > r.v AND r.v > 0) ORDER BY 1; +---- +1 +2 + +query I +SELECT l.id FROM ex_l l WHERE NOT EXISTS (SELECT 1 FROM ex_r r WHERE l.v > r.v AND r.v > 0) ORDER BY 1; +---- +3 +4 + +# Confirms the streamed side really is repartitioned above (RoundRobinBatch(4)), which is +# what makes the multi-partition final-pass coordination reachable from SQL. +query TT +EXPLAIN SELECT l.id FROM ex_l l WHERE EXISTS (SELECT 1 FROM ex_r r WHERE l.v > r.v AND r.v > 0) ORDER BY 1; +---- +logical_plan +01)Sort: l.id ASC NULLS LAST +02)--Projection: l.id +03)----LeftSemi Join: Filter: l.v > __correlated_sq_1.v +04)------SubqueryAlias: l +05)--------TableScan: ex_l projection=[id, v] +06)------SubqueryAlias: __correlated_sq_1 +07)--------SubqueryAlias: r +08)----------Filter: ex_r.v > Int32(0) +09)------------TableScan: ex_r projection=[v] +physical_plan +01)SortPreservingMergeExec: [id@0 ASC NULLS LAST] +02)--SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true] +03)----ProjectionExec: expr=[id@0 as id] +04)------PiecewiseMergeJoin: operator=Gt, join_type=LeftSemi, on=(v > v) +05)--------SortExec: expr=[v@1 ASC], preserve_partitioning=[false] +06)----------DataSourceExec: partitions=1, partition_sizes=[1] +07)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +08)----------FilterExec: v@0 > 0 +09)------------DataSourceExec: partitions=1, partition_sizes=[1] + +statement ok +CREATE TABLE ex_empty(v INT); + +# Empty buffered (left) side: nothing to mark and nothing to emit either way. +query I +SELECT l.v FROM ex_empty l WHERE EXISTS (SELECT 1 FROM ex_r r WHERE l.v > r.v) ORDER BY 1; +---- + +query I +SELECT l.v FROM ex_empty l WHERE NOT EXISTS (SELECT 1 FROM ex_r r WHERE l.v > r.v) ORDER BY 1; +---- + +# Empty streamed (right) side: the watermark never moves, so EXISTS is empty and NOT EXISTS keeps +# every buffered row. +query I +SELECT l.id FROM ex_l l WHERE EXISTS (SELECT 1 FROM ex_empty r WHERE l.v > r.v) ORDER BY 1; +---- + +query I +SELECT l.id FROM ex_l l WHERE NOT EXISTS (SELECT 1 FROM ex_empty r WHERE l.v > r.v) ORDER BY 1; +---- +1 +2 +3 +4 + +statement ok +CREATE TABLE ex_r_null(v INT); + +statement ok +INSERT INTO ex_r_null VALUES (NULL), (NULL); + +# Every streamed key is NULL, so no comparison can match: same outcome as an empty +# streamed side. +query I +SELECT l.id FROM ex_l l WHERE EXISTS (SELECT 1 FROM ex_r_null r WHERE l.v > r.v) ORDER BY 1; +---- + +query I +SELECT l.id FROM ex_l l WHERE NOT EXISTS (SELECT 1 FROM ex_r_null r WHERE l.v > r.v) ORDER BY 1; +---- +1 +2 +3 +4 + +statement ok +CREATE TABLE ex_dl(id INT, d DATE); + +statement ok +INSERT INTO ex_dl VALUES (1, DATE '2022-04-23'), (2, DATE '2022-04-28'), (3, DATE '2022-04-18'); + +statement ok +CREATE TABLE ex_dr(d DATE); + +statement ok +INSERT INTO ex_dr VALUES (DATE '2022-04-20'), (DATE '2022-04-26'); + +# Non-integer key: existence joins share the comparator with the matching joins, so a +# Date32 key behaves identically. 04-23 > 04-20; 04-28 > both; 04-18 has none. +query I +SELECT l.id FROM ex_dl l WHERE EXISTS (SELECT 1 FROM ex_dr r WHERE l.d > r.d) ORDER BY 1; +---- +1 +2 + +query I +SELECT l.id FROM ex_dl l WHERE NOT EXISTS (SELECT 1 FROM ex_dr r WHERE l.d > r.d) ORDER BY 1; +---- +3 + +# Stopping at the first matching buffered row is load-bearing: it relies on both sides +# being sorted in the same direction, so that the first match yields the maximal matching +# buffered suffix and later streamed rows could only re-mark a subset. This interleaved +# case (buffered {2,4,6,8} vs streamed {1,3,5,7}, all in one batch) is the shape that would +# expose an off-by-one there: every buffered row has some smaller streamed value, so all +# four must survive EXISTS and none may survive NOT EXISTS. +statement ok +CREATE TABLE ex_interleave_l(id INT, v INT); + +statement ok +INSERT INTO ex_interleave_l VALUES (1, 2), (2, 4), (3, 6), (4, 8); + +statement ok +CREATE TABLE ex_interleave_r(v INT); + +statement ok +INSERT INTO ex_interleave_r VALUES (1), (3), (5), (7); + +query I +SELECT l.id FROM ex_interleave_l l WHERE EXISTS (SELECT 1 FROM ex_interleave_r r WHERE l.v > r.v) ORDER BY 1; +---- +1 +2 +3 +4 + +query I +SELECT l.id FROM ex_interleave_l l WHERE NOT EXISTS (SELECT 1 FROM ex_interleave_r r WHERE l.v > r.v) ORDER BY 1; +---- + +# Only the extreme key of each streamed batch is compared against the buffered side, so +# that key must be picked correctly out of an unsorted batch that also contains a NULL. +# One INSERT -> one batch of {7, NULL, 9, 3}: the deciding key sits at neither end, and a +# NULL must never be chosen. For `>` the decider is min=3 (buffered 5,4 exceed it; 2,1 do +# not); for `<` it is max=9 (every buffered row is below it). +statement ok +CREATE TABLE ex_extreme_r(v INT); + +statement ok +INSERT INTO ex_extreme_r VALUES (7), (NULL), (9), (3); + +query I +SELECT l.id FROM ex_l l WHERE EXISTS (SELECT 1 FROM ex_extreme_r r WHERE l.v > r.v) ORDER BY 1; +---- +1 +2 + +query I +SELECT l.id FROM ex_l l WHERE NOT EXISTS (SELECT 1 FROM ex_extreme_r r WHERE l.v > r.v) ORDER BY 1; +---- +3 +4 + +query I +SELECT l.id FROM ex_l l WHERE EXISTS (SELECT 1 FROM ex_extreme_r r WHERE l.v < r.v) ORDER BY 1; +---- +1 +2 +3 +4 + +query I +SELECT l.id FROM ex_l l WHERE NOT EXISTS (SELECT 1 FROM ex_extreme_r r WHERE l.v < r.v) ORDER BY 1; +---- + +# ------------------------------------------------------------------ +# Existence joins: streamed-side batch and partition layout +# ------------------------------------------------------------------ +# The existence path lowers the shared watermark incrementally as streamed batches arrive, so +# the batch and partition layout of the streamed side is load-bearing. Each INSERT below +# adds one batch, and the EXPLAINs assert the layout the row expectations rely on. + +statement ok +CREATE TABLE ex_layout_l(id INT, v INT); + +statement ok +INSERT INTO ex_layout_l VALUES (1, 1), (2, 3), (3, 5), (4, 7); + +statement ok +CREATE TABLE ex_layout_r(v INT); + +# Only the ordering within a streamed batch is enforced, so batches may arrive in any +# order. This deliberately non-monotonic arrival order drives the cross-batch low-water +# mark through all three of its cases: +# batch 1 (v=2) matches low -> marks buffered v={3,5,7}, watermark drops to index 1 +# batch 2 (v=6) matches above the watermark -> scan stops at it, nothing written +# batch 3 (v=0) matches lower still-> marks the remaining v=1 +# So all four buffered rows match. Re-marking greedily or skipping batch 3 both change this. +statement ok +INSERT INTO ex_layout_r VALUES (2); + +statement ok +INSERT INTO ex_layout_r VALUES (6); + +statement ok +INSERT INTO ex_layout_r VALUES (0); + +query I +SELECT l.id FROM ex_layout_l l WHERE EXISTS (SELECT 1 FROM ex_layout_r r WHERE l.v > r.v) ORDER BY 1; +---- +1 +2 +3 +4 + +query I +SELECT l.id FROM ex_layout_l l WHERE NOT EXISTS (SELECT 1 FROM ex_layout_r r WHERE l.v > r.v) ORDER BY 1; +---- + +# Asserts the streamed side really arrives as three batches in a single partition +# (`partition_sizes=[3]`), which is what the batch ordering above depends on. +query TT +EXPLAIN SELECT l.id FROM ex_layout_l l WHERE EXISTS (SELECT 1 FROM ex_layout_r r WHERE l.v > r.v) ORDER BY 1; +---- +logical_plan +01)Sort: l.id ASC NULLS LAST +02)--Projection: l.id +03)----LeftSemi Join: Filter: l.v > __correlated_sq_1.v +04)------SubqueryAlias: l +05)--------TableScan: ex_layout_l projection=[id, v] +06)------SubqueryAlias: __correlated_sq_1 +07)--------SubqueryAlias: r +08)----------TableScan: ex_layout_r projection=[v] +physical_plan +01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false] +02)--ProjectionExec: expr=[id@0 as id] +03)----PiecewiseMergeJoin: operator=Gt, join_type=LeftSemi, on=(v > v) +04)------SortExec: expr=[v@1 ASC], preserve_partitioning=[false] +05)--------DataSourceExec: partitions=1, partition_sizes=[1] +06)------DataSourceExec: partitions=1, partition_sizes=[3] + +# Same three batches, now also repartitioned across streamed partitions. The filter admits +# {2,6}, so buffered 3,5,7 match and only v=1 does not. This covers marking across batches +# and the cross-partition final-pass counter together, for both semi and anti. +query I +SELECT l.id FROM ex_layout_l l WHERE EXISTS (SELECT 1 FROM ex_layout_r r WHERE l.v > r.v AND r.v > 0) ORDER BY 1; +---- +2 +3 +4 + +query I +SELECT l.id FROM ex_layout_l l WHERE NOT EXISTS (SELECT 1 FROM ex_layout_r r WHERE l.v > r.v AND r.v > 0) ORDER BY 1; +---- +1 + +query TT +EXPLAIN SELECT l.id FROM ex_layout_l l WHERE NOT EXISTS (SELECT 1 FROM ex_layout_r r WHERE l.v > r.v AND r.v > 0) ORDER BY 1; +---- +logical_plan +01)Sort: l.id ASC NULLS LAST +02)--Projection: l.id +03)----LeftAnti Join: Filter: l.v > __correlated_sq_1.v +04)------SubqueryAlias: l +05)--------TableScan: ex_layout_l projection=[id, v] +06)------SubqueryAlias: __correlated_sq_1 +07)--------SubqueryAlias: r +08)----------Filter: ex_layout_r.v > Int32(0) +09)------------TableScan: ex_layout_r projection=[v] +physical_plan +01)SortPreservingMergeExec: [id@0 ASC NULLS LAST] +02)--SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true] +03)----ProjectionExec: expr=[id@0 as id] +04)------PiecewiseMergeJoin: operator=Gt, join_type=LeftAnti, on=(v > v) +05)--------SortExec: expr=[v@1 ASC], preserve_partitioning=[false] +06)----------DataSourceExec: partitions=1, partition_sizes=[1] +07)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +08)----------FilterExec: v@0 > 0 +09)------------DataSourceExec: partitions=1, partition_sizes=[3] + +# The first matching buffered row is located by binary search, so the boundary cases matter. +# `ex_layout_l` holds v = {1,3,5,7} and nothing has lowered the watermark for these tables, +# so each search runs over the full buffered range. + +# Boundary at the LAST buffered row: only 7 > 6. +statement ok +CREATE TABLE ex_bs_last(v INT); + +statement ok +INSERT INTO ex_bs_last VALUES (6); + +query I +SELECT l.id FROM ex_layout_l l WHERE EXISTS (SELECT 1 FROM ex_bs_last r WHERE l.v > r.v) ORDER BY 1; +---- +4 + +query I +SELECT l.id FROM ex_layout_l l WHERE NOT EXISTS (SELECT 1 FROM ex_bs_last r WHERE l.v > r.v) ORDER BY 1; +---- +1 +2 +3 + +# No boundary at all: no buffered row exceeds 9, so the search must run off the end. +statement ok +CREATE TABLE ex_bs_none(v INT); + +statement ok +INSERT INTO ex_bs_none VALUES (9); + +query I +SELECT l.id FROM ex_layout_l l WHERE EXISTS (SELECT 1 FROM ex_bs_none r WHERE l.v > r.v) ORDER BY 1; +---- + +query I +SELECT l.id FROM ex_layout_l l WHERE NOT EXISTS (SELECT 1 FROM ex_bs_none r WHERE l.v > r.v) ORDER BY 1; +---- +1 +2 +3 +4 + +# ------------------------------------------------------------------ +# Existence joins: duplicate buffered keys across the match boundary +# ------------------------------------------------------------------ +# Every fixture above uses distinct buffered keys, so the boundary always fell between two +# different values. The binary search must return the FIRST index of a run of equal keys -- +# landing on a later one silently drops the earlier duplicates from EXISTS. `ex_dup_l` holds +# v = {5,5,3,3,1} and the deciding streamed key is 3, so for `<=` and `>=` the boundary falls +# *inside* the run of 3s. +statement ok +CREATE TABLE ex_dup_l(id INT, v INT); + +statement ok +INSERT INTO ex_dup_l VALUES (1, 5), (2, 5), (3, 3), (4, 3), (5, 1); + +statement ok +CREATE TABLE ex_dup_r(v INT); + +statement ok +INSERT INTO ex_dup_r VALUES (3); + +# `<` : only v=1 is below 3. +query I +SELECT l.id FROM ex_dup_l l WHERE EXISTS (SELECT 1 FROM ex_dup_r r WHERE l.v < r.v) ORDER BY 1; +---- +5 + +query I +SELECT l.id FROM ex_dup_l l WHERE NOT EXISTS (SELECT 1 FROM ex_dup_r r WHERE l.v < r.v) ORDER BY 1; +---- +1 +2 +3 +4 + +# `<=` : the boundary is the first of the two v=3 rows, so BOTH must appear. +query I +SELECT l.id FROM ex_dup_l l WHERE EXISTS (SELECT 1 FROM ex_dup_r r WHERE l.v <= r.v) ORDER BY 1; +---- +3 +4 +5 + +query I +SELECT l.id FROM ex_dup_l l WHERE NOT EXISTS (SELECT 1 FROM ex_dup_r r WHERE l.v <= r.v) ORDER BY 1; +---- +1 +2 + +# `>` : only the two v=5 rows exceed 3. +query I +SELECT l.id FROM ex_dup_l l WHERE EXISTS (SELECT 1 FROM ex_dup_r r WHERE l.v > r.v) ORDER BY 1; +---- +1 +2 + +query I +SELECT l.id FROM ex_dup_l l WHERE NOT EXISTS (SELECT 1 FROM ex_dup_r r WHERE l.v > r.v) ORDER BY 1; +---- +3 +4 +5 + +# `>=` : the boundary is again inside the run of 3s, now from the ascending side. +query I +SELECT l.id FROM ex_dup_l l WHERE EXISTS (SELECT 1 FROM ex_dup_r r WHERE l.v >= r.v) ORDER BY 1; +---- +1 +2 +3 +4 + +query I +SELECT l.id FROM ex_dup_l l WHERE NOT EXISTS (SELECT 1 FROM ex_dup_r r WHERE l.v >= r.v) ORDER BY 1; +---- +5 + +# ------------------------------------------------------------------ +# Existence joins: NULL-only and multi-NULL buffered side +# ------------------------------------------------------------------ +# An all-NULL buffered side saturates the watermark before the first poll -- a different +# early-exit trigger from the "every row marked" case above -- so no streamed batch is read +# at all and EXISTS must still be empty. +statement ok +CREATE TABLE ex_l_allnull(id INT, v INT); + +statement ok +INSERT INTO ex_l_allnull VALUES (1, NULL), (2, NULL); + +query I +SELECT l.id FROM ex_l_allnull l WHERE EXISTS (SELECT 1 FROM ex_r r WHERE l.v < r.v) ORDER BY 1; +---- + +query I +SELECT l.id FROM ex_l_allnull l WHERE NOT EXISTS (SELECT 1 FROM ex_r r WHERE l.v < r.v) ORDER BY 1; +---- +1 +2 + +query I +SELECT l.id FROM ex_l_allnull l WHERE EXISTS (SELECT 1 FROM ex_r r WHERE l.v > r.v) ORDER BY 1; +---- + +query I +SELECT l.id FROM ex_l_allnull l WHERE NOT EXISTS (SELECT 1 FROM ex_r r WHERE l.v > r.v) ORDER BY 1; +---- +1 +2 + +# More than one NULL on the buffered side: the scan starts at `null_count()`, so a second +# NULL shifts that offset. Buffered v = {NULL,3,NULL,1} sorts to [NULL,NULL,3,1] for `<`. +statement ok +CREATE TABLE ex_multinull_l(id INT, v INT); + +statement ok +INSERT INTO ex_multinull_l VALUES (1, NULL), (2, 3), (3, NULL), (4, 1); + +statement ok +CREATE TABLE ex_multinull_r(v INT); + +statement ok +INSERT INTO ex_multinull_r VALUES (2); + +# `<` : only v=1 is below 2. Both NULLs stay in NOT EXISTS. +query I +SELECT l.id FROM ex_multinull_l l WHERE EXISTS (SELECT 1 FROM ex_multinull_r r WHERE l.v < r.v) ORDER BY 1; +---- +4 + +query I +SELECT l.id FROM ex_multinull_l l WHERE NOT EXISTS (SELECT 1 FROM ex_multinull_r r WHERE l.v < r.v) ORDER BY 1; +---- +1 +2 +3 + +# `>` : only v=3 exceeds 2. +query I +SELECT l.id FROM ex_multinull_l l WHERE EXISTS (SELECT 1 FROM ex_multinull_r r WHERE l.v > r.v) ORDER BY 1; +---- +2 + +query I +SELECT l.id FROM ex_multinull_l l WHERE NOT EXISTS (SELECT 1 FROM ex_multinull_r r WHERE l.v > r.v) ORDER BY 1; +---- +1 +3 +4 + +# ------------------------------------------------------------------ +# Existence joins: float keys and negative zero +# ------------------------------------------------------------------ +# The existence path shares `JoinKeyComparator`, which normalizes `-0.0` before building the +# comparator because arrow's `make_comparator` uses IEEE totalOrder and would rank `-0.0` +# below `+0.0`. SQL says they are equal, so `-0.0 < 0.0` must be false and `-0.0 <= 0.0` +# true. Skipping the normalization would put id 1 in the `<` result. +statement ok +CREATE TABLE ex_fl(id INT, v DOUBLE); + +statement ok +INSERT INTO ex_fl VALUES (1, -0.0), (2, 2.5); + +statement ok +CREATE TABLE ex_fr(v DOUBLE); + +statement ok +INSERT INTO ex_fr VALUES (0.0); + +query I +SELECT l.id FROM ex_fl l WHERE EXISTS (SELECT 1 FROM ex_fr r WHERE l.v < r.v) ORDER BY 1; +---- + +query I +SELECT l.id FROM ex_fl l WHERE NOT EXISTS (SELECT 1 FROM ex_fr r WHERE l.v < r.v) ORDER BY 1; +---- +1 +2 + +query I +SELECT l.id FROM ex_fl l WHERE EXISTS (SELECT 1 FROM ex_fr r WHERE l.v <= r.v) ORDER BY 1; +---- +1 + +query I +SELECT l.id FROM ex_fl l WHERE NOT EXISTS (SELECT 1 FROM ex_fr r WHERE l.v <= r.v) ORDER BY 1; +---- +2 + +# ------------------------------------------------------------------ +# Classic joins: final pass when the streamed side has fewer partitions than +# `target_partitions` +# ------------------------------------------------------------------ +# The final pass -- the only thing that emits unmatched buffered rows for `Left`/`Full` -- runs +# on the last streamed partition to finish, gated by a counter seeded with the number of +# partitions that will call `execute`. That is the streamed side's partition count, not +# `target_partitions`: the streamed side carries no distribution requirement, so a small input +# stays single-partition while `target_partitions` is 4 (asserted by the plan below). Seeding +# from `target_partitions` leaves the counter at 3, no partition ever emits, and `x = 10` +# disappears from both queries. +statement ok +CREATE TABLE pwmj_unmatched_l(x INT); + +statement ok +INSERT INTO pwmj_unmatched_l VALUES (1), (10); + +statement ok +CREATE TABLE pwmj_unmatched_r(y INT); + +statement ok +INSERT INTO pwmj_unmatched_r VALUES (5); + +query II +SELECT l.x, r.y FROM pwmj_unmatched_l l LEFT JOIN pwmj_unmatched_r r ON l.x < r.y ORDER BY 1; +---- +1 5 +10 NULL + +query II +SELECT l.x, r.y FROM pwmj_unmatched_l l FULL JOIN pwmj_unmatched_r r ON l.x < r.y ORDER BY 1; +---- +1 5 +10 NULL + +query TT +EXPLAIN SELECT l.x, r.y FROM pwmj_unmatched_l l LEFT JOIN pwmj_unmatched_r r ON l.x < r.y ORDER BY 1; +---- +logical_plan +01)Sort: l.x ASC NULLS LAST +02)--Left Join: Filter: l.x < r.y +03)----SubqueryAlias: l +04)------TableScan: pwmj_unmatched_l projection=[x] +05)----SubqueryAlias: r +06)------TableScan: pwmj_unmatched_r projection=[y] +physical_plan +01)SortExec: expr=[x@0 ASC NULLS LAST], preserve_partitioning=[false] +02)--PiecewiseMergeJoin: operator=Lt, join_type=Left, on=(x < y) +03)----SortExec: expr=[x@0 DESC], preserve_partitioning=[false] +04)------DataSourceExec: partitions=1, partition_sizes=[1] +05)----DataSourceExec: partitions=1, partition_sizes=[1] + +# ------------------------------------------------------------------ +# Existence joins: dictionary-encoded keys +# ------------------------------------------------------------------ +# Each streamed batch is reduced to its extreme key. Dictionaries have no typed arrow min/max +# kernel, so they take the generic `ScalarValue`-per-row path -- this is the only coverage of +# that path. +statement ok +CREATE TABLE ex_dict_l AS + SELECT column1 AS id, arrow_cast(column2, 'Dictionary(Int32, Utf8)') AS v + FROM (VALUES (1, 'a'), (2, 'c'), (3, 'e'), (4, NULL)); + +statement ok +CREATE TABLE ex_dict_r AS + SELECT arrow_cast(column1, 'Dictionary(Int32, Utf8)') AS v FROM (VALUES ('c')); + +query T +SELECT arrow_typeof(v) FROM ex_dict_l LIMIT 1; +---- +Dictionary(Int32, Utf8) + +# `>` : only 'e' exceeds 'c'. The NULL key is never marked, so it stays in NOT EXISTS. +query I +SELECT l.id FROM ex_dict_l l WHERE EXISTS (SELECT 1 FROM ex_dict_r r WHERE l.v > r.v) ORDER BY 1; +---- +3 + +query I +SELECT l.id FROM ex_dict_l l WHERE NOT EXISTS (SELECT 1 FROM ex_dict_r r WHERE l.v > r.v) ORDER BY 1; +---- +1 +2 +4 + +# `<=` : 'a' and 'c'. +query I +SELECT l.id FROM ex_dict_l l WHERE EXISTS (SELECT 1 FROM ex_dict_r r WHERE l.v <= r.v) ORDER BY 1; +---- +1 +2 + +query I +SELECT l.id FROM ex_dict_l l WHERE NOT EXISTS (SELECT 1 FROM ex_dict_r r WHERE l.v <= r.v) ORDER BY 1; +---- +3 +4 + +query TT +EXPLAIN SELECT l.id FROM ex_dict_l l WHERE EXISTS (SELECT 1 FROM ex_dict_r r WHERE l.v > r.v) ORDER BY 1; +---- +logical_plan +01)Sort: l.id ASC NULLS LAST +02)--Projection: l.id +03)----LeftSemi Join: Filter: l.v > __correlated_sq_1.v +04)------SubqueryAlias: l +05)--------TableScan: ex_dict_l projection=[id, v] +06)------SubqueryAlias: __correlated_sq_1 +07)--------SubqueryAlias: r +08)----------TableScan: ex_dict_r projection=[v] +physical_plan +01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false] +02)--ProjectionExec: expr=[id@0 as id] +03)----PiecewiseMergeJoin: operator=Gt, join_type=LeftSemi, on=(v > v) +04)------SortExec: expr=[v@1 ASC], preserve_partitioning=[false] +05)--------DataSourceExec: partitions=1, partition_sizes=[1] +06)------DataSourceExec: partitions=1, partition_sizes=[1] + statement ok set datafusion.optimizer.enable_piecewise_merge_join = false; From 2956cbaa9ab3117205e94f827ff2c0c1a0a9a557 Mon Sep 17 00:00:00 2001 From: Goutam Adwant <8672451+goutamadwant@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:36:59 +0000 Subject: [PATCH 27/50] feat(cli): add opt-in Spark function registration (#24179) ## Which issue does this PR close? - Closes #24146. ## Rationale for this change `datafusion-cli` does not currently enable the Spark SQL dialect or register the `datafusion-spark` function library, so Spark-compatible syntax and functions cannot be explored interactively. Enabling them by default could override core DataFusion functions with the same names and change existing CLI behavior. ## What changes are included in this PR? - Add a `--spark` CLI flag. - Select the Spark SQL dialect while constructing the session configuration. - Register Spark-compatible scalar, aggregate, and window functions only when the flag is enabled. - Add CLI integration coverage for both default and opt-in behavior. - Document the flag and its function-override behavior in the CLI and Spark documentation. ## Are these changes tested? Yes. The following checks pass: - `cargo test -p datafusion-cli --test cli_integration spark_features_require_opt_in` - `cargo test -p datafusion-cli --all-targets` - `cargo fmt --all -- --check` - `cargo clippy --all-targets --all-features -- -D warnings` - `./ci/scripts/doc_prettier_check.sh` ## Are there any user-facing changes? Yes. Passing `--spark` enables the Spark SQL dialect, Spark expression planner, and Spark-compatible functions for the CLI session. The option is disabled by default, so existing CLI behavior remains unchanged. --- Cargo.lock | 1 + datafusion-cli/Cargo.toml | 1 + datafusion-cli/src/main.rs | 24 ++++++++++++-- datafusion-cli/tests/cli_integration.rs | 31 +++++++++++++++++++ datafusion/spark/README.md | 12 +++++++ .../sqllogictest/test_files/spark/README.md | 4 +++ docs/source/user-guide/cli/usage.md | 14 +++++++++ 7 files changed, 85 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5365be12a1061..d7576659f0620 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1870,6 +1870,7 @@ dependencies = [ "ctor", "datafusion", "datafusion-common", + "datafusion-spark", "dirs", "env_logger", "futures", diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 62eedafe798d4..50b4002541f89 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -56,6 +56,7 @@ datafusion = { workspace = true, features = [ "unicode_expressions", ] } datafusion-common = { workspace = true } +datafusion-spark = { workspace = true, features = ["core"] } dirs = "6.0.0" env_logger = { workspace = true } futures = { workspace = true } diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index 20a2537d7c10c..4be6058aef8b6 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -23,6 +23,7 @@ use std::process::ExitCode; use std::sync::{Arc, LazyLock}; use datafusion::error::{DataFusionError, Result}; +use datafusion::execution::SessionStateBuilder; use datafusion::execution::context::SessionConfig; use datafusion::execution::memory_pool::{ FairSpillPool, GreedyMemoryPool, MemoryPool, TrackConsumersPool, @@ -46,9 +47,11 @@ use datafusion_cli::{ }; use clap::Parser; +use datafusion::common::config::Dialect; use datafusion::common::config_err; use datafusion::config::ConfigOptions; use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; +use datafusion_spark::SessionStateBuilderSpark; use mimalloc::MiMalloc; #[global_allocator] @@ -143,6 +146,12 @@ struct Args { #[clap(long, help = "Enables console syntax highlighting")] color: bool, + #[clap( + long, + help = "Enable Apache Spark-compatible SQL dialect, functions, and expression planning" + )] + spark: bool, + #[clap( short = 'd', long, @@ -243,9 +252,16 @@ async fn main_inner() -> Result<()> { let runtime_env = rt_builder.build_arc()?; + let mut state_builder = SessionStateBuilder::new() + .with_config(session_config) + .with_runtime_env(runtime_env) + .with_default_features(); + if args.spark { + state_builder = state_builder.with_spark_features(); + } + // enable dynamic file query - let ctx = SessionContext::new_with_config_rt(session_config, runtime_env) - .enable_url_table(); + let ctx = SessionContext::new_with_state(state_builder.build()).enable_url_table(); ctx.refresh_catalogs().await?; // install dynamic catalog provider that can register required object stores ctx.register_catalog_list(Arc::new(DynamicObjectStoreCatalog::new( @@ -350,6 +366,10 @@ fn get_session_config(args: &Args) -> Result { config_options.format.null = String::from("NULL"); } + if args.spark { + config_options.sql_parser.dialect = Dialect::Spark; + } + let mut session_config = SessionConfig::from(config_options).with_information_schema(true); diff --git a/datafusion-cli/tests/cli_integration.rs b/datafusion-cli/tests/cli_integration.rs index 4dc244445a2eb..9bfeb65278a6d 100644 --- a/datafusion-cli/tests/cli_integration.rs +++ b/datafusion-cli/tests/cli_integration.rs @@ -173,6 +173,37 @@ fn cli_quick_test<'a>( assert_cmd_snapshot!(cmd); } +#[test] +fn spark_features_require_opt_in() { + let dialect_query = "SELECT CAST(1 AS LONG);"; + let spark_query = "SELECT concat_ws(',', array(1, 2)), CAST(1 AS LONG);"; + + let output = cli() + .args(["-q", "--command", dialect_query]) + .output() + .expect("failed to run datafusion-cli without --spark"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!output.status.success(), "query unexpectedly succeeded"); + assert!( + stdout.contains("Unsupported SQL type LONG"), + "expected Spark SQL syntax to be unavailable without --spark, got:\n{stdout}" + ); + + let output = cli() + .args(["-q", "--spark", "--format", "csv", "--command", spark_query]) + .output() + .expect("failed to run datafusion-cli with --spark"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success(), + "query failed with --spark, got:\n{stdout}" + ); + assert!( + stdout.lines().any(|line| line == "\"1,2\",1"), + "expected Spark planner and dialect result, got:\n{stdout}" + ); +} + /// Read data piped into the CLI via the `/dev/stdin` pseudo-path. /// /// Unix-only: `/dev/stdin` does not exist on Windows. This drives the real diff --git a/datafusion/spark/README.md b/datafusion/spark/README.md index 7cb24084cd228..0ad21604ce62e 100644 --- a/datafusion/spark/README.md +++ b/datafusion/spark/README.md @@ -23,6 +23,18 @@ under the License. This crate is a submodule of DataFusion that provides [Apache Spark] compatible expressions for use with DataFusion. +## Using the DataFusion CLI + +Pass `--spark` to `datafusion-cli` to enable the Spark SQL dialect and register +the Spark-compatible scalar, aggregate, and window functions for the session: + +```shell +datafusion-cli --spark +``` + +The option is disabled by default because Spark-compatible functions can +override DataFusion functions with the same name. + [apache arrow]: https://arrow.apache.org/ [apache datafusion]: https://datafusion.apache.org/ [apache spark]: https://spark.apache.org/ diff --git a/datafusion/sqllogictest/test_files/spark/README.md b/datafusion/sqllogictest/test_files/spark/README.md index e61001c6e42e5..a4550213fa679 100644 --- a/datafusion/sqllogictest/test_files/spark/README.md +++ b/datafusion/sqllogictest/test_files/spark/README.md @@ -52,6 +52,10 @@ statement ok set datafusion.execution.enable_ansi_mode = false; ``` +Run `datafusion-cli --spark` to explore Spark-compatible functions +interactively. This enables the Spark SQL dialect and registers the +Spark-compatible function library for the CLI session. + ### Finding Test Cases To verify and compare function behavior at a minimum, you can refer to the following documentation sources: diff --git a/docs/source/user-guide/cli/usage.md b/docs/source/user-guide/cli/usage.md index 75c6698f007a5..24811a3ce9099 100644 --- a/docs/source/user-guide/cli/usage.md +++ b/docs/source/user-guide/cli/usage.md @@ -52,6 +52,8 @@ Options: [possible values: numbers(0/10/...), inf(no limit)] [default: 40] --color Enables console syntax highlighting + --spark + Enable Apache Spark-compatible SQL dialect, functions, and expression planning -d, --disk-limit Available disk space for spilling queries (e.g. '10g'), default to None (uses DataFusion's default value of '100g') --object-store-profiling @@ -63,6 +65,18 @@ Options: Print version ``` +## Apache Spark compatibility + +Use `--spark` to enable the Apache Spark SQL dialect, expression planner, and +function library for the CLI session: + +```bash +datafusion-cli --spark -c "SELECT next_day('2015-07-27'::DATE, 'Sun'::STRING);" +``` + +Spark compatibility is disabled by default. Enabling it overrides DataFusion +functions with the same name for the duration of the session. + ## Commands Available commands inside DataFusion CLI are: From 7aa5c8da4f1e06a636d0a9da2c5817f23edec9f9 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 14 Aug 2026 13:59:26 +0000 Subject: [PATCH 28/50] Docs: document manual catalog impls and how compilation got faster (#24362) ## Which issue does this PR close? - Follow on to https://github.com/apache/datafusion/pull/24325 - Follow on to https://github.com/apache/datafusion/pull/24326 - Follow on to https://github.com/apache/datafusion/pull/24329 - Follow on to https://github.com/apache/datafusion/pull/24330 ## Rationale for this change @Dandandan came up with a very clever way to improve compile time, see https://github.com/apache/datafusion/issues/13814#issuecomment-5281799225 However, the workaround (to make a special future and implement the async trait manually) is not obvious (at least to me) and I worry it might get removed in the future by accident, as we don't have any checks to prevent the regression Thus I think some comments explaining the pattern and how it would work will help avoid such a problem ## What changes are included in this PR? Add comments explaining why the code is done this way ## Are these changes tested? Comments only ## Are there any user-facing changes? No --- datafusion/catalog-listing/src/table.rs | 6 ++++++ datafusion/catalog/src/cte_worktable.rs | 4 ++++ datafusion/catalog/src/memory/table.rs | 8 ++++++++ datafusion/catalog/src/stream.rs | 8 ++++++++ datafusion/catalog/src/streaming.rs | 2 ++ datafusion/catalog/src/view.rs | 2 ++ datafusion/core/src/dataframe/mod.rs | 2 ++ datafusion/core/src/datasource/dynamic_file.rs | 2 ++ .../core/src/datasource/file_format/options.rs | 12 ++++++++++++ datafusion/core/src/datasource/listing/table.rs | 4 ++++ .../core/src/datasource/listing_table_factory.rs | 2 ++ datafusion/core/src/datasource/provider.rs | 2 ++ datafusion/core/src/execution/session_state.rs | 4 ++++ datafusion/core/src/physical_planner.rs | 2 ++ datafusion/core/src/test_util/mod.rs | 2 ++ datafusion/session/src/table.rs | 6 ++++++ 16 files changed, 68 insertions(+) diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs index 46fab1f9cbff4..498640ba0ff32 100644 --- a/datafusion/catalog-listing/src/table.rs +++ b/datafusion/catalog-listing/src/table.rs @@ -490,6 +490,8 @@ impl TableProvider for ListingTable { TableType::Base } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, @@ -507,6 +509,8 @@ impl TableProvider for ListingTable { self.scan_boxed(state, projection, filters, limit) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn scan_with_args<'a, 'life0, 'life1, 'async_trait>( &'life0 self, state: &'life1 dyn Session, @@ -549,6 +553,8 @@ impl TableProvider for ListingTable { self.definition.as_deref() } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn insert_into<'life0, 'life1, 'async_trait>( &'life0 self, state: &'life1 dyn Session, diff --git a/datafusion/catalog/src/cte_worktable.rs b/datafusion/catalog/src/cte_worktable.rs index 37f66b8c56a3e..6690fad24c736 100644 --- a/datafusion/catalog/src/cte_worktable.rs +++ b/datafusion/catalog/src/cte_worktable.rs @@ -80,6 +80,8 @@ impl TableProvider for CteWorkTable { TableType::Temporary } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, @@ -97,6 +99,8 @@ impl TableProvider for CteWorkTable { self.scan_boxed(state, projection, filters, limit) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn scan_with_args<'a, 'life0, 'life1, 'async_trait>( &'life0 self, state: &'life1 dyn Session, diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rs index 79c6e44933110..8bb45e945d242 100644 --- a/datafusion/catalog/src/memory/table.rs +++ b/datafusion/catalog/src/memory/table.rs @@ -186,6 +186,8 @@ impl TableProvider for MemTable { TableType::Base } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, @@ -217,6 +219,8 @@ impl TableProvider for MemTable { /// * A plan that returns the number of rows written. /// /// [`SessionState`]: https://docs.rs/datafusion/latest/datafusion/execution/session_state/struct.SessionState.html + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn insert_into<'life0, 'life1, 'async_trait>( &'life0 self, state: &'life1 dyn Session, @@ -235,6 +239,8 @@ impl TableProvider for MemTable { self.column_defaults.get(column) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn delete_from<'life0, 'life1, 'async_trait>( &'life0 self, state: &'life1 dyn Session, @@ -248,6 +254,8 @@ impl TableProvider for MemTable { self.delete_from_boxed(state, filters) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn update<'life0, 'life1, 'async_trait>( &'life0 self, state: &'life1 dyn Session, diff --git a/datafusion/catalog/src/stream.rs b/datafusion/catalog/src/stream.rs index f01874612e4d5..86bd4e81dd134 100644 --- a/datafusion/catalog/src/stream.rs +++ b/datafusion/catalog/src/stream.rs @@ -49,6 +49,8 @@ pub struct StreamTableFactory {} #[async_trait] impl TableProviderFactory for StreamTableFactory { + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn create<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, state: &'life1 dyn Session, @@ -348,6 +350,8 @@ impl TableProvider for StreamTable { TableType::Base } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, @@ -365,6 +369,8 @@ impl TableProvider for StreamTable { self.scan_boxed(state, projection, filters, limit) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn insert_into<'life0, 'life1, 'async_trait>( &'life0 self, state: &'life1 dyn Session, @@ -489,6 +495,8 @@ impl DataSink for StreamWrite { self.0.source.schema() } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn write_all<'life0, 'life1, 'async_trait>( &'life0 self, data: SendableRecordBatchStream, diff --git a/datafusion/catalog/src/streaming.rs b/datafusion/catalog/src/streaming.rs index bb70270ce9157..7b466d8a59c7f 100644 --- a/datafusion/catalog/src/streaming.rs +++ b/datafusion/catalog/src/streaming.rs @@ -123,6 +123,8 @@ impl TableProvider for StreamingTable { TableType::View } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, diff --git a/datafusion/catalog/src/view.rs b/datafusion/catalog/src/view.rs index ac0f74c181956..32af208984d95 100644 --- a/datafusion/catalog/src/view.rs +++ b/datafusion/catalog/src/view.rs @@ -96,6 +96,8 @@ impl TableProvider for ViewTable { Ok(vec![TableProviderFilterPushDown::Exact; filters.len()]) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs index 8be23f927edba..5ca40c854b4df 100644 --- a/datafusion/core/src/dataframe/mod.rs +++ b/datafusion/core/src/dataframe/mod.rs @@ -2717,6 +2717,8 @@ impl TableProvider for DataFrameTableProvider { self.table_type } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, diff --git a/datafusion/core/src/datasource/dynamic_file.rs b/datafusion/core/src/datasource/dynamic_file.rs index f670c9cc444aa..0212222cdd1cb 100644 --- a/datafusion/core/src/datasource/dynamic_file.rs +++ b/datafusion/core/src/datasource/dynamic_file.rs @@ -54,6 +54,8 @@ impl DynamicListTableFactory { #[async_trait] impl UrlTableFactory for DynamicListTableFactory { + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn try_new<'life0, 'life1, 'async_trait>( &'life0 self, url: &'life1 str, diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs index 4babf4a421f4e..f2903405c8204 100644 --- a/datafusion/core/src/datasource/file_format/options.rs +++ b/datafusion/core/src/datasource/file_format/options.rs @@ -604,6 +604,8 @@ pub trait ReadOptions<'a> { } /// helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function. + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn _get_resolved_schema<'life0, 'async_trait>( &'a self, config: &'life0 SessionConfig, @@ -662,6 +664,8 @@ impl ReadOptions<'_> for CsvReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn get_resolved_schema<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 SessionConfig, @@ -713,6 +717,8 @@ impl ReadOptions<'_> for ParquetReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn get_resolved_schema<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 SessionConfig, @@ -751,6 +757,8 @@ impl ReadOptions<'_> for JsonReadOptions<'_> { .with_file_sort_order(self.file_sort_order.clone()) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn get_resolved_schema<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 SessionConfig, @@ -785,6 +793,8 @@ impl ReadOptions<'_> for AvroReadOptions<'_> { .with_table_partition_cols(self.table_partition_cols.clone()) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn get_resolved_schema<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 SessionConfig, @@ -818,6 +828,8 @@ impl ReadOptions<'_> for ArrowReadOptions<'_> { .with_table_partition_cols(self.table_partition_cols.clone()) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn get_resolved_schema<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 SessionConfig, diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index 7b435822f275f..f5a76c2cf579d 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -48,6 +48,8 @@ pub trait ListingTableConfigExt { #[async_trait] impl ListingTableConfigExt for ListingTableConfig { + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn infer_options<'life0, 'async_trait>( self, state: &'life0 dyn Session, @@ -59,6 +61,8 @@ impl ListingTableConfigExt for ListingTableConfig { infer_options_boxed(self, state) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn infer<'life0, 'async_trait>( self, state: &'life0 dyn Session, diff --git a/datafusion/core/src/datasource/listing_table_factory.rs b/datafusion/core/src/datasource/listing_table_factory.rs index 116b8985b4115..1e597e38fb5b1 100644 --- a/datafusion/core/src/datasource/listing_table_factory.rs +++ b/datafusion/core/src/datasource/listing_table_factory.rs @@ -51,6 +51,8 @@ impl ListingTableFactory { #[async_trait] impl TableProviderFactory for ListingTableFactory { + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn create<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, state: &'life1 dyn Session, diff --git a/datafusion/core/src/datasource/provider.rs b/datafusion/core/src/datasource/provider.rs index 84bb338835c86..e574042813a7b 100644 --- a/datafusion/core/src/datasource/provider.rs +++ b/datafusion/core/src/datasource/provider.rs @@ -49,6 +49,8 @@ impl DefaultTableFactory { #[async_trait] impl TableProviderFactory for DefaultTableFactory { + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn create<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, state: &'life1 dyn Session, diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 99b16857bb1bc..a9d6198f953ed 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -292,6 +292,8 @@ impl Session for SessionState { SessionState::statistics_registry(self) } + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn create_physical_plan<'life0, 'life1, 'async_trait>( &'life0 self, logical_plan: &'life1 LogicalPlan, @@ -2358,6 +2360,8 @@ struct DefaultQueryPlanner {} #[async_trait] impl QueryPlanner for DefaultQueryPlanner { /// Given a `LogicalPlan`, create an [`ExecutionPlan`] suitable for execution + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, logical_plan: &'life1 LogicalPlan, diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 6e50ec8aa4298..77f8238eddd4c 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -154,6 +154,8 @@ pub struct DefaultPhysicalPlanner { #[async_trait] impl PhysicalPlanner for DefaultPhysicalPlanner { /// Create a physical plan from a logical plan + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn create_physical_plan<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, logical_plan: &'life1 LogicalPlan, diff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rs index 036988c8fcc76..bcb38b97ceab8 100644 --- a/datafusion/core/src/test_util/mod.rs +++ b/datafusion/core/src/test_util/mod.rs @@ -184,6 +184,8 @@ pub struct TestTableFactory {} #[async_trait] impl TableProviderFactory for TestTableFactory { + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn create<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, session: &'life1 dyn Session, diff --git a/datafusion/session/src/table.rs b/datafusion/session/src/table.rs index 9dbda40b9bc7a..c1a1cd45401c0 100644 --- a/datafusion/session/src/table.rs +++ b/datafusion/session/src/table.rs @@ -353,6 +353,8 @@ pub trait TableProvider: Any + Debug + Sync + Send { /// /// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64). /// Empty `filters` deletes all rows. + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn delete_from<'life0, 'life1, 'async_trait>( &'life0 self, _state: &'life1 dyn Session, @@ -373,6 +375,8 @@ pub trait TableProvider: Any + Debug + Sync + Send { /// /// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64). /// Empty `filters` updates all rows. + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn update<'life0, 'life1, 'async_trait>( &'life0 self, _state: &'life1 dyn Session, @@ -409,6 +413,8 @@ pub trait TableProvider: Any + Debug + Sync + Send { /// The `clauses` describe the WHEN MATCHED / WHEN NOT MATCHED actions. /// /// Returns an [`ExecutionPlan`] producing a single row with `count` (UInt64). + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // fn merge_into<'life0, 'life1, 'async_trait>( &'life0 self, _state: &'life1 dyn Session, From a7cebaec0846270ec0cda4055d04a467d498eb2f Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Fri, 14 Aug 2026 15:00:29 +0000 Subject: [PATCH 29/50] fix(parquet): don't runtime-prune row groups while a page-index RowSelection is live (#24355) (#24359) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #24355. ## Rationale for this change With `pushdown_filters = true` + dynamic filter pushdown (on by default), a query `SELECT b FROM t WHERE ORDER BY b LIMIT k` can silently return **wrong results** — rows satisfying the predicate are dropped and replaced by later ones, no error. Root cause (thanks to @adriangb's report + fixture in #24355): the push decoder carries one flat `RowSelection` over the concatenation of the *remaining* row groups. At a row-group boundary the runtime pruner drops row groups the dynamic predicate proves unwinnable and rebuilds the decoder: ```rust decoder.into_builder()?.with_row_groups(new_indices).build() ``` `with_row_groups(new_indices)` removes row groups **without slicing the carried `RowSelection` to match**, so the selectors intended for a dropped RG are applied to the next surviving one. In the fixture, page-index pruning leaves RG 1 with `skip 50, select 50`; after the TopK threshold prunes RG 1 and RG 2, the survivor RG 3 is decoded under RG 1's selection and its first 50 rows (`b = 0..49`, the correct answer) are wrongly skipped. This is a second, independent instance of the drift family in #24352/#24354; it is **not** fixed by #24354. ## What changes are included in this PR? - `opener/mod.rs`: **decline to build the runtime `RowGroupPruner` when a page-index `RowSelection` is present.** With no pruner there is no boundary rebuild, so the carried selection is never applied to the wrong row groups. This mirrors `PreparedAccessPlan::reorder_by_statistics`, which already bails when a row selection is present (`"Skipping RG reorder: row_selection present"`) because remapping the selection is too complex. This is the minimal, DataFusion-side stop-the-bleeding fix. The proper fix is upstream in arrow-rs: apache/arrow-rs#10624 proposes letting the push decoder carry **row-group-local** `RowSelection`s (`with_row_group_selections`) that are preserved across rebuilds, so dropping a row group keeps every survivor's selection aligned by construction — no global-selection slicing to get wrong, and no parallel `rg_plan` to drift (the #24352 path). DataFusion tracks that migration in #24358; this guard is removed once it lands. ## Are these changes tested? - Adds an slt regression test in `dynamic_row_group_pruning.slt` (the reporter's fixture via `generate_series` + `COPY`). It **fails on `main`** (returns `50..54` instead of `0..4`) and passes with this change. - Updates the existing rust integration test that previously asserted the runtime pruner **coexists** with a page-index selection (`dynamic_rg_pruning_coexists_with_page_index_row_selection`, `row_groups_pruned_dynamic_filter >= 1`). Since this PR intentionally disables the pruner in that case, it is renamed to `dynamic_rg_pruning_disabled_when_page_index_row_selection_present` and now asserts `row_groups_pruned_dynamic_filter == 0` while results stay correct and page-index pruning still runs. (That old test passed only because its scenario happened not to expose the bug — the misapplied selection fell outside the top-k.) - The other dynamic-prune tests are unaffected — they have no row selection, so the pruner is created as before. ## Are there any user-facing changes? Fixes silently-wrong results. Runtime row-group pruning is skipped for scans that also have a page-index row selection (correctness over a pruning optimization); this is undone once #24358 lands. cc @alamb @adriangb @hhhizzz --- .../parquet/dynamic_row_group_pruning.rs | 72 ++++++++------ .../datasource-parquet/src/opener/mod.rs | 58 ++++++++---- .../test_files/dynamic_row_group_pruning.slt | 93 +++++++++++++++++++ 3 files changed, 176 insertions(+), 47 deletions(-) diff --git a/datafusion/core/tests/parquet/dynamic_row_group_pruning.rs b/datafusion/core/tests/parquet/dynamic_row_group_pruning.rs index 917faaaa5ce64..5ee42b30674bf 100644 --- a/datafusion/core/tests/parquet/dynamic_row_group_pruning.rs +++ b/datafusion/core/tests/parquet/dynamic_row_group_pruning.rs @@ -35,6 +35,8 @@ use std::sync::Arc; use arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema}; +use datafusion::prelude::SessionConfig; + use crate::parquet::Unit::RowGroup; use crate::parquet::{ContextWithParquet, Scenario}; @@ -297,9 +299,18 @@ fn build_five_thousand_row_rgs(schema: &Arc) -> Vec { .collect() } -/// Co-existence test for **page-index `RowSelection`** + dynamic RG -/// pruning. Tests that the `into_builder` rebuild preserves the -/// `RowSelection` derived from page-index pruning across RG drops. +/// Regression test for : +/// when a page-index `RowSelection` is live, the runtime dynamic row-group +/// pruner is intentionally **not built**, so its `into_builder` rebuild can +/// never drop a row group without slicing the carried selection (which would +/// silently return wrong rows). Correctness is bought at the cost of the +/// dynamic-pruning optimization for this scan. +/// +/// The behavior asserted below (pruner disabled → +/// `row_groups_pruned_dynamic_filter == 0`) is expected to change once the +/// proper upstream fix lands, which keeps both mechanisms: +/// (tracked on the +/// DataFusion side in ). /// /// Layout: 5 RGs × 1000 rows, with `data_page_row_count_limit=100` so /// each RG has 10 pages of 100 rows. @@ -309,17 +320,14 @@ fn build_five_thousand_row_rgs(schema: &Arc) -> Vec { /// first 5 pages (values 0..500) are pruned, the last 5 (500..1000) /// are scanned. RGs 1..4 keep all their pages (every page has /// `max >= 500`). The decoder receives a `RowSelection` that masks -/// out those first 5 pages of RG 0. -/// - `ORDER BY v DESC LIMIT 5` fills the TopK heap from RG 4 -/// (`max=4999`); the tightened threshold (≥ 4995) then proves RGs -/// 0..3 unreachable and the runtime pruner drops them in one -/// `into_builder` rebuild. -/// -/// If `into_builder` did **not** preserve the row selection (or -/// truncated / shifted it incorrectly), either the result rows would -/// drift or the count of pruned pages would drop to zero. +/// out those first 5 pages of RG 0 — its presence is what suppresses +/// the runtime pruner. +/// - `ORDER BY v DESC LIMIT 5` would let the tightened TopK threshold +/// (≥ 4995) prune RGs 0..3, but because a row selection is present the +/// runtime pruner is never created, so `row_groups_pruned_dynamic_filter` +/// stays 0. Results are still correct and page-index pruning still runs. #[tokio::test] -async fn dynamic_rg_pruning_coexists_with_page_index_row_selection() { +async fn dynamic_rg_pruning_disabled_when_page_index_row_selection_present() { let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int64, false)])); let batches = build_five_thousand_row_rgs(&schema); @@ -348,12 +356,9 @@ async fn dynamic_rg_pruning_coexists_with_page_index_row_selection() { ); } - // Page-index pruning must have engaged: RG 0's first 5 pages are - // entirely < 500. If `into_builder` dropped the row-selection state, - // this metric would still report the original count (it is captured - // at file open). Combined with the dynamic-pruner assertion below it - // proves both mechanisms were active and that the rebuild left the - // selection coherent — otherwise the result rows above would drift. + // Page-index pruning still engages: RG 0's first 5 pages are entirely + // < 500. #24355 only suppresses the *runtime* row-group pruner, not + // page-index pruning, so this must remain non-zero. let pages_pruned = output.metric_value("page_index_pages_pruned").unwrap_or(0); assert!( pages_pruned >= 5, @@ -362,13 +367,18 @@ async fn dynamic_rg_pruning_coexists_with_page_index_row_selection() { output.description(), ); + // The runtime dynamic pruner must be disabled while a page-index row + // selection is live (#24355): with no pruner there is no rebuild that + // could misapply the carried selection. Before the fix the pruner ran + // and this metric was >= 1. let pruned = output .row_groups_pruned_dynamic_filter() .expect("`row_groups_pruned_dynamic_filter` metric must be registered"); - assert!( - pruned >= 1, - "with TopK + tight threshold the runtime pruner must skip at least \ - one row group; pruned={pruned}\n{}", + assert_eq!( + pruned, + 0, + "runtime row-group pruning must be skipped when a page-index row \ + selection is present; pruned={pruned}\n{}", output.description(), ); } @@ -647,12 +657,20 @@ async fn topk_pushdown_does_not_reread_delivered_row_group() { // `RowGroup(2048)` writes one row group per 2048-row batch (4 RGs) and // enables `pushdown_filters`, required for the dynamic filter to reach the - // parquet scan. - let mut ctx = ContextWithParquet::with_custom_data( + // parquet scan. Page-index reading is disabled: this test exercises the + // #24352 empty-row-group / rg_plan-sync path, which is row-filter-driven and + // does not need the page index. With the page index on, `search_phrase <> ''` + // produces an intra-row-group `RowSelection`, and #24355 disables the runtime + // pruner whenever a row selection is present — which would stop this test + // from exercising the dynamic pruner at all. + let mut config = SessionConfig::new(); + config.options_mut().execution.parquet.enable_page_index = false; + let mut ctx = ContextWithParquet::with_config( Scenario::Int, RowGroup(2048), - Arc::clone(&schema), - batches, + config, + Some(Arc::clone(&schema)), + Some(batches), ) .await; diff --git a/datafusion/datasource-parquet/src/opener/mod.rs b/datafusion/datasource-parquet/src/opener/mod.rs index a57f4695b55e3..693e9bd2cbf31 100644 --- a/datafusion/datasource-parquet/src/opener/mod.rs +++ b/datafusion/datasource-parquet/src/opener/mod.rs @@ -1435,7 +1435,7 @@ impl RowGroupsPrunedParquetOpen { prepared.virtual_state.as_deref(), )?; - let (decoder, rg_plan) = { + let (decoder, rg_plan, has_row_selection) = { let pushdown_predicate = prepared .pushdown_filters .then_some(prepared.predicate.as_ref()) @@ -1464,6 +1464,18 @@ impl RowGroupsPrunedParquetOpen { }; let prepared_access_plan = prepare_access_plan(access_plan)?; + // #24355: a row selection (from page-index pruning, or an externally + // supplied `ParquetRowSelection`) is carried by the decoder as one + // flat selection over the concatenation of the remaining row groups. + // The runtime pruner's `into_builder().with_row_groups(...)` rebuild + // drops row groups without slicing that selection to match, so record + // whether a selection is present and disable runtime pruning below + // when it is (mirroring `reorder_by_statistics`, which also bails when + // a row selection is present). The proper fix that keeps pruning + // under a live selection is tracked in + // https://github.com/apache/arrow-rs/issues/10624 / + // https://github.com/apache/datafusion/issues/24358. + let has_row_selection = prepared_access_plan.row_selection.is_some(); let rg_plan: VecDeque = prepared_access_plan .row_group_indexes .iter() @@ -1482,7 +1494,7 @@ impl RowGroupsPrunedParquetOpen { } } - (builder.build()?, rg_plan) + (builder.build()?, rg_plan, has_row_selection) }; let predicate_cache_inner_records = @@ -1504,24 +1516,30 @@ impl RowGroupsPrunedParquetOpen { // via the `DynamicFilterTracker` watch channel (#22460), so detecting // a threshold change is a single atomic load — not a tree walk per // RG check. - let row_group_pruner = match (&prepared.predicate, rg_plan.len() > 1) { - (Some(predicate), true) - if matches!( - DynamicFilterTracking::classify(predicate), - DynamicFilterTracking::Watching(_) - ) => - { - Some(RowGroupPruner::new( - Arc::clone(predicate), - Arc::clone(&prepared.physical_file_schema), - Arc::clone(reader_metadata.metadata()), - prepared.predicate_creation_errors.clone(), - prepared.file_metrics.predicate_evaluation_errors.clone(), - prepared.max_in_list_size, - )) - } - _ => None, - }; + // Also disabled when a row selection is live (#24355) — page-index + // pruning is the common source: the pruner rebuilds the decoder via + // `with_row_groups(...)`, which drops row groups without slicing the + // carried selection to match, so pruning under a live selection returns + // wrong results. Decline to prune in that case. + let row_group_pruner = + match (&prepared.predicate, rg_plan.len() > 1, has_row_selection) { + (Some(predicate), true, false) + if matches!( + DynamicFilterTracking::classify(predicate), + DynamicFilterTracking::Watching(_) + ) => + { + Some(RowGroupPruner::new( + Arc::clone(predicate), + Arc::clone(&prepared.physical_file_schema), + Arc::clone(reader_metadata.metadata()), + prepared.predicate_creation_errors.clone(), + prepared.file_metrics.predicate_evaluation_errors.clone(), + prepared.max_in_list_size, + )) + } + _ => None, + }; let row_groups_pruned_dynamic = prepared .file_metrics .row_groups_pruned_dynamic_filter diff --git a/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt b/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt index 81d9511839725..c6700ebf0b97c 100644 --- a/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt +++ b/datafusion/sqllogictest/test_files/dynamic_row_group_pruning.slt @@ -194,3 +194,96 @@ RESET datafusion.optimizer.enable_dynamic_filter_pushdown; statement ok RESET datafusion.optimizer.enable_topk_dynamic_filter_pushdown; + +# Regression test for a scan where two pruning mechanisms are live at once: +# page-index pruning leaves an intra-row-group `RowSelection`, and a TopK +# dynamic filter prunes row groups at runtime. The property under test is that +# a `WHERE a >= 50 ORDER BY b ASC LIMIT 5` query returns the correct top-5 by +# `b` while the dynamic predicate prunes a row group during the application of +# multiple predicates. Layout (RG size 100): +# RG 0: b=1000..1099, a=100..199 (a>=50 keeps all) +# RG 1: b=2000..2099, a=0..99 (a>=50 keeps rows 50..99 — page-index prunes +# the first 5 pages, leaving `skip 50, select 50`) +# RG 2: b=3000..3099, a=100..199 (keeps all) +# RG 3: b=0..99, a=100..199 (keeps all) +# The correct top-5 by `b` (0..4) lives entirely in RG 3. +# `data_page_row_count_limit`/`write_batch_size` force multiple pages per RG so +# page-index pruning can produce the intra-RG selection. +# Tracking issue for the behavior change (keeping both mechanisms): +# https://github.com/apache/arrow-rs/issues/10624 / +# https://github.com/apache/datafusion/issues/24358 +statement ok +set datafusion.execution.target_partitions = 1; + +statement ok +set datafusion.execution.parquet.pushdown_filters = true; + +statement ok +CREATE TABLE rgsel_src AS +SELECT + CAST(CASE WHEN i / 100 = 1 THEN i % 100 ELSE 100 + (i % 100) END AS BIGINT) AS a, + CAST(CASE + WHEN i < 100 THEN 1000 + i + WHEN i < 200 THEN 2000 + (i - 100) + WHEN i < 300 THEN 3000 + (i - 200) + ELSE (i - 300) + END AS BIGINT) AS b +FROM generate_series(0, 399) AS t(i); + +statement ok +COPY (SELECT * FROM rgsel_src) +TO 'test_files/scratch/dynamic_row_group_pruning/rgsel.parquet' +STORED AS PARQUET +OPTIONS ( + 'format.max_row_group_size' '100', + 'format.data_page_row_count_limit' '10', + 'format.write_batch_size' '10' +); + +statement ok +drop table rgsel_src; + +statement ok +CREATE EXTERNAL TABLE rgsel (a BIGINT NOT NULL, b BIGINT NOT NULL) +STORED AS PARQUET +LOCATION 'test_files/scratch/dynamic_row_group_pruning/rgsel.parquet'; + +# The correct top-5 by `b` among rows with `a >= 50` is b = 0..4 (they live in +# RG 3, all of whose rows satisfy `a >= 50`). +query I +SELECT b FROM rgsel WHERE a >= 50 ORDER BY b ASC LIMIT 5; +---- +0 +1 +2 +3 +4 + +# The same query without filter pushdown never engages the runtime pruner, so +# its answer is the ground truth the pushdown path above must match. +statement ok +set datafusion.execution.parquet.pushdown_filters = false; + +query I +SELECT b FROM rgsel WHERE a >= 50 ORDER BY b ASC LIMIT 5; +---- +0 +1 +2 +3 +4 + +statement ok +set datafusion.execution.parquet.pushdown_filters = true; + +statement ok +drop table rgsel; + +# The SLT runner sets `target_partitions` to 4 instead of using the default, so +# restore it explicitly rather than RESET (which would revert to the system +# default = num_cpus and leak modified config out of this file). +statement ok +set datafusion.execution.target_partitions = 4; + +statement ok +RESET datafusion.execution.parquet.pushdown_filters; From 22518be3c0bacd42893bc5db0bac78a82209f275 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 14 Aug 2026 17:07:04 +0000 Subject: [PATCH 30/50] fix: preserve the input list's inner field in array_append/prepend/replace (#24365) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #24347. ## Rationale for this change `array_append`, `array_prepend`, `array_replace`, `array_replace_n` and `array_replace_all` promise the input list type verbatim — inner field name, nullability and metadata included — but their kernels rebuilt the output's inner field from scratch with `Field::new_list_field(..., true)`. On debug builds this trips the return-type assertion from #17515; on release builds it silently yields a batch whose inner field disagrees with the schema the planner recorded. This is the other half of #24341, whose `array_slice` part was fixed in #24345. The field-name symptom is a 55.0.0 regression from the same commit (5b228570, #20945); the non-nullable symptom is not a regression. Unlike `array_slice`, threading the input's field through is not sufficient here: the appended, prepended or replacement element can itself be null, so a promise cloned from a `List(non-null T)` input is wrong at the source and arrow rejects the array with `Non-nullable field of ListArray cannot contain nulls`. ## What changes are included in this PR? Each of the five functions now implements `return_field_from_args`, carrying the input field's name and metadata through while widening `nullable` when the new element's argument is nullable. The kernels build their output from `args.return_field` instead of deriving a field of their own, so promise and payload come from a single source. Nullability is therefore only widened when the result can genuinely contain a null: ```sql array_append(List(non-null Int64), 3) -> List(non-null Int64) array_append(List(non-null Int64), NULL) -> List(Int64) ``` Two paths beyond those listed in the issue turned out to have the same defect and are fixed too: `array_append` / `array_prepend` with a **nested** value type (which delegates to `concat_internal`), and the `LargeList` variants of all five. Since these five now implement `return_field_from_args`, their `return_type` becomes unreachable and returns `internal_err!("return_field_from_args should be used instead")`, matching the guidance on `ScalarUDFImpl::return_type` and the existing convention in `remove.rs` and `map_values.rs`. Two small cleanups while in here: - The `List`/`LargeList` inner-field extraction added to `general_array_slice` by #24345 is now shared as `utils::list_inner_field`, used by all three files. Its error text is unchanged. The `ListView` variant in `general_list_view_array_slice` is deliberately left alone — folding all four variants into one helper would let `general_array_slice` silently accept a `ListView` that its match currently rejects. - `array_concat` is **not** affected and its behaviour is unchanged: it derives a fresh return type via `type_union_resolution` rather than cloning an input's, so it keeps passing `None` to `concat_internal` and deriving the field from the aligned inputs. Behaviour for `Null`-typed array arguments is unchanged in all five functions (`array_replace*` return `Null`, `array_append` / `array_prepend` return `List(element)`). ## Are these changes tested? Yes — 14 new SLT tests across `array_append.slt`, `array_prepend.slt`, `array_replace.slt`, plus two guard cases in `array_concat.slt` pinning down that it is unaffected. `arrow_cast` can express a named inner field (`'List(Int64, field: ''element'')'`), so these reproduce the field-name half of the bug without needing the Spark dialect. Coverage: inner field name preserved, non-nullable inner field preserved, nullability widened only when the new element is nullable (both literal `NULL` and a nullable column), `LargeList`, nested value types, the `max <= 0` short circuit, and a `NULL` `max`. Every one of these queries fails on `main` with the return-type assertion. Also run: `cargo clippy --all-targets --all-features -- -D warnings`, the full sqllogictest suite, and the extended workspace test suite (68 test binaries, 0 failures). The `array_replace` and `array_concat` benchmarks show no regression against `main`. ## Are there any user-facing changes? The five functions now return the inner field they promise instead of a rebuilt one, which is the bug fix. As a consequence, appending or replacing with a nullable element widens the declared inner nullability of the result (`List(non-null Int64)` -> `List(Int64)`), which is required for the result to be representable at all. No breaking changes to public APIs. --- datafusion/functions-nested/src/concat.rs | 140 +++++++++++----- datafusion/functions-nested/src/extract.rs | 11 +- datafusion/functions-nested/src/replace.rs | 157 +++++++++++++----- datafusion/functions-nested/src/utils.rs | 53 +++++- .../test_files/array/array_append.slt | 62 +++++++ .../test_files/array/array_concat.slt | 19 +++ .../test_files/array/array_prepend.slt | 62 +++++++ .../test_files/array/array_replace.slt | 87 ++++++++++ 8 files changed, 505 insertions(+), 86 deletions(-) diff --git a/datafusion/functions-nested/src/concat.rs b/datafusion/functions-nested/src/concat.rs index 5dc437b3c20b5..1f03a0b17014e 100644 --- a/datafusion/functions-nested/src/concat.rs +++ b/datafusion/functions-nested/src/concat.rs @@ -20,26 +20,29 @@ use std::sync::Arc; use crate::make_array::make_array_inner; -use crate::utils::{align_array_dimensions, check_datatypes, make_scalar_function}; +use crate::utils::{ + align_array_dimensions, check_datatypes, list_inner_field, list_type_with_element, + make_scalar_function, +}; use arrow::array::{ Array, ArrayData, ArrayRef, Capacities, GenericListArray, MutableArrayData, OffsetSizeTrait, }; use arrow::buffer::{NullBuffer, OffsetBuffer}; -use arrow::datatypes::{DataType, Field}; +use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::Result; use datafusion_common::utils::{ ListCoercion, base_type, coerced_type_with_base_type_only, }; use datafusion_common::{ cast::as_generic_list_array, - exec_err, plan_err, + exec_err, internal_err, plan_err, utils::{list_ndims, take_function_args}, }; use datafusion_expr::binary::type_union_resolution; use datafusion_expr::{ - ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + Signature, Volatility, }; use datafusion_macros::user_doc; use itertools::Itertools; @@ -104,17 +107,26 @@ impl ScalarUDFImpl for ArrayAppend { &self.signature } - fn return_type(&self, arg_types: &[DataType]) -> Result { - let [array_type, element_type] = take_function_args(self.name(), arg_types)?; - if array_type.is_null() { - Ok(DataType::new_list(element_type.clone(), true)) - } else { - Ok(array_type.clone()) - } + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!("return_field_from_args should be used instead") + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let [array_field, element_field] = + take_function_args(self.name(), args.arg_fields)?; + let data_type = append_prepend_return_type( + array_field.data_type(), + element_field.data_type(), + element_field.is_nullable(), + ); + Ok(Arc::new(Field::new(self.name(), data_type, true))) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - make_scalar_function(array_append_inner)(&args.args) + let return_type = args.return_field.data_type().clone(); + make_scalar_function(|args: &[ArrayRef]| array_append_inner(args, &return_type))( + &args.args, + ) } fn aliases(&self) -> &[String] { @@ -186,17 +198,26 @@ impl ScalarUDFImpl for ArrayPrepend { &self.signature } - fn return_type(&self, arg_types: &[DataType]) -> Result { - let [element_type, array_type] = take_function_args(self.name(), arg_types)?; - if array_type.is_null() { - Ok(DataType::new_list(element_type.clone(), true)) - } else { - Ok(array_type.clone()) - } + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!("return_field_from_args should be used instead") + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let [element_field, array_field] = + take_function_args(self.name(), args.arg_fields)?; + let data_type = append_prepend_return_type( + array_field.data_type(), + element_field.data_type(), + element_field.is_nullable(), + ); + Ok(Arc::new(Field::new(self.name(), data_type, true))) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - make_scalar_function(array_prepend_inner)(&args.args) + let return_type = args.return_field.data_type().clone(); + make_scalar_function(|args: &[ArrayRef]| array_prepend_inner(args, &return_type))( + &args.args, + ) } fn aliases(&self) -> &[String] { @@ -375,13 +396,38 @@ pub fn array_concat_inner(args: &[ArrayRef]) -> Result { args[0].len(), ))) } else if large_list { - concat_internal::(args) + concat_internal::(args, None) + } else { + concat_internal::(args, None) + } +} + +/// Return type shared by `array_append` and `array_prepend`: the input list +/// type, except that its inner field is nullable whenever the appended or +/// prepended element may be null. +fn append_prepend_return_type( + array_type: &DataType, + element_type: &DataType, + element_nullable: bool, +) -> DataType { + if array_type.is_null() { + DataType::new_list(element_type.clone(), true) } else { - concat_internal::(args) + list_type_with_element(array_type, element_nullable) } } -fn concat_internal(args: &[ArrayRef]) -> Result { +/// Concatenates the list arrays in `args` row-wise. +/// +/// `field` is the list field the output must carry. `array_concat` passes `None` +/// because its `return_type` derives a fresh field from the unified element +/// types, which is what deriving the field from the aligned inputs reproduces. +/// `array_append` / `array_prepend` promise their input's field verbatim and so +/// must pass it in explicitly. +fn concat_internal( + args: &[ArrayRef], + field: Option<&FieldRef>, +) -> Result { let args = align_array_dimensions::(args.to_vec())?; let list_arrays = args @@ -438,11 +484,14 @@ fn concat_internal(args: &[ArrayRef]) -> Result { offsets.push(O::usize_as(mutable.len())); } - let data_type = list_arrays[0].value_type(); + let field = match field { + Some(field) => Arc::clone(field), + None => Arc::new(Field::new_list_field(list_arrays[0].value_type(), true)), + }; let data = mutable.freeze(); Ok(Arc::new(GenericListArray::::try_new( - Arc::new(Field::new_list_field(data_type, true)), + field, OffsetBuffer::new(offsets.into()), arrow::array::make_array(data), valid, @@ -451,22 +500,26 @@ fn concat_internal(args: &[ArrayRef]) -> Result { // Kernel functions -fn array_append_inner(args: &[ArrayRef]) -> Result { +fn array_append_inner(args: &[ArrayRef], return_type: &DataType) -> Result { let [array, values] = take_function_args("array_append", args)?; match array.data_type() { DataType::Null => make_array_inner(&[Arc::clone(values)]), - DataType::List(_) => general_append_and_prepend::(args, true), - DataType::LargeList(_) => general_append_and_prepend::(args, true), + DataType::List(_) => general_append_and_prepend::(args, true, return_type), + DataType::LargeList(_) => { + general_append_and_prepend::(args, true, return_type) + } arg_type => exec_err!("array_append does not support type {arg_type}"), } } -fn array_prepend_inner(args: &[ArrayRef]) -> Result { +fn array_prepend_inner(args: &[ArrayRef], return_type: &DataType) -> Result { let [values, array] = take_function_args("array_prepend", args)?; match array.data_type() { DataType::Null => make_array_inner(&[Arc::clone(values)]), - DataType::List(_) => general_append_and_prepend::(args, false), - DataType::LargeList(_) => general_append_and_prepend::(args, false), + DataType::List(_) => general_append_and_prepend::(args, false, return_type), + DataType::LargeList(_) => { + general_append_and_prepend::(args, false, return_type) + } arg_type => exec_err!("array_prepend does not support type {arg_type}"), } } @@ -474,6 +527,7 @@ fn array_prepend_inner(args: &[ArrayRef]) -> Result { fn general_append_and_prepend( args: &[ArrayRef], is_append: bool, + return_type: &DataType, ) -> Result where i64: TryInto, @@ -490,14 +544,22 @@ where (list_array, element_array) }; + let name = if is_append { + "array_append" + } else { + "array_prepend" + }; + let field = list_inner_field(name, return_type)?; + let res = match list_array.value_type() { - DataType::List(_) => concat_internal::(args)?, - DataType::LargeList(_) => concat_internal::(args)?, - data_type => { + DataType::List(_) | DataType::LargeList(_) => { + concat_internal::(args, Some(&field))? + } + _ => { return generic_append_and_prepend::( list_array, element_array, - &data_type, + field, is_append, ); } @@ -516,7 +578,7 @@ where /// /// * `list_array` - A reference to the ListArray to which elements will be appended/prepended. /// * `element_array` - A reference to the Array containing elements to be appended/prepended. -/// * `field` - A reference to the Field describing the data type of the arrays. +/// * `field` - The list field the output must carry, taken from the promised return type. /// * `is_append` - A boolean flag indicating whether to append (`true`) or prepend (`false`) elements. /// /// # Examples @@ -528,7 +590,7 @@ where fn generic_append_and_prepend( list_array: &GenericListArray, element_array: &ArrayRef, - data_type: &DataType, + field: FieldRef, is_append: bool, ) -> Result where @@ -565,7 +627,7 @@ where let data = mutable.freeze(); Ok(Arc::new(GenericListArray::::try_new( - Arc::new(Field::new_list_field(data_type.to_owned(), true)), + field, OffsetBuffer::new(offsets.into()), arrow::array::make_array(data), None, diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs index 8f2ea1f40dcb2..cb7a316b289a9 100644 --- a/datafusion/functions-nested/src/extract.rs +++ b/datafusion/functions-nested/src/extract.rs @@ -47,7 +47,7 @@ use datafusion_expr::{ use datafusion_macros::user_doc; use std::sync::Arc; -use crate::utils::make_scalar_function; +use crate::utils::{list_inner_field, make_scalar_function}; // Create static instances of ScalarUDFs for each function make_udf_expr_and_func!( @@ -624,14 +624,7 @@ where // Carry the input's list field through to the output so that the returned // type matches the one promised by `return_type` / `return_field_from_args`, // including the field name, nullability and metadata. - let field = match array.data_type() { - List(field) | LargeList(field) => Arc::clone(field), - other => { - return internal_err!( - "general_array_slice got unexpected data type: {other}" - ); - } - }; + let field = list_inner_field("general_array_slice", array.data_type())?; // `use_nulls` is false because we never call `try_extend_nulls`: null rows are // emitted as empty slices. Arrow still allocates a validity buffer on its own diff --git a/datafusion/functions-nested/src/replace.rs b/datafusion/functions-nested/src/replace.rs index 71d6f578158f4..4bfd0c0dbecfe 100644 --- a/datafusion/functions-nested/src/replace.rs +++ b/datafusion/functions-nested/src/replace.rs @@ -22,17 +22,20 @@ use arrow::array::{ NullBufferBuilder, OffsetSizeTrait, Scalar, new_null_array, }; use arrow::buffer::OffsetBuffer; -use arrow::datatypes::{DataType, Field}; +use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::cast::as_int64_array; use datafusion_common::utils::ListCoercion; -use datafusion_common::{Result, ScalarValue, exec_err, utils::take_function_args}; +use datafusion_common::{ + Result, ScalarValue, exec_err, internal_err, utils::take_function_args, +}; use datafusion_expr::{ ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, Documentation, - ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, + ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, + Volatility, }; use datafusion_macros::user_doc; -use crate::utils::compare_element_to_list; +use crate::utils::{compare_element_to_list, list_inner_field, list_type_with_element}; use std::sync::Arc; @@ -118,21 +121,28 @@ impl ScalarUDFImpl for ArrayReplace { &self.signature } - fn return_type(&self, args: &[DataType]) -> Result { - Ok(args[0].clone()) + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!("return_field_from_args should be used instead") + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + replace_return_field(self.name(), args.arg_fields) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let return_type = args.return_field.data_type().clone(); let [list_arg, from_arg, to_arg] = take_function_args(self.name(), &args.args)?; let num_rows = args.number_rows; let list_array = list_arg.to_array(num_rows)?; match (from_arg, to_arg) { (ColumnarValue::Scalar(scalar_from), ColumnarValue::Scalar(scalar_to)) => { let result = array_replace_with_scalar_args( + self.name(), &list_array, scalar_from, scalar_to, 1i64, + &return_type, )?; Ok(ColumnarValue::Array(result)) } @@ -140,10 +150,12 @@ impl ScalarUDFImpl for ArrayReplace { let from_array = from_arg.to_array(num_rows)?; let to_array = to_arg.to_array(num_rows)?; let result = array_replace_internal( + self.name(), &list_array, &from_array, &to_array, &[Some(1)], + &return_type, )?; Ok(ColumnarValue::Array(result)) } @@ -217,11 +229,16 @@ impl ScalarUDFImpl for ArrayReplaceN { &self.signature } - fn return_type(&self, args: &[DataType]) -> Result { - Ok(args[0].clone()) + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!("return_field_from_args should be used instead") + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + replace_return_field(self.name(), args.arg_fields) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let return_type = args.return_field.data_type().clone(); let [list_arg, from_arg, to_arg, max_arg] = take_function_args(self.name(), &args.args)?; let num_rows = args.number_rows; @@ -234,15 +251,17 @@ impl ScalarUDFImpl for ArrayReplaceN { ) => { let ScalarValue::Int64(Some(n)) = scalar_max else { return Ok(ColumnarValue::Array(new_null_array( - list_array.data_type(), + &return_type, num_rows, ))); }; let result = array_replace_with_scalar_args( + self.name(), &list_array, scalar_from, scalar_to, *n, + &return_type, )?; Ok(ColumnarValue::Array(result)) } @@ -251,10 +270,12 @@ impl ScalarUDFImpl for ArrayReplaceN { let to_array = to_arg.to_array(num_rows)?; let max_array = max_arg.to_array(num_rows)?; let result = array_replace_n_inner( + self.name(), &list_array, &from_array, &to_array, &max_array, + &return_type, )?; Ok(ColumnarValue::Array(result)) } @@ -326,21 +347,28 @@ impl ScalarUDFImpl for ArrayReplaceAll { &self.signature } - fn return_type(&self, args: &[DataType]) -> Result { - Ok(args[0].clone()) + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!("return_field_from_args should be used instead") + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + replace_return_field(self.name(), args.arg_fields) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let return_type = args.return_field.data_type().clone(); let [list_arg, from_arg, to_arg] = take_function_args(self.name(), &args.args)?; let num_rows = args.number_rows; let list_array = list_arg.to_array(num_rows)?; match (from_arg, to_arg) { (ColumnarValue::Scalar(scalar_from), ColumnarValue::Scalar(scalar_to)) => { let result = array_replace_with_scalar_args( + self.name(), &list_array, scalar_from, scalar_to, i64::MAX, + &return_type, )?; Ok(ColumnarValue::Array(result)) } @@ -348,10 +376,12 @@ impl ScalarUDFImpl for ArrayReplaceAll { let from_array = from_arg.to_array(num_rows)?; let to_array = to_arg.to_array(num_rows)?; let result = array_replace_internal( + self.name(), &list_array, &from_array, &to_array, &[Some(i64::MAX)], + &return_type, )?; Ok(ColumnarValue::Array(result)) } @@ -367,6 +397,24 @@ impl ScalarUDFImpl for ArrayReplaceAll { } } +/// Return field shared by `array_replace`, `array_replace_n` and +/// `array_replace_all`: the input list type, except that its inner field is +/// nullable whenever the replacement element may be null. +fn replace_return_field(name: &str, arg_fields: &[FieldRef]) -> Result { + // `array` is at index 0 and `to` at index 2 for all three functions. + // `from` never contributes values to the output, so `to` is the only + // argument besides `array` that can affect the output's type. + let [array_field, _from_field, to_field, ..] = arg_fields else { + return exec_err!( + "{name} expects at least 3 arguments, got {}", + arg_fields.len() + ); + }; + let data_type = + list_type_with_element(array_field.data_type(), to_field.is_nullable()); + Ok(Arc::new(Field::new(name, data_type, true))) +} + /// For each element of `list_array[i]`, replaces up to `arr_n[i]` occurrences /// of `from_array[i]`, `to_array[i]`. /// @@ -389,6 +437,7 @@ fn general_replace( from_array: &ArrayRef, to_array: &ArrayRef, arr_n: &[Option], + field: FieldRef, ) -> Result { // Build up the offsets for the final output array let mut offsets: Vec = Vec::with_capacity(list_array.len() + 1); @@ -502,7 +551,7 @@ fn general_replace( let data = mutable.freeze(); Ok(Arc::new(GenericListArray::::try_new( - Arc::new(Field::new_list_field(list_array.value_type(), true)), + field, OffsetBuffer::::new(offsets.into()), arrow::array::make_array(data), valid.finish(), @@ -520,10 +569,17 @@ fn general_replace_with_scalar( needle: &Scalar, scalar_to: &ScalarValue, max_replacements: i64, + field: FieldRef, ) -> Result { - // No replacement needed - return unchanged. + // No replacement needed, but the output still has to carry the promised + // field, which may be more nullable than the input's. if max_replacements <= 0 { - return Ok(Arc::new(list_array.clone())); + return Ok(Arc::new(GenericListArray::::try_new( + field, + list_array.offsets().clone(), + Arc::clone(list_array.values()), + list_array.nulls().cloned(), + )?)); } let first_offset = list_array.offsets()[0].to_usize().unwrap(); @@ -598,7 +654,7 @@ fn general_replace_with_scalar( let data = mutable.freeze(); Ok(Arc::new(GenericListArray::::try_new( - Arc::new(Field::new_list_field(list_array.value_type(), true)), + field, OffsetBuffer::new(offsets.into()), arrow::array::make_array(data), list_array.nulls().cloned(), @@ -609,10 +665,12 @@ fn general_replace_with_scalar( /// /// Uses a single bulk `not_distinct` comparison instead of per-row comparisons. fn array_replace_with_scalar_args( + name: &str, list_array: &ArrayRef, scalar_from: &ScalarValue, scalar_to: &ScalarValue, max_replacements: i64, + return_type: &DataType, ) -> Result { // `not_distinct` doesn't support nested types, fall back to the generic array path. if scalar_from.data_type().is_nested() { @@ -620,56 +678,74 @@ fn array_replace_with_scalar_args( let from_array = scalar_from.to_array_of_size(num_rows)?; let to_array = scalar_to.to_array_of_size(num_rows)?; return array_replace_internal( + name, list_array, &from_array, &to_array, &vec![Some(max_replacements); num_rows], + return_type, ); } let needle = Scalar::new(scalar_from.to_array_of_size(1)?); match list_array.data_type() { - DataType::List(_) => { - let list = list_array.as_list::(); - general_replace_with_scalar::(list, &needle, scalar_to, max_replacements) - } - DataType::LargeList(_) => { - let list = list_array.as_list::(); - general_replace_with_scalar::(list, &needle, scalar_to, max_replacements) - } - DataType::Null => Ok(new_null_array(list_array.data_type(), list_array.len())), - array_type => exec_err!("array_replace does not support type '{array_type}'."), + DataType::List(_) => general_replace_with_scalar::( + list_array.as_list::(), + &needle, + scalar_to, + max_replacements, + list_inner_field(name, return_type)?, + ), + DataType::LargeList(_) => general_replace_with_scalar::( + list_array.as_list::(), + &needle, + scalar_to, + max_replacements, + list_inner_field(name, return_type)?, + ), + DataType::Null => Ok(new_null_array(return_type, list_array.len())), + array_type => exec_err!("{name} does not support type '{array_type}'."), } } fn array_replace_internal( + name: &str, array: &ArrayRef, from: &ArrayRef, to: &ArrayRef, arr_n: &[Option], + return_type: &DataType, ) -> Result { match array.data_type() { - DataType::List(_) => { - let list_array = array.as_list::(); - general_replace::(list_array, from, to, arr_n) - } - DataType::LargeList(_) => { - let list_array = array.as_list::(); - general_replace::(list_array, from, to, arr_n) - } - DataType::Null => Ok(new_null_array(array.data_type(), array.len())), - array_type => exec_err!("array_replace does not support type '{array_type}'."), + DataType::List(_) => general_replace::( + array.as_list::(), + from, + to, + arr_n, + list_inner_field(name, return_type)?, + ), + DataType::LargeList(_) => general_replace::( + array.as_list::(), + from, + to, + arr_n, + list_inner_field(name, return_type)?, + ), + DataType::Null => Ok(new_null_array(return_type, array.len())), + array_type => exec_err!("{name} does not support type '{array_type}'."), } } fn array_replace_n_inner( + name: &str, array: &ArrayRef, from: &ArrayRef, to: &ArrayRef, max: &ArrayRef, + return_type: &DataType, ) -> Result { let arr_n = as_int64_array(max)?.iter().collect::>(); - array_replace_internal(array, from, to, &arr_n) + array_replace_internal(name, array, from, to, &arr_n, return_type) } #[cfg(test)] @@ -696,7 +772,14 @@ mod tests { Some(NullBuffer::from(vec![true, false])), )); - let result = array_replace_n_inner(&array, &from, &to, &max)?; + let result = array_replace_n_inner( + "array_replace_n", + &array, + &from, + &to, + &max, + array.data_type(), + )?; let expected = ListArray::from_iter_primitive::(vec![ Some(vec![Some(1), Some(9), Some(3)]), None, diff --git a/datafusion/functions-nested/src/utils.rs b/datafusion/functions-nested/src/utils.rs index 8b413686abcab..9822b6121e695 100644 --- a/datafusion/functions-nested/src/utils.rs +++ b/datafusion/functions-nested/src/utils.rs @@ -19,7 +19,7 @@ use std::sync::Arc; -use arrow::datatypes::{DataType, Field, Fields}; +use arrow::datatypes::{DataType, Field, FieldRef, Fields}; use arrow::array::{ Array, ArrayRef, BooleanArray, Float64Array, GenericListArray, NullBufferBuilder, @@ -35,6 +35,57 @@ use datafusion_common::{Result, ScalarValue, exec_err, internal_err, plan_err}; use datafusion_expr::ColumnarValue; use itertools::Itertools as _; +/// Computes the return type of a function that produces a list with the same +/// inner field as `array_type`, plus an element that may be null when +/// `element_nullable` is set. +/// +/// The inner field is carried over from `array_type` verbatim — name, metadata +/// and all — so that the type promised at planning time is the one the kernel +/// can actually build. Its nullability is widened when `element_nullable` is +/// set, because a nullable new element may introduce nulls into a list whose +/// elements were previously declared non-nullable. +/// +/// Types other than `List`/`LargeList` are returned unchanged; callers handle +/// `Null` themselves and the kernels reject anything else at execution time. +pub(crate) fn list_type_with_element( + array_type: &DataType, + element_nullable: bool, +) -> DataType { + match array_type { + DataType::List(field) => { + DataType::List(widen_nullability(field, element_nullable)) + } + DataType::LargeList(field) => { + DataType::LargeList(widen_nullability(field, element_nullable)) + } + other => other.clone(), + } +} + +fn widen_nullability(field: &FieldRef, nullable: bool) -> FieldRef { + if nullable && !field.is_nullable() { + Arc::new(field.as_ref().clone().with_nullable(true)) + } else { + Arc::clone(field) + } +} + +/// Extracts the inner field of a `List`/`LargeList` type, so that a kernel can +/// build a list array carrying exactly that field. +/// +/// Used both on an input's type and on the type promised by +/// [`ScalarUDFImpl::return_field_from_args`]. Anything else is a bug in the +/// caller's dispatch, hence the internal error; `context` names the kernel so +/// that error identifies where the bad dispatch happened. +/// +/// [`ScalarUDFImpl::return_field_from_args`]: datafusion_expr::ScalarUDFImpl::return_field_from_args +pub(crate) fn list_inner_field(context: &str, data_type: &DataType) -> Result { + match data_type { + DataType::List(field) | DataType::LargeList(field) => Ok(Arc::clone(field)), + other => internal_err!("{context} got unexpected data type: {other}"), + } +} + pub(crate) fn check_datatypes(name: &str, args: &[&ArrayRef]) -> Result<()> { let data_type = args[0].data_type(); if !args.iter().all(|arg| { diff --git a/datafusion/sqllogictest/test_files/array/array_append.slt b/datafusion/sqllogictest/test_files/array/array_append.slt index 50949948c890e..0758a09a4925b 100644 --- a/datafusion/sqllogictest/test_files/array/array_append.slt +++ b/datafusion/sqllogictest/test_files/array/array_append.slt @@ -269,5 +269,67 @@ select array_append(column1, arrow_cast(make_array(1, 11, 111), 'FixedSizeList(3 [[1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [7, 8, 9]] [[4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7], [1, 11, 111]] [[1, 2, 3], [11, 12, 13], [10, 11, 12]] +# the input list's inner field is carried through to the output, so that the +# returned type matches the one promised at planning time +query ?T +select + array_append(arrow_cast(column1, 'List(Int64, field: ''element'')'), 4), + arrow_typeof(array_append(arrow_cast(column1, 'List(Int64, field: ''element'')'), 4)) +from values (make_array(1, 2, 3)); +---- +[1, 2, 3, 4] List(Int64, field: 'element') + +query ?T +select + array_append(arrow_cast(column1, 'LargeList(Int64, field: ''element'')'), 4), + arrow_typeof(array_append(arrow_cast(column1, 'LargeList(Int64, field: ''element'')'), 4)) +from values (make_array(1, 2, 3)); +---- +[1, 2, 3, 4] LargeList(Int64, field: 'element') + +# nested value types go through a different kernel path +query ?T +select + array_append(arrow_cast(column1, 'List(List(Int64), field: ''element'')'), make_array(2)), + arrow_typeof(array_append(arrow_cast(column1, 'List(List(Int64), field: ''element'')'), make_array(2))) +from values (make_array(make_array(1))); +---- +[[1], [2]] List(List(Int64), field: 'element') + +# a non-nullable inner field stays non-nullable when the appended element cannot be null +query ??TT +select + array_append(column1, 4), + array_append(arrow_cast(column1, 'LargeList(non-null Int64)'), 4), + arrow_typeof(array_append(column1, 4)), + arrow_typeof(array_append(arrow_cast(column1, 'LargeList(non-null Int64)'), 4)) +from values + (arrow_cast(make_array(), 'List(non-null Int64)')), + (arrow_cast(NULL, 'List(non-null Int64)')), + (arrow_cast(make_array(1, 2, 3), 'List(non-null Int64)')) +; +---- +[4] [4] List(non-null Int64) LargeList(non-null Int64) +[4] [4] List(non-null Int64) LargeList(non-null Int64) +[1, 2, 3, 4] [1, 2, 3, 4] List(non-null Int64) LargeList(non-null Int64) + +# ... and is widened when the appended element is nullable, since the result +# genuinely contains a null element +query ?T +select + array_append(arrow_cast(make_array(1, 2), 'List(non-null Int64)'), NULL), + arrow_typeof(array_append(arrow_cast(make_array(1, 2), 'List(non-null Int64)'), NULL)); +---- +[1, 2, NULL] List(Int64) + +query ?T +select + array_append(arrow_cast(make_array(1, 2), 'List(non-null Int64)'), column1), + arrow_typeof(array_append(arrow_cast(make_array(1, 2), 'List(non-null Int64)'), column1)) +from values (3), (NULL); +---- +[1, 2, 3] List(Int64) +[1, 2, NULL] List(Int64) + include ./cleanup.slt.part diff --git a/datafusion/sqllogictest/test_files/array/array_concat.slt b/datafusion/sqllogictest/test_files/array/array_concat.slt index 168b307a1e636..5b7985ef6d194 100644 --- a/datafusion/sqllogictest/test_files/array/array_concat.slt +++ b/datafusion/sqllogictest/test_files/array/array_concat.slt @@ -419,5 +419,24 @@ select array_concat(make_array(column3), column1, column2) from arrays_values_v2 [NULL, 11, 12] [NULL] +# array_concat derives a fresh return type from the unified element types rather +# than cloning an input's, so its output field is always the default nullable +# `item` regardless of the inputs' inner fields +query ?T +select + array_concat(arrow_cast(column1, 'List(non-null Int64)'), make_array(3)), + arrow_typeof(array_concat(arrow_cast(column1, 'List(non-null Int64)'), make_array(3))) +from values (make_array(1, 2)); +---- +[1, 2, 3] List(Int64) + +query ?T +select + array_concat(arrow_cast(column1, 'List(Int64, field: ''element'')'), make_array(3)), + arrow_typeof(array_concat(arrow_cast(column1, 'List(Int64, field: ''element'')'), make_array(3))) +from values (make_array(1, 2)); +---- +[1, 2, 3] List(Int64) + include ./cleanup.slt.part diff --git a/datafusion/sqllogictest/test_files/array/array_prepend.slt b/datafusion/sqllogictest/test_files/array/array_prepend.slt index 14b53e93b3d0d..bfb61ab4f9f91 100644 --- a/datafusion/sqllogictest/test_files/array/array_prepend.slt +++ b/datafusion/sqllogictest/test_files/array/array_prepend.slt @@ -273,5 +273,67 @@ select array_prepend(arrow_cast(make_array(1, 11, 111), 'FixedSizeList(3, Int64) [[1, 11, 111], [1, 2, 3], [2, 9, 1], [7, 8, 9], [1, 2, 3], [1, 7, 4], [4, 5, 6]] [[7, 8, 9], [1, 2, 3], [11, 12, 13]] [[1, 11, 111], [4, 5, 6], [10, 11, 12], [4, 9, 8], [7, 8, 9], [10, 11, 12], [1, 8, 7]] [[10, 11, 12], [1, 2, 3], [11, 12, 13]] +# the input list's inner field is carried through to the output, so that the +# returned type matches the one promised at planning time +query ?T +select + array_prepend(0, arrow_cast(column1, 'List(Int64, field: ''element'')')), + arrow_typeof(array_prepend(0, arrow_cast(column1, 'List(Int64, field: ''element'')'))) +from values (make_array(1, 2, 3)); +---- +[0, 1, 2, 3] List(Int64, field: 'element') + +query ?T +select + array_prepend(0, arrow_cast(column1, 'LargeList(Int64, field: ''element'')')), + arrow_typeof(array_prepend(0, arrow_cast(column1, 'LargeList(Int64, field: ''element'')'))) +from values (make_array(1, 2, 3)); +---- +[0, 1, 2, 3] LargeList(Int64, field: 'element') + +# nested value types go through a different kernel path +query ?T +select + array_prepend(make_array(0), arrow_cast(column1, 'List(List(Int64), field: ''element'')')), + arrow_typeof(array_prepend(make_array(0), arrow_cast(column1, 'List(List(Int64), field: ''element'')'))) +from values (make_array(make_array(1))); +---- +[[0], [1]] List(List(Int64), field: 'element') + +# a non-nullable inner field stays non-nullable when the prepended element cannot be null +query ??TT +select + array_prepend(0, column1), + array_prepend(0, arrow_cast(column1, 'LargeList(non-null Int64)')), + arrow_typeof(array_prepend(0, column1)), + arrow_typeof(array_prepend(0, arrow_cast(column1, 'LargeList(non-null Int64)'))) +from values + (arrow_cast(make_array(), 'List(non-null Int64)')), + (arrow_cast(NULL, 'List(non-null Int64)')), + (arrow_cast(make_array(1, 2, 3), 'List(non-null Int64)')) +; +---- +[0] [0] List(non-null Int64) LargeList(non-null Int64) +[0] [0] List(non-null Int64) LargeList(non-null Int64) +[0, 1, 2, 3] [0, 1, 2, 3] List(non-null Int64) LargeList(non-null Int64) + +# ... and is widened when the prepended element is nullable, since the result +# genuinely contains a null element +query ?T +select + array_prepend(NULL, arrow_cast(make_array(1, 2), 'List(non-null Int64)')), + arrow_typeof(array_prepend(NULL, arrow_cast(make_array(1, 2), 'List(non-null Int64)'))); +---- +[NULL, 1, 2] List(Int64) + +query ?T +select + array_prepend(column1, arrow_cast(make_array(1, 2), 'List(non-null Int64)')), + arrow_typeof(array_prepend(column1, arrow_cast(make_array(1, 2), 'List(non-null Int64)'))) +from values (0), (NULL); +---- +[0, 1, 2] List(Int64) +[NULL, 1, 2] List(Int64) + include ./cleanup.slt.part diff --git a/datafusion/sqllogictest/test_files/array/array_replace.slt b/datafusion/sqllogictest/test_files/array/array_replace.slt index cab84007bcd53..ce45e6440dddf 100644 --- a/datafusion/sqllogictest/test_files/array/array_replace.slt +++ b/datafusion/sqllogictest/test_files/array/array_replace.slt @@ -753,6 +753,93 @@ select ---- [3, 5, NULL] [3, 5, 5] [3, 5, 5] +# the input list's inner field is carried through to the output, so that the +# returned type matches the one promised at planning time +query ???TTT +select + array_replace(arrow_cast(column1, 'List(Int64, field: ''element'')'), 2, 9), + array_replace_n(arrow_cast(column1, 'List(Int64, field: ''element'')'), 2, 9, 1), + array_replace_all(arrow_cast(column1, 'List(Int64, field: ''element'')'), 2, 9), + arrow_typeof(array_replace(arrow_cast(column1, 'List(Int64, field: ''element'')'), 2, 9)), + arrow_typeof(array_replace_n(arrow_cast(column1, 'List(Int64, field: ''element'')'), 2, 9, 1)), + arrow_typeof(array_replace_all(arrow_cast(column1, 'List(Int64, field: ''element'')'), 2, 9)) +from values (make_array(1, 2, 2)); +---- +[1, 9, 2] [1, 9, 2] [1, 9, 9] List(Int64, field: 'element') List(Int64, field: 'element') List(Int64, field: 'element') + +query ???TTT +select + array_replace(arrow_cast(column1, 'LargeList(Int64, field: ''element'')'), 2, 9), + array_replace_n(arrow_cast(column1, 'LargeList(Int64, field: ''element'')'), 2, 9, 1), + array_replace_all(arrow_cast(column1, 'LargeList(Int64, field: ''element'')'), 2, 9), + arrow_typeof(array_replace(arrow_cast(column1, 'LargeList(Int64, field: ''element'')'), 2, 9)), + arrow_typeof(array_replace_n(arrow_cast(column1, 'LargeList(Int64, field: ''element'')'), 2, 9, 1)), + arrow_typeof(array_replace_all(arrow_cast(column1, 'LargeList(Int64, field: ''element'')'), 2, 9)) +from values (make_array(1, 2, 2)); +---- +[1, 9, 2] [1, 9, 2] [1, 9, 9] LargeList(Int64, field: 'element') LargeList(Int64, field: 'element') LargeList(Int64, field: 'element') + +# nested from/to values fall back to the generic comparison path +query ?T +select + array_replace(arrow_cast(column1, 'List(List(Int64), field: ''element'')'), make_array(2), make_array(9)), + arrow_typeof(array_replace(arrow_cast(column1, 'List(List(Int64), field: ''element'')'), make_array(2), make_array(9))) +from values (make_array(make_array(1), make_array(2))); +---- +[[1], [9]] List(List(Int64), field: 'element') + +# a non-nullable inner field stays non-nullable when the replacement cannot be null +query ???TTT +select + array_replace(column1, 2, 9), + array_replace_n(column1, 2, 9, 1), + array_replace_all(arrow_cast(column1, 'LargeList(non-null Int64)'), 2, 9), + arrow_typeof(array_replace(column1, 2, 9)), + arrow_typeof(array_replace_n(column1, 2, 9, 1)), + arrow_typeof(array_replace_all(arrow_cast(column1, 'LargeList(non-null Int64)'), 2, 9)) +from values + (arrow_cast(make_array(), 'List(non-null Int64)')), + (arrow_cast(NULL, 'List(non-null Int64)')), + (arrow_cast(make_array(1, 2, 2), 'List(non-null Int64)')) +; +---- +[] [] [] List(non-null Int64) List(non-null Int64) LargeList(non-null Int64) +NULL NULL NULL List(non-null Int64) List(non-null Int64) LargeList(non-null Int64) +[1, 9, 2] [1, 9, 2] [1, 9, 9] List(non-null Int64) List(non-null Int64) LargeList(non-null Int64) + +# ... and is widened when the replacement is nullable, since the result +# genuinely contains a null element +query ???TTT +select + array_replace(arrow_cast(column1, 'List(non-null Int64)'), 2, NULL), + array_replace_n(arrow_cast(column1, 'List(non-null Int64)'), 2, NULL, 1), + array_replace_all(arrow_cast(column1, 'List(non-null Int64)'), 2, NULL), + arrow_typeof(array_replace(arrow_cast(column1, 'List(non-null Int64)'), 2, NULL)), + arrow_typeof(array_replace_n(arrow_cast(column1, 'List(non-null Int64)'), 2, NULL, 1)), + arrow_typeof(array_replace_all(arrow_cast(column1, 'List(non-null Int64)'), 2, NULL)) +from values (make_array(1, 2, 2)); +---- +[1, NULL, 2] [1, NULL, 2] [1, NULL, NULL] List(Int64) List(Int64) List(Int64) + +# a max of 0 short circuits without replacing anything, but must still return +# the promised (widened) type +query ?T +select + array_replace_n(arrow_cast(column1, 'List(non-null Int64)'), 2, NULL, 0), + arrow_typeof(array_replace_n(arrow_cast(column1, 'List(non-null Int64)'), 2, NULL, 0)) +from values (make_array(1, 2, 2)); +---- +[1, 2, 2] List(Int64) + +# a NULL max yields a NULL row of the promised type +query ?T +select + array_replace_n(arrow_cast(column1, 'List(non-null Int64)'), 2, 9, NULL), + arrow_typeof(array_replace_n(arrow_cast(column1, 'List(non-null Int64)'), 2, 9, NULL)) +from values (make_array(1, 2, 2)); +---- +NULL List(non-null Int64) + statement ok From 68b8fd4cfb414e3c8a620bf28866d0a9fa849870 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 14 Aug 2026 20:14:31 +0000 Subject: [PATCH 31/50] fix: don't report equi-key equivalence for PiecewiseMergeJoin range joins (#24361) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #24360. ## Rationale for this change `PiecewiseMergeJoinExec::compute_properties` passed the join's `on` pair to `join_equivalence_properties` as if it were an equijoin key. For an `INNER` join that registers `left_on == right_on` as an output equivalence — but PWMJ's `on` is a **range** predicate (`l.v < r.v`), not equality, so the equivalence is false. It let the optimizer treat a sort on the left key as also sorting the right key and drop a required `ORDER BY`, returning wrongly ordered rows. ```sql set datafusion.optimizer.enable_piecewise_merge_join = true; create table l(v int) as values (1),(2),(3),(5),(8); create table r(v int) as values (4),(6),(9),(2); select l.v, r.v from l join r on l.v < r.v where l.v = 2 order by r.v; -- PWMJ: 2,9 / 2,6 / 2,4 (wrong order) -- NLJ: 2,4 / 2,6 / 2,9 (correct) ``` The plans differ: PWMJ sorts only on `l.v` (`SortExec: expr=[v@0 ASC]`); NLJ sorts on both (`[v@0 ASC, v@1 ASC]`). ## What changes are included in this PR? - `compute_properties` no longer passes the range `on` pair to `join_equivalence_properties` (a range join adds no column equivalences). The now-unused `join_on` parameter is dropped. - Existing `pwmj.slt` plans/results updated to the corrected (fully sorted) output — they previously encoded the wrong ordering — and a regression test is added. ## Are these changes tested? Yes. - New regression test in `pwmj.slt` (INNER `l.v < r.v`, `WHERE l.v = 2`, `ORDER BY r.v`) asserting the correctly ordered result. - The existing `pwmj.slt` cases that asserted the wrong ordering / a single-column sort plan are updated to the correct values (verified against `NestedLoopJoin`). - `piecewise` unit tests and `joins.slt` still pass. Only `INNER` is affected. ## Are there any user-facing changes? `INNER` range joins via `PiecewiseMergeJoin` (behind `enable_piecewise_merge_join`, default off) no longer drop a required sort, matching `NestedLoopJoin`. No API changes. --- .../src/joins/piecewise_merge_join/exec.rs | 9 ++-- datafusion/sqllogictest/test_files/pwmj.slt | 46 +++++++++++++++---- 2 files changed, 43 insertions(+), 12 deletions(-) diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs index a91903934ac9c..5183d3aa0feb7 100644 --- a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs +++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs @@ -363,7 +363,6 @@ impl PiecewiseMergeJoinExec { &streamed, Arc::clone(&schema), join_type, - &on, )?; Ok(Self { @@ -425,7 +424,6 @@ impl PiecewiseMergeJoinExec { streamed: &Arc, schema: SchemaRef, join_type: JoinType, - join_on: &(PhysicalExprRef, PhysicalExprRef), ) -> Result { let eq_properties = join_equivalence_properties( buffered.equivalence_properties().clone(), @@ -434,7 +432,12 @@ impl PiecewiseMergeJoinExec { schema, &Self::maintains_input_order(join_type), Some(Self::probe_side(&join_type)), - std::slice::from_ref(join_on), + // `PiecewiseMergeJoin`'s `on` is a range predicate (e.g. `l < r`), + // not an equijoin key. Passing it here would register a false + // `left == right` output equivalence, letting the optimizer drop a + // required sort and return wrongly ordered results. Range joins add + // no column equivalences, so pass none. + &[], )?; let output_partitioning = diff --git a/datafusion/sqllogictest/test_files/pwmj.slt b/datafusion/sqllogictest/test_files/pwmj.slt index 42be6c6b5cd88..f71a1b667243e 100644 --- a/datafusion/sqllogictest/test_files/pwmj.slt +++ b/datafusion/sqllogictest/test_files/pwmj.slt @@ -129,7 +129,7 @@ logical_plan 09)----------TableScan: join_t2 projection=[t2_id, t2_int] physical_plan 01)SortPreservingMergeExec: [t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST] -02)--SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true] +02)--SortExec: expr=[t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST], preserve_partitioning=[true] 03)----PiecewiseMergeJoin: operator=GtEq, join_type=Inner, on=(t1_id >= t2_id) 04)------SortExec: expr=[t1_id@0 ASC], preserve_partitioning=[false] 05)--------FilterExec: t1_id@0 >= 22 @@ -146,12 +146,12 @@ JOIN join_t2 t2 WHERE t2.t2_int >= 3 ORDER BY 1,2; ---- -11 55 11 44 -22 55 +11 55 22 44 -33 55 +22 55 33 44 +33 55 44 55 query TT @@ -174,7 +174,7 @@ logical_plan 08)----------TableScan: join_t2 projection=[t2_id, t2_int] physical_plan 01)SortPreservingMergeExec: [t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST] -02)--SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true] +02)--SortExec: expr=[t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST], preserve_partitioning=[true] 03)----PiecewiseMergeJoin: operator=Lt, join_type=Inner, on=(t1_id < t2_id) 04)------SortExec: expr=[t1_id@0 DESC], preserve_partitioning=[false] 05)--------DataSourceExec: partitions=1, partition_sizes=[1] @@ -238,11 +238,11 @@ WHERE t1.t1_id IN (11, 44) AND t2.t2_name <> 'y' ORDER BY 1,2; ---- -11 55 -11 44 11 11 -44 55 +11 44 +11 55 44 44 +44 55 query TT EXPLAIN @@ -266,7 +266,7 @@ logical_plan 09)----------TableScan: join_t2 projection=[t2_id, t2_name] physical_plan 01)SortPreservingMergeExec: [t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST] -02)--SortExec: expr=[t1_id@0 ASC NULLS LAST], preserve_partitioning=[true] +02)--SortExec: expr=[t1_id@0 ASC NULLS LAST, t2_id@1 ASC NULLS LAST], preserve_partitioning=[true] 03)----PiecewiseMergeJoin: operator=LtEq, join_type=Inner, on=(t1_id <= t2_id) 04)------SortExec: expr=[t1_id@0 DESC], preserve_partitioning=[false] 05)--------FilterExec: t1_id@0 = 11 OR t1_id@0 = 44 @@ -387,6 +387,34 @@ ORDER BY 1,2; 20 100 NULL NULL +# Regression test: a range predicate is NOT an equi-key, so an INNER +# PiecewiseMergeJoin must not report `l.v == r.v` as an output equivalence. +# Doing so let the optimizer treat a sort on `l.v` as also sorting `r.v` and +# drop the `ORDER BY r.v` sort, returning wrongly ordered rows. +statement ok +CREATE TABLE eq_t1 (v INT); + +statement ok +CREATE TABLE eq_t2 (v INT); + +statement ok +INSERT INTO eq_t1 VALUES (1), (2), (3), (5), (8); + +statement ok +INSERT INTO eq_t2 VALUES (4), (6), (9), (2); + +query II +SELECT t1.v AS lv, t2.v AS rv +FROM eq_t1 t1 +JOIN eq_t2 t2 + ON t1.v < t2.v +WHERE t1.v = 2 +ORDER BY t2.v; +---- +2 4 +2 6 +2 9 + # ------------------------------------------------------------------ # Existence joins (LeftSemi / LeftAnti) via PiecewiseMergeJoin # ------------------------------------------------------------------ From c77da3bba88b186de1662981016e6278f1ce03bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Fri, 14 Aug 2026 21:19:44 +0000 Subject: [PATCH 32/50] perf: Update `TableProvider` to take scan projection as `Option<&[usize]>` (#24376) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Addresses https://github.com/apache/datafusion/issues/13814 - Follow-on to #24325, #24326, #24329, #24330 ## Rationale for this change `TableProvider::scan` took its projection as `Option<&Vec>`. Besides being the signature `clippy::ptr_arg` exists to discourage, it forced `TableProvider::scan_with_args`'s default body to allocate: ```rust let projection = args.projection().map(|p| p.to_vec()); let plan = self.scan(state, projection.as_ref(), filters, limit).await?; ``` `ScanArgs::projection()` already yields `Option<&[usize]>`, so that `Vec` existed only to be borrowed. Worse, it cannot outlive a hoisted future, so the body had to stay an `async fn` — and that made it **the single most expensive thing to compile in `datafusion-session`**. Proving its coroutine `Send` walks the whole `Expr`/`LogicalPlan` type graph, and `#[async_trait]`'s `'life0: 'async_trait` bounds stop rustc serving that proof from its global cache. I measured this directly before writing the change: stubbing the body out dropped `datafusion-session`'s `evaluate_obligation` from **617ms to 16.5ms** and the whole crate from 0.86s to 0.235s, while the rest of the crate's trait solving attributed to just 4ms. So this one default body was ~97% of what remained after #24326. ## What changes are included in this PR? The projection becomes `Option<&[usize]>`, and the default body hands it straight through, capturing only the already-boxed future it awaits: ```rust let plan = self.scan( state, args.projection(), args.filters().unwrap_or(&[]), args.limit(), ); Box::pin(async move { Ok(plan.await?.into()) }) ``` Also updated, because their projections feed the same call paths: `StreamingTableExec::try_new`, `FilterExec::with_projection`, `batch_filter`. `datafusion_common::project_schema` now takes `Option<&T> where T: AsRef<[usize]> + ?Sized` instead of `Option<&impl AsRef<[usize]>>`, so it accepts a `Vec` *or* a slice — no caller of it needs to change. Call sites get simpler rather than noisier: `Some(&vec![2, 1])` becomes `Some(&[2, 1])`, and three `vec!` allocations in tests disappear (clippy pointed those out on its own). ## Are these changes tested? Yes ## Are there any user-facing changes? Yes — this is a **breaking change** for `TableProvider` implementors, so it has an upgrade-guide entry. Implementations only need the signature updated; two patterns inside a body need adjusting: ```rust // Before // After projection.cloned() projection.map(|p| p.to_vec()) ``` and callers holding an `Option>` pass `.as_deref()` rather than `.as_ref()`. In tree that was 47 signatures across 36 files, all mechanical. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 --- datafusion-cli/src/functions.rs | 16 +++--- .../custom_data_source/custom_datasource.rs | 8 +-- .../default_column_values.rs | 4 +- .../data_io/parquet_advanced_index.rs | 4 +- .../data_io/parquet_embedded_index.rs | 2 +- .../examples/data_io/parquet_index.rs | 4 +- .../examples/data_io/remote_catalog.rs | 4 +- .../examples/udf/simple_udtf.rs | 4 +- datafusion/catalog-listing/src/table.rs | 8 +-- datafusion/catalog/src/async.rs | 2 +- datafusion/catalog/src/cte_worktable.rs | 8 +-- .../catalog/src/default_table_source.rs | 2 +- datafusion/catalog/src/empty.rs | 2 +- datafusion/catalog/src/memory/table.rs | 13 +++-- datafusion/catalog/src/stream.rs | 6 +-- datafusion/catalog/src/streaming.rs | 11 ++-- datafusion/catalog/src/view.rs | 8 +-- datafusion/common/src/utils/mod.rs | 4 +- datafusion/core/src/dataframe/mod.rs | 6 +-- datafusion/core/src/datasource/memory_test.rs | 2 +- datafusion/core/src/physical_planner.rs | 4 +- datafusion/core/src/test_util/mod.rs | 2 +- .../custom_sources_cases/dml_planning.rs | 6 +-- .../core/tests/custom_sources_cases/mod.rs | 6 ++- .../provider_filter_pushdown.rs | 2 +- .../tests/custom_sources_cases/statistics.rs | 4 +- datafusion/core/tests/memory_limit/mod.rs | 4 +- .../physical_optimizer/projection_pushdown.rs | 2 +- .../tests/user_defined/insert_operation.rs | 2 +- .../tests/user_defined/statistics_requests.rs | 6 +-- .../user_defined_table_functions.rs | 4 +- datafusion/ffi/src/table_provider.rs | 6 +-- datafusion/ffi/src/tests/async_provider.rs | 2 +- datafusion/ffi/src/tests/mod.rs | 2 +- datafusion/ffi/src/tests/query_planner.rs | 7 ++- .../functions-table/src/generate_series.rs | 4 +- datafusion/physical-plan/src/filter.rs | 2 +- datafusion/physical-plan/src/streaming.rs | 6 +-- datafusion/session/src/table.rs | 34 ++++++++----- datafusion/sqllogictest/src/test_context.rs | 2 +- .../custom-table-providers.md | 10 ++-- .../library-user-guide/upgrading/55.0.0.md | 51 +++++++++++++++++++ 42 files changed, 176 insertions(+), 110 deletions(-) diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rs index 164af2559d2f6..409eedee750a3 100644 --- a/datafusion-cli/src/functions.rs +++ b/datafusion-cli/src/functions.rs @@ -241,14 +241,14 @@ impl TableProvider for ParquetMetadataTable { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { Ok(MemorySourceConfig::try_new_exec( &[vec![self.batch.clone()]], TableProvider::schema(self), - projection.cloned(), + projection.map(|p| p.to_vec()), )?) } } @@ -487,14 +487,14 @@ impl TableProvider for MetadataCacheTable { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { Ok(MemorySourceConfig::try_new_exec( &[vec![self.batch.clone()]], TableProvider::schema(self), - projection.cloned(), + projection.map(|p| p.to_vec()), )?) } } @@ -604,14 +604,14 @@ impl TableProvider for StatisticsCacheTable { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { Ok(MemorySourceConfig::try_new_exec( &[vec![self.batch.clone()]], TableProvider::schema(self), - projection.cloned(), + projection.map(|p| p.to_vec()), )?) } } @@ -753,14 +753,14 @@ impl TableProvider for ListFilesCacheTable { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { Ok(MemorySourceConfig::try_new_exec( &[vec![self.batch.clone()]], TableProvider::schema(self), - projection.cloned(), + projection.map(|p| p.to_vec()), )?) } } diff --git a/datafusion-examples/examples/custom_data_source/custom_datasource.rs b/datafusion-examples/examples/custom_data_source/custom_datasource.rs index 6f176f8b46609..91fb219f34736 100644 --- a/datafusion-examples/examples/custom_data_source/custom_datasource.rs +++ b/datafusion-examples/examples/custom_data_source/custom_datasource.rs @@ -148,7 +148,7 @@ impl Debug for CustomDataSource { impl CustomDataSource { pub(crate) fn create_physical_plan( &self, - projections: Option<&Vec>, + projections: Option<&[usize]>, schema: SchemaRef, ) -> Result> { Ok(Arc::new(CustomExec::new(projections, schema, self.clone()))) @@ -203,7 +203,7 @@ impl TableProvider for CustomDataSource { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, // filters and limit can be used here to inject some push-down operations if needed _filters: &[Expr], _limit: Option, @@ -223,7 +223,7 @@ struct CustomExec { impl CustomExec { #[expect(clippy::needless_pass_by_value)] fn new( - projections: Option<&Vec>, + projections: Option<&[usize]>, schema: SchemaRef, db: CustomDataSource, ) -> Self { @@ -231,7 +231,7 @@ impl CustomExec { let cache = Self::compute_properties(projected_schema.clone()); Self { db, - projection: projections.cloned(), + projection: projections.map(|p| p.to_vec()), projected_schema, cache: Arc::new(cache), } diff --git a/datafusion-examples/examples/custom_data_source/default_column_values.rs b/datafusion-examples/examples/custom_data_source/default_column_values.rs index 633b98244367e..d2024621aad76 100644 --- a/datafusion-examples/examples/custom_data_source/default_column_values.rs +++ b/datafusion-examples/examples/custom_data_source/default_column_values.rs @@ -218,7 +218,7 @@ impl TableProvider for DefaultValueTableProvider { async fn scan( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result> { @@ -253,7 +253,7 @@ impl TableProvider for DefaultValueTableProvider { ObjectStoreUrl::parse("memory://")?, Arc::new(parquet_source), ) - .with_projection_indices(projection.cloned())? + .with_projection_indices(projection.map(|p| p.to_vec()))? .with_limit(limit) .with_file_group(file_group) .with_expr_adapter(Some(Arc::new(DefaultValuePhysicalExprAdapterFactory) as _)); diff --git a/datafusion-examples/examples/data_io/parquet_advanced_index.rs b/datafusion-examples/examples/data_io/parquet_advanced_index.rs index b6440eb3e2078..47c57a4804c5c 100644 --- a/datafusion-examples/examples/data_io/parquet_advanced_index.rs +++ b/datafusion-examples/examples/data_io/parquet_advanced_index.rs @@ -464,7 +464,7 @@ impl TableProvider for IndexTableProvider { async fn scan( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result> { @@ -500,7 +500,7 @@ impl TableProvider for IndexTableProvider { ); let file_scan_config = FileScanConfigBuilder::new(object_store_url, file_source) .with_limit(limit) - .with_projection_indices(projection.cloned())? + .with_projection_indices(projection.map(|p| p.to_vec()))? .with_file(partitioned_file) .build(); diff --git a/datafusion-examples/examples/data_io/parquet_embedded_index.rs b/datafusion-examples/examples/data_io/parquet_embedded_index.rs index a8a3c97fa11f8..5e48600650702 100644 --- a/datafusion-examples/examples/data_io/parquet_embedded_index.rs +++ b/datafusion-examples/examples/data_io/parquet_embedded_index.rs @@ -405,7 +405,7 @@ impl TableProvider for DistinctIndexTable { async fn scan( &self, _ctx: &dyn Session, - _proj: Option<&Vec>, + _proj: Option<&[usize]>, filters: &[Expr], _limit: Option, ) -> Result> { diff --git a/datafusion-examples/examples/data_io/parquet_index.rs b/datafusion-examples/examples/data_io/parquet_index.rs index 753d1b30fc0e8..c8ae7e14c4d37 100644 --- a/datafusion-examples/examples/data_io/parquet_index.rs +++ b/datafusion-examples/examples/data_io/parquet_index.rs @@ -218,7 +218,7 @@ impl TableProvider for IndexTableProvider { async fn scan( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result> { @@ -242,7 +242,7 @@ impl TableProvider for IndexTableProvider { Arc::new(ParquetSource::new(self.schema()).with_predicate(predicate)); let mut file_scan_config_builder = FileScanConfigBuilder::new(object_store_url, source) - .with_projection_indices(projection.cloned())? + .with_projection_indices(projection.map(|p| p.to_vec()))? .with_limit(limit); // Transform to the format needed to pass to DataSourceExec diff --git a/datafusion-examples/examples/data_io/remote_catalog.rs b/datafusion-examples/examples/data_io/remote_catalog.rs index a24ca2238181d..49157c6b1f5b7 100644 --- a/datafusion-examples/examples/data_io/remote_catalog.rs +++ b/datafusion-examples/examples/data_io/remote_catalog.rs @@ -237,7 +237,7 @@ impl TableProvider for RemoteTable { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { @@ -257,7 +257,7 @@ impl TableProvider for RemoteTable { let exec = MemorySourceConfig::try_new_exec( &[batches], self.schema.clone(), - projection.cloned(), + projection.map(|p| p.to_vec()), )?; Ok(exec) } diff --git a/datafusion-examples/examples/udf/simple_udtf.rs b/datafusion-examples/examples/udf/simple_udtf.rs index af123ab7e5d4a..3b55a0456a0aa 100644 --- a/datafusion-examples/examples/udf/simple_udtf.rs +++ b/datafusion-examples/examples/udf/simple_udtf.rs @@ -96,7 +96,7 @@ impl TableProvider for LocalCsvTable { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { @@ -122,7 +122,7 @@ impl TableProvider for LocalCsvTable { Ok(MemorySourceConfig::try_new_exec( &[batches], TableProvider::schema(self), - projection.cloned(), + projection.map(|p| p.to_vec()), )?) } } diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs index 498640ba0ff32..1749cdac2f311 100644 --- a/datafusion/catalog-listing/src/table.rs +++ b/datafusion/catalog-listing/src/table.rs @@ -495,7 +495,7 @@ impl TableProvider for ListingTable { fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, - projection: Option<&'life2 Vec>, + projection: Option<&'life2 [usize]>, filters: &'life3 [Expr], limit: Option, ) -> BoxFuture<'async_trait, datafusion_common::Result>> @@ -753,7 +753,7 @@ impl ListingTable { fn scan_boxed<'a>( &'a self, state: &'a dyn Session, - projection: Option<&'a Vec>, + projection: Option<&'a [usize]>, filters: &'a [Expr], limit: Option, ) -> BoxFuture<'a, datafusion_common::Result>> { @@ -763,12 +763,12 @@ impl ListingTable { async fn scan_inner( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> datafusion_common::Result> { let options = ScanArgs::default() - .with_projection(projection.map(|p| p.as_slice())) + .with_projection(projection) .with_filters(Some(filters)) .with_limit(limit); Ok(self.scan_with_args(state, options).await?.into_inner()) diff --git a/datafusion/catalog/src/async.rs b/datafusion/catalog/src/async.rs index 87b7b7c3431a1..6f8e25cafd1c0 100644 --- a/datafusion/catalog/src/async.rs +++ b/datafusion/catalog/src/async.rs @@ -444,7 +444,7 @@ mod tests { async fn scan( &self, _state: &dyn Session, - _projection: Option<&Vec>, + _projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { diff --git a/datafusion/catalog/src/cte_worktable.rs b/datafusion/catalog/src/cte_worktable.rs index 6690fad24c736..5f5094793c01d 100644 --- a/datafusion/catalog/src/cte_worktable.rs +++ b/datafusion/catalog/src/cte_worktable.rs @@ -85,7 +85,7 @@ impl TableProvider for CteWorkTable { fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, - projection: Option<&'life2 Vec>, + projection: Option<&'life2 [usize]>, filters: &'life3 [Expr], limit: Option, ) -> BoxFuture<'async_trait, Result>> @@ -143,7 +143,7 @@ impl CteWorkTable { fn scan_boxed<'a>( &'a self, state: &'a dyn Session, - projection: Option<&'a Vec>, + projection: Option<&'a [usize]>, filters: &'a [Expr], limit: Option, ) -> BoxFuture<'a, Result>> { @@ -153,12 +153,12 @@ impl CteWorkTable { async fn scan_inner( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result> { let options = ScanArgs::default() - .with_projection(projection.map(|p| p.as_slice())) + .with_projection(projection) .with_filters(Some(filters)) .with_limit(limit); Ok(self.scan_with_args(state, options).await?.into_inner()) diff --git a/datafusion/catalog/src/default_table_source.rs b/datafusion/catalog/src/default_table_source.rs index 60f85891d66e6..3342db54de92f 100644 --- a/datafusion/catalog/src/default_table_source.rs +++ b/datafusion/catalog/src/default_table_source.rs @@ -118,7 +118,7 @@ fn preserves_table_type() { async fn scan( &self, _: &dyn crate::Session, - _: Option<&Vec>, + _: Option<&[usize]>, _: &[Expr], _: Option, ) -> Result, DataFusionError> diff --git a/datafusion/catalog/src/empty.rs b/datafusion/catalog/src/empty.rs index 1ff36ecf360a2..66b62bfcec452 100644 --- a/datafusion/catalog/src/empty.rs +++ b/datafusion/catalog/src/empty.rs @@ -66,7 +66,7 @@ impl TableProvider for EmptyTable { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rs index 8bb45e945d242..d817aa8b7788a 100644 --- a/datafusion/catalog/src/memory/table.rs +++ b/datafusion/catalog/src/memory/table.rs @@ -191,7 +191,7 @@ impl TableProvider for MemTable { fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, - projection: Option<&'life2 Vec>, + projection: Option<&'life2 [usize]>, filters: &'life3 [Expr], limit: Option, ) -> BoxFuture<'async_trait, Result>> @@ -275,7 +275,7 @@ impl MemTable { fn scan_boxed<'a>( &'a self, state: &'a dyn Session, - projection: Option<&'a Vec>, + projection: Option<&'a [usize]>, filters: &'a [Expr], limit: Option, ) -> BoxFuture<'a, Result>> { @@ -285,7 +285,7 @@ impl MemTable { async fn scan_inner( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { @@ -295,8 +295,11 @@ impl MemTable { partitions.push(inner_vec.clone()) } - let mut source = - MemorySourceConfig::try_new(&partitions, self.schema(), projection.cloned())?; + let mut source = MemorySourceConfig::try_new( + &partitions, + self.schema(), + projection.map(|p| p.to_vec()), + )?; let show_sizes = state.config_options().explain.show_sizes; source = source.with_show_sizes(show_sizes); diff --git a/datafusion/catalog/src/stream.rs b/datafusion/catalog/src/stream.rs index 86bd4e81dd134..97e5293765afb 100644 --- a/datafusion/catalog/src/stream.rs +++ b/datafusion/catalog/src/stream.rs @@ -355,7 +355,7 @@ impl TableProvider for StreamTable { fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, - projection: Option<&'life2 Vec>, + projection: Option<&'life2 [usize]>, filters: &'life3 [Expr], limit: Option, ) -> BoxFuture<'async_trait, Result>> @@ -390,7 +390,7 @@ impl StreamTable { fn scan_boxed<'a>( &'a self, state: &'a dyn Session, - projection: Option<&'a Vec>, + projection: Option<&'a [usize]>, filters: &'a [Expr], limit: Option, ) -> BoxFuture<'a, Result>> { @@ -400,7 +400,7 @@ impl StreamTable { fn scan_inner( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], limit: Option, ) -> Result> { diff --git a/datafusion/catalog/src/streaming.rs b/datafusion/catalog/src/streaming.rs index 7b466d8a59c7f..c2b5d691866b5 100644 --- a/datafusion/catalog/src/streaming.rs +++ b/datafusion/catalog/src/streaming.rs @@ -95,10 +95,7 @@ impl StreamingTable { self } - fn output_partitioning( - &self, - projection: Option<&Vec>, - ) -> Result { + fn output_partitioning(&self, projection: Option<&[usize]>) -> Result { let Some(output_partitioning) = &self.output_partitioning else { return Ok(Partitioning::UnknownPartitioning(self.partitions.len())); }; @@ -128,7 +125,7 @@ impl TableProvider for StreamingTable { fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, - projection: Option<&'life2 Vec>, + projection: Option<&'life2 [usize]>, filters: &'life3 [Expr], limit: Option, ) -> BoxFuture<'async_trait, Result>> @@ -147,7 +144,7 @@ impl StreamingTable { fn scan_boxed<'a>( &'a self, state: &'a dyn Session, - projection: Option<&'a Vec>, + projection: Option<&'a [usize]>, filters: &'a [Expr], limit: Option, ) -> BoxFuture<'a, Result>> { @@ -157,7 +154,7 @@ impl StreamingTable { fn scan_inner( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], limit: Option, ) -> Result> { diff --git a/datafusion/catalog/src/view.rs b/datafusion/catalog/src/view.rs index 32af208984d95..795d2985c8c44 100644 --- a/datafusion/catalog/src/view.rs +++ b/datafusion/catalog/src/view.rs @@ -101,7 +101,7 @@ impl TableProvider for ViewTable { fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, - projection: Option<&'life2 Vec>, + projection: Option<&'life2 [usize]>, filters: &'life3 [Expr], limit: Option, ) -> BoxFuture<'async_trait, Result>> @@ -120,7 +120,7 @@ impl ViewTable { fn scan_boxed<'a>( &'a self, state: &'a dyn Session, - projection: Option<&'a Vec>, + projection: Option<&'a [usize]>, filters: &'a [Expr], limit: Option, ) -> BoxFuture<'a, Result>> { @@ -130,7 +130,7 @@ impl ViewTable { async fn scan_inner( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result> { @@ -146,7 +146,7 @@ impl ViewTable { // avoiding adding a redundant projection (e.g. SELECT * FROM view) let current_projection = (0..plan.schema().fields().len()).collect::>(); - if projection == ¤t_projection { + if projection == current_projection { plan } else { let fields: Vec = projection diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rs index 73772b319351c..8df160187ac7d 100644 --- a/datafusion/common/src/utils/mod.rs +++ b/datafusion/common/src/utils/mod.rs @@ -80,9 +80,9 @@ use std::thread::available_parallelism; /// /// assert_eq!(projected_schema, expected_schema); /// ``` -pub fn project_schema( +pub fn project_schema + ?Sized>( schema: &SchemaRef, - projection: Option<&impl AsRef<[usize]>>, + projection: Option<&T>, ) -> Result { let schema = match projection { Some(columns) => Arc::new(schema.project(columns.as_ref())?), diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs index 5ca40c854b4df..333f94dc8f7e0 100644 --- a/datafusion/core/src/dataframe/mod.rs +++ b/datafusion/core/src/dataframe/mod.rs @@ -2722,7 +2722,7 @@ impl TableProvider for DataFrameTableProvider { fn scan<'life0, 'life1, 'life2, 'life3, 'async_trait>( &'life0 self, state: &'life1 dyn Session, - projection: Option<&'life2 Vec>, + projection: Option<&'life2 [usize]>, filters: &'life3 [Expr], limit: Option, ) -> BoxFuture<'async_trait, Result>> @@ -2741,7 +2741,7 @@ impl DataFrameTableProvider { fn scan_boxed<'a>( &'a self, state: &'a dyn Session, - projection: Option<&'a Vec>, + projection: Option<&'a [usize]>, filters: &'a [Expr], limit: Option, ) -> BoxFuture<'a, Result>> { @@ -2751,7 +2751,7 @@ impl DataFrameTableProvider { async fn scan_inner( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result> { diff --git a/datafusion/core/src/datasource/memory_test.rs b/datafusion/core/src/datasource/memory_test.rs index d7311c1d9c960..9e37f77d8b49c 100644 --- a/datafusion/core/src/datasource/memory_test.rs +++ b/datafusion/core/src/datasource/memory_test.rs @@ -60,7 +60,7 @@ mod tests { // scan with projection let exec = provider - .scan(&session_ctx.state(), Some(&vec![2, 1]), &[], None) + .scan(&session_ctx.state(), Some(&[2, 1]), &[], None) .await?; let mut it = exec.execute(0, task_ctx)?; diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 77f8238eddd4c..7002a4b04d957 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -3538,7 +3538,7 @@ mod tests { async fn scan( &self, _state: &dyn Session, - _projection: Option<&Vec>, + _projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { @@ -5526,7 +5526,7 @@ digraph { async fn scan( &self, _state: &dyn Session, - _projection: Option<&Vec>, + _projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { diff --git a/datafusion/core/src/test_util/mod.rs b/datafusion/core/src/test_util/mod.rs index bcb38b97ceab8..ab7edef885b40 100644 --- a/datafusion/core/src/test_util/mod.rs +++ b/datafusion/core/src/test_util/mod.rs @@ -250,7 +250,7 @@ impl TableProvider for TestTableProvider { async fn scan( &self, _state: &dyn Session, - _projection: Option<&Vec>, + _projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { diff --git a/datafusion/core/tests/custom_sources_cases/dml_planning.rs b/datafusion/core/tests/custom_sources_cases/dml_planning.rs index 24a3df7e0a8fa..6ae942bfbb88b 100644 --- a/datafusion/core/tests/custom_sources_cases/dml_planning.rs +++ b/datafusion/core/tests/custom_sources_cases/dml_planning.rs @@ -101,7 +101,7 @@ impl TableProvider for CaptureDeleteProvider { async fn scan( &self, _state: &dyn Session, - _projection: Option<&Vec>, + _projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { @@ -197,7 +197,7 @@ impl TableProvider for CaptureUpdateProvider { async fn scan( &self, _state: &dyn Session, - _projection: Option<&Vec>, + _projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { @@ -271,7 +271,7 @@ impl TableProvider for CaptureTruncateProvider { async fn scan( &self, _state: &dyn Session, - _projection: Option<&Vec>, + _projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { diff --git a/datafusion/core/tests/custom_sources_cases/mod.rs b/datafusion/core/tests/custom_sources_cases/mod.rs index 7abbcd6e9578c..ea3cdaa34df54 100644 --- a/datafusion/core/tests/custom_sources_cases/mod.rs +++ b/datafusion/core/tests/custom_sources_cases/mod.rs @@ -247,11 +247,13 @@ impl TableProvider for CustomTableProvider { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { - Ok(Arc::new(CustomExecutionPlan::new(projection.cloned()))) + Ok(Arc::new(CustomExecutionPlan::new( + projection.map(|p| p.to_vec()), + ))) } } diff --git a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs index a8f7f09ad016b..51d31360f682d 100644 --- a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs +++ b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs @@ -191,7 +191,7 @@ impl TableProvider for CustomProvider { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], _: Option, ) -> Result> { diff --git a/datafusion/core/tests/custom_sources_cases/statistics.rs b/datafusion/core/tests/custom_sources_cases/statistics.rs index 6213be8e2d24f..7701ca0279125 100644 --- a/datafusion/core/tests/custom_sources_cases/statistics.rs +++ b/datafusion/core/tests/custom_sources_cases/statistics.rs @@ -91,7 +91,7 @@ impl TableProvider for StatisticsValidation { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], // limit is ignored because it is not mandatory for a `TableProvider` to honor it _limit: Option, @@ -102,7 +102,7 @@ impl TableProvider for StatisticsValidation { filters.len(), "Unsupported expressions should not be pushed down" ); - let projection = match projection.cloned() { + let projection = match projection.map(|p| p.to_vec()) { Some(p) => p, None => (0..self.schema.fields().len()).collect(), }; diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rs index 84d7e9c4508b5..472ff07aacf7e 100644 --- a/datafusion/core/tests/memory_limit/mod.rs +++ b/datafusion/core/tests/memory_limit/mod.rs @@ -1216,14 +1216,14 @@ impl TableProvider for SortedTableProvider { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { let mem_conf = MemorySourceConfig::try_new( &self.batches, self.schema(), - projection.cloned(), + projection.map(|p| p.to_vec()), )? .try_with_sort_information(self.sort_information.clone())?; diff --git a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs index 3a8d82f111145..113552c462f76 100644 --- a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs +++ b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs @@ -561,7 +561,7 @@ fn test_streaming_table_after_projection() -> Result<()> { Field::new("e", DataType::Int32, true), ])), }) as _], - Some(&vec![0_usize, 2, 4, 3]), + Some(&[0_usize, 2, 4, 3]), vec![ [ PhysicalSortExpr { diff --git a/datafusion/core/tests/user_defined/insert_operation.rs b/datafusion/core/tests/user_defined/insert_operation.rs index 0eefcdb551a65..49d0ecc1aeb23 100644 --- a/datafusion/core/tests/user_defined/insert_operation.rs +++ b/datafusion/core/tests/user_defined/insert_operation.rs @@ -101,7 +101,7 @@ impl TableProvider for TestInsertTableProvider { async fn scan( &self, _state: &dyn Session, - _projection: Option<&Vec>, + _projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { diff --git a/datafusion/core/tests/user_defined/statistics_requests.rs b/datafusion/core/tests/user_defined/statistics_requests.rs index 64c6676c23746..079639baf1db3 100644 --- a/datafusion/core/tests/user_defined/statistics_requests.rs +++ b/datafusion/core/tests/user_defined/statistics_requests.rs @@ -113,14 +113,14 @@ impl TableProvider for RecordingTable { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { Ok(MemorySourceConfig::try_new_exec( &[vec![self.batch.clone()]], Arc::clone(&self.schema), - projection.cloned(), + projection.map(|p| p.to_vec()), )?) } @@ -134,7 +134,7 @@ impl TableProvider for RecordingTable { let plan = self .scan( state, - args.projection().map(|p| p.to_vec()).as_ref(), + args.projection(), args.filters().unwrap_or(&[]), args.limit(), ) diff --git a/datafusion/core/tests/user_defined/user_defined_table_functions.rs b/datafusion/core/tests/user_defined/user_defined_table_functions.rs index c8ded3a6fce3f..24205cf8c4010 100644 --- a/datafusion/core/tests/user_defined/user_defined_table_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_table_functions.rs @@ -129,7 +129,7 @@ impl TableProvider for SimpleCsvTable { async fn scan( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { @@ -156,7 +156,7 @@ impl TableProvider for SimpleCsvTable { Ok(MemorySourceConfig::try_new_exec( &[batches], TableProvider::schema(self), - projection.cloned(), + projection.map(|p| p.to_vec()), )?) } } diff --git a/datafusion/ffi/src/table_provider.rs b/datafusion/ffi/src/table_provider.rs index f1a79added0ab..b1d54b5f33096 100644 --- a/datafusion/ffi/src/table_provider.rs +++ b/datafusion/ffi/src/table_provider.rs @@ -320,7 +320,7 @@ unsafe extern "C" fn scan_fn_wrapper( let plan = sresult_return!( internal_provider - .scan(session, projections.as_ref(), &filters, limit.into()) + .scan(session, projections.as_deref(), &filters, limit.into()) .await ); @@ -653,7 +653,7 @@ impl TableProvider for ForeignTableProvider { async fn scan( &self, session: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result> { @@ -874,7 +874,7 @@ mod tests { async fn scan( &self, session: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result> { diff --git a/datafusion/ffi/src/tests/async_provider.rs b/datafusion/ffi/src/tests/async_provider.rs index 83057d8c45db3..cb5d27c6e666f 100644 --- a/datafusion/ffi/src/tests/async_provider.rs +++ b/datafusion/ffi/src/tests/async_provider.rs @@ -139,7 +139,7 @@ impl TableProvider for AsyncTableProvider { async fn scan( &self, state: &dyn Session, - _projection: Option<&Vec>, + _projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rs index ce922de6647d2..fbc3e83ba49fc 100644 --- a/datafusion/ffi/src/tests/mod.rs +++ b/datafusion/ffi/src/tests/mod.rs @@ -259,7 +259,7 @@ impl TableProvider for TableWithStats { async fn scan( &self, session: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result> { diff --git a/datafusion/ffi/src/tests/query_planner.rs b/datafusion/ffi/src/tests/query_planner.rs index 90d713f8a6336..40c237c39ecdf 100644 --- a/datafusion/ffi/src/tests/query_planner.rs +++ b/datafusion/ffi/src/tests/query_planner.rs @@ -57,7 +57,12 @@ impl QueryPlanner for TestQueryPlanner { return exec_err!("library B's provider was not foreign to library C"); } let library_b_plan = provider - .scan(session, scan.projection.as_ref(), &scan.filters, scan.fetch) + .scan( + session, + scan.projection.as_deref(), + &scan.filters, + scan.fetch, + ) .await?; if !library_b_plan.is::() { diff --git a/datafusion/functions-table/src/generate_series.rs b/datafusion/functions-table/src/generate_series.rs index f5e4df13899df..8d9327c5acfa2 100644 --- a/datafusion/functions-table/src/generate_series.rs +++ b/datafusion/functions-table/src/generate_series.rs @@ -552,14 +552,14 @@ impl TableProvider for GenerateSeriesTable { async fn scan( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], _limit: Option, ) -> Result> { let batch_size = state.config_options().execution.batch_size.get(); let generator = self.as_generator(batch_size)?; let mut exec = LazyMemoryExec::try_new(self.schema(), vec![generator])? - .with_projection(projection.cloned()); + .with_projection(projection.map(|p| p.to_vec())); if let Some(ordering) = self.output_ordering(exec.schema().as_ref()) { exec.add_ordering([ordering]); diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 5df5482fb75de..c62db109ead1c 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -1236,7 +1236,7 @@ pub fn batch_filter( fn filter_and_project( batch: &RecordBatch, predicate: &Arc, - projection: Option<&Vec>, + projection: Option<&[usize]>, ) -> Result { predicate .evaluate(batch) diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rs index 7b0058e79887c..dd57f958e83ec 100644 --- a/datafusion/physical-plan/src/streaming.rs +++ b/datafusion/physical-plan/src/streaming.rs @@ -81,7 +81,7 @@ impl StreamingTableExec { pub fn try_new( schema: SchemaRef, partitions: Vec>, - projection: Option<&Vec>, + projection: Option<&[usize]>, projected_output_ordering: impl IntoIterator, infinite: bool, limit: Option, @@ -112,7 +112,7 @@ impl StreamingTableExec { Ok(Self { partitions, projected_schema, - projection: projection.cloned().map(Into::into), + projection: projection.map(|p| p.to_vec().into()), projected_output_ordering, infinite, limit, @@ -475,7 +475,7 @@ mod test { StreamingTableExec::try_new( self.schema.unwrap(), self.partitions, - self.projection.as_ref(), + self.projection.as_deref(), self.projected_output_ordering, self.infinite, self.limit, diff --git a/datafusion/session/src/table.rs b/datafusion/session/src/table.rs index c1a1cd45401c0..a6cddd10ce80a 100644 --- a/datafusion/session/src/table.rs +++ b/datafusion/session/src/table.rs @@ -187,7 +187,7 @@ pub trait TableProvider: Any + Debug + Sync + Send { async fn scan( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result>; @@ -209,18 +209,26 @@ pub trait TableProvider: Any + Debug + Sync + Send { /// A [`ScanResult`] containing the [`ExecutionPlan`] for scanning the table /// /// See [`Self::scan`] for detailed documentation about projection, filters, and limits. - async fn scan_with_args<'a>( - &self, - state: &dyn Session, + // Hand-written `#[async_trait]` expansion to reduce compile time. See + // . + fn scan_with_args<'a, 'life0, 'life1, 'async_trait>( + &'life0 self, + state: &'life1 dyn Session, args: ScanArgs<'a>, - ) -> Result { - let filters = args.filters().unwrap_or(&[]); - let projection = args.projection().map(|p| p.to_vec()); - let limit = args.limit(); - let plan = self - .scan(state, projection.as_ref(), filters, limit) - .await?; - Ok(plan.into()) + ) -> BoxFuture<'async_trait, Result> + where + 'a: 'async_trait, + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + let plan = self.scan( + state, + args.projection(), + args.filters().unwrap_or(&[]), + args.limit(), + ); + Box::pin(async move { Ok(plan.await?.into()) }) } /// Specify if DataFusion should provide filter expressions to the @@ -269,7 +277,7 @@ pub trait TableProvider: Any + Debug + Sync + Send { /// impl TableProvider for TestDataSource { /// # fn schema(&self) -> SchemaRef { todo!() } /// # fn table_type(&self) -> TableType { todo!() } - /// # async fn scan(&self, s: &dyn Session, p: Option<&Vec>, f: &[Expr], l: Option) -> Result> { + /// # async fn scan(&self, s: &dyn Session, p: Option<&[usize]>, f: &[Expr], l: Option) -> Result> { /// todo!() /// # } /// // Override the supports_filters_pushdown to evaluate which expressions diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs index d85ca2db76268..39aa2b09e5685 100644 --- a/datafusion/sqllogictest/src/test_context.rs +++ b/datafusion/sqllogictest/src/test_context.rs @@ -388,7 +388,7 @@ pub fn register_temp_table(ctx: &SessionContext) { async fn scan( &self, _state: &dyn Session, - _: Option<&Vec>, + _: Option<&[usize]>, _: &[Expr], _: Option, ) -> Result, DataFusionError> { diff --git a/docs/source/library-user-guide/custom-table-providers.md b/docs/source/library-user-guide/custom-table-providers.md index c094f8bf7eb1b..dbbdf9d022716 100644 --- a/docs/source/library-user-guide/custom-table-providers.md +++ b/docs/source/library-user-guide/custom-table-providers.md @@ -170,7 +170,7 @@ impl TableProvider for MyTable { async fn scan( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result> { @@ -323,7 +323,7 @@ expects to work with: async fn scan( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result> { @@ -528,7 +528,7 @@ To opt in, implement `supports_filters_pushdown`: # impl TableProvider for MyFilterTable { # fn schema(&self) -> SchemaRef { todo!() } # fn table_type(&self) -> TableType { TableType::Base } -# async fn scan(&self, _: &dyn Session, _: Option<&Vec>, _: &[Expr], _: Option) -> Result> { todo!() } +# async fn scan(&self, _: &dyn Session, _: Option<&[usize]>, _: &[Expr], _: Option) -> Result> { todo!() } # fn supports_filters_pushdown( &self, @@ -700,7 +700,7 @@ impl TableProvider for DatePartitionedTable { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> Result> { @@ -848,7 +848,7 @@ impl TableProvider for CountingTable { async fn scan( &self, _state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, _filters: &[Expr], limit: Option, ) -> Result> { diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md index d64f287ea0b52..21def5a1fc02c 100644 --- a/docs/source/library-user-guide/upgrading/55.0.0.md +++ b/docs/source/library-user-guide/upgrading/55.0.0.md @@ -25,6 +25,57 @@ in this section pertains to features and changes that have already been merged to the main branch and are awaiting release in this version. +### `TableProvider::scan` takes `Option<&[usize]>` for the projection + +`TableProvider::scan` and the related APIs listed below previously took the +projection as `Option<&Vec>`: + +```rust,ignore +// Before +async fn scan( + &self, + state: &dyn Session, + projection: Option<&Vec>, + filters: &[Expr], + limit: Option, +) -> Result>; + +// After +async fn scan( + &self, + state: &dyn Session, + projection: Option<&[usize]>, + filters: &[Expr], + limit: Option, +) -> Result>; +``` + +Implementations only need the signature updated. Two patterns inside a `scan` +body need adjusting, because `Option<&[usize]>` is not an `Option<&T>` over a +sized type: + +```rust,ignore +// Before // After +projection.cloned() projection.map(|p| p.to_vec()) +``` + +Callers holding an `Option>` should pass `.as_deref()` rather than +`.as_ref()`. + +The same change applies to `StreamingTableExec::try_new`, +`FilterExec::with_projection` and `batch_filter`, which took the projection in +the same shape. + +`datafusion_common::project_schema` now accepts unsized projections +(`projection: Option<&T> where T: AsRef<[usize]> + ?Sized`), so it takes both +`Option<&Vec>` and `Option<&[usize]>`; no caller changes are required. + +Besides being the more idiomatic signature, this lets +`TableProvider::scan_with_args`'s default body hand the projection straight to +`scan` instead of allocating a `Vec` that then has to be captured by the +returned future -- which is what made that method expensive to compile +(`datafusion-session` compiles ~3.3x faster as a result). + ### `DataFrame::fill_null` now borrows its arguments `DataFrame::fill_null` previously took its arguments by value: From b095406ed43038b2ce84b5e4b52e88d3c95be22b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20=C5=9Een?= Date: Sat, 15 Aug 2026 11:04:19 +0000 Subject: [PATCH 33/50] fix(proto): preserve Arrow IPC stream format (#24224) ## Which issue does this PR close? - Closes #24196. ## Rationale for this change `ArrowSource` distinguishes between Arrow IPC file and stream formats, but this format was not serialized. As a result, a stream scan round-tripped through protobuf as a file scan, selecting the wrong opener and potentially allowing unsupported range-based repartitioning. ## What changes are included in this PR? - Add an `ArrowIpcFormat` discriminator to `ArrowScanExecNode`. - Serialize and restore the `ArrowSource` IPC format. - Decode payloads without the new field as file format, preserving the previous behavior. - Regenerate the prost and pbjson models. - Add regression coverage for file, stream, and older payloads without the format field. ## Are these changes tested? Yes: - `cargo test -p datafusion-proto --test proto_integration roundtrip_arrow` - `cargo test -p datafusion-proto --test proto_integration arrow_scan_without_format_field_decodes_as_file_format` - `cargo fmt --all` - Targeted all-feature clippy for `datafusion-datasource-arrow` and `datafusion-proto` ## Are there any user-facing changes? Arrow IPC stream scans now preserve their format across protobuf round trips. The binary protobuf change is additive. Older payloads continue to decode as file format. Stream preservation requires both producer and consumer to include this change because older versions do not carry or read the discriminator. The generated Rust `ArrowScanExecNode` struct gains a `format` field. --- datafusion/datasource-arrow/src/source.rs | 32 ++++-- .../proto-models/proto/datafusion.proto | 10 ++ .../proto-models/src/generated/pbjson.rs | 90 ++++++++++++++++ .../proto-models/src/generated/prost.rs | 32 ++++++ datafusion/proto/tests/cases/plans/sources.rs | 101 +++++++++++++++++- 5 files changed, 256 insertions(+), 9 deletions(-) diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rs index dba99a9758886..dd973c03ae19f 100644 --- a/datafusion/datasource-arrow/src/source.rs +++ b/datafusion/datasource-arrow/src/source.rs @@ -403,10 +403,8 @@ impl FileSource for ArrowSource { datafusion_physical_plan::apply_expression_roots(self.projection.source.iter(), f) } - /// Emit an `ArrowScan` node wrapping the shared base config. - /// - /// Decoding defaults to the IPC file format because protobuf does not - /// distinguish it from the IPC stream format. + /// Emit an `ArrowScan` node wrapping the shared base config and recording + /// which Arrow IPC format (file or stream) this source reads. #[cfg(feature = "proto")] fn try_to_proto( &self, @@ -416,10 +414,16 @@ impl FileSource for ArrowSource { use datafusion_proto_models::protobuf; use protobuf::physical_plan_node::PhysicalPlanType; + let format = match self.format { + ArrowFormat::File => protobuf::ArrowIpcFormat::File, + ArrowFormat::Stream => protobuf::ArrowIpcFormat::Stream, + }; + Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some(PhysicalPlanType::ArrowScan( protobuf::ArrowScanExecNode { base_conf: Some(base.try_to_proto(ctx)?), + format: format as i32, }, )), })) @@ -430,8 +434,8 @@ impl FileSource for ArrowSource { impl ArrowSource { /// Reconstructs a `DataSourceExec` from a protobuf `ArrowScan`. /// - /// Defaults to the IPC file format because protobuf does not distinguish it - /// from the IPC stream format. + /// Payloads encoded before the `format` field existed leave it unset; + /// those decode as the IPC file format, matching the historical behavior. pub fn try_from_proto( node: &datafusion_proto_models::protobuf::PhysicalPlanNode, ctx: &datafusion_physical_plan::proto::ExecutionPlanDecodeCtx<'_>, @@ -455,9 +459,21 @@ impl ArrowSource { ) })?; + let format = protobuf::ArrowIpcFormat::try_from(scan.format).map_err(|_| { + datafusion_common::internal_datafusion_err!( + "Unknown ArrowIpcFormat: {}", + scan.format + ) + })?; + let table_schema = FileScanConfig::parse_table_schema_from_proto(base_conf)?; - let source = Arc::new(ArrowSource::new_file_source(table_schema)); - let scan_conf = FileScanConfig::try_from_proto(base_conf, ctx, source)?; + let source = match format { + protobuf::ArrowIpcFormat::Stream => { + ArrowSource::new_stream_file_source(table_schema) + } + protobuf::ArrowIpcFormat::File => ArrowSource::new_file_source(table_schema), + }; + let scan_conf = FileScanConfig::try_from_proto(base_conf, ctx, Arc::new(source))?; Ok(DataSourceExec::from_data_source(scan_conf)) } } diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto index 43a90264c2b1f..f29d63c4762b9 100644 --- a/datafusion/proto-models/proto/datafusion.proto +++ b/datafusion/proto-models/proto/datafusion.proto @@ -1296,8 +1296,18 @@ message AvroScanExecNode { FileScanExecConf base_conf = 1; } +// Identifies which Arrow IPC format an ArrowScanExecNode reads. +enum ArrowIpcFormat { + // Arrow IPC file format (with footer, supports range-based parallel reading). + // This is the default for payloads encoded before the format field existed. + ARROW_IPC_FORMAT_FILE = 0; + // Arrow IPC stream format (without footer, sequential reading only) + ARROW_IPC_FORMAT_STREAM = 1; +} + message ArrowScanExecNode { FileScanExecConf base_conf = 1; + ArrowIpcFormat format = 2; } message MemoryScanExecNode { diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs index 908f9752b7f18..19121b2dbee9b 100644 --- a/datafusion/proto-models/src/generated/pbjson.rs +++ b/datafusion/proto-models/src/generated/pbjson.rs @@ -1445,6 +1445,77 @@ impl<'de> serde::Deserialize<'de> for AnalyzedLogicalPlanType { deserializer.deserialize_struct("datafusion.AnalyzedLogicalPlanType", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for ArrowIpcFormat { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + let variant = match self { + Self::File => "ARROW_IPC_FORMAT_FILE", + Self::Stream => "ARROW_IPC_FORMAT_STREAM", + }; + serializer.serialize_str(variant) + } +} +impl<'de> serde::Deserialize<'de> for ArrowIpcFormat { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "ARROW_IPC_FORMAT_FILE", + "ARROW_IPC_FORMAT_STREAM", + ]; + + struct GeneratedVisitor; + + impl serde::de::Visitor<'_> for GeneratedVisitor { + type Value = ArrowIpcFormat; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + fn visit_i64(self, v: i64) -> std::result::Result + where + E: serde::de::Error, + { + i32::try_from(v) + .ok() + .and_then(|x| x.try_into().ok()) + .ok_or_else(|| { + serde::de::Error::invalid_value(serde::de::Unexpected::Signed(v), &self) + }) + } + + fn visit_u64(self, v: u64) -> std::result::Result + where + E: serde::de::Error, + { + i32::try_from(v) + .ok() + .and_then(|x| x.try_into().ok()) + .ok_or_else(|| { + serde::de::Error::invalid_value(serde::de::Unexpected::Unsigned(v), &self) + }) + } + + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "ARROW_IPC_FORMAT_FILE" => Ok(ArrowIpcFormat::File), + "ARROW_IPC_FORMAT_STREAM" => Ok(ArrowIpcFormat::Stream), + _ => Err(serde::de::Error::unknown_variant(value, FIELDS)), + } + } + } + deserializer.deserialize_any(GeneratedVisitor) + } +} impl serde::Serialize for ArrowScanExecNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result @@ -1456,10 +1527,18 @@ impl serde::Serialize for ArrowScanExecNode { if self.base_conf.is_some() { len += 1; } + if self.format != 0 { + len += 1; + } let mut struct_ser = serializer.serialize_struct("datafusion.ArrowScanExecNode", len)?; if let Some(v) = self.base_conf.as_ref() { struct_ser.serialize_field("baseConf", v)?; } + if self.format != 0 { + let v = ArrowIpcFormat::try_from(self.format) + .map_err(|_| serde::ser::Error::custom(format!("Invalid variant {}", self.format)))?; + struct_ser.serialize_field("format", &v)?; + } struct_ser.end() } } @@ -1472,11 +1551,13 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode { const FIELDS: &[&str] = &[ "base_conf", "baseConf", + "format", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { BaseConf, + Format, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -1499,6 +1580,7 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode { { match value { "baseConf" | "base_conf" => Ok(GeneratedField::BaseConf), + "format" => Ok(GeneratedField::Format), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -1519,6 +1601,7 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode { V: serde::de::MapAccess<'de>, { let mut base_conf__ = None; + let mut format__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::BaseConf => { @@ -1527,10 +1610,17 @@ impl<'de> serde::Deserialize<'de> for ArrowScanExecNode { } base_conf__ = map_.next_value()?; } + GeneratedField::Format => { + if format__.is_some() { + return Err(serde::de::Error::duplicate_field("format")); + } + format__ = Some(map_.next_value::()? as i32); + } } } Ok(ArrowScanExecNode { base_conf: base_conf__, + format: format__.unwrap_or_default(), }) } } diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs index ba00577ab9a1b..a6fe825ff6f0e 100644 --- a/datafusion/proto-models/src/generated/prost.rs +++ b/datafusion/proto-models/src/generated/prost.rs @@ -1996,6 +1996,8 @@ pub struct AvroScanExecNode { pub struct ArrowScanExecNode { #[prost(message, optional, tag = "1")] pub base_conf: ::core::option::Option, + #[prost(enumeration = "ArrowIpcFormat", tag = "2")] + pub format: i32, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct MemoryScanExecNode { @@ -2794,6 +2796,36 @@ impl InsertOp { } } } +/// Identifies which Arrow IPC format an ArrowScanExecNode reads. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ArrowIpcFormat { + /// Arrow IPC file format (with footer, supports range-based parallel reading). + /// This is the default for payloads encoded before the format field existed. + File = 0, + /// Arrow IPC stream format (without footer, sequential reading only) + Stream = 1, +} +impl ArrowIpcFormat { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::File => "ARROW_IPC_FORMAT_FILE", + Self::Stream => "ARROW_IPC_FORMAT_STREAM", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "ARROW_IPC_FORMAT_FILE" => Some(Self::File), + "ARROW_IPC_FORMAT_STREAM" => Some(Self::Stream), + _ => None, + } + } +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum PartitionMode { diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rs index 04708dec6439c..5cdd42a0acc3b 100644 --- a/datafusion/proto/tests/cases/plans/sources.rs +++ b/datafusion/proto/tests/cases/plans/sources.rs @@ -57,10 +57,15 @@ use datafusion_datasource::{TableSchema, TableSchemaBuilder}; use datafusion_expr::ColumnarValue; use datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx; use datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx; +use datafusion_proto::bytes::{ + physical_plan_from_bytes_with_proto_converter, + physical_plan_to_bytes_with_proto_converter, +}; use datafusion_proto::physical_plan::{ AsExecutionPlan, DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalProtoConverterExtension, }; +use datafusion_proto::protobuf; use datafusion_proto::protobuf::PhysicalPlanNode; use prost::Message; use std::collections::HashMap; @@ -156,6 +161,22 @@ fn roundtrip_parquet_exec_attaches_cached_reader_factory_after_roundtrip() -> Re Ok(()) } +/// Returns `FileSource::file_type` of a `DataSourceExec` file scan, e.g. +/// "arrow" vs "arrow_stream". The two Arrow IPC formats print identically in +/// plan debug output, so roundtrip tests must inspect the source directly. +fn scan_file_type(plan: &Arc) -> Result { + let data_source = plan.downcast_ref::().ok_or_else(|| { + internal_datafusion_err!("Expected DataSourceExec after roundtrip") + })?; + let file_scan = data_source + .data_source() + .downcast_ref::() + .ok_or_else(|| { + internal_datafusion_err!("Expected FileScanConfig after roundtrip") + })?; + Ok(file_scan.file_source().file_type().to_string()) +} + #[test] fn roundtrip_arrow_scan() -> Result<()> { let file_schema = @@ -177,7 +198,85 @@ fn roundtrip_arrow_scan() -> Result<()> { }) .build(); - roundtrip_test(DataSourceExec::from_data_source(scan_config)) + let roundtripped = roundtrip_test_and_return( + DataSourceExec::from_data_source(scan_config), + &SessionContext::new(), + &DefaultPhysicalExtensionCodec {}, + &DefaultPhysicalProtoConverter {}, + )?; + assert_eq!(scan_file_type(&roundtripped)?, "arrow"); + Ok(()) +} + +#[test] +fn roundtrip_arrow_stream_scan() -> Result<()> { + let file_schema = + Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)])); + let file_source = Arc::new(ArrowSource::new_stream_file_source(TableSchema::from( + &file_schema, + ))); + let scan_config = + FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source) + .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new( + "/path/to/file.arrows".to_string(), + 1024, + )])]) + .build(); + + let roundtripped = roundtrip_test_and_return( + DataSourceExec::from_data_source(scan_config), + &SessionContext::new(), + &DefaultPhysicalExtensionCodec {}, + &DefaultPhysicalProtoConverter {}, + )?; + assert_eq!(scan_file_type(&roundtripped)?, "arrow_stream"); + Ok(()) +} + +#[test] +fn arrow_scan_without_format_field_decodes_as_file_format() -> Result<()> { + // Payloads encoded before `ArrowScanExecNode.format` existed carry no + // format discriminator; they must keep decoding as the IPC file format. + let file_schema = + Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)])); + let file_source = Arc::new(ArrowSource::new_file_source(TableSchema::from( + &file_schema, + ))); + let scan_config = + FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source) + .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new( + "/path/to/file.arrow".to_string(), + 1024, + )])]) + .build(); + + let codec = DefaultPhysicalExtensionCodec {}; + let proto_converter = DefaultPhysicalProtoConverter {}; + let bytes = physical_plan_to_bytes_with_proto_converter( + DataSourceExec::from_data_source(scan_config), + &codec, + &proto_converter, + )?; + + let mut node = PhysicalPlanNode::decode(bytes.as_ref()).map_err(|e| { + internal_datafusion_err!("Failed to decode PhysicalPlanNode: {e}") + })?; + match node.physical_plan_type.as_mut() { + Some(protobuf::physical_plan_node::PhysicalPlanType::ArrowScan(scan)) => { + scan.format = protobuf::ArrowIpcFormat::File as i32; + } + other => return internal_err!("Expected ArrowScan node, got {other:?}"), + } + + let ctx = SessionContext::new(); + let decoded = physical_plan_from_bytes_with_proto_converter( + &node.encode_to_vec(), + ctx.task_ctx().as_ref(), + &codec, + &proto_converter, + )?; + assert_eq!(scan_file_type(&decoded)?, "arrow"); + Ok(()) } #[test] From fb1ca94c629dacfafccfbd0524957befe7ca2e8e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 15 Aug 2026 11:04:48 +0000 Subject: [PATCH 34/50] docs: Document `WindowFn` and add comments (#24118) ## Which issue does this PR close? - N/A (documentation only) ## Rationale for this change The `WindowFn` enum in `datafusion-physical-expr` has no documentation, and its `Builtin` variant name is misleading: ## What changes are included in this PR? Doc comments on `WindowFn` and its two variants, including links to the `WindowExpr` implementations that create each variant. ## Are these changes tested? By CI ## Are there any user-facing changes? Improved API documentation only; no code changes. --------- Co-authored-by: Claude Fable 5 --- datafusion/physical-expr/src/window/window_expr.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/datafusion/physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rs index 1c52ea4ea6d0e..7fe768cfbe329 100644 --- a/datafusion/physical-expr/src/window/window_expr.rs +++ b/datafusion/physical-expr/src/window/window_expr.rs @@ -610,9 +610,16 @@ pub(crate) fn get_orderby_values(order_by_columns: Vec) -> Vec), + /// An aggregate function used as a window function, such as `avg` or + /// `sum`, which is evaluated via an [`Accumulator`]. Aggregate(Box), } From b186633b9f6f7f882b5a81f1bb7e3d0e59fe369d Mon Sep 17 00:00:00 2001 From: RIchard Baah <137434454+Rich-T-kid@users.noreply.github.com> Date: Sat, 15 Aug 2026 11:06:08 +0000 Subject: [PATCH 35/50] Perf: remove extra allocations for hex operations (#24057) ## Which issue does this PR close? - Closes #23810. ## Rationale for this change The digest functions hex-encode their output by allocating one String per row and then copying each of those into the output array (#23810) ## What changes are included in this PR? - This PR reduces per-row allocation and string-conversion overhead in the Spark crypto/hash paths by switching to the buffer-based hex encoder APIs. - Updates call sites to re-use vector buffers. ## Are these changes tested? existing test cover behavior. ## Are there any user-facing changes? yes, `encode_bytes()` is being deprecated. This was apart of the public API --------- Co-authored-by: rich-T-kid --- datafusion/common/src/utils/hex.rs | 2 + datafusion/functions/src/crypto/md5.rs | 43 ++++++-- datafusion/functions/src/encoding/inner.rs | 4 +- datafusion/spark/src/function/hash/sha1.rs | 7 +- datafusion/spark/src/function/hash/sha2.rs | 121 +++++++++++++++------ 5 files changed, 131 insertions(+), 46 deletions(-) diff --git a/datafusion/common/src/utils/hex.rs b/datafusion/common/src/utils/hex.rs index 872d54f40c6f7..672cd8463f0a0 100644 --- a/datafusion/common/src/utils/hex.rs +++ b/datafusion/common/src/utils/hex.rs @@ -161,6 +161,8 @@ pub fn encode_bytes_to_slice(bytes: &[u8], case: HexCase, out: &mut [u8]) -> Res /// Returns the hex encoding of `bytes` as an owned `String`. /// +/// Prefer [`encode_bytes_into`] when you already have a reusable output buffer. +/// /// # Example /// /// ``` diff --git a/datafusion/functions/src/crypto/md5.rs b/datafusion/functions/src/crypto/md5.rs index b1206d2e423cc..c9aba15e6ef22 100644 --- a/datafusion/functions/src/crypto/md5.rs +++ b/datafusion/functions/src/crypto/md5.rs @@ -15,13 +15,16 @@ // specific language governing permissions and limitations // under the License. -use arrow::{array::StringViewArray, datatypes::DataType}; +use arrow::{ + array::{Array, BinaryViewBuilder}, + datatypes::DataType, +}; use datafusion_common::{ Result, ScalarValue, cast::as_binary_array, internal_err, types::{logical_binary, logical_string}, - utils::hex::{HexCase, encode_bytes}, + utils::hex::{HexCase, encode_bytes_into}, utils::take_function_args, }; use datafusion_expr::{ @@ -107,15 +110,35 @@ fn md5(args: &[ColumnarValue]) -> Result { Ok(match value { ColumnarValue::Array(array) => { let binary_array = as_binary_array(&array)?; - let string_array: StringViewArray = binary_array - .iter() - .map(|opt| opt.map(|b| encode_bytes(b, HexCase::Lower))) - .collect(); - ColumnarValue::Array(Arc::new(string_array)) + let mut byte_builder = BinaryViewBuilder::with_capacity(binary_array.len()); + let mut hex_bytes = Vec::with_capacity(32); + + for i in 0..binary_array.len() { + if binary_array.is_null(i) { + byte_builder.append_null(); + continue; + } + + hex_bytes.clear(); + let digest = binary_array.value(i); + encode_bytes_into(digest, HexCase::Lower, &mut hex_bytes); + byte_builder.append_value(&hex_bytes); + } + + let str_array = unsafe { + // Safe: `encode_bytes_into` only writes ASCII hex digits, so the bytes are valid UTF-8. + byte_builder.finish().to_string_view_unchecked() + }; + ColumnarValue::Array(Arc::new(str_array)) + } + ColumnarValue::Scalar(ScalarValue::Binary(opt)) => { + ColumnarValue::Scalar(ScalarValue::Utf8View(opt.map(|b| { + let mut hex_bytes = Vec::with_capacity(b.len() * 2); + encode_bytes_into(&b, HexCase::Lower, &mut hex_bytes); + // Safe: `encode_bytes_into` only writes ASCII hex digits, so the bytes are valid UTF-8. + unsafe { String::from_utf8_unchecked(hex_bytes) } + }))) } - ColumnarValue::Scalar(ScalarValue::Binary(opt)) => ColumnarValue::Scalar( - ScalarValue::Utf8View(opt.map(|b| encode_bytes(&b, HexCase::Lower))), - ), _ => return internal_err!("Impossibly got invalid results from digest"), }) } diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rs index 850e312abdb40..8b57033fa0de6 100644 --- a/datafusion/functions/src/encoding/inner.rs +++ b/datafusion/functions/src/encoding/inner.rs @@ -34,7 +34,7 @@ use datafusion_common::{ not_impl_err, plan_err, types::{NativeType, logical_string}, utils::{ - hex::{HexCase, encode_bytes as encode_hex, encode_bytes_to_slice}, + hex::{HexCase, encode_bytes, encode_bytes_to_slice}, take_function_args, }, }; @@ -373,7 +373,7 @@ impl Encoding { match self { Self::Base64 => BASE64_ENGINE.encode(value), Self::Base64Padded => BASE64_ENGINE_PADDED.encode(value), - Self::Hex => encode_hex(value, HexCase::Lower), + Self::Hex => encode_bytes(value, HexCase::Lower), } } diff --git a/datafusion/spark/src/function/hash/sha1.rs b/datafusion/spark/src/function/hash/sha1.rs index 05a224f33f25a..64c52a7108283 100644 --- a/datafusion/spark/src/function/hash/sha1.rs +++ b/datafusion/spark/src/function/hash/sha1.rs @@ -24,7 +24,7 @@ use datafusion_common::cast::{ as_large_binary_array, }; use datafusion_common::types::{NativeType, logical_string}; -use datafusion_common::utils::hex::{HexCase, encode_bytes}; +use datafusion_common::utils::hex::{HexCase, encode_bytes_into}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ @@ -92,7 +92,10 @@ impl ScalarUDFImpl for SparkSha1 { #[inline] fn spark_sha1_digest(value: &[u8]) -> String { - encode_bytes(&Sha1::digest(value), HexCase::Lower) + let mut out = Vec::with_capacity(40); + // Safe: `encode_bytes_into` only writes ASCII hex digits, which are valid UTF-8. + encode_bytes_into(&Sha1::digest(value), HexCase::Lower, &mut out); + unsafe { String::from_utf8_unchecked(out) } } fn spark_sha1_impl<'a>(input: impl Iterator>) -> ArrayRef { diff --git a/datafusion/spark/src/function/hash/sha2.rs b/datafusion/spark/src/function/hash/sha2.rs index 541df2957669e..53c1ff9465781 100644 --- a/datafusion/spark/src/function/hash/sha2.rs +++ b/datafusion/spark/src/function/hash/sha2.rs @@ -15,12 +15,14 @@ // specific language governing permissions and limitations // under the License. -use arrow::array::{ArrayRef, AsArray, BinaryArrayType, Int32Array, StringArray}; +use arrow::array::{ + ArrayRef, AsArray, BinaryArrayType, BinaryBuilder, Int32Array, StringArray, +}; use arrow::datatypes::{DataType, Int32Type}; use datafusion_common::types::{ NativeType, logical_binary, logical_int32, logical_string, }; -use datafusion_common::utils::hex::{HexCase, encode_bytes}; +use datafusion_common::utils::hex::{HexCase, encode_bytes_into}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, internal_err}; use datafusion_expr::{ @@ -113,22 +115,58 @@ impl ScalarUDFImpl for SparkSha2 { 224 => { let mut digest = sha2::Sha224::default(); digest.update(bytes); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) + let mut hex_bytes = Vec::with_capacity(56); + encode_bytes_into( + &digest.finalize(), + HexCase::Lower, + &mut hex_bytes, + ); + Some( + String::from_utf8(hex_bytes) + .expect("ASCII hex is valid UTF-8"), + ) } 0 | 256 => { let mut digest = sha2::Sha256::default(); digest.update(bytes); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) + let mut hex_bytes = Vec::with_capacity(64); + encode_bytes_into( + &digest.finalize(), + HexCase::Lower, + &mut hex_bytes, + ); + Some( + String::from_utf8(hex_bytes) + .expect("ASCII hex is valid UTF-8"), + ) } 384 => { let mut digest = sha2::Sha384::default(); digest.update(bytes); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) + let mut hex_bytes = Vec::with_capacity(96); + encode_bytes_into( + &digest.finalize(), + HexCase::Lower, + &mut hex_bytes, + ); + Some( + String::from_utf8(hex_bytes) + .expect("ASCII hex is valid UTF-8"), + ) } 512 => { let mut digest = sha2::Sha512::default(); digest.update(bytes); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) + let mut hex_bytes = Vec::with_capacity(128); + encode_bytes_into( + &digest.finalize(), + HexCase::Lower, + &mut hex_bytes, + ); + Some( + String::from_utf8(hex_bytes) + .expect("ASCII hex is valid UTF-8"), + ) } _ => None, }; @@ -216,33 +254,52 @@ where BinaryArrType: BinaryArrayType<'a>, I: Iterator>, { - let array = values + let mut byte_builder = BinaryBuilder::with_capacity(values.len(), values.len() * 2); + let mut hex_bytes = Vec::with_capacity(128); + + values .iter() .zip(bit_lengths) - .map(|(value, bit_length)| match (value, bit_length) { - (Some(value), Some(224)) => { - let mut digest = sha2::Sha224::default(); - digest.update(value); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) - } - (Some(value), Some(0 | 256)) => { - let mut digest = sha2::Sha256::default(); - digest.update(value); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) - } - (Some(value), Some(384)) => { - let mut digest = sha2::Sha384::default(); - digest.update(value); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) - } - (Some(value), Some(512)) => { - let mut digest = sha2::Sha512::default(); - digest.update(value); - Some(encode_bytes(&digest.finalize(), HexCase::Lower)) + .for_each(|(value, bit_length)| { + match (value, bit_length) { + (Some(value), Some(224)) => { + let mut digest = sha2::Sha224::default(); + digest.update(value); + hex_bytes.clear(); + encode_bytes_into(&digest.finalize(), HexCase::Lower, &mut hex_bytes); + byte_builder.append_value(&hex_bytes); + } + (Some(value), Some(0 | 256)) => { + let mut digest = sha2::Sha256::default(); + digest.update(value); + hex_bytes.clear(); + encode_bytes_into(&digest.finalize(), HexCase::Lower, &mut hex_bytes); + byte_builder.append_value(&hex_bytes); + } + (Some(value), Some(384)) => { + let mut digest = sha2::Sha384::default(); + digest.update(value); + hex_bytes.clear(); + encode_bytes_into(&digest.finalize(), HexCase::Lower, &mut hex_bytes); + byte_builder.append_value(&hex_bytes); + } + (Some(value), Some(512)) => { + let mut digest = sha2::Sha512::default(); + digest.update(value); + hex_bytes.clear(); + encode_bytes_into(&digest.finalize(), HexCase::Lower, &mut hex_bytes); + byte_builder.append_value(&hex_bytes); + } + // Unknown bit-lengths go to null, same as in Spark + _ => byte_builder.append_null(), } - // Unknown bit-lengths go to null, same as in Spark - _ => None, - }) - .collect::(); - Arc::new(array) + }); + + let str_array = unsafe { + let binary_array = byte_builder.finish(); + let (offsets, values, nulls) = binary_array.into_parts(); + // Safe: `encode_bytes_into` only writes ASCII hex digits, so the bytes are valid UTF-8. + StringArray::new_unchecked(offsets, values, nulls) + }; + Arc::new(str_array) } From 2d20db76d5ff4576693f3fb2f4bfeabaab05257e Mon Sep 17 00:00:00 2001 From: sakshichitnis27 <156598682+sakshichitnis27@users.noreply.github.com> Date: Sat, 15 Aug 2026 11:08:30 +0000 Subject: [PATCH 36/50] Document decimal AVG wrapping arithmetic (#23685) ## Which issue does this PR close? - Closes #23665 ## Rationale for this change Decimal AVG intentionally uses wrapping arithmetic with a widened intermediate sum type. The existing direct `add_wrapping` and `sub_wrapping` calls made that invariant difficult to audit. ## What changes are included in this PR? - Added small private helpers documenting the widened/headroom contract. - Updated scalar, grouped, merge, retract, and `AVG(DISTINCT)` decimal paths to use the helpers. - Kept the existing state types, return types, and arithmetic behavior unchanged. ## Are these changes tested? - `cargo fmt --all -- --check` - `cargo test -p datafusion-functions-aggregate avg --lib` - `cargo test -p datafusion-functions-aggregate-common avg_distinct --lib` - `cargo clippy -p datafusion-functions-aggregate-common --all-targets -- -D warnings` ## Are there any user-facing changes? No. Co-authored-by: Adam Gutglick --- .../src/aggregate/avg_distinct/decimal.rs | 15 +++++++- datafusion/functions-aggregate/src/average.rs | 37 +++++++++++++++---- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/datafusion/functions-aggregate-common/src/aggregate/avg_distinct/decimal.rs b/datafusion/functions-aggregate-common/src/aggregate/avg_distinct/decimal.rs index 1b2fb01189feb..d9e06d39523b4 100644 --- a/datafusion/functions-aggregate-common/src/aggregate/avg_distinct/decimal.rs +++ b/datafusion/functions-aggregate-common/src/aggregate/avg_distinct/decimal.rs @@ -65,6 +65,19 @@ impl DecimalDistinctAvgAccumulat } } +/// Adds a distinct input value to AVG's widened intermediate sum. +/// Wrapping is intentional because the caller selects `S` with the same +/// `avg_sum_data_type` headroom contract as the non-distinct AVG path. +#[inline] +fn add_avg_distinct_sum(sum: S::Native, value: I::Native) -> S::Native +where + I: ArrowNumericType, + S: ArrowNumericType, + I::Native: Into, +{ + sum.add_wrapping(value.into()) +} + impl Accumulator for DecimalDistinctAvgAccumulator where I: DecimalType + ArrowNumericType + Debug, @@ -95,7 +108,7 @@ where // overflow the input's native width (mirrors the non-distinct path). let mut sum = S::Native::usize_as(0); for value in self.sum_accumulator.distinct_values() { - sum = sum.add_wrapping(value.into()); + sum = add_avg_distinct_sum::(sum, value); } let Some(count) = S::Native::from_usize(count) else { diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rs index e5030bf39e409..89b633085240c 100644 --- a/datafusion/functions-aggregate/src/average.rs +++ b/datafusion/functions-aggregate/src/average.rs @@ -700,6 +700,29 @@ where /// Wraps on overflow, matching the `sum` aggregate and [`arrow::compute::sum`]. /// [`avg_sum_data_type`] gives `S` enough headroom that this is unreachable for /// any realistic row count. +#[inline] +fn add_avg_sum(sum: S::Native, value: I::Native) -> S::Native +where + I: ArrowNumericType, + S: ArrowNumericType, + I::Native: Into, +{ + sum.add_wrapping(value.into()) +} + +/// Subtracts a value already represented by AVG's widened state. The wrapping +/// operation is intentional because `avg_sum_data_type` reserves the +/// documented headroom for the intermediate sum. +#[inline] +fn sub_avg_sum(sum: S::Native, value: I::Native) -> S::Native +where + I: ArrowNumericType, + S: ArrowNumericType, + I::Native: Into, +{ + sum.sub_wrapping(value.into()) +} + fn decimal_sum_as(values: &PrimitiveArray) -> Option where I: DecimalType + ArrowNumericType, @@ -714,11 +737,11 @@ where let mut sum = S::Native::default(); if values.null_count() == 0 { for value in values.values() { - sum = sum.add_wrapping((*value).into()); + sum = add_avg_sum::(sum, *value); } } else { for value in values.iter().flatten() { - sum = sum.add_wrapping(value.into()); + sum = add_avg_sum::(sum, value); } } @@ -738,7 +761,7 @@ where if let Some(x) = decimal_sum_as::(values) { let v = self.sum.unwrap_or_default(); - self.sum = Some(v.add_wrapping(x)); + self.sum = Some(add_avg_sum::(v, x)); } Ok(()) } @@ -774,7 +797,7 @@ where // sums are summed if let Some(x) = sum(states[1].as_primitive::()) { let v = self.sum.unwrap_or_default(); - self.sum = Some(v.add_wrapping(x)); + self.sum = Some(add_avg_sum::(v, x)); } Ok(()) } @@ -783,7 +806,7 @@ where self.count -= (values.len() - values.null_count()) as u64; if let Some(x) = decimal_sum_as::(values) { let v = self.sum.unwrap_or_default(); - self.sum = Some(v.sub_wrapping(x)); + self.sum = Some(sub_avg_sum::(v, x)); } Ok(()) } @@ -990,7 +1013,7 @@ where |group_index, new_value| { // SAFETY: group_index is guaranteed to be in bounds let sum = unsafe { self.sums.get_unchecked_mut(group_index) }; - *sum = sum.add_wrapping(new_value.into()); + *sum = add_avg_sum::(*sum, new_value); self.counts[group_index] += 1; }, @@ -1090,7 +1113,7 @@ where |group_index, new_value: ::Native| { // SAFETY: group_index is guaranteed to be in bounds let sum = unsafe { self.sums.get_unchecked_mut(group_index) }; - *sum = sum.add_wrapping(new_value); + *sum = add_avg_sum::(*sum, new_value); }, ); From 855a47b648cc3f5de918a7757f5db6f2cb856264 Mon Sep 17 00:00:00 2001 From: Alvaro Balbin <153404840+AlvaroBalbin@users.noreply.github.com> Date: Sat, 15 Aug 2026 11:16:18 +0000 Subject: [PATCH 37/50] docs(upgrading): document RecursiveQuery schema field in upgrade guide (#23991) Adding the `schema` field to `RecursiveQuery` broke downstream code that built the node with a struct literal, and the change was never written up for people upgrading. I added an upgrade guide entry for 55.0.0 that describes what changed, notes it also shipped in 54.1.0 so users on the 54 series find it, and shows how to move to `RecursiveQuery::try_new`. It is documentation only, so whether to revert the change on `branch-54` for 54.2.0 is still an open question. I pushed this to my fork first and the project's own CI workflows pass on the commit. Fixes #23886. Co-authored-by: Andrew Lamb --- .../library-user-guide/upgrading/55.0.0.md | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md index 21def5a1fc02c..b0f2602d31f5a 100644 --- a/docs/source/library-user-guide/upgrading/55.0.0.md +++ b/docs/source/library-user-guide/upgrading/55.0.0.md @@ -1109,6 +1109,41 @@ let plan = deserialize_bytes(&proto_bytes)?; See [PR #23827](https://github.com/apache/datafusion/pull/23827) for details. +### `RecursiveQuery` gained a `schema` field + +`datafusion_expr::logical_plan::RecursiveQuery` exposed the static (anchor) term's +schema to parent plans, which ignored columns that only the recursive term makes +nullable. The node now stores the schema reconciled from both terms explicitly, so +it can no longer be built with an exhaustive struct literal. + +**Who is affected:** + +- Code constructing `RecursiveQuery` with a struct literal. This change also + shipped in `54.1.0`, so the same migration applies within the `54` series. + +**Migration guide:** + +Build the node with `RecursiveQuery::try_new`, which computes the schema and +returns an error if the two terms do not have the same number of columns: + +```rust,ignore +// Before +let query = RecursiveQuery { + name, + static_term, + recursive_term, + is_distinct, +}; + +// After +let query = RecursiveQuery::try_new(name, static_term, recursive_term, is_distinct)?; +``` + +`RecursiveQueryExec::try_new` takes the reconciled schema as its second argument +for the same reason, rather than deriving it from its children. + +See [PR #22552](https://github.com/apache/datafusion/pull/22552) for details. + ### `ExecutionPlan::apply_expressions` is now a required method `apply_expressions` has been added as a **required** method on the `ExecutionPlan`, `FileSource`, and `DataSource` traits. Any custom implementation of From 143b56b9b4f30d1abb250b8662007641ef3d2dbd Mon Sep 17 00:00:00 2001 From: Reid Kaufmann <73494261+reidkaufmann@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:11:31 +0000 Subject: [PATCH 38/50] Improve planning speed: Fast path for `union_schema` when all children share a schema (#24389) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this close? Complements #19792. Fits with the wide-`UnionExec` planning-cost work, but originates from an InfluxDB issue. ## Rationale for this change `union_schema` builds the output schema for `UnionExec` and `InterleaveExec` by coercing field metadata and nullability across **every** child. That merge is quadratic in the number of children: for each output field it walks all inputs, and for each input it walks *every other* input to union field-level metadata. For a union of `n` children with `f` fields the construction cost is `O(n^2 * f)` (worse when fields carry metadata). For narrow unions this is insignificant. It matters when a plan fans a single source out into many identical-schema children and unions them back together -- e.g. a union assembled from repartitioned copies of the same input. An instance like this occurred with InfluxDB: every child schema was the same, so the merge, guaranteed to reproduce the first child's schema, unnecessarily incurred the planning latency penalty from `O(n^2 * f)` complexity. ### Relationship to #19792 `UnionExec` construction has two quadratic halves: - **`with_new_children` / `PlanProperties`** -- addressed by #19792 (`with_new_children_and_same_properties`, `Arc`, the properties fast path). Already on `main`. - **`union_schema`** -- *not* covered by #19792 and still quadratic on `main`. This PR complements it by making `union_schema` skip the merge when it can't change the result. It deliberately doesn't touch `with_new_children`; that path is already handled. ## What changes are included? A fast path at the top of `union_schema`: after taking `inputs[0].schema()`, if every remaining child's schema is either the **same allocation** (`Arc::ptr_eq`) or **structurally equal** (`==`) to the first, return the first schema immediately. Otherwise we fall through to the existing full merge, so behavior for genuinely heterogeneous unions is byte-for-byte unchanged. ```rust let first_schema = inputs[0].schema(); if inputs[1..].iter().all(|input| { let schema = input.schema(); Arc::ptr_eq(&schema, &first_schema) || schema == first_schema }) { return Ok(first_schema); } ``` `InterleaveExec` shares `union_schema`, so it gets the same speedup for free. ## On the cost of the deep compare... The natural objection (which came up before this PR): **doesn't the deep `==` make the *unequal* case slower?** I'll paraphrase the prior conclusions, risking verbosity to avoid rehashing the discussion. Spoiler: it's not an issue. - The equal case avoids the merge, and its check is cheap. The shared-`Arc` case is settled by pointer comparison. The distinct-but-equal case runs `Schema::eq`, which is allocation-free and short-circuits on the first difference. Benchmarks show a small loss versus a pointer-equality-only control (the theoretical floor) but it still beats the full merge by a wide margin, and that advantage grows with schema complexity. - The adversarial worst case is bounded. The one shape where the scan is pure overhead is `last_differs`: children `0..n-1` are equal and the last diverges, so we scan `n` schemas, fail on the last, then merge anyway. That's a single linear `==` pass bounded by the merge that follows -- a constant fraction, not another factor of `n` -- and it takes *thousands* of near-identical children differing only in the last to hit. - Ordinary unequal unions fail fast. `SELECT a ... UNION ALL SELECT b ...` differs at field 0, so `==` rejects on the first field (see `names_differ`). And `UnionExec::try_new` already rejects misaligned children, so the only divergence `union_schema` ever sees is top-level (caught in the first pass). ## Benchmark results New bench `datafusion/physical-plan/benches/union_schema.rs` measures `UnionExec::try_new` construction over a flat schema and a nested/struct schema, for the four child shapes above. Run interleaved (baseline / patched alternated per cell) on a fixed-clock T2D VM. ### `union_schema` construction (lower is better) | scenario | n | baseline (ms) | patched (ms) | change | |---|---|---|---|---| | union_exec_try_new/shared_arc | 100 | 0.263 | 0.042 | 6.2× | | union_exec_try_new/shared_arc | 1000 | 2.60 | 0.429 | 6.1× | | union_exec_try_new/shared_arc | 4000 | 10.5 | 1.74 | 6.0× | | union_exec_try_new/content_equal | 100 | 0.262 | 0.042 | 6.2× | | union_exec_try_new/content_equal | 1000 | 2.61 | 0.433 | 6.0× | | union_exec_try_new/content_equal | 4000 | 10.5 | 1.74 | 6.0× | | union_exec_try_new/last_differs | 4000 | ~98 | ~97 | flat (±1.5%) | | union_exec_try_new/names_differ | 4000 | ~87 | ~87 | flat (±1%) | | union_exec_try_new_nested/content_equal | 1000 | 2.68 | 0.431 | 6.2× | | union_exec_try_new_nested/content_equal | 4000 | 10.8 | 1.73 | 6.2× | The `last_differs` (adversarial: N-1 children equal, deep compare then full merge) and `names_differ` (typical unequal: fails on the first field) cells were re-measured with tight per-cell interleaving (baseline/patched adjacent, 4 rounds) to control for variance: both are within ±1.5%, straddling zero. Interpretation: the **deep compare cost isn't observable end to end**. ### End-to-end planning: no regression (`sql_planner`) `cargo bench --bench sql_planner` (TPC-H + ClickBench) run baseline vs patched on the fixed-clock T2D VM. Every case lands within ±1% -- run-to-run noise -- with no case regressing beyond that noise. Notable rows, including the union-heavy `sorted_union` cases the fast path is meant to help: | case | baseline (ms) | patched (ms) | |---|---|---| | physical_plan_tpcds_all | 995.9 ± 1.4 | 991.9 ± 1.9 | | physical_plan_tpch_all | 60.4 ± 0.2 | 60.3 ± 0.1 | | physical_sorted_union_order_by_50 | 349.9 ± 2.4 | 346.3 ± 2.7 | | physical_sorted_union_order_by_10 | 12.3 ± 0.03 | 12.2 ± 0.07 | | physical_select_all_from_1000 | 30.8 ± 0.25 | 30.7 ± 0.08 | The full TPC-H q1-q22 and ClickBench sets are all flat (ratio is 1.00-1.01 in both directions). Separately, interleaving tests per-cell (baseline and patched back-to-back, so run-to-run variance -- e.g. thermal -- cancels rather than favoring one) for `physical_join_distinct` + eight ClickBench queries (4 rounds) confirmed the same thing: patched and baseline straddle zero; no systematic regression from the deep compare. ## Testing - `cargo test -p datafusion-physical-plan --lib union` -- all pass, including a new `test_union_schema_fast_path_content_equal` that exercises the `==` branch with pointer-distinct-but-equal schemas and asserts the result matches the shared schema (i.e. identical to the slow-path merge). - `cargo clippy -p datafusion-physical-plan --lib -- -D warnings` -- clean. - `cargo bench --bench union_schema` -- compiles and runs. ## Are there any user-facing changes? No: planning-time performance change only, results and schema are identical. --------- Signed-off-by: Reid Kaufmann --- datafusion/physical-plan/Cargo.toml | 4 + .../physical-plan/benches/union_schema.rs | 159 ++++++++++++++++++ datafusion/physical-plan/src/union.rs | 46 +++++ 3 files changed, 209 insertions(+) create mode 100644 datafusion/physical-plan/benches/union_schema.rs diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index 0f72b74840d01..552b979c785a4 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -108,6 +108,10 @@ tokio = { workspace = true, features = [ harness = false name = "partial_ordering" +[[bench]] +harness = false +name = "union_schema" + [[bench]] harness = false name = "spill_io" diff --git a/datafusion/physical-plan/benches/union_schema.rs b/datafusion/physical-plan/benches/union_schema.rs new file mode 100644 index 0000000000000..8cffe474663f3 --- /dev/null +++ b/datafusion/physical-plan/benches/union_schema.rs @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Benchmark for `UnionExec` construction cost as a function of child count. +//! +//! Scenarios (run against a flat and a nested/struct schema): +//! - `shared_arc`: every child returns the same `Arc` +//! - `content_equal`: pointer-distinct but identical schemas per child +//! - `last_differs`: all children equal except extra metadata on the last +//! child's last top-level field. Worst case for the equality fast path: +//! the scan runs, fails on the final child, then the full merge runs anyway. +//! - `names_differ`: the first field is renamed in every child but the first, +//! the common real-world unequal union where equality fails immediately. + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use datafusion_physical_plan::ExecutionPlan; +use datafusion_physical_plan::empty::EmptyExec; +use datafusion_physical_plan::union::UnionExec; + +const NUM_FIELDS: usize = 10; +const NESTED_CHILDREN: usize = 5; +const METADATA_PER_FIELD: usize = 2; + +fn metadata(tag: &str, i: usize) -> HashMap { + (0..METADATA_PER_FIELD) + .map(|m| (format!("key_{m}"), format!("value_{tag}_{i}_{m}"))) + .collect() +} + +fn flat_schema() -> Schema { + let fields: Vec = (0..NUM_FIELDS) + .map(|i| { + Field::new(format!("col_{i}"), DataType::Int64, true) + .with_metadata(metadata("f", i)) + }) + .collect(); + Schema::new(fields) +} + +fn nested_schema() -> Schema { + let fields: Vec = (0..NUM_FIELDS) + .map(|i| { + let children: Vec = (0..NESTED_CHILDREN) + .map(|c| { + Field::new(format!("sub_{i}_{c}"), DataType::Int64, true) + .with_metadata(metadata("n", i * NESTED_CHILDREN + c)) + }) + .collect(); + Field::new(format!("col_{i}"), DataType::Struct(children.into()), true) + .with_metadata(metadata("s", i)) + }) + .collect(); + Schema::new(fields) +} + +/// Clone `schema` with extra metadata on its last field. Divergence must stay +/// at the top level: differing nested fields change the field `DataType` +/// itself, which `UnionExec::try_new` rejects ("Schemas have to be aligned"). +fn divergent(schema: &Schema) -> Schema { + let mut fields: Vec = + schema.fields().iter().map(|f| f.as_ref().clone()).collect(); + let last = fields.pop().unwrap(); + let mut md = last.metadata().clone(); + md.insert("divergent".to_string(), "true".to_string()); + fields.push(last.with_metadata(md)); + Schema::new(fields) +} + +fn child(schema: SchemaRef) -> Arc { + Arc::new(EmptyExec::new(schema)) +} + +fn children_shared_arc(schema: &Schema, n: usize) -> Vec> { + let schema: SchemaRef = Arc::new(schema.clone()); + (0..n).map(|_| child(Arc::clone(&schema))).collect() +} + +fn children_content_equal(schema: &Schema, n: usize) -> Vec> { + (0..n).map(|_| child(Arc::new(schema.clone()))).collect() +} + +/// First child keeps `schema`; the rest rename the first field — the common +/// real-world unequal union (`SELECT a .. UNION ALL SELECT b ..`), where any +/// equality check fails immediately. +fn children_names_differ(schema: &Schema, n: usize) -> Vec> { + let mut fields: Vec = + schema.fields().iter().map(|f| f.as_ref().clone()).collect(); + let first = fields.remove(0); + let renamed = first.clone().with_name(format!("renamed_{}", first.name())); + fields.insert(0, renamed); + let alt = Schema::new(fields); + let mut children = vec![child(Arc::new(schema.clone()))]; + children.extend((1..n).map(|_| child(Arc::new(alt.clone())))); + children +} + +fn children_last_differs(schema: &Schema, n: usize) -> Vec> { + let mut children = children_content_equal(schema, n - 1); + children.push(child(Arc::new(divergent(schema)))); + children +} + +fn bench_union_construction(c: &mut Criterion) { + for (suffix, schema, sizes) in [ + ("", flat_schema(), &[100usize, 1000, 4000][..]), + ("_nested", nested_schema(), &[1000, 4000][..]), + ] { + let mut group = c.benchmark_group(format!("union_exec_try_new{suffix}")); + for &n in sizes { + let shared = children_shared_arc(&schema, n); + let content = children_content_equal(&schema, n); + let differs = children_last_differs(&schema, n); + + group.bench_with_input( + BenchmarkId::new("shared_arc", n), + &shared, + |b, ch| b.iter(|| UnionExec::try_new(ch.clone()).unwrap()), + ); + group.bench_with_input( + BenchmarkId::new("content_equal", n), + &content, + |b, ch| b.iter(|| UnionExec::try_new(ch.clone()).unwrap()), + ); + group.bench_with_input( + BenchmarkId::new("last_differs", n), + &differs, + |b, ch| b.iter(|| UnionExec::try_new(ch.clone()).unwrap()), + ); + let names = children_names_differ(&schema, n); + group.bench_with_input( + BenchmarkId::new("names_differ", n), + &names, + |b, ch| b.iter(|| UnionExec::try_new(ch.clone()).unwrap()), + ); + } + group.finish(); + } +} + +criterion_group!(benches, bench_union_construction); +criterion_main!(benches); diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index c1cc5da31abaf..160772dc22314 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -916,6 +916,22 @@ fn union_schema(inputs: &[Arc]) -> Result { } let first_schema = inputs[0].schema(); + + // Fast path: when every input already shares the first input's schema, the + // field-by-field metadata/nullability merge below is redundant work that + // scales as O(n^2 * fields). This is common in practice: unions built from + // repartitioned copies of the same plan (e.g. observed in InfluxDB) hand us + // children that all carry the exact same schema. A pointer-equality check + // catches the shared-`Arc` case for free, and a content `==` comparison + // catches distinct-but-equal schemas; both let us return early and hand back + // the first schema unchanged. + if inputs[1..].iter().all(|input| { + let schema = input.schema(); + Arc::ptr_eq(&schema, &first_schema) || schema == first_schema + }) { + return Ok(first_schema); + } + let first_field_count = first_schema.fields().len(); // validate that all inputs have the same number of fields @@ -1627,6 +1643,36 @@ mod tests { Ok(()) } + #[test] + fn test_union_schema_fast_path_content_equal() -> Result<()> { + // Inputs whose schemas are pointer-distinct but structurally equal must + // take the content-equality (`==`) fast path and still produce a schema + // equal to the shared one, matching the slow-path merge exactly. + let schema = create_test_schema()?; + let distinct: SchemaRef = Arc::new((*schema).clone()); + // Guard the branch under test: these must NOT be the same allocation, so + // the fast path is reached via `==` rather than `Arc::ptr_eq`. + assert!(!Arc::ptr_eq(&schema, &distinct)); + + let memory_exec1 = + Arc::new(TestMemoryExec::try_new(&[], Arc::clone(&schema), None)?); + let memory_exec2 = + Arc::new(TestMemoryExec::try_new(&[], Arc::clone(&distinct), None)?); + let memory_exec3 = + Arc::new(TestMemoryExec::try_new(&[], Arc::clone(&distinct), None)?); + + // Capture the first child's schema before it is moved into the union. + let first_input_schema = memory_exec1.schema(); + let union_plan = + UnionExec::try_new(vec![memory_exec1, memory_exec2, memory_exec3])?; + + // The fast path returns the first child's schema Arc unchanged. Assert + // pointer equality (not just `==`): a slow-path merge would build a new, + // merely-equal Schema, so only ptr-eq proves the merge was skipped. + assert!(Arc::ptr_eq(&union_plan.schema(), &first_input_schema)); + Ok(()) + } + #[test] fn test_union_schema_mismatch() { // Test that UnionExec properly rejects inputs with different field counts From 095276ad2baa7ee5bc7cfca5f31b361943d09b3f Mon Sep 17 00:00:00 2001 From: Subham Singhal Date: Sat, 15 Aug 2026 14:15:03 +0000 Subject: [PATCH 39/50] bench: pwmj left semi/anti join (#24160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this close? Benchmark companion to #23870 (LeftSemi / LeftAnti support for `PiecewiseMergeJoinExec`), part of EPIC #17427. Closes no issue on its own. ## Rationale for this change #23870 routes existence subqueries with an inequality correlation (`WHERE EXISTS (SELECT 1 FROM rhs WHERE lhs.key < rhs.key)`) to `PiecewiseMergeJoinExec` instead of `NestedLoopJoinExec`. That claim needs a benchmark that can be run on either side of the change, so this PR adds the benchmark separately from the operator work — it is bench code only, no functional change. ## What changes are included in this PR? `datafusion/core/benches/pwmj_semi_anti_sql.rs` (+ its `Cargo.toml` entry), a Criterion benchmark with two arms over 20k × 20k `Int32` rows, `target_partitions=1`: - **`pwmj_enabled`** — flag on - **`nlj`** — flag off, which can only plan `NestedLoopJoinExec` Axes: join type (`EXISTS` → Semi, `NOT EXISTS` → Anti) × match regime (`all_match` 100%, `no_match` 0%, `half_match` ~50%, so the buffered side is only partially marked and scan depth varies per streamed row). ## Are these changes tested? ## Are there any user-facing changes? No. Benchmark code only. --- datafusion/core/Cargo.toml | 4 + datafusion/core/benches/pwmj_semi_anti_sql.rs | 289 ++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 datafusion/core/benches/pwmj_semi_anti_sql.rs diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 81e0418dd2b26..b51d981f6c950 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -228,6 +228,10 @@ name = "distinct_query_sql" harness = false name = "push_down_filter" +[[bench]] +harness = false +name = "pwmj_semi_anti_sql" + [[bench]] harness = false name = "sort_limit_query_sql" diff --git a/datafusion/core/benches/pwmj_semi_anti_sql.rs b/datafusion/core/benches/pwmj_semi_anti_sql.rs new file mode 100644 index 0000000000000..f84b8bfced210 --- /dev/null +++ b/datafusion/core/benches/pwmj_semi_anti_sql.rs @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Criterion benchmark for existence subqueries (`EXISTS` / `NOT EXISTS`) whose +//! correlation is a range predicate (`lhs.key < rhs.key`). The query is planned end to +//! end from SQL, with `datafusion.optimizer.enable_piecewise_merge_join` toggled to pick +//! the operator under test: +//! +//! - **`pwmj_enabled`**: on a build that routes existence joins to PWMJ, the subquery is +//! decorrelated to a `LeftSemi` / `LeftAnti` join and planned as `PiecewiseMergeJoin`, +//! together with the `SortExec` the planner inserts on the buffered side. The sort is +//! included because it is a real cost of that plan. +//! - **`nlj`**: the same join is planned as `NestedLoopJoinExec`, which is O(n*m). +//! +//! Both arms compute the same result, so the comparison measures the win from routing an +//! inequality-correlated `EXISTS` / `NOT EXISTS` to PWMJ instead of the nested-loop join. +//! +//! The arms are named after the config they set, not the operator they get, because the +//! enabled arm's operator is build-dependent: while the planner still excludes semi/anti +//! join types from PWMJ, setting the flag changes nothing and both arms plan +//! `NestedLoopJoinExec`. So the enabled arm accepts either operator instead of aborting the +//! run. Benchmark ids stay the same either way, which is what makes such a run useful as a +//! baseline: a later build that does route these joins to PWMJ compares straight against it. +//! +//! What every arm does pin down is *which* operator it planned, printed before the +//! timings, and it says so outright when both arms landed on the same one. Without that a +//! planning change would quietly compare `NestedLoopJoinExec` against itself and the +//! resulting ~1.0x would read as "PWMJ is no faster". +//! +//! ## Axes +//! - **join type**: `EXISTS` (LeftSemi) and `NOT EXISTS` (LeftAnti). +//! - **match regime**: the fraction of left rows that have at least one matching right +//! row, set by shifting the right-side key range relative to the left one: +//! `all_match` (100%), `no_match` (0%) and `half_match` (~50%, where the buffered side +//! ends up only partially marked). Semi output size grows with that fraction; Anti +//! output size shrinks. + +use std::sync::Arc; + +use arrow::array::{Int32Array, RecordBatch}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_main}; +use datafusion::datasource::MemTable; +use datafusion::physical_plan::{ExecutionPlan, collect, displayable}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use tokio::runtime::Runtime; + +const LEFT_ROWS: usize = 20_000; +const RIGHT_ROWS: usize = 20_000; +const KEY_SPAN: i32 = 10_000; + +/// Operators the `pwmj_enabled` arm is allowed to plan. Two of them, because the planner +/// only hands existence joins to PWMJ once PWMJ accepts semi/anti join types; before that +/// the flag is a no-op and the nested-loop join stays. +const PWMJ_OR_NLJ: &[&str] = &["PiecewiseMergeJoin", "NestedLoopJoinExec"]; + +/// With the flag off, nothing but the nested-loop join can plan this query. +const NLJ_ONLY: &[&str] = &["NestedLoopJoinExec"]; + +/// Two-column schema: (`key`, `payload`). +fn schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("key", DataType::Int32, false), + Field::new("payload", DataType::Int32, false), + ])) +} + +/// Build the batches for a single-partition table of `num_rows` rows. Keys are drawn from +/// `[key_offset, key_offset + KEY_SPAN)` in a fixed, reproducible pattern (no RNG so the +/// benchmark is deterministic). +fn build_batches( + num_rows: usize, + key_offset: i32, + schema: &SchemaRef, +) -> Vec { + let keys: Vec = (0..num_rows) + .map(|i| { + key_offset + + (i as i32) + .wrapping_mul(2_654_435_761u32 as i32) + .rem_euclid(KEY_SPAN) + }) + .collect(); + let payload: Vec = (0..num_rows as i32).collect(); + let batch = RecordBatch::try_new( + Arc::clone(schema), + vec![ + Arc::new(Int32Array::from(keys)), + Arc::new(Int32Array::from(payload)), + ], + ) + .unwrap(); + + // Slice into 8192-row batches to mirror a realistic scan. + let batch_size = 8192; + let mut batches = Vec::new(); + let mut offset = 0; + while offset < batch.num_rows() { + let len = (batch.num_rows() - offset).min(batch_size); + batches.push(batch.slice(offset, len)); + offset += len; + } + batches +} + +/// Register `lhs` and `rhs` in a context with PWMJ planning on or off. +fn create_context(right_offset: i32, pwmj: bool, schema: &SchemaRef) -> SessionContext { + let config = SessionConfig::new() + // Pinned so results are comparable across machines, and so the comparison + // isolates the join operator rather than how much repartitioning surrounds it. + .with_target_partitions(1) + .set_bool("datafusion.optimizer.enable_piecewise_merge_join", pwmj); + let ctx = SessionContext::new_with_config(config); + + for (name, key_offset, num_rows) in + [("lhs", 0, LEFT_ROWS), ("rhs", right_offset, RIGHT_ROWS)] + { + let table = MemTable::try_new( + Arc::clone(schema), + vec![build_batches(num_rows, key_offset, schema)], + ) + .unwrap(); + ctx.register_table(name, Arc::new(table)).unwrap(); + } + ctx +} + +/// `EXISTS` / `NOT EXISTS` over the range correlation `lhs.key < rhs.key`. +fn query(exists: bool) -> String { + let negation = if exists { "" } else { "NOT " }; + format!( + "SELECT lhs.key, lhs.payload FROM lhs \ + WHERE {negation}EXISTS (SELECT 1 FROM rhs WHERE lhs.key < rhs.key)" + ) +} + +fn physical_plan( + ctx: &SessionContext, + rt: &Runtime, + sql: &str, +) -> Arc { + rt.block_on(async { + ctx.sql(sql) + .await + .unwrap() + .create_physical_plan() + .await + .unwrap() + }) +} + +/// Check that the plan contains every fragment in `required` and exactly one of the +/// operators in `one_of`, returning the one it found. +/// +/// An arm named after a config flag says nothing about what ran, so each one reports the +/// operator it actually planned. `one_of` has two entries for the enabled arm because +/// whether existence joins reach PWMJ depends on the build; a single-entry list pins the +/// arm down completely. +fn assert_plan_operator<'a>( + plan: &Arc, + required: &[&str], + one_of: &[&'a str], + label: &str, +) -> &'a str { + let displayed = displayable(plan.as_ref()).indent(false).to_string(); + for fragment in required { + assert!( + displayed.contains(fragment), + "{label}: expected `{fragment}` in the physical plan, got:\n{displayed}" + ); + } + + let found: Vec<&str> = one_of + .iter() + .copied() + .filter(|operator| displayed.contains(operator)) + .collect(); + assert_eq!( + found.len(), + 1, + "{label}: expected exactly one of {one_of:?} in the physical plan, found \ + {found:?} in:\n{displayed}" + ); + found[0] +} + +fn run(plan: Arc, ctx: &SessionContext, rt: &Runtime) -> usize { + rt.block_on(async { + let batches = collect(plan, ctx.task_ctx()).await.unwrap(); + batches.iter().map(|b| b.num_rows()).sum() + }) +} + +fn bench_pwmj_semi_anti_sql(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let s = schema(); + + // An existence join needs only *one* match per left row, so all that matters is where + // `max(rhs.key)` falls inside the left range: a left row survives `EXISTS` iff + // `lhs.key < max(rhs.key)`. Shifting the right range up therefore saturates at + // all-match; only shifting it *down* leaves part of the left side unmatched. + let regimes: [(&str, i32); 3] = [ + // Right keys entirely above the left range: every left row matches. + ("all_match", KEY_SPAN), + // Right keys entirely below the left range: no left row matches, so the scan + // walks the whole buffered side without marking anything. + ("no_match", -KEY_SPAN), + // Ranges half-overlap: ~50% of left rows match, so only a suffix of the buffered + // side gets marked and the scan depth varies per streamed row. + ("half_match", -KEY_SPAN / 2), + ]; + + let mut group = c.benchmark_group("pwmj_vs_nlj_semi_anti_sql"); + // Nested-loop is O(n*m); keep sample counts modest so the suite finishes. + group.sample_size(10); + + for (regime, right_offset) in regimes { + // Only the Semi/Anti half of the join type is pinned, not the side: with PWMJ + // disabled the planner swaps the nested-loop inputs, so the same `EXISTS` plans as + // `RightSemi` there and `LeftSemi` under PWMJ. + for (exists, join_type) in [(true, "Semi"), (false, "Anti")] { + let sql = query(exists); + let label = if exists { "semi" } else { "anti" }; + + // Plan and check both arms before timing either, so the operators they picked + // are on screen ahead of the numbers those operators qualify. + let arms: Vec<(String, SessionContext, &str)> = [ + ("pwmj_enabled", true, PWMJ_OR_NLJ), + ("nlj", false, NLJ_ONLY), + ] + .into_iter() + .map(|(arm, pwmj, operators)| { + let ctx = create_context(right_offset, pwmj, &s); + let name = format!("{arm}_{label}_{regime}"); + let planned = assert_plan_operator( + &physical_plan(&ctx, &rt, &sql), + &[join_type], + operators, + &name, + ); + println!("{name}: planned {planned}"); + (name, ctx, planned) + }) + .collect(); + + if arms.iter().all(|(_, _, planned)| *planned == arms[0].2) { + println!( + "note: {label}_{regime}: both arms planned {}, so the ratio between \ + them measures nothing about PWMJ", + arms[0].2 + ); + } + + for (name, ctx, _) in &arms { + // Plan afresh in the untimed setup rather than reusing one plan: the + // buffered side of `PiecewiseMergeJoinExec` (its visited-indices bitmap + // and final-pass partition counter) is cached in a `OnceAsync` on the + // exec, so a second `collect` over the same instance would not repeat + // the work. + group.bench_function(BenchmarkId::new(name.as_str(), RIGHT_ROWS), |b| { + b.iter_batched( + || physical_plan(ctx, &rt, &sql), + |plan| run(plan, ctx, &rt), + BatchSize::SmallInput, + ) + }); + } + } + } + + group.finish(); +} + +criterion_group!(benches, bench_pwmj_semi_anti_sql); +criterion_main!(benches); From 8d54633ed74d88e6eefaa8fc643d637490868343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20=C5=9Een?= Date: Sat, 15 Aug 2026 14:18:41 +0000 Subject: [PATCH 40/50] fix: drain PiecewiseMergeJoin output before state transitions (#24298) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #24297 ## Rationale for this change `ClassicPWMJStream` transitions phases while its output `BatchCoalescer` still holds completed batches. The Left/Full unmatched pass falls back into its producer when the drained queue is an exact multiple of `batch_size` — emitting duplicate rows forever — and queued unmatched Right/Full batches are dropped at terminal transitions. A zero-row placeholder batch also escapes as stream output on empty results. ## What changes are included in this PR? - Add `BatchProcessState::next_drained_batch()` (`finish_buffered_batch` + `next_completed_batch`); `None` guarantees the coalescer is empty - `process_stream_batch` and `process_unmatched_buffered_batch` drain one batch per poll and transition only after a confirmed-empty pop, so a finished producer can never re-run and no queued batch is lost - Scan completion returns `StatefulStreamResult::Continue` instead of the empty placeholder batch; empty joins now yield no batches ## Are these changes tested? Yes. The existing unmatched Left/Right tests now run with `batch_size = 1` (the Left one bounded with `take(6)` so the old duplicate loop fails as a snapshot mismatch instead of hanging) — both fail on main and pass here. The empty-join tests assert no batches are emitted. ## Are there any user-facing changes? No (PWMJ is experimental and off by default). --- .../piecewise_merge_join/classic_join.rs | 123 +++++++----------- 1 file changed, 49 insertions(+), 74 deletions(-) diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs index 1c89927087fdb..bc69ae5140831 100644 --- a/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs +++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs @@ -277,6 +277,18 @@ impl ClassicPWMJStream { return Ok(StatefulStreamResult::Ready(Some(batch))); } + // A finished scan can leave several completed batches queued; emit + // them one per poll and transition only once the queue is empty, so + // no output is lost. + if !self.batch_process_state.continue_process { + if let Some(batch) = self.batch_process_state.next_drained_batch()? { + return Ok(StatefulStreamResult::Ready(Some(batch))); + } + + self.state = PiecewiseMergeJoinStreamState::FetchStreamBatch; + return Ok(StatefulStreamResult::Continue); + } + // Produce more work let batch = resolve_classic_join( buffered_side, @@ -289,25 +301,8 @@ impl ClassicPWMJStream { )?; if !self.batch_process_state.continue_process { - // We finished scanning this stream batch. - self.batch_process_state - .output_batches - .finish_buffered_batch()?; - if let Some(b) = self - .batch_process_state - .output_batches - .next_completed_batch() - { - self.state = PiecewiseMergeJoinStreamState::FetchStreamBatch; - return Ok(StatefulStreamResult::Ready(Some(b))); - } - - // Nothing pending; hand back whatever `resolve` returned (often empty) and move on. - if self.batch_process_state.output_batches.is_empty() { - self.state = PiecewiseMergeJoinStreamState::FetchStreamBatch; - - return Ok(StatefulStreamResult::Ready(Some(batch))); - } + // Scan finished; re-enter through the drain guard above. + return Ok(StatefulStreamResult::Continue); } Ok(StatefulStreamResult::Ready(Some(batch))) @@ -324,25 +319,13 @@ impl ClassicPWMJStream { } if !self.batch_process_state.continue_process { - if let Some(batch) = self - .batch_process_state - .output_batches - .next_completed_batch() - { + if let Some(batch) = self.batch_process_state.next_drained_batch()? { return Ok(StatefulStreamResult::Ready(Some(batch))); } - self.batch_process_state - .output_batches - .finish_buffered_batch()?; - if let Some(batch) = self - .batch_process_state - .output_batches - .next_completed_batch() - { - self.state = PiecewiseMergeJoinStreamState::Completed; - return Ok(StatefulStreamResult::Ready(Some(batch))); - } + // Fully drained; finish instead of re-running the pass. + self.state = PiecewiseMergeJoinStreamState::Completed; + return Ok(StatefulStreamResult::Continue); } let buffered_data = @@ -372,29 +355,8 @@ impl ClassicPWMJStream { self.batch_process_state.output_batches.push_batch(batch)?; self.batch_process_state.continue_process = false; - if let Some(batch) = self - .batch_process_state - .output_batches - .next_completed_batch() - { - return Ok(StatefulStreamResult::Ready(Some(batch))); - } - - self.batch_process_state - .output_batches - .finish_buffered_batch()?; - if let Some(batch) = self - .batch_process_state - .output_batches - .next_completed_batch() - { - self.state = PiecewiseMergeJoinStreamState::Completed; - return Ok(StatefulStreamResult::Ready(Some(batch))); - } - - self.state = PiecewiseMergeJoinStreamState::Completed; - self.batch_process_state.reset(); - Ok(StatefulStreamResult::Ready(None)) + // Re-enter through the drain guard above. + Ok(StatefulStreamResult::Continue) } } @@ -438,6 +400,13 @@ impl BatchProcessState { self.continue_process = true; self.processed_null_count = false; } + + // `None` guarantees the coalescer holds no pending rows, so the caller + // may safely transition state without losing output. + fn next_drained_batch(&mut self) -> Result> { + self.output_batches.finish_buffered_batch()?; + Ok(self.output_batches.next_completed_batch()) + } } impl Stream for ClassicPWMJStream { @@ -674,7 +643,9 @@ mod tests { use arrow_schema::{DataType, Field}; use datafusion_common::test_util::batches_to_string; use datafusion_execution::TaskContext; + use datafusion_execution::config::SessionConfig; use datafusion_physical_expr::{PhysicalExpr, expressions::Column}; + use futures::TryStreamExt; use insta::assert_snapshot; use std::sync::Arc; @@ -964,12 +935,8 @@ mod tests { ); let (_, batches) = join_collect(left, right, on, Operator::LtEq, JoinType::Inner).await?; - assert_snapshot!(batches_to_string(&batches), @r" - +----+----+----+----+----+----+ - | a1 | b1 | c1 | a2 | b1 | c2 | - +----+----+----+----+----+----+ - +----+----+----+----+----+----+ - "); + // An empty join result produces no batches at all, not an empty batch. + assert!(batches.is_empty()); Ok(()) } @@ -1298,8 +1265,16 @@ mod tests { Arc::new(Column::new_with_schema("b1", &right.schema())?) as _, ); - let (_, batches) = - join_collect(left, right, on, Operator::LtEq, JoinType::Left).await?; + let task_ctx = Arc::new( + TaskContext::default() + .with_session_config(SessionConfig::new().with_batch_size(1)), + ); + // Bound collection so the old duplicate loop becomes a snapshot mismatch. + let batches = join(left, right, on, Operator::LtEq, JoinType::Left)? + .execute(0, task_ctx)? + .take(6) + .try_collect::>() + .await?; assert_snapshot!(batches_to_string(&batches), @r" +----+----+----+----+----+----+ @@ -1344,8 +1319,12 @@ mod tests { Arc::new(Column::new_with_schema("b1", &right.schema())?) as _, ); - let (_, batches) = - join_collect(left, right, on, Operator::GtEq, JoinType::Right).await?; + let task_ctx = Arc::new( + TaskContext::default() + .with_session_config(SessionConfig::new().with_batch_size(1)), + ); + let join = join(left, right, on, Operator::GtEq, JoinType::Right)?; + let batches = common::collect(join.execute(0, task_ctx)?).await?; assert_snapshot!(batches_to_string(&batches), @r" +----+----+----+----+----+----+ @@ -1408,12 +1387,8 @@ mod tests { let (_, batches) = join_collect(left, right, on, Operator::Gt, JoinType::Inner).await?; - assert_snapshot!(batches_to_string(&batches), @r" - +----+----+----+----+----+----+ - | a1 | b1 | c1 | a2 | b1 | c2 | - +----+----+----+----+----+----+ - +----+----+----+----+----+----+ - "); + // An empty join result produces no batches at all, not an empty batch. + assert!(batches.is_empty()); Ok(()) } From 35c9214b196f115a5135718b66a0825fd3b0924a Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Sat, 15 Aug 2026 16:10:37 +0000 Subject: [PATCH 41/50] fix: preserve NULL semantics when simplifying col ~ '.*' (#24380) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #24379. ## Rationale for this change `simplify_regex_expr` rewrites `col ~ '.*'` to `col IS NOT NULL`. For a NULL input that returns `false`, but `NULL ~ '.*'` is `NULL` under three-valued logic — so the rewrite produces wrong results in a projection context: ```sql SELECT s, s ~ '.*' FROM (VALUES (CAST(NULL AS VARCHAR)), ('x')) t(s); -- NULL row currently returns `false`; it should be NULL ``` The `!~` (`RegexNotMatch`) branch of the same rule is already NULL-aware (`col IS NULL AND NULL`); only the `~` branch dropped the NULL. ## What changes are included in this PR? - Rewrite `col ~ '.*'` to `col IS NOT NULL OR NULL` — `true` for a non-NULL string, `NULL` for a NULL input. - In a WHERE filter both FALSE and NULL reject the row, so filter *results* are unchanged; only the plan text and projection-context values differ. Existing filter-plan expectations in `simplify_expr.slt` and the `test_simplify_regex_special_cases` unit test are updated accordingly, and a projection regression test is added. ## Are these changes tested? Yes. - New projection regression test in `simplify_expr.slt` asserting `col ~ '.*'` returns `true`/`true`/`NULL` for `'foo'`/`''`/`NULL`. - Updated the two filter-context plan expectations (logical + physical) that previously encoded the `IS NOT NULL` rewrite. - `simplify_expr.slt`, the `regexp/*` SLTs, and the optimizer simplify unit tests all pass. ## Are there any user-facing changes? `col ~ '.*'` in a projection now returns `NULL` for a NULL input instead of `false`, matching SQL semantics. No API changes. --- .../src/simplify_expressions/regex.rs | 9 +++++--- .../simplify_expressions/simplify_exprs.rs | 4 ++-- .../sqllogictest/test_files/simplify_expr.slt | 22 +++++++++++++++++-- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/datafusion/optimizer/src/simplify_expressions/regex.rs b/datafusion/optimizer/src/simplify_expressions/regex.rs index f04d9476c42fe..7dccb5b1b952d 100644 --- a/datafusion/optimizer/src/simplify_expressions/regex.rs +++ b/datafusion/optimizer/src/simplify_expressions/regex.rs @@ -38,7 +38,7 @@ const ANY_CHAR_REGEX_PATTERN: &str = ".*"; /// - full anchored regex patterns (e.g. `^foo$`) to `= 'foo'` /// - partial anchored regex patterns (e.g. `^foo`) to `LIKE 'foo%'` /// - combinations (alternatives) of the above, will be concatenated with `OR` or `AND` -/// - `EQ .*` to NotNull +/// - `EQ .*` to `col IS NOT NULL OR NULL` (true for any string, NULL if col is NULL) /// - `NE .*` to col IS NULL AND Boolean(NULL) (false for any string, or NULL if col is NULL) /// /// Dev note: unit tests of this function are in `expr_simplifier.rs`, case `test_simplify_regex`. @@ -75,8 +75,11 @@ pub fn simplify_regex_expr( right: Box::new(null_bool), }) } else { - // not null - left.is_not_null() + // `col ~ '.*'` matches every non-NULL string and yields NULL for a + // NULL input (three-valued logic). Mirror the `not` branch so the + // NULL row is preserved (`NULL`) instead of collapsing to `false`. + let null_bool = lit(ScalarValue::Boolean(None)); + left.is_not_null().or(null_bool) }; return Ok(Transformed::yes(new_expr)); } diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs index 0e72a17abc9f7..1c5a4a1869ddb 100644 --- a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs +++ b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs @@ -1026,7 +1026,7 @@ mod tests { ]); let table_scan = table_scan(Some("test"), &schema, None)?.build()?; - // Test `~ ".*"` transforms to true for any non-NULL string + // Test `~ ".*"` is TRUE for any non-NULL string and NULL for a NULL input let plan = LogicalPlanBuilder::from(table_scan.clone()) .filter(binary_expr(col("a"), Operator::RegexMatch, lit(".*")))? .build()?; @@ -1034,7 +1034,7 @@ mod tests { assert_optimized_plan_equal!( plan, @ r" - Filter: test.a IS NOT NULL + Filter: test.a IS NOT NULL OR Boolean(NULL) TableScan: test " )?; diff --git a/datafusion/sqllogictest/test_files/simplify_expr.slt b/datafusion/sqllogictest/test_files/simplify_expr.slt index 57dc440407dc0..158096328e960 100644 --- a/datafusion/sqllogictest/test_files/simplify_expr.slt +++ b/datafusion/sqllogictest/test_files/simplify_expr.slt @@ -34,10 +34,10 @@ query TT explain select b from t where b ~ '.*' ---- logical_plan -01)Filter: t.b IS NOT NULL +01)Filter: t.b IS NOT NULL OR Boolean(NULL) 02)--TableScan: t projection=[b] physical_plan -01)FilterExec: b@0 IS NOT NULL +01)FilterExec: b@0 IS NOT NULL OR NULL 02)--DataSourceExec: partitions=1, partition_sizes=[1] query TT @@ -50,6 +50,24 @@ physical_plan 01)FilterExec: b@0 IS NULL AND NULL 02)--DataSourceExec: partitions=1, partition_sizes=[1] +# `col ~ '.*'` is TRUE for any non-NULL string and NULL for a NULL input. +# The `.*` -> IS NOT NULL simplification must preserve that NULL (returning +# `false` for the NULL row would be wrong in a projection context). +query TB +WITH vals(id, col) AS ( + VALUES + (1, 'foo'::text), + (2, ''::text), + (3, NULL::text) +) +SELECT col, col ~ '.*' +FROM vals +ORDER BY id +---- +foo true +(empty) true +NULL NULL + query TB WITH vals(id, col) AS ( VALUES From b24581315b2d1fd1e03a9e516903368dc1088f5c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 15 Aug 2026 18:02:38 +0000 Subject: [PATCH 42/50] docs: add an agent skill for auditing datafusion-spark expressions (#23893) ## Which issue does this PR close? - Closes #. No issue tracks this. It is tooling, proposed for discussion. Related: https://github.com/apache/datafusion/issues/15914 (the `datafusion-spark` epic) and https://github.com/apache/datafusion/issues/23887 (version-specific SLT expectations, which this skill's cross-version findings link to). ## Rationale for this change The `datafusion-spark` crate reimplements Spark's built-in expressions. Correctness against Spark is the entire point of the crate, but there is no routine process for checking an implementation against the Spark source, and no record of which functions have been checked. The testing situation makes this harder than it looks. Comet, the sibling project, tests by differential execution: it runs each query in both engines and compares, so expected values never need to be written down. DataFusion's `.slt` files hard-code every expected result, with no Spark in the loop. Two consequences follow: 1. Someone has to determine what Spark actually returns before writing the expectation. Reading `nullSafeEval` and predicting the output is how wrong golden values get written, and once written they look like verified behavior. 2. A hard-coded expectation encodes exactly one Spark version, so every file has to be written with one version in mind. Many files under `test_files/spark/` were machine-generated from Sail's gold data and their values were never checked against a running Spark. An existing passing expectation is a claim, not evidence. ## What changes are included in this PR? One file: `.ai/skills/audit-datafusion-spark-expression/SKILL.md`, following the `.ai/skills/` convention already described in the repository `CLAUDE.md`. No production code changes. The skill audits one function per invocation, in eight steps: read the Spark source across 3.5.8, 4.0.4, 4.1.3 and 4.2.0 to detect cross-version divergence; harvest Spark's own unit tests and `sql-tests/results/*.sql.out` golden files; review the Rust implementation's signature and coercion, return nullability, null propagation, overflow, type dispatch and ANSI handling; cross-check Comet and Sail; review existing SLT coverage; build a gap matrix; establish ground truth; then apply findings. Three decisions are worth calling out for review: **Spark 4.2.0 is the declared baseline.** Older versions are read only to detect that behavior changed, never to set an expectation. **Expected values come from a real Spark.** The skill sets up a `pyspark==4.2.0` venv and runs every candidate query through it under both ANSI modes, including expectations that already exist in the file. If PySpark cannot be installed the audit continues from source-derived values, but every such expectation is marked unverified rather than presented as observed. **`--complete` is forbidden for files under `test_files/spark/`.** That mode fills expectations with whatever DataFusion currently returns, which on an audit would silently cement the bug being audited as the golden answer. The skill also establishes a convention for divergences it cannot fix: a commented-out SLT query carrying the correct Spark result and a link to the tracking issue, where uncommenting it is the contract for verifying a fix. That convention is new to this tree and is the piece most worth arguing about. ## Are these changes tested? The skill was run end to end against `next_day`, which is how it was debugged. That run produced https://github.com/apache/datafusion/pull/23892 and issues #23889, #23890 and #23891. Running it surfaced eleven defects in the skill itself, which are fixed here. Two are worth citing as evidence that the exercise was not circular: - The instruction to `unset SPARK_HOME` sat in a different code block from the command that needed it, so it never took effect and the run reproduced the exact `TypeError: 'JavaPackage' object is not callable` failure the text was written to prevent. - The ground-truth harness reported a Python `ValueError` as though it were a Spark error, which would have written a fabricated divergence into the `.slt` as observed fact. Review of the resulting `next_day` work then found a genuine SQL-reachable panic in the audited function that the audit had missed, and the gap that hid it, which is now a rule in the skill: cross the argument shapes with the NULL cases and with both ANSI modes rather than checking each axis alone. The skill carries no automated test. Its verification is that following it produces a correct audit, which is what #23892 demonstrates. It has since been run a second time, against `pmod`, which found three real bugs now fixed in https://github.com/apache/datafusion/pull/23898 and four more divergences filed as #23894, #23895, #23896 and #23897. That run surfaced four further skill gaps, folded in here. The most valuable is a section on Arrow kernel semantics not matching Java's: total-ordered float comparison hiding `-0.0`, checked rather than wrapping arithmetic, and Java's `byte`/`short` promotion to `int` each caused one of the `pmod` bugs, and the skill had said nothing about any of them. The `datafusion-spark` README already points contributors to Comet and Sail when implementing a function. This is the same idea applied to checking one. --- .../SKILL.md | 996 ++++++++++++++++++ 1 file changed, 996 insertions(+) create mode 100644 .ai/skills/audit-datafusion-spark-expression/SKILL.md diff --git a/.ai/skills/audit-datafusion-spark-expression/SKILL.md b/.ai/skills/audit-datafusion-spark-expression/SKILL.md new file mode 100644 index 0000000000000..258b4032361a5 --- /dev/null +++ b/.ai/skills/audit-datafusion-spark-expression/SKILL.md @@ -0,0 +1,996 @@ +--- +name: audit-datafusion-spark-expression +description: Audit a datafusion-spark function implementation for correctness against Apache Spark 4.2.0. Studies the Spark source across versions, reviews the Rust implementation and its signature, verifies expected values against a real PySpark, fixes divergences, and captures the rest as issues and disabled tests. +argument-hint: +--- + + + +Audit the `datafusion-spark` implementation of the `$ARGUMENTS` function for +correctness against Apache Spark. + +## Baseline Version + +The baseline is **Spark 4.2.0**. Every expected value written into a `.slt` +file asserts Spark 4.2.0 behavior. + +Spark 3.5.8, 4.0.4, and 4.1.3 are also read, but only to detect that behavior +changed between versions. They never set an expectation. There is currently no +mechanism for version-specific expectations in DataFusion's SLT files, which is +tracked by https://github.com/apache/datafusion/issues/23887. + +Older versions are read rather than ignored because `datafusion-spark` is meant +to be reusable by engines that target more than one Spark release. Comet +supports Spark 3.4 through 4.0. A function audited against 4.2.0 alone can be +correct there and silently wrong for such a consumer, and nothing in the crate +records which of the two it is. Finding the divergence is useful even though +the test suite cannot yet assert it. + +## Overview + +1. Study the Spark implementation across versions +2. Harvest Spark's own test coverage +3. Review the DataFusion implementation +4. Cross-check Comet and Sail +5. Review existing SLT coverage +6. Gap analysis +7. Establish ground truth with PySpark +8. Apply findings + +Audit one function per invocation. + +## Paths in This Skill Are Hints + +Every file path below was correct when this skill was written and several will +rot. `FunctionRegistry.scala`, the golden-file directories, and +`datafusion/common/src/config.rs` all move on someone else's schedule. + +**A path that does not resolve is a stop condition, not a step to skip.** The +failure mode this exists to prevent is a command that quietly matches nothing, +an audit that reads the silence as "no ANSI branch" or "Spark does not test +this", and a confident report built on an empty grep. A `grep` over a +nonexistent directory and a `grep` that genuinely found nothing are the same +output. + +So when a path in this skill does not exist, or a command returns nothing where +something was expected: + +1. Say so out loud in the audit report. Name the path and the step. +2. Search for where the thing moved to, by content rather than by path. The + grep patterns here are more durable than the directories they are pointed at. +3. If it cannot be found, mark every conclusion that depended on that step as + unestablished. Do not let an unread source become an absence of findings. + +The same rule covers a `.slt` file, an implementation file, or a Spark class +that is not where this skill says it is. + +## Step 1: Study the Spark Implementation + +Shallow-clone the four version tags. Skip any already present. + +```bash +set -eu +for tag in v3.5.8 v4.0.4 v4.1.3 v4.2.0; do + dir="/tmp/spark-${tag}" + [ -d "$dir" ] || git clone --depth 1 --branch "$tag" \ + https://github.com/apache/spark.git "$dir" +done +``` + +Find the expression class in each version. Spark registers SQL function names +in `FunctionRegistry.scala`, so start there to map `$ARGUMENTS` to its Scala +class name, then locate the class. + +```bash +for tag in v3.5.8 v4.0.4 v4.1.3 v4.2.0; do + dir="/tmp/spark-${tag}" + echo "=== $tag ===" + grep -rn "\"$ARGUMENTS\"" \ + "$dir/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala" +done +``` + +With the class name in hand, read its source in each version: + +```bash +for tag in v3.5.8 v4.0.4 v4.1.3 v4.2.0; do + dir="/tmp/spark-${tag}" + echo "=== $tag ===" + find "$dir/sql/catalyst/src/main/scala" -name "*.scala" \ + | xargs grep -lE "case class [ (\[]" 2>/dev/null +done +``` + +A Scala `case class` name is always followed by a space, a `(`, or a `[`, so +that bracket expression is the portable way to avoid matching a longer class +name that starts with the same prefix. Do not reach for `\b`, which is a GNU +and ugrep extension and is not available in POSIX or BSD `grep`. + +If it is not in catalyst, widen the search to `$dir/sql`. + +For each version, record: + +- `eval` / `nullSafeEval` / `doGenCode` logic +- `inputTypes` and `dataType` +- `nullable`, and how nulls propagate +- ANSI branches (`ansiEnabled`, `failOnError`) +- guards, `require` assertions, thrown exceptions +- any SQLConf the expression reads +- for string functions in 4.0+, whether it routes through + `CollationSupport.X.exec(..., collationId)` or declares + `StringTypeWithCollation` inputs + +Produce a diff summary across 3.5.8 → 4.0.4 → 4.1.3 → 4.2.0. Note new or +removed input types, changed edge-case behavior, new ANSI branches, new +parameters, and collation changes. + +Two changes affect nearly every function and are easy to miss: + +- ANSI mode became the **default** in Spark 4.0. A function whose behavior + differs under ANSI has a different default behavior in 3.5 than in 4.x. +- Collation support landed for many string functions in 4.0. Non-default + collations are a divergence axis that does not exist in 3.5. + +## Step 2: Harvest Spark's Own Test Coverage + +Spark's tests are a ready-made edge-case inventory. Two sources matter. + +Unit tests: + +```bash +dir=/tmp/spark-v4.2.0 +find "$dir/sql" -name "*.scala" -path "*/test/*" \ + | xargs grep -ln "$ARGUMENTS" 2>/dev/null +``` + +Golden files, which carry Spark's own verified query results and are the +highest-value source: + +```bash +for tag in v3.5.8 v4.0.4 v4.1.3 v4.2.0; do + dir="/tmp/spark-${tag}" + echo "=== $tag ===" + grep -rln "$ARGUMENTS" "$dir/sql/core/src/test/resources/sql-tests/results/" 2>/dev/null +done +``` + +**The ANSI and non-ANSI golden directories swapped meaning in Spark 4.0.** Read +the version banner before trusting a path: + +| Version | ANSI-mode results | Non-ANSI results | +| ------- | ------------------------ | --------------------------- | +| 3.5.8 | `results/ansi/X.sql.out` | `results/X.sql.out` | +| 4.x | `results/X.sql.out` | `results/nonansi/X.sql.out` | + +ANSI became the default in 4.0, so the unqualified path flipped from meaning +non-ANSI to meaning ANSI. An auditor who learns the layout on 3.5.8 and then +reads `results/X.sql.out` on 4.2.0 will record exactly inverted expectations, +and they will look plausible. Confirm the mode by checking whether the file +contains error output or `NULL` for a known-invalid input. + +Read every match and extract: + +- input types covered +- edge cases exercised: null, empty, overflow, negative, boundary, special + characters +- ANSI-mode cases +- error cases and their exact messages + +This list is the reference for the gap matrix in Step 6. + +## Step 3: Review the DataFusion Implementation + +```bash +grep -rln "$ARGUMENTS" datafusion/spark/src/function/ --include="*.rs" +``` + +The implementation lives at +`datafusion/spark/src/function//.rs`. **Read every other +file that grep returns as well.** In particular +`datafusion/spark/src/function//mod.rs` carries the `export_functions!` +registration and the user-facing doc string, and unimplemented-behavior `TODO` +comments are written there too, not only in the function's own file. An audit +that reads only `.rs` will miss them and will leave a doc string that +contradicts the code after a fix. + +**A `.slt` file under `test_files/spark/` does not imply an implementation in +`datafusion/spark`.** `SessionStateBuilderSpark::with_spark_features` layers the +Spark functions on top of `with_default_features`, so a name the spark crate +does not define resolves to DataFusion's own function of that name. `md5`, +`coalesce`, and `reverse` are served this way today, and `date_format` resolves +through an alias on core's `to_char`. The query under audit still runs, and the +code it runs is not in the directory the grep above searched. If that grep comes +back empty, widen it before concluding the function is unimplemented: + +```bash +grep -rln "\"$ARGUMENTS\"" \ + datafusion/functions/src datafusion/functions-nested/src \ + datafusion/functions-aggregate/src datafusion/functions-window/src \ + datafusion/functions-table/src --include="*.rs" +``` + +A hit may be in a file named after something else, because core registers +aliases. `date_format` lives in `to_char.rs`. Read the `aliases` vector, not +just the file name. + +If that grep is also empty, the function is genuinely unimplemented and the +`.slt` file is a porting stub with every query commented out. Check with +`grep -cE '^(query|statement)' .slt`, which returns zero for a stub. +There is no implementation to audit in that case. Say so and stop, rather than +reporting a clean audit of nothing. + +Otherwise audit what the grep found against Spark with the rest of this step's +checklist. The findings land differently. A core function has non-Spark users, +so Spark's semantics cannot be imposed on it. Where the two agree, record the +coverage and leave the core function alone. Where they diverge, the fix is a +Spark-specific implementation under `datafusion/spark/src/function//` +that shadows the core one, never an edit to the core function. +`datafusion/spark/src/function/math/abs.rs` shadowing +`datafusion/functions/src/math/abs.rs` is the in-repo precedent. Say in the +audit report which crate the audited code came from, because a reader who +assumes `datafusion/spark` will look for a file that does not exist. + +Check each of the following. The first two have no Comet analog and are the +most common source of silent divergence. + +**Signature and coercion.** Compare `Signature` (and `coerce_types`, if +implemented) against Spark's `inputTypes`. Two failure directions, both real: + +- DataFusion rejects a type Spark accepts. The user gets a coercion error + where Spark returns a value. +- DataFusion accepts a type Spark rejects. The user gets a value where Spark + raises an analysis error. + +Note that `Signature::exact` is narrow. A Spark function accepting any string +type needs all of `Utf8`, `LargeUtf8`, and `Utf8View` handled, and a function +accepting any integral type needs the full width range. + +**Return nullability.** Compare `return_field_from_args` (or `return_type`) +against Spark's `nullable`. A function that reports non-nullable while Spark +can return NULL produces wrong plans, not merely wrong values: downstream +optimizer rules will fold away null checks that were load-bearing. + +**Null propagation.** Does every input-null combination return what Spark +returns? Check the order in which nullness and validity are tested. Spark +expressions that are null intolerant (a `NullIntolerant` mixin in 3.5, a +`nullIntolerant` override in 4.x) short circuit to NULL *before* validating the +other arguments, so a NULL in one argument suppresses an error the other +argument would otherwise raise, even under ANSI mode. An implementation that +validates first and checks nullness second raises where Spark returns NULL. + +**Arrow kernel semantics are not Java semantics.** A `datafusion-spark` +function is usually a thin arrangement of Arrow kernels, and the kernels do not +share Java's rules. Check each one the implementation leans on: + +- **Comparison kernels order floats by total order**, so `-0.0` compares as + *less than* `0.0` rather than equal to it. Any `eq`/`lt` used to detect a + special value, such as a zero divisor, silently misses `-0.0`. Java compares + them as equal. This is easy to miss because `-0.0` almost never appears in + hand-written test values, so write it in deliberately. +- **Arithmetic kernels are checked, not wrapping.** Java wraps silently on + overflow, so a checked Arrow kernel raises where Spark returns a wrapped + value. Where Spark's own expression is written in `Int` arithmetic, a + wrapping kernel is the faithful choice, not the reckless one. +- **Java promotes `byte` and `short` operands to `int`.** A Spark expression + whose Scala source reads `(r + n)` on `Byte` operands is evaluated at `Int` + width and cannot overflow, while the same source on `Int` operands can. + Reproducing it at `Int8` width wraps where Spark does not. +- **Kernel coverage is type dependent.** Arrow's `rem` raises on a zero divisor + for integers and decimals but returns `NaN` for floats, so an ANSI check that + relies on the kernel to raise silently does nothing on the float path. + +Read the Scala source as arithmetic to be reproduced, not as pseudocode to be +paraphrased. Then test each of these against a live Spark rather than reasoning +about them. + +**`ColumnarValue` shape dispatch.** A two-argument function has four shape +combinations, and each needs its own branch: +`(Scalar, Scalar)`, `(Array, Scalar)`, `(Scalar, Array)`, `(Array, Array)`. +A missing arm falls through to a catch-all `exec_err!` and turns a working +Spark query into a hard error. `(Scalar, Array)` is the one most often +forgotten, because the natural test query puts the column first. Write a query +for every combination and run it. Do not infer coverage from reading the +`match`. + +**How to run an ad-hoc query.** The only in-repo way to execute a +`datafusion-spark` function is to add the query to +`datafusion/sqllogictest/test_files/spark//$ARGUMENTS.slt` and run + +```bash +cargo test --test sqllogictests -- spark//$ARGUMENTS +``` + +`datafusion-cli` does **not** register the `datafusion-spark` function library, +so reaching for it fails in a way that looks like the function does not exist: + +``` +$ ./target/debug/datafusion-cli -c "SELECT next_day(arrow_cast(95026236,'Date32'),'Mon');" +Error during planning: Invalid function 'next_day'. Did you mean 'today'? +``` + +Giving the CLI an opt-in for the Spark library is tracked by +https://github.com/apache/datafusion/issues/24146. Until that lands, treat the +error above as a registration gap and not as evidence about the function. + +For a throwaway probe that should not end up in the committed file, add a +temporary `.slt` alongside it, run it by name, and delete it afterwards. To +exercise a raw storage value that has no literal syntax, such as a `Date32` +past the last date DataFusion can render, wrap it in `arrow_cast`. + +**Cross the shapes with the NULL cases, and with ANSI mode.** These are not +three independent checklists to tick off separately, they are one product to +enumerate. The bug this instruction exists to catch is a branch that gets NULL +handling right in the row-wise path and wrong in the vectorized one, so the two +shapes disagree with each other while each looks correct in isolation. For +every shape, ask what happens when the *other* argument is NULL, and when it is +NULL in only some rows. A shape that takes an array and a scalar has to make +the scalar's validation depend on the array's null mask, because a per-row null +intolerant function skips validation on the rows it short circuits. All-NULL, +some-NULL, and no-NULL versions of the same array are three different tests. + +Note that a `(Array, Scalar)` branch returning `ColumnarValue::Scalar` is *not* +by itself a bug. In DataFusion a returned `Scalar` means "this value for every +row" and the caller broadcasts it to the batch length. Before reporting one, +run an N-row query and count the rows that come back. Report it only if the +row count is actually wrong, or if the scalar's value depends on per-row input. +Otherwise it is at most a readability cleanup, and reporting it as a +correctness bug costs the audit credibility. + +**Overflow and underflow.** Does it return `Err`, wrap, or panic? Spark's +answer depends on ANSI mode. A panic is always a bug. + +**Type dispatch.** Every type the signature admits must have a branch. An +`exec_err!` fallthrough for a type the signature accepts is a bug. + +**ANSI handling.** Compare against `datafusion.execution.enable_ansi_mode` +(defined in `datafusion/common/src/config.rs`). The crate has scattered `TODO` +comments about unimplemented ANSI paths. Treat every one found in this +function as a finding, not as a known-acceptable state. Grep for them across +every file Step 3's grep returned, including `mod.rs`: + +```bash +grep -rn "TODO" datafusion/spark/src/function//$ARGUMENTS.rs \ + datafusion/spark/src/function//mod.rs +``` + +Wiring ANSI up is usually mechanical rather than a semantics decision. The flag +is already threaded through `ScalarFunctionArgs`, so the implementation reads +it directly: + +```rust +let ScalarFunctionArgs { args, config_options, .. } = args; +let ansi_mode = config_options.execution.enable_ansi_mode; +``` + +`datafusion/spark/src/function/math/abs.rs` and +`datafusion/spark/src/function/math/modulus.rs` are the in-repo precedents for +both the plumbing and the test layout. DataFusion does not model Spark's error +classes or SQLSTATE values, so say that in the audit either way. + +Whether to match Spark's message text depends on who constructs the error: + +- **The function constructs it** (an `exec_err!` you write). Match Spark's text + exactly, including trailing punctuation, and assert it in the `.slt`. +- **An Arrow kernel constructs it** (`rem`, `add`, a cast). You do not control + the wording. Assert only that an error is raised, note the divergence in a + comment, and file an issue for the message text rather than contorting the + implementation to restate an Arrow error in Spark's words. + +Do not read the `abs.rs` precedent as licence to invent DataFusion-flavoured +text for an error you *do* construct. `abs` reports `Int32 overflow on +abs(-2147483648)` where Spark 4.2.0 reports `[ARITHMETIC_OVERFLOW] overflow.`, +which is a known divergence tracked by +https://github.com/apache/datafusion/issues/24145. Follow its ANSI plumbing and +test layout, not its wording. + +`abs` also shows why the "who constructs it" question is worth asking rather +than assuming. Its scalar path builds the message in the spark crate, but its +array path borrows `make_try_abs_function!` from +`datafusion/functions/src/math/abs.rs`, so that half of the text is owned by +core DataFusion and shared with core's own `abs`. Wherever a spark function +reuses a core macro or kernel, Spark-specific error text is not available +without un-sharing the code first. Check where the message is actually built +before promising to match Spark's. + +**Rust unit tests.** Read the `#[test]` blocks in the same file. Per the crate +README, direct invocation through `test_scalar_function!()` bypasses input +coercion, so passing unit tests never establish end-to-end correctness. Note +which cases are covered by unit tests only. + +## Step 4: Cross-Check Comet and Sail + +The `datafusion-spark` README recommends checking both projects for existing +implementations. Diff against them to surface edge cases they handle that this +crate misses. + +Neither project needs to be checked out. Both are searched over HTTP, by +content rather than by path, because both have moved their directory layouts +more than once: + +```bash +for repo in apache/datafusion-comet lakehq/sail; do + echo "=== $repo ===" + gh api -X GET search/code -f q="$ARGUMENTS repo:$repo" --jq '.items[].path' +done + +# Then read a specific hit +gh api "repos//contents/" --jq '.content' | base64 -d +``` + +If a clone happens to be on disk, grep it instead. It is faster and it is not +rate limited. Do not silence the failure with `2>/dev/null`, though: a grep over +a directory that is not there prints an error, and swallowing it turns "Comet +not checked out" into "Comet has no implementation". + +```bash +[ -d ../datafusion-comet ] \ + && grep -rln "$ARGUMENTS" ../datafusion-comet/native/spark-expr/src/ --include="*.rs" \ + || echo "no local Comet clone, use the API above" +``` + +`gh api search/code` requires authentication and is rate limited to a handful of +requests per minute. If it fails, say so in the audit report and mark Step 4 as +not performed rather than reporting no differences found. + +As of this writing the implementations live under +`crates/sail-function/src/scalar//spark_.rs`, and Sail also +keeps behavior fixtures at +`python/pysail/tests/spark/function/features/.feature`. Treat both +paths as hints, not as addresses. + +Sail matters more than a generic peer here: many `datafusion-spark` functions +were ported from Sail, so Sail's current code is often the same code with +subsequent fixes applied. A difference between the two is a strong lead. + +Treat both as peer implementations, not as authority. Each has its own audit +backlog, and a shared assumption between them is not evidence of correctness. +Spark's source and a running Spark are the only authorities. + +## Step 5: Review Existing SLT Coverage + +```bash +ls datafusion/sqllogictest/test_files/spark/*/$ARGUMENTS.slt +``` + +Read the file and list the queries, the types exercised, which cases are +commented out, and whether the file has an audit header comment from a +previous audit. + +Also read `datafusion/sqllogictest/test_files/spark/README.md`. It is the +binding local convention for this tree and it prescribes things this skill does +not repeat: every function must be tested on both scalar and array inputs, +every literal needs an explicit cast because DataFusion and Spark infer types +differently, test cases contain `SELECT` statements only, and a function whose +behavior differs under ANSI is wrapped in + +```sql +statement ok +set datafusion.execution.enable_ansi_mode = true; + +# queries + +statement ok +set datafusion.execution.enable_ansi_mode = false; +``` + +Reset the flag to `false` at the end of the block. The setting leaks to +whatever follows it in the file otherwise. + +Many files in this tree were machine-generated from Sail's gold data. Their +expected values were not necessarily verified against Spark, so an existing +passing expectation is not evidence of correctness. Re-verify in Step 7. + +## Step 6: Gap Analysis + +Build this matrix, filling each cell with yes, no, or n/a: + +| Dimension | Spark tests it | DataFusion SLT | Rust unit test | Gap? | +| ---------------------------------------------- | -------------- | -------------- | -------------- | ---- | +| All-scalar arguments | | | | | +| All-array arguments | | | | | +| Array first argument, scalar second | | | | | +| Scalar first argument, array second | | | | | +| NULL input, each argument independently | | | | | +| Empty string / array / map | | | | | +| Zero, negative zero, negatives | | | | | +| Overflow, underflow | | | | | +| Boundary values (INT_MIN, INT_MAX, i64 bounds) | | | | | +| NaN, Infinity, -Infinity, subnormal | | | | | +| Multibyte / special UTF-8 | | | | | +| ANSI mode enabled | | | | | +| ANSI mode disabled | | | | | +| Every type in Spark's `inputTypes` | | | | | +| Types Spark rejects | | | | | +| Return nullability | | | | | +| Cross-version divergence | | | | | + +Rows marked n/a need a one-line justification, for example "no float inputs" +against the NaN row. + +Both ANSI rows matter independently. DataFusion defaults `enable_ansi_mode` to +`false` while Spark 4.x defaults ANSI to `true`, so the mode users actually hit +on Spark 4 is the one most likely to be untested here. + +## Step 7: Establish Ground Truth with PySpark + +No expected value goes into a `.slt` file without being observed from a real +Spark. Reading `nullSafeEval` and predicting the output is exactly how wrong +golden values get written in the first place. + +### Set up the environment + +Requires a JDK 17 or later on `PATH`. Use `uv` to create the venv rather than +`python3 -m venv`: `pyspark==4.2.0` requires Python 3.10 or later, and the +system `python3` on many machines is older. `uv` is already used by this +repository and will fetch a suitable interpreter if none is installed. + +Unset `SPARK_HOME` and `PYSPARK_PYTHON` before doing anything else. If either +is set, a local Spark install on `SPARK_HOME` silently overrides the +pip-installed `pyspark` JARs, and `PYSPARK_PYTHON` can point PySpark's workers +at a different Python than the venv you just built. The `SPARK_HOME` case is +not hypothetical: it reproduces reliably on a machine with any other Spark +install on `PATH`, and it fails late and unhelpfully, inside `getOrCreate()`, +as `TypeError: 'JavaPackage' object is not callable`. If you hit that exact +error, this is the cause: check `echo $SPARK_HOME` and unset it. + +**`unset` does not persist between tool calls.** Each shell invocation starts +from the user's profile, which is where `SPARK_HOME` was exported in the first +place. Unsetting it once in the setup block and then running the harness in a +later call reintroduces the failure. Put `unset SPARK_HOME PYSPARK_PYTHON` in +the same command as every JVM-starting invocation, not in a preceding one. + +```bash +unset SPARK_HOME PYSPARK_PYTHON +uv venv --python 3.12 /tmp/pyspark-4.2.0-venv +uv pip install --python /tmp/pyspark-4.2.0-venv --quiet 'pyspark==4.2.0' +/tmp/pyspark-4.2.0-venv/bin/python -c "import pyspark; print(pyspark.__version__)" +``` + +The last command must print `4.2.0`. If it prints anything else, stop and fix +the environment: an audit against the wrong Spark version is worse than no +audit, because it writes confident wrong expectations. + +That check is necessary but not sufficient. It only reads the installed +package's metadata: it never starts a JVM, so it passes cleanly even on a +machine primed to hit the `SPARK_HOME` shadowing failure above. The check +that actually proves which Spark you are talking to is the +`# Spark 4.2.0 tz=UTC` banner the harness prints from a live `SparkSession` in +the next section. Do not trust the environment until you have seen that +banner. + +### The harness + +Write this to the scratchpad as `spark_ground_truth.py`. It is deliberately not +committed to the repo: `pyspark` is too heavy a dependency to add to the `uv` +workspace in `dev/`, and `datafusion/spark/scripts/` is claimed by other +in-flight work. + +Write it unconditionally, overwriting whatever is already there. A previous +audit may have left an older copy in the same scratchpad, and silently reusing +it means auditing with a harness whose fixes you do not have. + +```python +#!/usr/bin/env python3 +"""Run SQL queries against a local PySpark and print SLT-ready results. + +Usage: spark_ground_truth.py QUERIES.sql + +Input is a file of SQL statements, each terminated by a semicolon at the end +of a line. Each query is run under both spark.sql.ansi.enabled=true and +=false, and the observed result is printed in a form that can be pasted into +a .slt file. + +The session time zone is pinned to UTC so that results do not depend on the +auditing machine's local zone. DataFusion's sqllogictest runner leaves +execution.time_zone unset, which is effectively UTC, so anything time zone +sensitive would otherwise produce golden values that are wrong everywhere +except on the machine that generated them. + +Statement splitting is naive: it splits on a semicolon followed by a newline. +That keeps a semicolon inside a string literal intact, as in +SELECT ascii(';'), as long as the literal is not the last thing on the line. +A query whose string literal ends a line with a semicolon still splits +wrongly. Put such a query on a single line with trailing text after the +literal, or run it on its own. +""" + +import sys + +from pyspark.sql import SparkSession + + +def format_value(value): + """Render one cell the way sqllogictest expects it.""" + if value is None: + return "NULL" + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +def run_query(spark, sql): + try: + rows = spark.sql(sql).collect() + except Exception as exc: # noqa: BLE001 - any failure is a result here + first_line = str(exc).strip().split("\n")[0] + return f"ERROR: {type(exc).__name__}: {first_line}" + if not rows: + return "(no rows)" + return "\n".join("\t".join(format_value(v) for v in row) for row in rows) + + +def main(path): + spark = ( + SparkSession.builder.appName("slt-ground-truth") + .master("local[1]") + .getOrCreate() + ) + spark.conf.set("spark.sql.session.timeZone", "UTC") + spark.sparkContext.setLogLevel("ERROR") + print(f"# Spark {spark.version} tz={spark.conf.get('spark.sql.session.timeZone')}") + + raw = open(path).read() + if not raw.endswith("\n"): + raw += "\n" + queries = [q.strip() for q in raw.split(";\n") if q.strip()] + + for sql in queries: + for ansi in (True, False): + spark.conf.set("spark.sql.ansi.enabled", str(ansi).lower()) + print(f"### ansi={str(ansi).lower()}") + print(f"# {sql}") + print(run_query(spark, sql)) + print() + + spark.stop() + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.exit("usage: spark_ground_truth.py QUERIES.sql") + main(sys.argv[1]) +``` + +### Using it + +Write every candidate query, including the ones already in the `.slt` file, to +a scratch `.sql` file and run: + +```bash +cd +unset SPARK_HOME PYSPARK_PYTHON +/tmp/pyspark-4.2.0-venv/bin/python spark_ground_truth.py queries.sql \ + > ground_truth.txt 2> spark_logs.txt +``` + +Keep the streams separate. Spark writes log4j output, including structured JSON +error records, on a stream that will interleave with the results if you merge +them with `2>&1`. The merged form is what an agent reaches for by reflex and it +makes the output unreadable exactly on the error cases that matter most. + +Verify the banner line reads `# Spark 4.2.0 tz=UTC` before trusting any +output. Both halves matter. The version half proves you are not talking to a +Spark install that shadowed the pip-installed one, and the time zone half +proves the session is not silently inheriting the auditing machine's local +zone. DataFusion's sqllogictest runner leaves `execution.time_zone` unset, +which behaves as UTC, so a session on any other zone generates golden values +that only reproduce on the machine that generated them. This is invisible for +functions that take a `DATE` and never touch a time zone, and it is a silent +corruption for `to_utc_timestamp`, `date_format`, `from_unixtime`, and every +other timestamp function. If the banner shows anything but `tz=UTC`, stop and +fix the harness before recording a single expected value. + +**Wrap date, timestamp, and interval results in `CAST(... AS STRING)`.** The +harness calls `collect()`, which converts every cell to a Python object, and +Python's `datetime.date` cannot represent everything Spark's `DateType` can. A +value outside Python's range surfaces as a line like + +``` +ERROR: ValueError: year 10000 is out of range +``` + +which reads exactly like a Spark error and is not one. Spark returned a value +fine. Any `ValueError` or `OverflowError` in the output is a harness artifact, +never a Spark result. Re-run the query wrapped in a cast to see what Spark +actually produced: + +```sql +SELECT CAST(next_day(DATE '9999-12-31', 'Mon') AS STRING); +-- +10000-01-03 +``` + +Re-verify expectations that already exist in the `.slt` file. Many files in +this tree were machine-generated and their values were never checked against +Spark. An existing expectation is a claim, not evidence. + +Write the queries in Spark SQL. The `.slt` file needs DataFusion syntax, which +differs: DataFusion uses `'2015-01-14'::DATE` where Spark SQL uses +`DATE '2015-01-14'`. Translate when transcribing, and keep the semantics +identical. If a query cannot be expressed in both dialects, note it as a +finding rather than silently testing something different. + +### Never use `--complete` + +The sqllogictest runner supports `--complete`, which fills in expected results +with whatever DataFusion currently returns. + +**Never run it on files under `test_files/spark/`.** On an audit it would +silently cement the bug being audited as the golden answer, which inverts the +purpose of the whole exercise. Expected values come from PySpark and are +written by hand. + +### If PySpark cannot be installed + +Continue the audit using values derived from the Spark source and the golden +files from Step 2, and mark every such expectation as unverified in both the +audit report and a comment in the `.slt` file: + +```sql +# UNVERIFIED: derived from Spark source, not confirmed against a running Spark +``` + +Never present a derived value as a verified one. An audit that overstates its +confidence is worse than one that admits a gap. + +## Step 8: Apply Findings + +Findings do not survive as prose. A recommendation in a PR description is +buried the moment the PR merges. Every finding becomes code, a test, or a +filed issue. + +### Fix what can be fixed + +Divergences from Spark 4.2.0 are fixed in the audit PR. Attempt the fix +regardless of size. Fall back to filing an issue only when the fix needs a +semantics decision a human should make, for example choosing which of two +defensible behaviors DataFusion should adopt when Spark's own behavior is +inconsistent across versions. + +Every fix is locked in by an SLT test that fails before the fix and passes +after. A test that passes before the fix was testing something else. + +Verify both directions before committing. Stash only the implementation +change, so the new `.slt` query runs against the unfixed code: + +```bash +# Stash the implementation only, leaving the new .slt query in place +git stash push -- datafusion/spark/src/function//.rs +cargo test --test sqllogictests -- spark//$ARGUMENTS # expect FAIL +git stash pop +cargo test --test sqllogictests -- spark//$ARGUMENTS # expect PASS +``` + +This verifies SLT tests only. Rust unit tests live inside the implementation +file, so stashing that file stashes the tests with it and the run proves +nothing about them. For a unit test, get the fail-before evidence by reverting +the implementation body by hand, or by writing the test first and watching it +fail before the fix exists. + +Read the failure list from the first run and confirm it names the queries you +added. Do not judge the run by an exit code you captured through a pipe: +`cargo test ... | head` reports the exit status of `head`, not of `cargo`, so a +piped command that "succeeded" tells you nothing. Either run it unpiped or +inspect the `External error: N errors in file ...` block directly. + +If the first run passes, the test is not exercising the fix. Either the +query was stashed along with the implementation, or the assertion does not +actually cover the divergent case. Investigate before continuing: a test +that passes against the unfixed code locks in nothing. + +If the fix also changed a `mod.rs` doc string or a helper in another file, that +file is not covered by the stash above. Add it to the `git stash push` argument +list, or the "expect FAIL" run may fail for the wrong reason. + +Run this before committing the fix. If `git stash push` reports +`No local changes to save`, the change is already committed and the check is +invalid, because the "expect FAIL" run just tested the fixed code. Redo it +against the pre-fix commit by checking that one file out of history and back, +which touches only the working tree and never moves `HEAD` or a branch: + +```bash +# is the commit before the one that introduced the fix +git checkout -- datafusion/spark/src/function//.rs +cargo test --test sqllogictests -- spark//$ARGUMENTS # expect FAIL +git checkout HEAD -- datafusion/spark/src/function//.rs +cargo test --test sqllogictests -- spark//$ARGUMENTS # expect PASS +``` + +Find `` with `git log --oneline -- ` and +take the commit before the fix. If the fix also touched other files, list every +one of them in both `git checkout` commands. + +Do not reach for `git reset`, `git revert`, or `git rebase` here. None of them +are needed and all of them can leave the branch in a state the audit then has +to repair. + +### Capture what cannot be fixed + +A deferred divergence becomes a commented-out SLT query carrying the correct +Spark 4.2.0 result and the issue link: + +```sql +# Spark 4.2.0 raises ILLEGAL_DAY_OF_WEEK under ANSI mode; DataFusion returns NULL +# https://github.com/apache/datafusion/issues/23889 <- the real number, not a placeholder +# query error +# SELECT next_day('2015-01-14'::DATE, 'NOT_A_DAY'::string); +``` + +Uncommenting the block is the contract for verifying the eventual fix. Note +that DataFusion `.slt` files use `#` for comments, not `--`. + +**File the issue first, then write the block.** The block cannot be written +until its issue number exists, so do the filing described below before typing +the comment. Never commit a placeholder in place of the number. A string like +``, `NNNNN`, or `TBD` inside a URL ships a dead link, and +because the surrounding block is commented out no test ever catches it. + +If the issue cannot be filed at all, for example because `gh` is not +authenticated or the user asked that nothing be filed yet, omit the link line +entirely. A block with no link is honest and self-contained. Then say so +explicitly in the audit report: name the divergence, state that no issue was +filed and why, and ask the user to file it. Do not leave the reader to +discover the gap from a broken URL. + +When you do assert an error, remember that the runner normalizes whitespace in +the `query error` regex, so literal leading, trailing, and repeated spaces in +the expected message cannot be written out. An input like `' MO '` produces +`Illegal input for day of week: MO .` with two spaces, and a regex containing +those two spaces is collapsed to one before matching. Write `\s+` instead, +which carries no literal whitespace and survives normalization: + +```sql +query error Illegal input for day of week:\s+MO \. +SELECT next_day('2015-01-14'::DATE, ' MO '::string); +``` + +That asserts the padding is present without pinning its exact width. Anchor +the right-hand end of the match on a non-whitespace character from Spark's +message, such as the trailing period in Spark 4.x error templates, so the +assertion still fails if the padding runs into the rest of the message. If +even that cannot be expressed, fall back to the stable prefix and say in a +comment why the assertion stops where it does. + +Match Spark's message text exactly, including trailing punctuation. Spark 4.x +error templates in `error-conditions.json` generally end in a period, and the +pre-4.0 exception strings they replaced generally did not. Copying the older +string leaves the implementation asserting a version the audit does not claim, +and a `query error` prefix match will not catch it. + +Before filing, search for an existing issue: + +```bash +gh issue list --repo apache/datafusion --search "$ARGUMENTS in:title,body" --state all --limit 5 +``` + +If a candidate comes back, open it with `gh issue view --repo apache/datafusion` +and confirm it actually describes this divergence and is still open. A closed +issue cited as a known divergence is worse than no citation, because the reader +follows the link and finds a fix that already shipped. + +If no issue exists, file one with the `bug` label plus `spark`. Title format: +`[Bug] `. The body includes the Spark version +range affected, a minimal repro in both Spark SQL and DataFusion SQL, the +divergent result, and the relevant file and line. + +### Capture cross-version divergences + +A behavior difference between Spark versions cannot be tested today. Capture it +in two places. + +A comment at the query site naming the versions: + +```sql +# Spark 3.5.8 returns NULL here; Spark 4.0.4 and later throw under ANSI mode. +# Expectation below asserts 4.2.0. Version-specific expectations are tracked by +# https://github.com/apache/datafusion/issues/23887 +``` + +And its own filed issue, so the divergence is scheduled once the version +mechanism lands. + +### Record the audit + +Add a header comment block to the `.slt` file, directly below the license +header and the existing provenance comment: + +```sql +# Audited against Spark 4.2.0 on YYYY-MM-DD. +# Expected values verified with pyspark==4.2.0. +# Known divergences are captured as commented-out queries below. +``` + +Use today's real date. If PySpark was unavailable, say so on the second line +instead of claiming verification. + +### Attribution + +Every issue and PR the skill files ends with: + +``` +Surfaced by the audit-datafusion-spark-expression skill. +``` + +Do not mention Claude, AI, or automated generation anywhere in an issue or PR. + +### Before committing + +Per the repository `CLAUDE.md`: + +```bash +cargo fmt --all +cargo clippy --all-targets --all-features -- -D warnings +cargo test -p datafusion-spark --lib $ARGUMENTS +cargo test --test sqllogictests -- spark//$ARGUMENTS +``` + +The `--lib` run is not optional. `cargo test --test sqllogictests` does not +compile or run the `#[test]` blocks inside `datafusion/spark/src/`, so a +refactor that renames or changes the signature of a helper those tests call +passes the SLT gate and fails CI. + +Do not use `cargo build -p datafusion-spark` as a quick compile check. The +crate does not currently build in isolation and the resulting error is +unrelated to your change. Use `cargo test -p datafusion-spark --lib` or the +sqllogictest target instead. + +If the audit touched documentation: + +```bash +./ci/scripts/doc_prettier_check.sh --write --allow-dirty +``` + +Follow `.github/pull_request_template.md` for the PR. + +## Output Format + +Present the audit as: + +1. **Function summary.** What `$ARGUMENTS` does, its input and output types, + and its null behavior. +2. **Spark version differences.** Across 3.5.8, 4.0.4, 4.1.3, and 4.2.0. Say + explicitly if there are none. +3. **DataFusion implementation notes.** Including signature, coercion, and + nullability findings. +4. **Gap matrix.** From Step 6, plus implementation gaps. +5. **Findings.** Split into fixed in this PR, deferred with a filed issue, and + low-risk coverage gaps. +6. **Offer to add the low-risk coverage tests.** + +Low-risk coverage gaps are the only category that pauses for user input. +Correctness findings are always either fixed or filed. + +The line between the two is the code you touched. Coverage for a path this PR +changed goes into the `.slt` file without asking: it is part of the fix. The +offer is for cases on paths the PR did not touch and that already look correct, +where adding tests is optional scope the user may not want. Phrase the offer +as: + +> Spark exercises the following cases that have no DataFusion test, on code +> paths that appear correct. Would you like me to add them? +> +> - [list each case] + +## Tone and Style + +- Write in clear, concise prose. +- Use backticks around function names, file paths, type names, and config keys. +- Acknowledge what is already well covered before raising gaps. +- Avoid em dashes and semicolons. Use separate sentences. +- Do not overstate confidence. An unverified expectation is labeled as one. From 67c77b8e2ad7e286dd9f4d1072eeb3f802f39513 Mon Sep 17 00:00:00 2001 From: Goutam Adwant <8672451+goutamadwant@users.noreply.github.com> Date: Sun, 16 Aug 2026 01:13:03 +0000 Subject: [PATCH 43/50] feat: union nested Parquet leaves across projection accesses (#24130) ## Which issue does this PR close? - Closes #24121. ## Rationale for this change When one projected nested column is consumed through multiple narrowing casts, or through both a narrowing cast and `get_field`, the Parquet read plan currently falls back to reading the entire root column. The leaves required by those consumers can be combined, so reading the full root performs unnecessary I/O. ## What changes are included in this PR? - Accumulate and deduplicate the leaf offsets required by every narrowing cast on a root column. - Add leaves required by `get_field` accesses to the same union. - Derive the Arrow type emitted by the reader from the merged leaf set while retaining a full-read fallback for unsupported partial wrapper shapes. - Add coverage for disjoint, overlapping, and repeated casts, cast plus `get_field`, and casts that diverge below `List`. ## Are these changes tested? Yes. The following checks pass: - `cargo fmt --all -- --check` - `cargo test -p datafusion-datasource-parquet --lib` - `cargo test --profile=ci --test sqllogictests -- parquet_nested_schema_pruning.slt` - `cargo clippy -p datafusion-datasource-parquet --all-targets --all-features -- -D warnings` The SQL logic test also verifies that the merged projections return the expected values and scan fewer bytes than a full-root read. ## Are there any user-facing changes? No API or SQL behavior changes. Queries with multiple nested accesses to the same Parquet root can read fewer leaf columns. --- .../src/nested_schema_pruning.rs | 109 +++++++++ .../src/projection_read_plan.rs | 213 ++++++++++++------ .../parquet_nested_schema_pruning.slt | 44 +++- 3 files changed, 287 insertions(+), 79 deletions(-) diff --git a/datafusion/datasource-parquet/src/nested_schema_pruning.rs b/datafusion/datasource-parquet/src/nested_schema_pruning.rs index 9768282c3bab0..9d1534eff61eb 100644 --- a/datafusion/datasource-parquet/src/nested_schema_pruning.rs +++ b/datafusion/datasource-parquet/src/nested_schema_pruning.rs @@ -128,6 +128,35 @@ pub(crate) fn clip_for_cast( Some((kept, pruned_type)) } +/// Rebuilds `physical` for a sorted, deduplicated subset of its leaf offsets. +/// +/// This is the second half of merging several nested accesses to one root: +/// callers union the offsets required by each access, then use this function +/// to derive the Arrow type the Parquet reader emits for that union. Partial +/// projection is supported through the same struct, list, and large-list +/// shapes as [`clip_for_cast`]. Any partial selection below another wrapper +/// returns `None`, preserving the total-fallback property of cast clipping. +pub(crate) fn type_for_leaf_subset( + physical: &DataType, + kept: &[usize], +) -> Option { + let total = count_leaves(physical); + if kept.is_empty() + || kept.len() >= total + || kept.windows(2).any(|pair| pair[0] >= pair[1]) + || kept.last().is_some_and(|last| *last >= total) + { + return None; + } + + let mut next_leaf = 0; + let projected = project_type_for_leaves(physical, kept, &mut next_leaf); + if projected.is_ok() { + debug_assert_eq!(next_leaf, total, "leaf accounting must cover the type"); + } + projected.ok().flatten() +} + /// Number of Parquet leaf columns a (Parquet-derived) Arrow type occupies. pub(crate) fn count_leaves(dt: &DataType) -> usize { match dt { @@ -249,6 +278,54 @@ fn clip_type( } } +/// Rebuilds one physical type subtree for `kept` leaf offsets. +/// +/// `Ok(None)` means no selected leaf lies below this subtree. `Err(())` means +/// only part of an unsupported wrapper was selected, so the caller must fall +/// back to reading the full root. +fn project_type_for_leaves( + physical: &DataType, + kept: &[usize], + next_leaf: &mut usize, +) -> Result, ()> { + match physical { + DataType::Struct(fields) => { + let mut projected = Vec::with_capacity(fields.len()); + for field in fields { + if let Some(dt) = + project_type_for_leaves(field.data_type(), kept, next_leaf)? + { + projected.push(field_with_type(field, dt)); + } + } + Ok((!projected.is_empty()).then(|| DataType::Struct(projected.into()))) + } + DataType::List(item) => { + Ok(project_type_for_leaves(item.data_type(), kept, next_leaf)? + .map(|projected| DataType::List(field_with_type(item, projected)))) + } + DataType::LargeList(item) => { + Ok(project_type_for_leaves(item.data_type(), kept, next_leaf)? + .map(|projected| DataType::LargeList(field_with_type(item, projected)))) + } + _ => { + let start = *next_leaf; + let end = start + count_leaves(physical); + let selected_start = kept.partition_point(|leaf| *leaf < start); + let selected_end = kept.partition_point(|leaf| *leaf < end); + let selected = selected_end - selected_start; + *next_leaf = end; + if selected == 0 { + Ok(None) + } else if selected == end - start { + Ok(Some(physical.clone())) + } else { + Err(()) + } + } + } +} + /// Keep every leaf of `dt` (no pruning below this point); returns `dt` /// unchanged since nothing was clipped. fn keep_all_leaves( @@ -455,6 +532,38 @@ mod tests { assert_eq!(emitted, list_of(struct_of(vec![int64("x"), utf8("y")]))); } + /// A union assembled from two casts below a shared List prefix + /// reconstructs one emitted type in physical field order. + #[test] + fn type_for_leaf_subset_unions_nested_cast_leaves() { + let physical = list_of(struct_of(vec![int64("x"), utf8("y"), utf8("pad")])); + let emitted = type_for_leaf_subset(&physical, &[0, 1]).unwrap(); + assert_eq!(emitted, list_of(struct_of(vec![int64("x"), utf8("y")]))); + } + + /// A complete leaf set has nothing to prune, so callers must retain the + /// original root type rather than treating it as a clipped projection. + #[test] + fn type_for_leaf_subset_rejects_complete_selection() { + let physical = struct_of(vec![int64("x"), utf8("y"), utf8("pad")]); + assert!(type_for_leaf_subset(&physical, &[0, 1, 2]).is_none()); + } + + /// Partial selection below a wrapper cast clipping does not understand + /// must fall back instead of predicting a type the reader may not emit. + #[test] + fn type_for_leaf_subset_rejects_unsupported_partial_wrapper() { + let physical = DataType::FixedSizeList( + Arc::new(Field::new( + "item", + struct_of(vec![int64("x"), utf8("pad")]), + true, + )), + 2, + ); + assert!(type_for_leaf_subset(&physical, &[0]).is_none()); + } + /// Two levels of `list` nesting, the inner one also narrowed, /// the `events: array>>>` shape /// reported in `datafusion-comet#4859`, where a sibling struct field at diff --git a/datafusion/datasource-parquet/src/projection_read_plan.rs b/datafusion/datasource-parquet/src/projection_read_plan.rs index 038180851125a..c00478f5627b1 100644 --- a/datafusion/datasource-parquet/src/projection_read_plan.rs +++ b/datafusion/datasource-parquet/src/projection_read_plan.rs @@ -45,6 +45,7 @@ use datafusion_physical_expr::{PhysicalExpr, ScalarFunctionExpr}; use crate::nested_schema_pruning::{ CastColumnAccess, clip_for_cast, contains_struct, count_leaves, field_with_type, + type_for_leaf_subset, }; /// The result of resolving which Parquet leaf columns and Arrow schema fields @@ -577,22 +578,16 @@ pub(crate) fn build_projection_read_plan( /// - roots referenced as whole columns keep every leaf and their full /// physical field (whole-column reads take precedence; cast accesses on /// such roots were already dropped by the caller); -/// - roots consumed through a cast, and not also through a `get_field` -/// access on the same root, keep only the leaves the cast target names -/// (see `crate::nested_schema_pruning`); +/// - roots consumed through one or more casts keep the union of the leaves +/// their targets name (see `crate::nested_schema_pruning`); +/// - a root consumed through both casts and `get_field` accesses keeps the +/// union of both access kinds; /// - roots consumed only through `get_field` accesses keep the union of the /// leaves those accesses reach, as before; /// - any other referenced root, a cast that can't be safely clipped (see -/// `nested_schema_pruning::clip_for_cast`), a root reached by two casts -/// with *different* targets (a projection can consume the same column -/// through more than one narrowing cast, e.g. -/// `SELECT CAST(s AS STRUCT(a)), CAST(s AS STRUCT(b)) FROM t`; clipping to -/// either target alone would starve the other), or a root reached by both a -/// cast and a `get_field` access (not produced by -/// `DefaultPhysicalExprAdapter`, which always routes a `get_field` over a -/// narrowed column through the same cast rather than a separate access, -/// but a custom `PhysicalExprAdapter` could in principle inject both), -/// falls back to a full read of that root. +/// `nested_schema_pruning::clip_for_cast`), or a merged leaf set whose +/// emitted Arrow type can't be derived safely, falls back to a full read of +/// that root. fn build_read_plan_with_cast_clipping( file_schema: &Schema, schema_descr: &SchemaDescriptor, @@ -601,45 +596,22 @@ fn build_read_plan_with_cast_clipping( cast_accesses: &[CastColumnAccess], ) -> ParquetReadPlan { let whole_roots: BTreeSet = whole_root_indices.iter().copied().collect(); - let struct_access_roots: BTreeSet = - struct_accesses.iter().map(|a| a.root_index).collect(); // Every referenced root's Parquet leaves, grouped in one pass over the // schema descriptor rather than one `leaf_indices_for_roots` scan per // root (this function may look up several roots). let leaves_by_root = leaves_grouped_by_root(schema_descr); - // Root -> (absolute kept leaf indices, cast-clipped Arrow type) for - // roots successfully clipped via a cast. - let mut clipped_by_root: BTreeMap, DataType)> = BTreeMap::new(); + // Root -> relative leaf offsets required by every narrowing cast on that + // root. A set makes repeated and overlapping targets a natural union. + let mut kept_offsets_by_root: BTreeMap> = BTreeMap::new(); // Roots with a cast access that must fall back to a full read. let mut fallback_roots: BTreeSet = BTreeSet::new(); - // The cast target already clipped for a root, so a second cast on the - // same root can be recognised as either a repeat (same target: nothing to - // do) or a conflict (different target: neither clip is valid on its own). - let mut clipped_target_by_root: BTreeMap = BTreeMap::new(); for access in cast_accesses { let root = access.root_index; if whole_roots.contains(&root) || fallback_roots.contains(&root) { continue; } - if let Some(previous) = clipped_target_by_root.get(&root) { - if **previous != access.target_type { - // The projection consumes this root through two different - // narrowing casts. Each cast only needs its own leaves, but - // the mask is per column: clipping to the first target would - // silently null-fill whatever the second one needs. Read the - // whole root instead. - clipped_by_root.remove(&root); - clipped_target_by_root.remove(&root); - fallback_roots.insert(root); - } - continue; - } - if struct_access_roots.contains(&root) { - fallback_roots.insert(root); - continue; - } let physical_type = file_schema.field(root).data_type(); let root_leaves = leaves_by_root.get(&root).map_or(&[][..], Vec::as_slice); @@ -654,33 +626,73 @@ fn build_read_plan_with_cast_clipping( } match clip_for_cast(physical_type, &access.target_type) { - Some((kept_offsets, pruned_type)) => { - let start = root_leaves[0]; - let absolute = kept_offsets.into_iter().map(|o| start + o).collect(); - clipped_by_root.insert(root, (absolute, pruned_type)); - clipped_target_by_root.insert(root, &access.target_type); + Some((kept_offsets, _pruned_type)) => { + kept_offsets_by_root + .entry(root) + .or_default() + .extend(kept_offsets); } // Nothing prunable for this cast: every leaf is consumed. None => { + kept_offsets_by_root.remove(&root); fallback_roots.insert(root); } } } + // Add leaves reached through `get_field` to cast roots. The resolver + // returns absolute Parquet leaf indices; convert them back to offsets in + // their root so they can share the same union as cast clipping. + let struct_access_tree = StructAccessTree::from_accesses(struct_accesses); + for leaf in resolve_struct_field_leaves(&struct_access_tree, schema_descr) { + let root = schema_descr.get_column_root_idx(leaf); + if !kept_offsets_by_root.contains_key(&root) { + continue; + } + let Some(offset) = leaves_by_root + .get(&root) + .and_then(|root_leaves| root_leaves.binary_search(&leaf).ok()) + else { + kept_offsets_by_root.remove(&root); + fallback_roots.insert(root); + continue; + }; + kept_offsets_by_root + .get_mut(&root) + .expect("root presence checked above") + .insert(offset); + } + + // Derive the reader's one emitted Arrow type from each merged leaf set. + // Any unsupported partial wrapper retains the total fallback guarantee. + let mut clipped_by_root: BTreeMap, DataType)> = BTreeMap::new(); + for (root, kept_offsets) in kept_offsets_by_root { + if fallback_roots.contains(&root) { + continue; + } + let physical_type = file_schema.field(root).data_type(); + let root_leaves = leaves_by_root.get(&root).map_or(&[][..], Vec::as_slice); + let kept_offsets = kept_offsets.into_iter().collect::>(); + let Some(pruned_type) = type_for_leaf_subset(physical_type, &kept_offsets) else { + fallback_roots.insert(root); + continue; + }; + let absolute = kept_offsets + .into_iter() + .map(|offset| root_leaves[offset]) + .collect(); + clipped_by_root.insert(root, (absolute, pruned_type)); + } + // `get_field` accesses on roots not already read in full (as a whole // column, or as a cast that fell back) keep the existing (non-cast) leaf // resolution. let get_field_accesses: Vec = struct_accesses .iter() .filter(|a| { - // A root carrying a `get_field` access is put into - // `fallback_roots` before any clip is attempted (see the loop - // above), so it can never also be clipped. Assert that rather - // than re-testing it here, so a future reordering trips the - // assert instead of silently changing which leaves are read. - debug_assert!(!clipped_by_root.contains_key(&a.root_index)); !whole_roots.contains(&a.root_index) && !fallback_roots.contains(&a.root_index) + && !clipped_by_root.contains_key(&a.root_index) }) .cloned() .collect(); @@ -1358,10 +1370,9 @@ mod test { ); } - /// Once conflicting cast targets have demoted a root to a full read, a - /// *third* cast on it must not resurrect the clip. + /// A repeated third cast does not duplicate leaves or widen the union. #[test] - fn build_projection_read_plan_keeps_full_read_after_a_third_cast() { + fn build_projection_read_plan_keeps_union_after_a_third_cast() { let (file_schema, metadata) = write_id_struct_file(); let schema_descr = metadata.file_metadata().schema_descr(); @@ -1374,10 +1385,19 @@ mod test { assert_eq!( read_plan.projection_mask, - ProjectionMask::leaves(schema_descr, [1, 2, 3]) + ProjectionMask::leaves(schema_descr, [1, 2]) ); let s_field = read_plan.projected_schema.field_with_name("s").unwrap(); - assert_eq!(s_field.data_type(), file_schema.field(1).data_type()); + assert_eq!( + s_field.data_type(), + &DataType::Struct( + vec![ + Arc::new(Field::new("value", DataType::Int32, false)), + Arc::new(Field::new("label", DataType::Utf8, false)), + ] + .into() + ) + ); } /// A whole-column reference wins over a `get_field` access on the same @@ -1490,12 +1510,10 @@ mod test { ); } - /// Two casts on the same root with *different* targets cannot both be - /// served by one mask: clipping to either target alone would null-fill - /// whatever the other one needs (or fail its runtime struct-compatibility - /// check outright). Read the whole root instead. + /// Two casts on the same root with disjoint targets share the union of + /// their leaves, while an unreferenced sibling remains pruned. #[test] - fn build_projection_read_plan_falls_back_on_conflicting_cast_targets() { + fn build_projection_read_plan_unions_disjoint_cast_targets() { let (file_schema, metadata) = write_id_struct_file(); let schema_descr = metadata.file_metadata().schema_descr(); @@ -1515,11 +1533,68 @@ mod test { assert_eq!( read_plan.projection_mask, - ProjectionMask::leaves(schema_descr, [1, 2, 3]), - "every leaf of `s` must be read so both casts see their fields" + ProjectionMask::leaves(schema_descr, [1, 2]), + "the union must serve both casts without reading `s.pad`" ); let s_field = read_plan.projected_schema.field_with_name("s").unwrap(); - assert_eq!(s_field.data_type(), file_schema.field(1).data_type()); + assert_eq!( + s_field.data_type(), + &DataType::Struct( + vec![ + Arc::new(Field::new("value", DataType::Int32, false)), + Arc::new(Field::new("label", DataType::Utf8, false)), + ] + .into() + ) + ); + } + + /// A union that covers every leaf falls back to the full root type. + #[test] + fn build_projection_read_plan_falls_back_for_complete_cast_union() { + let (file_schema, metadata) = write_id_struct_file(); + let schema_descr = metadata.file_metadata().schema_descr(); + + let exprs = vec![ + cast_to_struct("s", 1, vec![("value", DataType::Int32)]), + cast_to_struct( + "s", + 1, + vec![("label", DataType::Utf8), ("pad", DataType::Utf8)], + ), + ]; + let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr); + + assert_eq!( + read_plan.projection_mask, + ProjectionMask::leaves(schema_descr, [1, 2, 3]) + ); + assert_eq!( + read_plan.projected_schema.field_with_name("s").unwrap(), + file_schema.field(1) + ); + } + + /// Overlapping cast targets deduplicate their shared leaves. + #[test] + fn build_projection_read_plan_unions_overlapping_cast_targets() { + let (file_schema, metadata) = write_id_struct_file(); + let schema_descr = metadata.file_metadata().schema_descr(); + + let exprs = vec![ + cast_to_struct("s", 1, vec![("value", DataType::Int32)]), + cast_to_struct( + "s", + 1, + vec![("value", DataType::Int32), ("label", DataType::Utf8)], + ), + ]; + let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr); + + assert_eq!( + read_plan.projection_mask, + ProjectionMask::leaves(schema_descr, [1, 2]) + ); } /// The struct fast-path gate looks at the *projected* columns, not at @@ -1546,12 +1621,10 @@ mod test { assert_eq!(read_plan.projected_schema.fields().len(), 1); } - /// A root reached by both a narrowing cast and a `get_field` access (not - /// producible by `DefaultPhysicalExprAdapter`, but a custom - /// `PhysicalExprAdapter` could inject both) falls back to a full read of - /// that root rather than attempting to union the two leaf sets. + /// A root reached by both a narrowing cast and a disjoint `get_field` + /// access shares the union of their leaves. #[test] - fn build_projection_read_plan_falls_back_when_cast_and_get_field_share_a_root() { + fn build_projection_read_plan_unions_cast_and_get_field_on_one_root() { let (file_schema, metadata) = write_id_struct_file(); let schema_descr = metadata.file_metadata().schema_descr(); @@ -1575,8 +1648,7 @@ mod test { let read_plan = build_projection_read_plan(exprs, &file_schema, schema_descr); - // Every leaf of `s` is read (full fallback), not just value/label. - let expected_mask = ProjectionMask::leaves(schema_descr, [1, 2, 3]); + let expected_mask = ProjectionMask::leaves(schema_descr, [1, 2]); assert_eq!(read_plan.projection_mask, expected_mask); let s_field = read_plan.projected_schema.field_with_name("s").unwrap(); @@ -1586,7 +1658,6 @@ mod test { vec![ Arc::new(Field::new("value", DataType::Int32, false)), Arc::new(Field::new("label", DataType::Utf8, false)), - Arc::new(Field::new("pad", DataType::Utf8, false)), ] .into() ), diff --git a/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt b/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt index d936a89beb9f7..0aa171824fb5f 100644 --- a/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt +++ b/datafusion/sqllogictest/test_files/parquet_nested_schema_pruning.slt @@ -473,10 +473,8 @@ set datafusion.execution.parquet.pushdown_filters = false; # Query-level casts. `ProjectionExec` is merged into the scan, so a `CAST` # written in the query reaches the same read-plan analysis as an # adapter-inserted one — including one column consumed through two *different* -# cast targets, which no single clipped read can serve. Clipping to one -# target's leaves would leave the other cast reading a struct that is missing -# the fields it names, which `cast_column` either null-fills (wrong results) -# or, for disjoint targets, rejects outright. +# cast targets. The read plan unions the targets' leaves so both casts receive +# the fields they name without reading unreferenced siblings. # # `exact` infers its schema from the file, so no adapter cast is interposed # and the casts below are the only ones the scan sees. @@ -515,6 +513,25 @@ FROM exact ORDER BY id; {x: 200} {x: 200, y: s2} NULL NULL +# A cast plus a separate get_field access on the same physical root. +query ?T +SELECT CAST(s AS STRUCT) AS q0, s['y'] AS q1 +FROM exact ORDER BY id; +---- +{x: 100} s1 +{x: 200} s2 +NULL NULL + +# The targets diverge below a shared ARRAY> prefix. +query ?? +SELECT CAST(events AS ARRAY>) AS q0, + CAST(events AS ARRAY>) AS q1 +FROM exact ORDER BY id; +---- +[{x: 10}] [{y: a1}] +[{x: 20}, {x: 21}] [{y: b1}, {y: b2}] +NULL NULL + # Repeated identical targets still clip. query ?? SELECT CAST(s AS STRUCT) AS q0, @@ -534,14 +551,25 @@ FROM exact ORDER BY id; {x: 200} {x: 200, y: s2, pad: sp2} NULL NULL -# The conflicting-target fallback reads the whole column -- exactly what a -# scan with no clipping at all reads, and never more. These two must match: -# the first falls back, the second never clips in the first place. +# The disjoint targets read their two-leaf union and skip `pad`. query TT explain analyze select CAST(s AS STRUCT) AS q0, CAST(s AS STRUCT) AS q1 from exact; ---- -Plan with Metrics DataSourceExec: metrics=[output_rows=3, bytes_scanned=219] +Plan with Metrics DataSourceExec: metrics=[output_rows=3, bytes_scanned=148] + +# The cast/get_field union likewise skips `pad`. +query TT +explain analyze select CAST(s AS STRUCT) AS q0, s['y'] AS q1 from exact; +---- +Plan with Metrics DataSourceExec: metrics=[output_rows=3, bytes_scanned=146] + +# The nested union reads x/y but skips both pad leaves. +query TT +explain analyze select CAST(events AS ARRAY>) AS q0, CAST(events AS ARRAY>) AS q1 from exact; +---- +Plan with Metrics DataSourceExec: metrics=[output_rows=3, bytes_scanned=172] +# A full-root baseline remains larger than every union above. query TT explain analyze select s from exact; ---- From 0e58b52de493e39a49f14c6f9ca4340c0c5ec112 Mon Sep 17 00:00:00 2001 From: kid <19265318+u70b3@users.noreply.github.com> Date: Sun, 16 Aug 2026 01:23:51 +0000 Subject: [PATCH 44/50] fix(spark): correct mod ANSI zero divisor and negative zero handling (#23987) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #23894 ## Rationale for this change `datafusion-spark`'s `mod` routes through the shared `try_rem` helper in `datafusion/spark/src/function/math/modulus.rs`, which has two gaps in its zero-divisor handling (both verified against Spark 3.5.8 – 4.2.0 in the issue): 1. **ANSI mode, floating-point divisor.** `try_rem` delegates to Arrow's `rem` kernel in ANSI mode, but Arrow only reports division by zero for integer and decimal types. Floating-point divisors follow IEEE 754 and quietly produce `NaN`, while Spark raises `REMAINDER_BY_ZERO` for a zero divisor of any numeric type: ```sql set datafusion.execution.enable_ansi_mode = true; SELECT mod(10.5::float8, 0.0::float8); -- NaN, Spark raises ``` 2. **`-0.0` divisor, both modes.** The legacy path nulls out zero divisors via `eq(right, 0)`, but Arrow's floating-point comparisons use a total order in which `-0.0` is distinct from `0.0`, so a `-0.0` divisor goes unrecognised. Spark's `isZero` is a numeric comparison and treats `-0.0` as zero: ```sql SELECT mod(10.5::float8, -0.0::float8); -- NaN, Spark returns NULL (legacy) ``` ## What changes are included in this PR? `try_rem` now detects zero divisors itself, mirroring the shape #23898 established for `pmod`: - A new `is_zero` helper counts `-0.0` as zero for the floating-point types (via a `negative_zero` companion, same as #23898). - In ANSI mode, any row with a zero divisor raises `ArrowError::DivideByZero` — the same error Arrow's `rem` already produces for integers today, so the message stays uniform across types. The check is masked by the validity of the dividend because Spark's remainder expressions are null intolerant: a NULL dividend short-circuits to NULL before the divisor is validated, so `mod(NULL, 0)` must return NULL rather than raise. - Both modes substitute NULL for zero divisors before calling Arrow's `rem`, so the kernel never sees a zero divisor: legacy mode gets NULLs, and ANSI mode has already raised on the rows that required it. Note on overlap with #23898: that PR rewrites `pmod` to no longer use `try_rem` and adds identical `is_zero`/`negative_zero` helpers. This PR is independent of it — `mod` is fixed either way — but whichever lands second should dedupe the helpers in a rebase. Until #23898 lands, `pmod` also picks up the `-0.0` and ANSI floating-point zero-divisor fixes through the shared helper. Out of scope: #23897 (reproducing Spark's exact ANSI error text) is a repository-wide error-message policy question, as noted in that issue. ## Are these changes tested? Yes: - New unit tests in `modulus.rs`: ANSI floating-point zero divisor raises; `-0.0` divisor returns NULL in legacy mode and raises in ANSI mode; a NULL dividend with a zero divisor returns NULL in ANSI mode (integer and float). - New sqllogictest cases in `spark/math/mod.slt` covering the same behavior at SQL level. - Verified `cargo test -p datafusion-spark`, the `spark/math/mod.slt` and `spark/math/pmod.slt` sqllogictests, `./dev/rust_lint.sh`, and the extended workspace suite (ci profile with `avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption`) — all green. ## Are there any user-facing changes? Only the bug fixes, and only for the Spark `mod`/`pmod` functions: in ANSI mode a floating-point zero divisor now raises instead of returning NaN, and a `-0.0` divisor is treated as zero in both modes (NULL in legacy mode, an error in ANSI mode), matching Spark. No API changes. --- datafusion/spark/src/function/math/modulus.rs | 265 ++++++++++++++++-- .../test_files/spark/math/mod.slt | 31 ++ .../test_files/spark/math/pmod.slt | 25 ++ 3 files changed, 302 insertions(+), 19 deletions(-) diff --git a/datafusion/spark/src/function/math/modulus.rs b/datafusion/spark/src/function/math/modulus.rs index 97f59c2cbb0cb..c37513c12cd6b 100644 --- a/datafusion/spark/src/function/math/modulus.rs +++ b/datafusion/spark/src/function/math/modulus.rs @@ -15,41 +15,77 @@ // specific language governing permissions and limitations // under the License. -use arrow::array::{Scalar, new_null_array}; +use arrow::array::{ArrayRef, BooleanArray, Scalar, new_null_array}; use arrow::compute::kernels::numeric::add; use arrow::compute::kernels::{ + boolean::{and, is_not_null, or}, cmp::{eq, lt}, - numeric::rem, + numeric::{neg, rem}, zip::zip, }; use arrow::datatypes::DataType; +use arrow::error::ArrowError; use datafusion_common::{Result, ScalarValue, assert_eq_or_internal_err}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; +/// Returns a one element array holding negative zero, for the floating point +/// types only. +/// +/// Arrow's comparison kernels order floating point values totally, so `-0.0` +/// compares as distinct from, and less than, `0.0`. Java, and therefore Spark, +/// treats `-0.0` as equal to zero. The helper below uses this to restore the +/// IEEE 754 answer. +fn negative_zero(data_type: &DataType) -> Result> { + match data_type { + DataType::Float16 | DataType::Float32 | DataType::Float64 => { + let zero = ScalarValue::new_zero(data_type)?.to_array()?; + Ok(Some(neg(zero.as_ref())?)) + } + _ => Ok(None), + } +} + +/// Rows of `values` that equal zero, counting `-0.0` as zero. +fn is_zero(values: &ArrayRef) -> Result { + let zero = ScalarValue::new_zero(values.data_type())?.to_array()?; + let mask = eq(values, &Scalar::new(zero))?; + match negative_zero(values.data_type())? { + Some(negative_zero) => Ok(or(&mask, &eq(values, &Scalar::new(negative_zero))?)?), + None => Ok(mask), + } +} + /// Computes `rem(left, right)` with divide-by-zero handling. -/// In ANSI mode, any zero divisor causes an error. -/// In legacy mode (ANSI off), zero divisors are replaced with NULL before -/// computing the remainder, so those positions return NULL while others -/// compute normally. +/// In ANSI mode, a zero divisor of any numeric type causes an error, with +/// `-0.0` counting as zero; a row whose dividend is NULL never raises, to +/// match Spark's null-intolerant remainder. In legacy mode (ANSI off), zero +/// divisors are replaced with NULL before computing the remainder, so those +/// positions return NULL while others compute normally. fn try_rem( - left: &arrow::array::ArrayRef, - right: &arrow::array::ArrayRef, + left: &ArrayRef, + right: &ArrayRef, enable_ansi_mode: bool, -) -> Result { +) -> Result { + let divisor_is_zero = is_zero(right)?; + // Null out zero divisors so that the remainder kernels never see one: + // division by zero then returns NULL instead of erroring (integers) or + // returning NaN (floats). ANSI mode reports the error itself below, so + // this substitution is harmless on rows that must raise. + let null = Scalar::new(new_null_array(right.data_type(), 1)); + let safe_right = zip(&divisor_is_zero, &null, right)?; if enable_ansi_mode { - Ok(rem(left, right)?) - } else { - // In legacy mode, null out zero divisors so that division by zero - // returns NULL instead of erroring (integers) or returning NaN (floats). - let zero = ScalarValue::new_zero(right.data_type())?.to_array()?; - let zero = Scalar::new(zero); - let null = Scalar::new(new_null_array(right.data_type(), 1)); - let is_zero = eq(right, &zero)?; - let safe_right = zip(&is_zero, &null, right)?; - Ok(rem(left, &safe_right)?) + // Spark's remainder expressions are null intolerant, so a row whose + // dividend is NULL evaluates to NULL and never raises, even when the + // divisor on that row is zero. Mask the check by the validity of the + // dividend to match. + let raises = and(&divisor_is_zero, &is_not_null(left.as_ref())?)?; + if raises.iter().flatten().any(|raises| raises) { + return Err(ArrowError::DivideByZero.into()); + } } + Ok(rem(left, &safe_right)?) } /// Spark-compatible `mod` function @@ -418,6 +454,101 @@ mod test { assert!(result.is_err()); } + #[test] + fn test_mod_zero_division_ansi_float() { + // In ANSI mode a zero divisor of any numeric type must raise, + // including floating point, where Arrow's `rem` follows IEEE 754 + // and quietly returns NaN (#23894) + let left = Float64Array::from(vec![Some(10.5), Some(7.2)]); + let right = Float64Array::from(vec![Some(0.0), Some(2.0)]); + + let left_value = ColumnarValue::Array(Arc::new(left)); + let right_value = ColumnarValue::Array(Arc::new(right)); + + let result = spark_mod(&[left_value, right_value], true); + assert!(result.is_err()); + } + + #[test] + fn test_mod_negative_zero_divisor_legacy() { + // `-0.0` counts as a zero divisor, so it returns NULL in legacy + // mode rather than NaN (#23894) + let left = Float64Array::from(vec![Some(10.5), Some(7.5)]); + let right = Float64Array::from(vec![Some(-0.0), Some(2.0)]); + + let left_value = ColumnarValue::Array(Arc::new(left)); + let right_value = ColumnarValue::Array(Arc::new(right)); + + let result = spark_mod(&[left_value, right_value], false).unwrap(); + + if let ColumnarValue::Array(result_array) = result { + let result_float64 = result_array + .as_any() + .downcast_ref::() + .unwrap(); + assert!(result_float64.is_null(0)); // 10.5 % -0.0 = NULL + assert_eq!(result_float64.value(1), 1.5); // 7.5 % 2.0 = 1.5 + } else { + panic!("Expected array result"); + } + } + + #[test] + fn test_mod_negative_zero_divisor_ansi() { + // `-0.0` counts as a zero divisor, so it raises in ANSI mode (#23894) + let left = Float64Array::from(vec![Some(10.5)]); + let right = Float64Array::from(vec![Some(-0.0)]); + + let left_value = ColumnarValue::Array(Arc::new(left)); + let right_value = ColumnarValue::Array(Arc::new(right)); + + let result = spark_mod(&[left_value, right_value], true); + assert!(result.is_err()); + } + + #[test] + fn test_mod_zero_division_ansi_null_dividend() { + // Spark's remainder expressions are null intolerant: a NULL dividend + // short-circuits to NULL before the divisor is validated, so a zero + // divisor on such a row must not raise, even in ANSI mode (#23894) + let left = Int32Array::from(vec![None, Some(10)]); + let right = Int32Array::from(vec![Some(0), Some(3)]); + + let left_value = ColumnarValue::Array(Arc::new(left)); + let right_value = ColumnarValue::Array(Arc::new(right)); + + let result = spark_mod(&[left_value, right_value], true).unwrap(); + + if let ColumnarValue::Array(result_array) = result { + let result_int32 = + result_array.as_any().downcast_ref::().unwrap(); + assert!(result_int32.is_null(0)); // NULL % 0 = NULL (no error) + assert_eq!(result_int32.value(1), 1); // 10 % 3 = 1 + } else { + panic!("Expected array result"); + } + + // Same for floating point + let left = Float64Array::from(vec![None, Some(10.5)]); + let right = Float64Array::from(vec![Some(0.0), Some(2.0)]); + + let left_value = ColumnarValue::Array(Arc::new(left)); + let right_value = ColumnarValue::Array(Arc::new(right)); + + let result = spark_mod(&[left_value, right_value], true).unwrap(); + + if let ColumnarValue::Array(result_array) = result { + let result_float64 = result_array + .as_any() + .downcast_ref::() + .unwrap(); + assert!(result_float64.is_null(0)); // NULL % 0.0 = NULL (no error) + assert!((result_float64.value(1) - 0.5).abs() < f64::EPSILON); // 10.5 % 2.0 = 0.5 + } else { + panic!("Expected array result"); + } + } + // PMOD tests #[test] fn test_pmod_int32() { @@ -645,6 +776,102 @@ mod test { assert!(result.is_err()); } + #[test] + fn test_pmod_zero_division_ansi_float() { + // pmod routes through `try_rem` twice, so it needs the same coverage + // as mod: in ANSI mode a zero divisor of any numeric type must + // raise, including floating point, where Arrow's `rem` follows + // IEEE 754 and quietly returns NaN (#23894) + let left = Float64Array::from(vec![Some(10.5), Some(7.2)]); + let right = Float64Array::from(vec![Some(0.0), Some(2.0)]); + + let left_value = ColumnarValue::Array(Arc::new(left)); + let right_value = ColumnarValue::Array(Arc::new(right)); + + let result = spark_pmod(&[left_value, right_value], true); + assert!(result.is_err()); + } + + #[test] + fn test_pmod_negative_zero_divisor_legacy() { + // `-0.0` counts as a zero divisor, so it returns NULL in legacy + // mode rather than NaN (#23894) + let left = Float64Array::from(vec![Some(10.5), Some(-7.5)]); + let right = Float64Array::from(vec![Some(-0.0), Some(2.0)]); + + let left_value = ColumnarValue::Array(Arc::new(left)); + let right_value = ColumnarValue::Array(Arc::new(right)); + + let result = spark_pmod(&[left_value, right_value], false).unwrap(); + + if let ColumnarValue::Array(result_array) = result { + let result_float64 = result_array + .as_any() + .downcast_ref::() + .unwrap(); + assert!(result_float64.is_null(0)); // 10.5 pmod -0.0 = NULL + assert!((result_float64.value(1) - 0.5).abs() < f64::EPSILON); // -7.5 pmod 2.0 = 0.5 + } else { + panic!("Expected array result"); + } + } + + #[test] + fn test_pmod_negative_zero_divisor_ansi() { + // `-0.0` counts as a zero divisor, so it raises in ANSI mode (#23894) + let left = Float64Array::from(vec![Some(10.5)]); + let right = Float64Array::from(vec![Some(-0.0)]); + + let left_value = ColumnarValue::Array(Arc::new(left)); + let right_value = ColumnarValue::Array(Arc::new(right)); + + let result = spark_pmod(&[left_value, right_value], true); + assert!(result.is_err()); + } + + #[test] + fn test_pmod_zero_division_ansi_null_dividend() { + // Spark's remainder expressions are null intolerant: a NULL dividend + // short-circuits to NULL before the divisor is validated, so a zero + // divisor on such a row must not raise, even in ANSI mode (#23894) + let left = Int32Array::from(vec![None, Some(10)]); + let right = Int32Array::from(vec![Some(0), Some(3)]); + + let left_value = ColumnarValue::Array(Arc::new(left)); + let right_value = ColumnarValue::Array(Arc::new(right)); + + let result = spark_pmod(&[left_value, right_value], true).unwrap(); + + if let ColumnarValue::Array(result_array) = result { + let result_int32 = + result_array.as_any().downcast_ref::().unwrap(); + assert!(result_int32.is_null(0)); // NULL pmod 0 = NULL (no error) + assert_eq!(result_int32.value(1), 1); // 10 pmod 3 = 1 + } else { + panic!("Expected array result"); + } + + // Same for floating point + let left = Float64Array::from(vec![None, Some(10.5)]); + let right = Float64Array::from(vec![Some(0.0), Some(2.0)]); + + let left_value = ColumnarValue::Array(Arc::new(left)); + let right_value = ColumnarValue::Array(Arc::new(right)); + + let result = spark_pmod(&[left_value, right_value], true).unwrap(); + + if let ColumnarValue::Array(result_array) = result { + let result_float64 = result_array + .as_any() + .downcast_ref::() + .unwrap(); + assert!(result_float64.is_null(0)); // NULL pmod 0.0 = NULL (no error) + assert!((result_float64.value(1) - 0.5).abs() < f64::EPSILON); // 10.5 pmod 2.0 = 0.5 + } else { + panic!("Expected array result"); + } + } + #[test] fn test_pmod_negative_divisor() { // PMOD with negative divisor should still work like regular mod diff --git a/datafusion/sqllogictest/test_files/spark/math/mod.slt b/datafusion/sqllogictest/test_files/spark/math/mod.slt index 8229bb065152d..b9a8499939841 100644 --- a/datafusion/sqllogictest/test_files/spark/math/mod.slt +++ b/datafusion/sqllogictest/test_files/spark/math/mod.slt @@ -160,6 +160,18 @@ SELECT MOD(10.5::float8, 0.0::float8) as mod_div_zero_float; ---- NULL +# A negative zero divisor counts as zero and returns NULL in legacy mode +query R +SELECT MOD(10.5::float8, -0.0::float8) as mod_div_zero_neg_float; +---- +NULL + +# A NULL dividend evaluates to NULL regardless of the divisor +query I +SELECT MOD(NULL::int, 0::int) as mod_null_dividend_legacy; +---- +NULL + # Division by zero errors in ANSI mode statement ok set datafusion.execution.enable_ansi_mode = true; @@ -170,6 +182,25 @@ SELECT MOD(10::int, 0::int); statement error DataFusion error: Arrow error: Divide by zero error SELECT MOD(-7::int, 0::int); +# A zero divisor of any numeric type raises, including floating point +statement error DataFusion error: Arrow error: Divide by zero error +SELECT MOD(10.5::float8, 0.0::float8); + +# A negative zero divisor counts as zero and raises +statement error DataFusion error: Arrow error: Divide by zero error +SELECT MOD(10.5::float8, -0.0::float8); + +# A NULL dividend short-circuits to NULL before the divisor is validated +query I +SELECT MOD(NULL::int, 0::int) as mod_null_dividend_ansi; +---- +NULL + +query R +SELECT MOD(NULL::float8, 0.0::float8) as mod_null_dividend_ansi_float; +---- +NULL + statement ok set datafusion.execution.enable_ansi_mode = false; diff --git a/datafusion/sqllogictest/test_files/spark/math/pmod.slt b/datafusion/sqllogictest/test_files/spark/math/pmod.slt index aa4a197ba470f..1165fdfba3f7c 100644 --- a/datafusion/sqllogictest/test_files/spark/math/pmod.slt +++ b/datafusion/sqllogictest/test_files/spark/math/pmod.slt @@ -74,6 +74,12 @@ SELECT pmod(-7::int, 0::int) as pmod_zero_3; ---- NULL +# A negative zero divisor counts as zero and returns NULL in legacy mode +query R +SELECT pmod(10.5::float8, -0.0::float8) as pmod_div_zero_neg_float; +---- +NULL + # Division by zero errors in ANSI mode statement ok set datafusion.execution.enable_ansi_mode = true; @@ -84,6 +90,25 @@ SELECT pmod(10::int, 0::int); statement error DataFusion error: Arrow error: Divide by zero error SELECT pmod(-7::int, 0::int); +# A zero divisor of any numeric type raises, including floating point +statement error DataFusion error: Arrow error: Divide by zero error +SELECT pmod(10.5::float8, 0.0::float8); + +# A negative zero divisor counts as zero and raises +statement error DataFusion error: Arrow error: Divide by zero error +SELECT pmod(10.5::float8, -0.0::float8); + +# A NULL dividend short-circuits to NULL before the divisor is validated +query I +SELECT pmod(NULL::int, 0::int) as pmod_null_dividend_ansi; +---- +NULL + +query R +SELECT pmod(NULL::float8, 0.0::float8) as pmod_null_dividend_ansi_float; +---- +NULL + statement ok set datafusion.execution.enable_ansi_mode = false; From d71efe49beac5b78e93c21383956f1c5607350d1 Mon Sep 17 00:00:00 2001 From: Nagato Yuzuru Date: Sun, 16 Aug 2026 02:31:43 +0000 Subject: [PATCH 45/50] fix: consider nulls_first when propagating ordered SortProperties (#24206) ## Which issue does this PR close? - Closes #11596. ## Rationale for this change In the original implementation, cases where `null_first` is not same were not correctly handled and were still treated as `ordered`. Example: `a` sorted `ASC NULLS FIRST`, `b` sorted `ASC NULLS LAST`: | a | b | a + b | | ---- | ---- | ----- | | NULL | 1 | NULL | | 1 | 2 | 3 | | 2 | 4 | 6 | | 3 | NULL | NULL | Since `a + b` is `NULL` wherever either input is, the result has nulls at both ends. The previous code returned `Ordered(ASC, nulls_first: true)` for this case; it now returns `Unordered`. ## What changes are included in this PR? Unit Tests were added and issues were fixed. Additional processing was performed on `and_or` (see https://github.com/apache/datafusion/issues/11596#issuecomment-5232788766 ) SortProperties::{add, sub, gt_or_gteq, and_or} now propagate an Ordered result only when both operands agree on nulls_first; otherwise they return Unordered. This is conservative. If one input were known to be non-null the old result could be valid. But SortProperties carries no nullability information, so Unordered is the only sound answer at this layer. ## Are these changes tested? Yes. ## Are there any user-facing changes? No. --- Cargo.lock | 1 + datafusion/expr-common/Cargo.toml | 1 + datafusion/expr-common/src/sort_properties.rs | 267 +++++++++++++++++- datafusion/sqllogictest/test_files/order.slt | 84 +++++- 4 files changed, 344 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7576659f0620..cca0d1fdbfb76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2196,6 +2196,7 @@ dependencies = [ "indexmap 2.14.0", "insta", "itertools 0.15.0", + "rstest", ] [[package]] diff --git a/datafusion/expr-common/Cargo.toml b/datafusion/expr-common/Cargo.toml index 072c8f14da503..026c1e0bc316b 100644 --- a/datafusion/expr-common/Cargo.toml +++ b/datafusion/expr-common/Cargo.toml @@ -48,3 +48,4 @@ itertools = { workspace = true } [dev-dependencies] insta = { workspace = true } +rstest = { workspace = true } diff --git a/datafusion/expr-common/src/sort_properties.rs b/datafusion/expr-common/src/sort_properties.rs index 74d644f79faef..6f18a8a46a33f 100644 --- a/datafusion/expr-common/src/sort_properties.rs +++ b/datafusion/expr-common/src/sort_properties.rs @@ -50,11 +50,12 @@ impl SortProperties { (Self::Singleton, _) => *rhs, (_, Self::Singleton) => *self, (Self::Ordered(lhs), Self::Ordered(rhs)) - if lhs.descending == rhs.descending => + if lhs.descending == rhs.descending + && lhs.nulls_first == rhs.nulls_first => { Self::Ordered(SortOptions { descending: lhs.descending, - nulls_first: lhs.nulls_first || rhs.nulls_first, + nulls_first: lhs.nulls_first, }) } _ => Self::Unordered, @@ -70,11 +71,12 @@ impl SortProperties { }), (_, Self::Singleton) => *self, (Self::Ordered(lhs), Self::Ordered(rhs)) - if lhs.descending != rhs.descending => + if lhs.descending != rhs.descending + && lhs.nulls_first == rhs.nulls_first => { Self::Ordered(SortOptions { descending: lhs.descending, - nulls_first: lhs.nulls_first || rhs.nulls_first, + nulls_first: lhs.nulls_first, }) } _ => Self::Unordered, @@ -89,7 +91,8 @@ impl SortProperties { }), (_, Self::Singleton) => *self, (Self::Ordered(lhs), Self::Ordered(rhs)) - if lhs.descending != rhs.descending => + if lhs.descending != rhs.descending + && lhs.nulls_first == rhs.nulls_first => { *self } @@ -100,11 +103,12 @@ impl SortProperties { pub fn and_or(&self, rhs: &Self) -> Self { match (self, rhs) { (Self::Ordered(lhs), Self::Ordered(rhs)) - if lhs.descending == rhs.descending => + if lhs.descending == rhs.descending + && lhs.nulls_first == rhs.nulls_first => { Self::Ordered(SortOptions { descending: lhs.descending, - nulls_first: lhs.nulls_first || rhs.nulls_first, + nulls_first: lhs.nulls_first, }) } (Self::Ordered(opt), Self::Singleton) @@ -118,6 +122,255 @@ impl SortProperties { } } +#[cfg(test)] +mod sort_properties_test { + use super::{SortOptions, SortProperties}; + use rstest::rstest; + + const fn ordered(descending: bool, nulls_first: bool) -> SortProperties { + SortProperties::Ordered(SortOptions { + descending, + nulls_first, + }) + } + + const ASC_NF: SortProperties = ordered(false, true); + const ASC_NL: SortProperties = ordered(false, false); + const DESC_NF: SortProperties = ordered(true, true); + const DESC_NL: SortProperties = ordered(true, false); + const UNORDERED: SortProperties = SortProperties::Unordered; + const SINGLETON: SortProperties = SortProperties::Singleton; + + type BinOp = fn(&SortProperties, &SortProperties) -> SortProperties; + + /// Each method's direction rule and its `Singleton` arms. + /// + /// Operands that *disagree* on null placement are deliberately absent: + /// that half is covered exhaustively by + /// [`conflicting_null_placement_is_never_ordered`]. + #[test] + fn ordering_propagation() { + let cases: &[(&str, BinOp, SortProperties, SortProperties, SortProperties)] = &[ + // `add` preserves ordering when both operands run in the same + // direction. It is commutative, so one argument order suffices. + ( + "add: same direction is preserved", + SortProperties::add, + ASC_NF, + ASC_NF, + ASC_NF, + ), + ( + "add: nulls_last placement is preserved", + SortProperties::add, + ASC_NL, + ASC_NL, + ASC_NL, + ), + ( + "add: opposing directions are unordered", + SortProperties::add, + ASC_NF, + DESC_NF, + UNORDERED, + ), + ( + "add: literal with ordered", + SortProperties::add, + SINGLETON, + ASC_NF, + ASC_NF, + ), + ( + "add: two literals stay a literal", + SortProperties::add, + SINGLETON, + SINGLETON, + SINGLETON, + ), + // `and_or` backs both `AND` and `OR`, whose rules coincide. Same + // shape as `add`, and likewise commutative. + ( + "and_or: same direction is preserved", + SortProperties::and_or, + ASC_NF, + ASC_NF, + ASC_NF, + ), + ( + "and_or: nulls_last placement is preserved", + SortProperties::and_or, + DESC_NL, + DESC_NL, + DESC_NL, + ), + ( + "and_or: opposing directions are unordered", + SortProperties::and_or, + ASC_NF, + DESC_NF, + UNORDERED, + ), + ( + "and_or: literal with ordered", + SortProperties::and_or, + SINGLETON, + ASC_NF, + ASC_NF, + ), + ( + "and_or: two literals stay a literal", + SortProperties::and_or, + SINGLETON, + SINGLETON, + SINGLETON, + ), + // `sub` needs the *opposite* rule: an ascending column minus a + // descending one still ascends. It is not commutative + ( + "sub: opposing directions are preserved", + SortProperties::sub, + ASC_NF, + DESC_NF, + ASC_NF, + ), + ( + "sub: result follows the left operand", + SortProperties::sub, + DESC_NF, + ASC_NF, + DESC_NF, + ), + ( + "sub: nulls_last placement is preserved", + SortProperties::sub, + ASC_NL, + DESC_NL, + ASC_NL, + ), + ( + "sub: same direction is unordered", + SortProperties::sub, + ASC_NF, + ASC_NF, + UNORDERED, + ), + ( + "sub: literal minus ordered flips the direction", + SortProperties::sub, + SINGLETON, + ASC_NF, + DESC_NF, + ), + ( + "sub: ordered minus literal keeps the direction", + SortProperties::sub, + ASC_NF, + SINGLETON, + ASC_NF, + ), + ( + "sub: two literals stay a literal", + SortProperties::sub, + SINGLETON, + SINGLETON, + SINGLETON, + ), + // `gt_or_gteq` compares into a boolean column, which is ordered by + // `false < true`. Same direction rule as `sub`, also asymmetric. + ( + "gt_or_gteq: opposing directions are preserved", + SortProperties::gt_or_gteq, + ASC_NF, + DESC_NF, + ASC_NF, + ), + ( + "gt_or_gteq: result follows the left operand", + SortProperties::gt_or_gteq, + DESC_NF, + ASC_NF, + DESC_NF, + ), + ( + "gt_or_gteq: nulls_last placement is preserved", + SortProperties::gt_or_gteq, + DESC_NL, + ASC_NL, + DESC_NL, + ), + ( + "gt_or_gteq: same direction is unordered", + SortProperties::gt_or_gteq, + ASC_NF, + ASC_NF, + UNORDERED, + ), + ( + "gt_or_gteq: literal on the left flips the direction", + SortProperties::gt_or_gteq, + SINGLETON, + ASC_NF, + DESC_NF, + ), + ( + "gt_or_gteq: literal on the right keeps the direction", + SortProperties::gt_or_gteq, + ASC_NF, + SINGLETON, + ASC_NF, + ), + ( + "gt_or_gteq: two literals stay a literal", + SortProperties::gt_or_gteq, + SINGLETON, + SINGLETON, + SINGLETON, + ), + ]; + + for &(name, op, lhs, rhs, expected) in cases { + assert_eq!(op(&lhs, &rhs), expected, "case: {name}"); + } + + // `add` and `and_or` are commutative, which is what lets the table + // above cover only one argument order for them. + for (lhs, rhs) in [(ASC_NF, DESC_NL), (ASC_NF, SINGLETON), (ASC_NL, DESC_NF)] { + assert_eq!(lhs.add(&rhs), rhs.add(&lhs), "add is commutative"); + assert_eq!(lhs.and_or(&rhs), rhs.and_or(&lhs), "and_or is commutative"); + } + } + + /// If two ordered operands disagree on null placement, the result is + /// always Unordered, no matter which operator or direction is used. + /// Checked below for every combination. + /// + /// Nulls propagate: the result is null wherever either operand is + /// null. `nulls_first` treats those rows as a prefix, `nulls_last` as + /// a suffix. A set that's both can't be described by any `SortOptions`. + /// + /// The assertion only checks "not Ordered", not which ordering + /// results. That keeps the test from just repeating the logic it's + /// checking. + #[rstest] + #[case::add("add", SortProperties::add)] + #[case::sub("sub", SortProperties::sub)] + #[case::gt_or_gteq("gt_or_gteq", SortProperties::gt_or_gteq)] + #[case::and_or("and_or", SortProperties::and_or)] + fn conflicting_null_placement_is_never_ordered( + #[values(false, true)] l_descending: bool, + #[values(false, true)] r_descending: bool, + #[values(false, true)] l_nulls_first: bool, + #[case] op_name: &str, + #[case] op: BinOp, + ) { + // Negating `l_nulls_first` makes the operands disagree by construction. + let lhs = ordered(l_descending, l_nulls_first); + let rhs = ordered(r_descending, !l_nulls_first); + assert_eq!(op(&lhs, &rhs), UNORDERED, "{op_name}: {lhs:?} and {rhs:?}"); + } +} + impl Neg for SortProperties { type Output = Self; diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt index 4b136d24b0751..dfe5c0faab714 100644 --- a/datafusion/sqllogictest/test_files/order.slt +++ b/datafusion/sqllogictest/test_files/order.slt @@ -1485,7 +1485,9 @@ physical_plan 03)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1, maintains_sort_order=true 04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true -# Boolean to integer casts preserve the order. +# The cast preserves the order, but the comparison does not: `inc_col` is +# ASC NULLS LAST while `desc_col` is DESC NULLS FIRST, so `inc_col > desc_col` +# is null at both ends and the sort has to stay. statement ok CREATE EXTERNAL TABLE annotated_data_finite ( ts INTEGER, @@ -1507,9 +1509,87 @@ logical_plan 03)----TableScan: annotated_data_finite projection=[inc_col, desc_col] physical_plan 01)SortPreservingMergeExec: [c@0 ASC NULLS LAST] +02)--SortExec: expr=[c@0 ASC NULLS LAST], preserve_partitioning=[true] +03)----ProjectionExec: expr=[CAST(inc_col@0 > desc_col@1 AS Int32) as c] +04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1, maintains_sort_order=true +05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_1.csv]]}, projection=[inc_col, desc_col], output_orderings=[[inc_col@0 ASC NULLS LAST], [desc_col@1 DESC]], file_type=csv, has_header=true + +# With matching null placement the comparison keeps its order: no sort needed. +statement ok +CREATE EXTERNAL TABLE annotated_data_finite_nulls_last ( + ts INTEGER, + inc_col INTEGER, + desc_col INTEGER, +) +STORED AS CSV +WITH ORDER (inc_col ASC NULLS LAST) +WITH ORDER (desc_col DESC NULLS LAST) +LOCATION '../core/tests/data/window_1.csv' +OPTIONS ('format.has_header' 'true'); + +query TT +EXPLAIN SELECT CAST((inc_col>desc_col) as integer) as c from annotated_data_finite_nulls_last order by c; +---- +logical_plan +01)Sort: c ASC NULLS LAST +02)--Projection: CAST(annotated_data_finite_nulls_last.inc_col > annotated_data_finite_nulls_last.desc_col AS Int32) AS c +03)----TableScan: annotated_data_finite_nulls_last projection=[inc_col, desc_col] +physical_plan +01)SortPreservingMergeExec: [c@0 ASC NULLS LAST] 02)--ProjectionExec: expr=[CAST(inc_col@0 > desc_col@1 AS Int32) as c] 03)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1, maintains_sort_order=true -04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_1.csv]]}, projection=[inc_col, desc_col], output_orderings=[[inc_col@0 ASC NULLS LAST], [desc_col@1 DESC]], file_type=csv, has_header=true +04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_1.csv]]}, projection=[inc_col, desc_col], output_orderings=[[inc_col@0 ASC NULLS LAST], [desc_col@1 DESC NULLS LAST]], file_type=csv, has_header=true + +# Regression test for #11596. `a` is ASC NULLS FIRST and `b` is ASC NULLS LAST, +# so `a + b` is null at both ends of the scan: (NULL, 3, 6, NULL). No nulls +# placement describes that. +query I +COPY (VALUES (NULL, 1), (1, 2), (2, 4), (3, NULL)) +TO 'test_files/scratch/order/mixed_null_placement.csv' +OPTIONS ('format.has_header' 'true'); +---- +4 + +statement ok +CREATE EXTERNAL TABLE mixed_null_placement ( + a BIGINT, + b BIGINT +) +STORED AS CSV +WITH ORDER (a ASC NULLS FIRST) +WITH ORDER (b ASC NULLS LAST) +LOCATION 'test_files/scratch/order/mixed_null_placement.csv' +OPTIONS ('format.has_header' 'true'); + +# The sort has to happen. Without it, this comes back in scan order. +query I +SELECT a + b AS s FROM mixed_null_placement ORDER BY s; +---- +3 +6 +NULL +NULL + +# Same at the plan level. Two separate things keep this sort: `add()` won't +# merge mismatched `nulls_first`, and `arithmetic_sort_properties` gives up on +# any `col + col` with unbounded ranges, which is every column read from a +# file. The second masks the first today. +query TT +EXPLAIN SELECT a + b AS s FROM mixed_null_placement ORDER BY s; +---- +logical_plan +01)Sort: s ASC NULLS LAST +02)--Projection: mixed_null_placement.a + mixed_null_placement.b AS s +03)----TableScan: mixed_null_placement projection=[a, b] +physical_plan +01)SortPreservingMergeExec: [s@0 ASC NULLS LAST] +02)--SortExec: expr=[s@0 ASC NULLS LAST], preserve_partitioning=[true] +03)----ProjectionExec: expr=[a@0 + b@1 as s] +04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1, maintains_sort_order=true +05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/order/mixed_null_placement.csv]]}, projection=[a, b], output_orderings=[[a@0 ASC], [b@1 ASC NULLS LAST]], file_type=csv, has_header=true + +statement ok +DROP TABLE mixed_null_placement; # Union a query with the actual data and one with a constant query I From 17c1694bfb0d3957f0fb7913cc8aca82dbe1ed07 Mon Sep 17 00:00:00 2001 From: RIchard Baah <137434454+Rich-T-kid@users.noreply.github.com> Date: Sun, 16 Aug 2026 02:55:39 +0000 Subject: [PATCH 46/50] Feat: add dictionaries as a supported group column type (#23187) ## Which issue does this PR close? - works towards closing #22682. - replacement for https://github.com/apache/datafusion/pull/21765 - - https://github.com/apache/datafusion/pull/21765#pullrequestreview-4546759419 ## Rationale for this change This PR introduces a specialized `GroupColumn` implementation for dictionary-typed columns inside `GroupValuesColumn`, allowing dictionary columns to participate in the columnar, vectorized aggregation path instead of the row-based fallback. **The Implementation is only about 175+ lines of code**. the remaining LOC is adding extensive test at the `GroupColumn` trait level as well as testing the `GroupValuesColumn` GroupValues trait and how it inter-opts with multi-dictionary group by's. ## What changes are included in this PR? - Adds a `DictionaryGroupValueBuilder` struct implementing the `GroupColumn` trait for `Dictionary`-typed group-by columns, supporting a configurable subset of value types - Extends the type-check gate in `GroupValuesColumn::try_new` (the `matches!` block) to accept `Dictionary(_, value_type)` where `value_type` is already supported. - Adds schema-level support so emitted dictionary group key columns round-trip through the output schema correctly - [removes casting ](https://github.com/apache/datafusion/blob/9e8dd76d6deb6736c51962d9c97e04be4e3f1fc9/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs#L1200)thats done for each dictionary array in `emit` ## Are these changes tested? yes. a majority of this PR is test ## Are there any user-facing changes? no. this is a pure perf boost for users. --- .../group_values/multi_group_by/dictionary.rs | 841 ++++++++++++++++++ .../group_values/multi_group_by/mod.rs | 113 ++- 2 files changed, 936 insertions(+), 18 deletions(-) create mode 100644 datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs new file mode 100644 index 0000000000000..501b13d0cd183 --- /dev/null +++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs @@ -0,0 +1,841 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::aggregates::group_values::multi_group_by::GroupColumn; +use arrow::array::{ + Array, ArrayRef, AsArray, BooleanBufferBuilder, DictionaryArray, Int64Array, + PrimitiveArray, +}; +use arrow::compute::take; +use arrow::datatypes::{ArrowDictionaryKeyType, ArrowNativeType, DataType, Field}; +use arrow::error::ArrowError; +use datafusion_common::hash_utils::{RandomState, create_hashes}; +use datafusion_common::{DataFusionError, Result, exec_err}; +use datafusion_execution::memory_pool::proxy::HashTableAllocExt; +use hashbrown::hash_table::HashTable; +use std::marker::PhantomData; +use std::mem::size_of; +use std::sync::Arc; + +use crate::aggregates::AGGREGATION_HASH_SEED; + +/// [`GroupColumn`] for dictionary-encoded columns with key type `K`. +/// +/// `inner` holds one slot per distinct value seen across all batches. +/// `group_to_inner[group_idx]` maps each group to its slot in `inner`, +/// so groups with the same value share a slot rather than duplicating data. +pub struct DictionaryGroupValuesColumn { + /// Deduplicated store of distinct values. + inner: Box, + /// Unary null array (length 1) reused for every null appended to `inner`. + null_array: ArrayRef, + /// Maps each group index to its slot in `inner`. + group_to_inner: Vec, + /// Lookup table mapping `(value_hash, inner_slot)` for each non-null distinct value. + value_dedup: HashTable<(u64, usize)>, + /// Tracked allocation size of `value_dedup` for memory accounting via `size()`. + value_dedup_size: usize, + /// Slot in `inner` for the null group; `None` until the first null is seen. + null_inner_slot: Option, + /// Hash seed — must match `create_hashes` so hashes are consistent across calls. + random_state: RandomState, + /// Reusable scratch buffer mapping `val_idx → inner_slot` across batches. + val_to_inner: Vec, + /// Reusable hash buffer for the dictionary values array. + val_hashes: Vec, + _phantom: PhantomData, +} + +impl DictionaryGroupValuesColumn { + pub fn new(inner: Box, field: &Field) -> Self { + let null_array = arrow::array::new_null_array(field.data_type(), 1); + Self { + inner, + null_array, + group_to_inner: Vec::new(), + value_dedup: HashTable::new(), + value_dedup_size: 0, + null_inner_slot: None, + random_state: AGGREGATION_HASH_SEED, + val_to_inner: Vec::default(), + val_hashes: Vec::default(), + _phantom: PhantomData, + } + } + + /// Build a `DictionaryArray` from `values` (all inner slots) and the + /// per-group slot mapping. The null inner slot, if any, is excluded from + /// the values array and its groups emit a null key — so it never consumes + /// a key index regardless of where it sits in `inner`. + fn into_dict( + values: ArrayRef, + group_to_inner: &[usize], + null_inner_slot: Option, + ) -> ArrayRef { + let Some(null_slot) = null_inner_slot else { + // Fast path: no null group — raw slot indices are valid keys. + let keys: PrimitiveArray = group_to_inner + .iter() + .map(|&slot| Some(K::Native::usize_as(slot))) + .collect(); + return Arc::new(DictionaryArray::::new(keys, values)); + }; + + // Build a compact remap: each non-null slot gets a contiguous key + // starting from 0; the null slot is skipped entirely. + let n = values.len(); + let mut remap = vec![0usize; n]; + let mut next = 0usize; + for (i, mapped) in remap.iter_mut().enumerate() { + if i != null_slot { + *mapped = next; + next += 1; + } + } + + let keys: PrimitiveArray = group_to_inner + .iter() + .map(|&slot| { + if slot == null_slot { + None + } else { + Some(K::Native::usize_as(remap[slot])) + } + }) + .collect(); + + // Compact values array: drop the null slot so key indices stay tight. + let compact_indices: Int64Array = (0..n) + .filter(|&i| i != null_slot) + .map(|i| i as i64) + .collect(); + let compact = + take(&*values, &compact_indices, None).expect("compact values in into_dict"); + Arc::new(DictionaryArray::::new(keys, compact)) + } + + // https://github.com/apache/datafusion/issues/23127 + // Null groups emit a null key (None), not a slot index, so the null inner + // slot never consumes a key index regardless of its position in inner. + fn check_key_overflow(&self) -> Result<()> { + let non_null_count = self.inner.len() - self.null_inner_slot.is_some() as usize; + if !Self::key_type_fits(non_null_count) { + return exec_err!( + "Dictionary key type {:?} cannot represent {} distinct values", + K::DATA_TYPE, + non_null_count + ); + } + Ok(()) + } + + fn key_type_fits(num_values: usize) -> bool { + let max: usize = match K::DATA_TYPE { + DataType::Int8 => i8::MAX as usize, + DataType::Int16 => i16::MAX as usize, + DataType::Int32 => i32::MAX as usize, + DataType::Int64 => i64::MAX as usize, + DataType::UInt8 => u8::MAX as usize, + DataType::UInt16 => u16::MAX as usize, + DataType::UInt32 => u32::MAX as usize, + DataType::UInt64 => usize::MAX, + _ => return false, + }; + num_values == 0 || num_values - 1 <= max + } + + fn hash_values(&mut self, values: &ArrayRef) { + self.val_hashes.clear(); + self.val_hashes.resize(values.len(), 0); + create_hashes( + std::slice::from_ref(values), + &self.random_state, + &mut self.val_hashes, + ) + .unwrap(); + } + + fn find_or_insert_value( + &mut self, + dict_values: &ArrayRef, + val_idx: usize, + hash: u64, + ) -> Result { + let inner = &*self.inner; + let existing = self + .value_dedup + .find(hash, |&(entry_hash, slot)| { + entry_hash == hash && inner.equal_to(slot, dict_values, val_idx) + }) + .map(|&(_, slot)| slot); + + match existing { + Some(slot) => Ok(slot), + None => { + let slot = self.inner.len(); + self.inner.append_val(dict_values, val_idx)?; + self.value_dedup.insert_accounted( + (hash, slot), + |&(entry_hash, _)| entry_hash, + &mut self.value_dedup_size, + ); + Ok(slot) + } + } + } + + fn find_or_insert_null(&mut self) -> Result { + if let Some(slot) = self.null_inner_slot { + return Ok(slot); + } + let slot = self.inner.len(); + self.inner.append_val(&self.null_array, 0)?; + self.null_inner_slot = Some(slot); + Ok(slot) + } + + fn build_lookup_table( + &self, + dict_values: &ArrayRef, + val_hashes: &[u64], + ) -> Vec { + let num_distinct = dict_values.len(); + let mut table = vec![usize::MAX; num_distinct + 1]; + let inner = &*self.inner; + for val_idx in 0..num_distinct { + if dict_values.is_null(val_idx) { + table[val_idx] = self.null_inner_slot.unwrap_or(usize::MAX); + } else { + let hash = val_hashes[val_idx]; + if let Some(&(_, slot)) = + self.value_dedup.find(hash, |&(entry_hash, slot)| { + entry_hash == hash && inner.equal_to(slot, dict_values, val_idx) + }) + { + table[val_idx] = slot; + } + } + } + table[num_distinct] = self.null_inner_slot.unwrap_or(usize::MAX); + table + } + + /// Per-row fallback for `vectorized_equal_to` used when the number of rows + /// to check is smaller than the dictionary cardinality, making the O(D) + /// lookup-table build more expensive than direct value comparison. + /// + /// `#[cold]` + `#[inline(never)]` keeps this code out of the hot + /// lookup-table loops in `vectorized_equal_to` so LLVM can pipeline them. + #[cold] + #[inline(never)] + fn equal_to_per_row( + &self, + lhs_rows: &[usize], + dict_values: &ArrayRef, + dict: &DictionaryArray, + rhs_rows: &[usize], + equal_to_results: &mut BooleanBufferBuilder, + ) { + let group_to_inner = self.group_to_inner.as_slice(); + for (idx, (&lhs_row, &rhs_row)) in + lhs_rows.iter().zip(rhs_rows.iter()).enumerate() + { + if !equal_to_results.get_bit(idx) { + continue; + } + let lhs_slot = group_to_inner[lhs_row]; + let equal = match dict.key(rhs_row) { + None => self.inner.equal_to(lhs_slot, &self.null_array, 0), + Some(val_idx) if dict_values.is_null(val_idx) => { + self.inner.equal_to(lhs_slot, &self.null_array, 0) + } + Some(val_idx) => self.inner.equal_to(lhs_slot, dict_values, val_idx), + }; + if !equal { + equal_to_results.set_bit(idx, false); + } + } + } +} + +impl GroupColumn + for DictionaryGroupValuesColumn +{ + fn equal_to(&self, lhs_row: usize, array: &ArrayRef, rhs_row: usize) -> bool { + let lhs_slot = self.group_to_inner[lhs_row]; + let dict = array.as_dictionary::(); + match dict.key(rhs_row) { + None => self.inner.equal_to(lhs_slot, &self.null_array, 0), + Some(val_idx) if dict.values().is_null(val_idx) => { + self.inner.equal_to(lhs_slot, &self.null_array, 0) + } + Some(val_idx) => self.inner.equal_to(lhs_slot, dict.values(), val_idx), + } + } + + fn append_val(&mut self, array: &ArrayRef, row: usize) -> Result<()> { + let dict = array.as_dictionary::(); + let inner_slot = match dict.key(row) { + None => self.find_or_insert_null()?, + Some(val_idx) if dict.values().is_null(val_idx) => { + self.find_or_insert_null()? + } + Some(val_idx) => { + let dict_values = dict.values(); + let single = dict_values.slice(val_idx, 1); + self.hash_values(&single); + self.find_or_insert_value(dict_values, val_idx, self.val_hashes[0])? + } + }; + self.group_to_inner.push(inner_slot); + self.check_key_overflow() + } + + fn vectorized_equal_to( + &self, + lhs_rows: &[usize], + array: &ArrayRef, + rhs_rows: &[usize], + equal_to_results: &mut BooleanBufferBuilder, + ) { + let dict = array.as_dictionary::(); + let dict_keys = dict.keys(); + let dict_values = dict.values(); + let num_distinct = dict_values.len(); + + // The fallback is in a separate #[cold] function so its code does not + // appear inline here and cannot prevent LLVM from pipelining / unrolling + // the hot lookup-table loops below. + if rhs_rows.len() < num_distinct { + self.equal_to_per_row( + lhs_rows, + dict_values, + dict, + rhs_rows, + equal_to_results, + ); + return; + } + + let mut val_hashes = vec![0u64; dict_values.len()]; + create_hashes( + std::slice::from_ref(dict_values), + &self.random_state, + &mut val_hashes, + ) + .unwrap(); + let lookup = self.build_lookup_table(dict_values, &val_hashes); + + let group_to_inner = self.group_to_inner.as_slice(); + + if dict_keys.null_count() == 0 { + // No null keys : skip the get_bit guard: we only ever write false, + // so overwriting an already-false bit is a no-op. + let raw_keys = dict_keys.values(); + for (idx, (&lhs_row, &rhs_row)) in + lhs_rows.iter().zip(rhs_rows.iter()).enumerate() + { + let rhs_slot = lookup[raw_keys[rhs_row].as_usize()]; + if rhs_slot == usize::MAX || group_to_inner[lhs_row] != rhs_slot { + equal_to_results.set_bit(idx, false); + } + } + } else { + let null_buf = dict_keys.nulls().unwrap(); + let raw_keys = dict_keys.values(); + for (idx, (&lhs_row, &rhs_row)) in + lhs_rows.iter().zip(rhs_rows.iter()).enumerate() + { + if equal_to_results.get_bit(idx) { + let val_idx = if null_buf.is_null(rhs_row) { + num_distinct + } else { + raw_keys[rhs_row].as_usize() + }; + let rhs_slot = lookup[val_idx]; + if rhs_slot == usize::MAX || group_to_inner[lhs_row] != rhs_slot { + equal_to_results.set_bit(idx, false); + } + } + } + } + } + + fn vectorized_append(&mut self, array: &ArrayRef, rows: &[usize]) -> Result<()> { + let dict = array.as_dictionary::(); + let dict_keys = dict.keys(); + let dict_values = dict.values(); + let num_distinct = dict_values.len(); + + self.hash_values(dict_values); + self.val_to_inner.clear(); + self.val_to_inner.resize(num_distinct, usize::MAX); + + self.group_to_inner.try_reserve(rows.len()).map_err(|e| { + DataFusionError::ArrowError( + Box::new(ArrowError::MemoryError(e.to_string())), + None, + ) + })?; + + if dict_keys.null_count() == 0 { + let raw_keys = dict_keys.values(); + for &row in rows { + let val_idx = raw_keys[row].as_usize(); + if self.val_to_inner[val_idx] == usize::MAX { + // A non-null key can still point to a null value in the values array. + self.val_to_inner[val_idx] = if dict_values.is_null(val_idx) { + self.find_or_insert_null()? + } else { + self.find_or_insert_value( + dict_values, + val_idx, + self.val_hashes[val_idx], + )? + }; + } + self.group_to_inner.push(self.val_to_inner[val_idx]); + } + } else { + let raw_keys = dict_keys.values(); + let null_buf = dict_keys.nulls().unwrap(); + for &row in rows { + let slot = if null_buf.is_null(row) { + self.find_or_insert_null()? + } else { + let val_idx = raw_keys[row].as_usize(); + if self.val_to_inner[val_idx] == usize::MAX { + self.val_to_inner[val_idx] = if dict_values.is_null(val_idx) { + self.find_or_insert_null()? + } else { + self.find_or_insert_value( + dict_values, + val_idx, + self.val_hashes[val_idx], + )? + }; + } + self.val_to_inner[val_idx] + }; + self.group_to_inner.push(slot); + } + } + + self.check_key_overflow() + } + + fn len(&self) -> usize { + self.group_to_inner.len() + } + + fn size(&self) -> usize { + self.inner.size() + + self.value_dedup_size + + self.group_to_inner.capacity() * size_of::() + + self.val_to_inner.capacity() * size_of::() + + self.val_hashes.capacity() * size_of::() + + self.null_array.get_array_memory_size() + + size_of::() + } + + fn build(self: Box) -> ArrayRef { + let null_inner_slot = self.null_inner_slot; + let values = self.inner.build(); + Self::into_dict(values, &self.group_to_inner, null_inner_slot) + } + + fn take_n(&mut self, n: usize) -> ArrayRef { + let old_inner_len = self.inner.len(); + let all_inner_values = self.inner.take_n(old_inner_len); + + let mut emit_old_to_new = vec![usize::MAX; old_inner_len]; + let mut emit_new_to_old: Vec = Vec::new(); + for &old in &self.group_to_inner[..n] { + // Null groups emit a null key (None) and need no slot in the + // values array, so excluding them keeps key indices tight and + // prevents overflow at key-type capacity. + if all_inner_values.is_null(old) { + continue; + } + if emit_old_to_new[old] == usize::MAX { + emit_old_to_new[old] = emit_new_to_old.len(); + emit_new_to_old.push(old); + } + } + let emit_indices = + Int64Array::from_iter(emit_new_to_old.iter().map(|&i| i as i64)); + let compact_emit_values = + take(&*all_inner_values, &emit_indices, None).expect("take emit values"); + let emitted_keys: PrimitiveArray = self.group_to_inner[..n] + .iter() + .map(|&old| { + if all_inner_values.is_null(old) { + None + } else { + Some(K::Native::usize_as(emit_old_to_new[old])) + } + }) + .collect(); + let emitted: ArrayRef = + Arc::new(DictionaryArray::::new(emitted_keys, compact_emit_values)); + + // Null deferred to last so null_inner_slot is always the highest index + // and check_key_overflow can subtract it without a false overflow. + let remaining = self.group_to_inner[n..].to_vec(); + let mut old_to_new = vec![usize::MAX; old_inner_len]; + let mut new_to_old: Vec = Vec::new(); + let mut null_old_slot: Option = None; + for &old in &remaining { + if all_inner_values.is_null(old) { + if null_old_slot.is_none() { + null_old_slot = Some(old); + } + continue; + } + if old_to_new[old] == usize::MAX { + old_to_new[old] = new_to_old.len(); + new_to_old.push(old); + } + } + if let Some(old) = null_old_slot { + old_to_new[old] = new_to_old.len(); + new_to_old.push(old); + } + + self.value_dedup = HashTable::new(); + self.value_dedup_size = 0; + self.null_inner_slot = None; + + self.hash_values(&all_inner_values); + + for (new_slot, &old_slot) in new_to_old.iter().enumerate() { + if all_inner_values.is_null(old_slot) { + self.inner + .append_val(&self.null_array, 0) + .expect("append null failed in take_n"); + self.null_inner_slot = Some(new_slot); + } else { + self.inner + .append_val(&all_inner_values, old_slot) + .expect("append value failed in take_n"); + self.value_dedup.insert_accounted( + (self.val_hashes[old_slot], new_slot), + |&(entry_hash, _)| entry_hash, + &mut self.value_dedup_size, + ); + } + } + + self.group_to_inner = remaining.iter().map(|&old| old_to_new[old]).collect(); + self.check_key_overflow().expect("key overflow in take_n"); + + emitted + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::aggregates::group_values::multi_group_by::bytes::ByteGroupValueBuilder; + use arrow::array::{ + Array, ArrayRef, BooleanBufferBuilder, DictionaryArray, Int32Array, StringArray, + UInt8Array, + }; + use arrow::compute::cast; + use arrow::datatypes::{DataType, Int8Type, Int32Type, UInt8Type}; + use datafusion_physical_expr::binary_map::OutputType; + use std::sync::Arc; + + fn utf8_col() -> DictionaryGroupValuesColumn { + let f = Field::new("", DataType::Utf8, true); + DictionaryGroupValuesColumn::new( + Box::new(ByteGroupValueBuilder::::new(OutputType::Utf8)), + &f, + ) + } + + fn int8_col() -> DictionaryGroupValuesColumn { + let f = Field::new("", DataType::Utf8, true); + DictionaryGroupValuesColumn::new( + Box::new(ByteGroupValueBuilder::::new(OutputType::Utf8)), + &f, + ) + } + + fn uint8_col() -> DictionaryGroupValuesColumn { + let f = Field::new("", DataType::Utf8, true); + DictionaryGroupValuesColumn::new( + Box::new(ByteGroupValueBuilder::::new(OutputType::Utf8)), + &f, + ) + } + + fn i32_dict(keys: &[Option], values: &[Option<&str>]) -> ArrayRef { + Arc::new(DictionaryArray::::new( + Int32Array::from(keys.to_vec()), + Arc::new(StringArray::from(values.to_vec())), + )) + } + + fn i8_dict(keys: &[Option], values: &[Option<&str>]) -> ArrayRef { + use arrow::array::Int8Array; + Arc::new(DictionaryArray::::new( + Int8Array::from(keys.to_vec()), + Arc::new(StringArray::from(values.to_vec())), + )) + } + + fn u8_dict(keys: &[Option], values: &[Option<&str>]) -> ArrayRef { + Arc::new(DictionaryArray::::new( + UInt8Array::from(keys.to_vec()), + Arc::new(StringArray::from(values.to_vec())), + )) + } + + fn str_values(arr: &ArrayRef) -> Vec> { + let plain = cast(arr.as_ref(), &DataType::Utf8).unwrap(); + plain + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .map(|v| v.map(|s| s.to_owned())) + .collect() + } + + fn bool_vec(buf: &BooleanBufferBuilder) -> Vec { + (0..buf.len()).map(|i| buf.get_bit(i)).collect() + } + + fn all_true(len: usize) -> BooleanBufferBuilder { + let mut buf = BooleanBufferBuilder::new(len); + buf.append_n(len, true); + buf + } + + // Builds an Int8-keyed dict of `end-start` distinct strings "v{start}".."v{end-1}". + fn distinct_i8_dict(start: usize, end: usize) -> ArrayRef { + let strs: Vec = (start..end).map(|i| format!("v{i}")).collect(); + let refs: Vec> = strs.iter().map(|s| Some(s.as_str())).collect(); + i8_dict( + &(0..strs.len()).map(|i| Some(i as i8)).collect::>(), + &refs, + ) + } + + // Builds a UInt8-keyed dict of `count` distinct strings "u0".."u{count-1}". + fn distinct_u8_dict(count: usize) -> ArrayRef { + let strs: Vec = (0..count).map(|i| format!("u{i}")).collect(); + let refs: Vec> = strs.iter().map(|s| Some(s.as_str())).collect(); + u8_dict( + &(0..count).map(|i| Some(i as u8)).collect::>(), + &refs, + ) + } + + #[test] + fn repeated_values_are_deduplicated_in_inner_store() { + let mut col = utf8_col(); + let arr = i32_dict( + &[Some(0), Some(1), Some(0), Some(1), Some(0)], + &[Some("a"), Some("b")], + ); + col.vectorized_append(&arr, &[0, 1, 2, 3, 4]).unwrap(); + let out = Box::new(col).build(); + assert_eq!(out.as_dictionary::().values().len(), 2); + assert_eq!( + str_values(&out), + vec![ + Some("a".into()), + Some("b".into()), + Some("a".into()), + Some("b".into()), + Some("a".into()), + ] + ); + } + + #[test] + fn null_key_and_null_valued_entry_both_map_to_null_group() { + let mut col = utf8_col(); + let input = i32_dict(&[None, Some(0), Some(1)], &[None, Some("b")]); + for row in 0..3 { + col.append_val(&input, row).unwrap(); + } + assert!(col.equal_to(0, &input, 1)); + assert!(!col.equal_to(0, &input, 2)); + let out = Box::new(col).build(); + assert_eq!(out.as_dictionary::().values().len(), 1); + assert_eq!(str_values(&out), vec![None, None, Some("b".into())]); + } + + #[test] + fn take_n_compacts_emitted_values_and_remaps_remaining_slots() { + let mut col = utf8_col(); + let b1 = i32_dict( + &[Some(0), Some(1), None, Some(2)], + &[Some("a"), Some("b"), Some("c")], + ); + col.vectorized_append(&b1, &[0, 1, 2, 3]).unwrap(); + + let emitted = col.take_n(2); + assert_eq!(emitted.as_dictionary::().values().len(), 2); + assert_eq!( + str_values(&emitted), + vec![Some("a".into()), Some("b".into())] + ); + + let b2 = i32_dict(&[None, Some(0)], &[Some("z")]); + col.vectorized_append(&b2, &[0, 1]).unwrap(); + + let mut buf = all_true(2); + col.vectorized_equal_to(&[0, 1], &b2, &[0, 1], &mut buf); + assert_eq!(bool_vec(&buf), vec![true, false]); + + let out = Box::new(col).build(); + assert_eq!( + str_values(&out), + vec![None, Some("c".into()), None, Some("z".into())] + ); + } + + #[test] + fn vectorized_equal_to_does_not_use_stale_hashes_from_prior_append() { + let mut col = utf8_col(); + col.vectorized_append(&i32_dict(&[Some(0)], &[Some("a"), Some("b")]), &[0]) + .unwrap(); + let batch2 = i32_dict(&[Some(1)], &[Some("z"), Some("a")]); + let mut buf = all_true(1); + col.vectorized_equal_to(&[0], &batch2, &[0], &mut buf); + assert_eq!(bool_vec(&buf), vec![true]); + } + + #[test] + fn null_does_not_consume_a_key_slot_int8_null_first_mid_and_last() { + let rows128 = (0..128).collect::>(); + + let mut col = int8_col(); // null-last: 128 non-null + null — ok + col.vectorized_append(&distinct_i8_dict(0, 128), &rows128) + .unwrap(); + col.append_val(&i8_dict(&[None], &[Some("x")]), 0).unwrap(); + + let mut col = int8_col(); // null-first: null + 128 non-null — ok; 129th — error + col.append_val(&i8_dict(&[None], &[Some("x")]), 0).unwrap(); + col.vectorized_append(&distinct_i8_dict(0, 128), &rows128) + .unwrap(); + assert!( + col.append_val(&i8_dict(&[Some(0)], &[Some("overflow")]), 0) + .is_err() + ); + + let mut col = int8_col(); // null-mid: 100 + null + 28 = 128 total — ok; 129th — error + col.vectorized_append(&distinct_i8_dict(0, 100), &(0..100).collect::>()) + .unwrap(); + col.append_val(&i8_dict(&[None], &[Some("x")]), 0).unwrap(); + col.vectorized_append(&distinct_i8_dict(100, 128), &(0..28).collect::>()) + .unwrap(); + assert!( + col.append_val(&i8_dict(&[Some(0)], &[Some("v128")]), 0) + .is_err() + ); + + let mut col = int8_col(); // build() null-first: 128 values (null excluded), null → None + col.append_val(&i8_dict(&[None], &[Some("x")]), 0).unwrap(); + col.vectorized_append(&distinct_i8_dict(0, 128), &rows128) + .unwrap(); + let out = Box::new(col).build(); + assert_eq!(out.as_dictionary::().values().len(), 128); + assert_eq!(str_values(&out)[0], None); + assert_eq!(str_values(&out)[1], Some("v0".into())); + } + + #[test] + fn null_does_not_consume_a_key_slot_uint8_null_first_and_last() { + let rows256 = (0..256).collect::>(); + + let mut col = uint8_col(); // null-first: null + 256 non-null — ok; 257th — error + col.append_val(&u8_dict(&[None], &[Some("x")]), 0).unwrap(); + col.vectorized_append(&distinct_u8_dict(256), &rows256) + .unwrap(); + assert!( + col.append_val(&u8_dict(&[Some(0)], &[Some("overflow")]), 0) + .is_err() + ); + + let mut col = uint8_col(); // build() null-first: 256 values (null excluded), last correct + col.append_val(&u8_dict(&[None], &[Some("x")]), 0).unwrap(); + col.vectorized_append(&distinct_u8_dict(256), &rows256) + .unwrap(); + let out = Box::new(col).build(); + assert_eq!(out.as_dictionary::().values().len(), 256); + assert_eq!(str_values(&out)[0], None); + assert_eq!(str_values(&out)[256], Some("u255".into())); + } + + #[test] + fn take_n_null_does_not_steal_key_slot_at_capacity() { + let rows128 = (0..128).collect::>(); + + // Int8 null-first + 128 non-null; emit all 129 — no panic, null → None + let mut col = int8_col(); + col.append_val(&i8_dict(&[None], &[Some("x")]), 0).unwrap(); + col.vectorized_append(&distinct_i8_dict(0, 128), &rows128) + .unwrap(); + let emitted = col.take_n(129); + assert!(emitted.as_dictionary::().key(0).is_none()); + assert_eq!(str_values(&emitted)[1], Some("v0".into())); + assert_eq!(str_values(&emitted)[128], Some("v127".into())); + + // UInt8 null-first + 256 non-null; emit all 257 — last must be "u255" not "u0" (wrap guard) + let mut col = uint8_col(); + col.append_val(&u8_dict(&[None], &[Some("x")]), 0).unwrap(); + col.vectorized_append(&distinct_u8_dict(256), &(0..256).collect::>()) + .unwrap(); + let emitted = col.take_n(257); + assert!(emitted.as_dictionary::().key(0).is_none()); + assert_eq!(str_values(&emitted)[1], Some("u0".into())); + assert_eq!(str_values(&emitted)[256], Some("u255".into())); + } + + #[test] + fn take_n_repeated_emissions_null_at_int8_capacity() { + let rows128 = (0..128).collect::>(); + let mut col = int8_col(); + col.vectorized_append(&distinct_i8_dict(0, 128), &rows128) + .unwrap(); + col.append_val(&i8_dict(&[None], &[Some("x")]), 0).unwrap(); + col.vectorized_append(&distinct_i8_dict(0, 128), &rows128) + .unwrap(); + + let first_half = col.take_n(64); + assert_eq!(str_values(&first_half)[0], Some("v0".into())); + assert_eq!(str_values(&first_half)[63], Some("v63".into())); + assert_eq!(first_half.as_dictionary::().values().len(), 64); + + let second_half = col.take_n(64); + assert_eq!(str_values(&second_half)[0], Some("v64".into())); + assert_eq!(str_values(&second_half)[63], Some("v127".into())); + + let null_group = col.take_n(1); + assert!(null_group.as_dictionary::().key(0).is_none()); + + let out = Box::new(col).build(); + assert_eq!(str_values(&out)[0], Some("v0".into())); + assert_eq!(str_values(&out)[127], Some("v127".into())); + assert_eq!(out.as_dictionary::().values().len(), 128); + } +} diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs index 5b474f3bae075..409e159216df6 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs @@ -20,6 +20,7 @@ mod boolean; mod bytes; pub mod bytes_view; +mod dictionary; mod fixed_size_binary; pub mod primitive; pub mod row_backed; @@ -34,7 +35,6 @@ use crate::aggregates::group_values::multi_group_by::{ primitive::PrimitiveGroupValueBuilder, row_backed::RowsGroupColumn, }; use arrow::array::{Array, ArrayRef, BooleanBufferBuilder}; -use arrow::compute::cast; use arrow::datatypes::{ BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type, Decimal256Type, DurationMicrosecondType, DurationMillisecondType, DurationNanosecondType, @@ -48,7 +48,7 @@ use arrow::datatypes::{ }; use datafusion_common::hash_utils::RandomState; use datafusion_common::hash_utils::create_hashes; -use datafusion_common::{Result, internal_datafusion_err, not_impl_err}; +use datafusion_common::{Result, not_impl_err}; use datafusion_execution::memory_pool::proxy::{HashTableAllocExt, VecAllocExt}; use datafusion_expr::EmitTo; use datafusion_physical_expr::binary_map::OutputType; @@ -980,7 +980,7 @@ fn group_column_supported_type(data_type: &DataType) -> bool { | DataType::Utf8View | DataType::BinaryView | DataType::Boolean - ) + ) || matches!(data_type, DataType::Dictionary(_,v ) if group_column_supported_type(v)) } /// Build a [`GroupColumn`] for a single schema field. @@ -1130,6 +1130,33 @@ fn make_group_column(field: &Field) -> Result> { v.push(Box::new(BooleanGroupValueBuilder::::new())); } } + DataType::Dictionary(ref key_dt, ref value_dt) => { + let new_field = Field::new("", *value_dt.clone(), true); + let inner = make_group_column(&new_field)?; + macro_rules! dict_col { + ($T:ty) => { + Box::new(dictionary::DictionaryGroupValuesColumn::<$T>::new( + inner, &new_field, + )) + }; + } + let col: Box = match key_dt.as_ref() { + DataType::Int8 => dict_col!(Int8Type), + DataType::Int16 => dict_col!(Int16Type), + DataType::Int32 => dict_col!(Int32Type), + DataType::Int64 => dict_col!(Int64Type), + DataType::UInt8 => dict_col!(UInt8Type), + DataType::UInt16 => dict_col!(UInt16Type), + DataType::UInt32 => dict_col!(UInt32Type), + DataType::UInt64 => dict_col!(UInt64Type), + _ => { + return not_impl_err!( + "Dictionary key type {key_dt} not supported in GroupValuesColumn" + ); + } + }; + v.push(col) + } // Generic fallback for nested types (Struct / List / LargeList / // FixedSizeList, recursively) that lack a type-specialized builder but // can be encoded by arrow's row format. This is what lets a mixed @@ -1178,7 +1205,7 @@ impl GroupValues for GroupValuesColumn { } fn emit(&mut self, emit_to: EmitTo) -> Result> { - let mut output = match emit_to { + let output = match emit_to { EmitTo::All => { // Replace the column builders with a fresh set so the // aggregator is immediately reusable after the drain. @@ -1268,20 +1295,6 @@ impl GroupValues for GroupValuesColumn { } }; - // TODO: Materialize dictionaries in group keys (#7647) - for (field, array) in self.schema.fields.iter().zip(&mut output) { - let expected = field.data_type(); - if let DataType::Dictionary(_, v) = expected { - let actual = array.data_type(); - if v.as_ref() != actual { - return Err(internal_datafusion_err!( - "Converted group rows expected dictionary of {v} got {actual}" - )); - } - *array = cast(array.as_ref(), expected)?; - } - } - Ok(output) } @@ -1645,6 +1658,20 @@ mod tests { DataType::Interval(arrow::datatypes::IntervalUnit::YearMonth), DataType::Interval(arrow::datatypes::IntervalUnit::DayTime), DataType::Interval(arrow::datatypes::IntervalUnit::MonthDayNano), + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Int64)), + DataType::Dictionary( + Box::new(DataType::UInt16), + Box::new(DataType::LargeUtf8), + ), + DataType::Dictionary( + Box::new(DataType::Int32), + Box::new(DataType::Timestamp( + arrow::datatypes::TimeUnit::Nanosecond, + None, + )), + ), + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Float16)), ]; for dt in &supported_cases { @@ -1905,6 +1932,56 @@ mod tests { } } + // https://github.com/apache/datafusion/issues/23127 + // validate DictionaryGroupColumn deduplicates values — only k distinct keys appear + // in the values array even when there are more than 128 groups total. + #[test] + fn multi_col_groupby_dict_many_groups_two_values() { + use arrow::array::{AsArray, DictionaryArray, Int8Array}; + use arrow::datatypes::Int8Type; + + let n_groups = 129_usize; + let dict_vocab: ArrayRef = Arc::new(StringArray::from(vec!["cat", "dog"])); + + // Each row has a unique label (forcing a new group) and alternates + // between the two dictionary values. Int8 keys are used; only 2 + // distinct values exist so the key type never overflows. + let labels: ArrayRef = Arc::new(StringArray::from( + (0..n_groups).map(|i| format!("g{i}")).collect::>(), + )); + let dict_keys = Int8Array::from( + (0..n_groups) + .map(|i| Some((i % 2) as i8)) + .collect::>(), + ); + let categories: ArrayRef = Arc::new(DictionaryArray::::new( + dict_keys, + Arc::clone(&dict_vocab), + )); + + let schema = Arc::new(Schema::new(vec![ + Field::new("label", DataType::Utf8, false), + Field::new( + "category", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + false, + ), + ])); + + let mut gv = GroupValuesColumn::::try_new(Arc::clone(&schema)).unwrap(); + gv.intern(&[labels, categories], &mut vec![]).unwrap(); + let out = gv.emit(EmitTo::All).unwrap(); + + assert_eq!(out[0].len(), n_groups); + assert!(matches!( + out[1].data_type(), + DataType::Dictionary(k, v) + if k.as_ref() == &DataType::Int8 && v.as_ref() == &DataType::Utf8 + )); + // Both vectorized and streaming paths now deduplicate dict values. + assert_eq!(out[1].as_dictionary::().values().len(), 2); + } + #[test] fn test_intern_for_vectorized_group_values() { let data_set = VectorizedTestDataSet::new(); From c581d660f0081c748f5053ea469abe39e1b665a3 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Sun, 16 Aug 2026 07:04:16 +0000 Subject: [PATCH 47/50] fix: skip WindowTopN rewrite when fetch is 0 to avoid PartitionedTopK panic (#24405) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #24404. ## Rationale for this change With `datafusion.optimizer.enable_window_topn = true`, a query filtering a partitioned `ROW_NUMBER()`/`RANK()` with `rn < 1` (or the flipped `1 > rn`) panics with `PartitionedTopK requires k > 0`. `ROW_NUMBER`/`RANK` are always `>= 1`, so such a predicate matches no rows and the correct result is empty — it must not panic. The `WindowTopN` rule's `extract_window_limit` maps `rn < K` to a fetch of `K - 1`; for `K = 1` that fetch is `0`, which is passed to `PartitionedTopKExec::try_new`, whose `assert!(k > 0)` panics. With the optimization disabled the same query correctly returns no rows. ## What changes are included in this PR? - In `WindowTopN::try_transform`, bail out of the rewrite when the computed limit is `0`, letting the regular `FilterExec` produce the (empty) result. - Add regression cases to `window_topn.slt` for `rn < 1`, `1 > rn`, and `rn <= 0` (all must return empty without panicking). ## Are these changes tested? Yes — the new sqllogictest cases fail (panic) without the fix and pass with it. Existing `window_topn` tests continue to pass. ## Are there any user-facing changes? No API changes. `rn < 1` / `1 > rn` now returns an empty result instead of panicking when `enable_window_topn` is on, matching the behavior when it is off. --- .../physical-optimizer/src/window_topn.rs | 9 ++++++++ .../sqllogictest/test_files/window_topn.slt | 23 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/datafusion/physical-optimizer/src/window_topn.rs b/datafusion/physical-optimizer/src/window_topn.rs index 20bd8b0d38a1c..aa46a5a9a7ed0 100644 --- a/datafusion/physical-optimizer/src/window_topn.rs +++ b/datafusion/physical-optimizer/src/window_topn.rs @@ -141,6 +141,15 @@ impl WindowTopN { // Step 2: Extract limit from predicate (rn <= K, rn < K, etc.) let (col_idx, limit_n) = extract_window_limit(filter.predicate())?; + // A predicate such as `rn < 1` (or the flipped `1 > rn`) yields a fetch of + // 0. `ROW_NUMBER`/`RANK` are always >= 1, so no row can satisfy it and the + // correct result is empty. `PartitionedTopKExec` requires `k > 0` and would + // panic on `k = 0`, so bail out here and let the regular `FilterExec` produce + // the (empty) result instead of rewriting. + if limit_n == 0 { + return None; + } + // Step 3: Walk through optional ProjectionExec and RepartitionExec to find BoundedWindowAggExec let child = filter.input(); let (window_exec, intermediates) = find_window_below(child)?; diff --git a/datafusion/sqllogictest/test_files/window_topn.slt b/datafusion/sqllogictest/test_files/window_topn.slt index 44cb31153b004..a9a52a654f08c 100644 --- a/datafusion/sqllogictest/test_files/window_topn.slt +++ b/datafusion/sqllogictest/test_files/window_topn.slt @@ -84,6 +84,29 @@ SELECT id, pk, val FROM ( 8 3 100 9 3 50 +# Test 3b: rn < 1 has fetch = 0. ROW_NUMBER is always >= 1 so no row qualifies and +# the result must be empty. The rewrite must not build a PartitionedTopKExec with k = 0 +# (which panics); it falls back to the regular filter. Regression for the k = 0 panic. +query III rowsort +SELECT id, pk, val FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY pk ORDER BY val) as rn FROM window_topn_t +) WHERE rn < 1; +---- + +# Test 3c: flipped form `1 > rn` (fetch = 0) — same empty result, no panic. +query III rowsort +SELECT id, pk, val FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY pk ORDER BY val) as rn FROM window_topn_t +) WHERE 1 > rn; +---- + +# Test 3d: rn <= 0 (fetch = 0) — empty, no panic. +query III rowsort +SELECT id, pk, val FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY pk ORDER BY val) as rn FROM window_topn_t +) WHERE rn <= 0; +---- + # Test 4: Without PARTITION BY — should NOT optimize query II rowsort SELECT id, val FROM ( From d2f65a747699c04d885b9617235229a66591b1d2 Mon Sep 17 00:00:00 2001 From: Jamison-J Date: Sun, 16 Aug 2026 07:07:40 +0000 Subject: [PATCH 48/50] feat: add implicit native coercion constructor (#24282) ## Which issue does this PR close? - Closes #24135. ## Rationale for this change When a coercion's desired type is a native logical type, callers currently have to repeat that type as both `TypeSignatureClass::Native(...)` and `default_casted_type`. This duplication is confusing and can allow the two values to disagree. ## What changes are included in this PR? - Add `Coercion::new_implicit_native`, which derives `default_casted_type` from the desired logical type and delegates to the existing `new_implicit` constructor. - Keep `Coercion::new_implicit` unchanged for backward compatibility and non-native desired type classes. - Migrate the representative `reverse` and `approx_percentile_cont` call sites linked from the issue. - Update the `Coercion` rustdoc example and add a constructor-equivalence unit test. ## Are these changes tested? Yes: - `cargo test -p datafusion-expr-common test_new_implicit_native --lib` - `cargo test -p datafusion-functions reverse --lib` - `cargo test -p datafusion-functions-aggregate approx_percentile_cont --lib` - `cargo test -p datafusion-expr-common --doc` - `cargo clippy -p datafusion-expr-common -p datafusion-functions -p datafusion-functions-aggregate --all-targets --all-features -- -D warnings` - `cargo fmt --all` The required full-workspace `cargo clippy --all-targets --all-features -- -D warnings` was also attempted on Windows. It reached an unrelated MSVC native-link failure while building `protobuf-src` (`LNK2005`/`LNK4098` in `protoc-gen-upb`); affected-crate clippy passed with warnings denied, and CI will run the full Linux workspace checks. No benchmark was run because this is an API convenience constructor that delegates to the existing constructor and does not change runtime execution. ## Are there any user-facing changes? Yes. This adds the public `Coercion::new_implicit_native` convenience API. It is additive and backward compatible; existing `new_implicit` callers continue to work unchanged. --------- Co-authored-by: jiangzhen --- datafusion/expr-common/src/signature.rs | 43 +++++++++++++++++-- .../src/approx_percentile_cont.rs | 10 ++--- datafusion/functions/src/unicode/reverse.rs | 7 ++- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rs index f0010f0a05014..2bde7c89a8259 100644 --- a/datafusion/expr-common/src/signature.rs +++ b/datafusion/expr-common/src/signature.rs @@ -1025,17 +1025,16 @@ fn get_data_types(native_type: &NativeType) -> Vec { /// # Examples /// /// ``` -/// use datafusion_common::types::{logical_binary, logical_string, NativeType}; +/// use datafusion_common::types::{logical_binary, logical_string}; /// use datafusion_expr_common::signature::{Coercion, TypeSignatureClass}; /// /// // Exact coercion that only accepts timestamp types /// let exact = Coercion::new_exact(TypeSignatureClass::Timestamp); /// /// // Implicit coercion that accepts string types but can coerce from binary types -/// let implicit = Coercion::new_implicit( -/// TypeSignatureClass::Native(logical_string()), +/// let implicit = Coercion::new_implicit_native( +/// logical_string(), /// vec![TypeSignatureClass::Native(logical_binary())], -/// NativeType::String, /// ); /// ``` /// @@ -1118,6 +1117,25 @@ impl Coercion { } } + /// Create a new coercion to a native logical type with implicit coercion rules. + /// + /// This is a convenience method for coercing to a specific logical type, + /// avoiding the need to separately pass the matching native default cast type. + /// + /// The native type of `desired_type` is used as the default type when coercing + /// from `allowed_source_types`. + pub fn new_implicit_native( + desired_type: LogicalTypeRef, + allowed_source_types: Vec, + ) -> Self { + let default_casted_type = desired_type.native().clone(); + Self::new_implicit( + TypeSignatureClass::Native(desired_type), + allowed_source_types, + default_casted_type, + ) + } + pub fn with_encoding_preservation( mut self, encoding_preservation: EncodingPreservation, @@ -2245,6 +2263,23 @@ mod tests { assert_snapshot!(implicit_with_multiple_sources, @"Int64"); } + #[test] + fn test_new_implicit_native() { + let allowed_source_types = vec![TypeSignatureClass::Numeric]; + + assert_eq!( + Coercion::new_implicit_native( + logical_float64(), + allowed_source_types.clone(), + ), + Coercion::new_implicit( + TypeSignatureClass::Native(logical_float64()), + allowed_source_types, + NativeType::Float64, + ) + ); + } + #[test] fn test_coercion_encoding_preservation_affects_equality() { assert!(!EncodingPreservation::default().preserve_dictionary()); diff --git a/datafusion/functions-aggregate/src/approx_percentile_cont.rs b/datafusion/functions-aggregate/src/approx_percentile_cont.rs index ea8fea1b1bc29..4af3574d8bd74 100644 --- a/datafusion/functions-aggregate/src/approx_percentile_cont.rs +++ b/datafusion/functions-aggregate/src/approx_percentile_cont.rs @@ -139,10 +139,9 @@ impl ApproxPercentileCont { vec![TypeSignatureClass::Numeric], NativeType::Float64, ), - Coercion::new_implicit( - TypeSignatureClass::Native(logical_float64()), + Coercion::new_implicit_native( + logical_float64(), vec![TypeSignatureClass::Numeric], - NativeType::Float64, ), ]), // 3 args - numeric, percentile (float), number of centroid for T-Digest (integer) @@ -152,10 +151,9 @@ impl ApproxPercentileCont { vec![TypeSignatureClass::Numeric], NativeType::Float64, ), - Coercion::new_implicit( - TypeSignatureClass::Native(logical_float64()), + Coercion::new_implicit_native( + logical_float64(), vec![TypeSignatureClass::Numeric], - NativeType::Float64, ), Coercion::new_implicit( TypeSignatureClass::Integer, diff --git a/datafusion/functions/src/unicode/reverse.rs b/datafusion/functions/src/unicode/reverse.rs index 9dfc25fbdfe07..e42240ba6697b 100644 --- a/datafusion/functions/src/unicode/reverse.rs +++ b/datafusion/functions/src/unicode/reverse.rs @@ -25,7 +25,7 @@ use DataType::{LargeUtf8, Utf8, Utf8View}; use arrow::array::{Array, ArrayRef, AsArray, StringArrayType}; use arrow::datatypes::DataType; use datafusion_common::Result; -use datafusion_common::types::{NativeType, logical_string}; +use datafusion_common::types::logical_string; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, @@ -62,10 +62,9 @@ impl ReverseFunc { Self { signature: Signature::coercible( vec![ - Coercion::new_implicit( - TypeSignatureClass::Native(logical_string()), + Coercion::new_implicit_native( + logical_string(), vec![TypeSignatureClass::Any], - NativeType::String, ) .with_encoding_preservation(EncodingPreservation::dictionary()), ], From 486e5ac2177c5f44d5be22ddee43f88c7fba85fb Mon Sep 17 00:00:00 2001 From: Narendra Date: Thu, 13 Aug 2026 21:09:24 +0530 Subject: [PATCH 49/50] feat: add sphinx-sitemap dependency and configuration for sitemap generation --- docs/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/pyproject.toml b/docs/pyproject.toml index 48740f2b33308..52828d6db810c 100644 --- a/docs/pyproject.toml +++ b/docs/pyproject.toml @@ -11,4 +11,5 @@ dependencies = [ "jinja2>=3.1.6,<4", "sphinx-sitemap>=2.9.0,<3", "setuptools>=84.0.0,<85", + "sphinx-sitemap>=2.9.0,<3", ] From d938297a52a9459758badb359e1f931623dbbbe8 Mon Sep 17 00:00:00 2001 From: Narendra Date: Sun, 16 Aug 2026 13:43:42 +0530 Subject: [PATCH 50/50] Removing repeated entry --- docs/pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/pyproject.toml b/docs/pyproject.toml index 52828d6db810c..5dceea95223f6 100644 --- a/docs/pyproject.toml +++ b/docs/pyproject.toml @@ -9,7 +9,6 @@ dependencies = [ "myst-parser>=5.1.0,<6", "maturin>=1.14.1,<2", "jinja2>=3.1.6,<4", - "sphinx-sitemap>=2.9.0,<3", "setuptools>=84.0.0,<85", "sphinx-sitemap>=2.9.0,<3", ]