From c734db339aac3ca8970f26622616060628958f83 Mon Sep 17 00:00:00 2001 From: Dave Sirockin Date: Fri, 17 Jul 2026 14:43:31 +0200 Subject: [PATCH 1/2] feat(duckdb): convert ARRAY_AGG IGNORE NULLS to FILTER clause BigQuery's `ARRAY_AGG(x IGNORE NULLS ...)` silently drops NULLs from the aggregation. DuckDB does not support `IGNORE NULLS` on aggregate functions, but the same semantics can be expressed with a `FILTER` clause: `ARRAY_AGG(x ...) FILTER(WHERE x IS NOT NULL)`. Previously, the DuckDB generator silently dropped the `IGNORE NULLS` modifier on `ARRAY_AGG`, producing incorrect results when the input contained NULL values. Adds: - `ArrayAgg`-specific handling in `ignorenulls_sql` that sets a `nulls_excluded` flag, which the existing `_add_arrayagg_null_filter` infrastructure converts to a `FILTER(WHERE col IS NOT NULL)` clause - `arrayagg_sql` override to unwrap `Order`/`Limit` nodes so the FILTER clause references the correct column expression - Updated test expectations for ARRAY_AGG DISTINCT IGNORE NULLS --- sqlglot/generators/duckdb.py | 18 ++++++++++++++++++ tests/dialects/test_bigquery.py | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/sqlglot/generators/duckdb.py b/sqlglot/generators/duckdb.py index ce31a22c12..c269550f0c 100644 --- a/sqlglot/generators/duckdb.py +++ b/sqlglot/generators/duckdb.py @@ -3953,6 +3953,16 @@ def unnest_sql(self, expression: exp.Unnest) -> str: return super().unnest_sql(expression) + def arrayagg_sql(self, expression: exp.ArrayAgg) -> str: + array_agg = self.function_fallback_sql(expression) + # The argument inside ARRAY_AGG may be wrapped in Limit, Order, and/or + # Distinct nodes. Unwrap to get the bare column expression so the + # FILTER clause is just "WHERE x IS NOT NULL". + column_expr = expression.this + while isinstance(column_expr, (exp.Limit, exp.Order)): + column_expr = column_expr.this + return self._add_arrayagg_null_filter(array_agg, expression, column_expr) + def ignorenulls_sql(self, expression: exp.IgnoreNulls) -> str: this = expression.this @@ -3961,6 +3971,14 @@ def ignorenulls_sql(self, expression: exp.IgnoreNulls) -> str: # window functions that accept it e.g. FIRST_VALUE(... IGNORE NULLS) OVER (...) return super().ignorenulls_sql(expression) + # For ARRAY_AGG(expr IGNORE NULLS ...), convert IGNORE NULLS to a + # FILTER(WHERE expr IS NOT NULL) clause by setting nulls_excluded on + # the ArrayAgg. The existing _add_arrayagg_null_filter method will + # emit the FILTER clause during arrayagg_sql / withingroup_sql. + if isinstance(this, exp.ArrayAgg): + this.set("nulls_excluded", True) + return self.sql(this) + if isinstance(this, exp.First): this = exp.AnyValue(this=this.this) diff --git a/tests/dialects/test_bigquery.py b/tests/dialects/test_bigquery.py index 9e7834b6a7..7a689ffdf4 100644 --- a/tests/dialects/test_bigquery.py +++ b/tests/dialects/test_bigquery.py @@ -737,7 +737,7 @@ def test_bigquery(self): "SELECT ARRAY_AGG(DISTINCT x IGNORE NULLS ORDER BY a, b DESC LIMIT 10) AS x", write={ "bigquery": "SELECT ARRAY_AGG(DISTINCT x IGNORE NULLS ORDER BY a, b DESC LIMIT 10) AS x", - "duckdb": "SELECT ARRAY_AGG(DISTINCT x ORDER BY a NULLS FIRST, b DESC LIMIT 10) AS x", + "duckdb": "SELECT ARRAY_AGG(DISTINCT x ORDER BY a NULLS FIRST, b DESC LIMIT 10) FILTER(WHERE x IS NOT NULL) AS x", "spark": "SELECT COLLECT_LIST(DISTINCT x ORDER BY a, b DESC LIMIT 10) IGNORE NULLS AS x", }, ) @@ -745,7 +745,7 @@ def test_bigquery(self): "SELECT ARRAY_AGG(DISTINCT x IGNORE NULLS ORDER BY a, b DESC LIMIT 1, 10) AS x", write={ "bigquery": "SELECT ARRAY_AGG(DISTINCT x IGNORE NULLS ORDER BY a, b DESC LIMIT 1, 10) AS x", - "duckdb": "SELECT ARRAY_AGG(DISTINCT x ORDER BY a NULLS FIRST, b DESC LIMIT 1, 10) AS x", + "duckdb": "SELECT ARRAY_AGG(DISTINCT x ORDER BY a NULLS FIRST, b DESC LIMIT 1, 10) FILTER(WHERE x IS NOT NULL) AS x", "spark": "SELECT COLLECT_LIST(DISTINCT x ORDER BY a, b DESC LIMIT 1, 10) IGNORE NULLS AS x", }, ) From 48a72bd97940c650191fc2cf65784473735982c7 Mon Sep 17 00:00:00 2001 From: Dave Sirockin Date: Mon, 20 Jul 2026 15:01:33 +0200 Subject: [PATCH 2/2] fix(duckdb): address review comments on ARRAY_AGG IGNORE NULLS PR - Move Order/Limit unwrapping from DuckDB override to base generator's arrayagg_sql so all dialects benefit - Emit unsupported warning for LIMIT inside ARRAY_AGG in DuckDB - Remove invalid LIMIT 1, 10 test (unsupported BigQuery syntax) - Simplify DuckDB arrayagg_sql to delegate to super() Co-Authored-By: Claude Opus 4.6 --- sqlglot/generator.py | 10 +++++++++- sqlglot/generators/duckdb.py | 11 +++-------- tests/dialects/test_bigquery.py | 10 +--------- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/sqlglot/generator.py b/sqlglot/generator.py index 2ab295e471..8f3e52ae95 100644 --- a/sqlglot/generator.py +++ b/sqlglot/generator.py @@ -5668,7 +5668,15 @@ def _add_arrayagg_null_filter( def arrayagg_sql(self, expression: exp.ArrayAgg) -> str: array_agg = self.function_fallback_sql(expression) - return self._add_arrayagg_null_filter(array_agg, expression, expression.this) + # The argument inside ARRAY_AGG may be wrapped in Order and/or Limit + # nodes. Unwrap to get the bare column expression so the FILTER + # clause is just "WHERE x IS NOT NULL". + column_expr = expression.this + if isinstance(column_expr, exp.Limit): + column_expr = column_expr.this + if isinstance(column_expr, exp.Order): + column_expr = column_expr.this + return self._add_arrayagg_null_filter(array_agg, expression, column_expr) def slice_sql(self, expression: exp.Slice) -> str: step = self.sql(expression, "step") diff --git a/sqlglot/generators/duckdb.py b/sqlglot/generators/duckdb.py index c269550f0c..4532caea3c 100644 --- a/sqlglot/generators/duckdb.py +++ b/sqlglot/generators/duckdb.py @@ -3954,14 +3954,9 @@ def unnest_sql(self, expression: exp.Unnest) -> str: return super().unnest_sql(expression) def arrayagg_sql(self, expression: exp.ArrayAgg) -> str: - array_agg = self.function_fallback_sql(expression) - # The argument inside ARRAY_AGG may be wrapped in Limit, Order, and/or - # Distinct nodes. Unwrap to get the bare column expression so the - # FILTER clause is just "WHERE x IS NOT NULL". - column_expr = expression.this - while isinstance(column_expr, (exp.Limit, exp.Order)): - column_expr = column_expr.this - return self._add_arrayagg_null_filter(array_agg, expression, column_expr) + if isinstance(expression.this, exp.Limit): + self.unsupported("LIMIT inside ARRAY_AGG is not supported in DuckDB") + return super().arrayagg_sql(expression) def ignorenulls_sql(self, expression: exp.IgnoreNulls) -> str: this = expression.this diff --git a/tests/dialects/test_bigquery.py b/tests/dialects/test_bigquery.py index 7a689ffdf4..f0e3151773 100644 --- a/tests/dialects/test_bigquery.py +++ b/tests/dialects/test_bigquery.py @@ -737,18 +737,10 @@ def test_bigquery(self): "SELECT ARRAY_AGG(DISTINCT x IGNORE NULLS ORDER BY a, b DESC LIMIT 10) AS x", write={ "bigquery": "SELECT ARRAY_AGG(DISTINCT x IGNORE NULLS ORDER BY a, b DESC LIMIT 10) AS x", - "duckdb": "SELECT ARRAY_AGG(DISTINCT x ORDER BY a NULLS FIRST, b DESC LIMIT 10) FILTER(WHERE x IS NOT NULL) AS x", + "duckdb": UnsupportedError, "spark": "SELECT COLLECT_LIST(DISTINCT x ORDER BY a, b DESC LIMIT 10) IGNORE NULLS AS x", }, ) - self.validate_all( - "SELECT ARRAY_AGG(DISTINCT x IGNORE NULLS ORDER BY a, b DESC LIMIT 1, 10) AS x", - write={ - "bigquery": "SELECT ARRAY_AGG(DISTINCT x IGNORE NULLS ORDER BY a, b DESC LIMIT 1, 10) AS x", - "duckdb": "SELECT ARRAY_AGG(DISTINCT x ORDER BY a NULLS FIRST, b DESC LIMIT 1, 10) FILTER(WHERE x IS NOT NULL) AS x", - "spark": "SELECT COLLECT_LIST(DISTINCT x ORDER BY a, b DESC LIMIT 1, 10) IGNORE NULLS AS x", - }, - ) self.validate_all( "SELECT * FROM Produce UNPIVOT((first_half_sales, second_half_sales) FOR semesters IN ((Q1, Q2) AS 'semester_1', (Q3, Q4) AS 'semester_2'))", read={