From bd23fde5c1292f125c3e2d18157c12aaff6740f5 Mon Sep 17 00:00:00 2001 From: Adam Reeve Date: Mon, 16 Nov 2026 22:11:29 +1300 Subject: [PATCH 1/4] Add FilteredDataset in C++ and use for unions of filtered datasets Dataset.filter() stores a filter on the Python Dataset wrapper, but a UnionDataset is represented in C++, so filtered Datasets could not be unioned. This adds a new FilteredDataset class that holds a filter in the C++ layer, and converts any datasets with filters to a FilteredDataset when they are used to create a UnionDataset. --- cpp/src/arrow/dataset/dataset.cc | 122 ++++++++++++++++++ cpp/src/arrow/dataset/dataset.h | 61 +++++++++ cpp/src/arrow/dataset/dataset_test.cc | 126 +++++++++++++++++++ docs/source/python/api/dataset.rst | 1 + python/pyarrow/_dataset.pyx | 57 +++++++++ python/pyarrow/dataset.py | 14 +-- python/pyarrow/includes/libarrow_dataset.pxd | 9 ++ python/pyarrow/tests/test_dataset.py | 92 +++++++++++++- 8 files changed, 470 insertions(+), 12 deletions(-) diff --git a/cpp/src/arrow/dataset/dataset.cc b/cpp/src/arrow/dataset/dataset.cc index 2925cbf0003d..0b81088a8bd1 100644 --- a/cpp/src/arrow/dataset/dataset.cc +++ b/cpp/src/arrow/dataset/dataset.cc @@ -19,6 +19,7 @@ #include #include "arrow/acero/util.h" +#include "arrow/compute/api_vector.h" #include "arrow/dataset/dataset.h" #include "arrow/dataset/dataset_internal.h" #include "arrow/dataset/projector.h" @@ -321,6 +322,127 @@ Result UnionDataset::GetFragmentsImpl(compute::Expression pred return GetFragmentsFromDatasets(children_, predicate); } +FilteredFragment::FilteredFragment(std::shared_ptr fragment, + compute::Expression filter) + : Fragment(fragment->partition_expression(), /*physical_schema=*/nullptr), + fragment_(std::move(fragment)), + filter_(std::move(filter)) {} + +Result> FilteredFragment::ReadPhysicalSchemaImpl() { + return fragment_->ReadPhysicalSchema(); +} + +Status FilteredFragment::ClearCachedMetadata() { + RETURN_NOT_OK(Fragment::ClearCachedMetadata()); + return fragment_->ClearCachedMetadata(); +} + +std::string FilteredFragment::ToString() const { + return "filtered(" + fragment_->ToString() + ", " + filter_.ToString() + ")"; +} + +Result FilteredFragment::ScanBatchesAsync( + const std::shared_ptr& options) { + const auto& dataset_schema = options->dataset_schema; + ARROW_ASSIGN_OR_RAISE(auto filter, filter_.Bind(*dataset_schema)); + ARROW_ASSIGN_OR_RAISE(filter, + SimplifyWithGuarantee(std::move(filter), partition_expression_)); + if (!filter.IsSatisfiable()) { + return MakeEmptyGenerator>(); + } + + // Combine any existing filter in the scan options with the fragment's filter, + // so the wrapped fragment can skip data that cannot match. + auto fragment_options = std::make_shared(*options); + ARROW_ASSIGN_OR_RAISE(fragment_options->filter, + compute::and_(options->filter, filter).Bind(*dataset_schema)); + ARROW_ASSIGN_OR_RAISE(auto batches, fragment_->ScanBatchesAsync(fragment_options)); + + // The filter might simplify to true if it only references partition fields, + // in which case we can skip filtering batches. + if (filter == compute::literal(true)) { + return batches; + } + + // Apply the fragment's filter to scanned rows. + // The filter in the scan options is only used for data pruning at this point + // and will be used to filter rows later, so don't duplicate that work. + auto guarantee = partition_expression_; + auto pool = options->pool; + return MakeMappedGenerator( + std::move(batches), + [filter, dataset_schema, guarantee, pool](const std::shared_ptr& batch) + -> Result> { + compute::ExecContext exec_context(pool); + ARROW_ASSIGN_OR_RAISE(auto exec_batch, + compute::MakeExecBatch(*dataset_schema, batch, guarantee)); + ARROW_ASSIGN_OR_RAISE(Datum mask, compute::ExecuteScalarExpression( + filter, exec_batch, &exec_context)); + if (mask.is_scalar()) { + const auto& selected = mask.scalar_as(); + return selected.is_valid && selected.value ? batch : batch->Slice(0, 0); + } + ARROW_ASSIGN_OR_RAISE( + Datum filtered, + compute::Filter(batch, mask, compute::FilterOptions::Defaults(), + &exec_context)); + return filtered.record_batch(); + }); +} + +Future> FilteredFragment::CountRows( + compute::Expression predicate, const std::shared_ptr& options) { + ARROW_ASSIGN_OR_RAISE( + auto combined, + compute::and_(std::move(predicate), filter_).Bind(*options->dataset_schema)); + return fragment_->CountRows(std::move(combined), options); +} + +FilteredDataset::FilteredDataset(std::shared_ptr child, + compute::Expression filter) + : Dataset(child->schema(), child->partition_expression()), + child_(std::move(child)), + filter_(std::move(filter)) {} + +Result> FilteredDataset::Make( + std::shared_ptr child, compute::Expression filter) { + // Verify the filter produces a bool + ARROW_ASSIGN_OR_RAISE(auto bound, filter.Bind(*child->schema())); + if (!bound.type()->Equals(*boolean())) { + return Status::TypeError("FilteredDataset filter must be boolean, got ", + *bound.type()); + } + // Store the unbound filter so it can be bound on-demand on the unified schema of a + // UnionDataset. + return std::shared_ptr( + new FilteredDataset(std::move(child), std::move(filter))); +} + +Result> FilteredDataset::ReplaceSchema( + std::shared_ptr schema) const { + ARROW_ASSIGN_OR_RAISE(auto child, child_->ReplaceSchema(std::move(schema))); + ARROW_ASSIGN_OR_RAISE(auto filtered, Make(std::move(child), filter_)); + return std::static_pointer_cast(std::move(filtered)); +} + +Result FilteredDataset::GetFragmentsImpl( + compute::Expression predicate) { + // Compute a combined predicate that can be used for predicate-pushdown to select + // fragments from the child Dataset. + ARROW_ASSIGN_OR_RAISE(auto combined_predicate, + compute::and_(std::move(predicate), filter_).Bind(*schema_)); + ARROW_ASSIGN_OR_RAISE(auto fragments, + child_->GetFragments(std::move(combined_predicate))); + + // Convert the child's fragments to FilteredFragments that apply the additional filter. + auto filter = filter_; + return MakeMapIterator( + [filter](std::shared_ptr fragment) -> std::shared_ptr { + return std::make_shared(std::move(fragment), filter); + }, + std::move(fragments)); +} + namespace { struct MissingColumn { diff --git a/cpp/src/arrow/dataset/dataset.h b/cpp/src/arrow/dataset/dataset.h index 5c788ef5581c..70ab582fc78a 100644 --- a/cpp/src/arrow/dataset/dataset.h +++ b/cpp/src/arrow/dataset/dataset.h @@ -485,6 +485,67 @@ class ARROW_DS_EXPORT UnionDataset : public Dataset { friend class UnionDatasetFactory; }; +/// \brief A Fragment which wraps another Fragment and applies a filter. +class ARROW_DS_EXPORT FilteredFragment : public Fragment { + public: + FilteredFragment(std::shared_ptr fragment, compute::Expression filter); + + Result ScanBatchesAsync( + const std::shared_ptr& options) override; + + Future> CountRows( + compute::Expression predicate, + const std::shared_ptr& options) override; + + Status ClearCachedMetadata() override; + + std::string type_name() const override { return "filtered"; } + + std::string ToString() const override; + + const std::shared_ptr& fragment() const { return fragment_; } + + const compute::Expression& filter() const { return filter_; } + + protected: + Result> ReadPhysicalSchemaImpl() override; + + std::shared_ptr fragment_; + compute::Expression filter_; +}; + +/// \brief A Dataset which wraps another Dataset and applies a filter. +/// +/// Unlike a filter in ScanOptions, the filter is part of the Dataset itself, so it +/// is preserved when the Dataset is combined with others, e.g. as a child of a +/// UnionDataset whose other children have different filters. +class ARROW_DS_EXPORT FilteredDataset : public Dataset { + public: + /// \brief Construct a FilteredDataset. + /// + /// \param[in] child the Dataset to filter. + /// \param[in] filter a boolean expression used to filter rows of the child Dataset. + static Result> Make(std::shared_ptr child, + compute::Expression filter); + + const std::shared_ptr& child() const { return child_; } + + const compute::Expression& filter() const { return filter_; } + + std::string type_name() const override { return "filtered"; } + + Result> ReplaceSchema( + std::shared_ptr schema) const override; + + protected: + Result GetFragmentsImpl(compute::Expression predicate) override; + + FilteredDataset(std::shared_ptr child, compute::Expression filter); + + std::shared_ptr child_; + compute::Expression filter_; +}; + /// @} } // namespace dataset diff --git a/cpp/src/arrow/dataset/dataset_test.cc b/cpp/src/arrow/dataset/dataset_test.cc index cd23429bf5ef..613785332bfe 100644 --- a/cpp/src/arrow/dataset/dataset_test.cc +++ b/cpp/src/arrow/dataset/dataset_test.cc @@ -277,6 +277,132 @@ TEST_F(TestUnionDataset, TrivialScan) { AssertDatasetEquals(reader.get(), dataset.get()); } +class TestFilteredDataset : public DatasetFixtureMixin {}; + +TEST_F(TestFilteredDataset, UnionOfDifferentFilters) { + SetSchema({field("i", int32())}); + // Include a null value to test handling when the filter evaluates to null. + auto batch = RecordBatchFromJSON(schema_, R"([[1], [2], [3], [null], [4]])"); + auto child = std::make_shared(schema_, RecordBatchVector{batch}); + + ASSERT_OK_AND_ASSIGN(auto small, + FilteredDataset::Make(child, compute::less(compute::field_ref("i"), + compute::literal(2)))); + ASSERT_OK_AND_ASSIGN( + auto large, FilteredDataset::Make(child, compute::greater(compute::field_ref("i"), + compute::literal(3)))); + ASSERT_OK_AND_ASSIGN(auto dataset, UnionDataset::Make(schema_, {small, large})); + + ASSERT_OK_AND_ASSIGN(auto builder, dataset->NewScan()); + ASSERT_OK_AND_ASSIGN(auto scanner, builder->Finish()); + ASSERT_OK_AND_ASSIGN(auto table, scanner->ToTable()); + AssertTablesEqual(*TableFromJSON(schema_, {R"([[1], [4]])"}), *table, + /*same_chunk_layout=*/false); + ASSERT_OK_AND_EQ(2, scanner->CountRows()); + + // A filter on the union applies on top of each child's filter. + ASSERT_OK_AND_ASSIGN(builder, dataset->NewScan()); + ASSERT_OK( + builder->Filter(compute::not_equal(compute::field_ref("i"), compute::literal(1)))); + ASSERT_OK_AND_ASSIGN(scanner, builder->Finish()); + ASSERT_OK_AND_ASSIGN(table, scanner->ToTable()); + AssertTablesEqual(*TableFromJSON(schema_, {R"([[4]])"}), *table, + /*same_chunk_layout=*/false); + ASSERT_OK_AND_EQ(1, scanner->CountRows()); +} + +TEST_F(TestFilteredDataset, ReplaceSchema) { + SetSchema({field("i", int32())}); + auto batch = RecordBatchFromJSON(schema_, R"([[1], [2]])"); + auto child = std::make_shared(schema_, RecordBatchVector{batch}); + ASSERT_OK_AND_ASSIGN( + auto filtered, FilteredDataset::Make(child, compute::equal(compute::field_ref("i"), + compute::literal(2)))); + + auto wider = schema({field("i", int32()), field("j", utf8())}); + ASSERT_OK_AND_ASSIGN(auto replaced, filtered->ReplaceSchema(wider)); + ASSERT_EQ("filtered", replaced->type_name()); + ASSERT_OK_AND_ASSIGN(auto builder, replaced->NewScan()); + ASSERT_OK_AND_ASSIGN(auto scanner, builder->Finish()); + ASSERT_OK_AND_ASSIGN(auto table, scanner->ToTable()); + AssertTablesEqual(*TableFromJSON(wider, {R"([[2, null]])"}), *table, + /*same_chunk_layout=*/false); + + // Check we get an error if attempting to replace the schema with one + // that the filter won't bind against. + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("No match for FieldRef"), + filtered->ReplaceSchema(schema({field("j", utf8())}))); +} + +TEST_F(TestFilteredDataset, FilterOnPartitionField) { + SetSchema({field("part", int32()), field("i", int32())}); + auto make_fragment = [this](int32_t part, const std::string& json) { + return std::make_shared( + schema_, RecordBatchVector{RecordBatchFromJSON(schema_, json)}, + compute::equal(compute::field_ref("part"), compute::literal(part))); + }; + auto child = std::make_shared( + schema_, FragmentVector{make_fragment(1, R"([[1, 10], [1, 11]])"), + make_fragment(2, R"([[2, 20], [2, 21]])"), + make_fragment(3, R"([[3, 30], [3, 31]])")}); + + // This filter is fully guaranteed by the matching fragment's partition expression, + // so it simplifies to true and no per-batch filtering is needed. + ASSERT_OK_AND_ASSIGN( + auto first, FilteredDataset::Make(child, compute::equal(compute::field_ref("part"), + compute::literal(1)))); + // This filter only partially simplifies, leaving i > 20 to apply to each batch. + ASSERT_OK_AND_ASSIGN( + auto second, + FilteredDataset::Make( + child, compute::and_( + compute::equal(compute::field_ref("part"), compute::literal(2)), + compute::greater(compute::field_ref("i"), compute::literal(20))))); + + // Fragments whose partition expression contradicts the filter are pruned. + ASSERT_OK_AND_ASSIGN(auto fragment_it, first->GetFragments()); + ASSERT_OK_AND_ASSIGN(auto fragments, fragment_it.ToVector()); + ASSERT_EQ(1, fragments.size()); + + ASSERT_OK_AND_ASSIGN(auto dataset, UnionDataset::Make(schema_, {first, second})); + ASSERT_OK_AND_ASSIGN(auto builder, dataset->NewScan()); + ASSERT_OK_AND_ASSIGN(auto scanner, builder->Finish()); + ASSERT_OK_AND_ASSIGN(auto table, scanner->ToTable()); + AssertTablesEqual(*TableFromJSON(schema_, {R"([[1, 10], [1, 11], [2, 21]])"}), *table, + /*same_chunk_layout=*/false); + ASSERT_OK_AND_EQ(3, scanner->CountRows()); +} + +TEST_F(TestFilteredDataset, ScalarFilterResult) { + // The batches lack column j, so it is filled with a null scalar when filtering and + // filters that only reference j produce a scalar mask. + SetSchema({field("i", int32()), field("j", utf8())}); + auto batch = RecordBatchFromJSON(schema({field("i", int32())}), R"([[1], [2]])"); + auto child = std::make_shared(schema_, RecordBatchVector{batch}); + + // A true mask keeps every row. + ASSERT_OK_AND_ASSIGN( + auto keep_all, + FilteredDataset::Make(child, compute::is_null(compute::field_ref("j")))); + // A null mask drops every row. + ASSERT_OK_AND_ASSIGN(auto drop_null, FilteredDataset::Make( + child, compute::equal(compute::field_ref("j"), + compute::literal("x")))); + // A false mask drops every row. + ASSERT_OK_AND_ASSIGN( + auto drop_false, + FilteredDataset::Make(child, compute::is_valid(compute::field_ref("j")))); + + ASSERT_OK_AND_ASSIGN(auto dataset, + UnionDataset::Make(schema_, {keep_all, drop_null, drop_false})); + ASSERT_OK_AND_ASSIGN(auto builder, dataset->NewScan()); + ASSERT_OK_AND_ASSIGN(auto scanner, builder->Finish()); + ASSERT_OK_AND_ASSIGN(auto table, scanner->ToTable()); + AssertTablesEqual(*TableFromJSON(schema_, {R"([[1, null], [2, null]])"}), *table, + /*same_chunk_layout=*/false); + ASSERT_OK_AND_EQ(2, scanner->CountRows()); +} + TEST(TestProjector, CheckProjectable) { struct Assert { explicit Assert(FieldVector from) : from_(from) {} diff --git a/docs/source/python/api/dataset.rst b/docs/source/python/api/dataset.rst index 4867c7bd7164..2b63c410f694 100644 --- a/docs/source/python/api/dataset.rst +++ b/docs/source/python/api/dataset.rst @@ -61,6 +61,7 @@ Classes FileSystemFactoryOptions FileSystemDatasetFactory UnionDataset + FilteredDataset Fragment FragmentScanOptions TaggedRecordBatch diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 4820027ee821..212023696d66 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -182,6 +182,7 @@ cdef class Dataset(_Weakrefable): classes = { 'union': UnionDataset, + 'filtered': FilteredDataset, 'filesystem': FileSystemDataset, 'in-memory': InMemoryDataset, } @@ -844,6 +845,17 @@ cdef class Dataset(_Weakrefable): filtered_dataset._scan_options = dict(filter=new_filter) return filtered_dataset + def _with_filter_attached(self): + """ + Return a Dataset which carries this Dataset's filter, if any, in its + C++ representation, so that it survives being combined with other + Datasets (e.g. in a UnionDataset). + """ + current_filter = self._scan_options.get("filter") + if current_filter is None: + return self + return FilteredDataset(Dataset.wrap(self.wrapped), current_filter) + def sort_by(self, sorting, **kwargs): """ Sort the Dataset by one or multiple columns. @@ -1097,6 +1109,51 @@ cdef class UnionDataset(Dataset): return [Dataset.wrap(children[i]) for i in range(children.size())] +cdef class FilteredDataset(Dataset): + """ + A Dataset viewing only the rows of a child dataset that match a filter. + + Unlike :meth:`Dataset.filter`, the filter is part of the dataset itself, so + it is preserved when this dataset is unioned with datasets that have + different filters. Rows for which the filter evaluates to null are dropped. + + Parameters + ---------- + child : Dataset + The dataset to filter. Any filter already applied to it with + :meth:`Dataset.filter` is combined with ``filter``. + filter : Expression + A boolean expression referencing fields of the child's schema. + """ + + cdef: + CFilteredDataset* filtered_dataset + + def __init__(self, Dataset child not None, Expression filter not None): + current_filter = child._scan_options.get("filter") + if current_filter is not None: + filter = current_filter & filter + self.init( GetResultValue( + CFilteredDataset.Make(child.wrapped, ( filter).unwrap()))) + + cdef void init(self, const shared_ptr[CDataset]& sp): + Dataset.init(self, sp) + self.filtered_dataset = sp.get() + + def __reduce__(self): + return FilteredDataset, (self.child, self.filter) + + @property + def child(self): + """The unfiltered child Dataset.""" + return Dataset.wrap(self.filtered_dataset.child()) + + @property + def filter(self): + """The filter applied to the child Dataset.""" + return Expression.wrap(self.filtered_dataset.filter()) + + cdef class FileSystemDataset(Dataset): """ A Dataset of file fragments. diff --git a/python/pyarrow/dataset.py b/python/pyarrow/dataset.py index f696d5f03ba5..49b1c33f1fab 100644 --- a/python/pyarrow/dataset.py +++ b/python/pyarrow/dataset.py @@ -48,6 +48,7 @@ Scanner, TaggedRecordBatch, UnionDataset, + FilteredDataset, UnionDatasetFactory, WrittenFile, get_partition_keys, @@ -500,16 +501,9 @@ def _union_dataset(children, schema=None, **kwargs): # unify the children datasets' schemas schema = pa.unify_schemas([child.schema for child in children]) - for child in children: - if getattr(child, "_scan_options", None): - raise ValueError( - "Creating an UnionDataset from filtered or projected Datasets " - "is currently not supported. Union the unfiltered datasets " - "and apply the filter to the resulting union." - ) - - # create datasets with the requested schema - children = [child.replace_schema(schema) for child in children] + # create datasets with the requested schema, keeping each child's own filter + children = [child._with_filter_attached().replace_schema(schema) + for child in children] return UnionDataset(schema, children) diff --git a/python/pyarrow/includes/libarrow_dataset.pxd b/python/pyarrow/includes/libarrow_dataset.pxd index 12cf82733935..42f164ef3ce0 100644 --- a/python/pyarrow/includes/libarrow_dataset.pxd +++ b/python/pyarrow/includes/libarrow_dataset.pxd @@ -154,6 +154,15 @@ cdef extern from "arrow/dataset/api.h" namespace "arrow::dataset" nogil: const CDatasetVector& children() const + cdef cppclass CFilteredDataset "arrow::dataset::FilteredDataset"( + CDataset): + @staticmethod + CResult[shared_ptr[CFilteredDataset]] Make(shared_ptr[CDataset] child, + CExpression filter) + + const shared_ptr[CDataset]& child() const + const CExpression& filter() const + cdef cppclass CInspectOptions "arrow::dataset::InspectOptions": int fragments CField.CMergeOptions field_merge_options diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index fd5a418aaf2d..23208bf923af 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -5627,8 +5627,96 @@ def test_union_dataset_filter(tempdir, dstype): filtered_ds1 = ds1.filter(pc.field("colA") < 3) filtered_ds2 = ds2.filter(pc.field("colA") < 10) - with pytest.raises(ValueError, match="currently not supported"): - ds.dataset((filtered_ds1, filtered_ds2)) + union_of_filtered = ds.dataset((filtered_ds1, filtered_ds2)) + assert union_of_filtered.to_table() == pa.table({ + "colA": [1, 2, 9], + "col2": ["a", "b", "h"] + }) + assert union_of_filtered.count_rows() == 3 + assert union_of_filtered.filter(pc.field("col2") != "a").to_table() == pa.table({ + "colA": [2, 9], + "col2": ["b", "h"] + }) + assert all(isinstance(child, ds.FilteredDataset) + for child in union_of_filtered.children) + + # Each child keeps its own filter, even where they overlap or select nulls. + t3 = pa.table({"colA": [1, None, 9], "col2": ["x", "y", "z"]}) + union_of_filtered = ds.dataset([ + ds.dataset(t3).filter(pc.field("colA") < 3), + ds.dataset(t3).filter(pc.field("colA") > 3), + ds.dataset(t3).filter(pc.field("colA").is_null()), + ]) + assert union_of_filtered.to_table() == pa.table({ + "colA": [1, 9, None], + "col2": ["x", "z", "y"] + }) + + +@pytest.mark.parquet +def test_filtered_dataset(tempdir, pickle_module): + table = pa.table({"part": [1, 1, 2, 2], "value": [1, 2, 3, 4]}) + ds.write_dataset(table, tempdir, format="parquet", partitioning=["part"]) + dataset = ds.dataset(tempdir, format="parquet", partitioning=["part"]) + + filtered = ds.FilteredDataset(dataset, pc.field("value") > 1) + assert filtered.filter.equals(pc.field("value") > 1) + assert filtered.child.to_table().num_rows == 4 + assert filtered.to_table().sort_by("value")["value"].to_pylist() == [2, 3, 4] + assert filtered.count_rows() == 3 + + # Fragments that cannot match the filter are pruned. + on_part = ds.FilteredDataset(dataset, pc.field("part") == 2) + assert len(list(on_part.get_fragments())) == 1 + assert on_part.to_table()["value"].to_pylist() == [3, 4] + + # A lazy filter on the child is folded into the FilteredDataset's filter. + combined = ds.FilteredDataset(dataset.filter(pc.field("value") < 4), + pc.field("value") > 1) + assert combined.to_table().sort_by("value")["value"].to_pylist() == [2, 3] + + assert pickle_module.loads( + pickle_module.dumps(filtered)).to_table().num_rows == 3 + + with pytest.raises(pa.ArrowInvalid): + ds.FilteredDataset(dataset, pc.field("missing") > 1) + with pytest.raises(TypeError): + ds.FilteredDataset(dataset, pc.field("value") + 1) + + +@pytest.mark.parquet +def test_union_of_filtered_datasets_pushes_down_outer_filter(tempdir): + """ + When scanning a union of filtered datasets with a predicate, + both the outer and inner predicates must be applied. + """ + # Write a Hive partitioned dataset with data covering 1000 days, + # with 10 rows per day. + day = pa.array([i // 10 for i in range(10_000)]) + for (i, part) in enumerate(["A", "B"]): + part_dir = tempdir / f"part={part}" + part_dir.mkdir() + table = pa.table({ + "day": day, + "v": pc.cast(pc.multiply(day, i + 2), "float64"), + }) + pq.write_table(table, part_dir / "part.parquet", row_group_size=500) + base = ds.dataset(tempdir, format="parquet", partitioning="hive") + + low_days = base.filter(pc.field("day") < 300) + high_days = base.filter(pc.field("day") >= 700) + union = ds.dataset([low_days, high_days]) + outer_filter = (pc.field("day") >= 250) & (pc.field("day") < 750) + + expected = pa.concat_tables( + [low_days.to_table(), high_days.to_table()]).filter(outer_filter) + assert expected.num_rows == 2_000 + result = union.filter(outer_filter) + sorting = [ + ("part", "ascending", "at_end"), + ("day", "ascending", "at_end"), + ] + assert result.to_table().sort_by(sorting).equals(expected.sort_by(sorting)) def test_parquet_dataset_filter(tempdir): From 9bcd294d8238c8e824abafc4b331aabb728de44f Mon Sep 17 00:00:00 2001 From: Adam Reeve Date: Wed, 18 Nov 2026 17:40:58 +1300 Subject: [PATCH 2/4] Add C++ type forwards --- cpp/src/arrow/dataset/type_fwd.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/src/arrow/dataset/type_fwd.h b/cpp/src/arrow/dataset/type_fwd.h index d58781e038de..c2db45b53cdd 100644 --- a/cpp/src/arrow/dataset/type_fwd.h +++ b/cpp/src/arrow/dataset/type_fwd.h @@ -67,6 +67,9 @@ enum class ExistingDataBehavior : int8_t { class InMemoryDataset; +class FilteredDataset; +class FilteredFragment; + class CsvFileFormat; class CsvFileWriter; class CsvFileWriteOptions; From 8d2b1ecae7104d8970df5fb003ce9a4ce9c41a15 Mon Sep 17 00:00:00 2001 From: Adam Reeve Date: Wed, 18 Nov 2026 17:41:56 +1300 Subject: [PATCH 3/4] Fix filter method shadowing --- python/pyarrow/_dataset.pyx | 4 ++-- python/pyarrow/tests/test_dataset.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx index 212023696d66..7579a951af81 100644 --- a/python/pyarrow/_dataset.pyx +++ b/python/pyarrow/_dataset.pyx @@ -1141,7 +1141,7 @@ cdef class FilteredDataset(Dataset): self.filtered_dataset = sp.get() def __reduce__(self): - return FilteredDataset, (self.child, self.filter) + return FilteredDataset, (self.child, self.filter_expression) @property def child(self): @@ -1149,7 +1149,7 @@ cdef class FilteredDataset(Dataset): return Dataset.wrap(self.filtered_dataset.child()) @property - def filter(self): + def filter_expression(self): """The filter applied to the child Dataset.""" return Expression.wrap(self.filtered_dataset.filter()) diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 23208bf923af..62899bb3c6f5 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -5660,11 +5660,15 @@ def test_filtered_dataset(tempdir, pickle_module): dataset = ds.dataset(tempdir, format="parquet", partitioning=["part"]) filtered = ds.FilteredDataset(dataset, pc.field("value") > 1) - assert filtered.filter.equals(pc.field("value") > 1) + assert filtered.filter_expression.equals(pc.field("value") > 1) assert filtered.child.to_table().num_rows == 4 assert filtered.to_table().sort_by("value")["value"].to_pylist() == [2, 3, 4] assert filtered.count_rows() == 3 + # The inherited Dataset.filter method applies a further filter. + refiltered = filtered.filter(pc.field("value") < 4) + assert refiltered.to_table().sort_by("value")["value"].to_pylist() == [2, 3] + # Fragments that cannot match the filter are pruned. on_part = ds.FilteredDataset(dataset, pc.field("part") == 2) assert len(list(on_part.get_fragments())) == 1 From 3ccc9df3de97b0b67eef33ac673f500a4f799353 Mon Sep 17 00:00:00 2001 From: Adam Reeve Date: Wed, 18 Nov 2026 18:15:24 +1300 Subject: [PATCH 4/4] Fix handling of the __filename field --- cpp/src/arrow/dataset/dataset.cc | 4 +++- cpp/src/arrow/dataset/dataset_test.cc | 4 ++++ python/pyarrow/tests/test_dataset.py | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/dataset/dataset.cc b/cpp/src/arrow/dataset/dataset.cc index 0b81088a8bd1..f5ab25b1150a 100644 --- a/cpp/src/arrow/dataset/dataset.cc +++ b/cpp/src/arrow/dataset/dataset.cc @@ -338,7 +338,9 @@ Status FilteredFragment::ClearCachedMetadata() { } std::string FilteredFragment::ToString() const { - return "filtered(" + fragment_->ToString() + ", " + filter_.ToString() + ")"; + // The scanner uses ToString() for the __filename augmented field, so this must + // match the wrapped fragment. + return fragment_->ToString(); } Result FilteredFragment::ScanBatchesAsync( diff --git a/cpp/src/arrow/dataset/dataset_test.cc b/cpp/src/arrow/dataset/dataset_test.cc index 613785332bfe..84ba9df42314 100644 --- a/cpp/src/arrow/dataset/dataset_test.cc +++ b/cpp/src/arrow/dataset/dataset_test.cc @@ -363,6 +363,10 @@ TEST_F(TestFilteredDataset, FilterOnPartitionField) { ASSERT_OK_AND_ASSIGN(auto fragment_it, first->GetFragments()); ASSERT_OK_AND_ASSIGN(auto fragments, fragment_it.ToVector()); ASSERT_EQ(1, fragments.size()); + ASSERT_EQ("filtered", fragments[0]->type_name()); + // ToString() is used for the __filename field, so must match the wrapped fragment. + ASSERT_EQ(checked_cast(*fragments[0]).fragment()->ToString(), + fragments[0]->ToString()); ASSERT_OK_AND_ASSIGN(auto dataset, UnionDataset::Make(schema_, {first, second})); ASSERT_OK_AND_ASSIGN(auto builder, dataset->NewScan()); diff --git a/python/pyarrow/tests/test_dataset.py b/python/pyarrow/tests/test_dataset.py index 62899bb3c6f5..e3b831df2a0b 100644 --- a/python/pyarrow/tests/test_dataset.py +++ b/python/pyarrow/tests/test_dataset.py @@ -5665,6 +5665,10 @@ def test_filtered_dataset(tempdir, pickle_module): assert filtered.to_table().sort_by("value")["value"].to_pylist() == [2, 3, 4] assert filtered.count_rows() == 3 + # The __filename field holds the paths of the underlying files. + filenames = filtered.to_table(columns=["__filename"])["__filename"] + assert set(filenames.to_pylist()) == {f.path for f in dataset.get_fragments()} + # The inherited Dataset.filter method applies a further filter. refiltered = filtered.filter(pc.field("value") < 4) assert refiltered.to_table().sort_by("value")["value"].to_pylist() == [2, 3]