Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 124 additions & 0 deletions cpp/src/arrow/dataset/dataset.cc
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include <utility>

#include "arrow/acero/util.h"
#include "arrow/compute/api_vector.h"
#include "arrow/dataset/dataset.h"
#include "arrow/dataset/dataset_internal.h"
#include "arrow/dataset/projector.h"
Expand Down Expand Up @@ -321,6 +322,129 @@ Result<FragmentIterator> UnionDataset::GetFragmentsImpl(compute::Expression pred
return GetFragmentsFromDatasets(children_, predicate);
}

FilteredFragment::FilteredFragment(std::shared_ptr<Fragment> fragment,
compute::Expression filter)
: Fragment(fragment->partition_expression(), /*physical_schema=*/nullptr),
fragment_(std::move(fragment)),
filter_(std::move(filter)) {}

Result<std::shared_ptr<Schema>> FilteredFragment::ReadPhysicalSchemaImpl() {
return fragment_->ReadPhysicalSchema();
}

Status FilteredFragment::ClearCachedMetadata() {
RETURN_NOT_OK(Fragment::ClearCachedMetadata());
return fragment_->ClearCachedMetadata();
}

std::string FilteredFragment::ToString() const {
// The scanner uses ToString() for the __filename augmented field, so this must
// match the wrapped fragment.
return fragment_->ToString();
}

Result<RecordBatchGenerator> FilteredFragment::ScanBatchesAsync(
const std::shared_ptr<ScanOptions>& options) {
const auto& dataset_schema = options->dataset_schema;
ARROW_ASSIGN_OR_RAISE(auto filter, filter_.Bind(*dataset_schema));
ARROW_ASSIGN_OR_RAISE(filter,
SimplifyWithGuarantee(std::move(filter), partition_expression_));
if (!filter.IsSatisfiable()) {
return MakeEmptyGenerator<std::shared_ptr<RecordBatch>>();
}

// Combine any existing filter in the scan options with the fragment's filter,
// so the wrapped fragment can skip data that cannot match.
auto fragment_options = std::make_shared<ScanOptions>(*options);
ARROW_ASSIGN_OR_RAISE(fragment_options->filter,
compute::and_(options->filter, filter).Bind(*dataset_schema));
ARROW_ASSIGN_OR_RAISE(auto batches, fragment_->ScanBatchesAsync(fragment_options));

// The filter might simplify to true if it only references partition fields,
// in which case we can skip filtering batches.
if (filter == compute::literal(true)) {
return batches;
}

// Apply the fragment's filter to scanned rows.
// The filter in the scan options is only used for data pruning at this point
// and will be used to filter rows later, so don't duplicate that work.
auto guarantee = partition_expression_;
auto pool = options->pool;
return MakeMappedGenerator(
std::move(batches),
[filter, dataset_schema, guarantee, pool](const std::shared_ptr<RecordBatch>& batch)
-> Result<std::shared_ptr<RecordBatch>> {
compute::ExecContext exec_context(pool);
ARROW_ASSIGN_OR_RAISE(auto exec_batch,
compute::MakeExecBatch(*dataset_schema, batch, guarantee));
ARROW_ASSIGN_OR_RAISE(Datum mask, compute::ExecuteScalarExpression(
filter, exec_batch, &exec_context));
if (mask.is_scalar()) {
const auto& selected = mask.scalar_as<BooleanScalar>();
return selected.is_valid && selected.value ? batch : batch->Slice(0, 0);
}
ARROW_ASSIGN_OR_RAISE(
Datum filtered,
compute::Filter(batch, mask, compute::FilterOptions::Defaults(),
&exec_context));
return filtered.record_batch();
});
}

Future<std::optional<int64_t>> FilteredFragment::CountRows(
compute::Expression predicate, const std::shared_ptr<ScanOptions>& options) {
ARROW_ASSIGN_OR_RAISE(
auto combined,
compute::and_(std::move(predicate), filter_).Bind(*options->dataset_schema));
return fragment_->CountRows(std::move(combined), options);
}

FilteredDataset::FilteredDataset(std::shared_ptr<Dataset> child,
compute::Expression filter)
: Dataset(child->schema(), child->partition_expression()),
child_(std::move(child)),
filter_(std::move(filter)) {}

Result<std::shared_ptr<FilteredDataset>> FilteredDataset::Make(
std::shared_ptr<Dataset> child, compute::Expression filter) {
// Verify the filter produces a bool
ARROW_ASSIGN_OR_RAISE(auto bound, filter.Bind(*child->schema()));
if (!bound.type()->Equals(*boolean())) {
return Status::TypeError("FilteredDataset filter must be boolean, got ",
*bound.type());
}
// Store the unbound filter so it can be bound on-demand on the unified schema of a
// UnionDataset.
return std::shared_ptr<FilteredDataset>(
new FilteredDataset(std::move(child), std::move(filter)));
}

Result<std::shared_ptr<Dataset>> FilteredDataset::ReplaceSchema(
std::shared_ptr<Schema> schema) const {
ARROW_ASSIGN_OR_RAISE(auto child, child_->ReplaceSchema(std::move(schema)));
ARROW_ASSIGN_OR_RAISE(auto filtered, Make(std::move(child), filter_));
return std::static_pointer_cast<Dataset>(std::move(filtered));
}

Result<FragmentIterator> FilteredDataset::GetFragmentsImpl(
compute::Expression predicate) {
// Compute a combined predicate that can be used for predicate-pushdown to select
// fragments from the child Dataset.
ARROW_ASSIGN_OR_RAISE(auto combined_predicate,
compute::and_(std::move(predicate), filter_).Bind(*schema_));
ARROW_ASSIGN_OR_RAISE(auto fragments,
child_->GetFragments(std::move(combined_predicate)));

// Convert the child's fragments to FilteredFragments that apply the additional filter.
auto filter = filter_;
return MakeMapIterator(
[filter](std::shared_ptr<Fragment> fragment) -> std::shared_ptr<Fragment> {
return std::make_shared<FilteredFragment>(std::move(fragment), filter);
},
std::move(fragments));
}

namespace {

struct MissingColumn {
Expand Down
61 changes: 61 additions & 0 deletions cpp/src/arrow/dataset/dataset.h
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,67 @@ class ARROW_DS_EXPORT UnionDataset : public Dataset {
friend class UnionDatasetFactory;
};

/// \brief A Fragment which wraps another Fragment and applies a filter.
class ARROW_DS_EXPORT FilteredFragment : public Fragment {
public:
FilteredFragment(std::shared_ptr<Fragment> fragment, compute::Expression filter);

Result<RecordBatchGenerator> ScanBatchesAsync(
const std::shared_ptr<ScanOptions>& options) override;

Future<std::optional<int64_t>> CountRows(
compute::Expression predicate,
const std::shared_ptr<ScanOptions>& options) override;

Status ClearCachedMetadata() override;

std::string type_name() const override { return "filtered"; }

std::string ToString() const override;

const std::shared_ptr<Fragment>& fragment() const { return fragment_; }

const compute::Expression& filter() const { return filter_; }

protected:
Result<std::shared_ptr<Schema>> ReadPhysicalSchemaImpl() override;

std::shared_ptr<Fragment> fragment_;
compute::Expression filter_;
};

/// \brief A Dataset which wraps another Dataset and applies a filter.
///
/// Unlike a filter in ScanOptions, the filter is part of the Dataset itself, so it
/// is preserved when the Dataset is combined with others, e.g. as a child of a
/// UnionDataset whose other children have different filters.
class ARROW_DS_EXPORT FilteredDataset : public Dataset {
Comment thread
adamreeve marked this conversation as resolved.
Resolved
public:
/// \brief Construct a FilteredDataset.
///
/// \param[in] child the Dataset to filter.
/// \param[in] filter a boolean expression used to filter rows of the child Dataset.
static Result<std::shared_ptr<FilteredDataset>> Make(std::shared_ptr<Dataset> child,
compute::Expression filter);

const std::shared_ptr<Dataset>& child() const { return child_; }

const compute::Expression& filter() const { return filter_; }

std::string type_name() const override { return "filtered"; }

Result<std::shared_ptr<Dataset>> ReplaceSchema(
std::shared_ptr<Schema> schema) const override;

protected:
Result<FragmentIterator> GetFragmentsImpl(compute::Expression predicate) override;

FilteredDataset(std::shared_ptr<Dataset> child, compute::Expression filter);

std::shared_ptr<Dataset> child_;
compute::Expression filter_;
};

/// @}

} // namespace dataset
Expand Down
130 changes: 130 additions & 0 deletions cpp/src/arrow/dataset/dataset_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,136 @@ TEST_F(TestUnionDataset, TrivialScan) {
AssertDatasetEquals(reader.get(), dataset.get());
}

class TestFilteredDataset : public DatasetFixtureMixin {};

TEST_F(TestFilteredDataset, UnionOfDifferentFilters) {
SetSchema({field("i", int32())});
// Include a null value to test handling when the filter evaluates to null.
auto batch = RecordBatchFromJSON(schema_, R"([[1], [2], [3], [null], [4]])");
auto child = std::make_shared<InMemoryDataset>(schema_, RecordBatchVector{batch});

ASSERT_OK_AND_ASSIGN(auto small,
FilteredDataset::Make(child, compute::less(compute::field_ref("i"),
compute::literal(2))));
ASSERT_OK_AND_ASSIGN(
auto large, FilteredDataset::Make(child, compute::greater(compute::field_ref("i"),
compute::literal(3))));
ASSERT_OK_AND_ASSIGN(auto dataset, UnionDataset::Make(schema_, {small, large}));

ASSERT_OK_AND_ASSIGN(auto builder, dataset->NewScan());
ASSERT_OK_AND_ASSIGN(auto scanner, builder->Finish());
ASSERT_OK_AND_ASSIGN(auto table, scanner->ToTable());
AssertTablesEqual(*TableFromJSON(schema_, {R"([[1], [4]])"}), *table,
/*same_chunk_layout=*/false);
ASSERT_OK_AND_EQ(2, scanner->CountRows());

// A filter on the union applies on top of each child's filter.
ASSERT_OK_AND_ASSIGN(builder, dataset->NewScan());
ASSERT_OK(
builder->Filter(compute::not_equal(compute::field_ref("i"), compute::literal(1))));
ASSERT_OK_AND_ASSIGN(scanner, builder->Finish());
ASSERT_OK_AND_ASSIGN(table, scanner->ToTable());
AssertTablesEqual(*TableFromJSON(schema_, {R"([[4]])"}), *table,
/*same_chunk_layout=*/false);
ASSERT_OK_AND_EQ(1, scanner->CountRows());
}

TEST_F(TestFilteredDataset, ReplaceSchema) {
SetSchema({field("i", int32())});
auto batch = RecordBatchFromJSON(schema_, R"([[1], [2]])");
auto child = std::make_shared<InMemoryDataset>(schema_, RecordBatchVector{batch});
ASSERT_OK_AND_ASSIGN(
auto filtered, FilteredDataset::Make(child, compute::equal(compute::field_ref("i"),
compute::literal(2))));

auto wider = schema({field("i", int32()), field("j", utf8())});
ASSERT_OK_AND_ASSIGN(auto replaced, filtered->ReplaceSchema(wider));
ASSERT_EQ("filtered", replaced->type_name());
ASSERT_OK_AND_ASSIGN(auto builder, replaced->NewScan());
ASSERT_OK_AND_ASSIGN(auto scanner, builder->Finish());
ASSERT_OK_AND_ASSIGN(auto table, scanner->ToTable());
AssertTablesEqual(*TableFromJSON(wider, {R"([[2, null]])"}), *table,
/*same_chunk_layout=*/false);

// Check we get an error if attempting to replace the schema with one
// that the filter won't bind against.
EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("No match for FieldRef"),
filtered->ReplaceSchema(schema({field("j", utf8())})));
}

TEST_F(TestFilteredDataset, FilterOnPartitionField) {
SetSchema({field("part", int32()), field("i", int32())});
auto make_fragment = [this](int32_t part, const std::string& json) {
return std::make_shared<InMemoryFragment>(
schema_, RecordBatchVector{RecordBatchFromJSON(schema_, json)},
compute::equal(compute::field_ref("part"), compute::literal(part)));
};
auto child = std::make_shared<FragmentDataset>(
schema_, FragmentVector{make_fragment(1, R"([[1, 10], [1, 11]])"),
make_fragment(2, R"([[2, 20], [2, 21]])"),
make_fragment(3, R"([[3, 30], [3, 31]])")});

// This filter is fully guaranteed by the matching fragment's partition expression,
// so it simplifies to true and no per-batch filtering is needed.
ASSERT_OK_AND_ASSIGN(
auto first, FilteredDataset::Make(child, compute::equal(compute::field_ref("part"),
compute::literal(1))));
// This filter only partially simplifies, leaving i > 20 to apply to each batch.
ASSERT_OK_AND_ASSIGN(
auto second,
FilteredDataset::Make(
child, compute::and_(
compute::equal(compute::field_ref("part"), compute::literal(2)),
compute::greater(compute::field_ref("i"), compute::literal(20)))));

// Fragments whose partition expression contradicts the filter are pruned.
ASSERT_OK_AND_ASSIGN(auto fragment_it, first->GetFragments());
ASSERT_OK_AND_ASSIGN(auto fragments, fragment_it.ToVector());
ASSERT_EQ(1, fragments.size());
ASSERT_EQ("filtered", fragments[0]->type_name());
// ToString() is used for the __filename field, so must match the wrapped fragment.
ASSERT_EQ(checked_cast<const FilteredFragment&>(*fragments[0]).fragment()->ToString(),
fragments[0]->ToString());

ASSERT_OK_AND_ASSIGN(auto dataset, UnionDataset::Make(schema_, {first, second}));
ASSERT_OK_AND_ASSIGN(auto builder, dataset->NewScan());
ASSERT_OK_AND_ASSIGN(auto scanner, builder->Finish());
ASSERT_OK_AND_ASSIGN(auto table, scanner->ToTable());
AssertTablesEqual(*TableFromJSON(schema_, {R"([[1, 10], [1, 11], [2, 21]])"}), *table,
/*same_chunk_layout=*/false);
ASSERT_OK_AND_EQ(3, scanner->CountRows());
}

TEST_F(TestFilteredDataset, ScalarFilterResult) {
// The batches lack column j, so it is filled with a null scalar when filtering and
// filters that only reference j produce a scalar mask.
SetSchema({field("i", int32()), field("j", utf8())});
auto batch = RecordBatchFromJSON(schema({field("i", int32())}), R"([[1], [2]])");
auto child = std::make_shared<InMemoryDataset>(schema_, RecordBatchVector{batch});

// A true mask keeps every row.
ASSERT_OK_AND_ASSIGN(
auto keep_all,
FilteredDataset::Make(child, compute::is_null(compute::field_ref("j"))));
// A null mask drops every row.
ASSERT_OK_AND_ASSIGN(auto drop_null, FilteredDataset::Make(
child, compute::equal(compute::field_ref("j"),
compute::literal("x"))));
// A false mask drops every row.
ASSERT_OK_AND_ASSIGN(
auto drop_false,
FilteredDataset::Make(child, compute::is_valid(compute::field_ref("j"))));

ASSERT_OK_AND_ASSIGN(auto dataset,
UnionDataset::Make(schema_, {keep_all, drop_null, drop_false}));
ASSERT_OK_AND_ASSIGN(auto builder, dataset->NewScan());
ASSERT_OK_AND_ASSIGN(auto scanner, builder->Finish());
ASSERT_OK_AND_ASSIGN(auto table, scanner->ToTable());
AssertTablesEqual(*TableFromJSON(schema_, {R"([[1, null], [2, null]])"}), *table,
/*same_chunk_layout=*/false);
ASSERT_OK_AND_EQ(2, scanner->CountRows());
}

TEST(TestProjector, CheckProjectable) {
struct Assert {
explicit Assert(FieldVector from) : from_(from) {}
Expand Down
3 changes: 3 additions & 0 deletions cpp/src/arrow/dataset/type_fwd.h
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@ enum class ExistingDataBehavior : int8_t {

class InMemoryDataset;

class FilteredDataset;
class FilteredFragment;

class CsvFileFormat;
class CsvFileWriter;
class CsvFileWriteOptions;
Expand Down
1 change: 1 addition & 0 deletions docs/source/python/api/dataset.rst
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ Classes
FileSystemFactoryOptions
FileSystemDatasetFactory
UnionDataset
FilteredDataset
Fragment
FragmentScanOptions
TaggedRecordBatch
Expand Down
Loading
Loading