Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 20 additions & 18 deletions src/iceberg/expression/strict_metrics_evaluator.cc
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@

#include "iceberg/expression/strict_metrics_evaluator.h"

#include <ranges>
#include <algorithm>
#include <optional>

#include "iceberg/expression/binder.h"
#include "iceberg/expression/expression_visitor.h"
Expand Down Expand Up @@ -400,28 +401,29 @@ class StrictMetricsVisitor : public BoundVisitor<bool> {
}
lower_bound = std::move(lower);
}
auto literals_view = literal_set | std::views::filter([&](const Literal& lit) {
return lower_bound.has_value() && lower_bound.value() <= lit;
});
// if all values are less than lower bound, rows must
// match (notIn).
if (lower_bound.has_value() && literals_view.empty()) {
return kRowsMustMatch;
}

std::optional<Literal> upper_bound;
auto upper_it = data_file_.upper_bounds.find(id);
if (upper_it != data_file_.upper_bounds.cend()) {
ICEBERG_ASSIGN_OR_RAISE(auto upper, ParseBound(expr, upper_it->second));
auto filtered_view = literals_view | std::views::filter([&](const Literal& lit) {
return upper >= lit;
});
if (filtered_view.empty()) {
// if all remaining values are greater than upper bound,
// rows must match
// (notIn).
return kRowsMustMatch;
}
upper_bound = std::move(upper);
}

if (!lower_bound.has_value() && !upper_bound.has_value()) {
return kRowsMightNotMatch;
}

// Treat a missing bound as unbounded on that side. If no value in literal_set
// falls within [lower_bound, upper_bound], no row from file can be in the set,
// so all rows satisfy NOT IN.
auto within_bounds = [&](const Literal& lit) {
return (!lower_bound.has_value() || *lower_bound <= lit) &&
(!upper_bound.has_value() || *upper_bound >= lit);
};
if (std::ranges::none_of(literal_set, within_bounds)) {
return kRowsMustMatch;
}

return kRowsMightNotMatch;
}

Expand Down
80 changes: 80 additions & 0 deletions src/iceberg/test/strict_metrics_evaluator_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -944,4 +944,84 @@ TEST(StrictMetricsEvaluatorRegressionTest, MissingNanCountForField) {
evaluate(Expressions::GreaterThan("no_nan_stats", Literal::Double(-1.0)));
}

TEST(StrictMetricsEvaluatorRegressionTest, NotInWithMissingLowerBound) {
auto schema = std::make_shared<Schema>(
std::vector<SchemaField>{SchemaField::MakeRequired(1, "id", int32())},
/*schema_id=*/0);

auto data_file = std::make_shared<DataFile>();
data_file->file_path = "missing_lower_bound.parquet";
data_file->file_format = FileFormatType::kParquet;
data_file->record_count = 1;
data_file->value_counts = {{1, 1L}};
data_file->null_value_counts = {{1, 0L}};
data_file->upper_bounds = {{1, Literal::Int(50).Serialize().value()}};

auto expect_eval = [&](const std::shared_ptr<Expression>& expr, bool expected) {
ICEBERG_UNWRAP_OR_FAIL(auto eval, StrictMetricsEvaluator::Make(expr, schema, true));
ICEBERG_UNWRAP_OR_FAIL(auto result, eval->Evaluate(*data_file));
EXPECT_EQ(result, expected) << expr->ToString();
};

// Lower bound is absent (unbounded below), so any literal <= 50 may exist in the file.
expect_eval(Expressions::NotIn("id", {Literal::Int(40), Literal::Int(50)}),
kRowsMightNotMatch);
expect_eval(Expressions::NotIn("id", {Literal::Int(20), Literal::Int(30)}),
kRowsMightNotMatch);
expect_eval(Expressions::NotIn("id", {Literal::Int(50), Literal::Int(70)}),
kRowsMightNotMatch);
// All literals are strictly above upper_bound, so no row can be in the set.
expect_eval(Expressions::NotIn("id", {Literal::Int(51), Literal::Int(70)}),
kRowsMustMatch);
}

TEST(StrictMetricsEvaluatorRegressionTest, NotInWithMissingUpperBound) {
auto schema = std::make_shared<Schema>(
std::vector<SchemaField>{SchemaField::MakeRequired(1, "id", int32())},
/*schema_id=*/0);

auto data_file = std::make_shared<DataFile>();
data_file->file_path = "missing_upper_bound.parquet";
data_file->file_format = FileFormatType::kParquet;
data_file->record_count = 1;
data_file->value_counts = {{1, 1L}};
data_file->null_value_counts = {{1, 0L}};
data_file->lower_bounds = {{1, Literal::Int(50).Serialize().value()}};

auto expect_eval = [&](const std::shared_ptr<Expression>& expr, bool expected) {
ICEBERG_UNWRAP_OR_FAIL(auto eval, StrictMetricsEvaluator::Make(expr, schema, true));
ICEBERG_UNWRAP_OR_FAIL(auto result, eval->Evaluate(*data_file));
EXPECT_EQ(result, expected) << expr->ToString();
};

// Upper bound is absent (unbounded above), so any literal >= 50 may exist in the file.
expect_eval(Expressions::NotIn("id", {Literal::Int(50), Literal::Int(60)}),
kRowsMightNotMatch);
expect_eval(Expressions::NotIn("id", {Literal::Int(30), Literal::Int(50)}),
kRowsMightNotMatch);
// All literals are strictly below lower_bound, so no row can be in the set.
expect_eval(Expressions::NotIn("id", {Literal::Int(20), Literal::Int(49)}),
kRowsMustMatch);
}

TEST(StrictMetricsEvaluatorRegressionTest, NotInWithMissingBounds) {
auto schema = std::make_shared<Schema>(
std::vector<SchemaField>{SchemaField::MakeRequired(1, "id", int32())},
/*schema_id=*/0);

auto data_file = std::make_shared<DataFile>();
data_file->file_path = "missing_bounds.parquet";
data_file->file_format = FileFormatType::kParquet;
data_file->record_count = 1;
data_file->value_counts = {{1, 1L}};
data_file->null_value_counts = {{1, 0L}};

ICEBERG_UNWRAP_OR_FAIL(
auto eval,
StrictMetricsEvaluator::Make(
Expressions::NotIn("id", {Literal::Int(40), Literal::Int(50)}), schema, true));
ICEBERG_UNWRAP_OR_FAIL(auto result, eval->Evaluate(*data_file));
EXPECT_EQ(result, kRowsMightNotMatch);
}

} // namespace iceberg
Loading