Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cpp/src/arrow/engine/substrait/extension_set.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1079,9 +1079,9 @@ ExtensionIdRegistry::SubstraitAggregateToArrow DecodeBasicAggregate(
}
int ddof;
if (prefs[0] == "POPULATION") {
ddof = 1;
} else if (prefs[0] == "SAMPLE") {
ddof = 0;
} else if (prefs[0] == "SAMPLE") {
ddof = 1;
} else {
return Status::Invalid("unknown distribution preference ", prefs[0]);
}
Expand Down
331 changes: 331 additions & 0 deletions cpp/src/arrow/engine/substrait/serde_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3837,6 +3837,119 @@ TEST(SubstraitRoundTrip, AggregateRelOptions) {
internal::SubstraitFromJSON("Plan", substrait_json,
/*ignore_unknown_fields=*/false));
auto output_schema = schema({field("keys", int32()), field("aggregates", float64())});
auto expected_table = TableFromJSON(output_schema, {R"([
[10, 100],
[20, 100],
[30, 0]
])"});

NamedTableProvider table_provider = AlwaysProvideSameTable(std::move(input_table));

ConversionOptions conversion_options;
conversion_options.named_table_provider = std::move(table_provider);

CheckRoundTripResult(std::move(expected_table), buf,
/*include_columns=*/{}, conversion_options);
}

TEST(SubstraitRoundTrip, AggregateRelOptionsSample) {
auto dummy_schema =
schema({field("A", int32()), field("B", int32()), field("C", int32())});

// Same dataset as AggregateRelOptions, but requesting SAMPLE distribution.
auto input_table = TableFromJSON(dummy_schema, {R"([
[10, 1, 10],
[20, 2, 10],
[30, 3, 20],
[30, 1, 20],
[20, 2, 30],
[10, 3, 30]
])"});

std::string substrait_json = R"({
"version": { "major_number": 9999, "minor_number": 9999, "patch_number": 9999 },
"relations": [{
"rel": {
"aggregate": {
"input": {
"read": {
"base_schema": {
"names": ["A", "B", "C"],
"struct": {
"types": [{
"i32": {}
}, {
"i32": {}
}, {
"i32": {}
}]
}
},
"namedTable" : {
"names": ["A"]
}
}
},
"groupings": [{
"groupingExpressions": [{
"selection": {
"directReference": {
"structField": {
"field": 0
}
}
}
}]
}],
"measures": [{
"measure": {
"functionReference": 0,
"arguments": [{
"value": {
"selection": {
"directReference": {
"structField": {
"field": 2
}
}
}
}
}],
"options": [{
"name": "distribution",
"preference": [
"SAMPLE"
]
}],
"sorts": [],
"phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT",
"invocation": "AGGREGATION_INVOCATION_ALL",
"outputType": {
"i64": {}
}
}
}]
}
}
}],
"extensionUris": [{
"extension_uri_anchor": 0,
"uri": "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml"
}],
"extensions": [{
"extension_function": {
"extension_uri_reference": 0,
"function_anchor": 0,
"name": "variance"
}
}],
})";

ASSERT_OK_AND_ASSIGN(auto buf,
internal::SubstraitFromJSON("Plan", substrait_json,
/*ignore_unknown_fields=*/false));
auto output_schema = schema({field("keys", int32()), field("aggregates", float64())});
// Sample variance (ddof=1): [10, 30] -> 200, [10, 30] -> 200, [20, 20] -> 0.
auto expected_table = TableFromJSON(output_schema, {R"([
[10, 200],
[20, 200],
Expand All @@ -3852,6 +3965,224 @@ TEST(SubstraitRoundTrip, AggregateRelOptions) {
/*include_columns=*/{}, conversion_options);
}

TEST(SubstraitRoundTrip, AggregateRelVarianceSingleRowGroup) {
auto dummy_schema =
schema({field("A", int32()), field("B", int32()), field("C", int32())});

// Group A=10 has a single row: population variance (ddof=0) is 0.0 there,
// while sample variance (ddof=1) is NULL. See GH-55575.
auto input_table = TableFromJSON(dummy_schema, {R"([
[10, 1, 5],
[20, 2, 10],
[20, 3, 30]
])"});

std::string substrait_json = R"({
"version": { "major_number": 9999, "minor_number": 9999, "patch_number": 9999 },
"relations": [{
"rel": {
"aggregate": {
"input": {
"read": {
"base_schema": {
"names": ["A", "B", "C"],
"struct": {
"types": [{
"i32": {}
}, {
"i32": {}
}, {
"i32": {}
}]
}
},
"namedTable" : {
"names": ["A"]
}
}
},
"groupings": [{
"groupingExpressions": [{
"selection": {
"directReference": {
"structField": {
"field": 0
}
}
}
}]
}],
"measures": [{
"measure": {
"functionReference": 0,
"arguments": [{
"value": {
"selection": {
"directReference": {
"structField": {
"field": 2
}
}
}
}
}],
"options": [{
"name": "distribution",
"preference": [
"POPULATION"
]
}],
"sorts": [],
"phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT",
"invocation": "AGGREGATION_INVOCATION_ALL",
"outputType": {
"i64": {}
}
}
}]
}
}
}],
"extensionUris": [{
"extension_uri_anchor": 0,
"uri": "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml"
}],
"extensions": [{
"extension_function": {
"extension_uri_reference": 0,
"function_anchor": 0,
"name": "variance"
}
}],
})";

ASSERT_OK_AND_ASSIGN(auto buf,
internal::SubstraitFromJSON("Plan", substrait_json,
/*ignore_unknown_fields=*/false));
auto output_schema = schema({field("keys", int32()), field("aggregates", float64())});
auto expected_table = TableFromJSON(output_schema, {R"([
[10, 0],
[20, 100]
])"});

NamedTableProvider table_provider = AlwaysProvideSameTable(std::move(input_table));

ConversionOptions conversion_options;
conversion_options.named_table_provider = std::move(table_provider);

CheckRoundTripResult(std::move(expected_table), buf,
/*include_columns=*/{}, conversion_options);
}

TEST(SubstraitRoundTrip, AggregateRelVarianceSingleRowGroupSample) {
auto dummy_schema =
schema({field("A", int32()), field("B", int32()), field("C", int32())});

// Same dataset as AggregateRelVarianceSingleRowGroup, but requesting SAMPLE
// distribution: the single-row group yields NULL instead of 0.0. See GH-55575.
auto input_table = TableFromJSON(dummy_schema, {R"([
[10, 1, 5],
[20, 2, 10],
[20, 3, 30]
])"});

std::string substrait_json = R"({
"version": { "major_number": 9999, "minor_number": 9999, "patch_number": 9999 },
"relations": [{
"rel": {
"aggregate": {
"input": {
"read": {
"base_schema": {
"names": ["A", "B", "C"],
"struct": {
"types": [{
"i32": {}
}, {
"i32": {}
}, {
"i32": {}
}]
}
},
"namedTable" : {
"names": ["A"]
}
}
},
"groupings": [{
"groupingExpressions": [{
"selection": {
"directReference": {
"structField": {
"field": 0
}
}
}
}]
}],
"measures": [{
"measure": {
"functionReference": 0,
"arguments": [{
"value": {
"selection": {
"directReference": {
"structField": {
"field": 2
}
}
}
}
}],
"options": [{
"name": "distribution",
"preference": [
"SAMPLE"
]
}],
"sorts": [],
"phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT",
"invocation": "AGGREGATION_INVOCATION_ALL",
"outputType": {
"i64": {}
}
}
}]
}
}
}],
"extensionUris": [{
"extension_uri_anchor": 0,
"uri": "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml"
}],
"extensions": [{
"extension_function": {
"extension_uri_reference": 0,
"function_anchor": 0,
"name": "variance"
}
}],
})";

ASSERT_OK_AND_ASSIGN(auto buf,
internal::SubstraitFromJSON("Plan", substrait_json,
/*ignore_unknown_fields=*/false));
auto output_schema = schema({field("keys", int32()), field("aggregates", float64())});
auto expected_table = TableFromJSON(output_schema, {R"([
[10, null],
[20, 200]
])"});

NamedTableProvider table_provider = AlwaysProvideSameTable(std::move(input_table));

ConversionOptions conversion_options;
conversion_options.named_table_provider = std::move(table_provider);

CheckRoundTripResult(std::move(expected_table), buf,
/*include_columns=*/{}, conversion_options);
}

TEST(SubstraitRoundTrip, AggregateRelEmit) {
auto dummy_schema =
schema({field("A", int32()), field("B", int32()), field("C", int32())});
Expand Down
Loading