From e8100d256e8b9ea0e115522900a1fa12ce5edbeb Mon Sep 17 00:00:00 2001 From: Mola-maker <2249464964@qq.com> Date: Fri, 13 Nov 2026 03:17:35 +0800 Subject: [PATCH] GH-55575: [C++][Substrait] Fix inverted ddof for POPULATION/SAMPLE in DecodeBasicAggregate DecodeBasicAggregate mapped POPULATION to ddof=1 and SAMPLE to ddof=0, which is backwards: VarianceOptions uses N - ddof as the divisor, so population variance needs ddof=0 and sample variance needs ddof=1. The swap made a POPULATION request silently compute sample variance (and return NULL on a 1-row group), and vice versa. Fix the mapping and correct the AggregateRelOptions test, which encoded the buggy behavior, and add coverage for the SAMPLE distribution and for a single-row group (0.0 for POPULATION, NULL for SAMPLE). stddev shares this code path, so it is fixed by the same change. --- .../arrow/engine/substrait/extension_set.cc | 4 +- cpp/src/arrow/engine/substrait/serde_test.cc | 331 ++++++++++++++++++ 2 files changed, 333 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/engine/substrait/extension_set.cc b/cpp/src/arrow/engine/substrait/extension_set.cc index 78188dd8d72e..ecd6e62a37e8 100644 --- a/cpp/src/arrow/engine/substrait/extension_set.cc +++ b/cpp/src/arrow/engine/substrait/extension_set.cc @@ -1079,9 +1079,9 @@ ExtensionIdRegistry::SubstraitAggregateToArrow DecodeBasicAggregate( } int ddof; if (prefs[0] == "POPULATION") { - ddof = 1; - } else if (prefs[0] == "SAMPLE") { ddof = 0; + } else if (prefs[0] == "SAMPLE") { + ddof = 1; } else { return Status::Invalid("unknown distribution preference ", prefs[0]); } diff --git a/cpp/src/arrow/engine/substrait/serde_test.cc b/cpp/src/arrow/engine/substrait/serde_test.cc index 8c3f5ca69876..941b3c47b831 100644 --- a/cpp/src/arrow/engine/substrait/serde_test.cc +++ b/cpp/src/arrow/engine/substrait/serde_test.cc @@ -3837,6 +3837,119 @@ TEST(SubstraitRoundTrip, AggregateRelOptions) { internal::SubstraitFromJSON("Plan", substrait_json, /*ignore_unknown_fields=*/false)); auto output_schema = schema({field("keys", int32()), field("aggregates", float64())}); + auto expected_table = TableFromJSON(output_schema, {R"([ + [10, 100], + [20, 100], + [30, 0] + ])"}); + + NamedTableProvider table_provider = AlwaysProvideSameTable(std::move(input_table)); + + ConversionOptions conversion_options; + conversion_options.named_table_provider = std::move(table_provider); + + CheckRoundTripResult(std::move(expected_table), buf, + /*include_columns=*/{}, conversion_options); +} + +TEST(SubstraitRoundTrip, AggregateRelOptionsSample) { + auto dummy_schema = + schema({field("A", int32()), field("B", int32()), field("C", int32())}); + + // Same dataset as AggregateRelOptions, but requesting SAMPLE distribution. + auto input_table = TableFromJSON(dummy_schema, {R"([ + [10, 1, 10], + [20, 2, 10], + [30, 3, 20], + [30, 1, 20], + [20, 2, 30], + [10, 3, 30] + ])"}); + + std::string substrait_json = R"({ + "version": { "major_number": 9999, "minor_number": 9999, "patch_number": 9999 }, + "relations": [{ + "rel": { + "aggregate": { + "input": { + "read": { + "base_schema": { + "names": ["A", "B", "C"], + "struct": { + "types": [{ + "i32": {} + }, { + "i32": {} + }, { + "i32": {} + }] + } + }, + "namedTable" : { + "names": ["A"] + } + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 0 + } + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 0, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + } + } + } + }], + "options": [{ + "name": "distribution", + "preference": [ + "SAMPLE" + ] + }], + "sorts": [], + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "invocation": "AGGREGATION_INVOCATION_ALL", + "outputType": { + "i64": {} + } + } + }] + } + } + }], + "extensionUris": [{ + "extension_uri_anchor": 0, + "uri": "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml" + }], + "extensions": [{ + "extension_function": { + "extension_uri_reference": 0, + "function_anchor": 0, + "name": "variance" + } + }], + })"; + + ASSERT_OK_AND_ASSIGN(auto buf, + internal::SubstraitFromJSON("Plan", substrait_json, + /*ignore_unknown_fields=*/false)); + auto output_schema = schema({field("keys", int32()), field("aggregates", float64())}); + // Sample variance (ddof=1): [10, 30] -> 200, [10, 30] -> 200, [20, 20] -> 0. auto expected_table = TableFromJSON(output_schema, {R"([ [10, 200], [20, 200], @@ -3852,6 +3965,224 @@ TEST(SubstraitRoundTrip, AggregateRelOptions) { /*include_columns=*/{}, conversion_options); } +TEST(SubstraitRoundTrip, AggregateRelVarianceSingleRowGroup) { + auto dummy_schema = + schema({field("A", int32()), field("B", int32()), field("C", int32())}); + + // Group A=10 has a single row: population variance (ddof=0) is 0.0 there, + // while sample variance (ddof=1) is NULL. See GH-55575. + auto input_table = TableFromJSON(dummy_schema, {R"([ + [10, 1, 5], + [20, 2, 10], + [20, 3, 30] + ])"}); + + std::string substrait_json = R"({ + "version": { "major_number": 9999, "minor_number": 9999, "patch_number": 9999 }, + "relations": [{ + "rel": { + "aggregate": { + "input": { + "read": { + "base_schema": { + "names": ["A", "B", "C"], + "struct": { + "types": [{ + "i32": {} + }, { + "i32": {} + }, { + "i32": {} + }] + } + }, + "namedTable" : { + "names": ["A"] + } + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 0 + } + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 0, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + } + } + } + }], + "options": [{ + "name": "distribution", + "preference": [ + "POPULATION" + ] + }], + "sorts": [], + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "invocation": "AGGREGATION_INVOCATION_ALL", + "outputType": { + "i64": {} + } + } + }] + } + } + }], + "extensionUris": [{ + "extension_uri_anchor": 0, + "uri": "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml" + }], + "extensions": [{ + "extension_function": { + "extension_uri_reference": 0, + "function_anchor": 0, + "name": "variance" + } + }], + })"; + + ASSERT_OK_AND_ASSIGN(auto buf, + internal::SubstraitFromJSON("Plan", substrait_json, + /*ignore_unknown_fields=*/false)); + auto output_schema = schema({field("keys", int32()), field("aggregates", float64())}); + auto expected_table = TableFromJSON(output_schema, {R"([ + [10, 0], + [20, 100] + ])"}); + + NamedTableProvider table_provider = AlwaysProvideSameTable(std::move(input_table)); + + ConversionOptions conversion_options; + conversion_options.named_table_provider = std::move(table_provider); + + CheckRoundTripResult(std::move(expected_table), buf, + /*include_columns=*/{}, conversion_options); +} + +TEST(SubstraitRoundTrip, AggregateRelVarianceSingleRowGroupSample) { + auto dummy_schema = + schema({field("A", int32()), field("B", int32()), field("C", int32())}); + + // Same dataset as AggregateRelVarianceSingleRowGroup, but requesting SAMPLE + // distribution: the single-row group yields NULL instead of 0.0. See GH-55575. + auto input_table = TableFromJSON(dummy_schema, {R"([ + [10, 1, 5], + [20, 2, 10], + [20, 3, 30] + ])"}); + + std::string substrait_json = R"({ + "version": { "major_number": 9999, "minor_number": 9999, "patch_number": 9999 }, + "relations": [{ + "rel": { + "aggregate": { + "input": { + "read": { + "base_schema": { + "names": ["A", "B", "C"], + "struct": { + "types": [{ + "i32": {} + }, { + "i32": {} + }, { + "i32": {} + }] + } + }, + "namedTable" : { + "names": ["A"] + } + } + }, + "groupings": [{ + "groupingExpressions": [{ + "selection": { + "directReference": { + "structField": { + "field": 0 + } + } + } + }] + }], + "measures": [{ + "measure": { + "functionReference": 0, + "arguments": [{ + "value": { + "selection": { + "directReference": { + "structField": { + "field": 2 + } + } + } + } + }], + "options": [{ + "name": "distribution", + "preference": [ + "SAMPLE" + ] + }], + "sorts": [], + "phase": "AGGREGATION_PHASE_INITIAL_TO_RESULT", + "invocation": "AGGREGATION_INVOCATION_ALL", + "outputType": { + "i64": {} + } + } + }] + } + } + }], + "extensionUris": [{ + "extension_uri_anchor": 0, + "uri": "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml" + }], + "extensions": [{ + "extension_function": { + "extension_uri_reference": 0, + "function_anchor": 0, + "name": "variance" + } + }], + })"; + + ASSERT_OK_AND_ASSIGN(auto buf, + internal::SubstraitFromJSON("Plan", substrait_json, + /*ignore_unknown_fields=*/false)); + auto output_schema = schema({field("keys", int32()), field("aggregates", float64())}); + auto expected_table = TableFromJSON(output_schema, {R"([ + [10, null], + [20, 200] + ])"}); + + NamedTableProvider table_provider = AlwaysProvideSameTable(std::move(input_table)); + + ConversionOptions conversion_options; + conversion_options.named_table_provider = std::move(table_provider); + + CheckRoundTripResult(std::move(expected_table), buf, + /*include_columns=*/{}, conversion_options); +} + TEST(SubstraitRoundTrip, AggregateRelEmit) { auto dummy_schema = schema({field("A", int32()), field("B", int32()), field("C", int32())});