Skip to content

Commit ffe72de

Browse files
committed
GH-55448: [C++][Parquet] Optimize ALP and make vector sizes configurable
1 parent 7299fdc commit ffe72de

16 files changed

Lines changed: 362 additions & 251 deletions

‎cpp/src/arrow/util/alp/alp_codec.cc‎

Lines changed: 29 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -153,61 +153,45 @@ Result<int64_t> EncodePage(std::span<const T> inputs, const AlpEncodingPreset& p
153153
std::span<uint8_t> body = output.subspan(AlpHeader::kSize);
154154
const int64_t element_count = static_cast<int64_t>(inputs.size());
155155

156-
// Phase 1: compress all vectors and collect them.
157-
std::vector<AlpEncodedVector<T>> encoded_vectors;
158156
const int64_t num_vectors = ::arrow::bit_util::CeilDiv(element_count, vector_size);
159-
encoded_vectors.reserve(num_vectors);
160-
161-
int64_t input_offset = 0;
162-
for (int64_t remaining_elements = element_count; remaining_elements > 0;) {
163-
const int64_t elements_to_encode =
164-
std::min(static_cast<int64_t>(vector_size), remaining_elements);
165-
encoded_vectors.push_back(AlpCompression<T>::Compress(
166-
inputs.subspan(static_cast<size_t>(input_offset),
167-
static_cast<size_t>(elements_to_encode)),
168-
preset));
169-
input_offset += elements_to_encode;
170-
remaining_elements -= elements_to_encode;
171-
}
172-
173-
// Phase 2: calculate sizes and offsets.
174157
const int64_t per_vector_metadata_size =
175158
AlpInfo::kStoredSize + GetIntegerEncodingMetadataSize<T>(preset.integer_encoding);
176159
const int64_t offsets_section_size =
177160
num_vectors * static_cast<int64_t>(sizeof(AlpFormatConstants::OffsetType));
178-
179-
std::vector<AlpFormatConstants::OffsetType> vector_offsets;
180-
vector_offsets.reserve(num_vectors);
161+
if (offsets_section_size > static_cast<int64_t>(body.size())) {
162+
return Status::Invalid("ALP output buffer too small for offsets: ", body.size(),
163+
" < ", offsets_section_size);
164+
}
181165

182166
int64_t current_offset = offsets_section_size;
183-
for (const auto& vec : encoded_vectors) {
167+
int64_t input_offset = 0;
168+
for (int64_t vector_index = 0; vector_index < num_vectors; ++vector_index) {
169+
const int64_t remaining_elements = element_count - input_offset;
170+
const int64_t elements_to_encode =
171+
std::min(static_cast<int64_t>(vector_size), remaining_elements);
172+
const auto vec = AlpCompression<T>::Compress(
173+
inputs.subspan(static_cast<size_t>(input_offset),
174+
static_cast<size_t>(elements_to_encode)),
175+
preset);
176+
const int64_t stored_size = per_vector_metadata_size + vec.GetDataStoredSize();
184177
if (current_offset >
185178
static_cast<int64_t>(
186179
std::numeric_limits<AlpFormatConstants::OffsetType>::max())) {
187180
return Status::Invalid("ALP encoded data exceeds the uint32 offset range");
188181
}
189-
vector_offsets.push_back(static_cast<AlpFormatConstants::OffsetType>(current_offset));
190-
current_offset += per_vector_metadata_size + vec.GetDataStoredSize();
182+
if (stored_size > static_cast<int64_t>(body.size()) - current_offset) {
183+
return Status::Invalid("ALP output buffer too small: ", body.size(), " < ",
184+
current_offset + stored_size);
185+
}
186+
const auto offset = static_cast<AlpFormatConstants::OffsetType>(current_offset);
187+
util::SafeStore(body.data() + vector_index * sizeof(AlpFormatConstants::OffsetType),
188+
bit_util::ToLittleEndian(offset));
189+
vec.Store(body.subspan(static_cast<size_t>(current_offset),
190+
static_cast<size_t>(stored_size)));
191+
current_offset += stored_size;
192+
input_offset += elements_to_encode;
191193
}
192194
const int64_t body_size = current_offset;
193-
if (body_size > static_cast<int64_t>(body.size())) {
194-
return Status::Invalid("ALP output buffer too small: ", body.size(), " < ",
195-
body_size);
196-
}
197-
198-
// Phase 3: write the offset table.
199-
uint8_t* offset_ptr = body.data();
200-
for (const auto& offset : vector_offsets) {
201-
util::SafeStore(offset_ptr, bit_util::ToLittleEndian(offset));
202-
offset_ptr += sizeof(AlpFormatConstants::OffsetType);
203-
}
204-
205-
// Phase 4: write interleaved vectors [AlpInfo | ForInfo | Data].
206-
for (size_t i = 0; i < encoded_vectors.size(); ++i) {
207-
const auto& vec = encoded_vectors[i];
208-
const int64_t stored_size = per_vector_metadata_size + vec.GetDataStoredSize();
209-
vec.Store(body.subspan(vector_offsets[i], static_cast<size_t>(stored_size)));
210-
}
211195
AlpHeader header{};
212196
header.compression_mode = static_cast<uint8_t>(AlpMode::kAlp);
213197
header.integer_encoding = static_cast<uint8_t>(AlpIntegerEncoding::kForBitPack);
@@ -228,8 +212,11 @@ Result<int64_t> EncodePage(std::span<const T> inputs, const AlpEncodingPreset& p
228212
template <AlpFloatingType T>
229213
Result<AlpEncodingPreset> AlpCodec<T>::MakePreset(std::span<const T> inputs) {
230214
RETURN_NOT_OK(ValidateElementCount(inputs.size()));
215+
if (inputs.empty()) {
216+
return AlpEncodingPreset::MakeDefault();
217+
}
231218

232-
AlpSampler<T> sampler;
219+
AlpSampler<T> sampler(static_cast<int64_t>(inputs.size()));
233220
sampler.AddSample(inputs);
234221
return sampler.MakePreset();
235222
}

‎cpp/src/arrow/util/alp/alp_compression.cc‎

Lines changed: 60 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "arrow/util/alp/alp_compression_internal.h"
1919

2020
#include <algorithm>
21+
#include <array>
2122
#include <cmath>
2223
#include <cstring>
2324
#include <limits>
@@ -58,6 +59,20 @@ class AlpInlines {
5859
using EncodedUnsigned = typename Constants::EncodedUnsigned;
5960
using EncodedSigned = typename Constants::EncodedSigned;
6061

62+
struct Multipliers {
63+
T encode_exponent;
64+
T encode_factor;
65+
T decode_factor;
66+
T decode_exponent;
67+
};
68+
69+
static Multipliers GetMultipliers(const AlpExponentAndFactor exponent_and_factor) {
70+
return {Constants::GetExponent(exponent_and_factor.exponent),
71+
Constants::GetFactor(exponent_and_factor.factor),
72+
Constants::GetExponent(exponent_and_factor.factor),
73+
Constants::GetFactor(exponent_and_factor.exponent)};
74+
}
75+
6176
// Returns true when the scaled value cannot be encoded.
6277
static bool IsImpossibleToEncode(const T n) {
6378
// The two limit comparisons already cover both infinities: the encoding
@@ -89,22 +104,15 @@ class AlpInlines {
89104
return FastRound(n);
90105
}
91106

92-
// Encodes one value using the exponent/factor pair.
93-
static EncodedSigned EncodeValue(const T value,
94-
const AlpExponentAndFactor exponent_and_factor) {
95-
const T tmp_encoded_value = value *
96-
Constants::GetExponent(exponent_and_factor.exponent) *
97-
Constants::GetFactor(exponent_and_factor.factor);
98-
return NumberToInt(tmp_encoded_value);
107+
static EncodedSigned EncodeValue(const T value, const Multipliers& multipliers) {
108+
return NumberToInt(value * multipliers.encode_exponent * multipliers.encode_factor);
99109
}
100110

101-
// Decodes one encoded integer using the exponent/factor pair.
102111
static T DecodeValue(const EncodedSigned encoded_value,
103-
const AlpExponentAndFactor exponent_and_factor) {
112+
const Multipliers& multipliers) {
104113
// The cast to T is needed to prevent a signed integer overflow.
105-
return static_cast<T>(encoded_value) *
106-
Constants::GetExponent(exponent_and_factor.factor) *
107-
Constants::GetFactor(exponent_and_factor.exponent);
114+
return static_cast<T>(encoded_value) * multipliers.decode_factor *
115+
multipliers.decode_exponent;
108116
}
109117
};
110118

@@ -149,6 +157,7 @@ std::optional<int64_t> EstimateCompressedSizeBits(
149157
static constexpr int kBatchSize = 8;
150158
const size_t n = inputs.size();
151159
int64_t num_exceptions = 0;
160+
const auto multipliers = AlpInlines<T>::GetMultipliers(exponent_and_factor);
152161

153162
size_t i = 0;
154163
for (; i + kBatchSize <= n; i += kBatchSize) {
@@ -157,12 +166,11 @@ std::optional<int64_t> EstimateCompressedSizeBits(
157166

158167
// Pass 1: Encode (vectorizable: multiply + magic-number round)
159168
for (int j = 0; j < kBatchSize; j++) {
160-
encoded_values[j] = AlpInlines<T>::EncodeValue(inputs[i + j], exponent_and_factor);
169+
encoded_values[j] = AlpInlines<T>::EncodeValue(inputs[i + j], multipliers);
161170
}
162171
// Pass 2: Decode (vectorizable: int->float cast + multiply)
163172
for (int j = 0; j < kBatchSize; j++) {
164-
decoded_values[j] =
165-
AlpInlines<T>::DecodeValue(encoded_values[j], exponent_and_factor);
173+
decoded_values[j] = AlpInlines<T>::DecodeValue(encoded_values[j], multipliers);
166174
}
167175
// Pass 3: Compare and accumulate min/max/exceptions
168176
for (int j = 0; j < kBatchSize; j++) {
@@ -177,9 +185,8 @@ std::optional<int64_t> EstimateCompressedSizeBits(
177185
// Scalar tail for remaining elements
178186
for (; i < n; i++) {
179187
const EncodedSigned encoded_value =
180-
AlpInlines<T>::EncodeValue(inputs[i], exponent_and_factor);
181-
const T decoded_value =
182-
AlpInlines<T>::DecodeValue(encoded_value, exponent_and_factor);
188+
AlpInlines<T>::EncodeValue(inputs[i], multipliers);
189+
const T decoded_value = AlpInlines<T>::DecodeValue(encoded_value, multipliers);
183190
if (decoded_value == inputs[i]) {
184191
max_encoded_value = std::max(encoded_value, max_encoded_value);
185192
min_encoded_value = std::min(encoded_value, min_encoded_value);
@@ -304,21 +311,6 @@ AlpEncodingPreset AlpCompression<T>::MakePreset(
304311
return AlpEncodingPreset{std::move(combinations), estimated_compressed_size_bytes};
305312
}
306313

307-
template <AlpFloatingType T>
308-
std::vector<T> AlpCompression<T>::CreateSample(std::span<const T> inputs) {
309-
// Pick an equidistant stride so the sample count is capped at kSamplesPerChunk.
310-
const int32_t idx_increments = std::max<int32_t>(
311-
1, static_cast<int32_t>(std::ceil(static_cast<double>(inputs.size()) /
312-
AlpSamplingOptions::kSamplesPerChunk)));
313-
const size_t increment = static_cast<size_t>(idx_increments);
314-
std::vector<T> sample;
315-
sample.reserve((inputs.size() + increment - 1) / increment);
316-
for (size_t i = 0; i < inputs.size(); i += idx_increments) {
317-
sample.push_back(inputs[i]);
318-
}
319-
return sample;
320-
}
321-
322314
template <AlpFloatingType T>
323315
AlpExponentAndFactor AlpCompression<T>::FindBestExponentAndFactor(
324316
std::span<const T> inputs, const std::vector<AlpExponentAndFactor>& combinations) {
@@ -331,7 +323,17 @@ AlpExponentAndFactor AlpCompression<T>::FindBestExponentAndFactor(
331323
return combinations.front();
332324
}
333325

334-
const std::vector<T> sample = CreateSample(inputs);
326+
// The sample is capped at 256 values. Keep it on the stack to avoid a heap
327+
// allocation for every vector parameter search.
328+
std::array<T, AlpSamplingOptions::kSamplesPerChunk> sample_storage;
329+
const int32_t idx_increments = std::max<int32_t>(
330+
1, static_cast<int32_t>(std::ceil(static_cast<double>(inputs.size()) /
331+
AlpSamplingOptions::kSamplesPerChunk)));
332+
size_t sample_count = 0;
333+
for (size_t i = 0; i < inputs.size(); i += idx_increments) {
334+
sample_storage[sample_count++] = inputs[i];
335+
}
336+
const std::span<const T> sample(sample_storage.data(), sample_count);
335337

336338
AlpExponentAndFactor best_exponent_and_factor;
337339
int64_t best_total_bits = std::numeric_limits<int64_t>::max();
@@ -375,6 +377,7 @@ typename AlpCompression<T>::EncodingResult AlpCompression<T>::EncodeVector(
375377
for_deltas.reserve(inputs.size());
376378
std::vector<T> exceptions;
377379
std::vector<AlpFormatConstants::PositionType> exception_positions;
380+
const auto multipliers = AlpInlines<T>::GetMultipliers(exponent_and_factor);
378381

379382
EncodedSigned min_encoded_value = std::numeric_limits<EncodedSigned>::max();
380383
EncodedSigned max_encoded_value = std::numeric_limits<EncodedSigned>::min();
@@ -383,10 +386,8 @@ typename AlpCompression<T>::EncodingResult AlpCompression<T>::EncodeVector(
383386
// while the remaining values define the FOR range.
384387
int64_t input_offset = 0;
385388
for (const T input : inputs) {
386-
const EncodedSigned encoded_value =
387-
AlpInlines<T>::EncodeValue(input, exponent_and_factor);
388-
const T decoded_value =
389-
AlpInlines<T>::DecodeValue(encoded_value, exponent_and_factor);
389+
const EncodedSigned encoded_value = AlpInlines<T>::EncodeValue(input, multipliers);
390+
const T decoded_value = AlpInlines<T>::DecodeValue(encoded_value, multipliers);
390391
for_deltas.push_back(static_cast<EncodedUnsigned>(encoded_value));
391392

392393
if (decoded_value != input) {
@@ -524,16 +525,25 @@ template <typename TargetType>
524525
void AlpCompression<T>::PatchExceptions(
525526
std::span<const T> exceptions,
526527
std::span<const AlpFormatConstants::PositionType> exception_positions,
527-
std::span<TargetType> outputs) {
528+
const bool positions_strictly_increasing, std::span<TargetType> outputs) {
528529
ARROW_DCHECK_EQ(exceptions.size(), exception_positions.size());
529-
// `ivdep` assumes the positions are distinct. The encoder writes them in
530-
// order, and loading a page rejects positions that do not increase.
531-
int64_t exception_idx = 0;
532-
ARROW_ALP_UNROLL_AND_ASSUME_INDEPENDENT
533-
for (const AlpFormatConstants::PositionType exception_position : exception_positions) {
534-
ARROW_DCHECK_LT(exception_position, outputs.size());
535-
outputs[exception_position] = static_cast<T>(exceptions[exception_idx]);
536-
exception_idx++;
530+
const size_t num_exceptions = exception_positions.size();
531+
const auto patch_exception = [&](size_t i) {
532+
const auto position = exception_positions[i];
533+
ARROW_DCHECK_LT(position, outputs.size());
534+
outputs[position] = static_cast<T>(exceptions[i]);
535+
};
536+
if (positions_strictly_increasing) {
537+
// The encoder emits strictly increasing positions. Use the vectorized
538+
// fast path for that common case.
539+
ARROW_ALP_UNROLL_AND_ASSUME_INDEPENDENT
540+
for (size_t i = 0; i < num_exceptions; ++i) {
541+
patch_exception(i);
542+
}
543+
} else {
544+
for (size_t i = 0; i < num_exceptions; ++i) {
545+
patch_exception(i);
546+
}
537547
}
538548
}
539549

@@ -563,17 +573,18 @@ void AlpCompression<T>::Decompress(const AlpEncodedVectorView<T>& encoded_view,
563573
// Read the exponent/factor once. AlpInfo stores them as uint8_t, which may
564574
// alias the output stores, so reading them inside the loop blocks vectorization.
565575
const AlpExponentAndFactor exponent_and_factor = alp_info.GetExponentAndFactor();
576+
const auto multipliers = AlpInlines<T>::GetMultipliers(exponent_and_factor);
566577
for (size_t i = 0; i < output.size(); ++i) {
567578
const EncodedUnsigned unfored_value = scratch[i] + frame_of_ref;
568579
const EncodedSigned signed_value = util::SafeCopy<EncodedSigned>(unfored_value);
569-
output[i] = AlpInlines<T>::DecodeValue(signed_value, exponent_and_factor);
580+
output[i] = AlpInlines<T>::DecodeValue(signed_value, multipliers);
570581
}
571582

572583
PatchExceptions<TargetType>(
573584
{encoded_view.exceptions().data(), encoded_view.exceptions().size()},
574585
{encoded_view.exception_positions().data(),
575586
encoded_view.exception_positions().size()},
576-
output);
587+
encoded_view.exception_positions_strictly_increasing(), output);
577588
}
578589

579590
// Template instantiations

‎cpp/src/arrow/util/alp/alp_compression_internal.h‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,6 @@ class ARROW_EXPORT AlpCompression {
7979
std::span<EncodedUnsigned> integer_scratch);
8080

8181
private:
82-
static std::vector<T> CreateSample(std::span<const T> inputs);
83-
8482
static AlpExponentAndFactor FindBestExponentAndFactor(
8583
std::span<const T> inputs, const std::vector<AlpExponentAndFactor>& combinations);
8684

@@ -113,7 +111,7 @@ class ARROW_EXPORT AlpCompression {
113111
static void PatchExceptions(
114112
std::span<const T> exceptions,
115113
std::span<const AlpFormatConstants::PositionType> exception_positions,
116-
std::span<TargetType> outputs);
114+
bool positions_strictly_increasing, std::span<TargetType> outputs);
117115
};
118116

119117
// The marker on a class template does not reach its member templates: each is

‎cpp/src/arrow/util/alp/alp_constants_internal.h‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,6 @@ enum class AlpMode : uint8_t { kAlp = 0 };
3838
enum class AlpIntegerEncoding : uint8_t { kForBitPack = 0 };
3939

4040
struct AlpFormatConstants {
41-
/// Recommended default vector size.
42-
static constexpr int32_t kDefaultVectorSize = 1024;
43-
4441
/// Minimum log2(vector size), corresponding to 8 elements.
4542
static constexpr uint8_t kMinLogVectorSize = 3;
4643

@@ -63,14 +60,14 @@ struct AlpSamplingOptions {
6360
/// Input elements per sampling chunk.
6461
static constexpr int64_t kChunkSize = 4096;
6562

66-
/// Elements per sampling interval.
67-
static constexpr int64_t kIntervalSize = 122880;
63+
/// Maximum prefix length examined in each sampled chunk.
64+
static constexpr int64_t kSampleWindowSize = 1024;
6865

6966
/// Samples collected per chunk.
7067
static constexpr int64_t kSamplesPerChunk = 256;
7168

72-
/// Chunks sampled per interval.
73-
static constexpr int64_t kSampleChunksPerInterval = 8;
69+
/// Target number of sampled chunks per window.
70+
static constexpr int64_t kSampleChunksPerWindow = 8;
7471

7572
/// Consecutive non-improving combinations before an early exit.
7673
static constexpr uint8_t kEarlyExitThreshold = 4;

0 commit comments

Comments
 (0)