From d1bbea7bf0e2e062107c6b349e86ea405d72cf35 Mon Sep 17 00:00:00 2001 From: Kugan Vivekanandarajah Date: Fri, 14 Nov 2025 18:16:41 +1100 Subject: [PATCH] Add hierarchical discriminator multiplicity support Added support for mutiplicity and copy_id for gcc gcov. This should matches the GCC implementation. I am posting patches to GCC to match this. This patch also adds support for two-pass aggregation when using hierarchical discriminator encoding (gcov_version=3). The two-pass approach correctly handles duplicate samples from code duplication (loop unrolling, etc.). * GetBaseDiscriminator() - Extract bits 0-7 * GetMultiplicity() - Extract bits 8-14 (duplication factor) * GetCopyID() - Extract bits 15-25 (unused but reserved) With the patch we now: 1. Extract multiplicity and copy_id from discriminator 2. Multiply sample count by multiplicity 4. Add samples using copy_id. This was already supported for LLVM. --- CMakeLists.txt | 24 ++ create_gcov.cc | 7 + ...rchical_discriminator_multiplicity_test.cc | 326 ++++++++++++++++++ gcov_discriminator_encoding.h | 66 ++++ profile_creator.cc | 8 + source_info.h | 28 +- symbol_map.cc | 80 ++++- symbol_map.h | 14 + testdata/hierarchical_discriminator_test.c | 42 +++ .../hierarchical_discriminator_test.x86_64 | Bin 0 -> 37224 bytes ...chical_discriminator_test.x86_64.perf.data | Bin 0 -> 73612 bytes 11 files changed, 590 insertions(+), 5 deletions(-) create mode 100644 gcc_hierarchical_discriminator_multiplicity_test.cc create mode 100644 gcov_discriminator_encoding.h create mode 100644 testdata/hierarchical_discriminator_test.c create mode 100755 testdata/hierarchical_discriminator_test.x86_64 create mode 100644 testdata/hierarchical_discriminator_test.x86_64.perf.data diff --git a/CMakeLists.txt b/CMakeLists.txt index 419ad66..1a78a3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,6 +117,30 @@ function (build_gcov) quipper_perf ) + # Same sources as create_gcov_lib except create_gcov.cc (which defines main), + # so we can link gtest_main without duplicate main(). + add_executable(gcc_hierarchical_discriminator_multiplicity_test + gcc_hierarchical_discriminator_multiplicity_test.cc + gcov.cc + instruction_map.cc + profile.cc + profile_creator.cc + profile_writer.cc + sample_reader.cc + simple_spe_sample_reader.cc + symbol_map.cc) + target_link_libraries(gcc_hierarchical_discriminator_multiplicity_test + absl::flags + absl::flags_parse + gtest + gtest_main + perf_proto + addr2line_lib + glog + quipper_perf) + add_test(NAME gcc_hierarchical_discriminator_multiplicity_test + COMMAND gcc_hierarchical_discriminator_multiplicity_test) + add_library(profile_merger_lib OBJECT gcov.cc instruction_map.cc diff --git a/create_gcov.cc b/create_gcov.cc index cce528d..0b71425 100644 --- a/create_gcov.cc +++ b/create_gcov.cc @@ -41,6 +41,13 @@ int main(int argc, char **argv) { absl::GetFlag(FLAGS_gcov_version)); devtools_crosstool_autofdo::ProfileCreator creator( absl::GetFlag(FLAGS_binary)); + + // Enable discriminator encoding and two-pass aggregation only for gcov version 3 + if (absl::GetFlag(FLAGS_gcov_version) >= 3) { + absl::SetFlag(&FLAGS_use_discriminator_encoding, true); + absl::SetFlag(&FLAGS_use_two_pass_aggregation, true); + } + if (creator.CreateProfile(absl::GetFlag(FLAGS_profile), absl::GetFlag(FLAGS_profiler), &writer, absl::GetFlag(FLAGS_gcov))) { diff --git a/gcc_hierarchical_discriminator_multiplicity_test.cc b/gcc_hierarchical_discriminator_multiplicity_test.cc new file mode 100644 index 0000000..71ea452 --- /dev/null +++ b/gcc_hierarchical_discriminator_multiplicity_test.cc @@ -0,0 +1,326 @@ + +// hierarchical_discriminator_test +// +// Targeted checks on the checked-in binary + perf.data: +// +// 1) test_unroll: a pos_counts sample whose packed discriminator has non-zero +// copy_id (HasCopyID). Full-profile keys use the raw word; two-pass pass 1 +// uses StripMultiplicity (before CollapseCopyIDs). Assert the merged bucket +// exists on test_unroll and PERFDATA MAX count matches manual MAX over full +// keys stripping to that bucket. +// +// 2) test_vector_loop: a pos_counts sample with hierarchical multiplicity +// (HasMultiplicity). Same two-pass strip + MAX count check on test_vector_loop. +// +// 3) Pass 2: SymbolMap::CollapseCopyIDs() (as at gcov write) strips copy_id to +// base-only keys and SUMs ProfileInfo. For the same test_unroll copy_id +// sample line+base, assert the collapsed bucket count equals the SUM of +// pass-1 counts for all buckets sharing that (line, base). +#include "profile_creator.h" +#include "gcov_discriminator_encoding.h" +#include "source_info.h" +#include "symbol_map.h" + +#include +#include +#include + +#include "gtest/gtest.h" +#include "third_party/abseil/absl/flags/flag.h" +#include "third_party/abseil/absl/strings/match.h" +#include "third_party/abseil/absl/strings/str_cat.h" +#include "third_party/abseil/absl/strings/string_view.h" + +ABSL_DECLARE_FLAG(bool, use_lbr); +ABSL_DECLARE_FLAG(bool, use_discriminator_encoding); +ABSL_DECLARE_FLAG(bool, use_two_pass_aggregation); + +namespace { + +using ::devtools_crosstool_autofdo::GetBaseDiscriminator; +using ::devtools_crosstool_autofdo::GetCopyID; +using ::devtools_crosstool_autofdo::GetMultiplicity; +using ::devtools_crosstool_autofdo::HasCopyID; +using ::devtools_crosstool_autofdo::HasMultiplicity; +using ::devtools_crosstool_autofdo::ProfileCreator; +using ::devtools_crosstool_autofdo::SourceInfo; +using ::devtools_crosstool_autofdo::StripMultiplicity; +using ::devtools_crosstool_autofdo::PositionCountMap; +using ::devtools_crosstool_autofdo::Symbol; +using ::devtools_crosstool_autofdo::SymbolMap; + +const char kTestDataDir[] = "/testdata/"; + +constexpr char kFixtureElf[] = "hierarchical_discriminator_test.x86_64"; +constexpr char kFixturePerf[] = + "hierarchical_discriminator_test.x86_64.perf.data"; + +std::string TestBinaryPath() { + return absl::StrCat(::testing::SrcDir(), kTestDataDir, kFixtureElf); +} + +std::string TestPerfDataPath() { + return absl::StrCat(::testing::SrcDir(), kTestDataDir, kFixturePerf); +} + +const Symbol *FindSymbolByStem(const SymbolMap &symbol_map, + absl::string_view name_stem) { + const auto &name_to_symbol = symbol_map.map(); + const std::string exact(name_stem); + auto direct = name_to_symbol.find(exact); + if (direct != name_to_symbol.end()) return direct->second; + for (const auto &entry : name_to_symbol) { + if (absl::StrContains(entry.first, name_stem)) return entry.second; + } + return nullptr; +} + +struct PosCountSample { + uint32_t raw_packed_discriminator = 0; + uint64_t pos_count_offset_key = 0; +}; + +uint64_t SampleCountAtBucket(const Symbol *symbol, uint32_t line_relative, + uint32_t packed_discriminator) { + if (symbol == nullptr) return 0; + const uint64_t key = + SourceInfo::GenerateOffset(line_relative, packed_discriminator); + auto it = symbol->pos_counts.find(key); + if (it == symbol->pos_counts.end()) return 0; + return it->second.count; +} + +uint64_t MaxFullProfileCountMergingToStripBucket(const Symbol *full_symbol, + uint32_t line_relative, + uint32_t stripped_discriminator) { + if (full_symbol == nullptr) return 0; + uint64_t max_count = 0; + for (const auto &row : full_symbol->pos_counts) { + if (SourceInfo::GetLineNumberFromOffset(row.first) != line_relative) + continue; + const uint32_t full_packed = + SourceInfo::GetDiscriminatorFromOffset(row.first); + if (StripMultiplicity(full_packed) != stripped_discriminator) continue; + max_count = std::max(max_count, row.second.count); + } + return max_count; +} + +bool TwoPassHasBucket(const Symbol *two_pass_symbol, uint32_t line_relative, + uint32_t stripped_discriminator) { + if (two_pass_symbol == nullptr) return false; + for (const auto &row : two_pass_symbol->pos_counts) { + if (SourceInfo::GetLineNumberFromOffset(row.first) == line_relative && + SourceInfo::GetDiscriminatorFromOffset(row.first) == + stripped_discriminator) { + return true; + } + } + return false; +} + +// Pass 1 two-pass: bucket (line, StripMultiplicity(raw)) exists and .count +// equals MAX of full-profile .count over full keys on that line stripping there. +void VerifyPass1TwoPassBucketMaxMatchesFullProfile( + const Symbol *full_symbol, const Symbol *two_pass_symbol, + const PosCountSample &full_sample, absl::string_view label) { + const uint32_t line_relative = + SourceInfo::GetLineNumberFromOffset(full_sample.pos_count_offset_key); + const uint32_t stripped = + StripMultiplicity(full_sample.raw_packed_discriminator); + ASSERT_TRUE(TwoPassHasBucket(two_pass_symbol, line_relative, stripped)) + << label << ": missing two-pass bucket (line_rel=" << line_relative + << ", StripMultiplicity(raw)=" << stripped << ")"; + const uint64_t expected_max = MaxFullProfileCountMergingToStripBucket( + full_symbol, line_relative, stripped); + const uint64_t actual = + SampleCountAtBucket(two_pass_symbol, line_relative, stripped); + ASSERT_EQ(actual, expected_max) + << label << ": two-pass count must equal MAX of full-profile counts for " + "full keys on this line stripping to this bucket"; +} + +// Pass 1 pos_counts: SUM counts for rows on line_relative whose packed +// discriminator has the same 8-bit base (these rows merge in CollapseCopyIDs). +uint64_t SumPass1CountsMergingToBaseOnLine( + const PositionCountMap &pass1_pos_counts, uint32_t line_relative, + uint32_t base_discriminator) { + uint64_t sum = 0; + for (const auto &row : pass1_pos_counts) { + if (SourceInfo::GetLineNumberFromOffset(row.first) != line_relative) continue; + const uint32_t packed = + SourceInfo::GetDiscriminatorFromOffset(row.first); + if (GetBaseDiscriminator(packed) != base_discriminator) continue; + sum += row.second.count; + } + return sum; +} + +// Snapshot pass-1 test_unroll pos_counts, run SymbolMap::CollapseCopyIDs(), then +// compare the collapsed (line, base-only) bucket to the manual SUM above. +// After CollapseCopyIDs: base-only bucket count equals SUM of pass-1 counts +// merged into that (line, base) on test_unroll for the captured copy_id row. +void VerifyCollapseCopyIdsSumsPass1IntoBaseOnlyUnrollBucket( + SymbolMap *two_pass_map, const Symbol *unroll_two_pass_symbol, + const PositionCountMap &unroll_pass1_pos_counts_snapshot, + const PosCountSample &unroll_copy_id_sample) { + const uint32_t line_relative = SourceInfo::GetLineNumberFromOffset( + unroll_copy_id_sample.pos_count_offset_key); + const uint32_t base_only = GetBaseDiscriminator( + unroll_copy_id_sample.raw_packed_discriminator); + const uint64_t expected_sum_after_collapse = + SumPass1CountsMergingToBaseOnLine(unroll_pass1_pos_counts_snapshot, + line_relative, base_only); + + two_pass_map->CollapseCopyIDs(); + + ASSERT_TRUE(TwoPassHasBucket(unroll_two_pass_symbol, line_relative, base_only)) + << "CollapseCopyIDs: expected base-only bucket on test_unroll"; + const uint64_t actual_count = SampleCountAtBucket( + unroll_two_pass_symbol, line_relative, base_only); + ASSERT_EQ(actual_count, expected_sum_after_collapse) + << "CollapseCopyIDs: pos_counts.count at (line_rel=" << line_relative + << ", base=" << base_only + << ") should equal SUM of pass-1 counts for that line and base"; + + EXPECT_EQ(GetCopyID(base_only), 0u) + << "after CollapseCopyIDs discriminator field should be base-only (no copy_id)"; +} + +// First symbol keyed like `stem` (any map entry whose name contains `stem`) +// with a pos_counts row satisfying `predicate`; pairs full + two-pass by exact +// profile name. +bool FindFirstMatchingSymbolPair( + const SymbolMap &full_map, const SymbolMap &two_pass_map, + absl::string_view name_substring, + const std::function &predicate, + const Symbol **full_symbol_out, const Symbol **two_pass_symbol_out, + std::string *matched_name_out) { + for (const auto &name_and_symbol : full_map.map()) { + if (!absl::StrContains(name_and_symbol.first, name_substring)) continue; + const Symbol *full_sym = name_and_symbol.second; + if (full_sym == nullptr) continue; + auto two_pass_it = two_pass_map.map().find(name_and_symbol.first); + if (two_pass_it == two_pass_map.map().end() || + two_pass_it->second == nullptr) { + continue; + } + for (const auto &row : full_sym->pos_counts) { + const uint32_t packed = + SourceInfo::GetDiscriminatorFromOffset(row.first); + if (!predicate(packed)) continue; + *full_symbol_out = full_sym; + *two_pass_symbol_out = two_pass_it->second; + *matched_name_out = name_and_symbol.first; + return true; + } + } + return false; +} + +// test_unroll*: first pos_counts hit with copy_id — pass-1 strip bucket .count +// vs full-profile MAX for that strip bucket. +void VerifyUnrollFirstCopyIdHitPass1StripAgainstFullProfile( + const SymbolMap &full_map, const SymbolMap &two_pass_map, + PosCountSample *copy_id_sample_out, const Symbol **two_pass_symbol_out) { + const Symbol *full_sym = nullptr; + const Symbol *tp_sym = nullptr; + std::string matched_name; + ASSERT_TRUE(FindFirstMatchingSymbolPair( + full_map, two_pass_map, "test_unroll", + [](uint32_t packed) { return HasCopyID(packed); }, &full_sym, &tp_sym, + &matched_name)) + << "fixture must provide a pos_counts row on a symbol keyed like " + "test_unroll* with HasCopyID (try rebuilding testdata if compiler " + "moved discriminators to a clone symbol)"; + *two_pass_symbol_out = tp_sym; + for (const auto &row : full_sym->pos_counts) { + const uint32_t packed = + SourceInfo::GetDiscriminatorFromOffset(row.first); + if (!HasCopyID(packed)) continue; + *copy_id_sample_out = {packed, row.first}; + ASSERT_GT(GetCopyID(packed), 0u); + VerifyPass1TwoPassBucketMaxMatchesFullProfile( + full_sym, tp_sym, *copy_id_sample_out, + absl::StrCat("copy_id ", matched_name)); + return; + } + FAIL() << "internal error: predicate matched but scan did not"; +} + +// test_vector_loop*: first pos_counts hit with multiplicity — pass-1 strip +// bucket .count vs full-profile MAX. +void VerifyVectorLoopFirstMultiplicityHitPass1StripAgainstFullProfile( + const SymbolMap &full_map, const SymbolMap &two_pass_map) { + const Symbol *full_sym = nullptr; + const Symbol *tp_sym = nullptr; + std::string matched_name; + ASSERT_TRUE(FindFirstMatchingSymbolPair( + full_map, two_pass_map, "test_vector_loop", + [](uint32_t packed) { return HasMultiplicity(packed); }, &full_sym, + &tp_sym, &matched_name)) + << "fixture must provide a pos_counts row on a symbol keyed like " + "test_vector_loop* with HasMultiplicity"; + + for (const auto &row : full_sym->pos_counts) { + const uint32_t packed = + SourceInfo::GetDiscriminatorFromOffset(row.first); + if (!HasMultiplicity(packed)) continue; + const PosCountSample sample{packed, row.first}; + ASSERT_GT(GetMultiplicity(packed), 1u); + VerifyPass1TwoPassBucketMaxMatchesFullProfile( + full_sym, tp_sym, sample, + absl::StrCat("multiplicity ", matched_name)); + return; + } + FAIL() << "internal error: predicate matched but scan did not"; +} + +TEST(GccHierarchicalDiscriminatorMultiplicityTest, + UnrollCopyIdAndVectorMultiplicityTwoPass) { + ProfileCreator profile_creator(TestBinaryPath()); + ASSERT_TRUE(profile_creator.ReadSample(TestPerfDataPath(), "perf")); + + absl::SetFlag(&FLAGS_use_lbr, false); + absl::SetFlag(&FLAGS_use_discriminator_encoding, false); + absl::SetFlag(&FLAGS_use_two_pass_aggregation, false); + + SymbolMap full_map(TestBinaryPath()); + full_map.ReadLoadableExecSegmentInfo(/*is_kernel=*/false); + ASSERT_TRUE(profile_creator.ComputeProfile(&full_map, + /*check_lbr_entry=*/false)); + + ASSERT_NE(FindSymbolByStem(full_map, "test_unroll"), nullptr) + << "expected test_unroll (or clone) in profile map"; + ASSERT_NE(FindSymbolByStem(full_map, "test_vector_loop"), nullptr) + << "expected test_vector_loop (or clone) in profile map"; + + // Two-pass pass 1 (ComputeProfile only). + absl::SetFlag(&FLAGS_use_discriminator_encoding, true); + absl::SetFlag(&FLAGS_use_two_pass_aggregation, true); + + SymbolMap two_pass_map(TestBinaryPath()); + two_pass_map.ReadLoadableExecSegmentInfo(/*is_kernel=*/false); + ASSERT_TRUE(profile_creator.ComputeProfile(&two_pass_map, + /*check_lbr_entry=*/false)); + + ASSERT_NE(FindSymbolByStem(two_pass_map, "test_unroll"), nullptr); + ASSERT_NE(FindSymbolByStem(two_pass_map, "test_vector_loop"), nullptr); + + PosCountSample unroll_copy_id_sample{}; + const Symbol *unroll_two_pass_for_copy_id = nullptr; + VerifyUnrollFirstCopyIdHitPass1StripAgainstFullProfile( + full_map, two_pass_map, &unroll_copy_id_sample, &unroll_two_pass_for_copy_id); + VerifyVectorLoopFirstMultiplicityHitPass1StripAgainstFullProfile(full_map, + two_pass_map); + + // Pass 2 (whole map): compare CollapseCopyIDs output for the same profile + // symbol that held the copy_id row — base-only bucket vs SUM of pass-1 rows. + const PositionCountMap pass1_pos_counts_before_collapse = + unroll_two_pass_for_copy_id->pos_counts; + VerifyCollapseCopyIdsSumsPass1IntoBaseOnlyUnrollBucket( + &two_pass_map, unroll_two_pass_for_copy_id, pass1_pos_counts_before_collapse, + unroll_copy_id_sample); +} + +} // namespace diff --git a/gcov_discriminator_encoding.h b/gcov_discriminator_encoding.h new file mode 100644 index 0000000..63b645d --- /dev/null +++ b/gcov_discriminator_encoding.h @@ -0,0 +1,66 @@ +// Discriminator encoding utilities for AutoFDO +// Similar to LLVM's discriminator encoding scheme +// +// Discriminator format: [Base:8][Multiplicity:7][CopyID:11][Unused:6] +// - Base discriminator (bits 0-7): Distinguishes instructions at same line +// - Multiplicity (bits 8-14): Duplication factor for unrolling/vectorization +// - CopyID (bits 15-25): Unique identifier for code copies +// - Unused (bits 26-31): Reserved + +#ifndef AUTOFDO_DISCRIMINATOR_ENCODING_H_ +#define AUTOFDO_DISCRIMINATOR_ENCODING_H_ + +#include + +namespace devtools_crosstool_autofdo { + +// Extract base discriminator (bits 0-7) +inline uint32_t GetBaseDiscriminator(uint32_t discriminator) { + return discriminator & 0xFF; +} + +// Extract multiplicity/duplication factor (bits 8-14) +// Returns 1 if multiplicity bits are 0 (no duplication) +inline uint32_t GetMultiplicity(uint32_t discriminator) { + uint32_t mult = (discriminator >> 8) & 0x7F; + return (mult == 0) ? 1 : mult; +} + +// Extract copy ID (bits 15-25) +inline uint32_t GetCopyID(uint32_t discriminator) { + return (discriminator >> 15) & 0x7FF; +} + +// Encode discriminator from components +inline uint32_t EncodeDiscriminator(uint32_t base, uint32_t multiplicity, + uint32_t copy_id) { + // Validate ranges + if (base > 0xFF || multiplicity > 127 || copy_id > 0x7FF) { + return base; // Fallback to just base if encoding fails + } + return base | (multiplicity << 8) | (copy_id << 15); +} + +// Check if discriminator has multiplicity encoded +inline bool HasMultiplicity(uint32_t discriminator) { + return GetMultiplicity(discriminator) > 1; +} + +// Check if discriminator has copy ID encoded +inline bool HasCopyID(uint32_t discriminator) { + return GetCopyID(discriminator) != 0; +} + +// Strip only multiplicity bits, keeping base and copy_id +// Used for MAX aggregation where we want to aggregate per (line, base, copy_id) +// but not create separate entries for different multiplicities +inline uint32_t StripMultiplicity(uint32_t discriminator) { + uint32_t base = GetBaseDiscriminator(discriminator); + uint32_t copy_id = GetCopyID(discriminator); + return base | (copy_id << 15); +} + +} // namespace devtools_crosstool_autofdo + +#endif // AUTOFDO_DISCRIMINATOR_ENCODING_H_ + diff --git a/profile_creator.cc b/profile_creator.cc index 09d68b6..4b17888 100644 --- a/profile_creator.cc +++ b/profile_creator.cc @@ -133,6 +133,14 @@ bool ProfileCreator::CreateProfile(const std::string &input_profile_name, if (!ComputeProfile(&symbol_map, check_lbr_entry)) return false; } +#if !defined(HAVE_LLVM) + // GCC GCOV only (gcov_version >= 3): pass 2 SUM after stripping copy_id. + // Done here (not in ProfileWriter) so writers keep a const SymbolMap*. + if (absl::GetFlag(FLAGS_use_two_pass_aggregation)) { + symbol_map.CollapseCopyIDs(); + } +#endif + #if defined(HAVE_LLVM) // Create prof_sym_list after symbol_map is populated because prof_sym_list // is expected not to contain any symbol showing up in the profile in diff --git a/source_info.h b/source_info.h index a8f4a8f..e431719 100644 --- a/source_info.h +++ b/source_info.h @@ -8,9 +8,13 @@ #include #include "base/integral_types.h" +#include "base/logging.h" #include "base/macros.h" #if defined(HAVE_LLVM) #include "llvm/IR/DebugInfoMetadata.h" +#else +// Include discriminator encoding utilities only when LLVM is not available +#include "gcov_discriminator_encoding.h" #endif namespace devtools_crosstool_autofdo { @@ -49,7 +53,25 @@ struct SourceInfo { discriminator) : discriminator)); #else - return (static_cast(line - start_line) << 32) | discriminator; + // Profile stores only base discriminator (bits 0-7). + uint32_t disc = use_discriminator_encoding + ? GetBaseDiscriminator(discriminator) + : discriminator; + return (static_cast(line - start_line) << 32) | disc; +#endif + } + + // Offset that keeps copy_id but strips multiplicity + // Used for MAX aggregation where we want to aggregate per (line, base, copy_id) + uint64_t OffsetWithCopyID() const { +#if defined(HAVE_LLVM) + // LLVM doesn't use two-pass aggregation, so just return regular offset. + // This should never be called in practice (use_two_pass_aggregation=false for LLVM). + return Offset(false); // Use full discriminator +#else + // Strip only multiplicity bits, keep base and copy_id + uint32_t disc = StripMultiplicity(discriminator); + return (static_cast(line - start_line) << 32) | disc; #endif } @@ -59,7 +81,9 @@ struct SourceInfo { return llvm::DILocation::getDuplicationFactorFromDiscriminator( discriminator); #else - return 1; + // Only extract multiplicity if discriminator is non-zero + if (discriminator == 0) return 1; + return GetMultiplicity(discriminator); #endif } diff --git a/symbol_map.cc b/symbol_map.cc index 2a8d949..b6b756f 100644 --- a/symbol_map.cc +++ b/symbol_map.cc @@ -56,6 +56,10 @@ ABSL_FLAG(bool, demangle_symbol_names, false, ABSL_FLAG(bool, use_discriminator_encoding, false, "Tell the symbol map that the discriminator encoding is enabled in " "the profile."); +ABSL_FLAG(bool, use_two_pass_aggregation, false, + "GCOV-only (gcov_version>=3, non-LLVM): pass 1 MAX per " + "(line, base, copy_id); pass 2 SUM after stripping copy_id in " + "ProfileCreator before write."); ABSL_FLAG(bool, use_discriminator_multiply_factor, true, "Tell the symbol map whether to use discriminator multiply factors."); #if defined(HAVE_LLVM) @@ -162,6 +166,43 @@ void Symbol::Merge(const Symbol *other) { } } +void Symbol::CollapseCopyIDs() { + // Pass 2: Strip copy_id from discriminators and SUM counts + // Aggregates across different copy_ids to get total execution count + + PositionCountMap new_pos_counts; + + for (const auto &pos_count : pos_counts) { + uint64_t old_offset = pos_count.first; + + // Extract line and discriminator using helpers + uint32_t line = SourceInfo::GetLineNumberFromOffset(old_offset); + uint32_t discriminator = SourceInfo::GetDiscriminatorFromOffset(old_offset); + + // Strip copy_id and multiplicity, keep only base discriminator +#if defined(HAVE_LLVM) + uint32_t base = llvm::DILocation::getBaseDiscriminatorFromDiscriminator( + discriminator); +#else + uint32_t base = GetBaseDiscriminator(discriminator); +#endif + + // Create new offset with only base discriminator + uint64_t new_offset = SourceInfo::GenerateOffset(line, base); + + // SUM counts from different copy_ids + new_pos_counts[new_offset] += pos_count.second; + } + + pos_counts = std::move(new_pos_counts); + + // Recursively collapse copy_ids in all callsites + for (auto &callsite_symbol : callsites) { + if (callsite_symbol.second) { + callsite_symbol.second->CollapseCopyIDs(); + } + } +} void Symbol::EstimateHeadCount() { if (head_count != 0) return; @@ -340,6 +381,15 @@ void SymbolMap::ElideSuffixesAndMerge() { } } +void SymbolMap::CollapseCopyIDs() { + // Pass 2: Collapse copy_ids for all symbols + for (auto &name_symbol : map_) { + if (name_symbol.second) { + name_symbol.second->CollapseCopyIDs(); + } + } +} + void SymbolMap::AddSymbol(absl::string_view name) { std::pair ret = map_.insert(NameSymbolMap::value_type(name, nullptr)); @@ -564,10 +614,26 @@ void SymbolMap::AddSourceCount(absl::string_view symbol_name, if (!symbol) return; bool need_conversion = (data_source == PERFDATA || data_source == AFDOPROTO); if (need_conversion && src[0].HasInvalidInfo()) return; - uint64_t offset = src[0].Offset(use_discriminator_encoding); // If it is to convert perf data or afdoproto to afdo profile, select the // MAX count if there are multiple records mapping to the same offset. // If it is just to read afdo profile, merge those counts. + + // Two-pass aggregation for discriminator encoding: + // Pass 1 (here): MAX per (line, base, copy_id) - keep copy_id in offset + // Pass 2 (before write): SUM across copy_ids - strip copy_id + bool use_two_pass = absl::GetFlag(FLAGS_use_two_pass_aggregation); + + // Offset calculation: keep copy_id only for two-pass aggregation + uint64_t offset; + if (use_two_pass) { + // Pass 1: Keep copy_id for MAX aggregation per copy + offset = src[0].OffsetWithCopyID(); + } else { + // Profile reading: strip copy_id + offset = src[0].Offset(use_discriminator_encoding); + } + + // Aggregation method: MAX for raw data conversion, SUM for profile merging if (need_conversion) { if (count > symbol->pos_counts[offset].count) { symbol->pos_counts[offset].count = count; @@ -584,13 +650,21 @@ bool SymbolMap::AddIndirectCallTarget(absl::string_view symbol_name, DataSource data_source) { bool use_discriminator_encoding = absl::GetFlag(FLAGS_use_discriminator_encoding); + bool use_two_pass = absl::GetFlag(FLAGS_use_two_pass_aggregation); Symbol *symbol = TraverseInlineStack(symbol_name, src, 0, data_source); if (!symbol) return false; if ((data_source == PERFDATA || data_source == AFDOPROTO) && src[0].HasInvalidInfo()) return false; - symbol->pos_counts[src[0].Offset(use_discriminator_encoding)] - .target_map[GetOriginalName(target)] = count; + + uint64_t offset; + if (use_two_pass) { + offset = src[0].OffsetWithCopyID(); // Keep copy_id during Pass 1 + } else { + offset = src[0].Offset(use_discriminator_encoding); // Strip copy_id + } + + symbol->pos_counts[offset].target_map[GetOriginalName(target)] = count; return true; } diff --git a/symbol_map.h b/symbol_map.h index 6613f9e..043442c 100644 --- a/symbol_map.h +++ b/symbol_map.h @@ -36,6 +36,12 @@ // Whether to use discriminator encoding. ABSL_DECLARE_FLAG(bool, use_discriminator_encoding); +// GCOV-only (gcov_version >= 3, non-LLVM build): two-pass copy_id aggregation. +// Pass 1 (AddSourceCount): MAX per (line, base, copy_id). +// Pass 2 (CollapseCopyIDs before write): SUM after stripping copy_id. +// LLVM profiles use a different discriminator path and do not enable this. +ABSL_DECLARE_FLAG(bool, use_two_pass_aggregation); + #if defined(HAVE_LLVM) // Whether to use FS discriminator. ABSL_DECLARE_FLAG(bool, use_fs_discriminator); @@ -191,6 +197,10 @@ class Symbol { // Merges profile stored in src symbol with this symbol. void Merge(const Symbol *src); + // Pass 2: Strip copy_id and SUM across different copies. + // Called before writing profile to collapse discriminators. + void CollapseCopyIDs(); + // Get an estimation of head count from the starting source or callsite // locations. void EstimateHeadCount(); @@ -420,6 +430,10 @@ class SymbolMap { // profile data. void ElideSuffixesAndMerge(); + // Pass 2: Strip copy_id and SUM across all copies. + // Called before writing profile to get final aggregated counts. + void CollapseCopyIDs(); + // Increments symbol's entry count. void AddSymbolEntryCount(absl::string_view symbol, uint64_t head_count, uint64_t total_count = 0); diff --git a/testdata/hierarchical_discriminator_test.c b/testdata/hierarchical_discriminator_test.c new file mode 100644 index 0000000..2c27975 --- /dev/null +++ b/testdata/hierarchical_discriminator_test.c @@ -0,0 +1,42 @@ +#include + +volatile int a[1024]; + +__attribute__((noinline, noclone)) +int test_unroll(void) { + int sum = 0; + int i, j; + + for (i = 0; i < 32; i++) { + for (j = 0; j < 4; j++) { + sum += a[i + j * 4] * i; + } + } + + return sum; +} + +float vx[2048] = {1.0f}; +float vy[2048] = {2.0f}; +float vz[2048] = {0.0f}; + +/* SIMD-friendly loop */ +__attribute__((noinline, noclone)) +float test_vector_loop(void) { + int j; + for (j = 0; j < 2048; j++) { + vz[j] = vx[j] + vy[j]; + } + return vz[100]; +} + +int main(void) { + volatile int sink_i = 0; + volatile float sink_f = 0.f; + for (int j = 0; j < 100000; j++) { + sink_i += test_unroll(); + sink_f += test_vector_loop(); + } + printf("%d %g\n", sink_i, (double)sink_f); + return 0; +} diff --git a/testdata/hierarchical_discriminator_test.x86_64 b/testdata/hierarchical_discriminator_test.x86_64 new file mode 100755 index 0000000000000000000000000000000000000000..366682c9f1739372ec0960f736bcb1d107bf0e89 GIT binary patch literal 37224 zcmeHQdyHJwc|Uh%?(D30y-ypjp~aW5=8<$~pI!{M<7<0u?}EVwOlt^mcXsB^?%;Vb zb7#G_NZ80g;vg$FLg_zN?WC1b^^u?^RY{WuQ{zC=s4`9eKvmQ_O(MBULS3MhV-VBd zch31{XD416BDG4SbL@M+^L^)go!>cg@3}LpbD!M4Ye&ek1ecKbxFGIahbu+uEX1FF zMzUJAUMv@`xLMpJqL4&zxUz+)sxf^dq#BsUv|bpL?0Tposk?~&j^u=3${7+RyM^NF zs-iu%Vws8##gm-E2 z$>&RovK^&Cc9*o>C2hy_ypG3|@uVE-Sg-M1A9leJuc$)|iX+cLG}`UccKftG(=n~j zlwwf1|EEFf*NL}B+qF8xV zJ^%fe|Dk&JYk&54UE4C9pZO8{ARE%5Li#$T8c%v8C%yVMsi05P#z(#UYIU;dx`I;o z67mg@J6F&bo(IH|UO7mV0xy@yjcV!~}y?A$f6b(=R7ABwN5)rUu5Ih`xy zwWSB&A@r8=qGEx&7L>=ca&4xW(CF2&uOI&crbS2$kTS{}n_Wq1uCNahoUfDXsV?<| z>q^>$^HSIKRMCI@T*6DnVPRIo-Zy zOo$7gz$)BA(}gdga`y5*w4S}(iSw4~Yn30Jz1)H7vq5j{rHJE-7ir0U_VPogFFX(5 zgQt;4&zW?*ea%zk5eW#&xBV**_rCY+Snqqw-FYF7jP+h>|H|IpOJ^uQY9IW?#F5t@ z9Y6B#}b__X3LAC0shy))L(-n!6U{d;H}zU)-~_RO)t({&xZdiB`Z_imYYCtY#&y?yhp z3+N7?cFw+c>%7EclW@?jA7YU0rwSO@O9#$S{&K$6W*A{?ib232U=T0}7z7Lg1_6VB zLBJqj5HJY*Un4-judPivYp0jtdGosP1{VS8x3Co#t5tgF9s&Jl&_4nFIp`~(4}-n~ zS_J+2e^sl00UCL$T0IR)zj;!z4&5iLxen{P)~47oJk`_hcl2BSB(B#NSO6g=F(M zHbOQDN`>-y9>+fTOOjx$%?Ul)79(BC`vu7PZw#o}6oY_4z#w1{FbEg~3<3rLgMdN6 zAn@CYzzdzOO#KD?_(^46^YlY275dvsDy@p(Ay&#QDHw9 z_1`P=GgbZH8I#_ruuDYz_xn}+A}GXbn&;=g@rX-e`8Uu+FX(nWp2~8krY)NGXu48U zCj2*^7dE?66w(c>&(Ep!tV?B;BKU(|SJqg5PUpk@7|?q7BUD+~-}_b7pHo^NKjO;% zG0u3zmHczMU7P>E5&rw}#(<-Gyc^{of_FcJu2D4z7z7Lg1_6VBLBJqj5HJWB1PlTO z0fT@+z#w1{FbEg~3<3rLgMdN6AYc$M2p9wm0tNwtfI;9l5`p>6zmaK~qznQE0fT@+ zz#w1{FbEg~3<3rLgMdN6AYc$M2p9wm0tNwtfI+|@U=T0}7z7Lg1_6VBLBJqj5ct3d z?A*5P4(ArUz3f(J@V59se83qR7#bZ|H#q3r;?EWRQYPo;gGBaLu@OGiE(~ffh10w< zg^oqr^HY(#^vT-d)9--|iS2Nq?0H8E-f3d7wh&vOX$wm#FFTCZ-wp4)CMCoTa9zmIxz*c@b{?o(GIHrBaYX|G@>S)uzE#H%g0;7z%Vo- z!m&GUur{>Z-ZIf0TJ<+uu{%k4k#O_sP!DX^i3!YB+p;FQEmWd6fbU0^O>t}U>TsL( zar5el?oO+#$?Z9`n$%)&#cl8pN6q8nlh%`HTGnZ;ZR>71X0;r*qTw~yt$AQor`3V* zEjm3+ht`{9)aC6+n8LT|U@aT$+gm8sMj55umTDbvFFIQ+N`QiQjaw>I_k;-bd*m$<-6X_xLk-2q0?Oh|4493J$uN1htasa% z6M>MXCMs(}s0w=FVF>m1@M>b^iRP(guM*86o@lG=aCi3#yHkWBAr^dnx2Vb=*{h*uMK>sQ7_xSxrXAU+wIfSwM{C+q){ZW%9jj@_ zHrhRuzzH29Hqe-mZ0_W~g&(!8Xt#ZRbdnhp?5L z?nBhO>RCsIdXaU*568YKgW2!1m3*m? z&5Bw>awbs{^2M3bpr$wLCxb%C%N7bnQJyK3g2le{TQ<|#LL$)ZWQseovu$YjDmP`P zFPA7KXEx0w$_M;x7HlpzhjXz|Oy&br%4J`jhZGr>r0gr|kHiO+d?Zc^iCHqnnV5W` z57r|IRZZs-BdRu6D53&hvCv*OIFtr6mCLBJbmf=96qDIph0=mEr8v8Yn>tY6mU_IN zOQ!0%S>?UvC)1F?tm^V&7i6TnGM&y-cCepSNj=`6n&)_55sc z2m|Gp${1fIA5P^9+@X|2aw?}D+Dwa$RL?bXDgTrMQxYN9+={mLw>1%dUB3~ z`T56_If4S=lSInX<*9@m*7RJtr0iz0a+nh{G!BT6JwRiWJ)|ZtQRU%050W z(pxFeIFzxTOk^{MbT;K_9j-j1XH$8m9PC#{0fQHm=d!7wq{XudIkS)>rCZ#ub38DW zFJ$tw6mYJbnAHJfl~9wF8sL(n`3WzH++1Ifk@#s2qq=^fZDH6eXkffce%aTfw_nfb z1DTZ1b`Nw+Gk#wgGiMXI zYiz(fo^|Uea_82edb7cG!=nS}S5V3O3)f4`p55t0`9oZ9>Ehm(DpaPje&2d>$WvB2 zp7!&Y;xyyrby>Z2+UY|W^?x^#=Odj=_Z3UHDkOtUA-^eIp$m9=s*=g3GKhiqE>9=n zDNVx@B$E4Cxln?zNI|aPL112b)FZsZIhYZ-xn#M7mthx@iERH&A?Nq+pUuqr`xE&@ zK9vwN8NWn{X3*@VGUX&5_cHke-LL2#7f*`*O1adZ$tSawlrPFbDpQEhhLh~6)|iW?*42Bf^v5nOll`?{y|747+I(d5GytqQ1w zM2w$sL}=h7UNjx!N0AWm5xH=nWuhgby!eZV2-+f%Ro0%!s++C*A~)ZL+HDi4P3*Sz zkP+7J;a@e_Vci?qF%f?J$&;jbZ@uqVI$g4YZ8!PuzA-d-GIHZ^Xy{~QczbC0WMunY zp^=kv`+UwF3lA7Xzo8$n5H)l>h=}IqNQWMt*rtj5@PgIEtFWZN zGJQdVS3OHTXG0f-LPxL}zhvr81gnLXC&I3jI`XXEIesPm5wu@Pz7_J2Xcvvy(39Ae z^gp5XJ2Z}9WqAZ=^5ZthX_ZIv`3CtjkXzW@lHO-Wh5AY5IUJT)DR{q2Da7d;IAop; ze$Gn$6@vHAWc{V`|B2M^7Ogk9tXiX2>hD`p{~FP_&*Qt&s!OywZq1%5ZRm(4dIax} z$&I~`cQI2(E08;Q`f*&hPKs|?Anyb|{H+6&WEqq4)q=mZVEInSJK%@$WM}t4-WBQ) zyx%Cx_mUjr!t<1$=qII|wH9bu`)|y19&%?vtycRyp~F&Feuh!| zbI2(lzA8-0Rw2G6_0cc(rnEWw0xKJ z4?m}r98P8Vq4wX1=LPL&;X|$Te?flG-$wmkl70xgzUsJ28)saL2jd0dd5Niv7bK=X zz%_sI<$LiB1#RV^f)`6BManPv)0r}sz+RB^lDIMYWyDDpyye2M4zsYH+vGR;Mgf9mnAW-t_AM;(N&atCHQSIBGM z6e?9Ncr&>9W_{sJ>_!7__8!((Ddo2QX%Sx(0*sXKQdzI$(|t1V$y1O?dc_Rvy{U3p z$B_@7p11ezZ43j!gm1XY2XjGU3N$DwiWRo1VIAm~iXxuJ5+siMZ5)dceBm1$tP{yw zbYCW=MO(H`^wAuW>UikHQXq9Oj}VFmCDnqpYZ*(LMU00wbRt0px>n2vA}-U62mT!D zGzrjFkXXn489nSXDcCZnEY$cb3$DX)B9}=bY5~4*ipv1X8ITfjOhfuM8_D`pm1)=| z^3#~Jn$6_X1+FKirb_-SbF{qVI?&@Pi2?rtUr1*0d8j!o<);;2`!N3c722;EA9389 zROt!3veKhBN^ZQnlWu*v5e$7V%=YWG{d#Th)GMQ|mQxGDQB>>g_i6im+P-%|Ro3`= z%i@U!dtOgGu|)hvds+|BHJ0-~rQ@G!h)+3E;r!XpA3*j$*u$CadEe^Sv_0e3XfK`N zl)yoIRr>i(Ku2oKpA$HW3aPR^@2h>T!JgxDKHUF5)b?B1 zq3x=gzHzc1*;BpI{)>WfaRCOUyyy}hID9c}N_G*v~5`8F-S z++ff5WmhW>FfiJWt0|MeM5R7Huak;hN`v=nXZ^WWii#dV_J z#6j`dp4XGD4umLUiqy`H@xP0fdiy;({vJn}oK~cE=J8{}4^Y8RBl0sqGG0fY?<0=Z zSc?nWGyOL-W3^SY=XLuY8N)eb9|=1zl8l-y0D}C zt0J{C+c8aR`=9R%kJu%mUU4et= mJ>M6aA>+BLEY#U~O3`XVzy_9{1g~hn<{4%GOoKrKEB+U*#@*@w literal 0 HcmV?d00001 diff --git a/testdata/hierarchical_discriminator_test.x86_64.perf.data b/testdata/hierarchical_discriminator_test.x86_64.perf.data new file mode 100644 index 0000000000000000000000000000000000000000..27cba94dc577a05efc2f03670c169337e0f0c76d GIT binary patch literal 73612 zcmeHQ33wD$wyv;+5rwd?fxrmEmK`FXjx9UFKx7dSnb7Gpq$TN&-3eh)2?z)(8W9IYHMw*#<2Vw7Kr5g<5#Z~kSW`mk8i_iTTa_?+Md%6 zoW^t7k<(6`CUDxB(=MDECtGsL8OxfeOzdw|+Ja?uH!gCXqV|)Y4S#}!2B03N@?m2r z?Wc9(eA~hoyBR;1Joe{FKOFttafVMMxSSz`l*XLR64c)bMgH;PLUm=RR~(NNa@2Pn zJ`O45;Gr(dkwOlB>aiRtv9KK=UlIjL#-rT9qEjPe-Y0=Iu!qI;IxWzssC8B0GKYIIz}$>VZ8Zokv- z%JsOMd09E06ZVe%_oA2YzN5Zo)Jq+0)d^gSb10*Fx#`2HQUAs@uC1f0mSIwR9sB*VTTnnDCUwEm0mgPqrLkQCeiUzSq}WGP72RpIq)aC zDtrsefxp{L;pHp`ek#N(-#CB1(2xI}va^Qeuv5bM<}8Pu#+(lb{fV5fC-jdq z?hyLta{m7~h5gSs|B)!?Zq9!o^i#RMwg`RIj+UI@QOlftY*Iicv7E0^w4)0*A2Ddx zwmtaVRYrQE9Tf%q$$6ga#tna{|68C+!~@JD?GIE9~loY}3^h(iSeKfw40oZ`IGZ)i#9gB
    3qH_=USFS|KIL*RGyA32i}(Rr7Q=2 zCm$at^dB4Amh5B-`#A4Pmcve{=LKJHG{`dXwPW>A;;a1>Wf^>}AFHl+3+tmE${4?w zQ*G3ZVtg*Az-M#*I)Puw`C@^0=X|ojpWMUe75LEy6n-De zao(G@jYtf3%*vLuk^}TANEHxUdSo@KAH0a zIEDU_&((P3-~eS0c=N*w|Nf{-f$wE}(T8R{`;f{%$8zZJVEj@}fj53`E?ws~fp@D= zxW;nmKgD_nHYhppL5yz_c0N6*$`dQ_?OeW6mc!0s#tTIGC64+;JigRBhWrM6)qU?0 z;_HQVYGw$&p!as3(#Lrb=O72}eOAS}B#wi?dpvY0oey&0_cZNDcqf(vZ^!u(mIJ?Q z^J3DE6Z(xfKb7Us&!~te{f0unF6ZM|4*fZgb)~oiIq)AJROO$_a^N1m-UBQLzGI83 z2PeyccipD&l`IE7;~9njmgT_VU&EU@O!2Z_r#}#1*NrtIu4YV2>66WR4o+dm#($am z0biMdufvPR5MSxLY`zR8tkOP%@wK}1SmLYUd1|H0_AduH(CLmR;7i8CXTBIud@X#^ z=Ic!GuM3Y&RQ=a))Jn1KU)J`orSX%AukQxid|A=I4mDNn2jlsduKXkM6_{z}>uesE z>|}k!a~n5U)wF2W1NyJ=VtlCm!%2LlmYMmo$L)%j(^U`POO7{6JGzOlm)@}XG8nSj zcz&0Nhp+rKm-t%vwVAJI`@3yl`ERg#fPcyU>xEu<#Mgwz#!Bm4dlT0KdJEd=BO0-(~LjQDeV7;^~xBBeg)$%3OjB2ypIU{{`cF_ zb2Uv^4m)!@s_R_Ga^TAn6h4CGz`H#&l+D=-K_3gF2dDx*ot6{2qL*dUX}?b!>r|FX&liK2qDt1HR;VQpKzyF>18n(pTf8D=Z>9uEl#M@S!pUQIJAB=8I@%*zF)p5X^?Naf7FUx_y z_ng8%;S~7uthZO-4aO+@{}gu4<$PV1!_JS4m;ON=2YmVKs(jxGJ8!XgwfG@c|fA^$~#MgG=U(qg)ZNF;X!IvCw4D9&?@wMCTzbx5*MC=Uqa9|fyV@FkyTTl>W;#8-#8*6=T@wew*0 z8EqVV$@$mhr(P$%uDZ_5SG3Dx+ppSB!IykLC40qxh_3zp$EptNAtflK%CN zDen?rkIysnWw-yT+kdp{2Yku(!vQ_tC%(1_|B7~bZ2MKW2k<5RYu?v~iLV3p^EOn0 z7(f5q{3zP(Ef`;}T}Oy7_d+`!8iZM$udS;;Q}OV6n=dQ!h4zE_@+Q}p#8(U3zbqLK z<$U?=#AC$QCt|(#bd9&IHm;6#KNZZr(SI4wZ#4e>qu>a$d>#a+U+fcth^L;ootb_*!z4nXl93 zU-o$^EYttlsO2W_N#sea>VDuy#N1T zmIHr|@t4H^2e^p!$_2h@pR(Ua*!i6CwVcBKe$Ia?@CMg*pyvkn3H->#3O{;{N?~W0 zL*=)#9Qeb(SNLNr2Y!T)FJn3I$44mrcZGfUmwdl+VO$g9>vG$_EIFQ+>rq4BZbp2) zV?Td*M)0-s{#L}-y&@h)yFazq303yV=Z_9j~g6SJAE?@b#_Oe`DeYU5PKJ?O#NYRs0ux z$^PrurXIvsiS1vO?7w6@Tw0n)d~LDE8)pPx=Vqr8UwiEFye0XP`_GR|?@N3wu=%na zU&j5f)$`iWeHg4C=GW^_d`%R5B`i>Si&-D*UEUIf$FUrEChLu3Iq)9usP}^RUZ;)& z-jv@HPGC9kulT*<@_(D(gI>h%P5)QCcYG1!)5Uw%cf6#IPZ#+3xe8y-a-4S==ga1p z&zs1{-yrm({eCyb8?yge@#+P{*XQZSMFHrAim}czJkr;Y@BEfd7s!& z?+3xZ`#^#(Io_Bua5(XGlenJ}?fJWHU$uu|^#H!= ziT5kNX)=oV>MG`6wtrRaU(v1~@Fo50tOH|+ukQq3!Fadvs{JBT^#i`-`0%Roal}`= zy&gqGV7(6G{7BV*sCEp#jQLlaRt|bj0rRi6^CuEty#!wwO_W}FOO>MhU-No#3d?~< zJ3gE#_9Hb83FcpoE}u+%1?+feIsd9`{&h*$EaI!&9?uhBHNBq_Y5zieMm)T>hl}{S z(;m-T(!b>M0AGGPjrgjVW4^z9rt}xT)^ei%lKb0SwDl_DtH|anBJnWP_7g0Af-m|0 z@|at^#Meae{~badf7<#)cf|Iw1YdH#Jm?BP@s(-szhTM#OZwNQo->KBOT_=B=G zfAR4D^@-zkKg>@XbKWif?@zG!3ICGgjkms@O?-_Pd<*8}*H!O^ob5uQ) z3A|5=Di7qakMeh5Iq=q;m*qc_tnB_0rXk5?`w;Y`(0>ms~%5^U{sP*NBJBe1*EcXX_hmYazd9 z6s+D559NLitJ0nzzCOGEm*uOrt=~qf|ME0_n)sS=kD0Gf?c4gwzamu+@UNrde#$jp zJVSg9{;QdU}2$?@UxvR%YilT~KE zBIVb%o$%r$_>#{@6&LRzz8bGI^A)LfV%rHXeu6J~Kc)S+eZ<$_mYeyq{USWSh*Z6S zFS$QJ@9wV>Un_1k^L4uX!XD?Vb^^X+|MlqSuM=NmZ!z-~sdi%93C~Z#*Oy{E?|t?^ z#8>@NGhdCUrX)%0m#3s692)M^skst zKO??A6why1t-aN@@_;WnKK%CSFNv>u;(4}E$A?za54FDtb^o+zw;%9@dd2wg`EAFD zuS4Q_QRqdh|2Qm@ikiT6>Wd7 zZT3U$_Yn`}^8hiwZB2aLvcT@Y3q&eKw)wK8 zf64XU#toB*uZP5bMBo+k>(l+dVQni9{7a5E9)C5J`107}LreOXT;C{ss4wyLzcyc% zuYO{_T-)N0)vg!uP_EC<8F?Y`6)XC$+O}SFx~@0aIu-bm>rpFPUQB!? ziFkOr%5R@njq|~meBNf?`-6$E?`;2~ii+0yVK6@mw%;cBlK)?C(}v;1SMx>odaprP zwD<}&Zh$X2K0LT&6!EoM@MX3Bx3-nX;LF&r@~IAEO#6M!x-3)iCE{V+IHgz0`j{{8 zxkBM(V^s?LtD$4b?ky|_-e8!*53xLkqK>A$$NBFBzLbwIVH|cAPFCleD(v4sNu%=? zupIgWcAiWBmoA0nz!8qiohaAQiaSHuf*;NK5|Q67@(qtGeHDE*EspcBH(umd3H>t8x8U+@(A1B9oIetyer)CozptLk zU#U*3)vM2nBHx<}lyVO1$0>!I<5ixs+8WLe=i?7?Ue&Lr9Txc`oL|GIUD8SYfIgcs zB{3yA+Hw7}r&s4)omTc-zP# zS1(hC&rvqz;No5@UfBBl@g0w@f3n#iPtzOTHO`To=W{vpl5>6e?&KLWJu}@ioL;9l z$C;e#ar>QqSFT4FXXSVTF25(=<8>DL{8@$WK;fxp$2s1JT|9cj)^c~fO}WduzIJ%n zq#p~G(K(6&{^UGQcJi!#8Ce;967xLXqFIU4y+z4+Ih@fYi5Y#80=}fQQ!StTjdL7K z9eKq!m+YR~aOmp$+O6IF?A&$x(x|Rx<^+7rWV|yeF z^7-A}$N!;6g6F)%zWvftMkWl(oRHv1NKZ;n88&>5u3G8Gp`*jx-Xf3JO{RKh^v+1{ z9ohvRDm+HE4SzE1C8JHL##HF@B0HEg8y?pICi%GC@T>Am|`F03YM*yz(cHKm&Bm8uyw z`u46FH!#eUYjqf?%PIQj`cUMwy84l7L*n#5n??B$7r*Z}kdATae{0&F<8>Q|v#x?7 zvK#Inqh1~QH{0)a&M5GCyrz*u^{sYfJFY;Xv(TNT2X1%u4?ab+li5zejOnKcL-=XL zNQeHNQQ-GwhgTFj+gWTUKj5c)xPKDZPJyr3T@y=a&jH`G!eVEbm8_S=@H2=0E%ZBG z?i%pklI;|E4a&1z*}hp>;RKadmDmX<=(n!KPG}<;-GKgSqkkRx_tfi3*Ylx%+Uf^5 z^zW&5`h;pk*KenP9s2iFJAL(hsGr9A0S^7kc5 z{f1!^edAzfp;OO>{c-3=rl*Yz`*`C(hyJbUxV*A!8{rKU`ioqJo-h{+jB;7^+N{D` zP3J;Z<}Di6W)9y6-$4AB(Nz;E|JUPMZndZ?qdIP@J zrw^VrGWt9I&@IYS@&w<=LZru!UqsJW7|&SH2RD6K4|SLAcY0m9?f}CfCl39H)SBJ; zBi{0zv$6{Go6?>@I4ft0`>+-hPq|{_S_<-uvb@gx@Cu