diff --git a/.ctrace-analyzer.cfg b/.ctrace-analyzer.cfg index 2cacfe2..212a469 100644 --- a/.ctrace-analyzer.cfg +++ b/.ctrace-analyzer.cfg @@ -11,6 +11,7 @@ buffer-model=models/buffer-overflow/generic.txt analysis-profile=full jobs=auto compile-ir-cache-dir=.cache/compile-ir +compile-ir-format=bc # Output behavior warnings-only=false diff --git a/CMakeLists.txt b/CMakeLists.txt index b5ce98d..a6a2643 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,7 +67,10 @@ endif() set(STACK_ANALYZER_SOURCES src/analyzer/AnalysisPipeline.cpp src/analyzer/DiagnosticEmitter.cpp + src/analyzer/HotspotProfiler.cpp + src/analyzer/IRFactCollector.cpp src/analyzer/LocationResolver.cpp + src/analyzer/PerFunctionInstructionCache.cpp src/analyzer/ModulePreparationService.cpp src/app/AnalyzerApp.cpp src/cli/ArgParser.cpp diff --git a/README.md b/README.md index d6afb27..05159f9 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,7 @@ Ready-to-adapt workflow examples: ./stack_usage_analyzer main.cpp --compile-commands=build/compile_commands.json ./stack_usage_analyzer main.cpp -I./include --only-file=./main.cpp --only-function=main ./stack_usage_analyzer main.cpp --dump-ir=./debug/main.ll +./stack_usage_analyzer main.cpp --compile-ir-format=ll ./stack_usage_analyzer a.c b.c --dump-ir=./debug ``` @@ -240,7 +241,8 @@ Ready-to-adapt workflow examples: --resource-summary-cache-dir= sets cache directory for cross-TU resource summaries (default: .cache/resource-lifetime) --resource-summary-cache-memory-only keeps cross-TU summary cache in memory only (process-local, no files) --compile-ir-cache-dir= enables dependency-aware LLVM IR compile cache for unchanged source files ---timing prints compile/analysis timings to stderr +--compile-ir-format=bc|ll selects source compilation IR format (`bc` default, `ll` for textual LLVM IR) +--timing prints compile/analysis timings to stderr, including aggregated hotspot ranking --config= loads optional key=value config file (CLI flags override config values) --print-effective-config prints resolved runtime config to stderr --smt=on|off enables or disables SMT-assisted reasoning (default: off) @@ -271,6 +273,19 @@ sanitizers, profiling) while keeping include paths and macros. For multi-file runs, `--jobs=` parallelizes input loading; with cross-TU enabled it also parallelizes summary construction. `--compile-ir-cache-dir=` reuses compiled LLVM IR for unchanged translation units based on source/dependency stamps, which reduces repeated C/C++ frontend cost across runs. +`--compile-ir-format=bc|ll` controls source compilation output format before module load: +- `bc` (default): compile to LLVM bitcode then parse bitcode. +- `ll`: compile to textual LLVM IR then parse text IR. +`--timing` now also includes pipeline traversal estimates per step +(`module/function/instruction` estimates) and by execution model +(`subscriber-compatible` vs `independent`) to help identify repeated scans. +It also prints a process-level hotspot summary sorted by cumulative time. +You can control the number of printed hotspots with `CTRACE_HOTSPOT_TOP=` +(default: 20, max: 200). +For rollout/A-B checks of the subscriber path, set: +`CTRACE_PIPELINE_SUBSCRIBERS=1`. +Reusable A/B benchmark helper: +`./scripts/bench/pipeline_subscriber_ab.sh ./build/stack_usage_analyzer`. When inputs are auto-discovered from `compile_commands.json`, `_deps` entries are skipped by default to keep analysis focused on project code; use `--include-compdb-deps` to opt back in. @@ -309,6 +324,7 @@ Supported keys: - `resource-summary-cache-dir` - `resource-summary-cache-memory-only` - `compile-ir-cache-dir` +- `compile-ir-format` (`bc` or `ll`) Example file: @@ -320,6 +336,7 @@ compile-commands=build/compile_commands.json analysis-profile=full jobs=auto compile-ir-cache-dir=.cache/compile-ir +compile-ir-format=bc smt=on smt-backend=z3 smt-rules=recursion,integer-overflow,size-minus-k,stack-buffer,oob-read diff --git a/archi_plan.md b/archi_plan.md new file mode 100644 index 0000000..913c2a6 --- /dev/null +++ b/archi_plan.md @@ -0,0 +1,32 @@ +# Plan + +Ce plan vise a reduire le cout algorithmique des analyses LLVM en mutualisant les parcours et les artefacts intermediaires, sans changer la semantique des diagnostics. L'approche priorise un moteur de pipeline avec dependances `requires/provides`, puis un collecteur de faits partage pour diminuer les traverses redondantes. + +## Decisions +- Objectif prioritaire: reduction du temps `real` (wall-clock). +- Strategie de delivery: migration en 2 phases. +- Selection des analyses subscriber vs independantes: revue module par module en phase 1. + +## Scope +- In: Evolution de l'architecture d'analyse (orchestration des modules, partage de donnees inter-analyses, cache d'artefacts derives, validation perf/fonctionnelle). +- Out: Changement des regles metier de diagnostics, modification des fichiers de tests, migration complete de toutes les analyses en une seule iteration. + +## Action items +[x] Phase 1 - Etablir la baseline perf cible `real` (timing + xctrace) et definir KPI primaires/secondaires (`real` prioritaire, `user/sys` en support). +[x] Phase 1 - Ajouter une instrumentation technique des traverses (compteurs `Module/Function/Instruction` par analyse) pour mesurer le niveau de redondance avant refactor. (fait: detail par step via `Traversal estimate detail` + agregation par modele) +[x] Phase 1 - Revoir chaque analyse pour classifier: `subscriber-compatible` vs `independante` (dataflow/fixpoint). (fait: `docs/architecture/pipeline-subscriber-classification.md`) +[x] Phase 1 - Refactorer `AnalysisPipeline` vers un modele `requires/provides` pour declarer explicitement les dependances entre analyses. +[x] Phase 1 - Introduire un squelette minimal `AnalysisArtifactStore` type-safe (API + invalidation par module/config), sans figer tous les artefacts. +[x] Phase 1 - Ajouter un `IRFactCollector` comme premier pass du pipeline (parcours unique Module/Function/BB/Instruction) exposant des `IRFacts` partages. +[x] Phase 2 - Introduire un mecanisme subscriber (Observer) pour les analyses classees `subscriber-compatible`. +[x] Phase 2 - Migrer d'abord les analyses pilotes vers les faits partages (priorite: `StackBuffer`, `ResourceLifetime`, `Uninitialized`; candidats additionnels: `IntegerOverflow`, `GlobalReadBeforeWrite`) et mesurer le gain incremental sur `real`. (fait: `StackBuffer` + `ResourceLifetime` migrees; `Uninitialized` classee `independante` apres validation de non-regression) +[x] Phase 2 - Etendre `PreparedModule`/cache avec artefacts derivables stables (debug index, symbol maps, type facts) avec cle versionnee. (fait: `DerivedModuleArtifacts` schema `derived-module-artifacts-v1`) +[x] Phase 2 - Ajouter des garde-fous d'architecture: tests d'integration pipeline, checks d'invalidation cache, et assertions de non-regression diagnostics. (fait: checks `run_test.py` + isolement des caches cross-TU; suite complete verte) +[x] Phase 2 - Valider avant/apres et rollout progressif derriere feature flag, pilote par la reduction du temps `real`. (fait: flag `CTRACE_PIPELINE_SUBSCRIBERS`, benchmark A/B, validation complete) + +## Validation Summary +- Build: `cmake --build build -j4` OK. +- Tests: `python3 -u run_test.py --jobs=4` => `Passed 1661/1661 tests`. +- Benchmark A/B (`scripts/bench/pipeline_subscriber_ab.sh`, 5 inputs): + - baseline: `real=0.24s user=0.19s sys=0.03s` + - subscriber: `real=0.24s user=0.19s sys=0.03s` diff --git a/docs/architecture/analyzer-modules.md b/docs/architecture/analyzer-modules.md index baa3b28..5c19832 100644 --- a/docs/architecture/analyzer-modules.md +++ b/docs/architecture/analyzer-modules.md @@ -15,12 +15,16 @@ This document describes the module split introduced around `StackUsageAnalyzer` Role: - Entry point for module-level analysis execution. - Coordinates preparation, analysis passes, and diagnostic emission. +- Declares step-level `requires/provides` artifact dependencies. +- Tracks per-step traversal estimates (`module/function/instruction`) for timing mode. Pattern: - `Facade` over lower-level analysis services. +- `Pipeline Orchestrator` with explicit dependency metadata. Why: - A single coordinator makes control flow explicit while avoiding a very large `StackUsageAnalyzer.cpp`. +- Dependency declarations reduce accidental ordering coupling and make rollout of shared passes safer. ### `src/analyzer/ModulePreparationService.cpp` @@ -33,6 +37,30 @@ Pattern: Why: - Preparation logic is pure module state derivation and should be reusable without triggering diagnostic side effects. +- Precomputed `DerivedModuleArtifacts` provide reusable debug/symbol/type indexes with a versioned schema key. + +### `src/analyzer/IRFactCollector.cpp` + +Role: +- Executes a single shared pass over selected IR to collect shared counters (`IRFacts`). +- Supports optional subscriber notifications during the same traversal. + +Pattern: +- `Collector` + `Observer dispatcher` integration point. + +Why: +- Centralizing common IR facts avoids repeated lightweight scans in individual analyses. + +### `include/analyzer/InstructionSubscriber.hpp` + +Role: +- Defines subscriber callbacks for instruction categories used by rollout passes. + +Pattern: +- `Observer` (`Subscriber/Registry`). + +Why: +- Allows incremental migration of analysis pre-filters to a shared event stream without changing diagnostic semantics. ### `src/analyzer/LocationResolver.cpp` diff --git a/docs/architecture/pipeline-subscriber-classification.md b/docs/architecture/pipeline-subscriber-classification.md new file mode 100644 index 0000000..9d14def --- /dev/null +++ b/docs/architecture/pipeline-subscriber-classification.md @@ -0,0 +1,62 @@ +# Pipeline Subscriber Classification + +This document records the Phase-1 classification of analysis steps for the +single-pass subscriber architecture. + +## Classification Rules + +- `subscriber-compatible`: primarily instruction-pattern driven checks that can + consume event streams (`alloca`, `load`, `store`, `call`, `invoke`, + `memintrinsic`) without requiring a fixpoint over inter-procedural summaries. +- `independent`: analyses that rely on iterative dataflow/fixpoint solving, + cross-function summary propagation, or specialized graph/state convergence. +- `utility`: orchestration/preparation steps (not diagnostics analyses). + +The split mirrors LLVM pass-planning guidance (explicit dependencies and +analysis invalidation boundaries) and keeps behavior deterministic during +rollout. + +## Current Mapping + +| Pipeline step | Class | Reason | +|---|---|---| +| Function attrs pass | utility | Canonicalization pass, no diagnostic rule. | +| Prepare module | utility | Context/call graph/recursion derivation. | +| Collect IR facts | utility | Shared fact collection pass. | +| Build results | utility | Output scaffolding. | +| Emit summary diagnostics | utility | Aggregation-only emit phase. | +| Compute alloca threshold | utility | Config-derived threshold materialization. | +| Stack buffer overflows | subscriber-compatible | Event-driven from stack writes and allocation patterns. | +| Dynamic allocas | subscriber-compatible | Direct `alloca`-shape detection. | +| Alloca usage | independent | Depends on recursion/global stack metadata coupling. | +| Mem intrinsic overflows | subscriber-compatible | Mem intrinsic rule matching over instruction stream. | +| Integer overflows | independent | Range/dataflow-heavy with conservative propagation. | +| Size-minus-k writes | independent | Wrapper-summary propagation to fixpoint. | +| Multiple stores | subscriber-compatible | Store-pattern correlation over stack slots. | +| Duplicate if conditions | subscriber-compatible | Intra-function condition canonicalization and matching. | +| Uninitialized local reads | independent | Inter-procedural summary + CFG/dataflow reasoning. | +| Global reads before writes | independent | Global state flow + summary-based tracking. | +| Invalid base reconstructions | subscriber-compatible | Local reconstruction chains, no inter-proc fixpoint loop. | +| Stack pointer escapes | independent | Inter-procedural fixed-point with conservative caps. | +| Const params | subscriber-compatible | Use-def traversal around argument write/read patterns. | +| Null pointer dereferences | independent | Branch-sensitive path/state reasoning. | +| Out-of-bounds reads | independent | Bounds/range refinement over dataflow paths. | +| Command injection | subscriber-compatible | Call-site taint/pattern checks from instruction events. | +| TOCTOU | subscriber-compatible | API call ordering checks in function-level streams. | +| Type confusion | subscriber-compatible | Type/layout pattern checks over pointer casts/GEPs. | +| Resource lifetime | independent | Summary propagation across calls and class lifecycle states. | + +## Migration Notes + +- Fully migrated to subscriber-shared gating: + - `Stack buffer overflows` + - `Resource lifetime` (safe call-site absence short-circuit) +- `Uninitialized local reads` remains `independent` after validation showed + that a simple `load`-based skip is unsound. +- Candidate next migrations (subscriber-compatible class): + - `Dynamic allocas` + - `Mem intrinsic overflows` + - `Multiple stores` + - `Duplicate if conditions` + - `Command injection` + - `TOCTOU` diff --git a/include/StackUsageAnalyzer.hpp b/include/StackUsageAnalyzer.hpp index b4e93e8..4cfcf5a 100644 --- a/include/StackUsageAnalyzer.hpp +++ b/include/StackUsageAnalyzer.hpp @@ -41,6 +41,12 @@ namespace ctrace::stack Full = 1 }; + enum class CompileIRFormat : std::uint8_t + { + BC = 0, + LL = 1 + }; + // Analysis configuration (mode + stack limit). struct AnalysisConfig { @@ -70,29 +76,31 @@ namespace ctrace::stack std::string resourceSummaryCacheDir = ".cache/resource-lifetime"; std::uint32_t smtTimeoutMs = 50; - std::uint32_t jobs = 1; + std::uint32_t jobs = 0; // 0 = auto (hardware_concurrency) analysis::smt::SolverMode smtMode = analysis::smt::SolverMode::Single; AnalysisMode mode = AnalysisMode::IR; AnalysisProfile profile = AnalysisProfile::Full; - - bool compdbFast : 1 = false; - bool demangle : 1 = false; - bool dumpFilter : 1 = false; - bool dumpIRIsDir : 1 = false; - bool includeSTL : 1 = false; - bool requireCompilationDatabase : 1 = false; - bool jobsAuto : 1 = false; - bool quiet : 1 = false; - bool smtEnabled : 1 = false; - bool timing : 1 = false; - bool uninitializedCrossTU : 1 = true; - bool resourceCrossTU : 1 = true; - bool resourceSummaryMemoryOnly : 1 = false; - bool warningsOnly : 1 = false; - bool reservedFlags0 : 1 = false; - bool reservedFlags1 : 1 = false; - std::uint32_t reservedPadding = 0; + CompileIRFormat compileIRFormat = CompileIRFormat::BC; + std::uint8_t reservedBytePadding = 0; + + // Keep flags in one 32-bit storage unit: + // 4x u8 enums above + this u32 block keeps tail alignment compact on 64-bit builds. + std::uint32_t compdbFast : 1 = 0; + std::uint32_t demangle : 1 = 0; + std::uint32_t dumpFilter : 1 = 0; + std::uint32_t dumpIRIsDir : 1 = 0; + std::uint32_t includeSTL : 1 = 0; + std::uint32_t requireCompilationDatabase : 1 = 0; + std::uint32_t jobsAuto : 1 = 1; + std::uint32_t quiet : 1 = 0; + std::uint32_t smtEnabled : 1 = 0; + std::uint32_t timing : 1 = 0; + std::uint32_t uninitializedCrossTU : 1 = 1; + std::uint32_t resourceCrossTU : 1 = 1; + std::uint32_t resourceSummaryMemoryOnly : 1 = 0; + std::uint32_t warningsOnly : 1 = 0; + std::uint32_t reservedFlags : 18 = 0; }; // Per-function result diff --git a/include/analysis/CommandInjectionAnalysis.hpp b/include/analysis/CommandInjectionAnalysis.hpp index edab2f2..a8fa829 100644 --- a/include/analysis/CommandInjectionAnalysis.hpp +++ b/include/analysis/CommandInjectionAnalysis.hpp @@ -6,8 +6,10 @@ namespace llvm { + class CallInst; class Function; class Instruction; + class InvokeInst; class Module; } // namespace llvm @@ -24,4 +26,9 @@ namespace ctrace::stack::analysis std::vector analyzeCommandInjection(llvm::Module& mod, const std::function& shouldAnalyze); + + std::vector + analyzeCommandInjectionCached(const llvm::Function& function, + const std::vector& calls, + const std::vector& invokes); } // namespace ctrace::stack::analysis diff --git a/include/analysis/DynamicAlloca.hpp b/include/analysis/DynamicAlloca.hpp index ec6d852..b6d4c01 100644 --- a/include/analysis/DynamicAlloca.hpp +++ b/include/analysis/DynamicAlloca.hpp @@ -24,4 +24,8 @@ namespace ctrace::stack::analysis std::vector analyzeDynamicAllocas(llvm::Module& mod, const std::function& shouldAnalyze); + + std::vector + analyzeDynamicAllocasCached(const llvm::Function& function, + const std::vector& allocas); } // namespace ctrace::stack::analysis diff --git a/include/analysis/MemIntrinsicOverflow.hpp b/include/analysis/MemIntrinsicOverflow.hpp index 657f35e..a0a3d9c 100644 --- a/include/analysis/MemIntrinsicOverflow.hpp +++ b/include/analysis/MemIntrinsicOverflow.hpp @@ -6,12 +6,15 @@ #include #include "StackUsageAnalyzer.hpp" +#include "analysis/BufferWriteModel.hpp" namespace llvm { + class CallInst; class DataLayout; class Function; class Instruction; + class InvokeInst; class Module; } // namespace llvm @@ -33,4 +36,11 @@ namespace ctrace::stack::analysis analyzeMemIntrinsicOverflows(llvm::Module& mod, const llvm::DataLayout& DL, const std::function& shouldAnalyze, const std::string& bufferModelPath = ""); + + std::vector + analyzeMemIntrinsicOverflowsCached(const llvm::Function& function, const llvm::DataLayout& DL, + const std::vector& calls, + const std::vector& invokes, + const BufferWriteModel* externalModel, + BufferWriteRuleMatcher* ruleMatcher); } // namespace ctrace::stack::analysis diff --git a/include/analysis/ResourceLifetimeAnalysis.hpp b/include/analysis/ResourceLifetimeAnalysis.hpp index 08b9ee7..dc3c141 100644 --- a/include/analysis/ResourceLifetimeAnalysis.hpp +++ b/include/analysis/ResourceLifetimeAnalysis.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace llvm @@ -70,6 +71,10 @@ namespace ctrace::stack::analysis bool resourceSummaryIndexEquals(const ResourceSummaryIndex& lhs, const ResourceSummaryIndex& rhs); + std::unordered_set + computeChangedResourceFunctionNames(const ResourceSummaryIndex& prev, + const ResourceSummaryIndex& next); + std::vector analyzeResourceLifetime( llvm::Module& mod, const std::function& shouldAnalyze, const std::string& modelPath, const ResourceSummaryIndex* externalSummaries = nullptr); diff --git a/include/analysis/TOCTOUAnalysis.hpp b/include/analysis/TOCTOUAnalysis.hpp index 6f56f16..6195390 100644 --- a/include/analysis/TOCTOUAnalysis.hpp +++ b/include/analysis/TOCTOUAnalysis.hpp @@ -6,8 +6,10 @@ namespace llvm { + class CallInst; class Function; class Instruction; + class InvokeInst; class Module; } // namespace llvm @@ -25,4 +27,9 @@ namespace ctrace::stack::analysis std::vector analyzeTOCTOU(llvm::Module& mod, const std::function& shouldAnalyze); + + std::vector + analyzeTOCTOUCached(const llvm::Function& function, + const std::vector& calls, + const std::vector& invokes); } // namespace ctrace::stack::analysis diff --git a/include/analysis/UninitializedVarAnalysis.hpp b/include/analysis/UninitializedVarAnalysis.hpp index 4d2e5a4..3c382b4 100644 --- a/include/analysis/UninitializedVarAnalysis.hpp +++ b/include/analysis/UninitializedVarAnalysis.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace llvm @@ -107,6 +108,13 @@ namespace ctrace::stack::analysis bool uninitializedSummaryIndexEquals(const UninitializedSummaryIndex& lhs, const UninitializedSummaryIndex& rhs); + std::unordered_set + computeChangedUninitializedFunctionNames(const UninitializedSummaryIndex& prev, + const UninitializedSummaryIndex& next); + + std::unordered_set + getCanonicalCalleeNames(const PreparedUninitializedModuleContext& prepared); + std::vector analyzeUninitializedLocalReads(llvm::Module& mod, const std::function& shouldAnalyze, diff --git a/include/analyzer/AnalysisArtifactStore.hpp b/include/analyzer/AnalysisArtifactStore.hpp new file mode 100644 index 0000000..578df7d --- /dev/null +++ b/include/analyzer/AnalysisArtifactStore.hpp @@ -0,0 +1,60 @@ +#pragma once + +#include +#include +#include +#include + +namespace ctrace::stack::analyzer +{ + class AnalysisArtifactStore + { + public: + template T& emplace(Args&&... args) + { + auto artifact = std::make_shared(std::forward(args)...); + auto key = std::type_index(typeid(T)); + artifacts_[key] = artifact; + return *artifact; + } + + template void set(const T& value) + { + (void)emplace(value); + } + + template void set(T&& value) + { + (void)emplace(std::move(value)); + } + + template [[nodiscard]] bool has() const + { + return artifacts_.find(std::type_index(typeid(T))) != artifacts_.end(); + } + + template [[nodiscard]] T* get() + { + auto it = artifacts_.find(std::type_index(typeid(T))); + if (it == artifacts_.end()) + return nullptr; + return static_cast(it->second.get()); + } + + template [[nodiscard]] const T* get() const + { + auto it = artifacts_.find(std::type_index(typeid(T))); + if (it == artifacts_.end()) + return nullptr; + return static_cast(it->second.get()); + } + + void clear() + { + artifacts_.clear(); + } + + private: + std::unordered_map> artifacts_; + }; +} // namespace ctrace::stack::analyzer diff --git a/include/analyzer/DerivedModuleArtifacts.hpp b/include/analyzer/DerivedModuleArtifacts.hpp new file mode 100644 index 0000000..2606eb9 --- /dev/null +++ b/include/analyzer/DerivedModuleArtifacts.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +namespace ctrace::stack::analyzer +{ + inline constexpr std::uint32_t kDerivedModuleArtifactsSchemaVersion = 1; + inline constexpr std::string_view kDerivedModuleArtifactsSchemaKey = + "derived-module-artifacts-v1"; + + struct DebugMetadataIndex + { + std::uint64_t allDefinedFunctionsWithSubprogram = 0; + std::uint64_t selectedFunctionsWithSubprogram = 0; + std::uint64_t distinctSourceFiles = 0; + std::unordered_map functionsPerSourceFile; + }; + + struct FunctionSymbolIndex + { + std::uint64_t totalDefinedFunctions = 0; + std::uint64_t distinctMangledNames = 0; + std::unordered_map mangledNameFrequency; + }; + + struct TypeFactIndex + { + std::uint64_t pointerReturnFunctionCount = 0; + std::uint64_t aggregateReturnFunctionCount = 0; + std::uint64_t pointerParameterCount = 0; + std::uint64_t aggregateParameterCount = 0; + }; + + struct DerivedModuleArtifacts + { + std::uint32_t schemaVersion = kDerivedModuleArtifactsSchemaVersion; + std::uint32_t reservedPadding = 0; + DebugMetadataIndex debugIndex; + FunctionSymbolIndex symbolIndex; + TypeFactIndex typeFacts; + + [[nodiscard]] bool hasCompatibleSchema() const + { + return schemaVersion == kDerivedModuleArtifactsSchemaVersion; + } + + [[nodiscard]] static constexpr std::string_view schemaKey() + { + return kDerivedModuleArtifactsSchemaKey; + } + }; +} // namespace ctrace::stack::analyzer diff --git a/include/analyzer/HotspotProfiler.hpp b/include/analyzer/HotspotProfiler.hpp new file mode 100644 index 0000000..754b6ab --- /dev/null +++ b/include/analyzer/HotspotProfiler.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace ctrace::stack::analyzer +{ + + struct HotspotSample + { + std::uint64_t calls = 0; + std::uint64_t totalNs = 0; + std::uint64_t maxNs = 0; + }; + + class HotspotProfiler + { + public: + static void record(std::string_view name, std::chrono::nanoseconds elapsed); + static void dumpTop(std::ostream& os, std::size_t topN); + static void dumpTop(std::ostream& os); + }; + + class ScopedHotspot + { + public: + explicit ScopedHotspot(bool enabled, std::string_view name); + ~ScopedHotspot(); + + ScopedHotspot(const ScopedHotspot&) = delete; + ScopedHotspot& operator=(const ScopedHotspot&) = delete; + + private: + std::chrono::steady_clock::time_point start_{}; + std::string_view name_; + std::uint8_t enabled_ = 0; + std::uint8_t reservedPadding_[7] = {}; + }; + + void dumpHotspotSummary(std::ostream& os, bool enabled); + +} // namespace ctrace::stack::analyzer diff --git a/include/analyzer/IRFactCollector.hpp b/include/analyzer/IRFactCollector.hpp new file mode 100644 index 0000000..6ae1db9 --- /dev/null +++ b/include/analyzer/IRFactCollector.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +namespace ctrace::stack::analyzer +{ + struct ModuleAnalysisContext; + class InstructionSubscriberRegistry; + + struct IRFacts + { + std::uint64_t allDefinedFunctionCount = 0; + std::uint64_t selectedFunctionCount = 0; + std::uint64_t basicBlockCountAllDefined = 0; + std::uint64_t basicBlockCountSelected = 0; + std::uint64_t instructionCountAllDefined = 0; + std::uint64_t instructionCountSelected = 0; + + std::uint64_t callInstCount = 0; + std::uint64_t invokeInstCount = 0; + std::uint64_t allocaInstCount = 0; + std::uint64_t loadInstCount = 0; + std::uint64_t storeInstCount = 0; + std::uint64_t memIntrinsicCount = 0; + std::uint64_t debugLocCount = 0; + }; + + IRFacts collectIRFacts(const ModuleAnalysisContext& ctx, + const InstructionSubscriberRegistry* subscribers = nullptr); +} // namespace ctrace::stack::analyzer diff --git a/include/analyzer/InstructionSubscriber.hpp b/include/analyzer/InstructionSubscriber.hpp new file mode 100644 index 0000000..41e7983 --- /dev/null +++ b/include/analyzer/InstructionSubscriber.hpp @@ -0,0 +1,97 @@ +#pragma once + +#include + +namespace llvm +{ + class AllocaInst; + class CallInst; + class Function; + class InvokeInst; + class LoadInst; + class MemIntrinsic; + class StoreInst; +} // namespace llvm + +namespace ctrace::stack::analyzer +{ + class InstructionSubscriber + { + public: + virtual ~InstructionSubscriber() = default; + + virtual void onFunctionBegin(const llvm::Function&) {} + virtual void onFunctionEnd(const llvm::Function&) {} + virtual void onAlloca(const llvm::AllocaInst&) {} + virtual void onLoad(const llvm::LoadInst&) {} + virtual void onStore(const llvm::StoreInst&) {} + virtual void onCall(const llvm::CallInst&) {} + virtual void onInvoke(const llvm::InvokeInst&) {} + virtual void onMemIntrinsic(const llvm::MemIntrinsic&) {} + }; + + class InstructionSubscriberRegistry + { + public: + void add(InstructionSubscriber& subscriber) + { + subscribers_.push_back(&subscriber); + } + + [[nodiscard]] bool empty() const + { + return subscribers_.empty(); + } + + void notifyFunctionBegin(const llvm::Function& F) const + { + for (InstructionSubscriber* subscriber : subscribers_) + subscriber->onFunctionBegin(F); + } + + void notifyFunctionEnd(const llvm::Function& F) const + { + for (InstructionSubscriber* subscriber : subscribers_) + subscriber->onFunctionEnd(F); + } + + void notifyAlloca(const llvm::AllocaInst& inst) const + { + for (InstructionSubscriber* subscriber : subscribers_) + subscriber->onAlloca(inst); + } + + void notifyLoad(const llvm::LoadInst& inst) const + { + for (InstructionSubscriber* subscriber : subscribers_) + subscriber->onLoad(inst); + } + + void notifyStore(const llvm::StoreInst& inst) const + { + for (InstructionSubscriber* subscriber : subscribers_) + subscriber->onStore(inst); + } + + void notifyCall(const llvm::CallInst& inst) const + { + for (InstructionSubscriber* subscriber : subscribers_) + subscriber->onCall(inst); + } + + void notifyInvoke(const llvm::InvokeInst& inst) const + { + for (InstructionSubscriber* subscriber : subscribers_) + subscriber->onInvoke(inst); + } + + void notifyMemIntrinsic(const llvm::MemIntrinsic& inst) const + { + for (InstructionSubscriber* subscriber : subscribers_) + subscriber->onMemIntrinsic(inst); + } + + private: + std::vector subscribers_; + }; +} // namespace ctrace::stack::analyzer diff --git a/include/analyzer/ModulePreparationService.hpp b/include/analyzer/ModulePreparationService.hpp index 2f70756..22b60ed 100644 --- a/include/analyzer/ModulePreparationService.hpp +++ b/include/analyzer/ModulePreparationService.hpp @@ -1,6 +1,7 @@ #pragma once #include "StackUsageAnalyzer.hpp" +#include "analyzer/DerivedModuleArtifacts.hpp" #include "analysis/FunctionFilter.hpp" #include "analysis/StackComputation.hpp" @@ -38,6 +39,7 @@ namespace ctrace::stack::analyzer struct PreparedModule { ModuleAnalysisContext ctx; + DerivedModuleArtifacts derivedArtifacts; LocalStackMap localStack; analysis::CallGraph callGraph; analysis::InternalAnalysisState recursionState; diff --git a/include/analyzer/PerFunctionInstructionCache.hpp b/include/analyzer/PerFunctionInstructionCache.hpp new file mode 100644 index 0000000..39a0037 --- /dev/null +++ b/include/analyzer/PerFunctionInstructionCache.hpp @@ -0,0 +1,48 @@ +#pragma once + +#include "analyzer/InstructionSubscriber.hpp" + +#include + +#include + +namespace llvm +{ + class Function; +} // namespace llvm + +namespace ctrace::stack::analyzer +{ + struct PerFunctionData + { + std::vector allocas; + std::vector stores; + std::vector calls; + std::vector invokes; + std::vector loads; + std::vector memIntrinsics; + }; + + class PerFunctionInstructionCache : public InstructionSubscriber + { + public: + void onFunctionBegin(const llvm::Function& F) override; + void onFunctionEnd(const llvm::Function& F) override; + void onAlloca(const llvm::AllocaInst& inst) override; + void onLoad(const llvm::LoadInst& inst) override; + void onStore(const llvm::StoreInst& inst) override; + void onCall(const llvm::CallInst& inst) override; + void onInvoke(const llvm::InvokeInst& inst) override; + void onMemIntrinsic(const llvm::MemIntrinsic& inst) override; + + [[nodiscard]] const llvm::DenseMap& data() const + { + return data_; + } + + private: + llvm::DenseMap data_; + const llvm::Function* currentFunction_ = nullptr; + PerFunctionData currentData_; + }; +} // namespace ctrace::stack::analyzer diff --git a/main.cpp b/main.cpp index da4faab..94635bd 100644 --- a/main.cpp +++ b/main.cpp @@ -72,6 +72,7 @@ static void printHelp() << " --resource-summary-cache-dir= Cache directory for cross-TU summaries\n" << " --compile-ir-cache-dir= Cache directory for compiled LLVM IR per source " "file\n" + << " --compile-ir-format=bc|ll Compilation IR format for source inputs (default: bc)\n" << " --resource-summary-cache-memory-only Use in-memory cache only for cross-TU " "summaries\n" << " --uninitialized-cross-tu Enable cross-TU uninitialized summaries (default: on)\n" @@ -117,6 +118,18 @@ static const char* solverModeName(ctrace::stack::analysis::smt::SolverMode mode) return "single"; } +static const char* compileIRFormatName(CompileIRFormat format) +{ + switch (format) + { + case CompileIRFormat::BC: + return "bc"; + case CompileIRFormat::LL: + return "ll"; + } + return "bc"; +} + static std::string joinCsv(const std::vector& values) { if (values.empty()) @@ -157,6 +170,7 @@ static void printEffectiveConfig(const ctrace::stack::cli::ParsedArguments& pars << (cfg.bufferModelPath.empty() ? "" : cfg.bufferModelPath) << "\n"; llvm::errs() << "compile-ir-cache-dir: " << (cfg.compileIRCacheDir.empty() ? "" : cfg.compileIRCacheDir) << "\n"; + llvm::errs() << "compile-ir-format: " << compileIRFormatName(cfg.compileIRFormat) << "\n"; llvm::errs() << "smt-enabled: " << (cfg.smtEnabled ? "true" : "false") << "\n"; llvm::errs() << "smt-backend: " << cfg.smtBackend << "\n"; llvm::errs() << "smt-secondary-backend: " diff --git a/run_test.py b/run_test.py index 38b38fb..5349d17 100755 --- a/run_test.py +++ b/run_test.py @@ -177,12 +177,13 @@ def _collect_cache_dependencies(args): return deps -def _cache_key_for_args(args): +def _cache_key_for_args(args, env_overrides: Optional[dict[str, str]] = None): payload = { "analyzer": str(RUN_CONFIG.analyzer.resolve()), "args": list(args), "cwd": str(Path.cwd()), "deps": _collect_cache_dependencies(args), + "env": dict(sorted((env_overrides or {}).items())), } encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") return hashlib.sha256(encoded).hexdigest() @@ -588,13 +589,13 @@ def _effective_analyzer_args(args): return base -def run_analyzer(args) -> subprocess.CompletedProcess: +def run_analyzer(args, env_overrides: Optional[dict[str, str]] = None) -> subprocess.CompletedProcess: """ Run analyzer with custom args and return the CompletedProcess. """ effective_args = _effective_analyzer_args(args) cmd = [str(RUN_CONFIG.analyzer)] + effective_args - key = _cache_key_for_args(effective_args) + key = _cache_key_for_args(effective_args, env_overrides) with _CACHE_LOCK: in_memory = _MEM_CACHE.get(key) @@ -617,7 +618,10 @@ def run_analyzer(args) -> subprocess.CompletedProcess: } return cached - result = subprocess.run(cmd, capture_output=True, text=True) + env = os.environ.copy() + if env_overrides: + env.update(env_overrides) + result = subprocess.run(cmd, capture_output=True, text=True, env=env) with _CACHE_LOCK: _MEM_CACHE[key] = { "returncode": result.returncode, @@ -628,13 +632,16 @@ def run_analyzer(args) -> subprocess.CompletedProcess: return result -def run_analyzer_uncached(args) -> subprocess.CompletedProcess: +def run_analyzer_uncached(args, env_overrides: Optional[dict[str, str]] = None) -> subprocess.CompletedProcess: """ Run analyzer with custom args and bypass run_test.py cache layer. Useful for checks that assert filesystem side effects. """ cmd = [str(RUN_CONFIG.analyzer)] + _effective_analyzer_args(args) - return subprocess.run(cmd, capture_output=True, text=True) + env = os.environ.copy() + if env_overrides: + env.update(env_overrides) + return subprocess.run(cmd, capture_output=True, text=True, env=env) def fail_check(message: str, output: str = "") -> bool: @@ -1329,6 +1336,7 @@ def run_success_case(label: str, args: list[str], required: Optional[list[str]] ("--escape-model", "Missing argument for --escape-model"), ("--buffer-model", "Missing argument for --buffer-model"), ("--resource-summary-cache-dir", "Missing argument for --resource-summary-cache-dir"), + ("--compile-ir-format", "Missing argument for --compile-ir-format"), ("--compile-commands", "Missing argument for --compile-commands"), ("--compdb", "Missing argument for --compdb"), ("--base-dir", "Missing argument for --base-dir"), @@ -1379,6 +1387,7 @@ def run_success_case(label: str, args: list[str], required: Optional[list[str]] (["--jobs=x", str(sample)], "Invalid --jobs value:"), (["--jobs=-1", str(sample)], "Invalid --jobs value:"), (["--analysis-profile=unknown", str(sample)], "Invalid --analysis-profile value:"), + (["--compile-ir-format=foo", str(sample)], "Invalid --compile-ir-format value:"), (["--stack-limit=oops", str(sample)], "Invalid --stack-limit value:"), (["--mode=unknown", str(sample)], "Unknown mode: unknown (expected 'ir' or 'abi')"), ] @@ -1439,6 +1448,8 @@ def run_success_case(label: str, args: list[str], required: Optional[list[str]] ("--analysis-profile equals", [str(sample), "--analysis-profile=full", "--only-function=transition"], ["Function:"], "text"), ("--jobs space", [str(sample), "--jobs", "2", "--only-function=transition"], ["Function:"], "text"), ("--jobs equals", [str(sample), "--jobs=2", "--only-function=transition"], ["Function:"], "text"), + ("--compile-ir-format=bc", [str(sample_c), "--compile-ir-format=bc"], ["Function:"], "text"), + ("--compile-ir-format=ll", [str(sample_c), "--compile-ir-format=ll"], ["Function:"], "text"), ("--timing", [str(sample), "--timing", "--only-function=transition"], ["Function:"], "text"), ("--resource-model space", [str(sample), "--resource-model", str(resource_model), "--only-function=transition"], ["Function:"], "text"), ("--resource-model equals", [str(sample), f"--resource-model={resource_model}", "--only-function=transition"], ["Function:"], "text"), @@ -1492,6 +1503,174 @@ def run_success_case(label: str, args: list[str], required: Optional[list[str]] return ok +def check_compile_ir_format_switch() -> bool: + """ + Validate that --compile-ir-format selects the expected source compile IR path + and that cache keys invalidate correctly when switching format. + """ + print("=== Testing compile IR format switch ===") + sample_c = RUN_CONFIG.test_dir / "alloca/oversized-constant.c" + ok = True + with tempfile.TemporaryDirectory(prefix="ct_compile_ir_format_") as tmp: + cache_dir = Path(tmp) / "compile-ir-cache" + cache_arg = f"--compile-ir-cache-dir={cache_dir}" + + def run_and_capture(fmt: str) -> tuple[bool, str]: + result = run_analyzer_uncached( + [str(sample_c), "--timing", cache_arg, f"--compile-ir-format={fmt}"] + ) + output = (result.stdout or "") + (result.stderr or "") + if result.returncode != 0: + print(f" ❌ --compile-ir-format={fmt} failed (code {result.returncode})") + print(output) + return False, output + return True, output + + bc_ok, bc_output = run_and_capture("bc") + ok = ok and bc_ok + if bc_ok: + if "Bitcode parse done in" not in bc_output: + print(" ❌ --compile-ir-format=bc did not use bitcode parse path") + print(bc_output) + ok = False + else: + print(" ✅ --compile-ir-format=bc uses bitcode parse path") + + ll_ok, ll_output = run_and_capture("ll") + ok = ok and ll_ok + if ll_ok: + if "IR parse done in" not in ll_output: + print(" ❌ --compile-ir-format=ll did not use textual IR parse path") + print(ll_output) + ok = False + elif "Bitcode parse done in" in ll_output: + print(" ❌ --compile-ir-format=ll unexpectedly used bitcode parse path") + print(ll_output) + ok = False + else: + print(" ✅ --compile-ir-format=ll uses textual IR parse path") + + # Re-run BC with the same cache directory to ensure format switch is + # versioned in cache identity and does not pin the LL parse path. + bc2_ok, bc2_output = run_and_capture("bc") + ok = ok and bc2_ok + if bc2_ok: + if "Bitcode parse done in" not in bc2_output: + print(" ❌ cache invalidation failed when switching back to bc") + print(bc2_output) + ok = False + elif "IR parse done in" in bc2_output: + print(" ❌ bc run unexpectedly reused textual IR parse path") + print(bc2_output) + ok = False + else: + print(" ✅ cache invalidation across bc/ll switch OK") + + print() + return ok + + +def check_pipeline_subscriber_rollout_parity() -> bool: + """ + Integration check: diagnostics must remain stable when toggling the + subscriber rollout flag. + """ + print("=== Testing pipeline subscriber rollout parity ===") + fixtures = [ + RUN_CONFIG.test_dir / "alloca/oversized-constant.c", + RUN_CONFIG.test_dir / "resource-lifetime/local-double-release.c", + RUN_CONFIG.test_dir / "integer-overflow/cross-tu-tricky-use.c", + RUN_CONFIG.test_dir / "uninitialized-variable/uninitialized-local-unused.c", + RUN_CONFIG.test_dir / "diagnostics/duplicate-else-if-basic.c", + ] + + ok = True + for fixture in fixtures: + args = [str(fixture), "--warnings-only", "--format=json"] + baseline = run_analyzer(args) + baseline_output = (baseline.stdout or "") + (baseline.stderr or "") + if baseline.returncode != 0: + print(f" ❌ baseline run failed for {fixture} (code {baseline.returncode})") + print(baseline_output) + ok = False + continue + try: + baseline_payload = json.loads(baseline.stdout or "") + except json.JSONDecodeError as exc: + print(f" ❌ baseline JSON parse failed for {fixture}: {exc}") + print(baseline.stdout or "") + ok = False + continue + + subscriber = run_analyzer(args, env_overrides={"CTRACE_PIPELINE_SUBSCRIBERS": "1"}) + subscriber_output = (subscriber.stdout or "") + (subscriber.stderr or "") + if subscriber.returncode != 0: + print(f" ❌ subscriber run failed for {fixture} (code {subscriber.returncode})") + print(subscriber_output) + ok = False + continue + try: + subscriber_payload = json.loads(subscriber.stdout or "") + except json.JSONDecodeError as exc: + print(f" ❌ subscriber JSON parse failed for {fixture}: {exc}") + print(subscriber.stdout or "") + ok = False + continue + + baseline_norm = json.dumps(baseline_payload, sort_keys=True, separators=(",", ":")) + subscriber_norm = json.dumps(subscriber_payload, sort_keys=True, separators=(",", ":")) + if baseline_norm != subscriber_norm: + print(f" ❌ parity mismatch with subscriber rollout for {fixture}") + print(" --- baseline ---") + print(baseline.stdout or "") + print(" --- subscriber ---") + print(subscriber.stdout or "") + ok = False + continue + + print(f" ✅ parity OK with subscriber rollout for {fixture}") + + print() + return ok + + +def check_pipeline_timing_traversal_instrumentation() -> bool: + """ + Integration check: timing output must include traversal instrumentation + and derived-artifact metadata. + """ + print("=== Testing pipeline traversal instrumentation ===") + sample = RUN_CONFIG.test_dir / "alloca/oversized-constant.c" + result = run_analyzer_uncached( + [str(sample), "--timing", "--quiet"], + env_overrides={"CTRACE_PIPELINE_SUBSCRIBERS": "1"}, + ) + output = (result.stdout or "") + (result.stderr or "") + + if result.returncode != 0: + print(f" ❌ analyzer failed (code {result.returncode})") + print(output) + print() + return False + + required = [ + "IR facts mode: subscriber", + "Derived artifacts schema: derived-module-artifacts-v1", + "Traversal estimate detail: step='Stack buffer overflows'", + "Traversal estimate by model:", + ] + + for needle in required: + if needle not in output: + print(f" ❌ missing traversal instrumentation token: {needle}") + print(output) + print() + return False + + print(" ✅ traversal instrumentation output OK\n") + return True + + def check_only_func_uninitialized() -> bool: """ Regression: --only-func must keep interprocedural uninitialized warnings. @@ -1727,145 +1906,169 @@ def check_resource_lifetime_cross_tu() -> bool: """ print("=== Testing resource lifetime cross-TU summaries ===") model = "models/resource-lifetime/generic.txt" + with tempfile.TemporaryDirectory(prefix="ct_resource_cross_tu_") as tmp: + tmpdir = Path(tmp) + compile_cache_dir = tmpdir / "compile-ir-cache" + resource_cache_dir = tmpdir / "resource-cache" + compile_cache_arg = f"--compile-ir-cache-dir={compile_cache_dir}" + default_resource_cache_arg = f"--resource-summary-cache-dir={resource_cache_dir}" - wrapper_use = RUN_CONFIG.test_dir / "resource-lifetime/cross-tu-wrapper-use.c" - result = run_analyzer([str(wrapper_use), f"--resource-model={model}", "--warnings-only"]) - output = (result.stdout or "") + (result.stderr or "") - if not expect_returncode_zero(result, output, "single-file wrapper run failed"): - return False - if not expect_contains( - output, - "Resource inter-procedural analysis: unavailable", - "missing inter-proc unavailable status message in single-file mode", - ): - return False - if not expect_contains( - output, - "inter-procedural resource analysis incomplete: handle 'h'", - "missing IncompleteInterproc warning in single-file wrapper case", - ): - return False - if not expect_not_contains( - output, - "potential double release: 'GenericHandle' handle 'h'", - "unexpected double release in single-file wrapper case", - ): - return False + wrapper_use = RUN_CONFIG.test_dir / "resource-lifetime/cross-tu-wrapper-use.c" + result = run_analyzer( + [str(wrapper_use), f"--resource-model={model}", "--warnings-only", compile_cache_arg, default_resource_cache_arg] + ) + output = (result.stdout or "") + (result.stderr or "") + if not expect_returncode_zero(result, output, "single-file wrapper run failed"): + return False + if not expect_contains( + output, + "Resource inter-procedural analysis: unavailable", + "missing inter-proc unavailable status message in single-file mode", + ): + return False + if not expect_contains( + output, + "inter-procedural resource analysis incomplete: handle 'h'", + "missing IncompleteInterproc warning in single-file wrapper case", + ): + return False + if not expect_not_contains( + output, + "potential double release: 'GenericHandle' handle 'h'", + "unexpected double release in single-file wrapper case", + ): + return False - wrapper_def = RUN_CONFIG.test_dir / "resource-lifetime/cross-tu-wrapper-def.c" - result = run_analyzer( - [ - str(wrapper_def), - str(wrapper_use), - f"--resource-model={model}", - "--jobs=2", - "--warnings-only", - ] - ) - output = (result.stdout or "") + (result.stderr or "") - if not expect_returncode_zero(result, output, "wrapper cross-TU run failed"): - return False - if not expect_contains( - output, - "Resource inter-procedural analysis: enabled (cross-TU summaries across 2 files", - "missing inter-proc enabled status message in cross-TU mode", - ): - return False - if not expect_contains(output, "jobs: 2", "missing jobs count in inter-proc enabled status message"): - return False - if not expect_not_contains( - output, - "potential double release: 'GenericHandle' handle 'h'", - "unexpected double release in cross-TU wrapper release case", - ): - return False + wrapper_def = RUN_CONFIG.test_dir / "resource-lifetime/cross-tu-wrapper-def.c" + result = run_analyzer( + [ + str(wrapper_def), + str(wrapper_use), + f"--resource-model={model}", + "--jobs=2", + "--warnings-only", + compile_cache_arg, + default_resource_cache_arg, + ] + ) + output = (result.stdout or "") + (result.stderr or "") + if not expect_returncode_zero(result, output, "wrapper cross-TU run failed"): + return False + if not expect_contains( + output, + "Resource inter-procedural analysis: enabled (cross-TU summaries across 2 files", + "missing inter-proc enabled status message in cross-TU mode", + ): + return False + if not expect_contains(output, "jobs: 2", "missing jobs count in inter-proc enabled status message"): + return False + if not expect_not_contains( + output, + "potential double release: 'GenericHandle' handle 'h'", + "unexpected double release in cross-TU wrapper release case", + ): + return False - wrapper_leak = RUN_CONFIG.test_dir / "resource-lifetime/cross-tu-wrapper-leak-use.c" - result = run_analyzer( - [str(wrapper_def), str(wrapper_leak), f"--resource-model={model}", "--warnings-only"] - ) - output = (result.stdout or "") + (result.stderr or "") - if not expect_returncode_zero(result, output, "wrapper leak cross-TU run failed"): - return False - if not expect_contains( - output, - "potential resource leak: 'GenericHandle' acquired in handle 'h'", - "missing leak warning in cross-TU wrapper leak case", - ): - return False + wrapper_leak = RUN_CONFIG.test_dir / "resource-lifetime/cross-tu-wrapper-leak-use.c" + result = run_analyzer( + [ + str(wrapper_def), + str(wrapper_leak), + f"--resource-model={model}", + "--warnings-only", + compile_cache_arg, + default_resource_cache_arg, + ] + ) + output = (result.stdout or "") + (result.stderr or "") + if not expect_returncode_zero(result, output, "wrapper leak cross-TU run failed"): + return False + if not expect_contains( + output, + "potential resource leak: 'GenericHandle' acquired in handle 'h'", + "missing leak warning in cross-TU wrapper leak case", + ): + return False - ret_def = RUN_CONFIG.test_dir / "resource-lifetime/cross-tu-return-def.c" - ret_use = RUN_CONFIG.test_dir / "resource-lifetime/cross-tu-return-use.c" - result = run_analyzer( - [str(ret_def), str(ret_use), f"--resource-model={model}", "--warnings-only"] - ) - output = (result.stdout or "") + (result.stderr or "") - if not expect_returncode_zero(result, output, "return cross-TU run failed"): - return False - if not expect_not_contains( - output, - "potential double release: 'HeapAlloc' handle 'p'", - "unexpected double release in cross-TU acquire_ret case", - ): - return False + ret_def = RUN_CONFIG.test_dir / "resource-lifetime/cross-tu-return-def.c" + ret_use = RUN_CONFIG.test_dir / "resource-lifetime/cross-tu-return-use.c" + result = run_analyzer( + [ + str(ret_def), + str(ret_use), + f"--resource-model={model}", + "--warnings-only", + compile_cache_arg, + default_resource_cache_arg, + ] + ) + output = (result.stdout or "") + (result.stderr or "") + if not expect_returncode_zero(result, output, "return cross-TU run failed"): + return False + if not expect_not_contains( + output, + "potential double release: 'HeapAlloc' handle 'p'", + "unexpected double release in cross-TU acquire_ret case", + ): + return False - result = run_analyzer( - [ - str(ret_def), - str(ret_use), - f"--resource-model={model}", - "--no-resource-cross-tu", - "--warnings-only", - ] - ) - output = (result.stdout or "") + (result.stderr or "") - if not expect_returncode_zero(result, output, "return cross-TU disabled run failed"): - return False - if not expect_contains( - output, - "inter-procedural resource analysis incomplete: handle 'p'", - "expected local-only incomplete inter-proc warning is missing with --no-resource-cross-tu", - ): - return False + result = run_analyzer( + [ + str(ret_def), + str(ret_use), + f"--resource-model={model}", + "--no-resource-cross-tu", + "--warnings-only", + compile_cache_arg, + default_resource_cache_arg, + ] + ) + output = (result.stdout or "") + (result.stderr or "") + if not expect_returncode_zero(result, output, "return cross-TU disabled run failed"): + return False + if not expect_contains( + output, + "inter-procedural resource analysis incomplete: handle 'p'", + "expected local-only incomplete inter-proc warning is missing with --no-resource-cross-tu", + ): + return False - cache_dir = Path(".cache/run_test_resource_summary") - if cache_dir.exists(): - shutil.rmtree(cache_dir, ignore_errors=True) - result = run_analyzer_uncached( - [ - str(ret_def), - str(ret_use), - f"--resource-model={model}", - f"--resource-summary-cache-dir={cache_dir}", - "--warnings-only", - ] - ) - output = (result.stdout or "") + (result.stderr or "") - if not expect_returncode_zero(result, output, "return cross-TU cache run failed"): - return False - if not list(cache_dir.glob("*.json")): - return fail_check("cross-TU cache directory was not populated", output) + cache_dir = tmpdir / "resource-summary-disk-cache" + result = run_analyzer_uncached( + [ + str(ret_def), + str(ret_use), + f"--resource-model={model}", + f"--resource-summary-cache-dir={cache_dir}", + "--warnings-only", + compile_cache_arg, + ] + ) + output = (result.stdout or "") + (result.stderr or "") + if not expect_returncode_zero(result, output, "return cross-TU cache run failed"): + return False + if not list(cache_dir.glob("*.json")): + return fail_check("cross-TU cache directory was not populated", output) - memory_only_cache_dir = Path(".cache/run_test_resource_summary_memory_only") - if memory_only_cache_dir.exists(): - shutil.rmtree(memory_only_cache_dir, ignore_errors=True) - result = run_analyzer_uncached( - [ - str(ret_def), - str(ret_use), - f"--resource-model={model}", - "--resource-summary-cache-memory-only", - f"--resource-summary-cache-dir={memory_only_cache_dir}", - "--warnings-only", - ] - ) - output = (result.stdout or "") + (result.stderr or "") - if not expect_returncode_zero(result, output, "return cross-TU memory-only cache run failed"): - return False - if not expect_contains(output, "cache: memory-only", "missing memory-only cache status message"): - return False - if list(memory_only_cache_dir.glob("*.json")): - return fail_check("memory-only cache mode unexpectedly wrote summary files", output) + memory_only_cache_dir = tmpdir / "resource-summary-memory-only-cache" + result = run_analyzer_uncached( + [ + str(ret_def), + str(ret_use), + f"--resource-model={model}", + "--resource-summary-cache-memory-only", + f"--resource-summary-cache-dir={memory_only_cache_dir}", + "--warnings-only", + compile_cache_arg, + ] + ) + output = (result.stdout or "") + (result.stderr or "") + if not expect_returncode_zero(result, output, "return cross-TU memory-only cache run failed"): + return False + if not expect_contains(output, "cache: memory-only", "missing memory-only cache status message"): + return False + if list(memory_only_cache_dir.glob("*.json")): + return fail_check("memory-only cache mode unexpectedly wrote summary files", output) print(" ✅ cross-TU resource summaries OK\n") return True @@ -2117,84 +2320,93 @@ def check_use_after_free_advanced_inter_tu() -> bool: return False model = "models/resource-lifetime/generic.txt" - result = run_analyzer( - [ - str(def_file), - str(use_file), - "--jobs=2", - "--analysis-profile=full", - "--resource-cross-tu", - f"--resource-model={model}", - "--warnings-only", - ] - ) - output = (result.stdout or "") + (result.stderr or "") + with tempfile.TemporaryDirectory(prefix="ct_uaf_cross_tu_") as tmp: + tmpdir = Path(tmp) + resource_cache_arg = f"--resource-summary-cache-dir={tmpdir / 'resource-cache'}" + compile_cache_arg = f"--compile-ir-cache-dir={tmpdir / 'compile-ir-cache'}" - if not expect_returncode_zero(result, output, "use-after-free inter-TU run failed"): - return False - if not expect_contains( - output, - "Resource inter-procedural analysis: enabled (cross-TU summaries across 2 files", - "missing resource cross-TU enabled status in use-after-free inter-TU run", - ): - return False - if not expect_contains( - output, - "Function: io_cross_uaf_nested_if", - "missing nested-if cross-TU UAF function in output", - ): - return False - if not expect_contains( - output, - "potential use-after-release: 'GenericHandle' handle 'h'", - "missing cross-TU use-after-release diagnostic", - ): - return False - if not expect_contains( - output, - "Function: io_cross_double_release_nested_loop", - "missing nested-loop cross-TU double-release function in output", - ): - return False - if not expect_contains( - output, - "potential double release: 'GenericHandle' handle 'h'", - "missing cross-TU double-release diagnostic", - ): - return False - if not expect_not_contains( - output, - "inter-procedural resource analysis incomplete: handle 'h'", - "unexpected IncompleteInterproc warning in cross-TU enabled run", - ): - return False + result = run_analyzer( + [ + str(def_file), + str(use_file), + "--jobs=2", + "--analysis-profile=full", + "--resource-cross-tu", + f"--resource-model={model}", + resource_cache_arg, + compile_cache_arg, + "--warnings-only", + ] + ) + output = (result.stdout or "") + (result.stderr or "") - result = run_analyzer( - [ - str(def_file), - str(use_file), - "--jobs=2", - "--analysis-profile=full", - "--no-resource-cross-tu", - f"--resource-model={model}", - "--warnings-only", - ] - ) - output = (result.stdout or "") + (result.stderr or "") - if not expect_returncode_zero(result, output, "use-after-free cross-TU disabled run failed"): - return False - if not expect_not_contains( - output, - "potential use-after-release: 'GenericHandle' handle 'h'", - "unexpected cross-TU use-after-release diagnostic with --no-resource-cross-tu", - ): - return False - if not expect_not_contains( - output, - "potential double release: 'GenericHandle' handle 'h'", - "unexpected cross-TU double-release diagnostic with --no-resource-cross-tu", - ): - return False + if not expect_returncode_zero(result, output, "use-after-free inter-TU run failed"): + return False + if not expect_contains( + output, + "Resource inter-procedural analysis: enabled (cross-TU summaries across 2 files", + "missing resource cross-TU enabled status in use-after-free inter-TU run", + ): + return False + if not expect_contains( + output, + "Function: io_cross_uaf_nested_if", + "missing nested-if cross-TU UAF function in output", + ): + return False + if not expect_contains( + output, + "potential use-after-release: 'GenericHandle' handle 'h'", + "missing cross-TU use-after-release diagnostic", + ): + return False + if not expect_contains( + output, + "Function: io_cross_double_release_nested_loop", + "missing nested-loop cross-TU double-release function in output", + ): + return False + if not expect_contains( + output, + "potential double release: 'GenericHandle' handle 'h'", + "missing cross-TU double-release diagnostic", + ): + return False + if not expect_not_contains( + output, + "inter-procedural resource analysis incomplete: handle 'h'", + "unexpected IncompleteInterproc warning in cross-TU enabled run", + ): + return False + + result = run_analyzer( + [ + str(def_file), + str(use_file), + "--jobs=2", + "--analysis-profile=full", + "--no-resource-cross-tu", + f"--resource-model={model}", + resource_cache_arg, + compile_cache_arg, + "--warnings-only", + ] + ) + output = (result.stdout or "") + (result.stderr or "") + if not expect_returncode_zero(result, output, "use-after-free cross-TU disabled run failed"): + return False + if not expect_not_contains( + output, + "potential use-after-release: 'GenericHandle' handle 'h'", + "unexpected cross-TU use-after-release diagnostic with --no-resource-cross-tu", + ): + return False + if not expect_not_contains( + output, + "potential double release: 'GenericHandle' handle 'h'", + "unexpected cross-TU double-release diagnostic with --no-resource-cross-tu", + ): + return False print(" ✅ nested use-after-free diagnostics OK in inter-TU mode\n") return True @@ -2890,6 +3102,9 @@ def record_ok(ok: bool): check_multi_file_total_summary, check_multi_file_failure, check_cli_parsing_and_filters, + check_compile_ir_format_switch, + check_pipeline_subscriber_rollout_parity, + check_pipeline_timing_traversal_instrumentation, check_only_func_uninitialized, check_warnings_only_filters_function_listing, check_uninitialized_verbose_ctor_trace, diff --git a/scripts/bench/pipeline_subscriber_ab.sh b/scripts/bench/pipeline_subscriber_ab.sh new file mode 100755 index 0000000..3cc7e15 --- /dev/null +++ b/scripts/bench/pipeline_subscriber_ab.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +ANALYZER="${1:-./build/stack_usage_analyzer}" +if [[ $# -gt 0 ]]; then + shift +fi + +if [[ $# -gt 0 ]]; then + FILES=("$@") +else + FILES=( + "test/alloca/oversized-constant.c" + "test/resource-lifetime/local-double-release.c" + "test/uninitialized-variable/uninitialized-local-unused.c" + "test/diagnostics/duplicate-else-if-basic.c" + "test/integer-overflow/cross-tu-tricky-use.c" + ) +fi + +for file in "${FILES[@]}"; do + if [[ ! -f "$file" ]]; then + echo "missing input file: $file" >&2 + exit 1 + fi +done + +run_case() { + local label="$1" + local subscriber_flag="$2" + + local tmp + tmp="$(mktemp "/tmp/ct_pipeline_ab_${label}.XXXXXX")" + ( + for file in "${FILES[@]}"; do + if [[ "$subscriber_flag" == "1" ]]; then + CTRACE_PIPELINE_SUBSCRIBERS=1 "$ANALYZER" "$file" --warnings-only >/dev/null 2>&1 + else + "$ANALYZER" "$file" --warnings-only >/dev/null 2>&1 + fi + done + ) > /dev/null 2>&1 + + /usr/bin/time -p bash -c ' + set -euo pipefail + for file in "$@"; do + if [[ "'"$subscriber_flag"'" == "1" ]]; then + CTRACE_PIPELINE_SUBSCRIBERS=1 "'"$ANALYZER"'" "$file" --warnings-only >/dev/null 2>&1 + else + "'"$ANALYZER"'" "$file" --warnings-only >/dev/null 2>&1 + fi + done + ' _ "${FILES[@]}" 2>"$tmp" + + local real user sys + real="$(awk "/^real /{print \$2}" "$tmp")" + user="$(awk "/^user /{print \$2}" "$tmp")" + sys="$(awk "/^sys /{print \$2}" "$tmp")" + rm -f "$tmp" + + echo "$label: real=${real}s user=${user}s sys=${sys}s" +} + +echo "Analyzer: $ANALYZER" +echo "Inputs (${#FILES[@]}): ${FILES[*]}" +run_case "baseline" "0" +run_case "subscriber" "1" diff --git a/src/StackUsageAnalyzer.cpp b/src/StackUsageAnalyzer.cpp index 5e2a2b1..8f100ff 100644 --- a/src/StackUsageAnalyzer.cpp +++ b/src/StackUsageAnalyzer.cpp @@ -1,6 +1,7 @@ #include "StackUsageAnalyzer.hpp" #include "analyzer/AnalysisPipeline.hpp" +#include "analyzer/HotspotProfiler.hpp" #include "analysis/InputPipeline.hpp" #include @@ -13,6 +14,7 @@ namespace ctrace::stack AnalysisResult analyzeModule(llvm::Module& mod, const AnalysisConfig& config) { + const analyzer::ScopedHotspot hotspot(config.timing, "analyze.module_total"); analyzer::AnalysisPipeline pipeline(config); return pipeline.run(mod); } @@ -20,8 +22,12 @@ namespace ctrace::stack AnalysisResult analyzeFile(const std::string& filename, const AnalysisConfig& config, llvm::LLVMContext& ctx, llvm::SMDiagnostic& err) { - analysis::ModuleLoadResult load = - analysis::loadModuleForAnalysis(filename, config, ctx, err); + const analyzer::ScopedHotspot fileHotspot(config.timing, "analyze.file_total"); + analysis::ModuleLoadResult load; + { + const analyzer::ScopedHotspot hotspot(config.timing, "analyze.load_module"); + load = analysis::loadModuleForAnalysis(filename, config, ctx, err); + } if (!load.module) { if (!load.error.empty()) @@ -34,7 +40,11 @@ namespace ctrace::stack std::cerr << "Analyzing " << filename << "...\n"; const auto analyzeStart = Clock::now(); - AnalysisResult result = analyzeModule(*load.module, config); + AnalysisResult result; + { + const analyzer::ScopedHotspot hotspot(config.timing, "analyze.pipeline"); + result = analyzeModule(*load.module, config); + } if (!load.frontendDiagnostics.empty()) { result.diagnostics.insert(result.diagnostics.end(), load.frontendDiagnostics.begin(), diff --git a/src/analysis/CommandInjectionAnalysis.cpp b/src/analysis/CommandInjectionAnalysis.cpp index 7463063..2880672 100644 --- a/src/analysis/CommandInjectionAnalysis.cpp +++ b/src/analysis/CommandInjectionAnalysis.cpp @@ -130,4 +130,48 @@ namespace ctrace::stack::analysis return issues; } + + namespace + { + template + static void + collectCommandInjectionFromCallBases(const llvm::Function& function, + const std::vector& callBases, + std::vector& issues) + { + for (const CallBaseT* call : callBases) + { + const llvm::Function* callee = getDirectCallee(*call); + if (!callee) + continue; + + const llvm::StringRef canonicalName = canonicalCalleeName(callee->getName()); + const std::optional commandArg = shellCommandArgIndex(canonicalName); + if (!commandArg || *commandArg >= call->arg_size()) + continue; + + const llvm::Value* commandValue = call->getArgOperand(*commandArg); + if (isCompileTimeConstantString(commandValue)) + continue; + + CommandInjectionIssue issue; + issue.funcName = function.getName().str(); + issue.filePath = getFunctionSourcePath(function); + issue.sinkName = canonicalName.str(); + issue.inst = call; + issues.push_back(std::move(issue)); + } + } + } // namespace + + std::vector + analyzeCommandInjectionCached(const llvm::Function& function, + const std::vector& calls, + const std::vector& invokes) + { + std::vector issues; + collectCommandInjectionFromCallBases(function, calls, issues); + collectCommandInjectionFromCallBases(function, invokes, issues); + return issues; + } } // namespace ctrace::stack::analysis diff --git a/src/analysis/DynamicAlloca.cpp b/src/analysis/DynamicAlloca.cpp index fb0c322..297d5df 100644 --- a/src/analysis/DynamicAlloca.cpp +++ b/src/analysis/DynamicAlloca.cpp @@ -89,4 +89,42 @@ namespace ctrace::stack::analysis return out; } + + std::vector + analyzeDynamicAllocasCached(const llvm::Function& function, + const std::vector& allocas) + { + std::vector out; + using namespace llvm; + + for (const AllocaInst* AI : allocas) + { + const Value* arraySizeVal = AI->getArraySize(); + + if (llvm::isa(arraySizeVal)) + continue; + + if (tryGetConstFromValue(arraySizeVal, function) != nullptr) + continue; + + DynamicAllocaIssue issue; + issue.funcName = function.getName().str(); + issue.varName = deriveAllocaName(AI); + if (AI->getAllocatedType()) + { + std::string tyStr; + llvm::raw_string_ostream rso(tyStr); + AI->getAllocatedType()->print(rso); + issue.typeName = rso.str(); + } + else + { + issue.typeName = ""; + } + issue.allocaInst = AI; + out.push_back(std::move(issue)); + } + + return out; + } } // namespace ctrace::stack::analysis diff --git a/src/analysis/InputPipeline.cpp b/src/analysis/InputPipeline.cpp index 4e59ad8..9ea21dc 100644 --- a/src/analysis/InputPipeline.cpp +++ b/src/analysis/InputPipeline.cpp @@ -1,6 +1,7 @@ #include "analysis/InputPipeline.hpp" #include "analysis/CompileCommands.hpp" #include "analysis/FrontendDiagnostics.hpp" +#include "analyzer/HotspotProfiler.hpp" #include #include @@ -16,9 +17,12 @@ #include #include +#include +#include #include #include #include +#include #include #include #include @@ -50,6 +54,42 @@ namespace ctrace::stack::analysis args.push_back(flag); } + static void removeOutputPathArgs(std::vector& args) + { + std::vector filtered; + filtered.reserve(args.size()); + for (std::size_t i = 0; i < args.size(); ++i) + { + const std::string& arg = args[i]; + if (arg == "-o") + { + if (i + 1 < args.size()) + ++i; + continue; + } + if (arg.rfind("-o=", 0) == 0) + continue; + if (arg.size() > 2 && arg.rfind("-o", 0) == 0) + continue; + filtered.push_back(arg); + } + args.swap(filtered); + } + + static std::vector + buildBitcodeCompileArgs(const std::vector& baseArgs, + const std::filesystem::path& outputBitcodePath) + { + std::vector args = baseArgs; + args.erase(std::remove(args.begin(), args.end(), "-S"), args.end()); + removeOutputPathArgs(args); + appendIfMissing(args, "-emit-llvm"); + appendIfMissing(args, "-c"); + args.push_back("-o"); + args.push_back(outputBitcodePath.string()); + return args; + } + bool hasDebugFlag(const std::vector& args) { for (const auto& arg : args) @@ -152,7 +192,7 @@ namespace ctrace::stack::analysis } } - constexpr llvm::StringLiteral kCompileIRCacheSchema = "compile-ir-cache-v1"; + constexpr llvm::StringLiteral kCompileIRCacheSchema = "compile-ir-cache-v2"; struct FileSnapshot { @@ -165,6 +205,7 @@ namespace ctrace::stack::analysis { std::filesystem::path directory; std::filesystem::path metaFile; + std::filesystem::path bcFile; std::filesystem::path irFile; std::filesystem::path depFile; std::uint64_t enabled : 1 = false; @@ -173,6 +214,7 @@ namespace ctrace::stack::analysis struct CompileIRCachePayload { + std::string llvmBitcode; std::string llvmIR; std::string diagnostics; }; @@ -306,6 +348,8 @@ namespace ctrace::stack::analysis std::ostringstream keyPayload; keyPayload << std::string(kCompileIRCacheSchema) << "\n"; keyPayload << "language:" << static_cast(language) << "\n"; + keyPayload << "compileIRFormat:" + << (config.compileIRFormat == CompileIRFormat::LL ? "ll" : "bc") << "\n"; keyPayload << "file:" << makeAbsolutePathFrom(filename, workingDir) << "\n"; keyPayload << "workingDir:" << makeAbsolutePathFrom(workingDir, "") << "\n"; for (const std::string& arg : args) @@ -316,6 +360,7 @@ namespace ctrace::stack::analysis paths.enabled = true; paths.directory = directory; paths.metaFile = directory / (key + ".json"); + paths.bcFile = directory / (key + ".bc"); paths.irFile = directory / (key + ".ll"); paths.depFile = directory / (key + ".d"); return paths; @@ -457,11 +502,16 @@ namespace ctrace::stack::analysis return std::nullopt; } + std::string llvmBitcode; + (void)readTextFile(cachePaths.bcFile, llvmBitcode); + std::string llvmIR; - if (!readTextFile(cachePaths.irFile, llvmIR)) + (void)readTextFile(cachePaths.irFile, llvmIR); + if (llvmBitcode.empty() && llvmIR.empty()) return std::nullopt; CompileIRCachePayload payload; + payload.llvmBitcode = std::move(llvmBitcode); payload.llvmIR = std::move(llvmIR); if (const auto diagnostics = root->getString("diagnostics")) payload.diagnostics = diagnostics->str(); @@ -472,7 +522,8 @@ namespace ctrace::stack::analysis const FileSnapshot& sourceSnapshot, const std::vector& dependencySnapshots, const std::string& diagnostics, - const std::string& llvmIR) + const std::string& llvmIR, + const std::string& llvmBitcode) { if (!cachePaths.enabled) return false; @@ -496,6 +547,8 @@ namespace ctrace::stack::analysis metadataStream << llvm::formatv("{0:2}", llvm::json::Value(std::move(root))); metadataStream.flush(); + if (!llvmBitcode.empty() && !writeTextFile(cachePaths.bcFile, llvmBitcode)) + return false; if (!writeTextFile(cachePaths.irFile, llvmIR)) return false; if (!writeTextFile(cachePaths.metaFile, metadataText)) @@ -691,41 +744,65 @@ namespace ctrace::stack::analysis std::uint64_t active_ : 1 = false; std::uint64_t reservedFlags_ : 63 = 0; }; + + class ScopedFileCleanup + { + public: + explicit ScopedFileCleanup(const std::filesystem::path& path) : path_(path) {} + + ~ScopedFileCleanup() + { + std::error_code ec; + if (!path_.empty()) + std::filesystem::remove(path_, ec); + } + + private: + std::filesystem::path path_; + }; } // namespace + static std::string extractExtension(const std::string& path) + { + const auto pos = path.find_last_of('.'); + if (pos == std::string::npos || pos + 1 >= path.size()) + return {}; + return path.substr(pos + 1); + } + + static std::string lowercaseCopy(std::string value) + { + std::transform(value.begin(), value.end(), value.begin(), + [](unsigned char c) { return std::tolower(c); }); + return value; + } + LanguageType detectFromExtension(const std::string& path) { - auto pos = path.find_last_of('.'); - if (pos == std::string::npos) + const std::string rawExt = extractExtension(path); + if (rawExt.empty()) return LanguageType::Unknown; - std::string ext = path.substr(pos + 1); - std::transform(ext.begin(), ext.end(), ext.begin(), - [](unsigned char c) { return std::tolower(c); }); + // Preserve historic ".C" semantics as C++ while still matching lower-case variants. + if (rawExt == "C") + return LanguageType::CXX; + + const std::string ext = lowercaseCopy(rawExt); - if (ext == "ll") + if (ext == "ll" || ext == "bc") return LanguageType::LLVM_IR; if (ext == "c") return LanguageType::C; - if (ext == "cpp" || ext == "cc" || ext == "cxx" || ext == "c++" || ext == "cp" || - ext == "C") + if (ext == "cpp" || ext == "cc" || ext == "cxx" || ext == "c++" || ext == "cp") return LanguageType::CXX; return LanguageType::Unknown; } - LanguageType detectLanguageFromFile(const std::string& path, llvm::LLVMContext& ctx) + LanguageType detectLanguageFromFile(const std::string& path, llvm::LLVMContext&) { - { - llvm::SMDiagnostic diag; - if (auto mod = llvm::parseIRFile(path, diag, ctx)) - { - return LanguageType::LLVM_IR; - } - } - return detectFromExtension(path); } @@ -733,12 +810,17 @@ namespace ctrace::stack::analysis const AnalysisConfig& config, llvm::LLVMContext& ctx, llvm::SMDiagnostic& err) { + using ctrace::stack::analyzer::ScopedHotspot; ModuleLoadResult result; + const ScopedHotspot totalHotspot(config.timing, "input.load_module.total"); std::error_code cwdErr; std::filesystem::path baseDir = std::filesystem::current_path(cwdErr); using Clock = std::chrono::steady_clock; auto compileStart = Clock::now(); - result.language = detectLanguageFromFile(filename, ctx); + { + const ScopedHotspot hotspot(config.timing, "input.detect_language"); + result.language = detectLanguageFromFile(filename, ctx); + } if (result.language == LanguageType::Unknown) { @@ -752,8 +834,13 @@ namespace ctrace::stack::analysis std::string workingDir; std::string compileError; std::string compileDiagnosticsText; - if (!buildCompileArgs(filename, result.language, config, args, workingDir, - compileError)) + bool compileArgsReady = false; + { + const ScopedHotspot hotspot(config.timing, "input.build_compile_args"); + compileArgsReady = buildCompileArgs(filename, result.language, config, args, + workingDir, compileError); + } + if (!compileArgsReady) { result.error = compileError + "\n"; return result; @@ -761,16 +848,32 @@ namespace ctrace::stack::analysis if (config.timing) coretrace::log(coretrace::Level::Info, "Compiling {}...\n", filename); - compilerlib::OutputMode mode = compilerlib::OutputMode::ToMemory; + const bool preferBitcodeCompile = (config.compileIRFormat == CompileIRFormat::BC); const CompileIRCachePaths cachePaths = buildCompileIRCachePaths(config, filename, result.language, args, workingDir); + std::error_code tempDirErr; + std::filesystem::path tempDir = std::filesystem::temp_directory_path(tempDirErr); + if (tempDirErr) + tempDir = std::filesystem::path("."); + const std::string tempBitcodeName = + "coretrace-compile-ir-" + + md5Hex(filename + "|" + std::to_string(compileStart.time_since_epoch().count())) + + ".bc"; + const std::filesystem::path tempBitcodePath = tempDir / tempBitcodeName; + const ScopedFileCleanup tempBitcodeCleanup(tempBitcodePath); + const std::vector bitcodeArgs = + buildBitcodeCompileArgs(args, tempBitcodePath); + auto compileWithOptionalWorkingDir = - [&](const std::vector& compileArgs, + [&](const std::vector& compileArgs, compilerlib::OutputMode outputMode, bool useWorkingDir) -> std::optional { if (!useWorkingDir) - return compilerlib::compile(compileArgs, mode); + { + const ScopedHotspot hotspot(config.timing, "input.compiler.invoke"); + return compilerlib::compile(compileArgs, outputMode); + } std::string cwdError; ScopedCurrentPath cwdGuard(workingDir, cwdError); @@ -779,11 +882,12 @@ namespace ctrace::stack::analysis result.error = cwdError + "\n"; return std::nullopt; } - return compilerlib::compile(compileArgs, mode); + const ScopedHotspot hotspot(config.timing, "input.compiler.invoke.cwd"); + return compilerlib::compile(compileArgs, outputMode); }; auto compileWithConfiguredWorkingDir = - [&](const std::vector& compileArgs, + [&](const std::vector& compileArgs, compilerlib::OutputMode outputMode, bool& retriedWithWorkingDir) -> std::optional { retriedWithWorkingDir = false; @@ -793,26 +897,31 @@ namespace ctrace::stack::analysis { // Optimistic fast path for multi-job runs: most compdb commands use absolute // paths. - res = compileWithOptionalWorkingDir(compileArgs, false); + res = compileWithOptionalWorkingDir(compileArgs, outputMode, false); if (!res || !res->success) { // Fallback keeps correctness for relative include paths and avoids process // cwd races. std::lock_guard lock(gCompileWorkingDirMutex); - res = compileWithOptionalWorkingDir(compileArgs, true); + res = compileWithOptionalWorkingDir(compileArgs, outputMode, true); retriedWithWorkingDir = true; } } else { - res = compileWithOptionalWorkingDir(compileArgs, hasWorkingDir); + res = compileWithOptionalWorkingDir(compileArgs, outputMode, hasWorkingDir); } return res; }; if (cachePaths.enabled) { - if (auto cached = loadCompileIRCachePayload(cachePaths)) + auto cached = [&]() -> std::optional + { + const ScopedHotspot hotspot(config.timing, "input.cache.lookup.compile"); + return loadCompileIRCachePayload(cachePaths); + }(); + if (cached) { if (config.timing) coretrace::log(coretrace::Level::Info, "Compilation cache hit for {}\n", @@ -822,17 +931,74 @@ namespace ctrace::stack::analysis if (!compileDiagnosticsText.empty() && !config.quiet) logText(coretrace::Level::Warn, compileDiagnosticsText); - auto buffer = llvm::MemoryBuffer::getMemBuffer(cached->llvmIR, "cached_ir"); - llvm::SMDiagnostic diag; - auto parseStart = Clock::now(); - result.module = llvm::parseIR(buffer->getMemBufferRef(), diag, ctx); - if (config.timing) + const auto parseStart = Clock::now(); + auto tryParseCachedBitcode = [&]() { - const auto parseEnd = Clock::now(); - const auto ms = std::chrono::duration_cast( - parseEnd - parseStart) - .count(); - coretrace::log(coretrace::Level::Info, "IR parse done in {} ms\n", ms); + if (result.module || cached->llvmBitcode.empty()) + return; + auto bcBuffer = llvm::MemoryBuffer::getMemBufferCopy( + llvm::StringRef(cached->llvmBitcode.data(), cached->llvmBitcode.size()), + "cached_ir_bc"); + auto bitcodeModule = [&]() + { + const ScopedHotspot hotspot(config.timing, + "input.cache.parse_bitcode_payload"); + return llvm::parseBitcodeFile(bcBuffer->getMemBufferRef(), ctx); + }(); + if (bitcodeModule) + { + result.module = std::move(*bitcodeModule); + if (config.timing) + { + const auto parseEnd = Clock::now(); + const auto ms = + std::chrono::duration_cast( + parseEnd - parseStart) + .count(); + coretrace::log(coretrace::Level::Info, + "Bitcode parse done in {} ms\n", ms); + } + return; + } + if (config.timing) + { + std::string bitcodeError = llvm::toString(bitcodeModule.takeError()); + coretrace::log(coretrace::Level::Warn, + "Compilation cache bitcode invalid for {}; " + "falling back to textual IR ({})\n", + filename, bitcodeError); + } + }; + auto tryParseCachedTextIR = [&]() + { + if (result.module || cached->llvmIR.empty()) + return; + auto buffer = llvm::MemoryBuffer::getMemBuffer(cached->llvmIR, "cached_ir"); + llvm::SMDiagnostic diag; + { + const ScopedHotspot hotspot(config.timing, + "input.cache.parse_text_payload"); + result.module = llvm::parseIR(buffer->getMemBufferRef(), diag, ctx); + } + if (config.timing) + { + const auto parseEnd = Clock::now(); + const auto ms = std::chrono::duration_cast( + parseEnd - parseStart) + .count(); + coretrace::log(coretrace::Level::Info, "IR parse done in {} ms\n", ms); + } + }; + + if (preferBitcodeCompile) + { + tryParseCachedBitcode(); + tryParseCachedTextIR(); + } + else + { + tryParseCachedTextIR(); + tryParseCachedBitcode(); } if (result.module) @@ -843,7 +1009,13 @@ namespace ctrace::stack::analysis compileDiagnosticsText, *result.module, filename); } - if (!dumpModuleIR(*result.module, filename, config, baseDir, result.error)) + bool dumpOk = false; + { + const ScopedHotspot hotspot(config.timing, "input.dump_module_ir"); + dumpOk = dumpModuleIR(*result.module, filename, config, baseDir, + result.error); + } + if (!dumpOk) return result; return result; } @@ -864,6 +1036,7 @@ namespace ctrace::stack::analysis } bool retriedWithWorkingDir = false; + bool compiledViaBitcode = false; std::optional res; std::optional sourceSnapshot; std::optional> dependencySnapshots; @@ -871,36 +1044,71 @@ namespace ctrace::stack::analysis { std::error_code removeErr; std::filesystem::remove(cachePaths.depFile, removeErr); - std::vector cacheCompileArgs = args; + std::vector cacheCompileArgs = + preferBitcodeCompile ? bitcodeArgs : args; appendDependencyCaptureArgs(cacheCompileArgs, cachePaths.depFile); bool retriedForDependencyCompile = false; - res = - compileWithConfiguredWorkingDir(cacheCompileArgs, retriedForDependencyCompile); + res = compileWithConfiguredWorkingDir(cacheCompileArgs, + preferBitcodeCompile + ? compilerlib::OutputMode::ToFile + : compilerlib::OutputMode::ToMemory, + retriedForDependencyCompile); retriedWithWorkingDir = retriedForDependencyCompile; - if (!res || !res->success) + if (preferBitcodeCompile && (!res || !res->success)) { bool retriedForFallbackCompile = false; - auto fallbackResult = - compileWithConfiguredWorkingDir(args, retriedForFallbackCompile); + auto fallbackResult = compileWithConfiguredWorkingDir( + args, compilerlib::OutputMode::ToMemory, retriedForFallbackCompile); retriedWithWorkingDir = retriedWithWorkingDir || retriedForFallbackCompile; if (fallbackResult) res = std::move(fallbackResult); } else { - const auto dependencies = - parseDepfileDependencies(cachePaths.depFile, workingDir); + compiledViaBitcode = preferBitcodeCompile; + const auto dependencies = [&]() + { + const ScopedHotspot hotspot(config.timing, + "input.cache.parse_dependencies"); + return parseDepfileDependencies(cachePaths.depFile, workingDir); + }(); const std::string sourcePath = makeAbsolutePathFrom(filename, workingDir); - sourceSnapshot = captureFileSnapshot(sourcePath); + sourceSnapshot = [&]() -> std::optional + { + const ScopedHotspot hotspot(config.timing, + "input.cache.capture_source_snapshot"); + return captureFileSnapshot(sourcePath); + }(); if (dependencies && sourceSnapshot) + { + const ScopedHotspot hotspot(config.timing, + "input.cache.build_dependency_snapshots"); dependencySnapshots = buildDependencySnapshots(*dependencies); + } } } else { - res = compileWithConfiguredWorkingDir(args, retriedWithWorkingDir); + res = compileWithConfiguredWorkingDir(preferBitcodeCompile ? bitcodeArgs : args, + preferBitcodeCompile + ? compilerlib::OutputMode::ToFile + : compilerlib::OutputMode::ToMemory, + retriedWithWorkingDir); + if (res && res->success) + { + compiledViaBitcode = preferBitcodeCompile; + } + else if (preferBitcodeCompile) + { + bool retriedForFallbackCompile = false; + auto fallbackResult = compileWithConfiguredWorkingDir( + args, compilerlib::OutputMode::ToMemory, retriedForFallbackCompile); + retriedWithWorkingDir = retriedWithWorkingDir || retriedForFallbackCompile; + if (fallbackResult) + res = std::move(fallbackResult); + } } if (!res) @@ -917,12 +1125,6 @@ namespace ctrace::stack::analysis } compileDiagnosticsText = res->diagnostics; - if (res->llvmIR.empty()) - { - result.error = "No LLVM IR produced by compilerlib::compile\n"; - return result; - } - if (config.timing) { auto compileEnd = Clock::now(); @@ -933,27 +1135,106 @@ namespace ctrace::stack::analysis retriedWithWorkingDir ? " (retry with working directory)" : ""); } - auto buffer = llvm::MemoryBuffer::getMemBuffer(res->llvmIR, "in_memory_ll"); - - llvm::SMDiagnostic diag; - auto parseStart = Clock::now(); - result.module = llvm::parseIR(buffer->getMemBufferRef(), diag, ctx); - if (config.timing) + std::string llvmIRForCache; + std::string llvmBitcodeForCache; + if (compiledViaBitcode) { - auto parseEnd = Clock::now(); - auto ms = - std::chrono::duration_cast(parseEnd - parseStart) - .count(); - coretrace::log(coretrace::Level::Info, "IR parse done in {} ms\n", ms); + if (readTextFile(tempBitcodePath, llvmBitcodeForCache) && + !llvmBitcodeForCache.empty()) + { + const auto parseStart = Clock::now(); + auto bcBuffer = llvm::MemoryBuffer::getMemBufferCopy( + llvm::StringRef(llvmBitcodeForCache.data(), llvmBitcodeForCache.size()), + "compiled_bc"); + auto bitcodeModule = [&]() + { + const ScopedHotspot hotspot(config.timing, + "input.compile.parse_bitcode_output"); + return llvm::parseBitcodeFile(bcBuffer->getMemBufferRef(), ctx); + }(); + if (bitcodeModule) + { + result.module = std::move(*bitcodeModule); + if (config.timing) + { + const auto parseEnd = Clock::now(); + const auto ms = std::chrono::duration_cast( + parseEnd - parseStart) + .count(); + coretrace::log(coretrace::Level::Info, "Bitcode parse done in {} ms\n", + ms); + } + } + else if (config.timing) + { + std::string bitcodeError = llvm::toString(bitcodeModule.takeError()); + coretrace::log(coretrace::Level::Warn, + "Bitcode output invalid for {}; " + "falling back to textual IR ({})\n", + filename, bitcodeError); + } + } + else if (config.timing) + { + coretrace::log(coretrace::Level::Warn, + "Missing bitcode output for {}; falling back to textual IR\n", + filename); + } } if (!result.module) { - std::string msg; - llvm::raw_string_ostream os(msg); - diag.print("in_memory_ll", os); - result.error = "Failed to parse in-memory LLVM IR:\n" + os.str(); - return result; + if (compiledViaBitcode) + { + bool retriedForTextFallback = false; + auto textFallback = compileWithConfiguredWorkingDir( + args, compilerlib::OutputMode::ToMemory, retriedForTextFallback); + retriedWithWorkingDir = retriedWithWorkingDir || retriedForTextFallback; + if (!textFallback) + return result; + res = std::move(textFallback); + if (!res->success) + { + result.error = "Compilation failed:\n" + res->diagnostics + '\n'; + return result; + } + if (!res->diagnostics.empty() && !config.quiet) + logText(coretrace::Level::Warn, res->diagnostics); + compileDiagnosticsText = res->diagnostics; + } + + if (res->llvmIR.empty()) + { + result.error = "No LLVM IR produced by compilerlib::compile\n"; + return result; + } + + llvmIRForCache = res->llvmIR; + auto buffer = llvm::MemoryBuffer::getMemBuffer(llvmIRForCache, "in_memory_ll"); + + llvm::SMDiagnostic diag; + const auto parseStart = Clock::now(); + { + const ScopedHotspot hotspot(config.timing, "input.compile.parse_text_output"); + result.module = llvm::parseIR(buffer->getMemBufferRef(), diag, ctx); + } + if (config.timing) + { + const auto parseEnd = Clock::now(); + const auto ms = + std::chrono::duration_cast(parseEnd - parseStart) + .count(); + coretrace::log(coretrace::Level::Info, "IR parse done in {} ms\n", ms); + } + + if (!result.module) + { + std::string msg; + llvm::raw_string_ostream os(msg); + diag.print("in_memory_ll", os); + result.error = "Failed to parse in-memory LLVM IR:\n" + os.str(); + return result; + } } if (!compileDiagnosticsText.empty()) { @@ -963,9 +1244,20 @@ namespace ctrace::stack::analysis if (cachePaths.enabled && sourceSnapshot && dependencySnapshots) { - const bool stored = - storeCompileIRCachePayload(cachePaths, *sourceSnapshot, *dependencySnapshots, - compileDiagnosticsText, res->llvmIR); + if (llvmBitcodeForCache.empty()) + { + llvm::raw_string_ostream bitcodeStream(llvmBitcodeForCache); + llvm::WriteBitcodeToFile(*result.module, bitcodeStream); + bitcodeStream.flush(); + } + + const bool stored = [&]() + { + const ScopedHotspot hotspot(config.timing, "input.cache.store_compile"); + return storeCompileIRCachePayload(cachePaths, *sourceSnapshot, + *dependencySnapshots, compileDiagnosticsText, + llvmIRForCache, llvmBitcodeForCache); + }(); if (config.timing && stored) { coretrace::log(coretrace::Level::Info, @@ -978,26 +1270,172 @@ namespace ctrace::stack::analysis std::filesystem::remove(cachePaths.depFile, removeErr); } - if (!dumpModuleIR(*result.module, filename, config, baseDir, result.error)) + bool dumpOk = false; + { + const ScopedHotspot hotspot(config.timing, "input.dump_module_ir"); + dumpOk = dumpModuleIR(*result.module, filename, config, baseDir, result.error); + } + if (!dumpOk) return result; return result; } + const std::string inputExt = lowercaseCopy(extractExtension(filename)); + const bool isTextIRInput = (inputExt == "ll"); + const std::string cacheWorkingDir = + cwdErr ? std::string() : baseDir.lexically_normal().generic_string(); + const CompileIRCachePaths cachePaths = + isTextIRInput + ? buildCompileIRCachePaths(config, filename, result.language, {}, cacheWorkingDir) + : CompileIRCachePaths{}; + + if (isTextIRInput && cachePaths.enabled) + { + auto cached = [&]() -> std::optional + { + const ScopedHotspot hotspot(config.timing, "input.cache.lookup_ir_input"); + return loadCompileIRCachePayload(cachePaths); + }(); + if (cached) + { + if (config.timing) + coretrace::log(coretrace::Level::Info, "IR parse cache hit for {}\n", filename); + + const auto cacheParseStart = Clock::now(); + if (!cached->llvmBitcode.empty()) + { + auto bcBuffer = llvm::MemoryBuffer::getMemBufferCopy( + llvm::StringRef(cached->llvmBitcode.data(), cached->llvmBitcode.size()), + "cached_input_ir_bc"); + auto bitcodeModule = [&]() + { + const ScopedHotspot hotspot(config.timing, + "input.cache.parse_bitcode_ir_input"); + return llvm::parseBitcodeFile(bcBuffer->getMemBufferRef(), ctx); + }(); + if (bitcodeModule) + { + result.module = std::move(*bitcodeModule); + if (config.timing) + { + const auto cacheParseEnd = Clock::now(); + const auto ms = std::chrono::duration_cast( + cacheParseEnd - cacheParseStart) + .count(); + coretrace::log(coretrace::Level::Info, "Bitcode parse done in {} ms\n", + ms); + } + } + else if (config.timing) + { + std::string bitcodeError = llvm::toString(bitcodeModule.takeError()); + coretrace::log(coretrace::Level::Warn, + "IR parse cache bitcode invalid for {}; " + "falling back to textual IR ({})\n", + filename, bitcodeError); + } + } + + if (!result.module && !cached->llvmIR.empty()) + { + auto buffer = + llvm::MemoryBuffer::getMemBuffer(cached->llvmIR, "cached_input_ir"); + llvm::SMDiagnostic diag; + { + const ScopedHotspot hotspot(config.timing, + "input.cache.parse_text_ir_input"); + result.module = llvm::parseIR(buffer->getMemBufferRef(), diag, ctx); + } + if (config.timing) + { + const auto cacheParseEnd = Clock::now(); + const auto ms = std::chrono::duration_cast( + cacheParseEnd - cacheParseStart) + .count(); + coretrace::log(coretrace::Level::Info, "IR parse done in {} ms\n", ms); + } + } + + if (result.module) + { + bool dumpOk = false; + { + const ScopedHotspot hotspot(config.timing, "input.dump_module_ir"); + dumpOk = + dumpModuleIR(*result.module, filename, config, baseDir, result.error); + } + if (!dumpOk) + return result; + return result; + } + + if (config.timing) + { + coretrace::log(coretrace::Level::Warn, + "IR parse cache entry invalid for {}; reparsing input\n", + filename); + } + result.module.reset(); + } + else if (config.timing) + { + coretrace::log(coretrace::Level::Info, "IR parse cache miss for {}\n", filename); + } + } + if (config.timing) coretrace::log(coretrace::Level::Info, "Parsing IR {}...\n", filename); - auto parseStart = Clock::now(); - result.module = llvm::parseIRFile(filename, err, ctx); + const auto parseStart = Clock::now(); + { + const ScopedHotspot hotspot(config.timing, "input.parse_ir_file"); + result.module = llvm::parseIRFile(filename, err, ctx); + } if (config.timing) { - auto parseEnd = Clock::now(); - auto ms = std::chrono::duration_cast(parseEnd - parseStart) - .count(); + const auto parseEnd = Clock::now(); + const auto ms = + std::chrono::duration_cast(parseEnd - parseStart) + .count(); coretrace::log(coretrace::Level::Info, "IR parse done in {} ms\n", ms); } if (result.module) { - if (!dumpModuleIR(*result.module, filename, config, baseDir, result.error)) + if (isTextIRInput && cachePaths.enabled) + { + if (const auto sourceSnapshot = captureFileSnapshot(filename)) + { + std::string sourceIR; + (void)readTextFile(filename, sourceIR); + + std::string llvmBitcode; + llvm::raw_string_ostream bitcodeStream(llvmBitcode); + llvm::WriteBitcodeToFile(*result.module, bitcodeStream); + bitcodeStream.flush(); + + std::vector dependencySnapshots; + dependencySnapshots.push_back(*sourceSnapshot); + const bool stored = [&]() + { + const ScopedHotspot hotspot(config.timing, "input.cache.store_ir_parse"); + return storeCompileIRCachePayload(cachePaths, *sourceSnapshot, + dependencySnapshots, "", sourceIR, + llvmBitcode); + }(); + if (config.timing && stored) + { + coretrace::log(coretrace::Level::Info, + "Stored IR parse cache entry for {}\n", filename); + } + } + } + + bool dumpOk = false; + { + const ScopedHotspot hotspot(config.timing, "input.dump_module_ir"); + dumpOk = dumpModuleIR(*result.module, filename, config, baseDir, result.error); + } + if (!dumpOk) return result; } return result; diff --git a/src/analysis/MemIntrinsicOverflow.cpp b/src/analysis/MemIntrinsicOverflow.cpp index a32eae8..d252c74 100644 --- a/src/analysis/MemIntrinsicOverflow.cpp +++ b/src/analysis/MemIntrinsicOverflow.cpp @@ -279,4 +279,88 @@ namespace ctrace::stack::analysis } return issues; } + + namespace + { + template + static void analyzeMemIntrinsicFromCallBases(const llvm::Function& function, + const llvm::DataLayout& DL, + const std::vector& callBases, + const BufferWriteModel* externalModel, + BufferWriteRuleMatcher* ruleMatcher, + std::vector& out) + { + using namespace llvm; + + for (const CallBaseT* constCB : callBases) + { + // The existing helpers take non-const CallBase* but do not + // modify the instruction. Use const_cast for API compat. + auto* CB = const_cast(constCB); + + ResolvedSink sink = resolveBuiltInSink(CB); + const ResolvedSink modeledSink = resolveModelSink(CB, externalModel, ruleMatcher); + if (modeledSink.valid) + sink = modeledSink; + + if (!sink.valid) + continue; + + if (CB->arg_size() <= sink.destArgIndex) + continue; + + Value* dest = CB->getArgOperand(sink.destArgIndex); + const AllocaInst* AI = resolveStackDestinationAlloca(dest); + if (!AI) + continue; + + auto maybeSize = getAllocaTotalSizeBytes(AI, DL); + if (!maybeSize) + continue; + StackSize destBytes = *maybeSize; + + MemIntrinsicIssue issue; + issue.funcName = function.getName().str(); + issue.varName = AI->hasName() ? AI->getName().str() : std::string(""); + issue.destSizeBytes = destBytes; + issue.inst = constCB; + issue.intrinsicName = sink.displayName; + + if (sink.hasExplicitLength) + { + if (CB->arg_size() <= sink.sizeArgIndex) + continue; + Value* lenV = CB->getArgOperand(sink.sizeArgIndex); + auto* lenC = dyn_cast(lenV); + if (!lenC) + continue; + + const uint64_t len = lenC->getZExtValue(); + if (len <= destBytes) + continue; + issue.lengthBytes = len; + issue.hasExplicitLength = true; + } + else + { + issue.hasExplicitLength = false; + } + + out.push_back(std::move(issue)); + } + } + } // namespace + + std::vector + analyzeMemIntrinsicOverflowsCached(const llvm::Function& function, const llvm::DataLayout& DL, + const std::vector& calls, + const std::vector& invokes, + const BufferWriteModel* externalModel, + BufferWriteRuleMatcher* ruleMatcher) + { + std::vector issues; + analyzeMemIntrinsicFromCallBases(function, DL, calls, externalModel, ruleMatcher, issues); + analyzeMemIntrinsicFromCallBases(function, DL, invokes, externalModel, ruleMatcher, issues); + return issues; + } } // namespace ctrace::stack::analysis diff --git a/src/analysis/ResourceLifetimeAnalysis.cpp b/src/analysis/ResourceLifetimeAnalysis.cpp index bdd4b95..2c6263c 100644 --- a/src/analysis/ResourceLifetimeAnalysis.cpp +++ b/src/analysis/ResourceLifetimeAnalysis.cpp @@ -2378,6 +2378,67 @@ namespace ctrace::stack::analysis return true; } + static bool resourceSummaryFunctionEquals(const ResourceSummaryFunction& lhs, + const ResourceSummaryFunction& rhs) + { + std::vector leftKeys; + std::vector rightKeys; + leftKeys.reserve(lhs.effects.size()); + rightKeys.reserve(rhs.effects.size()); + + for (const ResourceSummaryEffect& effect : lhs.effects) + { + ParamLifetimeEffect tmp; + tmp.action = fromPublicSummaryAction(effect.action); + tmp.argIndex = effect.argIndex; + tmp.offset = effect.offset; + tmp.viaPointerSlot = effect.viaPointerSlot; + tmp.resourceKind = effect.resourceKind; + leftKeys.push_back(encodeSummaryEffectKey(tmp)); + } + for (const ResourceSummaryEffect& effect : rhs.effects) + { + ParamLifetimeEffect tmp; + tmp.action = fromPublicSummaryAction(effect.action); + tmp.argIndex = effect.argIndex; + tmp.offset = effect.offset; + tmp.viaPointerSlot = effect.viaPointerSlot; + tmp.resourceKind = effect.resourceKind; + rightKeys.push_back(encodeSummaryEffectKey(tmp)); + } + + std::sort(leftKeys.begin(), leftKeys.end()); + std::sort(rightKeys.begin(), rightKeys.end()); + return leftKeys == rightKeys; + } + + std::unordered_set + computeChangedResourceFunctionNames(const ResourceSummaryIndex& prev, + const ResourceSummaryIndex& next) + { + std::unordered_set changed; + + for (const auto& entry : next.functions) + { + auto prevIt = prev.functions.find(entry.first); + if (prevIt == prev.functions.end()) + { + changed.insert(entry.first); + continue; + } + if (!resourceSummaryFunctionEquals(entry.second, prevIt->second)) + changed.insert(entry.first); + } + + for (const auto& entry : prev.functions) + { + if (next.functions.find(entry.first) == next.functions.end()) + changed.insert(entry.first); + } + + return changed; + } + std::vector analyzeResourceLifetime( llvm::Module& mod, const std::function& shouldAnalyze, const std::string& modelPath, const ResourceSummaryIndex* externalSummaries) diff --git a/src/analysis/TOCTOUAnalysis.cpp b/src/analysis/TOCTOUAnalysis.cpp index c2682d7..8fd2766 100644 --- a/src/analysis/TOCTOUAnalysis.cpp +++ b/src/analysis/TOCTOUAnalysis.cpp @@ -263,4 +263,88 @@ namespace ctrace::stack::analysis return issues; } + + namespace + { + template + static void collectTOCTOUEventsFromCallBases(const std::vector& callBases, + std::vector& checks, + std::vector& uses, unsigned& order) + { + for (const CallBaseT* call : callBases) + { + ++order; + + const llvm::Function* callee = getDirectCallee(*call); + if (!callee) + continue; + + const llvm::StringRef canonicalName = canonicalCalleeName(callee->getName()); + const std::optional checkArg = checkPathArgIndex(canonicalName); + const std::optional useArg = usePathArgIndex(canonicalName); + if (!checkArg && !useArg) + continue; + + const unsigned argIndex = checkArg ? *checkArg : *useArg; + if (argIndex >= call->arg_size()) + continue; + + const llvm::Value* pathValue = + peelPointerFromSingleStoreSlot(call->getArgOperand(argIndex)); + PathEvent event; + event.inst = call; + event.root = llvm::getUnderlyingObject(pathValue, 32); + event.literal = tryExtractStringLiteral(pathValue).value_or(""); + event.api = canonicalName.str(); + event.order = order; + + if (checkArg) + checks.push_back(std::move(event)); + else + uses.push_back(std::move(event)); + } + } + } // namespace + + std::vector + analyzeTOCTOUCached(const llvm::Function& function, + const std::vector& calls, + const std::vector& invokes) + { + std::vector issues; + + std::vector checks; + std::vector uses; + unsigned order = 0; + + // Calls and invokes are stored in traversal order individually. + // Process calls first, then invokes. Relative ordering within each + // list is correct; cross-list ordering is approximate but sufficient + // since TOCTOU-relevant APIs (access/stat/open/fopen) are virtually + // always CallInst, never InvokeInst. + collectTOCTOUEventsFromCallBases(calls, checks, uses, order); + collectTOCTOUEventsFromCallBases(invokes, checks, uses, order); + + for (const PathEvent& useEvent : uses) + { + for (const PathEvent& checkEvent : checks) + { + if (checkEvent.order >= useEvent.order) + continue; + if (!likelySamePath(checkEvent, useEvent)) + continue; + + TOCTOUIssue issue; + issue.funcName = function.getName().str(); + issue.filePath = getFunctionSourcePath(function); + issue.checkApi = checkEvent.api; + issue.useApi = useEvent.api; + issue.inst = useEvent.inst; + issues.push_back(std::move(issue)); + break; + } + } + + return issues; + } } // namespace ctrace::stack::analysis diff --git a/src/analysis/UninitializedVarAnalysis.cpp b/src/analysis/UninitializedVarAnalysis.cpp index c1ecead..f3eea8a 100644 --- a/src/analysis/UninitializedVarAnalysis.cpp +++ b/src/analysis/UninitializedVarAnalysis.cpp @@ -3285,7 +3285,64 @@ namespace ctrace::stack::analysis } const unsigned reachableBlocks = static_cast(outState.size()); + + // Leaf function optimization: if no call in this function targets a + // function in summaries or externalSummariesByName, the dataflow + // converges in exactly 1 iteration (no interprocedural effects). + bool isLeaf = true; + for (const llvm::BasicBlock& BB : F) + { + if (!isLeaf) + break; + if (!reachable.lookup(&BB)) + continue; + for (const llvm::Instruction& I : BB) + { + const auto* CB = llvm::dyn_cast(&I); + if (!CB) + continue; + const llvm::Function* callee = CB->getCalledFunction(); + if (!callee) + continue; + if (summaries.find(callee) != summaries.end()) + { + isLeaf = false; + break; + } + if (externalSummariesByName && canonicalCalleeNames) + { + auto nameIt = canonicalCalleeNames->find(callee); + if (nameIt != canonicalCalleeNames->end() && + externalSummariesByName->find(nameIt->second) != + externalSummariesByName->end()) + { + isLeaf = false; + break; + } + } + } + } + + // Leaf functions have no inter-procedural deps so the *summary* + // converges in one outer iteration, but the intra-function + // dataflow (loops, phi-nodes) still needs the full BB iteration + // budget to stabilize. const unsigned maxIterations = std::max(64u, reachableBlocks * 16u); + + // When in issue-collection mode (outSummary == nullptr), we fuse + // the issue collection into the fixpoint loop. On each iteration + // we collect issues opportunistically; if the fixpoint hasn't + // converged yet, we reset and recollect on the next iteration. + // When it converges, the collected issues are final, saving one + // full function traversal. + const bool fuseIssueCollection = (outSummary == nullptr && outIssues != nullptr); + + llvm::BitVector writeSeen(trackedCount, false); + llvm::BitVector constructedSeen(trackedCount, false); + llvm::BitVector defaultCtorSeen(trackedCount, false); + llvm::BitVector readBeforeInitSeen(trackedCount, false); + std::vector pendingIssues; + bool changed = true; unsigned iteration = 0; while (changed && iteration < maxIterations) @@ -3293,6 +3350,15 @@ namespace ctrace::stack::analysis ++iteration; changed = false; + if (fuseIssueCollection) + { + writeSeen.reset(); + constructedSeen.reset(); + defaultCtorSeen.reset(); + readBeforeInitSeen.reset(); + pendingIssues.clear(); + } + for (const llvm::BasicBlock& BB : F) { if (!reachable.lookup(&BB)) @@ -3302,11 +3368,24 @@ namespace ctrace::stack::analysis computeInState(BB, &F.getEntryBlock(), reachable, outState, tracked); InitRangeState state = newIn; - for (const llvm::Instruction& I : BB) + if (fuseIssueCollection) { - transferInstruction(I, tracked, DL, summaries, externalSummariesByName, - canonicalCalleeNames, state, nullptr, nullptr, nullptr, - nullptr, nullptr, nullptr); + for (const llvm::Instruction& I : BB) + { + transferInstruction(I, tracked, DL, summaries, externalSummariesByName, + canonicalCalleeNames, state, &writeSeen, + &constructedSeen, &defaultCtorSeen, + &readBeforeInitSeen, nullptr, &pendingIssues); + } + } + else + { + for (const llvm::Instruction& I : BB) + { + transferInstruction(I, tracked, DL, summaries, externalSummariesByName, + canonicalCalleeNames, state, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr); + } } InitRangeState& oldIn = inState[&BB]; @@ -3342,25 +3421,18 @@ namespace ctrace::stack::analysis return; } - llvm::BitVector writeSeen(trackedCount, false); - llvm::BitVector constructedSeen(trackedCount, false); - llvm::BitVector defaultCtorSeen(trackedCount, false); - llvm::BitVector readBeforeInitSeen(trackedCount, false); - - for (const llvm::BasicBlock& BB : F) + if (!fuseIssueCollection) { - if (!reachable.lookup(&BB)) - continue; - - InitRangeState state = inState[&BB]; - for (const llvm::Instruction& I : BB) - { - transferInstruction(I, tracked, DL, summaries, externalSummariesByName, - canonicalCalleeNames, state, &writeSeen, &constructedSeen, - &defaultCtorSeen, &readBeforeInitSeen, nullptr, outIssues); - } + // Fallback: separate issue collection traversal + // (only reached if outIssues == nullptr, i.e. no-op mode) + return; } + // Issues were collected during the last fixpoint iteration. + // Transfer pendingIssues to the output. + if (outIssues) + outIssues->insert(outIssues->end(), pendingIssues.begin(), pendingIssues.end()); + for (unsigned idx = 0; idx < trackedCount; ++idx) { const TrackedMemoryObject& obj = tracked.objects[idx]; @@ -3427,6 +3499,7 @@ namespace ctrace::stack::analysis const CanonicalCalleeNameMap* canonicalCalleeNames) { FunctionSummaryMap summaries; + llvm::SmallVector analysisFunctions; for (const llvm::Function& F : mod) { if (F.isDeclaration()) @@ -3434,8 +3507,33 @@ namespace ctrace::stack::analysis if (!shouldAnalyze(F)) continue; summaries[&F] = makeEmptySummary(F); + analysisFunctions.push_back(&F); + } + + // Build reverse call graph: callerOf[callee] = {callers...} + llvm::DenseMap> + callerOf; + for (const llvm::Function* func : analysisFunctions) + { + for (const llvm::BasicBlock& BB : *func) + { + for (const llvm::Instruction& I : BB) + { + const auto* CB = llvm::dyn_cast(&I); + if (!CB) + continue; + const llvm::Function* callee = CB->getCalledFunction(); + if (callee && summaries.find(callee) != summaries.end()) + callerOf[callee].push_back(func); + } + } } + // Track which functions need re-analysis. + llvm::DenseSet dirtyFunctions; + for (const llvm::Function* func : analysisFunctions) + dirtyFunctions.insert(func); + bool changed = true; unsigned guard = 0; while (changed && guard < 64) @@ -3443,23 +3541,30 @@ namespace ctrace::stack::analysis changed = false; ++guard; - for (const llvm::Function& F : mod) + llvm::DenseSet nextDirty; + for (const llvm::Function* func : analysisFunctions) { - if (F.isDeclaration()) - continue; - if (!shouldAnalyze(F)) + if (dirtyFunctions.find(func) == dirtyFunctions.end()) continue; - FunctionSummary next = makeEmptySummary(F); - analyzeFunction(F, mod.getDataLayout(), summaries, externalSummariesByName, + FunctionSummary next = makeEmptySummary(*func); + analyzeFunction(*func, mod.getDataLayout(), summaries, externalSummariesByName, canonicalCalleeNames, &next, nullptr); - FunctionSummary& cur = summaries[&F]; + FunctionSummary& cur = summaries[func]; if (!(cur == next)) { cur = std::move(next); changed = true; + // Mark callers as dirty for the next iteration. + auto callersIt = callerOf.find(func); + if (callersIt != callerOf.end()) + { + for (const llvm::Function* caller : callersIt->second) + nextDirty.insert(caller); + } } } + dirtyFunctions = std::move(nextDirty); } return summaries; @@ -3933,6 +4038,44 @@ namespace ctrace::stack::analysis return true; } + std::unordered_set + computeChangedUninitializedFunctionNames(const UninitializedSummaryIndex& prev, + const UninitializedSummaryIndex& next) + { + std::unordered_set changed; + + for (const auto& entry : next.functions) + { + auto prevIt = prev.functions.find(entry.first); + if (prevIt == prev.functions.end()) + { + changed.insert(entry.first); + continue; + } + if (!publicFunctionSummaryEquals(entry.second, prevIt->second)) + changed.insert(entry.first); + } + + for (const auto& entry : prev.functions) + { + if (next.functions.find(entry.first) == next.functions.end()) + changed.insert(entry.first); + } + + return changed; + } + + std::unordered_set + getCanonicalCalleeNames(const PreparedUninitializedModuleContext& prepared) + { + std::unordered_set result; + if (!prepared.opaque) + return result; + for (const auto& entry : prepared.opaque->canonicalCalleeNames) + result.insert(entry.second); + return result; + } + std::vector analyzeUninitializedLocalReads(llvm::Module& mod, const std::function& shouldAnalyze, diff --git a/src/analyzer/AnalysisPipeline.cpp b/src/analyzer/AnalysisPipeline.cpp index 3f731e5..f38b8f2 100644 --- a/src/analyzer/AnalysisPipeline.cpp +++ b/src/analyzer/AnalysisPipeline.cpp @@ -1,7 +1,13 @@ #include "analyzer/AnalysisPipeline.hpp" +#include "analyzer/AnalysisArtifactStore.hpp" +#include "analyzer/DerivedModuleArtifacts.hpp" #include "analyzer/DiagnosticEmitter.hpp" +#include "analyzer/HotspotProfiler.hpp" +#include "analyzer/IRFactCollector.hpp" +#include "analyzer/InstructionSubscriber.hpp" #include "analyzer/ModulePreparationService.hpp" +#include "analyzer/PerFunctionInstructionCache.hpp" #include "analysis/AllocaUsage.hpp" #include "analysis/ConstParamAnalysis.hpp" @@ -25,9 +31,15 @@ #include "passes/ModulePasses.hpp" #include +#include +#include +#include #include #include #include +#include +#include +#include #include #include @@ -36,14 +48,149 @@ namespace ctrace::stack::analyzer { namespace { + enum class ArtifactId : std::uint64_t + { + None = 0, + PreparedModule = 1ull << 0, + IRFacts = 1ull << 1, + AllocaLargeThreshold = 1ull << 2, + PipelineSubscriberSignals = 1ull << 3, + DerivedModuleArtifacts = 1ull << 4 + }; + + using ArtifactMask = std::uint64_t; + + enum class ExecutionModel : std::uint8_t + { + Utility = 0, + SubscriberCompatible = 1, + Independent = 2 + }; + + constexpr ArtifactMask maskOf(ArtifactId id) + { + return static_cast(id); + } + + struct TraversalEstimate + { + std::uint64_t fullTraversalPasses = 0; + std::uint64_t estimatedInstructionVisits = 0; + }; + + struct StepTraversalStats + { + const char* label = ""; + std::uint64_t moduleVisits = 0; + std::uint64_t functionVisits = 0; + std::uint64_t instructionVisits = 0; + std::int64_t durationMs = 0; + std::uint32_t executionModel = static_cast(ExecutionModel::Utility); + std::uint32_t reservedPadding = 0; + }; + + struct PipelineSubscriberSignals + { + std::uint64_t callSiteCount = 0; + std::uint64_t loadCount = 0; + std::uint64_t bufferRelevantCount = 0; + }; + + class PipelineSignalSubscriber final : public InstructionSubscriber + { + public: + explicit PipelineSignalSubscriber(PipelineSubscriberSignals& signals) + : signals_(signals) + { + } + + void onAlloca(const llvm::AllocaInst&) override + { + ++signals_.bufferRelevantCount; + } + void onLoad(const llvm::LoadInst&) override + { + ++signals_.loadCount; + } + void onStore(const llvm::StoreInst&) override + { + ++signals_.bufferRelevantCount; + } + void onCall(const llvm::CallInst&) override + { + ++signals_.callSiteCount; + } + void onInvoke(const llvm::InvokeInst&) override + { + ++signals_.callSiteCount; + } + void onMemIntrinsic(const llvm::MemIntrinsic&) override + { + ++signals_.bufferRelevantCount; + } + + private: + PipelineSubscriberSignals& signals_; + }; + + static PipelineSubscriberSignals derivePipelineSignals(const IRFacts& facts) + { + PipelineSubscriberSignals signals; + signals.callSiteCount = facts.callInstCount + facts.invokeInstCount; + signals.loadCount = facts.loadInstCount; + signals.bufferRelevantCount = + facts.allocaInstCount + facts.storeInstCount + facts.memIntrinsicCount; + return signals; + } + + static bool parseBooleanEnvFlag(const char* name, bool defaultValue) + { + const char* raw = std::getenv(name); + if (!raw) + return defaultValue; + + std::string value(raw); + for (char& ch : value) + ch = static_cast(std::tolower(static_cast(ch))); + + if (value == "1" || value == "true" || value == "yes" || value == "on") + return true; + if (value == "0" || value == "false" || value == "no" || value == "off") + return false; + return defaultValue; + } + + static bool usePipelineSubscribers() + { + static const bool enabled = parseBooleanEnvFlag("CTRACE_PIPELINE_SUBSCRIBERS", true); + return enabled; + } + + static const char* executionModelName(ExecutionModel model) + { + switch (model) + { + case ExecutionModel::Utility: + return "utility"; + case ExecutionModel::SubscriberCompatible: + return "subscriber-compatible"; + case ExecutionModel::Independent: + return "independent"; + } + return "utility"; + } + struct PipelineData { llvm::Module& mod; const AnalysisConfig& config; + AnalysisArtifactStore artifacts; std::unique_ptr prepared; FunctionAuxData aux; AnalysisResult result; StackSize allocaLargeThreshold = 0; + TraversalEstimate traversalEstimate; + std::vector stepStats; PipelineData(llvm::Module& module, const AnalysisConfig& cfg) : mod(module), config(cfg) { @@ -54,7 +201,22 @@ namespace ctrace::stack::analyzer { const char* label; std::function run; + ArtifactMask requiredArtifacts = maskOf(ArtifactId::None); + ArtifactMask producedArtifacts = maskOf(ArtifactId::None); + bool contributesFullTraversalEstimate = false; + ExecutionModel executionModel = ExecutionModel::Utility; + std::uint8_t reservedPadding[6] = {}; }; + + static PipelineStep* findStep(std::vector& steps, std::string_view label) + { + for (PipelineStep& step : steps) + { + if (std::string_view(step.label) == label) + return &step; + } + return nullptr; + } } // namespace AnalysisPipeline::AnalysisPipeline(const AnalysisConfig& config) : config_(config) {} @@ -64,27 +226,84 @@ namespace ctrace::stack::analyzer using Clock = std::chrono::steady_clock; PipelineData data(mod, config_); - - auto logDuration = [&](const char* label, Clock::time_point start) - { - if (!config_.timing) - return; - const auto end = Clock::now(); - const auto ms = - std::chrono::duration_cast(end - start).count(); - std::cerr << label << " done in " << ms << " ms\n"; - }; + const ScopedHotspot pipelineHotspot(config_.timing, "pipeline.total"); + const bool subscribersEnabled = usePipelineSubscribers(); std::vector steps; steps.push_back({"Function attrs pass", [](const PipelineData& state) { runFunctionAttrsPass(state.mod); }}); - steps.push_back({"Prepare module", [](PipelineData& state) - { - ModulePreparationService preparationService; - state.prepared = std::make_unique( - preparationService.prepare(state.mod, state.config)); - }}); + steps.push_back( + {"Prepare module", [](PipelineData& state) + { + ModulePreparationService preparationService; + state.prepared = std::make_unique( + preparationService.prepare(state.mod, state.config)); + state.artifacts.set(state.prepared.get()); + state.artifacts.set( + &state.prepared->derivedArtifacts); + + if (state.config.timing) + { + const DerivedModuleArtifacts& derived = state.prepared->derivedArtifacts; + if (!derived.hasCompatibleSchema()) + { + std::cerr << "Derived artifacts schema mismatch: expected " + << DerivedModuleArtifacts::schemaKey() << ", got version " + << derived.schemaVersion << "\n"; + } + std::cerr << "Derived artifacts schema: " + << DerivedModuleArtifacts::schemaKey() << "\n"; + std::cerr << "Derived artifacts: debug_functions=" + << derived.debugIndex.allDefinedFunctionsWithSubprogram + << ", selected_debug_functions=" + << derived.debugIndex.selectedFunctionsWithSubprogram + << ", source_files=" << derived.debugIndex.distinctSourceFiles + << ", symbols=" << derived.symbolIndex.distinctMangledNames + << ", ptr_params=" << derived.typeFacts.pointerParameterCount + << ", aggregate_params=" << derived.typeFacts.aggregateParameterCount + << "\n"; + } + }}); + + steps.push_back( + {"Collect IR facts", [subscribersEnabled](PipelineData& state) + { + IRFacts facts; + PipelineSubscriberSignals signals; + + if (subscribersEnabled) + { + InstructionSubscriberRegistry registry; + PipelineSignalSubscriber signalSubscriber(signals); + registry.add(signalSubscriber); + PerFunctionInstructionCache instCache; + registry.add(instCache); + facts = collectIRFacts(state.prepared->ctx, ®istry); + state.artifacts.set(std::move(instCache)); + } + else + { + facts = collectIRFacts(state.prepared->ctx); + signals = derivePipelineSignals(facts); + } + + state.artifacts.set(facts); + state.artifacts.set(signals); + + if (state.config.timing) + { + std::cerr << "IR facts mode: " + << (subscribersEnabled ? "subscriber" : "direct") << "\n"; + std::cerr << "IR facts: selected funcs=" << facts.selectedFunctionCount + << ", selected BB=" << facts.basicBlockCountSelected + << ", selected inst=" << facts.instructionCountSelected + << ", alloca=" << facts.allocaInstCount + << ", loads=" << facts.loadInstCount + << ", stores=" << facts.storeInstCount + << ", memintrinsics=" << facts.memIntrinsicCount << "\n"; + } + }}); steps.push_back({"Build results", [](PipelineData& state) { state.result = buildResults(*state.prepared, state.aux); }}); @@ -96,26 +315,55 @@ namespace ctrace::stack::analyzer { state.allocaLargeThreshold = analysis::computeAllocaLargeThreshold(state.config); + state.artifacts.set(state.allocaLargeThreshold); }}); - steps.push_back({"Stack buffer overflows", [](PipelineData& state) - { - auto shouldAnalyze = [&](const llvm::Function& F) -> bool - { return state.prepared->ctx.shouldAnalyze(F); }; - const std::vector issues = - analysis::analyzeStackBufferOverflows(state.mod, shouldAnalyze, - state.config); - appendStackBufferDiagnostics(state.result, issues); - }}); + steps.push_back( + {"Stack buffer overflows", [](PipelineData& state) + { + if (const auto* signals = state.artifacts.get()) + { + const bool noBufferRelevantInsts = signals->bufferRelevantCount == 0; + if (noBufferRelevantInsts) + { + if (state.config.timing) + std::cerr << "Stack buffer overflows skipped: no relevant " + "alloca/store/memintrinsic\n"; + return; + } + } - steps.push_back({"Dynamic allocas", [](PipelineData& state) - { - auto shouldAnalyze = [&](const llvm::Function& F) -> bool - { return state.prepared->ctx.shouldAnalyze(F); }; - const std::vector issues = - analysis::analyzeDynamicAllocas(state.mod, shouldAnalyze); - appendDynamicAllocaDiagnostics(state.result, issues); - }}); + auto shouldAnalyze = [&](const llvm::Function& F) -> bool + { return state.prepared->ctx.shouldAnalyze(F); }; + const std::vector issues = + analysis::analyzeStackBufferOverflows(state.mod, shouldAnalyze, state.config); + appendStackBufferDiagnostics(state.result, issues); + }}); + + steps.push_back( + {"Dynamic allocas", [](PipelineData& state) + { + if (const auto* cache = state.artifacts.get()) + { + std::vector issues; + for (const auto& [func, data] : cache->data()) + { + if (!func || func->isDeclaration()) + continue; + auto funcIssues = + analysis::analyzeDynamicAllocasCached(*func, data.allocas); + issues.insert(issues.end(), funcIssues.begin(), funcIssues.end()); + } + appendDynamicAllocaDiagnostics(state.result, issues); + return; + } + + auto shouldAnalyze = [&](const llvm::Function& F) -> bool + { return state.prepared->ctx.shouldAnalyze(F); }; + const std::vector issues = + analysis::analyzeDynamicAllocas(state.mod, shouldAnalyze); + appendDynamicAllocaDiagnostics(state.result, issues); + }}); steps.push_back( {"Alloca usage", [](PipelineData& state) @@ -131,17 +379,52 @@ namespace ctrace::stack::analyzer state.allocaLargeThreshold, issues); }}); - steps.push_back({"Mem intrinsic overflows", [](PipelineData& state) + steps.push_back( + {"Mem intrinsic overflows", [](PipelineData& state) + { + if (const auto* cache = state.artifacts.get()) + { + const llvm::DataLayout& dataLayout = *state.prepared->ctx.dataLayout; + + // Parse model once for all functions. + analysis::BufferWriteModel externalModel; + analysis::BufferWriteRuleMatcher ruleMatcher; + const analysis::BufferWriteModel* modelPtr = nullptr; + if (!state.config.bufferModelPath.empty()) + { + std::string parseError; + if (analysis::parseBufferWriteModel(state.config.bufferModelPath, + externalModel, parseError)) { - auto shouldAnalyze = [&](const llvm::Function& F) -> bool - { return state.prepared->ctx.shouldAnalyze(F); }; - const llvm::DataLayout& dataLayout = *state.prepared->ctx.dataLayout; - const std::vector issues = - analysis::analyzeMemIntrinsicOverflows( - state.mod, dataLayout, shouldAnalyze, - state.config.bufferModelPath); - appendMemIntrinsicDiagnostics(state.result, issues); - }}); + modelPtr = &externalModel; + } + else + { + std::cerr << "Buffer model load error: " << parseError << "\n"; + } + } + + std::vector issues; + for (const auto& [func, data] : cache->data()) + { + if (!func || func->isDeclaration()) + continue; + auto funcIssues = analysis::analyzeMemIntrinsicOverflowsCached( + *func, dataLayout, data.calls, data.invokes, modelPtr, &ruleMatcher); + issues.insert(issues.end(), funcIssues.begin(), funcIssues.end()); + } + appendMemIntrinsicDiagnostics(state.result, issues); + return; + } + + auto shouldAnalyze = [&](const llvm::Function& F) -> bool + { return state.prepared->ctx.shouldAnalyze(F); }; + const llvm::DataLayout& dataLayout = *state.prepared->ctx.dataLayout; + const std::vector issues = + analysis::analyzeMemIntrinsicOverflows(state.mod, dataLayout, shouldAnalyze, + state.config.bufferModelPath); + appendMemIntrinsicDiagnostics(state.result, issues); + }}); steps.push_back({"Integer overflows", [](PipelineData& state) { @@ -255,23 +538,55 @@ namespace ctrace::stack::analyzer appendOOBReadDiagnostics(state.result, issues); }}); - steps.push_back({"Command injection", [](PipelineData& state) - { - auto shouldAnalyze = [&](const llvm::Function& F) -> bool - { return state.prepared->ctx.shouldAnalyze(F); }; - const std::vector issues = - analysis::analyzeCommandInjection(state.mod, shouldAnalyze); - appendCommandInjectionDiagnostics(state.result, issues); - }}); + steps.push_back( + {"Command injection", [](PipelineData& state) + { + if (const auto* cache = state.artifacts.get()) + { + std::vector issues; + for (const auto& [func, data] : cache->data()) + { + if (!func || func->isDeclaration()) + continue; + auto funcIssues = analysis::analyzeCommandInjectionCached( + *func, data.calls, data.invokes); + issues.insert(issues.end(), funcIssues.begin(), funcIssues.end()); + } + appendCommandInjectionDiagnostics(state.result, issues); + return; + } - steps.push_back({"TOCTOU", [](PipelineData& state) - { - auto shouldAnalyze = [&](const llvm::Function& F) -> bool - { return state.prepared->ctx.shouldAnalyze(F); }; - const std::vector issues = - analysis::analyzeTOCTOU(state.mod, shouldAnalyze); - appendTOCTOUDiagnostics(state.result, issues); - }}); + auto shouldAnalyze = [&](const llvm::Function& F) -> bool + { return state.prepared->ctx.shouldAnalyze(F); }; + const std::vector issues = + analysis::analyzeCommandInjection(state.mod, shouldAnalyze); + appendCommandInjectionDiagnostics(state.result, issues); + }}); + + steps.push_back( + {"TOCTOU", [](PipelineData& state) + { + if (const auto* cache = state.artifacts.get()) + { + std::vector issues; + for (const auto& [func, data] : cache->data()) + { + if (!func || func->isDeclaration()) + continue; + auto funcIssues = + analysis::analyzeTOCTOUCached(*func, data.calls, data.invokes); + issues.insert(issues.end(), funcIssues.begin(), funcIssues.end()); + } + appendTOCTOUDiagnostics(state.result, issues); + return; + } + + auto shouldAnalyze = [&](const llvm::Function& F) -> bool + { return state.prepared->ctx.shouldAnalyze(F); }; + const std::vector issues = + analysis::analyzeTOCTOU(state.mod, shouldAnalyze); + appendTOCTOUDiagnostics(state.result, issues); + }}); steps.push_back({"Type confusion", [](PipelineData& state) { @@ -284,22 +599,169 @@ namespace ctrace::stack::analyzer appendTypeConfusionDiagnostics(state.result, issues); }}); - steps.push_back({"Resource lifetime", [](PipelineData& state) - { - auto shouldAnalyze = [&](const llvm::Function& F) -> bool - { return state.prepared->ctx.shouldAnalyze(F); }; - const std::vector issues = - analysis::analyzeResourceLifetime( - state.mod, shouldAnalyze, state.config.resourceModelPath, - state.config.resourceSummaryIndex.get()); - appendResourceLifetimeDiagnostics(state.result, issues); - }}); + steps.push_back( + {"Resource lifetime", [](PipelineData& state) + { + if (const auto* signals = state.artifacts.get()) + { + if (signals->callSiteCount == 0) + { + if (state.config.timing) + std::cerr << "Resource lifetime skipped: no call sites\n"; + return; + } + } + + auto shouldAnalyze = [&](const llvm::Function& F) -> bool + { return state.prepared->ctx.shouldAnalyze(F); }; + const std::vector issues = + analysis::analyzeResourceLifetime(state.mod, shouldAnalyze, + state.config.resourceModelPath, + state.config.resourceSummaryIndex.get()); + appendResourceLifetimeDiagnostics(state.result, issues); + }}); + + const ArtifactMask kNone = maskOf(ArtifactId::None); + const ArtifactMask kPrepared = maskOf(ArtifactId::PreparedModule); + const ArtifactMask kIRFacts = maskOf(ArtifactId::IRFacts); + const ArtifactMask kAllocaThreshold = maskOf(ArtifactId::AllocaLargeThreshold); + const ArtifactMask kPipelineSignals = maskOf(ArtifactId::PipelineSubscriberSignals); + const ArtifactMask kDerivedArtifacts = maskOf(ArtifactId::DerivedModuleArtifacts); + + auto setStepMeta = [&](std::string_view label, ArtifactMask requiredMask, + ArtifactMask providedMask, bool traversalEstimate, + ExecutionModel executionModel) + { + if (PipelineStep* step = findStep(steps, label)) + { + step->requiredArtifacts = requiredMask; + step->producedArtifacts = providedMask; + step->contributesFullTraversalEstimate = traversalEstimate; + step->executionModel = executionModel; + } + }; + setStepMeta("Prepare module", kNone, kPrepared | kDerivedArtifacts, false, + ExecutionModel::Utility); + setStepMeta("Collect IR facts", kPrepared, kIRFacts | kPipelineSignals, true, + ExecutionModel::Utility); + setStepMeta("Build results", kPrepared, kNone, false, ExecutionModel::Utility); + setStepMeta("Emit summary diagnostics", kPrepared, kNone, false, ExecutionModel::Utility); + setStepMeta("Compute alloca threshold", kNone, kAllocaThreshold, false, + ExecutionModel::Utility); + + setStepMeta("Stack buffer overflows", kPrepared | kPipelineSignals, kNone, true, + ExecutionModel::SubscriberCompatible); + setStepMeta("Dynamic allocas", kPrepared, kNone, true, + ExecutionModel::SubscriberCompatible); + setStepMeta("Alloca usage", kPrepared | kAllocaThreshold, kNone, true, + ExecutionModel::Independent); + setStepMeta("Mem intrinsic overflows", kPrepared, kNone, true, + ExecutionModel::SubscriberCompatible); + setStepMeta("Integer overflows", kPrepared, kNone, true, ExecutionModel::Independent); + setStepMeta("Size-minus-k writes", kPrepared, kNone, true, ExecutionModel::Independent); + setStepMeta("Multiple stores", kPrepared, kNone, true, + ExecutionModel::SubscriberCompatible); + setStepMeta("Duplicate if conditions", kPrepared, kNone, true, + ExecutionModel::SubscriberCompatible); + setStepMeta("Uninitialized local reads", kPrepared, kNone, true, + ExecutionModel::Independent); + setStepMeta("Global reads before writes", kPrepared, kNone, true, + ExecutionModel::Independent); + setStepMeta("Invalid base reconstructions", kPrepared, kNone, true, + ExecutionModel::SubscriberCompatible); + setStepMeta("Stack pointer escapes", kPrepared, kNone, true, ExecutionModel::Independent); + setStepMeta("Const params", kPrepared, kNone, true, ExecutionModel::SubscriberCompatible); + setStepMeta("Null pointer dereferences", kPrepared, kNone, true, + ExecutionModel::Independent); + setStepMeta("Out-of-bounds reads", kPrepared, kNone, true, ExecutionModel::Independent); + setStepMeta("Command injection", kPrepared, kNone, true, + ExecutionModel::SubscriberCompatible); + setStepMeta("TOCTOU", kPrepared, kNone, true, ExecutionModel::SubscriberCompatible); + setStepMeta("Type confusion", kPrepared, kNone, true, ExecutionModel::SubscriberCompatible); + setStepMeta("Resource lifetime", kPrepared | kPipelineSignals, kNone, true, + ExecutionModel::Independent); + + ArtifactMask availableArtifacts = kNone; for (const PipelineStep& step : steps) { + if ((availableArtifacts & step.requiredArtifacts) != step.requiredArtifacts) + { + std::cerr << "Pipeline dependency violation before step '" << step.label + << "': required artifacts are missing\n"; + return AnalysisResult{config_, {}, {}}; + } + const auto start = Clock::now(); step.run(data); - logDuration(step.label, start); + const auto end = Clock::now(); + const auto elapsed = end - start; + const auto durationMs = + std::chrono::duration_cast(elapsed).count(); + if (config_.timing) + { + const std::string hotspotName = std::string("pipeline.step.") + step.label; + HotspotProfiler::record( + hotspotName, std::chrono::duration_cast(elapsed)); + std::cerr << step.label << " done in " << durationMs << " ms\n"; + } + availableArtifacts |= step.producedArtifacts; + + StepTraversalStats stats; + stats.label = step.label; + stats.executionModel = static_cast(step.executionModel); + stats.durationMs = durationMs; + if (step.contributesFullTraversalEstimate) + { + if (const auto* facts = data.artifacts.get()) + { + stats.moduleVisits = 1; + stats.functionVisits = facts->selectedFunctionCount; + stats.instructionVisits = facts->instructionCountSelected; + ++data.traversalEstimate.fullTraversalPasses; + data.traversalEstimate.estimatedInstructionVisits += + facts->instructionCountAllDefined; + } + } + data.stepStats.push_back(std::move(stats)); + } + + if (config_.timing) + { + std::cerr << "Traversal estimate: full-traversal passes=" + << data.traversalEstimate.fullTraversalPasses + << ", estimated instruction visits=" + << data.traversalEstimate.estimatedInstructionVisits << "\n"; + + std::uint64_t utilityInstructionVisits = 0; + std::uint64_t subscriberInstructionVisits = 0; + std::uint64_t independentInstructionVisits = 0; + for (const StepTraversalStats& stats : data.stepStats) + { + std::cerr << "Traversal estimate detail: step='" << stats.label << "', model=" + << executionModelName(static_cast(stats.executionModel)) + << ", modules=" << stats.moduleVisits + << ", functions=" << stats.functionVisits + << ", instructions=" << stats.instructionVisits + << ", duration_ms=" << stats.durationMs << "\n"; + + switch (static_cast(stats.executionModel)) + { + case ExecutionModel::Utility: + utilityInstructionVisits += stats.instructionVisits; + break; + case ExecutionModel::SubscriberCompatible: + subscriberInstructionVisits += stats.instructionVisits; + break; + case ExecutionModel::Independent: + independentInstructionVisits += stats.instructionVisits; + break; + } + } + + std::cerr << "Traversal estimate by model: utility=" << utilityInstructionVisits + << ", subscriber-compatible=" << subscriberInstructionVisits + << ", independent=" << independentInstructionVisits << "\n"; } return data.result; diff --git a/src/analyzer/HotspotProfiler.cpp b/src/analyzer/HotspotProfiler.cpp new file mode 100644 index 0000000..ce05b4e --- /dev/null +++ b/src/analyzer/HotspotProfiler.cpp @@ -0,0 +1,167 @@ +#include "analyzer/HotspotProfiler.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ctrace::stack::analyzer +{ + namespace + { + struct HotspotStorage + { + std::mutex mutex; + std::unordered_map samples; + }; + + struct NamedHotspotSample + { + std::string name; + HotspotSample sample; + }; + + static HotspotStorage& hotspotStorage() + { + static HotspotStorage storage; + return storage; + } + + static std::size_t parsePositiveSize(const char* rawValue, std::size_t fallback) + { + if (!rawValue || *rawValue == '\0') + return fallback; + + std::uint64_t parsed = 0; + for (const char* p = rawValue; *p != '\0'; ++p) + { + const unsigned char ch = static_cast(*p); + if (!std::isdigit(ch)) + return fallback; + parsed = parsed * 10u + static_cast(ch - '0'); + if (parsed > static_cast(std::numeric_limits::max())) + return fallback; + } + + if (parsed == 0) + return fallback; + return static_cast(parsed); + } + + static std::size_t hotspotTopCountFromEnv() + { + constexpr std::size_t kDefaultTopCount = 20; + constexpr std::size_t kMaxTopCount = 200; + + const std::size_t parsed = + parsePositiveSize(std::getenv("CTRACE_HOTSPOT_TOP"), kDefaultTopCount); + return std::min(parsed, kMaxTopCount); + } + + static double nsToMs(std::uint64_t ns) + { + return static_cast(ns) / 1000000.0; + } + } // namespace + + void HotspotProfiler::record(std::string_view name, std::chrono::nanoseconds elapsed) + { + if (name.empty()) + return; + + const std::uint64_t elapsedNs = static_cast(elapsed.count()); + HotspotStorage& storage = hotspotStorage(); + std::lock_guard lock(storage.mutex); + HotspotSample& sample = storage.samples[std::string(name)]; + ++sample.calls; + sample.totalNs += elapsedNs; + sample.maxNs = std::max(sample.maxNs, elapsedNs); + } + + void HotspotProfiler::dumpTop(std::ostream& os, std::size_t topN) + { + if (topN == 0) + topN = 1; + + std::vector rows; + std::uint64_t trackedTotalNs = 0; + { + HotspotStorage& storage = hotspotStorage(); + std::lock_guard lock(storage.mutex); + rows.reserve(storage.samples.size()); + for (const auto& kv : storage.samples) + { + rows.push_back(NamedHotspotSample{kv.first, kv.second}); + trackedTotalNs += kv.second.totalNs; + } + } + + std::sort(rows.begin(), rows.end(), + [](const NamedHotspotSample& lhs, const NamedHotspotSample& rhs) + { + if (lhs.sample.totalNs != rhs.sample.totalNs) + return lhs.sample.totalNs > rhs.sample.totalNs; + return lhs.name < rhs.name; + }); + + const std::size_t count = std::min(topN, rows.size()); + os << "Hotspot summary (top " << count << " / " << rows.size() + << ", tracked_total_ms=" << std::fixed << std::setprecision(3) << nsToMs(trackedTotalNs) + << "):\n"; + + if (rows.empty()) + return; + + for (std::size_t idx = 0; idx < count; ++idx) + { + const NamedHotspotSample& row = rows[idx]; + const double totalMs = nsToMs(row.sample.totalNs); + const double avgMs = + row.sample.calls == 0 ? 0.0 : totalMs / static_cast(row.sample.calls); + const double maxMs = nsToMs(row.sample.maxNs); + const double share = trackedTotalNs == 0 + ? 0.0 + : (100.0 * static_cast(row.sample.totalNs) / + static_cast(trackedTotalNs)); + os << " " << (idx + 1) << ". " << row.name << " total_ms=" << std::fixed + << std::setprecision(3) << totalMs << " avg_ms=" << avgMs << " max_ms=" << maxMs + << " calls=" << row.sample.calls << " share_pct=" << share << "\n"; + } + } + + void HotspotProfiler::dumpTop(std::ostream& os) + { + dumpTop(os, hotspotTopCountFromEnv()); + } + + ScopedHotspot::ScopedHotspot(bool enabled, std::string_view name) + : name_(name), enabled_(enabled ? 1u : 0u) + { + if (enabled_ != 0) + start_ = std::chrono::steady_clock::now(); + } + + ScopedHotspot::~ScopedHotspot() + { + if (enabled_ == 0) + return; + const auto end = std::chrono::steady_clock::now(); + HotspotProfiler::record(name_, + std::chrono::duration_cast(end - start_)); + } + + void dumpHotspotSummary(std::ostream& os, bool enabled) + { + if (!enabled) + return; + HotspotProfiler::dumpTop(os); + } + +} // namespace ctrace::stack::analyzer diff --git a/src/analyzer/IRFactCollector.cpp b/src/analyzer/IRFactCollector.cpp new file mode 100644 index 0000000..a09007f --- /dev/null +++ b/src/analyzer/IRFactCollector.cpp @@ -0,0 +1,88 @@ +#include "analyzer/IRFactCollector.hpp" + +#include "analyzer/InstructionSubscriber.hpp" +#include "analyzer/ModulePreparationService.hpp" + +#include +#include +#include + +namespace ctrace::stack::analyzer +{ + IRFacts collectIRFacts(const ModuleAnalysisContext& ctx, + const InstructionSubscriberRegistry* subscribers) + { + IRFacts facts; + facts.allDefinedFunctionCount = static_cast(ctx.allDefinedFunctions.size()); + facts.selectedFunctionCount = static_cast(ctx.functions.size()); + + for (const llvm::Function* function : ctx.allDefinedFunctions) + { + if (!function) + continue; + + const bool selected = ctx.shouldAnalyze(*function); + if (selected && subscribers) + subscribers->notifyFunctionBegin(*function); + for (const llvm::BasicBlock& block : *function) + { + ++facts.basicBlockCountAllDefined; + if (selected) + ++facts.basicBlockCountSelected; + + for (const llvm::Instruction& instruction : block) + { + ++facts.instructionCountAllDefined; + if (selected) + ++facts.instructionCountSelected; + + if (!selected) + continue; + + if (const auto* call = llvm::dyn_cast(&instruction)) + { + ++facts.callInstCount; + if (subscribers) + subscribers->notifyCall(*call); + } + if (const auto* invoke = llvm::dyn_cast(&instruction)) + { + ++facts.invokeInstCount; + if (subscribers) + subscribers->notifyInvoke(*invoke); + } + if (const auto* alloca = llvm::dyn_cast(&instruction)) + { + ++facts.allocaInstCount; + if (subscribers) + subscribers->notifyAlloca(*alloca); + } + if (const auto* load = llvm::dyn_cast(&instruction)) + { + ++facts.loadInstCount; + if (subscribers) + subscribers->notifyLoad(*load); + } + if (const auto* store = llvm::dyn_cast(&instruction)) + { + ++facts.storeInstCount; + if (subscribers) + subscribers->notifyStore(*store); + } + if (const auto* memIntrinsic = llvm::dyn_cast(&instruction)) + { + ++facts.memIntrinsicCount; + if (subscribers) + subscribers->notifyMemIntrinsic(*memIntrinsic); + } + if (instruction.getDebugLoc()) + ++facts.debugLocCount; + } + } + if (selected && subscribers) + subscribers->notifyFunctionEnd(*function); + } + + return facts; + } +} // namespace ctrace::stack::analyzer diff --git a/src/analyzer/ModulePreparationService.cpp b/src/analyzer/ModulePreparationService.cpp index ce243b9..23678cf 100644 --- a/src/analyzer/ModulePreparationService.cpp +++ b/src/analyzer/ModulePreparationService.cpp @@ -1,11 +1,14 @@ #include "analyzer/ModulePreparationService.hpp" +#include "analyzer/HotspotProfiler.hpp" #include "analysis/FunctionFilter.hpp" #include +#include #include #include #include +#include namespace ctrace::stack::analyzer { @@ -48,6 +51,81 @@ namespace ctrace::stack::analyzer return localStack; } + static std::string debugSourcePath(const llvm::Function& F) + { + const llvm::DISubprogram* subprogram = F.getSubprogram(); + if (!subprogram) + return {}; + + if (const llvm::DIFile* file = subprogram->getFile()) + { + const std::string dir = file->getDirectory().str(); + const std::string filename = file->getFilename().str(); + if (filename.empty()) + return {}; + if (dir.empty()) + return filename; + return dir + "/" + filename; + } + + const std::string filename = subprogram->getFilename().str(); + if (!filename.empty()) + return filename; + return {}; + } + + static DerivedModuleArtifacts buildDerivedArtifacts(const ModuleAnalysisContext& ctx) + { + DerivedModuleArtifacts artifacts; + auto& debugIndex = artifacts.debugIndex; + auto& symbolIndex = artifacts.symbolIndex; + auto& typeFacts = artifacts.typeFacts; + + symbolIndex.totalDefinedFunctions = + static_cast(ctx.allDefinedFunctions.size()); + symbolIndex.mangledNameFrequency.reserve(ctx.allDefinedFunctions.size()); + + for (const llvm::Function* function : ctx.allDefinedFunctions) + { + if (!function) + continue; + + ++symbolIndex.mangledNameFrequency[function->getName().str()]; + + if (function->getSubprogram()) + { + ++debugIndex.allDefinedFunctionsWithSubprogram; + if (ctx.shouldAnalyze(*function)) + ++debugIndex.selectedFunctionsWithSubprogram; + + const std::string sourcePath = debugSourcePath(*function); + if (!sourcePath.empty()) + ++debugIndex.functionsPerSourceFile[sourcePath]; + } + + const llvm::FunctionType* functionType = function->getFunctionType(); + if (functionType->getReturnType()->isPointerTy()) + ++typeFacts.pointerReturnFunctionCount; + if (functionType->getReturnType()->isAggregateType()) + ++typeFacts.aggregateReturnFunctionCount; + + for (const llvm::Argument& arg : function->args()) + { + const llvm::Type* argType = arg.getType(); + if (argType->isPointerTy()) + ++typeFacts.pointerParameterCount; + if (argType->isAggregateType()) + ++typeFacts.aggregateParameterCount; + } + } + + symbolIndex.distinctMangledNames = + static_cast(symbolIndex.mangledNameFrequency.size()); + debugIndex.distinctSourceFiles = + static_cast(debugIndex.functionsPerSourceFile.size()); + return artifacts; + } + static analysis::CallGraph buildCallGraphFiltered(const ModuleAnalysisContext& ctx) { analysis::CallGraph graph; @@ -109,14 +187,41 @@ namespace ctrace::stack::analyzer PreparedModule ModulePreparationService::prepare(llvm::Module& mod, const AnalysisConfig& config) const { - ModuleAnalysisContext ctx = buildContext(mod, config); - LocalStackMap localStack = computeLocalStacks(ctx); - analysis::CallGraph callGraph = buildCallGraphFiltered(ctx); - analysis::InternalAnalysisState recursionState = - computeRecursionState(ctx, callGraph, localStack); - - return PreparedModule{std::move(ctx), std::move(localStack), std::move(callGraph), - std::move(recursionState)}; + const bool timingEnabled = config.timing; + const ScopedHotspot totalHotspot(timingEnabled, "prepare.total"); + + ModuleAnalysisContext ctx = [&]() + { + const ScopedHotspot hotspot(timingEnabled, "prepare.build_context"); + return buildContext(mod, config); + }(); + + DerivedModuleArtifacts derivedArtifacts = [&]() + { + const ScopedHotspot hotspot(timingEnabled, "prepare.derived_artifacts"); + return buildDerivedArtifacts(ctx); + }(); + + LocalStackMap localStack = [&]() + { + const ScopedHotspot hotspot(timingEnabled, "prepare.local_stacks"); + return computeLocalStacks(ctx); + }(); + + analysis::CallGraph callGraph = [&]() + { + const ScopedHotspot hotspot(timingEnabled, "prepare.call_graph"); + return buildCallGraphFiltered(ctx); + }(); + + analysis::InternalAnalysisState recursionState = [&]() + { + const ScopedHotspot hotspot(timingEnabled, "prepare.recursion_state"); + return computeRecursionState(ctx, callGraph, localStack); + }(); + + return PreparedModule{std::move(ctx), std::move(derivedArtifacts), std::move(localStack), + std::move(callGraph), std::move(recursionState)}; } } // namespace ctrace::stack::analyzer diff --git a/src/analyzer/PerFunctionInstructionCache.cpp b/src/analyzer/PerFunctionInstructionCache.cpp new file mode 100644 index 0000000..6fc479b --- /dev/null +++ b/src/analyzer/PerFunctionInstructionCache.cpp @@ -0,0 +1,52 @@ +#include "analyzer/PerFunctionInstructionCache.hpp" + +#include +#include +#include + +namespace ctrace::stack::analyzer +{ + void PerFunctionInstructionCache::onFunctionBegin(const llvm::Function& F) + { + currentFunction_ = &F; + currentData_ = {}; + } + + void PerFunctionInstructionCache::onFunctionEnd(const llvm::Function& F) + { + if (currentFunction_ == &F) + data_.try_emplace(&F, std::move(currentData_)); + currentFunction_ = nullptr; + currentData_ = {}; + } + + void PerFunctionInstructionCache::onAlloca(const llvm::AllocaInst& inst) + { + currentData_.allocas.push_back(&inst); + } + + void PerFunctionInstructionCache::onLoad(const llvm::LoadInst& inst) + { + currentData_.loads.push_back(&inst); + } + + void PerFunctionInstructionCache::onStore(const llvm::StoreInst& inst) + { + currentData_.stores.push_back(&inst); + } + + void PerFunctionInstructionCache::onCall(const llvm::CallInst& inst) + { + currentData_.calls.push_back(&inst); + } + + void PerFunctionInstructionCache::onInvoke(const llvm::InvokeInst& inst) + { + currentData_.invokes.push_back(&inst); + } + + void PerFunctionInstructionCache::onMemIntrinsic(const llvm::MemIntrinsic& inst) + { + currentData_.memIntrinsics.push_back(&inst); + } +} // namespace ctrace::stack::analyzer diff --git a/src/app/AnalyzerApp.cpp b/src/app/AnalyzerApp.cpp index 6998ed9..2c2ea01 100644 --- a/src/app/AnalyzerApp.cpp +++ b/src/app/AnalyzerApp.cpp @@ -1,6 +1,7 @@ #include "app/AnalyzerApp.hpp" #include "StackUsageAnalyzer.hpp" +#include "analyzer/HotspotProfiler.hpp" #include "cli/ArgParser.hpp" #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -141,6 +143,144 @@ static constexpr std::size_t kDestructiveCacheLineBytes = static constexpr std::size_t kDestructiveCacheLineBytes = 64; #endif +// ── Tarjan SCC algorithm on module indices ── +// Used by both resource and uninit cross-TU loops to compute strongly +// connected components of the inter-module call graph. +struct ModuleTarjan +{ + std::vector index; + std::vector lowlink; + std::vector onStack; + std::vector stack; + std::vector> sccs; + int nextIndex = 0; + + void run(std::size_t N, const std::vector>& edges) + { + index.assign(N, -1); + lowlink.assign(N, -1); + onStack.assign(N, false); + stack.reserve(N); + for (std::size_t v = 0; v < N; ++v) + { + if (index[v] < 0) + strongConnect(v, edges); + } + } + + void strongConnect(std::size_t v, const std::vector>& edges) + { + index[v] = lowlink[v] = nextIndex++; + stack.push_back(v); + onStack[v] = true; + + for (std::size_t w : edges[v]) + { + if (index[w] < 0) + { + strongConnect(w, edges); + lowlink[v] = std::min(lowlink[v], lowlink[w]); + } + else if (onStack[w]) + { + lowlink[v] = std::min(lowlink[v], index[w]); + } + } + + if (lowlink[v] == index[v]) + { + std::vector component; + std::size_t w; + do + { + w = stack.back(); + stack.pop_back(); + onStack[w] = false; + component.push_back(w); + } while (w != v); + // Sort for deterministic output. + std::sort(component.begin(), component.end()); + sccs.push_back(std::move(component)); + } + } +}; + +// Build the single-def filtered inter-module edge graph. +// Only edges through functions with exactly one definition across all modules +// are included. Multi-def functions (inline, template, weak) are excluded +// because their summaries are identical in all TUs and don't create real +// cross-module data dependencies. +static std::vector> buildSingleDefFilteredEdges( + std::size_t N, const std::vector>& moduleCalleeNames, + const std::unordered_map>& definedBy) +{ + std::vector> edges(N); + for (std::size_t i = 0; i < N; ++i) + { + for (const std::string& callee : moduleCalleeNames[i]) + { + auto it = definedBy.find(callee); + if (it == definedBy.end() || it->second.size() != 1) + continue; // skip multi-def and unresolved + const std::size_t j = it->second[0]; + if (j != i) + edges[i].insert(j); + } + } + return edges; +} + +// Run Tarjan on the filtered edges and return SCCs in dependency order +// (callees before callers). +// With edge direction caller→callee, Tarjan's DFS reaches callees first +// and emits their SCCs before caller SCCs. This is already the correct +// processing order — no reversal needed. +static std::vector> +computeTopologicalSCCOrder(std::size_t N, + const std::vector>& filteredEdges) +{ + ModuleTarjan tarjan; + tarjan.run(N, filteredEdges); + return std::move(tarjan.sccs); +} + +// Compute topological levels for SCCs given the filtered edge graph. +// SCCs at the same level are independent and can be processed in parallel. +// Returns a vector of levels (one per SCC in sccOrder), plus fills +// levelGroups: levelGroups[level] = {indices into sccOrder}. +static std::vector +computeSCCLevels(const std::vector>& sccOrder, + const std::vector>& filteredEdges, std::size_t N) +{ + // Map each module index to its SCC index in sccOrder. + std::vector moduleToSCC(N); + for (std::size_t s = 0; s < sccOrder.size(); ++s) + for (std::size_t m : sccOrder[s]) + moduleToSCC[m] = s; + + std::vector level(sccOrder.size(), 0); + // Process in topological order: each SCC's level is 1 + max(predecessor levels). + for (std::size_t s = 0; s < sccOrder.size(); ++s) + { + unsigned maxPred = 0; + bool hasPred = false; + for (std::size_t m : sccOrder[s]) + { + for (std::size_t dep : filteredEdges[m]) + { + const std::size_t depSCC = moduleToSCC[dep]; + if (depSCC != s) + { + hasPred = true; + maxPred = std::max(maxPred, level[depSCC]); + } + } + } + level[s] = hasPred ? maxPred + 1 : 0; + } + return level; +} + template static void runParallelWork(std::size_t workItemCount, unsigned maxJobs, WorkFn&& workFn) { @@ -799,11 +939,14 @@ static AppStatus analyzeWithSharedModuleLoading(const std::vector& bool needsCrossTUGlobalReadBeforeWriteSummaries, std::vector& results) { + const analyzer::ScopedHotspot totalHotspot(cfg.timing, "app.shared_loading.total"); std::vector loadedModules(inputFilenames.size()); std::vector loadErrors(inputFilenames.size()); std::vector loadSucceeded(inputFilenames.size(), 0); auto loadSingleModule = [&](std::size_t index) { + const analyzer::ScopedHotspot moduleLoadHotspot(cfg.timing, + "app.shared_loading.load_module"); const std::string& inputFilename = inputFilenames[index]; auto moduleContext = std::make_unique(); llvm::SMDiagnostic localErr; @@ -863,18 +1006,31 @@ static AppStatus analyzeWithSharedModuleLoading(const std::vector& loadedModules.swap(orderedLoadedModules); if (needsCrossTUResourceSummaries) + { + const analyzer::ScopedHotspot hotspot(cfg.timing, "app.shared_loading.cross_tu_resource"); cfg.resourceSummaryIndex = buildCrossTUSummaryIndex(loadedModules, cfg); + } if (needsCrossTUUninitializedSummaries) + { + const analyzer::ScopedHotspot hotspot(cfg.timing, + "app.shared_loading.cross_tu_uninitialized"); cfg.uninitializedSummaryIndex = buildCrossTUUninitializedSummaryIndex(loadedModules, cfg); + } if (needsCrossTUGlobalReadBeforeWriteSummaries) { + const analyzer::ScopedHotspot hotspot(cfg.timing, + "app.shared_loading.cross_tu_global_read"); cfg.globalReadBeforeWriteSummaryIndex = buildCrossTUGlobalReadBeforeWriteSummaryIndex(loadedModules, cfg); } for (auto& loaded : loadedModules) { - AnalysisResult result = analyzeModule(*loaded.module, cfg); + AnalysisResult result; + { + const analyzer::ScopedHotspot hotspot(cfg.timing, "app.shared_loading.analyze_module"); + result = analyzeModule(*loaded.module, cfg); + } if (!loaded.frontendDiagnostics.empty()) { result.diagnostics.insert(result.diagnostics.end(), loaded.frontendDiagnostics.begin(), @@ -893,11 +1049,13 @@ static AppStatus analyzeWithoutSharedModuleLoading(const std::vector& results) { + const analyzer::ScopedHotspot totalHotspot(cfg.timing, "app.direct_loading.total"); const unsigned parallelJobs = resolveConfiguredJobs(cfg); if (parallelJobs <= 1 || inputFilenames.size() <= 1) { for (const auto& inputFilename : inputFilenames) { + const analyzer::ScopedHotspot fileHotspot(cfg.timing, "app.direct_loading.file"); llvm::LLVMContext localContext; llvm::SMDiagnostic localErr; analysis::ModuleLoadResult load = @@ -922,7 +1080,12 @@ static AppStatus analyzeWithoutSharedModuleLoading(const std::vector slots(inputFilenames.size()); - runParallelWork(inputFilenames.size(), parallelJobs, - [&](std::size_t index) - { - const std::string& inputFilename = inputFilenames[index]; - llvm::LLVMContext localContext; - llvm::SMDiagnostic localErr; - analysis::ModuleLoadResult load = analysis::loadModuleForAnalysis( - inputFilename, cfg, localContext, localErr); - if (!load.module) - { - std::string err; - if (!load.error.empty()) - err += load.error; - if (localErr.getLineNo() != 0 || !localErr.getFilename().empty()) - { - std::string diagText; - llvm::raw_string_ostream os(diagText); - localErr.print("stack_usage_analyzer", os); - os.flush(); - err += diagText; - } - slots[index].loadError = std::move(err); - return; - } + runParallelWork( + inputFilenames.size(), parallelJobs, + [&](std::size_t index) + { + const analyzer::ScopedHotspot fileHotspot(cfg.timing, "app.direct_loading.file"); + const std::string& inputFilename = inputFilenames[index]; + llvm::LLVMContext localContext; + llvm::SMDiagnostic localErr; + analysis::ModuleLoadResult load = + analysis::loadModuleForAnalysis(inputFilename, cfg, localContext, localErr); + if (!load.module) + { + std::string err; + if (!load.error.empty()) + err += load.error; + if (localErr.getLineNo() != 0 || !localErr.getFilename().empty()) + { + std::string diagText; + llvm::raw_string_ostream os(diagText); + localErr.print("stack_usage_analyzer", os); + os.flush(); + err += diagText; + } + slots[index].loadError = std::move(err); + return; + } - AnalysisResult result = analyzeModule(*load.module, cfg); - if (!load.frontendDiagnostics.empty()) - { - result.diagnostics.insert(result.diagnostics.end(), - load.frontendDiagnostics.begin(), - load.frontendDiagnostics.end()); - } - stampResultFilePaths(result, inputFilename); - slots[index].noFunctionMsg = - noFunctionMessage(result, inputFilename, hasFilter); - slots[index].result = std::make_unique(std::move(result)); - }); + AnalysisResult result; + { + const analyzer::ScopedHotspot hotspot(cfg.timing, + "app.direct_loading.analyze_module"); + result = analyzeModule(*load.module, cfg); + } + if (!load.frontendDiagnostics.empty()) + { + result.diagnostics.insert(result.diagnostics.end(), + load.frontendDiagnostics.begin(), + load.frontendDiagnostics.end()); + } + stampResultFilePaths(result, inputFilename); + slots[index].noFunctionMsg = noFunctionMessage(result, inputFilename, hasFilter); + slots[index].result = std::make_unique(std::move(result)); + }); for (std::size_t index = 0; index < inputFilenames.size(); ++index) { @@ -1441,6 +1610,7 @@ static std::shared_ptr buildCrossTUSummaryIndex(const std::vector& loadedModules, const AnalysisConfig& cfg) { + const analyzer::ScopedHotspot totalHotspot(cfg.timing, "app.cross_tu.resource_summary.total"); if (!cfg.resourceCrossTU || cfg.resourceModelPath.empty() || loadedModules.size() < 2) return nullptr; @@ -1474,40 +1644,155 @@ buildCrossTUSummaryIndex(const std::vector& loadedModules, moduleCompileArgsHashes.push_back(computeCompileArgsSignature(cfg, loaded.filename)); } - // Empirical safeguard: cross-TU summaries usually stabilize in a few rounds. - // Keep a bounded worst-case runtime on very large dependency graphs. + // Build inter-module dependency metadata for filtered dirty-marking. + // + // The naive dirty-check marks a module as dirty if *any* callee's summary + // changed. This is too sensitive because inline/template/weak functions + // (linkonce_odr, weak_odr, available_externally, internal) are emitted in + // every TU that uses them, creating O(N²) artificial cross-module edges. + // Their summaries converge identically in all TUs after the first iteration, + // so re-analyzing modules because of them is wasted work. + // + // Generic criterion: a callee represents a *real* inter-module dependency + // only if it has ExternalLinkage in its definition. This is a property of + // LLVM IR semantics, not specific to any project: + // - ExternalLinkage: unique strong definition, real cross-TU dependency. + // - LinkOnceODRLinkage: inline/template (C++), identical in all TUs. + // - WeakODRLinkage: weak symbols, COMDAT — same as ODR for our purposes. + // - AvailableExternallyLinkage: inlined copies kept for optimization. + // - InternalLinkage / PrivateLinkage: static functions, TU-local. + std::unordered_map> definedBy; + for (std::size_t i = 0; i < loadedModules.size(); ++i) + { + for (const llvm::Function& F : *loadedModules[i].module) + { + if (F.isDeclaration() || !F.hasName() || F.getName().empty()) + continue; + const std::string canon = ctrace_tools::canonicalizeMangledName(F.getName().str()); + definedBy[canon].push_back(i); + } + } + + // Build single-def name set: functions defined in exactly one module. + // This is the same criterion used by buildSingleDefFilteredEdges for + // SCC edge construction. Using it for dirty-marking ensures consistency: + // an SCC edge exists iff the corresponding callee can trigger dirty-marking. + std::unordered_set singleDefNames; + for (const auto& [name, modules] : definedBy) + { + if (modules.size() == 1) + singleDefNames.insert(name); + } + + // Pre-compute per-module callee name sets for delta-based convergence. + std::vector> resourceModuleCalleeNames(loadedModules.size()); + // Filtered version: only callees with single-def definitions. + // Consistent with SCC edge criterion (buildSingleDefFilteredEdges). + std::vector> filteredResourceCalleeNames(loadedModules.size()); + for (std::size_t i = 0; i < loadedModules.size(); ++i) + { + const LoadedInputModule& loaded = loadedModules[i]; + const analysis::FunctionFilter filter = analysis::buildFunctionFilter(*loaded.module, cfg); + for (const llvm::Function& F : *loaded.module) + { + if (F.isDeclaration() || !filter.shouldAnalyze(F)) + continue; + for (const llvm::BasicBlock& BB : F) + { + for (const llvm::Instruction& I : BB) + { + const auto* CB = llvm::dyn_cast(&I); + if (!CB) + continue; + const llvm::Function* callee = CB->getCalledFunction(); + if (!callee || !callee->hasName() || callee->getName().empty()) + continue; + const std::string canon = + ctrace_tools::canonicalizeMangledName(callee->getName().str()); + resourceModuleCalleeNames[i].insert(canon); + if (singleDefNames.count(canon)) + filteredResourceCalleeNames[i].insert(canon); + } + } + } + } + + // ── SCC worklist for resource cross-TU convergence ── constexpr unsigned kCrossTUMaxIterations = 12; + const std::size_t N = loadedModules.size(); + + const auto filteredEdges = buildSingleDefFilteredEdges(N, resourceModuleCalleeNames, definedBy); + const auto sccOrder = computeTopologicalSCCOrder(N, filteredEdges); + const auto sccLevels = computeSCCLevels(sccOrder, filteredEdges, N); + + unsigned maxLevel = 0; + for (unsigned lvl : sccLevels) + maxLevel = std::max(maxLevel, lvl); + + std::vector> levelGroups(maxLevel + 1); + for (std::size_t s = 0; s < sccOrder.size(); ++s) + levelGroups[sccLevels[s]].push_back(s); + + // Classify SCCs for logging. + std::size_t trivialSCCCount = 0, cyclicSCCCount = 0; + for (std::size_t s = 0; s < sccOrder.size(); ++s) + { + const auto& scc = sccOrder[s]; + if (scc.size() == 1 && !filteredEdges[scc[0]].count(scc[0])) + ++trivialSCCCount; + else + ++cyclicSCCCount; + } + + if (cfg.timing) + { + coretrace::log(coretrace::Level::Info, + "Cross-TU resource SCC worklist: {} SCCs ({} trivial, {} cyclic) " + "in {} levels\n", + sccOrder.size(), trivialSCCCount, cyclicSCCCount, maxLevel + 1); + } + ctrace::stack::analysis::ResourceSummaryIndex globalIndex; - unsigned iterationsRan = 0; - bool converged = false; - for (unsigned iter = 0; iter < kCrossTUMaxIterations; ++iter) + std::vector moduleSummaries(N); + std::size_t totalModuleAnalyses = 0; + + for (unsigned level = 0; level <= maxLevel; ++level) { - const auto iterStart = Clock::now(); + const auto& group = levelGroups[level]; + if (group.empty()) + continue; + + const auto levelStart = Clock::now(); const std::string externalHash = hashSummaryIndex(globalIndex); - ctrace::stack::analysis::ResourceSummaryIndex nextGlobal; - std::vector moduleSummaries( - loadedModules.size()); - std::vector summaryReady(loadedModules.size(), 0); - std::vector cacheKeys(loadedModules.size()); - std::vector missingIndices; - missingIndices.reserve(loadedModules.size()); - for (std::size_t moduleIndex = 0; moduleIndex < loadedModules.size(); ++moduleIndex) + auto buildModuleSummary = + [&](std::size_t moduleIndex) -> ctrace::stack::analysis::ResourceSummaryIndex + { + const analyzer::ScopedHotspot hotspot(cfg.timing, + "app.cross_tu.resource_summary.build_module"); + const LoadedInputModule& loaded = loadedModules[moduleIndex]; + analysis::FunctionFilter filter = analysis::buildFunctionFilter(*loaded.module, cfg); + auto shouldAnalyze = [&](const llvm::Function& F) -> bool + { return filter.shouldAnalyze(F); }; + return analysis::buildResourceLifetimeSummaryIndex(*loaded.module, shouldAnalyze, + cfg.resourceModelPath, &globalIndex); + }; + + // Try cache for each module at this level, collect modules that need building. + auto tryCacheForModule = [&](std::size_t moduleIndex) -> bool { const std::string cacheKeyPayload = std::string(kCacheSchema) + "|" + modelHash + "|" + externalHash + "|" + filterHash + "|" + moduleCompileArgsHashes[moduleIndex] + "|" + moduleIRHashes[moduleIndex]; const std::string cacheKey = md5Hex(cacheKeyPayload); - cacheKeys[moduleIndex] = cacheKey; - bool loadedFromCache = false; if (const auto memIt = memoryCache.find(cacheKey); memIt != memoryCache.end()) { moduleSummaries[moduleIndex] = memIt->second; - loadedFromCache = true; + return true; } - else if (allowDiskCache) + if (allowDiskCache) { const std::filesystem::path cacheFile = std::filesystem::path(cfg.resourceSummaryCacheDir) / (cacheKey + ".json"); @@ -1516,111 +1801,180 @@ buildCrossTUSummaryIndex(const std::vector& loadedModules, { moduleSummaries[moduleIndex] = std::move(*cached); memoryCache.emplace(cacheKey, moduleSummaries[moduleIndex]); - loadedFromCache = true; + return true; } } + return false; + }; - if (loadedFromCache) + auto cacheAndStoreModule = [&](std::size_t moduleIndex) + { + const std::string cacheKeyPayload = std::string(kCacheSchema) + "|" + modelHash + "|" + + externalHash + "|" + filterHash + "|" + + moduleCompileArgsHashes[moduleIndex] + "|" + + moduleIRHashes[moduleIndex]; + const std::string cacheKey = md5Hex(cacheKeyPayload); + memoryCache.emplace(cacheKey, moduleSummaries[moduleIndex]); + if (allowDiskCache) { - summaryReady[moduleIndex] = 1; + const std::filesystem::path cacheFile = + std::filesystem::path(cfg.resourceSummaryCacheDir) / (cacheKey + ".json"); + (void)writeSummaryCacheFile(cacheFile, moduleSummaries[moduleIndex]); } + }; + + // Collect trivial and cyclic SCCs. + std::vector trivialModules; + std::vector cyclicSCCIndices; + + for (std::size_t sccIdx : group) + { + const auto& scc = sccOrder[sccIdx]; + const bool isTrivial = scc.size() == 1 && !filteredEdges[scc[0]].count(scc[0]); + if (isTrivial) + trivialModules.push_back(scc[0]); else - { - missingIndices.push_back(moduleIndex); - } + cyclicSCCIndices.push_back(sccIdx); } - auto buildModuleSummary = - [&](std::size_t moduleIndex) -> ctrace::stack::analysis::ResourceSummaryIndex + // Process trivial SCCs: try cache, then build missing ones in parallel. + std::vector missingTrivial; + for (std::size_t moduleIndex : trivialModules) { - const LoadedInputModule& loaded = loadedModules[moduleIndex]; - analysis::FunctionFilter filter = analysis::buildFunctionFilter(*loaded.module, cfg); - auto shouldAnalyze = [&](const llvm::Function& F) -> bool - { return filter.shouldAnalyze(F); }; - return analysis::buildResourceLifetimeSummaryIndex(*loaded.module, shouldAnalyze, - cfg.resourceModelPath, &globalIndex); - }; + if (!tryCacheForModule(moduleIndex)) + missingTrivial.push_back(moduleIndex); + } - if (!missingIndices.empty()) + if (!missingTrivial.empty()) { - if (maxJobs <= 1 || missingIndices.size() <= 1) + if (maxJobs <= 1 || missingTrivial.size() <= 1) { - for (std::size_t moduleIndex : missingIndices) - { + for (std::size_t moduleIndex : missingTrivial) moduleSummaries[moduleIndex] = buildModuleSummary(moduleIndex); - summaryReady[moduleIndex] = 1; - } } else { - std::vector computed( - loadedModules.size()); - std::vector computedReady(loadedModules.size(), 0); - runParallelWork(missingIndices.size(), maxJobs, + runParallelWork(missingTrivial.size(), maxJobs, [&](std::size_t slot) { - const std::size_t moduleIndex = missingIndices[slot]; - computed[moduleIndex] = buildModuleSummary(moduleIndex); - computedReady[moduleIndex] = 1; + const std::size_t moduleIndex = missingTrivial[slot]; + moduleSummaries[moduleIndex] = buildModuleSummary(moduleIndex); }); + } + for (std::size_t moduleIndex : missingTrivial) + cacheAndStoreModule(moduleIndex); + } + totalModuleAnalyses += missingTrivial.size(); + + // Merge trivial SCCs into globalIndex. + for (std::size_t moduleIndex : trivialModules) + (void)analysis::mergeResourceSummaryIndex(globalIndex, moduleSummaries[moduleIndex]); - for (std::size_t moduleIndex : missingIndices) + // Process cyclic SCCs with internal iteration. + for (std::size_t sccIdx : cyclicSCCIndices) + { + const auto& scc = sccOrder[sccIdx]; + std::vector sccPrevSummaries(N); + std::unordered_set sccChangedNames; + bool sccConverged = false; + + for (unsigned sccIter = 0; sccIter < kCrossTUMaxIterations; ++sccIter) + { + std::vector dirtyInSCC; + if (sccIter == 0) { - if (computedReady[moduleIndex] == 0) - continue; - moduleSummaries[moduleIndex] = std::move(computed[moduleIndex]); - summaryReady[moduleIndex] = 1; + dirtyInSCC = std::vector(scc.begin(), scc.end()); + } + else + { + for (std::size_t m : scc) + { + bool isDirty = false; + for (const std::string& callee : filteredResourceCalleeNames[m]) + { + if (sccChangedNames.count(callee)) + { + isDirty = true; + break; + } + } + if (isDirty) + dirtyInSCC.push_back(m); + else + moduleSummaries[m] = sccPrevSummaries[m]; + } } - } - for (std::size_t moduleIndex : missingIndices) - { - if (summaryReady[moduleIndex] == 0) - continue; - memoryCache.emplace(cacheKeys[moduleIndex], moduleSummaries[moduleIndex]); - if (allowDiskCache) + for (std::size_t m : dirtyInSCC) + moduleSummaries[m] = buildModuleSummary(m); + totalModuleAnalyses += dirtyInSCC.size(); + + analysis::ResourceSummaryIndex sccMerged; + for (std::size_t m : scc) + (void)analysis::mergeResourceSummaryIndex(sccMerged, moduleSummaries[m]); + + analysis::ResourceSummaryIndex prevSccMerged; + for (std::size_t m : scc) + (void)analysis::mergeResourceSummaryIndex(prevSccMerged, sccPrevSummaries[m]); + const bool iterConverged = + analysis::resourceSummaryIndexEquals(sccMerged, prevSccMerged); + + sccChangedNames = + analysis::computeChangedResourceFunctionNames(prevSccMerged, sccMerged); + + for (std::size_t m : scc) + sccPrevSummaries[m] = moduleSummaries[m]; + + if (cfg.timing) + { + coretrace::log(coretrace::Level::Info, + " Resource cyclic SCC (size={}) iteration {}{} (dirty={})\n", + scc.size(), sccIter + 1, iterConverged ? " converged" : "", + dirtyInSCC.size()); + } + + if (iterConverged) { - const std::filesystem::path cacheFile = - std::filesystem::path(cfg.resourceSummaryCacheDir) / - (cacheKeys[moduleIndex] + ".json"); - (void)writeSummaryCacheFile(cacheFile, moduleSummaries[moduleIndex]); + sccConverged = true; + break; } + + for (std::size_t m : scc) + (void)analysis::mergeResourceSummaryIndex(globalIndex, moduleSummaries[m]); } - } - for (std::size_t moduleIndex = 0; moduleIndex < loadedModules.size(); ++moduleIndex) - { - if (summaryReady[moduleIndex] == 0) - continue; - (void)analysis::mergeResourceSummaryIndex(nextGlobal, moduleSummaries[moduleIndex]); + if (!sccConverged) + { + coretrace::log(coretrace::Level::Warn, + "Resource cross-TU: cyclic SCC (size={}) reached " + "iteration cap ({})\n", + scc.size(), kCrossTUMaxIterations); + } + + for (std::size_t m : scc) + (void)analysis::mergeResourceSummaryIndex(globalIndex, moduleSummaries[m]); } - const bool iterConverged = analysis::resourceSummaryIndexEquals(nextGlobal, globalIndex); - ++iterationsRan; if (cfg.timing) { - const auto iterEnd = Clock::now(); + const auto levelEnd = Clock::now(); const auto ms = - std::chrono::duration_cast(iterEnd - iterStart).count(); - coretrace::log(coretrace::Level::Info, - "Cross-TU summary iteration {} done in {} ms{}\n", iterationsRan, ms, - iterConverged ? " (converged)" : ""); - } - - if (iterConverged) - { - converged = true; - break; + std::chrono::duration_cast(levelEnd - levelStart) + .count(); + coretrace::log( + coretrace::Level::Info, + " Resource level {}: {} trivial, {} cyclic ({} modules) in {} ms\n", level, + trivialModules.size(), cyclicSCCIndices.size(), + trivialModules.size() + + [&]() + { + std::size_t n = 0; + for (std::size_t s : cyclicSCCIndices) + n += sccOrder[s].size(); + return n; + }(), + ms); } - globalIndex = std::move(nextGlobal); - } - - if (!converged) - { - coretrace::log(coretrace::Level::Warn, - "Resource inter-procedural analysis: reached fixed-point iteration cap " - "({}); summary may be non-converged and conservative\n", - kCrossTUMaxIterations); } if (cfg.timing) @@ -1629,8 +1983,9 @@ buildCrossTUSummaryIndex(const std::vector& loadedModules, const auto ms = std::chrono::duration_cast(buildEnd - buildStart).count(); coretrace::log(coretrace::Level::Info, - "Cross-TU summary build done in {} ms ({} iteration(s))\n", ms, - iterationsRan); + "Cross-TU resource summary build done in {} ms " + "({} SCCs, {} module analyses)\n", + ms, sccOrder.size(), totalModuleAnalyses); } return std::make_shared(std::move(globalIndex)); @@ -1640,6 +1995,8 @@ static std::shared_ptr& loadedModules, const AnalysisConfig& cfg) { + const analyzer::ScopedHotspot totalHotspot(cfg.timing, + "app.cross_tu.global_read_before_write.total"); if (loadedModules.size() < 2) return nullptr; @@ -1658,6 +2015,8 @@ buildCrossTUGlobalReadBeforeWriteSummaryIndex(const std::vector analysis::GlobalReadBeforeWriteSummaryIndex { + const analyzer::ScopedHotspot hotspot(cfg.timing, + "app.cross_tu.global_read_before_write.build_module"); const LoadedInputModule& loaded = loadedModules[moduleIndex]; const analysis::FunctionFilter filter = analysis::buildFunctionFilter(*loaded.module, cfg); auto shouldAnalyze = [&](const llvm::Function& F) -> bool @@ -1700,6 +2059,7 @@ static std::shared_ptr buildCrossTUUninitializedSummaryIndex(const std::vector& loadedModules, const AnalysisConfig& cfg) { + const analyzer::ScopedHotspot totalHotspot(cfg.timing, "app.cross_tu.uninitialized.total"); if (!cfg.uninitializedCrossTU || loadedModules.size() < 2) return nullptr; @@ -1725,70 +2085,525 @@ buildCrossTUUninitializedSummaryIndex(const std::vector& load preparedModules.push_back( analysis::prepareUninitializedModuleContext(*loaded.module, shouldAnalyze)); } + // Pre-compute per-module callee name sets for delta-based convergence. + std::vector> moduleCalleeNames(loadedModules.size()); + for (std::size_t i = 0; i < loadedModules.size(); ++i) + moduleCalleeNames[i] = analysis::getCanonicalCalleeNames(preparedModules[i]); + + // Build definedBy map and single-def name set for SCC + dirty-marking. + // Same criterion as buildSingleDefFilteredEdges: a function is single-def + // if exactly one module defines it. Multi-def functions (inline, template, + // weak) produce identical summaries in all TUs and don't create real + // cross-module data dependencies. + const std::size_t N = loadedModules.size(); + std::unordered_map> definedBy; + for (std::size_t i = 0; i < N; ++i) + { + for (const llvm::Function& F : *loadedModules[i].module) + { + if (F.isDeclaration() || !F.hasName() || F.getName().empty()) + continue; + const std::string canon = ctrace_tools::canonicalizeMangledName(F.getName().str()); + definedBy[canon].push_back(i); + } + } + + std::unordered_set singleDefNames; + for (const auto& [name, modules] : definedBy) + { + if (modules.size() == 1) + singleDefNames.insert(name); + } + + // Per-module filtered callee sets: only callees with single-def + // definitions. Consistent with SCC edge criterion. + std::vector> filteredModuleCalleeNames(N); + for (std::size_t i = 0; i < N; ++i) + { + for (const std::string& callee : moduleCalleeNames[i]) + { + if (singleDefNames.count(callee)) + filteredModuleCalleeNames[i].insert(callee); + } + } + + // ── SCC instrumentation: diagnose inter-module call graph structure ── + if (cfg.timing) + { + // 2. Build inter-module edge graph: moduleEdges[i] = {j, ...} + // Module i depends on module j if i calls a function defined in j. + std::vector> moduleEdges(N); + std::size_t indirectCallModules = 0; + for (std::size_t i = 0; i < N; ++i) + { + for (const std::string& callee : moduleCalleeNames[i]) + { + auto it = definedBy.find(callee); + if (it != definedBy.end()) + { + for (std::size_t j : it->second) + { + if (j != i) + moduleEdges[i].insert(j); + } + } + // Unresolved callees (not in definedBy) are external or indirect; + // they don't create inter-module edges. + } + } + + // 3. Tarjan's algorithm on module indices → SCCs + // (Reuses the ModuleTarjan struct extracted to file scope.) + ModuleTarjan tarjan; + tarjan.run(N, moduleEdges); + + // 4. Classify and log SCC distribution. + std::size_t trivialSCCs = 0; // size == 1, no self-edge + std::size_t selfLoopSCCs = 0; // size == 1, has self-edge + std::size_t cyclicSCCs = 0; // size > 1 + std::size_t maxSCCSize = 0; + std::size_t totalModulesInCyclicSCCs = 0; + + for (const auto& scc : tarjan.sccs) + { + if (scc.size() == 1) + { + // Check self-edge (module calls itself via cross-module path) + if (moduleEdges[scc[0]].count(scc[0])) + ++selfLoopSCCs; + else + ++trivialSCCs; + } + else + { + ++cyclicSCCs; + totalModulesInCyclicSCCs += scc.size(); + maxSCCSize = std::max(maxSCCSize, scc.size()); + } + } + + coretrace::log(coretrace::Level::Info, + "Cross-TU inter-module call graph SCC analysis:\n" + " Modules: {}\n" + " Total SCCs: {}\n" + " Trivial (acyclic, 1 pass): {}\n" + " Self-loop (size=1, self-dep): {}\n" + " Cyclic (size>1, need iteration): {}\n" + " Max cyclic SCC size: {}\n" + " Total modules in cyclic SCCs: {}\n", + N, tarjan.sccs.size(), trivialSCCs, selfLoopSCCs, cyclicSCCs, maxSCCSize, + totalModulesInCyclicSCCs); + + // Log cyclic SCCs with module names for investigation. + for (const auto& scc : tarjan.sccs) + { + if (scc.size() > 1) + { + std::string members; + for (std::size_t idx : scc) + { + if (!members.empty()) + members += ", "; + members += loadedModules[idx].filename; + } + coretrace::log(coretrace::Level::Info, " Cyclic SCC (size={}): [{}]\n", scc.size(), + members); + } + } + + // Log total inter-module edges for density insight. + std::size_t totalEdges = 0; + for (const auto& e : moduleEdges) + totalEdges += e.size(); + coretrace::log(coretrace::Level::Info, " Inter-module edges: {} (density: {:.1f}%)\n", + totalEdges, N > 1 ? (100.0 * totalEdges / (N * (N - 1))) : 0.0); + + // 5. Identify "hub" functions that create the most cross-module edges. + // These are functions defined in many modules (inline/template) that + // every other module calls, inflating the density artificially. + struct HubInfo + { + std::string name; + std::size_t definedInModules; // how many modules define it + std::size_t calledByModules; // how many modules call it + std::size_t edgesCreated; // definedIn × calledBy (cross-module) + }; + std::vector hubs; + for (const auto& [funcName, defModules] : definedBy) + { + if (defModules.size() <= 1) + continue; // only interesting if defined in multiple modules + std::size_t callerCount = 0; + for (std::size_t i = 0; i < N; ++i) + { + if (moduleCalleeNames[i].count(funcName)) + ++callerCount; + } + if (callerCount > 0) + { + std::size_t edges = 0; + for (std::size_t i = 0; i < N; ++i) + { + if (!moduleCalleeNames[i].count(funcName)) + continue; + for (std::size_t j : defModules) + { + if (j != i) + ++edges; + } + } + hubs.push_back({funcName, defModules.size(), callerCount, edges}); + } + } + std::sort(hubs.begin(), hubs.end(), [](const HubInfo& a, const HubInfo& b) + { return a.edgesCreated > b.edgesCreated; }); + + coretrace::log(coretrace::Level::Info, " Multi-defined hub functions (top 15):\n"); + for (std::size_t i = 0; i < std::min(hubs.size(), std::size_t(15)); ++i) + { + coretrace::log(coretrace::Level::Info, + " {:4} edges | defined-in={:2} called-by={:2} | {}\n", + hubs[i].edgesCreated, hubs[i].definedInModules, hubs[i].calledByModules, + hubs[i].name); + } + + // Also count: how many edges come from single-def functions vs multi-def? + std::size_t edgesFromMultiDef = 0; + std::size_t edgesFromSingleDef = 0; + for (std::size_t i = 0; i < N; ++i) + { + for (const std::string& callee : moduleCalleeNames[i]) + { + auto it = definedBy.find(callee); + if (it == definedBy.end()) + continue; + for (std::size_t j : it->second) + { + if (j == i) + continue; + if (it->second.size() > 1) + ++edgesFromMultiDef; + else + ++edgesFromSingleDef; + } + } + } + coretrace::log(coretrace::Level::Info, " Edge source: single-def={} multi-def={}\n", + edgesFromSingleDef, edgesFromMultiDef); + + // 6. Filtered SCC analysis: only single-def functions (real dependencies). + // Multi-def functions (inline/template) are noise for summary propagation. + // (Uses buildSingleDefFilteredEdges helper extracted to file scope.) + const auto diagFilteredEdges = buildSingleDefFilteredEdges(N, moduleCalleeNames, definedBy); + + std::size_t filteredTotalEdges = 0; + for (const auto& e : diagFilteredEdges) + filteredTotalEdges += e.size(); + + ModuleTarjan filteredTarjan; + filteredTarjan.run(N, diagFilteredEdges); + + std::size_t fTrivial = 0, fSelfLoop = 0, fCyclic = 0; + std::size_t fMaxSCC = 0, fTotalInCyclic = 0; + for (const auto& scc : filteredTarjan.sccs) + { + if (scc.size() == 1) + { + if (diagFilteredEdges[scc[0]].count(scc[0])) + ++fSelfLoop; + else + ++fTrivial; + } + else + { + ++fCyclic; + fTotalInCyclic += scc.size(); + fMaxSCC = std::max(fMaxSCC, scc.size()); + } + } + + coretrace::log( + coretrace::Level::Info, + " FILTERED SCC (single-def only, {} edges, density {:.1f}%):\n" + " Total SCCs: {}\n" + " Trivial (acyclic): {}\n" + " Self-loop: {}\n" + " Cyclic (size>1): {}\n" + " Max cyclic SCC size: {}\n" + " Modules in cyclic SCCs: {}\n", + filteredTotalEdges, N > 1 ? (100.0 * filteredTotalEdges / (N * (N - 1))) : 0.0, + filteredTarjan.sccs.size(), fTrivial, fSelfLoop, fCyclic, fMaxSCC, fTotalInCyclic); + + for (const auto& scc : filteredTarjan.sccs) + { + if (scc.size() > 1) + { + std::string members; + for (std::size_t idx : scc) + { + if (!members.empty()) + members += ", "; + // Show just the filename, not the full path. + const std::string& path = loadedModules[idx].filename; + auto slash = path.rfind('/'); + members += (slash != std::string::npos) ? path.substr(slash + 1) : path; + } + coretrace::log(coretrace::Level::Info, " Cyclic SCC (size={}): [{}]\n", + scc.size(), members); + } + } + } + // ── End SCC instrumentation ── + + // ── SCC worklist: topological processing of inter-module dependencies ── + // Build single-def filtered edge graph and compute topological SCC order. + // This replaces the global fixed-point loop with ordered SCC processing: + // - Trivial SCCs (no cycles): processed once, no iteration needed + // - Cyclic SCCs: iterate internally until convergence + const auto filteredEdges = buildSingleDefFilteredEdges(N, moduleCalleeNames, definedBy); + const auto sccOrder = computeTopologicalSCCOrder(N, filteredEdges); + const auto sccLevels = computeSCCLevels(sccOrder, filteredEdges, N); + + // Note on indirect calls (function pointers): + // Virtually all C++ modules contain indirect calls (virtual dispatch, + // std::function, etc.), so flagging them as "conservatively cyclic" would + // negate the SCC worklist benefit. The uninit/resource analysis already + // handles indirect calls conservatively within each module (unresolved + // callees are treated as unknown). The SCC graph only tracks *resolved* + // inter-module edges, so indirect calls don't affect the topological order. + // No special treatment is needed. + + // Compute max level for grouping. + unsigned maxLevel = 0; + for (unsigned lvl : sccLevels) + maxLevel = std::max(maxLevel, lvl); + + // Group SCCs by topological level for parallel processing. + std::vector> levelGroups(maxLevel + 1); + for (std::size_t s = 0; s < sccOrder.size(); ++s) + levelGroups[sccLevels[s]].push_back(s); + + // Classify SCCs for logging. + std::size_t trivialSCCCount = 0; + std::size_t cyclicSCCCount = 0; + for (std::size_t s = 0; s < sccOrder.size(); ++s) + { + const auto& scc = sccOrder[s]; + const bool isTrivial = scc.size() == 1 && !filteredEdges[scc[0]].count(scc[0]); + if (isTrivial) + ++trivialSCCCount; + else + ++cyclicSCCCount; + } + + if (cfg.timing) + { + coretrace::log(coretrace::Level::Info, + "Cross-TU uninitialized SCC worklist: {} SCCs ({} trivial, {} cyclic) " + "in {} levels\n", + sccOrder.size(), trivialSCCCount, cyclicSCCCount, maxLevel + 1); + } + + // ── Process SCCs level by level ── analysis::UninitializedSummaryIndex globalIndex; - unsigned iterationsRan = 0; - bool converged = false; - for (unsigned iter = 0; iter < kCrossTUMaxIterations; ++iter) + std::vector moduleSummaries(N); + std::size_t totalModuleAnalyses = 0; + + for (unsigned level = 0; level <= maxLevel; ++level) { - const auto iterStart = Clock::now(); + const auto& group = levelGroups[level]; + if (group.empty()) + continue; + + const auto levelStart = Clock::now(); + + // Prepare external summaries once per level — all SCCs at this level + // see the same accumulated globalIndex (they are independent). const analysis::PreparedUninitializedExternalSummaries preparedExternal = analysis::prepareUninitializedExternalSummaries(&globalIndex); - analysis::UninitializedSummaryIndex nextGlobal; - std::vector moduleSummaries(loadedModules.size()); auto buildModuleSummary = [&](std::size_t moduleIndex) -> analysis::UninitializedSummaryIndex { + const analyzer::ScopedHotspot hotspot(cfg.timing, + "app.cross_tu.uninitialized.build_module"); const LoadedInputModule& loaded = loadedModules[moduleIndex]; return analysis::buildUninitializedSummaryIndex( *loaded.module, &preparedModules[moduleIndex], &preparedExternal); }; - if (maxJobs <= 1 || loadedModules.size() <= 1) - { - for (std::size_t moduleIndex = 0; moduleIndex < loadedModules.size(); ++moduleIndex) - moduleSummaries[moduleIndex] = buildModuleSummary(moduleIndex); - } - else + // Collect all trivial SCC modules at this level for parallel batch processing. + std::vector trivialModules; + std::vector cyclicSCCIndices; + + for (std::size_t sccIdx : group) { - runParallelWork(loadedModules.size(), maxJobs, [&](std::size_t moduleIndex) - { moduleSummaries[moduleIndex] = buildModuleSummary(moduleIndex); }); + const auto& scc = sccOrder[sccIdx]; + const bool isTrivial = scc.size() == 1 && !filteredEdges[scc[0]].count(scc[0]); + if (isTrivial) + trivialModules.push_back(scc[0]); + else + cyclicSCCIndices.push_back(sccIdx); } - for (const auto& moduleSummary : moduleSummaries) + // Process trivial SCCs in parallel. + if (!trivialModules.empty()) { - (void)analysis::mergeUninitializedSummaryIndex(nextGlobal, moduleSummary); + if (maxJobs <= 1 || trivialModules.size() <= 1) + { + for (std::size_t moduleIndex : trivialModules) + moduleSummaries[moduleIndex] = buildModuleSummary(moduleIndex); + } + else + { + runParallelWork(trivialModules.size(), maxJobs, + [&](std::size_t slot) + { + const std::size_t moduleIndex = trivialModules[slot]; + moduleSummaries[moduleIndex] = buildModuleSummary(moduleIndex); + }); + } + totalModuleAnalyses += trivialModules.size(); } - const bool iterConverged = - analysis::uninitializedSummaryIndexEquals(nextGlobal, globalIndex); - ++iterationsRan; - globalIndex = std::move(nextGlobal); + // Merge trivial SCC summaries into globalIndex. + for (std::size_t moduleIndex : trivialModules) + (void)analysis::mergeUninitializedSummaryIndex(globalIndex, + moduleSummaries[moduleIndex]); - if (cfg.timing) + // Process cyclic SCCs (or modules with indirect calls) with internal iteration. + for (std::size_t sccIdx : cyclicSCCIndices) { - const auto iterEnd = Clock::now(); - const auto ms = - std::chrono::duration_cast(iterEnd - iterStart).count(); - coretrace::log(coretrace::Level::Info, - "Cross-TU uninitialized summary iteration {} done in {} ms{}\n", - iterationsRan, ms, iterConverged ? " (converged)" : ""); + const auto& scc = sccOrder[sccIdx]; + + // Internal convergence loop for this SCC. + std::vector sccPrevSummaries(N); + std::unordered_set sccChangedNames; + bool sccConverged = false; + + for (unsigned sccIter = 0; sccIter < kCrossTUMaxIterations; ++sccIter) + { + // Re-prepare external summaries with current globalIndex + // (which includes upstream SCCs + any changes from prev SCC iterations). + const analysis::PreparedUninitializedExternalSummaries sccExternal = + analysis::prepareUninitializedExternalSummaries(&globalIndex); + + // Rebind buildModuleSummary to use sccExternal. + auto buildSCCModuleSummary = + [&](std::size_t moduleIndex) -> analysis::UninitializedSummaryIndex + { + const analyzer::ScopedHotspot hotspot( + cfg.timing, "app.cross_tu.uninitialized.build_module"); + const LoadedInputModule& loaded = loadedModules[moduleIndex]; + return analysis::buildUninitializedSummaryIndex( + *loaded.module, &preparedModules[moduleIndex], &sccExternal); + }; + + // Delta-based dirty-marking within the SCC. + std::vector dirtyInSCC; + if (sccIter == 0) + { + dirtyInSCC = std::vector(scc.begin(), scc.end()); + } + else + { + for (std::size_t m : scc) + { + bool isDirty = false; + for (const std::string& callee : filteredModuleCalleeNames[m]) + { + if (sccChangedNames.count(callee)) + { + isDirty = true; + break; + } + } + if (isDirty) + dirtyInSCC.push_back(m); + else + moduleSummaries[m] = sccPrevSummaries[m]; + } + } + + for (std::size_t m : dirtyInSCC) + moduleSummaries[m] = buildSCCModuleSummary(m); + totalModuleAnalyses += dirtyInSCC.size(); + + // Build SCC-local merged index to check convergence. + analysis::UninitializedSummaryIndex sccMerged; + for (std::size_t m : scc) + (void)analysis::mergeUninitializedSummaryIndex(sccMerged, moduleSummaries[m]); + + // Check if SCC summaries changed from previous iteration. + analysis::UninitializedSummaryIndex prevSccMerged; + for (std::size_t m : scc) + (void)analysis::mergeUninitializedSummaryIndex(prevSccMerged, + sccPrevSummaries[m]); + const bool iterConverged = + analysis::uninitializedSummaryIndexEquals(sccMerged, prevSccMerged); + + sccChangedNames = + analysis::computeChangedUninitializedFunctionNames(prevSccMerged, sccMerged); + + for (std::size_t m : scc) + sccPrevSummaries[m] = moduleSummaries[m]; + + if (cfg.timing) + { + coretrace::log(coretrace::Level::Info, + " Cyclic SCC (size={}) iteration {}{} (dirty={})\n", scc.size(), + sccIter + 1, iterConverged ? " converged" : "", + dirtyInSCC.size()); + } + + if (iterConverged) + { + sccConverged = true; + break; + } + + // Update globalIndex with intermediate SCC state for next iteration's + // prepareExternalSummaries. + for (std::size_t m : scc) + (void)analysis::mergeUninitializedSummaryIndex(globalIndex, moduleSummaries[m]); + } + + if (!sccConverged) + { + coretrace::log(coretrace::Level::Warn, + "Uninitialized cross-TU: cyclic SCC (size={}) reached " + "iteration cap ({})\n", + scc.size(), kCrossTUMaxIterations); + } + + // Merge final SCC summaries into globalIndex. + for (std::size_t m : scc) + (void)analysis::mergeUninitializedSummaryIndex(globalIndex, moduleSummaries[m]); } - if (iterConverged) + if (cfg.timing) { - converged = true; - break; + const auto levelEnd = Clock::now(); + const auto ms = + std::chrono::duration_cast(levelEnd - levelStart) + .count(); + coretrace::log( + coretrace::Level::Info, + " Level {}: {} trivial SCCs, {} cyclic SCCs ({} modules) in {} ms\n", level, + trivialModules.size(), cyclicSCCIndices.size(), + trivialModules.size() + + [&]() + { + std::size_t n = 0; + for (std::size_t s : cyclicSCCIndices) + n += sccOrder[s].size(); + return n; + }(), + ms); } } - if (!converged) - { - coretrace::log(coretrace::Level::Warn, - "Uninitialized inter-procedural analysis: reached fixed-point iteration " - "cap ({}); summary may be non-converged and conservative\n", - kCrossTUMaxIterations); - } + // No indirect-call cleanup pass needed — see note above on indirect calls. if (cfg.timing) { @@ -1796,8 +2611,9 @@ buildCrossTUUninitializedSummaryIndex(const std::vector& load const auto ms = std::chrono::duration_cast(buildEnd - buildStart).count(); coretrace::log(coretrace::Level::Info, - "Cross-TU uninitialized summary build done in {} ms ({} iteration(s))\n", ms, - iterationsRan); + "Cross-TU uninitialized summary build done in {} ms " + "({} SCCs, {} module analyses)\n", + ms, sccOrder.size(), totalModuleAnalyses); } return std::make_shared(std::move(globalIndex)); @@ -2026,7 +2842,9 @@ class AnalyzerApp return AppResult::failure(std::move(executionStatus.error)); std::unique_ptr outputStrategy = makeOutputStrategy(plan.outputFormat); - return AppResult::success(outputStrategy->emit(plan, results)); + const int exitCode = outputStrategy->emit(plan, results); + analyzer::dumpHotspotSummary(std::cerr, plan.cfg.timing); + return AppResult::success(exitCode); } }; diff --git a/src/cli/ArgParser.cpp b/src/cli/ArgParser.cpp index 55c3dfc..14ac583 100644 --- a/src/cli/ArgParser.cpp +++ b/src/cli/ArgParser.cpp @@ -45,7 +45,7 @@ namespace ctrace::stack::cli } private: - static constexpr std::array kCandidates = { + static constexpr std::array kCandidates = { {{"-h", "-h"}, {"--help", "--help"}, {"--demangle", "--demangle"}, @@ -87,6 +87,9 @@ namespace ctrace::stack::cli {"--resource-summary-cache-dir", "--resource-summary-cache-dir"}, {"--resource-summary-cache-memory-only", "--resource-summary-cache-memory-only"}, {"--compile-ir-cache-dir", "--compile-ir-cache-dir"}, + {"--compile-ir-format", "--compile-ir-format"}, + {"--compile-ir-format=bc", "--compile-ir-format=bc"}, + {"--compile-ir-format=ll", "--compile-ir-format=ll"}, {"--config", "--config"}, {"--print-effective-config", "--print-effective-config"}, {"--compile-commands", "--compile-commands"}, @@ -390,6 +393,32 @@ namespace ctrace::stack::cli return false; } + bool parseCompileIRFormat(const std::string& input, CompileIRFormat& out, + std::string& error) + { + std::string trimmed = trimCopy(input); + if (!trimmed.empty() && trimmed.front() == '.') + trimmed.erase(trimmed.begin()); + + std::string lowered; + lowered.reserve(trimmed.size()); + for (char c : trimmed) + lowered.push_back(static_cast(std::tolower(static_cast(c)))); + + if (lowered == "bc") + { + out = CompileIRFormat::BC; + return true; + } + if (lowered == "ll") + { + out = CompileIRFormat::LL; + return true; + } + error = "expected 'bc' or 'll'"; + return false; + } + bool parseStackLimitValue(const std::string& input, StackSize& out, std::string& error) { std::string trimmed = trimCopy(input); @@ -952,6 +981,16 @@ namespace ctrace::stack::cli cfg.compileIRCacheDir = resolveConfigRelativePath(value, configDir); return true; } + if (key == "compile-ir-format") + { + std::string localError; + if (!parseCompileIRFormat(value, cfg.compileIRFormat, localError)) + { + error = "invalid compile-ir-format value: " + localError; + return false; + } + return true; + } error = "unknown key '" + key + "'"; return false; @@ -1458,6 +1497,19 @@ namespace ctrace::stack::cli continue; } } + { + std::string value; + std::string error; + if (consumeLongOptionValue(argStr, "--compile-ir-format", i, argc, argv, value, + error)) + { + if (!error.empty()) + return makeError(error); + if (!parseCompileIRFormat(value, cfg.compileIRFormat, error)) + return makeError("Invalid --compile-ir-format value: " + error); + continue; + } + } if (argStr == "--resource-summary-cache-memory-only") { cfg.resourceSummaryMemoryOnly = true;