From d983c8390c891308358e9cf0a92b70119c292e9f Mon Sep 17 00:00:00 2001 From: Levi Cook Date: Tue, 10 Jun 2025 23:36:20 -0600 Subject: [PATCH 1/3] feat: add compute unit benchmarking framework with trait-based design - Add InstructionBenchmark trait for clean separation of concerns - Benchmark owns: SVM setup, keypairs, signing - Framework owns: unsigned tx building, CU measurement, statistics - Implement benchmark_instruction() runner with SVM state accumulation - Convert SOL and SPL token transfer benchmarks to use new framework - Add solana-message dependency for unsigned transaction creation - Simplify benchmark output to single summary line + JSON data - Eliminate 150+ lines of boilerplate from benchmark implementations - Maintain identical CU measurements (300 CU SOL, 4794 CU SPL) The framework enables measuring any Solana instruction's compute unit usage with minimal code while providing structured estimates similar to Helius Priority Fee API. --- Cargo.lock | 1 + Cargo.toml | 1 + crates/litesvm-testing/Cargo.toml | 1 + .../benches/cu_bench_sol_transfer.rs | 168 +++------ .../benches/cu_bench_spl_transfer.rs | 336 ++++++++---------- crates/litesvm-testing/src/cu_bench/mod.rs | 74 ++++ 6 files changed, 284 insertions(+), 297 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e5596e..fc43e2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1757,6 +1757,7 @@ dependencies = [ "solana-compute-budget-interface", "solana-instruction", "solana-keypair", + "solana-message", "solana-pubkey", "solana-signer", "solana-system-interface", diff --git a/Cargo.toml b/Cargo.toml index aafbd97..b4d1682 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ serde_json = "1.0.140" solana-compute-budget-interface = "2.2" solana-instruction = "2.2" solana-keypair = "2.2" +solana-message = "2.2" solana-pubkey = "2.2" solana-signer = "2.2" solana-system-interface = "1" diff --git a/crates/litesvm-testing/Cargo.toml b/crates/litesvm-testing/Cargo.toml index 0745c86..0f9d3f1 100644 --- a/crates/litesvm-testing/Cargo.toml +++ b/crates/litesvm-testing/Cargo.toml @@ -45,6 +45,7 @@ serde_json = { workspace = true } solana-compute-budget-interface = { workspace = true } solana-instruction = { workspace = true } solana-keypair = { workspace = true } +solana-message = { workspace = true } solana-pubkey = { workspace = true } solana-signer = { workspace = true } solana-system-interface = { workspace = true } diff --git a/crates/litesvm-testing/benches/cu_bench_sol_transfer.rs b/crates/litesvm-testing/benches/cu_bench_sol_transfer.rs index 9d6a572..f825e55 100644 --- a/crates/litesvm-testing/benches/cu_bench_sol_transfer.rs +++ b/crates/litesvm-testing/benches/cu_bench_sol_transfer.rs @@ -1,130 +1,80 @@ -use litesvm_testing::cu_bench::{ComputeUnitEstimate, CuLevel}; +use litesvm::LiteSVM; +use litesvm_testing::cu_bench::{benchmark_instruction, InstructionBenchmark}; use litesvm_testing::prelude::*; +use solana_instruction::Instruction; use solana_keypair::Keypair; +use solana_pubkey::Pubkey; use solana_signer::Signer; +use solana_transaction::Transaction; -fn main() { - println!("=== SOL Transfer CU Benchmark ==="); - - // Run multiple measurements to collect data - let mut cu_measurements = Vec::new(); +/// SOL transfer benchmark using the new framework +struct SolTransferBenchmark { + sender: Keypair, + recipient: Keypair, + transfer_amount: u64, +} - for i in 0..100 { - // More samples for better statistics - let cu_used = measure_sol_transfer(); - cu_measurements.push(cu_used); - if (i + 1) % 10 == 0 { - println!("Completed {} measurements...", i + 1); +impl SolTransferBenchmark { + fn new() -> Self { + Self { + sender: Keypair::new(), + recipient: Keypair::new(), + transfer_amount: 500_000, // Smaller transfer amount for multiple measurements } } +} - // Create structured estimate from our measurements - let estimate = ComputeUnitEstimate::from_measurements( - "sol_transfer".to_string(), - &cu_measurements, - vec!["litesvm".to_string()], - ); +impl InstructionBenchmark for SolTransferBenchmark { + fn instruction_name(&self) -> &'static str { + "sol_transfer" + } - // Print basic stats like before - let min = *cu_measurements.iter().min().unwrap(); - let max = *cu_measurements.iter().max().unwrap(); - let avg = cu_measurements.iter().sum::() / cu_measurements.len() as u64; + fn setup_svm(&self) -> LiteSVM { + let mut svm = LiteSVM::new(); - println!("\n=== Raw Statistics ==="); - println!("Samples: {}", cu_measurements.len()); - println!("Min: {} CU", min); - println!("Max: {} CU", max); - println!("Avg: {} CU", avg); - println!("Variance: {} CU", max - min); + // Fund the sender account with enough for 200+ transfers (including fees) + svm.airdrop(&self.sender.pubkey(), 200_000_000).unwrap(); - // Print our structured estimate - println!("\n=== Structured Estimate ==="); - println!("Instruction: {}", estimate.instruction_type); - println!( - "Min (0th percentile): {} CU", - estimate.get_cu_for_level(CuLevel::Min) - ); - println!( - "Conservative (25th): {} CU", - estimate.get_cu_for_level(CuLevel::Conservative) - ); - println!( - "Balanced (50th): {} CU", - estimate.get_cu_for_level(CuLevel::Balanced) - ); - println!( - "Safe (75th): {} CU", - estimate.get_cu_for_level(CuLevel::Safe) - ); - println!( - "Very High (95th): {} CU", - estimate.get_cu_for_level(CuLevel::VeryHigh) - ); - println!( - "Unsafe Max (100th): {} CU", - estimate.get_cu_for_level(CuLevel::UnsafeMax) - ); + svm + } - // Show custom levels - println!("\n=== Custom Levels ==="); - println!( - "Custom(350): {} CU", - estimate.get_cu_for_level(CuLevel::Custom(350)) - ); - println!( - "Multiplier(1.2): {} CU", - estimate.get_cu_for_level(CuLevel::Multiplier(1.2)) - ); - println!( - "Multiplier(1.5): {} CU", - estimate.get_cu_for_level(CuLevel::Multiplier(1.5)) - ); + fn build_instruction(&self, _svm: &mut LiteSVM) -> (Instruction, Vec) { + let transfer_ix = solana_system_interface::instruction::transfer( + &self.sender.pubkey(), + &self.recipient.pubkey(), + self.transfer_amount, + ); - // Output JSON for potential consumption - println!("\n=== JSON Output ==="); - let json = serde_json::to_string_pretty(&estimate).expect("Failed to serialize"); - println!("{}", json); -} + let signer_pubkeys = vec![self.sender.pubkey()]; + (transfer_ix, signer_pubkeys) + } -fn measure_sol_transfer() -> u64 { - // Set up fresh environment - let (mut svm, fee_payer) = litesvm_testing::setup_svm_and_fee_payer(); + fn sign_transaction(&self, mut unsigned_tx: Transaction) -> Transaction { + let signers = vec![&self.sender]; + unsigned_tx.sign(&signers, unsigned_tx.message.recent_blockhash); + unsigned_tx + } +} - // Create sender with some SOL - let sender = Keypair::new(); - svm.airdrop(&sender.pubkey(), 10_000_000).unwrap(); +fn main() { + println!("=== SOL Transfer CU Benchmark ==="); - // Create recipient - let recipient = Keypair::new(); + let benchmark = SolTransferBenchmark::new(); + let estimate = benchmark_instruction(benchmark, 100); - // Build transaction with high CU limit for measurement - let transfer_ix = solana_system_interface::instruction::transfer( - &sender.pubkey(), - &recipient.pubkey(), - 1_000_000, // 1M lamports + println!( + "Measured {} samples: {} CU ({}% variance)", + estimate.sample_size, + estimate.balanced, + if estimate.min == estimate.unsafe_max { + 0 + } else { + ((estimate.unsafe_max - estimate.min) * 100) / estimate.balanced + } ); - // Set high CU limit so we don't hit limits during measurement - use solana_compute_budget_interface::ComputeBudgetInstruction; - let cu_limit_ix = ComputeBudgetInstruction::set_compute_unit_limit(200_000); - - let tx = solana_transaction::Transaction::new_signed_with_payer( - &[cu_limit_ix, transfer_ix], - Some(&fee_payer.pubkey()), - &[&fee_payer, &sender], - svm.latest_blockhash(), + println!( + "{}", + serde_json::to_string_pretty(&estimate).expect("Failed to serialize") ); - - // Execute and extract CU usage - let result = svm.send_transaction(tx); - - match result { - Ok(meta) => { - // CU usage is in the metadata - meta.compute_units_consumed - } - Err(meta) => { - panic!("Transaction failed: {:?}", meta); - } - } } diff --git a/crates/litesvm-testing/benches/cu_bench_spl_transfer.rs b/crates/litesvm-testing/benches/cu_bench_spl_transfer.rs index ebab029..d48ab9b 100644 --- a/crates/litesvm-testing/benches/cu_bench_spl_transfer.rs +++ b/crates/litesvm-testing/benches/cu_bench_spl_transfer.rs @@ -1,211 +1,171 @@ -use litesvm_testing::cu_bench::{ComputeUnitEstimate, CuLevel}; +use litesvm::LiteSVM; +use litesvm_testing::cu_bench::{benchmark_instruction, InstructionBenchmark}; use litesvm_testing::prelude::*; +use solana_instruction::Instruction; use solana_keypair::Keypair; +use solana_pubkey::Pubkey; use solana_signer::Signer; +use solana_transaction::Transaction; use spl_token::solana_program::program_pack::Pack; -fn main() { - println!("=== SPL Token Transfer CU Benchmark ==="); +/// SPL token transfer benchmark using the new framework +struct SplTokenTransferBenchmark { + mint_authority: Keypair, + mint: Keypair, + sender: Keypair, + recipient: Keypair, + sender_ata: Pubkey, + recipient_ata: Pubkey, + transfer_amount: u64, +} + +impl SplTokenTransferBenchmark { + fn new() -> Self { + let sender = Keypair::new(); + let recipient = Keypair::new(); + let mint = Keypair::new(); - // Run multiple measurements to collect data - let mut cu_measurements = Vec::new(); + let sender_ata = spl_associated_token_account::get_associated_token_address( + &sender.pubkey(), + &mint.pubkey(), + ); + let recipient_ata = spl_associated_token_account::get_associated_token_address( + &recipient.pubkey(), + &mint.pubkey(), + ); - for i in 0..100 { - // More samples for better statistics - let cu_used = measure_spl_token_transfer(); - cu_measurements.push(cu_used); - if (i + 1) % 10 == 0 { - println!("Completed {} measurements...", i + 1); + Self { + mint_authority: Keypair::new(), + mint, + sender, + recipient, + sender_ata, + recipient_ata, + transfer_amount: 100_000, // 0.1 tokens (with 6 decimals) } } - - // Create structured estimate from our measurements - let estimate = ComputeUnitEstimate::from_measurements( - "spl_token_transfer".to_string(), - &cu_measurements, - vec!["litesvm".to_string()], - ); - - // Print basic stats - let min = *cu_measurements.iter().min().unwrap(); - let max = *cu_measurements.iter().max().unwrap(); - let avg = cu_measurements.iter().sum::() / cu_measurements.len() as u64; - - println!("\n=== Raw Statistics ==="); - println!("Samples: {}", cu_measurements.len()); - println!("Min: {} CU", min); - println!("Max: {} CU", max); - println!("Avg: {} CU", avg); - println!("Variance: {} CU", max - min); - - // Print our structured estimate - println!("\n=== Structured Estimate ==="); - println!("Instruction: {}", estimate.instruction_type); - println!( - "Min (0th percentile): {} CU", - estimate.get_cu_for_level(CuLevel::Min) - ); - println!( - "Conservative (25th): {} CU", - estimate.get_cu_for_level(CuLevel::Conservative) - ); - println!( - "Balanced (50th): {} CU", - estimate.get_cu_for_level(CuLevel::Balanced) - ); - println!( - "Safe (75th): {} CU", - estimate.get_cu_for_level(CuLevel::Safe) - ); - println!( - "Very High (95th): {} CU", - estimate.get_cu_for_level(CuLevel::VeryHigh) - ); - println!( - "Unsafe Max (100th): {} CU", - estimate.get_cu_for_level(CuLevel::UnsafeMax) - ); - - // Show custom levels - println!("\n=== Custom Levels ==="); - println!( - "Custom(5000): {} CU", - estimate.get_cu_for_level(CuLevel::Custom(5000)) - ); - println!( - "Multiplier(1.2): {} CU", - estimate.get_cu_for_level(CuLevel::Multiplier(1.2)) - ); - println!( - "Multiplier(1.5): {} CU", - estimate.get_cu_for_level(CuLevel::Multiplier(1.5)) - ); - - // Output JSON for potential consumption - println!("\n=== JSON Output ==="); - let json = serde_json::to_string_pretty(&estimate).expect("Failed to serialize"); - println!("{}", json); } -fn measure_spl_token_transfer() -> u64 { - // Set up fresh environment - let (mut svm, fee_payer) = litesvm_testing::setup_svm_and_fee_payer(); - - // Create mint authority - let mint_authority = Keypair::new(); - svm.airdrop(&mint_authority.pubkey(), 10_000_000).unwrap(); - - // Create sender and recipient - let sender = Keypair::new(); - let recipient = Keypair::new(); - svm.airdrop(&sender.pubkey(), 10_000_000).unwrap(); - svm.airdrop(&recipient.pubkey(), 10_000_000).unwrap(); - - // Create mint - let mint = Keypair::new(); - let create_mint_ix = spl_token::instruction::initialize_mint( - &spl_token::ID, - &mint.pubkey(), - &mint_authority.pubkey(), - None, // No freeze authority - 6, // 6 decimals - ) - .unwrap(); - - let create_mint_account_ix = solana_system_interface::instruction::create_account( - &fee_payer.pubkey(), - &mint.pubkey(), - 10_000_000, // Much more lamports to ensure rent exemption - spl_token::state::Mint::LEN as u64, - &spl_token::ID, - ); - - // Create associated token accounts - let sender_ata = spl_associated_token_account::get_associated_token_address( - &sender.pubkey(), - &mint.pubkey(), - ); - let recipient_ata = spl_associated_token_account::get_associated_token_address( - &recipient.pubkey(), - &mint.pubkey(), - ); +impl InstructionBenchmark for SplTokenTransferBenchmark { + fn instruction_name(&self) -> &'static str { + "spl_token_transfer" + } - let create_sender_ata_ix = - spl_associated_token_account::instruction::create_associated_token_account( - &fee_payer.pubkey(), - &sender.pubkey(), - &mint.pubkey(), + fn setup_svm(&self) -> LiteSVM { + let mut svm = LiteSVM::new(); + + // Airdrop to accounts that need SOL + svm.airdrop(&self.mint_authority.pubkey(), 50_000_000) + .unwrap(); + svm.airdrop(&self.sender.pubkey(), 10_000_000).unwrap(); + svm.airdrop(&self.recipient.pubkey(), 10_000_000).unwrap(); + + // Create mint account + let create_mint_account_ix = solana_system_interface::instruction::create_account( + &self.mint_authority.pubkey(), + &self.mint.pubkey(), + 5_000_000, // Rent exemption + spl_token::state::Mint::LEN as u64, &spl_token::ID, ); - let create_recipient_ata_ix = - spl_associated_token_account::instruction::create_associated_token_account( - &fee_payer.pubkey(), - &recipient.pubkey(), - &mint.pubkey(), + // Initialize mint + let create_mint_ix = spl_token::instruction::initialize_mint( + &spl_token::ID, + &self.mint.pubkey(), + &self.mint_authority.pubkey(), + None, // No freeze authority + 6, // 6 decimals + ) + .unwrap(); + + // Create associated token accounts + let create_sender_ata_ix = + spl_associated_token_account::instruction::create_associated_token_account( + &self.mint_authority.pubkey(), + &self.sender.pubkey(), + &self.mint.pubkey(), + &spl_token::ID, + ); + + let create_recipient_ata_ix = + spl_associated_token_account::instruction::create_associated_token_account( + &self.mint_authority.pubkey(), + &self.recipient.pubkey(), + &self.mint.pubkey(), + &spl_token::ID, + ); + + // Mint tokens to sender (enough for 200+ transfers) + let mint_to_ix = spl_token::instruction::mint_to( &spl_token::ID, + &self.mint.pubkey(), + &self.sender_ata, + &self.mint_authority.pubkey(), + &[], + 50_000_000, // 50 tokens (with 6 decimals) + ) + .unwrap(); + + // Execute setup transaction + let setup_tx = solana_transaction::Transaction::new_signed_with_payer( + &[ + create_mint_account_ix, + create_mint_ix, + create_sender_ata_ix, + create_recipient_ata_ix, + mint_to_ix, + ], + Some(&self.mint_authority.pubkey()), + &[&self.mint_authority, &self.mint], + svm.latest_blockhash(), ); - // Mint tokens to sender - let mint_to_ix = spl_token::instruction::mint_to( - &spl_token::ID, - &mint.pubkey(), - &sender_ata, - &mint_authority.pubkey(), - &[], - 1_000_000, // 1 token (with 6 decimals) - ) - .unwrap(); - - // Set up all accounts first - let setup_tx = solana_transaction::Transaction::new_signed_with_payer( - &[ - create_mint_account_ix, - create_mint_ix, - create_sender_ata_ix, - create_recipient_ata_ix, - mint_to_ix, - ], - Some(&fee_payer.pubkey()), - &[&fee_payer, &mint, &mint_authority], - svm.latest_blockhash(), - ); + svm.send_transaction(setup_tx).unwrap(); + svm + } - // Execute setup (not measured) - svm.send_transaction(setup_tx).unwrap(); - - // Now measure just the transfer - let transfer_ix = spl_token::instruction::transfer( - &spl_token::ID, - &sender_ata, - &recipient_ata, - &sender.pubkey(), - &[], - 500_000, // 0.5 tokens (with 6 decimals) - ) - .unwrap(); - - // Set high CU limit so we don't hit limits during measurement - use solana_compute_budget_interface::ComputeBudgetInstruction; - let cu_limit_ix = ComputeBudgetInstruction::set_compute_unit_limit(200_000); - - let transfer_tx = solana_transaction::Transaction::new_signed_with_payer( - &[cu_limit_ix, transfer_ix], - Some(&fee_payer.pubkey()), - &[&fee_payer, &sender], - svm.latest_blockhash(), - ); + fn build_instruction(&self, _svm: &mut LiteSVM) -> (Instruction, Vec) { + let transfer_ix = spl_token::instruction::transfer( + &spl_token::ID, + &self.sender_ata, + &self.recipient_ata, + &self.sender.pubkey(), + &[], + self.transfer_amount, + ) + .unwrap(); + + let signer_pubkeys = vec![self.sender.pubkey()]; + (transfer_ix, signer_pubkeys) + } + + fn sign_transaction(&self, mut unsigned_tx: Transaction) -> Transaction { + let signers = vec![&self.sender]; + unsigned_tx.sign(&signers, unsigned_tx.message.recent_blockhash); + unsigned_tx + } +} - // Execute and extract CU usage - let result = svm.send_transaction(transfer_tx); +fn main() { + println!("=== SPL Token Transfer CU Benchmark ==="); - match result { - Ok(meta) => { - // CU usage is in the metadata - meta.compute_units_consumed - } - Err(meta) => { - panic!("Transaction failed: {:?}", meta); + let benchmark = SplTokenTransferBenchmark::new(); + let estimate = benchmark_instruction(benchmark, 100); + + println!( + "Measured {} samples: {} CU ({}% variance)", + estimate.sample_size, + estimate.balanced, + if estimate.min == estimate.unsafe_max { + 0 + } else { + ((estimate.unsafe_max - estimate.min) * 100) / estimate.balanced } - } + ); + + println!( + "{}", + serde_json::to_string_pretty(&estimate).expect("Failed to serialize") + ); } diff --git a/crates/litesvm-testing/src/cu_bench/mod.rs b/crates/litesvm-testing/src/cu_bench/mod.rs index c5db989..39448df 100644 --- a/crates/litesvm-testing/src/cu_bench/mod.rs +++ b/crates/litesvm-testing/src/cu_bench/mod.rs @@ -9,6 +9,80 @@ use std::collections::HashMap; #[cfg(feature = "cu_bench")] use serde::{Deserialize, Serialize}; +use litesvm::LiteSVM; +use solana_instruction::Instruction; +use solana_transaction::Transaction; + +/// Trait for benchmarking the CU usage of specific instructions +pub trait InstructionBenchmark { + /// Human-readable name for this instruction type + fn instruction_name(&self) -> &'static str; + + /// Set up SVM with necessary programs and initial state (called once per benchmark run) + fn setup_svm(&self) -> LiteSVM; + + /// Build the instruction to measure, returning instruction and required signer pubkeys + fn build_instruction(&self, svm: &mut LiteSVM) -> (Instruction, Vec); + + /// Sign the unsigned transaction containing the instruction + fn sign_transaction(&self, unsigned_tx: Transaction) -> Transaction; +} + +/// Universal benchmark runner for any instruction implementing InstructionBenchmark +pub fn benchmark_instruction( + benchmark: T, + samples: usize, +) -> ComputeUnitEstimate { + let mut cu_measurements = Vec::new(); + + // Set up SVM once - it will accumulate state across measurements + let mut svm = benchmark.setup_svm(); + + for i in 0..samples { + let cu_used = measure_instruction(&benchmark, &mut svm); + cu_measurements.push(cu_used); + + if (i + 1) % 10 == 0 { + println!("Completed {} measurements...", i + 1); + } + } + + // Create structured estimate from measurements + ComputeUnitEstimate::from_measurements( + benchmark.instruction_name().to_string(), + &cu_measurements, + vec!["litesvm".to_string()], + ) +} + +/// Measure CU usage for a single instruction +fn measure_instruction(benchmark: &T, svm: &mut LiteSVM) -> u64 { + // 1. Get target instruction and signer pubkeys from benchmark + let (target_ix, signer_pubkeys) = benchmark.build_instruction(svm); + + // 2. Framework creates unsigned transaction with CU limit + use solana_compute_budget_interface::ComputeBudgetInstruction; + let cu_limit_ix = ComputeBudgetInstruction::set_compute_unit_limit(200_000); + let instructions = vec![cu_limit_ix, target_ix]; + + // 3. Build unsigned transaction (framework responsibility) + use solana_message::Message; + + // Get fresh blockhash for each measurement to avoid AlreadyProcessed + svm.expire_blockhash(); + + let message = Message::new(&instructions, Some(&signer_pubkeys[0])); + let mut unsigned_tx = Transaction::new_unsigned(message); + unsigned_tx.message.recent_blockhash = svm.latest_blockhash(); + + // 4. Benchmark signs the transaction + let signed_tx = benchmark.sign_transaction(unsigned_tx); + + // 5. Send transaction and measure CU usage + let result = svm.send_transaction(signed_tx).unwrap(); + result.compute_units_consumed +} + /// Confidence level for CU estimates, similar to Helius Priority Fee API levels #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] pub enum CuLevel { From 2025ef8b14b5b1a469d4c1cb74f781f2a5a3436f Mon Sep 17 00:00:00 2001 From: Levi Cook Date: Wed, 11 Jun 2025 12:56:08 -0600 Subject: [PATCH 2/3] feat(cu_bench): complete compute unit benchmarking framework Add comprehensive CU benchmarking framework with dual instruction/transaction paradigms: Framework Features: - InstructionBenchmark: Pure instruction CU measurement (no framework overhead) - TransactionBenchmark: Complete workflow measurement with multi-program context - Rich execution context discovery through simulation - Percentile-based CU estimates (min/conservative/balanced/safe/very_high/unsafe_max) - Professional logging with env_logger integration - Clean JSON output with proper domain modeling Key Design Decisions: - Remove automatic ComputeBudgetInstruction from instruction benchmarks for transparency - Two-phase measurement: simulation for context + execution for statistics - SVM state accumulation across measurements for realism - StatType enum for clean instruction vs transaction distinction - Comprehensive unit tests for percentile calculations Benchmarks: - SOL transfer: 150 CU (pure instruction) - SPL token transfer: instruction-level benchmark - Token setup workflow: 28,322-38,822 CU transaction benchmark This provides systematic, reproducible CU analysis for both research and production planning. --- Cargo.lock | 142 +++++++- Cargo.toml | 4 + crates/litesvm-testing/Cargo.toml | 12 +- ...ransfer.rs => cu_bench_sol_transfer_ix.rs} | 30 +- ...ransfer.rs => cu_bench_spl_transfer_ix.rs} | 82 +++-- .../benches/cu_bench_token_setup_tx.rs | 165 +++++++++ .../src/cu_bench/DESIGN_NOTES.md | 94 +++++ .../litesvm-testing/src/cu_bench/context.rs | 218 ++++++++++++ .../litesvm-testing/src/cu_bench/estimate.rs | 324 ++++++++++++++++++ crates/litesvm-testing/src/cu_bench/mod.rs | 241 ++----------- crates/litesvm-testing/src/cu_bench/runner.rs | 128 +++++++ crates/litesvm-testing/src/lib.rs | 5 + 12 files changed, 1196 insertions(+), 249 deletions(-) rename crates/litesvm-testing/benches/{cu_bench_sol_transfer.rs => cu_bench_sol_transfer_ix.rs} (67%) rename crates/litesvm-testing/benches/{cu_bench_spl_transfer.rs => cu_bench_spl_transfer_ix.rs} (64%) create mode 100644 crates/litesvm-testing/benches/cu_bench_token_setup_tx.rs create mode 100644 crates/litesvm-testing/src/cu_bench/DESIGN_NOTES.md create mode 100644 crates/litesvm-testing/src/cu_bench/context.rs create mode 100644 crates/litesvm-testing/src/cu_bench/estimate.rs create mode 100644 crates/litesvm-testing/src/cu_bench/runner.rs diff --git a/Cargo.lock b/Cargo.lock index fc43e2e..cdf544d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,6 +293,56 @@ dependencies = [ "winapi", ] +[[package]] +name = "anstream" +version = "0.6.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" version = "1.0.98" @@ -753,6 +803,12 @@ dependencies = [ "inout", ] +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "combine" version = "3.8.1" @@ -1031,6 +1087,16 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + [[package]] name = "env_logger" version = "0.9.3" @@ -1044,6 +1110,19 @@ dependencies = [ "termcolor", ] +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1552,6 +1631,12 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -1585,6 +1670,30 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jiff" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -1750,11 +1859,15 @@ name = "litesvm-testing" version = "0.1.1" dependencies = [ "chrono", + "env_logger 0.11.8", "litesvm", + "log", "num-traits", "serde", "serde_json", + "solana-clock", "solana-compute-budget-interface", + "solana-hash", "solana-instruction", "solana-keypair", "solana-message", @@ -1978,6 +2091,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -2141,6 +2260,21 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.2" @@ -3399,7 +3533,7 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db8e777ec1afd733939b532a42492d888ec7c88d8b4127a5d867eb45c6eb5cd5" dependencies = [ - "env_logger", + "env_logger 0.9.3", "lazy_static", "libc", "log", @@ -5111,6 +5245,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/Cargo.toml b/Cargo.toml index b4d1682..799bb7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,13 +27,17 @@ anchor-lang = "0.31.1" chrono = "0.4.41" litesvm = "0.6.1" litesvm-testing = { path = "crates/litesvm-testing" } +log = "0.4.27" +env_logger = "0.11.8" num-traits = "0.2.19" pinocchio = "0.8.4" pinocchio-log = "0.4.0" pinocchio-pubkey = "0.2.4" serde = "1.0.219" serde_json = "1.0.140" +solana-clock = "2.2" solana-compute-budget-interface = "2.2" +solana-hash = "2.2" solana-instruction = "2.2" solana-keypair = "2.2" solana-message = "2.2" diff --git a/crates/litesvm-testing/Cargo.toml b/crates/litesvm-testing/Cargo.toml index 0f9d3f1..e5a1bda 100644 --- a/crates/litesvm-testing/Cargo.toml +++ b/crates/litesvm-testing/Cargo.toml @@ -29,20 +29,28 @@ cu_bench = [] pinocchio = [] [[bench]] -name = "cu_bench_sol_transfer" +name = "cu_bench_sol_transfer_ix" harness = false [[bench]] -name = "cu_bench_spl_transfer" +name = "cu_bench_spl_transfer_ix" +harness = false + +[[bench]] +name = "cu_bench_token_setup_tx" harness = false [dependencies] chrono = { workspace = true } +env_logger = { workspace = true } litesvm = { workspace = true } +log = { workspace = true } num-traits = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +solana-clock = { workspace = true } solana-compute-budget-interface = { workspace = true } +solana-hash = { workspace = true } solana-instruction = { workspace = true } solana-keypair = { workspace = true } solana-message = { workspace = true } diff --git a/crates/litesvm-testing/benches/cu_bench_sol_transfer.rs b/crates/litesvm-testing/benches/cu_bench_sol_transfer_ix.rs similarity index 67% rename from crates/litesvm-testing/benches/cu_bench_sol_transfer.rs rename to crates/litesvm-testing/benches/cu_bench_sol_transfer_ix.rs index f825e55..206d2ad 100644 --- a/crates/litesvm-testing/benches/cu_bench_sol_transfer.rs +++ b/crates/litesvm-testing/benches/cu_bench_sol_transfer_ix.rs @@ -1,6 +1,7 @@ use litesvm::LiteSVM; use litesvm_testing::cu_bench::{benchmark_instruction, InstructionBenchmark}; use litesvm_testing::prelude::*; +use log::info; use solana_instruction::Instruction; use solana_keypair::Keypair; use solana_pubkey::Pubkey; @@ -54,27 +55,40 @@ impl InstructionBenchmark for SolTransferBenchmark { unsigned_tx.sign(&signers, unsigned_tx.message.recent_blockhash); unsigned_tx } + + fn address_book(&self) -> std::collections::HashMap { + let mut book = std::collections::HashMap::new(); + book.insert( + solana_system_interface::program::ID, + "system_program".to_string(), + ); + book.insert(self.sender.pubkey(), "sender".to_string()); + book.insert(self.recipient.pubkey(), "recipient".to_string()); + book + } } fn main() { - println!("=== SOL Transfer CU Benchmark ==="); + env_logger::init(); + info!("=== SOL Transfer CU Benchmark ==="); let benchmark = SolTransferBenchmark::new(); - let estimate = benchmark_instruction(benchmark, 100); + let result = benchmark_instruction(benchmark, 100); - println!( + info!( "Measured {} samples: {} CU ({}% variance)", - estimate.sample_size, - estimate.balanced, - if estimate.min == estimate.unsafe_max { + result.cu_estimate.sample_size, + result.cu_estimate.balanced, + if result.cu_estimate.min == result.cu_estimate.unsafe_max { 0 } else { - ((estimate.unsafe_max - estimate.min) * 100) / estimate.balanced + ((result.cu_estimate.unsafe_max - result.cu_estimate.min) * 100) + / result.cu_estimate.balanced } ); println!( "{}", - serde_json::to_string_pretty(&estimate).expect("Failed to serialize") + serde_json::to_string_pretty(&result).expect("Failed to serialize") ); } diff --git a/crates/litesvm-testing/benches/cu_bench_spl_transfer.rs b/crates/litesvm-testing/benches/cu_bench_spl_transfer_ix.rs similarity index 64% rename from crates/litesvm-testing/benches/cu_bench_spl_transfer.rs rename to crates/litesvm-testing/benches/cu_bench_spl_transfer_ix.rs index d48ab9b..55d47c7 100644 --- a/crates/litesvm-testing/benches/cu_bench_spl_transfer.rs +++ b/crates/litesvm-testing/benches/cu_bench_spl_transfer_ix.rs @@ -1,11 +1,17 @@ +use std::collections::HashMap; + use litesvm::LiteSVM; use litesvm_testing::cu_bench::{benchmark_instruction, InstructionBenchmark}; use litesvm_testing::prelude::*; +use log::info; use solana_instruction::Instruction; use solana_keypair::Keypair; use solana_pubkey::Pubkey; use solana_signer::Signer; +use solana_system_interface::instruction::create_account; use solana_transaction::Transaction; +use spl_associated_token_account::instruction::create_associated_token_account; +use spl_token::instruction::{initialize_mint, mint_to}; use spl_token::solana_program::program_pack::Pack; /// SPL token transfer benchmark using the new framework @@ -29,6 +35,7 @@ impl SplTokenTransferBenchmark { &sender.pubkey(), &mint.pubkey(), ); + let recipient_ata = spl_associated_token_account::get_associated_token_address( &recipient.pubkey(), &mint.pubkey(), @@ -61,7 +68,7 @@ impl InstructionBenchmark for SplTokenTransferBenchmark { svm.airdrop(&self.recipient.pubkey(), 10_000_000).unwrap(); // Create mint account - let create_mint_account_ix = solana_system_interface::instruction::create_account( + let create_mint_account_ix = create_account( &self.mint_authority.pubkey(), &self.mint.pubkey(), 5_000_000, // Rent exemption @@ -70,7 +77,7 @@ impl InstructionBenchmark for SplTokenTransferBenchmark { ); // Initialize mint - let create_mint_ix = spl_token::instruction::initialize_mint( + let create_mint_ix = initialize_mint( &spl_token::ID, &self.mint.pubkey(), &self.mint_authority.pubkey(), @@ -80,24 +87,22 @@ impl InstructionBenchmark for SplTokenTransferBenchmark { .unwrap(); // Create associated token accounts - let create_sender_ata_ix = - spl_associated_token_account::instruction::create_associated_token_account( - &self.mint_authority.pubkey(), - &self.sender.pubkey(), - &self.mint.pubkey(), - &spl_token::ID, - ); - - let create_recipient_ata_ix = - spl_associated_token_account::instruction::create_associated_token_account( - &self.mint_authority.pubkey(), - &self.recipient.pubkey(), - &self.mint.pubkey(), - &spl_token::ID, - ); + let create_sender_ata_ix = create_associated_token_account( + &self.mint_authority.pubkey(), + &self.sender.pubkey(), + &self.mint.pubkey(), + &spl_token::ID, + ); + + let create_recipient_ata_ix = create_associated_token_account( + &self.mint_authority.pubkey(), + &self.recipient.pubkey(), + &self.mint.pubkey(), + &spl_token::ID, + ); // Mint tokens to sender (enough for 200+ transfers) - let mint_to_ix = spl_token::instruction::mint_to( + let mint_to_ix = mint_to( &spl_token::ID, &self.mint.pubkey(), &self.sender_ata, @@ -108,7 +113,7 @@ impl InstructionBenchmark for SplTokenTransferBenchmark { .unwrap(); // Execute setup transaction - let setup_tx = solana_transaction::Transaction::new_signed_with_payer( + let setup_tx = Transaction::new_signed_with_payer( &[ create_mint_account_ix, create_mint_ix, @@ -122,6 +127,7 @@ impl InstructionBenchmark for SplTokenTransferBenchmark { ); svm.send_transaction(setup_tx).unwrap(); + svm } @@ -145,27 +151,49 @@ impl InstructionBenchmark for SplTokenTransferBenchmark { unsigned_tx.sign(&signers, unsigned_tx.message.recent_blockhash); unsigned_tx } + + fn address_book(&self) -> HashMap { + HashMap::from_iter(vec![ + (spl_token::ID, "spl_token".to_string()), + ( + spl_associated_token_account::ID, + "spl_associated_token_account".to_string(), + ), + (self.mint.pubkey(), "test_mint".to_string()), + (self.sender_ata, "sender_ata".to_string()), + (self.recipient_ata, "recipient_ata".to_string()), + (self.sender.pubkey(), "sender".to_string()), + (self.recipient.pubkey(), "recipient".to_string()), + (self.mint_authority.pubkey(), "mint_authority".to_string()), + ( + solana_system_interface::program::ID, + "system_program".to_string(), + ), + ]) + } } fn main() { - println!("=== SPL Token Transfer CU Benchmark ==="); + env_logger::init(); + info!("=== SPL Token Transfer CU Benchmark ==="); let benchmark = SplTokenTransferBenchmark::new(); - let estimate = benchmark_instruction(benchmark, 100); + let result = benchmark_instruction(benchmark, 100); - println!( + info!( "Measured {} samples: {} CU ({}% variance)", - estimate.sample_size, - estimate.balanced, - if estimate.min == estimate.unsafe_max { + result.cu_estimate.sample_size, + result.cu_estimate.balanced, + if result.cu_estimate.min == result.cu_estimate.unsafe_max { 0 } else { - ((estimate.unsafe_max - estimate.min) * 100) / estimate.balanced + ((result.cu_estimate.unsafe_max - result.cu_estimate.min) * 100) + / result.cu_estimate.balanced } ); println!( "{}", - serde_json::to_string_pretty(&estimate).expect("Failed to serialize") + serde_json::to_string_pretty(&result).expect("Failed to serialize") ); } diff --git a/crates/litesvm-testing/benches/cu_bench_token_setup_tx.rs b/crates/litesvm-testing/benches/cu_bench_token_setup_tx.rs new file mode 100644 index 0000000..1115ea2 --- /dev/null +++ b/crates/litesvm-testing/benches/cu_bench_token_setup_tx.rs @@ -0,0 +1,165 @@ +use std::collections::HashMap; + +use litesvm_testing::prelude::*; + +use litesvm::LiteSVM; +use litesvm_testing::cu_bench::{benchmark_transaction, TransactionBenchmark}; +use log::info; +use solana_compute_budget_interface::ComputeBudgetInstruction; +use solana_message::Message; +use solana_transaction::Transaction; +use spl_token::solana_program::program_pack::Pack; + +/// Benchmark for a complete token setup transaction +/// This represents a realistic workflow: create mint + ATA + mint initial supply +struct TokenSetupTransactionBenchmark { + mint_authority: Keypair, + mint: Keypair, + token_account_owner: Keypair, + mint_amount: u64, +} + +impl TokenSetupTransactionBenchmark { + fn new() -> Self { + let mint_authority = Keypair::new(); + let mint = Keypair::new(); + let token_account_owner = Keypair::new(); + + Self { + mint_authority, + mint, + token_account_owner, + mint_amount: 1_000_000, // 1M tokens with 6 decimals + } + } +} + +impl TransactionBenchmark for TokenSetupTransactionBenchmark { + fn transaction_name(&self) -> &'static str { + "token_setup_complete" + } + + fn setup_svm(&self) -> LiteSVM { + // Create and configure SVM with necessary accounts + let mut svm = LiteSVM::new(); + + // Airdrop SOL to accounts that need to pay fees/rent (generous amounts for many measurements) + svm.airdrop(&self.mint_authority.pubkey(), 1_000_000_000) + .unwrap(); // 10 SOL + svm.airdrop(&self.token_account_owner.pubkey(), 1_000_000_000) + .unwrap(); // 10 SOL + + svm + } + + fn build_transaction(&mut self, svm: &mut LiteSVM) -> Transaction { + // Use a fresh mint keypair for each transaction to avoid "account already exists" errors + self.mint = Keypair::new(); + + // Get fresh blockhash from the provided SVM + svm.expire_blockhash(); + let recent_blockhash = svm.latest_blockhash(); + + // Calculate rent for mint account + let mint_rent = svm.minimum_balance_for_rent_exemption(spl_token::state::Mint::LEN); + + // Get associated token account address + let ata_address = spl_associated_token_account::get_associated_token_address( + &self.token_account_owner.pubkey(), + &self.mint.pubkey(), + ); + + // Build all instructions for the transaction + let instructions = vec![ + // 1. Set compute unit limit + ComputeBudgetInstruction::set_compute_unit_limit(150_000), + // 2. Create mint account + solana_system_interface::instruction::create_account( + &self.mint_authority.pubkey(), + &self.mint.pubkey(), + mint_rent, + spl_token::state::Mint::LEN as u64, + &spl_token::ID, + ), + // 3. Initialize mint + spl_token::instruction::initialize_mint( + &spl_token::ID, + &self.mint.pubkey(), + &self.mint_authority.pubkey(), + Some(&self.mint_authority.pubkey()), + 6, // decimals + ) + .unwrap(), + // 4. Create associated token account + spl_associated_token_account::instruction::create_associated_token_account( + &self.token_account_owner.pubkey(), // fee payer + &self.token_account_owner.pubkey(), // wallet + &self.mint.pubkey(), + &spl_token::ID, + ), + // 5. Mint initial supply to the ATA + spl_token::instruction::mint_to( + &spl_token::ID, + &self.mint.pubkey(), + &ata_address, + &self.mint_authority.pubkey(), + &[&self.mint_authority.pubkey()], + self.mint_amount, + ) + .unwrap(), + ]; + + // Create and sign transaction + let message = Message::new(&instructions, Some(&self.mint_authority.pubkey())); + let mut transaction = Transaction::new_unsigned(message); + transaction.message.recent_blockhash = recent_blockhash; + + // Sign with all required signers + transaction.sign( + &[&self.mint_authority, &self.mint, &self.token_account_owner], + recent_blockhash, + ); + + transaction + } + + fn address_book(&self) -> HashMap { + HashMap::from_iter(vec![ + (system_program::ID, "system_program".to_string()), + (spl_token::ID, "spl_token".to_string()), + ( + spl_associated_token_account::ID, + "spl_associated_token_account".to_string(), + ), + ( + solana_compute_budget_interface::ID, + "compute_budget".to_string(), + ), + ]) + } +} + +fn main() { + env_logger::init(); + info!("=== Token Setup Transaction CU Benchmark ==="); + + let benchmark = TokenSetupTransactionBenchmark::new(); + let result = benchmark_transaction(benchmark, 100); + + info!( + "Measured {} samples: {} CU ({}% variance)", + result.cu_estimate.sample_size, + result.cu_estimate.balanced, + if result.cu_estimate.min == result.cu_estimate.unsafe_max { + 0 + } else { + ((result.cu_estimate.unsafe_max - result.cu_estimate.min) * 100) + / result.cu_estimate.balanced + } + ); + + println!( + "{}", + serde_json::to_string_pretty(&result).expect("Failed to serialize") + ); +} diff --git a/crates/litesvm-testing/src/cu_bench/DESIGN_NOTES.md b/crates/litesvm-testing/src/cu_bench/DESIGN_NOTES.md new file mode 100644 index 0000000..b9d81f5 --- /dev/null +++ b/crates/litesvm-testing/src/cu_bench/DESIGN_NOTES.md @@ -0,0 +1,94 @@ +# CU Benchmarking Framework - Design Notes + +## Core Principles + +### Why Systematic CU Benchmarking? + +- **Gap in Ecosystem**: Existing tools focus on RPC performance/tx success, not instruction-level CU optimization +- **Developer Pain Point**: Manual CU testing is ad-hoc, non-reproducible, and time-consuming +- **Economic Impact**: Better CU estimates → lower fees → more efficient network usage + +### Design Philosophy + +- **Helius-Inspired Confidence Levels**: Familiar model from priority fee API +- **Reproducible by Default**: Same benchmark should give same results across environments +- **Context-Aware**: Environment/program context affects CU usage significantly +- **Community-Extensible**: Easy for developers to add their own benchmarks + +## Methodology Decisions + +### Measurement Strategy + +- **Stateful Accumulation**: SVM maintains state across measurements (more realistic) +- **Fresh Blockhashes**: Each measurement gets new blockhash to avoid AlreadyProcessed errors +- **Statistical Approach**: Multiple samples → percentile-based estimates +- **Framework Responsibility**: Framework handles tx construction, signing, and CU extraction + +### Environment Standardization + +- **Feature Set Tracking**: LiteSVM features affect CU (see feature list) +- **Version Pinning**: Solana version changes can affect CU usage +- **Program Dependencies**: Track what programs are loaded/interacted with + +### Benchmark Separation + +- **Instruction-Level**: Single instruction CU usage (current focus) +- **Transaction-Level**: Multiple instructions + interaction effects (future) +- **Scenario-Based**: Real-world usage patterns (future) + +## Current API Design + +### InstructionBenchmark Trait + +```rust +pub trait InstructionBenchmark { + fn instruction_name(&self) -> &'static str; + fn setup_svm(&self) -> LiteSVM; + fn build_instruction(&self, svm: &mut LiteSVM) -> (Instruction, Vec); + fn sign_transaction(&self, unsigned_tx: Transaction) -> Transaction; +} +``` + +**Key Design Decisions:** + +- **Benchmark owns setup**: Each benchmark controls its SVM environment +- **Framework owns measurement**: Consistent CU extraction across all benchmarks +- **Separation of concerns**: Benchmark builds instruction, framework measures CU + +## Future API Evolution + +### TransactionBenchmark (Future) + +```rust +pub trait TransactionBenchmark { + fn transaction_name(&self) -> &'static str; + fn setup_svm(&self) -> LiteSVM; + fn build_transaction(&self, svm: &mut LiteSVM) -> (Transaction, Vec); + // Note: Different from instruction - builds full transaction +} +``` + +### Unified Benchmark Enum (Consideration) + +```rust +pub enum BenchmarkType<'a> { + Instruction(&'a dyn InstructionBenchmark), + Transaction(&'a dyn TransactionBenchmark), +} +``` + +## Open Questions/TODOs + +- [ ] How to handle CPI effects in instruction benchmarks? +- [ ] Should we validate benchmark reproducibility automatically? +- [ ] Database/sharing mechanism for community estimates? +- [ ] How to handle version evolution of programs? + +## Feature Impact Considerations + +Based on LiteSVM feature set, these could affect CU measurements: + +- Signature verification settings +- Precompiled programs enabled +- SPL program availability +- Feature gates (curve25519, etc.) diff --git a/crates/litesvm-testing/src/cu_bench/context.rs b/crates/litesvm-testing/src/cu_bench/context.rs new file mode 100644 index 0000000..f5e3999 --- /dev/null +++ b/crates/litesvm-testing/src/cu_bench/context.rs @@ -0,0 +1,218 @@ +use std::collections::HashMap; + +use litesvm::{types::SimulatedTransactionInfo, LiteSVM}; +use serde::{Deserialize, Serialize}; +use solana_hash::Hash; +use solana_message::Message; +use solana_pubkey::Pubkey; +use solana_transaction::Transaction; + +use crate::cu_bench::InstructionBenchmark; + +/// Execution context discovered through simulation (for instructions) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InstructionExecutionContext { + pub svm_context: SVMContext, + pub program_context: ProgramContext, + pub execution_stats: ExecutionStats, +} + +/// Execution context discovered through simulation (for transactions/workflows) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TransactionExecutionContext { + pub svm_context: SVMContext, + pub workflow_context: WorkflowContext, + pub execution_stats: ExecutionStats, +} + +/// SVM environment context +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SVMContext { + pub current_slot: u64, + #[serde(serialize_with = "serialize_hash")] + pub latest_blockhash: Hash, + // Future additions when available: + // pub feature_set: Option, + // pub compute_budget: Option, + // pub rent_config: Option, +} + +/// Information about the primary program and its dependencies +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProgramContext { + #[serde(serialize_with = "serialize_pubkey")] + pub program_id: Pubkey, + pub program_name: String, + pub cpi_count: usize, +} + +/// Information about a multi-program workflow (for transactions) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WorkflowContext { + pub workflow_name: String, + pub involved_programs: Vec, + pub cpi_sequence: Vec, + pub total_cpi_calls: usize, +} + +/// Information about a program involved in a workflow +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProgramInfo { + #[serde(serialize_with = "serialize_pubkey")] + pub program_id: Pubkey, + pub program_name: String, + pub instruction_count: usize, // How many instructions call this program +} + +/// Statistics about the instruction execution +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExecutionStats { + pub logs: Vec, + pub simulated_cu: u64, +} + +/// Discover execution context by simulating the pure instruction +pub fn discover_instruction_context( + benchmark: &T, + svm: &mut LiteSVM, +) -> InstructionExecutionContext { + let (target_ix, signer_pubkeys) = benchmark.build_instruction(svm); + + // Build transaction with just the target instruction (no CU budget) + svm.expire_blockhash(); + + let message = Message::new(&[target_ix], Some(&signer_pubkeys[0])); + let mut unsigned_tx = Transaction::new_unsigned(message); + unsigned_tx.message.recent_blockhash = svm.latest_blockhash(); + let signed_tx = benchmark.sign_transaction(unsigned_tx); + + // Simulate to extract context + let simulation = svm.simulate_transaction(signed_tx.clone()).unwrap(); + let address_book = benchmark.address_book(); + + InstructionExecutionContext { + svm_context: SVMContext { + current_slot: svm.get_sysvar::().slot, + latest_blockhash: svm.latest_blockhash(), + }, + program_context: extract_program_context(&signed_tx, &simulation, &address_book), + execution_stats: extract_execution_stats(&simulation), + } +} + +fn extract_program_context( + transaction: &Transaction, + simulation: &SimulatedTransactionInfo, + address_book: &HashMap, +) -> ProgramContext { + let target_instruction = &transaction.message.instructions[0]; // Only instruction + let program_id = transaction.message.account_keys[target_instruction.program_id_index as usize]; + + ProgramContext { + program_id, + program_name: lookup_program_name(program_id, address_book), + cpi_count: simulation.meta.inner_instructions.len(), + } +} + +fn extract_execution_stats(simulation: &SimulatedTransactionInfo) -> ExecutionStats { + ExecutionStats { + logs: simulation.meta.logs.clone(), + simulated_cu: simulation.meta.compute_units_consumed, + } +} + +fn lookup_program_name(program_id: Pubkey, address_book: &HashMap) -> String { + address_book + .get(&program_id) + .cloned() + .unwrap_or_else(|| program_id.to_string()) +} + +/// Discover execution context for a transaction workflow +pub fn discover_transaction_context( + transaction: &Transaction, + workflow_name: String, + svm: &mut LiteSVM, + address_book: &HashMap, +) -> TransactionExecutionContext { + // Simulate the transaction to extract context + let simulation = svm.simulate_transaction(transaction.clone()).unwrap(); + + // Extract workflow context from the transaction and simulation + let workflow_context = + extract_workflow_context(transaction, &simulation, workflow_name, address_book); + + TransactionExecutionContext { + svm_context: SVMContext { + current_slot: svm.get_sysvar::().slot, + latest_blockhash: svm.latest_blockhash(), + }, + workflow_context, + execution_stats: extract_execution_stats(&simulation), + } +} + +fn extract_workflow_context( + transaction: &Transaction, + simulation: &SimulatedTransactionInfo, + workflow_name: String, + address_book: &HashMap, +) -> WorkflowContext { + // Extract all unique programs involved + let mut program_usage: HashMap = HashMap::new(); + let mut cpi_sequence: Vec = Vec::new(); + + // Count direct instruction calls + for instruction in &transaction.message.instructions { + let program_id = transaction.message.account_keys[instruction.program_id_index as usize]; + *program_usage.entry(program_id).or_insert(0) += 1; + + let program_name = lookup_program_name(program_id, address_book); + cpi_sequence.push(program_name); + } + + // Add CPI calls from simulation logs (extracted from inner instructions) + for inner_instruction_set in &simulation.meta.inner_instructions { + for inner_instruction in inner_instruction_set { + let program_id = transaction.message.account_keys + [inner_instruction.instruction.program_id_index as usize]; + *program_usage.entry(program_id).or_insert(0) += 1; + + let program_name = lookup_program_name(program_id, address_book); + cpi_sequence.push(format!("{}_cpi", program_name)); + } + } + + // Convert to program info list + let involved_programs: Vec = program_usage + .into_iter() + .map(|(program_id, instruction_count)| ProgramInfo { + program_id, + program_name: lookup_program_name(program_id, address_book), + instruction_count, + }) + .collect(); + + WorkflowContext { + workflow_name, + involved_programs, + cpi_sequence, + total_cpi_calls: simulation.meta.inner_instructions.len(), + } +} + +// Custom serialization helpers for better display +fn serialize_hash(hash: &Hash, serializer: S) -> Result +where + S: serde::Serializer, +{ + serializer.serialize_str(&hash.to_string()) +} + +fn serialize_pubkey(pubkey: &Pubkey, serializer: S) -> Result +where + S: serde::Serializer, +{ + serializer.serialize_str(&pubkey.to_string()) +} diff --git a/crates/litesvm-testing/src/cu_bench/estimate.rs b/crates/litesvm-testing/src/cu_bench/estimate.rs new file mode 100644 index 0000000..1b2cf34 --- /dev/null +++ b/crates/litesvm-testing/src/cu_bench/estimate.rs @@ -0,0 +1,324 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use super::context::InstructionExecutionContext; + +/// Type of benchmark being measured +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "benchmark_type", content = "benchmark_name")] +pub enum StatType { + #[serde(rename = "instruction")] + Instruction(String), + #[serde(rename = "transaction")] + Transaction(String), +} + +/// Enhanced benchmark result with execution context +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InstructionBenchmarkResult { + pub instruction_name: String, + pub cu_estimate: ComputeUnitStats, + pub execution_context: InstructionExecutionContext, + pub generated_at: String, + pub generated_by: String, +} + +/// Confidence level for CU estimates, similar to Helius Priority Fee API levels +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub enum ComputeUnitLevel { + /// Minimum observed CU usage (0th percentile) - absolute minimum + Min, + /// Conservative estimate (25th percentile) - safe for most cases + Conservative, + /// Balanced estimate (50th percentile) - good default + Balanced, + /// Safe estimate (75th percentile) - high reliability + Safe, + /// Very high estimate (95th percentile) - very reliable + VeryHigh, + /// Maximum observed (100th percentile) - may be unnecessarily high + UnsafeMax, + /// Custom CU value for exact control + Custom(u64), + /// Apply multiplier to balanced estimate + Multiplier(f32), +} + +/// CU usage statistics for a specific benchmark type +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComputeUnitStats { + /// Type and name of the benchmark + #[serde(flatten)] + pub stat_type: StatType, + /// Minimum observed CU usage (0th percentile) + pub min: u64, + /// Conservative estimate (25th percentile) + pub conservative: u64, + /// Balanced estimate (50th percentile) + pub balanced: u64, + /// Safe estimate (75th percentile) + pub safe: u64, + /// Very high estimate (95th percentile) + pub very_high: u64, + /// Maximum observed CU usage (100th percentile) + pub unsafe_max: u64, + /// Number of samples used to generate this estimate + pub sample_size: usize, +} + +impl ComputeUnitStats { + /// Get CU estimate for the specified confidence level + pub fn get_cu_for_level(&self, level: ComputeUnitLevel) -> u64 { + match level { + ComputeUnitLevel::Min => self.min, + ComputeUnitLevel::Conservative => self.conservative, + ComputeUnitLevel::Balanced => self.balanced, + ComputeUnitLevel::Safe => self.safe, + ComputeUnitLevel::VeryHigh => self.very_high, + ComputeUnitLevel::UnsafeMax => self.unsafe_max, + ComputeUnitLevel::Custom(cu) => cu, + ComputeUnitLevel::Multiplier(mult) => (self.balanced as f32 * mult) as u64, + } + } + + /// Create estimate from a series of CU measurements + pub fn from_measurements(stat_type: StatType, measurements: &[u64]) -> Self { + let mut sorted = measurements.to_vec(); + sorted.sort_unstable(); + + let len = sorted.len(); + let min = sorted[0]; + let unsafe_max = sorted[len - 1]; + + // Calculate percentiles (use len-1 for proper indexing) + let conservative = sorted[(len - 1) * 25 / 100]; + let balanced = sorted[(len - 1) * 50 / 100]; + let safe = sorted[(len - 1) * 75 / 100]; + let very_high = sorted[(len - 1) * 95 / 100]; + + Self { + stat_type, + min, + conservative, + balanced, + safe, + very_high, + unsafe_max, + sample_size: len, + } + } +} + +/// Database of CU estimates for different instruction types +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComputeUnitDatabase { + pub estimates: HashMap, + pub generated_at: String, // ISO timestamp +} + +impl ComputeUnitDatabase { + /// Create new empty database + pub fn new() -> Self { + Self { + estimates: HashMap::new(), + generated_at: chrono::Utc::now().to_rfc3339(), + } + } + + /// Get estimate for instruction type + pub fn get_estimate(&self, instruction_type: &str) -> Option<&ComputeUnitStats> { + self.estimates.get(instruction_type) + } + + /// Get CU estimate for instruction type at specified level + pub fn get_cu_estimate(&self, instruction_type: &str, level: ComputeUnitLevel) -> Option { + self.get_estimate(instruction_type) + .map(|est| est.get_cu_for_level(level)) + } +} + +impl Default for ComputeUnitDatabase { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_percentiles_simple_case() { + // Test with 100 values: 1, 2, 3, ..., 100 + let measurements: Vec = (1..=100).collect(); + let stats = ComputeUnitStats::from_measurements( + StatType::Instruction("test".to_string()), + &measurements, + ); + + // For 1-100, percentiles should be: + // 25th percentile: index (100-1)*25/100 = 99*25/100 = 24 -> value 25 + // 50th percentile: index (100-1)*50/100 = 99*50/100 = 49 -> value 50 + // 75th percentile: index (100-1)*75/100 = 99*75/100 = 74 -> value 75 + // 95th percentile: index (100-1)*95/100 = 99*95/100 = 94 -> value 95 + + assert_eq!(stats.min, 1); + assert_eq!(stats.conservative, 25); + assert_eq!(stats.balanced, 50); + assert_eq!(stats.safe, 75); + assert_eq!(stats.very_high, 95); + assert_eq!(stats.unsafe_max, 100); + assert_eq!(stats.sample_size, 100); + } + + #[test] + fn test_percentiles_small_dataset() { + // Test with 4 values: [10, 20, 30, 40] + let measurements = vec![10, 20, 30, 40]; + let stats = ComputeUnitStats::from_measurements( + StatType::Transaction("small_test".to_string()), + &measurements, + ); + + // For 4 values, percentiles should be: + // 25th percentile: index (4-1)*25/100 = 3*25/100 = 0 -> value 10 + // 50th percentile: index (4-1)*50/100 = 3*50/100 = 1 -> value 20 + // 75th percentile: index (4-1)*75/100 = 3*75/100 = 2 -> value 30 + // 95th percentile: index (4-1)*95/100 = 3*95/100 = 2 -> value 30 + + assert_eq!(stats.min, 10); + assert_eq!(stats.conservative, 10); + assert_eq!(stats.balanced, 20); + assert_eq!(stats.safe, 30); + assert_eq!(stats.very_high, 30); + assert_eq!(stats.unsafe_max, 40); + assert_eq!(stats.sample_size, 4); + } + + #[test] + fn test_percentiles_single_value() { + let measurements = vec![42]; + let stats = ComputeUnitStats::from_measurements( + StatType::Instruction("single".to_string()), + &measurements, + ); + + // All percentiles should be the same value + assert_eq!(stats.min, 42); + assert_eq!(stats.conservative, 42); + assert_eq!(stats.balanced, 42); + assert_eq!(stats.safe, 42); + assert_eq!(stats.very_high, 42); + assert_eq!(stats.unsafe_max, 42); + assert_eq!(stats.sample_size, 1); + } + + #[test] + fn test_percentiles_duplicate_values() { + // Test with duplicates: [5, 5, 5, 10, 10, 15, 20, 20, 20, 20] + let measurements = vec![5, 5, 5, 10, 10, 15, 20, 20, 20, 20]; + let stats = ComputeUnitStats::from_measurements( + StatType::Transaction("duplicates".to_string()), + &measurements, + ); + + // Sorted: [5, 5, 5, 10, 10, 15, 20, 20, 20, 20] + // Indices: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 + // 25th percentile: index (10-1)*25/100 = 9*25/100 = 2 -> value 5 + // 50th percentile: index (10-1)*50/100 = 9*50/100 = 4 -> value 10 + // 75th percentile: index (10-1)*75/100 = 9*75/100 = 6 -> value 20 + // 95th percentile: index (10-1)*95/100 = 9*95/100 = 8 -> value 20 + + assert_eq!(stats.min, 5); + assert_eq!(stats.conservative, 5); + assert_eq!(stats.balanced, 10); + assert_eq!(stats.safe, 20); + assert_eq!(stats.very_high, 20); + assert_eq!(stats.unsafe_max, 20); + assert_eq!(stats.sample_size, 10); + } + + #[test] + fn test_percentiles_unsorted_input() { + // Test that input gets sorted correctly + let measurements = vec![100, 10, 50, 30, 80, 20, 90, 40, 70, 60]; + let stats = ComputeUnitStats::from_measurements( + StatType::Instruction("unsorted".to_string()), + &measurements, + ); + + // Should be sorted to: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100] + // 25th percentile: index (10-1)*25/100 = 2 -> value 30 + // 50th percentile: index (10-1)*50/100 = 4 -> value 50 + // 75th percentile: index (10-1)*75/100 = 6 -> value 70 + // 95th percentile: index (10-1)*95/100 = 8 -> value 90 + + assert_eq!(stats.min, 10); + assert_eq!(stats.conservative, 30); + assert_eq!(stats.balanced, 50); + assert_eq!(stats.safe, 70); + assert_eq!(stats.very_high, 90); + assert_eq!(stats.unsafe_max, 100); + assert_eq!(stats.sample_size, 10); + } + + #[test] + fn test_stat_type_serialization() { + let instruction_stats = ComputeUnitStats::from_measurements( + StatType::Instruction("test_instruction".to_string()), + &[100, 200, 300], + ); + + let transaction_stats = ComputeUnitStats::from_measurements( + StatType::Transaction("test_transaction".to_string()), + &[100, 200, 300], + ); + + // Test that we can serialize/deserialize the stat types + let instruction_json = serde_json::to_string(&instruction_stats).unwrap(); + let transaction_json = serde_json::to_string(&transaction_stats).unwrap(); + + assert!(instruction_json.contains("\"benchmark_type\":\"instruction\"")); + assert!(instruction_json.contains("\"benchmark_name\":\"test_instruction\"")); + + assert!(transaction_json.contains("\"benchmark_type\":\"transaction\"")); + assert!(transaction_json.contains("\"benchmark_name\":\"test_transaction\"")); + } + + #[test] + fn test_get_cu_for_level() { + let measurements = vec![10, 20, 30, 40, 50]; + let stats = ComputeUnitStats::from_measurements( + StatType::Instruction("level_test".to_string()), + &measurements, + ); + + assert_eq!(stats.get_cu_for_level(ComputeUnitLevel::Min), stats.min); + assert_eq!( + stats.get_cu_for_level(ComputeUnitLevel::Conservative), + stats.conservative + ); + assert_eq!( + stats.get_cu_for_level(ComputeUnitLevel::Balanced), + stats.balanced + ); + assert_eq!(stats.get_cu_for_level(ComputeUnitLevel::Safe), stats.safe); + assert_eq!( + stats.get_cu_for_level(ComputeUnitLevel::VeryHigh), + stats.very_high + ); + assert_eq!( + stats.get_cu_for_level(ComputeUnitLevel::UnsafeMax), + stats.unsafe_max + ); + assert_eq!(stats.get_cu_for_level(ComputeUnitLevel::Custom(999)), 999); + + // Test multiplier (2x balanced) + let expected_multiplied = (stats.balanced as f32 * 2.0) as u64; + assert_eq!( + stats.get_cu_for_level(ComputeUnitLevel::Multiplier(2.0)), + expected_multiplied + ); + } +} diff --git a/crates/litesvm-testing/src/cu_bench/mod.rs b/crates/litesvm-testing/src/cu_bench/mod.rs index 39448df..28f182d 100644 --- a/crates/litesvm-testing/src/cu_bench/mod.rs +++ b/crates/litesvm-testing/src/cu_bench/mod.rs @@ -6,13 +6,25 @@ use std::collections::HashMap; -#[cfg(feature = "cu_bench")] -use serde::{Deserialize, Serialize}; - use litesvm::LiteSVM; use solana_instruction::Instruction; +use solana_pubkey::Pubkey; use solana_transaction::Transaction; +pub mod context; +pub mod estimate; +pub mod runner; + +// Re-export main types for convenience +pub use context::{ + ExecutionStats, InstructionExecutionContext, ProgramContext, ProgramInfo, SVMContext, + TransactionExecutionContext, WorkflowContext, +}; +pub use estimate::{ + ComputeUnitDatabase, ComputeUnitLevel, ComputeUnitStats, InstructionBenchmarkResult, StatType, +}; +pub use runner::{benchmark_instruction, benchmark_transaction, TransactionBenchmarkResult}; + /// Trait for benchmarking the CU usage of specific instructions pub trait InstructionBenchmark { /// Human-readable name for this instruction type @@ -22,223 +34,30 @@ pub trait InstructionBenchmark { fn setup_svm(&self) -> LiteSVM; /// Build the instruction to measure, returning instruction and required signer pubkeys - fn build_instruction(&self, svm: &mut LiteSVM) -> (Instruction, Vec); + fn build_instruction(&self, svm: &mut LiteSVM) -> (Instruction, Vec); /// Sign the unsigned transaction containing the instruction fn sign_transaction(&self, unsigned_tx: Transaction) -> Transaction; -} - -/// Universal benchmark runner for any instruction implementing InstructionBenchmark -pub fn benchmark_instruction( - benchmark: T, - samples: usize, -) -> ComputeUnitEstimate { - let mut cu_measurements = Vec::new(); - - // Set up SVM once - it will accumulate state across measurements - let mut svm = benchmark.setup_svm(); - - for i in 0..samples { - let cu_used = measure_instruction(&benchmark, &mut svm); - cu_measurements.push(cu_used); - - if (i + 1) % 10 == 0 { - println!("Completed {} measurements...", i + 1); - } - } - - // Create structured estimate from measurements - ComputeUnitEstimate::from_measurements( - benchmark.instruction_name().to_string(), - &cu_measurements, - vec!["litesvm".to_string()], - ) -} - -/// Measure CU usage for a single instruction -fn measure_instruction(benchmark: &T, svm: &mut LiteSVM) -> u64 { - // 1. Get target instruction and signer pubkeys from benchmark - let (target_ix, signer_pubkeys) = benchmark.build_instruction(svm); - - // 2. Framework creates unsigned transaction with CU limit - use solana_compute_budget_interface::ComputeBudgetInstruction; - let cu_limit_ix = ComputeBudgetInstruction::set_compute_unit_limit(200_000); - let instructions = vec![cu_limit_ix, target_ix]; - - // 3. Build unsigned transaction (framework responsibility) - use solana_message::Message; - - // Get fresh blockhash for each measurement to avoid AlreadyProcessed - svm.expire_blockhash(); - - let message = Message::new(&instructions, Some(&signer_pubkeys[0])); - let mut unsigned_tx = Transaction::new_unsigned(message); - unsigned_tx.message.recent_blockhash = svm.latest_blockhash(); - - // 4. Benchmark signs the transaction - let signed_tx = benchmark.sign_transaction(unsigned_tx); - - // 5. Send transaction and measure CU usage - let result = svm.send_transaction(signed_tx).unwrap(); - result.compute_units_consumed -} - -/// Confidence level for CU estimates, similar to Helius Priority Fee API levels -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -pub enum CuLevel { - /// Minimum observed CU usage (0th percentile) - absolute minimum - Min, - /// Conservative estimate (25th percentile) - safe for most cases - Conservative, - /// Balanced estimate (50th percentile) - good default - Balanced, - /// Safe estimate (75th percentile) - high reliability - Safe, - /// Very high estimate (95th percentile) - very reliable - VeryHigh, - /// Maximum observed (100th percentile) - may be unnecessarily high - UnsafeMax, - /// Custom CU value for exact control - Custom(u64), - /// Apply multiplier to balanced estimate - Multiplier(f32), -} - -/// CU usage statistics for a specific instruction type -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComputeUnitEstimate { - /// Instruction type identifier - pub instruction_type: String, - /// Minimum observed CU usage (0th percentile) - pub min: u64, - /// Conservative estimate (25th percentile) - pub conservative: u64, - /// Balanced estimate (50th percentile) - pub balanced: u64, - /// Safe estimate (75th percentile) - pub safe: u64, - /// Very high estimate (95th percentile) - pub very_high: u64, - /// Maximum observed CU usage (100th percentile) - pub unsafe_max: u64, - /// Number of samples used to generate this estimate - pub sample_size: usize, - /// Testing environments used (e.g., ["litesvm", "mollusk"]) - pub environments: Vec, -} - -impl ComputeUnitEstimate { - /// Get CU estimate for the specified confidence level - pub fn get_cu_for_level(&self, level: CuLevel) -> u64 { - match level { - CuLevel::Min => self.min, - CuLevel::Conservative => self.conservative, - CuLevel::Balanced => self.balanced, - CuLevel::Safe => self.safe, - CuLevel::VeryHigh => self.very_high, - CuLevel::UnsafeMax => self.unsafe_max, - CuLevel::Custom(cu) => cu, - CuLevel::Multiplier(mult) => (self.balanced as f32 * mult) as u64, - } - } - - /// Create estimate from a series of CU measurements - pub fn from_measurements( - instruction_type: String, - measurements: &[u64], - environments: Vec, - ) -> Self { - let mut sorted = measurements.to_vec(); - sorted.sort_unstable(); - - let len = sorted.len(); - let min = sorted[0]; - let unsafe_max = sorted[len - 1]; - - // Calculate percentiles - let conservative = sorted[len * 25 / 100]; - let balanced = sorted[len * 50 / 100]; - let safe = sorted[len * 75 / 100]; - let very_high = sorted[len * 95 / 100]; - Self { - instruction_type, - min, - conservative, - balanced, - safe, - very_high, - unsafe_max, - sample_size: len, - environments, - } + /// Provide names for programs/accounts this benchmark interacts with + fn address_book(&self) -> HashMap { + HashMap::new() } } -/// Database of CU estimates for different instruction types -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComputeUnitDatabase { - pub estimates: HashMap, - pub generated_at: String, // ISO timestamp -} - -impl ComputeUnitDatabase { - /// Create new empty database - pub fn new() -> Self { - Self { - estimates: HashMap::new(), - generated_at: chrono::Utc::now().to_rfc3339(), - } - } +/// Trait for benchmarking the CU usage of a transaction +pub trait TransactionBenchmark { + /// Human-readable name for this transaction type + fn transaction_name(&self) -> &'static str; - /// Get estimate for instruction type - pub fn get_estimate(&self, instruction_type: &str) -> Option<&ComputeUnitEstimate> { - self.estimates.get(instruction_type) - } + /// Set up SVM with necessary programs and initial state (called once per benchmark run) + fn setup_svm(&self) -> LiteSVM; - /// Get CU estimate for instruction type at specified level - pub fn get_cu_estimate(&self, instruction_type: &str, level: CuLevel) -> Option { - self.get_estimate(instruction_type) - .map(|est| est.get_cu_for_level(level)) - } -} + /// Build the transaction to measure using the provided SVM + fn build_transaction(&mut self, svm: &mut LiteSVM) -> Transaction; -impl Default for ComputeUnitDatabase { - fn default() -> Self { - Self::new() + /// Provide names for programs/accounts this benchmark interacts with + fn address_book(&self) -> HashMap { + HashMap::new() } } - -// // Core trait -// trait CuBenchInstruction { ... } - -// // Runner -// struct CuBenchRunner { ... } - -// // Database/estimates -// struct CuBenchDatabase { ... } -// struct CuBenchEstimate { ... } - -// // TX builder integration -// let estimates = CuBenchDatabase::load(); -// let tx_builder = TxBuilder::new() -// .with_cubench_estimates(estimates); - -// // benches/cu_measurements_sol_transfer.rs -// use litesvm_testing::*; - -// #[derive(Clone, Debug, Serialize, Deserialize)] -// pub struct SolTransfer { -// pub amount: u64, -// pub from_balance: u64, -// } - -// impl BenchmarkableInstruction for SolTransfer { -// // trait implementation -// } - -// fn main() { -// let mut runner = BenchmarkRunner::new(); -// let results = runner.benchmark_instruction::(); -// results.write_reports("sol_transfer").unwrap(); -// } diff --git a/crates/litesvm-testing/src/cu_bench/runner.rs b/crates/litesvm-testing/src/cu_bench/runner.rs new file mode 100644 index 0000000..9ef9ff3 --- /dev/null +++ b/crates/litesvm-testing/src/cu_bench/runner.rs @@ -0,0 +1,128 @@ +use chrono::Utc; +use litesvm::LiteSVM; +use log::info; +use solana_message::Message; +use solana_transaction::Transaction; + +use super::context::{ + discover_instruction_context, discover_transaction_context, TransactionExecutionContext, +}; +use super::estimate::{ComputeUnitStats, InstructionBenchmarkResult, StatType}; +use crate::cu_bench::{InstructionBenchmark, TransactionBenchmark}; + +/// Enhanced benchmark result for transactions +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct TransactionBenchmarkResult { + pub transaction_name: String, + pub cu_estimate: ComputeUnitStats, + pub execution_context: TransactionExecutionContext, + pub generated_at: String, + pub generated_by: String, +} + +/// Universal benchmark runner for any instruction implementing InstructionBenchmark +pub fn benchmark_instruction( + benchmark: T, + samples: usize, +) -> InstructionBenchmarkResult { + // Set up SVM once - it will accumulate state across measurements + let mut svm = benchmark.setup_svm(); + + // Phase 1: Discover context through simulation + let execution_context = discover_instruction_context(&benchmark, &mut svm); + + // Phase 2: Measure CU usage through actual execution + let mut cu_measurements = Vec::new(); + for i in 0..samples { + let cu_used = measure_instruction(&benchmark, &mut svm); + cu_measurements.push(cu_used); + + if (i + 1) % 10 == 0 { + info!("Completed {} measurements...", i + 1); + } + } + + // Create enhanced result + InstructionBenchmarkResult { + instruction_name: benchmark.instruction_name().to_string(), + cu_estimate: ComputeUnitStats::from_measurements( + StatType::Instruction(benchmark.instruction_name().to_string()), + &cu_measurements, + ), + execution_context, + generated_at: Utc::now().to_rfc3339(), + generated_by: generated_by(), + } +} + +/// Universal benchmark runner for any transaction implementing TransactionBenchmark +pub fn benchmark_transaction( + mut benchmark: T, + samples: usize, +) -> TransactionBenchmarkResult { + // Set up SVM once using benchmark's configuration - it will accumulate state across measurements + let mut svm = benchmark.setup_svm(); + + // Phase 1: Discover context through simulation + let context_tx = benchmark.build_transaction(&mut svm); + let workflow_name = benchmark.transaction_name().to_string(); + let address_book = benchmark.address_book(); + let execution_context = + discover_transaction_context(&context_tx, workflow_name, &mut svm, &address_book); + + // Phase 2: Measure CU usage through actual execution + let mut cu_measurements = Vec::new(); + for i in 0..samples { + let tx = benchmark.build_transaction(&mut svm); + let cu_used = measure_transaction_cu(&tx, &mut svm); + cu_measurements.push(cu_used); + + if (i + 1) % 10 == 0 { + info!("Completed {} measurements...", i + 1); + } + } + + // Create enhanced result + TransactionBenchmarkResult { + transaction_name: benchmark.transaction_name().to_string(), + cu_estimate: ComputeUnitStats::from_measurements( + StatType::Transaction(benchmark.transaction_name().to_string()), + &cu_measurements, + ), + execution_context, + generated_at: Utc::now().to_rfc3339(), + generated_by: generated_by(), + } +} + +/// Measure CU usage for a transaction using the provided SVM +fn measure_transaction_cu(transaction: &Transaction, svm: &mut LiteSVM) -> u64 { + // Execute transaction and measure CU usage + let result = svm.send_transaction(transaction.clone()).unwrap(); + result.compute_units_consumed +} + +/// Measure CU usage for a single instruction +fn measure_instruction(benchmark: &T, svm: &mut LiteSVM) -> u64 { + // 1. Get target instruction and signer pubkeys from benchmark + let (target_ix, signer_pubkeys) = benchmark.build_instruction(svm); + + // 2. Build unsigned transaction with just the target instruction + // Get fresh blockhash for each measurement to avoid AlreadyProcessed + svm.expire_blockhash(); + + let message = Message::new(&[target_ix], Some(&signer_pubkeys[0])); + let mut unsigned_tx = Transaction::new_unsigned(message); + unsigned_tx.message.recent_blockhash = svm.latest_blockhash(); + + // 3. Benchmark signs the transaction + let signed_tx = benchmark.sign_transaction(unsigned_tx); + + // 4. Send transaction and measure CU usage + let result = svm.send_transaction(signed_tx).unwrap(); + result.compute_units_consumed +} + +fn generated_by() -> String { + format!("{}@{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")) +} diff --git a/crates/litesvm-testing/src/lib.rs b/crates/litesvm-testing/src/lib.rs index 9d9f58f..01e5c00 100644 --- a/crates/litesvm-testing/src/lib.rs +++ b/crates/litesvm-testing/src/lib.rs @@ -106,6 +106,11 @@ pub mod prelude { pub use spl_associated_token_account; pub use spl_token; + pub use solana_keypair::Keypair; + pub use solana_pubkey::Pubkey; + pub use solana_signer::Signer; + pub use solana_system_interface::program as system_program; + pub use super::{ demand_instruction_error, // demand_instruction_error_at_index, From 27a5586c20147dec22911dc46e6b993058c51c73 Mon Sep 17 00:00:00 2001 From: Levi Cook Date: Wed, 11 Jun 2025 13:07:49 -0600 Subject: [PATCH 3/3] feat(docs): elevate CU benchmarking as core value proposition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transform project positioning from "testing framework" to "testing and benchmarking framework" with comprehensive documentation: Documentation Additions: - Add BENCHMARKING.md: Complete guide with living examples and best practices - Enhance README: Prominently feature CU benchmarking alongside testing - Create clear learning path: README → BENCHMARKING.md → benchmark files Key Documentation Features: - Dual paradigm explanation (instruction vs transaction benchmarking) - Statistical output interpretation (percentile-based estimates) - Production integration patterns for fee estimation - Troubleshooting guide for common benchmark issues - Living documentation that references actual working benchmark files Project Positioning: - README hero section now highlights both testing AND benchmarking capabilities - CU benchmarking quick start with concrete examples (SOL transfer: 150 CU, Token setup: 28K-38K CU) - Updated roadmap showing completed benchmarking framework - Enhanced examples section showcasing benchmark files as primary documentation This establishes systematic CU analysis as a unique differentiator alongside the existing comprehensive testing utilities. --- README.md | 61 +++- crates/litesvm-testing/BENCHMARKING.md | 274 ++++++++++++++++++ .../src/cu_bench/DESIGN_NOTES.md | 94 ------ 3 files changed, 327 insertions(+), 102 deletions(-) create mode 100644 crates/litesvm-testing/BENCHMARKING.md delete mode 100644 crates/litesvm-testing/src/cu_bench/DESIGN_NOTES.md diff --git a/README.md b/README.md index 4a163df..a81ee66 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,13 @@ # litesvm-testing -A comprehensive testing framework for Solana programs using [LiteSVM](https://github.com/LiteSVM/litesvm). Provides ergonomic, type-safe assertions for transaction results, logs, and all levels of Solana errors. +A comprehensive testing and benchmarking framework for Solana programs using [LiteSVM](https://github.com/LiteSVM/litesvm). Provides ergonomic, type-safe assertions for transaction results, logs, and all levels of Solana errors, plus systematic compute unit (CU) analysis for performance optimization. -> **⚠️ Development Status**: This library is currently in active development. The API may change before the first stable release. We plan to publish to [crates.io](https://crates.io) once the API stabilizes. +> **⚠️ Development Status**: This library is currently in active development. The API may change before the first stable release. ## ✨ Features +### 🧪 Testing Framework + - 🎯 **Complete Error Testing**: Transaction, instruction, and system program errors with type safety - 📋 **Log Assertions**: Detailed log content verification with helpful error messages - 🔧 **Build System Integration**: Automatic program compilation for Anchor and Pinocchio @@ -13,7 +15,20 @@ A comprehensive testing framework for Solana programs using [LiteSVM](https://gi - 🎪 **Precision Control**: "Anywhere" matching vs surgical instruction-index targeting - 🛡️ **Type Safety**: Work with `SystemError` enums instead of raw error codes - 📚 **Educational Examples**: Learn API progression from verbose to elegant -- 🚀 **Framework Support**: Anchor, Pinocchio, with more coming + +### 📊 CU Benchmarking Framework + +- ⚡ **Systematic CU Analysis**: Measure compute unit usage with statistical accuracy +- 🔬 **Dual Paradigms**: Pure instruction benchmarking vs complete transaction workflows +- 📈 **Percentile-Based Estimates**: Min, conservative, balanced, safe, very high, and max CU usage +- 🎯 **Production-Ready**: Generate CU estimates for real-world fee planning +- 📝 **Rich Context**: Execution logs, program details, and SVM environment state +- 🔄 **Reproducible**: Consistent results across environments and runs + +### 🚀 Framework Support + +- **Testing**: Anchor, Pinocchio, with more coming +- **Benchmarking**: Universal framework for any Solana program ## 🚀 Quick Start @@ -222,23 +237,40 @@ Both styles provide identical functionality - choose what feels right for your t This repository includes comprehensive, documented examples: -### Anchor Framework +### Testing Framework + +#### Anchor Framework - **Program**: [`examples/anchor/simple-anchor-program/`](examples/anchor/simple-anchor-program/) - **Tests**: [`examples/anchor/simple-anchor-tests/`](examples/anchor/simple-anchor-tests/) - **Features**: IDL integration, automatic compilation, complete build documentation -### Pinocchio Framework +#### Pinocchio Framework - **Program**: [`examples/pinocchio/simple-pinocchio-program/`](examples/pinocchio/simple-pinocchio-program/) - **Tests**: [`examples/pinocchio/simple-pinocchio-tests/`](examples/pinocchio/simple-pinocchio-tests/) - **Features**: Minimal boilerplate, lightweight setup, direct BPF compilation -### Educational Test Suite +#### Educational Test Suite - **API Progression**: [`tests/test_system_error_insufficient_funds.rs`](crates/litesvm-testing/tests/test_system_error_insufficient_funds.rs) - **Features**: Good → Better → Best → Best+ progression, demonstrates all API styles +### CU Benchmarking Framework + +#### Instruction Benchmarks + +- **SOL Transfer**: [`benches/cu_bench_sol_transfer_ix.rs`](crates/litesvm-testing/benches/cu_bench_sol_transfer_ix.rs) - Pure system program instruction (150 CU) +- **SPL Token Transfer**: [`benches/cu_bench_spl_transfer_ix.rs`](crates/litesvm-testing/benches/cu_bench_spl_transfer_ix.rs) - Complex multi-account instruction + +#### Transaction Benchmarks + +- **Token Setup Workflow**: [`benches/cu_bench_token_setup_tx.rs`](crates/litesvm-testing/benches/cu_bench_token_setup_tx.rs) - Complete 5-instruction workflow (28K-38K CU) + +#### Documentation + +- **Complete Guide**: [`BENCHMARKING.md`](crates/litesvm-testing/BENCHMARKING.md) - Comprehensive benchmarking documentation + ## 🏃‍♂️ Running Examples ```bash @@ -255,6 +287,11 @@ cargo test -p simple-pinocchio-tests -- --show-output # Run educational test suite cargo test -p litesvm-testing test_system_error -- --show-output + +# Run CU benchmarks with progress logging +cd crates/litesvm-testing +RUST_LOG=info cargo bench --bench cu_bench_sol_transfer_ix --features cu_bench +RUST_LOG=info cargo bench --bench cu_bench_token_setup_tx --features cu_bench ``` ## 🛠️ Prerequisites @@ -319,6 +356,8 @@ litesvm-testing/ ## 🗺️ Roadmap +### ✅ Completed Features + - [x] **Core log assertion utilities** - [x] **Complete error testing framework** (transaction, instruction, system) - [x] **Type-safe system error handling** @@ -326,8 +365,14 @@ litesvm-testing/ - [x] **Working examples for both frameworks** with educational progression - [x] **Dual API styles** (direct functions + fluent method syntax) - [x] **Precision control** ("anywhere" vs "surgical" assertions) +- [x] **CU benchmarking framework** with instruction and transaction paradigms +- [x] **Statistical CU analysis** with percentile-based estimates +- [x] **Rich benchmarking context** (execution logs, program details, SVM state) + +### 🔄 In Progress + - [ ] **Steel framework support** -- [ ] **Additional testing utilities** (compute unit checks, account state, etc.) +- [ ] **Additional testing utilities** (account state verification, etc.) - [ ] **First stable release (v0.1.0) to crates.io** - [ ] **Integration with popular Solana testing patterns** @@ -349,4 +394,4 @@ This project is dual licensed under GPL-3.0-or-later and CC BY-SA 4.0. See LICEN - [LiteSVM](https://github.com/LiteSVM/litesvm) - Fast Solana VM for testing - [Anchor](https://github.com/coral-xyz/anchor) - Solana development framework - [Pinocchio](https://github.com/anza-xyz/pinocchio) - Lightweight Solana SDK -test + test diff --git a/crates/litesvm-testing/BENCHMARKING.md b/crates/litesvm-testing/BENCHMARKING.md new file mode 100644 index 0000000..4acf742 --- /dev/null +++ b/crates/litesvm-testing/BENCHMARKING.md @@ -0,0 +1,274 @@ +# Compute Unit Benchmarking Guide + +> **Systematic CU analysis for Solana programs using LiteSVM** + +This guide shows you how to measure compute unit (CU) usage for Solana instructions and transactions with reproducible, statistical accuracy. + +## Quick Start + +```bash +# Run an instruction benchmark +RUST_LOG=info cargo bench --bench cu_bench_sol_transfer_ix --features cu_bench + +# Run a transaction benchmark +RUST_LOG=info cargo bench --bench cu_bench_token_setup_tx --features cu_bench +``` + +## When to Use What + +### Instruction Benchmarking + +**Use when:** You want to measure the pure CU cost of a single instruction + +- ✅ Research and analysis +- ✅ Understanding base program costs +- ✅ Comparing instruction efficiency +- ✅ Building CU cost models + +**Example:** How much does a SOL transfer cost? → **150 CU** + +### Transaction Benchmarking + +**Use when:** You want to measure real-world workflow costs + +- ✅ Production fee estimation +- ✅ Multi-instruction workflows +- ✅ Cross-program invocation analysis +- ✅ Complete user journeys + +**Example:** How much does token setup cost? → **28,322-38,822 CU** + +## Implementing Instruction Benchmarks + +> **See [`benches/cu_bench_sol_transfer_ix.rs`](benches/cu_bench_sol_transfer_ix.rs) for a complete working example** + +### 1. Implement the InstructionBenchmark Trait + +```rust +use litesvm_testing::cu_bench::{benchmark_instruction, InstructionBenchmark}; + +struct SolTransferBenchmark { + sender: Keypair, + recipient: Keypair, + transfer_amount: u64, +} + +impl InstructionBenchmark for SolTransferBenchmark { + fn instruction_name(&self) -> &'static str { + "sol_transfer" + } + + fn setup_svm(&self) -> LiteSVM { + let mut svm = LiteSVM::new(); + svm.airdrop(&self.sender.pubkey(), 200_000_000).unwrap(); + svm + } + + fn build_instruction(&self, _svm: &mut LiteSVM) -> (Instruction, Vec) { + let transfer_ix = solana_system_interface::instruction::transfer( + &self.sender.pubkey(), + &self.recipient.pubkey(), + self.transfer_amount, + ); + (transfer_ix, vec![self.sender.pubkey()]) + } + + fn sign_transaction(&self, mut unsigned_tx: Transaction) -> Transaction { + unsigned_tx.sign(&[&self.sender], unsigned_tx.message.recent_blockhash); + unsigned_tx + } +} +``` + +### 2. Run the Benchmark + +```rust +fn main() { + env_logger::init(); + let benchmark = SolTransferBenchmark::new(); + let result = benchmark_instruction(benchmark, 100); + + println!("{}", serde_json::to_string_pretty(&result).unwrap()); +} +``` + +## Implementing Transaction Benchmarks + +> **See [`benches/cu_bench_token_setup_tx.rs`](benches/cu_bench_token_setup_tx.rs) for a complete multi-program workflow example** + +### 1. Implement the TransactionBenchmark Trait + +```rust +use litesvm_testing::cu_bench::{benchmark_transaction, TransactionBenchmark}; + +impl TransactionBenchmark for TokenSetupTransactionBenchmark { + fn transaction_name(&self) -> &'static str { + "token_setup_complete" + } + + fn setup_svm(&self) -> LiteSVM { + let mut svm = LiteSVM::new(); + svm.airdrop(&self.mint_authority.pubkey(), 1_000_000_000).unwrap(); + svm + } + + fn build_transaction(&mut self, svm: &mut LiteSVM) -> Transaction { + // Create fresh mint to avoid collisions + self.mint = Keypair::new(); + + let instructions = vec![ + ComputeBudgetInstruction::set_compute_unit_limit(150_000), + // ... create mint, initialize, create ATA, mint tokens + ]; + + // Build and sign complete transaction + // (see full example in benchmark file) + } +} +``` + +## Understanding Results + +### Percentile-Based Estimates + +```json +{ + "cu_estimate": { + "min": 150, // 0th percentile - absolute minimum + "conservative": 150, // 25th percentile - safe for most cases + "balanced": 150, // 50th percentile - good default + "safe": 175, // 75th percentile - high reliability + "very_high": 190, // 95th percentile - very reliable + "unsafe_max": 200, // 100th percentile - maximum observed + "sample_size": 100 + } +} +``` + +### Execution Context + +Rich context about what happened during execution: + +```json +{ + "execution_context": { + "svm_context": { + "current_slot": 0, + "latest_blockhash": "..." + }, + "program_context": { + "program_id": "11111111111111111111111111111111", + "program_name": "system_program", + "cpi_count": 1 + }, + "execution_stats": { + "logs": ["Program 11111111111111111111111111111111 invoke [1]", "..."], + "simulated_cu": 150 + } + } +} +``` + +## Best Practices + +### 1. **Fresh State for Each Measurement** + +- Use `svm.expire_blockhash()` to avoid `AlreadyProcessed` errors +- Generate fresh keypairs when needed to avoid account collisions +- Fund accounts generously for multiple measurements + +### 2. **Realistic Scenarios** + +- Mirror production account states and balances +- Include necessary setup instructions +- Use representative data sizes + +### 3. **Statistical Rigor** + +- Run 100+ samples for stable percentiles +- Use appropriate confidence levels: + - `conservative` (25th) for safety margins + - `balanced` (50th) for typical estimates + - `safe` (75th) for high-reliability scenarios + +### 4. **Environment Consistency** + +- Pin Solana versions for reproducible results +- Document LiteSVM features that affect measurements +- Use consistent SVM configuration across benchmarks + +## Advanced Examples + +### Multi-Program Workflows + +See [`benches/cu_bench_token_setup_tx.rs`](benches/cu_bench_token_setup_tx.rs) for: + +- Cross-program invocations (CPI) +- Account creation and initialization +- Complex transaction construction +- Address book for human-readable program names + +### Complex Instructions + +See [`benches/cu_bench_spl_transfer_ix.rs`](benches/cu_bench_spl_transfer_ix.rs) for: + +- Multi-step SVM setup +- Associated token account handling +- Mint and token account management + +## Integration with Production Code + +### Using Results for Fee Estimation + +```rust +// Load benchmark results +let sol_transfer_result: InstructionBenchmarkResult = + serde_json::from_str(include_str!("../results/sol_transfer.json"))?; + +// Get conservative estimate for fee calculation +let cu_estimate = sol_transfer_result.cu_estimate.conservative; + +// Build transaction with appropriate CU limit +let compute_budget_ix = ComputeBudgetInstruction::set_compute_unit_limit(cu_estimate); +``` + +### Building CU Databases + +```rust +use litesvm_testing::cu_bench::ComputeUnitDatabase; + +let mut db = ComputeUnitDatabase::new(); +// Add estimates from various benchmarks +// Save as JSON for application use +``` + +## Troubleshooting + +### Common Issues + +**"AlreadyProcessed" Errors** + +- Ensure `svm.expire_blockhash()` before each measurement +- Don't reuse transactions across measurements + +**"Account Already Exists" Errors** + +- Generate fresh keypairs in `build_transaction()` +- Don't reuse account addresses across measurements + +**"Insufficient Funds" Errors** + +- Increase airdrop amounts in `setup_svm()` +- Account for fees across many measurements + +**Inconsistent Results** + +- Check for state accumulation effects +- Verify SVM setup consistency +- Ensure measurements are independent + +## Further Reading + +- **Benchmark Examples**: [`benches/`](benches/) directory +- **API Documentation**: [docs.rs/litesvm-testing](https://docs.rs/litesvm-testing) +- **Core Framework**: [`src/cu_bench/`](src/cu_bench/) module diff --git a/crates/litesvm-testing/src/cu_bench/DESIGN_NOTES.md b/crates/litesvm-testing/src/cu_bench/DESIGN_NOTES.md deleted file mode 100644 index b9d81f5..0000000 --- a/crates/litesvm-testing/src/cu_bench/DESIGN_NOTES.md +++ /dev/null @@ -1,94 +0,0 @@ -# CU Benchmarking Framework - Design Notes - -## Core Principles - -### Why Systematic CU Benchmarking? - -- **Gap in Ecosystem**: Existing tools focus on RPC performance/tx success, not instruction-level CU optimization -- **Developer Pain Point**: Manual CU testing is ad-hoc, non-reproducible, and time-consuming -- **Economic Impact**: Better CU estimates → lower fees → more efficient network usage - -### Design Philosophy - -- **Helius-Inspired Confidence Levels**: Familiar model from priority fee API -- **Reproducible by Default**: Same benchmark should give same results across environments -- **Context-Aware**: Environment/program context affects CU usage significantly -- **Community-Extensible**: Easy for developers to add their own benchmarks - -## Methodology Decisions - -### Measurement Strategy - -- **Stateful Accumulation**: SVM maintains state across measurements (more realistic) -- **Fresh Blockhashes**: Each measurement gets new blockhash to avoid AlreadyProcessed errors -- **Statistical Approach**: Multiple samples → percentile-based estimates -- **Framework Responsibility**: Framework handles tx construction, signing, and CU extraction - -### Environment Standardization - -- **Feature Set Tracking**: LiteSVM features affect CU (see feature list) -- **Version Pinning**: Solana version changes can affect CU usage -- **Program Dependencies**: Track what programs are loaded/interacted with - -### Benchmark Separation - -- **Instruction-Level**: Single instruction CU usage (current focus) -- **Transaction-Level**: Multiple instructions + interaction effects (future) -- **Scenario-Based**: Real-world usage patterns (future) - -## Current API Design - -### InstructionBenchmark Trait - -```rust -pub trait InstructionBenchmark { - fn instruction_name(&self) -> &'static str; - fn setup_svm(&self) -> LiteSVM; - fn build_instruction(&self, svm: &mut LiteSVM) -> (Instruction, Vec); - fn sign_transaction(&self, unsigned_tx: Transaction) -> Transaction; -} -``` - -**Key Design Decisions:** - -- **Benchmark owns setup**: Each benchmark controls its SVM environment -- **Framework owns measurement**: Consistent CU extraction across all benchmarks -- **Separation of concerns**: Benchmark builds instruction, framework measures CU - -## Future API Evolution - -### TransactionBenchmark (Future) - -```rust -pub trait TransactionBenchmark { - fn transaction_name(&self) -> &'static str; - fn setup_svm(&self) -> LiteSVM; - fn build_transaction(&self, svm: &mut LiteSVM) -> (Transaction, Vec); - // Note: Different from instruction - builds full transaction -} -``` - -### Unified Benchmark Enum (Consideration) - -```rust -pub enum BenchmarkType<'a> { - Instruction(&'a dyn InstructionBenchmark), - Transaction(&'a dyn TransactionBenchmark), -} -``` - -## Open Questions/TODOs - -- [ ] How to handle CPI effects in instruction benchmarks? -- [ ] Should we validate benchmark reproducibility automatically? -- [ ] Database/sharing mechanism for community estimates? -- [ ] How to handle version evolution of programs? - -## Feature Impact Considerations - -Based on LiteSVM feature set, these could affect CU measurements: - -- Signature verification settings -- Precompiled programs enabled -- SPL program availability -- Feature gates (curve25519, etc.)