diff --git a/CHANGELOG.md b/CHANGELOG.md index ae6b25e..3188a2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,47 @@ and the crate uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). `WIRE_VERSION = 1`, public Rust API additions and small breaking changes may still occur within `0.1.x` until `0.2.0`. +## [0.1.5] — 2026-05-21 + +`0.1.5` is a docs-and-testing release. It adds a Linux device-mapper +power-loss simulation harness and brings the README in line with the +post-alpha state of the crate. Public Rust API unchanged; wire +format (`WIRE_VERSION = 1`) unchanged; corpus fixtures unchanged. + +### Added + +- **Power-loss simulation harness (Linux, device-mapper)** (issue + [#31](https://github.com/deepcausa/datawal/issues/31)). New + examples `power_loss_workload` and `power_loss_validate`, plus + driver scripts `scripts/power_loss_dm_flakey.sh` and + `scripts/power_loss_cleanup.sh`, exercise `DataWal` under + `dm-flakey`-induced write loss on ext4. The workload appends + `put`/`delete` ops with `fsync` after every op and mirrors them + into an fsync-ordered JSONL oracle on a separate filesystem. + After the dm table is flipped to `error_writes` and the volume is + force-unmounted and remounted, the validator reopens the store + and checks that the keydir is a per-key prefix of the oracle + (invariants 3, 4, 5). Linux-only, root-only, **not** part of CI. + See [`docs/power-loss-testing.md`](docs/power-loss-testing.md) for + the harness contract and + [`docs/power-loss-results.md`](docs/power-loss-results.md) for a + sanitized record of a verified run. +- **`Durability evidence` section in `README.md`** linking the fuzz + targets, `proptest` invariants, SIGKILL crash-injection suite, and + the new `dm-flakey` harness. + +### Changed + +- **`README.md` aligned to post-alpha state.** Removed references to + "alpha crate", "alpha release", "alpha limits", "not + production-ready", and other wording inherited from + `0.1.0-alpha.1`. Updated "What is in" / "Limits" / "Evidence + stack" / "Formal models" to reflect `RecordLogReader`, + `scan_iter`, the `datawal` CLI, and the four TLA+ models + (`RecordLog`, `KeydirProjection`, `Compaction`, `ReadWhileWrite`). + Reframed status as "pre-1.0 crate suitable for local recoverable + logs with documented limits; not a general-purpose database". + ## [0.1.4] — 2026-05-21 `0.1.4` is the first non-alpha release of `datawal`. It is suitable diff --git a/Cargo.lock b/Cargo.lock index 8f1cbce..e1a1dc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -253,7 +253,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "datawal" -version = "0.1.4" +version = "0.1.5" dependencies = [ "anyhow", "base64", @@ -269,7 +269,7 @@ dependencies = [ [[package]] name = "datawal-cli" -version = "0.1.4" +version = "0.1.5" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 75371f6..548cd85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ exclude = [ ] [workspace.package] -version = "0.1.4" +version = "0.1.5" edition = "2021" license = "MIT OR Apache-2.0" repository = "https://github.com/deepcausa/datawal" diff --git a/README.md b/README.md index b41062b..01df16d 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,12 @@ datawal is a local record store: a framed append-only `RecordLog` plus an optional last-write-wins `DataWal` KV projection. -> `0.1.4` is the first non-alpha datawal release. It is suitable for -> local recoverable logs where JSONL would otherwise be used, with the -> documented limits in [`docs/canon.md`](docs/canon.md). `0.1.x` may -> still introduce small breaking changes before `0.2`; the on-disk -> wire format (`WIRE_VERSION = 1`) is frozen and locked by corpus -> fixtures. +> datawal is a pre-1.0 crate suitable for local recoverable logs where +> JSONL would otherwise be used, with the documented limits in +> [`docs/canon.md`](docs/canon.md). It is not a general-purpose +> database. `0.1.x` may still introduce small breaking changes before +> `0.2`; the on-disk wire format (`WIRE_VERSION = 1`) is frozen and +> locked by corpus fixtures. **MSRV:** Rust 1.75.0 @@ -61,22 +61,28 @@ optional last-write-wins `DataWal` KV projection. ## Current status -**datawal is currently an alpha crate:** functional and model-checked at -the protocol level, but not production-ready. See +**datawal is a pre-1.0 crate suitable for local recoverable logs where +JSONL would otherwise be used, with documented limits.** It is not a +general-purpose database. `0.1.x` may still introduce small breaking +changes before `0.2`; the on-disk wire format (`WIRE_VERSION = 1`) is +frozen and locked by corpus fixtures. See [`docs/roadmap.md`](docs/roadmap.md) for the exact release scope. -It is published on crates.io as an alpha release; the on-disk wire format -is frozen by a corpus fixture set, but workload coverage is still only at -the unit and integration level. - What is in: -- 58 tests green (`cargo test --workspace`). -- 3 TLA+ models model-checked with TLC 2.19. -- Wire-format corpus: 6 binary fixture directories, 11 corpus tests. -- 4 runnable examples. -- Real **CRC-32C** (Castagnoli, `0x1EDC6F41`) per record, pinned by a - known-vector test. +- Framed `RecordLog` with **CRC-32C** (Castagnoli, `0x1EDC6F41`) and + longest-valid-prefix recovery. +- `DataWal` bytes-in / bytes-out KV projection with tombstones and + `compact_to`. +- `RecordLogReader` snapshot-at-open reader API (no live tailing). +- `datawal` CLI for inspection and export + (`crates/datawal-cli/`). +- Wire-format corpus locked by binary fixtures. +- TLA+ models for `RecordLog`, `KeydirProjection`, `Compaction`, + and `ReadWhileWrite`. +- Fuzz targets, `proptest` invariants, crash-injection tests, + ENOSPC tests, soak harness, and `dm-flakey` power-loss harness. +- Criterion benchmarks with a reference run. - **fs2 fd-based advisory lock**: held by a file descriptor, not by the existence of the sentinel file. Released on `Drop` / process exit. A stale `.lock` from a crashed previous process is not a problem. @@ -95,28 +101,28 @@ What is not in: - Multi-writer. - Query / secondary indexes. - In-place compaction. -- Reader API / concurrent reads. +- Group commit / configurable fsync policy. ## Limits `datawal` is bytes-first, but not unbounded. Neither the `RecordLog` nor the `DataWal` projection interprets the bytes — no JSON, no UTF-8, no -MessagePack parsing in the core. Current alpha limits: - -| Limit | Value / status | Notes | -| ------------------ | ---------------------: | ---------------------------------------------------------------------------------------------------- | -| Max key size | 64 KiB | Per record. Larger keys are rejected. | -| Max payload size | 64 MiB | Per record. For larger values, use an external blob store and store references. | -| Writers | Single writer | Enforced with an advisory fd lock. No multi-writer semantics. | -| Readers | No reader API yet | `scan()` is the only read path; tracked in [#5](https://github.com/deepcausa/datawal/issues/5). | -| `scan()` memory | `Vec` | Not streaming yet; large logs need [#3](https://github.com/deepcausa/datawal/issues/3). | -| `DataWal` keydir | values in memory | Live values are held in the in-memory keydir; offset-based variant is [#4](https://github.com/deepcausa/datawal/issues/4). | -| Durability | explicit `fsync()` | `append()` is recoverable; `append() + fsync()` is durable under documented assumptions. | -| Compaction | `compact_to` only | Snapshot-style rebuild into a target directory. No in-place `compact()`. | -| CAS / blob | not included | Planned as a separate crate / layer; tracked in [#7](https://github.com/deepcausa/datawal/issues/7). | -| Compression | not included | `flags` must be zero in v0.1. | -| Query | not included | No SQL, indexes, joins, range scans, or planner. See [#13](https://github.com/deepcausa/datawal/issues/13). | -| Production status | alpha | Functional, tested, model-checked at the protocol level; not production-ready. | +MessagePack parsing in the core. Current limits: + +| Limit | Value / status | Notes | +| ------------------ | ----------------------------: | -------------------------------------------------------------------------------------------------------------------------------------- | +| Max key size | 64 KiB | Per record. Larger keys are rejected. | +| Max payload size | 64 MiB | Per record. For larger values, use an external blob store and store references. | +| Writers | Single writer | Enforced with an advisory fd lock. No multi-writer semantics. | +| Readers | snapshot-at-open reader | `RecordLogReader` can inspect a store without taking the writer lock; no live tailing API. | +| `scan()` memory | eager `Vec` | Use `scan_iter()` for record-level lazy iteration; it is segment-buffered, not zero-copy. | +| `DataWal` keydir | offsets in memory | Live keys map to `RecordRef`; `get()` performs I/O and CRC verification. | +| Durability | explicit `fsync()` | `append()` is recoverable; `append() + fsync()` is durable under documented assumptions. | +| Compaction | `compact_to` only | Snapshot-style rebuild into a target directory. No in-place `compact()`. | +| CAS / blob | not included | Planned as a separate crate / layer; tracked in [#7](https://github.com/deepcausa/datawal/issues/7). | +| Compression | not included | `flags` must be zero in v0.1. | +| Query | not included | No SQL, indexes, joins, range scans, or planner. See [#13](https://github.com/deepcausa/datawal/issues/13). | +| Production status | scoped production use | Suitable for local single-writer recoverable logs; not a general-purpose database. | What is **not** limited inside those bounds: the byte composition of keys and payloads. Any sequence is legal, including all-zero, all-`0xFF`, @@ -159,19 +165,44 @@ db.export_jsonl(Path::new("/tmp/my-kv.jsonl"))?; The protocol has been validated at multiple levels: -| Layer | Evidence | -| ------------------- | ----------------------------------------------------- | -| Specification | `docs/canon.md` — 14 binding clauses; byte layout | -| Code | `crates/datawal-core/src/` — ~1900 LOC Rust | -| Unit + integration | 58 tests across `tests/*.rs` and embedded `#[test]`s | -| Wire-format corpus | 6 binary fixture dirs, 11 corpus tests | -| Formal models | 3 TLA+ models, model-checked with TLC 2.19 | -| Runnable examples | 4 examples under `crates/datawal-core/examples/` | +| Layer | Evidence | +| ------------------- | ---------------------------------------------------------------------------------------------- | +| Specification | `docs/canon.md`; documented byte layout and limits | +| Wire format | binary corpus fixtures locked by CI | +| Formal models | TLA+ models for `RecordLog`, `KeydirProjection`, `Compaction`, `ReadWhileWrite` | +| Parser robustness | `cargo-fuzz` targets and `proptest` invariants | +| Recovery behavior | crash-injection tests, ENOSPC tests, `dm-flakey` power-loss harness | +| Long-run behavior | soak harness | +| Performance | Criterion benchmarks and reference run | +| Operations | `datawal` CLI for inspection and export | **Formal models wording:** model-checked under documented assumptions. Not "formally verified". Models do not check the Rust implementation. See `formal/README.md` for invariants and how to run TLC. +## Durability evidence + +DataWal is exercised under several layers of failure-mode testing: + +- Fuzz tests on the record decoder (see [Fuzzing](#fuzzing)). +- `proptest` invariants on append-then-recover sequences. +- A SIGKILL-based crash-injection suite in `tests/crash_injection.rs` + that spawns the test binary as a child, kills it at named points + (`append_no_fsync`, `append_fsync`, `rotate`, `compact_to`, + `export_jsonl`), then reopens the store and checks invariants. +- A `dm-flakey` power-loss simulation harness on Linux (root, not CI) + that routes ext4 over a device-mapper layer, flips the layer to + `error_writes`, force-unmounts, remounts the layer healthy, reopens + the store, and validates that the reopened state matches an + fsync-ordered oracle. See [`docs/power-loss-testing.md`](docs/power-loss-testing.md) + for the harness contract and prerequisites, and + [`docs/power-loss-results.md`](docs/power-loss-results.md) for a + sample verified run. + +This is stricter than process-level crash testing but is not a +substitute for real power-cut testing on real hardware. DataWal trusts +the storage stack below it to honor `fsync`. + ## Layout ``` @@ -220,6 +251,7 @@ cargo test --workspace cargo run -p datawal --example record_log_demo cargo run -p datawal --example datawal_kv_demo cargo run -p datawal --example tail_recovery_demo +cargo run -p datawal-cli -- --help cargo doc --workspace --no-deps ``` @@ -293,12 +325,14 @@ CI verifies the targets *compile* on nightly; it does not run them. ## Formal models -Three small TLA+ models live under `formal/` and are checked with +Four small TLA+ models live under `formal/` and are checked with [TLC](https://github.com/tlaplus/tlaplus/) 2.19+: - `RecordLog.tla` — append / fsync / crash; durable is a monotonic prefix. - `KeydirProjection.tla` — last-write-wins keydir from a put/del log. - `Compaction.tla` — `compact_to` preserves the live projection. +- `ReadWhileWrite.tla` — snapshot-at-open reader behavior under + concurrent writer progress. **model-checked under documented assumptions** — not "formally verified", does not check the Rust implementation. See `formal/README.md`. @@ -331,7 +365,7 @@ from them. - `docs/canon.md` — binding decisions and the byte-layout of a record. - `docs/technical-decisions.md` — TD-NNN entries documenting choices. -- `docs/roadmap.md` — current alpha scope, what is frozen, and the tracked roadmap issues. +- `docs/roadmap.md` — current release scope, what is frozen, and the tracked roadmap issues. - `formal/README.md` — the TLA+ models and how to run TLC. ## License diff --git a/crates/datawal-core/examples/power_loss_validate.rs b/crates/datawal-core/examples/power_loss_validate.rs new file mode 100644 index 0000000..875f1c9 --- /dev/null +++ b/crates/datawal-core/examples/power_loss_validate.rs @@ -0,0 +1,362 @@ +//! Power-loss simulation **validator** for `datawal`. +//! +//! **This is a manual validation tool, NOT a usage example.** It is the +//! consumer half of the dm-flakey power-loss harness; the producer half +//! is the sibling `power_loss_workload` example, and the two are wired +//! together by `scripts/power_loss_dm_flakey.sh`. None of this is part +//! of CI and none of it is part of the published artefact's documented +//! behaviour. +//! +//! # What the validator does +//! +//! Open the `DataWal` directory **after** a fault injection (typically a +//! dm-flakey table reload that drops writes, followed by `umount -f` and +//! `mount` again) and assert the following invariants: +//! +//! 1. `DataWal::open(&dir)` succeeds. `RecordLog`'s longest-valid-prefix +//! recovery is allowed to truncate a partial tail, but it must not +//! error out. CRC mismatch in a *sealed* segment is a hard error and +//! will surface as a failed `open`. +//! 2. The `RecoveryReport` returned by reopening the underlying +//! `RecordLog` is recorded for inspection (segments scanned, +//! tail-bytes discarded, mid-stream errors). +//! 3. For every **put** line in the oracle file (lines that the workload +//! flushed *after* a successful `DataWal::fsync`): +//! - `DataWal::contains_key(key)` must be `true`, **unless** the +//! same key is overwritten or tombstoned by a later oracle line. +//! - `DataWal::get(key)` must succeed (CRC-valid) and return the +//! expected payload bytes, where "expected" is the most recent +//! oracle line for that key. +//! 4. For every **del** oracle line, the key must not be live in the +//! reopened store *unless* a later oracle line puts it back. +//! 5. The reopened keydir must contain **no extra live keys** beyond +//! those implied by the oracle. The harness's contract is that the +//! oracle is the upper bound of what the workload claimed durable; +//! a reopened store may have *less* (tail dropped) but not *more*. +//! A larger live set means the recovery surfaced state we never +//! fsync'd — that is a bug. +//! +//! Together these say: **the reopened store is a prefix of the oracle**. +//! That is the property dm-flakey-class faults should preserve under +//! correct fsync semantics. +//! +//! # CLI +//! +//! ```text +//! cargo run --release -p datawal --example power_loss_validate -- \ +//! --work-dir /mnt/datawal-test/wal \ +//! --oracle /tmp/datawal-powerloss-/oracle.jsonl +//! ``` +//! +//! # Exit codes +//! +//! - `0` reopened store is a valid prefix of the oracle. +//! - `1` invariant violated. The validator prints exactly which +//! invariant failed and a small sample of mismatched keys. +//! - `2` setup error (missing arg, oracle unreadable, etc.). +//! +//! Exit codes mirror `soak.rs` and `power_loss_workload.rs` so the +//! orchestrator can treat all three examples uniformly. +//! +//! # Honest claims +//! +//! - The validator does not re-derive payloads from `--seed`; it reads +//! the bytes the workload claimed durable from the oracle. This keeps +//! the validator independent of the workload's PRNG choice. +//! - "Prefix of the oracle" is checked *per key*, not in oracle order. +//! The workload writes oracle lines after each successful fsync, but +//! the validator does not assume the surviving tail of the WAL is a +//! contiguous prefix of the oracle by `seq`: a tail may end anywhere +//! the kernel cared to flush. What it does assume is that any *single* +//! surviving record is the **most recent** put for its key, because +//! `RecordLog` recovery uses the longest valid prefix per segment. + +use std::collections::HashMap; +use std::env; +use std::fs; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use anyhow::{Context, Result}; +use base64::Engine; +use datawal::{DataWal, RecordLog}; +use serde::Deserialize; + +struct Cli { + work_dir: PathBuf, + oracle: PathBuf, + /// Max samples to print when reporting mismatches. Defaults to 5. + sample: usize, +} + +fn parse_args() -> Result { + let mut work_dir: Option = None; + let mut oracle: Option = None; + let mut sample: usize = 5; + + let args: Vec = env::args().skip(1).collect(); + let mut i = 0; + while i < args.len() { + let arg = &args[i]; + let next = || -> Result<&String> { + args.get(i + 1) + .with_context(|| format!("{} requires a value", arg)) + }; + match arg.as_str() { + "--work-dir" => { + work_dir = Some(PathBuf::from(next()?)); + i += 2; + } + "--oracle" => { + oracle = Some(PathBuf::from(next()?)); + i += 2; + } + "--sample" => { + sample = next()?.parse().context("parse --sample")?; + i += 2; + } + other => anyhow::bail!("unknown arg `{}`", other), + } + } + Ok(Cli { + work_dir: work_dir.context("--work-dir is required")?, + oracle: oracle.context("--oracle is required")?, + sample, + }) +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "op")] +enum OracleLine { + #[serde(rename = "put")] + Put { + seq: u64, + key: String, + payload: String, + }, + #[serde(rename = "del")] + Del { seq: u64, key: String }, +} + +/// Effective oracle state after replay: the most-recent op per key. +enum OracleEffect { + Live(Vec), + Dead, +} + +fn b64_decode(s: &str) -> Result> { + base64::engine::general_purpose::STANDARD + .decode(s) + .context("base64 decode oracle field") +} + +/// Oracle aggregate: final effect per key, plus counters for reporting. +/// +/// Fields: +/// - `effect` : last write seen per key (`Live(payload)` or `Dead`). +/// - `last_seq` : highest sequence number observed. +/// - `puts` : count of `put` lines parsed. +/// - `dels` : count of `del` lines parsed. +struct OracleAggregate { + effect: HashMap, OracleEffect>, + last_seq: u64, + puts: u64, + dels: u64, +} + +fn load_oracle(path: &Path) -> Result { + let f = fs::File::open(path).with_context(|| format!("open oracle {}", path.display()))?; + let mut effect: HashMap, OracleEffect> = HashMap::new(); + let mut last_seq: u64 = 0; + let mut puts: u64 = 0; + let mut dels: u64 = 0; + for (lineno, line) in BufReader::new(f).lines().enumerate() { + let line = line?; + if line.trim().is_empty() { + continue; + } + let parsed: OracleLine = serde_json::from_str(&line) + .with_context(|| format!("{}:{} parse oracle line", path.display(), lineno + 1))?; + match parsed { + OracleLine::Put { seq, key, payload } => { + let key = b64_decode(&key)?; + let payload = b64_decode(&payload)?; + effect.insert(key, OracleEffect::Live(payload)); + last_seq = last_seq.max(seq); + puts += 1; + } + OracleLine::Del { seq, key } => { + let key = b64_decode(&key)?; + effect.insert(key, OracleEffect::Dead); + last_seq = last_seq.max(seq); + dels += 1; + } + } + } + Ok(OracleAggregate { + effect, + last_seq, + puts, + dels, + }) +} + +fn main() -> ExitCode { + let cli = match parse_args() { + Ok(c) => c, + Err(e) => { + eprintln!("power_loss_validate: setup error: {:#}", e); + return ExitCode::from(2); + } + }; + match run(cli) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("power_loss_validate: invariant violated: {:#}", e); + ExitCode::from(1) + } + } +} + +fn run(cli: Cli) -> Result<()> { + let OracleAggregate { + effect, + last_seq, + puts: oracle_puts, + dels: oracle_dels, + } = load_oracle(&cli.oracle)?; + eprintln!( + "power_loss_validate: oracle loaded path={} last_seq={} puts={} dels={} effective_keys={}", + cli.oracle.display(), + last_seq, + oracle_puts, + oracle_dels, + effect.len() + ); + + // First, inspect recovery via RecordLog directly so we can print the + // RecoveryReport (DataWal::open does not surface it). RecordLog::open + // holds the lock; drop it before opening DataWal on the same dir. + { + let rlog = RecordLog::open(&cli.work_dir) + .with_context(|| format!("RecordLog::open {}", cli.work_dir.display()))?; + let report = rlog + .recovery_report() + .context("RecordLog::recovery_report")?; + eprintln!( + "power_loss_validate: recovery files_scanned={} records_replayed={} tail_truncated_segs={} tail_bytes_discarded={} mid_stream_errors={} last_txid_seen={}", + report.files_scanned, + report.records_replayed, + report.tail_truncated, + report.tail_bytes_discarded, + report.mid_stream_errors, + report.last_txid_seen, + ); + if report.mid_stream_errors > 0 { + anyhow::bail!( + "RecoveryReport.mid_stream_errors = {} (v0.1.x is supposed to abort on the first; this is a bug)", + report.mid_stream_errors + ); + } + } + + // Now reopen as DataWal and run the invariants. + let mut store = DataWal::open(&cli.work_dir) + .with_context(|| format!("DataWal::open {}", cli.work_dir.display()))?; + + // Collect the live keydir for fast intersection. + let observed: std::collections::HashSet> = store.keys().into_iter().collect(); + let oracle_live: std::collections::HashSet> = effect + .iter() + .filter_map(|(k, eff)| match eff { + OracleEffect::Live(_) => Some(k.clone()), + OracleEffect::Dead => None, + }) + .collect(); + + // Invariant 5: no extra live keys. + let extras: Vec> = observed.difference(&oracle_live).cloned().collect(); + if !extras.is_empty() { + let sample: Vec = extras + .iter() + .take(cli.sample) + .map(|k| format!("len={} sha256_prefix={}", k.len(), short_fingerprint(k))) + .collect(); + anyhow::bail!( + "reopened store has {} live key(s) not present (live) in oracle; sample={:?}", + extras.len(), + sample + ); + } + + // Invariants 3 & 4: for every live oracle entry, the reopened store + // either has the same payload or is missing the key (tail dropped). + let mut survived_live: u64 = 0; + let mut dropped_live: u64 = 0; + let mut mismatches: Vec = Vec::new(); + for (key, eff) in &effect { + if let OracleEffect::Live(expected) = eff { + match store.get(key).context("DataWal::get for oracle key")? { + Some(got) => { + if &got != expected { + if mismatches.len() < cli.sample { + mismatches.push(format!( + "key_len={} expected_len={} got_len={} key_fp={}", + key.len(), + expected.len(), + got.len(), + short_fingerprint(key), + )); + } + } else { + survived_live += 1; + } + } + None => { + dropped_live += 1; + } + } + } + } + if !mismatches.is_empty() { + anyhow::bail!( + "payload mismatch for {} live oracle key(s); sample={:?}", + mismatches.len(), + mismatches + ); + } + + // For oracle Dead entries we already enforced "no extras" above; a + // tombstone that survived is consistent. We do not require Dead + // entries to have left a tombstone on disk because the validator + // cannot distinguish "tombstone applied" from "put + delete were + // both lost together" without the RecordLog scan, which is out of + // scope here. + + let dead_in_oracle = effect + .values() + .filter(|e| matches!(e, OracleEffect::Dead)) + .count(); + eprintln!( + "power_loss_validate: OK observed_live={} oracle_live={} survived={} dropped={} oracle_dead={} (extras=0)", + observed.len(), + oracle_live.len(), + survived_live, + dropped_live, + dead_in_oracle, + ); + Ok(()) +} + +/// Stable fingerprint for logs — first 8 bytes hex of a non-cryptographic +/// hash. Just enough to disambiguate sample lines without dumping bytes. +fn short_fingerprint(b: &[u8]) -> String { + // FNV-1a 64-bit — fine for log labels, not a security primitive. + let mut h: u64 = 0xcbf29ce484222325; + for &byte in b { + h ^= byte as u64; + h = h.wrapping_mul(0x100000001b3); + } + format!("{:016x}", h) +} diff --git a/crates/datawal-core/examples/power_loss_workload.rs b/crates/datawal-core/examples/power_loss_workload.rs new file mode 100644 index 0000000..cccae4f --- /dev/null +++ b/crates/datawal-core/examples/power_loss_workload.rs @@ -0,0 +1,376 @@ +//! Power-loss simulation **workload generator** for `datawal`. +//! +//! **This is a manual validation tool, NOT a usage example.** It is the +//! producer half of the dm-flakey power-loss harness; the consumer half +//! is the sibling `power_loss_validate` example, and the two are wired +//! together by `scripts/power_loss_dm_flakey.sh`. None of this is part +//! of CI and none of it is part of the published artefact's documented +//! behaviour. +//! +//! # What the workload does +//! +//! Open (or create) a `DataWal` under `--work-dir`, then run a deterministic +//! loop of `put` / `delete` operations seeded by `--seed`. For every +//! operation whose effect we want to claim is durable, we: +//! +//! 1. Call `DataWal::put` / `DataWal::delete`. +//! 2. Call `DataWal::fsync`. +//! 3. **Only after `fsync` returns Ok**, append a single JSONL line to +//! `--oracle` describing the operation and flush the oracle file +//! (`write_all` + `flush` + `sync_data`). +//! +//! The oracle is the contract: every line in `--oracle` is an operation +//! the workload claims is durable. The validator reopens the store after +//! a fault injection and asserts that the on-disk state honours every +//! oracle line. +//! +//! The oracle file lives on a *separate filesystem* from the WAL under +//! test (the orchestrator script pins it to `/tmp` while the WAL is on +//! the dm-flakey-backed mount). This is deliberate: the dm-flakey device +//! drops writes to the WAL filesystem, not to the oracle path. +//! +//! # CLI +//! +//! ```text +//! cargo run --release -p datawal --example power_loss_workload -- \ +//! --work-dir /mnt/datawal-test/wal \ +//! --oracle /tmp/datawal-powerloss-/oracle.jsonl \ +//! --seed 42 \ +//! --ops 100000 \ +//! --duration 0 +//! ``` +//! +//! `--duration` is wall-clock seconds; `--ops` is an operation budget. +//! The workload stops as soon as **either** limit is reached. `--ops 0` +//! disables the op budget; `--duration 0` disables the wall-clock budget. +//! At least one must be non-zero. +//! +//! # Oracle line format +//! +//! Each oracle line is a JSON object on its own line, base64-encoding any +//! bytes: +//! +//! ```json +//! {"op":"put","seq":1,"key":"","payload":""} +//! {"op":"del","seq":2,"key":""} +//! ``` +//! +//! `seq` is the op index of the **next** op that will be attempted; lines +//! are written in `seq` order. The validator treats the maximum recorded +//! `seq` as "the highest op the workload claims is durable". +//! +//! # Exit codes +//! +//! - `0` clean exit (budget reached, oracle flushed). +//! - `1` workload error (anything `DataWal` rejected before fsync). +//! - `2` setup error (missing arg, unwritable path, etc.). +//! +//! Exit codes mirror `soak.rs` so the orchestrator can treat the two +//! examples uniformly. +//! +//! # Honest claims +//! +//! - The workload is single-threaded and single-writer. +//! - The oracle is fsync-ordered: a line at `seq=N` means the workload +//! **observed** `DataWal::fsync` return `Ok(())` for the op at `seq=N`. +//! It does **not** mean the underlying block device fsync'd +//! successfully; under dm-flakey that is exactly the property we are +//! probing. +//! - This example does not inject faults. Fault injection is the +//! orchestrator script's job (dm-flakey table reload). + +use std::env; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use base64::Engine; +use datawal::DataWal; +use serde::Serialize; + +const DEFAULT_KEY_LEN: usize = 16; +const DEFAULT_VALUE_LEN: usize = 96; +const DELETE_EVERY: u64 = 23; +const FSYNC_BATCH: u64 = 1; + +struct Cli { + work_dir: PathBuf, + oracle: PathBuf, + seed: u64, + ops: u64, + duration: Duration, + key_len: usize, + value_len: usize, +} + +fn parse_args() -> Result { + let mut work_dir: Option = None; + let mut oracle: Option = None; + let mut seed: u64 = 1; + let mut ops: u64 = 0; + let mut duration_s: u64 = 0; + let mut key_len: usize = DEFAULT_KEY_LEN; + let mut value_len: usize = DEFAULT_VALUE_LEN; + + let args: Vec = env::args().skip(1).collect(); + let mut i = 0; + while i < args.len() { + let arg = &args[i]; + let next = || -> Result<&String> { + args.get(i + 1) + .with_context(|| format!("{} requires a value", arg)) + }; + match arg.as_str() { + "--work-dir" => { + work_dir = Some(PathBuf::from(next()?)); + i += 2; + } + "--oracle" => { + oracle = Some(PathBuf::from(next()?)); + i += 2; + } + "--seed" => { + seed = next()?.parse().context("parse --seed")?; + i += 2; + } + "--ops" => { + ops = next()?.parse().context("parse --ops")?; + i += 2; + } + "--duration" => { + duration_s = next()?.parse().context("parse --duration")?; + i += 2; + } + "--key-len" => { + key_len = next()?.parse().context("parse --key-len")?; + i += 2; + } + "--value-len" => { + value_len = next()?.parse().context("parse --value-len")?; + i += 2; + } + other => anyhow::bail!("unknown arg `{}`", other), + } + } + + let work_dir = work_dir.context("--work-dir is required")?; + let oracle = oracle.context("--oracle is required")?; + if ops == 0 && duration_s == 0 { + anyhow::bail!("at least one of --ops or --duration must be non-zero"); + } + if !(1..=512).contains(&key_len) { + anyhow::bail!("--key-len must be in 1..=512 (got {})", key_len); + } + if !(1..=65_536).contains(&value_len) { + anyhow::bail!("--value-len must be in 1..=65536 (got {})", value_len); + } + + Ok(Cli { + work_dir, + oracle, + seed, + ops, + duration: Duration::from_secs(duration_s), + key_len, + value_len, + }) +} + +/// Deterministic PRNG so a `(seed, op_idx)` pair always produces the +/// same key/payload bytes. We use `splitmix64` for the state advance +/// and slice the resulting bytes to fill key/value buffers; this is +/// not cryptographic and does not need to be. +#[derive(Clone, Copy)] +struct SplitMix64(u64); + +impl SplitMix64 { + fn new(seed: u64) -> Self { + Self(seed.wrapping_add(0x9E3779B97F4A7C15)) + } + fn next_u64(&mut self) -> u64 { + let mut z = self.0.wrapping_add(0x9E3779B97F4A7C15); + self.0 = z; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + z ^ (z >> 31) + } + fn fill(&mut self, out: &mut [u8]) { + let mut i = 0; + while i < out.len() { + let chunk = self.next_u64().to_le_bytes(); + let take = (out.len() - i).min(8); + out[i..i + take].copy_from_slice(&chunk[..take]); + i += take; + } + } +} + +fn key_for(seed: u64, op: u64, key_len: usize) -> Vec { + // Universe of ~4096 distinct keys per seed so deletes hit existing + // keys reasonably often without growing the keydir without bound. + let key_id = op.wrapping_mul(0x100000001B3) & 0xFFF; + let mut rng = SplitMix64::new(seed ^ key_id); + let mut out = vec![0u8; key_len]; + rng.fill(&mut out); + out +} + +fn value_for(seed: u64, op: u64, value_len: usize) -> Vec { + let mut rng = SplitMix64::new(seed ^ op ^ 0xDEADBEEFCAFEBABE); + let mut out = vec![0u8; value_len]; + rng.fill(&mut out); + out +} + +#[derive(Serialize)] +#[serde(tag = "op")] +enum OracleLine<'a> { + #[serde(rename = "put")] + Put { + seq: u64, + key: &'a str, + payload: &'a str, + }, + #[serde(rename = "del")] + Del { seq: u64, key: &'a str }, +} + +fn b64(b: &[u8]) -> String { + base64::engine::general_purpose::STANDARD.encode(b) +} + +fn write_oracle_line(file: &mut fs::File, line: &OracleLine<'_>) -> Result<()> { + let mut buf = serde_json::to_vec(line).context("encode oracle line")?; + buf.push(b'\n'); + file.write_all(&buf).context("write oracle line")?; + file.flush().context("flush oracle line")?; + // sync_data on the oracle file — this is on a *separate* filesystem + // from the WAL under test (orchestrator places it on /tmp); we want + // the line itself durable so the validator can trust it. + file.sync_data().context("fdatasync oracle file")?; + Ok(()) +} + +fn ensure_parent_dir(p: &Path) -> Result<()> { + if let Some(parent) = p.parent() { + if !parent.as_os_str().is_empty() { + fs::create_dir_all(parent) + .with_context(|| format!("create parent of {}", p.display()))?; + } + } + Ok(()) +} + +fn main() -> ExitCode { + let cli = match parse_args() { + Ok(c) => c, + Err(e) => { + eprintln!("power_loss_workload: setup error: {:#}", e); + return ExitCode::from(2); + } + }; + match run(cli) { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("power_loss_workload: failed: {:#}", e); + ExitCode::from(1) + } + } +} + +// `FSYNC_BATCH` is a knob for future tuning; today it is fixed at 1 so +// every op is `fsync`'d before the oracle line is appended. clippy's +// `modulo_one` is correct at this value but the structure is intentional +// (the modulo guard stays meaningful if the constant is ever raised). +#[allow(clippy::modulo_one)] +fn run(cli: Cli) -> Result<()> { + fs::create_dir_all(&cli.work_dir) + .with_context(|| format!("create work dir {}", cli.work_dir.display()))?; + ensure_parent_dir(&cli.oracle)?; + + // Truncate the oracle on each run. The workload is the sole writer. + let mut oracle = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(&cli.oracle) + .with_context(|| format!("open oracle {}", cli.oracle.display()))?; + + eprintln!( + "power_loss_workload: work_dir={} oracle={} seed={} ops={} duration={}s key_len={} value_len={}", + cli.work_dir.display(), + cli.oracle.display(), + cli.seed, + cli.ops, + cli.duration.as_secs(), + cli.key_len, + cli.value_len, + ); + + let mut log = DataWal::open(&cli.work_dir).context("DataWal::open work_dir")?; + let start = Instant::now(); + let ops_budget = if cli.ops == 0 { u64::MAX } else { cli.ops }; + let duration_budget = cli.duration; + + let mut seq: u64 = 0; + let mut puts: u64 = 0; + let mut dels: u64 = 0; + + loop { + if seq >= ops_budget { + break; + } + if !duration_budget.is_zero() && start.elapsed() >= duration_budget { + break; + } + + let do_delete = seq % DELETE_EVERY == DELETE_EVERY - 1; + let key = key_for(cli.seed, seq, cli.key_len); + if do_delete { + log.delete(&key).context("DataWal::delete")?; + // We sync every op to make oracle truthful. + if seq % FSYNC_BATCH == 0 { + log.fsync().context("DataWal::fsync (delete)")?; + let key_b64 = b64(&key); + write_oracle_line(&mut oracle, &OracleLine::Del { seq, key: &key_b64 })?; + dels += 1; + } + } else { + let value = value_for(cli.seed, seq, cli.value_len); + log.put(&key, &value).context("DataWal::put")?; + if seq % FSYNC_BATCH == 0 { + log.fsync().context("DataWal::fsync (put)")?; + let key_b64 = b64(&key); + let payload_b64 = b64(&value); + write_oracle_line( + &mut oracle, + &OracleLine::Put { + seq, + key: &key_b64, + payload: &payload_b64, + }, + )?; + puts += 1; + } + } + + seq = seq.wrapping_add(1); + } + + // Final fsync on the WAL and the oracle. The orchestrator may still + // pull the rug afterwards, but the workload's claim ends here. + log.fsync().context("final fsync (wal)")?; + oracle.sync_data().context("final fdatasync (oracle)")?; + + eprintln!( + "power_loss_workload: clean exit ops={} puts={} dels={} elapsed={}s", + seq, + puts, + dels, + start.elapsed().as_secs() + ); + Ok(()) +} diff --git a/docs/power-loss-results.md b/docs/power-loss-results.md new file mode 100644 index 0000000..3b22bb7 --- /dev/null +++ b/docs/power-loss-results.md @@ -0,0 +1,131 @@ +# Power-loss harness: verified runs + +Sanitized snapshot of `dm-flakey` power-loss simulation harness runs. See +[`power-loss-testing.md`](power-loss-testing.md) for the harness contract, +what it does, and what it explicitly does not show. + +The numbers below are reproducible from `scripts/power_loss_dm_flakey.sh` +with the same seed and ops on any host that satisfies the prerequisites in +`power-loss-testing.md`. + +## Run 1 — 2026-05-21 + +### Host shape, sanitized + +| Field | Value | +|---|---| +| OS family | Linux x86_64, apt-based distribution | +| Kernel | 6.12 series | +| device-mapper | flakey target available | +| Backing | loopback file under temporary local filesystem | +| Filesystem under test | ext4 | + +No hostname, user, paths, kernel patchlevel, mount table, partition UUID, +device IDs, container IDs, environment, or raw `uname`/`mount`/`env` +output is published. + +### Workload + +| Field | Value | +|---|---| +| Ops | 50000 | +| Key length | 16 bytes | +| Value length | 96 bytes | +| Seed | 42 (deterministic) | +| Fsync policy | every op | +| Effective key space | ~4096 (deletes target live keys by construction) | +| Wall time | ~9 s | +| Puts | 47827 | +| Deletes | 2173 | + +### Fault injection + +| Phase | Action | +|---|---| +| 1 | `dm-flakey` table `up=600 down=0` (healthy) | +| 2 | Workload writes to ext4 on top of `dm-flakey`, fsync per op; oracle written on a separate local filesystem | +| 3 | `dm-flakey` reloaded to `error_writes` (every subsequent write is rejected by the device) | +| 4 | `umount -f` (errors expected and tolerated) | +| 5 | `dm-flakey` reloaded to healthy table | +| 6 | Remount, reopen `DataWal`, run validator | + +### Result + +#### Recovery report on reopen + +| Metric | Value | +|---|---| +| Files scanned | 1 | +| Records replayed | 50000 | +| Tail segments truncated | 0 | +| Tail bytes discarded | 0 | +| Mid-stream errors | 0 | +| Last txid seen | 50000 | + +#### Validator output + +| Metric | Value | +|---|---| +| Observed live keys (on disk) | 3918 | +| Oracle live keys (expected) | 3918 | +| Survived (live in both) | 3918 | +| Dropped (oracle live missing on disk) | 0 | +| Oracle dead (tombstoned, correctly absent) | 178 | +| Extras (on disk but not in oracle) | 0 | +| Exit code | 0 | + +### Reading + +- Per-key prefix invariant held. Every key the oracle marked live after a + successful `fsync` was present on disk with the exact expected payload. + Every tombstoned key was correctly absent. No on-disk key appeared that + was not produced by the workload. +- No tail truncation was required: every record whose `fsync` returned + before the fault was durable past the `umount -f` and remount cycle, as + expected when `fsync` is honored end-to-end by the storage stack under + test. +- This run exercises the "happy" path of `dm-flakey error_writes` with + `fsync` per op. To exercise the prefix property more aggressively, a + future workload variant will buffer ops between fsyncs so the fault can + catch in-flight writes that have not yet reached the device. That is + expected to drive `dropped > 0` and `tail_bytes_discarded > 0` while + still keeping `extras = 0`. + +### Scope: what this run shows and does not show + +This run shows: under a controlled `dm-flakey` `error_writes` fault on +ext4, the reopened `DataWal` matches the fsync-ordered oracle, with no +extras and no corrupted payloads, for a 50k-operation workload. + +This run does not show: + +- behaviour under a real power cut on physical hardware, +- behaviour on lying storage (drives that ack `fsync` before persisting), +- behaviour on networked filesystems, +- behaviour on filesystems other than ext4, +- the `dm-log-writes` intermediate-state replay property (see roadmap). + +This is stricter than process-level crash testing but is not a substitute +for real power-cut testing on real hardware. DataWal trusts the storage +stack below it to honor `fsync`. + +### How to reproduce + +On a Linux host with the prerequisites listed in `power-loss-testing.md`: + +```sh +sudo \ + DATAWAL_POWERLOSS_OPS=50000 \ + DATAWAL_POWERLOSS_SEED=42 \ + DATAWAL_POWERLOSS_KEY_LEN=16 \ + DATAWAL_POWERLOSS_VALUE_LEN=96 \ + ./scripts/power_loss_dm_flakey.sh +``` + +Exit code 0 with the validator line + +```text +power_loss_validate: OK observed_live=3918 oracle_live=3918 survived=3918 dropped=0 oracle_dead=178 (extras=0) +``` + +is the expected outcome with these inputs. diff --git a/docs/power-loss-testing.md b/docs/power-loss-testing.md new file mode 100644 index 0000000..cbd7c44 --- /dev/null +++ b/docs/power-loss-testing.md @@ -0,0 +1,272 @@ +# Power-loss testing (Linux device-mapper) + +This page documents the **device-mapper power-loss harness**, which +exercises `DataWal` under simulated power-loss-class faults on Linux +using the kernel's `dm-flakey` target. + +The harness is shipped as two runnable examples +(`examples/power_loss_workload.rs`, `examples/power_loss_validate.rs`) +plus two driver scripts (`scripts/power_loss_dm_flakey.sh`, +`scripts/power_loss_cleanup.sh`). It is intentionally **not** part of +the default test suite, **not** part of CI, and **not** run by +`cargo test`. It is a tool you run by hand, as root, on a Linux box +you trust to spawn loop devices and to leave a directory under +`/tmp/datawal-powerloss-*` lying around if something goes wrong. + +It is **not** a real power-loss test. The host is up the whole time; +the kernel page cache, controller cache, and physical disk are +unaffected. Only the writes that reach the device-mapper layer of the +test stack are dropped. This catches a strict superset of the +`SIGKILL` cases already covered by +`crates/datawal-core/tests/crash_injection.rs`, but it is **not** a +substitute for testing on a machine where you actually pull the +power cord. + +See issue [#31](https://github.com/deepcausa/datawal/issues/31) for +the design rationale. + +## What it does + +The orchestrator script does, in order: + +1. Allocates a sparse backing file under `/tmp/datawal-powerloss-${ID}` + (default size `256 MiB`). +2. Attaches it to a loop device (`losetup --show -f`). +3. Builds a `dm-flakey` device on top of the loop device, initially in + the **healthy** mapping (`flakey ... 0 600 0` — 600 seconds up, 0 + seconds down). +4. Formats the dm device with `ext4` and mounts it under + `${WORK_ROOT}/mnt`. +5. Runs `power_loss_workload` against `${WORK_ROOT}/mnt/wal`. The + workload appends a deterministic stream of `put` / `delete` ops to + the `DataWal`, calling `fsync` after every op. After each + successful `fsync` it appends one base64 JSONL line to an oracle + file kept on a **separate** filesystem (`${WORK_ROOT}/oracle.jsonl`, + on the host's `/tmp`, never on the dm device). +6. Issues a host-side `sync` and then reloads the dm-flakey table to + the **fault** mapping (`flakey ... 0 0 60 1 error_writes` — 0 + seconds up, 60 seconds down, error every write). All writes that + the kernel has not yet drained to the device are now failed back + to the caller. +7. Force-unmounts the mountpoint (`umount -f`, falling back to + `umount -l` if necessary). I/O errors from the fault layer are + expected at this point. +8. Reloads the dm-flakey table back to the **healthy** mapping and + remounts the filesystem. This is the post-power-loss reopen path. +9. Runs `power_loss_validate` against the same `${WORK_ROOT}/mnt/wal`. + The validator opens the store, runs `RecoveryReport`, then checks + that the reopened keydir is a **prefix of the fsync-ordered + oracle** per key. + +The workload and the validator are independent binaries that +communicate only through (a) the on-disk store and (b) the JSONL +oracle. There is no shared state. + +## What is being checked + +The validator enforces three on-disk invariants against the oracle: + +- **Inv 3 (per-key prefix).** For every key that the oracle says is + live at some `fsync`-ordered prefix, the reopened store must + return either the same payload or no value at all. +- **Inv 4 (no payload corruption).** A returned value must match the + oracle's payload bytes exactly. A wrong payload is a hard failure. +- **Inv 5 (no extras).** No key in the reopened keydir may be absent + from `oracle_live ∪ oracle_dead`. The store is a **truncation** of + the workload, never a fabrication. + +It also surfaces `RecoveryReport` from `RecordLog::open`: +`files_scanned`, `records_replayed`, `tail_truncated`, +`tail_bytes_discarded`, `mid_stream_errors`, `last_txid_seen`. A +non-zero `mid_stream_errors` is treated as a hard failure (datawal +`0.1.x` aborts recovery on the first CRC mismatch in a sealed +segment). + +The validator deliberately does **not** assert anything about +keys whose oracle effect is `Dead` (a `delete` was issued). It cannot +distinguish "tombstone applied" from "put + delete both lost" using +only the on-disk state, and treating either outcome as wrong would +overconstrain the harness. + +## Prerequisites + +The orchestrator refuses to run unless all of these hold: + +- Linux. macOS and Windows are explicitly out of scope. +- `root`. `dm-flakey` and `losetup` both require it. +- These binaries on `$PATH`: `losetup`, `dmsetup`, `mkfs.ext4`, + `mount`, `umount`, `blockdev`, `cargo`, `dd`. +- Kernel module `dm-flakey` available + (`modprobe dm-flakey || lsmod | grep dm-flakey`). +- Free space under `/tmp` of at least `DATAWAL_POWERLOSS_SIZE_MB`. + +The harness only writes to: +- `/tmp/datawal-powerloss-${ID}/...` (backing file, mountpoint, oracle). +- The dm device named `datawal-test-${ID}`. +- The loop device returned by `losetup --show -f` for that backing file. + +The cleanup script enforces the same prefix guard; it refuses to +touch anything that does not match. + +## Environment variables + +| Variable | Default | Meaning | +| --- | --- | --- | +| `DATAWAL_POWERLOSS_SIZE_MB` | `256` | Backing-file size in MiB. Min 32. | +| `DATAWAL_POWERLOSS_FS` | `ext4` | Filesystem on the dm device. Only `ext4` is supported in v1. | +| `DATAWAL_POWERLOSS_OPS` | `50000` | Op count for the workload. `0` = unbounded (must combine with `DURATION`). | +| `DATAWAL_POWERLOSS_DURATION` | `0` | Wall-clock budget in seconds. `0` = unbounded (must combine with `OPS`). | +| `DATAWAL_POWERLOSS_SEED` | `42` | PRNG seed for key and payload generation. | +| `DATAWAL_POWERLOSS_KEY_LEN` | `16` | Key length in bytes. `1..=512`. | +| `DATAWAL_POWERLOSS_VALUE_LEN` | `96` | Payload length in bytes. `1..=65536`. | +| `DATAWAL_POWERLOSS_KEEP_ARTIFACTS` | `0` | If `1`, do not delete `/tmp/datawal-powerloss-${ID}` on success. Always kept on failure. | + +At least one of `OPS` or `DURATION` must be non-zero. + +## Exit codes + +| Code | Meaning | +| --- | --- | +| `0` | Reopen check passed: the post-fault store is a per-key prefix of the fsync-ordered oracle. | +| `1` | Invariant violated. Artefacts under `/tmp/datawal-powerloss-${ID}` are kept for inspection regardless of `KEEP_ARTIFACTS`. | +| `2` | Setup error: missing tool, not Linux, not root, dm-flakey unavailable, mkfs failed, mount failed, build failed. | + +Both Rust binaries follow the same convention. The shell driver +propagates the highest-priority code it has seen: setup error beats +invariant violation beats success. + +## Running + +The simplest invocation, as root, from the repo root: + +```bash +sudo ./scripts/power_loss_dm_flakey.sh +``` + +Override the workload size: + +```bash +sudo DATAWAL_POWERLOSS_OPS=200000 \ + DATAWAL_POWERLOSS_VALUE_LEN=1024 \ + ./scripts/power_loss_dm_flakey.sh +``` + +Keep the artefacts on success (for cargo-fmt-style inspection of the +on-disk segments after a clean run): + +```bash +sudo DATAWAL_POWERLOSS_KEEP_ARTIFACTS=1 \ + ./scripts/power_loss_dm_flakey.sh +``` + +To run the binaries by hand without the dm-flakey wrapper (useful for +sanity-checking the oracle / validator path on a real disk, with no +fault injection at all): + +```bash +cargo build --release \ + --example power_loss_workload \ + --example power_loss_validate + +mkdir -p /tmp/datawal-noflakey/wal +target/release/examples/power_loss_workload \ + --work-dir /tmp/datawal-noflakey/wal \ + --oracle /tmp/datawal-noflakey/oracle.jsonl \ + --ops 10000 + +target/release/examples/power_loss_validate \ + --work-dir /tmp/datawal-noflakey/wal \ + --oracle /tmp/datawal-noflakey/oracle.jsonl +``` + +Without fault injection the validator should always print `OK` and +exit `0`. If it does not, the bug is in the harness itself, not in +`datawal`. + +## Cleanup + +The orchestrator already cleans up on `EXIT`, `INT`, and `TERM`. The +separate cleanup script is for the case where the orchestrator was +killed before its trap ran (`SIGKILL`, host reboot, OOM kill, etc.): + +```bash +sudo ./scripts/power_loss_cleanup.sh +``` + +It is idempotent. It only touches: +- Mountpoints whose path starts with `/tmp/datawal-powerloss-`. +- dm devices whose name starts with `datawal-test-`. +- Loop devices whose backing file path starts with + `/tmp/datawal-powerloss-`. + +Any other dm device, mount, or loop device on the host is left +untouched. The script returns `0` whenever it finishes; "something +was dirty but I cleaned it" is success. + +## Interpreting a run + +A clean run prints, in order: + +- `[setup]` lines from the orchestrator (loop device, dm device, + filesystem, mount). +- A workload progress line per ~10k ops. +- `[fault]` lines when the dm-flakey table is flipped. +- `[validate]` lines showing the `RecoveryReport` and the per-key + totals. +- A final `OK observed_live=… oracle_live=… survived=… dropped=… + oracle_dead=… (extras=0)` line, then exit `0`. + +A failed run leaves the work directory in place. Useful next steps: + +- Re-read the JSONL oracle: it lists every op in fsync order with + base64-encoded key and payload. The last `seq` value tells you how + far the workload got before the table flipped. +- Re-run the validator by hand against the on-disk store and the + oracle. The same binary that ran inside the harness can be invoked + from outside it. +- Inspect segments with `datawal scan` or `datawal report` from + `datawal-cli` (shipped in `0.1.4`). +- Open the segment files directly with `xxd` and check the trailing + bytes against the corpus layout described in `format.rs`. + +## What this is not + +- **Not a real power-loss test.** A real power-loss test pulls the + power cord (or `echo b > /proc/sysrq-trigger`, or yanks a VM's + virtio-blk). The host kernel survives this harness; only the + device-mapper layer drops writes. +- **Not a substitute for `crash_injection.rs`.** The SIGKILL tests in + `tests/crash_injection.rs` exercise the in-process write path and + catch logic bugs that `dm-flakey` cannot reach (the workload never + gets killed mid-`append`). Both layers are useful. +- **Not a substitute for `dm-log-writes` replay.** `dm-flakey` drops + writes; it does not let you **replay** the I/O log and check + every intermediate state. A future track will add a + `dm-log-writes`-based harness; see the roadmap. +- **Not safe on a shared host.** It creates real loop devices and + real dm devices in the global kernel namespaces. Run it on a + scratch VM or a dedicated box. +- **Not safe on NFS, tmpfs, or any filesystem that does not honour + `fsync`.** The whole harness assumes the underlying storage stack + treats `fsync` as a hard durability barrier. The dm-flakey device + is built on top of a loop device on top of a real local filesystem + for that reason; do not move the backing file to NFS. +- **Not part of CI.** The example and the scripts only have to + **compile** in the default workspace; they are not executed + automatically. CI signals remain unchanged. +- **Not a benchmark.** Throughput numbers from this harness are not + comparable across machines and do not reflect real workloads. + +## Honest claim wording + +If you want to cite this harness in release notes or external docs, +use language no stronger than: + +> Exercised under a Linux device-mapper `dm-flakey` harness with +> ext4. After dropping every write that had not reached the +> device, the reopened store was a per-key prefix of the +> fsync-ordered oracle. This is a stricter test than `SIGKILL`-only +> crash injection; it is not a substitute for testing on a machine +> with a real power cord. + +Anything stronger overclaims. diff --git a/docs/roadmap.md b/docs/roadmap.md index d6e2922..62991f2 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -389,6 +389,12 @@ Hardening the Rust library for real workloads and publication: mutation. - Soak test: 24h append/rotate/fsync/reopen cycle. - Crash injection: kill during `append`, `rotate`, `compact_to`. +- Power-loss simulation via Linux device-mapper (issue #31). v1 is a + `dm-flakey` harness that drops every write that has not reached the + device, then reopens and checks per-key prefix invariants against an + fsync-ordered oracle. v2 would add a `dm-log-writes`-based replay + harness that captures the I/O log and asserts every intermediate + state, not just the final one. Linux-only, root-only, never CI. - `fdatasync` / `fsync_policy` if a real workload demands it. - Streaming `scan` that does not allocate the full record list. - Keydir by offset (not by full value) to reduce memory pressure. @@ -447,6 +453,7 @@ two disagree, the issues win — please open a PR against this section. | [#12](https://github.com/deepcausa/datawal/issues/12) WAL-engine features and related work | scope | open (notes) | Reference list (incl. `okaywal`). Features adopted only when justified. | | [#13](https://github.com/deepcausa/datawal/issues/13) Guardrails: datawal is JSONL, not a DB | scope | open (charter) | Canonical scope statement; SQL / query / multi-writer / server remain non-goals. | | [#14](https://github.com/deepcausa/datawal/issues/14) Derived tag index for simple filtering | exploratory | open (research) | Must remain a deterministic projection. No SQL drift, no general indexes. | +| [#31](https://github.com/deepcausa/datawal/issues/31) Power-loss simulation via Linux device-mapper | Track A | in progress | v1: `dm-flakey` harness (this PR). v2: `dm-log-writes` replay. Linux-only, root-only, not CI. See `docs/power-loss-testing.md`. | Bodies on GitHub stay authoritative for acceptance criteria and open questions. This table only records track, status and the one-line cut. diff --git a/scripts/power_loss_cleanup.sh b/scripts/power_loss_cleanup.sh new file mode 100755 index 0000000..74f7645 --- /dev/null +++ b/scripts/power_loss_cleanup.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Idempotent teardown for any device-mapper / loop / mount artefacts left +# behind by an interrupted `power_loss_dm_flakey.sh` run. +# +# Safety: +# - Only touches dm-mapper devices named `datawal-test-*`. +# - Only touches mount points / work dirs under `/tmp/datawal-powerloss-*`. +# - Only detaches loop devices whose backing file matches the pattern above. +# - Refuses to run on non-Linux or as non-root (no-op exit 2). +# +# Exit codes: +# 0 — nothing was wrong, or everything was cleaned up cleanly. +# 2 — preflight failed (not Linux / not root / missing tool). +# +# This script never returns 1; "something was dirty but I cleaned it" +# is still a success. + +set -euo pipefail + +readonly SCRIPT_NAME="${0##*/}" +readonly DM_PREFIX="datawal-test-" +readonly WORK_PREFIX="/tmp/datawal-powerloss-" + +log() { + printf '%s: %s\n' "$SCRIPT_NAME" "$*" >&2 +} + +die_setup() { + log "setup error: $*" + exit 2 +} + +if [[ "$(uname -s)" != "Linux" ]]; then + die_setup "Linux-only (uname=$(uname -s))" +fi +if [[ "$(id -u)" -ne 0 ]]; then + die_setup "must run as root" +fi +for tool in dmsetup losetup umount findmnt awk; do + if ! command -v "$tool" >/dev/null 2>&1; then + die_setup "missing required tool: $tool" + fi +done + +# 1. Unmount anything under /tmp/datawal-powerloss-*/mnt +while IFS= read -r line; do + [[ -n "$line" ]] || continue + log "umount $line" + umount -f "$line" 2>/dev/null || umount -l "$line" 2>/dev/null || log " (failed; continuing)" +done < <(findmnt -rn -o TARGET | awk -v p="$WORK_PREFIX" 'index($0, p)==1') + +# 2. Remove dm-flakey devices named datawal-test-* +while IFS= read -r name; do + [[ -n "$name" ]] || continue + log "dmsetup remove $name" + dmsetup remove "$name" 2>/dev/null || log " (failed; continuing)" +done < <(dmsetup ls 2>/dev/null | awk '{print $1}' | awk -v p="$DM_PREFIX" 'index($0, p)==1') + +# 3. Detach loop devices whose backing file lives under the work prefix +while IFS= read -r line; do + [[ -n "$line" ]] || continue + dev="$(awk '{print $1}' <<< "$line" | tr -d ':')" + log "losetup -d $dev" + losetup -d "$dev" 2>/dev/null || log " (failed; continuing)" +done < <(losetup -a 2>/dev/null | awk -v p="$WORK_PREFIX" '$0 ~ p') + +# 4. Remove work directories that match the pattern. Only after we have +# unmounted everything that could have been pointing at them. +shopt -s nullglob +for d in "${WORK_PREFIX}"*; do + if [[ -d "$d" ]]; then + if findmnt -rn "$d" >/dev/null 2>&1 || findmnt -rn "$d/mnt" >/dev/null 2>&1; then + log "$d still has mounts after step 1; leaving it alone" + continue + fi + log "rm -rf $d" + rm -rf "$d" + fi +done + +log "cleanup complete" +exit 0 diff --git a/scripts/power_loss_dm_flakey.sh b/scripts/power_loss_dm_flakey.sh new file mode 100755 index 0000000..6151e0f --- /dev/null +++ b/scripts/power_loss_dm_flakey.sh @@ -0,0 +1,302 @@ +#!/usr/bin/env bash +# Orchestrate a dm-flakey-based power-loss test for `datawal`. +# +# The harness: +# 1. Creates a sparse backing file under /tmp. +# 2. Wraps it in a loop device. +# 3. Stacks a dm-flakey target on top with `up_interval` only (no faults). +# 4. mkfs.ext4 on the flakey device, mounts it under /tmp/datawal-powerloss-/mnt. +# 5. Runs `power_loss_workload` against that mount. +# 6. After the workload exits, reloads the flakey table with +# `error_writes` for a configurable window — every subsequent write +# to the device errors. Then forces umount and remount. +# 7. Runs `power_loss_validate` against the remounted filesystem. +# 8. Cleans up (umount, dmsetup remove, losetup -d, rm). +# +# This is a *manual validation tool*, not a CI job. It requires root, +# Linux, dmsetup, losetup, mkfs.ext4. It refuses to run on non-Linux, +# without root, or without the required tools. It does NOT operate on +# operator-supplied block devices in v1 — the backing file is always a +# sparse file under /tmp scoped to a `datawal-powerloss-` directory. +# +# Required env vars (no defaults — the script refuses to invent them): +# none (sensible defaults below; override via env). +# +# Optional env vars: +# DATAWAL_POWERLOSS_SIZE_MB size of backing file (default: 256) +# DATAWAL_POWERLOSS_FS filesystem (default: ext4; only ext4 in v1) +# DATAWAL_POWERLOSS_OPS workload op budget (default: 50000) +# DATAWAL_POWERLOSS_DURATION workload wall-clock seconds (default: 0 = no limit) +# DATAWAL_POWERLOSS_SEED workload PRNG seed (default: 42) +# DATAWAL_POWERLOSS_KEY_LEN bytes per key (default: 16) +# DATAWAL_POWERLOSS_VALUE_LEN bytes per value (default: 96) +# DATAWAL_POWERLOSS_KEEP_ARTIFACTS retain work dir & oracle on success +# (default: 0; set to 1 to keep) +# +# Exit codes mirror the Rust examples: +# 0 — workload + validate both clean. +# 1 — validator surfaced an invariant violation (this is the +# interesting failure mode and the reason the harness exists). +# 2 — setup error (missing tool, not Linux, not root, mkfs failed, etc.). +# +# Safety: every artefact lives under /tmp/datawal-powerloss-. The +# dm-mapper device name is `datawal-test-`. The cleanup trap fires +# on any exit path. If the script is killed mid-flight, run +# `scripts/power_loss_cleanup.sh` to reclaim devices and mounts. + +set -euo pipefail + +readonly SCRIPT_NAME="${0##*/}" +readonly ID="$$" +readonly WORK_ROOT="/tmp/datawal-powerloss-${ID}" +readonly DM_NAME="datawal-test-${ID}" +readonly BACKING_FILE="${WORK_ROOT}/backing.img" +readonly MNT="${WORK_ROOT}/mnt" +readonly WAL_DIR="${MNT}/wal" +readonly ORACLE="${WORK_ROOT}/oracle.jsonl" + +readonly SIZE_MB="${DATAWAL_POWERLOSS_SIZE_MB:-256}" +readonly FS_TYPE="${DATAWAL_POWERLOSS_FS:-ext4}" +readonly OPS="${DATAWAL_POWERLOSS_OPS:-50000}" +readonly DURATION="${DATAWAL_POWERLOSS_DURATION:-0}" +readonly SEED="${DATAWAL_POWERLOSS_SEED:-42}" +readonly KEY_LEN="${DATAWAL_POWERLOSS_KEY_LEN:-16}" +readonly VALUE_LEN="${DATAWAL_POWERLOSS_VALUE_LEN:-96}" +readonly KEEP="${DATAWAL_POWERLOSS_KEEP_ARTIFACTS:-0}" + +LOOP_DEV="" +DM_CREATED=0 +MOUNTED=0 +WORK_CREATED=0 + +log() { + printf '%s: %s\n' "$SCRIPT_NAME" "$*" >&2 +} + +die_setup() { + log "setup error: $*" + exit 2 +} + +cleanup() { + local rc=$? + set +e + if (( MOUNTED )); then + umount -f "$MNT" 2>/dev/null || umount -l "$MNT" 2>/dev/null + fi + if (( DM_CREATED )); then + dmsetup remove "$DM_NAME" 2>/dev/null + fi + if [[ -n "$LOOP_DEV" ]]; then + losetup -d "$LOOP_DEV" 2>/dev/null + fi + if (( WORK_CREATED )) && [[ "$KEEP" != "1" || $rc -ne 0 ]]; then + # On non-zero exit OR when KEEP is off, wipe the artefacts. + # On rc==0 with KEEP=1, leave them in place for inspection. + if [[ $rc -eq 0 && "$KEEP" == "1" ]]; then + log "keeping artefacts under $WORK_ROOT (DATAWAL_POWERLOSS_KEEP_ARTIFACTS=1)" + else + rm -rf "$WORK_ROOT" 2>/dev/null + fi + fi + exit "$rc" +} +trap cleanup EXIT INT TERM + +# --- preflight --- + +if [[ "$(uname -s)" != "Linux" ]]; then + die_setup "Linux-only harness (uname=$(uname -s))" +fi + +if [[ "$(id -u)" -ne 0 ]]; then + die_setup "must run as root (need losetup, dmsetup, mount); try sudo $0" +fi + +# Try to locate cargo from invoking user (sudo on Debian/Ubuntu re-writes PATH +# via secure_path, so 'sudo -E' is not enough; do not depend on PATH passthrough). +if ! command -v cargo >/dev/null 2>&1; then + found="" + if [[ -n "${SUDO_USER:-}" ]]; then + user_home=$(getent passwd "$SUDO_USER" | cut -d: -f6 2>/dev/null || true) + for cand in "${user_home:-}/.cargo/bin/cargo" "/home/${SUDO_USER}/.cargo/bin/cargo"; do + [[ -n "$cand" && -x "$cand" ]] && { found="$cand"; break; } + done + fi + if [[ -z "$found" ]]; then + for cand in /home/*/.cargo/bin/cargo /root/.cargo/bin/cargo; do + [[ -x "$cand" ]] && { found="$cand"; break; } + done + fi + if [[ -n "$found" ]]; then + export PATH="$(dirname "$found"):$PATH" + log "preflight: using cargo from $found" + fi +fi + +# The rustup shim resolves the toolchain from $RUSTUP_HOME (default $HOME/.rustup). +# When invoked via sudo, $HOME is /root and the default toolchain is unset, +# producing 'no default is configured'. Re-point at the invoking user's home. +if [[ -n "${SUDO_USER:-}" ]]; then + sudo_home=$(getent passwd "$SUDO_USER" | cut -d: -f6 2>/dev/null || true) + if [[ -n "$sudo_home" && -d "$sudo_home/.rustup" ]]; then + export RUSTUP_HOME="$sudo_home/.rustup" + export CARGO_HOME="$sudo_home/.cargo" + log "preflight: RUSTUP_HOME=$RUSTUP_HOME" + fi +fi +if ! cargo --version >/dev/null 2>&1; then + export RUSTUP_TOOLCHAIN="${RUSTUP_TOOLCHAIN:-stable}" + log "preflight: cargo had no default; forcing RUSTUP_TOOLCHAIN=$RUSTUP_TOOLCHAIN" +fi + +for tool in losetup dmsetup mkfs."$FS_TYPE" mount umount blockdev cargo dd; do + if ! command -v "$tool" >/dev/null 2>&1; then + die_setup "missing required tool: $tool" + fi +done + +if [[ "$FS_TYPE" != "ext4" ]]; then + die_setup "only ext4 is supported in v1 (got $FS_TYPE)" +fi + +if [[ "$SIZE_MB" -lt 32 ]]; then + die_setup "DATAWAL_POWERLOSS_SIZE_MB must be >= 32 (got $SIZE_MB)" +fi + +if [[ "$OPS" -eq 0 && "$DURATION" -eq 0 ]]; then + die_setup "set at least one of DATAWAL_POWERLOSS_OPS / DATAWAL_POWERLOSS_DURATION non-zero" +fi + +# --- build the harness binaries up front (no compile during the run) --- + +log "preflight: building examples in release mode" +cd "$(dirname "$0")/.." +cargo build --release \ + --example power_loss_workload \ + --example power_loss_validate \ + >&2 || die_setup "cargo build of examples failed" + +readonly WORKLOAD_BIN="$(pwd)/target/release/examples/power_loss_workload" +readonly VALIDATE_BIN="$(pwd)/target/release/examples/power_loss_validate" + +[[ -x "$WORKLOAD_BIN" ]] || die_setup "workload binary missing at $WORKLOAD_BIN" +[[ -x "$VALIDATE_BIN" ]] || die_setup "validate binary missing at $VALIDATE_BIN" + +# --- stage the work root --- + +mkdir -p "$WORK_ROOT" +WORK_CREATED=1 +mkdir -p "$MNT" + +log "stage: WORK_ROOT=$WORK_ROOT DM_NAME=$DM_NAME size=${SIZE_MB}MiB fs=$FS_TYPE" + +# Create sparse backing file. +truncate -s "${SIZE_MB}M" "$BACKING_FILE" || die_setup "truncate backing file failed" + +# Attach to a loop device. +LOOP_DEV="$(losetup --show -f "$BACKING_FILE")" || die_setup "losetup attach failed" +log "loop: $LOOP_DEV" + +# Build the initial dm-flakey table. Sector count = file size in 512-byte sectors. +SECTORS="$(blockdev --getsz "$LOOP_DEV")" +[[ "$SECTORS" -gt 0 ]] || die_setup "blockdev --getsz returned $SECTORS" + +# Healthy table: up_interval=600 down_interval=0 -> never faults, just a +# passthrough we can later reload with a faulting variant. +HEALTHY_TABLE="0 $SECTORS flakey $LOOP_DEV 0 600 0" +dmsetup create "$DM_NAME" --table "$HEALTHY_TABLE" || die_setup "dmsetup create failed" +DM_CREATED=1 +log "dm-flakey created: /dev/mapper/$DM_NAME table='$HEALTHY_TABLE'" + +# mkfs and mount. +mkfs."$FS_TYPE" -q -F "/dev/mapper/$DM_NAME" || die_setup "mkfs.$FS_TYPE failed" +mount -t "$FS_TYPE" "/dev/mapper/$DM_NAME" "$MNT" || die_setup "mount failed" +MOUNTED=1 +log "mount: /dev/mapper/$DM_NAME on $MNT ($FS_TYPE)" + +mkdir -p "$WAL_DIR" + +# --- workload phase --- + +log "workload: starting ops=$OPS duration=${DURATION}s seed=$SEED key_len=$KEY_LEN value_len=$VALUE_LEN" +WL_RC=0 +"$WORKLOAD_BIN" \ + --work-dir "$WAL_DIR" \ + --oracle "$ORACLE" \ + --seed "$SEED" \ + --ops "$OPS" \ + --duration "$DURATION" \ + --key-len "$KEY_LEN" \ + --value-len "$VALUE_LEN" \ + || WL_RC=$? + +if (( WL_RC != 0 )); then + log "workload exited with code $WL_RC; aborting before fault injection" + exit "$WL_RC" +fi + +[[ -s "$ORACLE" ]] || die_setup "oracle file is empty at $ORACLE — workload produced no claims" + +# --- fault injection --- +# +# Flip the dm-flakey table to error every write for a window long enough +# to cover the umount flush. The kernel will see EIO on writes; the +# subsequent umount may report errors (we use umount -f / -l to detach +# regardless). Then we flip back to the healthy table and remount, so +# the validator runs against a filesystem whose post-workload writes +# were dropped — the proxy for power loss. + +FAULT_TABLE="0 $SECTORS flakey $LOOP_DEV 0 0 60 1 error_writes" +log "fault: reloading dm-flakey to error_writes; sync may fail (expected)" + +# `sync` first so dirty pages from the workload's final fsync are +# already pushed; what we then drop is anything generated by umount +# itself. +sync || true + +dmsetup suspend "$DM_NAME" +dmsetup reload "$DM_NAME" --table "$FAULT_TABLE" +dmsetup resume "$DM_NAME" + +# Force umount; errors are expected because writes now EIO. +log "umount -f (errors expected)" +if ! umount -f "$MNT" 2>&1 | sed -e "s/^/$SCRIPT_NAME: umount: /" >&2; then + log "umount -f failed; retrying with -l (lazy)" + umount -l "$MNT" 2>/dev/null || true +fi +MOUNTED=0 + +# Restore the healthy table so the remount can flush its journal. +HEALTHY_TABLE="0 $SECTORS flakey $LOOP_DEV 0 600 0" +dmsetup suspend "$DM_NAME" +dmsetup reload "$DM_NAME" --table "$HEALTHY_TABLE" +dmsetup resume "$DM_NAME" +log "fault: reloaded to healthy table" + +# Remount. +mount -t "$FS_TYPE" "/dev/mapper/$DM_NAME" "$MNT" || die_setup "remount failed" +MOUNTED=1 +log "remount: /dev/mapper/$DM_NAME on $MNT" + +# --- validate phase --- + +log "validate: starting against $WAL_DIR (oracle=$ORACLE)" +VR_RC=0 +"$VALIDATE_BIN" \ + --work-dir "$WAL_DIR" \ + --oracle "$ORACLE" \ + || VR_RC=$? + +if (( VR_RC == 0 )); then + log "validate: OK" + exit 0 +elif (( VR_RC == 1 )); then + log "validate: INVARIANT VIOLATED (rc=1); artefacts kept at $WORK_ROOT" + # Force-keep on failure regardless of KEEP flag. + KEEP=1 + exit 1 +else + log "validate: setup error (rc=$VR_RC)" + exit "$VR_RC" +fi