From 57876435413a0f41a0c9826077509e8f4baa03a6 Mon Sep 17 00:00:00 2001 From: YifanXu1999 Date: Tue, 17 Nov 2026 22:03:01 +0800 Subject: [PATCH] =?UTF-8?q?feat(store):=20open=20SQLite=20Stores=20Normal?= =?UTF-8?q?=20and=20default=20PostgreSQL=20to=20local=20(layerfs-project#1?= =?UTF-8?q?55=20=C2=A77)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner direction 2026-11-17: enable the Normal mode on both backends now, ahead of its safety and performance gate; troubleshooting follows. SQLite - New profile SqlitePersistenceProfile::Normal: WAL, synchronous=1 (NORMAL), fullfsync=0, wal_autocheckpoint=0, identity sqlite-wal-normal-v2. - Every product opener selects it: the daemon (install and owner), the Store service (install and SDK open), project add and discovery, Store identity, compact, SQLite transfer and backup restore. - The session's checkpoint thread opened its connection at synchronous=0. It now takes the session's value and reads it back; under NORMAL a pass at 0 would mark log frames copied that never reached the disk. - The Init, live-add and install gates accept Normal or Disposable and still refuse Durable. Disposable stays selectable, so existing tests that select it keep running at synchronous=OFF. PostgreSQL - synchronous_commit defaults to local (it was off): SynchronousCommit's default, PostgresSelection::new, store.connect and the open arguments. A Store connected earlier keeps its recorded value; a recorded "off" is still honoured. Not in this change (known gaps, #155 §7) - The safety proof (§7.5) and the performance gate (§7.6) have not run. No power-loss guarantee is claimed. - Store files written outside SQLite are still not synchronized (§7.3): install, transfer, restore, snapshot and home records. - The manifest wire byte (2) and the record label ("disposable") are unchanged; store.get still reports profile "disposable". - No PostgreSQL server prerequisite check (fsync, full_page_writes). - PersistenceConfig::sqlite still defaults to Durable. Tests - background_checkpoint.rs: the same long-log case now also runs under Normal and asserts the read-back profile (synchronous=1). - open_args.rs: the default setting is "local". - No test deleted. Docs: root and engine AGENTS.md persistence-profile rule, the SQLite architecture page, the PostgreSQL default in the Store pages and the SDK skill reference. Production LOC: 187380 -> 187394 (delta +14) Scope and method: tools/production_loc.py at this commit, first parent 9203d99db0 against the committed tree; core 40408 -> 40408, engine 89372 -> 89386, control 51073 -> 51073, api 6527 -> 6527; reference absent (retired R9c; last counted 65,417 at tree 498dd1917812). Co-Authored-By: Claude Fable 5.2 --- .../references/methods/store.md | 2 +- AGENTS.md | 39 ++++++++++++------- .../layerfs-bucket/src/backup/restore.rs | 2 +- control/crates/layerfs-store/src/compact.rs | 2 +- control/crates/layerfs-store/src/identity.rs | 2 +- control/crates/layerfs-store/src/open_args.rs | 2 +- .../layerfs-store/src/postgresql/connect.rs | 2 +- .../layerfs-store/src/postgresql/open.rs | 11 +++--- .../crates/layerfs-store/src/project/add.rs | 4 +- .../layerfs-store/src/project/discovery.rs | 2 +- .../src/sqlite/transfer/steps.rs | 2 +- .../crates/layerfs-store/tests/open_args.rs | 2 +- docs/architecture/api/sdk/store.md | 2 +- .../sqlite-and-postgresql.md | 2 +- docs/architecture/engine/store/sqlite.md | 10 ++--- docs/versioned/2.0.0/machines-and-stores.md | 2 +- engine/AGENTS.md | 9 +++-- engine/crates/layerfs-bridge/src/provision.rs | 4 +- .../layerfs-daemon/src/application/owner.rs | 2 +- engine/crates/layerfs-daemon/src/install.rs | 3 +- .../src/backend/sqlite/session/background.rs | 22 ++++++++--- .../src/backend/sqlite/session/connection.rs | 4 +- .../src/backend/sqlite/session/profile.rs | 14 +++++-- .../layerfs-persistence/src/store/config.rs | 25 +++++++++--- .../tests/background_checkpoint.rs | 29 ++++++++++---- .../crates/layerfs-project/src/store/init.rs | 8 ++-- .../layerfs-project/src/store/install.rs | 7 +--- .../crates/layerfs-project/src/store/live.rs | 12 +++--- .../layerfs-store-service/src/install.rs | 2 +- .../src/store_service.rs | 3 +- 30 files changed, 145 insertions(+), 87 deletions(-) diff --git a/.agents/skills/layerfs-js-sdk/references/methods/store.md b/.agents/skills/layerfs-js-sdk/references/methods/store.md index c053178544..19d1bc2530 100644 --- a/.agents/skills/layerfs-js-sdk/references/methods/store.md +++ b/.agents/skills/layerfs-js-sdk/references/methods/store.md @@ -80,7 +80,7 @@ Store on another machine is read through its worker; a stopped worker is | Call | What it does | Notes | | --- | --- | --- | | `create(name)` | An empty sqlite Store on `native` | Live result: the record above, with `projects: []`. `Exists` for a taken name | -| `connect(name, url, …)` | Attaches a PostgreSQL database | `url` is `postgresql://@:/[?sslmode=…]` without a password (one in the user part or in any query parameter whose name contains `password` is `InvalidInput`); `password` is a reference, `env:LAYERFS_` or `file:/path`. `synchronous_commit` defaults to `off`. The same database under a second name is `Exists` | +| `connect(name, url, …)` | Attaches a PostgreSQL database | `url` is `postgresql://@:/[?sslmode=…]` without a password (one in the user part or in any query parameter whose name contains `password` is `InvalidInput`); `password` is a reference, `env:LAYERFS_` or `file:/path`. `synchronous_commit` defaults to `local`. The same database under a second name is `Exists` | | `attach(name, machine, …)` | Lets a machine run Sandboxes of a PostgreSQL Store | Checks from that machine that it reaches the database and its role may write; `url` and `password` replace the Store's for that machine, either alone. `InvalidInput` for sqlite; `Unsupported` when the platform runs no PostgreSQL Sandboxes; `Unavailable` (not reached, machine stopped); `Denied` (login or privilege); `Conflict` while the Store is disconnected or reconnected | | `detach(name, machine)` | Stops a machine running Sandboxes of a PostgreSQL Store | Dials nothing; detaching `native` is allowed. `InvalidInput` (sqlite, or not attached there); `InUse` while a Sandbox of the Store is recorded on the machine | | `push(name, machine, …)` | Sends a sqlite Store's file from `native` to `machine` | `InvalidInput` unless the Store is on `native` (pull it first). Progress `{ phase, done, total }` on `on_progress` / `--progress`: `done` and `total` are bytes during `send`, null in the other phases (`check`, `snapshot`, `verify`, `install`, `record`, `retire`). `InUse` while the Store is held (Sandbox records, processes with it open; `detail.related` lists them as `{ kind, name }`); `Capacity` when the target has less free disk than the Store plus a headroom of 1 GiB (or a tenth of the Store), checked before anything changes; `Unavailable` naming `layerfs machine start ` for a stopped machine; while it runs, Store calls and operations on it are `Busy`, from the caller's process too. An uncertain push whose process is gone (it ended during install or record, or a step's answer was lost) leaves every later call `Unresolved`, naming `store status` and the push that clears it: a new `push` (or `pull`) first deletes the copy on the machine the record does not name, then runs. A push whose `retire` did not finish leaves the Store in use on `machine` and `last_transfer.retired: false`; the next `pull` deletes the old copy first | diff --git a/AGENTS.md b/AGENTS.md index 3ad415877e..cc856ba6eb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,7 @@ assignment. Earlier wording of these guides is in the | Rule | Where | | --- | --- | -| Disposable / WAL / synchronous=OFF is the only SQLite Store profile; Durable execution is `NOT_RUN` | [Persistence profile](#active-persistence-profile--owner-direction-2026-10-07) | +| Normal / WAL / synchronous=NORMAL is the product SQLite Store profile; Durable execution is `NOT_RUN` | [Persistence profile](#active-persistence-profile--owner-direction-2026-11-17) | | One attempted operation: no automatic retry, busy handler, refresh or replay | [§4](#4-code-build-and-docs) | | Build tests with `--no-run` first; every test command stops at 120 s, and reaching it is a hang | [§4](#4-code-build-and-docs) | | CI reports and never gates; verify your own change and report checks and gaps | [§4](#4-code-build-and-docs) | @@ -98,24 +98,33 @@ assignment. Earlier wording of these guides is in the not resume old loops, reopen closed rows or run old campaigns unless the task selects that work, and do not choose authority by issue number alone. -## Active persistence profile — owner direction 2026-10-07 - -- **Disposable / WAL / synchronous=OFF is the only permitted profile for a - SQLite Store.** Durable execution is disabled indefinitely, until the owner - explicitly allows it again; this supersedes earlier instructions to exercise - both profiles or resume Durable. A milestone, old benchmark selection, - default argument or passing build does not reauthorize Durable. -- Select Disposable explicitly for every new Store, application run, test, - diagnostic and measurement; never execute Durable or fall back to it. - Retained Durable code may compile; report its execution as +## Active persistence profile — owner direction 2026-11-17 + +- **Normal / WAL / synchronous=NORMAL is the profile every product path opens + a SQLite Store with** (owner, 2026-11-17; it was Disposable / synchronous=OFF + from 2026-10-07). The owner enabled it ahead of its safety and performance + gate ([layerfs-project#155](https://github.com/Ephemeral-AI-Lab/layerfs-project/issues/155) + §7.5 and §7.6), which has **not** been run: claim no power-loss guarantee + from it yet. Store files written outside SQLite (install, transfer, restore, + snapshot) are still not synchronized (#155 §7.3). +- Disposable / synchronous=OFF remains selectable, and existing tests, + diagnostics and measurements that select it explicitly keep doing so. A + Store's manifest and record still carry the `disposable` label; it names the + non-durable WAL Store, not the connection's synchronization. +- Durable execution is disabled indefinitely, until the owner explicitly + allows it again. A milestone, old benchmark selection, default argument or + passing build does not reauthorize Durable; never execute it or fall back to + it. Retained Durable code may compile; report its execution as `NOT_RUN — disabled by owner until explicit reauthorization`. Historical Durable receipts keep their verdicts. This policy does not claim the Durable API was removed. -- Disposable's COMMIT does not checkpoint (`wal_autocheckpoint=0`); its writing - session's own thread checkpoints the WAL off the commit path (owner approval, - 2026-11-07; [SQLite checkpoints](docs/architecture/engine/store/sqlite.md#checkpoints)). +- A Normal or Disposable COMMIT does not checkpoint (`wal_autocheckpoint=0`); + the writing session's own thread checkpoints the WAL off the commit path + with the session's synchronization (owner approval, 2026-11-07; + [SQLite checkpoints](docs/architecture/engine/store/sqlite.md#checkpoints)). - A PostgreSQL Store has no profile; its write setting is `synchronous_commit` - (default `off`), recorded by `store.connect`. + (default `local` since 2026-11-17, `off` before), recorded by + `store.connect`. A Store connected earlier keeps its recorded value. - Each Workspace's daemon overlay is a separate MEMORY/OFF/EXCLUSIVE database file. diff --git a/control/crates/layerfs-bucket/src/backup/restore.rs b/control/crates/layerfs-bucket/src/backup/restore.rs index f2a87bcf3a..be0223834b 100644 --- a/control/crates/layerfs-bucket/src/backup/restore.rs +++ b/control/crates/layerfs-bucket/src/backup/restore.rs @@ -240,7 +240,7 @@ fn fetch( progress::begin("verify", None); let opened = open_store_observed( PersistenceConfig::sqlite(&temporary) - .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + .with_sqlite_profile(SqlitePersistenceProfile::Normal), &binding, cursor_key, 1, diff --git a/control/crates/layerfs-store/src/compact.rs b/control/crates/layerfs-store/src/compact.rs index fb9a10def6..7f24751708 100644 --- a/control/crates/layerfs-store/src/compact.rs +++ b/control/crates/layerfs-store/src/compact.rs @@ -119,7 +119,7 @@ pub fn compact(home: &Path, _events: &mut Events<'_>, store: &str) -> Outcome { }; let identity = identity::identity(&args)?; let handles = SqliteHandles::open_writable( - PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable), + PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Normal), &identity.binding, identity.cursor_key, ) diff --git a/control/crates/layerfs-store/src/identity.rs b/control/crates/layerfs-store/src/identity.rs index 90ead8fad6..f8b19e1827 100644 --- a/control/crates/layerfs-store/src/identity.rs +++ b/control/crates/layerfs-store/src/identity.rs @@ -255,7 +255,7 @@ fn create_empty( let sealed = ProjectApi::new() .create_store(CreateRequest { store: PersistenceConfig::sqlite(&temporary) - .with_sqlite_profile(SqlitePersistenceProfile::Disposable) + .with_sqlite_profile(SqlitePersistenceProfile::Normal) .into(), policy, catalog: HistoryCatalogConfig { diff --git a/control/crates/layerfs-store/src/open_args.rs b/control/crates/layerfs-store/src/open_args.rs index fdc317522b..4274f6df67 100644 --- a/control/crates/layerfs-store/src/open_args.rs +++ b/control/crates/layerfs-store/src/open_args.rs @@ -105,7 +105,7 @@ impl PgSelection { schema_of(self.schema.as_deref()) } - /// Its `synchronous_commit`, `off` by default. + /// Its `synchronous_commit`, `local` by default. pub fn setting(&self) -> Result { setting_of(self.synchronous_commit.as_deref()) } diff --git a/control/crates/layerfs-store/src/postgresql/connect.rs b/control/crates/layerfs-store/src/postgresql/connect.rs index e0b3929a33..08adce740c 100644 --- a/control/crates/layerfs-store/src/postgresql/connect.rs +++ b/control/crates/layerfs-store/src/postgresql/connect.rs @@ -195,7 +195,7 @@ pub fn connect(home: &Path, request: &StoreConnect) -> Outcome { let name = request.name.as_str(); let given = request.url.as_str(); refuse_password(given)?; - let sync = request.synchronous_commit.as_deref().unwrap_or("off"); + let sync = request.synchronous_commit.as_deref().unwrap_or("local"); if !matches!( sync, "off" | "local" | "remote_write" | "on" | "remote_apply" diff --git a/control/crates/layerfs-store/src/postgresql/open.rs b/control/crates/layerfs-store/src/postgresql/open.rs index 5fa15e5e77..ebc755c0ea 100644 --- a/control/crates/layerfs-store/src/postgresql/open.rs +++ b/control/crates/layerfs-store/src/postgresql/open.rs @@ -14,10 +14,11 @@ use layerfs_home::outcome::{refused, Outcome}; use layerfs_store_service::StoreService; use std::path::Path; -/// `synchronous_commit` as the call or record names it: off (default), -/// local, remote_write, on or remote_apply. +/// `synchronous_commit` as the call or record names it: off, local (the +/// default since 2026-11-17), remote_write, on or remote_apply. A record +/// keeps the value it was connected with. pub fn setting_of(given: Option<&str>) -> Result { - let given = given.unwrap_or("off"); + let given = given.unwrap_or("local"); match given { "off" | "local" | "remote_write" | "on" | "remote_apply" => Ok(given.to_owned()), other => Err(refused( @@ -63,11 +64,11 @@ pub fn url_of(url: Option<&str>) -> Result<&str, SdkRefusal> { pub fn selection(url: &str, schema: &str, sync: &str) -> layerfs_persistence::PostgresSelection { use layerfs_persistence::SynchronousCommit as S; let setting = match sync { - "local" => S::Local, + "off" => S::Off, "remote_write" => S::RemoteWrite, "on" => S::On, "remote_apply" => S::RemoteApply, - _ => S::Off, + _ => S::Local, }; layerfs_persistence::PostgresSelection::new(url, schema).with_synchronous_commit(setting) } diff --git a/control/crates/layerfs-store/src/project/add.rs b/control/crates/layerfs-store/src/project/add.rs index 31594a5c40..7519a1a492 100644 --- a/control/crates/layerfs-store/src/project/add.rs +++ b/control/crates/layerfs-store/src/project/add.rs @@ -60,7 +60,7 @@ pub fn save_entry( let project = Project { store: SealedStore { path: entry.join("sealed.sqlite"), - profile: SqlitePersistenceProfile::Disposable, + profile: SqlitePersistenceProfile::Normal, sqlite_version: manifest.host_sqlite.clone(), bytes: manifest.bytes, }, @@ -170,7 +170,7 @@ pub fn add_project(request: &ProjectAdd) -> Outcome { let request = AddRequest { source, store: PersistenceConfig::sqlite(&store_file) - .with_sqlite_profile(SqlitePersistenceProfile::Disposable) + .with_sqlite_profile(SqlitePersistenceProfile::Normal) .into(), binding: identity.binding.clone(), cursor_key: identity.cursor_key, diff --git a/control/crates/layerfs-store/src/project/discovery.rs b/control/crates/layerfs-store/src/project/discovery.rs index c9d34793e3..7a5015d723 100644 --- a/control/crates/layerfs-store/src/project/discovery.rs +++ b/control/crates/layerfs-store/src/project/discovery.rs @@ -291,7 +291,7 @@ pub fn project(home: Option<&Path>, store: Option<&str>, name: Option<&str>) -> Ok(Project { store: SealedStore { path: path.join("sealed.sqlite"), - profile: SqlitePersistenceProfile::Disposable, + profile: SqlitePersistenceProfile::Normal, sqlite_version: manifest.host_sqlite.clone(), bytes: manifest.bytes, }, diff --git a/control/crates/layerfs-store/src/sqlite/transfer/steps.rs b/control/crates/layerfs-store/src/sqlite/transfer/steps.rs index fdca5f159a..8da2a91770 100644 --- a/control/crates/layerfs-store/src/sqlite/transfer/steps.rs +++ b/control/crates/layerfs-store/src/sqlite/transfer/steps.rs @@ -88,7 +88,7 @@ fn checkpoint_leftover_wal(file: &Path, frame: &StepFrame) -> Result<(), SdkRefu let identity = crate::identity::identity_of(record)?; if let Ok(handles) = layerfs_persistence::SqliteHandles::open_writable( layerfs_persistence::PersistenceConfig::sqlite(file) - .with_sqlite_profile(layerfs_persistence::SqlitePersistenceProfile::Disposable), + .with_sqlite_profile(layerfs_persistence::SqlitePersistenceProfile::Normal), &identity.binding, identity.cursor_key, ) { diff --git a/control/crates/layerfs-store/tests/open_args.rs b/control/crates/layerfs-store/tests/open_args.rs index b039308a96..179f55ad08 100644 --- a/control/crates/layerfs-store/tests/open_args.rs +++ b/control/crates/layerfs-store/tests/open_args.rs @@ -25,7 +25,7 @@ fn reads_the_routing_fields_and_refuses_a_missing_store_path() { ); let postgresql = open.postgresql.as_ref().unwrap(); assert_eq!(postgresql.url().unwrap(), "postgresql://u:pw@x/db"); - assert_eq!(postgresql.setting().unwrap(), "off"); + assert_eq!(postgresql.setting().unwrap(), "local"); assert_eq!( postgresql.schema().unwrap_err().message, "postgresql.schema is required" diff --git a/docs/architecture/api/sdk/store.md b/docs/architecture/api/sdk/store.md index a0e5dd34a7..c1a222c86d 100644 --- a/docs/architecture/api/sdk/store.md +++ b/docs/architecture/api/sdk/store.md @@ -214,7 +214,7 @@ CLI layerfs store connect [--password ] [--synchronou | --- | --- | --- | | `url` | Text | `postgresql://` or `postgres://`, holding no password (no `user:pass@`, no query parameter whose name contains `password`); names its user when `password` is given | | `password` option | Text | a reference, `env:NAME` or `file:/absolute/path` (trailing newline dropped), never the password; it must resolve now and is resolved again in the caller on every later call ([`secrets.rs`](../../../../control/crates/layerfs-home/src/secrets.rs)) | -| `synchronous_commit` option | Text | `off` (default), `local`, `remote_write`, `on` or `remote_apply`; applied to the Store's write transactions | +| `synchronous_commit` option | Text | `off`, `local` (default), `remote_write`, `on` or `remote_apply`; applied to the Store's write transactions | ```text { name, kind: "postgresql", schema: "layerfs", synchronous_commit, diff --git a/docs/architecture/control/stores-and-projects/sqlite-and-postgresql.md b/docs/architecture/control/stores-and-projects/sqlite-and-postgresql.md index afec073bd8..3cb4f22304 100644 --- a/docs/architecture/control/stores-and-projects/sqlite-and-postgresql.md +++ b/docs/architecture/control/stores-and-projects/sqlite-and-postgresql.md @@ -77,7 +77,7 @@ another operation holds the Store ```text store.connect(name, url, {password: "env:PG", synchronous_commit}) refuse_password(url) user:pass@ or any *password* param -> InvalidInput - synchronous_commit off (default) | local | remote_write | on + synchronous_commit off | local (default) | remote_write | on | remote_apply, else InvalidInput resolve(password) checked once now Exists? same name, or same server+database, or diff --git a/docs/architecture/engine/store/sqlite.md b/docs/architecture/engine/store/sqlite.md index 91dee2620b..b689a298a7 100644 --- a/docs/architecture/engine/store/sqlite.md +++ b/docs/architecture/engine/store/sqlite.md @@ -28,7 +28,7 @@ The SQLite backend of the [Store](README.md): one WAL file every opener shares t | --- | --- | --- | | `sqlite_pack_layout` | `Monolithic` (`user_version` 36): whole `pack.body`; `GroupRows` (37): `control` row plus one `pack_unit` row per group; `GroupRowsIndexed` (38): adds index `pack_unit_mapping` | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs), [`objects_units.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/objects_units.sql) | | `sqlite_acquisition` | `Absent` (+0) or `Tables` (+6, versions 42–44, created with `auto_vacuum=INCREMENTAL`) | [`config.rs`](../../../../engine/crates/layerfs-persistence/src/store/config.rs), [`acquisition/schema.sql`](../../../../engine/crates/layerfs-persistence/sql/sqlite/acquisition/schema.sql) | -| `sqlite_profile` | `Disposable` in every product opener (see [Store](README.md)) | [`profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs) | +| `sqlite_profile` | `Normal` in every product opener since 2026-11-17 (see [Store](README.md)); a test may select `Disposable` | [`profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs) | Every table is `STRICT`; history keys are `WITHOUT ROWID` blobs whose first byte tags the kind (`0x31` stack, `0x32` Layer, `0x12` Commit, `0x11` @@ -55,13 +55,13 @@ UPDATE and DELETE on `pack` and `pack_unit`. Open checks `application_id`, | Open | `READ_WRITE` or `READ_ONLY`, `NO_MUTEX`; statement cache 64 | [`connection.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs) | | Guard | `busy_timeout=0`; `SQLITE_DBCONFIG_DEFENSIVE` on | same | | Settings | `profile::apply`, then read back into `ConnectionProfile`; a mismatch is `Integrity`; an existing Store not in `wal` mode is refused | [`profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs) | -| Checkpoint thread | writable Disposable session only: a second connection on its own thread; failure to start it refuses the open | [`background.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/background.rs) | +| Checkpoint thread | writable Normal or Disposable session only: a second connection on its own thread, opened with the session's `synchronous` value and read back; failure to start it refuses the open | [`background.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/background.rs) | | Last close | A writable session's close waits for `Writer::acquire(path)` like any write (or reuses the lane an uncertain transaction already holds), so its close, which may be SQLite's last, checkpoints and removes `-wal`/`-shm` under the lane; a pull's verify close and a restore's in-use check rely on that. If the wait itself fails (interrupted, or the directory cannot be opened), the close sets `SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE` first, so SQLite takes no exclusive lock, runs no checkpoint and removes nothing; there is no second attempt. If even that setting fails, the connection is kept until process exit. A read-only or diagnostic session cannot checkpoint or remove those files and takes no lane, so it never waits behind a writer kept after an uncertain write. The checkpoint thread's connection closes first and never takes it. One exception: an open whose checkpoint thread fails to start drops its new connection after the open's lane is released | [`lifecycle.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/lifecycle.rs) | -| Setting | Disposable | Source | +| Setting | Normal (Disposable where it differs) | Source | | --- | --- | --- | -| identity | `sqlite-wal-off-v2` | [`profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs) | -| `journal_mode`, `synchronous`, `fullfsync` | `wal`, 0 (OFF), 0 | same | +| identity | `sqlite-wal-normal-v2` (`sqlite-wal-off-v2`) | [`profile.rs`](../../../../engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs) | +| `journal_mode`, `synchronous`, `fullfsync` | `wal`, 1 (NORMAL; Disposable 0, OFF), 0 | same | | `wal_autocheckpoint` | 0: the session's checkpoint thread copies the log | same | | shared | `foreign_keys=1`, `checkpoint_fullfsync=1`, `journal_size_limit=4194304`, `cache_size=-2048`, `mmap_size=0`, `temp_store=2`, `page_size=4096`, `busy_timeout=0` | same | diff --git a/docs/versioned/2.0.0/machines-and-stores.md b/docs/versioned/2.0.0/machines-and-stores.md index 7f00bc4ed6..ec3d7b7b3f 100644 --- a/docs/versioned/2.0.0/machines-and-stores.md +++ b/docs/versioned/2.0.0/machines-and-stores.md @@ -426,7 +426,7 @@ layerfs project init pg-main/load ./load (`password`, `sslpassword`, any case). Give the password as a reference with `--password` (`env:NAME` or `file:/absolute/path`). It is resolved on each call and never written to the home. -- `synchronous_commit` is `off` (the default), `local`, `remote_write`, +- `synchronous_commit` is `off`, `local` (the default), `remote_write`, `on` or `remote_apply`. - The Store lives in the fixed schema `layerfs`. An empty database gets the schema with a new identity. A database that already holds the `layerfs` diff --git a/engine/AGENTS.md b/engine/AGENTS.md index 577f58ab6f..89ddb82dfe 100644 --- a/engine/AGENTS.md +++ b/engine/AGENTS.md @@ -228,10 +228,11 @@ is the working procedure. A docs-only change needs no campaign. uncertain custody: no resend, rollback, deletion or success claim from an unfenced read. An unsupported required platform or capability fails explicitly, never as a silent no-op or fallback. -- **SQLite Stores are Disposable/WAL/OFF only**, on macOS and Linux - ([profile](../AGENTS.md#active-persistence-profile--owner-direction-2026-10-07)); - select it explicitly, never by default argument. Disposable claims - process-crash survival only. Open verifies WAL and never converts an existing +- **Product paths open SQLite Stores Normal/WAL/NORMAL**, on macOS and Linux + ([profile](../AGENTS.md#active-persistence-profile--owner-direction-2026-11-17)); + tests may still select Disposable/WAL/OFF. Select a profile explicitly, + never by default argument. Both claim process-crash survival only until + Normal's safety gate is run. Open verifies WAL and never converts an existing Store; select profile and layout before open. WAL stays on throughout Init and Commit (owner supersession 2026-10-07): no private MEMORY import or journal promotion. Checkpointing is described on the diff --git a/engine/crates/layerfs-bridge/src/provision.rs b/engine/crates/layerfs-bridge/src/provision.rs index 576c43a915..46b6ddf7ba 100644 --- a/engine/crates/layerfs-bridge/src/provision.rs +++ b/engine/crates/layerfs-bridge/src/provision.rs @@ -15,7 +15,9 @@ pub enum ProviderKind { pub enum StoreProfile { /// WAL with FULL synchronization. Durable, - /// WAL with synchronization disabled; process-crash survival only. + /// The non-durable WAL Store. Its wire byte and name predate the Normal + /// profile: every product opener now opens such a Store with NORMAL + /// synchronization (owner, 2026-11-17); only a test selects OFF. Disposable, } /// Bounded control metadata. File bytes are streamed separately without a size cap. diff --git a/engine/crates/layerfs-daemon/src/application/owner.rs b/engine/crates/layerfs-daemon/src/application/owner.rs index 8f4bdc21d9..edf01f513c 100644 --- a/engine/crates/layerfs-daemon/src/application/owner.rs +++ b/engine/crates/layerfs-daemon/src/application/owner.rs @@ -520,7 +520,7 @@ impl Application { } => Some(( layerfs_persistence::StoreSelection::Sqlite( PersistenceConfig::sqlite(locator) - .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + .with_sqlite_profile(SqlitePersistenceProfile::Normal), ), m.binding.as_slice(), m.cursor_key, diff --git a/engine/crates/layerfs-daemon/src/install.rs b/engine/crates/layerfs-daemon/src/install.rs index 8499d88f2d..b6984b9616 100644 --- a/engine/crates/layerfs-daemon/src/install.rs +++ b/engine/crates/layerfs-daemon/src/install.rs @@ -134,7 +134,8 @@ fn install_manifest( retained.phase = InstallPhase::Open; let profile = match manifest.profile { StoreProfile::Durable => SqlitePersistenceProfile::Durable, - StoreProfile::Disposable => SqlitePersistenceProfile::Disposable, + // The manifest's non-durable label; its connections open Normal. + StoreProfile::Disposable => SqlitePersistenceProfile::Normal, }; retained.opened = Some( open_store_observed( diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/background.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/background.rs index 6efb4375cb..3866ef906d 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/background.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/background.rs @@ -34,7 +34,12 @@ impl Checkpointer { /// session's own connection has already opened in WAL mode, and starts /// its thread. Failure refuses the session's open: without it the log /// would never be copied while the session lives. - pub(crate) fn start(path: &Path) -> Result { + /// + /// `synchronous` is the session profile's value. A checkpoint copies log + /// frames into the database file, so it must synchronize exactly as the + /// session does: under NORMAL a pass at 0 would mark frames copied that + /// never reached the disk, and the writer would restart the log over them. + pub(crate) fn start(path: &Path, synchronous: i64) -> Result { let connection = Connection::open_with_flags( path, OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX, @@ -48,11 +53,18 @@ impl Checkpointer { .map_err(rows::error)?; let work = std::cell::RefCell::new(super::connection::SqlWork::default()); for pragma in [ - "PRAGMA synchronous=0", - "PRAGMA fullfsync=0", - "PRAGMA wal_autocheckpoint=0", + format!("PRAGMA synchronous={synchronous}"), + "PRAGMA fullfsync=0".to_owned(), + "PRAGMA wal_autocheckpoint=0".to_owned(), ] { - query::run(&connection, pragma, vec![], &work)?; + query::run(&connection, &pragma, vec![], &work)?; + } + let applied = query::run(&connection, "PRAGMA synchronous", vec![], &work)? + .first() + .ok_or(BackendError::Integrity)? + .get::(0)?; + if applied != synchronous { + return Err(BackendError::Integrity); } let mode = query::run(&connection, "PRAGMA journal_mode", vec![], &work)? .first() diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs index 608efa6d47..dcffe6025f 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/connection.rs @@ -332,8 +332,8 @@ impl Session { writer.release().map_err(|_| BackendError::Unknown)?; } super::thread_work::add(&work.borrow()); - let background = (writable && selected == SqlitePersistenceProfile::Disposable) - .then(|| super::background::Checkpointer::start(path)) + let background = (writable && selected.background_checkpoint()) + .then(|| super::background::Checkpointer::start(path, selected.synchronous())) .transpose()?; // §10: the record every later transaction is fenced against. let identity = FileIdentity::capture(path).map_err(|_| BackendError::Integrity)?; diff --git a/engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs b/engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs index a1fb54acef..6458b9673a 100644 --- a/engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs +++ b/engine/crates/layerfs-persistence/src/backend/sqlite/session/profile.rs @@ -9,6 +9,7 @@ impl SqlitePersistenceProfile { match self { Self::Durable => "sqlite-wal-full-v2", Self::Disposable => "sqlite-wal-off-v2", + Self::Normal => "sqlite-wal-normal-v2", } } pub(crate) fn journal(self) -> &'static str { @@ -18,20 +19,25 @@ impl SqlitePersistenceProfile { match self { Self::Durable => 2, Self::Disposable => 0, + Self::Normal => 1, } } pub(crate) fn fullfsync(self) -> i64 { i64::from(self == Self::Durable) } - /// Pages after which a COMMIT checkpoints the log itself. Disposable - /// leaves it to its session's checkpoint thread, off the commit path - /// (`background.rs`; owner approval, 2026-11-07). + /// Pages after which a COMMIT checkpoints the log itself. Disposable and + /// Normal leave it to their session's checkpoint thread, off the commit + /// path (`background.rs`; owner approval, 2026-11-07). pub(crate) fn wal_autocheckpoint(self) -> i64 { match self { Self::Durable => 1000, - Self::Disposable => 0, + Self::Disposable | Self::Normal => 0, } } + /// Whether a writable session of this profile runs the checkpoint thread. + pub(crate) fn background_checkpoint(self) -> bool { + self.wal_autocheckpoint() == 0 + } } pub(crate) fn apply( c: &Connection, diff --git a/engine/crates/layerfs-persistence/src/store/config.rs b/engine/crates/layerfs-persistence/src/store/config.rs index 8f1d32ec26..27ecf8d63c 100644 --- a/engine/crates/layerfs-persistence/src/store/config.rs +++ b/engine/crates/layerfs-persistence/src/store/config.rs @@ -1,13 +1,15 @@ //! Explicit backend and local database selection. use std::path::PathBuf; /// PostgreSQL `synchronous_commit` for every write transaction, applied with -/// `SET LOCAL`. `Off` is the default (owner, 2026-10-29). +/// `SET LOCAL`. `Local` is the default (owner, 2026-11-17; it was `Off` +/// from 2026-10-29): a server crash loses no acknowledged write. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub enum SynchronousCommit { - /// Acknowledge before the WAL record is flushed. - #[default] + /// Acknowledge before the WAL record is flushed. A server crash can lose + /// acknowledged writes; an explicit choice only. Off, /// Wait for the local WAL flush only. + #[default] Local, /// Wait for standbys to receive the WAL. RemoteWrite, @@ -65,12 +67,12 @@ pub struct PostgresSelection { pub check: PostgresCheck, } impl PostgresSelection { - /// A selection with `synchronous_commit=off` and a 30 s lane wait. + /// A selection with `synchronous_commit=local` and a 30 s lane wait. pub fn new(url: impl Into, schema: impl Into) -> Self { Self { url: url.into(), schema: schema.into(), - synchronous_commit: SynchronousCommit::Off, + synchronous_commit: SynchronousCommit::Local, lock_timeout_ms: 30_000, check: PostgresCheck::Full, } @@ -105,7 +107,20 @@ pub enum SqlitePersistenceProfile { Durable, /// WAL/OFF; survives process termination, without kernel-crash durability. /// A kernel or VM crash can corrupt the Store or lose acknowledged data. + /// Tests and diagnostics may still select it; the product opens Normal. Disposable, + /// WAL/NORMAL: the log is synchronized before each checkpoint and after a + /// log restart, not at every COMMIT. A kernel or VM crash can lose the + /// commits since the last synchronized checkpoint. The profile every + /// product opener selects (owner, 2026-11-17). + Normal, +} +impl SqlitePersistenceProfile { + /// Whether a product path may open a Store with this profile: Normal, or + /// Disposable for a test or diagnostic. Durable stays disabled. + pub const fn permitted(self) -> bool { + matches!(self, Self::Normal | Self::Disposable) + } } /// Explicit physical layout for a newly created embedded Store. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] diff --git a/engine/crates/layerfs-persistence/tests/background_checkpoint.rs b/engine/crates/layerfs-persistence/tests/background_checkpoint.rs index 80562dc90f..7c56a69d84 100644 --- a/engine/crates/layerfs-persistence/tests/background_checkpoint.rs +++ b/engine/crates/layerfs-persistence/tests/background_checkpoint.rs @@ -1,7 +1,7 @@ -//! A Disposable session's checkpoint thread (owner approval, 2026-11-07): -//! COMMIT does not checkpoint (`wal_autocheckpoint=0`); a log longer than -//! 4 MiB is copied into the Store file while the session stays open, with -//! no idle checkpoint, seal or close. +//! A Normal or Disposable session's checkpoint thread (owner approval, +//! 2026-11-07; Normal since 2026-11-17): COMMIT does not checkpoint +//! (`wal_autocheckpoint=0`); a log longer than 4 MiB is copied into the Store +//! file while the session stays open, with no idle checkpoint, seal or close. #![cfg(any(target_os = "macos", target_os = "linux"))] mod support; @@ -16,9 +16,9 @@ use support::Temp; const BYTES: usize = 24 << 20; -fn create(path: &Path) -> SqliteHandles { +fn create(path: &Path, profile: SqlitePersistenceProfile) -> SqliteHandles { SqliteHandles::create( - PersistenceConfig::sqlite(path).with_sqlite_profile(SqlitePersistenceProfile::Disposable), + PersistenceConfig::sqlite(path).with_sqlite_profile(profile), StoragePolicy::frozen_default(), &HistoryCatalogConfig { binding_key: b"background-checkpoint".to_vec(), @@ -49,9 +49,24 @@ fn size(path: &Path) -> u64 { #[test] fn a_long_log_is_copied_into_the_store_while_the_session_stays_open() { + copied(SqlitePersistenceProfile::Disposable, 0); +} + +/// The product profile: the session and its checkpoint thread run at +/// `synchronous=1` (NORMAL), and the log is still copied off the commit path. +#[test] +fn a_normal_session_copies_its_log_with_normal_synchronization() { + copied(SqlitePersistenceProfile::Normal, 1); +} + +fn copied(profile: SqlitePersistenceProfile, synchronous: i64) { let temp = Temp::new("background-checkpoint"); let path = temp.join("store.sqlite"); - let handles = create(&path); + let handles = create(&path, profile); + assert_eq!(handles.profile().persistence, profile); + assert_eq!(handles.profile().journal_mode, "wal"); + assert_eq!(handles.profile().synchronous, synchronous); + assert_eq!(handles.profile().fullfsync, 0); assert_eq!(handles.profile().wal_autocheckpoint, 0); let storage = Storage::new(handles.storage.clone()).unwrap(); let bytes = random(BYTES); diff --git a/engine/crates/layerfs-project/src/store/init.rs b/engine/crates/layerfs-project/src/store/init.rs index 7bbc6457d9..9ad3c57c53 100644 --- a/engine/crates/layerfs-project/src/store/init.rs +++ b/engine/crates/layerfs-project/src/store/init.rs @@ -109,9 +109,9 @@ pub(crate) fn initialize( let mut initialized = None; let mut branch = None; let result = (|| { - if request.store.sqlite_profile != SqlitePersistenceProfile::Disposable { + if !request.store.sqlite_profile.permitted() { return Err(InitError::Manifest( - "explicit Disposable Store profile required", + "explicit Normal or Disposable Store profile required", )); } check_destination( @@ -185,7 +185,9 @@ pub(crate) fn initialize( locator: request.locator.clone(), profile: match request.store.sqlite_profile { SqlitePersistenceProfile::Durable => StoreProfile::Durable, - SqlitePersistenceProfile::Disposable => StoreProfile::Disposable, + SqlitePersistenceProfile::Disposable | SqlitePersistenceProfile::Normal => { + StoreProfile::Disposable + } }, binding: request.catalog.binding_key.clone(), cursor_key: request.catalog.cursor_key, diff --git a/engine/crates/layerfs-project/src/store/install.rs b/engine/crates/layerfs-project/src/store/install.rs index fe8f02d8a3..c65ff1cb4a 100644 --- a/engine/crates/layerfs-project/src/store/install.rs +++ b/engine/crates/layerfs-project/src/store/install.rs @@ -5,7 +5,6 @@ use layerfs_bridge::{ duplex::{Connection, MAX_PLAINTEXT_BYTES}, provision::StoreProfile, }; -use layerfs_persistence::SqlitePersistenceProfile; use std::{ fs::File, io::{self, Read}, @@ -33,11 +32,9 @@ pub fn install_file( let mut accepted = false; let mut acknowledged = None; let result = (|| { - if manifest.profile != StoreProfile::Disposable - || store.profile != SqlitePersistenceProfile::Disposable - { + if manifest.profile != StoreProfile::Disposable || !store.profile.permitted() { return Err(InstallError::Protocol(ControlError( - "explicit Disposable Store profile required", + "explicit Normal or Disposable Store profile required", ))); } if manifest.daemon_sqlite.is_some() { diff --git a/engine/crates/layerfs-project/src/store/live.rs b/engine/crates/layerfs-project/src/store/live.rs index c3207e7a16..1e3194edf5 100644 --- a/engine/crates/layerfs-project/src/store/live.rs +++ b/engine/crates/layerfs-project/src/store/live.rs @@ -18,9 +18,7 @@ use layerfs_history::{ BranchId, BranchSnapshot, ForkRequest, ForkSource, HistoryCatalog, HistoryCatalogConfig, HistoryError, HistoryName, HistoryResult, LayerStackId, LayerStackRecord, Page, }; -use layerfs_persistence::{ - SealedStore, SqliteAcquisitionSchema, SqliteHandles, SqlitePersistenceProfile, StoreSelection, -}; +use layerfs_persistence::{SealedStore, SqliteAcquisitionSchema, SqliteHandles, StoreSelection}; use layerfs_storage::port::{acquisition::AcquisitionError, PersistenceError}; use layerfs_storage::{Storage, StoragePolicy}; use layerfs_telemetry::timer::{Active, TimingScope}; @@ -96,15 +94,15 @@ impl std::fmt::Display for AddFailure { } impl std::error::Error for AddFailure {} -/// The Disposable profile rule applies to a SQLite Store only; a PostgreSQL -/// Store has no SQLite profile and passes. +/// The profile rule (Normal, or Disposable in a test) applies to a SQLite +/// Store only; a PostgreSQL Store has no SQLite profile and passes. fn disposable(store: &StoreSelection) -> Result<(), InitError> { let StoreSelection::Sqlite(store) = store else { return Ok(()); }; - if store.sqlite_profile != SqlitePersistenceProfile::Disposable { + if !store.sqlite_profile.permitted() { return Err(InitError::Manifest( - "explicit Disposable Store profile required", + "explicit Normal or Disposable Store profile required", )); } Ok(()) diff --git a/engine/crates/layerfs-store-service/src/install.rs b/engine/crates/layerfs-store-service/src/install.rs index 5c27660c7e..33fbd23c5b 100644 --- a/engine/crates/layerfs-store-service/src/install.rs +++ b/engine/crates/layerfs-store-service/src/install.rs @@ -232,7 +232,7 @@ pub fn install_local( left.temporary = Some(target.clone()); let opened = open_store_observed( PersistenceConfig::sqlite(&target) - .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + .with_sqlite_profile(SqlitePersistenceProfile::Normal), &manifest.binding, manifest.cursor_key, 1, diff --git a/engine/crates/layerfs-store-service/src/store_service.rs b/engine/crates/layerfs-store-service/src/store_service.rs index ae334cd7f5..52085dc8d2 100644 --- a/engine/crates/layerfs-store-service/src/store_service.rs +++ b/engine/crates/layerfs-store-service/src/store_service.rs @@ -106,8 +106,7 @@ impl StoreService { ))); } let opened = open_store_observed( - PersistenceConfig::sqlite(store) - .with_sqlite_profile(SqlitePersistenceProfile::Disposable), + PersistenceConfig::sqlite(store).with_sqlite_profile(SqlitePersistenceProfile::Normal), binding, cursor_key, CALLS,