Repository navigation
fix(repo): per-signer signature index and index lock (#257, #260) #265
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
c02d7c3
40c43e9
5507204
28b8809
2604e17
8115b2d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -54,6 +54,77 @@ impl RepoLock { | |
| } | ||
| } | ||
|
|
||
| thread_local! { | ||
| /// Index locks this thread holds (by path), so a nested critical section | ||
| /// on the same thread reuses the held lock instead of deadlocking on it. | ||
| static HELD_INDEX_LOCKS: std::cell::RefCell<Vec<PathBuf>> = const { std::cell::RefCell::new(Vec::new()) }; | ||
| } | ||
|
|
||
| /// How long [`with_index_lock`] waits for another holder before giving up. | ||
| /// Tests get a long wait: their threads queue behind durable (fsynced) writes | ||
| /// on a loaded machine; the timeout path is tested with `with_index_lock_for`. | ||
| const INDEX_LOCK_WAIT: std::time::Duration = | ||
| std::time::Duration::from_secs(if cfg!(test) { 300 } else { 30 }); | ||
|
|
||
| /// Run `f` holding the short-lived index lock (`.sc/index.lock`, ADR-0038 | ||
| /// "Index lock", #260). It serializes the read-modify-write of the derived | ||
| /// index files (`signatures.v2`, `transcripts`, `transcript-pending`...) | ||
| /// between concurrent serve-path ingests and local commands, WITHOUT being | ||
| /// [`RepoLock`]: it is held only around one index update (or one coupled | ||
| /// group of them), never for a command's lifetime, so it never turns a busy repo into `Locked` for a | ||
| /// local `sc` command. | ||
| /// | ||
| /// An advisory `flock` on a file that is never removed: the OS drops it when | ||
| /// the holder dies, so there is no stale-lock breaking (and none of its | ||
| /// races). Polls up to [`INDEX_LOCK_WAIT`], then `Error::IndexBusy`. Reentrant per | ||
| /// thread. Lock order: this lock is a leaf. Never acquire [`RepoLock`] (or | ||
| /// wait on anything else) while holding it; taking it under `RepoLock` is fine. | ||
| pub(crate) fn with_index_lock<T>(layout: &Layout, f: impl FnOnce() -> Result<T>) -> Result<T> { | ||
| with_index_lock_for(layout, INDEX_LOCK_WAIT, f) | ||
| } | ||
|
|
||
| fn with_index_lock_for<T>( | ||
| layout: &Layout, | ||
| wait: std::time::Duration, | ||
| f: impl FnOnce() -> Result<T>, | ||
| ) -> Result<T> { | ||
| let path = layout.dot_sc.join("index.lock"); | ||
| if HELD_INDEX_LOCKS.with(|h| h.borrow().contains(&path)) { | ||
| return f(); | ||
| } | ||
| let file = std::fs::OpenOptions::new() | ||
| .read(true) | ||
| .write(true) | ||
| .create(true) | ||
| .truncate(false) | ||
| .open(&path)?; | ||
| let deadline = std::time::Instant::now() + wait; | ||
| loop { | ||
| match file.try_lock() { | ||
| Ok(()) => break, | ||
| Err(std::fs::TryLockError::WouldBlock) if std::time::Instant::now() < deadline => { | ||
| std::thread::sleep(std::time::Duration::from_millis(2)); | ||
| } | ||
| Err(std::fs::TryLockError::WouldBlock) => { | ||
| return Err(Error::IndexBusy(path.display().to_string())) | ||
| } | ||
|
Comment on lines
+108
to
+110
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When index-lock contention lasts longer than ten seconds, this reuses Useful? React with 👍 / 👎. |
||
| Err(std::fs::TryLockError::Error(e)) => return Err(e.into()), | ||
| } | ||
| } | ||
| HELD_INDEX_LOCKS.with(|h| h.borrow_mut().push(path.clone())); | ||
| // Pops the entry (and unlocks via `file`'s drop) even if `f` panics. | ||
| struct Release(PathBuf); | ||
| impl Drop for Release { | ||
| fn drop(&mut self) { | ||
| HELD_INDEX_LOCKS.with(|h| h.borrow_mut().retain(|p| p != &self.0)); | ||
| } | ||
| } | ||
| let _release = Release(path); | ||
| let out = f(); | ||
| drop(file); | ||
| out | ||
| } | ||
|
|
||
| /// True only when the lock file names a PID we can *prove* is not running. | ||
| /// Unreadable file, no parseable PID, or an inconclusive probe all return | ||
| /// false (respect the lock). | ||
|
|
@@ -143,4 +214,53 @@ mod tests { | |
| assert_eq!(content.trim(), std::process::id().to_string()); | ||
| std::fs::remove_dir_all(&layout.root).unwrap(); | ||
| } | ||
|
|
||
| #[test] | ||
| fn index_lock_is_not_the_repo_lock() { | ||
| // Holding either never makes the other fail: a local command keeps | ||
| // `RepoLock` for its lifetime while a serve-path ingest takes the short | ||
| // index lock, and vice versa (#260). | ||
| let layout = tmp_layout("idx-indep"); | ||
| let repo_lock = RepoLock::acquire(&layout).unwrap(); | ||
| with_index_lock(&layout, || Ok(())).unwrap(); | ||
| assert!(matches!(RepoLock::acquire(&layout), Err(Error::Locked(_)))); | ||
| drop(repo_lock); | ||
| with_index_lock(&layout, || { | ||
| // Reentrant on the same thread, and RepoLock is still free. | ||
| with_index_lock(&layout, || Ok(()))?; | ||
| drop(RepoLock::acquire(&layout)?); | ||
| Ok(()) | ||
| }) | ||
| .unwrap(); | ||
| std::fs::remove_dir_all(&layout.root).unwrap(); | ||
| assert!(!layout.root.exists()); | ||
| } | ||
|
|
||
| #[test] | ||
| fn index_lock_serializes_threads_and_times_out_with_locked() { | ||
| let layout = tmp_layout("idx-wait"); | ||
| let (held_tx, held_rx) = std::sync::mpsc::channel(); | ||
| let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); | ||
| let l2 = layout.clone(); | ||
| let holder = std::thread::spawn(move || { | ||
| with_index_lock(&l2, || { | ||
| held_tx.send(()).unwrap(); | ||
| release_rx.recv().unwrap(); | ||
| Ok(()) | ||
| }) | ||
| .unwrap(); | ||
| }); | ||
| held_rx.recv().unwrap(); | ||
| // Another thread of the same process waits, then gives up. | ||
| assert!(matches!( | ||
| with_index_lock_for(&layout, std::time::Duration::from_millis(100), || Ok(())), | ||
| Err(Error::IndexBusy(_)) | ||
| )); | ||
| release_tx.send(()).unwrap(); | ||
| holder.join().unwrap(); | ||
| // Released: free again, and the failed waiter left no held entry. | ||
| with_index_lock(&layout, || Ok(())).unwrap(); | ||
| std::fs::remove_dir_all(&layout.root).unwrap(); | ||
| assert!(!layout.root.exists()); | ||
| } | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -99,6 +99,11 @@ impl Repo { | |
| fn open_layout(layout: Layout, budget_bytes: usize) -> Result<Repo> { | ||
| let lock = RepoLock::acquire(&layout)?; | ||
| let mut store = Store::open_persistent(layout.objects_dir(), budget_bytes)?; | ||
| // Import a pre-#257 signature index first (best effort; every writer | ||
| // and gc retry it and fail rather than start a v2 file without it): the | ||
| // re-evaluation below reads that index, and a pass that saw it empty | ||
| // would record the trust as judged. | ||
| let _ = crate::signatures::migrate_legacy(&layout, &mut store); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When the legacy import fails transiently—particularly with Useful? React with 👍 / 👎. |
||
| // Honor held tombstones once the trusted-signer set changed (#230). Best | ||
| // effort: a failure must not stop an open, and the next open retries. | ||
| let _ = crate::transcripts::reevaluate_if_trust_changed(&layout, &mut store); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
A serve-path ingest still releases the index lock after
signatures::index_incoming(transport.rs:634) and reacquires it separately intranscripts::index_incoming(transport.rs:637). If GC enters this new combined section between those calls, transcript pruning cannot root the newly stored transcript, so signature pruning removes its just-indexed signature; the ingest then adds the transcript entry without its provenance, and a signed tombstone can likewise become neither honored nor pending. The fresh evidence beyond the prior race is this remaining gap between the two ingest passes, so the ingest side also needs one encompassing index-lock transaction.Useful? React with 👍 / 👎.