From 3aafbc46f6f7533cd28057e0a8c198e317709e38 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 25 Aug 2026 11:03:23 +0200 Subject: [PATCH 01/24] fuse: give DLM ranges a lifecycle state A FUSE_DLM_WB_LOCK reply and a NOTIFY revoke run on different threads, so a revoke can arrive before the grant is recorded, find nothing, and leave a grant that is never taken back. revoke_gen caught this with a per-inode counter that cannot say which range was hit, so any revoke re-requested every grant in flight. Add enum fuse_dlm_range_state: REQUESTED or REVOKED on the new cache->pending list while in flight, GRANTED in cache->ranges. A revoke marks the pending requests it overlaps, fuse_dlm_request_commit() drops a marked grant, and unlinking the request and recording its grant is one step under the cache lock. Pending requests stay off the interval tree, so no tree walker needs a state filter. Signed-off-by: Horst Birthelmer --- fs/fuse/fuse_dlm_cache.c | 272 ++++++++++++++++++++++++++++----------- fs/fuse/fuse_dlm_cache.h | 50 +++++-- 2 files changed, 234 insertions(+), 88 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index bc6dbae2d5aeb0..a5d24e98c2169c 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -1,6 +1,27 @@ // SPDX-License-Identifier: GPL-2.0-only /* * FUSE page lock cache implementation + * + * cache->ranges records the grants the server has given this client. A + * grant still on the wire covers nothing and must not appear there, but + * a revoke has to be able to find it: otherwise a revoke processed + * before the grant is recorded removes nothing, and the grant recorded + * afterwards is never taken back. + * + * A range therefore carries enum fuse_dlm_range_state: + * + * - REQUESTED, on cache->pending, while its FUSE_DLM_WB_LOCK is in + * flight. + * + * - REVOKED, still on cache->pending, once a revoke has overlapped it. + * fuse_dlm_request_commit() drops such a grant instead of recording + * it. + * + * - GRANTED, in cache->ranges. The only state + * fuse_dlm_range_is_locked() reports as covered. + * + * In-flight requests are kept off the tree so the state is consulted + * only where a request is retired, not by every tree walker. */ #include "fuse_i.h" #include "fuse_dlm_cache.h" @@ -11,9 +32,19 @@ #include +/* Lifecycle of a range; see the file comment above */ +enum fuse_dlm_range_state { + /* FUSE_DLM_WB_LOCK in flight, on cache->pending */ + FUSE_DLM_RANGE_REQUESTED, + /* Revoked while in flight; the grant must not be recorded */ + FUSE_DLM_RANGE_REVOKED, + /* Recorded grant, in cache->ranges */ + FUSE_DLM_RANGE_GRANTED, +}; + /* A range of pages with a lock */ struct fuse_dlm_range { - /* Interval tree node */ + /* Interval tree node; only linked while GRANTED */ struct rb_node rb; /* Start page offset (inclusive) */ uint64_t start; @@ -21,9 +52,11 @@ struct fuse_dlm_range { uint64_t end; /* Subtree end value for interval tree */ uint64_t __subtree_end; - /* Lock mode */ + /* Lock mode, as FUSE_PCACHE_LK_READ / FUSE_PCACHE_LK_WRITE */ enum fuse_page_lock_mode mode; - /* Temporary list entry for operations */ + /* Lifecycle state; see enum fuse_dlm_range_state */ + enum fuse_dlm_range_state state; + /* Temporary list entry for operations, and the cache->pending link */ struct list_head list; }; @@ -46,6 +79,31 @@ INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, uint64_t, __subtree_end, fuse_dlm_range_start, fuse_dlm_range_last, static, fuse_page_it); +/** + * fuse_dlm_kill_pending - mark in-flight requests overlapping [start, end] + * @cache: The page cache + * @start: Start page offset of the revoked region + * @end: End page offset of the revoked region + * + * A revoke overlapping a request still on the wire has nothing to remove + * from the tree, since that grant is not recorded yet. Marking it makes + * fuse_dlm_request_commit() drop the grant instead of recording it. + * + * The nodes are owned by the threads waiting on their replies: mark + * only, never remove or free. + * + * Caller holds @cache->lock for write. + */ +static void fuse_dlm_kill_pending(struct fuse_dlm_cache *cache, + uint64_t start, uint64_t end) +{ + struct fuse_dlm_range *req; + + list_for_each_entry(req, &cache->pending, list) + if (req->start <= end && start <= req->end) + req->state = FUSE_DLM_RANGE_REVOKED; +} + /** * fuse_page_cache_init - Initialize a page cache lock manager * @cache: The cache to initialize @@ -63,7 +121,7 @@ int fuse_dlm_cache_init(struct fuse_inode *inode) init_rwsem(&cache->lock); cache->ranges = RB_ROOT_CACHED; - cache->revoke_gen = 0; + INIT_LIST_HEAD(&cache->pending); return 0; } @@ -85,7 +143,11 @@ void fuse_dlm_cache_release_locks(struct fuse_inode *inode) /* Release all locks */ down_write(&cache->lock); - WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); + /* + * Every grant goes, so every request in flight is revoked. Mark + * only; each node is owned by the thread waiting on its reply. + */ + fuse_dlm_kill_pending(cache, 0, U64_MAX); while ((node = rb_first_cached(&cache->ranges)) != NULL) { range = rb_entry(node, struct fuse_dlm_range, rb); fuse_page_it_remove(range, &cache->ranges); @@ -168,13 +230,11 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, } /** - * __fuse_dlm_lock_range - Lock a range of pages - * @cache: The page cache + * fuse_dlm_lock_range_locked - Record a granted range of pages + * @inode: The fuse inode * @start: Start page offset * @end: End page offset * @mode: Lock mode (read or write) - * @genp: If non-NULL, the revocation generation sampled before the grant - * was requested; recording fails with -EAGAIN if it has moved * * Add a locked range on the specified range of pages. * If parts of the range are already locked, only add the remaining parts. @@ -183,11 +243,16 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, * - READ locks are compatible with existing WRITE locks (downgrade not needed) * - WRITE locks need to upgrade existing READ locks * + * Everything inserted here is FUSE_DLM_RANGE_GRANTED: this runs only + * after the server has answered. + * + * Caller holds the cache lock for write. + * * Return: 0 on success, negative error code on failure */ -static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - const uint64_t *genp) +static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, + uint64_t end, + enum fuse_page_lock_mode mode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range, *new_range, *next; @@ -205,19 +270,6 @@ static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, lock_mode = (mode == FUSE_PAGE_LOCK_READ) ? FUSE_PCACHE_LK_READ : FUSE_PCACHE_LK_WRITE; - down_write(&cache->lock); - - /* - * A revoke was processed after @genp was sampled; the grant this - * record carries may be the very one it targeted (a revoke of a - * not-yet-recorded grant removes nothing and would never be - * retried). Refuse, the caller re-requests. - */ - if (genp && cache->revoke_gen != *genp) { - up_write(&cache->lock); - return -EAGAIN; - } - /* Find all ranges that overlap with [start, end] */ range = fuse_page_it_iter_first(&cache->ranges, start, end); while (range) { @@ -243,6 +295,7 @@ static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, new_range->start = current_start; new_range->end = range->start - 1; new_range->mode = lock_mode; + new_range->state = FUSE_DLM_RANGE_GRANTED; INIT_LIST_HEAD(&new_range->list); list_add_tail(&new_range->list, &to_lock); @@ -269,6 +322,7 @@ static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, new_range->start = current_start; new_range->end = end; new_range->mode = lock_mode; + new_range->state = FUSE_DLM_RANGE_GRANTED; INIT_LIST_HEAD(&new_range->list); list_add_tail(&new_range->list, &to_lock); @@ -289,7 +343,6 @@ static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, /* Try to merge adjacent ranges with the same mode */ fuse_dlm_try_merge(cache, start, end); - up_write(&cache->lock); return 0; out_free: @@ -309,37 +362,110 @@ static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, } } - up_write(&cache->lock); return ret; } int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, uint64_t end, enum fuse_page_lock_mode mode) { - return __fuse_dlm_lock_range(inode, start, end, mode, NULL); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + int ret; + + down_write(&cache->lock); + ret = fuse_dlm_lock_range_locked(inode, start, end, mode); + up_write(&cache->lock); + + return ret; +} + +/** + * fuse_dlm_request_begin - publish a lock request before it is sent + * @inode: the fuse inode + * @req: caller-owned storage for the request, live until commit or abort + * @start: start page offset being requested (inclusive) + * @end: end page offset being requested (inclusive) + * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE + * + * A FUSE_DLM_WB_LOCK reply and a NOTIFY revoke are serviced on different + * threads, so a revoke can be processed before the grant the reply + * carries is recorded. Publishing the request before it leaves gives + * that revoke a node to mark; without one it removes nothing, and the + * grant recorded afterwards is never taken back by any later NOTIFY. + * + * The request covers nothing while in flight, so it is kept off the + * tree. @req is reachable only through cache->pending, which both + * fuse_dlm_request_commit() and fuse_dlm_request_abort() unlink under + * the cache lock before the caller returns; stack storage is therefore + * fine and nothing is allocated here. + */ +void fuse_dlm_request_begin(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + RB_CLEAR_NODE(&req->rb); + req->start = start; + req->end = end; + req->mode = (mode == FUSE_PAGE_LOCK_READ) ? FUSE_PCACHE_LK_READ : + FUSE_PCACHE_LK_WRITE; + req->state = FUSE_DLM_RANGE_REQUESTED; + + down_write(&cache->lock); + list_add_tail(&req->list, &cache->pending); + up_write(&cache->lock); } -int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - uint64_t gen) +/** + * fuse_dlm_request_commit - retire a request and record its grant + * @inode: the fuse inode + * @req: the request published by fuse_dlm_request_begin() + * @start: start page offset the server granted (inclusive) + * @end: end page offset the server granted (inclusive) + * @mode: the mode that was requested + * + * Unlinking @req and recording the grant are one step under the cache + * lock, so a revoke lands either before it and is seen on @req, or after + * it and finds the grant in the tree. + * + * @req is retired in every case and may be reused. + * + * Return: -EAGAIN if a revoke overlapped @req while it was in flight, + * nothing recorded; otherwise the result of recording the grant. + */ +int fuse_dlm_request_commit(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { - return __fuse_dlm_lock_range(inode, start, end, mode, &gen); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool revoked; + int ret = 0; + + down_write(&cache->lock); + list_del(&req->list); + revoked = req->state == FUSE_DLM_RANGE_REVOKED; + if (!revoked) + ret = fuse_dlm_lock_range_locked(inode, start, end, mode); + up_write(&cache->lock); + + return revoked ? -EAGAIN : ret; } /** - * fuse_dlm_revoke_gen - sample the revocation generation + * fuse_dlm_request_abort - retire a request that got no usable reply * @inode: the fuse inode + * @req: the request published by fuse_dlm_request_begin() * - * Sampled before a FUSE_DLM_WB_LOCK request leaves the client. The - * reply and a NOTIFY revoke can be serviced on different threads, so a - * revoke may be processed between the reply arriving and its grant - * being recorded. fuse_dlm_lock_range_gen() re-checks the generation - * under the cache lock and refuses to record a grant such a revoke may - * have already killed. + * Nothing is recorded, so a mark left by a revoke does not matter. */ -uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode) +void fuse_dlm_request_abort(struct fuse_inode *inode, + struct fuse_dlm_range *req) { - return READ_ONCE(inode->dlm_locked_areas.revoke_gen); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + down_write(&cache->lock); + list_del(&req->list); + up_write(&cache->lock); } /** @@ -436,12 +562,11 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, down_write(&cache->lock); /* - * Unconditional, even when nothing overlaps: the revoke racing - * with an in-flight grant finds an empty tree precisely because - * the grant is not recorded yet, and the bump is what makes the - * recording side notice (see fuse_dlm_lock_range_gen()). + * Before touching the tree, and even when nothing in the tree + * overlaps: a revoke racing an in-flight grant finds no overlap + * because that grant is not recorded yet. */ - WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); + fuse_dlm_kill_pending(cache, start, end); /* Find all ranges that overlap with [start, end] */ range = fuse_page_it_iter_first(&cache->ranges, start, end); @@ -646,7 +771,7 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, FUSE_ARGS(args); struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; - uint64_t gen; + struct fuse_dlm_range req; int err; /* An empty range needs no lock. */ @@ -664,16 +789,6 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, if (fuse_dlm_lock_is_held(fi, offset, length, mode)) return 0; /* we already have this area locked */ - /* - * Sample the revocation generation before the request leaves. - * The reply and a NOTIFY revoke are serviced on different - * threads, so a revoke aimed at the grant this request returns - * can be processed before the grant is recorded below -- - * recording it anyway would resurrect a dead grant that no later - * NOTIFY will ever remove. - */ - gen = fuse_dlm_revoke_gen(fi); - memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; @@ -693,18 +808,23 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, args.out_numargs = 1; args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; + + /* Publish before sending; see fuse_dlm_request_begin() */ + fuse_dlm_request_begin(fi, &req, inarg.start, inarg.end, mode); + err = fuse_simple_request(fm, &args); - if (err == -ENOSYS) { - /* fuse server does not support dlm, save the info */ - fc->dlm = 0; + if (err) { + fuse_dlm_request_abort(fi, &req); + if (err == -ENOSYS) { + /* fuse server does not support dlm, save the info */ + fc->dlm = 0; + } return err; } - if (err) - return err; - if (inarg.start < outarg.start || inarg.end > outarg.end) { /* fuse server is seriously broken */ + fuse_dlm_request_abort(fi, &req); pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", inarg.start, inarg.end, outarg.start, outarg.end); fuse_abort_conn(fc); @@ -712,20 +832,24 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, } /* - * The server granted the lock; record it so - * fuse_dlm_lock_is_held() sees it. + * Retire the request and record the grant. A server may grant more + * than was asked for, and recording all of it is what lets later IO + * over the same neighbourhood skip the round trip. + * + * fuse_dlm_kill_pending() matches a request in flight on the bounds + * published to it, so a revoke aimed only at the part beyond them + * marks nothing here. It would have to be a revoke of a sub-range + * the server is granting in the same breath. */ - err = fuse_dlm_lock_range_gen(fi, outarg.start, outarg.end, mode, gen); + err = fuse_dlm_request_commit(fi, &req, outarg.start, outarg.end, mode); if (err == -EAGAIN) { /* - * A revoke was processed while the request was in flight; - * the grant may already be dead, so re-request instead of - * recording it. Retry until a grant survives long enough to - * be recorded: giving up here would hand the caller an error - * for a range no one else holds, and the write path turns - * that into a failed write. Each pass makes a fresh server - * round trip, so a revoke storm throttles this loop rather - * than spinning it. + * A revoke overlapping this range was processed while the + * request was in flight, so the grant is dead. Retry + * rather than fail: no one else holds the range, and the + * write path turns an error into a failed write. Each + * pass is a fresh round trip, so a revoke storm throttles + * the loop. */ goto restart; } diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 30fdbb26bd3daf..6f7c2c2fde0fd1 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -13,6 +13,7 @@ struct fuse_inode; +struct fuse_dlm_range; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; @@ -26,19 +27,25 @@ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; */ #define FUSE_DLM_GRANT_UNRECORDED 1 -/* Page cache lock manager */ +/* + * Page cache lock manager. + * + * @ranges holds the grants the client has been given and not had taken + * back. A request still on the wire covers nothing and lives on + * @pending instead, so tree walkers never filter on state. See enum + * fuse_dlm_range_state in fuse_dlm_cache.c. + */ struct fuse_dlm_cache { - /* Lock protecting the tree */ + /* Lock protecting the tree and the pending list */ struct rw_semaphore lock; - /* Interval tree of locked ranges */ + /* Interval tree of granted ranges (FUSE_DLM_RANGE_GRANTED) */ struct rb_root_cached ranges; /* - * Bumped under @lock by every revocation - * (fuse_dlm_unlock_range(), fuse_dlm_cache_release_locks()); - * lets fuse_get_dlm_lock() order recording a reply's grant - * against revokes processed while the reply was in flight. + * FUSE_DLM_WB_LOCK requests in flight (REQUESTED, or REVOKED once + * a revoke has overlapped one). Owned by the queueing thread; the + * revoke paths mark them only. */ - uint64_t revoke_gen; + struct list_head pending; }; /* Initialize a page cache lock manager */ @@ -51,13 +58,28 @@ void fuse_dlm_cache_release_locks(struct fuse_inode *inode); int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, uint64_t end, enum fuse_page_lock_mode mode); -/* As above, but refuse (-EAGAIN) if a revoke ran since @gen was sampled */ -int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - uint64_t gen); +/* + * Publish a FUSE_DLM_WB_LOCK for [start, end] before it is sent, so a + * revoke processed while the reply is on the wire can mark it. @req is + * caller-owned storage, live until the matching commit or abort. + */ +void fuse_dlm_request_begin(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode); + +/* + * Retire @req and record the grant [start, end] as one step under the + * cache lock. -EAGAIN means a revoke overlapped @req in flight and + * nothing was recorded; the caller must request again. @req is retired + * either way. + */ +int fuse_dlm_request_commit(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode); -/* Sample the revocation generation (see fuse_dlm_lock_range_gen()) */ -uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode); +/* Retire @req without recording anything */ +void fuse_dlm_request_abort(struct fuse_inode *inode, + struct fuse_dlm_range *req); /* Unlock a range of pages */ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, From 971a7eae4dca80355872e6fbb96fdcb96756bb16 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 25 Aug 2026 11:10:09 +0200 Subject: [PATCH 02/24] fuse: make the DLM lock mode part of the range state struct fuse_dlm_range carried a lifecycle state and a mode, the mode stored as FUSE_PCACHE_LK_READ/_WRITE (1 and 2) in a field typed enum fuse_page_lock_mode, whose enumerators are 0 and 1. The two are never independent, and fuse_dlm_range_is_locked() relied on the value ordering to let a write grant cover a read. Replace GRANTED with READ and WRITE and drop the mode field and the FUSE_PCACHE_LK_* values. fuse_dlm_state_satisfies() replaces the ordinal comparison. fuse_dlm_request_begin() no longer takes a mode: the one that reaches the tree is the one passed to the commit. Signed-off-by: Horst Birthelmer --- fs/fuse/fuse_dlm_cache.c | 99 +++++++++++++++++++++++----------------- fs/fuse/fuse_dlm_cache.h | 7 +-- 2 files changed, 62 insertions(+), 44 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index a5d24e98c2169c..82e18334299f70 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -17,8 +17,10 @@ * fuse_dlm_request_commit() drops such a grant instead of recording * it. * - * - GRANTED, in cache->ranges. The only state - * fuse_dlm_range_is_locked() reports as covered. + * - READ or WRITE, in cache->ranges. The only states + * fuse_dlm_range_is_locked() reports as covered; the mode is not a + * separate field, since a range is either not held or held in one + * definite mode. * * In-flight requests are kept off the tree so the state is consulted * only where a request is retired, not by every tree walker. @@ -38,13 +40,15 @@ enum fuse_dlm_range_state { FUSE_DLM_RANGE_REQUESTED, /* Revoked while in flight; the grant must not be recorded */ FUSE_DLM_RANGE_REVOKED, - /* Recorded grant, in cache->ranges */ - FUSE_DLM_RANGE_GRANTED, + /* Granted shared, in cache->ranges */ + FUSE_DLM_RANGE_READ, + /* Granted exclusive, in cache->ranges */ + FUSE_DLM_RANGE_WRITE, }; /* A range of pages with a lock */ struct fuse_dlm_range { - /* Interval tree node; only linked while GRANTED */ + /* Interval tree node; only linked once granted */ struct rb_node rb; /* Start page offset (inclusive) */ uint64_t start; @@ -52,17 +56,35 @@ struct fuse_dlm_range { uint64_t end; /* Subtree end value for interval tree */ uint64_t __subtree_end; - /* Lock mode, as FUSE_PCACHE_LK_READ / FUSE_PCACHE_LK_WRITE */ - enum fuse_page_lock_mode mode; - /* Lifecycle state; see enum fuse_dlm_range_state */ + /* Lifecycle and, once granted, the mode; see the enum above */ enum fuse_dlm_range_state state; /* Temporary list entry for operations, and the cache->pending link */ struct list_head list; }; -/* Lock modes for FUSE page cache */ -#define FUSE_PCACHE_LK_READ 1 /* Shared read lock */ -#define FUSE_PCACHE_LK_WRITE 2 /* Exclusive write lock */ +/* The state a grant in @mode is recorded under */ +static inline enum fuse_dlm_range_state +fuse_dlm_granted_state(enum fuse_page_lock_mode mode) +{ + return mode == FUSE_PAGE_LOCK_READ ? FUSE_DLM_RANGE_READ : + FUSE_DLM_RANGE_WRITE; +} + +/** + * fuse_dlm_state_satisfies - is a range in @held usable for @want + * @held: the state of a range recorded in the tree + * @want: FUSE_DLM_RANGE_READ or FUSE_DLM_RANGE_WRITE + * + * A WRITE grant is exclusive and so covers a READ request; nothing else + * substitutes for anything. The two pending states never appear in the + * tree and cover nothing. + */ +static inline bool fuse_dlm_state_satisfies(enum fuse_dlm_range_state held, + enum fuse_dlm_range_state want) +{ + return held == want || + (held == FUSE_DLM_RANGE_WRITE && want == FUSE_DLM_RANGE_READ); +} /* Interval tree definitions for page ranges */ static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) @@ -210,8 +232,8 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, struct fuse_dlm_range, rb); } - /* Try to merge with next range if adjacent and same mode */ - if (next && range->mode == next->mode && + /* Try to merge with next range if adjacent and same state */ + if (next && range->state == next->state && range->end + 1 == next->start) { /* Merge ranges: re-insert so __subtree_end is updated */ fuse_page_it_remove(next, &cache->ranges); @@ -243,8 +265,8 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, * - READ locks are compatible with existing WRITE locks (downgrade not needed) * - WRITE locks need to upgrade existing READ locks * - * Everything inserted here is FUSE_DLM_RANGE_GRANTED: this runs only - * after the server has answered. + * Everything inserted here is READ or WRITE: this runs only after the + * server has answered. * * Caller holds the cache lock for write. * @@ -256,7 +278,7 @@ static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range, *new_range, *next; - int lock_mode; + enum fuse_dlm_range_state want; bool covered_to_end = false; int ret = 0; LIST_HEAD(to_lock); @@ -266,9 +288,8 @@ static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, if (!cache || start > end) return -EINVAL; - /* Convert to lock mode */ - lock_mode = (mode == FUSE_PAGE_LOCK_READ) ? FUSE_PCACHE_LK_READ : - FUSE_PCACHE_LK_WRITE; + /* The state this grant records */ + want = fuse_dlm_granted_state(mode); /* Find all ranges that overlap with [start, end] */ range = fuse_page_it_iter_first(&cache->ranges, start, end); @@ -277,8 +298,8 @@ static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, next = fuse_page_it_iter_next(range, start, end); /* Check lock compatibility */ - if (lock_mode == FUSE_PCACHE_LK_WRITE && - lock_mode != range->mode) { + if (want == FUSE_DLM_RANGE_WRITE && + range->state != FUSE_DLM_RANGE_WRITE) { /* we own the lock but have to update it. */ list_add_tail(&range->list, &to_upgrade); } @@ -294,8 +315,7 @@ static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, new_range->start = current_start; new_range->end = range->start - 1; - new_range->mode = lock_mode; - new_range->state = FUSE_DLM_RANGE_GRANTED; + new_range->state = want; INIT_LIST_HEAD(&new_range->list); list_add_tail(&new_range->list, &to_lock); @@ -321,8 +341,7 @@ static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, new_range->start = current_start; new_range->end = end; - new_range->mode = lock_mode; - new_range->state = FUSE_DLM_RANGE_GRANTED; + new_range->state = want; INIT_LIST_HEAD(&new_range->list); list_add_tail(&new_range->list, &to_lock); @@ -331,7 +350,7 @@ static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, /* update locks, if any lock is in this list it has the wrong mode */ list_for_each_entry(range, &to_upgrade, list) { /* Update the lock mode */ - range->mode = lock_mode; + range->state = want; } /* Add all new ranges to the tree */ @@ -356,9 +375,9 @@ static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, /* Restore original lock modes for any partially upgraded locks */ list_for_each_entry(range, &to_upgrade, list) { - if (lock_mode == FUSE_PCACHE_LK_WRITE) { + if (want == FUSE_DLM_RANGE_WRITE) { /* We upgraded this lock but failed later, downgrade it back */ - range->mode = FUSE_PCACHE_LK_READ; + range->state = FUSE_DLM_RANGE_READ; } } @@ -384,7 +403,10 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, * @req: caller-owned storage for the request, live until commit or abort * @start: start page offset being requested (inclusive) * @end: end page offset being requested (inclusive) - * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE + * + * The mode is not recorded here: until the server answers the range is + * held in neither, and the mode that reaches the tree is the one passed + * to fuse_dlm_request_commit(). * * A FUSE_DLM_WB_LOCK reply and a NOTIFY revoke are serviced on different * threads, so a revoke can be processed before the grant the reply @@ -400,15 +422,13 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, */ void fuse_dlm_request_begin(struct fuse_inode *inode, struct fuse_dlm_range *req, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) + uint64_t end) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; RB_CLEAR_NODE(&req->rb); req->start = start; req->end = end; - req->mode = (mode == FUSE_PAGE_LOCK_READ) ? FUSE_PCACHE_LK_READ : - FUSE_PCACHE_LK_WRITE; req->state = FUSE_DLM_RANGE_REQUESTED; down_write(&cache->lock); @@ -623,17 +643,14 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range; - int lock_mode = 0; + enum fuse_dlm_range_state want; uint64_t current_start = start; if (!cache || start > end) return false; - /* Convert to lock mode if specified */ - if (mode == FUSE_PAGE_LOCK_READ) - lock_mode = FUSE_PCACHE_LK_READ; - else if (mode == FUSE_PAGE_LOCK_WRITE) - lock_mode = FUSE_PCACHE_LK_WRITE; + /* The state a range has to be in to cover this request */ + want = fuse_dlm_granted_state(mode); down_read(&cache->lock); @@ -650,7 +667,7 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, * re-requesting a READ lock for a range we already hold * a WRITE lock on (e.g. read-after-write). */ - if (lock_mode && range->mode < lock_mode) { + if (!fuse_dlm_state_satisfies(range->state, want)) { /* Held lock is weaker than requested */ up_read(&cache->lock); return false; @@ -708,7 +725,7 @@ bool fuse_dlm_write_grant_exists(struct fuse_inode *fi) down_read(&cache->lock); for (range = fuse_dlm_find_overlapping(cache, 0, U64_MAX); range; range = fuse_page_it_iter_next(range, 0, U64_MAX)) { - if (range->mode == FUSE_PCACHE_LK_WRITE) { + if (range->state == FUSE_DLM_RANGE_WRITE) { held = true; break; } @@ -810,7 +827,7 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, args.out_args[0].value = &outarg; /* Publish before sending; see fuse_dlm_request_begin() */ - fuse_dlm_request_begin(fi, &req, inarg.start, inarg.end, mode); + fuse_dlm_request_begin(fi, &req, inarg.start, inarg.end); err = fuse_simple_request(fm, &args); if (err) { diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 6f7c2c2fde0fd1..383a9a2174950f 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -38,7 +38,7 @@ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; struct fuse_dlm_cache { /* Lock protecting the tree and the pending list */ struct rw_semaphore lock; - /* Interval tree of granted ranges (FUSE_DLM_RANGE_GRANTED) */ + /* Interval tree of granted ranges (FUSE_DLM_RANGE_READ/_WRITE) */ struct rb_root_cached ranges; /* * FUSE_DLM_WB_LOCK requests in flight (REQUESTED, or REVOKED once @@ -61,11 +61,12 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, /* * Publish a FUSE_DLM_WB_LOCK for [start, end] before it is sent, so a * revoke processed while the reply is on the wire can mark it. @req is - * caller-owned storage, live until the matching commit or abort. + * caller-owned storage, live until the matching commit or abort. The + * mode is not recorded until the grant is, so only the commit takes it. */ void fuse_dlm_request_begin(struct fuse_inode *inode, struct fuse_dlm_range *req, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); + uint64_t end); /* * Retire @req and record the grant [start, end] as one step under the From c556abac177e8914197e43faedd9d5ca0c568667 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 27 Aug 2026 09:28:31 +0200 Subject: [PATCH 03/24] fuse: keep a revoked DLM range instead of deleting it fuse_dlm_unlock_range() removed the ranges it revoked, so a grant taken away looked exactly like a range that was never held. Nothing then stops writeback sending the page cache under it, and a writer that passed fuse_dlm_lock_is_held() before the revoke and dirtied the folio after it has its bytes sent for a range another node now holds. Mark such a range FUSE_DLM_RANGE_REVOKED instead, and keep it only for as long as there is page cache under it to describe; filemap_range_has_page() answers that, and one with nothing cached is still removed. A revoked range covers nothing, so the IO paths ask for the grant again, and writeback calls fuse_dlm_regrant_range() to take the range back before sending what it found revoked. Splitting at the revoke bounds replaces the trim and punch hole arithmetic, so fuse_dlm_punch_hole() goes and fuse_dlm_split_at() arrives in its place. fuse_dlm_ranges_dropped() frees the revoked ranges over page cache the caller has established is gone. The kernel-doc of the record said page offset throughout while every one of those arguments is a byte offset that happens to be page aligned, and the same file computes real page indices a few lines from some of them. Say byte offset. [hbi: adapted from ubuntu-hwe 1ef5278e3604, with the wording fix 248056598d62 and the record side of d75a174d339b and 9bb623317817 folded in. That branch grew a per range content bound and removed it again; this one never had it, so fuse_dlm_dirty_run() and the run classification it fed are not ported and writeback regrants whole runs.] Signed-off-by: Horst Birthelmer --- fs/fuse/fuse_dlm_cache.c | 366 ++++++++++++++++++++++++--------------- fs/fuse/fuse_dlm_cache.h | 23 ++- 2 files changed, 244 insertions(+), 145 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 82e18334299f70..473a3257729385 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -13,9 +13,13 @@ * - REQUESTED, on cache->pending, while its FUSE_DLM_WB_LOCK is in * flight. * - * - REVOKED, still on cache->pending, once a revoke has overlapped it. - * fuse_dlm_request_commit() drops such a grant instead of recording - * it. + * - REVOKED, in either place. On cache->pending it is a request a + * revoke overlapped while it was in flight, and + * fuse_dlm_request_commit() drops that grant instead of recording it. + * In cache->ranges it is a grant that was recorded and has since been + * taken away, kept because the page cache under it is still + * described. It covers nothing either way, so the IO paths ask + * again. * * - READ or WRITE, in cache->ranges. The only states * fuse_dlm_range_is_locked() reports as covered; the mode is not a @@ -24,11 +28,16 @@ * * In-flight requests are kept off the tree so the state is consulted * only where a request is retired, not by every tree walker. + * + * The record says nothing about the page cache under a range. What is + * cached there, and whether the server has seen it, is what the page + * cache itself answers. */ #include "fuse_i.h" #include "fuse_dlm_cache.h" #include +#include #include #include #include @@ -38,7 +47,11 @@ enum fuse_dlm_range_state { /* FUSE_DLM_WB_LOCK in flight, on cache->pending */ FUSE_DLM_RANGE_REQUESTED, - /* Revoked while in flight; the grant must not be recorded */ + /* + * On cache->pending, revoked in flight and the grant must not be + * recorded. In cache->ranges, granted once and taken away, kept to + * describe the page cache under it. Covers nothing either way. + */ FUSE_DLM_RANGE_REVOKED, /* Granted shared, in cache->ranges */ FUSE_DLM_RANGE_READ, @@ -50,9 +63,12 @@ enum fuse_dlm_range_state { struct fuse_dlm_range { /* Interval tree node; only linked once granted */ struct rb_node rb; - /* Start page offset (inclusive) */ + /* + * The range, as byte offsets, both inclusive. Grants arrive page + * aligned, and a range is split only at the bounds of another, so + * these are page aligned too. + */ uint64_t start; - /* End page offset (inclusive) */ uint64_t end; /* Subtree end value for interval tree */ uint64_t __subtree_end; @@ -104,8 +120,8 @@ INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, uint64_t, __subtree_end, /** * fuse_dlm_kill_pending - mark in-flight requests overlapping [start, end] * @cache: The page cache - * @start: Start page offset of the revoked region - * @end: End page offset of the revoked region + * @start: Start byte offset of the revoked region + * @end: End byte offset of the revoked region * * A revoke overlapping a request still on the wire has nothing to remove * from the tree, since that grant is not recorded yet. Marking it makes @@ -181,8 +197,8 @@ void fuse_dlm_cache_release_locks(struct fuse_inode *inode) /** * fuse_dlm_find_overlapping - Find a range that overlaps with [start, end] * @cache: The page cache - * @start: Start page offset - * @end: End page offset + * @start: Start byte offset + * @end: End byte offset * * Return: Pointer to the first overlapping range, or NULL if none found */ @@ -196,8 +212,8 @@ fuse_dlm_find_overlapping(struct fuse_dlm_cache *cache, uint64_t start, /** * fuse_page_try_merge - Try to merge ranges within a specific region * @cache: The page cache - * @start: Start page offset - * @end: End page offset + * @start: Start byte offset + * @end: End byte offset * * Attempt to merge ranges within and adjacent to the specified region * that have the same lock mode. @@ -232,7 +248,7 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, struct fuse_dlm_range, rb); } - /* Try to merge with next range if adjacent and same state */ + /* Merge neighbours the server has given us on the same terms */ if (next && range->state == next->state && range->end + 1 == next->start) { /* Merge ranges: re-insert so __subtree_end is updated */ @@ -254,8 +270,8 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, /** * fuse_dlm_lock_range_locked - Record a granted range of pages * @inode: The fuse inode - * @start: Start page offset - * @end: End page offset + * @start: Start byte offset + * @end: End byte offset * @mode: Lock mode (read or write) * * Add a locked range on the specified range of pages. @@ -297,17 +313,19 @@ static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, /* Get next overlapping range before we potentially modify the tree */ next = fuse_page_it_iter_next(range, start, end); - /* Check lock compatibility */ - if (want == FUSE_DLM_RANGE_WRITE && - range->state != FUSE_DLM_RANGE_WRITE) { - /* we own the lock but have to update it. */ + /* + * A revoked range is covered again by this grant, and a read + * range needs upgrading when a write is granted. + */ + if (range->state == FUSE_DLM_RANGE_REVOKED || + (want == FUSE_DLM_RANGE_WRITE && + range->state != FUSE_DLM_RANGE_WRITE)) list_add_tail(&range->list, &to_upgrade); - } /* If WRITE lock already exists - nothing to do */ /* If there's a gap before this range, we need to add the missing range */ if (current_start < range->start) { - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); + new_range = kmalloc(sizeof(*new_range), GFP_NOFS); if (!new_range) { ret = -ENOMEM; goto out_free; @@ -333,7 +351,7 @@ static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, /* If there's a gap after the last range to the end, extend the range */ if (!covered_to_end && current_start <= end) { - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); + new_range = kmalloc(sizeof(*new_range), GFP_NOFS); if (!new_range) { ret = -ENOMEM; goto out_free; @@ -347,11 +365,9 @@ static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, list_add_tail(&new_range->list, &to_lock); } - /* update locks, if any lock is in this list it has the wrong mode */ - list_for_each_entry(range, &to_upgrade, list) { - /* Update the lock mode */ + /* Everything on this list is now covered in @want */ + list_for_each_entry(range, &to_upgrade, list) range->state = want; - } /* Add all new ranges to the tree */ list_for_each_entry(new_range, &to_lock, list) { @@ -373,14 +389,10 @@ static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, kfree(new_range); } - /* Restore original lock modes for any partially upgraded locks */ - list_for_each_entry(range, &to_upgrade, list) { - if (want == FUSE_DLM_RANGE_WRITE) { - /* We upgraded this lock but failed later, downgrade it back */ - range->state = FUSE_DLM_RANGE_READ; - } - } - + /* + * Nothing to undo on @to_upgrade: every goto here is taken before + * the loop above runs, so no state has been changed yet. + */ return ret; } @@ -401,8 +413,8 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, * fuse_dlm_request_begin - publish a lock request before it is sent * @inode: the fuse inode * @req: caller-owned storage for the request, live until commit or abort - * @start: start page offset being requested (inclusive) - * @end: end page offset being requested (inclusive) + * @start: start byte offset being requested (inclusive) + * @end: end byte offset being requested (inclusive) * * The mode is not recorded here: until the server answers the range is * held in neither, and the mode that reaches the tree is the one passed @@ -430,6 +442,7 @@ void fuse_dlm_request_begin(struct fuse_inode *inode, req->start = start; req->end = end; req->state = FUSE_DLM_RANGE_REQUESTED; + /* Nothing reads this while the request is pending; publish it set */ down_write(&cache->lock); list_add_tail(&req->list, &cache->pending); @@ -440,8 +453,8 @@ void fuse_dlm_request_begin(struct fuse_inode *inode, * fuse_dlm_request_commit - retire a request and record its grant * @inode: the fuse inode * @req: the request published by fuse_dlm_request_begin() - * @start: start page offset the server granted (inclusive) - * @end: end page offset the server granted (inclusive) + * @start: start byte offset the server granted (inclusive) + * @end: end byte offset the server granted (inclusive) * @mode: the mode that was requested * * Unlinking @req and recording the grant are one step under the cache @@ -489,83 +502,116 @@ void fuse_dlm_request_abort(struct fuse_inode *inode, } /** - * fuse_dlm_punch_hole - Punch a hole in a locked range + * fuse_dlm_split_at - make @off start a range * @cache: The page cache - * @start: Start page offset of the hole - * @end: End page offset of the hole + * @off: byte offset to split at * - * Create a hole in a locked range by splitting it into two ranges. + * Splits the range containing @off in two, both halves keeping the state + * of the original, so a revoke can apply to one side only. A no-op when + * @off already starts a range or falls in a gap. * - * Return: 0 on success, negative error code on failure + * Caller holds @cache->lock for write. + * + * Cannot fail: the split decides which bytes a caller goes on to name, + * and both naming more than was written and lowering more than was sent + * lose data. iomap allocates the state it keeps per folio the same way. */ -static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +static void fuse_dlm_split_at(struct fuse_dlm_cache *cache, uint64_t off) { - struct fuse_dlm_range *range, *new_range; - int ret = 0; + struct fuse_dlm_range *range, *tail; - if (!cache || start > end) - return -EINVAL; + if (!off) + return; - /* Find a range that contains [start, end] */ - range = fuse_dlm_find_overlapping(cache, start, end); - if (!range) { - ret = -EINVAL; - goto out; - } + range = fuse_page_it_iter_first(&cache->ranges, off, off); + if (!range || range->start == off) + return; - /* If the hole is at the beginning of the range */ - if (start == range->start) { - fuse_page_it_remove(range, &cache->ranges); - range->start = end + 1; - fuse_page_it_insert(range, &cache->ranges); - goto out; - } + tail = kmalloc(sizeof(*tail), GFP_NOFS | __GFP_NOFAIL); - /* If the hole is at the end of the range */ - if (end == range->end) { - fuse_page_it_remove(range, &cache->ranges); - range->end = start - 1; - fuse_page_it_insert(range, &cache->ranges); - goto out; - } + *tail = *range; + INIT_LIST_HEAD(&tail->list); + tail->start = off; - /* The hole is in the middle, need to split */ - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out; - } + /* + * Bounds are never edited in place: the interval tree caches a + * subtree end that only insertion recomputes. + */ + fuse_page_it_remove(range, &cache->ranges); + range->end = off - 1; + fuse_page_it_insert(range, &cache->ranges); + fuse_page_it_insert(tail, &cache->ranges); +} + +/** + * fuse_dlm_ranges_dropped - the page cache under [start, end] is gone + * @inode: the fuse inode + * @start: start byte offset (inclusive) + * @end: end byte offset (inclusive) + * + * A revoked range exists only to describe page cache dirtied before the + * grant was taken away. Once that cache is gone the range has nothing + * left to say and is freed; a range still held goes back to describing + * nothing. + * + * The caller must have established that the range really is empty, not + * merely asked for it to be dropped: a folio that survived an + * invalidate is still there, and claiming otherwise would let writeback + * send it with no record of where it came from. + */ +void fuse_dlm_ranges_dropped(struct fuse_inode *inode, uint64_t start, + uint64_t end) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_range *range, *next; - /* Copy properties from original range */ - *new_range = *range; - INIT_LIST_HEAD(&new_range->list); + if (start > end) + return; - /* Adjust ranges */ - new_range->start = end + 1; - range->end = start - 1; + down_write(&cache->lock); - /* Update interval tree */ - fuse_page_it_remove(range, &cache->ranges); - fuse_page_it_insert(range, &cache->ranges); - fuse_page_it_insert(new_range, &cache->ranges); + fuse_dlm_split_at(cache, start); + if (end < U64_MAX) + fuse_dlm_split_at(cache, end + 1); -out: - return ret; + range = fuse_page_it_iter_first(&cache->ranges, start, end); + while (range) { + next = fuse_page_it_iter_next(range, start, end); + + if (range->state == FUSE_DLM_RANGE_REVOKED) { + fuse_page_it_remove(range, &cache->ranges); + kfree(range); + } + + range = next; + } + + fuse_dlm_try_merge(cache, start, end); + + up_write(&cache->lock); } /** - * fuse_dlm_unlock_range - Unlock a range of pages - * @cache: The page cache - * @start: Start page offset - * @end: End page offset + * fuse_dlm_unlock_range - Revoke the grants over a range of pages + * @inode: The fuse inode + * @start: Start byte offset + * @end: End byte offset * - * Release locks on the specified range of pages. An inverted range is - * rejected rather than silently removing nothing: the callers revoke - * coverage, and a revoke that quietly keeps the grant alive would let - * the re-validating IO paths trust a lock the server has taken away. - * To drop every grant use fuse_dlm_cache_release_locks() (there is no - * in-band sentinel range for it). + * The server has taken [start, end] back. A range that has nothing + * cached under it is removed; one that has is kept and marked + * FUSE_DLM_RANGE_REVOKED, so the page cache it covers stays described. + * Removing it instead would leave a gap, and a gap reads as "no record", + * which is what an untracked range looks like: writeback would then send + * folios dirtied under the grant that was just taken away. + * + * A revoked range covers nothing, so fuse_dlm_range_is_locked() reports + * it uncovered and the IO paths request again. + * + * An inverted range is rejected rather than silently revoking nothing: + * the callers revoke coverage, and a revoke that quietly keeps the grant + * alive would let the re-validating IO paths trust a lock the server has + * taken away. To drop every grant use fuse_dlm_cache_release_locks() + * (there is no in-band sentinel range for it). * * Return: 0 on success, negative error code on failure */ @@ -574,7 +620,6 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range, *next; - int ret = 0; if (!cache || start > end) return -EINVAL; @@ -588,32 +633,31 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, */ fuse_dlm_kill_pending(cache, start, end); - /* Find all ranges that overlap with [start, end] */ + /* Split so the revoked region has its own ranges */ + fuse_dlm_split_at(cache, start); + if (end < U64_MAX) + fuse_dlm_split_at(cache, end + 1); + range = fuse_page_it_iter_first(&cache->ranges, start, end); while (range) { - /* Get next overlapping range before we potentially modify the tree */ + /* Get next overlapping range before we modify the tree */ next = fuse_page_it_iter_next(range, start, end); - /* Check if we need to punch a hole */ - if (start > range->start && end < range->end) { - /* Punch a hole in the middle */ - ret = fuse_dlm_punch_hole(cache, start, end); - if (ret) - goto out; - /* After punching a hole, we're done */ - break; - } else if (start > range->start) { - /* Adjust the end of the range */ - fuse_page_it_remove(range, &cache->ranges); - range->end = start - 1; - fuse_page_it_insert(range, &cache->ranges); - } else if (end < range->end) { - /* Adjust the start of the range */ - fuse_page_it_remove(range, &cache->ranges); - range->start = end + 1; - fuse_page_it_insert(range, &cache->ranges); + /* + * A revoked range is kept only to say that the page cache + * under it was dirtied under a grant that has gone, so that + * writeback takes the range again before sending it. With + * nothing cached there it has nothing to say. + * + * A grant with no end is recorded to U64_MAX; the page cache + * is indexed by a signed offset, so ask it about as much of + * that as it can name. + */ + if (filemap_range_has_page(inode->inode.i_mapping, range->start, + min_t(uint64_t, range->end, + LLONG_MAX))) { + range->state = FUSE_DLM_RANGE_REVOKED; } else { - /* Complete overlap, remove the range */ fuse_page_it_remove(range, &cache->ranges); kfree(range); } @@ -621,20 +665,18 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, range = next; } -out: + fuse_dlm_try_merge(cache, start, end); + up_write(&cache->lock); - return ret; + return 0; } /** - * fuse_dlm_range_is_locked - Check if a page range is already locked - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * @mode: Lock mode to check for (or NULL to check for any lock) - * - * Check if the specified range of pages is already locked. - * The entire range must be locked for this to return true. + * fuse_dlm_range_is_locked - Check if a byte range is already locked + * @inode: The fuse inode + * @start: Start byte offset + * @end: End byte offset + * @mode: Lock mode to check for * * Return: true if the entire range is locked, false otherwise */ @@ -714,6 +756,11 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, * server has not seen, so its mtime and ctime run ahead of anything the * server can report. * + * A revoked range is not counted. The state does not keep the mode the + * range was granted in, so a revoked one cannot be told from a read that + * was taken away, and the dirty page cache such a range describes is + * what the caller checks the mapping for instead. + * * Return: true if at least one recorded range is held for write */ bool fuse_dlm_write_grant_exists(struct fuse_inode *fi) @@ -776,11 +823,10 @@ bool fuse_dlm_lock_is_held(struct fuse_inode *fi, loff_t offset, * re-validating the grant must not re-request on a nonzero return or * they would spin. */ -int fuse_get_dlm_lock(struct file *file, loff_t offset, - size_t length, enum fuse_page_lock_mode mode) +static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, + loff_t offset, size_t length, + enum fuse_page_lock_mode mode) { - struct fuse_file *ff = file->private_data; - struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_mount *fm = ff->fm; @@ -789,12 +835,21 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; struct fuse_dlm_range req; + uint64_t pg_start, pg_end; int err; /* An empty range needs no lock. */ if (!length) return 0; + /* + * note that the offset and length don't have to be page aligned + * here but since we only get here on writeback caching we will + * send out page aligned requests + */ + pg_start = (uint64_t)offset & PAGE_MASK; + pg_end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); + restart: /* note that this can be run from different processes * at the same time. It is intentionally not protected @@ -803,17 +858,19 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, * The early exit uses the same helper the callers re-validate * with, so this check and a later fuse_dlm_lock_is_held() can * never disagree about what counts as covered. */ - if (fuse_dlm_lock_is_held(fi, offset, length, mode)) - return 0; /* we already have this area locked */ + if (fuse_dlm_lock_is_held(fi, offset, length, mode)) { + /* + * Already covered, and the record says nothing beyond that, + * so this is one shared acquisition end to end. + */ + return 0; + } memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; - /* note that the offset and length don't have to be page aligned - * here but since we only get here on writeback caching we will - * send out page aligned requests */ - inarg.start = offset & PAGE_MASK; - inarg.end = (offset + length - 1) | (PAGE_SIZE - 1); + inarg.start = pg_start; + inarg.end = pg_end; inarg.type = (mode == FUSE_PAGE_LOCK_WRITE) ? FUSE_DLM_LOCK_WRITE : FUSE_DLM_LOCK_READ; @@ -884,3 +941,32 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, return 0; } + +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) +{ + return __fuse_get_dlm_lock(file->private_data, file_inode(file), + offset, length, mode); +} + +/** + * fuse_dlm_regrant_range - hold [start, end] again for writeback + * @ff: a fuse file open for writing on @inode + * @inode: the inode + * @start: start byte offset (inclusive) + * @end: end byte offset (inclusive) + * + * Writeback holds the range again before sending a folio, since a revoke + * may have arrived between the write and the send. Whatever the other + * holder wrote in between is overwritten, which for two writers that + * never synchronised is a legitimate order. + * + * A range still held is the ordinary case: the grant is found recorded + * and nothing is sent to the server. + */ +int fuse_dlm_regrant_range(struct fuse_file *ff, struct inode *inode, + uint64_t start, uint64_t end) +{ + return __fuse_get_dlm_lock(ff, inode, start, end - start + 1, + FUSE_PAGE_LOCK_WRITE); +} diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 383a9a2174950f..d1078cd84546d5 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -14,6 +14,7 @@ struct fuse_inode; struct fuse_dlm_range; +struct fuse_file; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; @@ -30,15 +31,16 @@ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; /* * Page cache lock manager. * - * @ranges holds the grants the client has been given and not had taken - * back. A request still on the wire covers nothing and lives on - * @pending instead, so tree walkers never filter on state. See enum - * fuse_dlm_range_state in fuse_dlm_cache.c. + * @ranges holds the grants the client has been given, and the ones it + * has had taken back that still describe page cache + * (FUSE_DLM_RANGE_REVOKED). A request still on the wire covers nothing + * and lives on @pending instead, so tree walkers never filter on state. + * See enum fuse_dlm_range_state in fuse_dlm_cache.c. */ struct fuse_dlm_cache { /* Lock protecting the tree and the pending list */ struct rw_semaphore lock; - /* Interval tree of granted ranges (FUSE_DLM_RANGE_READ/_WRITE) */ + /* Interval tree of recorded ranges, granted or revoked */ struct rb_root_cached ranges; /* * FUSE_DLM_WB_LOCK requests in flight (REQUESTED, or REVOKED once @@ -97,6 +99,17 @@ bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, /* Is any part of the file held for write? */ bool fuse_dlm_write_grant_exists(struct fuse_inode *inode); +/* + * The page cache under [start, end] is gone: free the revoked ranges over + * it. The caller must have established the range really is empty. + */ +void fuse_dlm_ranges_dropped(struct fuse_inode *inode, uint64_t start, + uint64_t end); + +/* Hold [start, end] again so writeback can send what it found revoked */ +int fuse_dlm_regrant_range(struct fuse_file *ff, struct inode *inode, + uint64_t start, uint64_t end); + /* This is the interface to the filesystem */ int fuse_get_dlm_lock(struct file *file, loff_t offset, size_t length, enum fuse_page_lock_mode mode); From 9d66b81f9f76334571b45131d35a923beb3098b4 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 27 Aug 2026 09:28:56 +0200 Subject: [PATCH 04/24] fuse: bound the re-request of a grant a revoke killed in flight fuse_dlm_request_commit() reports -EAGAIN when a revoke overlapped a request while it was on the wire, and __fuse_get_dlm_lock() went round again with nothing stopping it. A remote node revoking as fast as the grants are handed out keeps that going for as long as it likes, and writeback asks for a grant with a folio locked, so the loop is not merely slow, it holds a folio hostage and the task is unkillable while it does. Give it a count and a signal check. Both are generous: every pass is a whole round trip, so reaching either means the range is genuinely being fought over and the caller is better told than left spinning. Signed-off-by: Horst Birthelmer --- fs/fuse/fuse_dlm_cache.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 473a3257729385..2af629f1a3ad25 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -38,11 +38,18 @@ #include #include +#include #include #include #include +/* + * How often to ask again for a grant a revoke killed while it was in + * flight, before giving up on the range. Each pass is a round trip. + */ +#define FUSE_DLM_GRANT_RETRIES 16 + /* Lifecycle of a range; see the file comment above */ enum fuse_dlm_range_state { /* FUSE_DLM_WB_LOCK in flight, on cache->pending */ @@ -836,6 +843,7 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, struct fuse_dlm_lock_out outarg; struct fuse_dlm_range req; uint64_t pg_start, pg_end; + int tries = FUSE_DLM_GRANT_RETRIES; int err; /* An empty range needs no lock. */ @@ -921,10 +929,18 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, * A revoke overlapping this range was processed while the * request was in flight, so the grant is dead. Retry * rather than fail: no one else holds the range, and the - * write path turns an error into a failed write. Each - * pass is a fresh round trip, so a revoke storm throttles - * the loop. + * write path turns an error into a failed write. + * + * Not forever, though. Every pass is a whole round trip, + * which throttles the loop but does not end it, and + * writeback asks for a grant with a folio locked, so a node + * revoking as fast as the grants arrive would hold that + * folio and this task for as long as it kept going. */ + if (fatal_signal_pending(current)) + return -EINTR; + if (!tries--) + return -EIO; goto restart; } From 81cd737f1ad4c32ef925ce1f4a44e7df4455753b Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Tue, 25 Aug 2026 19:43:09 -0700 Subject: [PATCH 05/24] fuse: split DLM ranges at the bounds of a recorded grant fuse_dlm_lock_range_locked() put every range overlapping the grant on the upgrade list whole, without splitting at the grant bounds the way fuse_dlm_unlock_range() does. A range extending past the grant then had its uncovered part upgraded with it: holding a read grant on [0, 8191] and being granted write on [4096, 8191] recorded the whole node as held for write, so fuse_dlm_lock_is_held() reported [0, 4095] covered, a cached write there never sent FUSE_DLM_WB_LOCK, and the server went on thinking this client held only a read there -- cluster exclusion broken without any error to see. The revoked case is worse. Writeback re-grants only the run it is about to send, but the commit flipped the whole revoked range back to held, so revoked-dirty bytes outside the re-granted run reclassified as FUSE_DLM_RUN_DIRTY and were sent without taking the range again -- exactly the lost update FUSE_DLM_RANGE_REVOKED exists to prevent. Split at both grant bounds before walking, so upgrades apply only inside the grant. Both halves of a split keep state and content, and fuse_dlm_try_merge() at the end recoalesces whatever stayed equal. Signed-off-by: Allison Henderson --- fs/fuse/fuse_dlm_cache.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 2af629f1a3ad25..5bdb2c5045b3f3 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -124,6 +124,8 @@ INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, uint64_t, __subtree_end, fuse_dlm_range_start, fuse_dlm_range_last, static, fuse_page_it); +static void fuse_dlm_split_at(struct fuse_dlm_cache *cache, uint64_t off); + /** * fuse_dlm_kill_pending - mark in-flight requests overlapping [start, end] * @cache: The page cache @@ -314,6 +316,19 @@ static int fuse_dlm_lock_range_locked(struct fuse_inode *inode, uint64_t start, /* The state this grant records */ want = fuse_dlm_granted_state(mode); + /* + * Ranges are upgraded whole below, so split at the grant bounds + * first: a range extending past the grant would otherwise have its + * uncovered part upgraded with it, recording coverage the server + * never gave. A read range half-covered by a write grant would + * report the other half held for write, and a revoked range + * half-regranted would report its still-revoked bytes as held, so + * writeback would send them without taking the range again. + */ + fuse_dlm_split_at(cache, start); + if (end < U64_MAX) + fuse_dlm_split_at(cache, end + 1); + /* Find all ranges that overlap with [start, end] */ range = fuse_page_it_iter_first(&cache->ranges, start, end); while (range) { From 55e2dff1728cff7343372eeb81d37eeea02a3acb Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 21 Aug 2026 14:07:44 +0200 Subject: [PATCH 06/24] fuse: take a DLM read lock for the readahead window Readahead fills the page cache past the range fuse_cache_read_iter() locked, so those folios get no revoke when a remote node writes them. Request a read grant over the whole window in fuse_readahead() before any folio is consumed, and skip the window when the request fails. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 48247876177b5d..5221603ea73e60 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1100,6 +1100,34 @@ static void fuse_readahead(struct readahead_control *rac) if (fuse_is_bad(inode)) return; + /* + * Readahead fills the page cache past the range the reader locked, + * so take a DLM read grant over the whole window here too. Folios + * the server handed out no lock for are folios it will not revoke + * when a remote node writes them, and a later read would be served + * from stale cache. Take the grant before any folio is pulled off + * @rac, so the window is either fully covered or not populated. + * + * Speculative pages are not worth serving uncovered: on a failed + * request drop the window and let read_pages() clean up the folios + * left in @rac. A server without DLM support answers -ENOSYS and + * clears fc->dlm, which is not a failure. + * + * This can run inside the coherency gate, which + * fuse_cache_read_iter() holds across generic_file_read_iter(), so + * the round trip leans on the same server contract that lets a + * cache-miss FUSE_READ block there: replies are serviced on threads + * other than the one delivering a NOTIFY invalidate. + */ + if (fc->writeback_cache && fc->dlm) { + int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), + readahead_length(rac), + FUSE_PAGE_LOCK_READ); + + if (err < 0 && err != -ENOSYS) + return; + } + max_pages = min_t(unsigned int, fc->max_pages, fc->max_read / PAGE_SIZE); From 29d1f219ffe78c80923d20564557e5bbd5b3b10f Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 27 Aug 2026 09:32:04 +0200 Subject: [PATCH 07/24] fuse: let a NOTIFY invalidate ask what it has to do A data invalidation always dropped the range with invalidate_inode_pages2_range(), which launders dirty folios and waits for a FUSE_WRITE reply, even for a range holding nothing. Ask instead, under the gate so nothing populates or dirties in between. filemap_range_has_page() says whether anything is cached at all, and filemap_range_needs_writeback() whether laundering is needed; when it is not, invalidate_mapping_pages() drops the same folios without ever waiting for the server. [hbi: adapted from ubuntu-hwe ef3f058a4424 as it ends up after 9bb623317817. That branch asked a per range content bound (fuse_dlm_range_may_be_dirty()) and later replaced it with the page cache queries used here, so the bound is skipped and its fuse_dlm_range_touched() hunk in fuse_cache_write_iter() with it.] Signed-off-by: Horst Birthelmer --- fs/fuse/inode.c | 76 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 69694ed4ee9105..dc321d2f78f659 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -890,22 +890,25 @@ static void fuse_dlm_revoke_inval_range(struct fuse_inode *fi, loff_t offset, * Drop a page-cache range on behalf of a NOTIFY invalidate. * * invalidate_inode_pages2_range() waits out folios under writeback and - * launders dirty ones, both of which need a FUSE_WRITE reply. While - * writepages are frozen (fuse_set_nowrite(): truncate, O_TRUNC open, fsync, - * pre-SETATTR flush) no reply can arrive, because fuse_flush_writepages() - * parks the request on fi->queued_writes until fuse_release_nowrite(). A - * server that revokes from inside the handler it is revoking for then - * deadlocks against its own reply. fuse_do_setattr() states the same rule - * for its own invalidate. + * launders dirty ones, both of which need a FUSE_WRITE reply. It is only + * needed when the range can hold data the server has not seen. * - * So while frozen use invalidate_mapping_pages(), which skips dirty and - * under-writeback folios and never blocks. The stale clean folios still - * go, and the freezes that span a request drop the cache themselves once - * they complete: fuse_do_setattr() invalidates the mapping after releasing - * the freeze, the O_TRUNC open path calls truncate_pagecache(). + * @may_be_dirty false says it cannot. invalidate_mapping_pages() then + * drops the same folios without ever waiting for the server. + * + * The same substitution is forced while writepages are frozen + * (fuse_set_nowrite(): truncate, O_TRUNC open, fsync, pre-SETATTR flush), + * where no reply can arrive because fuse_flush_writepages() parks the + * request on fi->queued_writes until fuse_release_nowrite(). A server that + * revokes from inside the handler it is revoking for would otherwise + * deadlock against its own reply. fuse_do_setattr() states the same rule + * for its own invalidate. There the dirty folios are left behind, and the + * freezes that span a request drop the cache themselves once they complete: + * fuse_do_setattr() invalidates the mapping after releasing the freeze, the + * O_TRUNC open path calls truncate_pagecache(). */ static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, - pgoff_t end) + pgoff_t end, bool may_be_dirty) { struct fuse_inode *fi = get_fuse_inode(inode); bool frozen; @@ -914,7 +917,7 @@ static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, frozen = fi->writectr < 0; spin_unlock(&fi->lock); - if (frozen) + if (frozen || !may_be_dirty) invalidate_mapping_pages(inode->i_mapping, start, end); else invalidate_inode_pages2_range(inode->i_mapping, start, end); @@ -926,6 +929,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, struct percpu_rw_semaphore *wb_sem = NULL; struct fuse_inode *fi; struct inode *inode; + loff_t end_byte; pgoff_t pg_start; pgoff_t pg_end; @@ -960,6 +964,9 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, else pg_end = (offset + len - 1) >> PAGE_SHIFT; + /* Byte bounds of the same region, for the page cache queries */ + end_byte = len <= 0 ? LLONG_MAX : offset + len - 1; + /* * A data invalidation means another (remote) entity is modifying * the file. Two things happen here: @@ -1011,6 +1018,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, if (wb_sem) { bool hot, has_writer, latched = false; + bool may_be_dirty, has_pages; spin_lock(&fi->lock); hot = fuse_notify_inval_hot(fi); @@ -1035,6 +1043,22 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); + /* + * Ask what is left to do, under the gate so no + * reader can populate and no writer can dirty + * between the answer and the drop below. + * + * Nothing cached in the range means the drop is a + * no-op; the revoke above was the whole job. + * Otherwise the page cache says whether the drop has + * to launder, which is what makes it wait for a + * FUSE_WRITE reply. + */ + has_pages = filemap_range_has_page(inode->i_mapping, + offset, end_byte); + may_be_dirty = filemap_range_needs_writeback( + inode->i_mapping, offset, end_byte); + if (enable_notify_dio && hot && has_writer && !mapping_mapped(inode->i_mapping) && !fuse_inode_force_dio(inode)) { @@ -1049,14 +1073,17 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, /* * Latched: drop the whole mapping (dirty folios * outside the notified range would be invisible to - * the forced direct reads). Otherwise just the - * notified range. + * the forced direct reads), and nothing was asked + * about the rest of the file, so launder. Otherwise + * just the notified range, and only if anything is + * cached there. */ if (fuse_inode_force_dio(inode)) - fuse_notify_invalidate_range(inode, 0, -1); - else + fuse_notify_invalidate_range(inode, 0, -1, true); + else if (has_pages) fuse_notify_invalidate_range(inode, pg_start, - pg_end); + pg_end, + may_be_dirty); percpu_up_write(wb_sem); @@ -1064,12 +1091,17 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", nodeid); } else { - /* No gate on this inode (DAX, backing, non-regular, + /* + * No gate on this inode (DAX, backing, non-regular, * or the gate allocation failed): drop the lock - * range unserialized (best-effort), as before. */ + * range unserialized (best-effort), as before. The + * answers above were not taken either, so assume the + * range can hold unwritten data. + */ if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); - fuse_notify_invalidate_range(inode, pg_start, pg_end); + fuse_notify_invalidate_range(inode, pg_start, pg_end, + true); } } iput(inode); From a941a49d2dc4f316e7ce0b4aa2e48f2fdb4ec365 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 27 Aug 2026 09:33:48 +0200 Subject: [PATCH 08/24] fuse: hold the DLM grant again before writeback sends a folio A revoked range is kept now instead of being forgotten, but nothing acted on it: writeback queued whatever the page cache held, so bytes dirtied under a grant the server has since taken away still went out under no grant at all. Take the range back first. Every dirty folio here was written whole by this client, because the unaligned edges of a cached write go to the server directly and the interior covers whole pages, so there is nothing to classify; only the grant to make sure of. fuse_dlm_regrant_range() re-requests a range that has gone and walks the record once under the lock held for read when it has not. A server without DLM answers -ENOSYS, which is not a failure. On a real failure the folio goes back on the dirty list so the next writeback tries again. Neither caller does that for us: write_cache_pages() clears the folio before calling in, and fuse_launder_folio() clears it itself, so returning the error without redirtying would drop bytes the server has never seen. Both writeback entry points take it: fuse_writepages_fill() for ordinary writeback and fuse_writepage_locked(), which is what ->launder_folio sends through. [hbi: the record side of ubuntu-hwe 1ef5278e3604 as d75a174d339b leaves it. That branch has a single iomap writeback range op where this one has two entry points, and it ported through a run classification this branch never grew.] Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 5221603ea73e60..c604dd4e2ea612 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2628,6 +2628,7 @@ static int fuse_writepage_locked(struct folio *folio) struct address_space *mapping = folio->mapping; struct inode *inode = mapping->host; struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_writepage_args *wpa; struct fuse_args_pages *ap; struct fuse_file *ff; @@ -2637,6 +2638,22 @@ static int fuse_writepage_locked(struct folio *folio) if (!ff) goto err; + /* + * Hold the range again before sending it; see fuse_writepages_fill(). + * fuse_launder_folio() cleared the folio, so put it back on failure + * rather than lose the bytes; the invalidate that laundered it then + * reports the folio busy, as it does for any folio it cannot free. + */ + if (fc->dlm && fc->writeback_cache) { + error = fuse_dlm_regrant_range(ff, inode, folio_pos(folio), + folio_pos(folio) + + folio_size(folio) - 1); + if (error < 0 && error != -ENOSYS) { + filemap_dirty_folio(mapping, folio); + goto err_writepage_args; + } + } + wpa = fuse_writepage_args_setup(folio, ff); error = -ENOMEM; if (!wpa) @@ -2763,6 +2780,33 @@ static int fuse_writepages_fill(struct folio *folio, goto out_unlock; } + /* + * Everything dirty here was written whole by this client: the + * unaligned edges of a cached write go to the server directly and + * the interior covers whole pages, so nothing partly written is + * ever dirtied. There is nothing to classify, only the grant to + * make sure of: a revoke may have arrived since the write, and + * these bytes must not go out from under one. + * + * fuse_dlm_regrant_range() takes the range back when it has gone, + * and walks the record once under the lock held for read when it + * has not. On failure the folio goes back on the dirty list, so + * the next writeback tries again: write_cache_pages() cleared it + * before calling here and does not put it back itself, and these + * bytes are not on the server. A server with no DLM answers + * -ENOSYS, which is not a failure. + */ + if (fc->dlm && fc->writeback_cache) { + err = fuse_dlm_regrant_range(data->ff, inode, folio_pos(folio), + folio_pos(folio) + + folio_size(folio) - 1); + if (err < 0 && err != -ENOSYS) { + folio_redirty_for_writepage(wbc, folio); + goto out_unlock; + } + err = 0; + } + if (wpa && fuse_writepage_need_send(fc, &folio->page, ap, data, wbc)) { fuse_writepages_send(data); data->wpa = NULL; From 60579c9c6a6cefabff7282fa3c170eea95a7cf5e Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 27 Aug 2026 09:34:00 +0200 Subject: [PATCH 09/24] fuse: free the DLM ranges a NOTIFY invalidate emptied A revoked range is kept only to say that the page cache under it was dirtied under a grant that has gone. The invalidate that follows the revoke is what takes that cache away, so nothing was left to free the ranges and they accumulated for the life of the inode, each one costing writeback a grant request for a folio that is no longer there. Free them once the drop has happened, and only when it really happened: invalidate_mapping_pages() skips a busy folio and invalidate_inode_pages2_range() can fail on one, and a folio that survived still needs its record. Ask the page cache again rather than assume. The record is told about whole pages, since that is how a grant is recorded. [hbi: the inode.c side of ubuntu-hwe 1ef5278e3604.] Signed-off-by: Horst Birthelmer --- fs/fuse/inode.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index dc321d2f78f659..4fde69d86061fe 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -929,6 +929,8 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, struct percpu_rw_semaphore *wb_sem = NULL; struct fuse_inode *fi; struct inode *inode; + uint64_t pg_first; + uint64_t pg_last; loff_t end_byte; pgoff_t pg_start; pgoff_t pg_end; @@ -964,8 +966,17 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, else pg_end = (offset + len - 1) >> PAGE_SHIFT; - /* Byte bounds of the same region, for the page cache queries */ + /* + * Byte bounds of the same region, and the page aligned form + * the DLM record is told about. A grant is recorded page + * aligned, so the range handed to it has to cover whole + * pages or the record would keep a range the page cache no + * longer backs. + */ end_byte = len <= 0 ? LLONG_MAX : offset + len - 1; + pg_first = (uint64_t)offset & PAGE_MASK; + pg_last = len <= 0 ? U64_MAX : + (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); /* * A data invalidation means another (remote) entity is modifying @@ -1085,6 +1096,18 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, pg_end, may_be_dirty); + /* + * A revoked range exists to describe page cache + * dirtied before the grant went; with that cache gone + * it has nothing left to say. Only when it really + * went: an invalidate can leave a busy folio behind, + * and that folio still needs its record. + */ + if (has_pages && + !filemap_range_has_page(inode->i_mapping, offset, + end_byte)) + fuse_dlm_ranges_dropped(fi, pg_first, pg_last); + percpu_up_write(wb_sem); if (latched) From b87c11e7c0369ed756d55cd69fa1a0f548625aab Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 27 Aug 2026 09:38:22 +0200 Subject: [PATCH 10/24] fuse: drop the per-inode coherency gate wb_inval_rwsem fenced cached IO out for the whole of a NOTIFY invalidate, so a writer could not dirty the page cache under a grant being revoked. The record does that now: a revoke marks the range instead of forgetting it, and writeback takes the range again before sending anything marked that way. A read never needed the fence. Gone with it: the re-validation and retry loops that existed only because the gate had to be dropped around a FUSE_DLM_WB_LOCK round trip, fuse_cache_wr_dlm_lock()'s unrecorded flag, FUSE_DLM_READ_RETRIES, and the percpu_rw_semaphore itself with its per-inode allocation and the eviction-time free. setattr and the atomic O_TRUNC path lose it too, holding i_rwsem exclusive. Both arms of fuse_cache_write_iter() had entered the gate, the writeback one and the writethrough one the killpriv fallback reaches with the writeback cache still on; both keep the forced-direct-IO re-check that used to sit inside it. [hbi: port of ubuntu-hwe 02a9e2ace5de. Its rationale also names the per range content bound, which that branch has since removed and this one never had; the revoked range and the writeback regrant are what carry the argument. Comments left pointing at the gate after the removal are corrected here rather than kept.] Signed-off-by: Horst Birthelmer --- fs/fuse/dir.c | 35 ++----- fs/fuse/file.c | 255 +++++++++++++---------------------------------- fs/fuse/fuse_i.h | 25 ----- fs/fuse/inode.c | 105 +++++-------------- 4 files changed, 104 insertions(+), 316 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index ce398f117ef593..034bd7d799565d 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2082,34 +2082,24 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, WARN_ON(!(attr->ia_valid & ATTR_SIZE)); WARN_ON(attr->ia_size != 0); if (fc->atomic_o_trunc) { - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; - /* * No need to send request to userspace, since actual * truncation has already been done by OPEN. But still * need to truncate page cache. * - * Revoke and drop under the coherency gate write side, - * like the NOTIFY invalidate path: a gate reader that - * already re-validated its grant must not have the - * lock tree and the cache yanked mid-hold, or it - * would repopulate the truncated range trusting a - * grant that no longer exists. Waiting for gate - * readers here is safe: we hold i_rwsem exclusive, so - * no gate holder can be waiting on it (the write path - * takes i_rwsem before the gate, the read path never - * takes it). + * Dropping every grant here does not need a reader or + * writer fenced out: truncate_pagecache() discards the + * folios rather than writing them, and a write racing + * this is a write racing an O_TRUNC open, which has no + * order to preserve. i_rwsem is held exclusive + * anyway, so no cached write is in progress. */ - if (wb_sem) - percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_cache_release_locks(fi); spin_lock(&fi->lock); i_size_write(inode, 0); spin_unlock(&fi->lock); truncate_pagecache(inode, 0); - if (wb_sem) - percpu_up_write(wb_sem); goto out; } file = NULL; @@ -2213,23 +2203,16 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if ((is_truncate || !is_wb) && S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; - /* - * Revoke and drop under the coherency gate write side; see - * the atomic-O_TRUNC branch above. i_rwsem is held - * exclusive here as well (setattr), so waiting out gate - * readers cannot deadlock. + * Revoke past the new size and drop what is beyond it; see + * the atomic-O_TRUNC branch above for why this needs nothing + * fenced out. i_rwsem is held exclusive here as well. */ - if (wb_sem) - percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1); truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); - if (wb_sem) - percpu_up_write(wb_sem); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index c604dd4e2ea612..42fc72b2edd3b2 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -468,10 +468,10 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, * If this release dropped the last writer, fuse_prepare_release() * cleared the forced-direct-IO latch (under fi->lock). Drop any clean * folios a read racing the latch may have repopulated so they cannot be - * served stale once caching mode resumes. No inode lock or - * wb_inval_rwsem: release may run on the fuse server thread (async fput - * from aio completion), where blocking on a contended inode lock could - * stall the connection. Writes were routed direct while latched, so + * served stale once caching mode resumes. No inode lock: release may + * run on the fuse server thread (async fput from aio completion), + * where blocking on a contended inode lock could stall the + * connection. Writes were routed direct while latched, so * only clean folios exist and this invalidate is server-free; the last * writer is gone, so no forced-dio writer can race the drop. */ @@ -1113,11 +1113,9 @@ static void fuse_readahead(struct readahead_control *rac) * left in @rac. A server without DLM support answers -ENOSYS and * clears fc->dlm, which is not a failure. * - * This can run inside the coherency gate, which - * fuse_cache_read_iter() holds across generic_file_read_iter(), so - * the round trip leans on the same server contract that lets a - * cache-miss FUSE_READ block there: replies are serviced on threads - * other than the one delivering a NOTIFY invalidate. + * The round trip is taken before any folio of the window is locked + * and with nothing fenced out, so it holds up this reader and + * nothing else. */ if (fc->writeback_cache && fc->dlm) { int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), @@ -1165,21 +1163,12 @@ static void fuse_readahead(struct readahead_control *rac) static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); -/* - * Bound on re-requesting a revoked DLM grant before a cached read is - * served unlocked; see fuse_cache_read_iter(). - */ -#define FUSE_DLM_READ_RETRIES 3 - static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_inode *fi = get_fuse_inode(inode); - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; ssize_t res; - int lock_err = 0; /* * In auto invalidate mode, always update attributes on read. @@ -1197,65 +1186,21 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) /* if we have dlm support acquire a read lock for the area * we are reading from. */ if (fc->writeback_cache && fc->dlm) - lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ); + fuse_get_dlm_lock(file, iocb->ki_pos, iov_iter_count(to), + FUSE_PAGE_LOCK_READ); /* - * Fence the cache-serving read against a NOTIFY invalidate so we never - * hand back a folio the server has just superseded. The gate read side - * is per-CPU cheap; the NOTIFY holds the write side with priority. - * Re-check the forced-DIO latch under it: if a storm latched us while we - * waited on a pending writer, reroute to direct like the buffered write - * path, so we do not repopulate the cache the latch just dropped. - * wb_sem is NULL on non-writeback+dlm mounts (gate inactive). + * A NOTIFY invalidate racing this read drops the folios it + * supersedes, so the read either misses and refetches or returns + * data that was current when it was copied. There is nothing to + * fence: unlike a write, a read leaves nothing behind that could + * reach the server under a grant it no longer holds. */ - if (wb_sem) { - int tries = FUSE_DLM_READ_RETRIES; - -retry: - percpu_down_read(wb_sem); - if (fuse_inode_force_dio(inode)) { - percpu_up_read(wb_sem); - return fuse_direct_read_iter(iocb, to); - } - /* - * The DLM lock was requested before entering the gate, and - * the NOTIFY invalidate we may just have waited on revokes - * locks under the gate write side. Re-check the grant here - * and re-request with the gate dropped, so a - * FUSE_DLM_WB_LOCK round trip never parks a pending - * invalidate behind our own gate hold. Once the check - * passes the lock cannot go away for the rest of the gate - * hold. A failed or unrecorded request falls through - * unlocked, as before: the retry is taken even then (the - * latch must be re-checked under the re-entered gate), so - * lock_err has to stay sticky across it -- seeded by the - * pre-gate request above -- or a grant that failed would - * be re-requested forever. The retry is also bounded: a - * remote writer can revoke each successful grant before - * the gate is re-entered, and a reader-only inode has no - * force-DIO latch to end such a storm, so after - * FUSE_DLM_READ_RETRIES re-requests the read is served - * unlocked rather than looping without bound. - */ - if (!lock_err && fc->dlm && tries-- > 0 && - !fuse_dlm_lock_is_held(fi, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ)) { - percpu_up_read(wb_sem); - lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ); - goto retry; - } - } + if (fuse_inode_force_dio(inode)) + return fuse_direct_read_iter(iocb, to); res = generic_file_read_iter(iocb, to); - if (wb_sem) - percpu_up_read(wb_sem); - return res; } @@ -1658,19 +1603,13 @@ static void fuse_cache_wr_unlock(struct inode *inode, bool exclusive) * fc->dlm: the server has no DLM, proceed as a plain cached write. Any * other failure means the cache would be dirtied without DLM coverage - * the caller must fail the write instead. A granted-but-unrecorded - * lock (positive return) is covered cluster-wide; proceed, but flag it - * so the in-gate re-validation skips a check an invisible grant could - * never pass. + * lock (positive return) is covered cluster-wide; proceed. */ -static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len, - bool *unrecorded) +static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len) { int err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); - if (err < 0 && err != -ENOSYS) - return err; - *unrecorded = err > 0; - return 0; + return (err < 0 && err != -ENOSYS) ? err : 0; } /* @@ -1764,11 +1703,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) struct inode *inode = mapping->host; ssize_t err, count; struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_inode *fi = get_fuse_inode(inode); - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; - bool wb_guard = false; bool exclusive = true; - bool dlm_unrecorded = false; loff_t dlm_pos = 0; size_t dlm_len = 0; @@ -1827,17 +1762,18 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * is an unbounded cluster round trip, and holding the * writer-priority rwsem across it would park a truncate -- and * behind it every later writer -- for the duration. The - * grant-to-use window this leaves open is closed by the in-gate - * re-validation below. Only the append case must wait for the - * lock: its range depends on i_size, which is stable only under - * the exclusive inode lock. + * grant-to-use window this leaves open is closed on the way + * out instead: a revoke keeps the range rather than forgetting + * it, and writeback holds it again before sending anything. + * Only the append case must wait for the lock: its range + * depends on i_size, which is stable only under the exclusive + * inode lock. */ if (fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { dlm_pos = iocb->ki_pos; dlm_len = iov_iter_count(from); - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len); if (err) return err; @@ -1855,17 +1791,12 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) } /* - * Open-code generic_file_write_iter() so that the coherency - * gate can be held for read across the page-cache dirtying: a - * concurrent NOTIFY_INVAL_INODE -- which takes the write side - * of that gate (blocking, with priority) around its invalidate - * + latch set -- must not be able to strand the folios we are - * about to dirty. Re-check the latch under it (it may have been - * set while we blocked on the inode lock) and re-route to the - * direct path if it is now set; the DLM write lock taken above - * is harmless there, as the direct path does its own server - * coordination. wb_sem is NULL on mounts where the gate is - * inactive. + * Open-code generic_file_write_iter() so the DLM lock can be + * taken where the write really lands and the forced-direct-IO + * latch re-checked after the inode lock, which may have been + * set while we blocked on it. A re-route then goes to the + * direct path; the DLM write lock taken above is harmless + * there, as the direct path does its own server coordination. */ if (exclusive) inode_lock(inode); @@ -1884,8 +1815,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) dlm_pos = i_size_read(inode); dlm_len = iov_iter_count(from); - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len); if (err) goto wb_out; } @@ -1898,16 +1828,14 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * The exclusive inode lock does not pin i_size for the append: * attribute replies move it under fi->lock alone, so * generic_write_checks() may have put ki_pos past the granted - * range. Re-lock where the write really lands; dlm_pos tracks - * it so the in-gate re-validation below guards the same range. + * range. Re-lock where the write really lands. */ if (fc->dlm && (iocb->ki_flags & IOCB_APPEND) && iocb->ki_pos != dlm_pos) { dlm_pos = iocb->ki_pos; dlm_len = iov_iter_count(from); - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len); if (err) { written = err; goto wb_out; @@ -1915,8 +1843,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) } /* - * Kill suid/sgid and stamp the timestamps here, before the - * gate, instead of leaving them to + * Kill suid/sgid and stamp the timestamps here, ahead of the + * write itself, instead of leaving them to * __generic_file_write_iter(). file_remove_privs() is the one * that reaches the server: without handle_killpriv[_v2] * fuse_setattr() kills the bits by asking it (a FUSE_GETATTR to @@ -1924,11 +1852,10 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * inode first flushes and freezes writepages), and * security_inode_killpriv() can drop the capability xattr with * another round trip. A server may have to invalidate this - * inode from inside such a handler; its NOTIFY_INVAL_INODE then - * blocks in percpu_down_write() draining a gate reader that is - * itself waiting for the reply. Nothing held under the gate may - * wait for the server. file_update_time() only marks the inode - * dirty, but stays next to it to keep the VFS order. + * inode from inside such a handler, and it must not find this + * write holding anything it needs. file_update_time() only + * marks the inode dirty, but stays next to it to keep the VFS + * order. */ err = file_remove_privs(file); if (!err) @@ -1938,33 +1865,21 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) goto wb_out; } - if (wb_sem) { - wb_guard = true; -retry: - percpu_down_read(wb_sem); - if (fuse_inode_force_dio(inode)) { - percpu_up_read(wb_sem); - fuse_cache_wr_unlock(inode, exclusive); - return fuse_direct_write_iter(iocb, from); - } - if (fc->dlm && !dlm_unrecorded && - !fuse_dlm_lock_is_held(fi, dlm_pos, dlm_len, - FUSE_PAGE_LOCK_WRITE)) { - percpu_up_read(wb_sem); - err = fuse_cache_wr_dlm_lock(file, dlm_pos, - dlm_len, - &dlm_unrecorded); - if (err) { - /* The gate is already dropped; funnel - * the failure through the one audited - * exit. */ - written = err; - wb_guard = false; - goto wb_out; - } - goto retry; - } + if (fuse_inode_force_dio(inode)) { + fuse_cache_wr_unlock(inode, exclusive); + return fuse_direct_write_iter(iocb, from); } + + /* + * A NOTIFY invalidate can revoke the grant requested above + * between here and the dirtying below, and nothing stops it: + * the bytes are caught on the way out instead. + * fuse_dlm_unlock_range() keeps a revoked range for as long as + * there is page cache under it, and writeback holds the range + * again before sending anything. So a write racing a revoke + * costs a round trip, not coverage. + */ + if (iocb->ki_flags & IOCB_DIRECT) { written = generic_file_direct_write(iocb, from); if (written < 0 || !iov_iter_count(from)) @@ -1985,8 +1900,6 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) written = generic_perform_write(iocb, from); } wb_out: - if (wb_guard) - percpu_up_read(wb_sem); fuse_cache_wr_unlock(inode, exclusive); if (written > 0) written = generic_write_sync(iocb, written); @@ -2001,12 +1914,10 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) goto out; /* - * Kill suid/sgid and stamp the timestamps before entering the gate, - * for the reason given in the writeback branch: file_remove_privs() - * can issue a request, and a request must never be waited for under - * the gate. They run before the forced-DIO re-route below, so a - * re-routed write repeats them; neither has anything left to do the - * second time. + * Kill suid/sgid and stamp the timestamps here, for the reason given + * in the writeback branch. They run before the forced-DIO re-route + * below, so a re-routed write repeats them; neither has anything left + * to do the second time. */ err = file_remove_privs(file); if (err) @@ -2018,23 +1929,13 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) /* * The killpriv fallback lands here with the writeback cache still on, - * so it populates the page cache too and needs the same guard as the - * writeback branch above: hold the coherency gate for read across the - * page-cache population and re-check the latch under it, so a - * concurrent NOTIFY_INVAL_INODE cannot have the cache repopulated - * behind the invalidate it just did. Still taken before - * task_io_account_write() so a re-route is not double-counted. - * wb_sem is NULL on mounts where the gate is inactive, and such a - * connection never latches either. + * so it populates the page cache too and re-checks the latch like the + * writeback branch above. Still before task_io_account_write() so a + * re-route is not double-counted. */ - wb_guard = !!wb_sem; - if (wb_guard) { - percpu_down_read(wb_sem); - if (fuse_inode_force_dio(inode)) { - percpu_up_read(wb_sem); - inode_unlock(inode); - return fuse_direct_write_iter(iocb, from); - } + if (fuse_inode_force_dio(inode)) { + inode_unlock(inode); + return fuse_direct_write_iter(iocb, from); } task_io_account_write(count); @@ -2049,8 +1950,6 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) written = fuse_perform_write(iocb, from, false); } out: - if (wb_guard) - percpu_up_read(wb_sem); inode_unlock(inode); if (written > 0) written = generic_write_sync(iocb, written); @@ -3140,9 +3039,9 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) /* * If the inode was latched into forced direct IO after a remote-modify * notification, a mapping needs the page cache, so revert to caching - * mode. Revert without the inode lock or wb_inval_rwsem: ->mmap runs - * under mmap_lock and the buffered write path holds both across a fault - * on the user buffer (which takes mmap_lock), so taking either here + * mode. Revert without the inode lock: ->mmap runs under mmap_lock + * and the buffered write path holds both across a fault on the user + * buffer (which takes mmap_lock), so taking either here * would invert lock order (ABBA). Clearing the latch and dropping the * cache is sufficient -- writers re-check the latch and route to cached * IO once it is clear, and in-flight parallel dio drains itself. Cached @@ -3977,7 +3876,6 @@ static const struct address_space_operations fuse_file_aops = { void fuse_init_file_inode(struct inode *inode, unsigned int flags) { struct fuse_inode *fi = get_fuse_inode(inode); - struct fuse_conn *fc = get_fuse_conn(inode); inode->i_fop = &fuse_file_operations; inode->i_data.a_ops = &fuse_file_aops; @@ -3989,23 +3887,6 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->iocachectr = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); - /* - * Coherency gate for the forced-direct-IO feature; only writeback+dlm - * regular files need it. A percpu_rw_semaphore embeds per-CPU state, - * so allocate it out of line and only when the mount can use it rather - * than paying it on every inode. On failure leave it NULL: the gate - * stays inactive (best-effort invalidate) and the inode is still usable. - */ - fi->wb_inval_rwsem = NULL; - if (fc->writeback_cache && fc->dlm) { - struct percpu_rw_semaphore *sem = kmalloc(sizeof(*sem), GFP_KERNEL); - - if (sem && percpu_init_rwsem(sem)) { - kfree(sem); - sem = NULL; - } - fi->wb_inval_rwsem = sem; - } fi->notify_stamp = jiffies; fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 8dc81ff1f27452..e021b977fe828e 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -189,31 +189,6 @@ struct fuse_inode { /* dlm locked areas we have sent lock requests for */ struct fuse_dlm_cache dlm_locked_areas; - /* - * Per-inode read/write coherency gate for the - * forced-direct-IO feature. Cache-serving buffered reads - * and buffered writes hold it for read; being a - * percpu_rw_semaphore the read side is per-CPU cheap and - * scales on a shared file. The NOTIFY invalidate - * (fuse_reverse_inval_inode()) holds it for write, which - * BLOCKS so the coherency notify has priority: it fences - * cache-serving reads (and buffered writes) out for the - * whole invalidate, so no folio a remote modify has - * superseded is ever handed back. - * - * The write side may run on the server thread delivering - * the notify, so a blocking writer is safe only under a - * server that services request replies on threads other - * than the one delivering the notify (see the NOTIFY site). - * - * Allocated out of line only for writeback+dlm regular - * files (it shares storage with the readdir-cache union - * arm); NULL on other mounts and on allocation failure, - * where the gate is inactive and the invalidate falls back - * to best-effort. - */ - struct percpu_rw_semaphore *wb_inval_rwsem; - /* * Rate of FUSE_NOTIFY_INVAL_INODE data invalidations * for this whole file: notify_stamp is the jiffies of diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 4fde69d86061fe..a71921d6e402f2 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -208,23 +208,6 @@ static void fuse_evict_inode(struct inode *inode) WARN_ON(!list_empty(&fi->queued_writes)); fuse_dlm_cache_release_locks(fi); } - - /* - * Free the coherency gate here rather than in ->free_inode: that runs - * from an RCU callback, where percpu_free_rwsem() may sleep in - * rcu_sync_dtor() if the write side has not fully quiesced. No user - * can remain by eviction time: gate readers hold a file reference and - * a concurrent notify holds an inode reference. wb_inval_rwsem lives - * in the regular-file union arm and is only ever allocated for regular - * files, so gate on S_ISREG (but not fuse_is_bad() -- bad-marked - * regular files still own a gate); a directory's overlapping - * readdir-cache fields must not be misread. - */ - if (S_ISREG(inode->i_mode) && fi->wb_inval_rwsem) { - percpu_free_rwsem(fi->wb_inval_rwsem); - kfree(fi->wb_inval_rwsem); - fi->wb_inval_rwsem = NULL; - } } static int fuse_reconfigure(struct fs_context *fsc) @@ -926,7 +909,6 @@ static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { - struct percpu_rw_semaphore *wb_sem = NULL; struct fuse_inode *fi; struct inode *inode; uint64_t pg_first; @@ -934,6 +916,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t end_byte; pgoff_t pg_start; pgoff_t pg_end; + bool tracked; inode = fuse_ilookup(fc, nodeid, NULL); if (!inode) @@ -983,19 +966,13 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * the file. Two things happen here: * * 1. Coherency. Drop the affected page-cache range so no local - * read returns a folio the remote modify has superseded. This - * runs under the write side of the per-inode coherency gate - * (wb_inval_rwsem), which fences cache-serving buffered reads - * and buffered writes out for the whole invalidate. Unlike the - * old best-effort trylock this BLOCKS -- the notify has - * priority: percpu_down_write() parks new gate readers, drains - * in-flight ones, then invalidates. A blocking writer here is - * safe only under a server that services request replies on - * threads other than the one delivering this notify: the write - * side waits for gate readers to drain, and a cache-miss read - * holds the read side across its FUSE_READ round-trip. redfs' - * dlm server provides that contract; a server that cannot must - * not enable writeback+dlm. + * read returns a folio the remote modify has superseded. + * Nothing is fenced out for it. A read racing the drop + * either misses and refetches or returns data that was + * current when it was copied. A write racing it is caught + * on the way out instead: this revoke marks the range rather + * than forgetting it, and writeback holds the range again + * before sending anything it finds marked that way. * * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), under * fi->lock, updated for every data invalidation) of how fast @@ -1013,21 +990,17 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * already-latched inodes run out on the usual exits (last * writer closes, or mmap). * - * The gate (and the average) exist only for writeback+dlm regular - * files; elsewhere wb_sem is NULL and the invalidate runs - * unserialized (best-effort), as before. An mmapped inode - * keeps the gate -- fuse_cache_read_iter() and - * fuse_cache_write_iter() enter it unconditionally and rely - * on the revoke staying fenced -- but is never latched: - * a mapping needs the page cache, and fuse_file_mmap() - * reverts any latch it races with. + * The average and the latch exist only for writeback+dlm + * regular files; elsewhere there is no record to consult and + * the range is dropped as it always was. An mmapped inode is + * never latched: a mapping needs the page cache, and + * fuse_file_mmap() reverts any latch it races with. */ - if (S_ISREG(inode->i_mode) && fc->writeback_cache && - fc->dlm && !FUSE_IS_DAX(inode) && - !fuse_inode_backing(fi)) - wb_sem = fi->wb_inval_rwsem; + tracked = S_ISREG(inode->i_mode) && fc->writeback_cache && + fc->dlm && !FUSE_IS_DAX(inode) && + !fuse_inode_backing(fi); - if (wb_sem) { + if (tracked) { bool hot, has_writer, latched = false; bool may_be_dirty, has_pages; @@ -1037,39 +1010,19 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, spin_unlock(&fi->lock); /* - * Priority write side: park new gate readers, - * drain in-flight ones, then invalidate. Blocks - * (unlike the old trylock) -- see the contract in - * the comment above. - */ - percpu_down_write(wb_sem); - - /* - * Revoke the DLM lock range under the gate write - * side, atomically with the page drop: gate readers - * re-validate their grant right after entering, and - * a grant that passed that check must stay visible - * for their whole gate hold. - */ - if (fc->dlm && fc->writeback_cache) - fuse_dlm_revoke_inval_range(fi, offset, len); - - /* - * Ask what is left to do, under the gate so no - * reader can populate and no writer can dirty - * between the answer and the drop below. - * - * Nothing cached in the range means the drop is a - * no-op; the revoke above was the whole job. - * Otherwise the page cache says whether the drop has - * to launder, which is what makes it wait for a - * FUSE_WRITE reply. + * What this notify has to do. Nothing cached in the + * range means the drop is a no-op and the revoke is + * the whole job. Otherwise the page cache says + * whether the drop has to launder, which is what + * makes it wait for a FUSE_WRITE reply. */ has_pages = filemap_range_has_page(inode->i_mapping, offset, end_byte); may_be_dirty = filemap_range_needs_writeback( inode->i_mapping, offset, end_byte); + fuse_dlm_revoke_inval_range(fi, offset, len); + if (enable_notify_dio && hot && has_writer && !mapping_mapped(inode->i_mapping) && !fuse_inode_force_dio(inode)) { @@ -1108,18 +1061,14 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, end_byte)) fuse_dlm_ranges_dropped(fi, pg_first, pg_last); - percpu_up_write(wb_sem); - if (latched) pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", nodeid); } else { /* - * No gate on this inode (DAX, backing, non-regular, - * or the gate allocation failed): drop the lock - * range unserialized (best-effort), as before. The - * answers above were not taken either, so assume the - * range can hold unwritten data. + * No record on this inode (DAX, backing, non-regular, + * or no DLM), so assume the range can hold unwritten + * data and drop it as before. */ if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); From 870f3c25434ae8840f857601af1ec0cb3dd075eb Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 27 Aug 2026 09:39:47 +0200 Subject: [PATCH 11/24] fuse: flush the page cache before re-routing to direct IO The forced-direct-IO latch is checked once, so a buffered write that passed the check just before a notify set the latch dirties the page cache after the notify dropped the mapping. The coherency gate used to fence that. What is left behind is invisible to the direct path: a direct read misses the dirty folio, and a direct write lands underneath it, after which the invalidate launders rather than drops and puts the stale folio on the server on top. Write the range back before re-routing. [port of ubuntu-hwe 53140d0b12c0; this branch has two late re-route sites, the writeback arm and the writethrough one the killpriv fallback reaches, and both take the flush. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 42fc72b2edd3b2..7a6de5775497cd 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1196,8 +1196,22 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) * fence: unlike a write, a read leaves nothing behind that could * reach the server under a grant it no longer holds. */ - if (fuse_inode_force_dio(inode)) + if (fuse_inode_force_dio(inode)) { + size_t count = iov_iter_count(to); + + /* + * A write that passed this same check just before the latch + * took hold dirtied the page cache after the notify dropped + * it, and a direct read does not look there. Send it first. + */ + if (count) { + res = filemap_write_and_wait_range(inode->i_mapping, + iocb->ki_pos, iocb->ki_pos + count - 1); + if (res) + return res; + } return fuse_direct_read_iter(iocb, to); + } res = generic_file_read_iter(iocb, to); @@ -1866,7 +1880,22 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) } if (fuse_inode_force_dio(inode)) { + /* + * As on the read side, only worse: the direct write + * would land under whatever a write racing the latch + * left dirty, and the invalidate + * fuse_direct_write_iter() does after it launders + * rather than drops, putting that folio on the server + * on top. Send it first and the order is ordinary. + */ + count = iov_iter_count(from); + if (count) + err = filemap_write_and_wait_range( + inode->i_mapping, iocb->ki_pos, + iocb->ki_pos + count - 1); fuse_cache_wr_unlock(inode, exclusive); + if (err) + return err; return fuse_direct_write_iter(iocb, from); } @@ -1934,7 +1963,13 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) * re-route is not double-counted. */ if (fuse_inode_force_dio(inode)) { + /* Flush before re-routing; see the writeback branch above. */ + if (count) + err = filemap_write_and_wait_range(inode->i_mapping, + iocb->ki_pos, iocb->ki_pos + count - 1); inode_unlock(inode); + if (err) + return err; return fuse_direct_write_iter(iocb, from); } From 3b816f055441bc69fc30ebdc153f18ec1a7bbb6b Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 25 Aug 2026 20:21:15 +0200 Subject: [PATCH 12/24] fuse: free the DLM ranges a truncate emptied fuse_do_setattr() revokes the grants past the new size and discards the page cache there, but never tells the record. A revoked range is kept because it describes page cache dirtied before the grant went, so every truncate leaves ranges describing folios that no longer exist. Nothing frees them before the inode, and until then the inode reports unwritten data and every later invalidate over that region launders. Call fuse_dlm_ranges_dropped() from the first whole page above the new size; the page holding the new end of the file survives, so its record has to. Signed-off-by: Horst Birthelmer --- fs/fuse/dir.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 034bd7d799565d..f7ed184388e15e 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2209,10 +2209,21 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, * fenced out. i_rwsem is held exclusive here as well. */ if (fc->dlm && fc->writeback_cache) - fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1); + fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, + U64_MAX); truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); + + /* + * The cache above the new size is gone, so the ranges + * describing it have nothing left to say. From the first + * whole page above it: the page holding the new end of the + * file survives the truncate, and so does its record. + */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_ranges_dropped(fi, PAGE_ALIGN(outarg.attr.size), + U64_MAX); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); From 92bde30e4077c2a205b1908f16b39968dc886e97 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Tue, 25 Aug 2026 19:43:34 -0700 Subject: [PATCH 13/24] fuse: drop the DLM ranges a truncate's invalidate empties fuse_do_setattr() frees the record above the new size, but the invalidate_inode_pages2() it runs empties the mapping below it too, and the record there was left alone. A dirty folio is laundered by that invalidate, which puts its bytes on the server, but laundering lowers the record only for a folio the page cache does not consider valid (fuse_dlm_range_sent()), so an ordinary uptodate dirty folio leaves a DIRTY range describing page cache that no longer exists. The next unaligned write to such a page allocates a fresh folio, leaves it unfilled, and records only its own bytes. The stale DIRTY run over the rest of the page then makes fuse_read_folio_merge() keep folio bytes nobody wrote, and writeback classifies and sends them -- bad data both to a local reader and to the server. fsx reaches this with any truncate-then-partial-write sequence. Drop the record below the new size as well, and only when the drop really emptied it: a busy folio that survives the invalidate still needs its record, so ask filemap_range_has_page() first, the same rule the NOTIFY invalidate path applies before its drop. Signed-off-by: Allison Henderson --- fs/fuse/dir.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index f7ed184388e15e..2d5312bbbd7e48 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2224,6 +2224,22 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, if (fc->dlm && fc->writeback_cache) fuse_dlm_ranges_dropped(fi, PAGE_ALIGN(outarg.attr.size), U64_MAX); + + /* + * invalidate_inode_pages2() emptied the mapping below the new + * size too (laundering anything dirty first, so those bytes + * are on the server). A revoked range is kept only to make + * writeback take the grant again before sending the folios + * under it, so with those folios gone it describes nothing + * and would sit in the tree unfreed, keeping its neighbours + * from merging. Only when the drop really emptied it: a busy + * folio that survived still needs its record, and a fault + * populating after the check keeps its page visible to it. + */ + if (fc->dlm && fc->writeback_cache && outarg.attr.size && + !filemap_range_has_page(mapping, 0, outarg.attr.size - 1)) + fuse_dlm_ranges_dropped(fi, 0, + PAGE_ALIGN(outarg.attr.size) - 1); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); From 75618b77f7f146a14115dd4323631735a814907b Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Tue, 25 Aug 2026 19:43:58 -0700 Subject: [PATCH 14/24] fuse: forget the DLM record an O_TRUNC open discards The atomic O_TRUNC branch of fuse_do_setattr() releases every DLM range before dropping the cache, but the far more common path -- fuse_open() with fc->atomic_o_trunc -- called truncate_pagecache() and left the whole record standing. Ranges recorded DIRTY then described folios that no longer existed, and the next unaligned write to such a page left a fresh folio unfilled while the stale run made fuse_read_folio_merge() keep its unwritten bytes and writeback send them: bad data from something as plain as 'echo x > file' followed by a partial write. Release the record before the truncate, exactly as fuse_do_setattr() does; i_rwsem is held exclusive here (is_wb_truncate), so no cached write is between recording and dirtying. The FOPEN_KEEP_CACHE-less open one line below drops the mapping with invalidate_inode_pages2() and has the same problem. Drop the record there too, but only when the invalidate really emptied the mapping, under the same filemap_range_has_page() rule the NOTIFY path applies: a busy folio that survived keeps its record. Signed-off-by: Allison Henderson --- fs/fuse/dir.c | 12 ++++++++++-- fs/fuse/file.c | 26 ++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 2d5312bbbd7e48..0c9861bd462386 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -785,9 +785,17 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, fi = get_fuse_inode(inode); fuse_sync_release(fi, ff, flags); } else { - if (fm->fc->atomic_o_trunc && trunc) + if (fm->fc->atomic_o_trunc && trunc) { + /* + * Every grant goes with the cache, as on the + * fuse_open() O_TRUNC path: a record left behind + * would keep naming bytes the folios no longer hold. + */ + if (fm->fc->dlm && fm->fc->writeback_cache) + fuse_dlm_cache_release_locks( + get_fuse_inode(inode)); truncate_pagecache(inode, 0); - else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) + } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) invalidate_inode_pages2(inode->i_mapping); } return err; diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 7a6de5775497cd..81e14e3113704e 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -383,10 +383,32 @@ static int fuse_open(struct inode *inode, struct file *file) if (is_wb_truncate || dax_truncate) fuse_release_nowrite(inode); if (!err) { - if (is_truncate) + if (is_truncate) { + /* + * Every grant goes with the cache, as on the + * fuse_do_setattr() O_TRUNC path: a record left + * behind would keep naming bytes the folios no + * longer hold. i_rwsem is held exclusive + * (is_wb_truncate), so no cached write is mid-record. + */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_cache_release_locks(fi); truncate_pagecache(inode, 0); - else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) + } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { invalidate_inode_pages2(inode->i_mapping); + /* + * Only when the drop really emptied the mapping; a + * folio that survived still needs its record. This + * open holds no lock against concurrent IO, but + * neither does the invalidate above -- anything + * populated or dirtied after it keeps its page, and + * the check sees that page. + */ + if (fc->dlm && fc->writeback_cache && + !filemap_range_has_page(inode->i_mapping, 0, + LLONG_MAX)) + fuse_dlm_ranges_dropped(fi, 0, U64_MAX); + } } if (dax_truncate) filemap_invalidate_unlock(inode->i_mapping); From fb80d43981f591413d1fd8f3f25d0c9a9d6cf586 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Tue, 25 Aug 2026 19:44:25 -0700 Subject: [PATCH 15/24] fuse: drop the DLM ranges a punch hole empties FALLOC_FL_PUNCH_HOLE and FALLOC_FL_ZERO_RANGE flush the span, punch it on the server, and drop it from the page cache with truncate_pagecache_range() -- and told the record nothing. The flush lowers the record only for folios the page cache does not consider valid, so an uptodate dirty folio inside the hole left a DIRTY range describing page cache that no longer exists. The next unaligned write into the hole allocates a fresh folio, leaves it unfilled, and records its own bytes; the stale DIRTY run over the rest of the page makes fuse_read_folio_merge() keep folio bytes nobody wrote and writeback send them. fsx exercises punch hole against partial writes constantly, which is where the bad-data failures on this branch come from. Free the record over the whole pages inside the hole, under the same filemap_range_has_page() rule the NOTIFY path applies before its drop. The partial pages at the edges survive the truncate with their punched part zeroed -- those zeroes are real bytes the folio really holds -- so their record has to survive with them. Signed-off-by: Allison Henderson --- fs/fuse/file.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 81e14e3113704e..cbda688545182d 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3744,9 +3744,29 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, file_update_time(file); } - if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) + if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) { truncate_pagecache_range(inode, offset, offset + length - 1); + /* + * The whole pages inside the hole are gone, so a revoked + * range over them has nothing left to make writeback take + * the grant again for, and would sit in the tree unfreed. + * The pages straddling the ends survive with their punched + * part zeroed, so their record still names real (now zero) + * bytes and stays. Only when the drop really emptied the + * span: a busy folio that survived keeps its record. + */ + if (fm->fc->dlm && fm->fc->writeback_cache) { + uint64_t first = PAGE_ALIGN(offset); + uint64_t last = (uint64_t)(offset + length) & PAGE_MASK; + + if (first < last && + !filemap_range_has_page(inode->i_mapping, first, + last - 1)) + fuse_dlm_ranges_dropped(fi, first, last - 1); + } + } + fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); out: From 2bf46a561bbb710c21b72753a6a551bb5f73cab2 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Tue, 25 Aug 2026 19:44:49 -0700 Subject: [PATCH 16/24] fuse: drop the DLM ranges copy_file_range invalidates __fuse_copy_file_range() flushes the destination span, lets the server copy, and then drops the copied pages with truncate_inode_pages_range() as stale -- without telling the record. As on the truncate and punch hole paths, an uptodate dirty folio flushed and then dropped leaves a DIRTY range describing page cache that no longer exists, and the next partial write there keeps and flushes folio bytes nobody wrote. Free the record over the dropped span, gated on filemap_range_has_page() like the other drops: the copy runs under i_rwsem but faults do not, and a folio a fault put back after the truncate keeps its record. Signed-off-by: Allison Henderson --- fs/fuse/file.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index cbda688545182d..37f93a139bfb3d 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3870,9 +3870,24 @@ static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in, if (err) goto out; - truncate_inode_pages_range(inode_out->i_mapping, - ALIGN_DOWN(pos_out, PAGE_SIZE), - ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1); + { + loff_t lstart = ALIGN_DOWN(pos_out, PAGE_SIZE); + loff_t lend = ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1; + + truncate_inode_pages_range(inode_out->i_mapping, lstart, lend); + + /* + * The record over the dropped span has nothing left to + * describe, and left DIRTY it would make the next partial + * write there keep and flush folio bytes nobody wrote. + * Only when the drop really emptied it: a folio a + * concurrent fault put back keeps its record. + */ + if (fc->dlm && fc->writeback_cache && + !filemap_range_has_page(inode_out->i_mapping, lstart, + lend)) + fuse_dlm_ranges_dropped(fi_out, lstart, lend); + } file_update_time(file_out); fuse_write_update_attr(inode_out, pos_out + outarg.size, outarg.size); From 9a6184a6e59caff2ee7ed959bd9c7e505b36c1be Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Tue, 25 Aug 2026 20:25:38 -0700 Subject: [PATCH 17/24] fuse: keep the local size while the tail is dirty fuse_attr_cache_mask() trusted the local i_size over the server's only while a recorded write grant covered [attr->size, i_size). Two paths leave locally-extended data with no such record: a fault dirties pages under a page-mkwrite lock that is never recorded, and a local truncate revokes its own tail grants, after which cached writes extend the file again. On the next attribute refresh the check failed, the server's smaller size was applied, and truncate_pagecache() discarded dirty pages past it -- cached data destroyed by a GETATTR. The invalidate that follows then dropped the rest of the mapping with the record left standing, arming the stale-DIRTY bad-data path on every page of the file. generic/075 hits this within a few hundred fsx operations; the failure point moves with the attribute timeout, which is what made the runs look nondeterministic. Keep STATX_SIZE cached also while anything in [attr->size, i_size) is dirty or under writeback: those bytes exist only here, and the server cannot have a newer opinion about a size it has never seen. A remote truncate still lands, exactly as the design intends: its revoke launders and drops the tail first, so nothing is dirty there by the time the smaller size arrives. The no-grant early return learns the same rule, since a mapping dirtied only through page-mkwrite has no recorded grant at all. Signed-off-by: Allison Henderson --- fs/fuse/inode.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index a71921d6e402f2..2fc30aec0e335a 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -529,16 +529,36 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, !S_ISREG(inode->i_mode)) return cache_mask; - if (!fuse_dlm_write_grant_exists(fi)) + /* + * A dirty mapping keeps the local attributes authoritative even + * when no grant is recorded: a fault dirties pages under a + * page-mkwrite lock that is never recorded, and a truncate revokes + * the tail grants itself while cached writes above the new size + * are still waiting for writeback. + */ + if (!fuse_dlm_write_grant_exists(fi) && + !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) && + !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) return cache_mask; if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) || mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) cache_mask |= STATX_MTIME | STATX_CTIME; + /* + * The local size stays authoritative while the extension is + * covered by a write grant, and also while anything in + * [attr->size, size) is dirty or under writeback: those bytes + * exist only here, and taking the server's smaller size would + * truncate them away before they are ever sent. The grant check + * alone misses them, because a page-mkwrite grant is never + * recorded and a local truncate revokes its own tail grants. + */ if (have_size && size > (loff_t) attr->size && - fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, - FUSE_PAGE_LOCK_WRITE)) + (fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, + FUSE_PAGE_LOCK_WRITE) || + filemap_range_needs_writeback(inode->i_mapping, attr->size, + size - 1))) cache_mask |= STATX_SIZE; return cache_mask; From 380ff95e907c869d6adf8a38c942659258f3f596 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Tue, 25 Aug 2026 20:25:57 -0700 Subject: [PATCH 18/24] fuse: drop the DLM ranges an attribute change invalidates When an attribute reply is applied with the size not served from the cache, fuse_change_attributes_i() truncates the page cache to the server's size and, when the data may be stale, drops the whole mapping with invalidate_inode_pages2() -- and told the record nothing. This is the same hole just closed on the truncate, O_TRUNC open, punch hole and copy_file_range paths, reached from every GETATTR, and it is the drop behind the generic/075 corruption: the server-side trace shows a folio whose bytes were flushed correctly once, dropped here with its record left DIRTY, repopulated by a later partial write that leaves the folio unfilled, and then written back with the stale record naming the unfilled head -- zeroes sent over data the server already had. Free the record over what really went: unconditionally above the new size, where truncate_pagecache() leaves nothing behind, and over the rest only when the invalidate emptied the mapping, under the same filemap_range_has_page() rule as the other drops, so a folio that survived or was faulted back keeps its record. Signed-off-by: Allison Henderson --- fs/fuse/inode.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 2fc30aec0e335a..e851e9962721ad 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -646,6 +646,27 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr if (inval) invalidate_inode_pages2(inode->i_mapping); + + /* + * The DLM record has to follow the cache out, as on every + * other path that drops it. A revoked range exists only to + * make writeback take the grant again before sending the + * folios under it, so once they are gone it describes + * nothing and would sit in the tree unfreed. The pages above + * the new size are gone unconditionally; the rest only when + * the invalidate really emptied the mapping, so a folio that + * survived (or was faulted back) keeps its record. + */ + if (fc->dlm && fc->writeback_cache) { + if (have_size && oldsize != attr->size) + fuse_dlm_ranges_dropped(fi, + PAGE_ALIGN(attr->size), + U64_MAX); + if (inval && + !filemap_range_has_page(inode->i_mapping, 0, + LLONG_MAX)) + fuse_dlm_ranges_dropped(fi, 0, U64_MAX); + } } if (IS_ENABLED(CONFIG_FUSE_DAX)) From 2764f2b5200ba4b991f8b504bcc3fd00c3f2725a Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 27 Aug 2026 09:41:11 +0200 Subject: [PATCH 19/24] fuse: wait out the writeback a NOTIFY invalidate starts The handler revoked the range and left whatever was dirty under it to the drop that follows. Those bytes then have to go out under a grant that is already gone: writeback takes the range again first, a DLM round trip from inside the handler the server is waiting on. Send them before the revoke, while the grant still covers them, and wait for them there. do_writepages() runs in this context, so the grant is asked for while it is still held and the request never leaves the client; the drop that follows then finds nothing under writeback to block on, where laundering it would have waited for the same replies. A server that revokes from a thread it also needs to answer FUSE_WRITE on still deadlocks, the same contract fuse_notify_invalidate_range() states for a frozen inode. The error is left to the mapping, where fsync collects it. [hbi: ubuntu-hwe c9fab6c5b4c6 together with the writeback-before-revoke block it amends, which arrived there inside a259b4545b9a.] Signed-off-by: Horst Birthelmer --- fs/fuse/inode.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index e851e9962721ad..cab0f647f0caff 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1062,6 +1062,30 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, may_be_dirty = filemap_range_needs_writeback( inode->i_mapping, offset, end_byte); + /* + * Put unwritten data on the server while the grant + * still covers it, rather than leaving it to the drop + * below. After the revoke writeback would have to + * take the range again to send those bytes: a DLM + * round trip from inside the handler the server is + * waiting on. do_writepages() runs in this context, + * so the grant is asked for before the revoke. + * + * Waited out here rather than left to the drop, which + * launders when the range may be dirty and so waits + * for these same replies. One explicit wait, before + * the revoke, and the drop then finds nothing under + * writeback to block on. Either way a server that + * revokes from a thread it also needs to answer + * FUSE_WRITE on deadlocks here, the same contract + * fuse_notify_invalidate_range() states for a frozen + * inode. The error is left to the mapping, where + * fsync collects it. + */ + if (has_pages && may_be_dirty) + filemap_write_and_wait_range(inode->i_mapping, + offset, end_byte); + fuse_dlm_revoke_inval_range(fi, offset, len); if (enable_notify_dio && hot && has_writer && From 1560fc8bd1b739bc8556204ef476add30c68afe6 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Wed, 26 Aug 2026 10:08:46 -0700 Subject: [PATCH 20/24] fuse: do not hand O_APPEND to the server under the writeback cache With the writeback cache the kernel owns append positioning: a cached O_APPEND write is placed at the local i_size, and writeback later sends FUSE_WRITE requests with explicit offsets. Those offsets are only honoured if the server does not re-apply O_APPEND itself -- on Linux, pwrite(2) on a descriptor opened O_APPEND appends regardless of the offset argument. libfuse's passthrough examples know this and strip the flag, but only on FUSE_OPEN; a file created with FUSE_CREATE kept an O_APPEND backing descriptor, and every writeback run landed at the server's EOF instead of its offset. That is not just a theoretical hazard. Writeback legitimately covers the same bytes twice -- a folio whose record still names sent bytes is re-sent from its page start -- and with offsets honoured that re-send is idempotent. Appended instead, each flush's overlap with the previous one is duplicated at EOF and the file grows: generic/069 fails with exactly this shape (an appended file 230356 bytes too long, every byte of the excess a repeated flush overlap; server-side IO traces show each pwrite landing at EOF, sum of pwrites equal to the final file size). Strip O_APPEND from the flags sent in FUSE_OPEN, FUSE_CREATE and the compound open, the same way O_TRUNC is suppressed without atomic_o_trunc. The server cannot use the flag correctly under writeback caching anyway: appending server-side would order writeback runs by arrival, not by offset. Non-writeback mounts are unchanged, since there the server really does own append positioning. Signed-off-by: Allison Henderson --- fs/fuse/dir.c | 4 ++++ fs/fuse/file.c | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 0c9861bd462386..42d3cf8ee52b6d 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -723,6 +723,10 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, memset(&inarg, 0, sizeof(inarg)); memset(&outentry, 0, sizeof(outentry)); inarg.flags = flags; + + /* The kernel owns append positioning; see fuse_send_open() */ + if (fm->fc->writeback_cache) + inarg.flags &= ~O_APPEND; inarg.mode = mode; inarg.umask = current_umask(); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 37f93a139bfb3d..773bd23a2797df 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -70,6 +70,17 @@ static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, if (!fm->fc->atomic_o_trunc) inarg.flags &= ~O_TRUNC; + /* + * With the writeback cache the kernel owns append positioning: + * writeback sends FUSE_WRITE with explicit offsets, and a server + * that opens its backing file O_APPEND has pwrite(2) ignore them + * (Linux appends regardless of offset). Any re-sent or reordered + * run is then placed at EOF: duplicated data and a growing file. + * Do not hand the flag to the server at all. + */ + if (fm->fc->writeback_cache) + inarg.flags &= ~O_APPEND; + if (fm->fc->handle_killpriv_v2 && (inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) { inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID; @@ -173,6 +184,10 @@ static int fuse_compound_open_getattr(struct fuse_mount *fm, u64 nodeid, if (!fm->fc->atomic_o_trunc) open_in.flags &= ~O_TRUNC; + /* The kernel owns append positioning; see fuse_send_open() */ + if (fm->fc->writeback_cache) + open_in.flags &= ~O_APPEND; + if (fm->fc->handle_killpriv_v2 && (open_in.flags & O_TRUNC) && !capable(CAP_FSETID)) open_in.open_flags |= FUSE_OPEN_KILL_SUIDGID; From 3d797ceadb58058add0cf6732a430d0a4e84633b Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 27 Aug 2026 11:29:07 +0200 Subject: [PATCH 21/24] fuse: drop the open time page cache under the invalidate lock An open without FOPEN_KEEP_CACHE drops the mapping with invalidate_inode_pages2(), unserialised, so concurrent openers of one file all walk it and take each folio's lock in turn. They convoy folio by folio and each still pays for the whole walk: seven tasks reopening a cached file took 4.96s apiece, in lockstep, with the server idle and 86k lock sleeps each. Hold the invalidate lock across the walk. The first opener empties the mapping; the rest return from the mapping_empty() test in invalidate_inode_pages2_range() without touching a folio. Exclusive also fences faults, as truncate does over the same walk, and keeps the emptiness test the DLM record drop depends on from reading a mapping another opener is halfway through. IO can repopulate the mapping after the unlock, but such a folio is added after the drop and keeps its page, so a later reader finds both the page and its record. fuse_create_open() drops the cache the same way, so share a helper. Signed-off-by: Horst Birthelmer --- fs/fuse/dir.c | 13 +++++++++++-- fs/fuse/file.c | 43 +++++++++++++++++++++++++++++++++++-------- fs/fuse/fuse_i.h | 3 +++ 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 42d3cf8ee52b6d..fe4470e39fc095 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -799,8 +799,17 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, fuse_dlm_cache_release_locks( get_fuse_inode(inode)); truncate_pagecache(inode, 0); - } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) - invalidate_inode_pages2(inode->i_mapping); + } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { + /* + * Only when the drop really emptied the mapping, as + * in fuse_open(): a folio that survived still needs + * its record. + */ + if (fuse_open_drop_cache(inode) && + fm->fc->dlm && fm->fc->writeback_cache) + fuse_dlm_ranges_dropped(get_fuse_inode(inode), + 0, U64_MAX); + } } return err; diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 773bd23a2797df..5cd2ae54f986c0 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -341,6 +341,34 @@ int fuse_finish_open(struct inode *inode, struct file *file) return 0; } +/* + * Drop the page cache an open that did not get FOPEN_KEEP_CACHE must not keep, + * and report whether the mapping came out empty. + * + * Serialised on the mapping's invalidate lock. Every opener runs this same + * full-mapping walk and invalidate_inode_pages2_range() takes each folio's + * lock in turn, so openers racing on one shared file convoy on every folio + * and each pays for the whole walk: seven tasks reopening a cached file were + * measured at 4.96s apiece, in lockstep, with the server completely idle and + * ~86k lock sleeps per task. Under the lock the first opener empties the + * mapping and the rest fall straight back out of mapping_empty(). + * + * Holding it exclusive also fences faults for the duration, which is what + * truncate already does across this same walk, and it keeps the emptiness + * test from reading a mapping another opener is halfway through. + */ +bool fuse_open_drop_cache(struct inode *inode) +{ + bool emptied; + + filemap_invalidate_lock(inode->i_mapping); + invalidate_inode_pages2(inode->i_mapping); + emptied = !filemap_range_has_page(inode->i_mapping, 0, LLONG_MAX); + filemap_invalidate_unlock(inode->i_mapping); + + return emptied; +} + static void fuse_truncate_update_attr(struct inode *inode, struct file *file) { struct fuse_conn *fc = get_fuse_conn(inode); @@ -410,18 +438,17 @@ static int fuse_open(struct inode *inode, struct file *file) fuse_dlm_cache_release_locks(fi); truncate_pagecache(inode, 0); } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { - invalidate_inode_pages2(inode->i_mapping); /* * Only when the drop really emptied the mapping; a * folio that survived still needs its record. This - * open holds no lock against concurrent IO, but - * neither does the invalidate above -- anything - * populated or dirtied after it keeps its page, and - * the check sees that page. + * open holds no lock against buffered IO, which can + * repopulate the mapping once the invalidate lock is + * dropped -- but such a folio is populated after the + * drop and keeps its page, so a later reader finds + * both the page and the record it needs. */ - if (fc->dlm && fc->writeback_cache && - !filemap_range_has_page(inode->i_mapping, 0, - LLONG_MAX)) + if (fuse_open_drop_cache(inode) && + fc->dlm && fc->writeback_cache) fuse_dlm_ranges_dropped(fi, 0, U64_MAX); } } diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index e021b977fe828e..f483c1893b5bc8 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1237,6 +1237,9 @@ struct fuse_file *fuse_file_alloc(struct fuse_mount *fm, bool release); void fuse_file_free(struct fuse_file *ff); int fuse_finish_open(struct inode *inode, struct file *file); +/* Drop the page cache an open must not keep; true if it came out empty */ +bool fuse_open_drop_cache(struct inode *inode); + void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, unsigned int flags); From ae415cf9dd48f379b1bac43c34ebe19e9b7d323a Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 27 Aug 2026 11:29:41 +0200 Subject: [PATCH 22/24] fuse: size the readahead window by request mm sizes the readahead window from per-fd offset arithmetic, so threads walking one file at a stride each see their own hits as isolated, never ramp, and settle just above a single read. Every read then waits on a request of its own, pipeline one deep: 73-88KB per request against a server whose threads sat 86% idle. Round the window up to whole requests before any folio is fetched. Filling out a partial request costs no extra request, and readahead_expand() folds the growth into ra->size, so the next window starts from the wider shape. Only the trailing edge moves and the expansion stops at the first cached folio, so nothing is refetched and a window bounded by neighbouring data stays bounded. ra->ra_pages caps the unit, keeping read_ahead_kb authoritative. The DLM read grant moves below the expansion, so the covered window is the populated one. That request size is also what the connection carries in one go, which is what s_bdi->io_pages describes: left at its 128K default it clamps a large explicit read or a FADV_WILLNEED to a window meant only to bound guessing. Set it to fc->max_pages. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 43 ++++++++++++++++++++++++++++++++++++++----- fs/fuse/inode.c | 10 ++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 5cd2ae54f986c0..700ecec77d0aff 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1164,13 +1164,49 @@ static void fuse_readahead(struct readahead_control *rac) if (fuse_is_bad(inode)) return; + max_pages = min_t(unsigned int, fc->max_pages, + fc->max_read / PAGE_SIZE); + + /* + * Round the window up to whole requests before anything is fetched. + * + * mm sizes it from per-fd offset arithmetic, so several threads + * walking one file at a stride each see their own hits as isolated, + * never ramp, and settle on a window barely wider than one read. + * That leaves a request -- and a server round trip -- in the reader's + * path for every read, with the pipeline one deep: measured at + * 73-88KB per request against a server whose threads sat 86% idle. + * + * Requests are the granularity this connection already negotiated, so + * filling out a partial one costs no extra request, only a fuller one, + * and readahead_expand() folds the growth back into ra->size so the + * next window starts from the wider shape instead of collapsing again. + * Only the trailing edge moves, so nothing already read is refetched, + * and the expansion stops at the first folio already cached -- a + * window genuinely bounded by neighbouring data stays bounded. + * + * Never past what read_ahead_kb authorised for this fd: a mount that + * negotiated large requests does not get to prefetch beyond the + * window the admin allowed. + */ + if (rac->ra) { + unsigned int unit = min_t(unsigned int, max_pages, + rac->ra->ra_pages); + unsigned int nr = readahead_count(rac); + + if (unit > 1 && nr % unit) + readahead_expand(rac, readahead_pos(rac), + (size_t)roundup(nr, unit) << PAGE_SHIFT); + } + /* * Readahead fills the page cache past the range the reader locked, * so take a DLM read grant over the whole window here too. Folios * the server handed out no lock for are folios it will not revoke * when a remote node writes them, and a later read would be served - * from stale cache. Take the grant before any folio is pulled off - * @rac, so the window is either fully covered or not populated. + * from stale cache. Take the grant after the expansion above and + * before any folio is pulled off @rac, so the window that gets + * populated is the window that is covered. * * Speculative pages are not worth serving uncovered: on a failed * request drop the window and let read_pages() clean up the folios @@ -1190,9 +1226,6 @@ static void fuse_readahead(struct readahead_control *rac) return; } - max_pages = min_t(unsigned int, fc->max_pages, - fc->max_read / PAGE_SIZE); - for (;;) { struct fuse_io_args *ia; struct fuse_args_pages *ap; diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index cab0f647f0caff..bb869e68619a83 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -2000,6 +2000,16 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, else fm->sb->s_bdi->ra_pages = min(fm->sb->s_bdi->ra_pages, ra_pages); + /* + * ra_pages caps speculative readahead; io_pages is what the + * device can carry in one go, and mm uses it to let a read + * larger than the readahead window through at the size it + * was actually asked for instead of clamping it to ra_pages. + * For fuse that size is a request, so say so -- otherwise a + * large explicit read or a FADV_WILLNEED gets chopped up by a + * window that was only ever meant to bound guessing. + */ + fm->sb->s_bdi->io_pages = fc->max_pages; fc->minor = arg->minor; fc->max_write = arg->minor < 5 ? 4096 : arg->max_write; fc->max_write = max_t(unsigned, 4096, fc->max_write); From 0dacbfef3bc8b3feb0e3e614846f9a3788e0730b Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 27 Aug 2026 13:53:21 +0200 Subject: [PATCH 23/24] fuse: send the readahead mm built before growing the window fuse_readahead() expanded the window to a whole request before sending anything. readahead_expand() locks one folio per page, so that walk ran with nothing in flight and other readers of the range blocked on its first locked folio until the request finally went out. Send what mm already built, then expand once those folios are in flight. Same window width, different order. readahead_expand() appends at _index + _nr_pages and __readahead_batch() retires a batch by moving both, so expanding with a batch outstanding lands where it should. The grant covers the intended window rather than the realised one, which is the safe direction. Wait on the page the batch handed over rather than looking its index back up in the mapping. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 79 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 15 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 700ecec77d0aff..a94fff0cf2a501 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1160,6 +1160,8 @@ static void fuse_readahead(struct readahead_control *rac) struct inode *inode = rac->mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); unsigned int i, max_pages, nr_pages = 0; + pgoff_t target_end; + bool expanded = false; if (fuse_is_bad(inode)) return; @@ -1168,7 +1170,8 @@ static void fuse_readahead(struct readahead_control *rac) fc->max_read / PAGE_SIZE); /* - * Round the window up to whole requests before anything is fetched. + * Decide how wide this window is meant to be, rounded up to whole + * requests. Nothing is allocated here, only the end offset. * * mm sizes it from per-fd offset arithmetic, so several threads * walking one file at a stride each see their own hits as isolated, @@ -1189,14 +1192,14 @@ static void fuse_readahead(struct readahead_control *rac) * negotiated large requests does not get to prefetch beyond the * window the admin allowed. */ + target_end = readahead_index(rac) + readahead_count(rac); if (rac->ra) { unsigned int unit = min_t(unsigned int, max_pages, rac->ra->ra_pages); unsigned int nr = readahead_count(rac); if (unit > 1 && nr % unit) - readahead_expand(rac, readahead_pos(rac), - (size_t)roundup(nr, unit) << PAGE_SHIFT); + target_end = readahead_index(rac) + roundup(nr, unit); } /* @@ -1204,9 +1207,17 @@ static void fuse_readahead(struct readahead_control *rac) * so take a DLM read grant over the whole window here too. Folios * the server handed out no lock for are folios it will not revoke * when a remote node writes them, and a later read would be served - * from stale cache. Take the grant after the expansion above and - * before any folio is pulled off @rac, so the window that gets - * populated is the window that is covered. + * from stale cache. Take the grant before any folio is pulled off + * @rac, so the window that gets populated is the window that is + * covered. + * + * The grant covers the window this call intends rather than the one + * it ends up with, since the folios past what mm built are not + * allocated yet. readahead_expand() stops at the first folio already + * cached, so a short realisation leaves the tail covered but not + * populated. That is the harmless direction: coverage without cached + * data serves nothing stale, and a folio already cached is one an + * earlier grant already covers. * * Speculative pages are not worth serving uncovered: on a failed * request drop the window and let read_pages() clean up the folios @@ -1218,8 +1229,9 @@ static void fuse_readahead(struct readahead_control *rac) * nothing else. */ if (fc->writeback_cache && fc->dlm) { - int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), - readahead_length(rac), + size_t len = (size_t)(target_end - readahead_index(rac)) + << PAGE_SHIFT; + int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), len, FUSE_PAGE_LOCK_READ); if (err < 0 && err != -ENOSYS) @@ -1229,6 +1241,7 @@ static void fuse_readahead(struct readahead_control *rac) for (;;) { struct fuse_io_args *ia; struct fuse_args_pages *ap; + unsigned int avail; if (fc->num_background >= fc->congestion_threshold && rac->ra->async_size >= readahead_count(rac)) @@ -1238,19 +1251,55 @@ static void fuse_readahead(struct readahead_control *rac) */ break; - nr_pages = readahead_count(rac) - nr_pages; - if (nr_pages > max_pages) - nr_pages = max_pages; - if (nr_pages == 0) - break; + /* + * @_nr_pages still counts the batch handed out last round, + * which __readahead_batch() retires on its next call, so + * subtracting it gives what is built but not yet sent. + */ + avail = readahead_count(rac) - nr_pages; + if (!avail) { + /* + * Everything built is in flight. Grow the window now + * rather than before the first send. + * + * readahead_expand() allocates, inserts and locks one + * folio per page, and every reader wanting this range + * blocks on the first of them until the request that + * fills it completes. Building the whole window up + * front puts that walk, hundreds of folios on a mount + * with large requests, ahead of the first byte anyone + * asked for, with nothing in flight to cover it and + * every other reader fenced behind it. Sending what + * mm built first turns the walk into work done while + * the server is already answering. + * + * _index and _nr_pages are both stale by the + * outstanding batch, so their sum is still the window + * end and the growth lands where it should. + */ + if (expanded || + readahead_index(rac) + readahead_count(rac) >= + target_end) + break; + expanded = true; + readahead_expand(rac, readahead_pos(rac), + (size_t)(target_end - + readahead_index(rac)) + << PAGE_SHIFT); + avail = readahead_count(rac) - nr_pages; + if (!avail) + break; + } + + nr_pages = min(avail, max_pages); ia = fuse_io_alloc(NULL, nr_pages); if (!ia) return; ap = &ia->ap; nr_pages = __readahead_batch(rac, ap->pages, nr_pages); for (i = 0; i < nr_pages; i++) { - fuse_wait_on_page_writeback(inode, - readahead_index(rac) + i); + /* The batch holds a reference, so no lookup needed. */ + wait_on_page_writeback(ap->pages[i]); ap->descs[i].length = PAGE_SIZE; } ap->num_pages = nr_pages; From 6ce78c0d70183252445e529de87e526367d100ae Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 27 Aug 2026 19:19:28 +0200 Subject: [PATCH 24/24] fuse: keep the read lookahead on the inode mm keeps readahead state per open file. Several threads reading one shared file through their own descriptors each carry a separate idea of the stream, so only the first to reach a hole allocates anything and the rest find the folios present and issue nothing. The PG_readahead trigger is cleared by whichever thread reaches it first, and that thread's state usually describes a window another thread built, so page_cache_async_ra() takes its interleaved path, finds no hole within read_ahead_kb and issues nothing. Every window then costs a synchronous miss with all the readers waiting on it. The inode is what those readers share. Once the window is in flight, claim the range past it under fi->lock and send one request for it, so one reader primes it and the others skip it. Mark its first folio so a reader crossing into the range still gives mm a trigger, which now finds the hole this request stops at. Bounded by i_size, skipped when the connection is congested, and covered by the same DLM read grant the window takes. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 145 +++++++++++++++++++++++++++++++++++++++++++++-- fs/fuse/fuse_i.h | 9 +++ 2 files changed, 148 insertions(+), 6 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index a94fff0cf2a501..24195db1c34755 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1155,13 +1155,133 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) fuse_readpages_end(fm, &ap->args, err); } +/* + * Populate and send one request past the window mm asked for. + * + * mm keeps readahead state per open file, so several threads reading one + * shared file through their own descriptors each carry a separate idea of + * the stream. Only the first of them to reach a hole allocates anything; + * the rest find the folios present and return having issued nothing. The + * PG_readahead trigger that should start the next window is cleared by + * whichever thread reaches it first, and that thread's state usually + * describes a window some other thread built, so page_cache_async_ra() + * takes its interleaved path, finds no hole within read_ahead_kb and + * issues nothing at all. The chain dies there and every window after it + * costs a synchronous miss with all the readers waiting on it. + * + * The inode is what those readers actually share, so keep the lookahead + * there instead: one of them claims the range past the window and fills + * it while the others skip it. One request deep is enough to leave a + * fetch outstanding for the window just sent to be consumed against. + * + * Called from inside read_pages(), so the invalidate lock is held shared + * by the readahead that got us here. That is what makes adding folios + * safe against a concurrent invalidate. + */ +static void fuse_readahead_lookahead(struct file *file, struct inode *inode, + pgoff_t start, unsigned int nr) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct address_space *mapping = inode->i_mapping; + gfp_t gfp = readahead_gfp_mask(mapping); + struct fuse_io_args *ia; + struct fuse_args_pages *ap; + loff_t size = i_size_read(inode); + unsigned int i, added = 0; + pgoff_t last; + bool claimed; + + if (!nr || !size) + return; + + /* Speculative work is not worth queueing behind a backlog. */ + if (fc->num_background >= fc->congestion_threshold) + return; + + /* Nothing past the end of the file. */ + last = (size - 1) >> PAGE_SHIFT; + if (start > last) + return; + if (start + nr - 1 > last) + nr = last - start + 1; + + /* + * One claimant per range. A reader that finds the range already + * claimed has nothing to add, and one that jumped elsewhere claims + * afresh, so a re-read from the start is not locked out. + */ + spin_lock(&fi->lock); + claimed = fi->ra_lookahead != start; + if (claimed) + fi->ra_lookahead = start; + spin_unlock(&fi->lock); + if (!claimed) + return; + + /* + * Same grant the window itself takes: folios the server handed out + * no lock for are folios it will not revoke when a remote node + * writes them. + */ + if (fc->writeback_cache && fc->dlm) { + int err = fuse_get_dlm_lock(file, (loff_t)start << PAGE_SHIFT, + (size_t)nr << PAGE_SHIFT, + FUSE_PAGE_LOCK_READ); + + if (err < 0 && err != -ENOSYS) + return; + } + + ia = fuse_io_alloc(NULL, nr); + if (!ia) + return; + ap = &ia->ap; + + for (i = 0; i < nr; i++) { + struct folio *folio = filemap_alloc_folio(gfp, 0); + + if (!folio) + break; + /* + * Stop at the first folio already cached, so the request + * stays contiguous and nothing already held is refetched. + */ + if (filemap_add_folio(mapping, folio, start + i, gfp) < 0) { + folio_put(folio); + break; + } + /* + * Hand mm a trigger it can still use. The reader crossing + * into this range calls page_cache_async_ra(), which looks + * for the first hole ahead and finds the one this request + * stops at, so the next window starts while this one is + * still being consumed. + */ + if (!added) + folio_set_readahead(folio); + /* Freshly allocated, so there is no writeback to wait out. */ + ap->pages[added] = &folio->page; + ap->descs[added].length = PAGE_SIZE; + added++; + } + + if (!added) { + fuse_io_free(ia); + return; + } + + ap->num_pages = added; + fuse_send_readpages(ia, file); +} + static void fuse_readahead(struct readahead_control *rac) { struct inode *inode = rac->mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); - unsigned int i, max_pages, nr_pages = 0; + unsigned int i, max_pages, nr_pages = 0, unit; pgoff_t target_end; - bool expanded = false; + bool expanded = false, complete = false; if (fuse_is_bad(inode)) return; @@ -1192,12 +1312,12 @@ static void fuse_readahead(struct readahead_control *rac) * negotiated large requests does not get to prefetch beyond the * window the admin allowed. */ + unit = max_pages; target_end = readahead_index(rac) + readahead_count(rac); if (rac->ra) { - unsigned int unit = min_t(unsigned int, max_pages, - rac->ra->ra_pages); unsigned int nr = readahead_count(rac); + unit = min_t(unsigned int, max_pages, rac->ra->ra_pages); if (unit > 1 && nr % unit) target_end = readahead_index(rac) + roundup(nr, unit); } @@ -1279,16 +1399,20 @@ static void fuse_readahead(struct readahead_control *rac) */ if (expanded || readahead_index(rac) + readahead_count(rac) >= - target_end) + target_end) { + complete = true; break; + } expanded = true; readahead_expand(rac, readahead_pos(rac), (size_t)(target_end - readahead_index(rac)) << PAGE_SHIFT); avail = readahead_count(rac) - nr_pages; - if (!avail) + if (!avail) { + complete = true; break; + } } nr_pages = min(avail, max_pages); @@ -1305,6 +1429,14 @@ static void fuse_readahead(struct readahead_control *rac) ap->num_pages = nr_pages; fuse_send_readpages(ia, rac->file); } + + /* + * The whole window is in flight, so prime the next one before the + * readers get there. Skipped on a congestion or allocation exit, + * where there is already more queued than the connection wants. + */ + if (complete) + fuse_readahead_lookahead(rac->file, inode, target_end, unit); } static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); @@ -4103,6 +4235,7 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->iocachectr = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); + fi->ra_lookahead = 0; fi->notify_stamp = jiffies; fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index f483c1893b5bc8..1cf4974ddda3ea 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -201,6 +201,15 @@ struct fuse_inode { */ unsigned long notify_stamp; unsigned int notify_interval_ewma; + + /* + * Index most recently claimed by the inode wide read + * lookahead, so several readers walking one file do + * not each populate the same range. Protected by + * fi->lock; regular files only (shares the + * readdir-cache union arm). + */ + pgoff_t ra_lookahead; }; /* readdir cache (directory only) */