From b2daa2e3888cc2a0f1565a2c8de27629f198e3f0 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 25 Jul 2026 12:45:45 +0200 Subject: [PATCH 01/11] fuse: seed DLM grant merging with an interval-tree lookup fuse_dlm_try_merge() locates the first merge candidate by walking from rb_first_cached() until it reaches the region just granted. The walk runs under the write-held cache rwsem on every fuse_dlm_lock_range() call, and the tree it walks holds every cached grant of the inode. Strided writers (IOR hard-write) accumulate grants that cannot merge with each other, so the tree keeps growing and every new grant pays a scan of all grants below it -- quadratic over the run, with fuse_dlm_range_is_locked() readers blocked behind each scan. Seed the merge with fuse_page_it_iter_first() on the region widened by one unit to each side instead; finding the lowest overlapping range is what the interval tree is there for. This also repairs two edge cases of the linear scan: a region starting at offset 0 made 'start - 1' wrap so the scan degenerated and merging was silently skipped, and a region ending at U64_MAX overflowed 'end + 1' in the loop bound, ending the merge after the first range. Both bounds now saturate. Signed-off-by: Horst Birthelmer --- fs/fuse/fuse_dlm_cache.c | 57 ++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 40eda6daf75cae..2ec072b86312f4 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -120,26 +120,25 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, uint64_t end) { struct fuse_dlm_range *range, *next; - struct rb_node *node; + uint64_t first = start ? start - 1 : start; + uint64_t last = end < U64_MAX ? end + 1 : end; if (!cache) return; - /* Find the first range that might need merging */ - range = NULL; - node = rb_first_cached(&cache->ranges); - while (node) { - range = rb_entry(node, struct fuse_dlm_range, rb); - if (range->end >= start - 1) - break; - node = rb_next(node); - } - - if (!range || range->start > end + 1) - return; + /* + * Find the first range that might need merging. Directly adjacent + * ranges can merge, hence the region is widened by one unit to each + * side (saturating at the type bounds). This must stay an + * interval-tree lookup: the tree holds every cached grant of the + * inode and strided writers grow it for the lifetime of the file, + * so seeding the merge by walking from the tree minimum would make + * every new grant cost a full scan. + */ + range = fuse_page_it_iter_first(&cache->ranges, first, last); /* Try to merge ranges in and around the specified region */ - while (range && range->start <= end + 1) { + while (range && range->start <= last) { /* Get next range before we potentially modify the tree */ next = NULL; if (rb_next(&range->rb)) { @@ -150,11 +149,11 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, /* Try to merge with next range if adjacent and same mode */ if (next && range->mode == next->mode && range->end + 1 == next->start) { - /* Merge ranges */ - range->end = next->end; - - /* Remove next from tree */ + /* Merge ranges: re-insert so __subtree_end is updated */ fuse_page_it_remove(next, &cache->ranges); + fuse_page_it_remove(range, &cache->ranges); + range->end = next->end; + fuse_page_it_insert(range, &cache->ranges); kfree(next); /* Continue with the same range */ @@ -188,6 +187,7 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range, *new_range, *next; int lock_mode; + bool covered_to_end = false; int ret = 0; LIST_HEAD(to_lock); LIST_HEAD(to_upgrade); @@ -233,14 +233,17 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, } /* Move current_start past this range */ - current_start = max(current_start, range->end + 1); + if (range->end >= end) + covered_to_end = true; + else + current_start = max(current_start, range->end + 1); /* Move to next range */ range = next; } /* If there's a gap after the last range to the end, extend the range */ - if (current_start <= end) { + if (!covered_to_end && current_start <= end) { new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); if (!new_range) { ret = -ENOMEM; @@ -322,13 +325,17 @@ static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, /* If the hole is at the beginning of the range */ if (start == range->start) { + fuse_page_it_remove(range, &cache->ranges); range->start = end + 1; + fuse_page_it_insert(range, &cache->ranges); goto out; } /* If the hole is at the end of the range */ if (end == range->end) { + fuse_page_it_remove(range, &cache->ranges); range->end = start - 1; + fuse_page_it_insert(range, &cache->ranges); goto out; } @@ -400,10 +407,14 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, break; } else if (start > range->start) { /* Adjust the end of the range */ + fuse_page_it_remove(range, &cache->ranges); range->end = start - 1; + fuse_page_it_insert(range, &cache->ranges); } else if (end < range->end) { /* Adjust the start of the range */ + fuse_page_it_remove(range, &cache->ranges); range->start = end + 1; + fuse_page_it_insert(range, &cache->ranges); } else { /* Complete overlap, remove the range */ fuse_page_it_remove(range, &cache->ranges); @@ -475,6 +486,12 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, return false; } + /* Covered through the end of the requested range? */ + if (range->end >= end) { + up_read(&cache->lock); + return true; + } + /* Move current_start past this range */ current_start = range->end + 1; From 98ccde5facbe3c26d96b180ca2d046b1c40c1cb5 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 23 Jul 2026 10:29:12 +0200 Subject: [PATCH 02/11] fuse: re-validate the DLM grant after waiting on the coherency gate Both cached IO paths request their DLM lock first and then go to sleep on things a NOTIFY invalidate can be holding: the read path blocks on the coherency gate (writer priority), the write path additionally sleeps on a contended i_rwsem. A NOTIFY invalidate running in that window revokes exactly the lock just granted (fuse_dlm_unlock_range()), so the task wakes up and populates or dirties the page cache with no DLM coverage. Close the window without ever sending a FUSE_DLM_WB_LOCK request while holding the gate (a grant that had to wait on an invalidate delivered to this same client would deadlock against our own gate hold): - Drop the lock record under the gate write side in fuse_reverse_inval_inode(), so revocation and page drop are one atomic step with respect to the gate. - After entering the gate read side, re-check the grant against the live lock tree; if it was revoked while we waited, drop the gate, re-request, re-enter and check again. With the revoke now gated, passing the check means the lock cannot go away for the whole gate hold: a revoke arriving mid-operation parks until the IO is done. - Keep the write path's lock request ahead of the inode lock: the round trip must not capture the writer-priority i_rwsem for unbounded cluster-grant latency, and the in-gate re-validation already closes the grant-to-use window. Only O_APPEND moves below the lock, because its range is the current EOF -- stable only under the exclusive inode lock. This also fixes the append range itself: generic_write_checks() rewrites ki_pos to i_size for IOCB_APPEND, so the old 'i_size + ki_pos' double-counted (ki_pos is absolute, not relative) and locked a range disjoint from where the data lands. fuse_get_dlm_lock() now reports whether the grant is recorded, and the re-validation never re-requests a grant that failed, so it cannot spin (the read path seeds this from its pre-gate request instead of discarding that result). A grant the server issued but that could not be recorded (small-allocation -ENOMEM) reports FUSE_DLM_GRANT_UNRECORDED: coverage exists cluster-wide, so failing the IO would be wrong -- it proceeds, it just cannot re-validate. Empty ranges are trivially held, so a zero-length IO neither sends a doomed request nor spins in the retry loops. The write path returns a real failure to the caller instead of dirtying the cache without DLM coverage; only -ENOSYS still degrades to a plain cached write, since it means the server has no DLM at all and clears fc->dlm. The read path keeps falling through unlocked and additionally bounds its retry: a reader-only inode has no force-DIO latch to end a revoke storm, so after a few re-requests the read is served unlocked rather than looping in the kernel for the duration of the storm. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 168 ++++++++++++++++++++++++++++----------- fs/fuse/fuse_dlm_cache.c | 102 +++++++++++++++++------- fs/fuse/fuse_dlm_cache.h | 17 +++- fs/fuse/inode.c | 35 +++++--- 4 files changed, 236 insertions(+), 86 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index f22bb5f0ff6a21..6b0d414b5ced69 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1260,6 +1260,12 @@ static void fuse_readahead(struct readahead_control *rac) static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); +/* + * Bound on re-requesting a revoked DLM grant before a cached read is + * served unlocked; see fuse_cache_read_iter(). + */ +#define FUSE_DLM_READ_RETRIES 3 + static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; @@ -1268,6 +1274,7 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) struct fuse_inode *fi = get_fuse_inode(inode); struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; ssize_t res; + int lock_err = 0; /* * In auto invalidate mode, always update attributes on read. @@ -1285,8 +1292,9 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) /* if we have dlm support acquire a read lock for the area * we are reading from. */ if (fc->writeback_cache && fc->dlm) - fuse_get_dlm_lock(file, iocb->ki_pos, - iov_iter_count(to), FUSE_PAGE_LOCK_READ); + lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, + iov_iter_count(to), + FUSE_PAGE_LOCK_READ); /* * Fence the cache-serving read against a NOTIFY invalidate so we never @@ -1298,11 +1306,44 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) * wb_sem is NULL on non-writeback+dlm mounts (gate inactive). */ if (wb_sem) { + int tries = FUSE_DLM_READ_RETRIES; + +retry: percpu_down_read(wb_sem); if (fuse_inode_force_dio(inode)) { percpu_up_read(wb_sem); return fuse_direct_read_iter(iocb, to); } + /* + * The DLM lock was requested before entering the gate, and + * the NOTIFY invalidate we may just have waited on revokes + * locks under the gate write side. Re-check the grant here + * and re-request with the gate dropped, so a + * FUSE_DLM_WB_LOCK round trip never parks a pending + * invalidate behind our own gate hold. Once the check + * passes the lock cannot go away for the rest of the gate + * hold. A failed or unrecorded request falls through + * unlocked, as before: the retry is taken even then (the + * latch must be re-checked under the re-entered gate), so + * lock_err has to stay sticky across it -- seeded by the + * pre-gate request above -- or a grant that failed would + * be re-requested forever. The retry is also bounded: a + * remote writer can revoke each successful grant before + * the gate is re-entered, and a reader-only inode has no + * force-DIO latch to end such a storm, so after + * FUSE_DLM_READ_RETRIES re-requests the read is served + * unlocked rather than looping without bound. + */ + if (!lock_err && fc->dlm && tries-- > 0 && + !fuse_dlm_lock_is_held(fi, iocb->ki_pos, + iov_iter_count(to), + FUSE_PAGE_LOCK_READ)) { + percpu_up_read(wb_sem); + lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, + iov_iter_count(to), + FUSE_PAGE_LOCK_READ); + goto retry; + } } res = generic_file_read_iter(iocb, to); @@ -1807,6 +1848,26 @@ static void fuse_cache_wr_unlock(struct inode *inode, bool exclusive) inode_unlock_shared(inode); } +/* + * Request the DLM write lock covering a cached write. -ENOSYS cleared + * fc->dlm: the server has no DLM, proceed as a plain cached write. Any + * other failure means the cache would be dirtied without DLM coverage - + * the caller must fail the write instead. A granted-but-unrecorded + * lock (positive return) is covered cluster-wide; proceed, but flag it + * so the in-gate re-validation skips a check an invisible grant could + * never pass. + */ +static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len, + bool *unrecorded) +{ + int err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); + + if (err < 0 && err != -ENOSYS) + return err; + *unrecorded = err > 0; + return 0; +} + static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; @@ -1821,14 +1882,10 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) bool writeback = false; bool wb_guard = false; bool exclusive = true; + bool dlm_unrecorded = false; + loff_t dlm_pos = 0; + size_t dlm_len = 0; - /* - * The inode may have been latched into forced direct IO -- by a - * NOTIFY_INVAL_INODE arriving while this inode is open for writing here - * -- after this write was routed to the cached path but before it took - * any lock. Re-route to the direct path (before taking a DLM lock) so - * we do not repopulate the page cache the latch just dropped. - */ if (fuse_inode_force_dio(inode)) return fuse_direct_write_iter(iocb, from); @@ -1843,61 +1900,78 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) return err; if (!fc->handle_killpriv_v2 || - !setattr_should_drop_suidgid(idmap, file_inode(file))) { + !setattr_should_drop_suidgid(idmap, file_inode(file))) writeback = true; - - /* - * If we have dlm support acquire the lock for the area - * we are writing into. - * dlm lock is only needed as the write is cached and the - * fuse server is not notified otherwise - */ - if (fc->dlm) { - /* - * Note that a file opened with O_APPEND will have - * relative values in ki_pos. This code is here for - * convenience and for libfuse overlay test. - * Filesystems should handle O_APPEND with 'direct io' - * to additionally get the performance benefits of - * 'parallel direct writes'. - */ - loff_t pos = file->f_flags & O_APPEND ? - i_size_read(inode) + iocb->ki_pos : - iocb->ki_pos; - size_t length = iov_iter_count(from); - - fuse_get_dlm_lock(file, pos, length, - FUSE_PAGE_LOCK_WRITE); - } - } } exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); + + /* + * Request the DLM write lock before taking i_rwsem: the request is + * an unbounded cluster round trip, and holding the writer-priority + * rwsem across it would park a truncate -- and behind it every + * later writer -- for the duration. The grant-to-use window this + * leaves open is closed by the in-gate re-validation below. Only + * the append case must wait for the lock: its range depends on + * i_size, which is stable only under the exclusive inode lock. + */ + if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { + dlm_pos = iocb->ki_pos; + dlm_len = iov_iter_count(from); + + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, + &dlm_unrecorded); + if (err) + return err; + } + if (exclusive) inode_lock(inode); else inode_lock_shared(inode); - /* - * The forced-direct-IO latch feature is active under writeback+dlm; - * hold the coherency gate (wb_inval_rwsem) for read across the - * page-cache dirtying so a concurrent NOTIFY_INVAL_INODE -- which takes - * the write side (blocking, with priority) around its invalidate + latch - * set -- cannot strand the folios we are about to write. Re-check the - * latch under it (it may have been set while we blocked on the inode - * lock) and re-route to the direct path if set. Taken before - * task_io_account_write() so a re-route is not double-counted; the DLM - * write lock taken above is harmless as the direct path does its own - * server coordination. - */ + /* note that this small code dup will save us a lot of headache later + * when appends are done concurrently without using parallel direct writes */ + if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { + /* + * An append write lands at the current EOF no matter what + * ki_pos holds: generic_write_checks() rewrites ki_pos to + * i_size for IOCB_APPEND, and i_size is stable here because + * append writes hold the inode lock exclusive. Lock where + * the data will land. + */ + dlm_pos = i_size_read(inode); + dlm_len = iov_iter_count(from); + + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, + &dlm_unrecorded); + if (err) + goto out; + } + wb_guard = !!wb_sem; if (wb_guard) { +retry: percpu_down_read(wb_sem); if (fuse_inode_force_dio(inode)) { percpu_up_read(wb_sem); fuse_cache_wr_unlock(inode, exclusive); return fuse_direct_write_iter(iocb, from); } + if (writeback && fc->dlm && !dlm_unrecorded && + !fuse_dlm_lock_is_held(fi, dlm_pos, dlm_len, + FUSE_PAGE_LOCK_WRITE)) { + percpu_up_read(wb_sem); + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, + &dlm_unrecorded); + if (err) { + /* The gate is already dropped; funnel the + * failure through the one audited exit. */ + wb_guard = false; + goto out; + } + goto retry; + } } err = count = generic_write_checks(iocb, from); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 2ec072b86312f4..30d371bcf6e293 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -510,45 +510,83 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, return true; } +/** + * fuse_dlm_lock_is_held - check that a byte range is covered by a granted lock + * @fi: the fuse inode + * @offset: byte offset into the file (need not be page-aligned) + * @length: length of the region in bytes (need not be page-aligned) + * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE + * + * Re-validation helper for fuse_get_dlm_lock() callers: checks the same + * page-aligned range a fuse_get_dlm_lock() call with these arguments + * requests, against the live lock tree. + */ +bool fuse_dlm_lock_is_held(struct fuse_inode *fi, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) +{ + uint64_t end = (offset + length - 1) | (PAGE_SIZE - 1); + + /* + * An empty range needs no coverage. Reporting it held keeps the + * re-validating IO paths from re-requesting a lock the tree can + * never show (the page-aligned end would invert below). + */ + if (!length) + return true; + + return fuse_dlm_range_is_locked(fi, offset & PAGE_MASK, end, mode); +} + /** * fuse_get_dlm_lock - request a dlm lock from the fuse server * @file: the file being accessed * @offset: byte offset into the file (need not be page-aligned) * @length: length of the region in bytes (need not be page-aligned) * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE + * + * Return: 0 when the range is covered by a recorded grant on return, + * FUSE_DLM_GRANT_UNRECORDED when the server granted the lock but + * recording it failed (covered cluster-wide, invisible to + * fuse_dlm_lock_is_held()), a negative error code otherwise. Callers + * re-validating the grant must not re-request on a nonzero return or + * they would spin. */ -void fuse_get_dlm_lock(struct file *file, loff_t offset, - size_t length, enum fuse_page_lock_mode mode) +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) { struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_mount *fm = ff->fm; - uint64_t end = (offset + length - 1) | (PAGE_SIZE - 1); - - /* note that the offset and length don't have to be page aligned here - * but since we only get here on writeback caching we will send out - * page aligned requests */ - offset &= PAGE_MASK; FUSE_ARGS(args); struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; int err; + /* An empty range needs no lock. */ + if (!length) + return 0; + /* note that this can be run from different processes * at the same time. It is intentionally not protected * since a DLM implementation in the FUSE server should take care - * of any races in lock requests */ - if (fuse_dlm_range_is_locked(fi, offset, end, mode)) - return; /* we already have this area locked */ + * of any races in lock requests. + * The early exit uses the same helper the callers re-validate + * with, so this check and a later fuse_dlm_lock_is_held() can + * never disagree about what counts as covered. */ + if (fuse_dlm_lock_is_held(fi, offset, length, mode)) + return 0; /* we already have this area locked */ memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; - inarg.start = offset; - inarg.end = end; + /* note that the offset and length don't have to be page aligned + * here but since we only get here on writeback caching we will + * send out page aligned requests */ + inarg.start = offset & PAGE_MASK; + inarg.end = (offset + length - 1) | (PAGE_SIZE - 1); inarg.type = (mode == FUSE_PAGE_LOCK_WRITE) ? FUSE_DLM_LOCK_WRITE : FUSE_DLM_LOCK_READ; @@ -564,21 +602,31 @@ void fuse_get_dlm_lock(struct file *file, loff_t offset, if (err == -ENOSYS) { /* fuse server does not support dlm, save the info */ fc->dlm = 0; - return; + return err; } if (err) - return; - else - if (inarg.start < outarg.start || - inarg.end > outarg.end) { - /* fuse server is seriously broken */ - pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", - inarg.start, inarg.end, outarg.start, outarg.end); - fuse_abort_conn(fc); - return; - } else { - /* ignore any errors here, there is no way we can react appropriately */ - fuse_dlm_lock_range(fi, outarg.start, outarg.end, mode); - } + return err; + + if (inarg.start < outarg.start || inarg.end > outarg.end) { + /* fuse server is seriously broken */ + pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", + inarg.start, inarg.end, outarg.start, outarg.end); + fuse_abort_conn(fc); + return -EIO; + } + + /* + * The server granted the lock; record it so + * fuse_dlm_lock_is_held() sees it. A failure to record + * (small-allocation -ENOMEM) does not undo the grant: coverage + * exists cluster-wide, only the local bookkeeping is missing. + * Report that as FUSE_DLM_GRANT_UNRECORDED so callers neither + * fail an IO that is actually covered nor keep re-requesting a + * grant that will not become visible. + */ + if (fuse_dlm_lock_range(fi, outarg.start, outarg.end, mode)) + return FUSE_DLM_GRANT_UNRECORDED; + + return 0; } diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 5c3deaa3536866..b0b16c56e3b0b0 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -17,6 +17,15 @@ struct fuse_inode; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; +/* + * fuse_get_dlm_lock() result: the server granted the lock but recording + * it locally failed, leaving the grant invisible to + * fuse_dlm_lock_is_held(). The IO is covered cluster-wide; the caller + * must proceed without re-validating (a re-request would spin) instead + * of failing the IO. + */ +#define FUSE_DLM_GRANT_UNRECORDED 1 + /* Page cache lock manager */ struct fuse_dlm_cache { /* Lock protecting the tree */ @@ -43,8 +52,12 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, uint64_t end, enum fuse_page_lock_mode mode); +/* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ +bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, + size_t length, enum fuse_page_lock_mode mode); + /* This is the interface to the filesystem */ -void fuse_get_dlm_lock(struct file *file, loff_t offset, - size_t length, enum fuse_page_lock_mode mode); +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode); #endif /* _FS_FUSE_DLM_CACHE_H */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 8167ba55d3552c..0009d3b90b1273 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -858,16 +858,6 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, else pg_end = (offset + len - 1) >> PAGE_SHIFT; - if (fc->dlm && fc->writeback_cache) - /* Invalidate the range exactly as the fuse server requested - * except for the case where it sends -1. - * Note that this can lead to some inconsistencies if - * the fuse server sends unaligned data */ - fuse_dlm_unlock_range(fi, - offset, - pg_end == -1 ? 0 : - (offset + len - 1)); - /* * A data invalidation means another (remote) entity is modifying * the file. Two things happen here: @@ -923,6 +913,23 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, */ percpu_down_write(wb_sem); + /* + * Revoke the DLM lock range under the gate write + * side, atomically with the page drop: gate readers + * re-validate their grant right after entering, and + * a grant that passed that check must stay visible + * for their whole gate hold. + * The range is exactly what the fuse server + * requested except for the case where it sends -1. + * Note that this can lead to some inconsistencies + * if the fuse server sends unaligned data. + */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_unlock_range(fi, + offset, + pg_end == -1 ? 0 : + (offset + len - 1)); + if (hot && has_writer && !fuse_inode_force_dio(inode)) { spin_lock(&fi->lock); @@ -951,6 +958,14 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", nodeid); } else { + /* No gate on this inode (mmapped, DAX, backing or + * non-regular): drop the lock range unserialized, + * as before. */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_unlock_range(fi, + offset, + pg_end == -1 ? 0 : + (offset + len - 1)); invalidate_inode_pages2_range(inode->i_mapping, pg_start, pg_end); } From eb3464fec9d322a3330e4a4952545da644570a54 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 28 Jul 2026 09:42:53 +0200 Subject: [PATCH 03/11] fuse: fix the DLM revoke range of an inode invalidate The NOTIFY_INVAL_INODE revoke computed fuse_dlm_unlock_range(fi, offset, pg_end == -1 ? 0 : offset + len - 1) which is wrong at both degenerate ends: a to-EOF invalidate (len <= 0, e.g. a remote truncate) with offset > 0 becomes the inverted range [offset, 0] and removes nothing, so the revoked grant stays visible to the re-validating IO paths forever -- cached writes with no server-side lock, zero-filled RMW reads; and an invalidate of byte 0 (offset 0, len 1) becomes [0, 0], the "destroy everything" sentinel, wiping every grant of the inode. Map the range in one helper shared by the gated and the ungated branch: to-EOF revokes through U64_MAX, and the bounds widen to page boundaries to match how grants are recorded -- revoking too much only costs a re-request, too little leaves a stale grant. Drop the in-band (0, 0) sentinel: whole-file invalidates walk the normal removal path, release-all is fuse_dlm_cache_release_locks(), and an inverted range is rejected with -EINVAL instead of silently ignored. Signed-off-by: Horst Birthelmer --- fs/fuse/fuse_dlm_cache.c | 15 +++++++-------- fs/fuse/inode.c | 33 +++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 30d371bcf6e293..4714d48e6bc9b1 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -369,8 +369,12 @@ static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, * @start: Start page offset * @end: End page offset * - * Release locks on the specified range of pages. - * Note that if start and end are set to zero the cache is destroyed. + * Release locks on the specified range of pages. An inverted range is + * rejected rather than silently removing nothing: the callers revoke + * coverage, and a revoke that quietly keeps the grant alive would let + * the re-validating IO paths trust a lock the server has taken away. + * To drop every grant use fuse_dlm_cache_release_locks() (there is no + * in-band sentinel range for it). * * Return: 0 on success, negative error code on failure */ @@ -381,14 +385,9 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, struct fuse_dlm_range *range, *next; int ret = 0; - if (!cache) + if (!cache || start > end) return -EINVAL; - if (start == 0 && end == 0) { - fuse_dlm_cache_release_locks(inode); - return 0; - } - down_write(&cache->lock); /* Find all ranges that overlap with [start, end] */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 0009d3b90b1273..5fa78aa490876b 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -818,6 +818,25 @@ static bool fuse_notify_inval_hot(struct fuse_inode *fi) return avg < FUSE_NOTIFY_DIO_INTERVAL; } +/* + * Revoke the DLM grants backing an invalidated byte range. Grants are + * recorded page-aligned, so widen the revoke to page boundaries: dropping + * more than the server invalidated only costs a re-request, dropping less + * would leave a stale grant that fuse_dlm_lock_is_held() keeps trusting. + * len <= 0 means "invalidate to EOF" (see fuse_notify_inval_inode()) and + * revokes through U64_MAX -- it must not become an inverted range, which + * fuse_dlm_unlock_range() rejects without removing anything. + */ +static void fuse_dlm_revoke_inval_range(struct fuse_inode *fi, loff_t offset, + loff_t len) +{ + uint64_t start = (uint64_t)offset & PAGE_MASK; + uint64_t end = len <= 0 ? U64_MAX : + (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); + + fuse_dlm_unlock_range(fi, start, end); +} + int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { @@ -919,16 +938,9 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * re-validate their grant right after entering, and * a grant that passed that check must stay visible * for their whole gate hold. - * The range is exactly what the fuse server - * requested except for the case where it sends -1. - * Note that this can lead to some inconsistencies - * if the fuse server sends unaligned data. */ if (fc->dlm && fc->writeback_cache) - fuse_dlm_unlock_range(fi, - offset, - pg_end == -1 ? 0 : - (offset + len - 1)); + fuse_dlm_revoke_inval_range(fi, offset, len); if (hot && has_writer && !fuse_inode_force_dio(inode)) { @@ -962,10 +974,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * non-regular): drop the lock range unserialized, * as before. */ if (fc->dlm && fc->writeback_cache) - fuse_dlm_unlock_range(fi, - offset, - pg_end == -1 ? 0 : - (offset + len - 1)); + fuse_dlm_revoke_inval_range(fi, offset, len); invalidate_inode_pages2_range(inode->i_mapping, pg_start, pg_end); } From 3523bf681f460e9b6ce197009a04198e5f73c29e Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 28 Jul 2026 09:45:16 +0200 Subject: [PATCH 04/11] fuse: fence mmapped invalidates and local truncates with the coherency gate Two revocation paths bypassed the revoke-under-gate invariant the IO paths re-validate against: - fuse_reverse_inval_inode() skipped the gate once mapping_mapped() turned true, revoking concurrently with gate holders -- but fuse_cache_read_iter()/fuse_cache_write_iter() enter the gate unconditionally, so a single mmap() reopened the race. Keep the gate for mmapped inodes; only the force-DIO latch stays disabled for them (a mapping needs the page cache, and fuse_file_mmap() reverts any latch it races with). - The local truncates in fuse_do_setattr() -- the atomic-O_TRUNC open shortcut and the after-setattr trim -- revoked and dropped the cache with no gate at all, so an already re-validated reader could repopulate the truncated range. Take the gate write side around revoke + drop. This cannot deadlock: both run under exclusive i_rwsem, which no gate holder waits on (the write path takes i_rwsem before the gate, the read path never takes it). Signed-off-by: Horst Birthelmer --- fs/fuse/dir.c | 29 +++++++++++++++++++++++++++++ fs/fuse/inode.c | 19 ++++++++++++------- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 766d369657ba88..8d6ff00c52788a 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2100,11 +2100,26 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, WARN_ON(!(attr->ia_valid & ATTR_SIZE)); WARN_ON(attr->ia_size != 0); if (fc->atomic_o_trunc) { + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + /* * No need to send request to userspace, since actual * truncation has already been done by OPEN. But still * need to truncate page cache. + * + * Revoke and drop under the coherency gate write side, + * like the NOTIFY invalidate path: a gate reader that + * already re-validated its grant must not have the + * lock tree and the cache yanked mid-hold, or it + * would repopulate the truncated range trusting a + * grant that no longer exists. Waiting for gate + * readers here is safe: we hold i_rwsem exclusive, so + * no gate holder can be waiting on it (the write path + * takes i_rwsem before the gate, the read path never + * takes it). */ + if (wb_sem) + percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_cache_release_locks(fi); spin_lock(&fi->lock); @@ -2112,6 +2127,8 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, i_size_write(inode, 0); spin_unlock(&fi->lock); truncate_pagecache(inode, 0); + if (wb_sem) + percpu_up_write(wb_sem); goto out; } file = NULL; @@ -2222,11 +2239,23 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if ((is_truncate || !is_wb) && S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + + /* + * Revoke and drop under the coherency gate write side; see + * the atomic-O_TRUNC branch above. i_rwsem is held + * exclusive here as well (setattr), so waiting out gate + * readers cannot deadlock. + */ + if (wb_sem) + percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1); truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); + if (wb_sem) + percpu_up_write(wb_sem); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 5fa78aa490876b..41b7dc23c8d5a1 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -907,13 +907,17 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * (stale read / lost write). * * The gate (and the average) exist only for writeback+dlm regular - * files, and not while mmapped; elsewhere wb_sem is NULL and the - * invalidate runs unserialized (best-effort), as before. + * files; elsewhere wb_sem is NULL and the invalidate runs + * unserialized (best-effort), as before. An mmapped inode + * keeps the gate -- fuse_cache_read_iter() and + * fuse_cache_write_iter() enter it unconditionally and rely + * on the revoke staying fenced -- but is never latched: + * a mapping needs the page cache, and fuse_file_mmap() + * reverts any latch it races with. */ if (S_ISREG(inode->i_mode) && fc->writeback_cache && fc->dlm && !FUSE_IS_DAX(inode) && - !fuse_inode_backing(fi) && - !mapping_mapped(inode->i_mapping)) + !fuse_inode_backing(fi)) wb_sem = fi->wb_inval_rwsem; if (wb_sem) { @@ -943,6 +947,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, fuse_dlm_revoke_inval_range(fi, offset, len); if (hot && has_writer && + !mapping_mapped(inode->i_mapping) && !fuse_inode_force_dio(inode)) { spin_lock(&fi->lock); if (!list_empty(&fi->write_files)) { @@ -970,9 +975,9 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", nodeid); } else { - /* No gate on this inode (mmapped, DAX, backing or - * non-regular): drop the lock range unserialized, - * as before. */ + /* No gate on this inode (DAX, backing, non-regular, + * or the gate allocation failed): drop the lock + * range unserialized (best-effort), as before. */ if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); invalidate_inode_pages2_range(inode->i_mapping, From 5c94fbbeb3e2d00e0c9c4d75044016b15de6c088 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 28 Jul 2026 09:47:47 +0200 Subject: [PATCH 05/11] fuse: order grant recording against concurrent revokes A FUSE_DLM_WB_LOCK reply and a NOTIFY invalidate are serviced on different threads, so a revoke aimed at the grant a reply carries can be processed before fuse_get_dlm_lock() records it: the revoke finds nothing to remove, and the requester then records an already-dead grant that no later NOTIFY will target -- a permanent false positive for the re-validating IO paths. Add a revocation generation to the lock cache, bumped under the cache lock by every revoke path -- unconditionally, because the racing revoke sees an empty overlap precisely when the grant is in flight. fuse_get_dlm_lock() samples it before sending and records through fuse_dlm_lock_range_gen(), which refuses with -EAGAIN once the generation has moved; the grant is then re-requested instead of recorded, bounded so a revoke storm cannot pin the IO here (past the bound the failure reports like any request failure). Signed-off-by: Horst Birthelmer --- fs/fuse/fuse_dlm_cache.c | 110 +++++++++++++++++++++++++++++++++++---- fs/fuse/fuse_dlm_cache.h | 15 ++++++ 2 files changed, 115 insertions(+), 10 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 4714d48e6bc9b1..960d51e7836a3c 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -31,6 +31,12 @@ struct fuse_dlm_range { #define FUSE_PCACHE_LK_READ 1 /* Shared read lock */ #define FUSE_PCACHE_LK_WRITE 2 /* Exclusive write lock */ +/* + * Bound on re-requesting a grant whose recording lost against a + * concurrent revoke; see fuse_get_dlm_lock(). + */ +#define FUSE_DLM_RECORD_TRIES 3 + /* Interval tree definitions for page ranges */ static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) { @@ -63,6 +69,7 @@ int fuse_dlm_cache_init(struct fuse_inode *inode) init_rwsem(&cache->lock); cache->ranges = RB_ROOT_CACHED; + cache->revoke_gen = 0; return 0; } @@ -84,6 +91,7 @@ void fuse_dlm_cache_release_locks(struct fuse_inode *inode) /* Release all locks */ down_write(&cache->lock); + WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); while ((node = rb_first_cached(&cache->ranges)) != NULL) { range = rb_entry(node, struct fuse_dlm_range, rb); fuse_page_it_remove(range, &cache->ranges); @@ -166,11 +174,13 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, } /** - * fuse_dlm_lock_range - Lock a range of pages + * __fuse_dlm_lock_range - Lock a range of pages * @cache: The page cache * @start: Start page offset * @end: End page offset * @mode: Lock mode (read or write) + * @genp: If non-NULL, the revocation generation sampled before the grant + * was requested; recording fails with -EAGAIN if it has moved * * Add a locked range on the specified range of pages. * If parts of the range are already locked, only add the remaining parts. @@ -181,8 +191,9 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, * * Return: 0 on success, negative error code on failure */ -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode, + const uint64_t *genp) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range, *new_range, *next; @@ -202,6 +213,17 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, down_write(&cache->lock); + /* + * A revoke was processed after @genp was sampled; the grant this + * record carries may be the very one it targeted (a revoke of a + * not-yet-recorded grant removes nothing and would never be + * retried). Refuse, the caller re-requests. + */ + if (genp && cache->revoke_gen != *genp) { + up_write(&cache->lock); + return -EAGAIN; + } + /* Find all ranges that overlap with [start, end] */ range = fuse_page_it_iter_first(&cache->ranges, start, end); while (range) { @@ -297,6 +319,35 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, return ret; } +int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) +{ + return __fuse_dlm_lock_range(inode, start, end, mode, NULL); +} + +int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode, + uint64_t gen) +{ + return __fuse_dlm_lock_range(inode, start, end, mode, &gen); +} + +/** + * fuse_dlm_revoke_gen - sample the revocation generation + * @inode: the fuse inode + * + * Sampled before a FUSE_DLM_WB_LOCK request leaves the client. The + * reply and a NOTIFY revoke can be serviced on different threads, so a + * revoke may be processed between the reply arriving and its grant + * being recorded. fuse_dlm_lock_range_gen() re-checks the generation + * under the cache lock and refuses to record a grant such a revoke may + * have already killed. + */ +uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode) +{ + return READ_ONCE(inode->dlm_locked_areas.revoke_gen); +} + /** * fuse_dlm_punch_hole - Punch a hole in a locked range * @cache: The page cache @@ -390,6 +441,14 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, down_write(&cache->lock); + /* + * Unconditional, even when nothing overlaps: the revoke racing + * with an in-flight grant finds an empty tree precisely because + * the grant is not recorded yet, and the bump is what makes the + * recording side notice (see fuse_dlm_lock_range_gen()). + */ + WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); + /* Find all ranges that overlap with [start, end] */ range = fuse_page_it_iter_first(&cache->ranges, start, end); while (range) { @@ -562,12 +621,15 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, FUSE_ARGS(args); struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; + uint64_t gen; + int tries = FUSE_DLM_RECORD_TRIES; int err; /* An empty range needs no lock. */ if (!length) return 0; +restart: /* note that this can be run from different processes * at the same time. It is intentionally not protected * since a DLM implementation in the FUSE server should take care @@ -578,6 +640,16 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, if (fuse_dlm_lock_is_held(fi, offset, length, mode)) return 0; /* we already have this area locked */ + /* + * Sample the revocation generation before the request leaves. + * The reply and a NOTIFY revoke are serviced on different + * threads, so a revoke aimed at the grant this request returns + * can be processed before the grant is recorded below -- + * recording it anyway would resurrect a dead grant that no later + * NOTIFY will ever remove. + */ + gen = fuse_dlm_revoke_gen(fi); + memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; @@ -617,14 +689,32 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, /* * The server granted the lock; record it so - * fuse_dlm_lock_is_held() sees it. A failure to record - * (small-allocation -ENOMEM) does not undo the grant: coverage - * exists cluster-wide, only the local bookkeeping is missing. - * Report that as FUSE_DLM_GRANT_UNRECORDED so callers neither - * fail an IO that is actually covered nor keep re-requesting a - * grant that will not become visible. + * fuse_dlm_lock_is_held() sees it. */ - if (fuse_dlm_lock_range(fi, outarg.start, outarg.end, mode)) + err = fuse_dlm_lock_range_gen(fi, outarg.start, outarg.end, mode, gen); + if (err == -EAGAIN) { + /* + * A revoke was processed while the request was in flight; + * the grant may already be dead, so re-request instead of + * recording it. Bounded: a revoke storm must not pin the + * IO here -- past the bound the failure is reported like + * any other request failure (the write path fails the + * write, the read path serves unlocked). + */ + if (--tries) + goto restart; + return -EAGAIN; + } + + /* + * A failure to record (small-allocation -ENOMEM) does not undo + * the grant: coverage exists cluster-wide, only the local + * bookkeeping is missing. Report that as + * FUSE_DLM_GRANT_UNRECORDED so callers neither fail an IO that + * is actually covered nor keep re-requesting a grant that will + * not become visible. + */ + if (err) return FUSE_DLM_GRANT_UNRECORDED; return 0; diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index b0b16c56e3b0b0..647a8c37c36095 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -32,6 +32,13 @@ struct fuse_dlm_cache { struct rw_semaphore lock; /* Interval tree of locked ranges */ struct rb_root_cached ranges; + /* + * Bumped under @lock by every revocation + * (fuse_dlm_unlock_range(), fuse_dlm_cache_release_locks()); + * lets fuse_get_dlm_lock() order recording a reply's grant + * against revokes processed while the reply was in flight. + */ + uint64_t revoke_gen; }; /* Initialize a page cache lock manager */ @@ -44,6 +51,14 @@ void fuse_dlm_cache_release_locks(struct fuse_inode *inode); int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, uint64_t end, enum fuse_page_lock_mode mode); +/* As above, but refuse (-EAGAIN) if a revoke ran since @gen was sampled */ +int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode, + uint64_t gen); + +/* Sample the revocation generation (see fuse_dlm_lock_range_gen()) */ +uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode); + /* Unlock a range of pages */ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, uint64_t end); From ed55efd1bb9d3dda6d42524b8d4d5309078e1b66 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 30 Jul 2026 14:22:16 +0200 Subject: [PATCH 06/11] fuse: complete pinned-header sends in ring task context On the pinned-header send path fuse_uring_dispatch_ent() calls io_uring_cmd_done() directly from the request submitter's context with IO_URING_F_UNLOCKED. Every fuse-uring command is marked cancelable, so io_uring_cmd_del_cancelable() has to take ctx->uring_lock from that foreign task on every request. The ring task holds this mutex for the whole ->uring_cmd() issue path (io_uring_enter() submission), where it also wakes the submitter of the request it just committed - before releasing the lock. The freshly woken submitter usually preempts the ring task on the same CPU, and its next dispatch then blocks on the very mutex its victim still holds. The preempted owner is merely runnable and gets no wakeup boost, so under CPU pressure this convoy costs milliseconds per request while the daemon's actual work is a few microseconds. Keep the copies into the pinned pages in the submitter's context - that is the point of the pinning - but defer the command completion to ring task task-work, like the non-pinned path already does. There io_uring_cmd_del_cancelable() runs under the task-work batch's already held uring_lock, and the submitter no longer touches ctx->uring_lock at all on the fast path (only the rare copy-failure fallback still can). Signed-off-by: Horst Birthelmer --- fs/fuse/dev_uring.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index f235417dad466b..06800114f8fb11 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1500,6 +1500,26 @@ static void fuse_uring_send_in_task(struct io_uring_cmd *cmd, fuse_uring_send(ent, cmd, err, issue_flags); } +/* + * The request was already copied to the ring buffer in the submitter's + * context, only the io_uring cmd completion is left to do. + * io_uring_cmd_done() must not run in the submitter's context as it would + * have to take ctx->uring_lock (io_uring_cmd_del_cancelable()) - a mutex + * the ring task holds across its whole submission path and frequently gets + * preempted under while the just-woken submitter runs. + */ +static void fuse_uring_send_prepared_in_task(struct io_uring_cmd *cmd, + unsigned int issue_flags) +{ + struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd); + int err = 0; + + if (unlikely(issue_flags & IO_URING_F_TASK_DEAD)) + err = -ECANCELED; + + fuse_uring_send(ent, cmd, err, issue_flags); +} + static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring, bool background) { @@ -1599,7 +1619,9 @@ static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg) IO_URING_F_UNLOCKED); return; } - fuse_uring_send(ent, cmd, 0, IO_URING_F_UNLOCKED); + uring_cmd_set_ring_ent(cmd, ent); + io_uring_cmd_complete_in_task(cmd, + fuse_uring_send_prepared_in_task); } } From 33b1bc256251b46dbfe053cb65b44376e6d25aa8 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 30 Jul 2026 14:23:13 +0200 Subject: [PATCH 07/11] fuse: defer uring request completion out of the uring_lock section fuse_uring_commit_fetch() runs from the ->uring_cmd() issue path with ctx->uring_lock held by io_uring_enter(). Ending the committed request there means wake_up(&req->waitq) runs while the ring task still holds the mutex, and the freshly woken submitter typically preempts the ring task on the same CPU right away - leaving the lock held by a merely runnable task for however long the runqueue is backed up. wake_up_sync() would not help here: WF_SYNC only biases task placement in select_task_rq_fair(), check_preempt_wakeup_fair() never looks at it, so it cannot stop the wakee from preempting the waker. Instead defer fuse_request_end() to task work on the ring task. It runs after the submission path has dropped uring_lock - at latest on return to userspace, typically at io_cqring_wait() entry via io_run_task_work() - so the ring task finishes its critical section and the entry recycling in fuse_uring_next_fuse_req() before the submitter gets the CPU. This also moves the request ->end() callbacks off the locked section, shortening the uring_lock hold time. Fall back to completing in place when the command was issued from an unlocked context (io-wq punt) or task_work_add() fails because the task is exiting. Signed-off-by: Horst Birthelmer --- fs/fuse/dev_uring.c | 44 +++++++++++++++++++++++++++++++++++++++++--- fs/fuse/fuse_i.h | 2 ++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 06800114f8fb11..71249d1119533c 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -12,6 +12,7 @@ #include #include #include +#include static bool __read_mostly enable_uring; module_param(enable_uring, bool, 0644); @@ -99,8 +100,8 @@ static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) } } -static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, - int error) +static void __fuse_uring_req_end(struct fuse_ring_ent *ent, + struct fuse_req *req, int error) { struct fuse_ring_queue *queue = ent->queue; struct fuse_ring *ring = queue->ring; @@ -123,6 +124,43 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, req->out.h.error = error; clear_bit(FR_SENT, &req->flags); +} + +static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, + int error) +{ + __fuse_uring_req_end(ent, req, error); + fuse_request_end(req); +} + +static void fuse_uring_req_end_work(struct callback_head *work) +{ + struct fuse_req *req = container_of(work, struct fuse_req, + ring_end_work); + + fuse_request_end(req); +} + +/* + * On the commit path ->uring_cmd() runs with ctx->uring_lock held by + * io_uring_enter(). fuse_request_end() wakes the request submitter, which + * typically preempts the ring task right away (same CPU) - while the mutex + * is still held. Defer the completion to task work, which runs once the + * submission path has released the lock (in io_cqring_wait() or on return + * to userspace), so the ring task can finish its critical section first. + */ +static void fuse_uring_req_end_deferred(struct fuse_ring_ent *ent, + struct fuse_req *req, int error, + unsigned int issue_flags) +{ + __fuse_uring_req_end(ent, req, error); + + if (!(issue_flags & IO_URING_F_UNLOCKED)) { + init_task_work(&req->ring_end_work, fuse_uring_req_end_work); + if (!task_work_add(current, &req->ring_end_work, TWA_RESUME)) + return; + } + fuse_request_end(req); } @@ -1059,7 +1097,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, err = fuse_uring_copy_from_ring(ring, req, ent); out: - fuse_uring_req_end(ent, req, err); + fuse_uring_req_end_deferred(ent, req, err, issue_flags); } /* diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index f84b65f151d2bc..4806a1e015c57a 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -558,6 +558,8 @@ struct fuse_req { #ifdef CONFIG_FUSE_IO_URING void *ring_entry; void *ring_queue; + /** Defers fuse_request_end() to the ring task's task work */ + struct callback_head ring_end_work; #endif /** When (in jiffies) the request was created */ unsigned long create_time; From 8571475b8e3385202ec7b7fe1ebdab5300c540c8 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 11 Aug 2026 14:46:18 +0200 Subject: [PATCH 08/11] fuse: default the request timeout to the hung task timeout A connection whose server stops answering has no timeout at all unless the server asked for one at FUSE_INIT or the admin set one of the fs/fuse sysctls. The requesting task itself waits killably and is therefore skipped by the hung task detector, but everything queueing up behind it -- inode locks, folio locks, writeback, unmount -- sleeps uninterruptibly and does get reported, or panics the box where hung_task_panic is set. The result is a machine that logs (or dies on) a hang the filesystem could have turned into an I/O error by itself. Fall back to the hung task timeout in that case, i.e. exactly where the timeout is not otherwise set: a request outstanding for as long as the system is willing to tolerate a stuck task aborts the connection. The timeout checks only run every FUSE_TIMEOUT_TIMER_FREQ seconds, so aim one period below hung_task_timeout_secs to abort before the detector reports on the waiters. hung_task_timeout_secs of 0 (detector disabled, also the value used when CONFIG_DETECT_HUNG_TASK is off) keeps the old behaviour of no timeout. The precedence of an explicit configuration is unchanged: the server's FUSE_INIT timeout, then default_request_timeout, then the clamp against max_request_timeout all take effect first, and the fallback is only consulted when all of them leave the timeout at 0. The value is read once at connection setup, like every other timeout source. Signed-off-by: Horst Birthelmer --- Documentation/admin-guide/sysctl/fs.rst | 6 +++++ fs/fuse/fuse_i.h | 6 +++++ fs/fuse/inode.c | 33 ++++++++++++++++++++++--- fs/fuse/sysctl.c | 6 +---- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/Documentation/admin-guide/sysctl/fs.rst b/Documentation/admin-guide/sysctl/fs.rst index 6c54718c9d04ba..89d645546f34d0 100644 --- a/Documentation/admin-guide/sysctl/fs.rst +++ b/Documentation/admin-guide/sysctl/fs.rst @@ -367,6 +367,12 @@ greater than max_request_timeout, the system will use max_request_timeout as the timeout. 0 indicates no max request timeout. The maximum value that can be set is 65535. +If the server did not specify a timeout at mount and both of the above are set +to 0, the timeout is derived from ``/proc/sys/kernel/hung_task_timeout_secs`` +instead, so that a server that stops answering aborts the connection rather than +leaving the waiters around for the hung task detector to report on. Setting +hung_task_timeout_secs to 0 disables this fallback as well. + For timeouts, if the server does not respond to the request by the time the set timeout elapses, then the connection to the fuse server will be aborted. Please note that the timeouts are not 100% precise (eg you may set 60 seconds but diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 4806a1e015c57a..20ae3902c369b2 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -53,6 +53,12 @@ /* Frequency (in seconds) of request timeout checks, if opted into */ #define FUSE_TIMEOUT_TIMER_FREQ 15 +/* + * Upper bound (in seconds) for request timeouts. fuse_init_out request + * timeouts are u16, and this goes up to ~18 hours, which is plenty. + */ +#define FUSE_REQ_TIMEOUT_LIMIT 65535 + /** Frequency (in jiffies) of request timeout checks, if opted into */ extern const unsigned long fuse_timeout_timer_freq; diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 41b7dc23c8d5a1..19ffa71c72e207 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1688,11 +1689,31 @@ static void set_request_timeout(struct fuse_conn *fc, unsigned int timeout) fuse_timeout_timer_freq); } -static void init_server_timeout(struct fuse_conn *fc, unsigned int timeout) +/* + * Timeout to use when neither the server nor the admin asked for one: keep it + * in sync with the hung task detector, which is the system wide notion of "this + * has been stuck for too long". Returns 0 if the detector is disabled. + */ +static unsigned int hung_task_req_timeout(void) { - if (!timeout && !fuse_max_req_timeout && !fuse_default_req_timeout) - return; + unsigned long timeout = sysctl_hung_task_timeout_secs; + if (!timeout) + return 0; + + /* + * Requests are only checked every FUSE_TIMEOUT_TIMER_FREQ seconds, so + * aim one period below the hung task timeout in order to abort the + * connection before the detector reports on the waiters. + */ + if (timeout > FUSE_TIMEOUT_TIMER_FREQ) + timeout -= FUSE_TIMEOUT_TIMER_FREQ; + + return min_t(unsigned long, timeout, FUSE_REQ_TIMEOUT_LIMIT); +} + +static void init_server_timeout(struct fuse_conn *fc, unsigned int timeout) +{ if (!timeout) timeout = fuse_default_req_timeout; @@ -1703,6 +1724,12 @@ static void init_server_timeout(struct fuse_conn *fc, unsigned int timeout) timeout = fuse_max_req_timeout; } + if (!timeout) + timeout = hung_task_req_timeout(); + + if (!timeout) + return; + timeout = max(FUSE_TIMEOUT_TIMER_FREQ, timeout); set_request_timeout(fc, timeout); diff --git a/fs/fuse/sysctl.c b/fs/fuse/sysctl.c index e2d921abcb8832..107ecae0e4f9dd 100644 --- a/fs/fuse/sysctl.c +++ b/fs/fuse/sysctl.c @@ -13,11 +13,7 @@ static struct ctl_table_header *fuse_table_header; /* Bound by fuse_init_out max_pages, which is a u16 */ static unsigned int sysctl_fuse_max_pages_limit = 65535; -/* - * fuse_init_out request timeouts are u16. - * This goes up to ~18 hours, which is plenty for a timeout. - */ -static unsigned int sysctl_fuse_req_timeout_limit = 65535; +static unsigned int sysctl_fuse_req_timeout_limit = FUSE_REQ_TIMEOUT_LIMIT; static const struct ctl_table fuse_sysctl_table[] = { { From 2b6ed886e6488cbd29ef4e284c30b5b521b7c567 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Wed, 12 Aug 2026 12:08:30 +0200 Subject: [PATCH 09/11] fuse: retry grant recording until it wins against revokes fuse_get_dlm_lock() re-requests the DLM lock when fuse_dlm_lock_range_gen() returns -EAGAIN, i.e. a revoke was processed while the grant request was in flight and the grant it returned may already be dead. That restart was bounded by FUSE_DLM_RECORD_TRIES, and past the bound the function returned -EAGAIN. Reporting that as a request failure is wrong: no one else holds the range at that point, the caller simply lost a race with a revoke, and the write path turns the error into a failed write. Retry unconditionally instead. Every pass issues a fresh FUSE_DLM_WB_LOCK round trip to the server, so a revoke storm throttles the loop rather than spinning it, and the loop ends as soon as one grant survives long enough to be recorded. Drop the now-unused bound and its counter. Signed-off-by: Horst Birthelmer --- fs/fuse/fuse_dlm_cache.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 960d51e7836a3c..b0b17cbd3c3f7a 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -31,12 +31,6 @@ struct fuse_dlm_range { #define FUSE_PCACHE_LK_READ 1 /* Shared read lock */ #define FUSE_PCACHE_LK_WRITE 2 /* Exclusive write lock */ -/* - * Bound on re-requesting a grant whose recording lost against a - * concurrent revoke; see fuse_get_dlm_lock(). - */ -#define FUSE_DLM_RECORD_TRIES 3 - /* Interval tree definitions for page ranges */ static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) { @@ -622,7 +616,6 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; uint64_t gen; - int tries = FUSE_DLM_RECORD_TRIES; int err; /* An empty range needs no lock. */ @@ -696,14 +689,14 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, /* * A revoke was processed while the request was in flight; * the grant may already be dead, so re-request instead of - * recording it. Bounded: a revoke storm must not pin the - * IO here -- past the bound the failure is reported like - * any other request failure (the write path fails the - * write, the read path serves unlocked). + * recording it. Retry until a grant survives long enough to + * be recorded: giving up here would hand the caller an error + * for a range no one else holds, and the write path turns + * that into a failed write. Each pass makes a fresh server + * round trip, so a revoke storm throttles this loop rather + * than spinning it. */ - if (--tries) - goto restart; - return -EAGAIN; + goto restart; } /* From ac617b0b037f411433322f570ff3709cab057dbe Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Wed, 12 Aug 2026 16:59:59 +0200 Subject: [PATCH 10/11] fuse: gate the notify-driven direct-IO latch behind a module parameter fuse_reverse_inval_inode() latches an inode into direct IO when a remote writer keeps invalidating a file that is also open for writing here. That trades the writeback cache away for as long as the latch holds, which only pays off on workloads that actually see such invalidation storms. Make it opt-in through a new 'enable_notify_dio' module parameter, default off. FUSE_I_FORCE_DIO is set in exactly one place, so gating that single site is enough: every other reference only tests or clears the bit, and with the bit never set those paths behave as they did before the latch existed. The moving average is still folded on every invalidation while the parameter is off, so enabling it at runtime takes effect on the next storm instead of after a warm-up. Clearing it stops new latches but leaves already-latched inodes to run out on the usual exits (last writer closes, or mmap). Signed-off-by: Horst Birthelmer --- fs/fuse/inode.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 19ffa71c72e207..ead6ad3e5bdbcf 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -42,6 +42,19 @@ bool __read_mostly enable_large_folios = true; module_param(enable_large_folios, bool, 0644); MODULE_PARM_DESC(enable_large_folios, "Enable large folios support"); +/* + * Gate for the notify-driven direct-IO latch (see + * fuse_reverse_inval_inode()): when a remote writer keeps invalidating a + * file that is also open for writing here, the inode is switched to + * direct IO until its last writer closes. Off by default -- it trades + * the writeback cache away for the duration, which only pays off on + * workloads that actually see such storms. + */ +static bool __read_mostly enable_notify_dio; +module_param(enable_notify_dio, bool, 0644); +MODULE_PARM_DESC(enable_notify_dio, + "Latch a contended inode to direct IO on an invalidation notify storm"); + static struct kmem_cache *fuse_inode_cachep; struct list_head fuse_conn_list; DEFINE_MUTEX(fuse_mutex); @@ -905,7 +918,13 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * writer closes or it is mmapped. When latched, drop the whole * mapping rather than just the notified range, or dirty folios * outside it would be invisible to the forced direct reads - * (stale read / lost write). + * (stale read / lost write). Latching is opt-in via the + * enable_notify_dio module parameter and off by default; the + * average is kept up to date either way, so enabling it at + * runtime takes effect on the next storm rather than after a + * warm-up. Clearing it at runtime stops new latches but lets + * already-latched inodes run out on the usual exits (last + * writer closes, or mmap). * * The gate (and the average) exist only for writeback+dlm regular * files; elsewhere wb_sem is NULL and the invalidate runs @@ -947,7 +966,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); - if (hot && has_writer && + if (enable_notify_dio && hot && has_writer && !mapping_mapped(inode->i_mapping) && !fuse_inode_force_dio(inode)) { spin_lock(&fi->lock); From a53417f5b780a45b69c6cdc79c54f91fc4f074c5 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Tue, 11 Aug 2026 12:16:57 -0700 Subject: [PATCH 11/11] fuse: re-decide the write lock mode after the DLM probe fuse_cache_write_iter() picks between the exclusive and the relaxed shared inode lock with fuse_cache_wr_exclusive_lock(), which returns "shared" only when fc->dlm is set. Since "fuse: re-validate the DLM grant after waiting on the coherency gate" that decision is made before the DLM write lock is requested, and the request itself can clear fc->dlm: a server that does not implement FUSE_DLM_WB_LOCK answers -ENOSYS, which fuse_get_dlm_lock() handles by clearing fc->dlm and reporting success. The write then proceeds in a state that was unreachable before: the shared lock was chosen believing DLM was active, but DLM is now known to be absent. That combination is not benign. The shared path claims the i_size extension up front so iomap never updates i_size itself, which also stops iomap_block_needs_zeroing() from ever firing for the write's own range; the zero-fill that compensates for it in fuse_iomap_read_folio_range() is gated on fc->dlm and so no longer runs. An expanding write therefore falls through to fuse_do_readfolio() and sends a READ for a range past EOF that cannot hold data. Against a file the client opened write-only the server fails that read -- passthrough_hp returns EBADF -- and the write fails with it. Re-evaluate the lock mode after the request, while no lock is held yet, so a server without DLM support gets the exclusive path and iomap's own beyond-EOF zeroing back. This showed up as generic/105, 123, 215, 246, 378, 423, 519 and 597 all failing with EBADF on the first write to a newly created file, and bisected to the commit named above. Signed-off-by: Allison Henderson (cherry picked from commit 4a9ddcf9495bc08241f2ad8c4e465bcf29ef60a7) --- fs/fuse/file.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 6b0d414b5ced69..df8cbe9d6e435f 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1923,6 +1923,21 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) &dlm_unrecorded); if (err) return err; + + /* + * The request above may have found that the server has no DLM + * at all, in which case it cleared fc->dlm. The relaxed shared + * lock was chosen just before, while fc->dlm still read 1, and + * it is only sound under DLM: the shared path claims the i_size + * extension up front, which stops iomap from zeroing beyond + * EOF, and the zero-fill that replaces it in + * fuse_iomap_read_folio_range() is itself gated on fc->dlm. + * Left as chosen, an expanding write would fall through to a + * READ of a range that cannot hold data -- which fails outright + * on a handle the client opened write-only. Re-decide now, + * while no lock is held yet. + */ + exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); } if (exclusive)