From 49cad57274551baabc7071c252af0b2dcf9638a0 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Wed, 2 Apr 2025 16:17:23 +0200 Subject: [PATCH 01/77] fuse: fine-grained request ftraces Rename trace_fuse_request_send to trace_fuse_request_enqueue Add trace_fuse_request_send Add trace_fuse_request_bg_enqueue Add trace_fuse_request_enqueue This helps to track entire request time and time in different queues. Signed-off-by: Bernd Schubert (imported from commit 4a7f14274fc223e50b36f428e1b6acd661b73f53) --- fs/fuse/dev.c | 6 ++++++ fs/fuse/dev_uring.c | 1 + fs/fuse/fuse_trace.h | 51 +++++++++++++++++++++++++++++++++----------- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 0b0241f47170d4..c339ea2c87ade1 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -418,6 +418,9 @@ static void fuse_send_one(struct fuse_iqueue *fiq, struct fuse_req *req) req->in.h.len = sizeof(struct fuse_in_header) + fuse_len_args(req->args->in_numargs, (struct fuse_arg *) req->args->in_args); + + /* enqueue, as it is send to "fiq->ops queue" */ + trace_fuse_request_enqueue(req); fiq->ops->send_req(fiq, req); } @@ -732,6 +735,8 @@ static int fuse_request_queue_background(struct fuse_req *req) } __set_bit(FR_ISREPLY, &req->flags); + trace_fuse_request_bg_enqueue(req); + #ifdef CONFIG_FUSE_IO_URING if (fuse_uring_ready(fc)) return fuse_request_queue_background_uring(fc, req); @@ -1467,6 +1472,7 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, clear_bit(FR_PENDING, &req->flags); list_del_init(&req->list); spin_unlock(&fiq->lock); + trace_fuse_request_send(req); args = req->args; reqsize = req->in.h.len; diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 3a38b61aac26f7..5a22328c078a73 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1207,6 +1207,7 @@ static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, ent->cmd = NULL; spin_unlock(&queue->lock); + trace_fuse_request_send(ent->fuse_req); io_uring_cmd_done(cmd, ret, issue_flags); } diff --git a/fs/fuse/fuse_trace.h b/fs/fuse/fuse_trace.h index bbe9ddd8c71696..393c630e772635 100644 --- a/fs/fuse/fuse_trace.h +++ b/fs/fuse/fuse_trace.h @@ -77,30 +77,55 @@ OPCODES #define EM(a, b) {a, b}, #define EMe(a, b) {a, b} -TRACE_EVENT(fuse_request_send, +#define FUSE_REQ_TRACE_FIELDS \ + __field(dev_t, connection) \ + __field(uint64_t, unique) \ + __field(enum fuse_opcode, opcode) \ + __field(uint32_t, len) \ + +#define FUSE_REQ_TRACE_ASSIGN(req) \ + do { \ + __entry->connection = req->fm->fc->dev; \ + __entry->unique = req->in.h.unique; \ + __entry->opcode = req->in.h.opcode; \ + __entry->len = req->in.h.len; \ + } while (0) + + +TRACE_EVENT(fuse_request_enqueue, TP_PROTO(const struct fuse_req *req), + TP_ARGS(req), + TP_STRUCT__entry(FUSE_REQ_TRACE_FIELDS), + TP_fast_assign(FUSE_REQ_TRACE_ASSIGN(req)), + TP_printk("connection %u req %llu opcode %u (%s) len %u ", + __entry->connection, __entry->unique, __entry->opcode, + __print_symbolic(__entry->opcode, OPCODES), __entry->len) +); + +TRACE_EVENT(fuse_request_bg_enqueue, + TP_PROTO(const struct fuse_req *req), TP_ARGS(req), + TP_STRUCT__entry(FUSE_REQ_TRACE_FIELDS), + TP_fast_assign(FUSE_REQ_TRACE_ASSIGN(req)), - TP_STRUCT__entry( - __field(dev_t, connection) - __field(uint64_t, unique) - __field(enum fuse_opcode, opcode) - __field(uint32_t, len) - ), + TP_printk("connection %u req %llu opcode %u (%s) len %u ", + __entry->connection, __entry->unique, __entry->opcode, + __print_symbolic(__entry->opcode, OPCODES), __entry->len) +); - TP_fast_assign( - __entry->connection = req->fm->fc->dev; - __entry->unique = req->in.h.unique; - __entry->opcode = req->in.h.opcode; - __entry->len = req->in.h.len; - ), +TRACE_EVENT(fuse_request_send, + TP_PROTO(const struct fuse_req *req), + TP_ARGS(req), + TP_STRUCT__entry(FUSE_REQ_TRACE_FIELDS), + TP_fast_assign(FUSE_REQ_TRACE_ASSIGN(req)), TP_printk("connection %u req %llu opcode %u (%s) len %u ", __entry->connection, __entry->unique, __entry->opcode, __print_symbolic(__entry->opcode, OPCODES), __entry->len) ); + TRACE_EVENT(fuse_request_end, TP_PROTO(const struct fuse_req *req), From c8f3d45312d137f2185c08ca87004ade98c38b7d Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Wed, 8 Jan 2025 16:10:27 +0100 Subject: [PATCH 02/77] fuse: {uring} Pin the user buffer This is to allow copying into the buffer from the application without the need to copy in ring context (and with that, the need that the ring task is active in kernel space). Signed-off-by: Bernd Schubert (cherry picked from commit 43d1a63dec17d928609fb9725ac4ab9d6e09803f) (imported from commit ea01f94a55f91fa48cb3a0304b1e41a92707539a) --- fs/fuse/dev.c | 9 ++ fs/fuse/dev_uring.c | 209 +++++++++++++++++++++++++++++++++++++++--- fs/fuse/dev_uring_i.h | 4 + fs/fuse/fuse_dev_i.h | 2 + 4 files changed, 211 insertions(+), 13 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index c339ea2c87ade1..1f107b160778fb 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -917,6 +917,15 @@ static int fuse_copy_fill(struct fuse_copy_state *cs) cs->pipebufs++; cs->nr_segs++; } + } else if (cs->ring.pages) { + cs->pg = cs->ring.pages[cs->ring.page_idx++]; + /* + * non stricly needed, just to avoid a uring exception in + * fuse_copy_finish + */ + get_page(cs->pg); + cs->len = PAGE_SIZE; + cs->offset = 0; } else { size_t off; err = iov_iter_get_pages2(cs->iter, &page, PAGE_SIZE, 1, &off); diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 5a22328c078a73..e24b87f1df745d 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -11,6 +11,7 @@ #include #include +#include static bool __read_mostly enable_uring; module_param(enable_uring, bool, 0644); @@ -18,6 +19,8 @@ MODULE_PARM_DESC(enable_uring, "Enable userspace communication through io-uring"); #define FUSE_URING_IOV_SEGS 2 /* header and payload */ +#define FUSE_RING_HEADER_PG 0 +#define FUSE_RING_PAYLOAD_PG 1 bool fuse_uring_enabled(void) @@ -142,6 +145,21 @@ void fuse_uring_abort_end_requests(struct fuse_ring *ring) } } +/* + * Copy from memmap.c, should be exported + */ +static void io_pages_free(struct page ***pages, int npages) +{ + struct page **page_array = *pages; + + if (!page_array) + return; + + unpin_user_pages(page_array, npages); + kvfree(page_array); + *pages = NULL; +} + static bool ent_list_request_expired(struct fuse_conn *fc, struct list_head *list) { struct fuse_ring_ent *ent; @@ -208,6 +226,9 @@ void fuse_uring_destruct(struct fuse_conn *fc) list_for_each_entry_safe(ent, next, &queue->ent_released, list) { list_del_init(&ent->list); + io_pages_free(&ent->header_pages, ent->nr_header_pages); + io_pages_free(&ent->payload_pages, + ent->nr_payload_pages); kfree(ent); } @@ -598,12 +619,66 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, fuse_copy_init(&cs, false, &iter); cs.is_uring = true; cs.req = req; + if (ent->payload_pages) + cs.ring.pages = ent->payload_pages; err = fuse_copy_out_args(&cs, args, ring_in_out.payload_sz); fuse_copy_finish(&cs); return err; } +/* + * Copy data from the req to the ring buffer + * In order to be able to write into the ring buffer from the application, + * i.e. to avoid io_uring_cmd_complete_in_task(), the header needs to be + * pinned as well. + */ +static int fuse_uring_args_to_ring_pages(struct fuse_ring *ring, + struct fuse_req *req, + struct fuse_ring_ent *ent, + struct fuse_uring_req_header *headers) +{ + struct fuse_copy_state cs; + struct fuse_args *args = req->args; + struct fuse_in_arg *in_args = args->in_args; + int num_args = args->in_numargs; + int err; + + struct fuse_uring_ent_in_out ent_in_out = { + .flags = 0, + .commit_id = req->in.h.unique, + }; + + fuse_copy_init(&cs, 1, NULL); + cs.is_uring = 1; + cs.req = req; + cs.ring.pages = ent->payload_pages; + + if (num_args > 0) { + /* + * Expectation is that the first argument is the per op header. + * Some op code have that as zero size. + */ + if (args->in_args[0].size > 0) { + memcpy(&headers->op_in, in_args->value, in_args->size); + } + in_args++; + num_args--; + } + + /* copy the payload */ + err = fuse_copy_args(&cs, num_args, args->in_pages, + (struct fuse_arg *)in_args, 0); + if (err) { + pr_info_ratelimited("%s fuse_copy_args failed\n", __func__); + return err; + } + + ent_in_out.payload_sz = cs.ring.copied_sz; + memcpy(&headers->ring_ent_in_out, &ent_in_out, sizeof(ent_in_out)); + return err; +} + /* * Copy data from the req to the ring buffer */ @@ -630,6 +705,8 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, fuse_copy_init(&cs, true, &iter); cs.is_uring = true; cs.req = req; + if (ent->payload_pages) + cs.ring.pages = ent->payload_pages; if (num_args > 0) { /* @@ -670,6 +747,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, struct fuse_ring_queue *queue = ent->queue; struct fuse_ring *ring = queue->ring; int err; + struct fuse_uring_req_header *headers = NULL; err = -EIO; if (WARN_ON(ent->state != FRRS_FUSE_REQ)) { @@ -682,22 +760,29 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, if (WARN_ON(req->in.h.unique == 0)) return err; - /* copy the request */ - err = fuse_uring_args_to_ring(ring, req, ent); - if (unlikely(err)) { - pr_info_ratelimited("Copy to ring failed: %d\n", err); - return err; - } - /* copy fuse_in_header */ - err = copy_to_user(&ent->headers->in_out, &req->in.h, - sizeof(req->in.h)); - if (err) { - err = -EFAULT; - return err; + if (ent->header_pages) { + headers = kmap_local_page( + ent->header_pages[FUSE_RING_HEADER_PG]); + + memcpy(&headers->in_out, &req->in.h, sizeof(req->in.h)); + + err = fuse_uring_args_to_ring_pages(ring, req, ent, headers); + kunmap_local(headers); + } else { + /* copy the request */ + err = fuse_uring_args_to_ring(ring, req, ent); + if (unlikely(err)) { + pr_info_ratelimited("Copy to ring failed: %d\n", err); + return err; + } + err = copy_to_user(&ent->headers->in_out, &req->in.h, + sizeof(req->in.h)); + if (err) + err = -EFAULT; } - return 0; + return err; } static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, @@ -1006,6 +1091,45 @@ static void fuse_uring_do_register(struct fuse_ring_ent *ent, } } +/* + * Copy from memmap.c, should be exported there + */ +static struct page **io_pin_pages(unsigned long uaddr, unsigned long len, + int *npages) +{ + unsigned long start, end, nr_pages; + struct page **pages; + int ret; + + end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; + start = uaddr >> PAGE_SHIFT; + nr_pages = end - start; + if (WARN_ON_ONCE(!nr_pages)) + return ERR_PTR(-EINVAL); + + pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL); + if (!pages) + return ERR_PTR(-ENOMEM); + + ret = pin_user_pages_fast(uaddr, nr_pages, FOLL_WRITE | FOLL_LONGTERM, + pages); + /* success, mapped all pages */ + if (ret == nr_pages) { + *npages = nr_pages; + return pages; + } + + /* partial map, or didn't map anything */ + if (ret >= 0) { + /* if we did partial map, release any pages we did get */ + if (ret) + unpin_user_pages(pages, ret); + ret = -EFAULT; + } + kvfree(pages); + return ERR_PTR(ret); +} + /* * sqe->addr is a ptr to an iovec array, iov[0] has the headers, iov[1] * the payload @@ -1032,6 +1156,59 @@ static int fuse_uring_get_iovec_from_sqe(const struct io_uring_sqe *sqe, return 0; } +static int fuse_uring_pin_pages(struct fuse_ring_ent *ent) +{ + struct fuse_ring *ring = ent->queue->ring; + int err; + + /* + * This needs to do locked memory accounting, for now privileged servers + * only. + */ + if (!capable(CAP_SYS_ADMIN)) + return 0; + + /* Pin header pages */ + if (!PAGE_ALIGNED(ent->headers)) { + pr_info_ratelimited("ent->headers is not page-aligned: %p\n", + ent->headers); + return -EINVAL; + } + + ent->header_pages = io_pin_pages((unsigned long)ent->headers, + sizeof(struct fuse_uring_req_header), + &ent->nr_header_pages); + if (IS_ERR(ent->header_pages)) { + err = PTR_ERR(ent->header_pages); + pr_info_ratelimited("Failed to pin header pages, err=%d\n", + err); + ent->header_pages = NULL; + return err; + } + + if (ent->nr_header_pages != 1) { + pr_info_ratelimited("Header pages not pinned as one page\n"); + io_pages_free(&ent->header_pages, ent->nr_header_pages); + ent->header_pages = NULL; + return -EINVAL; + } + + /* Pin payload pages */ + ent->payload_pages = io_pin_pages((unsigned long)ent->payload, + ring->max_payload_sz, + &ent->nr_payload_pages); + if (IS_ERR(ent->payload_pages)) { + err = PTR_ERR(ent->payload_pages); + pr_info_ratelimited("Failed to pin payload pages, err=%d\n", + err); + io_pages_free(&ent->header_pages, ent->nr_header_pages); + ent->payload_pages = NULL; + return err; + } + + return 0; +} + static struct fuse_ring_ent * fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_queue *queue) @@ -1073,6 +1250,12 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, ent->headers = iov[0].iov_base; ent->payload = iov[1].iov_base; + err = fuse_uring_pin_pages(ent); + if (err) { + kfree(ent); + return ERR_PTR(err); + } + atomic_inc(&ring->queue_refs); return ent; } diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 51a563922ce141..c89c7dc27c76c1 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -40,7 +40,11 @@ enum fuse_ring_req_state { struct fuse_ring_ent { /* userspace buffer */ struct fuse_uring_req_header __user *headers; + struct page **header_pages; + int nr_header_pages; void __user *payload; + struct page **payload_pages; + int nr_payload_pages; /* the ring queue that owns the request */ struct fuse_ring_queue *queue; diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h index 134bf44aff0d39..4037fd7bdeee66 100644 --- a/fs/fuse/fuse_dev_i.h +++ b/fs/fuse/fuse_dev_i.h @@ -36,6 +36,8 @@ struct fuse_copy_state { bool is_uring:1; struct { unsigned int copied_sz; /* copied size into the user buffer */ + struct page **pages; + int page_idx; } ring; }; From 8a20f0b2b11d0a5d4e338ed22212bc1a3cdb2290 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Fri, 17 Jan 2025 22:06:30 +0100 Subject: [PATCH 03/77] fuse: {io-uring] Avoid complete-in-task if pinned pages are used If pinned pages are used the application can write into these pages and io_uring_cmd_complete_in_task() is not needed. Signed-off-by: Bernd Schubert (imported from commit 5f0264c1dc0100e274f3db37511bba0d8043de1c) --- fs/fuse/dev_uring.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index e24b87f1df745d..d5737245516b01 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1438,12 +1438,31 @@ static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) return queue; } -static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent) +static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg) { struct io_uring_cmd *cmd = ent->cmd; - uring_cmd_set_ring_ent(cmd, ent); - io_uring_cmd_complete_in_task(cmd, fuse_uring_send_in_task); + /* + * Task needed when pages are not pinned as the application doing IO + * is not allowed to write into fuse-server pages. + * Additionally for IO through io-uring as issue flags are unknown then. + * backgrounds requests might hold spin-locks, that conflict with + * io_uring_cmd_done() mutex lock. + */ + if (!ent->header_pages || current->io_uring || bg) { + uring_cmd_set_ring_ent(cmd, ent); + io_uring_cmd_complete_in_task(cmd, fuse_uring_send_in_task); + } else { + int err = fuse_uring_prepare_send(ent, ent->fuse_req); + struct fuse_ring_queue *queue = ent->queue; + + if (err) { + fuse_uring_next_fuse_req(ent, queue, + IO_URING_F_UNLOCKED); + return; + } + fuse_uring_send(ent, cmd, 0, IO_URING_F_UNLOCKED); + } } /* queue a fuse request and send it if a ring entry is available */ @@ -1478,7 +1497,7 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) spin_unlock(&queue->lock); if (ent) - fuse_uring_dispatch_ent(ent); + fuse_uring_dispatch_ent(ent, false); return; @@ -1531,7 +1550,7 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) fuse_uring_add_req_to_ring_ent(ent, req); spin_unlock(&queue->lock); - fuse_uring_dispatch_ent(ent); + fuse_uring_dispatch_ent(ent, true); } else { spin_unlock(&queue->lock); } From 1cb8f2d520f6aacc6580fef766a396bab6802307 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Wed, 7 May 2025 23:39:00 +0200 Subject: [PATCH 04/77] fuse: Use fuser-server provided read-ahead for CAP_SYS_ADMIN readhead is currently limited to bdi->ra_pages. One can change that after the mount with something like minor=$(stat -c "%d" /path/to/fuse) echo 1024 > /sys/class/bdi/0:$(minor)/read_ahead_kb Issue is that fuse-server cannot do that from its ->init method, as it has to know about device minor, which blocks before init is complete. Fuse already sets the bdi value, but upper limit is the current bdi value. For CAP_SYS_ADMIN we can allow higher values. Signed-off-by: Bernd Schubert (imported from commit 763c96da4bd6d1bb95d8e6bb7fd352389f3a17b9) --- fs/fuse/inode.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index c795abe47a4f4a..093310f669b522 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1464,7 +1464,10 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, init_server_timeout(fc, timeout); - fm->sb->s_bdi->ra_pages = + if (CAP_SYS_ADMIN) + fm->sb->s_bdi->ra_pages = ra_pages; + else + fm->sb->s_bdi->ra_pages = min(fm->sb->s_bdi->ra_pages, ra_pages); fc->minor = arg->minor; fc->max_write = arg->minor < 5 ? 4096 : arg->max_write; From 7c5216b61b38d42167c559a044ffc13733a39c2c Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Tue, 8 Apr 2025 16:44:55 +0200 Subject: [PATCH 05/77] fuse: Increase the default max pages limit to 8182 Due to user buffer misalignent we actually need one page more, i.e. 1025 instead of 1024, will be handled differently. For now we just bump up the max. (imported from commit 3f71501c9c4702ba976145ff15c4a053ecd1a3ee) --- fs/fuse/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 093310f669b522..e0f3f1ab08a1f4 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -39,7 +39,7 @@ DECLARE_WAIT_QUEUE_HEAD(fuse_dev_waitq); static int set_global_limit(const char *val, const struct kernel_param *kp); -unsigned int fuse_max_pages_limit = 256; +unsigned int fuse_max_pages_limit = 4097; /* default is no timeout */ unsigned int fuse_default_req_timeout; unsigned int fuse_max_req_timeout; From 9e44162e70c21af8866814cc12f4db3d3a478f44 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Fri, 20 Jun 2025 17:34:53 +0200 Subject: [PATCH 06/77] fuse: add DLM_LOCK opcode When having writeback cache enabled it is beneficial for data consistency to communicate to the FUSE server when the kernel prepares a page for caching. This lets the FUSE server react and lock the page. Additionally the kernel lets the FUSE server decide how much data it locks by the same call and keeps the given information in the dlm lock management. If the feature is not supported it will be disabled after first unsuccessful use. - Add DLM_LOCK fuse opcode - Add cache page lock caching for writeback cache functionality. This means sending out a FUSE call whenever the kernel prepares a page for writeback cache. The kernel will manage the cache so that it will keep track of already acquired locks. (except for the case that is documented in the code) - Use rb-trees for the management of the already 'locked' page ranges - Use rw_semaphore for synchronization in fuse_dlm_cache (imported from commit 287c8840b60d5cdcf806b16e8cc5722f2dbf0738) --- fs/fuse/Makefile | 2 +- fs/fuse/dir.c | 6 + fs/fuse/file.c | 13 + fs/fuse/fuse_dlm_cache.c | 551 ++++++++++++++++++++++++++++++++++++++ fs/fuse/fuse_dlm_cache.h | 50 ++++ fs/fuse/fuse_i.h | 18 ++ fs/fuse/fuse_trace.h | 1 + fs/fuse/inode.c | 11 + include/uapi/linux/fuse.h | 36 +++ 9 files changed, 687 insertions(+), 1 deletion(-) create mode 100644 fs/fuse/fuse_dlm_cache.c create mode 100644 fs/fuse/fuse_dlm_cache.h diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile index 22ad9538dfc4b8..64bc8682ae9659 100644 --- a/fs/fuse/Makefile +++ b/fs/fuse/Makefile @@ -11,7 +11,7 @@ obj-$(CONFIG_CUSE) += cuse.o obj-$(CONFIG_VIRTIO_FS) += virtiofs.o fuse-y := trace.o # put trace.o first so we see ftrace errors sooner -fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o +fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o fuse-y += iomode.o fuse-$(CONFIG_FUSE_DAX) += dax.o fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o backing.o diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 7ac6b232ef1232..c1179ce8fc96b2 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -6,6 +6,7 @@ See the file COPYING. */ +#include "fuse_dlm_cache.h" #include "fuse_i.h" #include @@ -2181,6 +2182,8 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, * truncation has already been done by OPEN. But still * need to truncate page cache. */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_cache_release_locks(fi); i_size_write(inode, 0); truncate_pagecache(inode, 0); goto out; @@ -2286,6 +2289,9 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if ((is_truncate || !is_wb) && S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { + if (fc->dlm && fc->writeback_cache) + fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1); + truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); } diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 676fd9856bfbf3..63b45e74356743 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -7,6 +7,7 @@ */ #include "fuse_i.h" +#include "fuse_dlm_cache.h" #include #include @@ -1489,6 +1490,17 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (!fc->handle_killpriv_v2 || !setattr_should_drop_suidgid(idmap, file_inode(file))) writeback = true; + /* if we have dlm support acquire the lock for the area + * we are writing into */ + if (fc->dlm) { + /* note that a file opened with O_APPEND will have relative values + * in ki_pos. This code is here for convenience and for libfuse overlay test. + * Filesystems should handle O_APPEND with 'direct io' to additionally + * get the performance benefits of 'parallel direct writes'. */ + loff_t pos = file->f_flags & O_APPEND ? i_size_read(inode) + iocb->ki_pos : iocb->ki_pos; + size_t length = iov_iter_count(from); + fuse_get_dlm_write_lock(file, pos, length); + } } inode_lock(inode); @@ -3206,6 +3218,7 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) INIT_LIST_HEAD(&fi->write_files); INIT_LIST_HEAD(&fi->queued_writes); + fuse_dlm_cache_init(fi); fi->writectr = 0; fi->iocachectr = 0; init_waitqueue_head(&fi->page_waitq); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c new file mode 100644 index 00000000000000..ea947f34a9f70a --- /dev/null +++ b/fs/fuse/fuse_dlm_cache.c @@ -0,0 +1,551 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * FUSE page lock cache implementation + */ +#include "fuse_i.h" +#include "fuse_dlm_cache.h" + +#include +#include +#include +#include + + +/* A range of pages with a lock */ +struct fuse_dlm_range { + /* Interval tree node */ + struct rb_node rb; + /* Start page offset (inclusive) */ + pgoff_t start; + /* End page offset (inclusive) */ + pgoff_t end; + /* Subtree end value for interval tree */ + pgoff_t __subtree_end; + /* Lock mode */ + enum fuse_page_lock_mode mode; + /* Temporary list entry for operations */ + struct list_head list; +}; + +/* Lock modes for FUSE page cache */ +#define FUSE_PCACHE_LK_READ 1 /* Shared read lock */ +#define FUSE_PCACHE_LK_WRITE 2 /* Exclusive write lock */ + +/* Interval tree definitions for page ranges */ +static inline pgoff_t fuse_dlm_range_start(struct fuse_dlm_range *range) +{ + return range->start; +} + +static inline pgoff_t fuse_dlm_range_last(struct fuse_dlm_range *range) +{ + return range->end; +} + +INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, pgoff_t, __subtree_end, + fuse_dlm_range_start, fuse_dlm_range_last, static, + fuse_page_it); + +/** + * fuse_page_cache_init - Initialize a page cache lock manager + * @cache: The cache to initialize + * + * Initialize a page cache lock manager for a FUSE inode. + * + * Return: 0 on success, negative error code on failure + */ +int fuse_dlm_cache_init(struct fuse_inode *inode) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + if (!cache) + return -EINVAL; + + init_rwsem(&cache->lock); + cache->ranges = RB_ROOT_CACHED; + + return 0; +} + +/** + * fuse_page_cache_destroy - Clean up a page cache lock manager + * @cache: The cache to clean up + * + * Release all locks and free all resources associated with the cache. + */ +void fuse_dlm_cache_release_locks(struct fuse_inode *inode) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_range *range; + struct rb_node *node; + + if (!cache) + return; + + /* Release all locks */ + down_write(&cache->lock); + while ((node = rb_first_cached(&cache->ranges)) != NULL) { + range = rb_entry(node, struct fuse_dlm_range, rb); + fuse_page_it_remove(range, &cache->ranges); + kfree(range); + } + up_write(&cache->lock); +} + +/** + * fuse_dlm_find_overlapping - Find a range that overlaps with [start, end] + * @cache: The page cache + * @start: Start page offset + * @end: End page offset + * + * Return: Pointer to the first overlapping range, or NULL if none found + */ +static struct fuse_dlm_range * +fuse_dlm_find_overlapping(struct fuse_dlm_cache *cache, pgoff_t start, + pgoff_t end) +{ + return fuse_page_it_iter_first(&cache->ranges, start, end); +} + +/** + * fuse_page_try_merge - Try to merge ranges within a specific region + * @cache: The page cache + * @start: Start page offset + * @end: End page offset + * + * Attempt to merge ranges within and adjacent to the specified region + * that have the same lock mode. + */ +static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, pgoff_t start, + pgoff_t end) +{ + struct fuse_dlm_range *range, *next; + struct rb_node *node; + + if (!cache) + return; + + /* Find the first range that might need merging */ + range = NULL; + node = rb_first_cached(&cache->ranges); + while (node) { + range = rb_entry(node, struct fuse_dlm_range, rb); + if (range->end >= start - 1) + break; + node = rb_next(node); + } + + if (!range || range->start > end + 1) + return; + + /* Try to merge ranges in and around the specified region */ + while (range && range->start <= end + 1) { + /* Get next range before we potentially modify the tree */ + next = NULL; + if (rb_next(&range->rb)) { + next = rb_entry(rb_next(&range->rb), + struct fuse_dlm_range, rb); + } + + /* Try to merge with next range if adjacent and same mode */ + if (next && range->mode == next->mode && + range->end + 1 == next->start) { + /* Merge ranges */ + range->end = next->end; + + /* Remove next from tree */ + fuse_page_it_remove(next, &cache->ranges); + kfree(next); + + /* Continue with the same range */ + continue; + } + + /* Move to next range */ + range = next; + } +} + +/** + * fuse_dlm_lock_range - Lock a range of pages + * @cache: The page cache + * @start: Start page offset + * @end: End page offset + * @mode: Lock mode (read or write) + * + * Add a locked range on the specified range of pages. + * If parts of the range are already locked, only add the remaining parts. + * For overlapping ranges, handle lock compatibility: + * - READ locks are compatible with existing READ locks + * - READ locks are compatible with existing WRITE locks (downgrade not needed) + * - WRITE locks need to upgrade existing READ locks + * + * Return: 0 on success, negative error code on failure + */ +int fuse_dlm_lock_range(struct fuse_inode *inode, pgoff_t start, + pgoff_t end, enum fuse_page_lock_mode mode) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_range *range, *new_range, *next; + int lock_mode; + int ret = 0; + LIST_HEAD(to_lock); + LIST_HEAD(to_upgrade); + pgoff_t current_start = start; + + if (!cache || start > end) + return -EINVAL; + + /* Convert to lock mode */ + lock_mode = (mode == FUSE_PAGE_LOCK_READ) ? FUSE_PCACHE_LK_READ : + FUSE_PCACHE_LK_WRITE; + + down_write(&cache->lock); + + /* Find all ranges that overlap with [start, end] */ + range = fuse_page_it_iter_first(&cache->ranges, start, end); + while (range) { + /* Get next overlapping range before we potentially modify the tree */ + next = fuse_page_it_iter_next(range, start, end); + + /* Check lock compatibility */ + if (lock_mode == FUSE_PCACHE_LK_WRITE && + lock_mode != range->mode) { + /* we own the lock but have to update it. */ + list_add_tail(&range->list, &to_upgrade); + } + /* If WRITE lock already exists - nothing to do */ + + /* If there's a gap before this range, we need to add the missing range */ + if (current_start < range->start) { + new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); + if (!new_range) { + ret = -ENOMEM; + goto out_free; + } + + new_range->start = current_start; + new_range->end = range->start - 1; + new_range->mode = lock_mode; + INIT_LIST_HEAD(&new_range->list); + + list_add_tail(&new_range->list, &to_lock); + } + + /* Move current_start past this range */ + current_start = max(current_start, range->end + 1); + + /* Move to next range */ + range = next; + } + + /* If there's a gap after the last range to the end, extend the range */ + if (current_start <= end) { + new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); + if (!new_range) { + ret = -ENOMEM; + goto out_free; + } + + new_range->start = current_start; + new_range->end = end; + new_range->mode = lock_mode; + INIT_LIST_HEAD(&new_range->list); + + list_add_tail(&new_range->list, &to_lock); + } + + /* update locks, if any lock is in this list it has the wrong mode */ + list_for_each_entry(range, &to_upgrade, list) { + /* Update the lock mode */ + range->mode = lock_mode; + } + + /* Add all new ranges to the tree */ + list_for_each_entry(new_range, &to_lock, list) { + /* Add to interval tree */ + fuse_page_it_insert(new_range, &cache->ranges); + } + + /* Try to merge adjacent ranges with the same mode */ + fuse_dlm_try_merge(cache, start, end); + + up_write(&cache->lock); + return 0; + +out_free: + /* Free any ranges we allocated but didn't insert */ + while (!list_empty(&to_lock)) { + new_range = + list_first_entry(&to_lock, struct fuse_dlm_range, list); + list_del(&new_range->list); + kfree(new_range); + } + + /* Restore original lock modes for any partially upgraded locks */ + list_for_each_entry(range, &to_upgrade, list) { + if (lock_mode == FUSE_PCACHE_LK_WRITE) { + /* We upgraded this lock but failed later, downgrade it back */ + range->mode = FUSE_PCACHE_LK_READ; + } + } + + up_write(&cache->lock); + return ret; +} + +/** + * fuse_dlm_punch_hole - Punch a hole in a locked range + * @cache: The page cache + * @start: Start page offset of the hole + * @end: End page offset of the hole + * + * Create a hole in a locked range by splitting it into two ranges. + * + * Return: 0 on success, negative error code on failure + */ +static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, pgoff_t start, + pgoff_t end) +{ + struct fuse_dlm_range *range, *new_range; + int ret = 0; + + if (!cache || start > end) + return -EINVAL; + + /* Find a range that contains [start, end] */ + range = fuse_dlm_find_overlapping(cache, start, end); + if (!range) { + ret = -EINVAL; + goto out; + } + + /* If the hole is at the beginning of the range */ + if (start == range->start) { + range->start = end + 1; + goto out; + } + + /* If the hole is at the end of the range */ + if (end == range->end) { + range->end = start - 1; + goto out; + } + + /* The hole is in the middle, need to split */ + new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); + if (!new_range) { + ret = -ENOMEM; + goto out; + } + + /* Copy properties from original range */ + *new_range = *range; + INIT_LIST_HEAD(&new_range->list); + + /* Adjust ranges */ + new_range->start = end + 1; + range->end = start - 1; + + /* Update interval tree */ + fuse_page_it_remove(range, &cache->ranges); + fuse_page_it_insert(range, &cache->ranges); + fuse_page_it_insert(new_range, &cache->ranges); + +out: + return ret; +} + +/** + * fuse_dlm_unlock_range - Unlock a range of pages + * @cache: The page cache + * @start: Start page offset + * @end: End page offset + * + * Release locks on the specified range of pages. + * + * Return: 0 on success, negative error code on failure + */ +int fuse_dlm_unlock_range(struct fuse_inode *inode, + pgoff_t start, pgoff_t end) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_range *range, *next; + int ret = 0; + + if (!cache) + return -EINVAL; + + down_write(&cache->lock); + + /* Find all ranges that overlap with [start, end] */ + range = fuse_page_it_iter_first(&cache->ranges, start, end); + while (range) { + /* Get next overlapping range before we potentially modify the tree */ + next = fuse_page_it_iter_next(range, start, end); + + /* Check if we need to punch a hole */ + if (start > range->start && end < range->end) { + /* Punch a hole in the middle */ + ret = fuse_dlm_punch_hole(cache, start, end); + if (ret) + goto out; + /* After punching a hole, we're done */ + break; + } else if (start > range->start) { + /* Adjust the end of the range */ + range->end = start - 1; + } else if (end < range->end) { + /* Adjust the start of the range */ + range->start = end + 1; + } else { + /* Complete overlap, remove the range */ + fuse_page_it_remove(range, &cache->ranges); + kfree(range); + } + + range = next; + } + +out: + up_write(&cache->lock); + return ret; +} + +/** + * fuse_dlm_range_is_locked - Check if a page range is already locked + * @cache: The page cache + * @start: Start page offset + * @end: End page offset + * @mode: Lock mode to check for (or NULL to check for any lock) + * + * Check if the specified range of pages is already locked. + * The entire range must be locked for this to return true. + * + * Return: true if the entire range is locked, false otherwise + */ +bool fuse_dlm_range_is_locked(struct fuse_inode *inode, pgoff_t start, + pgoff_t end, enum fuse_page_lock_mode mode) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_range *range; + int lock_mode = 0; + pgoff_t current_start = start; + + if (!cache || start > end) + return false; + + /* Convert to lock mode if specified */ + if (mode == FUSE_PAGE_LOCK_READ) + lock_mode = FUSE_PCACHE_LK_READ; + else if (mode == FUSE_PAGE_LOCK_WRITE) + lock_mode = FUSE_PCACHE_LK_WRITE; + + down_read(&cache->lock); + + /* Find the first range that overlaps with [start, end] */ + range = fuse_dlm_find_overlapping(cache, start, end); + + /* Check if the entire range is covered */ + while (range && current_start <= end) { + /* If we're checking for a specific mode, verify it matches */ + if (lock_mode && range->mode != lock_mode) { + /* Wrong lock mode */ + up_read(&cache->lock); + return false; + } + + /* Check if there's a gap before this range */ + if (current_start < range->start) { + /* Found a gap */ + up_read(&cache->lock); + return false; + } + + /* Move current_start past this range */ + current_start = range->end + 1; + + /* Get next overlapping range */ + range = fuse_page_it_iter_next(range, start, end); + } + + /* Check if we covered the entire range */ + if (current_start <= end) { + /* There's a gap at the end */ + up_read(&cache->lock); + return false; + } + + up_read(&cache->lock); + return true; +} + +/** + * request a dlm lock from the fuse server + */ +void fuse_get_dlm_write_lock(struct file *file, loff_t offset, + size_t length) +{ + struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_mount *fm = ff->fm; + loff_t end = (offset + length - 1) | (PAGE_SIZE - 1); + + /* note that the offset and length don't have to be page aligned here + * but since we only get here on writeback caching we will send out + * page aligned requests */ + offset &= PAGE_MASK; + + FUSE_ARGS(args); + struct fuse_dlm_lock_in inarg; + struct fuse_dlm_lock_out outarg; + int err; + + /* note that this can be run from different processes + * at the same time. It is intentionally not protected + * since a DLM implementation in the FUSE server should take care + * of any races in lock requests */ + if (fuse_dlm_range_is_locked(fi, offset, + end, FUSE_PAGE_LOCK_WRITE)) + return; /* we already have this area locked */ + + memset(&inarg, 0, sizeof(inarg)); + inarg.fh = ff->fh; + + inarg.offset = offset; + inarg.size = end - offset + 1; + inarg.type = FUSE_DLM_LOCK_WRITE; + + args.opcode = FUSE_DLM_WB_LOCK; + args.nodeid = get_node_id(inode); + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.out_numargs = 1; + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + err = fuse_simple_request(fm, &args); + if (err == -ENOSYS) { + /* fuse server does not support dlm, save the info */ + fc->dlm = 0; + return; + } + + if (outarg.locksize < end - offset + 1) { + /* fuse server is seriously broken */ + pr_warn("fuse: dlm lock request for %llu bytes returned %u bytes\n", + end - offset + 1, outarg.locksize); + fuse_abort_conn(fc); + return; + } + + if (err) + return; + else + /* ignore any errors here, there is no way we can react appropriately */ + fuse_dlm_lock_range(fi, offset, + offset + outarg.locksize - 1, + FUSE_PAGE_LOCK_WRITE); +} diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h new file mode 100644 index 00000000000000..98b27a2c15d8ba --- /dev/null +++ b/fs/fuse/fuse_dlm_cache.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * FUSE page cache lock implementation + */ + +#ifndef _FS_FUSE_DLM_CACHE_H +#define _FS_FUSE_DLM_CACHE_H + +#include +#include +#include +#include + + +struct fuse_inode; + +/* Lock modes for page ranges */ +enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; + +/* Page cache lock manager */ +struct fuse_dlm_cache { + /* Lock protecting the tree */ + struct rw_semaphore lock; + /* Interval tree of locked ranges */ + struct rb_root_cached ranges; +}; + +/* Initialize a page cache lock manager */ +int fuse_dlm_cache_init(struct fuse_inode *inode); + +/* Clean up a page cache lock manager */ +void fuse_dlm_cache_release_locks(struct fuse_inode *inode); + +/* Lock a range of pages */ +int fuse_dlm_lock_range(struct fuse_inode *inode, pgoff_t start, + pgoff_t end, enum fuse_page_lock_mode mode); + +/* Unlock a range of pages */ +int fuse_dlm_unlock_range(struct fuse_inode *inode, pgoff_t start, + pgoff_t end); + +/* Check if a page range is already locked */ +bool fuse_dlm_range_is_locked(struct fuse_inode *inode, pgoff_t start, + pgoff_t end, enum fuse_page_lock_mode mode); + +/* this is the interface to the filesystem */ +void fuse_get_dlm_write_lock(struct file *file, loff_t offset, + size_t length); + +#endif /* _FS_FUSE_DLM_CACHE_H */ diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 7f16049387d15e..eacd1e735dc5b6 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -31,6 +31,7 @@ #include #include #include +#include "fuse_dlm_cache.h" /** Default max number of pages that can be used in a single read request */ #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 @@ -113,6 +114,17 @@ struct fuse_backing { struct rcu_head rcu; }; +/** + * data structure to save the information that we have + * requested dlm locks for the given area from the fuse server +*/ +struct dlm_locked_area +{ + struct list_head list; + loff_t offset; + size_t size; +}; + /** FUSE inode */ struct fuse_inode { /** Inode data */ @@ -168,6 +180,9 @@ struct fuse_inode { /* waitq for direct-io completion */ wait_queue_head_t direct_io_waitq; + + /* dlm locked areas we have sent lock requests for */ + struct fuse_dlm_cache dlm_locked_areas; }; /* readdir cache (directory only) */ @@ -909,6 +924,9 @@ struct fuse_conn { /* Is statx not implemented by fs? */ unsigned int no_statx:1; + /* do we have support for dlm in the fs? */ + unsigned int dlm:1; + /** Passthrough support for read/write IO */ unsigned int passthrough:1; diff --git a/fs/fuse/fuse_trace.h b/fs/fuse/fuse_trace.h index 393c630e772635..9976e31a51a9c9 100644 --- a/fs/fuse/fuse_trace.h +++ b/fs/fuse/fuse_trace.h @@ -58,6 +58,7 @@ EM( FUSE_SYNCFS, "FUSE_SYNCFS") \ EM( FUSE_TMPFILE, "FUSE_TMPFILE") \ EM( FUSE_STATX, "FUSE_STATX") \ + EM( FUSE_DLM_WB_LOCK, "FUSE_DLM_WB_LOCK") \ EMe(CUSE_INIT, "CUSE_INIT") /* diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index e0f3f1ab08a1f4..f3ff39627a02bd 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -7,6 +7,7 @@ */ #include "fuse_i.h" +#include "fuse_dlm_cache.h" #include "fuse_dev_i.h" #include "dev_uring_i.h" @@ -195,6 +196,7 @@ static void fuse_evict_inode(struct inode *inode) WARN_ON(fi->iocachectr != 0); WARN_ON(!list_empty(&fi->write_files)); WARN_ON(!list_empty(&fi->queued_writes)); + fuse_dlm_cache_release_locks(fi); } } @@ -578,6 +580,14 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, pg_end = -1; else pg_end = (offset + len - 1) >> PAGE_SHIFT; + + if (fc->dlm && fc->writeback_cache) + /* invalidate the range from the beginning of the first page + * in the given range to the last byte of the last page */ + fuse_dlm_unlock_range(fi, + pg_start << PAGE_SHIFT, + (pg_end << PAGE_SHIFT) | (PAGE_SIZE - 1)); + invalidate_inode_pages2_range(inode->i_mapping, pg_start, pg_end); } @@ -991,6 +1001,7 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, fc->blocked = 0; fc->initialized = 0; fc->connected = 1; + fc->dlm = 1; atomic64_set(&fc->attr_version, 1); atomic64_set(&fc->evict_ctr, 1); get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index c13e1f9a2f12bd..d4139185c7491c 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -663,6 +663,7 @@ enum fuse_opcode { FUSE_TMPFILE = 51, FUSE_STATX = 52, FUSE_COPY_FILE_RANGE_64 = 53, + FUSE_DLM_WB_LOCK = 53, /* CUSE specific operations */ CUSE_INIT = 4096, @@ -1245,6 +1246,41 @@ struct fuse_supp_groups { uint32_t groups[]; }; +/** + * Type of the dlm lock requested + */ +enum fuse_dlm_lock_type { + FUSE_DLM_LOCK_NONE = 0, + FUSE_DLM_LOCK_READ = 1, + FUSE_DLM_LOCK_WRITE = 2 +}; + +/** + * struct fuse_dlm_lock_in - Lock request + * @fh: file handle + * @offset: offset into the file + * @size: size of the locked region + * @type: type of lock + */ +struct fuse_dlm_lock_in { + uint64_t fh; + uint64_t offset; + uint32_t size; + uint32_t type; + uint64_t reserved; +}; + +/** + * struct fuse_dlm_lock_out - Lock response + * @locksize: how many bytes where locked by the call + * (most of the time we want to lock more than is requested + * to reduce number of calls) + */ +struct fuse_dlm_lock_out { + uint32_t locksize; + uint32_t padding; +}; + /** * Size of the ring buffer header */ From ec05a1c73fa95bb0dd99c87bc6f3344600afe01c Mon Sep 17 00:00:00 2001 From: Cheng Ding Date: Thu, 17 Jul 2025 17:04:16 +0000 Subject: [PATCH 07/77] fuse: Renumber FUSE_DLM_WB_LOCK to 100 Renumber the operation code to a high value to avoid conflicts with upstream. (imported from commit 27a0e9ea714f7fcf3ee40f977be6a17c10766509) --- fs/fuse/fuse_trace.h | 2 +- include/uapi/linux/fuse.h | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/fs/fuse/fuse_trace.h b/fs/fuse/fuse_trace.h index 9976e31a51a9c9..e81c93b9614627 100644 --- a/fs/fuse/fuse_trace.h +++ b/fs/fuse/fuse_trace.h @@ -58,7 +58,7 @@ EM( FUSE_SYNCFS, "FUSE_SYNCFS") \ EM( FUSE_TMPFILE, "FUSE_TMPFILE") \ EM( FUSE_STATX, "FUSE_STATX") \ - EM( FUSE_DLM_WB_LOCK, "FUSE_DLM_WB_LOCK") \ + EM( FUSE_DLM_WB_LOCK, "FUSE_DLM_WB_LOCK") \ EMe(CUSE_INIT, "CUSE_INIT") /* diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index d4139185c7491c..6828ceb1216d3c 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -662,8 +662,10 @@ enum fuse_opcode { FUSE_SYNCFS = 50, FUSE_TMPFILE = 51, FUSE_STATX = 52, - FUSE_COPY_FILE_RANGE_64 = 53, - FUSE_DLM_WB_LOCK = 53, + FUSE_COPY_FILE_RANGE_64 = 53, + + /* Operations which have not been merged into upstream */ + FUSE_DLM_WB_LOCK = 100, /* CUSE specific operations */ CUSE_INIT = 4096, @@ -1252,7 +1254,7 @@ struct fuse_supp_groups { enum fuse_dlm_lock_type { FUSE_DLM_LOCK_NONE = 0, FUSE_DLM_LOCK_READ = 1, - FUSE_DLM_LOCK_WRITE = 2 + FUSE_DLM_LOCK_WRITE = 2, }; /** From d25d13aaa327b7de796b68cef3c9e5b4f86877e2 Mon Sep 17 00:00:00 2001 From: Yong Ze Chen Date: Tue, 8 Jul 2025 06:41:45 +0000 Subject: [PATCH 08/77] fuse: invalidate inode aliases when doing inode invalidation Add support to invalidate inode aliases when doing inode invalidation. This is useful for distributed file systems, which use DLM for cache coherency. So, when a client losts its inode lock, it should invalidate its inode cache and dentry cache since the other client may delete this file after getting inode lock. Signed-off-by: Yong Ze Chen (imported from commit 49720b5c84ada61feeb09da9ad4b9a0a40694792) --- fs/fuse/fuse_i.h | 6 +++++ fs/fuse/inode.c | 49 +++++++++++++++++++++++++++++++++++++++ include/uapi/linux/fuse.h | 4 ++++ 3 files changed, 59 insertions(+) diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index eacd1e735dc5b6..07c9704ac58672 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -787,6 +787,12 @@ struct fuse_conn { */ unsigned handle_killpriv_v2:1; + /* invalidate inode entries when doing inode invalidation */ + unsigned inval_inode_entries:1; + + /* expire inode entries when doing inode invalidation */ + unsigned expire_inode_entries:1; + /* * The following bitfields are only for optimization purposes * and hence races in setting them will not cause malfunction diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index f3ff39627a02bd..81c7bfd2184d34 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -555,6 +555,45 @@ struct inode *fuse_ilookup(struct fuse_conn *fc, u64 nodeid, return NULL; } +static void fuse_prune_aliases(struct inode *inode) +{ + struct dentry *dentry; + + spin_lock(&inode->i_lock); + hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) { + fuse_invalidate_entry_cache(dentry); + } + spin_unlock(&inode->i_lock); + + d_prune_aliases(inode); +} + +static void fuse_invalidate_inode_entry(struct inode *inode) +{ + struct dentry *dentry; + + if (S_ISDIR(inode->i_mode)) { + /* For directories, use d_invalidate to handle children and submounts */ + dentry = d_find_alias(inode); + if (dentry) { + d_invalidate(dentry); + fuse_invalidate_entry_cache(dentry); + dput(dentry); + } + } else { + /* For regular files, just unhash the dentry */ + spin_lock(&inode->i_lock); + hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) { + spin_lock(&dentry->d_lock); + if (!d_unhashed(dentry)) + __d_drop(dentry); + spin_unlock(&dentry->d_lock); + fuse_invalidate_entry_cache(dentry); + } + spin_unlock(&inode->i_lock); + } +} + int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { @@ -572,6 +611,11 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, fi->attr_version = atomic64_inc_return(&fc->attr_version); spin_unlock(&fi->lock); + if (fc->inval_inode_entries) + fuse_invalidate_inode_entry(inode); + else if (fc->expire_inode_entries) + fuse_prune_aliases(inode); + fuse_invalidate_attr(inode); forget_all_cached_acls(inode); if (offset >= 0) { @@ -1467,6 +1511,10 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, if (flags & FUSE_REQUEST_TIMEOUT) timeout = arg->request_timeout; + if (flags & FUSE_INVAL_INODE_ENTRY) + fc->inval_inode_entries = 1; + if (flags & FUSE_EXPIRE_INODE_ENTRY) + fc->expire_inode_entries = 1; } else { ra_pages = fc->max_read / PAGE_SIZE; fc->no_lock = 1; @@ -1519,6 +1567,7 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) FUSE_HANDLE_KILLPRIV_V2 | FUSE_SETXATTR_EXT | FUSE_INIT_EXT | FUSE_SECURITY_CTX | FUSE_CREATE_SUPP_GROUP | FUSE_HAS_EXPIRE_ONLY | FUSE_DIRECT_IO_ALLOW_MMAP | + FUSE_INVAL_INODE_ENTRY | FUSE_EXPIRE_INODE_ENTRY | FUSE_NO_EXPORT_SUPPORT | FUSE_HAS_RESEND | FUSE_ALLOW_IDMAP | FUSE_REQUEST_TIMEOUT; #ifdef CONFIG_FUSE_DAX diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index 6828ceb1216d3c..a9cdacfb76da48 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -448,6 +448,8 @@ struct fuse_file_lock { * FUSE_OVER_IO_URING: Indicate that client supports io-uring * FUSE_REQUEST_TIMEOUT: kernel supports timing out requests. * init_out.request_timeout contains the timeout (in secs) + * FUSE_INVAL_INODE_ENTRY: invalidate inode aliases when doing inode invalidation + * FUSE_EXPIRE_INODE_ENTRY: expire inode aliases when doing inode invalidation */ #define FUSE_ASYNC_READ (1 << 0) #define FUSE_POSIX_LOCKS (1 << 1) @@ -495,6 +497,8 @@ struct fuse_file_lock { #define FUSE_ALLOW_IDMAP (1ULL << 40) #define FUSE_OVER_IO_URING (1ULL << 41) #define FUSE_REQUEST_TIMEOUT (1ULL << 42) +#define FUSE_INVAL_INODE_ENTRY (1ULL << 60) +#define FUSE_EXPIRE_INODE_ENTRY (1ULL << 61) /** * CUSE INIT request/reply flags From 440ccd8458a1a546ecee398db04ca4be1b21db6b Mon Sep 17 00:00:00 2001 From: Cheng Ding Date: Wed, 16 Jul 2025 03:18:06 +0000 Subject: [PATCH 09/77] fuse: Send DLM_WB_LOCK request in page_mkwrite handler Send a DLM_WB_LOCK request in the page_mkwrite handler to enable FUSE filesystems to acquire a distributed lock manager (DLM) lock for protecting upcoming dirty pages when a previously read-only mapped page is about to be written. Signed-off-by: Cheng Ding (imported from commit ec36c455214837e9ce0d3f3385a0bb50dcfb51db) --- fs/fuse/file.c | 64 ++++++++++++++++++++++++++++++++++++++- include/uapi/linux/fuse.h | 1 + 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 63b45e74356743..75e9116194aaeb 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2339,6 +2339,57 @@ static void fuse_vma_close(struct vm_area_struct *vma) mapping_set_error(vma->vm_file->f_mapping, err); } +/** + * Request a DLM lock from the FUSE server. + * + * This routine is similar to fuse_get_dlm_write_lock(), but it + * does not cache the DLM lock in the kernel. + */ +static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t length) +{ + struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_mount *fm = ff->fm; + + FUSE_ARGS(args); + struct fuse_dlm_lock_in inarg; + struct fuse_dlm_lock_out outarg; + int err; + + if (WARN_ON_ONCE((offset & ~PAGE_MASK) || (length & ~PAGE_MASK))) + return -EIO; + + memset(&inarg, 0, sizeof(inarg)); + inarg.fh = ff->fh; + + inarg.offset = offset; + inarg.size = length; + inarg.type = FUSE_DLM_PAGE_MKWRITE; + + args.opcode = FUSE_DLM_WB_LOCK; + args.nodeid = get_node_id(inode); + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.out_numargs = 1; + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + err = fuse_simple_request(fm, &args); + if (err == -ENOSYS) { + fc->dlm = 0; + err = 0; + } + + if (!err && outarg.locksize < length) { + /* fuse server is seriously broken */ + pr_warn("fuse: dlm lock request for %lu bytes returned %u bytes\n", + length, outarg.locksize); + fuse_abort_conn(fc); + err = -EINVAL; + } + return err; +} /* * Wait for writeback against this page to complete before allowing it * to be marked dirty again, and hence written back again, possibly @@ -2357,7 +2408,18 @@ static void fuse_vma_close(struct vm_area_struct *vma) static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) { struct folio *folio = page_folio(vmf->page); - struct inode *inode = file_inode(vmf->vma->vm_file); + struct file *file = vmf->vma->vm_file; + struct inode *inode = file_inode(file); + struct fuse_mount *fm = get_fuse_mount(inode); + + if (fm->fc->dlm) { + loff_t pos = vmf->pgoff << PAGE_SHIFT; + size_t length = PAGE_SIZE; + int err = fuse_get_page_mkwrite_lock(file, pos, length); + if (err < 0) { + return vmf_error(err); + } + } file_update_time(vmf->vma->vm_file); folio_lock(folio); diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index a9cdacfb76da48..e3acfb4aa34269 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -1259,6 +1259,7 @@ enum fuse_dlm_lock_type { FUSE_DLM_LOCK_NONE = 0, FUSE_DLM_LOCK_READ = 1, FUSE_DLM_LOCK_WRITE = 2, + FUSE_DLM_PAGE_MKWRITE = 3, }; /** From dbfa78c1ff1e0b88be901fbad32e6e09f46655fe Mon Sep 17 00:00:00 2001 From: Cheng Ding Date: Wed, 16 Jul 2025 03:20:08 +0000 Subject: [PATCH 10/77] fuse: Allow read_folio to retry page fault and read operations Allow read_folio to return EAGAIN error and translate it to AOP_TRUNCATE_PAGE to retry page fault and read operations. This is used to prevent deadlock of folio lock/DLM lock order reversal: - Fault or read operations acquire folio lock first, then DLM lock. - FUSE daemon blocks new DLM lock acquisition while it invalidating page cache. invalidate_inode_pages2_range() acquires folio lock To prevent deadlock, the FUSE daemon will fail its DLM lock acquisition with EAGAIN if it detects an in-flight page cache invalidating operation. Signed-off-by: Cheng Ding (imported from commit 8ecf1182053891c6458b10be1272d2d562492fbd) --- fs/fuse/file.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 75e9116194aaeb..c029641d34f5d3 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -836,8 +836,11 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); res = fuse_simple_request(fm, &ia.ap.args); - if (res < 0) + if (res < 0) { + if (res == -EAGAIN) + res = AOP_TRUNCATED_PAGE; return res; + } /* * Short read means EOF. If file size is larger, truncate it */ From a6dc0c66a609ce3eab2ba8ff380cc64bcd79c788 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 17 Jul 2025 16:26:51 -0700 Subject: [PATCH 11/77] fuse: flush pending fuse events before aborting the connection generic/488 fails with fuse2fs in the following fashion: generic/488 _check_generic_filesystem: filesystem on /dev/sdf is inconsistent (see /var/tmp/fstests/generic/488.full for details) This test opens a large number of files, unlinks them (which really just renames them to fuse hidden files), closes the program, unmounts the filesystem, and runs fsck to check that there aren't any inconsistencies in the filesystem. Unfortunately, the 488.full file shows that there are a lot of hidden files left over in the filesystem, with incorrect link counts. Tracing fuse_request_* shows that there are a large number of FUSE_RELEASE commands that are queued up on behalf of the unlinked files at the time that fuse_conn_destroy calls fuse_abort_conn. Had the connection not aborted, the fuse server would have responded to the RELEASE commands by removing the hidden files; instead they stick around. Create a function to push all the background requests to the queue and then wait for the number of pending events to hit zero, and call this before fuse_abort_conn. That way, all the pending events are processed by the fuse server and we don't end up with a corrupt filesystem. Signed-off-by: Darrick J. Wong (imported from commit d4262f9cf5232394d518207863d1ad79f52b179e) --- fs/fuse/dev.c | 39 ++++++++++++++++++++++++++++++++++++++- fs/fuse/fuse_i.h | 6 ++++++ fs/fuse/inode.c | 1 + 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 1f107b160778fb..c076904e8e4b95 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include "fuse_trace.h" @@ -2445,6 +2445,43 @@ static void end_polls(struct fuse_conn *fc) } } +/* + * Flush all pending requests and wait for them. Only call this function when + * it is no longer possible for other threads to add requests. + */ +void fuse_flush_requests(struct fuse_conn *fc, unsigned long timeout) +{ + unsigned long deadline; + + spin_lock(&fc->lock); + if (!fc->connected) { + spin_unlock(&fc->lock); + return; + } + + /* Push all the background requests to the queue. */ + spin_lock(&fc->bg_lock); + fc->blocked = 0; + fc->max_background = UINT_MAX; + flush_bg_queue(fc); + spin_unlock(&fc->bg_lock); + spin_unlock(&fc->lock); + + /* + * Wait 30s for all the events to complete or abort. Touch the + * watchdog once per second so that we don't trip the hangcheck timer + * while waiting for the fuse server. + */ + deadline = jiffies + timeout; + smp_mb(); + while (fc->connected && + (!timeout || time_before(jiffies, deadline)) && + wait_event_timeout(fc->blocked_waitq, + !fc->connected || atomic_read(&fc->num_waiting) == 0, + HZ) == 0) + touch_softlockup_watchdog(); +} + /* * Abort all requests. * diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 07c9704ac58672..dbefbcf3c14d5f 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -1319,6 +1319,12 @@ void fuse_dentry_tree_cleanup(void); void fuse_epoch_work(struct work_struct *work); +/** + * Flush all pending requests and wait for them. Takes an optional timeout + * in jiffies. + */ +void fuse_flush_requests(struct fuse_conn *fc, unsigned long timeout); + /** * Invalidate inode attributes */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 81c7bfd2184d34..e2563152bd6727 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -2150,6 +2150,7 @@ void fuse_conn_destroy(struct fuse_mount *fm) { struct fuse_conn *fc = fm->fc; + fuse_flush_requests(fc, 30 * HZ); if (fc->destroy) fuse_send_destroy(fm); From 1f531ed564567e342cefdc5e39fa418f57a7932d Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Fri, 18 Jul 2025 17:24:42 +0200 Subject: [PATCH 12/77] fuse: Refactor io-uring bg queue flush and queue abort This is a preparation to allow fuse-io-uring bg queue flush from flush_bg_queue() This does two function renames: fuse_uring_flush_bg -> fuse_uring_flush_queue_bg fuse_uring_abort_end_requests -> fuse_uring_flush_bg And fuse_uring_abort_end_queue_requests() is moved to fuse_uring_stop_queues(). Signed-off-by: Bernd Schubert (imported from commit e70ef24251116bc7f591a9a856c371549cd5ae77) --- fs/fuse/dev_uring.c | 14 +++++++------- fs/fuse/dev_uring_i.h | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index d5737245516b01..04888f8b263592 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -51,7 +51,7 @@ static struct fuse_ring_ent *uring_cmd_to_ring_ent(struct io_uring_cmd *cmd) return pdu->ent; } -static void fuse_uring_flush_bg(struct fuse_ring_queue *queue) +static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) { struct fuse_ring *ring = queue->ring; struct fuse_conn *fc = ring->fc; @@ -93,7 +93,7 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, if (test_bit(FR_BACKGROUND, &req->flags)) { queue->active_background--; spin_lock(&fc->bg_lock); - fuse_uring_flush_bg(queue); + fuse_uring_flush_queue_bg(queue); spin_unlock(&fc->bg_lock); } @@ -122,11 +122,11 @@ static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue) fuse_dev_end_requests(&req_list); } -void fuse_uring_abort_end_requests(struct fuse_ring *ring) +void fuse_uring_flush_bg(struct fuse_conn *fc) { int qid; struct fuse_ring_queue *queue; - struct fuse_conn *fc = ring->fc; + struct fuse_ring *ring = fc->ring; for (qid = 0; qid < ring->nr_queues; qid++) { queue = READ_ONCE(ring->queues[qid]); @@ -138,10 +138,9 @@ void fuse_uring_abort_end_requests(struct fuse_ring *ring) WARN_ON_ONCE(ring->fc->max_background != UINT_MAX); spin_lock(&queue->lock); spin_lock(&fc->bg_lock); - fuse_uring_flush_bg(queue); + fuse_uring_flush_queue_bg(queue); spin_unlock(&fc->bg_lock); spin_unlock(&queue->lock); - fuse_uring_abort_end_queue_requests(queue); } } @@ -498,6 +497,7 @@ void fuse_uring_stop_queues(struct fuse_ring *ring) if (!queue) continue; + fuse_uring_abort_end_queue_requests(queue); fuse_uring_teardown_entries(queue); } @@ -1536,7 +1536,7 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) fc->num_background++; if (fc->num_background == fc->max_background) fc->blocked = 1; - fuse_uring_flush_bg(queue); + fuse_uring_flush_queue_bg(queue); spin_unlock(&fc->bg_lock); /* diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index c89c7dc27c76c1..ea86d4084e7676 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -142,7 +142,7 @@ struct fuse_ring { bool fuse_uring_enabled(void); void fuse_uring_destruct(struct fuse_conn *fc); void fuse_uring_stop_queues(struct fuse_ring *ring); -void fuse_uring_abort_end_requests(struct fuse_ring *ring); +void fuse_uring_flush_bg(struct fuse_conn *fc); int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags); void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req); bool fuse_uring_queue_bq_req(struct fuse_req *req); @@ -157,7 +157,7 @@ static inline void fuse_uring_abort(struct fuse_conn *fc) return; if (atomic_read(&ring->queue_refs) > 0) { - fuse_uring_abort_end_requests(ring); + fuse_uring_flush_bg(fc); fuse_uring_stop_queues(ring); } } From 07447370363cda5d88f7b50beb6fc801b47cf93f Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Fri, 18 Jul 2025 18:24:41 +0200 Subject: [PATCH 13/77] fuse: Flush the io-uring bg queue from fuse_uring_flush_bg This is useful to have a unique API to flush background requests. For example when the bg queue gets flushed before the remaining of fuse_conn_destroy(). Signed-off-by: Bernd Schubert (imported from commit fc4120cc58e7fbcb541bf2e9a72781b569561912) --- fs/fuse/dev.c | 2 ++ fs/fuse/dev_uring.c | 3 +++ fs/fuse/dev_uring_i.h | 8 ++++++++ 3 files changed, 13 insertions(+) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index c076904e8e4b95..d5e62f132e1dba 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -2467,6 +2467,8 @@ void fuse_flush_requests(struct fuse_conn *fc, unsigned long timeout) spin_unlock(&fc->bg_lock); spin_unlock(&fc->lock); + fuse_uring_flush_bg(fc); + /* * Wait 30s for all the events to complete or abort. Touch the * watchdog once per second so that we don't trip the hangcheck timer diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 04888f8b263592..fde1b6100d3218 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -128,6 +128,9 @@ void fuse_uring_flush_bg(struct fuse_conn *fc) struct fuse_ring_queue *queue; struct fuse_ring *ring = fc->ring; + if (!ring) + return; + for (qid = 0; qid < ring->nr_queues; qid++) { queue = READ_ONCE(ring->queues[qid]); if (!queue) diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index ea86d4084e7676..305c5869fde251 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -210,6 +210,14 @@ static inline bool fuse_uring_request_expired(struct fuse_conn *fc) return false; } +static inline bool fuse_uring_request_expired(struct fuse_conn *fc) +{ +} + +static inline void fuse_uring_flush_bg(struct fuse_conn *fc) +{ +} + #endif /* CONFIG_FUSE_IO_URING */ #endif /* _FS_FUSE_DEV_URING_I_H */ From 8c810fa32f95921ed46ddc01ecbd950fc50de10b Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 21 Jul 2025 15:54:09 +0200 Subject: [PATCH 14/77] fuse: fix unnecessary connection abort in dlm lock acquiring When calling the fuse server with a dlm request and the fuse server responds with some other error than ENOSYS most likely the lock size will be set to zero. In that case the kernel will abort the fuse connection. This is completely unnecessary. Signed-off-by: Horst Birthelmer (imported from commit 0bc2f9c39c52ad11a1753e5be376c424b06f43db) --- fs/fuse/fuse_dlm_cache.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index ea947f34a9f70a..a9cad2c1bd2174 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -533,19 +533,19 @@ void fuse_get_dlm_write_lock(struct file *file, loff_t offset, return; } - if (outarg.locksize < end - offset + 1) { - /* fuse server is seriously broken */ - pr_warn("fuse: dlm lock request for %llu bytes returned %u bytes\n", - end - offset + 1, outarg.locksize); - fuse_abort_conn(fc); - return; - } - if (err) return; else - /* ignore any errors here, there is no way we can react appropriately */ - fuse_dlm_lock_range(fi, offset, + if (outarg.locksize < end - offset + 1) { + /* fuse server is seriously broken */ + pr_warn("fuse: dlm lock request for %llu bytes returned %u bytes\n", + end - offset + 1, outarg.locksize); + fuse_abort_conn(fc); + return; + } else { + /* ignore any errors here, there is no way we can react appropriately */ + fuse_dlm_lock_range(fi, offset, offset + outarg.locksize - 1, FUSE_PAGE_LOCK_WRITE); + } } From 1d4a8b3c1847f38f8a2728eacbabcaba0d308ff5 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 21 Jul 2025 18:15:55 +0200 Subject: [PATCH 15/77] fuse: fix connection abort on mmap when fuse server returns ENOSYS Check whether dlm is still enabled when interpreting the returned error from fuse server. Signed-off-by: Horst Birthelmer (imported from commit f6fbf7c7bfb976ae2a30b4d699770a13e699ff04) --- fs/fuse/file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index c029641d34f5d3..bed01bf8121658 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2384,7 +2384,7 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l err = 0; } - if (!err && outarg.locksize < length) { + if (!err && fc->dlm && outarg.locksize < length) { /* fuse server is seriously broken */ pr_warn("fuse: dlm lock request for %lu bytes returned %u bytes\n", length, outarg.locksize); From 3ce2031a3fb71619d72d1b153ace0235be15a882 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Wed, 20 Aug 2025 16:56:43 +0200 Subject: [PATCH 16/77] fuse: change FUSE DLM_LOCK to request start and end of area - Increase the possible lock size to 64 bit. - change semantics of DLM locks to request start and end - change semantics of DLM request return to mark start and end of the locked area - better prepare dlm lock range cache rb-tree for unaligned byte range locks which could return any value as long as it is larger than the range requested - add the case where start and end are zero to destroy the cache Signed-off-by: Horst Birthelmer (imported from commit 87968c738b67b07084b19b5e727074c0604d7ba6) --- fs/fuse/file.c | 13 +++++--- fs/fuse/fuse_dlm_cache.c | 67 +++++++++++++++++++++------------------ fs/fuse/fuse_dlm_cache.h | 12 +++---- fs/fuse/inode.c | 11 ++++--- include/uapi/linux/fuse.h | 11 ++++--- 5 files changed, 64 insertions(+), 50 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index bed01bf8121658..aac4702b08cd77 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2366,8 +2366,8 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; - inarg.offset = offset; - inarg.size = length; + inarg.start = offset; + inarg.end = offset + length - 1; inarg.type = FUSE_DLM_PAGE_MKWRITE; args.opcode = FUSE_DLM_WB_LOCK; @@ -2384,10 +2384,13 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l err = 0; } - if (!err && fc->dlm && outarg.locksize < length) { + if (!err && + fc->dlm && + (outarg.start > inarg.start || + outarg.end < inarg.end)) { /* fuse server is seriously broken */ - pr_warn("fuse: dlm lock request for %lu bytes returned %u bytes\n", - length, outarg.locksize); + pr_warn("fuse: dlm lock request for %llu:%llu bytes returned %llu:%llu bytes\n", + inarg.start, inarg.end, outarg.start, outarg.end); fuse_abort_conn(fc); err = -EINVAL; } diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index a9cad2c1bd2174..d765dd8018cc6a 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -16,11 +16,11 @@ struct fuse_dlm_range { /* Interval tree node */ struct rb_node rb; /* Start page offset (inclusive) */ - pgoff_t start; + uint64_t start; /* End page offset (inclusive) */ - pgoff_t end; + uint64_t end; /* Subtree end value for interval tree */ - pgoff_t __subtree_end; + uint64_t __subtree_end; /* Lock mode */ enum fuse_page_lock_mode mode; /* Temporary list entry for operations */ @@ -32,19 +32,19 @@ struct fuse_dlm_range { #define FUSE_PCACHE_LK_WRITE 2 /* Exclusive write lock */ /* Interval tree definitions for page ranges */ -static inline pgoff_t fuse_dlm_range_start(struct fuse_dlm_range *range) +static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) { return range->start; } -static inline pgoff_t fuse_dlm_range_last(struct fuse_dlm_range *range) +static inline uint64_t fuse_dlm_range_last(struct fuse_dlm_range *range) { return range->end; } -INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, pgoff_t, __subtree_end, - fuse_dlm_range_start, fuse_dlm_range_last, static, - fuse_page_it); +INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, uint64_t, __subtree_end, + fuse_dlm_range_start, fuse_dlm_range_last, static, + fuse_page_it); /** * fuse_page_cache_init - Initialize a page cache lock manager @@ -101,8 +101,8 @@ void fuse_dlm_cache_release_locks(struct fuse_inode *inode) * Return: Pointer to the first overlapping range, or NULL if none found */ static struct fuse_dlm_range * -fuse_dlm_find_overlapping(struct fuse_dlm_cache *cache, pgoff_t start, - pgoff_t end) +fuse_dlm_find_overlapping(struct fuse_dlm_cache *cache, uint64_t start, + uint64_t end) { return fuse_page_it_iter_first(&cache->ranges, start, end); } @@ -116,8 +116,8 @@ fuse_dlm_find_overlapping(struct fuse_dlm_cache *cache, pgoff_t start, * Attempt to merge ranges within and adjacent to the specified region * that have the same lock mode. */ -static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, pgoff_t start, - pgoff_t end) +static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, + uint64_t end) { struct fuse_dlm_range *range, *next; struct rb_node *node; @@ -182,8 +182,8 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, pgoff_t start, * * Return: 0 on success, negative error code on failure */ -int fuse_dlm_lock_range(struct fuse_inode *inode, pgoff_t start, - pgoff_t end, enum fuse_page_lock_mode mode) +int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range, *new_range, *next; @@ -191,7 +191,7 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, pgoff_t start, int ret = 0; LIST_HEAD(to_lock); LIST_HEAD(to_upgrade); - pgoff_t current_start = start; + uint64_t current_start = start; if (!cache || start > end) return -EINVAL; @@ -304,8 +304,8 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, pgoff_t start, * * Return: 0 on success, negative error code on failure */ -static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, pgoff_t start, - pgoff_t end) +static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, + uint64_t end) { struct fuse_dlm_range *range, *new_range; int ret = 0; @@ -363,11 +363,12 @@ static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, pgoff_t start, * @end: End page offset * * Release locks on the specified range of pages. + * Note that if start and end are set to zero the cache is destroyed. * * Return: 0 on success, negative error code on failure */ int fuse_dlm_unlock_range(struct fuse_inode *inode, - pgoff_t start, pgoff_t end) + uint64_t start, uint64_t end) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range, *next; @@ -376,6 +377,11 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, if (!cache) return -EINVAL; + if (start == 0 && end == 0) { + fuse_dlm_cache_release_locks(inode); + return 0; + } + down_write(&cache->lock); /* Find all ranges that overlap with [start, end] */ @@ -424,13 +430,13 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, * * Return: true if the entire range is locked, false otherwise */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, pgoff_t start, - pgoff_t end, enum fuse_page_lock_mode mode) +bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range; int lock_mode = 0; - pgoff_t current_start = start; + uint64_t current_start = start; if (!cache || start > end) return false; @@ -491,7 +497,7 @@ void fuse_get_dlm_write_lock(struct file *file, loff_t offset, struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_mount *fm = ff->fm; - loff_t end = (offset + length - 1) | (PAGE_SIZE - 1); + uint64_t end = (offset + length - 1) | (PAGE_SIZE - 1); /* note that the offset and length don't have to be page aligned here * but since we only get here on writeback caching we will send out @@ -514,8 +520,8 @@ void fuse_get_dlm_write_lock(struct file *file, loff_t offset, memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; - inarg.offset = offset; - inarg.size = end - offset + 1; + inarg.start = offset; + inarg.end = end; inarg.type = FUSE_DLM_LOCK_WRITE; args.opcode = FUSE_DLM_WB_LOCK; @@ -536,16 +542,17 @@ void fuse_get_dlm_write_lock(struct file *file, loff_t offset, if (err) return; else - if (outarg.locksize < end - offset + 1) { + if (inarg.start < outarg.start || + inarg.end > outarg.end) { /* fuse server is seriously broken */ - pr_warn("fuse: dlm lock request for %llu bytes returned %u bytes\n", - end - offset + 1, outarg.locksize); + pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", + inarg.start, inarg.end, outarg.start, outarg.end); fuse_abort_conn(fc); return; } else { /* ignore any errors here, there is no way we can react appropriately */ - fuse_dlm_lock_range(fi, offset, - offset + outarg.locksize - 1, - FUSE_PAGE_LOCK_WRITE); + fuse_dlm_lock_range(fi, outarg.start, + outarg.end, + FUSE_PAGE_LOCK_WRITE); } } diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 98b27a2c15d8ba..438d31d28b666e 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -32,16 +32,16 @@ int fuse_dlm_cache_init(struct fuse_inode *inode); void fuse_dlm_cache_release_locks(struct fuse_inode *inode); /* Lock a range of pages */ -int fuse_dlm_lock_range(struct fuse_inode *inode, pgoff_t start, - pgoff_t end, enum fuse_page_lock_mode mode); +int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode); /* Unlock a range of pages */ -int fuse_dlm_unlock_range(struct fuse_inode *inode, pgoff_t start, - pgoff_t end); +int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end); /* Check if a page range is already locked */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, pgoff_t start, - pgoff_t end, enum fuse_page_lock_mode mode); +bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode); /* this is the interface to the filesystem */ void fuse_get_dlm_write_lock(struct file *file, loff_t offset, diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index e2563152bd6727..7d1f936f8a1ff2 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -626,11 +626,14 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, pg_end = (offset + len - 1) >> PAGE_SHIFT; if (fc->dlm && fc->writeback_cache) - /* invalidate the range from the beginning of the first page - * in the given range to the last byte of the last page */ + /* Invalidate the range exactly as the fuse server requested + * except for the case where it sends -1. + * Note that this can lead to some inconsistencies if + * the fuse server sends unaligned data */ fuse_dlm_unlock_range(fi, - pg_start << PAGE_SHIFT, - (pg_end << PAGE_SHIFT) | (PAGE_SIZE - 1)); + offset, + pg_end == -1 ? 0 : + (offset + len - 1)); invalidate_inode_pages2_range(inode->i_mapping, pg_start, pg_end); diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index e3acfb4aa34269..dd463d13585043 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -1271,10 +1271,10 @@ enum fuse_dlm_lock_type { */ struct fuse_dlm_lock_in { uint64_t fh; - uint64_t offset; - uint32_t size; + uint64_t start; + uint64_t end; uint32_t type; - uint64_t reserved; + uint32_t reserved; }; /** @@ -1284,8 +1284,9 @@ struct fuse_dlm_lock_in { * to reduce number of calls) */ struct fuse_dlm_lock_out { - uint32_t locksize; - uint32_t padding; + uint64_t start; + uint64_t end; + uint64_t reserved; }; /** From 13eef8050601d7f269fed15a26883f1a95a9d520 Mon Sep 17 00:00:00 2001 From: Cheng Ding Date: Wed, 24 Sep 2025 08:12:17 +0000 Subject: [PATCH 17/77] fuse: fix memory leak in fuse-over-io-uring argument copies Fix reference count leak of payload pages during fuse argument copies. Signed-off-by: Cheng Ding (imported from commit 8b75cf05a2efc20e8f46ba9e10664c502249ee21) --- fs/fuse/dev_uring.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index fde1b6100d3218..9c72223678e0be 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -674,11 +674,14 @@ static int fuse_uring_args_to_ring_pages(struct fuse_ring *ring, (struct fuse_arg *)in_args, 0); if (err) { pr_info_ratelimited("%s fuse_copy_args failed\n", __func__); - return err; + goto copy_finish; } ent_in_out.payload_sz = cs.ring.copied_sz; memcpy(&headers->ring_ent_in_out, &ent_in_out, sizeof(ent_in_out)); + +copy_finish: + fuse_copy_finish(&cs); return err; } @@ -735,12 +738,14 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, fuse_copy_finish(&cs); if (err) { pr_info_ratelimited("%s fuse_copy_args failed\n", __func__); - return err; + goto copy_finish; } ent_in_out.payload_sz = cs.ring.copied_sz; err = copy_to_user(&ent->headers->ring_ent_in_out, &ent_in_out, sizeof(ent_in_out)); +copy_finish: + fuse_copy_finish(&cs); return err ? -EFAULT : 0; } From cdd444eb3b074e2d5a678a9b00e81d72498fd526 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Mon, 2 Jun 2025 23:23:43 +0200 Subject: [PATCH 18/77] fuse: {io-uring} Add queue length counters This is another preparation and will be used for decision which queue to add a request to. Signed-off-by: Bernd Schubert Reviewed-by: Joanne Koong (imported from commit e4698faf912435f7f3f28c169f7bb8342d7b1edf) --- fs/fuse/dev_uring.c | 17 +++++++++++++++-- fs/fuse/dev_uring_i.h | 3 +++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 9c72223678e0be..9bb6573c826173 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -89,6 +89,7 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, lockdep_assert_not_held(&queue->lock); spin_lock(&queue->lock); ent->fuse_req = NULL; + queue->nr_reqs--; list_del_init(&req->list); if (test_bit(FR_BACKGROUND, &req->flags)) { queue->active_background--; @@ -96,7 +97,6 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, fuse_uring_flush_queue_bg(queue); spin_unlock(&fc->bg_lock); } - spin_unlock(&queue->lock); if (error) @@ -116,6 +116,7 @@ static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue) list_for_each_entry(req, &queue->fuse_req_queue, list) clear_bit(FR_PENDING, &req->flags); list_splice_init(&queue->fuse_req_queue, &req_list); + queue->nr_reqs = 0; spin_unlock(&queue->lock); /* must not hold queue lock to avoid order issues with fi->lock */ @@ -1498,10 +1499,13 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) req->ring_queue = queue; ent = list_first_entry_or_null(&queue->ent_avail_queue, struct fuse_ring_ent, list); + queue->nr_reqs++; + if (ent) fuse_uring_add_req_to_ring_ent(ent, req); else list_add_tail(&req->list, &queue->fuse_req_queue); + spin_unlock(&queue->lock); if (ent) @@ -1537,6 +1541,7 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) set_bit(FR_URING, &req->flags); req->ring_queue = queue; list_add_tail(&req->list, &queue->fuse_req_bg_queue); + queue->nr_reqs++; ent = list_first_entry_or_null(&queue->ent_avail_queue, struct fuse_ring_ent, list); @@ -1569,8 +1574,16 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) bool fuse_uring_remove_pending_req(struct fuse_req *req) { struct fuse_ring_queue *queue = req->ring_queue; + bool removed = fuse_remove_pending_req(req, &queue->lock); + + if (removed) { + /* Update counters after successful removal */ + spin_lock(&queue->lock); + queue->nr_reqs--; + spin_unlock(&queue->lock); + } - return fuse_remove_pending_req(req, &queue->lock); + return removed; } static const struct fuse_iqueue_ops fuse_io_uring_ops = { diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 305c5869fde251..f4e707a6711138 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -98,6 +98,9 @@ struct fuse_ring_queue { /* background fuse requests */ struct list_head fuse_req_bg_queue; + /* number of requests queued or in userspace */ + unsigned int nr_reqs; + struct fuse_pqueue fpq; unsigned int active_background; From 9f6de8b57a2b3bf8acb377f3fe3e2759743e9be7 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Fri, 13 Jun 2025 15:12:47 +0200 Subject: [PATCH 19/77] fuse: {io-uring} Rename ring->nr_queues to max_nr_queues This is preparation for follow up commits that allow to run with a reduced number of queues. Signed-off-by: Bernd Schubert (imported from commit 2e27c33ffcf65b434ada1364a4d2ea92b094f0c3) --- fs/fuse/dev_uring.c | 22 +++++++++++----------- fs/fuse/dev_uring_i.h | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 9bb6573c826173..7e1a51d5037bce 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -132,7 +132,7 @@ void fuse_uring_flush_bg(struct fuse_conn *fc) if (!ring) return; - for (qid = 0; qid < ring->nr_queues; qid++) { + for (qid = 0; qid < ring->max_nr_queues; qid++) { queue = READ_ONCE(ring->queues[qid]); if (!queue) continue; @@ -214,7 +214,7 @@ void fuse_uring_destruct(struct fuse_conn *fc) if (!ring) return; - for (qid = 0; qid < ring->nr_queues; qid++) { + for (qid = 0; qid < ring->max_nr_queues; qid++) { struct fuse_ring_queue *queue = ring->queues[qid]; struct fuse_ring_ent *ent, *next; @@ -277,7 +277,7 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc) init_waitqueue_head(&ring->stop_waitq); - ring->nr_queues = nr_queues; + ring->max_nr_queues = nr_queues; ring->fc = fc; ring->max_payload_sz = max_payload_size; smp_store_release(&fc->ring, ring); @@ -429,7 +429,7 @@ static void fuse_uring_log_ent_state(struct fuse_ring *ring) int qid; struct fuse_ring_ent *ent; - for (qid = 0; qid < ring->nr_queues; qid++) { + for (qid = 0; qid < ring->max_nr_queues; qid++) { struct fuse_ring_queue *queue = ring->queues[qid]; if (!queue) @@ -460,7 +460,7 @@ static void fuse_uring_async_stop_queues(struct work_struct *work) container_of(work, struct fuse_ring, async_teardown_work.work); /* XXX code dup */ - for (qid = 0; qid < ring->nr_queues; qid++) { + for (qid = 0; qid < ring->max_nr_queues; qid++) { struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); if (!queue) @@ -495,7 +495,7 @@ void fuse_uring_stop_queues(struct fuse_ring *ring) { int qid; - for (qid = 0; qid < ring->nr_queues; qid++) { + for (qid = 0; qid < ring->max_nr_queues; qid++) { struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); if (!queue) @@ -988,7 +988,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (!ring) return err; - if (qid >= ring->nr_queues) + if (qid >= ring->max_nr_queues) return -EINVAL; queue = ring->queues[qid]; @@ -1051,7 +1051,7 @@ static bool is_ring_ready(struct fuse_ring *ring, int current_qid) struct fuse_ring_queue *queue; bool ready = true; - for (qid = 0; qid < ring->nr_queues && ready; qid++) { + for (qid = 0; qid < ring->max_nr_queues && ready; qid++) { if (current_qid == qid) continue; @@ -1291,7 +1291,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, return err; } - if (qid >= ring->nr_queues) { + if (qid >= ring->max_nr_queues) { pr_info_ratelimited("fuse: Invalid ring qid %u\n", qid); return -EINVAL; } @@ -1436,9 +1436,9 @@ static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) qid = task_cpu(current); - if (WARN_ONCE(qid >= ring->nr_queues, + if (WARN_ONCE(qid >= ring->max_nr_queues, "Core number (%u) exceeds nr queues (%zu)\n", qid, - ring->nr_queues)) + ring->max_nr_queues)) qid = 0; queue = ring->queues[qid]; diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index f4e707a6711138..0a5e826100585f 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -117,7 +117,7 @@ struct fuse_ring { struct fuse_conn *fc; /* number of ring queues */ - size_t nr_queues; + size_t max_nr_queues; /* maximum payload/arg size */ size_t max_payload_sz; From 5a0f4bfbad64c9854cba9b97a829d45f7796da93 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Tue, 10 Jun 2025 16:23:28 +0200 Subject: [PATCH 20/77] fuse: {io-uring} Use bitmaps to track registered queues Add per-CPU and per-NUMA node bitmasks to track which io-uring queues are registered. Signed-off-by: Bernd Schubert (imported from commit be6edce441ecc37ee34a8937f07c01ab99bfb7f7) --- fs/fuse/dev_uring.c | 79 +++++++++++++++++++++++++++++++++++++++++-- fs/fuse/dev_uring_i.h | 20 +++++++++++ 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 7e1a51d5037bce..dbbf3a7f949614 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -163,6 +163,24 @@ static void io_pages_free(struct page ***pages, int npages) *pages = NULL; } + +static void fuse_ring_destruct_q_map(struct fuse_queue_map *q_map) +{ + free_cpumask_var(q_map->registered_q_mask); + kfree(q_map->cpu_to_qid); +} + +static void fuse_uring_destruct_q_masks(struct fuse_ring *ring) +{ + int node; + + fuse_ring_destruct_q_map(&ring->q_map); + + if (ring->numa_q_map) + for (node = 0; node < ring->nr_numa_nodes; node++) + fuse_ring_destruct_q_map(&ring->numa_q_map[node]); +} + static bool ent_list_request_expired(struct fuse_conn *fc, struct list_head *list) { struct fuse_ring_ent *ent; @@ -187,7 +205,7 @@ bool fuse_uring_request_expired(struct fuse_conn *fc) if (!ring) return false; - for (qid = 0; qid < ring->nr_queues; qid++) { + for (qid = 0; qid < ring->max_nr_queues; qid++) { queue = READ_ONCE(ring->queues[qid]); if (!queue) continue; @@ -240,11 +258,45 @@ void fuse_uring_destruct(struct fuse_conn *fc) ring->queues[qid] = NULL; } + fuse_uring_destruct_q_masks(ring); kfree(ring->queues); kfree(ring); fc->ring = NULL; } +static int fuse_uring_init_q_map(struct fuse_queue_map *q_map, size_t nr_cpu) +{ + if (!zalloc_cpumask_var(&q_map->registered_q_mask, GFP_KERNEL_ACCOUNT)) + return -ENOMEM; + + q_map->cpu_to_qid = kcalloc(nr_cpu, sizeof(*q_map->cpu_to_qid), + GFP_KERNEL_ACCOUNT); + + return 0; +} + +static int fuse_uring_create_q_masks(struct fuse_ring *ring) +{ + int err, node; + + err = fuse_uring_init_q_map(&ring->q_map, ring->max_nr_queues); + if (err) + return err; + + ring->numa_q_map = kcalloc(ring->nr_numa_nodes, + sizeof(*ring->numa_q_map), + GFP_KERNEL_ACCOUNT); + if (!ring->numa_q_map) + return -ENOMEM; + for (node = 0; node < ring->nr_numa_nodes; node++) { + err = fuse_uring_init_q_map(&ring->numa_q_map[node], + ring->max_nr_queues); + if (err) + return err; + } + return 0; +} + /* * Basic ring setup for this connection based on the provided configuration */ @@ -254,19 +306,26 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc) size_t nr_queues = num_possible_cpus(); struct fuse_ring *res = NULL; size_t max_payload_size; + int err; ring = kzalloc_obj(*fc->ring, GFP_KERNEL_ACCOUNT); if (!ring) return NULL; - ring->queues = kzalloc_objs(struct fuse_ring_queue *, nr_queues, - GFP_KERNEL_ACCOUNT); + ring->nr_numa_nodes = num_online_nodes(); + + ring->queues = kcalloc(nr_queues, sizeof(struct fuse_ring_queue *), + GFP_KERNEL_ACCOUNT); if (!ring->queues) goto out_err; max_payload_size = max(FUSE_MIN_READ_BUFFER, fc->max_write); max_payload_size = max(max_payload_size, fc->max_pages * PAGE_SIZE); + err = fuse_uring_create_q_masks(ring); + if (err) + goto out_err; + spin_lock(&fc->lock); if (fc->ring) { /* race, another thread created the ring in the meantime */ @@ -286,6 +345,7 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc) return ring; out_err: + fuse_uring_destruct_q_masks(ring); kfree(ring->queues); kfree(ring); return res; @@ -448,6 +508,7 @@ static void fuse_uring_log_ent_state(struct fuse_ring *ring) pr_info(" ent-commit-queue ring=%p qid=%d ent=%p state=%d\n", ring, qid, ent, ent->state); } + spin_unlock(&queue->lock); } ring->stop_debug_log = 1; @@ -494,6 +555,7 @@ static void fuse_uring_async_stop_queues(struct work_struct *work) void fuse_uring_stop_queues(struct fuse_ring *ring) { int qid; + int node; for (qid = 0; qid < ring->max_nr_queues; qid++) { struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); @@ -505,6 +567,13 @@ void fuse_uring_stop_queues(struct fuse_ring *ring) fuse_uring_teardown_entries(queue); } + /* Reset all queue masks, we won't process any more IO */ + cpumask_clear(ring->q_map.registered_q_mask); + for (node = 0; node < ring->nr_numa_nodes; node++) { + if (ring->numa_q_map) + cpumask_clear(ring->numa_q_map[node].registered_q_mask); + } + if (atomic_read(&ring->queue_refs) > 0) { ring->teardown_time = jiffies; INIT_DELAYED_WORK(&ring->async_teardown_work, @@ -1081,6 +1150,10 @@ static void fuse_uring_do_register(struct fuse_ring_ent *ent, struct fuse_ring *ring = queue->ring; struct fuse_conn *fc = ring->fc; struct fuse_iqueue *fiq = &fc->iq; + int node = cpu_to_node(queue->qid); + + if (WARN_ON_ONCE(node >= ring->nr_numa_nodes)) + node = 0; fuse_uring_prepare_cancel(cmd, issue_flags, ent); diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 0a5e826100585f..86fef37a863a1e 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -108,6 +108,17 @@ struct fuse_ring_queue { bool stopped; }; +struct fuse_queue_map { + /* Tracks which queues are registered */ + cpumask_var_t registered_q_mask; + + /* number of registered queues */ + size_t nr_queues; + + /* cpu to qid mapping */ + int *cpu_to_qid; +}; + /** * Describes if uring is for communication and holds alls the data needed * for uring communication @@ -119,6 +130,9 @@ struct fuse_ring { /* number of ring queues */ size_t max_nr_queues; + /* number of numa nodes */ + int nr_numa_nodes; + /* maximum payload/arg size */ size_t max_payload_sz; @@ -129,6 +143,12 @@ struct fuse_ring { */ unsigned int stop_debug_log : 1; + /* per numa node queue tracking */ + struct fuse_queue_map *numa_q_map; + + /* all queue tracking */ + struct fuse_queue_map q_map; + wait_queue_head_t stop_waitq; /* async tear down */ From a7f6de4e26a740eb9d5b0eb5adfdbd528094e926 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Wed, 4 Jun 2025 19:32:39 +0200 Subject: [PATCH 21/77] fuse: {io-uring} Allow reduced number of ring queues Queues selection (fuse_uring_get_queue) can handle reduced number queues - using io-uring is possible now even with a single queue and entry. The FUSE_URING_REDUCED_Q flag is being introduce tell fuse server that reduced queues are possible, i.e. if the flag is set, fuse server is free to reduce number queues. Signed-off-by: Bernd Schubert (imported from commit f620f3d35969bd9a04304b757a18a11a0787dedc) --- fs/fuse/dev_uring.c | 124 +++++++++++++++++++++++--------------- fs/fuse/inode.c | 8 +-- include/uapi/linux/fuse.h | 4 ++ 3 files changed, 84 insertions(+), 52 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index dbbf3a7f949614..b3927356be722a 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -271,15 +271,17 @@ static int fuse_uring_init_q_map(struct fuse_queue_map *q_map, size_t nr_cpu) q_map->cpu_to_qid = kcalloc(nr_cpu, sizeof(*q_map->cpu_to_qid), GFP_KERNEL_ACCOUNT); + if (!q_map->cpu_to_qid) + return -ENOMEM; return 0; } -static int fuse_uring_create_q_masks(struct fuse_ring *ring) +static int fuse_uring_create_q_masks(struct fuse_ring *ring, size_t nr_queues) { int err, node; - err = fuse_uring_init_q_map(&ring->q_map, ring->max_nr_queues); + err = fuse_uring_init_q_map(&ring->q_map, nr_queues); if (err) return err; @@ -290,7 +292,7 @@ static int fuse_uring_create_q_masks(struct fuse_ring *ring) return -ENOMEM; for (node = 0; node < ring->nr_numa_nodes; node++) { err = fuse_uring_init_q_map(&ring->numa_q_map[node], - ring->max_nr_queues); + nr_queues); if (err) return err; } @@ -322,7 +324,7 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc) max_payload_size = max(FUSE_MIN_READ_BUFFER, fc->max_write); max_payload_size = max(max_payload_size, fc->max_pages * PAGE_SIZE); - err = fuse_uring_create_q_masks(ring); + err = fuse_uring_create_q_masks(ring, nr_queues); if (err) goto out_err; @@ -351,12 +353,37 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc) return res; } +static void fuse_uring_cpu_qid_mapping(struct fuse_ring *ring, int qid, + struct fuse_queue_map *q_map) +{ + int cpu, qid_idx; + size_t nr_queues; + + cpumask_set_cpu(qid, q_map->registered_q_mask); + nr_queues = cpumask_weight(q_map->registered_q_mask); + for (cpu = 0; cpu < ring->max_nr_queues; cpu++) { + if (!q_map->cpu_to_qid) + return; + + /* + * Position of this CPU within the registered queue mask, + * handles non-contiguous CPU distributions across NUMA nodes. + */ + qid_idx = bitmap_weight( + cpumask_bits(q_map->registered_q_mask), cpu); + + q_map->cpu_to_qid[cpu] = cpumask_nth(qid_idx % nr_queues, + q_map->registered_q_mask); + } +} + static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, int qid) { struct fuse_conn *fc = ring->fc; struct fuse_ring_queue *queue; struct list_head *pq; + int node; queue = kzalloc_obj(*queue, GFP_KERNEL_ACCOUNT); if (!queue) @@ -394,6 +421,22 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, * write_once and lock as the caller mostly doesn't take the lock at all */ WRITE_ONCE(ring->queues[qid], queue); + + /* Static mapping from cpu to per numa queues */ + node = cpu_to_node(qid); + fuse_uring_cpu_qid_mapping(ring, qid, &ring->numa_q_map[node]); + + /* + * smp_store_release, as the variable is read without fc->lock and + * we need to avoid compiler re-ordering of updating the nr_queues + * and setting ring->numa_queues[node].cpu_to_qid above + */ + smp_store_release (&ring->numa_q_map[node].nr_queues, + ring->numa_q_map[node].nr_queues + 1); + + /* global mapping */ + fuse_uring_cpu_qid_mapping(ring, qid, &ring->q_map); + spin_unlock(&fc->lock); return queue; @@ -1114,31 +1157,6 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, return 0; } -static bool is_ring_ready(struct fuse_ring *ring, int current_qid) -{ - int qid; - struct fuse_ring_queue *queue; - bool ready = true; - - for (qid = 0; qid < ring->max_nr_queues && ready; qid++) { - if (current_qid == qid) - continue; - - queue = ring->queues[qid]; - if (!queue) { - ready = false; - break; - } - - spin_lock(&queue->lock); - if (list_empty(&queue->ent_avail_queue)) - ready = false; - spin_unlock(&queue->lock); - } - - return ready; -} - /* * fuse_uring_req_fetch command handling */ @@ -1163,13 +1181,9 @@ static void fuse_uring_do_register(struct fuse_ring_ent *ent, spin_unlock(&queue->lock); if (!ring->ready) { - bool ready = is_ring_ready(ring, queue->qid); - - if (ready) { - WRITE_ONCE(fiq->ops, &fuse_io_uring_ops); - WRITE_ONCE(ring->ready, true); - wake_up_all(&fc->blocked_waitq); - } + WRITE_ONCE(fiq->ops, &fuse_io_uring_ops); + WRITE_ONCE(ring->ready, true); + wake_up_all(&fc->blocked_waitq); } } @@ -1502,22 +1516,36 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) fuse_uring_send(ent, cmd, err, issue_flags); } -static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) +static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring) { unsigned int qid; - struct fuse_ring_queue *queue; + int node; + unsigned int nr_queues; + unsigned int cpu = task_cpu(current); - qid = task_cpu(current); + cpu = cpu % ring->max_nr_queues; - if (WARN_ONCE(qid >= ring->max_nr_queues, - "Core number (%u) exceeds nr queues (%zu)\n", qid, - ring->max_nr_queues)) - qid = 0; + /* numa local registered queue bitmap */ + node = cpu_to_node(cpu); + if (WARN_ONCE(node >= ring->nr_numa_nodes, + "Node number (%d) exceeds nr nodes (%d)\n", + node, ring->nr_numa_nodes)) { + node = 0; + } - queue = ring->queues[qid]; - WARN_ONCE(!queue, "Missing queue for qid %d\n", qid); + nr_queues = READ_ONCE(ring->numa_q_map[node].nr_queues); + if (nr_queues) { + qid = ring->numa_q_map[node].cpu_to_qid[cpu]; + if (WARN_ON_ONCE(qid >= ring->max_nr_queues)) + return NULL; + return READ_ONCE(ring->queues[qid]); + } - return queue; + /* global registered queue bitmap */ + qid = ring->q_map.cpu_to_qid[cpu]; + if (WARN_ON_ONCE(qid >= ring->max_nr_queues)) + return NULL; + return READ_ONCE(ring->queues[qid]); } static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg) @@ -1557,7 +1585,7 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) int err; err = -EINVAL; - queue = fuse_uring_task_to_queue(ring); + queue = fuse_uring_select_queue(ring); if (!queue) goto err; @@ -1601,7 +1629,7 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) struct fuse_ring_queue *queue; struct fuse_ring_ent *ent = NULL; - queue = fuse_uring_task_to_queue(ring); + queue = fuse_uring_select_queue(ring); if (!queue) return false; diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 7d1f936f8a1ff2..7e19accd8f27f2 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1557,8 +1557,7 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) ia->in.major = FUSE_KERNEL_VERSION; ia->in.minor = FUSE_KERNEL_MINOR_VERSION; ia->in.max_readahead = fm->sb->s_bdi->ra_pages * PAGE_SIZE; - flags = - FUSE_ASYNC_READ | FUSE_POSIX_LOCKS | FUSE_ATOMIC_O_TRUNC | + flags = FUSE_ASYNC_READ | FUSE_POSIX_LOCKS | FUSE_ATOMIC_O_TRUNC | FUSE_EXPORT_SUPPORT | FUSE_BIG_WRITES | FUSE_DONT_MASK | FUSE_SPLICE_WRITE | FUSE_SPLICE_MOVE | FUSE_SPLICE_READ | FUSE_FLOCK_LOCKS | FUSE_HAS_IOCTL_DIR | FUSE_AUTO_INVAL_DATA | @@ -1570,8 +1569,9 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) FUSE_HANDLE_KILLPRIV_V2 | FUSE_SETXATTR_EXT | FUSE_INIT_EXT | FUSE_SECURITY_CTX | FUSE_CREATE_SUPP_GROUP | FUSE_HAS_EXPIRE_ONLY | FUSE_DIRECT_IO_ALLOW_MMAP | - FUSE_INVAL_INODE_ENTRY | FUSE_EXPIRE_INODE_ENTRY | - FUSE_NO_EXPORT_SUPPORT | FUSE_HAS_RESEND | FUSE_ALLOW_IDMAP | + FUSE_NO_EXPORT_SUPPORT | FUSE_INVAL_INODE_ENTRY | + FUSE_EXPIRE_INODE_ENTRY | FUSE_URING_REDUCED_Q | + FUSE_EXPIRE_INODE_ENTRY | FUSE_REQUEST_TIMEOUT; #ifdef CONFIG_FUSE_DAX if (fm->fc->dax) diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index dd463d13585043..605c755c8c6331 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -450,6 +450,8 @@ struct fuse_file_lock { * init_out.request_timeout contains the timeout (in secs) * FUSE_INVAL_INODE_ENTRY: invalidate inode aliases when doing inode invalidation * FUSE_EXPIRE_INODE_ENTRY: expire inode aliases when doing inode invalidation + * FUSE_URING_REDUCED_Q: Client (kernel) supports less queues - Server is free + * to register between 1 and nr-core io-uring queues */ #define FUSE_ASYNC_READ (1 << 0) #define FUSE_POSIX_LOCKS (1 << 1) @@ -497,6 +499,8 @@ struct fuse_file_lock { #define FUSE_ALLOW_IDMAP (1ULL << 40) #define FUSE_OVER_IO_URING (1ULL << 41) #define FUSE_REQUEST_TIMEOUT (1ULL << 42) +#define FUSE_ALIGN_PG_ORDER (1ULL << 50) +#define FUSE_URING_REDUCED_Q (1ULL << 59) #define FUSE_INVAL_INODE_ENTRY (1ULL << 60) #define FUSE_EXPIRE_INODE_ENTRY (1ULL << 61) From 1fe1b00840f41b8c66f18abae473290f6188e9a4 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Wed, 24 Sep 2025 19:14:19 +0200 Subject: [PATCH 22/77] fuse: {io-uring} Queue background requests on a different core Running background IO on a different core makes quite a difference. fio --directory=/tmp/dest --name=iops.\$jobnum --rw=randread \ --bs=4k --size=1G --numjobs=1 --iodepth=4 --time_based\ --runtime=30s --group_reporting --ioengine=io_uring\ --direct=1 unpatched READ: bw=272MiB/s (285MB/s) ... patched READ: bw=650MiB/s (682MB/s) Reason is easily visible, the fio process is migrating between CPUs when requests are submitted on the queue for the same core. With --iodepth=8 unpatched READ: bw=466MiB/s (489MB/s) patched READ: bw=641MiB/s (672MB/s) Without io-uring (--iodepth=8) READ: bw=729MiB/s (764MB/s) Without fuse (--iodepth=8) READ: bw=2199MiB/s (2306MB/s) (Test were done with /example/passthrough_hp -o allow_other --nopassthrough \ [-o io_uring] /tmp/source /tmp/dest ) Additional notes: With FURING_NEXT_QUEUE_RETRIES=0 (--iodepth=8) READ: bw=903MiB/s (946MB/s) With just a random qid (--iodepth=8) READ: bw=429MiB/s (450MB/s) With --iodepth=1 unpatched READ: bw=195MiB/s (204MB/s) patched READ: bw=232MiB/s (243MB/s) With --iodepth=1 --numjobs=2 unpatched READ: bw=366MiB/s (384MB/s) patched READ: bw=472MiB/s (495MB/s) With --iodepth=1 --numjobs=8 unpatched READ: bw=1437MiB/s (1507MB/s) patched READ: bw=1529MiB/s (1603MB/s) fuse without io-uring READ: bw=1314MiB/s (1378MB/s), 1314MiB/s-1314MiB/s ... no-fuse READ: bw=2566MiB/s (2690MB/s), 2566MiB/s-2566MiB/s ... In summary, for async requests the core doing application IO is busy sending requests and processing IOs should be done on a different core. Spreading the load on random cores is also not desirable, as the core might be frequency scaled down and/or in C1 sleep states. Not shown here, but differnces are much smaller when the system uses performance govenor instead of schedutil (ubuntu default). Obviously at the cost of higher system power consumption for performance govenor - not desirable either. Results without io-uring (which uses fixed libfuse threads per queue) heavily depend on the current number of active threads. Libfuse uses default of max 10 threads, but actual nr max threads is a parameter. Also, no-fuse-io-uring results heavily depend on, if there was already running another workload before, as libfuse starts these threads dynamically - i.e. the more threads are active, the worse the performance. Signed-off-by: Bernd Schubert (imported from commit c6399ea79b104ac79758f2c36f1977b80a02358d) --- fs/fuse/dev_uring.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index b3927356be722a..e9dfff3c912851 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1516,13 +1516,21 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) fuse_uring_send(ent, cmd, err, issue_flags); } -static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring) +static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring, + bool background) { unsigned int qid; int node; unsigned int nr_queues; unsigned int cpu = task_cpu(current); + /* + * Background requests result in better performance on a different + * CPU, unless CPUs are already busy. + */ + if (background) + cpu++; + cpu = cpu % ring->max_nr_queues; /* numa local registered queue bitmap */ @@ -1585,7 +1593,7 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) int err; err = -EINVAL; - queue = fuse_uring_select_queue(ring); + queue = fuse_uring_select_queue(ring, false); if (!queue) goto err; @@ -1629,7 +1637,7 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req) struct fuse_ring_queue *queue; struct fuse_ring_ent *ent = NULL; - queue = fuse_uring_select_queue(ring); + queue = fuse_uring_select_queue(ring, true); if (!queue) return false; From 489b160a640a8570e1b9888f57d79dc4fd75793b Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Fri, 24 Oct 2025 19:05:07 +0200 Subject: [PATCH 23/77] fuse: Add retry attempts for numa local queues for load distribution This is to further improve performance. fio --directory=/tmp/dest --name=iops.\$jobnum --rw=randread \ --bs=4k --size=1G --numjobs=1 --iodepth=4 --time_based\ --runtime=30s --group_reporting --ioengine=io_uring\ --direct=1 unpatched READ: bw=650MiB/s (682MB/s) patched: READ: bw=995MiB/s (1043MB/s) with --iodepth=8 unpatched READ: bw=641MiB/s (672MB/s) patched READ: bw=966MiB/s (1012MB/s) Reason is that with --iodepth=x (x > 1) fio submits multiple async requests and a single queue might become CPU limited. I.e. spreading the load helps. (imported from commit 2e73b0be1f55d61c2d861a12bf6bb9963b9b877a) --- fs/fuse/dev_uring.c | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index e9dfff3c912851..f99418cfa698dd 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -22,6 +22,8 @@ MODULE_PARM_DESC(enable_uring, #define FUSE_RING_HEADER_PG 0 #define FUSE_RING_PAYLOAD_PG 1 +#define FUSE_URING_Q_THRESHOLD 2 + bool fuse_uring_enabled(void) { @@ -1520,9 +1522,10 @@ static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring, bool background) { unsigned int qid; - int node; + int node, retries = 0; unsigned int nr_queues; unsigned int cpu = task_cpu(current); + struct fuse_ring_queue *queue, *primary_queue = NULL; /* * Background requests result in better performance on a different @@ -1531,6 +1534,7 @@ static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring, if (background) cpu++; +retry: cpu = cpu % ring->max_nr_queues; /* numa local registered queue bitmap */ @@ -1546,12 +1550,35 @@ static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring, qid = ring->numa_q_map[node].cpu_to_qid[cpu]; if (WARN_ON_ONCE(qid >= ring->max_nr_queues)) return NULL; - return READ_ONCE(ring->queues[qid]); + queue = READ_ONCE(ring->queues[qid]); + + /* Might happen on teardown */ + if (unlikely(!queue)) + return NULL; + + if (queue->nr_reqs < FUSE_URING_Q_THRESHOLD) + return queue; + + /* Retries help for load balancing */ + if (retries < FUSE_URING_Q_THRESHOLD) { + if (!retries) + primary_queue = queue; + + /* Increase cpu, assuming it will map to a differet qid*/ + cpu++; + retries++; + goto retry; + } } + /* Retries exceeded, take the primary target queue */ + if (primary_queue) + return primary_queue; + /* global registered queue bitmap */ qid = ring->q_map.cpu_to_qid[cpu]; if (WARN_ON_ONCE(qid >= ring->max_nr_queues)) + /* Might happen on teardown */ return NULL; return READ_ONCE(ring->queues[qid]); } From 7f65f763e84b5aa3d81a28839193cee68ba3fc29 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Mon, 10 Nov 2025 13:17:38 +0100 Subject: [PATCH 24/77] fuse: Fetch a queued fuse request on command registration With the reduced queue feature io-uring is marked as ready after receiving the 1st ring entry. At this time other queues just might be in the process of registration and then a race happens fuse_uring_queue_fuse_req -> no queue entry registered yet list_add_tail -> fuse request gets queued So far fetching requests from the list only happened from FUSE_IO_URING_CMD_COMMIT_AND_FETCH, but without new requests on the same queue, it would actually never send requests from that queue - the request was stuck. (imported from commit 3bfb6cdc9b978a13eab59ebae592ddfa225c4c4a) --- fs/fuse/dev_uring.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index f99418cfa698dd..f88cc4c94aba8e 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1403,6 +1403,8 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, fuse_uring_do_register(ent, cmd, issue_flags); + fuse_uring_next_fuse_req(ent, queue, issue_flags); + return 0; } From 73c9855ebfdbb89a77b9301b11d47f4d2a51ff27 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 16 Sep 2025 13:31:45 +0200 Subject: [PATCH 25/77] fuse: add compound command to combine multiple requests fuse.h: add new opcode FUSE_COMPOUND fuse_compound.c: add new functionality to pack multiple fuse operations into one compound command file.c: add an implementation of open+getattr Signed-off-by: Horst Birthelmer (imported from commit d9e735140a3faccbe5786a7e75a4ad9a6a9aa2e0) (imported from commit 1607a03696693c4ceef7a61adf5759748a7ca9b0) (imported from commit 9df5e4cb96184aae03d7d49131b59a4767641d6b) (imported from commit 9921bcdc4e126a7606e036b04893a6bfd36b8c75) (imported from commit 09d6f59e98090b4de35bfe5344fd1ca5559d1c16) (imported from commit 41b40bdc0739af60f3fbabb4dd45006f801ebd0d) --- fs/fuse/Makefile | 2 +- fs/fuse/compound.c | 263 ++++++++++++++++++++++++++++++++++++++ fs/fuse/dev.c | 24 ++++ fs/fuse/dir.c | 9 +- fs/fuse/file.c | 154 ++++++++++++++++++---- fs/fuse/fuse_i.h | 24 +++- fs/fuse/inode.c | 6 + fs/fuse/ioctl.c | 2 +- include/uapi/linux/fuse.h | 38 ++++++ 9 files changed, 487 insertions(+), 35 deletions(-) create mode 100644 fs/fuse/compound.c diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile index 64bc8682ae9659..2407870803000d 100644 --- a/fs/fuse/Makefile +++ b/fs/fuse/Makefile @@ -11,7 +11,7 @@ obj-$(CONFIG_CUSE) += cuse.o obj-$(CONFIG_VIRTIO_FS) += virtiofs.o fuse-y := trace.o # put trace.o first so we see ftrace errors sooner -fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o +fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o compound.o fuse-y += iomode.o fuse-$(CONFIG_FUSE_DAX) += dax.o fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o backing.o diff --git a/fs/fuse/compound.c b/fs/fuse/compound.c new file mode 100644 index 00000000000000..bc52e22eff3123 --- /dev/null +++ b/fs/fuse/compound.c @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE: Filesystem in Userspace + * Copyright (C) 2025 + * + * This file implements compound operations for FUSE, allowing multiple + * operations to be batched into a single request to reduce round trips + * between kernel and userspace. + */ + +#include "fuse_i.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Compound request builder and state tracker and args pointer storage + */ +struct fuse_compound_req { + struct fuse_mount *fm; + struct fuse_compound_in compound_header; + struct fuse_compound_out result_header; + + /* Per-operation error codes */ + int op_errors[FUSE_MAX_COMPOUND_OPS]; + struct fuse_args *op_args[FUSE_MAX_COMPOUND_OPS]; +}; + +struct fuse_compound_req *fuse_compound_alloc(struct fuse_mount *fm, u32 flags) +{ + struct fuse_compound_req *compound; + + compound = kzalloc(sizeof(*compound), GFP_KERNEL); + if (!compound) + return ERR_PTR(-ENOMEM); + + compound->fm = fm; + compound->compound_header.flags = flags; + + return compound; +} + +int fuse_compound_add(struct fuse_compound_req *compound, + struct fuse_args *args) +{ + if (!compound || + compound->compound_header.count >= FUSE_MAX_COMPOUND_OPS) + return -EINVAL; + + if (args->in_pages) + return -EINVAL; + + compound->op_args[compound->compound_header.count] = args; + compound->compound_header.count++; + return 0; +} + +static void *fuse_copy_response_per_req(struct fuse_args *args, + char *resp) +{ + int i; + size_t copied = 0; + + for (i = 0; i < args->out_numargs; i++) { + struct fuse_arg current_arg = args->out_args[i]; + size_t arg_size = current_arg.size; + + if (current_arg.value && arg_size > 0) { + memcpy(current_arg.value, + (char *)resp + copied, arg_size); + copied += arg_size; + } + } + + return (char *)resp + copied; +} + +int fuse_compound_get_error(struct fuse_compound_req *compound, int op_idx) +{ + return compound->op_errors[op_idx]; +} + +static void *fuse_compound_parse_one_op(struct fuse_compound_req *compound, + int op_index, void *op_out_data, + void *response_end) +{ + struct fuse_out_header *op_hdr = op_out_data; + struct fuse_args *args = compound->op_args[op_index]; + + if (op_hdr->len < sizeof(struct fuse_out_header)) + return NULL; + + /* Check if the entire operation response fits in the buffer */ + if ((char *)op_out_data + op_hdr->len > (char *)response_end) + return NULL; + + if (op_hdr->error != 0) + compound->op_errors[op_index] = op_hdr->error; + + if (args && op_hdr->len > sizeof(struct fuse_out_header)) + return fuse_copy_response_per_req(args, op_out_data + + sizeof(struct fuse_out_header)); + + /* No response data, just advance past the header */ + return (char *)op_out_data + op_hdr->len; +} + +static int fuse_compound_parse_resp(struct fuse_compound_req *compound, + u32 count, void *response, + size_t response_size) +{ + void *op_out_data = response; + void *response_end = (char *)response + response_size; + int i; + + if (!response || response_size < sizeof(struct fuse_out_header)) + return -EIO; + + for (i = 0; i < count && i < compound->result_header.count; i++) { + op_out_data = fuse_compound_parse_one_op(compound, i, + op_out_data, + response_end); + if (!op_out_data) + return -EIO; + } + + return 0; +} + +ssize_t fuse_compound_send(struct fuse_compound_req *compound) +{ + struct fuse_args args = { + .opcode = FUSE_COMPOUND, + .nodeid = 0, + .in_numargs = 2, + .out_numargs = 2, + .out_argvar = true, + }; + size_t resp_buffer_size; + size_t actual_response_size; + size_t buffer_pos; + size_t total_expected_out_size; + void *buffer = NULL; + void *resp_payload; + ssize_t ret; + int i; + + if (!compound) { + pr_info_ratelimited("FUSE: compound request is NULL in %s\n", + __func__); + return -EINVAL; + } + + if (compound->compound_header.count == 0) { + pr_info_ratelimited("FUSE: compound request contains no operations\n"); + return -EINVAL; + } + + buffer_pos = 0; + total_expected_out_size = 0; + + for (i = 0; i < compound->compound_header.count; i++) { + struct fuse_args *op_args = compound->op_args[i]; + size_t needed_size = sizeof(struct fuse_in_header); + int j; + + for (j = 0; j < op_args->in_numargs; j++) + needed_size += op_args->in_args[j].size; + + buffer_pos += needed_size; + + for (j = 0; j < op_args->out_numargs; j++) + total_expected_out_size += op_args->out_args[j].size; + } + + buffer = kvmalloc(buffer_pos, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + + buffer_pos = 0; + for (i = 0; i < compound->compound_header.count; i++) { + struct fuse_args *op_args = compound->op_args[i]; + struct fuse_in_header *hdr; + size_t needed_size = sizeof(struct fuse_in_header); + int j; + + for (j = 0; j < op_args->in_numargs; j++) + needed_size += op_args->in_args[j].size; + + hdr = (struct fuse_in_header *)(buffer + buffer_pos); + memset(hdr, 0, sizeof(*hdr)); + hdr->len = needed_size; + hdr->opcode = op_args->opcode; + hdr->nodeid = op_args->nodeid; + hdr->uid = from_kuid(compound->fm->fc->user_ns, + current_fsuid()); + hdr->gid = from_kgid(compound->fm->fc->user_ns, + current_fsgid()); + hdr->pid = pid_nr_ns(task_pid(current), + compound->fm->fc->pid_ns); + buffer_pos += sizeof(*hdr); + + for (j = 0; j < op_args->in_numargs; j++) { + memcpy(buffer + buffer_pos, op_args->in_args[j].value, + op_args->in_args[j].size); + buffer_pos += op_args->in_args[j].size; + } + } + + resp_buffer_size = total_expected_out_size + + (compound->compound_header.count * + sizeof(struct fuse_out_header)); + + resp_payload = kvmalloc(resp_buffer_size, GFP_KERNEL | __GFP_ZERO); + if (!resp_payload) { + ret = -ENOMEM; + goto out_free_buffer; + } + + compound->compound_header.result_size = total_expected_out_size; + + args.in_args[0].size = sizeof(compound->compound_header); + args.in_args[0].value = &compound->compound_header; + args.in_args[1].size = buffer_pos; + args.in_args[1].value = buffer; + + args.out_args[0].size = sizeof(compound->result_header); + args.out_args[0].value = &compound->result_header; + args.out_args[1].size = resp_buffer_size; + args.out_args[1].value = resp_payload; + + ret = fuse_simple_request(compound->fm, &args); + if (ret < 0) + goto out; + + actual_response_size = args.out_args[1].size; + + if (actual_response_size < sizeof(struct fuse_compound_out)) { + pr_info_ratelimited("FUSE: compound response too small (%zu bytes, minimum %zu bytes)\n", + actual_response_size, + sizeof(struct fuse_compound_out)); + ret = -EINVAL; + goto out; + } + + ret = fuse_compound_parse_resp(compound, compound->result_header.count, + (char *)resp_payload, + actual_response_size); +out: + kvfree(resp_payload); +out_free_buffer: + kvfree(buffer); + return ret; +} diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index d5e62f132e1dba..703c56e5f63a3e 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -663,6 +663,30 @@ static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args) __set_bit(FR_ASYNC, &req->flags); } +ssize_t fuse_compound_request(struct fuse_mount *fm, struct fuse_args *args) +{ + struct fuse_req *req; + ssize_t ret; + + req = fuse_get_req(&invalid_mnt_idmap, fm, false); + if (IS_ERR(req)) + return PTR_ERR(req); + + fuse_args_to_req(req, args); + + if (!args->noreply) + __set_bit(FR_ISREPLY, &req->flags); + + __fuse_request_send(req); + ret = req->out.h.error; + if (!ret && args->out_argvar) { + BUG_ON(args->out_numargs == 0); + ret = args->out_args[args->out_numargs - 1].size; + } + fuse_put_request(req); + return ret; +} + ssize_t __fuse_simple_request(struct mnt_idmap *idmap, struct fuse_mount *fm, struct fuse_args *args) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index c1179ce8fc96b2..d67858330bd1f6 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -1493,14 +1493,7 @@ static int fuse_do_getattr(struct mnt_idmap *idmap, struct inode *inode, inarg.getattr_flags |= FUSE_GETATTR_FH; inarg.fh = ff->fh; } - args.opcode = FUSE_GETATTR; - args.nodeid = get_node_id(inode); - args.in_numargs = 1; - args.in_args[0].size = sizeof(inarg); - args.in_args[0].value = &inarg; - args.out_numargs = 1; - args.out_args[0].size = sizeof(outarg); - args.out_args[0].value = &outarg; + fuse_getattr_args_fill(&args, get_node_id(inode), &inarg, &outarg); err = fuse_simple_request(fm, &args); if (!err) { if (fuse_invalid_attr(&outarg.attr) || diff --git a/fs/fuse/file.c b/fs/fuse/file.c index aac4702b08cd77..5f331e6136931b 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -24,6 +24,39 @@ #include #include +/* + * Helper function to initialize fuse_args for OPEN/OPENDIR operations + */ +void fuse_open_args_fill(struct fuse_args *args, u64 nodeid, int opcode, + struct fuse_open_in *inarg, struct fuse_open_out *outarg) +{ + args->opcode = opcode; + args->nodeid = nodeid; + args->in_numargs = 1; + args->in_args[0].size = sizeof(*inarg); + args->in_args[0].value = inarg; + args->out_numargs = 1; + args->out_args[0].size = sizeof(*outarg); + args->out_args[0].value = outarg; +} + +/* + * Helper function to initialize fuse_args for GETATTR operations + */ +void fuse_getattr_args_fill(struct fuse_args *args, u64 nodeid, + struct fuse_getattr_in *inarg, + struct fuse_attr_out *outarg) +{ + args->opcode = FUSE_GETATTR; + args->nodeid = nodeid; + args->in_numargs = 1; + args->in_args[0].size = sizeof(*inarg); + args->in_args[0].value = inarg; + args->out_numargs = 1; + args->out_args[0].size = sizeof(*outarg); + args->out_args[0].value = outarg; +} + static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, unsigned int open_flags, int opcode, struct fuse_open_out *outargp) @@ -41,14 +74,7 @@ static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID; } - args.opcode = opcode; - args.nodeid = nodeid; - args.in_numargs = 1; - args.in_args[0].size = sizeof(inarg); - args.in_args[0].value = &inarg; - args.out_numargs = 1; - args.out_args[0].size = sizeof(*outargp); - args.out_args[0].value = outargp; + fuse_open_args_fill(&args, nodeid, opcode, &inarg, outargp); return fuse_simple_request(fm, &args); } @@ -127,8 +153,66 @@ static void fuse_file_put(struct fuse_file *ff, bool sync) } } +static int fuse_compound_open_getattr(struct fuse_mount *fm, u64 nodeid, + int flags, int opcode, + struct fuse_file *ff, + struct fuse_attr_out *outattrp, + struct fuse_open_out *outopenp) +{ + struct fuse_compound_req *compound; + struct fuse_args open_args = {}; + struct fuse_args getattr_args = {}; + struct fuse_open_in open_in = {}; + struct fuse_getattr_in getattr_in = {}; + int err; + + compound = fuse_compound_alloc(fm, 0); + if (IS_ERR(compound)) + return PTR_ERR(compound); + + open_in.flags = flags & ~(O_CREAT | O_EXCL | O_NOCTTY); + if (!fm->fc->atomic_o_trunc) + open_in.flags &= ~O_TRUNC; + + if (fm->fc->handle_killpriv_v2 && + (open_in.flags & O_TRUNC) && !capable(CAP_FSETID)) + open_in.open_flags |= FUSE_OPEN_KILL_SUIDGID; + + fuse_open_args_fill(&open_args, nodeid, opcode, &open_in, outopenp); + + err = fuse_compound_add(compound, &open_args); + if (err) + goto out; + + fuse_getattr_args_fill(&getattr_args, nodeid, &getattr_in, outattrp); + + err = fuse_compound_add(compound, &getattr_args); + if (err) + goto out; + + err = fuse_compound_send(compound); + if (err) + goto out; + + err = fuse_compound_get_error(compound, 0); + if (err) + goto out; + + err = fuse_compound_get_error(compound, 1); + if (err) + goto out; + + ff->fh = outopenp->fh; + ff->open_flags = outopenp->open_flags; + +out: + kfree(compound); + return err; +} + struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, - unsigned int open_flags, bool isdir) + struct inode *inode, + unsigned int open_flags, bool isdir) { struct fuse_conn *fc = fm->fc; struct fuse_file *ff; @@ -154,23 +238,46 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, if (open) { /* Store outarg for fuse_finish_open() */ struct fuse_open_out *outargp = &ff->args->open_outarg; - int err; + int err = -ENOSYS; + + if (inode && fc->compound_open_getattr) { + + struct fuse_attr_out attr_outarg; + + err = fuse_compound_open_getattr(fm, nodeid, open_flags, + opcode, ff, + &attr_outarg, outargp); + if (err == -ENOSYS) + fc->compound_open_getattr = 0; + if (!err) + fuse_change_attributes(inode, &attr_outarg.attr, + NULL, + ATTR_TIMEOUT(&attr_outarg), + fuse_get_attr_version(fc)); + } + if (err == -ENOSYS) { + err = fuse_send_open(fm, nodeid, open_flags, opcode, outargp); + if (!err) { + ff->fh = outargp->fh; + ff->open_flags = outargp->open_flags; + } + } - err = fuse_send_open(fm, nodeid, open_flags, opcode, outargp); - if (!err) { - ff->fh = outargp->fh; - ff->open_flags = outargp->open_flags; - } else if (err != -ENOSYS) { - fuse_file_free(ff); - return ERR_PTR(err); - } else { - if (isdir) { + if (err) { + if (err != -ENOSYS) { + /* err is not ENOSYS */ + fuse_file_free(ff); + return ERR_PTR(err); + } else { /* No release needed */ kfree(ff->args); ff->args = NULL; - fc->no_opendir = 1; - } else { - fc->no_open = 1; + + /* we don't have open */ + if (isdir) + fc->no_opendir = 1; + else + fc->no_open = 1; } } } @@ -186,11 +293,10 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file, bool isdir) { - struct fuse_file *ff = fuse_file_open(fm, nodeid, file->f_flags, isdir); + struct fuse_file *ff = fuse_file_open(fm, nodeid, file_inode(file), file->f_flags, isdir); if (!IS_ERR(ff)) file->private_data = ff; - return PTR_ERR_OR_ZERO(ff); } EXPORT_SYMBOL_GPL(fuse_do_open); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index dbefbcf3c14d5f..f982875b6b0ed8 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -948,6 +948,8 @@ struct fuse_conn { /* Use io_uring for communication */ unsigned int io_uring; + /* Does the filesystem support compound operations? */ + unsigned int compound_open_getattr:1; /** Maximum stack depth for passthrough backing files */ int max_stack_depth; @@ -1203,6 +1205,14 @@ struct fuse_io_args { void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, size_t count, int opcode); +/* + * Helper functions to initialize fuse_args for common operations + */ +void fuse_open_args_fill(struct fuse_args *args, u64 nodeid, int opcode, + struct fuse_open_in *inarg, struct fuse_open_out *outarg); +void fuse_getattr_args_fill(struct fuse_args *args, u64 nodeid, + struct fuse_getattr_in *inarg, + struct fuse_attr_out *outarg); struct fuse_file *fuse_file_alloc(struct fuse_mount *fm, bool release); void fuse_file_free(struct fuse_file *ff); @@ -1294,6 +1304,8 @@ static inline ssize_t fuse_simple_idmap_request(struct mnt_idmap *idmap, return __fuse_simple_request(idmap, fm, args); } +ssize_t fuse_compound_request(struct fuse_mount *fm, struct fuse_args *args); + int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, gfp_t gfp_flags); @@ -1301,6 +1313,14 @@ int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, * Assign a unique id to a fuse request */ void fuse_request_assign_unique(struct fuse_iqueue *fiq, struct fuse_req *req); +struct fuse_compound_req; + +struct fuse_compound_req *fuse_compound_alloc(struct fuse_mount *fm, uint32_t flags); +int fuse_compound_add(struct fuse_compound_req *compound, + struct fuse_args *args); +ssize_t fuse_compound_send(struct fuse_compound_req *compound); +int fuse_compound_get_error(struct fuse_compound_req * compound, + int op_idx); /** * End a finished request @@ -1573,7 +1593,9 @@ void fuse_file_io_release(struct fuse_file *ff, struct inode *inode); /* file.c */ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, - unsigned int open_flags, bool isdir); + struct inode *inode, + unsigned int open_flags, + bool isdir); void fuse_file_release(struct inode *inode, struct fuse_file *ff, unsigned int open_flags, fl_owner_t id, bool isdir); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 7e19accd8f27f2..56faa9dd278f01 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1049,6 +1049,12 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, fc->initialized = 0; fc->connected = 1; fc->dlm = 1; + + /* pretend fuse server supports compound operations + * until it tells us otherwise. + */ + fc->compound_open_getattr = 1; + atomic64_set(&fc->attr_version, 1); atomic64_set(&fc->evict_ctr, 1); get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); diff --git a/fs/fuse/ioctl.c b/fs/fuse/ioctl.c index fdc175e93f7474..07a02e47b2c3a6 100644 --- a/fs/fuse/ioctl.c +++ b/fs/fuse/ioctl.c @@ -494,7 +494,7 @@ static struct fuse_file *fuse_priv_ioctl_prepare(struct inode *inode) if (!S_ISREG(inode->i_mode) && !isdir) return ERR_PTR(-ENOTTY); - return fuse_file_open(fm, get_node_id(inode), O_RDONLY, isdir); + return fuse_file_open(fm, get_node_id(inode), NULL, O_RDONLY, isdir); } static void fuse_priv_ioctl_cleanup(struct inode *inode, struct fuse_file *ff) diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index 605c755c8c6331..30bb854fbc9408 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -675,6 +675,13 @@ enum fuse_opcode { /* Operations which have not been merged into upstream */ FUSE_DLM_WB_LOCK = 100, + /* A compound request works like multiple simple requests. + * This is a special case for calls that can be combined atomic on the + * fuse server. If the server actually does atomically execute the command is + * left to the fuse server implementation. + */ + FUSE_COMPOUND = 101, + /* CUSE specific operations */ CUSE_INIT = 4096, @@ -1281,6 +1288,7 @@ struct fuse_dlm_lock_in { uint32_t reserved; }; + /** * struct fuse_dlm_lock_out - Lock response * @locksize: how many bytes where locked by the call @@ -1293,6 +1301,36 @@ struct fuse_dlm_lock_out { uint64_t reserved; }; +/* + * Compound request header + * + * This header is followed by the fuse requests + */ +struct fuse_compound_in { + uint32_t count; /* Number of operations */ + uint32_t flags; /* Compound flags */ + + /* Total size of all results. + * This is needed for preallocating the whole result for all + * commands in this compound. + */ + uint32_t result_size; + uint64_t reserved; +}; + +/* + * Compound response header + * + * This header is followed by complete fuse responses + */ +struct fuse_compound_out { + uint32_t count; /* Number of results */ + uint32_t flags; /* Result flags */ + uint64_t reserved; +}; + +#define FUSE_MAX_COMPOUND_OPS 16 /* Maximum operations per compound */ + /** * Size of the ring buffer header */ From 901deb597a8ac7bd1c13029f7219eea2d627ff17 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Fri, 12 Dec 2025 14:13:10 +0100 Subject: [PATCH 26/77] RED-34640: Fix a startup teardown race There was a race between fuse_uring_cancel() and fuse_uring_register()/fuse_uring_next_fuse_req(), which comes from the queue reduction feature. Race was core-A core-B fuse_uring_register spin_lock(&queue->lock); fuse_uring_ent_avail() spin_unlock(&queue->lock); fuse_uring_cancel() spin_lock(&queue->lock); ent->state = FRRS_USERSPACE; list_move() fuse_uring_next_fuse_req() spin_lock(&queue->lock); fuse_uring_ent_avail(ent, queue); fuse_uring_send_next_to_ring() spin_unlock(&queue->lock); fuse_uring_send_next_to_ring I.e. fuse_uring_ent_avail() was called two times and the 2nd time when the entry was actually already handled by fuse_uring_cancel(). Solution is to not call fuse_uring_ent_avail() from fuse_uring_register. With that the entry is not in state FRRS_AVAILABLE and fuse_uring_cancel() will not touch it. fuse_uring_send_next_to_ring() will mark it as FRRS_AVAILABLE, and then either assign a request to it and change state again or will not touch it at all anymore - race fixed. This will be folded into the upstream queue reduction patches and therefore has the RED-34640 commit message. Also entirely removed is fuse_uring_do_register() as remaining work can be done by the caller. Signed-off-by: Bernd Schubert (imported from commit 932febaee72bfc10a391cdfa14a2b7f37549d967) --- fs/fuse/dev_uring.c | 43 ++++++++++++------------------------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index f88cc4c94aba8e..52882280763ab9 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -1159,36 +1159,6 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, return 0; } -/* - * fuse_uring_req_fetch command handling - */ -static void fuse_uring_do_register(struct fuse_ring_ent *ent, - struct io_uring_cmd *cmd, - unsigned int issue_flags) -{ - struct fuse_ring_queue *queue = ent->queue; - struct fuse_ring *ring = queue->ring; - struct fuse_conn *fc = ring->fc; - struct fuse_iqueue *fiq = &fc->iq; - int node = cpu_to_node(queue->qid); - - if (WARN_ON_ONCE(node >= ring->nr_numa_nodes)) - node = 0; - - fuse_uring_prepare_cancel(cmd, issue_flags, ent); - - spin_lock(&queue->lock); - ent->cmd = cmd; - fuse_uring_ent_avail(ent, queue); - spin_unlock(&queue->lock); - - if (!ring->ready) { - WRITE_ONCE(fiq->ops, &fuse_io_uring_ops); - WRITE_ONCE(ring->ready, true); - wake_up_all(&fc->blocked_waitq); - } -} - /* * Copy from memmap.c, should be exported there */ @@ -1370,6 +1340,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, struct fuse_ring *ring = smp_load_acquire(&fc->ring); struct fuse_ring_queue *queue; struct fuse_ring_ent *ent; + struct fuse_iqueue *fiq = &fc->iq; int err; unsigned int qid = READ_ONCE(cmd_req->qid); @@ -1401,8 +1372,18 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, if (IS_ERR(ent)) return PTR_ERR(ent); - fuse_uring_do_register(ent, cmd, issue_flags); + fuse_uring_prepare_cancel(cmd, issue_flags, ent); + if (!ring->ready) { + WRITE_ONCE(fiq->ops, &fuse_io_uring_ops); + WRITE_ONCE(ring->ready, true); + wake_up_all(&fc->blocked_waitq); + } + + spin_lock(&queue->lock); + ent->cmd = cmd; + spin_unlock(&queue->lock); + /* Marks the ring entry as ready */ fuse_uring_next_fuse_req(ent, queue, issue_flags); return 0; From 9bb2806ee2c557743f1ccbec45b00d662997bb52 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Mon, 20 Oct 2025 23:17:15 +0200 Subject: [PATCH 27/77] fuse: Move ring queues_refs decrement This is just to avoid code dup with an upcoming commit. Signed-off-by: Bernd Schubert (imported from commit ec3217f655d816ac9e3e29b1dc1506d7b195a0a5) --- fs/fuse/dev_uring.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 52882280763ab9..a6d3ecabcf632c 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -458,7 +458,7 @@ static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent) { struct fuse_req *req; struct io_uring_cmd *cmd; - + ssize_t queue_refs; struct fuse_ring_queue *queue = ent->queue; spin_lock(&queue->lock); @@ -486,15 +486,16 @@ static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent) if (req) fuse_uring_stop_fuse_req_end(req); + + queue_refs = atomic_dec_return(&queue->ring->queue_refs); + WARN_ON_ONCE(queue_refs < 0); } static void fuse_uring_stop_list_entries(struct list_head *head, struct fuse_ring_queue *queue, enum fuse_ring_req_state exp_state) { - struct fuse_ring *ring = queue->ring; struct fuse_ring_ent *ent, *next; - ssize_t queue_refs = SSIZE_MAX; LIST_HEAD(to_teardown); spin_lock(&queue->lock); @@ -511,11 +512,8 @@ static void fuse_uring_stop_list_entries(struct list_head *head, spin_unlock(&queue->lock); /* no queue lock to avoid lock order issues */ - list_for_each_entry_safe(ent, next, &to_teardown, list) { + list_for_each_entry_safe(ent, next, &to_teardown, list) fuse_uring_entry_teardown(ent); - queue_refs = atomic_dec_return(&ring->queue_refs); - WARN_ON_ONCE(queue_refs < 0); - } } static void fuse_uring_teardown_entries(struct fuse_ring_queue *queue) From c915d7f04573fdd96d375eb3f2dba201e36136c9 Mon Sep 17 00:00:00 2001 From: Jian Huang Li Date: Mon, 20 Oct 2025 23:23:11 +0200 Subject: [PATCH 28/77] fs/fuse: fix potential memory leak from fuse_uring_cancel This issue could be observed sometimes during libfuse xfstests, from dmseg prints some like "kernel: WARNING: CPU: 4 PID: 0 at fs/fuse/dev_uring.c:204 fuse_uring_destruct+0x1f5/0x200 [fuse]". The cause is, if when fuse daemon just submitted FUSE_IO_URING_CMD_REGISTER SQEs, then umount or fuse daemon quits at this very early stage. After all uring queues stopped, might have one or more unprocessed FUSE_IO_URING_CMD_REGISTER SQEs get processed then some new ring entities are created and added to ent_avail_queue, and immediately fuse_uring_cancel moved them to ent_in_userspace after SQEs get canceled. These ring entities were not moved to ent_released, and stayed in ent_in_userspace when fuse_uring_destruct was called. One way to solve it would be to also free 'ent_in_userspace' in fuse_uring_destruct(), but from code point of view it is hard to see why it is needed. As suggested by Joanne, another solution is to avoid moving entries in fuse_uring_cancel() to the 'ent_in_userspace' list and just releasing them directly. Fixes: b6236c8407cb ("fuse: {io-uring} Prevent mount point hang on fuse-server termination") Cc: Joanne Koong Cc: # v6.14 Signed-off-by: Jian Huang Li Signed-off-by: Bernd Schubert (imported from commit 30d0473dcc0eecac6b1e00d9d87b0892146086a9) --- debian/scripts/misc/kconfig/__init__.py | 0 fs/fuse/dev_uring.c | 21 +++++++++------------ 2 files changed, 9 insertions(+), 12 deletions(-) delete mode 100644 debian/scripts/misc/kconfig/__init__.py diff --git a/debian/scripts/misc/kconfig/__init__.py b/debian/scripts/misc/kconfig/__init__.py deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index a6d3ecabcf632c..197333cf53042a 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -454,7 +454,7 @@ static void fuse_uring_stop_fuse_req_end(struct fuse_req *req) /* * Release a request/entry on connection tear down */ -static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent) +static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent, int issue_flags) { struct fuse_req *req; struct io_uring_cmd *cmd; @@ -482,7 +482,7 @@ static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent) spin_unlock(&queue->lock); if (cmd) - io_uring_cmd_done(cmd, -ENOTCONN, IO_URING_F_UNLOCKED); + io_uring_cmd_done(cmd, -ENOTCONN, issue_flags); if (req) fuse_uring_stop_fuse_req_end(req); @@ -513,7 +513,7 @@ static void fuse_uring_stop_list_entries(struct list_head *head, /* no queue lock to avoid lock order issues */ list_for_each_entry_safe(ent, next, &to_teardown, list) - fuse_uring_entry_teardown(ent); + fuse_uring_entry_teardown(ent, IO_URING_F_UNLOCKED); } static void fuse_uring_teardown_entries(struct fuse_ring_queue *queue) @@ -639,7 +639,7 @@ static void fuse_uring_cancel(struct io_uring_cmd *cmd, { struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd); struct fuse_ring_queue *queue; - bool need_cmd_done = false; + bool teardown = false; /* * direct access on ent - it must not be destructed as long as @@ -648,17 +648,14 @@ static void fuse_uring_cancel(struct io_uring_cmd *cmd, queue = ent->queue; spin_lock(&queue->lock); if (ent->state == FRRS_AVAILABLE) { - ent->state = FRRS_USERSPACE; - list_move_tail(&ent->list, &queue->ent_in_userspace); - need_cmd_done = true; - ent->cmd = NULL; + ent->state = FRRS_TEARDOWN; + list_del_init(&ent->list); + teardown = true; } spin_unlock(&queue->lock); - if (need_cmd_done) { - /* no queue lock to avoid lock order issues */ - io_uring_cmd_done(cmd, -ENOTCONN, issue_flags); - } + if (teardown) + fuse_uring_entry_teardown(ent, issue_flags); } static void fuse_uring_prepare_cancel(struct io_uring_cmd *cmd, int issue_flags, From 9b49db7dd146f7513c6f601b2e8558ee92ad66cb Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Sun, 23 Nov 2025 17:43:40 +0100 Subject: [PATCH 29/77] fuse: Fix missing numa_q_map free in dev_uring This fixes a memory leak. (imported from commit f75b62fce0e6689b1cc57bdae4b6a93be1ca2168) --- fs/fuse/dev_uring.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 197333cf53042a..be64945ce64584 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -178,9 +178,11 @@ static void fuse_uring_destruct_q_masks(struct fuse_ring *ring) fuse_ring_destruct_q_map(&ring->q_map); - if (ring->numa_q_map) + if (ring->numa_q_map) { for (node = 0; node < ring->nr_numa_nodes; node++) fuse_ring_destruct_q_map(&ring->numa_q_map[node]); + kfree(ring->numa_q_map); + } } static bool ent_list_request_expired(struct fuse_conn *fc, struct list_head *list) From e32fd2522de2937b090904f80f78f4deefdb8721 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 19 Dec 2025 10:04:50 +0100 Subject: [PATCH 30/77] fuse: fix includes no functional changes Signed-off-by: Horst Birthelmer (imported from commit f0bccb2ea093d8bf703d535d34541b3000ec1d86) --- fs/fuse/compound.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/fs/fuse/compound.c b/fs/fuse/compound.c index bc52e22eff3123..5d84e3558a06f8 100644 --- a/fs/fuse/compound.c +++ b/fs/fuse/compound.c @@ -10,18 +10,6 @@ #include "fuse_i.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - /* * Compound request builder and state tracker and args pointer storage */ From 1ae938aab8800cd7120a7fdf38d98ab81197cc06 Mon Sep 17 00:00:00 2001 From: Feng Shuo Date: Tue, 30 Sep 2025 03:00:46 +0800 Subject: [PATCH 31/77] Create workflow the create pr for redfs in each branch Take actions on the PR merged event of this repo. Run copy-from-linux-branch.sh and create a PR for redfs. (cherry picked from commit f54872e99c6ebccc92c202e15c22eb68c26b10f6) (imported from commit 522fddfe975a361a411b853eb6b40c62e35ad39e) --- .github/workflows/create-redfs-pr.yml | 92 +++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 .github/workflows/create-redfs-pr.yml diff --git a/.github/workflows/create-redfs-pr.yml b/.github/workflows/create-redfs-pr.yml new file mode 100644 index 00000000000000..cd7e9717440c4e --- /dev/null +++ b/.github/workflows/create-redfs-pr.yml @@ -0,0 +1,92 @@ +# Automatially run copy-from-linux-branch.sh on branches and create PR for redfs. +name: Sync to redfs repo +on: + # Triggers the workflow on pull request merged. + pull_request: + branches: [ "*" ] + types: [ "closed" ] + +jobs: + create-redfs-pr: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + # Checks-out to a different directory to avoid following checkout removing it. + - uses: actions/checkout@v4 + with: + path: linux + + - name: Try to checkout sync-${{ github.ref_name }} if it exists + uses: actions/checkout@v4 + id: try-checkout + continue-on-error: true + with: + repository: DDNStorage/redfs + ref: sync-${{ github.ref_name }} + fetch-depth: 0 + path: redfs + token: ${{ secrets.REDFS_TOKEN }} + + - name: Fallback to checkout main + if: steps.try-checkout.outcome == 'failure' + uses: actions/checkout@v4 + with: + repository: DDNStorage/redfs + ref: main + fetch-depth: 0 + path: redfs + token: ${{ secrets.REDFS_TOKEN }} + + - name: Initialize git + run: | + git config --global user.name "DDNStorage RED Workflow" + git config --global user.email "red@ddn.com" + + - name: Create tracking branch based on main + if: steps.try-checkout.outcome == 'failure' + run: | + pushd redfs + git checkout -b sync-${{ github.ref_name }} + popd + + - name: Generate PR for redfs + run: | + declare -A MAP + MAP["redfs-rhel9_5-503.40.1"]="5.14.0-503.40.1.el9_5" + MAP["redfs-rhel9_6-570.12.1"]="5.14.0-570.26.1.el9_6" + MAP["redfs-ubuntu-noble-6.8.0-58.60"]="6.8.0-58.60.ubuntu" + kerver=${MAP["${{ github.ref_name }}"]} + if [ -z ${kerver} ]; then + echo "Cannot find target kernel version" + exit 1 + fi + pushd redfs + ./copy-from-linux-branch.sh $GITHUB_WORKSPACE/linux ${kerver} + git add src/$kerver + echo -e "Sync with ${{ github.repository }} branch ${{ github.ref_name }} \n" > ../commit.msg + echo -e "Sync with ${{ github.repository }} branch ${{ github.ref_name }} by commit" >> ../commit.msg + echo -e "${{ github.sha }}" >> ../commit.msg + RET=0 + git commit -F ../commit.msg 2> ../commit.log || RET=$?; + if [ -s ../commit.log ]; then + echo "Error detcted in commit:" + cat ../commit.log + exit 1 + elif [ $RET -eq 0 ]; then + echo "Done. Push the code to remote:" + git push origin sync-${{ github.ref_name }} 2> ../push.log ||: + else + echo "No changes to existed codes. Still try with PR." + fi + if [ -s ../push.log ]; then + echo "Error detected in push:" + cat ../push.log + fi + gh pr create --base main --fill || RET=$? + if [ $RET -eq 1 ]; then + echo "No pending changes for PR, returning $RET." + fi + popd + env: + GH_TOKEN: ${{ secrets.OPENUNIXPAT }} + From c4d803777e1472a25efd7d21282df1799f9a98f5 Mon Sep 17 00:00:00 2001 From: Feng Shuo Date: Tue, 30 Dec 2025 08:56:41 +0800 Subject: [PATCH 32/77] Fix the github actions PR trigger Switch to pull_request_target instead of pull_request as the github security requirement. Also limits the scope to protected PR. (cherry picked from commit b9980ad9af3598d465c72fb92f565415c8d4a006) (imported from commit e504e4a44abfa9cef941189e229cef0412c3f014) --- .github/workflows/create-redfs-pr.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/create-redfs-pr.yml b/.github/workflows/create-redfs-pr.yml index cd7e9717440c4e..1f7b99a60c7aff 100644 --- a/.github/workflows/create-redfs-pr.yml +++ b/.github/workflows/create-redfs-pr.yml @@ -3,7 +3,10 @@ name: Sync to redfs repo on: # Triggers the workflow on pull request merged. pull_request: - branches: [ "*" ] + branches: [ "redfs-*" ] + types: [ "closed" ] + pull_request_target: + branches: [ "redfs-*" ] types: [ "closed" ] jobs: @@ -52,8 +55,9 @@ jobs: - name: Generate PR for redfs run: | declare -A MAP + MAP["redfs-rhel9_4-427.42.1"]="5.14.0-427.42.1.el9_4" MAP["redfs-rhel9_5-503.40.1"]="5.14.0-503.40.1.el9_5" - MAP["redfs-rhel9_6-570.12.1"]="5.14.0-570.26.1.el9_6" + MAP["redfs-rhel9_6-570.12.1"]="5.14.0-570.12.1.el9_6" MAP["redfs-ubuntu-noble-6.8.0-58.60"]="6.8.0-58.60.ubuntu" kerver=${MAP["${{ github.ref_name }}"]} if [ -z ${kerver} ]; then @@ -63,7 +67,7 @@ jobs: pushd redfs ./copy-from-linux-branch.sh $GITHUB_WORKSPACE/linux ${kerver} git add src/$kerver - echo -e "Sync with ${{ github.repository }} branch ${{ github.ref_name }} \n" > ../commit.msg + echo -e "Sync with ${{ github.repository }} branch ${{ github.ref_name }}\n" > ../commit.msg echo -e "Sync with ${{ github.repository }} branch ${{ github.ref_name }} by commit" >> ../commit.msg echo -e "${{ github.sha }}" >> ../commit.msg RET=0 @@ -79,7 +83,7 @@ jobs: echo "No changes to existed codes. Still try with PR." fi if [ -s ../push.log ]; then - echo "Error detected in push:" + echo "Message detected in push:" cat ../push.log fi gh pr create --base main --fill || RET=$? @@ -88,5 +92,5 @@ jobs: fi popd env: - GH_TOKEN: ${{ secrets.OPENUNIXPAT }} + GH_TOKEN: ${{ secrets.REDFS_TOKEN }} From 27baabd0a9f09a1f2e27f0856ee3618f92920b68 Mon Sep 17 00:00:00 2001 From: Shuo Feng Date: Tue, 30 Dec 2025 11:18:10 +0800 Subject: [PATCH 33/77] Remove the pull_request_target from actions Remove the pull_request_target as it doesn't work. (cherry picked from commit 5328f660acf48ef3cf1f00ab8ae486aedf6874ee) (imported from commit 5277386783667357873cdd2819517b301a4b5063) --- .github/workflows/create-redfs-pr.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/create-redfs-pr.yml b/.github/workflows/create-redfs-pr.yml index 1f7b99a60c7aff..cc03d7e1219e9b 100644 --- a/.github/workflows/create-redfs-pr.yml +++ b/.github/workflows/create-redfs-pr.yml @@ -5,9 +5,6 @@ on: pull_request: branches: [ "redfs-*" ] types: [ "closed" ] - pull_request_target: - branches: [ "redfs-*" ] - types: [ "closed" ] jobs: create-redfs-pr: From b2263460189dd85654051f2f506ebc65c75000f8 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Tue, 13 Jan 2026 17:58:23 +0100 Subject: [PATCH 34/77] fuse: Make compounds a module option For now compounds are a module option and disabled by default Signed-off-by: Bernd Schubert (imported from commit f3b301ddccefec9e6363bb14e307c51462c0cc6a) --- fs/fuse/inode.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 56faa9dd278f01..08c14478c38328 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -33,6 +33,10 @@ MODULE_AUTHOR("Miklos Szeredi "); MODULE_DESCRIPTION("Filesystem in Userspace"); MODULE_LICENSE("GPL"); +static bool __read_mostly enable_compound; +module_param(enable_compound, bool, 0644); +MODULE_PARM_DESC(enable_uring, "Enable fuse compounds"); + static struct kmem_cache *fuse_inode_cachep; struct list_head fuse_conn_list; DEFINE_MUTEX(fuse_mutex); @@ -1050,10 +1054,8 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, fc->connected = 1; fc->dlm = 1; - /* pretend fuse server supports compound operations - * until it tells us otherwise. - */ - fc->compound_open_getattr = 1; + /* module option for now */ + fc->compound_open_getattr = enable_compound; atomic64_set(&fc->attr_version, 1); atomic64_set(&fc->evict_ctr, 1); From 026345bd6ca2172f3baecb48d989fbcf48962c65 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Wed, 4 Feb 2026 18:47:37 +0100 Subject: [PATCH 35/77] fuse: Fix the reduced queue assignment The use of bitmap_weight() didn't give the actual index, but always returned the current cpu, which resulted in a totally wrong mapping. It now just increases a counter for every mapping and ignores cores not in the given (numa) map and then find the index for that. Also added is a pr_debug(), which can be activated for example with echo "module redfs +p" >/proc/dynamic_debug/control (Pity that upstream is not open for such debug messages). (imported from commit bcbb684ad26c86cc77c04fdab1584ff1ed6bc270) --- fs/fuse/dev_uring.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index be64945ce64584..8254d2d8ff80ba 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -358,26 +358,25 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc) } static void fuse_uring_cpu_qid_mapping(struct fuse_ring *ring, int qid, - struct fuse_queue_map *q_map) + struct fuse_queue_map *q_map, + int node) { - int cpu, qid_idx; + int cpu, qid_idx, mapping_count = 0; size_t nr_queues; cpumask_set_cpu(qid, q_map->registered_q_mask); nr_queues = cpumask_weight(q_map->registered_q_mask); for (cpu = 0; cpu < ring->max_nr_queues; cpu++) { - if (!q_map->cpu_to_qid) - return; - - /* - * Position of this CPU within the registered queue mask, - * handles non-contiguous CPU distributions across NUMA nodes. - */ - qid_idx = bitmap_weight( - cpumask_bits(q_map->registered_q_mask), cpu); + if (node != -1 && cpu_to_node(cpu) != node) + continue; - q_map->cpu_to_qid[cpu] = cpumask_nth(qid_idx % nr_queues, + qid_idx = mapping_count % nr_queues; + q_map->cpu_to_qid[cpu] = cpumask_nth(qid_idx, q_map->registered_q_mask); + mapping_count++; + pr_debug("%s node=%d qid=%d qid_idx=%d nr_queues=%zu %d->%d\n", + __func__, node, qid, qid_idx, nr_queues, cpu, + q_map->cpu_to_qid[cpu]); } } @@ -428,7 +427,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, /* Static mapping from cpu to per numa queues */ node = cpu_to_node(qid); - fuse_uring_cpu_qid_mapping(ring, qid, &ring->numa_q_map[node]); + fuse_uring_cpu_qid_mapping(ring, qid, &ring->numa_q_map[node], node); /* * smp_store_release, as the variable is read without fc->lock and @@ -439,7 +438,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, ring->numa_q_map[node].nr_queues + 1); /* global mapping */ - fuse_uring_cpu_qid_mapping(ring, qid, &ring->q_map); + fuse_uring_cpu_qid_mapping(ring, qid, &ring->q_map, -1); spin_unlock(&fc->lock); From 8e7e5b9ae6c8206dcd8acd06dfa02ee1815c440b Mon Sep 17 00:00:00 2001 From: Feng Shuo Date: Fri, 26 Dec 2025 23:34:06 +0800 Subject: [PATCH 36/77] Fix the compiling error on aarch64 Fix the include sequence which causes a compiling error on aarch64. (imported from commit f5fed0e3f4ad6f98427baa53f5e7505df831dd81) --- fs/fuse/dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index d67858330bd1f6..db3d2a737a7e0e 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -6,8 +6,8 @@ See the file COPYING. */ -#include "fuse_dlm_cache.h" #include "fuse_i.h" +#include "fuse_dlm_cache.h" #include #include From ce5b16a788ee6b56668e07faa19de2b55d528499 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Wed, 11 Feb 2026 16:38:21 +0100 Subject: [PATCH 37/77] fuse: {io-uring} Prefer the current core over mapping Mapping might point to a totally different core due to random assignment. For performance using the current core might be beneficial Example (with core binding) unpatched WRITE: bw=841MiB/s patched WRITE: bw=1363MiB/s With fio --name=test --ioengine=psync --direct=1 \ --rw=write --bs=1M --iodepth=1 --numjobs=1 \ --filename_format=/redfs/testfile.\$jobnum --size=100G \ --thread --create_on_open=1 --runtime=30s --cpus_allowed=1 In order to get the good number `--cpus_allowed=1` is needed. This could be improved by a future change that avoids cpu migration in fuse_request_end() on wake_up() call. (imported from commit 32e0073d67cfc7bd602dc7675ae71fa825b04362) --- fs/fuse/dev_uring.c | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 8254d2d8ff80ba..9c8f7f38193796 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -22,8 +22,12 @@ MODULE_PARM_DESC(enable_uring, #define FUSE_RING_HEADER_PG 0 #define FUSE_RING_PAYLOAD_PG 1 +/* Threshold that determines if a better queue should be searched for */ #define FUSE_URING_Q_THRESHOLD 2 +/* Number of (re)tries to find a better queue */ +#define FUSE_URING_Q_TRIES 3 + bool fuse_uring_enabled(void) { @@ -1501,7 +1505,7 @@ static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring, bool background) { unsigned int qid; - int node, retries = 0; + int node, tries = 0; unsigned int nr_queues; unsigned int cpu = task_cpu(current); struct fuse_ring_queue *queue, *primary_queue = NULL; @@ -1526,26 +1530,36 @@ static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring, nr_queues = READ_ONCE(ring->numa_q_map[node].nr_queues); if (nr_queues) { + /* prefer the queue that corresponds to the current cpu */ + queue = READ_ONCE(ring->queues[cpu]); + if (queue) { + if (queue->nr_reqs <= FUSE_URING_Q_THRESHOLD) + return queue; + primary_queue = queue; + } + qid = ring->numa_q_map[node].cpu_to_qid[cpu]; if (WARN_ON_ONCE(qid >= ring->max_nr_queues)) return NULL; - queue = READ_ONCE(ring->queues[qid]); + if (qid != cpu) { + queue = READ_ONCE(ring->queues[qid]); - /* Might happen on teardown */ - if (unlikely(!queue)) - return NULL; + /* Might happen on teardown */ + if (unlikely(!queue)) + return NULL; - if (queue->nr_reqs < FUSE_URING_Q_THRESHOLD) - return queue; + if (queue->nr_reqs <= FUSE_URING_Q_THRESHOLD) + return queue; + } /* Retries help for load balancing */ - if (retries < FUSE_URING_Q_THRESHOLD) { - if (!retries) + if (tries < FUSE_URING_Q_TRIES && tries + 1 < nr_queues) { + if (!primary_queue) primary_queue = queue; - /* Increase cpu, assuming it will map to a differet qid*/ + /* Increase cpu, assuming it will map to a different qid*/ cpu++; - retries++; + tries++; goto retry; } } @@ -1556,9 +1570,10 @@ static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring, /* global registered queue bitmap */ qid = ring->q_map.cpu_to_qid[cpu]; - if (WARN_ON_ONCE(qid >= ring->max_nr_queues)) - /* Might happen on teardown */ + if (WARN_ON_ONCE(qid >= ring->max_nr_queues)) { + /* Might happen on teardown */ return NULL; + } return READ_ONCE(ring->queues[qid]); } From 2e4905aa111ff1b8b2f1e3fca08c4c383b21c10e Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 23 Feb 2026 16:58:19 +0100 Subject: [PATCH 38/77] fuse: enable large folios in inode initialization Add a module parameter to enable large folio support. Signed-off-by: Horst Birthelmer (imported from commit 475371c422ded852784924f998db8c181a077180) --- fs/fuse/file.c | 3 +++ fs/fuse/fuse_i.h | 2 ++ fs/fuse/inode.c | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 5f331e6136931b..f24be6fb886461 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3400,4 +3400,7 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); + + if (enable_large_folios) + mapping_set_large_folios(inode->i_mapping); } diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index f982875b6b0ed8..1e14a1102e3c01 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -84,6 +84,7 @@ extern struct mutex fuse_mutex; /** Module parameters */ extern unsigned int max_user_bgreq; extern unsigned int max_user_congthresh; +extern bool enable_large_folios; /* One forget request */ struct fuse_forget_link { @@ -948,6 +949,7 @@ struct fuse_conn { /* Use io_uring for communication */ unsigned int io_uring; + /* Does the filesystem support compound operations? */ unsigned int compound_open_getattr:1; /** Maximum stack depth for passthrough backing files */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 08c14478c38328..b32ea012d37395 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -37,6 +37,10 @@ static bool __read_mostly enable_compound; module_param(enable_compound, bool, 0644); MODULE_PARM_DESC(enable_uring, "Enable fuse compounds"); +static bool __read_mostly enable_large_folios = true; +module_param(enable_large_folios, bool, 0644); +MODULE_PARM_DESC(enable_large_folios, "Enable large folios support"); + static struct kmem_cache *fuse_inode_cachep; struct list_head fuse_conn_list; DEFINE_MUTEX(fuse_mutex); From 0b7039affdfaaef576203857d231402501f5e6a0 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Fri, 6 Mar 2026 19:50:14 +0100 Subject: [PATCH 39/77] fuse: Remove double define of 'enable_large_folios' module param compilation failed, due to external and static. The extern is actually not needed, static is enough. (imported from commit b2af4bd09d3b91dd0b95db4513bc7e291eb26661) (imported from commit 5df77fbe17cbda47647cbba82400fe31042bc104) --- fs/fuse/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index b32ea012d37395..24db2d8af82d3e 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -37,7 +37,7 @@ static bool __read_mostly enable_compound; module_param(enable_compound, bool, 0644); MODULE_PARM_DESC(enable_uring, "Enable fuse compounds"); -static bool __read_mostly enable_large_folios = true; +bool __read_mostly enable_large_folios = true; module_param(enable_large_folios, bool, 0644); MODULE_PARM_DESC(enable_large_folios, "Enable large folios support"); From 9ebffb0be731668ecda47860df5622b153976e53 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Mon, 9 Mar 2026 21:39:09 +0100 Subject: [PATCH 40/77] fuse: Remove unlock_request/lock_request This is a DDN patch only, as unlock_request()/lock_request() solve a deadlock issue for specially designed file systems, see Documentation/filesystems/fuse.rst, in the section **Scenario 2 - Tricky deadlock** This one needs a carefully crafted filesystem. It's a variation on the above, only the call back to the filesystem is not explicit, but is caused by a pagefault. :: | Kamikaze filesystem thread 1 | Kamikaze filesystem thread 2 In redfsd we do our best to not cause any kind of user issues and just want to be as fast as possible. Hence, we do not need the per page unlock/lock checks. Given that fuse is a generic file system, this can be a DDN commit only for now, until we find a better generic solution. The unlock_request/lock_request functions have been replaced by check_req_aborted(), which is run once per copied argument. (imported from commit dc7fa1cd35a0cf0caaca30c35ef09714f5fd3646) --- fs/fuse/dev.c | 45 +++++---------------------------------------- 1 file changed, 5 insertions(+), 40 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 703c56e5f63a3e..1c355999cf9b31 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -829,38 +829,14 @@ static int fuse_simple_notify_reply(struct fuse_mount *fm, return 0; } -/* - * Lock the request. Up to the next unlock_request() there mustn't be - * anything that could cause a page-fault. If the request was already - * aborted bail out. - */ -static int lock_request(struct fuse_req *req) -{ - int err = 0; - if (req) { - spin_lock(&req->waitq.lock); - if (test_bit(FR_ABORTED, &req->flags)) - err = -ENOENT; - else - set_bit(FR_LOCKED, &req->flags); - spin_unlock(&req->waitq.lock); - } - return err; -} -/* - * Unlock request. If it was aborted while locked, caller is responsible - * for unlocking and ending the request. - */ -static int unlock_request(struct fuse_req *req) +static int check_req_aborted(struct fuse_req *req) { int err = 0; - if (req) { + if (req && test_bit(FR_ABORTED, &req->flags)) { spin_lock(&req->waitq.lock); if (test_bit(FR_ABORTED, &req->flags)) err = -ENOENT; - else - clear_bit(FR_LOCKED, &req->flags); spin_unlock(&req->waitq.lock); } return err; @@ -902,7 +878,7 @@ static int fuse_copy_fill(struct fuse_copy_state *cs) struct page *page; int err; - err = unlock_request(cs->req); + err = check_req_aborted(cs->req); if (err) return err; @@ -961,7 +937,7 @@ static int fuse_copy_fill(struct fuse_copy_state *cs) cs->pg = page; } - return lock_request(cs->req); + return 0; } /* Do as much copy to/from userspace buffer as we can */ @@ -1022,9 +998,6 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop struct pipe_buffer *buf = cs->pipebufs; folio_get(oldfolio); - err = unlock_request(cs->req); - if (err) - goto out_put_old; fuse_copy_finish(cs); @@ -1110,9 +1083,7 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop cs->pg = buf->page; cs->offset = buf->offset; - err = lock_request(cs->req); - if (!err) - err = 1; + err = 1; goto out_put_old; } @@ -1121,17 +1092,11 @@ static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio, unsigned offset, unsigned count) { struct pipe_buffer *buf; - int err; if (cs->nr_segs >= cs->pipe->max_usage) return -EIO; folio_get(folio); - err = unlock_request(cs->req); - if (err) { - folio_put(folio); - return err; - } fuse_copy_finish(cs); From 09e3c7b1ab98055f2e59162855a307a3303f6d5b Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 16 Jan 2026 13:31:07 +0100 Subject: [PATCH 41/77] fuse: fix inode initialization race Fix a race between fuse_iget() and fuse_reverse_inval_inode() where invalidation can arrive while an inode is being initialized, causing the invalidation to be lost. Add a waitqueue to make fuse_reverse_inval_inode() wait when it encounters an inode with attr_version == 0 (still initializing). When fuse_change_attributes_common() completes initialization, it wakes waiting threads. This ensures invalidations are properly serialized with inode initialization, maintaining cache coherency. Signed-off-by: Horst Birthelmer (imported from commit 03eacfdec557c7574be00442563297d26ac50521) --- fs/fuse/fuse_i.h | 3 +++ fs/fuse/inode.c | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 1e14a1102e3c01..58dbd9dc7991ee 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -973,6 +973,9 @@ struct fuse_conn { /** Version counter for attribute changes */ atomic64_t attr_version; + /** Waitqueue for attr_version initialization */ + wait_queue_head_t attr_version_waitq; + /** Version counter for evict inode */ atomic64_t evict_ctr; diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 24db2d8af82d3e..e901446de52802 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -256,6 +256,7 @@ void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, set_mask_bits(&fi->inval_mask, STATX_BASIC_STATS, 0); fi->attr_version = atomic64_inc_return(&fc->attr_version); + wake_up_all(&fc->attr_version_waitq); fi->i_time = attr_valid; inode->i_ino = fuse_squash_ino(attr->ino); @@ -615,10 +616,17 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, return -ENOENT; fi = get_fuse_inode(inode); + spin_lock(&fi->lock); + while (fi->attr_version == 0) { + spin_unlock(&fi->lock); + wait_event(fc->attr_version_waitq, READ_ONCE(fi->attr_version) != 0); + spin_lock(&fi->lock); + } + fi->attr_version = atomic64_inc_return(&fc->attr_version); spin_unlock(&fi->lock); - + if (fc->inval_inode_entries) fuse_invalidate_inode_entry(inode); else if (fc->expire_inode_entries) @@ -1044,6 +1052,7 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, atomic_set(&fc->epoch, 1); INIT_WORK(&fc->epoch_work, fuse_epoch_work); init_waitqueue_head(&fc->blocked_waitq); + init_waitqueue_head(&fc->attr_version_waitq); fuse_iqueue_init(&fc->iq, fiq_ops, fiq_priv); INIT_LIST_HEAD(&fc->bg_queue); INIT_LIST_HEAD(&fc->entry); From f65445e2a0f451bd78f1f3cfb83afc565019ac6f Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 26 Mar 2026 11:12:54 +0100 Subject: [PATCH 42/77] fuse: debug print requests when we hang in fuse_wait_aborted() Signed-off-by: Horst Birthelmer (imported from commit ad21e5a936b5a7ad4050ebb4d118ee96a5635104) --- fs/fuse/dev.c | 113 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 2 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 1c355999cf9b31..90117fb13ec8e0 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -585,7 +585,8 @@ static void request_wait_answer(struct fuse_req *req) * Either request is already in userspace, or it was forced. * Wait it out. */ - wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags)); + wait_event(req->waitq, + test_bit(FR_FINISHED, &req->flags)); } static void __fuse_request_send(struct fuse_req *req) @@ -2565,11 +2566,119 @@ void fuse_abort_conn(struct fuse_conn *fc) } EXPORT_SYMBOL_GPL(fuse_abort_conn); +static void fuse_debug_print_outstanding_reqs(struct fuse_conn *fc) +{ + struct fuse_dev *fud; + + pr_warn("FUSE: fuse_wait_aborted: num_waiting=%d (should be 0)\n", + atomic_read(&fc->num_waiting)); + +#ifdef CONFIG_FUSE_IO_URING + /* Print io_uring state if enabled */ + if (fc->ring) { + struct fuse_ring *ring = fc->ring; + + pr_warn("FUSE: io_uring enabled - queue_refs=%d ready=%d\n", + atomic_read(&ring->queue_refs), ring->ready); + } +#endif + + /* Print all outstanding requests - lockless for debug */ + list_for_each_entry(fud, &fc->devices, entry) { + struct fuse_pqueue *fpq = &fud->pq; + struct fuse_req *req; + int i; + + /* Print all requests on fpq->io */ + if (!list_empty(&fpq->io)) { + pr_warn("FUSE: Outstanding requests on fpq->io:\n"); + list_for_each_entry(req, &fpq->io, list) { +#ifdef CONFIG_FUSE_IO_URING + if (test_bit(FR_URING, &req->flags) && + req->ring_entry) { + struct fuse_ring_ent *ent = req->ring_entry; + + pr_warn(" req %p: opcode=%u unique=%llu flags=0x%lx FR_WAITING=%d FR_LOCKED=%d FR_FORCE=%d FR_ABORTED=%d FR_URING=%d ring_ent=%p state=%d\n", + req, req->in.h.opcode, + req->in.h.unique, req->flags, + test_bit(FR_WAITING, &req->flags), + test_bit(FR_LOCKED, &req->flags), + test_bit(FR_FORCE, &req->flags), + test_bit(FR_ABORTED, &req->flags), + test_bit(FR_URING, &req->flags), + ent, ent->state); + } else { +#endif + pr_warn(" req %p: opcode=%u unique=%llu flags=0x%lx FR_WAITING=%d FR_LOCKED=%d FR_FORCE=%d FR_ABORTED=%d FR_URING=%d\n", + req, req->in.h.opcode, + req->in.h.unique, req->flags, + test_bit(FR_WAITING, &req->flags), + test_bit(FR_LOCKED, &req->flags), + test_bit(FR_FORCE, &req->flags), + test_bit(FR_ABORTED, &req->flags), + test_bit(FR_URING, &req->flags)); +#ifdef CONFIG_FUSE_IO_URING + } +#endif + } + } + + /* Print all requests on fpq->processing */ + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) { + if (list_empty(&fpq->processing[i])) + continue; + + pr_warn("FUSE: Outstanding requests on fpq->processing[%d]:\n", + i); + list_for_each_entry(req, &fpq->processing[i], list) { +#ifdef CONFIG_FUSE_IO_URING + if (test_bit(FR_URING, &req->flags) && + req->ring_entry) { + struct fuse_ring_ent *ent = req->ring_entry; + + pr_warn(" req %p: opcode=%u unique=%llu flags=0x%lx FR_WAITING=%d FR_LOCKED=%d FR_FORCE=%d FR_ABORTED=%d FR_URING=%d ring_ent=%p state=%d\n", + req, req->in.h.opcode, + req->in.h.unique, req->flags, + test_bit(FR_WAITING, &req->flags), + test_bit(FR_LOCKED, &req->flags), + test_bit(FR_FORCE, &req->flags), + test_bit(FR_ABORTED, &req->flags), + test_bit(FR_URING, &req->flags), + ent, ent->state); + } else { +#endif + pr_warn(" req %p: opcode=%u unique=%llu flags=0x%lx FR_WAITING=%d FR_LOCKED=%d FR_FORCE=%d FR_ABORTED=%d FR_URING=%d\n", + req, req->in.h.opcode, + req->in.h.unique, req->flags, + test_bit(FR_WAITING, &req->flags), + test_bit(FR_LOCKED, &req->flags), + test_bit(FR_FORCE, &req->flags), + test_bit(FR_ABORTED, &req->flags), + test_bit(FR_URING, &req->flags)); +#ifdef CONFIG_FUSE_IO_URING + } +#endif + } + } + } +} + void fuse_wait_aborted(struct fuse_conn *fc) { + unsigned int timeout = 20; + /* matches implicit memory barrier in fuse_drop_waiting() */ smp_mb(); - wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0); + +wait: + wait_event_timeout(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0, HZ * timeout); + + /* Debug: print info if we're waiting */ + if (atomic_read(&fc->num_waiting) > 0) { + fuse_debug_print_outstanding_reqs(fc); + timeout *= 3; + goto wait; + } fuse_uring_wait_stopped_queues(fc); } From 39d2998f1a951d4865cff10ca4d473ecf2811a20 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 2 Apr 2026 08:14:42 +0200 Subject: [PATCH 43/77] fuse: fix io_uring connection abort leaving requests stuck Fix uninterruptible sleep (D state) hangs during FUSE filesystem teardown when using io_uring. The issue manifests as processes stuck waiting for requests that are never completed, particularly affecting force requests like FUSE_FLUSH or when requests are created after fuse_abort_conn() already finished. If on daemon exit io_uring_try_cancel_requests() runs and calls fuse_uring_cancel() which will teardown the entries by calling fuse_uring_entry_teardown() before fuse_abort_conn() then we end up in fuse_uring_abort with queue_refs == 0 and the queues are never stopped. If the queues are stopped all new requests will be rejected, but that does not happen, so all new calls are stuck. Signed-off-by: Horst Birthelmer (imported from commit 9550b4d733625ff51bbff4fda533dee9cf8ae765) --- fs/fuse/dev_uring.c | 3 +-- fs/fuse/dev_uring_i.h | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 9c8f7f38193796..7fa79e68afd5c9 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -143,11 +143,10 @@ void fuse_uring_flush_bg(struct fuse_conn *fc) if (!queue) continue; - queue->stopped = true; - WARN_ON_ONCE(ring->fc->max_background != UINT_MAX); spin_lock(&queue->lock); spin_lock(&fc->bg_lock); + queue->stopped = true; fuse_uring_flush_queue_bg(queue); spin_unlock(&fc->bg_lock); spin_unlock(&queue->lock); diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 86fef37a863a1e..4518990e98bdd5 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -179,10 +179,8 @@ static inline void fuse_uring_abort(struct fuse_conn *fc) if (ring == NULL) return; - if (atomic_read(&ring->queue_refs) > 0) { - fuse_uring_flush_bg(fc); - fuse_uring_stop_queues(ring); - } + fuse_uring_flush_bg(fc); + fuse_uring_stop_queues(ring); } static inline void fuse_uring_wait_stopped_queues(struct fuse_conn *fc) From 17fd84583e4d035d2eaf0f59be2df908d6ee1694 Mon Sep 17 00:00:00 2001 From: Cheng Ding Date: Fri, 6 Mar 2026 17:16:09 +0800 Subject: [PATCH 44/77] fuse: invalidate page cache after sync and async direct writes Fixes xfstests generic/451, similar to how commit b359af8275a9 ("fuse: Invalidate the page cache after FOPEN_DIRECT_IO write") fixes xfstests generic/209. Signed-off-by: Cheng Ding (imported from commit 51e07998077c2ad0ac83ed3f00435a3a50ded2bf) --- fs/fuse/file.c | 59 +++++++++++++++++++++++++++++++++++++++--------- fs/fuse/fuse_i.h | 1 + 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index f24be6fb886461..170cf0b778189c 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -24,6 +24,8 @@ #include #include +int sb_init_dio_done_wq(struct super_block *sb); + /* * Helper function to initialize fuse_args for OPEN/OPENDIR operations */ @@ -736,6 +738,19 @@ static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io) return io->bytes < 0 ? io->size : io->bytes; } +static void fuse_aio_invalidate_worker(struct work_struct *work) +{ + struct fuse_io_priv *io = container_of(work, struct fuse_io_priv, work); + struct address_space *mapping = io->iocb->ki_filp->f_mapping; + ssize_t res = fuse_get_res_by_io(io); + pgoff_t start = io->offset >> PAGE_SHIFT; + pgoff_t end = (io->offset + res - 1) >> PAGE_SHIFT; + + invalidate_inode_pages2_range(mapping, start, end); + io->iocb->ki_complete(io->iocb, res); + kref_put(&io->refcnt, fuse_io_release); +} + /* * In case of short read, the caller sets 'pos' to the position of * actual end of fuse request in IO request. Otherwise, if bytes_requested @@ -768,10 +783,11 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) spin_unlock(&io->lock); if (!left && !io->blocking) { + struct inode *inode = file_inode(io->iocb->ki_filp); + struct address_space *mapping = io->iocb->ki_filp->f_mapping; ssize_t res = fuse_get_res_by_io(io); if (res >= 0) { - struct inode *inode = file_inode(io->iocb->ki_filp); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); @@ -780,6 +796,17 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) spin_unlock(&fi->lock); } + if (io->write && res > 0 && mapping->nrpages) { + /* + * As in generic_file_direct_write(), invalidate after the + * write, to invalidate read-ahead cache that may have competed + * with the write. + */ + INIT_WORK(&io->work, fuse_aio_invalidate_worker); + queue_work(inode->i_sb->s_dio_done_wq, &io->work); + return; + } + io->iocb->ki_complete(io->iocb, res); } @@ -1859,15 +1886,6 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, if (res > 0) *ppos = pos; - if (res > 0 && write && fopen_direct_io) { - /* - * As in generic_file_direct_write(), invalidate after the - * write, to invalidate read-ahead cache that may have competed - * with the write. - */ - invalidate_inode_pages2_range(mapping, idx_from, idx_to); - } - return res > 0 ? res : err; } EXPORT_SYMBOL_GPL(fuse_direct_io); @@ -1906,6 +1924,8 @@ static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct inode *inode = file_inode(iocb->ki_filp); + struct address_space *mapping = inode->i_mapping; + loff_t pos = iocb->ki_pos; ssize_t res; bool exclusive; @@ -1922,6 +1942,16 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) FUSE_DIO_WRITE); fuse_write_update_attr(inode, iocb->ki_pos, res); } + if (res > 0 && mapping->nrpages) { + /* + * As in generic_file_direct_write(), invalidate after + * write, to invalidate read-ahead cache that may have + * with the write. + */ + invalidate_inode_pages2_range(mapping, + pos >> PAGE_SHIFT, + (pos + res - 1) >> PAGE_SHIFT); + } } fuse_dio_unlock(iocb, exclusive); @@ -3012,6 +3042,7 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) size_t count = iov_iter_count(iter), shortened = 0; loff_t offset = iocb->ki_pos; struct fuse_io_priv *io; + bool async = ff->fm->fc->async_dio; pos = offset; inode = file->f_mapping->host; @@ -3020,6 +3051,12 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if ((iov_iter_rw(iter) == READ) && (offset >= i_size)) return 0; + if ((iov_iter_rw(iter) == WRITE) && async && !inode->i_sb->s_dio_done_wq) { + ret = sb_init_dio_done_wq(inode->i_sb); + if (ret < 0) + return ret; + } + io = kmalloc_obj(struct fuse_io_priv); if (!io) return -ENOMEM; @@ -3035,7 +3072,7 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) * By default, we want to optimize all I/Os with async request * submission to the client filesystem if supported. */ - io->async = ff->fm->fc->async_dio; + io->async = async; io->iocb = iocb; io->blocking = is_sync_kiocb(iocb); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 58dbd9dc7991ee..d6b68d12be9307 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -393,6 +393,7 @@ union fuse_file_args { /** The request IO state (for asynchronous processing) */ struct fuse_io_priv { struct kref refcnt; + struct work_struct work; int async; spinlock_t lock; unsigned reqs; From 4f790a1965d3903e7f8996ddbe5407191c11b5dd Mon Sep 17 00:00:00 2001 From: kchen Date: Tue, 21 Apr 2026 09:15:15 +0000 Subject: [PATCH 45/77] Invalidate selinux security label during inode invalidation. Add security_inode_invalidate_secctx() call to invalidate cached security context when inode attributes change. This ensures that SELinux security labels are properly refreshed and prevents stale security context from being used after inode modifications. Signed-off-by: Kevin Chen (imported from commit 6c9ec1dca008983d5eaecda066aed49edbd6cae7) --- fs/fuse/inode.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index e901446de52802..25fe7f81815b26 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -634,6 +634,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, fuse_invalidate_attr(inode); forget_all_cached_acls(inode); + security_inode_invalidate_secctx(inode); if (offset >= 0) { pg_start = offset >> PAGE_SHIFT; if (len <= 0) From b3218850c73de39b4402e459cac34a8f75cdbbed Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 28 Apr 2026 20:14:06 +0200 Subject: [PATCH 46/77] ubuntu: fix makefile compiler warning option to support LLVM Signed-off-by: Horst Birthelmer --- ubuntu/igh-ecat/master/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubuntu/igh-ecat/master/Makefile b/ubuntu/igh-ecat/master/Makefile index 8aef742ebb2f74..4f8fc539c6ab6c 100644 --- a/ubuntu/igh-ecat/master/Makefile +++ b/ubuntu/igh-ecat/master/Makefile @@ -1,5 +1,5 @@ ccflags-y := -I$(src)/../ \ - -Wmaybe-uninitialized + -Wuninitialized obj-$(CONFIG_IGH_ECAT) += ec_master.o From f1e07df8e15dba44117094828c4d1587535ad6b1 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 1 May 2026 11:01:24 +0200 Subject: [PATCH 47/77] fuse: fix DLM lock acquiring when writeback is not used Fix a logical error where the DLM lock was acquired regardless of whether the writeback part was actually called. This is necessarry after the move to iomap_file_buffered_write() Signed-off-by: Horst Birthelmer (cherry picked from commit c0127c53ec86cb73ba94ad03b8f8edd2a3735559) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 170cf0b778189c..632e12b7d1c535 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1624,18 +1624,31 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) return err; if (!fc->handle_killpriv_v2 || - !setattr_should_drop_suidgid(idmap, file_inode(file))) + !setattr_should_drop_suidgid(idmap, file_inode(file))) { writeback = true; - /* if we have dlm support acquire the lock for the area - * we are writing into */ - if (fc->dlm) { - /* note that a file opened with O_APPEND will have relative values - * in ki_pos. This code is here for convenience and for libfuse overlay test. - * Filesystems should handle O_APPEND with 'direct io' to additionally - * get the performance benefits of 'parallel direct writes'. */ - loff_t pos = file->f_flags & O_APPEND ? i_size_read(inode) + iocb->ki_pos : iocb->ki_pos; - size_t length = iov_iter_count(from); - fuse_get_dlm_write_lock(file, pos, length); + + /* + * If we have dlm support acquire the lock for the area + * we are writing into. + * dlm lock is only needed as the write is cached and the + * fuse server is not notified otherwise + */ + if (fc->dlm) { + /* + * Note that a file opened with O_APPEND will have + * relative values in ki_pos. This code is here for + * convenience and for libfuse overlay test. + * Filesystems should handle O_APPEND with 'direct io' + * to additionally get the performance benefits of + * 'parallel direct writes'. + */ + loff_t pos = file->f_flags & O_APPEND ? + i_size_read(inode) + iocb->ki_pos : + iocb->ki_pos; + size_t length = iov_iter_count(from); + + fuse_get_dlm_write_lock(file, pos, length); + } } } From 6c3bd485a08664c4c90888b6b0d31a19e38bb62b Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 1 May 2026 13:29:35 +0200 Subject: [PATCH 48/77] fuse: prepare EAGAIN translation fuse_do_readfolio() is called by fuse_iomap_read_folio_range() as well and is not supposed to return a positive value, thus the translation has to be done in another layer. Signed-off-by: Horst Birthelmer (cherry picked from commit ba4ab3b2ef3cba8aef8cb558d4a3c8e48335a9fc) Signed-off-by: Allison Henderson --- .../fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt | 577 ++++++++++++++++++ fs/fuse/file.c | 13 +- 2 files changed, 586 insertions(+), 4 deletions(-) create mode 100644 Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt diff --git a/Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt b/Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt new file mode 100644 index 00000000000000..3db1ac877cd2d5 --- /dev/null +++ b/Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt @@ -0,0 +1,577 @@ +================================================================================= +WHY FUSE CONVERTS -EAGAIN TO AOP_TRUNCATED_PAGE IN fuse_read_folio() +================================================================================= + +TLDR: To prevent ABBA deadlock between page locks and DLM (cluster) locks. + +================================================================================= +THE FUNDAMENTAL DEADLOCK SCENARIO - DETAILED CODE FLOW +================================================================================= + +This deadlock occurs when two CPUs hold locks in opposite order, creating +a circular dependency. Below is the exact code flow showing how this happens. + +───────────────────────────────────────────────────────────────────────────── +TIMELINE: CORE-0 (Application Read Path - Waiting for DLM Lock) +───────────────────────────────────────────────────────────────────────────── + +T0: Application calls read() + ├─ sys_read() [fs/read_write.c] + └─ vfs_read() [fs/read_write.c:440] + └─ generic_file_read_iter() [mm/filemap.c:2850] + +T1: Enter page cache read path + └─ filemap_read() [mm/filemap.c:2675] + │ ACQUIRES: filemap invalidate_lock (shared) [line 2678] + │ STATUS: Holding invalidate_lock + │ + └─ filemap_get_pages() [mm/filemap.c:2712] + └─ filemap_update_page() [mm/filemap.c:2625] + +T2: Attempt to lock page + └─ folio_trylock() [mm/filemap.c:2466] + │ ACQUIRES: page lock + │ STATUS: Holding invalidate_lock + page lock + │ + └─ filemap_read_folio() [mm/filemap.c:2497] + └─ filler(file, folio) [mm/filemap.c:2413] + │ Calls address_space_operations->read_folio + │ + └─ ocfs2_read_folio() [fs/ocfs2/aops.c:262] + +T3: Attempt to acquire DLM lock (CRITICAL POINT) + └─ ocfs2_inode_lock_with_folio() [fs/ocfs2/aops.c:271] + │ STATUS: Holding invalidate_lock + page lock + │ WANTS: DLM inode lock + │ + └─ ocfs2_inode_lock_full() [fs/ocfs2/dlmglue.c:2552] + └─ __ocfs2_cluster_lock() [fs/ocfs2/dlmglue.c:2465] + └─ dlm_lock() [fs/dlm/lock.c:3372] + │ + ❌ BLOCKS HERE - DLM lock held by Core-1 + │ Waiting for DLM lock while holding page lock + │ + DEADLOCK CONDITION: Cannot proceed + +───────────────────────────────────────────────────────────────────────────── +TIMELINE: CORE-1 (Memory Reclaim Path - Waiting for Page Lock) +───────────────────────────────────────────────────────────────────────────── + +T0: Memory pressure triggers reclaim + └─ kswapd() or direct memory reclaim [mm/vmscan.c] + └─ shrink_node_memcgs() [mm/vmscan.c] + +T1: Start reclaiming pages + └─ shrink_inactive_list() [mm/vmscan.c:2004] + └─ shrink_folio_list() [mm/vmscan.c:1098] + │ Iterating through pages to evict + │ + └─ Check if folio needs writeback [mm/vmscan.c:1227] + +T2: DLM lock already held during downconvert/writeback + │ CONTEXT: This thread is handling: + │ - DLM lock downconvert request from another node + │ - Metadata update requiring DLM lock + │ - Page writeback with cluster coordination + │ + │ STATUS: Holding DLM INODE LOCK (exclusive or shared) + │ + └─ pageout(folio, mapping, ...) [mm/vmscan.c:1452] + │ May trigger writepage callback + │ + └─ Filesystem writepage operations + │ These operations may update metadata + │ Already holding DLM lock for coordination + +T3: Attempt to acquire page lock (CRITICAL POINT) + └─ folio_trylock() [mm/vmscan.c:1129] + │ STATUS: Holding DLM lock + │ WANTS: page lock + │ + ❌ BLOCKS HERE - Page lock held by Core-0 + │ Waiting for page lock while holding DLM lock + │ + DEADLOCK CONDITION: Cannot proceed + +───────────────────────────────────────────────────────────────────────────── +RESULT: ABBA DEADLOCK (Circular Wait) +───────────────────────────────────────────────────────────────────────────── + +Lock Acquisition Order Violation: + + Core-0: invalidate_lock → page lock → [WANTS] DLM lock + Core-1: DLM lock → [WANTS] page lock + +Circular Dependency: + Core-0 waits for DLM lock (held by Core-1) + Core-1 waits for page lock (held by Core-0) + + ┌──────────────┐ ┌──────────────┐ + │ CORE-0 │ │ CORE-1 │ + │ │ │ │ + │ Holds: page │◄───────────────┤ Wants: page │ + │ Wants: DLM │ │ Holds: DLM │ + │ │────────────────►│ │ + └──────────────┘ └──────────────┘ + ▲ │ + │ │ + └──────────── DEADLOCK ────────────┘ + +Neither Core-0 nor Core-1 can proceed → System deadlock + +================================================================================= +WHY PAGE LOCKS ARE HELD WHEN CALLING read_folio() +================================================================================= + +From mm/filemap.c:do_read_cache_folio(): + 1. Page is allocated/looked up in page cache + 2. folio_trylock() acquires the page lock + 3. filler (read_folio) is called WITH page locked + 4. Lock prevents concurrent modifications during I/O + 5. Lock ensures atomic read operation + +The page lock must be held to prevent: + - Concurrent writes while reading + - Page truncation during read + - Multiple simultaneous reads to the same page + +================================================================================= +THE LOCK ORDERING RULE (From GFS2 Documentation) +================================================================================= + +From Documentation/filesystems/gfs2-glocks.rst: + + Lock ordering within GFS2: + 1. i_rwsem (if required) + 2. Rename glock + 3. Inode glock(s) + 4. Rgrp glock(s) + 5. Transaction glock + 6. i_rw_mutex (if required) + 7. Page lock (ALWAYS LAST!) + +**Critical Rule: PAGE LOCK IS ALWAYS ACQUIRED LAST** + +This means: + - All filesystem/cluster locks must be acquired BEFORE page lock + - NO filesystem lock can be acquired while holding a page lock + - Prevents lock inversion with memory reclaim + +================================================================================= +OCFS2'S SOLUTION: Breaking the Deadlock with AOP_TRUNCATED_PAGE +================================================================================= + +Location: fs/ocfs2/dlmglue.c:2547-2568 +Function: ocfs2_inode_lock_with_folio() + +───────────────────────────────────────────────────────────────────────────── +CODE: Deadlock Prevention Pattern +───────────────────────────────────────────────────────────────────────────── + + int ocfs2_inode_lock_with_folio(struct inode *inode, + struct buffer_head **ret_bh, int ex, struct folio *folio) + { + int ret; + + // LINE 2552: Try non-blocking DLM lock first + ret = ocfs2_inode_lock_full(inode, ret_bh, ex, OCFS2_LOCK_NONBLOCK); + + if (ret == -EAGAIN) { + // DLM lock not immediately available + // Would block, risking deadlock with Core-1 + + // LINE 2554: ✓ CRITICAL - Unlock page BEFORE waiting for DLM lock + folio_unlock(folio); + + /* + * Wait for DLM lock WITHOUT holding page lock + * This breaks the Core-0 → Core-1 dependency + * + * From comment (lines 2555-2560): + * "If we can't get inode lock immediately, we should not return + * directly here, since this will lead to a softlockup problem. + * The method is to get a blocking lock and immediately unlock + * before returning, this can avoid CPU resource waste due to + * lots of retries, and benefits fairness in getting lock." + */ + + // LINE 2562: Acquire lock in blocking mode (without page lock) + if (ocfs2_inode_lock(inode, ret_bh, ex) == 0) + ocfs2_inode_unlock(inode, ex); // Immediately release + + // LINE 2564: Signal VFS to retry entire operation + ret = AOP_TRUNCATED_PAGE; + } + + return ret; + } + +───────────────────────────────────────────────────────────────────────────── +HOW THIS BREAKS THE DEADLOCK +───────────────────────────────────────────────────────────────────────────── + +BEFORE (would deadlock): + Core-0: page lock → [WAITS for DLM lock] → BLOCKED by Core-1 + Core-1: DLM lock → [WAITS for page lock] → BLOCKED by Core-0 + Result: Circular wait, system hangs + +AFTER (with fix): + Core-0 at T3: + 1. Detects DLM lock unavailable (-EAGAIN) + 2. ✓ Releases page lock (line 2554) + 3. Waits for DLM lock (now safe, no page lock held) + 4. Gets DLM lock, immediately releases it (fair cycling) + 5. Returns AOP_TRUNCATED_PAGE + + Core-0 caller (mm/filemap.c): + 6. Sees AOP_TRUNCATED_PAGE + 7. Drops folio reference + 8. Jumps to retry label (e.g., do_read_cache_folio line 3971) + 9. Re-acquires page from cache + 10. Tries read again + + Core-1: + - While Core-0 released page lock (step 2) + - Core-1 can now acquire page lock + - Core-1 completes its work + - Core-1 releases DLM lock + + Core-0 retry: + - Now DLM lock is available + - Successfully acquires DLM lock + - Completes read operation + + Result: Both cores make progress, no deadlock + +───────────────────────────────────────────────────────────────────────────── +KEY INSIGHT FROM OCFS2 COMMENTS +───────────────────────────────────────────────────────────────────────────── + +From fs/ocfs2/dlmglue.c:1627-1632: + + "This is helping work around a lock inversion between the page lock + and dlm locks. One path holds the page lock while calling aops + which block acquiring dlm locks. The voting thread holds dlm + locks while acquiring page locks while down converting data locks. + This block is helping an aop path notice the inversion and back + off to unlock its page lock before trying the dlm lock again." + +Translation: + - "One path" = Core-0 (application read) + - "voting thread" = Core-1 (DLM downconvert/memory reclaim) + - "lock inversion" = ABBA deadlock scenario + - "back off" = Release page lock, return AOP_TRUNCATED_PAGE + +================================================================================= +FUSE'S SPECIFIC CONTEXT: Same Pattern, Different DLM +================================================================================= + +FUSE implements a custom "DLM cache" (fs/fuse/fuse_dlm_cache.c): + - NOT the kernel's DLM subsystem (used by GFS2/OCFS2) + - Tracks page-level locks for distributed coordination + - Sends FUSE_DLM_WB_LOCK operations to userspace daemon + - Userspace daemon handles cluster lock negotiation + +───────────────────────────────────────────────────────────────────────────── +FUSE READ PATH - CODE FLOW WITH DEADLOCK RISK +───────────────────────────────────────────────────────────────────────────── + +Core-0: Application reading from FUSE filesystem + + filemap_read() [mm/filemap.c:2675] + └─ filemap_get_pages() [line 2712] + └─ filemap_update_page() [line 2625] + └─ folio_trylock() [line 2466] + │ ACQUIRES: page lock + │ + └─ filemap_read_folio() [line 2497] + └─ fuse_read_folio() [fs/fuse/file.c:947] + └─ fuse_do_readfolio() [fs/fuse/file.c:956] + └─ fuse_simple_request() [fs/fuse/file.c:932] + │ + │ Sends FUSE_READ to userspace daemon + │ Daemon may need to acquire cluster lock + │ + └─ Might return -EAGAIN if DLM detects possible + deadlock due to concurrant page invalidation + + +───────────────────────────────────────────────────────────────────────────── +WHY FUSE NEEDS THIS CONVERSION +───────────────────────────────────────────────────────────────────────────── + +When fuse_simple_request() returns -EAGAIN: + - Userspace FUSE daemon encountered transient failure + - Could be: cluster lock contention, timeout, daemon busy + - The page might be stale/modified during the wait + - Page lock is STILL HELD at this point + +If -EAGAIN were returned directly: + ❌ VFS would see error and fail the read + ❌ Page would remain locked + ❌ No retry mechanism triggered + +By converting to AOP_TRUNCATED_PAGE: + ✓ Signals to VFS: "retry the entire page acquisition" + ✓ fuse_read_folio() unlocks page before returning (line 962) + ✓ Follows the OCFS2 pattern: unlock page, retry operation + ✓ Prevents potential deadlock with cluster lock operations + ✓ Uses VFS's built-in retry mechanism in mm/filemap.c + +───────────────────────────────────────────────────────────────────────────── +CODE: fuse_read_folio() - Ensures Page Unlock +───────────────────────────────────────────────────────────────────────────── + +Location: fs/fuse/file.c:947-964 + + static int fuse_read_folio(struct file *file, struct folio *folio) + { + struct inode *inode = folio->mapping->host; + int err; + + err = -EIO; + if (fuse_is_bad(inode)) + goto out; + + // LINE 956: Calls fuse_do_readfolio, may return AOP_TRUNCATED_PAGE + err = fuse_do_readfolio(file, folio, 0, folio_size(folio)); + if (!err) + folio_mark_uptodate(folio); + + fuse_invalidate_atime(inode); + out: + // LINE 962: ✓ CRITICAL - Always unlocks page before returning + folio_unlock(folio); + return err; // Returns AOP_TRUNCATED_PAGE if -EAGAIN occurred + } + +This matches OCFS2's pattern: + 1. Detect lock contention (-EAGAIN from daemon) + 2. Unlock the page (line 962) + 3. Return AOP_TRUNCATED_PAGE + 4. VFS retries the operation + +================================================================================= +THE CONTRACT: AOP_TRUNCATED_PAGE Semantics +================================================================================= + +From include/linux/fs.h: + "AOP_TRUNCATED_PAGE: The AOP method that was handed a locked page has + unlocked it and the page might have been truncated. The caller should + back up to acquiring a new page and trying again." + +Key requirements: + - The folio MUST be unlocked before returning AOP_TRUNCATED_PAGE + - Signals "retry from scratch" to the caller + - Caller will drop page reference and re-acquire + - Prevents livelock through filesystem's "reasonable precautions" + +Callers in mm/filemap.c handle it consistently: + - do_read_cache_folio(): "if (err == AOP_TRUNCATED_PAGE) goto repeat;" + - filemap_get_pages(): "if (err == AOP_TRUNCATED_PAGE) goto retry;" + - do_filemap_fault(): "if (error == AOP_TRUNCATED_PAGE) goto retry_find;" + +================================================================================= +WHY NOT JUST RETURN -EAGAIN? +================================================================================= + +From OCFS2 comments (fs/ocfs2/dlmglue.c:2555-2560): + "If we can't get inode lock immediately, we should not return + directly here, since this will lead to a softlockup problem. + The method is to get a blocking lock and immediately unlock + before returning, this can avoid CPU resource waste due to + lots of retries, and benefits fairness in getting lock." + +Returning -EAGAIN directly would cause: + - VFS to immediately retry without unlocking the page + - Busy-waiting loop (soft lockup) + - CPU resource waste + - Potential starvation of lock waiters + +AOP_TRUNCATED_PAGE forces: + - Full retry cycle (unlock, re-acquire, re-read) + - Allows other threads to make progress + - Yields CPU properly + - Fair lock acquisition + +================================================================================= +COMPLETE EXECUTION TIMELINE - DEADLOCK AND RESOLUTION +================================================================================= + +Without AOP_TRUNCATED_PAGE (DEADLOCK SCENARIO): +──────────────────────────────────────────────────────────────────────────── + +Time Core-0 (Read Path) Core-1 (Reclaim/DLM) +──── ───────────────────────────── ────────────────────────────────── +T0 Enter filemap_read() [Idle] + +T1 Lock: invalidate_lock [Idle] + Lock: page lock + +T2 Call: ocfs2_read_folio() DLM downconvert starts + Lock: DLM inode lock + +T3 Want: DLM inode lock Want: page lock + ❌ BLOCKED (held by Core-1) ❌ BLOCKED (held by Core-0) + +T4 [DEADLOCK] [DEADLOCK] + Holding: page lock Holding: DLM lock + Waiting: DLM lock Waiting: page lock + +∞ System hangs System hangs + +With AOP_TRUNCATED_PAGE (DEADLOCK PREVENTION): +──────────────────────────────────────────────────────────────────────────── + +Time Core-0 (Read Path) Core-1 (Reclaim/DLM) +──── ───────────────────────────── ────────────────────────────────── +T0 Enter filemap_read() [Idle] + +T1 Lock: invalidate_lock [Idle] + Lock: page lock + +T2 Call: ocfs2_read_folio() DLM downconvert starts + Lock: DLM inode lock + +T3 Try: DLM lock (NONBLOCK) Want: page lock + Returns: -EAGAIN ❌ BLOCKED (held by Core-0) + +T4 ✓ Unlock: page lock ✓ Acquires: page lock + Return: AOP_TRUNCATED_PAGE Continues: reclaim work + +T5 Caller sees AOP_TRUNCATED_PAGE Completes: page eviction + goto retry (line 3971) Unlock: page lock + Unlock: DLM lock + +T6 Re-acquire: page from cache [Completes] + Re-try: read operation + +T7 Try: DLM lock (NONBLOCK) + ✓ SUCCESS (Core-1 released it) + Lock: DLM inode lock + +T8 Complete: read operation + Unlock: DLM lock + Unlock: page lock + +T9 Return: success to application + +Result: Both cores complete successfully, no deadlock + +================================================================================= +VFS RETRY MECHANISM - HOW AOP_TRUNCATED_PAGE TRIGGERS RETRY +================================================================================= + +Location: mm/filemap.c + +───────────────────────────────────────────────────────────────────────────── +Caller 1: do_read_cache_folio() - lines 3967-3973 +───────────────────────────────────────────────────────────────────────────── + + repeat: + folio = __filemap_get_folio(...); + + filler: + err = filemap_read_folio(file, filler, folio); // Calls read_folio + if (err) { + folio_put(folio); + if (err == AOP_TRUNCATED_PAGE) + goto repeat; // ← RETRY from beginning + return ERR_PTR(err); + } + +Effect: Full retry of page acquisition and read + +───────────────────────────────────────────────────────────────────────────── +Caller 2: filemap_get_pages() - lines 2638-2639 +───────────────────────────────────────────────────────────────────────────── + + retry: + ... + err = filemap_update_page(...); + if (err < 0) + goto err; + ... + err: + if (err < 0) + folio_put(folio); + if (likely(--fbatch->nr)) + return 0; + if (err == AOP_TRUNCATED_PAGE) + goto retry; // ← RETRY from beginning + return err; + +Effect: Re-attempts page update after releasing folio + +───────────────────────────────────────────────────────────────────────────── +Caller 3: do_filemap_fault() - lines 3542-3543 +───────────────────────────────────────────────────────────────────────────── + + retry_find: + ... + error = filemap_read_folio(file, mapping->a_ops->read_folio, folio); + ... + if (!error || error == AOP_TRUNCATED_PAGE) + goto retry_find; // ← RETRY page fault handling + +Effect: Re-handles the page fault from scratch + +================================================================================= +SUMMARY: Why -EAGAIN → AOP_TRUNCATED_PAGE is Essential +================================================================================= + +The conversion -EAGAIN → AOP_TRUNCATED_PAGE in fuse_read_folio() is necessary: + +1. **Prevent Deadlock**: Avoids page lock vs cluster lock ABBA deadlock + - Core-0 holds page lock, wants DLM lock + - Core-1 holds DLM lock, wants page lock + - Conversion forces Core-0 to release page lock first + +2. **Follow Lock Ordering**: Page lock must be released before cluster locks + - Kernel rule: Page lock is ALWAYS acquired last + - Documented in Documentation/filesystems/gfs2-glocks.rst + - Prevents lock inversion with memory reclaim + +3. **Enable Retry**: Uses VFS's built-in retry mechanism properly + - AOP_TRUNCATED_PAGE is understood by mm/filemap.c + - Triggers automatic retry loops at multiple call sites + - -EAGAIN alone would just fail the operation + +4. **Ensure Fairness**: Allows fair lock acquisition through retry cycle + - Other threads get chance to acquire locks + - Prevents starvation of waiters + - Better system responsiveness + +This pattern is the established solution in distributed filesystems (OCFS2, GFS2) +for handling deadlock between page cache and cluster coordination locks. + +================================================================================= +REFERENCES - Exact Code Locations +================================================================================= + +Core-0 (Read Path): + - Entry: filemap_read() [mm/filemap.c:2675] + - Page lock: folio_trylock() [mm/filemap.c:2466] + - FUSE read: fuse_do_readfolio() [fs/fuse/file.c:905] + - -EAGAIN conversion [fs/fuse/file.c:934-935] + - Page unlock: folio_unlock() [fs/fuse/file.c:962] + +Core-1 (Reclaim Path): + - Entry: shrink_folio_list() [mm/vmscan.c:1098] + - Page lock attempt: folio_trylock() [mm/vmscan.c:1129] + +OCFS2 Solution: + - Detection: ocfs2_inode_lock_with_folio() [fs/ocfs2/dlmglue.c:2547] + - Page unlock: folio_unlock() [fs/ocfs2/dlmglue.c:2554] + - Return: AOP_TRUNCATED_PAGE [fs/ocfs2/dlmglue.c:2564] + +VFS Retry: + - do_read_cache_folio() retry [mm/filemap.c:3971] + - filemap_get_pages() retry [mm/filemap.c:2639] + - do_filemap_fault() retry [mm/filemap.c:3543] + +Lock Ordering Documentation: + - GFS2 lock ordering rules [Documentation/filesystems/gfs2-glocks.rst:110-120] + - OCFS2 deadlock comments [fs/ocfs2/dlmglue.c:1627-1632] + +================================================================================= diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 632e12b7d1c535..8958972fda5eb3 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -970,10 +970,16 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); res = fuse_simple_request(fm, &ia.ap.args); if (res < 0) { - if (res == -EAGAIN) + /* + * please refer to Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt + * why this is necessarry. + * READ can return -EAGAIN from DLM subsystem + */ + if (res == -EAGAIN && fm->fc->dlm) res = AOP_TRUNCATED_PAGE; return res; } + /* * Short read means EOF. If file size is larger, truncate it */ @@ -1127,7 +1133,6 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, { struct file *file = iter->private; size_t off = offset_in_folio(folio, pos); - return fuse_do_readfolio(file, folio, off, len); } @@ -1630,8 +1635,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) /* * If we have dlm support acquire the lock for the area * we are writing into. - * dlm lock is only needed as the write is cached and the - * fuse server is not notified otherwise + * dlm lock is only needed as the write is cached and the + * fuse server is not notified otherwise */ if (fc->dlm) { /* From e5427d09a6493269d4590be6f3f3a33776907819 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Fri, 1 May 2026 16:37:54 +0200 Subject: [PATCH 49/77] fuse: Add DLM retry workaround for iomap write failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the FUSE server returns -EAGAIN during write-back operations (signaled by DLM), the write fails with an IO error. This happens because: 1. Page invalidation holds DLM lock and needs folio lock 2. iomap write path holds folio lock and calls fuse_iomap_read_folio_range() 3. FUSE gets -EAGAIN from server (cannot acquire DLM lock - would deadlock) 4. fuse_do_readfolio() converts -EAGAIN to AOP_TRUNCATED_PAGE and unlocks folio (This prevents the deadlock by releasing the folio lock) 5. However, iomap doesn't understand AOP_TRUNCATED_PAGE and treats it as error 6. Result: Write fails with IO error, even though it's just temporary contention This is a FUSE-only workaround until mainline iomap gains AOP_TRUNCATED_PAGE retry support. The solution: 1. Stack-allocate retry state in fuse_cache_write_iter() 2. Register it in fuse_conn xarray before calling iomap (indexed by task pointer) 3. When fuse_iomap_read_folio_range() sees AOP_TRUNCATED_PAGE: - Mark the retry flag in the registered state - Convert to -EAGAIN for iomap 4. After iomap returns, check the retry flag 5. If set, retry the entire write operation 6. Remove from xarray when done (or keep for next retry iteration) This allows writes to succeed by retrying after the DLM lock contention clears, rather than failing with IO error. Technical flow showing why iov_iter is not advanced on -EAGAIN: fuse_cache_write_iter() total_written = 0 retry: iomap_file_buffered_write() iomap_write_iter() [write loop] iomap_write_begin() __iomap_write_begin() Need read? → Yes read_folio_range() FUSE server -EAGAIN? Yes → Set retry flag, return -EAGAIN No → Success Error → Break loop [iov_iter NOT advanced] Success → Continue copy_folio_from_iter_atomic() [Advances iov_iter] iomap_write_end() Advance iter.pos Loop while more data Update iocb->ki_pos = iter.pos Return bytes written if (written > 0) total_written += written if (retry_needed) goto retry return total_written Signed-off-by: Bernd Schubert (cherry picked from commit e37c84be2702496a223b3888df829e7c1618d106) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 118 ++++++++++++++++++++++++++++++++++++++++++----- fs/fuse/fuse_i.h | 18 ++++++++ fs/fuse/inode.c | 2 + 3 files changed, 127 insertions(+), 11 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 8958972fda5eb3..986a8186d8a7e6 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -972,8 +972,8 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, if (res < 0) { /* * please refer to Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt - * why this is necessarry. - * READ can return -EAGAIN from DLM subsystem + * why READ can return -EAGAIN from DLM subsystem. + * XXX find a better DLM specific error code */ if (res == -EAGAIN && fm->fc->dlm) res = AOP_TRUNCATED_PAGE; @@ -1132,8 +1132,45 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, size_t len) { struct file *file = iter->private; + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); size_t off = offset_in_folio(folio, pos); - return fuse_do_readfolio(file, folio, off, len); + int ret; + ret = fuse_do_readfolio(file, folio, off, len); + + /* + * TEMPORARY WORKAROUND for iomap write deadlock: + * + * When FUSE server returns -EAGAIN due to DLM, + * fuse_do_readfolio() converts it to AOP_TRUNCATED_PAGE and + * unlocks the folio (per AOP_TRUNCATED_PAGE contract). + * + * However, iomap doesn't understand AOP_TRUNCATED_PAGE. + * We need to: + * 1. Mark the retry flag (caller stored it in xarray) + * 2. Convert to -EAGAIN so iomap sees an error + * 3. Let fuse_cache_write_iter() detect and retry + * + * This breaks the ABBA deadlock: + * - Folio is unlocked (page invalidation can proceed) + * - Write will be retried at higher level + * + * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE support. + */ + if (ret == AOP_TRUNCATED_PAGE) { + struct fuse_dlm_retry *retry; + unsigned long task_key = (unsigned long)current; + + retry = xa_load(&fc->dlm_retry_tasks, task_key); + if (retry) { + retry->retry_needed = true; + } + + /* Convert to -EAGAIN for iomap */ + ret = -EAGAIN; + } + + return ret; } static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, @@ -1610,6 +1647,68 @@ static const struct iomap_write_ops fuse_iomap_write_ops = { .read_folio_range = fuse_iomap_read_folio_range, }; +static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, + struct iov_iter *from, + struct file *file) +{ + struct fuse_conn *fc = get_fuse_conn(file_inode(file)); + ssize_t written, total_written = 0; + + /* + * TEMPORARY WORKAROUND for iomap write deadlock: + * + * Stack-allocate retry state and register it before calling + * iomap. If fuse_iomap_read_folio_range() encounters + * AOP_TRUNCATED_PAGE, it will mark retry_needed. + * + * Stack allocation ensures no memory leaks - the state is + * valid for the duration of this function call and is + * automatically cleaned up. + */ + struct fuse_dlm_retry retry_state = { + .retry_needed = false, + }; + unsigned long task_key = (unsigned long)current; + int xa_ret; + + xa_ret = xa_err(xa_store(&fc->dlm_retry_tasks, task_key, + &retry_state, GFP_KERNEL)); + if (xa_ret) + return xa_ret; + +retry: + /* + * Use iomap so that we can do granular uptodate reads + * and granular dirty tracking for large folios. + */ + written = iomap_file_buffered_write(iocb, from, &fuse_iomap_ops, + &fuse_iomap_write_ops, file); + + if (written > 0) + total_written += written; + + /* + * If DLM lock contention occurred (AOP_TRUNCATED_PAGE), + * retry the entire write operation. + * + * The folio has been unlocked by fuse_do_readfolio(), + * breaking the ABBA deadlock with page invalidation. + * + * Keep the entry in xarray and reuse it for the retry. + * + * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE + * retry support. + */ + if (retry_state.retry_needed) { + retry_state.retry_needed = false; + goto retry; + } + + /* Remove from xarray now that we're done */ + xa_erase(&fc->dlm_retry_tasks, task_key); + + return written < 0 ? written : total_written; +} static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; @@ -1676,14 +1775,11 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) written = direct_write_fallback(iocb, from, written, fuse_perform_write(iocb, from)); } else if (writeback) { - /* - * Use iomap so that we can do granular uptodate reads - * and granular dirty tracking for large folios. - */ - written = iomap_file_buffered_write(iocb, from, - &fuse_iomap_ops, - &fuse_iomap_write_ops, - file); + written = fuse_writeback_write_iter(iocb, from, file); + if (written < 0) { + err = written; + goto out; + } } else { written = fuse_perform_write(iocb, from); } diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index d6b68d12be9307..1dea11240a915a 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -651,6 +651,17 @@ struct fuse_sync_bucket { struct rcu_head rcu; }; +/** + * DLM retry tracking for iomap write deadlock workaround. + * + * Temporary workaround until mainline iomap gains AOP_TRUNCATED_PAGE + * retry support. Tracks tasks that need to retry write operations due + * to DLM lock contention (-EAGAIN from FUSE server). + */ +struct fuse_dlm_retry { + bool retry_needed; +}; + /** * A Fuse connection. * @@ -1027,6 +1038,13 @@ struct fuse_conn { /* Request timeout (in jiffies). 0 = no timeout */ unsigned int req_timeout; } timeout; + + /** + * XArray tracking tasks that need DLM retry. + * Maps task pointer -> struct fuse_dlm_retry. + * Temporary workaround for iomap write deadlock. + */ + struct xarray dlm_retry_tasks; }; /* diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 25fe7f81815b26..e3e54eaf6ae3f3 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1071,6 +1071,7 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, /* module option for now */ fc->compound_open_getattr = enable_compound; + xa_init(&fc->dlm_retry_tasks); atomic64_set(&fc->attr_version, 1); atomic64_set(&fc->evict_ctr, 1); get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); @@ -1123,6 +1124,7 @@ void fuse_conn_put(struct fuse_conn *fc) } if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) fuse_backing_files_free(fc); + xa_destroy(&fc->dlm_retry_tasks); call_rcu(&fc->rcu, delayed_release); } EXPORT_SYMBOL_GPL(fuse_conn_put); From 5d0d41ccb7ca209929e802ca0c046fb3b391cf73 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Sat, 2 May 2026 09:34:25 +0200 Subject: [PATCH 50/77] fuse: Use -EDEADLK for DLM lock error instead of -EAGAIN -EAGAIN is semantically overloaded for a DLM error and not self describing, switch to -EDEADLK. In order to allow a graceful daemon change, -EAGAIN is kept for now. Signed-off-by: Bernd Schubert (cherry picked from commit 975dffcb574edec2a8d204cc1aa5534abd2c6585) Signed-off-by: Allison Henderson --- .../fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt | 24 ++++++++++--------- fs/fuse/file.c | 17 +++++++------ 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt b/Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt index 3db1ac877cd2d5..7ae02c1f1e88b7 100644 --- a/Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt +++ b/Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt @@ -1,5 +1,5 @@ ================================================================================= -WHY FUSE CONVERTS -EAGAIN TO AOP_TRUNCATED_PAGE IN fuse_read_folio() +WHY FUSE CONVERTS -EDEADLK TO AOP_TRUNCATED_PAGE IN fuse_read_folio() ================================================================================= TLDR: To prevent ABBA deadlock between page locks and DLM (cluster) locks. @@ -292,21 +292,22 @@ Core-0: Application reading from FUSE filesystem │ Sends FUSE_READ to userspace daemon │ Daemon may need to acquire cluster lock │ - └─ Might return -EAGAIN if DLM detects possible + └─ Might return -EDEADLK if DLM detects possible deadlock due to concurrant page invalidation + - For now -EAGAIN handled the same ───────────────────────────────────────────────────────────────────────────── WHY FUSE NEEDS THIS CONVERSION ───────────────────────────────────────────────────────────────────────────── -When fuse_simple_request() returns -EAGAIN: +When fuse_simple_request() returns -EDEADLK: - Userspace FUSE daemon encountered transient failure - Could be: cluster lock contention, timeout, daemon busy - The page might be stale/modified during the wait - Page lock is STILL HELD at this point -If -EAGAIN were returned directly: +If -EDEADLK were returned directly: ❌ VFS would see error and fail the read ❌ Page would remain locked ❌ No retry mechanism triggered @@ -342,11 +343,12 @@ Location: fs/fuse/file.c:947-964 out: // LINE 962: ✓ CRITICAL - Always unlocks page before returning folio_unlock(folio); - return err; // Returns AOP_TRUNCATED_PAGE if -EAGAIN occurred + return err; // Returns AOP_TRUNCATED_PAGE if -EDEADLK occurred } This matches OCFS2's pattern: 1. Detect lock contention (-EAGAIN from daemon) + - Note: confusing that OCFS2 uses -EAGAIN instead of -EDEADLK 2. Unlock the page (line 962) 3. Return AOP_TRUNCATED_PAGE 4. VFS retries the operation @@ -372,7 +374,7 @@ Callers in mm/filemap.c handle it consistently: - do_filemap_fault(): "if (error == AOP_TRUNCATED_PAGE) goto retry_find;" ================================================================================= -WHY NOT JUST RETURN -EAGAIN? +WHY NOT JUST RETURN -EDEADLK (-EAGAIN)? ================================================================================= From OCFS2 comments (fs/ocfs2/dlmglue.c:2555-2560): @@ -434,7 +436,7 @@ T2 Call: ocfs2_read_folio() DLM downconvert starts Lock: DLM inode lock T3 Try: DLM lock (NONBLOCK) Want: page lock - Returns: -EAGAIN ❌ BLOCKED (held by Core-0) + Returns: -EDEADLK ❌ BLOCKED (held by Core-0) T4 ✓ Unlock: page lock ✓ Acquires: page lock Return: AOP_TRUNCATED_PAGE Continues: reclaim work @@ -517,10 +519,10 @@ Caller 3: do_filemap_fault() - lines 3542-3543 Effect: Re-handles the page fault from scratch ================================================================================= -SUMMARY: Why -EAGAIN → AOP_TRUNCATED_PAGE is Essential +SUMMARY: Why -EDEADLK → AOP_TRUNCATED_PAGE is Essential ================================================================================= -The conversion -EAGAIN → AOP_TRUNCATED_PAGE in fuse_read_folio() is necessary: +The conversion -EDEADLK → AOP_TRUNCATED_PAGE in fuse_read_folio() is necessary: 1. **Prevent Deadlock**: Avoids page lock vs cluster lock ABBA deadlock - Core-0 holds page lock, wants DLM lock @@ -535,7 +537,7 @@ The conversion -EAGAIN → AOP_TRUNCATED_PAGE in fuse_read_folio() is necessary: 3. **Enable Retry**: Uses VFS's built-in retry mechanism properly - AOP_TRUNCATED_PAGE is understood by mm/filemap.c - Triggers automatic retry loops at multiple call sites - - -EAGAIN alone would just fail the operation + - -EDEADLK alone would just fail the operation 4. **Ensure Fairness**: Allows fair lock acquisition through retry cycle - Other threads get chance to acquire locks @@ -553,7 +555,7 @@ Core-0 (Read Path): - Entry: filemap_read() [mm/filemap.c:2675] - Page lock: folio_trylock() [mm/filemap.c:2466] - FUSE read: fuse_do_readfolio() [fs/fuse/file.c:905] - - -EAGAIN conversion [fs/fuse/file.c:934-935] + - -EDEADLK conversion [fs/fuse/file.c:934-935] - Page unlock: folio_unlock() [fs/fuse/file.c:962] Core-1 (Reclaim Path): diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 986a8186d8a7e6..057cd1726c9a46 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -971,11 +971,14 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, res = fuse_simple_request(fm, &ia.ap.args); if (res < 0) { /* - * please refer to Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt - * why READ can return -EAGAIN from DLM subsystem. - * XXX find a better DLM specific error code + * Please refer to Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt + * why READ can return -EDEADLK from DLM subsystem. + * + * -EDEADLK: Preferred error code indicating DLM lock ordering violation + * (would cause deadlock with page lock) + * -EAGAIN: Legacy error code, maintained for backward compatibility */ - if (res == -EAGAIN && fm->fc->dlm) + if ((res == -EDEADLK || res == -EAGAIN) && fm->fc->dlm) res = AOP_TRUNCATED_PAGE; return res; } @@ -1141,9 +1144,9 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, /* * TEMPORARY WORKAROUND for iomap write deadlock: * - * When FUSE server returns -EAGAIN due to DLM, - * fuse_do_readfolio() converts it to AOP_TRUNCATED_PAGE and - * unlocks the folio (per AOP_TRUNCATED_PAGE contract). + * When FUSE server returns -EDEADLK (or legacy -EAGAIN) due to DLM + * lock contention, fuse_do_readfolio() converts it to AOP_TRUNCATED_PAGE + * and unlocks the folio (per AOP_TRUNCATED_PAGE contract). * * However, iomap doesn't understand AOP_TRUNCATED_PAGE. * We need to: From 254befb1de7488f35d1e6b94a819ba8d9127e235 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Wed, 17 Jun 2026 09:07:51 +0200 Subject: [PATCH 51/77] fuse: avoid retry livelock when iomap drains iov_iter Reset retry_needed before each iomap call so it reflects only the most recent attempt. Fixes: 975dffcb5 Signed-off-by: Horst Birthelmer (cherry picked from commit 1c21fa43ec1f9b672ae229b447570bee618b395c) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 057cd1726c9a46..8a4aa8eb4634eb 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1680,6 +1680,16 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, return xa_ret; retry: + /* + * Reset before each iomap call so retry_needed only reflects what + * happened in the most recent call. iomap may set retry_needed + * during an internal iteration that then recovers and completes + * the write fully; without the reset the flag would survive into + * the next iteration with iov_iter already drained, and iomap + * would re-enter with len==0 and livelock on a 0-length mapping. + */ + retry_state.retry_needed = false; + /* * Use iomap so that we can do granular uptodate reads * and granular dirty tracking for large folios. @@ -1702,10 +1712,8 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE * retry support. */ - if (retry_state.retry_needed) { - retry_state.retry_needed = false; + if (retry_state.retry_needed && iov_iter_count(from)) goto retry; - } /* Remove from xarray now that we're done */ xa_erase(&fc->dlm_retry_tasks, task_key); From 43e47ca6e726df22442c73f5527342426416eb2d Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 22 Jun 2026 08:54:19 +0200 Subject: [PATCH 52/77] fuse: drop BDI_CAP_STRICTLIMIT from fuse bdi setup Signed-off-by: Horst Birthelmer (cherry picked from commit 0ea57a5496221f6b92de777b72953cccd6bacc0b) Signed-off-by: Allison Henderson --- fs/fuse/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index e3e54eaf6ae3f3..f637aecda9b37d 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1684,7 +1684,7 @@ static int fuse_bdi_init(struct fuse_conn *fc, struct super_block *sb) if (err) return err; - sb->s_bdi->capabilities |= BDI_CAP_STRICTLIMIT; + sb->s_bdi->capabilities &= ~BDI_CAP_STRICTLIMIT; /* * For a single fuse filesystem use max 1% of dirty + From 838b25495cec7bdab17a7259b34656056c414a3b Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 7 Jul 2026 12:02:37 +0200 Subject: [PATCH 53/77] fuse: switch to direct IO on an inode-invalidation notify storm A FUSE_NOTIFY_INVAL_INODE data invalidation means another (remote) entity is modifying the file. Rather than react to a single notify, keep a per-inode moving average of how fast data invalidations arrive for the whole file: an EWMA of the inter-arrival interval, updated under fi->lock on every notify (fuse_notify_inval_hot()). The inode is latched only once the average spacing drops below an internal threshold (FUSE_NOTIFY_DIO_INTERVAL) while a local writer is open; a lone or occasional notify keeps the average high and does not trip the switch. The heuristic has no external knob -- its parameters (EWMA weight, threshold, seed) are source-level constants. Introduce the forced-direct-IO latch (FUSE_I_FORCE_DIO): - fuse_reverse_inval_inode() folds each data invalidation into the moving average and sets the latch when it trips with a local writer present; - fuse_file_{read,write}_iter() and fuse_cache_write_iter() route to the direct path while latched; fuse_dio_{wr_exclusive_lock,lock,unlock}() use the shared parallel-dio path and bypass the cached/uncached accounting; - fuse_file_io_open() opens new files uncached so they do not re-enter caching mode; - fuse_prepare_release() clears the latch once the last writer is gone and fuse_file_release() drops any clean folios a racing read repopulated; fuse_file_mmap() reverts to caching mode (a mapping needs the page cache). Latching to direct IO is only coherent if no buffered write can deposit dirty folios into the page cache after it has been dropped. Add a per-inode rw_semaphore, wb_inval_rwsem, to serialise the buffered-write page-cache dirtying against the latch transition. The writeback path holds it for read around the dirtying and re-checks the latch under it; fuse_reverse_inval_inode() holds it for write around its invalidate + latch set. The notification may be delivered by the same server thread that still owes a reply to an in-flight write holding the inode lock, so it takes the rwsem with a trylock and never blocks: if the writer has gone it skips the latch and only invalidates the notified range. The writer's read-side section stays free of server round-trips because under fc->dlm the partial-write RMW read is skipped. Signed-off-by: Horst Birthelmer (cherry picked from commit eeec69bb75561f189ab0b68450dcccc64a2f9173) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 123 ++++++++++++++++++++++++++++++++++++++++++++--- fs/fuse/fuse_i.h | 56 +++++++++++++++++++++ fs/fuse/inode.c | 98 ++++++++++++++++++++++++++++++++++++- fs/fuse/iomode.c | 10 ++++ 4 files changed, 277 insertions(+), 10 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 8a4aa8eb4634eb..105c6a8a00df4b 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -423,6 +423,18 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, if (likely(fi)) { spin_lock(&fi->lock); list_del(&ff->write_entry); + /* + * Leave forced direct IO mode once the last writer is gone: with + * no local writer left there is no cached-write contention with + * the remote modifier that triggered the switch. Restore + * FUSE_I_CACHE_IO_MODE for any frozen cached opens. + */ + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + list_empty(&fi->write_files)) { + clear_bit(FUSE_I_FORCE_DIO, &fi->state); + if (fi->iocachectr > 0) + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + } spin_unlock(&fi->lock); } spin_lock(&fc->lock); @@ -461,9 +473,24 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_release_args *ra = &ff->args->release_args; int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE; + bool was_force_dio = test_bit(FUSE_I_FORCE_DIO, &fi->state); fuse_prepare_release(fi, ff, open_flags, opcode, false); + /* + * If this release dropped the last writer, fuse_prepare_release() + * cleared the forced-direct-IO latch (under fi->lock). Drop any clean + * folios a read racing the latch may have repopulated so they cannot be + * served stale once caching mode resumes. No inode lock or + * wb_inval_rwsem: release may run on the fuse server thread (async fput + * from aio completion), where blocking on a contended inode lock could + * stall the connection. Writes were routed direct while latched, so + * only clean folios exist and this invalidate is server-free; the last + * writer is gone, so no forced-dio writer can race the drop. + */ + if (was_force_dio && !test_bit(FUSE_I_FORCE_DIO, &fi->state)) + invalidate_inode_pages2(inode->i_mapping); + if (ra && ff->flock) { ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id); @@ -1582,9 +1609,15 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(iocb->ki_filp); struct fuse_inode *fi = get_fuse_inode(inode); + bool force_dio = test_bit(FUSE_I_FORCE_DIO, &fi->state); - /* Server side has to advise that it supports parallel dio writes. */ - if (!(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES)) + /* + * Server side has to advise that it supports parallel dio writes. + * When the inode is latched into forced direct IO, parallel writes are + * used unconditionally: the page cache has been flushed and is bypassed + * for this inode. + */ + if (!force_dio && !(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES)) return true; /* @@ -1595,7 +1628,7 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from return true; /* shared locks are not allowed with parallel page cache IO */ - if (test_bit(FUSE_I_CACHE_IO_MODE, &fi->state)) + if (!force_dio && test_bit(FUSE_I_CACHE_IO_MODE, &fi->state)) return true; /* Parallel dio beyond EOF is not supported, at least for now. */ @@ -1622,9 +1655,14 @@ static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from, * should be performed only after taking shared inode lock. * Previous past eof check was without inode lock and might * have raced, so check it again. + * + * Under the forced-dio latch the cached/uncached accounting is + * bypassed (the latch guarantees the cache is flushed and not + * repopulated), so only re-check the past-eof condition. */ if (fuse_io_past_eof(iocb, from) || - fuse_inode_uncached_io_start(fi, NULL) != 0) { + (!test_bit(FUSE_I_FORCE_DIO, &fi->state) && + fuse_inode_uncached_io_start(fi, NULL) != 0)) { inode_unlock_shared(inode); inode_lock(inode); *exclusive = true; @@ -1640,8 +1678,13 @@ static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive) if (exclusive) { inode_unlock(inode); } else { - /* Allow opens in caching mode after last parallel dio end */ - fuse_inode_uncached_io_end(fi); + /* + * Allow opens in caching mode after last parallel dio end. + * Skipped under the forced-dio latch, which never took an + * uncached_io reference in fuse_dio_lock(). + */ + if (!test_bit(FUSE_I_FORCE_DIO, &fi->state)) + fuse_inode_uncached_io_end(fi); inode_unlock_shared(inode); } } @@ -1720,6 +1763,9 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, return written < 0 ? written : total_written; } + +static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from); + static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; @@ -1729,7 +1775,19 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) struct inode *inode = mapping->host; ssize_t err, count; struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); bool writeback = false; + bool wb_guard = false; + + /* + * The inode may have been latched into forced direct IO -- by a + * NOTIFY_INVAL_INODE arriving while this inode is open for writing here + * -- after this write was routed to the cached path but before it took + * any lock. Re-route to the direct path (before taking a DLM lock) so + * we do not repopulate the page cache the latch just dropped. + */ + if (fuse_inode_force_dio(inode)) + return fuse_direct_write_iter(iocb, from); if (fc->writeback_cache) { /* Update size (EOF optimization) and mode (SUID clearing) */ @@ -1769,6 +1827,27 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) inode_lock(inode); + /* + * The forced-direct-IO latch feature is active under writeback+dlm; + * hold wb_inval_rwsem for read across the page-cache dirtying so a + * concurrent NOTIFY_INVAL_INODE -- which latches the inode under the + * write side of this lock via a non-blocking trylock -- cannot strand + * the folios we are about to write. Re-check the latch under it (it may + * have been set while we blocked on the inode lock) and re-route to the + * direct path if set. Taken before task_io_account_write() so a + * re-route is not double-counted; the DLM write lock taken above is + * harmless as the direct path does its own server coordination. + */ + wb_guard = fc->writeback_cache && fc->dlm; + if (wb_guard) { + down_read(&fi->wb_inval_rwsem); + if (fuse_inode_force_dio(inode)) { + up_read(&fi->wb_inval_rwsem); + inode_unlock(inode); + return fuse_direct_write_iter(iocb, from); + } + } + err = count = generic_write_checks(iocb, from); if (err <= 0) goto out; @@ -1795,6 +1874,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) written = fuse_perform_write(iocb, from); } out: + if (wb_guard) + up_read(&fi->wb_inval_rwsem); inode_unlock(inode); if (written > 0) written = generic_write_sync(iocb, written); @@ -2096,7 +2177,7 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) return fuse_dax_read_iter(iocb, to); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if (ff->open_flags & FOPEN_DIRECT_IO) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) return fuse_direct_read_iter(iocb, to); else if (fuse_file_passthrough(ff)) return fuse_passthrough_read_iter(iocb, to); @@ -2117,7 +2198,7 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) return fuse_dax_write_iter(iocb, from); /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ - if (ff->open_flags & FOPEN_DIRECT_IO) + if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode)) return fuse_direct_write_iter(iocb, from); else if (fuse_file_passthrough(ff)) return fuse_passthrough_write_iter(iocb, from); @@ -2727,6 +2808,29 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) else if (fuse_inode_backing(get_fuse_inode(inode))) return -ENODEV; + /* + * If the inode was latched into forced direct IO after a remote-modify + * notification, a mapping needs the page cache, so revert to caching + * mode. Revert without the inode lock or wb_inval_rwsem: ->mmap runs + * under mmap_lock and the buffered write path holds both across a fault + * on the user buffer (which takes mmap_lock), so taking either here + * would invert lock order (ABBA). Clearing the latch and dropping the + * cache is sufficient -- writers re-check the latch and route to cached + * IO once it is clear, and in-flight parallel dio drains itself. Cached + * opens frozen while latched are still counted in iocachectr, so restore + * FUSE_I_CACHE_IO_MODE for them. + */ + if (fuse_inode_force_dio(inode)) { + struct fuse_inode *fi = get_fuse_inode(inode); + + spin_lock(&fi->lock); + clear_bit(FUSE_I_FORCE_DIO, &fi->state); + if (fi->iocachectr > 0) + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + spin_unlock(&fi->lock); + invalidate_inode_pages2(file->f_mapping); + } + /* * FOPEN_DIRECT_IO handling is special compared to O_DIRECT, * as does not allow MAP_SHARED mmap without FUSE_DIRECT_IO_ALLOW_MMAP. @@ -3559,6 +3663,9 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->iocachectr = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); + init_rwsem(&fi->wb_inval_rwsem); + fi->notify_stamp = jiffies; + fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 1dea11240a915a..a4a2702c7926ff 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -126,6 +126,20 @@ struct dlm_locked_area size_t size; }; +/* + * Force-DIO switch trigger: an exponentially weighted moving average of the + * interval (in jiffies) between FUSE_NOTIFY_INVAL_INODE data invalidations for + * a file. When the average spacing falls below FUSE_NOTIFY_DIO_INTERVAL -- a + * remote writer streaming invalidations -- and the file is open for writing + * here, it is latched into direct IO. These are the source-level (not + * externally tunable) parameters of the heuristic: EWMA weight 1/2^SHIFT, + * seeded and capped at SEED so it takes a short burst rather than a single + * notify to trip. + */ +#define FUSE_NOTIFY_DIO_INTERVAL max_t(unsigned long, HZ / 10, 1) +#define FUSE_NOTIFY_EWMA_SHIFT 2 +#define FUSE_NOTIFY_EWMA_SEED (2 * FUSE_NOTIFY_DIO_INTERVAL) + /** FUSE inode */ struct fuse_inode { /** Inode data */ @@ -184,6 +198,34 @@ struct fuse_inode { /* dlm locked areas we have sent lock requests for */ struct fuse_dlm_cache dlm_locked_areas; + + /* + * Serializes buffered-write page-cache dirtying against + * the forced-direct-IO latch transition driven by + * NOTIFY_INVAL_INODE (fuse_reverse_inval_inode()), which + * may be delivered by the same server thread that still + * owes a reply to an in-flight write holding the inode + * lock. The buffered writer holds this for read around + * the dirtying and re-checks the latch under it; the + * NOTIFY latch site takes it for write (trylock, never + * blocking) around its page-cache invalidate + latch set. + * Only regular files initialise it -- it shares storage + * with the readdir-cache union arm. + */ + struct rw_semaphore wb_inval_rwsem; + + /* + * Rate of FUSE_NOTIFY_INVAL_INODE data invalidations + * for this whole file: notify_stamp is the jiffies of + * the last one, notify_interval_ewma the EWMA of the + * inter-arrival interval (jiffies, scaled by + * 2^FUSE_NOTIFY_EWMA_SHIFT). A rapid stream (short + * average interval) with a local writer latches the + * inode into direct IO. Protected by fi->lock; regular + * files only (shares the readdir-cache union arm). + */ + unsigned long notify_stamp; + unsigned int notify_interval_ewma; }; /* readdir cache (directory only) */ @@ -260,6 +302,14 @@ enum { * or the fuse server has an exclusive "lease" on distributed fs */ FUSE_I_EXCLUSIVE, + /* + * Latched into direct IO: a NOTIFY_INVAL_INODE arrived while the file + * was open for writing here, so another (remote) entity is modifying it + * concurrently. Reads and writes are routed direct (shared-lock + * parallel dio) until the last writer closes or the inode is mmapped. + * See fuse_reverse_inval_inode()/fuse_file_io_open(). + */ + FUSE_I_FORCE_DIO, }; struct fuse_conn; @@ -1615,6 +1665,12 @@ void fuse_inode_uncached_io_end(struct fuse_inode *fi); int fuse_file_io_open(struct file *file, struct inode *inode); void fuse_file_io_release(struct fuse_file *ff, struct inode *inode); +/* Inode latched into forced direct IO after a remote-modify notification */ +static inline bool fuse_inode_force_dio(struct inode *inode) +{ + return test_bit(FUSE_I_FORCE_DIO, &get_fuse_inode(inode)->state); +} + /* file.c */ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, struct inode *inode, diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index f637aecda9b37d..1303cc0eca0748 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -603,6 +603,35 @@ static void fuse_invalidate_inode_entry(struct inode *inode) } } +/* + * Fold one FUSE_NOTIFY_INVAL_INODE data invalidation into the per-inode + * moving average of the notification inter-arrival interval and report whether + * the file is now "hot" -- notifications are arriving fast enough (short + * average interval) that a remote writer is repeatedly invalidating it. The + * average is an EWMA (weight 1/2^FUSE_NOTIFY_EWMA_SHIFT); the sample is clamped + * to FUSE_NOTIFY_EWMA_SEED so a notify after a long idle only cools the average + * and cannot overflow the accumulator. Must be called under fi->lock; called + * for every data invalidation so the average stays current even while no local + * writer is open. + */ +static bool fuse_notify_inval_hot(struct fuse_inode *fi) +{ + unsigned long now = jiffies; + unsigned long sample; + unsigned int avg; + + sample = min_t(unsigned long, now - fi->notify_stamp, + FUSE_NOTIFY_EWMA_SEED); + fi->notify_stamp = now; + + /* E += sample - (E >> SHIFT); avg = E >> SHIFT */ + fi->notify_interval_ewma += sample - + (fi->notify_interval_ewma >> FUSE_NOTIFY_EWMA_SHIFT); + avg = fi->notify_interval_ewma >> FUSE_NOTIFY_EWMA_SHIFT; + + return avg < FUSE_NOTIFY_DIO_INTERVAL; +} + int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { @@ -652,8 +681,73 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, pg_end == -1 ? 0 : (offset + len - 1)); - invalidate_inode_pages2_range(inode->i_mapping, - pg_start, pg_end); + /* + * A data invalidation means another (remote) entity is modifying + * the file. Keep a moving average of how fast these notifications + * arrive for the whole inode; when they come in a rapid stream -- + * a remote writer repeatedly invalidating the file -- and it is + * also open for writing here, latch the inode into direct IO: + * reads and writes are served direct (from the server) until the + * last writer closes or the inode is mmapped. A lone or occasional + * notify keeps the average high and does not trip the switch. Only + * under writeback+dlm, where the buffered write's RMW read is + * skipped so its wb_inval_rwsem read-side section is free of server + * round-trips. + * + * fuse_notify_inval_hot() updates the average under fi->lock and is + * called for every data invalidation so it stays current even while + * no local writer is open. When it trips, take wb_inval_rwsem for + * write so the buffered write path -- which holds it for read across + * its dirtying and re-checks the latch under it -- cannot strand + * dirty folios after the cache is dropped. Use a trylock and never + * block: this may run on the server thread that still owes an + * in-flight write (holding the inode lock) its reply, so blocking on + * the rwsem or the inode lock would deadlock. If the writer has + * gone, skip the latch this round (best effort); the invalidate + * still runs. Only regular files initialise the average and the + * rwsem (they share storage with the readdir-cache union arm), so + * gate on S_ISREG. When latched, drop the whole mapping rather than + * just the notified range, or dirty folios outside it would be + * invisible to the forced direct reads (stale read / lost write). + */ + if (S_ISREG(inode->i_mode) && fc->writeback_cache && fc->dlm && + !FUSE_IS_DAX(inode) && !fuse_inode_backing(fi) && + !mapping_mapped(inode->i_mapping)) { + bool hot, has_writer; + + spin_lock(&fi->lock); + hot = fuse_notify_inval_hot(fi); + has_writer = !list_empty(&fi->write_files); + spin_unlock(&fi->lock); + + if (hot && has_writer && !fuse_inode_force_dio(inode) && + down_write_trylock(&fi->wb_inval_rwsem)) { + bool latched = false; + + spin_lock(&fi->lock); + if (!list_empty(&fi->write_files)) { + set_bit(FUSE_I_FORCE_DIO, &fi->state); + latched = true; + } + spin_unlock(&fi->lock); + + if (latched) { + pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", + nodeid); + invalidate_inode_pages2(inode->i_mapping); + } else { + invalidate_inode_pages2_range(inode->i_mapping, + pg_start, pg_end); + } + up_write(&fi->wb_inval_rwsem); + } else { + invalidate_inode_pages2_range(inode->i_mapping, + pg_start, pg_end); + } + } else { + invalidate_inode_pages2_range(inode->i_mapping, + pg_start, pg_end); + } } iput(inode); return 0; diff --git a/fs/fuse/iomode.c b/fs/fuse/iomode.c index 3728933188f307..1076d954de94eb 100644 --- a/fs/fuse/iomode.c +++ b/fs/fuse/iomode.c @@ -232,6 +232,16 @@ int fuse_file_io_open(struct file *file, struct inode *inode) !(ff->open_flags & FOPEN_PASSTHROUGH)) return 0; + /* + * The inode was latched into direct IO after a remote-modify + * notification arrived while it was open for writing here. Open this + * file uncached as well so its IO is routed direct and it does not + * re-enter caching mode. + */ + if (test_bit(FUSE_I_FORCE_DIO, &fi->state) && + !(ff->open_flags & FOPEN_PASSTHROUGH)) + return 0; + if (ff->open_flags & FOPEN_PASSTHROUGH) err = fuse_file_passthrough_open(inode, file); else From c2e13f99cceb6fb3997b49fdbc9e3fdcd8ccef3a Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 22 Jun 2026 15:14:46 +0200 Subject: [PATCH 54/77] fuse: bound folio order to the per-request page limit fuse_readahead() batches whole folios into a single request, capped at min(fc->max_pages, fc->max_read/PAGE_SIZE) pages, but fuse_init_file_inode() let the page cache build folios up to MAX_PAGECACHE_ORDER. A large sequential read could thus produce a folio bigger than one request can carry: the first loop iteration took the folio_pages > cur_pages path, fired WARN_ON(!pages), and broke with ap->num_folios == 0. fuse_send_readpages() was still called and dereferenced a NULL ap->folios[0] via folio_pos(), oopsing at CR2=0x20 (folio->index). Cap the folio order to the per-request page limit so the page cache can never build an unserviceable folio. There is a patch for upstream that does exactly the same. In later versions this is not needed but for 6.17 we need this. Signed-off-by: Horst Birthelmer (cherry picked from commit 85f949cc8ba9927d199b3789a4560ad66737a124) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 105c6a8a00df4b..8b18aefe4cf4e5 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3670,6 +3670,20 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); - if (enable_large_folios) - mapping_set_large_folios(inode->i_mapping); + if (enable_large_folios) { + /* + * Readahead and writeback batch whole folios into a single + * request, capped at min(fc->max_pages, fc->max_read/PAGE_SIZE) + * pages. The page cache must therefore never build a folio + * larger than that, or fuse_readahead() trips WARN_ON(!pages) + * and then dereferences a NULL ap->folios[0] in + * fuse_send_readpages(). Bound the folio order to the request + * limit instead of MAX_PAGECACHE_ORDER. + */ + unsigned int max_pages = min(fc->max_pages, + fc->max_read >> PAGE_SHIFT); + + mapping_set_folio_order_range(inode->i_mapping, 0, + ilog2(max_pages ?: 1)); + } } From a58871ef22ba28d3a17939c05b516aaba7daa139 Mon Sep 17 00:00:00 2001 From: Bernd Schubert Date: Tue, 30 Jun 2026 13:47:52 +0200 Subject: [PATCH 55/77] fuse: allow parallel direct writes past EOF Extending FOPEN_PARALLEL_DIRECT_WRITES writes were forced onto the exclusive inode lock, re-serializing the parallel phase. The exclusive lock only bundled "write + advance i_size + undo-on-failure" into one unit. But i_size is committed by fuse_write_update_attr() under fi->lock, only on a successful growing write and independent of the inode rwsem -- so shared-lock writers commit size correctly and have nothing to undo. Drop the past-EOF exclusive triggers and gate the whole-file fuse_do_truncate() rollback on holding the exclusive lock. Lock mode is passed to __fuse_direct_IO(); i_size is committed at the same point in every path, only the failure rollback differs: non-exclusive (relaxed, parallel): fuse_direct_write_iter fuse_dio_lock -> inode_lock_shared (exclusive=false) __fuse_direct_IO(.., false) fuse_direct_io() write to server fuse_write_update_attr() commit i_size (on success) no rollback exclusive (append / caching / !parallel): fuse_direct_write_iter fuse_dio_lock -> inode_lock (exclusive=true) __fuse_direct_IO(.., true) fuse_direct_io() write to server fuse_write_update_attr() commit i_size (on success) ret<0 & extend -> fuse_do_truncate() rollback exclusive (caching-mode O_DIRECT): fuse_cache_write_iter -> inode_lock (exclusive) generic_file_direct_write -> fuse_direct_IO __fuse_direct_IO(.., true) fuse_direct_io() write to server fuse_write_update_attr() commit i_size (on success) ret<0 & extend -> fuse_do_truncate() rollback Signed-off-by: Bernd Schubert (cherry picked from commit c99a893c90f7faff2e26fd4f47a99e9c4deb1476) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 53 ++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 8b18aefe4cf4e5..3b195ee8237297 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1593,13 +1593,6 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) return res; } -static bool fuse_io_past_eof(struct kiocb *iocb, struct iov_iter *iter) -{ - struct inode *inode = file_inode(iocb->ki_filp); - - return iocb->ki_pos + iov_iter_count(iter) > i_size_read(inode); -} - /* * @return true if an exclusive lock for direct IO writes is needed */ @@ -1631,10 +1624,6 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from if (!force_dio && test_bit(FUSE_I_CACHE_IO_MODE, &fi->state)) return true; - /* Parallel dio beyond EOF is not supported, at least for now. */ - if (fuse_io_past_eof(iocb, from)) - return true; - return false; } @@ -1653,16 +1642,15 @@ static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from, * New parallal dio allowed only if inode is not in caching * mode and denies new opens in caching mode. This check * should be performed only after taking shared inode lock. - * Previous past eof check was without inode lock and might - * have raced, so check it again. * - * Under the forced-dio latch the cached/uncached accounting is - * bypassed (the latch guarantees the cache is flushed and not - * repopulated), so only re-check the past-eof condition. + * Under the forced-dio latch the uncached-io accounting is + * bypassed entirely -- fuse_dio_unlock() likewise skips + * fuse_inode_uncached_io_end() -- so do not take a reference + * here. An unbalanced start would drive fi->iocachectr + * permanently negative and hang the next caching-mode open. */ - if (fuse_io_past_eof(iocb, from) || - (!test_bit(FUSE_I_FORCE_DIO, &fi->state) && - fuse_inode_uncached_io_start(fi, NULL) != 0)) { + if (!test_bit(FUSE_I_FORCE_DIO, &fi->state) && + fuse_inode_uncached_io_start(fi, NULL) != 0) { inode_unlock_shared(inode); inode_lock(inode); *exclusive = true; @@ -2110,14 +2098,16 @@ static ssize_t __fuse_direct_read(struct fuse_io_priv *io, return res; } -static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter); +static ssize_t __fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter, + bool exclusive); static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) { ssize_t res; if (!is_sync_kiocb(iocb)) { - res = fuse_direct_IO(iocb, to); + /* exclusive is unused on reads; rollback is write-only */ + res = __fuse_direct_IO(iocb, to, true); } else { struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); @@ -2140,7 +2130,7 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) if (res > 0) { task_io_account_write(res); if (!is_sync_kiocb(iocb)) { - res = fuse_direct_IO(iocb, from); + res = __fuse_direct_IO(iocb, from, exclusive); } else { struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); @@ -3259,7 +3249,7 @@ static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off) } static ssize_t -fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +__fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter, bool exclusive) { DECLARE_COMPLETION_ONSTACK(wait); ssize_t ret = 0; @@ -3353,14 +3343,27 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) if (iov_iter_rw(iter) == WRITE) { fuse_write_update_attr(inode, pos, ret); - /* For extending writes we already hold exclusive lock */ - if (ret < 0 && offset + count > i_size) + /* + * Whole-file rollback is only safe under an exclusive lock. + * Parallel writers commit i_size only on success (nothing to + * undo); the server owns failed-extend cleanup. + */ + if (exclusive && ret < 0 && offset + count > i_size) fuse_do_truncate(file); } return ret; } +static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + /* + * Only reached via generic_file_direct_write/read() + * (caching-mode O_DIRECT), which holds the inode lock exclusively. + */ + return __fuse_direct_IO(iocb, iter, true); +} + static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end) { int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX); From c8875413ba76ee97f18ecf53cca40a9c5d72857a Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 16 Jul 2026 14:02:56 +0200 Subject: [PATCH 56/77] fuse: balance uncached_io accounting under forced-DIO latch fuse_dio_lock() takes an uncached_io reference (via fuse_inode_uncached_io_start()) only when FUSE_I_FORCE_DIO is clear, while fuse_dio_unlock() decided whether to drop it by re-reading FUSE_I_FORCE_DIO. On this tree the latch is toggled asynchronously by the inode-invalidation notify-storm path, so the bit can differ between the lock and the unlock of a single direct write: - clear at lock (reference taken), set before unlock: the reference is never dropped, leaving fi->iocachectr permanently negative and hanging the next caching-mode open; - set at lock (no reference), cleared before unlock: fuse_inode_uncached_io_end() is called without a matching start, tripping WARN_ON(fi->iocachectr >= 0) and corrupting the counter. Record in fuse_dio_lock() whether a reference was actually taken and have fuse_dio_unlock() drop it based on that captured decision instead of re-testing the racy bit, so the accounting stays balanced regardless of any FORCE_DIO transition mid-write. Signed-off-by: Horst Birthelmer (cherry picked from commit 6bf7ec0a5f17050e002e311959e81e6af201d19b) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 3b195ee8237297..a61514f219c7aa 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1628,7 +1628,7 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from } static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from, - bool *exclusive) + bool *exclusive, bool *uncached) { struct inode *inode = file_inode(iocb->ki_filp); struct fuse_inode *fi = get_fuse_inode(inode); @@ -1642,23 +1642,20 @@ static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from, * New parallal dio allowed only if inode is not in caching * mode and denies new opens in caching mode. This check * should be performed only after taking shared inode lock. - * - * Under the forced-dio latch the uncached-io accounting is - * bypassed entirely -- fuse_dio_unlock() likewise skips - * fuse_inode_uncached_io_end() -- so do not take a reference - * here. An unbalanced start would drive fi->iocachectr - * permanently negative and hang the next caching-mode open. */ - if (!test_bit(FUSE_I_FORCE_DIO, &fi->state) && - fuse_inode_uncached_io_start(fi, NULL) != 0) { - inode_unlock_shared(inode); - inode_lock(inode); - *exclusive = true; + if (!test_bit(FUSE_I_FORCE_DIO, &fi->state)) { + if (fuse_inode_uncached_io_start(fi, NULL) != 0) { + inode_unlock_shared(inode); + inode_lock(inode); + *exclusive = true; + } else { + *uncached = true; + } } } } -static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive) +static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive, bool uncached) { struct inode *inode = file_inode(iocb->ki_filp); struct fuse_inode *fi = get_fuse_inode(inode); @@ -1666,12 +1663,7 @@ static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive) if (exclusive) { inode_unlock(inode); } else { - /* - * Allow opens in caching mode after last parallel dio end. - * Skipped under the forced-dio latch, which never took an - * uncached_io reference in fuse_dio_lock(). - */ - if (!test_bit(FUSE_I_FORCE_DIO, &fi->state)) + if (uncached) fuse_inode_uncached_io_end(fi); inode_unlock_shared(inode); } @@ -2122,10 +2114,11 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) struct inode *inode = file_inode(iocb->ki_filp); struct address_space *mapping = inode->i_mapping; loff_t pos = iocb->ki_pos; + bool exclusive = false; + bool uncached = false; ssize_t res; - bool exclusive; - fuse_dio_lock(iocb, from, &exclusive); + fuse_dio_lock(iocb, from, &exclusive, &uncached); res = generic_write_checks(iocb, from); if (res > 0) { task_io_account_write(res); @@ -2149,7 +2142,7 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) (pos + res - 1) >> PAGE_SHIFT); } } - fuse_dio_unlock(iocb, exclusive); + fuse_dio_unlock(iocb, exclusive, uncached); return res; } From 7184b5da548fcb6634edd1ac3c2fd40f48c6b7fd Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Wed, 8 Jul 2026 15:49:48 +0200 Subject: [PATCH 57/77] fuse: don't hold i_rwsem exclusively when doing buffered write Signed-off-by: Horst Birthelmer (cherry picked from commit 9d6feaaf47baf6c906dc91b0238baa869f6e51d6) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 109 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 106 insertions(+), 3 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index a61514f219c7aa..cf46581e5ebf93 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1746,6 +1746,45 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from); +/* + * @return true if an exclusive inode lock is needed for a cached (buffered) + * write. + * + * Buffered writes normally hold the inode rwsem exclusively, serialising all + * writers even on disjoint ranges. The DLM-serialised iomap writeback path is + * the exception: the DLM already excludes cluster-wide, and i_size is committed + * under fi->lock rather than the inode rwsem (see fuse_cache_write_iter()), so + * disjoint writers (MPI-IO / IOR) may share the lock. Mirrors + * fuse_dio_wr_exclusive_lock() for the direct path. + */ +static bool fuse_cache_wr_exclusive_lock(struct kiocb *iocb, bool writeback) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct fuse_conn *fc = get_fuse_conn(inode); + + /* Only the DLM-serialised iomap writeback path relaxes the lock. */ + if (!fc->dlm || !writeback) + return true; + + /* O_DIRECT writes fall back to generic_file_direct_write(). */ + if (iocb->ki_flags & IOCB_DIRECT) + return true; + + /* Append needs the eventual EOF - always needs an exclusive lock. */ + if (iocb->ki_flags & IOCB_APPEND) + return true; + + return false; +} + +static void fuse_cache_wr_unlock(struct inode *inode, bool exclusive) +{ + if (exclusive) + inode_unlock(inode); + else + inode_unlock_shared(inode); +} + static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; @@ -1758,6 +1797,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) struct fuse_inode *fi = get_fuse_inode(inode); bool writeback = false; bool wb_guard = false; + bool exclusive = true; /* * The inode may have been latched into forced direct IO -- by a @@ -1805,7 +1845,11 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) } } - inode_lock(inode); + exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); + if (exclusive) + inode_lock(inode); + else + inode_lock_shared(inode); /* * The forced-direct-IO latch feature is active under writeback+dlm; @@ -1823,7 +1867,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) down_read(&fi->wb_inval_rwsem); if (fuse_inode_force_dio(inode)) { up_read(&fi->wb_inval_rwsem); - inode_unlock(inode); + fuse_cache_wr_unlock(inode, exclusive); return fuse_direct_write_iter(iocb, from); } } @@ -1845,7 +1889,66 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) written = direct_write_fallback(iocb, from, written, fuse_perform_write(iocb, from)); } else if (writeback) { + loff_t pos = iocb->ki_pos; + loff_t end = pos + count; + loff_t orig_size = 0; + bool extended = false; + + /* + * i_size is not protected by the shared lock in inode->i_rwsem. + * So if iomap_write_iter() grew EOF past i_size via its normal + * unlocked read-modify-write, two concurrent writers could race + * and one's update would get lost. + * To avoid this, claim the extension up front under fi->lock, + * so iomap sees pos + written <= i_size and never touches i_size + * itself. The update can then safely happen here, the same way + * fuse_write_update_attr() commits size on the direct io path. + * + * The lockless pre-check below avoids needlessly locking fi->lock + * if writes fall within the existing i_size. + * Operations that grow the file size take fi->lock, whereas a + * truncate holds the inode->i_rwsem exclusive. A stale read + * may over trigger this slow path, but it won’t miss an extension + * beyond i_size. + * + * The exclusive path keeps the classic behavior + * iomap owns the i_size update, serialized by the inode lock. + */ + if (!exclusive && end > i_size_read(inode)) { + spin_lock(&fi->lock); + orig_size = i_size_read(inode); + if (end > orig_size) { + i_size_write(inode, end); + extended = true; + } + spin_unlock(&fi->lock); + + /* Zero the tail of the folio straddling the old EOF. */ + if (extended && orig_size < pos) + pagecache_isize_extended(inode, orig_size, pos); + } + written = fuse_writeback_write_iter(iocb, from, file); + + /* + * Reconcile the speculative extension with what was actually + * written (short write, error, or nothing written all retract + * to the reached position). Only retract if no concurrent + * extender has pushed i_size past our claim; otherwise + * [reached, end) is a legitimate hole inside their extension and + * must remain. + */ + if (extended) { + loff_t reached = written > 0 ? pos + written : orig_size; + + if (reached < end) { + spin_lock(&fi->lock); + if (i_size_read(inode) == end) + i_size_write(inode, reached); + spin_unlock(&fi->lock); + } + } + if (written < 0) { err = written; goto out; @@ -1856,7 +1959,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) out: if (wb_guard) up_read(&fi->wb_inval_rwsem); - inode_unlock(inode); + fuse_cache_wr_unlock(inode, exclusive); if (written > 0) written = generic_write_sync(iocb, written); From 98d943bfebcd4c1a9e1a31e70adc89879c60f85b Mon Sep 17 00:00:00 2001 From: Hai Zhong Zhou Date: Thu, 16 Jul 2026 09:55:59 +0000 Subject: [PATCH 58/77] fuse: acquire dlm lock for the normal buffer read Acquire the dlm lock from fuse server for the normal buffer read path to ensure the distributed page cache across different nodes can be co-existing and consistency. More importantly, this change will correct the DLM lock and folio locks ordering for the buffer read path, thus can avoid the potential deadlock between the buffer read and page cache invalidation processes. Signed-off-by Hai Zhong Zhou (cherry picked from commit cf349bc7b27864a36a092fbd4318bd8e80796f47) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 14 +++++++++++--- fs/fuse/fuse_dlm_cache.c | 22 ++++++++++++---------- fs/fuse/fuse_dlm_cache.h | 6 +++--- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index cf46581e5ebf93..ae60867ce90715 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1295,7 +1295,8 @@ static void fuse_readahead(struct readahead_control *rac) static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { - struct inode *inode = iocb->ki_filp->f_mapping->host; + struct file *file = iocb->ki_filp; + struct inode *inode = file->f_mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); /* @@ -1311,6 +1312,12 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) return err; } + /* if we have dlm support acquire a read lock for the area + * we are reading from. */ + if (fc->writeback_cache && fc->dlm) + fuse_get_dlm_lock(file, iocb->ki_pos, + iov_iter_count(to), FUSE_PAGE_LOCK_READ); + return generic_file_read_iter(iocb, to); } @@ -1840,7 +1847,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) iocb->ki_pos; size_t length = iov_iter_count(from); - fuse_get_dlm_write_lock(file, pos, length); + fuse_get_dlm_lock(file, pos, length, + FUSE_PAGE_LOCK_WRITE); } } } @@ -2773,7 +2781,7 @@ static void fuse_vma_close(struct vm_area_struct *vma) /** * Request a DLM lock from the FUSE server. * - * This routine is similar to fuse_get_dlm_write_lock(), but it + * This routine is similar to fuse_get_dlm_lock(), but it * does not cache the DLM lock in the kernel. */ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t length) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index d765dd8018cc6a..ea296a2e9ec89c 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -487,10 +487,14 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, } /** - * request a dlm lock from the fuse server + * fuse_get_dlm_lock - request a dlm lock from the fuse server + * @file: the file being accessed + * @offset: byte offset into the file (need not be page-aligned) + * @length: length of the region in bytes (need not be page-aligned) + * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE */ -void fuse_get_dlm_write_lock(struct file *file, loff_t offset, - size_t length) +void fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) { struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(file); @@ -500,7 +504,7 @@ void fuse_get_dlm_write_lock(struct file *file, loff_t offset, uint64_t end = (offset + length - 1) | (PAGE_SIZE - 1); /* note that the offset and length don't have to be page aligned here - * but since we only get here on writeback caching we will send out + * but since we only get here on writeback caching we will send out * page aligned requests */ offset &= PAGE_MASK; @@ -513,8 +517,7 @@ void fuse_get_dlm_write_lock(struct file *file, loff_t offset, * at the same time. It is intentionally not protected * since a DLM implementation in the FUSE server should take care * of any races in lock requests */ - if (fuse_dlm_range_is_locked(fi, offset, - end, FUSE_PAGE_LOCK_WRITE)) + if (fuse_dlm_range_is_locked(fi, offset, end, mode)) return; /* we already have this area locked */ memset(&inarg, 0, sizeof(inarg)); @@ -522,7 +525,8 @@ void fuse_get_dlm_write_lock(struct file *file, loff_t offset, inarg.start = offset; inarg.end = end; - inarg.type = FUSE_DLM_LOCK_WRITE; + inarg.type = (mode == FUSE_PAGE_LOCK_WRITE) ? + FUSE_DLM_LOCK_WRITE : FUSE_DLM_LOCK_READ; args.opcode = FUSE_DLM_WB_LOCK; args.nodeid = get_node_id(inode); @@ -551,8 +555,6 @@ void fuse_get_dlm_write_lock(struct file *file, loff_t offset, return; } else { /* ignore any errors here, there is no way we can react appropriately */ - fuse_dlm_lock_range(fi, outarg.start, - outarg.end, - FUSE_PAGE_LOCK_WRITE); + fuse_dlm_lock_range(fi, outarg.start, outarg.end, mode); } } diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 438d31d28b666e..5c3deaa3536866 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -43,8 +43,8 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, uint64_t end, enum fuse_page_lock_mode mode); -/* this is the interface to the filesystem */ -void fuse_get_dlm_write_lock(struct file *file, loff_t offset, - size_t length); +/* This is the interface to the filesystem */ +void fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode); #endif /* _FS_FUSE_DLM_CACHE_H */ From eb4d9492551f314536bc199cb58cba0d2a69a883 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 16 Jul 2026 14:55:18 +0200 Subject: [PATCH 59/77] fuse: fence cached reads against NOTIFY invalidate with a percpu gate A FUSE_NOTIFY_INVAL_INODE is a coherency event: once the server signals a remote modify, no local read may return a page it has superseded. The invalidate ran unserialized against cache-serving reads, so a buffered read could hand back a stale folio it still held a reference to. Convert the per-inode wb_inval_rwsem to a percpu_rw_semaphore and take its read side around the cache-serving read as well as the existing buffered write. The read side is per-CPU, so it scales on a shared file; the NOTIFY takes the write side blocking, giving the invalidate priority -- it parks new readers, drains in-flight ones, then drops the cache. Every gated invalidate now runs under the write side, not just the storm-latching one. The gate is allocated only for writeback+dlm regular files and is NULL elsewhere (best-effort invalidate, as before). The blocking write side may run on the notify-delivering server thread, so it is safe only under a server that services request replies on other threads; redfs' dlm server provides that contract. Signed-off-by: Horst Birthelmer (cherry picked from commit d6180a20384a2f0d4b0eff4f4f8ebd00a5998c60) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 73 ++++++++++++++++++++++----- fs/fuse/fuse_i.h | 35 ++++++++----- fs/fuse/inode.c | 129 ++++++++++++++++++++++++++++++----------------- 3 files changed, 165 insertions(+), 72 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index ae60867ce90715..cab324c4522e99 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1293,11 +1293,16 @@ static void fuse_readahead(struct readahead_control *rac) iomap_readahead(&fuse_iomap_ops, &ctx, NULL); } +static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); + static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + ssize_t res; /* * In auto invalidate mode, always update attributes on read. @@ -1318,7 +1323,29 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) fuse_get_dlm_lock(file, iocb->ki_pos, iov_iter_count(to), FUSE_PAGE_LOCK_READ); - return generic_file_read_iter(iocb, to); + /* + * Fence the cache-serving read against a NOTIFY invalidate so we never + * hand back a folio the server has just superseded. The gate read side + * is per-CPU cheap; the NOTIFY holds the write side with priority. + * Re-check the forced-DIO latch under it: if a storm latched us while we + * waited on a pending writer, reroute to direct like the buffered write + * path, so we do not repopulate the cache the latch just dropped. + * wb_sem is NULL on non-writeback+dlm mounts (gate inactive). + */ + if (wb_sem) { + percpu_down_read(wb_sem); + if (fuse_inode_force_dio(inode)) { + percpu_up_read(wb_sem); + return fuse_direct_read_iter(iocb, to); + } + } + + res = generic_file_read_iter(iocb, to); + + if (wb_sem) + percpu_up_read(wb_sem); + + return res; } static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, @@ -1802,6 +1829,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) ssize_t err, count; struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; bool writeback = false; bool wb_guard = false; bool exclusive = true; @@ -1861,20 +1889,21 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) /* * The forced-direct-IO latch feature is active under writeback+dlm; - * hold wb_inval_rwsem for read across the page-cache dirtying so a - * concurrent NOTIFY_INVAL_INODE -- which latches the inode under the - * write side of this lock via a non-blocking trylock -- cannot strand - * the folios we are about to write. Re-check the latch under it (it may - * have been set while we blocked on the inode lock) and re-route to the - * direct path if set. Taken before task_io_account_write() so a - * re-route is not double-counted; the DLM write lock taken above is - * harmless as the direct path does its own server coordination. + * hold the coherency gate (wb_inval_rwsem) for read across the + * page-cache dirtying so a concurrent NOTIFY_INVAL_INODE -- which takes + * the write side (blocking, with priority) around its invalidate + latch + * set -- cannot strand the folios we are about to write. Re-check the + * latch under it (it may have been set while we blocked on the inode + * lock) and re-route to the direct path if set. Taken before + * task_io_account_write() so a re-route is not double-counted; the DLM + * write lock taken above is harmless as the direct path does its own + * server coordination. */ - wb_guard = fc->writeback_cache && fc->dlm; + wb_guard = !!wb_sem; if (wb_guard) { - down_read(&fi->wb_inval_rwsem); + percpu_down_read(wb_sem); if (fuse_inode_force_dio(inode)) { - up_read(&fi->wb_inval_rwsem); + percpu_up_read(wb_sem); fuse_cache_wr_unlock(inode, exclusive); return fuse_direct_write_iter(iocb, from); } @@ -1966,7 +1995,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) } out: if (wb_guard) - up_read(&fi->wb_inval_rwsem); + percpu_up_read(wb_sem); fuse_cache_wr_unlock(inode, exclusive); if (written > 0) written = generic_write_sync(iocb, written); @@ -3770,7 +3799,23 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->iocachectr = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); - init_rwsem(&fi->wb_inval_rwsem); + /* + * Coherency gate for the forced-direct-IO feature; only writeback+dlm + * regular files need it. A percpu_rw_semaphore embeds per-CPU state, + * so allocate it out of line and only when the mount can use it rather + * than paying it on every inode. On failure leave it NULL: the gate + * stays inactive (best-effort invalidate) and the inode is still usable. + */ + fi->wb_inval_rwsem = NULL; + if (fc->writeback_cache && fc->dlm) { + struct percpu_rw_semaphore *sem = kmalloc(sizeof(*sem), GFP_KERNEL); + + if (sem && percpu_init_rwsem(sem)) { + kfree(sem); + sem = NULL; + } + fi->wb_inval_rwsem = sem; + } fi->notify_stamp = jiffies; fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index a4a2702c7926ff..1ad6c6b4d1dc77 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -200,19 +201,29 @@ struct fuse_inode { struct fuse_dlm_cache dlm_locked_areas; /* - * Serializes buffered-write page-cache dirtying against - * the forced-direct-IO latch transition driven by - * NOTIFY_INVAL_INODE (fuse_reverse_inval_inode()), which - * may be delivered by the same server thread that still - * owes a reply to an in-flight write holding the inode - * lock. The buffered writer holds this for read around - * the dirtying and re-checks the latch under it; the - * NOTIFY latch site takes it for write (trylock, never - * blocking) around its page-cache invalidate + latch set. - * Only regular files initialise it -- it shares storage - * with the readdir-cache union arm. + * Per-inode read/write coherency gate for the + * forced-direct-IO feature. Cache-serving buffered reads + * and buffered writes hold it for read; being a + * percpu_rw_semaphore the read side is per-CPU cheap and + * scales on a shared file. The NOTIFY invalidate + * (fuse_reverse_inval_inode()) holds it for write, which + * BLOCKS so the coherency notify has priority: it fences + * cache-serving reads (and buffered writes) out for the + * whole invalidate, so no folio a remote modify has + * superseded is ever handed back. + * + * The write side may run on the server thread delivering + * the notify, so a blocking writer is safe only under a + * server that services request replies on threads other + * than the one delivering the notify (see the NOTIFY site). + * + * Allocated out of line only for writeback+dlm regular + * files (it shares storage with the readdir-cache union + * arm); NULL on other mounts and on allocation failure, + * where the gate is inactive and the invalidate falls back + * to best-effort. */ - struct rw_semaphore wb_inval_rwsem; + struct percpu_rw_semaphore *wb_inval_rwsem; /* * Rate of FUSE_NOTIFY_INVAL_INODE data invalidations diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 1303cc0eca0748..45cc9f2a38e6c5 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -206,6 +206,23 @@ static void fuse_evict_inode(struct inode *inode) WARN_ON(!list_empty(&fi->queued_writes)); fuse_dlm_cache_release_locks(fi); } + + /* + * Free the coherency gate here rather than in ->free_inode: that runs + * from an RCU callback, where percpu_free_rwsem() may sleep in + * rcu_sync_dtor() if the write side has not fully quiesced. No user + * can remain by eviction time: gate readers hold a file reference and + * a concurrent notify holds an inode reference. wb_inval_rwsem lives + * in the regular-file union arm and is only ever allocated for regular + * files, so gate on S_ISREG (but not fuse_is_bad() -- bad-marked + * regular files still own a gate); a directory's overlapping + * readdir-cache fields must not be misread. + */ + if (S_ISREG(inode->i_mode) && fi->wb_inval_rwsem) { + percpu_free_rwsem(fi->wb_inval_rwsem); + kfree(fi->wb_inval_rwsem); + fi->wb_inval_rwsem = NULL; + } } static int fuse_reconfigure(struct fs_context *fsc) @@ -635,6 +652,7 @@ static bool fuse_notify_inval_hot(struct fuse_inode *fi) int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { + struct percpu_rw_semaphore *wb_sem = NULL; struct fuse_inode *fi; struct inode *inode; pgoff_t pg_start; @@ -677,73 +695,92 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * Note that this can lead to some inconsistencies if * the fuse server sends unaligned data */ fuse_dlm_unlock_range(fi, - offset, - pg_end == -1 ? 0 : - (offset + len - 1)); + offset, + pg_end == -1 ? 0 : + (offset + len - 1)); /* * A data invalidation means another (remote) entity is modifying - * the file. Keep a moving average of how fast these notifications - * arrive for the whole inode; when they come in a rapid stream -- - * a remote writer repeatedly invalidating the file -- and it is - * also open for writing here, latch the inode into direct IO: - * reads and writes are served direct (from the server) until the - * last writer closes or the inode is mmapped. A lone or occasional - * notify keeps the average high and does not trip the switch. Only - * under writeback+dlm, where the buffered write's RMW read is - * skipped so its wb_inval_rwsem read-side section is free of server - * round-trips. + * the file. Two things happen here: + * + * 1. Coherency. Drop the affected page-cache range so no local + * read returns a folio the remote modify has superseded. This + * runs under the write side of the per-inode coherency gate + * (wb_inval_rwsem), which fences cache-serving buffered reads + * and buffered writes out for the whole invalidate. Unlike the + * old best-effort trylock this BLOCKS -- the notify has + * priority: percpu_down_write() parks new gate readers, drains + * in-flight ones, then invalidates. A blocking writer here is + * safe only under a server that services request replies on + * threads other than the one delivering this notify: the write + * side waits for gate readers to drain, and a cache-miss read + * holds the read side across its FUSE_READ round-trip. redfs' + * dlm server provides that contract; a server that cannot must + * not enable writeback+dlm. * - * fuse_notify_inval_hot() updates the average under fi->lock and is - * called for every data invalidation so it stays current even while - * no local writer is open. When it trips, take wb_inval_rwsem for - * write so the buffered write path -- which holds it for read across - * its dirtying and re-checks the latch under it -- cannot strand - * dirty folios after the cache is dropped. Use a trylock and never - * block: this may run on the server thread that still owes an - * in-flight write (holding the inode lock) its reply, so blocking on - * the rwsem or the inode lock would deadlock. If the writer has - * gone, skip the latch this round (best effort); the invalidate - * still runs. Only regular files initialise the average and the - * rwsem (they share storage with the readdir-cache union arm), so - * gate on S_ISREG. When latched, drop the whole mapping rather than - * just the notified range, or dirty folios outside it would be - * invisible to the forced direct reads (stale read / lost write). + * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), under + * fi->lock, updated for every data invalidation) of how fast + * these arrive; when they come in a rapid stream -- a remote + * writer repeatedly invalidating -- and the inode is also open + * for writing here, latch it into direct IO until the last + * writer closes or it is mmapped. When latched, drop the whole + * mapping rather than just the notified range, or dirty folios + * outside it would be invisible to the forced direct reads + * (stale read / lost write). + * + * The gate (and the average) exist only for writeback+dlm regular + * files, and not while mmapped; elsewhere wb_sem is NULL and the + * invalidate runs unserialized (best-effort), as before. */ - if (S_ISREG(inode->i_mode) && fc->writeback_cache && fc->dlm && - !FUSE_IS_DAX(inode) && !fuse_inode_backing(fi) && - !mapping_mapped(inode->i_mapping)) { - bool hot, has_writer; + if (S_ISREG(inode->i_mode) && fc->writeback_cache && + fc->dlm && !FUSE_IS_DAX(inode) && + !fuse_inode_backing(fi) && + !mapping_mapped(inode->i_mapping)) + wb_sem = fi->wb_inval_rwsem; + + if (wb_sem) { + bool hot, has_writer, latched = false; spin_lock(&fi->lock); hot = fuse_notify_inval_hot(fi); has_writer = !list_empty(&fi->write_files); spin_unlock(&fi->lock); - if (hot && has_writer && !fuse_inode_force_dio(inode) && - down_write_trylock(&fi->wb_inval_rwsem)) { - bool latched = false; + /* + * Priority write side: park new gate readers, + * drain in-flight ones, then invalidate. Blocks + * (unlike the old trylock) -- see the contract in + * the comment above. + */ + percpu_down_write(wb_sem); + if (hot && has_writer && + !fuse_inode_force_dio(inode)) { spin_lock(&fi->lock); if (!list_empty(&fi->write_files)) { set_bit(FUSE_I_FORCE_DIO, &fi->state); latched = true; } spin_unlock(&fi->lock); + } - if (latched) { - pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", - nodeid); - invalidate_inode_pages2(inode->i_mapping); - } else { - invalidate_inode_pages2_range(inode->i_mapping, - pg_start, pg_end); - } - up_write(&fi->wb_inval_rwsem); - } else { + /* + * Latched: drop the whole mapping (dirty folios + * outside the notified range would be invisible to + * the forced direct reads). Otherwise just the + * notified range. + */ + if (fuse_inode_force_dio(inode)) + invalidate_inode_pages2(inode->i_mapping); + else invalidate_inode_pages2_range(inode->i_mapping, pg_start, pg_end); - } + + percpu_up_write(wb_sem); + + if (latched) + pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", + nodeid); } else { invalidate_inode_pages2_range(inode->i_mapping, pg_start, pg_end); From 670028b948ccf3f8f601ef5fb9d7cb101abe4878 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 17 Jul 2026 15:51:24 +0200 Subject: [PATCH 60/77] fuse: satisfy DLM read lock requests from a held write lock The buffered read path now acquires a DLM read lock via fuse_get_dlm_lock(..., FUSE_PAGE_LOCK_READ). Before sending the request to the server, fuse_get_dlm_lock() calls fuse_dlm_range_is_locked() to skip regions we already hold. That coverage check compared the held lock mode for exact equality (range->mode != lock_mode), so a range we already hold with an exclusive WRITE lock was reported as not-locked for a READ request. Because fuse_dlm_lock_range() intentionally does not downgrade a WRITE lock on a read, the region stays WRITE-locked and every subsequent read re-requests a DLM read lock from the server. This made read-after-write and re-read workloads flood the server with redundant FUSE_DLM_WB_LOCK requests, never converging. A held WRITE lock (exclusive) subsumes a READ lock. Treat a range as uncovered only when the held mode is strictly weaker than the requested mode (range->mode < lock_mode). READ requests are now satisfied by either a READ or a WRITE lock, while WRITE requests still require an existing WRITE lock (upgrade otherwise), matching the compatibility rules already documented in fuse_dlm_lock_range(). Signed-off-by: Horst Birthelmer (cherry picked from commit 4e3bef093ea1d715452e8f060c37d1c8f6881347) Signed-off-by: Allison Henderson --- fs/fuse/fuse_dlm_cache.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index ea296a2e9ec89c..40eda6daf75cae 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -454,9 +454,16 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, /* Check if the entire range is covered */ while (range && current_start <= end) { - /* If we're checking for a specific mode, verify it matches */ - if (lock_mode && range->mode != lock_mode) { - /* Wrong lock mode */ + /* + * The held lock must be at least as strong as the one + * requested. A WRITE lock (exclusive) satisfies a READ + * request, so only treat the range as uncovered when the + * held mode is weaker than what we ask for. This avoids + * re-requesting a READ lock for a range we already hold + * a WRITE lock on (e.g. read-after-write). + */ + if (lock_mode && range->mode < lock_mode) { + /* Held lock is weaker than requested */ up_read(&cache->lock); return false; } From 13c342da8f0113f58d53e4a68400853f9228c208 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 20 Jul 2026 19:08:41 +0200 Subject: [PATCH 61/77] fuse: zero-fill expanding writes instead of sending READ requests This fixes the problem where iomap was sending READ requests for file sections after EOF. For an expanding file we can assume zeros for the passage after EOF. Signed-off-by: Horst Birthelmer (cherry picked from commit ca1664e48545dc56ee498eab452a277cbd27e899) Signed-off-by: Allison Henderson --- fs/fuse/dir.c | 10 +++++++++ fs/fuse/file.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ fs/fuse/fuse_i.h | 46 ++++++++++++++++++++------------------ fs/fuse/inode.c | 17 ++++++++++++++ 4 files changed, 110 insertions(+), 21 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index db3d2a737a7e0e..badb9e59887c5f 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2177,7 +2177,10 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if (fc->dlm && fc->writeback_cache) fuse_dlm_cache_release_locks(fi); + spin_lock(&fi->lock); + fi->server_size = 0; i_size_write(inode, 0); + spin_unlock(&fi->lock); truncate_pagecache(inode, 0); goto out; } @@ -2269,6 +2272,13 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, /* see the comment in fuse_change_attributes() */ if (!is_wb || is_truncate) i_size_write(inode, outarg.attr.size); + /* + * A truncate settles the size on the server; only shrink the + * server-materialized bound: growing just exposes zeros, which the + * bound need not cover (see fuse_iomap_read_folio_range()). + */ + if (is_truncate && (loff_t) outarg.attr.size < fi->server_size) + fi->server_size = outarg.attr.size; if (is_truncate) { /* NOTE: this may release/reacquire fi->lock */ diff --git a/fs/fuse/file.c b/fs/fuse/file.c index cab324c4522e99..4f5012a52649a8 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -346,6 +346,7 @@ static void fuse_truncate_update_attr(struct inode *inode, struct file *file) spin_lock(&fi->lock); fi->attr_version = atomic64_inc_return(&fc->attr_version); + fi->server_size = 0; i_size_write(inode, 0); spin_unlock(&fi->lock); file_update_time(file); @@ -1164,8 +1165,41 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, struct file *file = iter->private; struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); size_t off = offset_in_folio(folio, pos); + bool hole; int ret; + + /* + * Expanding writes claim their new i_size up front (see + * fuse_cache_write_iter()), which keeps iomap's own beyond-EOF + * zeroing in iomap_block_needs_zeroing() from ever firing for the + * write's own range: every block of a file expansion would be read + * from the server although it cannot contain data. Zero-fill + * locally instead when the server is known to hold no data in the + * range and we hold the DLM write lock covering it: + * + * - fi->server_size bounds the data materialized on the server + * (writeback and direct write acknowledgements, server + * attributes), + * - local data not yet acknowledged sits in uptodate blocks, which + * iomap never passes to this callback, + * - the page-granular DLM write lock excludes data written by + * other nodes, re-checked against the live lock tree so a + * revoked lock falls back to reading. + */ + if (fc->dlm) { + spin_lock(&fi->lock); + hole = pos >= fi->server_size; + spin_unlock(&fi->lock); + + if (hole && fuse_dlm_range_is_locked(fi, pos, pos + len - 1, + FUSE_PAGE_LOCK_WRITE)) { + folio_zero_range(folio, off, len); + return 0; + } + } + ret = fuse_do_readfolio(file, folio, off, len); /* @@ -1417,6 +1451,15 @@ bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written) spin_lock(&fi->lock); fi->attr_version = atomic64_inc_return(&fc->attr_version); + if (written > 0 && S_ISREG(inode->i_mode)) { + /* + * The server acknowledged data up to @pos, keep the + * server-materialized bound in sync for the expansion + * zero-fill in fuse_iomap_read_folio_range(). + */ + if (pos > fi->server_size) + fi->server_size = pos; + } if (written > 0 && pos > inode->i_size) { i_size_write(inode, pos); ret = true; @@ -2479,6 +2522,20 @@ static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, if (!fc->writeback_cache) fuse_invalidate_attr_mask(inode, FUSE_STATX_MODIFY); spin_lock(&fi->lock); + if (!error) { + struct fuse_write_in *inarg = &wpa->ia.write.in; + + /* + * The server acknowledged this writeback, so data up to the + * end of the request is materialized on the server. Advance + * the bound before the folios end writeback below, i.e. + * before they can go clean and be reclaimed, so that + * fuse_iomap_read_folio_range() can never zero-fill a + * reclaimed range the server holds data in. + */ + if ((loff_t) (inarg->offset + inarg->size) > fi->server_size) + fi->server_size = inarg->offset + inarg->size; + } fi->writectr--; fuse_writepage_finish(wpa); spin_unlock(&fi->lock); @@ -3797,6 +3854,7 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fuse_dlm_cache_init(fi); fi->writectr = 0; fi->iocachectr = 0; + fi->server_size = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); /* diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 1ad6c6b4d1dc77..603e57200b1914 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -201,27 +201,31 @@ struct fuse_inode { struct fuse_dlm_cache dlm_locked_areas; /* - * Per-inode read/write coherency gate for the - * forced-direct-IO feature. Cache-serving buffered reads - * and buffered writes hold it for read; being a - * percpu_rw_semaphore the read side is per-CPU cheap and - * scales on a shared file. The NOTIFY invalidate - * (fuse_reverse_inval_inode()) holds it for write, which - * BLOCKS so the coherency notify has priority: it fences - * cache-serving reads (and buffered writes) out for the - * whole invalidate, so no folio a remote modify has - * superseded is ever handed back. - * - * The write side may run on the server thread delivering - * the notify, so a blocking writer is safe only under a - * server that services request replies on threads other - * than the one delivering the notify (see the NOTIFY site). - * - * Allocated out of line only for writeback+dlm regular - * files (it shares storage with the readdir-cache union - * arm); NULL on other mounts and on allocation failure, - * where the gate is inactive and the invalidate falls back - * to best-effort. + * Server-materialized size: an upper bound for how far + * the server holds file data. Seeded from + * server-reported attributes, advanced when the server + * acknowledges data (writeback completion, + * fuse_write_update_attr()), lowered again on + * truncate. A read-modify-write of a block starting + * at or past this bound needs no READ request under a + * held DLM write lock: the server has no data there + * (see fuse_iomap_read_folio_range()). Protected by + * fi->lock. + */ + loff_t server_size; + + /* + * Serializes buffered-write page-cache dirtying against + * the forced-direct-IO latch transition driven by + * NOTIFY_INVAL_INODE (fuse_reverse_inval_inode()), which + * may be delivered by the same server thread that still + * owes a reply to an in-flight write holding the inode + * lock. The buffered writer holds this for read around + * the dirtying and re-checks the latch under it; the + * NOTIFY latch site takes it for write (trylock, never + * blocking) around its page-cache invalidate + latch set. + * Only regular files initialise it -- it shares storage + * with the readdir-cache union arm. */ struct percpu_rw_semaphore *wb_inval_rwsem; diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 45cc9f2a38e6c5..06725fbf42281e 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -362,8 +362,12 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr u32 cache_mask; loff_t oldsize; struct timespec64 old_mtime; + bool have_size = !sx || (sx->mask & STATX_SIZE); + u64 srv_size; spin_lock(&fi->lock); + srv_size = attr->size; + /* * In case of writeback_cache enabled, writes update mtime, ctime and * may update i_size. In these cases trust the cached value in the @@ -388,6 +392,19 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr return; } + /* + * srv_size is the size the server reported before the writeback + * cache_mask above replaced attr->size with the local value. It + * bounds how far the server can hold data, letting the iomap write + * path zero-fill expansion read-modify-writes instead of sending + * READ requests, see fuse_iomap_read_folio_range(). Only ever grow + * it here: stale attributes were rejected above and truncation + * lowers it directly. + */ + if (have_size && S_ISREG(inode->i_mode) && + (loff_t) srv_size > fi->server_size) + fi->server_size = srv_size; + old_mtime = inode_get_mtime(inode); fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask, evict_ctr); From 7b956932bb9dc832ca9fd95e10aa40e302a53ba4 Mon Sep 17 00:00:00 2001 From: Hai Zhong Zhou Date: Wed, 29 Jul 2026 06:38:18 +0000 Subject: [PATCH 62/77] fuse: only refresh size on cached write when opened with O_APPEND In fuse_cache_write_iter, writeback_cache mode was always refreshing STATX_SIZE along with STATX_MODE before a buffered write. The size refresh is only needed for the O_APPEND path, where the kernel must know the current EOF before extending the file. For ordinary writes, fetching size is unnecessary work and can race with concurrent writes and then impact writeback performance. Keep refreshing STATX_MODE in all cases so SUID clearing still sees an up-to-date mode. Request STATX_SIZE only when the file is opened with O_APPEND. Signed-off-by Hai Zhong Zhou (cherry picked from commit a41215d061474c6f36834e4b2ebd9a9a61f13a34) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 4f5012a52649a8..a4e90182d2542b 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1888,9 +1888,12 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) return fuse_direct_write_iter(iocb, from); if (fc->writeback_cache) { - /* Update size (EOF optimization) and mode (SUID clearing) */ - err = fuse_update_attributes(mapping->host, file, - STATX_SIZE | STATX_MODE); + /* Update mode for SUID clearing, and also update size if the file + * is opened with O_APPEND mode. + */ + u32 request_mask = (file->f_flags & O_APPEND) ? + (STATX_SIZE | STATX_MODE) : STATX_MODE; + err = fuse_update_attributes(mapping->host, file, request_mask); if (err) return err; From c722576873266d4a6cdb8857dbf5af8e5f2d4147 Mon Sep 17 00:00:00 2001 From: Hai Zhong Zhou Date: Wed, 12 Aug 2026 09:32:34 +0000 Subject: [PATCH 63/77] fuse: disable local attribute cache override under DLM fuse_get_cache_mask() returned STATX_MTIME|CTIME|SIZE whenever writeback_cache was enabled, causing the kernel to trust its locally cached mtime/ctime/size over whatever the server returned in a GETATTR reply. This is unsafe under DLM: another node can hold a PW lock on the inode and modify its size/mtime independently, and the local writeback_cache values have no way of reflecting that. Under the DLM protocol, however, this override is unnecessary in the first place: a GETATTR always acquires a PR sattr lock, which forces every node holding a conflicting PW lock -- including the local node, for its own buffered writes -- to flush dirty pages before the server renders the reply. So under DLM the server's answer is always at least as fresh as anything cached locally, for all three attributes, not just size. Make fuse_get_cache_mask() return 0 whenever fc->dlm is set, when writeback_cache is enabled, so the kernel always trusts the server's attr/size reply in that case. The STATX_MTIME|CTIME|SIZE local-cache override remains only as a fallback for servers without DLM support, where no such flush-before-grant guarantee exists. Signed-off-by Hai Zhong Zhou (cherry picked from commit dc7336078eb3e2f21912358b877471757aaf1cf1) Signed-off-by: Allison Henderson --- fs/fuse/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 06725fbf42281e..9c737654d35359 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -347,7 +347,7 @@ u32 fuse_get_cache_mask(struct inode *inode) { struct fuse_conn *fc = get_fuse_conn(inode); - if (!fc->writeback_cache || !S_ISREG(inode->i_mode)) + if (!fc->writeback_cache || !S_ISREG(inode->i_mode) || fc->dlm) return 0; return STATX_MTIME | STATX_CTIME | STATX_SIZE; From 109645d7918a0aac80f597e8d70105fd7c184113 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Sat, 25 Jul 2026 12:45:45 +0200 Subject: [PATCH 64/77] fuse: seed DLM grant merging with an interval-tree lookup fuse_dlm_try_merge() locates the first merge candidate by walking from rb_first_cached() until it reaches the region just granted. The walk runs under the write-held cache rwsem on every fuse_dlm_lock_range() call, and the tree it walks holds every cached grant of the inode. Strided writers (IOR hard-write) accumulate grants that cannot merge with each other, so the tree keeps growing and every new grant pays a scan of all grants below it -- quadratic over the run, with fuse_dlm_range_is_locked() readers blocked behind each scan. Seed the merge with fuse_page_it_iter_first() on the region widened by one unit to each side instead; finding the lowest overlapping range is what the interval tree is there for. This also repairs two edge cases of the linear scan: a region starting at offset 0 made 'start - 1' wrap so the scan degenerated and merging was silently skipped, and a region ending at U64_MAX overflowed 'end + 1' in the loop bound, ending the merge after the first range. Both bounds now saturate. Signed-off-by: Horst Birthelmer (cherry picked from commit b2daa2e3888cc2a0f1565a2c8de27629f198e3f0) --- fs/fuse/fuse_dlm_cache.c | 57 ++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 40eda6daf75cae..2ec072b86312f4 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -120,26 +120,25 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, uint64_t end) { struct fuse_dlm_range *range, *next; - struct rb_node *node; + uint64_t first = start ? start - 1 : start; + uint64_t last = end < U64_MAX ? end + 1 : end; if (!cache) return; - /* Find the first range that might need merging */ - range = NULL; - node = rb_first_cached(&cache->ranges); - while (node) { - range = rb_entry(node, struct fuse_dlm_range, rb); - if (range->end >= start - 1) - break; - node = rb_next(node); - } - - if (!range || range->start > end + 1) - return; + /* + * Find the first range that might need merging. Directly adjacent + * ranges can merge, hence the region is widened by one unit to each + * side (saturating at the type bounds). This must stay an + * interval-tree lookup: the tree holds every cached grant of the + * inode and strided writers grow it for the lifetime of the file, + * so seeding the merge by walking from the tree minimum would make + * every new grant cost a full scan. + */ + range = fuse_page_it_iter_first(&cache->ranges, first, last); /* Try to merge ranges in and around the specified region */ - while (range && range->start <= end + 1) { + while (range && range->start <= last) { /* Get next range before we potentially modify the tree */ next = NULL; if (rb_next(&range->rb)) { @@ -150,11 +149,11 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, /* Try to merge with next range if adjacent and same mode */ if (next && range->mode == next->mode && range->end + 1 == next->start) { - /* Merge ranges */ - range->end = next->end; - - /* Remove next from tree */ + /* Merge ranges: re-insert so __subtree_end is updated */ fuse_page_it_remove(next, &cache->ranges); + fuse_page_it_remove(range, &cache->ranges); + range->end = next->end; + fuse_page_it_insert(range, &cache->ranges); kfree(next); /* Continue with the same range */ @@ -188,6 +187,7 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range, *new_range, *next; int lock_mode; + bool covered_to_end = false; int ret = 0; LIST_HEAD(to_lock); LIST_HEAD(to_upgrade); @@ -233,14 +233,17 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, } /* Move current_start past this range */ - current_start = max(current_start, range->end + 1); + if (range->end >= end) + covered_to_end = true; + else + current_start = max(current_start, range->end + 1); /* Move to next range */ range = next; } /* If there's a gap after the last range to the end, extend the range */ - if (current_start <= end) { + if (!covered_to_end && current_start <= end) { new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); if (!new_range) { ret = -ENOMEM; @@ -322,13 +325,17 @@ static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, /* If the hole is at the beginning of the range */ if (start == range->start) { + fuse_page_it_remove(range, &cache->ranges); range->start = end + 1; + fuse_page_it_insert(range, &cache->ranges); goto out; } /* If the hole is at the end of the range */ if (end == range->end) { + fuse_page_it_remove(range, &cache->ranges); range->end = start - 1; + fuse_page_it_insert(range, &cache->ranges); goto out; } @@ -400,10 +407,14 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, break; } else if (start > range->start) { /* Adjust the end of the range */ + fuse_page_it_remove(range, &cache->ranges); range->end = start - 1; + fuse_page_it_insert(range, &cache->ranges); } else if (end < range->end) { /* Adjust the start of the range */ + fuse_page_it_remove(range, &cache->ranges); range->start = end + 1; + fuse_page_it_insert(range, &cache->ranges); } else { /* Complete overlap, remove the range */ fuse_page_it_remove(range, &cache->ranges); @@ -475,6 +486,12 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, return false; } + /* Covered through the end of the requested range? */ + if (range->end >= end) { + up_read(&cache->lock); + return true; + } + /* Move current_start past this range */ current_start = range->end + 1; From 512fda2f1b20e53b4b2af299858cb2d69ab7c6b2 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Thu, 23 Jul 2026 10:29:12 +0200 Subject: [PATCH 65/77] fuse: re-validate the DLM grant after waiting on the coherency gate Both cached IO paths request their DLM lock first and then go to sleep on things a NOTIFY invalidate can be holding: the read path blocks on the coherency gate (writer priority), the write path additionally sleeps on a contended i_rwsem. A NOTIFY invalidate running in that window revokes exactly the lock just granted (fuse_dlm_unlock_range()), so the task wakes up and populates or dirties the page cache with no DLM coverage. Close the window without ever sending a FUSE_DLM_WB_LOCK request while holding the gate (a grant that had to wait on an invalidate delivered to this same client would deadlock against our own gate hold): - Drop the lock record under the gate write side in fuse_reverse_inval_inode(), so revocation and page drop are one atomic step with respect to the gate. - After entering the gate read side, re-check the grant against the live lock tree; if it was revoked while we waited, drop the gate, re-request, re-enter and check again. With the revoke now gated, passing the check means the lock cannot go away for the whole gate hold: a revoke arriving mid-operation parks until the IO is done. - Keep the write path's lock request ahead of the inode lock: the round trip must not capture the writer-priority i_rwsem for unbounded cluster-grant latency, and the in-gate re-validation already closes the grant-to-use window. Only O_APPEND moves below the lock, because its range is the current EOF -- stable only under the exclusive inode lock. This also fixes the append range itself: generic_write_checks() rewrites ki_pos to i_size for IOCB_APPEND, so the old 'i_size + ki_pos' double-counted (ki_pos is absolute, not relative) and locked a range disjoint from where the data lands. fuse_get_dlm_lock() now reports whether the grant is recorded, and the re-validation never re-requests a grant that failed, so it cannot spin (the read path seeds this from its pre-gate request instead of discarding that result). A grant the server issued but that could not be recorded (small-allocation -ENOMEM) reports FUSE_DLM_GRANT_UNRECORDED: coverage exists cluster-wide, so failing the IO would be wrong -- it proceeds, it just cannot re-validate. Empty ranges are trivially held, so a zero-length IO neither sends a doomed request nor spins in the retry loops. The write path returns a real failure to the caller instead of dirtying the cache without DLM coverage; only -ENOSYS still degrades to a plain cached write, since it means the server has no DLM at all and clears fc->dlm. The read path keeps falling through unlocked and additionally bounds its retry: a reader-only inode has no force-DIO latch to end a revoke storm, so after a few re-requests the read is served unlocked rather than looping in the kernel for the duration of the storm. Signed-off-by: Horst Birthelmer (cherry picked from commit 98ccde5facbe3c26d96b180ca2d046b1c40c1cb5) --- fs/fuse/file.c | 168 ++++++++++++++++++++++++++++----------- fs/fuse/fuse_dlm_cache.c | 102 +++++++++++++++++------- fs/fuse/fuse_dlm_cache.h | 17 +++- fs/fuse/inode.c | 35 +++++--- 4 files changed, 236 insertions(+), 86 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index a4e90182d2542b..7364955b38f88f 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1329,6 +1329,12 @@ static void fuse_readahead(struct readahead_control *rac) static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); +/* + * Bound on re-requesting a revoked DLM grant before a cached read is + * served unlocked; see fuse_cache_read_iter(). + */ +#define FUSE_DLM_READ_RETRIES 3 + static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; @@ -1337,6 +1343,7 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) struct fuse_inode *fi = get_fuse_inode(inode); struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; ssize_t res; + int lock_err = 0; /* * In auto invalidate mode, always update attributes on read. @@ -1354,8 +1361,9 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) /* if we have dlm support acquire a read lock for the area * we are reading from. */ if (fc->writeback_cache && fc->dlm) - fuse_get_dlm_lock(file, iocb->ki_pos, - iov_iter_count(to), FUSE_PAGE_LOCK_READ); + lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, + iov_iter_count(to), + FUSE_PAGE_LOCK_READ); /* * Fence the cache-serving read against a NOTIFY invalidate so we never @@ -1367,11 +1375,44 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) * wb_sem is NULL on non-writeback+dlm mounts (gate inactive). */ if (wb_sem) { + int tries = FUSE_DLM_READ_RETRIES; + +retry: percpu_down_read(wb_sem); if (fuse_inode_force_dio(inode)) { percpu_up_read(wb_sem); return fuse_direct_read_iter(iocb, to); } + /* + * The DLM lock was requested before entering the gate, and + * the NOTIFY invalidate we may just have waited on revokes + * locks under the gate write side. Re-check the grant here + * and re-request with the gate dropped, so a + * FUSE_DLM_WB_LOCK round trip never parks a pending + * invalidate behind our own gate hold. Once the check + * passes the lock cannot go away for the rest of the gate + * hold. A failed or unrecorded request falls through + * unlocked, as before: the retry is taken even then (the + * latch must be re-checked under the re-entered gate), so + * lock_err has to stay sticky across it -- seeded by the + * pre-gate request above -- or a grant that failed would + * be re-requested forever. The retry is also bounded: a + * remote writer can revoke each successful grant before + * the gate is re-entered, and a reader-only inode has no + * force-DIO latch to end such a storm, so after + * FUSE_DLM_READ_RETRIES re-requests the read is served + * unlocked rather than looping without bound. + */ + if (!lock_err && fc->dlm && tries-- > 0 && + !fuse_dlm_lock_is_held(fi, iocb->ki_pos, + iov_iter_count(to), + FUSE_PAGE_LOCK_READ)) { + percpu_up_read(wb_sem); + lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, + iov_iter_count(to), + FUSE_PAGE_LOCK_READ); + goto retry; + } } res = generic_file_read_iter(iocb, to); @@ -1862,6 +1903,26 @@ static void fuse_cache_wr_unlock(struct inode *inode, bool exclusive) inode_unlock_shared(inode); } +/* + * Request the DLM write lock covering a cached write. -ENOSYS cleared + * fc->dlm: the server has no DLM, proceed as a plain cached write. Any + * other failure means the cache would be dirtied without DLM coverage - + * the caller must fail the write instead. A granted-but-unrecorded + * lock (positive return) is covered cluster-wide; proceed, but flag it + * so the in-gate re-validation skips a check an invisible grant could + * never pass. + */ +static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len, + bool *unrecorded) +{ + int err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); + + if (err < 0 && err != -ENOSYS) + return err; + *unrecorded = err > 0; + return 0; +} + static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; @@ -1876,14 +1937,10 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) bool writeback = false; bool wb_guard = false; bool exclusive = true; + bool dlm_unrecorded = false; + loff_t dlm_pos = 0; + size_t dlm_len = 0; - /* - * The inode may have been latched into forced direct IO -- by a - * NOTIFY_INVAL_INODE arriving while this inode is open for writing here - * -- after this write was routed to the cached path but before it took - * any lock. Re-route to the direct path (before taking a DLM lock) so - * we do not repopulate the page cache the latch just dropped. - */ if (fuse_inode_force_dio(inode)) return fuse_direct_write_iter(iocb, from); @@ -1898,61 +1955,78 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) return err; if (!fc->handle_killpriv_v2 || - !setattr_should_drop_suidgid(idmap, file_inode(file))) { + !setattr_should_drop_suidgid(idmap, file_inode(file))) writeback = true; - - /* - * If we have dlm support acquire the lock for the area - * we are writing into. - * dlm lock is only needed as the write is cached and the - * fuse server is not notified otherwise - */ - if (fc->dlm) { - /* - * Note that a file opened with O_APPEND will have - * relative values in ki_pos. This code is here for - * convenience and for libfuse overlay test. - * Filesystems should handle O_APPEND with 'direct io' - * to additionally get the performance benefits of - * 'parallel direct writes'. - */ - loff_t pos = file->f_flags & O_APPEND ? - i_size_read(inode) + iocb->ki_pos : - iocb->ki_pos; - size_t length = iov_iter_count(from); - - fuse_get_dlm_lock(file, pos, length, - FUSE_PAGE_LOCK_WRITE); - } - } } exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); + + /* + * Request the DLM write lock before taking i_rwsem: the request is + * an unbounded cluster round trip, and holding the writer-priority + * rwsem across it would park a truncate -- and behind it every + * later writer -- for the duration. The grant-to-use window this + * leaves open is closed by the in-gate re-validation below. Only + * the append case must wait for the lock: its range depends on + * i_size, which is stable only under the exclusive inode lock. + */ + if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { + dlm_pos = iocb->ki_pos; + dlm_len = iov_iter_count(from); + + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, + &dlm_unrecorded); + if (err) + return err; + } + if (exclusive) inode_lock(inode); else inode_lock_shared(inode); - /* - * The forced-direct-IO latch feature is active under writeback+dlm; - * hold the coherency gate (wb_inval_rwsem) for read across the - * page-cache dirtying so a concurrent NOTIFY_INVAL_INODE -- which takes - * the write side (blocking, with priority) around its invalidate + latch - * set -- cannot strand the folios we are about to write. Re-check the - * latch under it (it may have been set while we blocked on the inode - * lock) and re-route to the direct path if set. Taken before - * task_io_account_write() so a re-route is not double-counted; the DLM - * write lock taken above is harmless as the direct path does its own - * server coordination. - */ + /* note that this small code dup will save us a lot of headache later + * when appends are done concurrently without using parallel direct writes */ + if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { + /* + * An append write lands at the current EOF no matter what + * ki_pos holds: generic_write_checks() rewrites ki_pos to + * i_size for IOCB_APPEND, and i_size is stable here because + * append writes hold the inode lock exclusive. Lock where + * the data will land. + */ + dlm_pos = i_size_read(inode); + dlm_len = iov_iter_count(from); + + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, + &dlm_unrecorded); + if (err) + goto out; + } + wb_guard = !!wb_sem; if (wb_guard) { +retry: percpu_down_read(wb_sem); if (fuse_inode_force_dio(inode)) { percpu_up_read(wb_sem); fuse_cache_wr_unlock(inode, exclusive); return fuse_direct_write_iter(iocb, from); } + if (writeback && fc->dlm && !dlm_unrecorded && + !fuse_dlm_lock_is_held(fi, dlm_pos, dlm_len, + FUSE_PAGE_LOCK_WRITE)) { + percpu_up_read(wb_sem); + err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, + &dlm_unrecorded); + if (err) { + /* The gate is already dropped; funnel the + * failure through the one audited exit. */ + wb_guard = false; + goto out; + } + goto retry; + } } err = count = generic_write_checks(iocb, from); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 2ec072b86312f4..30d371bcf6e293 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -510,45 +510,83 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, return true; } +/** + * fuse_dlm_lock_is_held - check that a byte range is covered by a granted lock + * @fi: the fuse inode + * @offset: byte offset into the file (need not be page-aligned) + * @length: length of the region in bytes (need not be page-aligned) + * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE + * + * Re-validation helper for fuse_get_dlm_lock() callers: checks the same + * page-aligned range a fuse_get_dlm_lock() call with these arguments + * requests, against the live lock tree. + */ +bool fuse_dlm_lock_is_held(struct fuse_inode *fi, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) +{ + uint64_t end = (offset + length - 1) | (PAGE_SIZE - 1); + + /* + * An empty range needs no coverage. Reporting it held keeps the + * re-validating IO paths from re-requesting a lock the tree can + * never show (the page-aligned end would invert below). + */ + if (!length) + return true; + + return fuse_dlm_range_is_locked(fi, offset & PAGE_MASK, end, mode); +} + /** * fuse_get_dlm_lock - request a dlm lock from the fuse server * @file: the file being accessed * @offset: byte offset into the file (need not be page-aligned) * @length: length of the region in bytes (need not be page-aligned) * @mode: FUSE_PAGE_LOCK_READ or FUSE_PAGE_LOCK_WRITE + * + * Return: 0 when the range is covered by a recorded grant on return, + * FUSE_DLM_GRANT_UNRECORDED when the server granted the lock but + * recording it failed (covered cluster-wide, invisible to + * fuse_dlm_lock_is_held()), a negative error code otherwise. Callers + * re-validating the grant must not re-request on a nonzero return or + * they would spin. */ -void fuse_get_dlm_lock(struct file *file, loff_t offset, - size_t length, enum fuse_page_lock_mode mode) +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) { struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_mount *fm = ff->fm; - uint64_t end = (offset + length - 1) | (PAGE_SIZE - 1); - - /* note that the offset and length don't have to be page aligned here - * but since we only get here on writeback caching we will send out - * page aligned requests */ - offset &= PAGE_MASK; FUSE_ARGS(args); struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; int err; + /* An empty range needs no lock. */ + if (!length) + return 0; + /* note that this can be run from different processes * at the same time. It is intentionally not protected * since a DLM implementation in the FUSE server should take care - * of any races in lock requests */ - if (fuse_dlm_range_is_locked(fi, offset, end, mode)) - return; /* we already have this area locked */ + * of any races in lock requests. + * The early exit uses the same helper the callers re-validate + * with, so this check and a later fuse_dlm_lock_is_held() can + * never disagree about what counts as covered. */ + if (fuse_dlm_lock_is_held(fi, offset, length, mode)) + return 0; /* we already have this area locked */ memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; - inarg.start = offset; - inarg.end = end; + /* note that the offset and length don't have to be page aligned + * here but since we only get here on writeback caching we will + * send out page aligned requests */ + inarg.start = offset & PAGE_MASK; + inarg.end = (offset + length - 1) | (PAGE_SIZE - 1); inarg.type = (mode == FUSE_PAGE_LOCK_WRITE) ? FUSE_DLM_LOCK_WRITE : FUSE_DLM_LOCK_READ; @@ -564,21 +602,31 @@ void fuse_get_dlm_lock(struct file *file, loff_t offset, if (err == -ENOSYS) { /* fuse server does not support dlm, save the info */ fc->dlm = 0; - return; + return err; } if (err) - return; - else - if (inarg.start < outarg.start || - inarg.end > outarg.end) { - /* fuse server is seriously broken */ - pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", - inarg.start, inarg.end, outarg.start, outarg.end); - fuse_abort_conn(fc); - return; - } else { - /* ignore any errors here, there is no way we can react appropriately */ - fuse_dlm_lock_range(fi, outarg.start, outarg.end, mode); - } + return err; + + if (inarg.start < outarg.start || inarg.end > outarg.end) { + /* fuse server is seriously broken */ + pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", + inarg.start, inarg.end, outarg.start, outarg.end); + fuse_abort_conn(fc); + return -EIO; + } + + /* + * The server granted the lock; record it so + * fuse_dlm_lock_is_held() sees it. A failure to record + * (small-allocation -ENOMEM) does not undo the grant: coverage + * exists cluster-wide, only the local bookkeeping is missing. + * Report that as FUSE_DLM_GRANT_UNRECORDED so callers neither + * fail an IO that is actually covered nor keep re-requesting a + * grant that will not become visible. + */ + if (fuse_dlm_lock_range(fi, outarg.start, outarg.end, mode)) + return FUSE_DLM_GRANT_UNRECORDED; + + return 0; } diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 5c3deaa3536866..b0b16c56e3b0b0 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -17,6 +17,15 @@ struct fuse_inode; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; +/* + * fuse_get_dlm_lock() result: the server granted the lock but recording + * it locally failed, leaving the grant invisible to + * fuse_dlm_lock_is_held(). The IO is covered cluster-wide; the caller + * must proceed without re-validating (a re-request would spin) instead + * of failing the IO. + */ +#define FUSE_DLM_GRANT_UNRECORDED 1 + /* Page cache lock manager */ struct fuse_dlm_cache { /* Lock protecting the tree */ @@ -43,8 +52,12 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, uint64_t end, enum fuse_page_lock_mode mode); +/* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ +bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, + size_t length, enum fuse_page_lock_mode mode); + /* This is the interface to the filesystem */ -void fuse_get_dlm_lock(struct file *file, loff_t offset, - size_t length, enum fuse_page_lock_mode mode); +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode); #endif /* _FS_FUSE_DLM_CACHE_H */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 9c737654d35359..a63c5dfe7fc765 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -706,16 +706,6 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, else pg_end = (offset + len - 1) >> PAGE_SHIFT; - if (fc->dlm && fc->writeback_cache) - /* Invalidate the range exactly as the fuse server requested - * except for the case where it sends -1. - * Note that this can lead to some inconsistencies if - * the fuse server sends unaligned data */ - fuse_dlm_unlock_range(fi, - offset, - pg_end == -1 ? 0 : - (offset + len - 1)); - /* * A data invalidation means another (remote) entity is modifying * the file. Two things happen here: @@ -771,6 +761,23 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, */ percpu_down_write(wb_sem); + /* + * Revoke the DLM lock range under the gate write + * side, atomically with the page drop: gate readers + * re-validate their grant right after entering, and + * a grant that passed that check must stay visible + * for their whole gate hold. + * The range is exactly what the fuse server + * requested except for the case where it sends -1. + * Note that this can lead to some inconsistencies + * if the fuse server sends unaligned data. + */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_unlock_range(fi, + offset, + pg_end == -1 ? 0 : + (offset + len - 1)); + if (hot && has_writer && !fuse_inode_force_dio(inode)) { spin_lock(&fi->lock); @@ -799,6 +806,14 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", nodeid); } else { + /* No gate on this inode (mmapped, DAX, backing or + * non-regular): drop the lock range unserialized, + * as before. */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_unlock_range(fi, + offset, + pg_end == -1 ? 0 : + (offset + len - 1)); invalidate_inode_pages2_range(inode->i_mapping, pg_start, pg_end); } From 83905a1391cd57c4d83f049e72f6dab27fa79b85 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 28 Jul 2026 09:42:53 +0200 Subject: [PATCH 66/77] fuse: fix the DLM revoke range of an inode invalidate The NOTIFY_INVAL_INODE revoke computed fuse_dlm_unlock_range(fi, offset, pg_end == -1 ? 0 : offset + len - 1) which is wrong at both degenerate ends: a to-EOF invalidate (len <= 0, e.g. a remote truncate) with offset > 0 becomes the inverted range [offset, 0] and removes nothing, so the revoked grant stays visible to the re-validating IO paths forever -- cached writes with no server-side lock, zero-filled RMW reads; and an invalidate of byte 0 (offset 0, len 1) becomes [0, 0], the "destroy everything" sentinel, wiping every grant of the inode. Map the range in one helper shared by the gated and the ungated branch: to-EOF revokes through U64_MAX, and the bounds widen to page boundaries to match how grants are recorded -- revoking too much only costs a re-request, too little leaves a stale grant. Drop the in-band (0, 0) sentinel: whole-file invalidates walk the normal removal path, release-all is fuse_dlm_cache_release_locks(), and an inverted range is rejected with -EINVAL instead of silently ignored. Signed-off-by: Horst Birthelmer (cherry picked from commit eb3464fec9d322a3330e4a4952545da644570a54) --- fs/fuse/fuse_dlm_cache.c | 15 +++++++-------- fs/fuse/inode.c | 33 +++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 30d371bcf6e293..4714d48e6bc9b1 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -369,8 +369,12 @@ static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, * @start: Start page offset * @end: End page offset * - * Release locks on the specified range of pages. - * Note that if start and end are set to zero the cache is destroyed. + * Release locks on the specified range of pages. An inverted range is + * rejected rather than silently removing nothing: the callers revoke + * coverage, and a revoke that quietly keeps the grant alive would let + * the re-validating IO paths trust a lock the server has taken away. + * To drop every grant use fuse_dlm_cache_release_locks() (there is no + * in-band sentinel range for it). * * Return: 0 on success, negative error code on failure */ @@ -381,14 +385,9 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, struct fuse_dlm_range *range, *next; int ret = 0; - if (!cache) + if (!cache || start > end) return -EINVAL; - if (start == 0 && end == 0) { - fuse_dlm_cache_release_locks(inode); - return 0; - } - down_write(&cache->lock); /* Find all ranges that overlap with [start, end] */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index a63c5dfe7fc765..df72cf1bb3df84 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -666,6 +666,25 @@ static bool fuse_notify_inval_hot(struct fuse_inode *fi) return avg < FUSE_NOTIFY_DIO_INTERVAL; } +/* + * Revoke the DLM grants backing an invalidated byte range. Grants are + * recorded page-aligned, so widen the revoke to page boundaries: dropping + * more than the server invalidated only costs a re-request, dropping less + * would leave a stale grant that fuse_dlm_lock_is_held() keeps trusting. + * len <= 0 means "invalidate to EOF" (see fuse_notify_inval_inode()) and + * revokes through U64_MAX -- it must not become an inverted range, which + * fuse_dlm_unlock_range() rejects without removing anything. + */ +static void fuse_dlm_revoke_inval_range(struct fuse_inode *fi, loff_t offset, + loff_t len) +{ + uint64_t start = (uint64_t)offset & PAGE_MASK; + uint64_t end = len <= 0 ? U64_MAX : + (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); + + fuse_dlm_unlock_range(fi, start, end); +} + int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { @@ -767,16 +786,9 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * re-validate their grant right after entering, and * a grant that passed that check must stay visible * for their whole gate hold. - * The range is exactly what the fuse server - * requested except for the case where it sends -1. - * Note that this can lead to some inconsistencies - * if the fuse server sends unaligned data. */ if (fc->dlm && fc->writeback_cache) - fuse_dlm_unlock_range(fi, - offset, - pg_end == -1 ? 0 : - (offset + len - 1)); + fuse_dlm_revoke_inval_range(fi, offset, len); if (hot && has_writer && !fuse_inode_force_dio(inode)) { @@ -810,10 +822,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * non-regular): drop the lock range unserialized, * as before. */ if (fc->dlm && fc->writeback_cache) - fuse_dlm_unlock_range(fi, - offset, - pg_end == -1 ? 0 : - (offset + len - 1)); + fuse_dlm_revoke_inval_range(fi, offset, len); invalidate_inode_pages2_range(inode->i_mapping, pg_start, pg_end); } From 0ce39f3df54d9c821c241005d41efb0bf8778032 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 28 Jul 2026 09:45:16 +0200 Subject: [PATCH 67/77] fuse: fence mmapped invalidates and local truncates with the coherency gate Two revocation paths bypassed the revoke-under-gate invariant the IO paths re-validate against: - fuse_reverse_inval_inode() skipped the gate once mapping_mapped() turned true, revoking concurrently with gate holders -- but fuse_cache_read_iter()/fuse_cache_write_iter() enter the gate unconditionally, so a single mmap() reopened the race. Keep the gate for mmapped inodes; only the force-DIO latch stays disabled for them (a mapping needs the page cache, and fuse_file_mmap() reverts any latch it races with). - The local truncates in fuse_do_setattr() -- the atomic-O_TRUNC open shortcut and the after-setattr trim -- revoked and dropped the cache with no gate at all, so an already re-validated reader could repopulate the truncated range. Take the gate write side around revoke + drop. This cannot deadlock: both run under exclusive i_rwsem, which no gate holder waits on (the write path takes i_rwsem before the gate, the read path never takes it). Signed-off-by: Horst Birthelmer (cherry picked from commit 3523bf681f460e9b6ce197009a04198e5f73c29e) Signed-off-by: Allison Henderson --- fs/fuse/dir.c | 29 +++++++++++++++++++++++++++++ fs/fuse/inode.c | 19 ++++++++++++------- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index badb9e59887c5f..ca50d1399c11ad 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2170,11 +2170,26 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, WARN_ON(!(attr->ia_valid & ATTR_SIZE)); WARN_ON(attr->ia_size != 0); if (fc->atomic_o_trunc) { + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + /* * No need to send request to userspace, since actual * truncation has already been done by OPEN. But still * need to truncate page cache. + * + * Revoke and drop under the coherency gate write side, + * like the NOTIFY invalidate path: a gate reader that + * already re-validated its grant must not have the + * lock tree and the cache yanked mid-hold, or it + * would repopulate the truncated range trusting a + * grant that no longer exists. Waiting for gate + * readers here is safe: we hold i_rwsem exclusive, so + * no gate holder can be waiting on it (the write path + * takes i_rwsem before the gate, the read path never + * takes it). */ + if (wb_sem) + percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_cache_release_locks(fi); spin_lock(&fi->lock); @@ -2182,6 +2197,8 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, i_size_write(inode, 0); spin_unlock(&fi->lock); truncate_pagecache(inode, 0); + if (wb_sem) + percpu_up_write(wb_sem); goto out; } file = NULL; @@ -2292,11 +2309,23 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if ((is_truncate || !is_wb) && S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + + /* + * Revoke and drop under the coherency gate write side; see + * the atomic-O_TRUNC branch above. i_rwsem is held + * exclusive here as well (setattr), so waiting out gate + * readers cannot deadlock. + */ + if (wb_sem) + percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1); truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); + if (wb_sem) + percpu_up_write(wb_sem); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index df72cf1bb3df84..7a00175fdd91aa 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -755,13 +755,17 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * (stale read / lost write). * * The gate (and the average) exist only for writeback+dlm regular - * files, and not while mmapped; elsewhere wb_sem is NULL and the - * invalidate runs unserialized (best-effort), as before. + * files; elsewhere wb_sem is NULL and the invalidate runs + * unserialized (best-effort), as before. An mmapped inode + * keeps the gate -- fuse_cache_read_iter() and + * fuse_cache_write_iter() enter it unconditionally and rely + * on the revoke staying fenced -- but is never latched: + * a mapping needs the page cache, and fuse_file_mmap() + * reverts any latch it races with. */ if (S_ISREG(inode->i_mode) && fc->writeback_cache && fc->dlm && !FUSE_IS_DAX(inode) && - !fuse_inode_backing(fi) && - !mapping_mapped(inode->i_mapping)) + !fuse_inode_backing(fi)) wb_sem = fi->wb_inval_rwsem; if (wb_sem) { @@ -791,6 +795,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, fuse_dlm_revoke_inval_range(fi, offset, len); if (hot && has_writer && + !mapping_mapped(inode->i_mapping) && !fuse_inode_force_dio(inode)) { spin_lock(&fi->lock); if (!list_empty(&fi->write_files)) { @@ -818,9 +823,9 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", nodeid); } else { - /* No gate on this inode (mmapped, DAX, backing or - * non-regular): drop the lock range unserialized, - * as before. */ + /* No gate on this inode (DAX, backing, non-regular, + * or the gate allocation failed): drop the lock + * range unserialized (best-effort), as before. */ if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); invalidate_inode_pages2_range(inode->i_mapping, From 0c6f60660f4bdbbd73c68fe8933e069964be20a8 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 28 Jul 2026 09:47:47 +0200 Subject: [PATCH 68/77] fuse: order grant recording against concurrent revokes A FUSE_DLM_WB_LOCK reply and a NOTIFY invalidate are serviced on different threads, so a revoke aimed at the grant a reply carries can be processed before fuse_get_dlm_lock() records it: the revoke finds nothing to remove, and the requester then records an already-dead grant that no later NOTIFY will target -- a permanent false positive for the re-validating IO paths. Add a revocation generation to the lock cache, bumped under the cache lock by every revoke path -- unconditionally, because the racing revoke sees an empty overlap precisely when the grant is in flight. fuse_get_dlm_lock() samples it before sending and records through fuse_dlm_lock_range_gen(), which refuses with -EAGAIN once the generation has moved; the grant is then re-requested instead of recorded, bounded so a revoke storm cannot pin the IO here (past the bound the failure reports like any request failure). Signed-off-by: Horst Birthelmer (cherry picked from commit 5c94fbbeb3e2d00e0c9c4d75044016b15de6c088) Signed-off-by: Allison Henderson --- fs/fuse/fuse_dlm_cache.c | 110 +++++++++++++++++++++++++++++++++++---- fs/fuse/fuse_dlm_cache.h | 15 ++++++ 2 files changed, 115 insertions(+), 10 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 4714d48e6bc9b1..960d51e7836a3c 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -31,6 +31,12 @@ struct fuse_dlm_range { #define FUSE_PCACHE_LK_READ 1 /* Shared read lock */ #define FUSE_PCACHE_LK_WRITE 2 /* Exclusive write lock */ +/* + * Bound on re-requesting a grant whose recording lost against a + * concurrent revoke; see fuse_get_dlm_lock(). + */ +#define FUSE_DLM_RECORD_TRIES 3 + /* Interval tree definitions for page ranges */ static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) { @@ -63,6 +69,7 @@ int fuse_dlm_cache_init(struct fuse_inode *inode) init_rwsem(&cache->lock); cache->ranges = RB_ROOT_CACHED; + cache->revoke_gen = 0; return 0; } @@ -84,6 +91,7 @@ void fuse_dlm_cache_release_locks(struct fuse_inode *inode) /* Release all locks */ down_write(&cache->lock); + WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); while ((node = rb_first_cached(&cache->ranges)) != NULL) { range = rb_entry(node, struct fuse_dlm_range, rb); fuse_page_it_remove(range, &cache->ranges); @@ -166,11 +174,13 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, } /** - * fuse_dlm_lock_range - Lock a range of pages + * __fuse_dlm_lock_range - Lock a range of pages * @cache: The page cache * @start: Start page offset * @end: End page offset * @mode: Lock mode (read or write) + * @genp: If non-NULL, the revocation generation sampled before the grant + * was requested; recording fails with -EAGAIN if it has moved * * Add a locked range on the specified range of pages. * If parts of the range are already locked, only add the remaining parts. @@ -181,8 +191,9 @@ static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, * * Return: 0 on success, negative error code on failure */ -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode, + const uint64_t *genp) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; struct fuse_dlm_range *range, *new_range, *next; @@ -202,6 +213,17 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, down_write(&cache->lock); + /* + * A revoke was processed after @genp was sampled; the grant this + * record carries may be the very one it targeted (a revoke of a + * not-yet-recorded grant removes nothing and would never be + * retried). Refuse, the caller re-requests. + */ + if (genp && cache->revoke_gen != *genp) { + up_write(&cache->lock); + return -EAGAIN; + } + /* Find all ranges that overlap with [start, end] */ range = fuse_page_it_iter_first(&cache->ranges, start, end); while (range) { @@ -297,6 +319,35 @@ int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, return ret; } +int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) +{ + return __fuse_dlm_lock_range(inode, start, end, mode, NULL); +} + +int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode, + uint64_t gen) +{ + return __fuse_dlm_lock_range(inode, start, end, mode, &gen); +} + +/** + * fuse_dlm_revoke_gen - sample the revocation generation + * @inode: the fuse inode + * + * Sampled before a FUSE_DLM_WB_LOCK request leaves the client. The + * reply and a NOTIFY revoke can be serviced on different threads, so a + * revoke may be processed between the reply arriving and its grant + * being recorded. fuse_dlm_lock_range_gen() re-checks the generation + * under the cache lock and refuses to record a grant such a revoke may + * have already killed. + */ +uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode) +{ + return READ_ONCE(inode->dlm_locked_areas.revoke_gen); +} + /** * fuse_dlm_punch_hole - Punch a hole in a locked range * @cache: The page cache @@ -390,6 +441,14 @@ int fuse_dlm_unlock_range(struct fuse_inode *inode, down_write(&cache->lock); + /* + * Unconditional, even when nothing overlaps: the revoke racing + * with an in-flight grant finds an empty tree precisely because + * the grant is not recorded yet, and the bump is what makes the + * recording side notice (see fuse_dlm_lock_range_gen()). + */ + WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); + /* Find all ranges that overlap with [start, end] */ range = fuse_page_it_iter_first(&cache->ranges, start, end); while (range) { @@ -562,12 +621,15 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, FUSE_ARGS(args); struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; + uint64_t gen; + int tries = FUSE_DLM_RECORD_TRIES; int err; /* An empty range needs no lock. */ if (!length) return 0; +restart: /* note that this can be run from different processes * at the same time. It is intentionally not protected * since a DLM implementation in the FUSE server should take care @@ -578,6 +640,16 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, if (fuse_dlm_lock_is_held(fi, offset, length, mode)) return 0; /* we already have this area locked */ + /* + * Sample the revocation generation before the request leaves. + * The reply and a NOTIFY revoke are serviced on different + * threads, so a revoke aimed at the grant this request returns + * can be processed before the grant is recorded below -- + * recording it anyway would resurrect a dead grant that no later + * NOTIFY will ever remove. + */ + gen = fuse_dlm_revoke_gen(fi); + memset(&inarg, 0, sizeof(inarg)); inarg.fh = ff->fh; @@ -617,14 +689,32 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, /* * The server granted the lock; record it so - * fuse_dlm_lock_is_held() sees it. A failure to record - * (small-allocation -ENOMEM) does not undo the grant: coverage - * exists cluster-wide, only the local bookkeeping is missing. - * Report that as FUSE_DLM_GRANT_UNRECORDED so callers neither - * fail an IO that is actually covered nor keep re-requesting a - * grant that will not become visible. + * fuse_dlm_lock_is_held() sees it. */ - if (fuse_dlm_lock_range(fi, outarg.start, outarg.end, mode)) + err = fuse_dlm_lock_range_gen(fi, outarg.start, outarg.end, mode, gen); + if (err == -EAGAIN) { + /* + * A revoke was processed while the request was in flight; + * the grant may already be dead, so re-request instead of + * recording it. Bounded: a revoke storm must not pin the + * IO here -- past the bound the failure is reported like + * any other request failure (the write path fails the + * write, the read path serves unlocked). + */ + if (--tries) + goto restart; + return -EAGAIN; + } + + /* + * A failure to record (small-allocation -ENOMEM) does not undo + * the grant: coverage exists cluster-wide, only the local + * bookkeeping is missing. Report that as + * FUSE_DLM_GRANT_UNRECORDED so callers neither fail an IO that + * is actually covered nor keep re-requesting a grant that will + * not become visible. + */ + if (err) return FUSE_DLM_GRANT_UNRECORDED; return 0; diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index b0b16c56e3b0b0..647a8c37c36095 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -32,6 +32,13 @@ struct fuse_dlm_cache { struct rw_semaphore lock; /* Interval tree of locked ranges */ struct rb_root_cached ranges; + /* + * Bumped under @lock by every revocation + * (fuse_dlm_unlock_range(), fuse_dlm_cache_release_locks()); + * lets fuse_get_dlm_lock() order recording a reply's grant + * against revokes processed while the reply was in flight. + */ + uint64_t revoke_gen; }; /* Initialize a page cache lock manager */ @@ -44,6 +51,14 @@ void fuse_dlm_cache_release_locks(struct fuse_inode *inode); int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, uint64_t end, enum fuse_page_lock_mode mode); +/* As above, but refuse (-EAGAIN) if a revoke ran since @gen was sampled */ +int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode, + uint64_t gen); + +/* Sample the revocation generation (see fuse_dlm_lock_range_gen()) */ +uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode); + /* Unlock a range of pages */ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, uint64_t end); From 71beba57ad1b09fca36321326876fd6ebd5389db Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Wed, 12 Aug 2026 12:08:30 +0200 Subject: [PATCH 69/77] fuse: retry grant recording until it wins against revokes fuse_get_dlm_lock() re-requests the DLM lock when fuse_dlm_lock_range_gen() returns -EAGAIN, i.e. a revoke was processed while the grant request was in flight and the grant it returned may already be dead. That restart was bounded by FUSE_DLM_RECORD_TRIES, and past the bound the function returned -EAGAIN. Reporting that as a request failure is wrong: no one else holds the range at that point, the caller simply lost a race with a revoke, and the write path turns the error into a failed write. Retry unconditionally instead. Every pass issues a fresh FUSE_DLM_WB_LOCK round trip to the server, so a revoke storm throttles the loop rather than spinning it, and the loop ends as soon as one grant survives long enough to be recorded. Drop the now-unused bound and its counter. Signed-off-by: Horst Birthelmer (cherry picked from commit 2b6ed886e6488cbd29ef4e284c30b5b521b7c567) Signed-off-by: Allison Henderson --- fs/fuse/fuse_dlm_cache.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 960d51e7836a3c..b0b17cbd3c3f7a 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -31,12 +31,6 @@ struct fuse_dlm_range { #define FUSE_PCACHE_LK_READ 1 /* Shared read lock */ #define FUSE_PCACHE_LK_WRITE 2 /* Exclusive write lock */ -/* - * Bound on re-requesting a grant whose recording lost against a - * concurrent revoke; see fuse_get_dlm_lock(). - */ -#define FUSE_DLM_RECORD_TRIES 3 - /* Interval tree definitions for page ranges */ static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) { @@ -622,7 +616,6 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; uint64_t gen; - int tries = FUSE_DLM_RECORD_TRIES; int err; /* An empty range needs no lock. */ @@ -696,14 +689,14 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, /* * A revoke was processed while the request was in flight; * the grant may already be dead, so re-request instead of - * recording it. Bounded: a revoke storm must not pin the - * IO here -- past the bound the failure is reported like - * any other request failure (the write path fails the - * write, the read path serves unlocked). + * recording it. Retry until a grant survives long enough to + * be recorded: giving up here would hand the caller an error + * for a range no one else holds, and the write path turns + * that into a failed write. Each pass makes a fresh server + * round trip, so a revoke storm throttles this loop rather + * than spinning it. */ - if (--tries) - goto restart; - return -EAGAIN; + goto restart; } /* From 5478785849e8280eb0a2add130369cb1e4baee99 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Wed, 12 Aug 2026 16:59:59 +0200 Subject: [PATCH 70/77] fuse: gate the notify-driven direct-IO latch behind a module parameter fuse_reverse_inval_inode() latches an inode into direct IO when a remote writer keeps invalidating a file that is also open for writing here. That trades the writeback cache away for as long as the latch holds, which only pays off on workloads that actually see such invalidation storms. Make it opt-in through a new 'enable_notify_dio' module parameter, default off. FUSE_I_FORCE_DIO is set in exactly one place, so gating that single site is enough: every other reference only tests or clears the bit, and with the bit never set those paths behave as they did before the latch existed. The moving average is still folded on every invalidation while the parameter is off, so enabling it at runtime takes effect on the next storm instead of after a warm-up. Clearing it stops new latches but leaves already-latched inodes to run out on the usual exits (last writer closes, or mmap). Signed-off-by: Horst Birthelmer (cherry picked from commit ac617b0b037f411433322f570ff3709cab057dbe) Signed-off-by: Allison Henderson --- fs/fuse/inode.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 7a00175fdd91aa..5c5bb7960634b2 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -41,6 +41,19 @@ bool __read_mostly enable_large_folios = true; module_param(enable_large_folios, bool, 0644); MODULE_PARM_DESC(enable_large_folios, "Enable large folios support"); +/* + * Gate for the notify-driven direct-IO latch (see + * fuse_reverse_inval_inode()): when a remote writer keeps invalidating a + * file that is also open for writing here, the inode is switched to + * direct IO until its last writer closes. Off by default -- it trades + * the writeback cache away for the duration, which only pays off on + * workloads that actually see such storms. + */ +static bool __read_mostly enable_notify_dio; +module_param(enable_notify_dio, bool, 0644); +MODULE_PARM_DESC(enable_notify_dio, + "Latch a contended inode to direct IO on an invalidation notify storm"); + static struct kmem_cache *fuse_inode_cachep; struct list_head fuse_conn_list; DEFINE_MUTEX(fuse_mutex); @@ -752,7 +765,13 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * writer closes or it is mmapped. When latched, drop the whole * mapping rather than just the notified range, or dirty folios * outside it would be invisible to the forced direct reads - * (stale read / lost write). + * (stale read / lost write). Latching is opt-in via the + * enable_notify_dio module parameter and off by default; the + * average is kept up to date either way, so enabling it at + * runtime takes effect on the next storm rather than after a + * warm-up. Clearing it at runtime stops new latches but lets + * already-latched inodes run out on the usual exits (last + * writer closes, or mmap). * * The gate (and the average) exist only for writeback+dlm regular * files; elsewhere wb_sem is NULL and the invalidate runs @@ -794,7 +813,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); - if (hot && has_writer && + if (enable_notify_dio && hot && has_writer && !mapping_mapped(inode->i_mapping) && !fuse_inode_force_dio(inode)) { spin_lock(&fi->lock); From 4a9ddcf9495bc08241f2ad8c4e465bcf29ef60a7 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Tue, 11 Aug 2026 12:16:57 -0700 Subject: [PATCH 71/77] fuse: re-decide the write lock mode after the DLM probe fuse_cache_write_iter() picks between the exclusive and the relaxed shared inode lock with fuse_cache_wr_exclusive_lock(), which returns "shared" only when fc->dlm is set. Since "fuse: re-validate the DLM grant after waiting on the coherency gate" that decision is made before the DLM write lock is requested, and the request itself can clear fc->dlm: a server that does not implement FUSE_DLM_WB_LOCK answers -ENOSYS, which fuse_get_dlm_lock() handles by clearing fc->dlm and reporting success. The write then proceeds in a state that was unreachable before: the shared lock was chosen believing DLM was active, but DLM is now known to be absent. That combination is not benign. The shared path claims the i_size extension up front so iomap never updates i_size itself, which also stops iomap_block_needs_zeroing() from ever firing for the write's own range; the zero-fill that compensates for it in fuse_iomap_read_folio_range() is gated on fc->dlm and so no longer runs. An expanding write therefore falls through to fuse_do_readfolio() and sends a READ for a range past EOF that cannot hold data. Against a file the client opened write-only the server fails that read -- passthrough_hp returns EBADF -- and the write fails with it. Re-evaluate the lock mode after the request, while no lock is held yet, so a server without DLM support gets the exclusive path and iomap's own beyond-EOF zeroing back. This showed up as generic/105, 123, 215, 246, 378, 423, 519 and 597 all failing with EBADF on the first write to a newly created file, and bisected to the commit named above. Signed-off-by: Allison Henderson --- fs/fuse/file.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 7364955b38f88f..05aa5abb974bf8 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1978,6 +1978,21 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) &dlm_unrecorded); if (err) return err; + + /* + * The request above may have found that the server has no DLM + * at all, in which case it cleared fc->dlm. The relaxed shared + * lock was chosen just before, while fc->dlm still read 1, and + * it is only sound under DLM: the shared path claims the i_size + * extension up front, which stops iomap from zeroing beyond + * EOF, and the zero-fill that replaces it in + * fuse_iomap_read_folio_range() is itself gated on fc->dlm. + * Left as chosen, an expanding write would fall through to a + * READ of a range that cannot hold data -- which fails outright + * on a handle the client opened write-only. Re-decide now, + * while no lock is held yet. + */ + exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); } if (exclusive) From db5896f03d6d9761d728d953093f79af3dd5057b Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Fri, 14 Aug 2026 18:36:14 -0700 Subject: [PATCH 72/77] fuse: unlink the DLM range before shortening it when splitting fuse_dlm_punch_hole() splits a grant by shortening the original range and adding a second one for the tail. It assigns range->end before calling fuse_page_it_remove(). The ranges live in an rbtree of intervals. Each node caches a maximum range->end which describes the max of its whole subtree. This enables lookups to skip a subtree whose intervals all end before the query starts. Removing a node also causes the tree's intervals to be re-balanced, and the cached maxima of the subtree nodes must be updated accordingly. But editing range->end before removing a node means that rebalancing is not computed correctly. Since the sub-nodes were not indexed by this value, the cached maxima no longer describe the remaining nodes. So a later fuse_page_it_iter_first() then either prunes a subtree that does hold an overlap, or descends into one that does not. Similar bugs were fixed in five other sites in this file in commit "fuse: seed DLM grant merging with an interval-tree lookup". Unfixed, the incorrect maxima can cause an oops under fsstress. This path is harder to expose since it needs a hole that is strictly interior to the grant, and was found by review, but it can also cause the same oops if left uncorrected. Apply the same ordering here: unlink first, then edit, then relink. The new_range is copied before the edit, so it still carries the original end and only its start needs adjusting. Signed-off-by: Allison Henderson --- fs/fuse/fuse_dlm_cache.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index b0b17cbd3c3f7a..01778325e8b30a 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -391,16 +391,19 @@ static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, goto out; } - /* Copy properties from original range */ + /* Copy properties from original range, keeping the original end */ *new_range = *range; INIT_LIST_HEAD(&new_range->list); - - /* Adjust ranges */ new_range->start = end + 1; - range->end = start - 1; - /* Update interval tree */ + /* + * Shorten the original end only while it is unlinked. range->end is + * used in calulating the interval tree's __subtree_end, so changes + * made while the node is still in the tree leaves every ancestor + * stale. Update range->end after fuse_page_it_remove() + */ fuse_page_it_remove(range, &cache->ranges); + range->end = start - 1; fuse_page_it_insert(range, &cache->ranges); fuse_page_it_insert(new_range, &cache->ranges); From 5941ee414d312060195863e1dbe88d83d558ca1e Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 14 Aug 2026 16:02:51 +0200 Subject: [PATCH 73/77] fuse: mark writeback-initiated SETATTR with FATTR_WRITEBACK fuse_write_inode() -> fuse_flush_times() pushes out the mtime/ctime that the kernel owns locally while the writeback cache is on. On the wire that request is indistinguishable from a userspace "touch -m": both arrive as SETATTR with FATTR_MTIME | FATTR_CTIME | FATTR_FH, because trust_local_cmtime makes iattr_to_fattr() send CTIME whenever the writeback cache is enabled. A server that wants to handle a cache flush differently from an explicit attribute change - skipping a cluster-wide lock, merging rather than overwriting - has no way to tell them apart. Add FATTR_WRITEBACK, a control bit in fuse_setattr_in.valid alongside the existing non-attribute bits FATTR_FH, FATTR_LOCKOWNER and FATTR_KILL_SUIDGID. It selects no attribute, it only states that the request originates from writeback. Bit 30 is used rather than the next free one. libfuse mirrors the wire bits into its own FUSE_SET_ATTR_* namespace, where bits 12 to 17 are already taken by library-internal flags, and it masks incoming requests against that namespace; a bit picked from the low end would collide there and need translating on the way in. Bit 30 is free on both sides and clear of the sign bit of the int that the libfuse setattr operation takes, so one value works end to end. The bit is negotiated at INIT time with FUSE_SETATTR_WRITEBACK and is only set on a connection whose server asked for it, so servers that do not know the bit never receive it. Only ->write_inode() is marked. The other kernel-initiated SETATTR, fuse_do_truncate() rolling back a failed extending direct-IO write, is deliberately left unmarked: it is a size correction rather than an attribute writeback, and conflating the two would make the flag ambiguous. Signed-off-by: Horst Birthelmer (cherry picked from commit c5196db5b4fe704f218b6904d11093663655ead7) Signed-off-by: Allison Henderson --- fs/fuse/dir.c | 6 ++++++ fs/fuse/fuse_i.h | 3 +++ fs/fuse/inode.c | 4 +++- include/uapi/linux/fuse.h | 14 ++++++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index ca50d1399c11ad..05c081ca0ca59d 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -2110,6 +2110,12 @@ int fuse_flush_times(struct inode *inode, struct fuse_file *ff) inarg.valid |= FATTR_FH; inarg.fh = ff->fh; } + /* + * This is ->write_inode() flushing times the kernel owns locally, not + * a userspace utimes(); let the server tell the two apart. + */ + if (fm->fc->setattr_writeback) + inarg.valid |= FATTR_WRITEBACK; fuse_setattr_fill(fm->fc, &args, inode, &inarg, &outarg); return fuse_simple_request(fm, &args); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 603e57200b1914..c18ded6cd394a6 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -871,6 +871,9 @@ struct fuse_conn { /* expire inode entries when doing inode invalidation */ unsigned expire_inode_entries:1; + /* mark writeback-initiated SETATTR requests with FATTR_WRITEBACK */ + unsigned setattr_writeback:1; + /* * The following bitfields are only for optimization purposes * and hence races in setting them will not cause malfunction diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 5c5bb7960634b2..0cdedc20ef7f3c 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1738,6 +1738,8 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, fc->inval_inode_entries = 1; if (flags & FUSE_EXPIRE_INODE_ENTRY) fc->expire_inode_entries = 1; + if (flags & FUSE_SETATTR_WRITEBACK) + fc->setattr_writeback = 1; } else { ra_pages = fc->max_read / PAGE_SIZE; fc->no_lock = 1; @@ -1792,7 +1794,7 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) FUSE_NO_EXPORT_SUPPORT | FUSE_INVAL_INODE_ENTRY | FUSE_EXPIRE_INODE_ENTRY | FUSE_URING_REDUCED_Q | FUSE_EXPIRE_INODE_ENTRY | - FUSE_REQUEST_TIMEOUT; + FUSE_REQUEST_TIMEOUT | FUSE_SETATTR_WRITEBACK; #ifdef CONFIG_FUSE_DAX if (fm->fc->dax) flags |= FUSE_MAP_ALIGNMENT; diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index 30bb854fbc9408..ebe2735c186e08 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -371,6 +371,17 @@ struct fuse_file_lock { #define FATTR_LOCKOWNER (1 << 9) #define FATTR_CTIME (1 << 10) #define FATTR_KILL_SUIDGID (1 << 11) +/* + * Not an attribute selector: marks the request as a kernel-initiated + * writeback of locally owned attributes rather than a userspace-initiated + * change. Only sent if the server negotiated FUSE_SETATTR_WRITEBACK. + * + * The bit is deliberately far above the sequentially allocated FATTR_* + * range: libfuse mirrors these bits into its own FUSE_SET_ATTR_* space, + * which has its own allocations from bit 12 upwards, and only a value that + * is free on both sides can be passed through without translation. + */ +#define FATTR_WRITEBACK (1 << 30) /** * Flags returned by the OPEN request @@ -452,6 +463,8 @@ struct fuse_file_lock { * FUSE_EXPIRE_INODE_ENTRY: expire inode aliases when doing inode invalidation * FUSE_URING_REDUCED_Q: Client (kernel) supports less queues - Server is free * to register between 1 and nr-core io-uring queues + * FUSE_SETATTR_WRITEBACK: kernel marks writeback-initiated SETATTR requests + * with FATTR_WRITEBACK */ #define FUSE_ASYNC_READ (1 << 0) #define FUSE_POSIX_LOCKS (1 << 1) @@ -500,6 +513,7 @@ struct fuse_file_lock { #define FUSE_OVER_IO_URING (1ULL << 41) #define FUSE_REQUEST_TIMEOUT (1ULL << 42) #define FUSE_ALIGN_PG_ORDER (1ULL << 50) +#define FUSE_SETATTR_WRITEBACK (1ULL << 58) #define FUSE_URING_REDUCED_Q (1ULL << 59) #define FUSE_INVAL_INODE_ENTRY (1ULL << 60) #define FUSE_EXPIRE_INODE_ENTRY (1ULL << 61) From ec910d0ef2ca3073d42b13ec85ff2844ee88fd25 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 14 Aug 2026 16:03:15 +0200 Subject: [PATCH 74/77] fuse: don't launder from a NOTIFY invalidate while writepages are frozen fuse_reverse_inval_inode() invalidates with invalidate_inode_pages2_range(), which waits out folios under writeback and launders dirty ones. Both need a FUSE_WRITE reply, and while fi->writectr < 0 none can arrive: fuse_flush_writepages() parks the request on fi->queued_writes until fuse_release_nowrite(). A truncate holds that freeze across its whole SETATTR, and the server revokes the truncated range from inside the SETATTR handler, so the notify blocks the very thread that owes the reply lifting the freeze. generic/014 deadlocks within seconds, in folio_wait_writeback() under fuse_launder_folio() under fuse_reverse_inval_inode(). fuse_do_setattr() already states the rule ("Only call invalidate_inode_pages2() after removing FUSE_NOWRITE, otherwise fuse_launder_folio() would deadlock"). Give the notify path the same: while frozen, use invalidate_mapping_pages(), which skips dirty and under-writeback folios instead of waiting on them. The stale clean folios still go, the DLM grant is revoked either way, and the freezes that span a request drop the cache themselves when they finish. Signed-off-by: Horst Birthelmer (cherry picked from commit b77b85cc5caf013dbc31104406fd35c73fee1adc) Signed-off-by: Allison Henderson --- fs/fuse/inode.c | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 0cdedc20ef7f3c..366d9b703225e3 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -698,6 +698,40 @@ static void fuse_dlm_revoke_inval_range(struct fuse_inode *fi, loff_t offset, fuse_dlm_unlock_range(fi, start, end); } +/* + * Drop a page-cache range on behalf of a NOTIFY invalidate. + * + * invalidate_inode_pages2_range() waits out folios under writeback and + * launders dirty ones, both of which need a FUSE_WRITE reply. While + * writepages are frozen (fuse_set_nowrite(): truncate, O_TRUNC open, fsync, + * pre-SETATTR flush) no reply can arrive, because fuse_flush_writepages() + * parks the request on fi->queued_writes until fuse_release_nowrite(). A + * server that revokes from inside the handler it is revoking for then + * deadlocks against its own reply. fuse_do_setattr() states the same rule + * for its own invalidate. + * + * So while frozen use invalidate_mapping_pages(), which skips dirty and + * under-writeback folios and never blocks. The stale clean folios still + * go, and the freezes that span a request drop the cache themselves once + * they complete: fuse_do_setattr() invalidates the mapping after releasing + * the freeze, the O_TRUNC open path calls truncate_pagecache(). + */ +static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, + pgoff_t end) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + bool frozen; + + spin_lock(&fi->lock); + frozen = fi->writectr < 0; + spin_unlock(&fi->lock); + + if (frozen) + invalidate_mapping_pages(inode->i_mapping, start, end); + else + invalidate_inode_pages2_range(inode->i_mapping, start, end); +} + int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { @@ -831,10 +865,10 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * notified range. */ if (fuse_inode_force_dio(inode)) - invalidate_inode_pages2(inode->i_mapping); + fuse_notify_invalidate_range(inode, 0, -1); else - invalidate_inode_pages2_range(inode->i_mapping, - pg_start, pg_end); + fuse_notify_invalidate_range(inode, pg_start, + pg_end); percpu_up_write(wb_sem); @@ -847,8 +881,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * range unserialized (best-effort), as before. */ if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); - invalidate_inode_pages2_range(inode->i_mapping, - pg_start, pg_end); + fuse_notify_invalidate_range(inode, pg_start, pg_end); } } iput(inode); From 2c5bf2198c7e6f01aa62b7a90e047baad272437a Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 14 Aug 2026 16:05:17 +0200 Subject: [PATCH 75/77] fuse: do not kill suid from inside the coherency gate fuse_cache_write_iter() holds wb_inval_rwsem for read across the write, and the kiocb_modified() -> file_remove_privs() that precedes it runs under that same gate. Without handle_killpriv[_v2] the privilege kill asks the server (GETATTR, then SETATTR), and a server that invalidates the inode from inside such a handler blocks in percpu_down_write() draining the gate reader that is waiting for its reply. generic/193 hangs there. The gate only has to fence the page-cache dirtying, so run the write checks, the privilege kill and the timestamp update before entering it. task_io_account_write() stays behind the gate, so a write that the forced-DIO re-check reroutes is not counted twice. Signed-off-by: Horst Birthelmer (cherry picked from commit 736375f73aa4d0960e11e83565325d85a44353ff) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 05aa5abb974bf8..b43551fe12c6de 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2019,6 +2019,30 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) goto out; } + err = count = generic_write_checks(iocb, from); + if (err <= 0) + goto out; + + /* + * Kill suid/sgid and stamp the timestamps here, before the gate, + * instead of leaving them next to the write itself. kiocb_modified() + * -> file_remove_privs() is the one that reaches the server: without + * handle_killpriv[_v2] fuse_setattr() kills the bits by asking it (a + * FUSE_GETATTR to refresh the mode, then a FUSE_SETATTR, which for a + * writeback inode first flushes and freezes writepages), and + * security_inode_killpriv() can drop the capability xattr with another + * round trip. A server may have to invalidate this inode from inside + * such a handler; its NOTIFY_INVAL_INODE then blocks in + * percpu_down_write() draining a gate reader that is itself waiting for + * the reply. Nothing held under the gate may wait for the server. + * + * This also runs before the forced-DIO re-route below, so a re-routed + * write repeats it; there is nothing left to do the second time. + */ + err = kiocb_modified(iocb); + if (err) + goto out; + wb_guard = !!wb_sem; if (wb_guard) { retry: @@ -2044,16 +2068,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) } } - err = count = generic_write_checks(iocb, from); - if (err <= 0) - goto out; - task_io_account_write(count); - err = kiocb_modified(iocb); - if (err) - goto out; - if (iocb->ki_flags & IOCB_DIRECT) { written = generic_file_direct_write(iocb, from); if (written < 0 || !iov_iter_count(from)) From 306422be044c4f5231a17bf1150c5ad4d7500ee6 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 14 Aug 2026 16:05:44 +0200 Subject: [PATCH 76/77] fuse: keep cached size and times under a DLM write grant fuse_get_cache_mask() returns 0 once the connection has DLM, so every GETATTR reply overwrites i_size, mtime and ctime, and truncate_pagecache() then drops the tail the client still holds dirty. The only way a server can make that answer true is to revoke the client from inside the handler, which deadlocks against the coherency gate. A write grant already guarantees that no other node can touch the range, so keep the cached values while one is held: the size when the server reports less than i_size and [srv_size, i_size) is fully granted, mtime and ctime while the cache under the grant is still dirty. A remote truncate has to revoke first, so the smaller size that follows is applied as usual. The attribute-driven invalidation now keys off STATX_SIZE instead of the whole mask, so a reply that does shrink i_size still truncates the page cache when only the timestamps were served from the cache. Signed-off-by: Horst Birthelmer (cherry picked from commit a6ca3d1103146556f36e2938a547866f5882a941) Signed-off-by: Allison Henderson --- fs/fuse/fuse_dlm_cache.c | 31 +++++++++++++++ fs/fuse/fuse_dlm_cache.h | 3 ++ fs/fuse/inode.c | 82 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 109 insertions(+), 7 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 01778325e8b30a..6531186d63b54b 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -565,6 +565,37 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, return true; } +/** + * fuse_dlm_write_grant_exists - does the inode hold an exclusive grant anywhere + * @fi: the fuse inode + * + * Unlike fuse_dlm_range_is_locked(), which asks whether one range is fully + * covered, this asks whether any part of the file is held exclusively. A + * client that holds a write grant may be sitting on dirty page cache the + * server has not seen, so its mtime and ctime run ahead of anything the + * server can report. + * + * Return: true if at least one recorded range is held for write + */ +bool fuse_dlm_write_grant_exists(struct fuse_inode *fi) +{ + struct fuse_dlm_cache *cache = &fi->dlm_locked_areas; + struct fuse_dlm_range *range; + bool held = false; + + down_read(&cache->lock); + for (range = fuse_dlm_find_overlapping(cache, 0, U64_MAX); range; + range = fuse_page_it_iter_next(range, 0, U64_MAX)) { + if (range->mode == FUSE_PCACHE_LK_WRITE) { + held = true; + break; + } + } + up_read(&cache->lock); + + return held; +} + /** * fuse_dlm_lock_is_held - check that a byte range is covered by a granted lock * @fi: the fuse inode diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 647a8c37c36095..30fdbb26bd3daf 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -71,6 +71,9 @@ bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, size_t length, enum fuse_page_lock_mode mode); +/* Is any part of the file held for write? */ +bool fuse_dlm_write_grant_exists(struct fuse_inode *inode); + /* This is the interface to the filesystem */ int fuse_get_dlm_lock(struct file *file, loff_t offset, size_t length, enum fuse_page_lock_mode mode); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 366d9b703225e3..431ae386872646 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -366,6 +366,68 @@ u32 fuse_get_cache_mask(struct inode *inode) return STATX_MTIME | STATX_CTIME | STATX_SIZE; } +/* + * Which cached attributes survive a server reply. + * + * Without DLM this is fuse_get_cache_mask(): with the writeback cache on, + * writes update mtime and ctime and may extend i_size locally, the server + * knows about none of it, so the cached values win. + * + * With DLM the server is the authority (fuse_get_cache_mask() returns 0), + * because another node may have changed the file behind us and only the + * server can say so. That holds for the parts of the file we do not own. A + * write grant means no other node can touch the range until we are revoked, + * so anything the server reports about it is at best as new as what we have, + * and older if we still have unwritten data there. Keep the cached values + * for exactly what the grant covers: + * + * - size, when the server reports less than i_size and the tail it does not + * know about, [srv_size, i_size), is entirely under a write grant. Taking + * the server's answer would shrink i_size and have truncate_pagecache() + * throw the unwritten tail away. + * - mtime and ctime, while a write grant covers unwritten data: our writes + * have stamped them locally and the server's stamps predate them. Only + * while the cache is actually dirty, not for as long as the grant lives: + * a grant is held until it is revoked or the inode is evicted, and past + * the writeback the server's stamps are the newer ones. Keeping ours + * beyond that would hide a remote chown or chmod indefinitely. + * + * A remote truncate cannot slip through. It has to revoke the grant first, + * and the revoke launders the tail and drops the grant, so by the time the + * smaller size is reported neither check holds and the server's answer is + * applied as usual. A grant the server made but that could not be recorded + * (FUSE_DLM_GRANT_UNRECORDED) is invisible to the lock tree and falls back to + * trusting the server, as before. + * + * Must be called without fi->lock: the lock tree query sleeps. + */ +static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, + bool have_size) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + u32 cache_mask = fuse_get_cache_mask(inode); + loff_t size = i_size_read(inode); + + if (cache_mask || !fc->dlm || !fc->writeback_cache || + !S_ISREG(inode->i_mode)) + return cache_mask; + + if (!fuse_dlm_write_grant_exists(fi)) + return cache_mask; + + if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) || + mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) + cache_mask |= STATX_MTIME | STATX_CTIME; + + if (have_size && size > (loff_t) attr->size && + fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, + FUSE_PAGE_LOCK_WRITE)) + cache_mask |= STATX_SIZE; + + return cache_mask; +} + static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr, struct fuse_statx *sx, u64 attr_valid, u64 attr_version, u64 evict_ctr) @@ -378,15 +440,11 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr bool have_size = !sx || (sx->mask & STATX_SIZE); u64 srv_size; + cache_mask = fuse_attr_cache_mask(inode, attr, have_size); + spin_lock(&fi->lock); srv_size = attr->size; - /* - * In case of writeback_cache enabled, writes update mtime, ctime and - * may update i_size. In these cases trust the cached value in the - * inode. - */ - cache_mask = fuse_get_cache_mask(inode); if (cache_mask & STATX_SIZE) attr->size = i_size_read(inode); @@ -432,7 +490,17 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr i_size_write(inode, attr->size); spin_unlock(&fi->lock); - if (!cache_mask && S_ISREG(inode->i_mode)) { + /* + * Only do page cache invalidation when the size was not served from + * the cache (writeback_cache disabled, or no grant covering the tail) + * AND the relevant attributes (SIZE/MTIME) were actually returned by + * the server. This has to key off STATX_SIZE alone: i_size_write() + * above took the server's size for any mask without that bit, and the + * cache has to be truncated to match it. The mtime branch neutralises + * itself when STATX_MTIME is set, since attr->mtime then holds the + * value old_mtime was read from. + */ + if (!(cache_mask & STATX_SIZE) && S_ISREG(inode->i_mode)) { bool inval = false; if (oldsize != attr->size) { From f40355d2f9fa3e48919e2149f8e7cbae53a4eb27 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 14 Aug 2026 16:06:09 +0200 Subject: [PATCH 77/77] fuse: take i_rwsem exclusive when a write drops suid/sgid Without handle_killpriv[_v2], fuse_setattr() kills the bits by asking the server, and fuse_do_setattr() freezes writepages around that SETATTR. fuse_set_nowrite() asserts BUG_ON(fi->writectr < 0) under fi->lock, which assumes the caller holds i_rwsem exclusive: with the writeback cache and DLM, buffered writes hold it only shared. Two writers to a suid file can then both pass dentry_needs_remove_privs() before either has cleared the bits, and the second one hits the assert. It oopses inside spin_lock(&fi->lock), so fi->lock stays held and the i_rwsem read count leaks: the inode wedges and the box follows. The race window is a full GETATTR plus SETATTR, so it is not narrow, and an unprivileged user can set the bit on a file it owns. Keep those writes off the writeback path, which is the one that relaxes i_rwsem to shared, the way handle_killpriv_v2 writes already are. Scoped to DLM connections, since every other configuration already holds i_rwsem exclusive for a buffered write, and only writes that still find the bits set pay for it. Note that the non-writeback path takes no DLM lock, so those writes leave clean folios in the page cache without a grant covering them. That gap already exists for handle_killpriv_v2 and is not addressed here. Signed-off-by: Horst Birthelmer (cherry picked from commit d109f0f6f6fb2d02091455432e01e8067ca93d78) Signed-off-by: Allison Henderson --- fs/fuse/file.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index b43551fe12c6de..f019dccaeff6d2 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1954,7 +1954,31 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (err) return err; - if (!fc->handle_killpriv_v2 || + /* + * A write that drops suid/sgid stays off the writeback path, + * so it holds i_rwsem exclusive. + * + * With handle_killpriv_v2 that is because the server does the + * killing from the WRITE itself. Without it, fuse_setattr() + * has to ask the server, and fuse_do_setattr() freezes + * writepages around that SETATTR: fuse_set_nowrite() asserts + * BUG_ON(fi->writectr < 0), which assumes an exclusive + * i_rwsem, and the DLM-relaxed buffered write path below holds + * it only shared. Two writers can both see the bits set + * before either has cleared them, and the second one would + * then oops inside spin_lock(&fi->lock). + * + * Only the DLM path needs the detour: everywhere else the + * buffered write already holds i_rwsem exclusive, so the two + * writers cannot overlap in the first place. + * + * The bits are read without the inode lock here, so a server + * attribute update can still set them between this test and + * file_remove_privs(). That leaves the same race, but only + * for writers whose mode changed underneath them, rather than + * for every write to a suid file. + */ + if (!(fc->handle_killpriv_v2 || fc->dlm) || !setattr_should_drop_suidgid(idmap, file_inode(file))) writeback = true; }