diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile index 64dad526c0b8ce..c0cf4cdadd4f91 100644 --- a/fs/fuse/Makefile +++ b/fs/fuse/Makefile @@ -10,7 +10,7 @@ obj-$(CONFIG_FUSE_FS) += fuse.o obj-$(CONFIG_CUSE) += cuse.o obj-$(CONFIG_VIRTIO_FS) += virtiofs.o -fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o compound.o +fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o compound.o fuse_gds.o fuse-y += iomode.o fuse-$(CONFIG_FUSE_DAX) += dax.o fuse-$(CONFIG_FUSE_IO_URING) += dev_uring.o diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 55ab5e9cf61715..e28c0b77266678 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -8,6 +8,7 @@ #include "fuse_i.h" #include "fuse_dlm_cache.h" +#include "fuse_gds.h" #include #include @@ -669,6 +670,23 @@ static int fuse_fsync(struct file *file, loff_t start, loff_t end, return err; } +static void fuse_readwrite_args_fill_gds_ext(struct fuse_io_args *ia) +{ + struct fuse_args *args = &ia->ap.args; + struct fuse_ext_header *ext_header = (struct fuse_ext_header *)ia->readwrite_in_gds_ext; + + /* + * The RDMA information data is already filled by the fuse_gds_get_gpu_sglist_rdma_info + * function. Fill the extension header here. + */ + ext_header->type = FUSE_EXT_READ_WRITE_GDS; + ext_header->size = FUSE_EXT_READ_WRITE_GDS_SIZE; + + args->is_ext = 1; + args->ext_idx = args->in_numargs++; + args->in_args[args->ext_idx].size = ext_header->size; + args->in_args[args->ext_idx].value = ext_header; +} void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, size_t count, int opcode) { @@ -687,6 +705,9 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, args->out_argvar = true; args->out_numargs = 1; args->out_args[0].size = count; + + if (ia->ap.args.use_gds) + fuse_readwrite_args_fill_gds_ext(ia); } static void fuse_release_user_pages(struct fuse_args_pages *ap, @@ -1126,16 +1147,25 @@ static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, ia->write.in.size = count; args->opcode = FUSE_WRITE; args->nodeid = ff->nodeid; - args->in_numargs = 2; + args->in_numargs = 0; if (ff->fm->fc->minor < 9) args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE; else args->in_args[0].size = sizeof(ia->write.in); args->in_args[0].value = &ia->write.in; - args->in_args[1].size = count; + args->in_numargs++; + + if (ia->ap.args.use_gds) + fuse_readwrite_args_fill_gds_ext(ia); + + /* inpage argument must be the last one */ + args->in_args[args->in_numargs++].size = count; + args->out_numargs = 1; args->out_args[0].size = sizeof(ia->write.out); args->out_args[0].value = &ia->write.out; + + } static unsigned int fuse_write_flags(struct kiocb *iocb) @@ -1663,6 +1693,14 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, if (err && !nbytes) break; + if (fc->gds_support && fuse_is_gds_buffer(&ia->ap)) { + err = fuse_gds_get_gpu_sglist_rdma_info(fc, write, ia); + if (err) { + fuse_release_user_pages(&ia->ap, io->should_dirty); + break; + } + } + if (write) { if (!capable(CAP_FSETID)) ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; diff --git a/fs/fuse/fuse_gds.c b/fs/fuse/fuse_gds.c new file mode 100644 index 00000000000000..72c7b87679a82c --- /dev/null +++ b/fs/fuse/fuse_gds.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE: Filesystem in Userspace + * Copyright (c) 2023-2026 DataDirect Networks. + */ + +#include +#include +#include "fuse_i.h" + +#define GDS_MOCK_TEST 0 + +/* NVIDIA GPU Direct Storage interface and operations */ + +static atomic_t nvfs_ops_refcnt = ATOMIC_INIT(0); +static struct nvfs_dma_rw_ops *nvfs_ops = NULL; + +static inline struct nvfs_dma_rw_ops* get_nvfs_dma_ops(void) +{ + struct nvfs_dma_rw_ops *ops; + + rcu_read_lock(); + ops = rcu_dereference(nvfs_ops); + if (ops) + atomic_inc(&nvfs_ops_refcnt); + rcu_read_unlock(); + return ops; +} + +static inline void put_nvfs_dma_ops(void) +{ + atomic_dec(&nvfs_ops_refcnt); +} + +int fuse_register_nvfs_dma_ops(struct nvfs_dma_rw_ops *ops) +{ + if (!ops) + return -EINVAL; + + rcu_assign_pointer(nvfs_ops, ops); + return 0; +} + +void fuse_unregister_nvfs_dma_ops(void) +{ + rcu_assign_pointer(nvfs_ops, NULL); + synchronize_rcu(); + while (atomic_read(&nvfs_ops_refcnt) > 0) + msleep(100); +} +EXPORT_SYMBOL_GPL(fuse_register_nvfs_dma_ops); +EXPORT_SYMBOL_GPL(fuse_unregister_nvfs_dma_ops); + +/* GPU buffer detection and DMA scatter-gather mapping operations */ + +#if !GDS_MOCK_TEST +static bool nvfs_dma_ops_is_gds_page(struct page *page) +{ + struct nvfs_dma_rw_ops *ops = get_nvfs_dma_ops(); + if (ops) { + bool ret = ops->nvfs_is_gpu_page(page); + put_nvfs_dma_ops(); + return ret; + } + return false; +} + +bool fuse_is_gds_buffer(struct fuse_args_pages *ap) +{ + struct page **pages = ap->pages; + unsigned int num_pages = ap->num_pages; + + return ap->args.user_pages && num_pages > 0 && nvfs_dma_ops_is_gds_page(pages[0]); +} + +static int nvfs_get_gpu_sglist_rdma_info(struct scatterlist *sglist, + int nents, + struct nvfs_rdma_info *rdma_infop) +{ + struct nvfs_dma_rw_ops *ops = get_nvfs_dma_ops(); + int rc = -EIO; + if (ops) { + /* The function returns the number of processed entries upon successful completion */ + rc = ops->nvfs_get_gpu_sglist_rdma_info(sglist, nents, rdma_infop); + put_nvfs_dma_ops(); + } + return rc > 0 ? 0: -EIO; +} + +int fuse_gds_get_gpu_sglist_rdma_info(struct fuse_conn *fc, bool write, + struct fuse_io_args *ia) +{ + struct sg_table sgt; + struct scatterlist *sgl; + int rc = 0; + unsigned int i; + + if (!ia->ap.num_pages) { + return -EINVAL; + } + + ia->ap.args.use_gds = 0; + if (sg_alloc_table(&sgt, ia->ap.num_pages, GFP_KERNEL)) { + rc = -ENOMEM; + goto out; + } + + sgl = sgt.sgl; + for(i = 0; i < ia->ap.num_pages; i++) { + sg_set_page(sgl, ia->ap.pages[i], ia->ap.descs[i].length, ia->ap.descs[i].offset); + sgl = sg_next(sgl); + } + + rc = nvfs_get_gpu_sglist_rdma_info(sgt.sgl, ia->ap.num_pages, fuse_get_readwrite_rdma_info(ia)); + if (rc) { + goto out_sgt; + } + + ia->ap.args.use_gds = 1; + +out_sgt: + sg_free_table(&sgt); +out: + return rc; +} + +#else +bool fuse_is_gds_buffer(struct fuse_args_pages *ap) +{ + return true; +} + +int fuse_gds_get_gpu_sglist_rdma_info(struct fuse_conn *fc, bool write, + struct fuse_io_args *ia) +{ + struct nvfs_rdma_info *rdma_infop = fuse_get_readwrite_rdma_info(ia); + unsigned int total_len = 0; + for (unsigned int i = 0; i < ia->ap.num_pages; i++) { + total_len += ia->ap.descs[i].length; + } + rdma_infop->version = 2; + rdma_infop->flags = 1; + rdma_infop->lid = 0; + rdma_infop->qp_num = 3; + rdma_infop->rem_vaddr = 4; + rdma_infop->size = total_len; + rdma_infop->rkey = 5; + rdma_infop->gid[0] = 6; + rdma_infop->gid[1] = 7; + rdma_infop->dc_key = 8; + + ia->ap.args.use_gds = 1; + return 0; +} +#endif diff --git a/fs/fuse/fuse_gds.h b/fs/fuse/fuse_gds.h new file mode 100644 index 00000000000000..4af097483c225b --- /dev/null +++ b/fs/fuse/fuse_gds.h @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE: Filesystem in Userspace + * Copyright (c) 2023-2026 DataDirect Networks. + */ + +#ifndef _FS_FUSE_GDS_H +#define _FS_FUSE_GDS_H + +#include + +struct fuse_args_pages; +struct fuse_conn; +struct fuse_io_args; +struct request; + +struct nvfs_rdma_info +{ + uint8_t version; /* to support future changes to structure */ + uint8_t flags; /* if bit 0 != 0, then gid field is valid */ + uint16_t lid; /* subnet local identifier of the client node port */ + uint32_t qp_num; /* QP number of DCT on the client node */ + uint64_t rem_vaddr; /* remote address */ + uint32_t size; + uint32_t rkey; + uint64_t gid[2]; /* 16-byte global identifier of the client node port */ + uint32_t dc_key; +}; + +struct nvfs_dma_rw_ops { + unsigned long long ft_bmap; /* feature bitmap */ + + int (*nvfs_blk_rq_map_sg) (struct request_queue *q, + struct request *req, + struct scatterlist *sglist); + + int (*nvfs_dma_map_sg_attrs) (struct device *device, + struct scatterlist *sglist, + int nents, + enum dma_data_direction dma_dir, + unsigned long attrs); + + int (*nvfs_dma_unmap_sg) (struct device *device, + struct scatterlist *sglist, + int nents, + enum dma_data_direction dma_dir); + + bool (*nvfs_is_gpu_page) (struct page *); + unsigned int (*nvfs_gpu_index) (struct page *page); + unsigned int (*nvfs_device_priority) (struct device *dev, unsigned int dev_index); + + int (*nvfs_get_gpu_sglist_rdma_info) (struct scatterlist *sglist, + int nents, + struct nvfs_rdma_info *rdma_infop); +}; + +bool fuse_is_gds_buffer(struct fuse_args_pages *ap); +int fuse_gds_get_gpu_sglist_rdma_info(struct fuse_conn *fc, bool write, + struct fuse_io_args *ia); +int fuse_register_nvfs_dma_ops(struct nvfs_dma_rw_ops *ops); +void fuse_unregister_nvfs_dma_ops(void); + +#endif diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index c8c0b5a1477968..1556c8da21e1cf 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -32,6 +32,7 @@ #include #include #include "fuse_dlm_cache.h" +#include "fuse_gds.h" /** Default max number of pages that can be used in a single read request */ #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 @@ -324,6 +325,7 @@ struct fuse_args { bool may_block:1; bool is_ext:1; bool is_pinned:1; + bool use_gds:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); @@ -879,6 +881,9 @@ struct fuse_conn { /* Is synchronous FUSE_INIT allowed? */ unsigned int sync_init:1; + /* Does the filesystem support GDS? */ + unsigned int gds_support:1; + /* Use io_uring for communication */ unsigned int io_uring; @@ -1115,6 +1120,10 @@ struct fuse_forget_link *fuse_alloc_forget(void); /* * Initialize READ or READDIR request */ + +#define FUSE_EXT_READ_WRITE_GDS_SIZE \ + FUSE_REC_ALIGN(sizeof(struct fuse_ext_header) + sizeof(struct nvfs_rdma_info)) + struct fuse_io_args { union { struct { @@ -1130,11 +1139,16 @@ struct fuse_io_args { struct fuse_args_pages ap; struct fuse_io_priv *io; struct fuse_file *ff; + uint8_t readwrite_in_gds_ext[FUSE_EXT_READ_WRITE_GDS_SIZE]; }; void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, size_t count, int opcode); +static inline struct nvfs_rdma_info *fuse_get_readwrite_rdma_info(struct fuse_io_args *ia) +{ + return (struct nvfs_rdma_info *)(ia->readwrite_in_gds_ext + sizeof(struct fuse_ext_header)); +} /* * Helper functions to initialize fuse_args for common operations */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 82c84a81b6e2d3..da81e0e99a8697 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1619,6 +1619,8 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, fc->inval_inode_entries = 1; if (flags & FUSE_EXPIRE_INODE_ENTRY) fc->expire_inode_entries = 1; + if (flags & FUSE_GDS_SUPPORT) + fc->gds_support = 1; } else { ra_pages = fc->max_read / PAGE_SIZE; fc->no_lock = 1; @@ -1669,7 +1671,8 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) FUSE_SECURITY_CTX | FUSE_CREATE_SUPP_GROUP | FUSE_HAS_EXPIRE_ONLY | FUSE_DIRECT_IO_ALLOW_MMAP | FUSE_NO_EXPORT_SUPPORT | FUSE_INVAL_INODE_ENTRY | - FUSE_EXPIRE_INODE_ENTRY | FUSE_URING_REDUCED_Q; + FUSE_EXPIRE_INODE_ENTRY | FUSE_URING_REDUCED_Q | + FUSE_GDS_SUPPORT; #ifdef CONFIG_FUSE_DAX if (fm->fc->dax) flags |= FUSE_MAP_ALIGNMENT; diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index d472df6370a400..b359b72844244b 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -482,6 +482,7 @@ struct fuse_file_lock { #define FUSE_URING_REDUCED_Q (1ULL << 59) #define FUSE_INVAL_INODE_ENTRY (1ULL << 60) #define FUSE_EXPIRE_INODE_ENTRY (1ULL << 61) +#define FUSE_GDS_SUPPORT (1ULL << 62) /** * CUSE INIT request/reply flags @@ -604,6 +605,7 @@ enum fuse_ext_type { /* Types 0..31 are reserved for fuse_secctx_header */ FUSE_MAX_NR_SECCTX = 31, FUSE_EXT_GROUPS = 32, + FUSE_EXT_READ_WRITE_GDS = 100, }; enum fuse_opcode { @@ -848,6 +850,10 @@ struct fuse_read_in { uint32_t padding; }; +struct fuse_gds_read_out { + uint64_t size; +}; + #define FUSE_COMPAT_WRITE_IN_SIZE 24 struct fuse_write_in {