fuse: add GPU Direct Storage (GDS) support

fs/fuse/Makefile

@@ -10,7 +10,7 @@ obj-$(CONFIG_FUSE_FS) += fuse.o
 obj-$(CONFIG_CUSE) += cuse.o
 obj-$(CONFIG_VIRTIO_FS) += virtiofs.o
 
-fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o compound.o
+fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o compound.o fuse_gds.o
 fuse-y += iomode.o
 fuse-$(CONFIG_FUSE_DAX) += dax.o
 fuse-$(CONFIG_FUSE_IO_URING) += dev_uring.o

fs/fuse/file.c

@@ -8,6 +8,7 @@
 
 #include "fuse_i.h"
 #include "fuse_dlm_cache.h"
+#include "fuse_gds.h"
 
 #include <linux/pagemap.h>
 #include <linux/slab.h>
@@ -669,6 +670,23 @@ static int fuse_fsync(struct file *file, loff_t start, loff_t end,
 	return err;
 }
 
+static void fuse_readwrite_args_fill_gds_ext(struct fuse_io_args *ia)
+{
+	struct fuse_args *args = &ia->ap.args;
+	struct fuse_ext_header *ext_header = (struct fuse_ext_header *)ia->readwrite_in_gds_ext;
+
+	/*
+	 * The RDMA information data is already filled by the fuse_gds_get_gpu_sglist_rdma_info
+	 * function. Fill the extension header here.
+	 */
+	ext_header->type = FUSE_EXT_READ_WRITE_GDS;
+	ext_header->size = FUSE_EXT_READ_WRITE_GDS_SIZE;
+
+	args->is_ext = 1;
+	args->ext_idx = args->in_numargs++;
+	args->in_args[args->ext_idx].size = ext_header->size;
+	args->in_args[args->ext_idx].value = ext_header;
+}
 void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
 			 size_t count, int opcode)
 {
@@ -687,6 +705,9 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
 	args->out_argvar = true;
 	args->out_numargs = 1;
 	args->out_args[0].size = count;
+
+	if (ia->ap.args.use_gds)
+		fuse_readwrite_args_fill_gds_ext(ia);
 }
 
 static void fuse_release_user_pages(struct fuse_args_pages *ap,
@@ -1126,16 +1147,25 @@ static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,
 	ia->write.in.size = count;
 	args->opcode = FUSE_WRITE;
 	args->nodeid = ff->nodeid;
-	args->in_numargs = 2;
+	args->in_numargs = 0;
 	if (ff->fm->fc->minor < 9)
 		args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
 	else
 		args->in_args[0].size = sizeof(ia->write.in);
 	args->in_args[0].value = &ia->write.in;
-	args->in_args[1].size = count;
+	args->in_numargs++;
+
+	if (ia->ap.args.use_gds)
+		fuse_readwrite_args_fill_gds_ext(ia);
+
+	/* inpage argument must be the last one */
+	args->in_args[args->in_numargs++].size = count;
+
 	args->out_numargs = 1;
 	args->out_args[0].size = sizeof(ia->write.out);
 	args->out_args[0].value = &ia->write.out;
+
+
 }
 
 static unsigned int fuse_write_flags(struct kiocb *iocb)
@@ -1663,6 +1693,14 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
 		if (err && !nbytes)
 			break;
 
+		if (fc->gds_support && fuse_is_gds_buffer(&ia->ap)) {
+			err = fuse_gds_get_gpu_sglist_rdma_info(fc, write, ia);
+			if (err) {
+				fuse_release_user_pages(&ia->ap, io->should_dirty);
+				break;
+			}
+		}
+
 		if (write) {
 			if (!capable(CAP_FSETID))
 				ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;

fs/fuse/fuse_gds.c

@@ -0,0 +1,155 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * FUSE: Filesystem in Userspace
+ * Copyright (c) 2023-2026 DataDirect Networks.
+ */
+
+#include <linux/delay.h>
+#include <linux/scatterlist.h>
+#include "fuse_i.h"
+
+#define GDS_MOCK_TEST 0
+
+/* NVIDIA GPU Direct Storage interface and operations */
+
+static atomic_t nvfs_ops_refcnt = ATOMIC_INIT(0);
+static struct nvfs_dma_rw_ops *nvfs_ops = NULL;
+
+static inline struct nvfs_dma_rw_ops* get_nvfs_dma_ops(void)
+{
+	struct nvfs_dma_rw_ops *ops;
+
+	rcu_read_lock();
+	ops = rcu_dereference(nvfs_ops);
+	if (ops)
+		atomic_inc(&nvfs_ops_refcnt);
+	rcu_read_unlock();
+	return ops;
+}
+
+static inline void put_nvfs_dma_ops(void)
+{
+	atomic_dec(&nvfs_ops_refcnt);
+}
+
+int fuse_register_nvfs_dma_ops(struct nvfs_dma_rw_ops *ops)
+{
+	if (!ops)
+		return -EINVAL;
+
+	rcu_assign_pointer(nvfs_ops, ops);
+	return 0;
+}
+
+void fuse_unregister_nvfs_dma_ops(void)
+{
+	rcu_assign_pointer(nvfs_ops, NULL);
+	synchronize_rcu();
+	while (atomic_read(&nvfs_ops_refcnt) > 0)
+		msleep(100);
+}
+EXPORT_SYMBOL_GPL(fuse_register_nvfs_dma_ops);
+EXPORT_SYMBOL_GPL(fuse_unregister_nvfs_dma_ops);
+
+/* GPU buffer detection and DMA scatter-gather mapping operations */
+
+#if !GDS_MOCK_TEST
+static bool nvfs_dma_ops_is_gds_page(struct page *page)
+{
+	struct nvfs_dma_rw_ops *ops = get_nvfs_dma_ops();
+	if (ops) {
+		bool ret = ops->nvfs_is_gpu_page(page);
+		put_nvfs_dma_ops();
+		return ret;
+	}
+	return false;
+}
+
+bool fuse_is_gds_buffer(struct fuse_args_pages *ap)
+{
+	struct page **pages = ap->pages;
+	unsigned int num_pages = ap->num_pages;
+
+	return ap->args.user_pages && num_pages > 0 && nvfs_dma_ops_is_gds_page(pages[0]);
+}
+
+static int nvfs_get_gpu_sglist_rdma_info(struct scatterlist *sglist,
+			int nents,
+			struct nvfs_rdma_info *rdma_infop)
+{
+	struct nvfs_dma_rw_ops *ops = get_nvfs_dma_ops();
+	int rc = -EIO;
+	if (ops) {
+		/* The function returns the number of processed entries upon successful completion */
+		rc = ops->nvfs_get_gpu_sglist_rdma_info(sglist, nents, rdma_infop);
+		put_nvfs_dma_ops();
+	}
+	return rc > 0 ? 0: -EIO;
+}
+
+int fuse_gds_get_gpu_sglist_rdma_info(struct fuse_conn *fc, bool write,
+			struct fuse_io_args *ia)
+{
+	struct sg_table sgt;
+	struct scatterlist *sgl;
+	int rc = 0;
+	unsigned int i;
+
+	if (!ia->ap.num_pages) {
+		return -EINVAL;
+	}
+
+	ia->ap.args.use_gds = 0;
+	if (sg_alloc_table(&sgt, ia->ap.num_pages, GFP_KERNEL)) {
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	sgl = sgt.sgl;
+	for(i = 0; i < ia->ap.num_pages; i++) {
+		sg_set_page(sgl, ia->ap.pages[i], ia->ap.descs[i].length, ia->ap.descs[i].offset);
+		sgl = sg_next(sgl);
+	}
+
+	rc = nvfs_get_gpu_sglist_rdma_info(sgt.sgl, ia->ap.num_pages, fuse_get_readwrite_rdma_info(ia));
+	if (rc) {
+		goto out_sgt;
+	}
+
+	ia->ap.args.use_gds = 1;
+
+out_sgt:
+	sg_free_table(&sgt);
+out:
+	return rc;
+}
+
+#else
+bool fuse_is_gds_buffer(struct fuse_args_pages *ap)
+{
+	return true;
+}
+
+int fuse_gds_get_gpu_sglist_rdma_info(struct fuse_conn *fc, bool write,
+			struct fuse_io_args *ia)
+{
+	struct nvfs_rdma_info *rdma_infop = fuse_get_readwrite_rdma_info(ia);
+	unsigned int total_len = 0;
+	for (unsigned int i = 0; i < ia->ap.num_pages; i++) {
+		total_len += ia->ap.descs[i].length;
+	}
+	rdma_infop->version = 2;
+	rdma_infop->flags = 1;
+	rdma_infop->lid = 0;
+	rdma_infop->qp_num = 3;
+	rdma_infop->rem_vaddr = 4;
+	rdma_infop->size = total_len;
+	rdma_infop->rkey = 5;
+	rdma_infop->gid[0] = 6;
+	rdma_infop->gid[1] = 7;
+	rdma_infop->dc_key = 8;
+
+	ia->ap.args.use_gds = 1;
+	return 0;
+}
+#endif

fs/fuse/fuse_gds.h

@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * FUSE: Filesystem in Userspace
+ * Copyright (c) 2023-2026 DataDirect Networks.
+ */
+
+#ifndef _FS_FUSE_GDS_H
+#define _FS_FUSE_GDS_H
+
+#include <linux/dma-direction.h>
+
+struct fuse_args_pages;
+struct fuse_conn;
+struct fuse_io_args;
+struct request;
+
+struct nvfs_rdma_info
+{
+        uint8_t    version;     /* to support future changes to structure */
+        uint8_t    flags;       /* if bit 0 != 0, then gid field is valid */
+        uint16_t   lid;         /* subnet local identifier of the client node port */
+        uint32_t   qp_num;      /* QP number of DCT on the client node */
+        uint64_t   rem_vaddr;   /* remote address */
+        uint32_t   size;
+        uint32_t   rkey;
+        uint64_t   gid[2];      /* 16-byte global identifier of the client node port */
+        uint32_t   dc_key;
+};
+
+struct nvfs_dma_rw_ops {
+	unsigned long long ft_bmap; /* feature bitmap */
+
+	int (*nvfs_blk_rq_map_sg) (struct request_queue *q,
+			struct request *req,
+			struct scatterlist *sglist);
+
+	int (*nvfs_dma_map_sg_attrs) (struct device *device,
+			struct scatterlist *sglist,
+			int nents,
+			enum dma_data_direction dma_dir,
+			unsigned long attrs);
+
+	int (*nvfs_dma_unmap_sg) (struct device *device,
+			struct scatterlist *sglist,
+			int nents,
+			enum dma_data_direction dma_dir);
+
+	bool (*nvfs_is_gpu_page) (struct page *);
+	unsigned int (*nvfs_gpu_index) (struct page *page);
+	unsigned int (*nvfs_device_priority) (struct device *dev, unsigned int dev_index);
+
+	int (*nvfs_get_gpu_sglist_rdma_info) (struct scatterlist *sglist,
+			int nents,
+			struct nvfs_rdma_info *rdma_infop);
+};
+
+bool fuse_is_gds_buffer(struct fuse_args_pages *ap);
+int fuse_gds_get_gpu_sglist_rdma_info(struct fuse_conn *fc, bool write,
+		struct fuse_io_args *ia);
+int fuse_register_nvfs_dma_ops(struct nvfs_dma_rw_ops *ops);
+void fuse_unregister_nvfs_dma_ops(void);
+
+#endif

fs/fuse/fuse_i.h

@@ -32,6 +32,7 @@
 #include <linux/refcount.h>
 #include <linux/user_namespace.h>
 #include "fuse_dlm_cache.h"
+#include "fuse_gds.h"
 
 /** Default max number of pages that can be used in a single read request */
 #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32
@@ -324,6 +325,7 @@ struct fuse_args {
 	bool may_block:1;
 	bool is_ext:1;
 	bool is_pinned:1;
+	bool use_gds:1;
 	struct fuse_in_arg in_args[4];
 	struct fuse_arg out_args[2];
 	void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error);
@@ -879,6 +881,9 @@ struct fuse_conn {
 	/* Is synchronous FUSE_INIT allowed? */
 	unsigned int sync_init:1;
 
+	/* Does the filesystem support GDS? */
+	unsigned int gds_support:1;
+
 	/* Use io_uring for communication */
 	unsigned int io_uring;
 
@@ -1115,6 +1120,10 @@ struct fuse_forget_link *fuse_alloc_forget(void);
 /*
  * Initialize READ or READDIR request
  */
+
+#define FUSE_EXT_READ_WRITE_GDS_SIZE \
+    FUSE_REC_ALIGN(sizeof(struct fuse_ext_header) + sizeof(struct nvfs_rdma_info))
+
 struct fuse_io_args {
 	union {
 		struct {
@@ -1130,11 +1139,16 @@ struct fuse_io_args {
 	struct fuse_args_pages ap;
 	struct fuse_io_priv *io;
 	struct fuse_file *ff;
+	uint8_t readwrite_in_gds_ext[FUSE_EXT_READ_WRITE_GDS_SIZE];
 };
 
 void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
 			 size_t count, int opcode);
 
+static inline struct nvfs_rdma_info *fuse_get_readwrite_rdma_info(struct fuse_io_args *ia)
+{
+	return (struct nvfs_rdma_info *)(ia->readwrite_in_gds_ext + sizeof(struct fuse_ext_header));
+}
 /*
  * Helper functions to initialize fuse_args for common operations
  */

fs/fuse/inode.c

@@ -1619,6 +1619,8 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args,
 				fc->inval_inode_entries = 1;
 			if (flags & FUSE_EXPIRE_INODE_ENTRY)
 				fc->expire_inode_entries = 1;
+			if (flags & FUSE_GDS_SUPPORT)
+				fc->gds_support = 1;
 		} else {
 			ra_pages = fc->max_read / PAGE_SIZE;
 			fc->no_lock = 1;
@@ -1669,7 +1671,8 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm)
 		FUSE_SECURITY_CTX | FUSE_CREATE_SUPP_GROUP |
 		FUSE_HAS_EXPIRE_ONLY | FUSE_DIRECT_IO_ALLOW_MMAP |
 		FUSE_NO_EXPORT_SUPPORT | FUSE_INVAL_INODE_ENTRY |
-		FUSE_EXPIRE_INODE_ENTRY | FUSE_URING_REDUCED_Q;
+		FUSE_EXPIRE_INODE_ENTRY | FUSE_URING_REDUCED_Q |
+		FUSE_GDS_SUPPORT;
 #ifdef CONFIG_FUSE_DAX
 	if (fm->fc->dax)
 		flags |= FUSE_MAP_ALIGNMENT;