fuse: serialize cached IO and invalidation with per-inode range locks

fs/fuse/Makefile

@@ -10,7 +10,7 @@ obj-$(CONFIG_FUSE_FS) += fuse.o
 obj-$(CONFIG_CUSE) += cuse.o
 obj-$(CONFIG_VIRTIO_FS) += virtiofs.o
 
-fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o compound.o
+fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o fuse_range_lock.o compound.o
 fuse-y += iomode.o
 fuse-$(CONFIG_FUSE_DAX) += dax.o
 fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o

fs/fuse/dev.c

@@ -433,9 +433,10 @@ static void flush_bg_queue(struct fuse_conn *fc)
  * This function is called when a request is finished.  Either a reply
  * has arrived or it was aborted (and not yet sent) or some error
  * occurred during communication with userspace, or the device file
- * was closed.  The requester thread is woken up (if still waiting),
- * the 'end' callback is called if given, else the reference to the
- * request is released
+ * was closed.  The 'complete' callback, if given, is called first, on
+ * this thread; only then is the requester thread woken up (if still
+ * waiting), the 'end' callback called if given, else the reference to
+ * the request is released.
  */
 void fuse_request_end(struct fuse_req *req)
 {
@@ -459,6 +460,10 @@ void fuse_request_end(struct fuse_req *req)
 	}
 	WARN_ON(test_bit(FR_PENDING, &req->flags));
 	WARN_ON(test_bit(FR_SENT, &req->flags));
+
+	if (req->args->complete)
+		req->args->complete(fm, req->args, req->out.h.error);
+
 	if (test_bit(FR_BACKGROUND, &req->flags)) {
 		spin_lock(&fc->bg_lock);
 		clear_bit(FR_BACKGROUND, &req->flags);

fs/fuse/dir.c

@@ -2106,34 +2106,41 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 		WARN_ON(!(attr->ia_valid & ATTR_SIZE));
 		WARN_ON(attr->ia_size != 0);
 		if (fc->atomic_o_trunc) {
-			struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem;
+			struct fuse_range_lock rlock;
+			bool range_locked = fc->writeback_cache && fc->dlm;
 
 			/*
 			 * No need to send request to userspace, since actual
 			 * truncation has already been done by OPEN.  But still
 			 * need to truncate page cache.
 			 *
-			 * Revoke and drop under the coherency gate write side,
-			 * like the NOTIFY invalidate path: a gate reader that
-			 * already re-validated its grant must not have the
-			 * lock tree and the cache yanked mid-hold, or it
-			 * would repopulate the truncated range trusting a
-			 * grant that no longer exists.  Waiting for gate
-			 * readers here is safe: we hold i_rwsem exclusive, so
-			 * no gate holder can be waiting on it (the write path
-			 * takes i_rwsem before the gate, the read path never
-			 * takes it).
+			 * Revoke and drop under the full-range IO range lock,
+			 * like the NOTIFY invalidate path
+			 * (fuse_reverse_inval_inode()): a reader/writer that
+			 * already reached READY state must not have the lock
+			 * tree and the cache yanked mid-hold, or it would
+			 * repopulate the truncated range trusting a grant that
+			 * no longer exists.  fuse_range_lock_acquire_ready()
+			 * ignores an overlapping INIT range (a read/write with
+			 * only a DLM request in flight), so waiting here is
+			 * bounded.  Blocking is also safe: we hold i_rwsem
+			 * exclusive, so no cached writer can be waiting on this
+			 * range lock (the write path takes i_rwsem before it,
+			 * the read path never takes i_rwsem at all).  Only
+			 * meaningful under DLM with the writeback cache; see
+			 * fuse_range_lock.h.
 			 */
-			if (wb_sem)
-				percpu_down_write(wb_sem);
+			if (range_locked)
+				fuse_range_lock_acquire_ready(fi, &rlock, 0, ~0ULL,
+							      FUSE_RANGE_LOCK_WRITE);
 			if (fc->dlm && fc->writeback_cache)
 				fuse_dlm_cache_release_locks(fi);
 			spin_lock(&fi->lock);
 			i_size_write(inode, 0);
 			spin_unlock(&fi->lock);
 			truncate_pagecache(inode, 0);
-			if (wb_sem)
-				percpu_up_write(wb_sem);
+			if (range_locked)
+				fuse_range_lock_release(fi, &rlock);
 			goto out;
 		}
 		file = NULL;
@@ -2237,23 +2244,26 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 	 */
 	if ((is_truncate || !is_wb) &&
 	    S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) {
-		struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem;
+		struct fuse_range_lock rlock;
+		bool range_locked = fc->writeback_cache && fc->dlm;
 
 		/*
-		 * Revoke and drop under the coherency gate write side; see
+		 * Revoke and drop under the full-range IO range lock; see
 		 * the atomic-O_TRUNC branch above.  i_rwsem is held
-		 * exclusive here as well (setattr), so waiting out gate
-		 * readers cannot deadlock.
+		 * exclusive here as well (setattr), so waiting out
+		 * in-progress READY IO cannot deadlock.  Only meaningful
+		 * under DLM with the writeback cache; see fuse_range_lock.h.
 		 */
-		if (wb_sem)
-			percpu_down_write(wb_sem);
+		if (range_locked)
+			fuse_range_lock_acquire_ready(fi, &rlock, 0, ~0ULL,
+						      FUSE_RANGE_LOCK_WRITE);
 		if (fc->dlm && fc->writeback_cache)
 			fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1);
 
 		truncate_pagecache(inode, outarg.attr.size);
 		invalidate_inode_pages2(mapping);
-		if (wb_sem)
-			percpu_up_write(wb_sem);
+		if (range_locked)
+			fuse_range_lock_release(fi, &rlock);
 	}
 
 	clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);