dma-mapping: add DMA_ATTR_NO_BOUNCE

Documentation/core-api/dma-attributes.rst

@@ -208,3 +208,50 @@ __DMA_ATTR_ALLOC_CC_SHARED differs from DMA_ATTR_CC_SHARED in that it is not
 a caller-visible DMA API attribute. DMA_ATTR_CC_SHARED describes an
 already-shared mapping and requires the caller to have prepared normal
 system memory before mapping it.
+
+DMA_ATTR_NO_BOUNCE
+------------------
+
+A mapping request with DMA_ATTR_NO_BOUNCE fails where the DMA core could
+only satisfy it through a bounce buffer (SWIOTLB): a successful mapping
+gives the device the memory that was passed in, never a copy of it. This
+is for callers whose device and CPU must see the same memory for as long
+as the mapping exists, with no synchronization points in between, such as
+long-lived mappings of pinned userspace memory.
+
+The request fails for memory that the device cannot address, where
+bouncing is forced (swiotlb=force, a restricted DMA pool, a device that
+needs unencrypted memory), for a kmalloc() buffer that is too small to be
+aligned for DMA to a device that is not DMA coherent, and for a buffer
+that is not aligned to the IOMMU granule of an untrusted device. It also
+fails for a device that is mapped by DMA operations of its own (struct
+dma_map_ops), since the DMA core cannot tell what those map onto.
+
+A refused request logs nothing, whether or not the system has a SWIOTLB
+that could have made the mapping: a driver may pass on requests of
+unprivileged callers for memory it knows nothing about. What each mapping
+function returns for it:
+
+- dma_map_page_attrs(), dma_map_single_attrs() and dma_map_phys():
+  DMA_MAPPING_ERROR, to be tested with dma_mapping_error().
+- dma_map_sg_attrs(): 0.
+- dma_map_sgtable(): -EIO where a bounce buffer would have been needed,
+  -EOPNOTSUPP for a device with DMA operations of its own. -EIO is the
+  error of any segment that cannot be mapped, so it does not tell a
+  refused bounce buffer from another mapping failure. Segments mapped
+  before the refused one are unmapped again.
+- dma_iova_link(): -EPERM.
+
+DMA_ATTR_NO_BOUNCE differs from DMA_ATTR_REQUIRE_COHERENT in that it does
+not require the device to be DMA coherent and does not change CPU cache
+maintenance: caches are synchronized at map, sync and unmap time as for
+any other mapping, unless DMA_ATTR_SKIP_CPU_SYNC is given as well. It
+says nothing about whether the device is coherent with the CPU caches.
+
+The attribute is a property of a mapping request and nothing else looks
+at it. The allocation functions (dma_alloc_attrs(), dma_alloc_pages(),
+dma_alloc_noncontiguous()), dma_mmap_attrs() and dma_get_sgtable_attrs()
+ignore it. So do the unmap and sync functions: no bounce buffer stands
+behind a mapping that was made with it. A DMA_ATTR_MMIO mapping
+(dma_map_resource()) is never bounced, and the attribute changes nothing
+for it on a device that the DMA core maps itself.

drivers/iommu/dma-iommu.c

@@ -1235,7 +1235,7 @@ dma_addr_t iommu_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,
 	 */
 	if (!(attrs & DMA_ATTR_MMIO) && dev_use_swiotlb(dev, size, dir) &&
 	    iova_unaligned(iovad, phys, size)) {
-		if (attrs & DMA_ATTR_REQUIRE_COHERENT)
+		if (attrs & (DMA_ATTR_REQUIRE_COHERENT | DMA_ATTR_NO_BOUNCE))
 			return DMA_MAPPING_ERROR;
 
 		phys = iommu_dma_map_swiotlb(dev, phys, size, dir, attrs);
@@ -2003,7 +2003,8 @@ int dma_iova_link(struct device *dev, struct dma_iova_state *state,
 
 	if (dev_use_swiotlb(dev, size, dir) &&
 	    iova_unaligned(iovad, phys, size)) {
-		if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
+		if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT |
+			     DMA_ATTR_NO_BOUNCE))
 			return -EPERM;
 
 		return iommu_dma_iova_link_swiotlb(dev, state, phys, offset,

include/linux/dma-mapping.h

@@ -111,6 +111,18 @@
  */
 #define __DMA_ATTR_ALLOC_CC_SHARED	(1UL << 14)
 
+/*
+ * DMA_ATTR_NO_BOUNCE: Indicates that the device must be given the memory that
+ * was passed in itself. A mapping request that could only be satisfied through
+ * a bounce buffer (SWIOTLB) fails instead, as does one for a device that is
+ * mapped by DMA operations of its own, where the DMA core cannot tell. Unlike
+ * DMA_ATTR_REQUIRE_COHERENT, it places no requirement on the device's cache
+ * coherence and leaves CPU cache maintenance as it is. Only the mapping
+ * functions look at it: allocations, dma_mmap_attrs() and
+ * dma_get_sgtable_attrs() ignore it.
+ */
+#define DMA_ATTR_NO_BOUNCE		(1UL << 15)
+
 /*
  * A dma_addr_t can hold any valid DMA or bus address for the platform.  It can
  * be given to a device to use as a DMA source or target.  It is specific to a

include/trace/events/dma.h

@@ -36,7 +36,8 @@ TRACE_DEFINE_ENUM(DMA_NONE);
 		{ DMA_ATTR_DEBUGGING_IGNORE_CACHELINES, "CACHELINES_OVERLAP" }, \
 		{ DMA_ATTR_REQUIRE_COHERENT, "REQUIRE_COHERENT" }, \
 		{ DMA_ATTR_CC_SHARED, "CC_SHARED" }, \
-		{ __DMA_ATTR_ALLOC_CC_SHARED, "ALLOC_CC_SHARED" })
+		{ __DMA_ATTR_ALLOC_CC_SHARED, "ALLOC_CC_SHARED" }, \
+		{ DMA_ATTR_NO_BOUNCE, "NO_BOUNCE" })
 
 DECLARE_EVENT_CLASS(dma_map,
 	TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr,

kernel/dma/.kunitconfig

@@ -0,0 +1,2 @@
+CONFIG_KUNIT=y
+CONFIG_DMA_NO_BOUNCE_KUNIT_TEST=y

kernel/dma/Kconfig

@@ -299,3 +299,17 @@ config DMA_MAP_BENCHMARK
 	  performance of dma_(un)map_page.
 
 	  See tools/testing/selftests/dma/dma_map_benchmark.c
+
+config DMA_NO_BOUNCE_KUNIT_TEST
+	bool "KUnit test of DMA_ATTR_NO_BOUNCE" if !KUNIT_ALL_TESTS
+	depends on KUNIT=y && HAS_DMA && !ARCH_HAS_PHYS_TO_DMA
+	default KUNIT_ALL_TESTS
+	help
+	  KUnit test of the DMA_ATTR_NO_BOUNCE mapping attribute: for a test
+	  device that cannot address one page, mappings of that page are
+	  refused instead of being made through a bounce buffer. The test
+	  device's addressing is set by its dma_range_map, which architectures
+	  with a phys_to_dma() of their own do not follow. The test is built
+	  in only, since it looks at the SWIOTLB state of the device.
+
+	  If unsure, say N.

kernel/dma/Makefile

@@ -10,3 +10,4 @@ obj-$(CONFIG_SWIOTLB)			+= swiotlb.o
 obj-$(CONFIG_DMA_COHERENT_POOL)		+= pool.o
 obj-$(CONFIG_MMU)			+= remap.o
 obj-$(CONFIG_DMA_MAP_BENCHMARK)		+= map_benchmark.o
+obj-$(CONFIG_DMA_NO_BOUNCE_KUNIT_TEST)	+= no_bounce_kunit.o

kernel/dma/direct.c

@@ -680,7 +680,8 @@ dma_addr_t dma_direct_map_phys(struct device *dev, phys_addr_t phys,
 	}
 
 	if (is_swiotlb_force_bounce(dev)) {
-		if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
+		if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT |
+			     DMA_ATTR_NO_BOUNCE))
 			return DMA_MAPPING_ERROR;
 
 		return swiotlb_map(dev, phys, size, dir, attrs);
@@ -699,6 +700,12 @@ dma_addr_t dma_direct_map_phys(struct device *dev, phys_addr_t phys,
 
 	if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs)) ||
 	    dma_kmalloc_needs_bounce(dev, size, dir)) {
+		/*
+		 * Not an overflow: the caller asked for this refusal, with
+		 * or without a SWIOTLB that could have made the mapping.
+		 */
+		if (attrs & DMA_ATTR_NO_BOUNCE)
+			return DMA_MAPPING_ERROR;
 		if (is_swiotlb_active(dev) &&
 		    !(attrs & DMA_ATTR_REQUIRE_COHERENT))
 			return swiotlb_map(dev, phys, size, dir, attrs);

kernel/dma/mapping.c

@@ -174,6 +174,9 @@ dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,
 		return DMA_MAPPING_ERROR;
 	else if (use_dma_iommu(dev))
 		addr = iommu_dma_map_phys(dev, phys, size, dir, attrs);
+	else if (attrs & DMA_ATTR_NO_BOUNCE)
+		/* What the device's own operations map is not known here */
+		return DMA_MAPPING_ERROR;
 	else if (ops->map_phys)
 		addr = ops->map_phys(dev, phys, size, dir, attrs);
 
@@ -256,6 +259,9 @@ static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
 		ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
 	else if (use_dma_iommu(dev))
 		ents = iommu_dma_map_sg(dev, sg, nents, dir, attrs);
+	else if (attrs & DMA_ATTR_NO_BOUNCE)
+		/* What the device's own operations map is not known here */
+		return -EOPNOTSUPP;
 	else
 		ents = ops->map_sg(dev, sg, nents, dir, attrs);
 
@@ -327,6 +333,11 @@ EXPORT_SYMBOL(dma_map_sg_attrs);
  *			DMA_MAPPING_ERROR.
  *   -EREMOTEIO		The DMA device cannot access P2PDMA memory specified
  *			in the sg_table. This will not succeed if retried.
+ *   -EOPNOTSUPP	The device cannot be given a mapping with a property
+ *			that @attrs require: DMA_ATTR_REQUIRE_COHERENT for a
+ *			device that is not DMA coherent, DMA_ATTR_NO_BOUNCE
+ *			for a device that is mapped by DMA operations of its
+ *			own. Will not succeed if retried.
  */
 int dma_map_sgtable(struct device *dev, struct sg_table *sgt,
 		    enum dma_data_direction dir, unsigned long attrs)