diff --git a/Documentation/core-api/dma-attributes.rst b/Documentation/core-api/dma-attributes.rst index eee743184acde..3f38be833663e 100644 --- a/Documentation/core-api/dma-attributes.rst +++ b/Documentation/core-api/dma-attributes.rst @@ -208,3 +208,50 @@ __DMA_ATTR_ALLOC_CC_SHARED differs from DMA_ATTR_CC_SHARED in that it is not a caller-visible DMA API attribute. DMA_ATTR_CC_SHARED describes an already-shared mapping and requires the caller to have prepared normal system memory before mapping it. + +DMA_ATTR_NO_BOUNCE +------------------ + +A mapping request with DMA_ATTR_NO_BOUNCE fails where the DMA core could +only satisfy it through a bounce buffer (SWIOTLB): a successful mapping +gives the device the memory that was passed in, never a copy of it. This +is for callers whose device and CPU must see the same memory for as long +as the mapping exists, with no synchronization points in between, such as +long-lived mappings of pinned userspace memory. + +The request fails for memory that the device cannot address, where +bouncing is forced (swiotlb=force, a restricted DMA pool, a device that +needs unencrypted memory), for a kmalloc() buffer that is too small to be +aligned for DMA to a device that is not DMA coherent, and for a buffer +that is not aligned to the IOMMU granule of an untrusted device. It also +fails for a device that is mapped by DMA operations of its own (struct +dma_map_ops), since the DMA core cannot tell what those map onto. + +A refused request logs nothing, whether or not the system has a SWIOTLB +that could have made the mapping: a driver may pass on requests of +unprivileged callers for memory it knows nothing about. What each mapping +function returns for it: + +- dma_map_page_attrs(), dma_map_single_attrs() and dma_map_phys(): + DMA_MAPPING_ERROR, to be tested with dma_mapping_error(). +- dma_map_sg_attrs(): 0. +- dma_map_sgtable(): -EIO where a bounce buffer would have been needed, + -EOPNOTSUPP for a device with DMA operations of its own. -EIO is the + error of any segment that cannot be mapped, so it does not tell a + refused bounce buffer from another mapping failure. Segments mapped + before the refused one are unmapped again. +- dma_iova_link(): -EPERM. + +DMA_ATTR_NO_BOUNCE differs from DMA_ATTR_REQUIRE_COHERENT in that it does +not require the device to be DMA coherent and does not change CPU cache +maintenance: caches are synchronized at map, sync and unmap time as for +any other mapping, unless DMA_ATTR_SKIP_CPU_SYNC is given as well. It +says nothing about whether the device is coherent with the CPU caches. + +The attribute is a property of a mapping request and nothing else looks +at it. The allocation functions (dma_alloc_attrs(), dma_alloc_pages(), +dma_alloc_noncontiguous()), dma_mmap_attrs() and dma_get_sgtable_attrs() +ignore it. So do the unmap and sync functions: no bounce buffer stands +behind a mapping that was made with it. A DMA_ATTR_MMIO mapping +(dma_map_resource()) is never bounced, and the attribute changes nothing +for it on a device that the DMA core maps itself. diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 82eccd24f0a2a..a6455b92d4eed 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1235,7 +1235,7 @@ dma_addr_t iommu_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, */ if (!(attrs & DMA_ATTR_MMIO) && dev_use_swiotlb(dev, size, dir) && iova_unaligned(iovad, phys, size)) { - if (attrs & DMA_ATTR_REQUIRE_COHERENT) + if (attrs & (DMA_ATTR_REQUIRE_COHERENT | DMA_ATTR_NO_BOUNCE)) return DMA_MAPPING_ERROR; phys = iommu_dma_map_swiotlb(dev, phys, size, dir, attrs); @@ -2003,7 +2003,8 @@ int dma_iova_link(struct device *dev, struct dma_iova_state *state, if (dev_use_swiotlb(dev, size, dir) && iova_unaligned(iovad, phys, size)) { - if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) + if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT | + DMA_ATTR_NO_BOUNCE)) return -EPERM; return iommu_dma_iova_link_swiotlb(dev, state, phys, offset, diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index a3e880649fa41..6741e754ffd57 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -111,6 +111,18 @@ */ #define __DMA_ATTR_ALLOC_CC_SHARED (1UL << 14) +/* + * DMA_ATTR_NO_BOUNCE: Indicates that the device must be given the memory that + * was passed in itself. A mapping request that could only be satisfied through + * a bounce buffer (SWIOTLB) fails instead, as does one for a device that is + * mapped by DMA operations of its own, where the DMA core cannot tell. Unlike + * DMA_ATTR_REQUIRE_COHERENT, it places no requirement on the device's cache + * coherence and leaves CPU cache maintenance as it is. Only the mapping + * functions look at it: allocations, dma_mmap_attrs() and + * dma_get_sgtable_attrs() ignore it. + */ +#define DMA_ATTR_NO_BOUNCE (1UL << 15) + /* * A dma_addr_t can hold any valid DMA or bus address for the platform. It can * be given to a device to use as a DMA source or target. It is specific to a diff --git a/include/trace/events/dma.h b/include/trace/events/dma.h index b06d8f99922de..db8e4f8739114 100644 --- a/include/trace/events/dma.h +++ b/include/trace/events/dma.h @@ -36,7 +36,8 @@ TRACE_DEFINE_ENUM(DMA_NONE); { DMA_ATTR_DEBUGGING_IGNORE_CACHELINES, "CACHELINES_OVERLAP" }, \ { DMA_ATTR_REQUIRE_COHERENT, "REQUIRE_COHERENT" }, \ { DMA_ATTR_CC_SHARED, "CC_SHARED" }, \ - { __DMA_ATTR_ALLOC_CC_SHARED, "ALLOC_CC_SHARED" }) + { __DMA_ATTR_ALLOC_CC_SHARED, "ALLOC_CC_SHARED" }, \ + { DMA_ATTR_NO_BOUNCE, "NO_BOUNCE" }) DECLARE_EVENT_CLASS(dma_map, TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr, diff --git a/kernel/dma/.kunitconfig b/kernel/dma/.kunitconfig new file mode 100644 index 0000000000000..59038070164b7 --- /dev/null +++ b/kernel/dma/.kunitconfig @@ -0,0 +1,2 @@ +CONFIG_KUNIT=y +CONFIG_DMA_NO_BOUNCE_KUNIT_TEST=y diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig index 3830a63ae032d..9618359bb3909 100644 --- a/kernel/dma/Kconfig +++ b/kernel/dma/Kconfig @@ -299,3 +299,17 @@ config DMA_MAP_BENCHMARK performance of dma_(un)map_page. See tools/testing/selftests/dma/dma_map_benchmark.c + +config DMA_NO_BOUNCE_KUNIT_TEST + bool "KUnit test of DMA_ATTR_NO_BOUNCE" if !KUNIT_ALL_TESTS + depends on KUNIT=y && HAS_DMA && !ARCH_HAS_PHYS_TO_DMA + default KUNIT_ALL_TESTS + help + KUnit test of the DMA_ATTR_NO_BOUNCE mapping attribute: for a test + device that cannot address one page, mappings of that page are + refused instead of being made through a bounce buffer. The test + device's addressing is set by its dma_range_map, which architectures + with a phys_to_dma() of their own do not follow. The test is built + in only, since it looks at the SWIOTLB state of the device. + + If unsure, say N. diff --git a/kernel/dma/Makefile b/kernel/dma/Makefile index 6977033444a37..1a892e256d659 100644 --- a/kernel/dma/Makefile +++ b/kernel/dma/Makefile @@ -10,3 +10,4 @@ obj-$(CONFIG_SWIOTLB) += swiotlb.o obj-$(CONFIG_DMA_COHERENT_POOL) += pool.o obj-$(CONFIG_MMU) += remap.o obj-$(CONFIG_DMA_MAP_BENCHMARK) += map_benchmark.o +obj-$(CONFIG_DMA_NO_BOUNCE_KUNIT_TEST) += no_bounce_kunit.o diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index da665ca22d5c0..a9d7da83a618b 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -680,7 +680,8 @@ dma_addr_t dma_direct_map_phys(struct device *dev, phys_addr_t phys, } if (is_swiotlb_force_bounce(dev)) { - if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) + if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT | + DMA_ATTR_NO_BOUNCE)) return DMA_MAPPING_ERROR; return swiotlb_map(dev, phys, size, dir, attrs); @@ -699,6 +700,12 @@ dma_addr_t dma_direct_map_phys(struct device *dev, phys_addr_t phys, if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs)) || dma_kmalloc_needs_bounce(dev, size, dir)) { + /* + * Not an overflow: the caller asked for this refusal, with + * or without a SWIOTLB that could have made the mapping. + */ + if (attrs & DMA_ATTR_NO_BOUNCE) + return DMA_MAPPING_ERROR; if (is_swiotlb_active(dev) && !(attrs & DMA_ATTR_REQUIRE_COHERENT)) return swiotlb_map(dev, phys, size, dir, attrs); diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index bf2651a70b7c2..18be108a8c906 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -174,6 +174,9 @@ dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, return DMA_MAPPING_ERROR; else if (use_dma_iommu(dev)) addr = iommu_dma_map_phys(dev, phys, size, dir, attrs); + else if (attrs & DMA_ATTR_NO_BOUNCE) + /* What the device's own operations map is not known here */ + return DMA_MAPPING_ERROR; else if (ops->map_phys) addr = ops->map_phys(dev, phys, size, dir, attrs); @@ -256,6 +259,9 @@ static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); else if (use_dma_iommu(dev)) ents = iommu_dma_map_sg(dev, sg, nents, dir, attrs); + else if (attrs & DMA_ATTR_NO_BOUNCE) + /* What the device's own operations map is not known here */ + return -EOPNOTSUPP; else ents = ops->map_sg(dev, sg, nents, dir, attrs); @@ -327,6 +333,11 @@ EXPORT_SYMBOL(dma_map_sg_attrs); * DMA_MAPPING_ERROR. * -EREMOTEIO The DMA device cannot access P2PDMA memory specified * in the sg_table. This will not succeed if retried. + * -EOPNOTSUPP The device cannot be given a mapping with a property + * that @attrs require: DMA_ATTR_REQUIRE_COHERENT for a + * device that is not DMA coherent, DMA_ATTR_NO_BOUNCE + * for a device that is mapped by DMA operations of its + * own. Will not succeed if retried. */ int dma_map_sgtable(struct device *dev, struct sg_table *sgt, enum dma_data_direction dir, unsigned long attrs) diff --git a/kernel/dma/no_bounce_kunit.c b/kernel/dma/no_bounce_kunit.c new file mode 100644 index 0000000000000..4f63efe3bac9f --- /dev/null +++ b/kernel/dma/no_bounce_kunit.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test of DMA_ATTR_NO_BOUNCE on the dma-direct path. + * + * The test device's bus reaches all memory, at its CPU address, except one + * page (a dma_range_map with a hole). The DMA layer can serve the device + * that page only through a bounce buffer. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct no_bounce { + struct device *dev; + /* A page the device addresses, and the one it does not. */ + struct page *near, *hidden; +}; + +static void no_bounce_free_page(void *page) +{ + __free_page(page); +} + +static struct page *no_bounce_page(struct kunit *test) +{ + struct page *page = alloc_page(GFP_KERNEL | __GFP_ZERO); + + KUNIT_ASSERT_NOT_NULL(test, page); + KUNIT_ASSERT_EQ(test, 0, + kunit_add_action_or_reset(test, no_bounce_free_page, page)); + return page; +} + +/* The DMA address of @page for the test device, as dma-direct makes it. */ +static dma_addr_t no_bounce_dma(struct no_bounce *nb, struct page *page) +{ + return phys_to_dma(nb->dev, page_to_phys(page)); +} + +static int no_bounce_init(struct kunit *test) +{ + struct no_bounce *nb = kunit_kzalloc(test, sizeof(*nb), GFP_KERNEL); + struct bus_dma_region *map; + phys_addr_t start, end; + dma_addr_t near; + + KUNIT_ASSERT_NOT_NULL(test, nb); + nb->near = no_bounce_page(test); + nb->hidden = no_bounce_page(test); + start = page_to_phys(nb->hidden); + end = start + PAGE_SIZE; + + nb->dev = kunit_device_register(test, "dma_no_bounce"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, nb->dev); + nb->dev->coherent_dma_mask = DMA_BIT_MASK(64); + nb->dev->dma_mask = &nb->dev->coherent_dma_mask; + + /* The driver core frees the map with the device. */ + map = kcalloc(3, sizeof(*map), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, map); + map[0] = (struct bus_dma_region){ + .cpu_start = end, .dma_start = end, .size = U64_MAX - end, + }; + map[1] = (struct bus_dma_region){ + .cpu_start = 0, .dma_start = 0, .size = start, + }; + nb->dev->dma_range_map = map; + test->priv = nb; + + /* + * The test needs dma-direct to map the device, the range map to decide + * what the device reaches, and nothing else to make a mapping bounce. + */ + if (get_dma_ops(nb->dev)) + kunit_skip(test, "the device is mapped by DMA operations"); + if (is_swiotlb_force_bounce(nb->dev)) + kunit_skip(test, "bouncing is forced for the device"); + if (force_dma_unencrypted(nb->dev)) + kunit_skip(test, "the device needs unencrypted memory, which bounces"); + near = no_bounce_dma(nb, nb->near); + if (!dma_capable(nb->dev, near, PAGE_SIZE, true, 0) || + dma_to_phys(nb->dev, near) != page_to_phys(nb->near) || + dma_capable(nb->dev, no_bounce_dma(nb, nb->hidden), PAGE_SIZE, + true, 0)) + kunit_skip(test, "phys_to_dma() does not follow the range map"); + return 0; +} + +/* + * Maps the page the device does not address with the attribute: true if the + * request is refused. A mapping made against that is undone. + */ +static bool no_bounce_hidden_refused(struct no_bounce *nb) +{ + dma_addr_t addr = dma_map_page_attrs(nb->dev, nb->hidden, 0, PAGE_SIZE, + DMA_TO_DEVICE, DMA_ATTR_NO_BOUNCE); + + if (dma_mapping_error(nb->dev, addr)) + return true; + dma_unmap_page_attrs(nb->dev, addr, PAGE_SIZE, DMA_TO_DEVICE, + DMA_ATTR_NO_BOUNCE); + return false; +} + +/* + * Left to itself the DMA layer bounces the page the device cannot address; + * with the attribute it fails the mapping, and maps a page the device does + * address as that page. + */ +static void no_bounce_refuses_a_bounce_buffer(struct kunit *test) +{ + struct no_bounce *nb = test->priv; + phys_addr_t start = page_to_phys(nb->hidden); + u8 *hidden = page_address(nb->hidden); + phys_addr_t bounce; + dma_addr_t addr; + + if (!is_swiotlb_active(nb->dev)) + kunit_skip(test, "no SWIOTLB to show the bounce buffer with"); + + /* + * An ordinary mapping lies outside the page, holds what the page held + * when it was mapped and does not follow a later store to the page. + */ + hidden[0] = 0xa5; + addr = dma_map_page_attrs(nb->dev, nb->hidden, 0, PAGE_SIZE, + DMA_TO_DEVICE, 0); + KUNIT_ASSERT_FALSE(test, dma_mapping_error(nb->dev, addr)); + bounce = dma_to_phys(nb->dev, addr); + KUNIT_EXPECT_NE(test, (phys_addr_t)-1, bounce); + KUNIT_EXPECT_TRUE(test, bounce < start || bounce >= start + PAGE_SIZE); + if (bounce != (phys_addr_t)-1) { + KUNIT_EXPECT_EQ(test, 0xa5, *(u8 *)phys_to_virt(bounce)); + hidden[0] = 0x5a; + KUNIT_EXPECT_EQ(test, 0xa5, *(u8 *)phys_to_virt(bounce)); + } + dma_unmap_page_attrs(nb->dev, addr, PAGE_SIZE, DMA_TO_DEVICE, 0); + + KUNIT_EXPECT_TRUE(test, no_bounce_hidden_refused(nb)); + + addr = dma_map_page_attrs(nb->dev, nb->near, 0, PAGE_SIZE, + DMA_TO_DEVICE, DMA_ATTR_NO_BOUNCE); + KUNIT_ASSERT_FALSE(test, dma_mapping_error(nb->dev, addr)); + KUNIT_EXPECT_EQ(test, no_bounce_dma(nb, nb->near), addr); + dma_unmap_page_attrs(nb->dev, addr, PAGE_SIZE, DMA_TO_DEVICE, + DMA_ATTR_NO_BOUNCE); +} + +/* + * A scatterlist fails as a whole when one of its segments would be bounced, + * also when a segment before it was mapped already. + */ +static void no_bounce_fails_the_whole_scatterlist(struct kunit *test) +{ + struct no_bounce *nb = test->priv; + struct scatterlist sgl[2]; + struct sg_table sgt = { .sgl = sgl, .nents = 2, .orig_nents = 2 }; + int ret; + + sg_init_table(sgl, 2); + sg_set_page(&sgl[0], nb->near, PAGE_SIZE, 0); + sg_set_page(&sgl[1], nb->hidden, PAGE_SIZE, 0); + ret = dma_map_sgtable(nb->dev, &sgt, DMA_TO_DEVICE, DMA_ATTR_NO_BOUNCE); + if (!ret) + dma_unmap_sgtable(nb->dev, &sgt, DMA_TO_DEVICE, + DMA_ATTR_NO_BOUNCE); + KUNIT_EXPECT_EQ(test, -EIO, ret); + ret = dma_map_sg_attrs(nb->dev, sgl, 2, DMA_TO_DEVICE, + DMA_ATTR_NO_BOUNCE); + if (ret) + dma_unmap_sg_attrs(nb->dev, sgl, 2, DMA_TO_DEVICE, + DMA_ATTR_NO_BOUNCE); + KUNIT_EXPECT_EQ(test, 0, ret); + + /* The segment the device addresses, alone, is mapped as itself. */ + sgt.nents = 1; + sgt.orig_nents = 1; + sg_init_table(sgl, 1); + sg_set_page(&sgl[0], nb->near, PAGE_SIZE, 0); + KUNIT_ASSERT_EQ(test, 0, + dma_map_sgtable(nb->dev, &sgt, DMA_TO_DEVICE, + DMA_ATTR_NO_BOUNCE)); + KUNIT_EXPECT_EQ(test, no_bounce_dma(nb, nb->near), sg_dma_address(sgl)); + dma_unmap_sgtable(nb->dev, &sgt, DMA_TO_DEVICE, DMA_ATTR_NO_BOUNCE); +} + +/* + * A device without SWIOTLB: a request without the attribute for the page it + * cannot address is an address overflow, with a warning. With the attribute + * it is a refusal like the others and must not warn. + */ +static void no_bounce_refusal_does_not_warn(struct kunit *test) +{ + struct no_bounce *nb = test->priv; + dma_addr_t addr; + +#ifdef CONFIG_SWIOTLB + nb->dev->dma_io_tlb_mem = NULL; +#endif + KUNIT_ASSERT_FALSE(test, is_swiotlb_active(nb->dev)); + if (test_taint(TAINT_WARN)) + kunit_skip(test, "the kernel has warned before"); + + KUNIT_EXPECT_TRUE(test, no_bounce_hidden_refused(nb)); + KUNIT_EXPECT_FALSE(test, test_taint(TAINT_WARN)); + + addr = dma_map_page_attrs(nb->dev, nb->near, 0, PAGE_SIZE, + DMA_TO_DEVICE, DMA_ATTR_NO_BOUNCE); + KUNIT_ASSERT_FALSE(test, dma_mapping_error(nb->dev, addr)); + KUNIT_EXPECT_EQ(test, no_bounce_dma(nb, nb->near), addr); + dma_unmap_page_attrs(nb->dev, addr, PAGE_SIZE, DMA_TO_DEVICE, + DMA_ATTR_NO_BOUNCE); +} + +static struct kunit_case no_bounce_cases[] = { + KUNIT_CASE(no_bounce_refuses_a_bounce_buffer), + KUNIT_CASE(no_bounce_fails_the_whole_scatterlist), + KUNIT_CASE(no_bounce_refusal_does_not_warn), + {} +}; + +static struct kunit_suite no_bounce_suite = { + .name = "dma_attr_no_bounce", + .init = no_bounce_init, + .test_cases = no_bounce_cases, +}; +kunit_test_suite(no_bounce_suite); + +MODULE_DESCRIPTION("KUnit test of DMA_ATTR_NO_BOUNCE"); +MODULE_LICENSE("GPL");