diff --git a/src/asahi/lib/agx_device.c b/src/asahi/lib/agx_device.c index 1d55f45f4cb7..567e6b9ce59c 100644 --- a/src/asahi/lib/agx_device.c +++ b/src/asahi/lib/agx_device.c @@ -1097,11 +1097,9 @@ agx_open_device(void *memctx, struct agx_device *dev) * decode, or that moves this layout again, has to explain that. * * Consequence, and it is not a conservative choice but a forced one: the - * user heap ends up at 32GiB rather than 64GiB, because the read-only - * sparse alias has to fit below the kernel range and the window must stay - * a power of two. Shrinking the kernel reservation buys nothing -- the - * power-of-two step dominates. 32GiB of VA against <=16GB of RAM is not - * a practical limit. */ + * window must stay a power of two. Shrinking the kernel reservation buys + * nothing -- the power-of-two step dominates. 32GiB of VA against <=16GB + * of RAM is not a practical limit. */ if (agx_chip_native_agx3(dev->chip)) dev->shader_base = AGX_G17P_USC_WINDOW_BASE; else if (agx_chip_g17_m5(dev->chip)) @@ -1147,10 +1145,10 @@ agx_open_device(void *memctx, struct agx_device *dev) /* * Start the low heap as high as it can go rather than at the reservation. - * agx_va_split_user_window() requires size < start and the sparse alias - * needs a second window of the same size, so the usable heap is maximised - * by start = shader_base/2: that yields size = shader_base/4 with the alias - * ending exactly at shader_base. Starting at the reservation instead would + * The split-window layout that chose start = shader_base/2 (size < start, + * plus a second window for the sparse alias, ending exactly at shader_base) + * no longer applies to this window; the selection stays as shipped and the + * rounding below sets the heap. Starting at the reservation instead would * waste three quarters of the range on unused low VA. */ uint64_t low_floor = @@ -1160,9 +1158,9 @@ agx_open_device(void *memctx, struct agx_device *dev) /* Bound by the NARROWER of the two G17P pointers, not by the USC * window: the compact VDM pointer reaches 0x9000000000, but the * absolute 36-bit shr(3) USC Texture/Sampler Buffer field stops at - * 2^39. Starting halfway through that aperture is as high as the split - * allows -- it needs size < start, and the sparse alias needs a second - * window of the same size -- and lands the alias exactly on the limit. + * 2^39. Halfway through that aperture keeps every payload inside both + * fields; the split-window rationale (size < start, a second window + * for the alias) no longer binds, but the selection stays as shipped. */ user_start = MAX2(low_floor, AGX_G17P_USER_LIMIT / 2); } else { @@ -1213,7 +1211,7 @@ agx_open_device(void *memctx, struct agx_device *dev) const uint64_t user_limit = agx_chip_g17_m5(dev->chip) ? AGX_M5_DATA_END : agx_va_user_limit(dev->chip, heap_below_usc, vm_create.kernel_start); - uint64_t user_size = user_limit - user_start; + uint64_t user_size; int ret = dev->ops.simple_ioctl(dev, DRM_IOCTL_ASAHI_VM_CREATE, &vm_create); if (ret) { @@ -1235,57 +1233,28 @@ agx_open_device(void *memctx, struct agx_device *dev) for (unsigned i = 0; i < ARRAY_SIZE(dev->bo_cache.buckets); ++i) list_inithead(&dev->bo_cache.buckets[i]); - /* Round the user VA window to a power-of-two window whose top half can be - * addressed by a single bit, so sparse buffers can be aliased read-only at - * +sparse_ro_offset. This is a hard failure rather than an assert: the - * reduction depends on the kernel-reported window, so a geometry we cannot - * split must be reported, not tripped over in a debug build and silently - * miscomputed in a release one. - */ - if (!agx_va_split_user_window(user_start, user_size, &user_start, - &user_size)) { - fprintf(stderr, - "Cannot split the kernel user VA window [%#llx, %#llx) for " - "sparse aliasing\n", - (unsigned long long)dev->params.vm_start, - (unsigned long long)dev->params.vm_end); - goto fail; - } - - /* Every vkAllocateMemory() binds a read-only shadow of the allocation at - * +sparse_ro_offset, unconditionally, "in case this is used for a sparse - * resident buffer" (hk_device_memory.c). So the ALIAS window - * [user_start + user_size, user_start + 2 * user_size) is live for every - * app allocation, not just sparse ones, and the kernel rejects a bind that - * leaves the user range or touches the kernel range (file.rs gem_bind: - * "not contained in user range" / "intrudes in kernel range", both EINVAL). - * - * agx_va_split_user_window() only sizes the window against its own start; - * it never sees kernel_start, so nothing has been checking that the alias - * half still fits underneath it. At the default base it fits with three - * orders of magnitude to spare, which is why this has never fired. Move - * the USC window to 2^40 and the split yields a 1TiB user heap at - * 0x20000000000 whose alias reaches 0x40000000000 -- past the top of the - * user range and straight through the kernel's 32GiB reservation -- and - * the FIRST app allocation fails with VK_ERROR_UNKNOWN before anything is - * ever submitted. - * - * Halve until the alias fits. This is a no-op for the default layout - * (0x2000000000 + 2 * 0x1000000000 = 0x4000000000, far below a kernel - * range starting at 0x3f7fff_f8000), so it does not perturb the baseline. */ - while (user_size > 0 && user_start + 2 * user_size > user_limit) - user_size /= 2; - + /* Honeykrisp refuses sparse buffers, so no read-only alias is needed and + * the main heap spans the whole rounded kernel user window instead of + * half of it. sparse_ro_offset stays zero, which gates every alias + * consumer (the queue shadow bind, the read-only scratch half of + * hk_bind_scratch, the read-only upper mapping when binding memory to a + * sparse buffer, and the read-only address fixup, which returns the + * address unchanged at offset zero). Unbound pages of sparse-binding or + * BDA capture-replay buffers now read whatever is mapped instead of + * zeroes; sparse residency is not exposed, so no conforming app can rely + * on the old behaviour. Restore this rounding-split with the sparse + * support itself: agx_va_split_user_window() rounds the window and cuts + * it in half so a single address bit distinguishes the read-only alias. + */ + user_start = util_next_power_of_two64(user_start); + user_size = user_start < user_limit ? user_limit - user_start : 0; if (user_size == 0) { - fprintf(stderr, - "User VA window at %#llx leaves no room for the sparse " - "read-only alias below %#llx\n", - (unsigned long long)user_start, - (unsigned long long)user_limit); + fprintf(stderr, "User VA window [%#llx, %#llx) leaves no heap\n", + (unsigned long long)user_start, (unsigned long long)user_limit); goto fail; } - dev->sparse_ro_offset = user_size; + dev->sparse_ro_offset = 0; dev->main_heap_base = user_start; dev->main_heap_size = user_size; diff --git a/src/asahi/lib/agx_va.c b/src/asahi/lib/agx_va.c index 1554c54787fc..6d7722207987 100644 --- a/src/asahi/lib/agx_va.c +++ b/src/asahi/lib/agx_va.c @@ -77,7 +77,9 @@ agx_va_free(struct agx_device *dev, struct agx_va *va, bool unbind) if (unbind) { /* Sparse/capture-replay VAs may contain a lower read-write mapping and * its upper read-only shadow. Clear both, including unbound holes, in - * one VM_BIND transaction so neither half can survive VA reuse. */ + * one VM_BIND transaction so neither half can survive VA reuse. With + * sparse_ro_offset unset the shadow is the same range, so send one + * UNBIND rather than a duplicate. */ assert(!(va->flags & AGX_VA_USC)); assert(va->size_B >= dev->guard_size); const uint64_t bound_size_B = va->size_B - dev->guard_size; @@ -95,7 +97,7 @@ agx_va_free(struct agx_device *dev, struct agx_va *va, bool unbind) }; /* Do not recycle an address if either stale mapping could remain. */ - if (dev->ops.bo_bind(dev, ops, ARRAY_SIZE(ops))) + if (dev->ops.bo_bind(dev, ops, dev->sparse_ro_offset ? 2 : 1)) return; } diff --git a/src/asahi/vulkan/hk_buffer.c b/src/asahi/vulkan/hk_buffer.c index a477a3d7089d..750fcec6aa56 100644 --- a/src/asahi/vulkan/hk_buffer.c +++ b/src/asahi/vulkan/hk_buffer.c @@ -138,11 +138,16 @@ hk_bind_scratch(struct hk_device *dev, struct agx_va *va, unsigned offset_B, if (ret) return errno == ENOMEM ? VK_ERROR_OUT_OF_DEVICE_MEMORY : VK_ERROR_UNKNOWN; - /* Map read-only scratch to the secondary (top half) VA range */ - ret = agx_bo_bind(&dev->dev, dev->dev.zero_bo, - addr + dev->dev.sparse_ro_offset, size_B, 0, flags); - if (ret) - return errno == ENOMEM ? VK_ERROR_OUT_OF_DEVICE_MEMORY : VK_ERROR_UNKNOWN; + /* Map read-only scratch to the secondary (top half) VA range. Skipped + * while sparse is unsupported: sparse_ro_offset == 0 means the whole + * window is the primary heap and there is no secondary range. */ + if (dev->dev.sparse_ro_offset) { + ret = agx_bo_bind(&dev->dev, dev->dev.zero_bo, + addr + dev->dev.sparse_ro_offset, size_B, 0, flags); + if (ret) + return errno == ENOMEM ? VK_ERROR_OUT_OF_DEVICE_MEMORY + : VK_ERROR_UNKNOWN; + } return VK_SUCCESS; } @@ -184,6 +189,13 @@ hk_CreateBuffer(VkDevice device, const VkBufferCreateInfo *pCreateInfo, if (pCreateInfo->size > HK_MAX_BUFFER_SIZE) return vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY); + if ((pCreateInfo->flags & + (VK_BUFFER_CREATE_SPARSE_BINDING_BIT | + VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT | + VK_BUFFER_CREATE_SPARSE_ALIASED_BIT)) && + !hk_device_physical(dev)->vk.supported_features.sparseBinding) + return vk_error(dev, VK_ERROR_FEATURE_NOT_PRESENT); + buffer = vk_buffer_create(&dev->vk, pCreateInfo, pAllocator, sizeof(*buffer)); if (!buffer) @@ -455,13 +467,16 @@ hk_BindBufferMemory2(VkDevice device, uint32_t bindInfoCount, if (ret) return VK_ERROR_UNKNOWN; - /* Upper mapping: read-only */ - ret = - agx_bo_bind(&dev->dev, mem->bo, - buffer->vk.device_address + dev->dev.sparse_ro_offset, - size, pBindInfos[i].memoryOffset, DRM_ASAHI_BIND_READ); - if (ret) - return VK_ERROR_UNKNOWN; + /* Upper mapping: read-only, only while the sparse alias exists. */ + if (dev->dev.sparse_ro_offset) { + ret = agx_bo_bind(&dev->dev, mem->bo, + buffer->vk.device_address + + dev->dev.sparse_ro_offset, + size, pBindInfos[i].memoryOffset, + DRM_ASAHI_BIND_READ); + if (ret) + return VK_ERROR_UNKNOWN; + } } else { assert(buffer->vk.device_address == 0); buffer->vk.device_address = diff --git a/src/asahi/vulkan/hk_device_memory.c b/src/asahi/vulkan/hk_device_memory.c index 0de931119c28..73c543cf8187 100644 --- a/src/asahi/vulkan/hk_device_memory.c +++ b/src/asahi/vulkan/hk_device_memory.c @@ -313,16 +313,21 @@ hk_AllocateMemory(VkDevice device, const VkMemoryAllocateInfo *pAllocateInfo, : mem->bo->va->addr; } - /* Shadow map in case this is used for a sparse resident buffer */ - int ret = agx_bo_bind(&dev->dev, mem->bo, - agx_rw_addr_to_ro(&dev->dev, mem->bo->va->addr), - mem->bo->size, 0, DRM_ASAHI_BIND_READ); - if (ret) { - /* The bind fails once the VM carries a sticky error. */ - agx_bo_unreference(&dev->dev, mem->bo); - mem->bo = NULL; - vk_device_memory_destroy(&dev->vk, pAllocator, &mem->vk); - return vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY); + /* Shadow map in case this is used for a sparse resident buffer. Skipped + * while sparse is unsupported: with sparse_ro_offset == 0 the shadow + * address equals the primary mapping, so binding it twice only doubles + * the per-allocation bind cost and halves the usable heap budget. */ + if (dev->dev.sparse_ro_offset) { + int ret = agx_bo_bind(&dev->dev, mem->bo, + agx_rw_addr_to_ro(&dev->dev, mem->bo->va->addr), + mem->bo->size, 0, DRM_ASAHI_BIND_READ); + if (ret) { + /* The bind fails once the VM carries a sticky error. */ + agx_bo_unreference(&dev->dev, mem->bo); + mem->bo = NULL; + vk_device_memory_destroy(&dev->vk, pAllocator, &mem->vk); + return vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY); + } } if (mem->bo->flags & (AGX_BO_SHAREABLE | AGX_BO_SHARED)) diff --git a/src/asahi/vulkan/hk_image.c b/src/asahi/vulkan/hk_image.c index 85cb97e1ab94..ec3aad73fee4 100644 --- a/src/asahi/vulkan/hk_image.c +++ b/src/asahi/vulkan/hk_image.c @@ -1771,6 +1771,13 @@ hk_CreateImage(VkDevice _device, const VkImageCreateInfo *pCreateInfo, return wsi_common_create_swapchain_image(&pdev->wsi_device, pCreateInfo, pImage); + if ((pCreateInfo->flags & + (VK_IMAGE_CREATE_SPARSE_BINDING_BIT | + VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT | + VK_IMAGE_CREATE_SPARSE_ALIASED_BIT)) && + !pdev->vk.supported_features.sparseBinding) + return vk_error(dev, VK_ERROR_FEATURE_NOT_PRESENT); + image = vk_zalloc2(&dev->vk.alloc, pAllocator, sizeof(*image), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); if (!image) diff --git a/src/asahi/vulkan/hk_physical_device.c b/src/asahi/vulkan/hk_physical_device.c index ac48a0805b15..b544e543ff49 100644 --- a/src/asahi/vulkan/hk_physical_device.c +++ b/src/asahi/vulkan/hk_physical_device.c @@ -327,6 +327,11 @@ hk_get_device_features( const struct vk_device_extension_table *supported_extensions, struct vk_features *features) { + /* The read-only sparse alias exists only while sparse_ro_offset is set. + * Without it an unbound sparse page maps the shared writable scratch BO, + * so non-resident-strict reads cannot hold; do not advertise sparse. */ + const bool sparse_usable = dev->sparse_ro_offset != 0; + *features = (struct vk_features){ /* Vulkan 1.0 */ .robustBufferAccess = true, @@ -384,9 +389,9 @@ hk_get_device_features( .shaderFloat64 = false, .shaderInt64 = true, .shaderInt16 = true, - .shaderResourceResidency = true, + .shaderResourceResidency = sparse_usable, .shaderResourceMinLod = true, - .sparseBinding = true, + .sparseBinding = sparse_usable, /* We probably could advertise multisampled sparse but we don't have a use * case yet and it isn't trivial. @@ -394,9 +399,9 @@ hk_get_device_features( .sparseResidency2Samples = false, .sparseResidency4Samples = false, .sparseResidency8Samples = false, - .sparseResidencyAliased = true, - .sparseResidencyImage2D = true, - .sparseResidencyBuffer = true, + .sparseResidencyAliased = sparse_usable, + .sparseResidencyImage2D = sparse_usable, + .sparseResidencyBuffer = sparse_usable, /* This needs investigation. */ .sparseResidencyImage3D = false, @@ -933,7 +938,8 @@ hk_get_device_features( * window is split for the RO alias exactly as AGX2 is. Sparse texture * updates flush the SSTT with a HAL200 CDM record, so other native * chips do not advertise it. */ - const bool sparse = dev->chip == AGX_CHIP_G17P; + const bool sparse = dev->chip == AGX_CHIP_G17P && + dev->sparse_ro_offset != 0; features->shaderResourceResidency = sparse; features->sparseBinding = sparse; features->sparseResidencyAliased = false; @@ -1300,7 +1306,7 @@ hk_get_device_properties(const struct agx_device *dev, .nonCoherentAtomSize = 64, /* Vulkan 1.0 sparse properties */ - .sparseResidencyNonResidentStrict = true, + .sparseResidencyNonResidentStrict = dev->sparse_ro_offset != 0, .sparseResidencyAlignedMipSize = false, .sparseResidencyStandard2DBlockShape = true, @@ -2097,8 +2103,9 @@ hk_create_drm_physical_device(struct vk_instance *_instance, assert(pdev->mem_type_count <= ARRAY_SIZE(pdev->mem_types)); VkQueueFlags queue_flags = - VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_TRANSFER_BIT | - VK_QUEUE_SPARSE_BINDING_BIT; + VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT | VK_QUEUE_TRANSFER_BIT; + if (pdev->vk.supported_features.sparseBinding) + queue_flags |= VK_QUEUE_SPARSE_BINDING_BIT; if (agx_native_graphics(&pdev->dev)) { /* Graphics, compute and transfers use the native HAL200 queue path. * Sparse binding rides the shared per-page VM_BIND path; advertise it diff --git a/src/asahi/vulkan/hk_queue.c b/src/asahi/vulkan/hk_queue.c index 4c7eb7eeee4f..74c7aba2ecbf 100644 --- a/src/asahi/vulkan/hk_queue.c +++ b/src/asahi/vulkan/hk_queue.c @@ -701,16 +701,20 @@ hk_flush_bind(struct hk_bind_builder *b) util_dynarray_append(&b->binds, op); - /* Shadow a read-only mapping to the upper half */ - op.flags &= ~DRM_ASAHI_BIND_WRITE; - op.addr = agx_rw_addr_to_ro(&b->dev->dev, op.addr); + /* Shadow a read-only mapping to the upper half. Skipped while sparse is + * unsupported: sparse_ro_offset == 0 makes the shadow address equal the + * primary mapping, so the duplicate bind buys nothing. */ + if (b->dev->dev.sparse_ro_offset) { + op.flags &= ~DRM_ASAHI_BIND_WRITE; + op.addr = agx_rw_addr_to_ro(&b->dev->dev, op.addr); + + if (!b->mem) { + op.handle = b->dev->dev.zero_bo->uapi_handle; + } - if (!b->mem) { - op.handle = b->dev->dev.zero_bo->uapi_handle; + util_dynarray_append(&b->binds, op); } - util_dynarray_append(&b->binds, op); - return VK_SUCCESS; }