From f9db4921e0cdabfc268026476350fcfc69696206 Mon Sep 17 00:00:00 2001 From: Philip Rebohle Date: Sat, 20 Aug 2022 21:57:46 +0200 Subject: [PATCH] [dxvk] Implement sparse memory allocator --- src/dxvk/dxvk_device.cpp | 5 ++ src/dxvk/dxvk_device.h | 7 ++ src/dxvk/dxvk_memory.cpp | 93 +++++++++++++++++++++--- src/dxvk/dxvk_memory.h | 24 +++++-- src/dxvk/dxvk_sparse.cpp | 152 +++++++++++++++++++++++++++++++++++++++ src/dxvk/dxvk_sparse.h | 149 +++++++++++++++++++++++++++++++++++++- 6 files changed, 415 insertions(+), 15 deletions(-) diff --git a/src/dxvk/dxvk_device.cpp b/src/dxvk/dxvk_device.cpp index 3608a66ce..4820cf778 100644 --- a/src/dxvk/dxvk_device.cpp +++ b/src/dxvk/dxvk_device.cpp @@ -216,6 +216,11 @@ namespace dxvk { } + Rc DxvkDevice::createSparsePageAllocator() { + return new DxvkSparsePageAllocator(this, m_objects.memoryManager()); + } + + DxvkStatCounters DxvkDevice::getStatCounters() { DxvkPipelineCount pipe = m_objects.pipelineManager().getPipelineCount(); diff --git a/src/dxvk/dxvk_device.h b/src/dxvk/dxvk_device.h index af89f1a17..9ee2744a9 100644 --- a/src/dxvk/dxvk_device.h +++ b/src/dxvk/dxvk_device.h @@ -22,6 +22,7 @@ #include "dxvk_renderpass.h" #include "dxvk_sampler.h" #include "dxvk_shader.h" +#include "dxvk_sparse.h" #include "dxvk_stats.h" #include "dxvk_unbound.h" #include "dxvk_marker.h" @@ -380,6 +381,12 @@ namespace dxvk { Rc createSampler( const DxvkSamplerCreateInfo& createInfo); + /** + * \brief Creates a sparse page allocator + * \returns Sparse page allocator + */ + Rc createSparsePageAllocator(); + /** * \brief Retrieves stat counters * diff --git a/src/dxvk/dxvk_memory.cpp b/src/dxvk/dxvk_memory.cpp index f0dfa6696..135bf9991 100644 --- a/src/dxvk/dxvk_memory.cpp +++ b/src/dxvk/dxvk_memory.cpp @@ -179,9 +179,8 @@ namespace dxvk { } - DxvkMemoryAllocator::DxvkMemoryAllocator(const DxvkDevice* device) - : m_vkd (device->vkd()), - m_device (device), + DxvkMemoryAllocator::DxvkMemoryAllocator(DxvkDevice* device) + : m_device (device), m_devProps (device->adapter()->deviceProperties()), m_memProps (device->adapter()->memoryProperties()) { for (uint32_t i = 0; i < m_memProps.memoryHeapCount; i++) { @@ -202,6 +201,9 @@ namespace dxvk { m_memTypes[i].memType = m_memProps.memoryTypes[i]; m_memTypes[i].memTypeId = i; } + + if (device->features().core.features.sparseBinding) + m_sparseMemoryTypes = determineSparseMemoryTypes(device); } @@ -366,6 +368,8 @@ namespace dxvk { VkDeviceSize size, DxvkMemoryProperties info, DxvkMemoryFlags hints) { + auto vk = m_device->vkd(); + bool useMemoryPriority = (info.flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) && (m_device->features().extMemoryPriority.memoryPriority); @@ -403,15 +407,15 @@ namespace dxvk { if (useMemoryPriority) priorityInfo.pNext = std::exchange(memoryInfo.pNext, &priorityInfo); - if (m_vkd->vkAllocateMemory(m_vkd->device(), &memoryInfo, nullptr, &result.memHandle) != VK_SUCCESS) + if (vk->vkAllocateMemory(vk->device(), &memoryInfo, nullptr, &result.memHandle)) return DxvkDeviceMemory(); if (info.flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) { - VkResult status = m_vkd->vkMapMemory(m_vkd->device(), result.memHandle, 0, VK_WHOLE_SIZE, 0, &result.memPointer); + VkResult status = vk->vkMapMemory(vk->device(), result.memHandle, 0, VK_WHOLE_SIZE, 0, &result.memPointer); - if (status != VK_SUCCESS) { + if (status) { Logger::err(str::format("DxvkMemoryAllocator: Mapping memory failed with ", status)); - m_vkd->vkFreeMemory(m_vkd->device(), result.memHandle, nullptr); + vk->vkFreeMemory(vk->device(), result.memHandle, nullptr); return DxvkDeviceMemory(); } } @@ -467,7 +471,9 @@ namespace dxvk { void DxvkMemoryAllocator::freeDeviceMemory( DxvkMemoryType* type, DxvkDeviceMemory memory) { - m_vkd->vkFreeMemory(m_vkd->device(), memory.memHandle, nullptr); + auto vk = m_device->vkd(); + vk->vkFreeMemory(vk->device(), memory.memHandle, nullptr); + type->heap->stats.memoryAllocated -= memory.memSize; m_device->adapter()->notifyHeapMemoryFree(type->heapId, memory.memSize); } @@ -542,4 +548,75 @@ namespace dxvk { } } + + uint32_t DxvkMemoryAllocator::determineSparseMemoryTypes( + DxvkDevice* device) const { + auto vk = device->vkd(); + + VkMemoryRequirements requirements = { }; + uint32_t typeMask = ~0u; + + // Create sparse dummy buffer to find available memory types + VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; + bufferInfo.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT + | VK_BUFFER_CREATE_SPARSE_ALIASED_BIT + | VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT; + bufferInfo.size = 65536; + bufferInfo.usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT + | VK_BUFFER_USAGE_INDEX_BUFFER_BIT + | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT + | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT + | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT + | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT + | VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT + | VK_BUFFER_USAGE_TRANSFER_DST_BIT + | VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + + VkBuffer buffer = VK_NULL_HANDLE; + + if (vk->vkCreateBuffer(vk->device(), &bufferInfo, nullptr, &buffer)) { + Logger::err("Failed to create dummy buffer to query sparse memory types"); + return 0; + } + + vk->vkGetBufferMemoryRequirements(vk->device(), buffer, &requirements); + vk->vkDestroyBuffer(vk->device(), buffer, nullptr); + typeMask &= requirements.memoryTypeBits; + + // Create sparse dummy image to find available memory types + VkImageCreateInfo imageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; + imageInfo.flags = VK_IMAGE_CREATE_SPARSE_BINDING_BIT + | VK_IMAGE_CREATE_SPARSE_ALIASED_BIT + | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT; + imageInfo.imageType = VK_IMAGE_TYPE_2D; + imageInfo.format = VK_FORMAT_R8G8B8A8_UNORM; + imageInfo.extent = { 256, 256, 1 }; + imageInfo.mipLevels = 1; + imageInfo.arrayLayers = 1; + imageInfo.samples = VK_SAMPLE_COUNT_1_BIT; + imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL; + imageInfo.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT + | VK_IMAGE_USAGE_SAMPLED_BIT + | VK_IMAGE_USAGE_STORAGE_BIT + | VK_IMAGE_USAGE_TRANSFER_DST_BIT + | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + + VkImage image = VK_NULL_HANDLE; + + if (vk->vkCreateImage(vk->device(), &imageInfo, nullptr, &image)) { + Logger::err("Failed to create dummy image to query sparse memory types"); + return 0; + } + + vk->vkGetImageMemoryRequirements(vk->device(), image, &requirements); + vk->vkDestroyImage(vk->device(), image, nullptr); + typeMask &= requirements.memoryTypeBits; + + Logger::log(typeMask ? LogLevel::Info : LogLevel::Error, + str::format("Memory type mask for sparse resources: 0x", std::hex, typeMask)); + return typeMask; + } + } diff --git a/src/dxvk/dxvk_memory.h b/src/dxvk/dxvk_memory.h index af377a95f..c5caaf88c 100644 --- a/src/dxvk/dxvk_memory.h +++ b/src/dxvk/dxvk_memory.h @@ -306,7 +306,7 @@ namespace dxvk { constexpr static VkDeviceSize SmallAllocationThreshold = 256 << 10; public: - DxvkMemoryAllocator(const DxvkDevice* device); + DxvkMemoryAllocator(DxvkDevice* device); ~DxvkMemoryAllocator(); /** @@ -320,7 +320,15 @@ namespace dxvk { VkDeviceSize bufferImageGranularity() const { return m_devProps.limits.bufferImageGranularity; } - + + /** + * \brief Memory type mask for sparse resources + * \returns Sparse resource memory types + */ + uint32_t getSparseMemoryTypes() const { + return m_sparseMemoryTypes; + } + /** * \brief Allocates device memory * @@ -348,15 +356,16 @@ namespace dxvk { private: - const Rc m_vkd; - const DxvkDevice* m_device; - const VkPhysicalDeviceProperties m_devProps; - const VkPhysicalDeviceMemoryProperties m_memProps; + DxvkDevice* m_device; + VkPhysicalDeviceProperties m_devProps; + VkPhysicalDeviceMemoryProperties m_memProps; dxvk::mutex m_mutex; std::array m_memHeaps; std::array m_memTypes; + uint32_t m_sparseMemoryTypes = 0u; + DxvkMemory tryAlloc( const DxvkMemoryRequirements& req, const DxvkMemoryProperties& info, @@ -403,6 +412,9 @@ namespace dxvk { void freeEmptyChunks( const DxvkMemoryHeap* heap); + uint32_t determineSparseMemoryTypes( + DxvkDevice* device) const; + }; } diff --git a/src/dxvk/dxvk_sparse.cpp b/src/dxvk/dxvk_sparse.cpp index 367eb4a34..9b0e2eaa5 100644 --- a/src/dxvk/dxvk_sparse.cpp +++ b/src/dxvk/dxvk_sparse.cpp @@ -5,6 +5,156 @@ namespace dxvk { + DxvkSparseMapping::DxvkSparseMapping() + : m_pool(nullptr), + m_page(nullptr) { + + } + + + DxvkSparseMapping::DxvkSparseMapping( + Rc allocator, + Rc page) + : m_pool(std::move(allocator)), + m_page(std::move(page)) { + + } + + + DxvkSparseMapping::DxvkSparseMapping( + DxvkSparseMapping&& other) + : m_pool(std::move(other.m_pool)), + m_page(std::move(other.m_page)) { + // No need to acquire here. The only place from which + // this constructor can be called does this atomically. + } + + + DxvkSparseMapping::DxvkSparseMapping( + const DxvkSparseMapping& other) + : m_pool(other.m_pool), + m_page(other.m_page) { + this->acquire(); + } + + + DxvkSparseMapping& DxvkSparseMapping::operator = ( + DxvkSparseMapping&& other) { + this->release(); + + m_pool = std::move(other.m_pool); + m_page = std::move(other.m_page); + return *this; + } + + + DxvkSparseMapping& DxvkSparseMapping::operator = ( + const DxvkSparseMapping& other) { + other.acquire(); + this->release(); + + m_pool = other.m_pool; + m_page = other.m_page; + return *this; + } + + + DxvkSparseMapping::~DxvkSparseMapping() { + this->release(); + } + + + void DxvkSparseMapping::acquire() const { + if (m_page != nullptr) + m_pool->acquirePage(m_page); + } + + + void DxvkSparseMapping::release() const { + if (m_page != nullptr) + m_pool->releasePage(m_page); + } + + + DxvkSparsePageAllocator::DxvkSparsePageAllocator( + DxvkDevice* device, + DxvkMemoryAllocator& memoryAllocator) + : m_device(device), m_memory(&memoryAllocator) { + + } + + + DxvkSparsePageAllocator::~DxvkSparsePageAllocator() { + + } + + + DxvkSparseMapping DxvkSparsePageAllocator::acquirePage( + uint32_t page) { + std::lock_guard lock(m_mutex); + + if (unlikely(page >= m_pageCount)) + return DxvkSparseMapping(); + + m_useCount += 1; + return DxvkSparseMapping(this, m_pages[page]); + } + + + void DxvkSparsePageAllocator::setCapacity( + uint32_t pageCount) { + std::lock_guard lock(m_mutex); + + if (pageCount < m_pageCount) { + if (!m_useCount) + m_pages.resize(pageCount); + } else if (pageCount > m_pageCount) { + while (m_pages.size() < pageCount) + m_pages.push_back(allocPage()); + } + + m_pageCount = pageCount; + } + + + Rc DxvkSparsePageAllocator::allocPage() { + DxvkMemoryRequirements memoryRequirements = { }; + memoryRequirements.core = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2 }; + + // We don't know what kind of resource the memory6 + // might be bound to, so just guess the memory types + auto& core = memoryRequirements.core.memoryRequirements; + core.size = SparseMemoryPageSize; + core.alignment = SparseMemoryPageSize; + core.memoryTypeBits = m_memory->getSparseMemoryTypes(); + + DxvkMemoryProperties memoryProperties = { }; + memoryProperties.flags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + + DxvkMemory memory = m_memory->alloc(memoryRequirements, + memoryProperties, DxvkMemoryFlag::GpuReadable); + + return new DxvkSparsePage(std::move(memory)); + } + + + void DxvkSparsePageAllocator::acquirePage( + const Rc& page) { + std::lock_guard lock(m_mutex); + m_useCount += 1; + } + + + void DxvkSparsePageAllocator::releasePage( + const Rc& page) { + std::lock_guard lock(m_mutex); + m_useCount -= 1; + + if (!m_useCount) + m_pages.resize(m_pageCount); + } + + DxvkSparsePageTable::DxvkSparsePageTable() { } @@ -20,6 +170,7 @@ namespace dxvk { // and consists of consecutive 64k pages size_t pageCount = align(bufferSize, SparseMemoryPageSize) / SparseMemoryPageSize; m_metadata.resize(pageCount); + m_mappings.resize(pageCount); for (size_t i = 0; i < pageCount; i++) { VkDeviceSize pageOffset = SparseMemoryPageSize * i; @@ -135,6 +286,7 @@ namespace dxvk { // Fill in page metadata m_metadata.reserve(totalPageCount); + m_mappings.resize(totalPageCount); for (uint32_t l = 0; l < image->info().numLayers; l++) { for (uint32_t m = 0; m < m_properties.pagedMipCount; m++) { diff --git a/src/dxvk/dxvk_sparse.h b/src/dxvk/dxvk_sparse.h index 1563e90e4..c0585fa3b 100644 --- a/src/dxvk/dxvk_sparse.h +++ b/src/dxvk/dxvk_sparse.h @@ -10,7 +10,7 @@ namespace dxvk { class DxvkBuffer; class DxvkImage; class DxvkSparsePage; - class DxvkSparsePagePool; + class DxvkSparsePageAllocator; constexpr static VkDeviceSize SparseMemoryPageSize = 1ull << 16; @@ -112,6 +112,152 @@ namespace dxvk { }; + /** + * \brief Sparse memory page + * + * Stores a single reference-counted page + * of memory. The page size is 64k. + */ + class DxvkSparsePage : public DxvkResource { + + public: + + DxvkSparsePage(DxvkMemory&& memory) + : m_memory(std::move(memory)) { } + + /** + * \brief Queries memory handle + * \returns Memory information + */ + DxvkSparsePageHandle getHandle() const { + DxvkSparsePageHandle result; + result.memory = m_memory.memory(); + result.offset = m_memory.offset(); + result.length = m_memory.length(); + return result; + } + + private: + + DxvkMemory m_memory; + + }; + + + /** + * \brief Sparse page mapping + * + * Stores a reference to a page as well as the pool that the page + * was allocated from, and automatically manages the use counter + * of the pool as the reference is being moved or copied around. + */ + class DxvkSparseMapping { + friend DxvkSparsePageAllocator; + public: + + DxvkSparseMapping(); + + DxvkSparseMapping(DxvkSparseMapping&& other); + DxvkSparseMapping(const DxvkSparseMapping& other); + + DxvkSparseMapping& operator = (DxvkSparseMapping&& other); + DxvkSparseMapping& operator = (const DxvkSparseMapping& other); + + ~DxvkSparseMapping(); + + Rc getPage() const { + return m_page; + } + + bool operator == (const DxvkSparseMapping& other) const { + // Pool is a function of the page, so no need to check both + return m_page == other.m_page; + } + + bool operator != (const DxvkSparseMapping& other) const { + return m_page != other.m_page; + } + + operator bool () const { + return m_page != nullptr; + } + + private: + + Rc m_pool; + Rc m_page; + + DxvkSparseMapping( + Rc allocator, + Rc page); + + void acquire() const; + + void release() const; + + }; + + + /** + * \brief Sparse memory allocator + * + * Provides an allocator for sparse pages with variable capacity. + * Pages are use-counted to make sure they are not removed from + * the allocator too early. + */ + class DxvkSparsePageAllocator : public RcObject { + friend DxvkSparseMapping; + public: + + DxvkSparsePageAllocator( + DxvkDevice* device, + DxvkMemoryAllocator& memoryAllocator); + + ~DxvkSparsePageAllocator(); + + /** + * \brief Acquires page at the given offset + * + * If the offset is valid, this will atomically + * increment the allocator's use count and return + * a reference to the page. + * \param [in] page Page index + * \returns Page mapping object + */ + DxvkSparseMapping acquirePage( + uint32_t page); + + /** + * \brief Changes the allocator's maximum capacity + * + * Allocates new pages as necessary, and frees existing + * pages if none of the pages are currently in use. + * \param [in] pageCount New capacity, in pages + */ + void setCapacity( + uint32_t pageCount); + + private: + + DxvkDevice* m_device; + DxvkMemoryAllocator* m_memory; + + dxvk::mutex m_mutex; + uint32_t m_pageCount = 0u; + uint32_t m_useCount = 0u; + std::vector> m_pages; + + Rc allocPage(); + + void acquirePage( + const Rc& page); + + void releasePage( + const Rc& page); + + }; + + /** * \brief Sparse page table * @@ -202,6 +348,7 @@ namespace dxvk { DxvkSparseImageProperties m_properties = { }; std::vector m_subresources; std::vector m_metadata; + std::vector m_mappings; };