From 4057937d2d8f3d5e8a25f2c66109a508b256600f Mon Sep 17 00:00:00 2001 From: Philip Rebohle Date: Tue, 27 Nov 2018 11:07:23 +0100 Subject: [PATCH] [dxvk] Refactor descriptor set allocation With this new approach, descriptor pools are decoupled from the command list they are used with. Instead, the DXVK context takes ownership of a single descriptor pool until it runs out of memory. This reduces the amount of memory wasted for under-utilized pools and should this reduce an application's memory footprint. --- src/dxvk/dxvk_cmdlist.cpp | 4 +- src/dxvk/dxvk_cmdlist.h | 16 +++--- src/dxvk/dxvk_context.cpp | 34 +++++++++---- src/dxvk/dxvk_context.h | 7 ++- src/dxvk/dxvk_descriptor.cpp | 97 +++++++++++++++++------------------- src/dxvk/dxvk_descriptor.h | 63 ++++++++++++++++------- src/dxvk/dxvk_device.cpp | 15 ++++++ src/dxvk/dxvk_device.h | 19 ++++++- 8 files changed, 166 insertions(+), 89 deletions(-) diff --git a/src/dxvk/dxvk_cmdlist.cpp b/src/dxvk/dxvk_cmdlist.cpp index 1ad0b8ad..d8e9dadb 100644 --- a/src/dxvk/dxvk_cmdlist.cpp +++ b/src/dxvk/dxvk_cmdlist.cpp @@ -8,7 +8,7 @@ namespace dxvk { uint32_t queueFamily) : m_vkd (device->vkd()), m_cmdBuffersUsed(0), - m_descAlloc (device->vkd()), + m_descriptorPoolTracker(device), m_stagingAlloc (device) { VkFenceCreateInfo fenceInfo; fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; @@ -127,7 +127,7 @@ namespace dxvk { m_eventTracker.reset(); m_queryTracker.reset(); m_stagingAlloc.reset(); - m_descAlloc.reset(); + m_descriptorPoolTracker.reset(); m_resources.reset(); } diff --git a/src/dxvk/dxvk_cmdlist.h b/src/dxvk/dxvk_cmdlist.h index 720fd685..cb9f7a78 100644 --- a/src/dxvk/dxvk_cmdlist.h +++ b/src/dxvk/dxvk_cmdlist.h @@ -156,6 +156,14 @@ namespace dxvk { void trackEvent(const DxvkEventRevision& event) { m_eventTracker.trackEvent(event); } + + /** + * \brief Tracks a descriptor pool + * \param [in] pool The descriptor pool + */ + void trackDescriptorPool(Rc pool) { + m_descriptorPoolTracker.trackDescriptorPool(pool); + } /** * \brief Signals tracked events @@ -188,12 +196,6 @@ namespace dxvk { */ void reset(); - VkDescriptorSet allocateDescriptorSet( - VkDescriptorSetLayout descriptorLayout) { - return m_descAlloc.alloc(descriptorLayout); - } - - void updateDescriptorSets( uint32_t descriptorWriteCount, const VkWriteDescriptorSet* pDescriptorWrites) { @@ -656,7 +658,7 @@ namespace dxvk { DxvkCmdBufferFlags m_cmdBuffersUsed; DxvkLifetimeTracker m_resources; - DxvkDescriptorAlloc m_descAlloc; + DxvkDescriptorPoolTracker m_descriptorPoolTracker; DxvkStagingAlloc m_stagingAlloc; DxvkQueryTracker m_queryTracker; DxvkEventTracker m_eventTracker; diff --git a/src/dxvk/dxvk_context.cpp b/src/dxvk/dxvk_context.cpp index 08c63af8..2ef665c2 100644 --- a/src/dxvk/dxvk_context.cpp +++ b/src/dxvk/dxvk_context.cpp @@ -296,8 +296,7 @@ namespace dxvk { // Create a descriptor set pointing to the view VkBufferView viewObject = bufferView->handle(); - VkDescriptorSet descriptorSet = - m_cmd->allocateDescriptorSet(pipeInfo.dsetLayout); + VkDescriptorSet descriptorSet = allocateDescriptorSet(pipeInfo.dsetLayout); VkWriteDescriptorSet descriptorWrite; descriptorWrite.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; @@ -963,7 +962,7 @@ namespace dxvk { descriptors.srcDepth = dView->getDescriptor(VK_IMAGE_VIEW_TYPE_2D_ARRAY, layout).image; descriptors.srcStencil = sView->getDescriptor(VK_IMAGE_VIEW_TYPE_2D_ARRAY, layout).image; - VkDescriptorSet dset = m_cmd->allocateDescriptorSet(pipeInfo.dsetLayout); + VkDescriptorSet dset = allocateDescriptorSet(pipeInfo.dsetLayout); m_cmd->updateDescriptorSetWithTemplate(dset, pipeInfo.dsetTemplate, &descriptors); // Since this is a meta operation, the image may be @@ -1290,7 +1289,7 @@ namespace dxvk { // Create descriptor set with the current source view descriptorImage.imageView = pass.srcView; - descriptorWrite.dstSet = m_cmd->allocateDescriptorSet(pipeInfo.dsetLayout); + descriptorWrite.dstSet = allocateDescriptorSet(pipeInfo.dsetLayout); m_cmd->updateDescriptorSets(1, &descriptorWrite); // Set up viewport and scissor rect @@ -1802,8 +1801,7 @@ namespace dxvk { imageView->type(), imageFormatInfo(imageView->info().format)->flags); // Create a descriptor set pointing to the view - VkDescriptorSet descriptorSet = - m_cmd->allocateDescriptorSet(pipeInfo.dsetLayout); + VkDescriptorSet descriptorSet = allocateDescriptorSet(pipeInfo.dsetLayout); VkDescriptorImageInfo viewInfo; viewInfo.sampler = VK_NULL_HANDLE; @@ -2077,7 +2075,7 @@ namespace dxvk { descriptorWrite.pBufferInfo = nullptr; descriptorWrite.pTexelBufferView = nullptr; - descriptorWrite.dstSet = m_cmd->allocateDescriptorSet(pipeInfo.dsetLayout); + descriptorWrite.dstSet = allocateDescriptorSet(pipeInfo.dsetLayout); m_cmd->updateDescriptorSets(1, &descriptorWrite); VkViewport viewport; @@ -2272,7 +2270,7 @@ namespace dxvk { descriptorWrite.pBufferInfo = nullptr; descriptorWrite.pTexelBufferView = nullptr; - descriptorWrite.dstSet = m_cmd->allocateDescriptorSet(pipeInfo.dsetLayout); + descriptorWrite.dstSet = allocateDescriptorSet(pipeInfo.dsetLayout); m_cmd->updateDescriptorSets(1, &descriptorWrite); // Set up viewport and scissor rect @@ -2814,7 +2812,7 @@ namespace dxvk { VkDescriptorSet descriptorSet = VK_NULL_HANDLE; if (layout->bindingCount() != 0) { - descriptorSet = m_cmd->allocateDescriptorSet( + descriptorSet = allocateDescriptorSet( layout->descriptorSetLayout()); m_cmd->updateDescriptorSetWithTemplate( @@ -3258,6 +3256,24 @@ namespace dxvk { 0, 1, &barrier, 0, nullptr, 0, nullptr); } + + VkDescriptorSet DxvkContext::allocateDescriptorSet( + VkDescriptorSetLayout layout) { + if (m_descPool == nullptr) + m_descPool = m_device->createDescriptorPool(); + + VkDescriptorSet set = m_descPool->alloc(layout); + + if (set == VK_NULL_HANDLE) { + m_cmd->trackDescriptorPool(std::move(m_descPool)); + + m_descPool = m_device->createDescriptorPool(); + set = m_descPool->alloc(layout); + } + + return set; + } + void DxvkContext::trackDrawBuffer() { if (m_flags.test(DxvkContextFlag::DirtyDrawBuffer)) { diff --git a/src/dxvk/dxvk_context.h b/src/dxvk/dxvk_context.h index ad3fca46..de476a18 100644 --- a/src/dxvk/dxvk_context.h +++ b/src/dxvk/dxvk_context.h @@ -768,7 +768,9 @@ namespace dxvk { const Rc m_metaPack; const Rc m_metaResolve; - Rc m_cmd; + Rc m_cmd; + Rc m_descPool; + DxvkContextFlags m_flags; DxvkContextState m_state; @@ -902,6 +904,9 @@ namespace dxvk { VkAccessFlags srcAccess, VkPipelineStageFlags dstStages, VkAccessFlags dstAccess); + + VkDescriptorSet allocateDescriptorSet( + VkDescriptorSetLayout layout); void trackDrawBuffer(); diff --git a/src/dxvk/dxvk_descriptor.cpp b/src/dxvk/dxvk_descriptor.cpp index db3691ee..d63c07e8 100644 --- a/src/dxvk/dxvk_descriptor.cpp +++ b/src/dxvk/dxvk_descriptor.cpp @@ -1,50 +1,10 @@ #include "dxvk_descriptor.h" +#include "dxvk_device.h" namespace dxvk { - DxvkDescriptorAlloc::DxvkDescriptorAlloc( - const Rc& vkd) + DxvkDescriptorPool::DxvkDescriptorPool(const Rc& vkd) : m_vkd(vkd) { - // Allocate one pool right away so that there - // is always at least one pool available when - // allocating a descriptor set - m_pools.push_back(createDescriptorPool()); - } - - - DxvkDescriptorAlloc::~DxvkDescriptorAlloc() { - for (auto p : m_pools) { - m_vkd->vkDestroyDescriptorPool( - m_vkd->device(), p, nullptr); - } - } - - - VkDescriptorSet DxvkDescriptorAlloc::alloc(VkDescriptorSetLayout layout) { - VkDescriptorSet set = allocFrom(m_pools[m_poolId], layout); - - if (set == VK_NULL_HANDLE) { - if (++m_poolId >= m_pools.size()) - m_pools.push_back(createDescriptorPool()); - - set = allocFrom(m_pools[m_poolId], layout); - } - - return set; - } - - - void DxvkDescriptorAlloc::reset() { - for (auto p : m_pools) { - m_vkd->vkResetDescriptorPool( - m_vkd->device(), p, 0); - } - - m_poolId = 0; - } - - - VkDescriptorPool DxvkDescriptorAlloc::createDescriptorPool() { constexpr uint32_t MaxSets = 2048; std::array pools = {{ @@ -67,21 +27,22 @@ namespace dxvk { info.poolSizeCount = pools.size(); info.pPoolSizes = pools.data(); - VkDescriptorPool pool = VK_NULL_HANDLE; - if (m_vkd->vkCreateDescriptorPool(m_vkd->device(), - &info, nullptr, &pool) != VK_SUCCESS) - throw DxvkError("DxvkDescriptorAlloc: Failed to create descriptor pool"); - return pool; + if (m_vkd->vkCreateDescriptorPool(m_vkd->device(), &info, nullptr, &m_pool) != VK_SUCCESS) + throw DxvkError("DxvkDescriptorPool: Failed to create descriptor pool"); } - VkDescriptorSet DxvkDescriptorAlloc::allocFrom( - VkDescriptorPool pool, - VkDescriptorSetLayout layout) const { + DxvkDescriptorPool::~DxvkDescriptorPool() { + m_vkd->vkDestroyDescriptorPool( + m_vkd->device(), m_pool, nullptr); + } + + + VkDescriptorSet DxvkDescriptorPool::alloc(VkDescriptorSetLayout layout) { VkDescriptorSetAllocateInfo info; info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; info.pNext = nullptr; - info.descriptorPool = pool; + info.descriptorPool = m_pool; info.descriptorSetCount = 1; info.pSetLayouts = &layout; @@ -91,4 +52,38 @@ namespace dxvk { return set; } + + void DxvkDescriptorPool::reset() { + m_vkd->vkResetDescriptorPool( + m_vkd->device(), m_pool, 0); + } + + + + + DxvkDescriptorPoolTracker::DxvkDescriptorPoolTracker(DxvkDevice* device) + : m_device(device) { + + } + + + DxvkDescriptorPoolTracker::~DxvkDescriptorPoolTracker() { + + } + + + void DxvkDescriptorPoolTracker::trackDescriptorPool(Rc pool) { + m_pools.push_back(std::move(pool)); + } + + + void DxvkDescriptorPoolTracker::reset() { + for (const auto& pool : m_pools) { + pool->reset(); + m_device->recycleDescriptorPool(pool); + } + + m_pools.clear(); + } + } \ No newline at end of file diff --git a/src/dxvk/dxvk_descriptor.h b/src/dxvk/dxvk_descriptor.h index e4afba68..243e6b0a 100644 --- a/src/dxvk/dxvk_descriptor.h +++ b/src/dxvk/dxvk_descriptor.h @@ -5,6 +5,8 @@ #include "dxvk_include.h" namespace dxvk { + + class DxvkDevice; /** * \brief Descriptor info @@ -20,21 +22,18 @@ namespace dxvk { /** - * \brief Descriptor set allocator + * \brief Descriptor pool * - * Creates descriptor pools on demand and - * allocates descriptor sets from those pools. + * Wrapper around a Vulkan descriptor pool that + * descriptor sets can be allocated from. */ - class DxvkDescriptorAlloc { + class DxvkDescriptorPool : public RcObject { public: - DxvkDescriptorAlloc( + DxvkDescriptorPool( const Rc& vkd); - ~DxvkDescriptorAlloc(); - - DxvkDescriptorAlloc (const DxvkDescriptorAlloc&) = delete; - DxvkDescriptorAlloc& operator = (const DxvkDescriptorAlloc&) = delete; + ~DxvkDescriptorPool(); /** * \brief Allocates a descriptor set @@ -56,16 +55,46 @@ namespace dxvk { private: Rc m_vkd; + VkDescriptorPool m_pool; - std::vector m_pools; - size_t m_poolId = 0; - - VkDescriptorPool createDescriptorPool(); - - VkDescriptorSet allocFrom( - VkDescriptorPool pool, - VkDescriptorSetLayout layout) const; + }; + + + /** + * \brief Descriptor pool tracker + * + * Tracks descriptor pools that are either full + * or no longer needed by the DXVK context. The + * command list will reset and recycle all pools + * once it has completed execution on the GPU. + */ + class DxvkDescriptorPoolTracker { + + public: + + DxvkDescriptorPoolTracker(DxvkDevice* device); + ~DxvkDescriptorPoolTracker(); + + /** + * \brief Adds a descriptor pool to track + * \param [in] pool The descriptor pool + */ + void trackDescriptorPool(Rc pool); + /** + * \brief Resets event tracker + * + * Resets all tracked descriptor pools + * and returns them to the device. + */ + void reset(); + + private: + + DxvkDevice* m_device; + + std::vector> m_pools; + }; } \ No newline at end of file diff --git a/src/dxvk/dxvk_device.cpp b/src/dxvk/dxvk_device.cpp index 26b2c33f..9ce3ec75 100644 --- a/src/dxvk/dxvk_device.cpp +++ b/src/dxvk/dxvk_device.cpp @@ -114,6 +114,16 @@ namespace dxvk { return cmdList; } + + + Rc DxvkDevice::createDescriptorPool() { + Rc pool = m_recycledDescriptorPools.retrieveObject(); + + if (pool == nullptr) + pool = new DxvkDescriptorPool(m_vkd); + + return pool; + } Rc DxvkDevice::createContext() { @@ -296,4 +306,9 @@ namespace dxvk { m_recycledCommandLists.returnObject(cmdList); } + + void DxvkDevice::recycleDescriptorPool(const Rc& pool) { + m_recycledDescriptorPools.returnObject(pool); + } + } diff --git a/src/dxvk/dxvk_device.h b/src/dxvk/dxvk_device.h index 7be4cc8b..3831dc35 100644 --- a/src/dxvk/dxvk_device.h +++ b/src/dxvk/dxvk_device.h @@ -58,6 +58,7 @@ namespace dxvk { class DxvkDevice : public RcObject { friend class DxvkContext; friend class DxvkSubmissionQueue; + friend class DxvkDescriptorPoolTracker; constexpr static VkDeviceSize DefaultStagingBufferSize = 4 * 1024 * 1024; public: @@ -178,6 +179,16 @@ namespace dxvk { */ Rc createCommandList(); + /** + * \brief Creates a descriptor pool + * + * Returns a previously recycled pool, or creates + * a new one if necessary. The context should take + * ownership of the returned pool. + * \returns Descriptor pool + */ + Rc createDescriptorPool(); + /** * \brief Creates a context * @@ -413,14 +424,18 @@ namespace dxvk { DxvkDeviceQueue m_graphicsQueue; DxvkDeviceQueue m_presentQueue; - DxvkRecycler m_recycledCommandLists; - DxvkRecycler m_recycledStagingBuffers; + DxvkRecycler m_recycledCommandLists; + DxvkRecycler m_recycledDescriptorPools; + DxvkRecycler m_recycledStagingBuffers; DxvkSubmissionQueue m_submissionQueue; void recycleCommandList( const Rc& cmdList); + void recycleDescriptorPool( + const Rc& pool); + /** * \brief Dummy buffer handle * \returns Use for unbound vertex buffers.