From cfc06405d2e574b1b76afe50879c0289e4c1d9a6 Mon Sep 17 00:00:00 2001
From: Philip Rebohle <philip.rebohle@tu-dortmund.de>
Date: Wed, 22 Jun 2022 16:12:05 +0200
Subject: [PATCH] [dxvk] Recycle Vulkan descriptor pools as well

Reduces the number of expensive reallocations when large descriptor
pools get reset and repopulated.
---
 src/dxvk/dxvk_descriptor.cpp | 96 +++++++++++++++++++++++-------------
 src/dxvk/dxvk_descriptor.h   | 28 ++++++++++-
 2 files changed, 88 insertions(+), 36 deletions(-)

diff --git a/src/dxvk/dxvk_descriptor.cpp b/src/dxvk/dxvk_descriptor.cpp
index bfaf0b8e9..3e2ab8e14 100644
--- a/src/dxvk/dxvk_descriptor.cpp
+++ b/src/dxvk/dxvk_descriptor.cpp
@@ -35,8 +35,8 @@ namespace dxvk {
 
   DxvkDescriptorPool::DxvkDescriptorPool(
           DxvkDevice*               device,
-          DxvkContextType           contextType)
-  : m_device(device), m_contextType(contextType),
+          DxvkDescriptorManager*    manager)
+  : m_device(device), m_manager(manager),
     m_cachedEntry(nullptr, nullptr) {
 
   }
@@ -100,13 +100,10 @@ namespace dxvk {
       // If most sets are no longer being used, reset and destroy
       // descriptor pools and reset all lookup tables in order to
       // accomodate more descriptors of different layouts.
-      auto vk = m_device->vkd();
-      vk->vkResetDescriptorPool(vk->device(), m_descriptorPools[0], 0);
+      for (auto pool : m_descriptorPools)
+        m_manager->recycleVulkanDescriptorPool(pool);
 
-      for (uint32_t i = 1; i < m_descriptorPools.size(); i++)
-        vk->vkDestroyDescriptorPool(vk->device(), m_descriptorPools[i], nullptr);
-
-      m_descriptorPools.resize(1);
+      m_descriptorPools.clear();
       m_setLists.clear();
       m_setMaps.clear();
 
@@ -205,8 +202,55 @@ namespace dxvk {
 
 
   VkDescriptorPool DxvkDescriptorPool::addPool() {
+    VkDescriptorPool pool = m_manager->createVulkanDescriptorPool();
+    m_descriptorPools.push_back(pool);
+    return pool;
+  }
+
+  
+  DxvkDescriptorManager::DxvkDescriptorManager(
+          DxvkDevice*                 device,
+          DxvkContextType             contextType)
+  : m_device(device), m_contextType(contextType) {
+
+  }
+
+
+  DxvkDescriptorManager::~DxvkDescriptorManager() {
     auto vk = m_device->vkd();
 
+    for (size_t i = 0; i < m_vkPoolCount; i++)
+      vk->vkDestroyDescriptorPool(vk->device(), m_vkPools[i], nullptr);
+  }
+
+
+  Rc<DxvkDescriptorPool> DxvkDescriptorManager::getDescriptorPool() {
+    Rc<DxvkDescriptorPool> pool = m_pools.retrieveObject();
+
+    if (pool == nullptr)
+      pool = new DxvkDescriptorPool(m_device, this);
+
+    return pool;
+  }
+
+
+  void DxvkDescriptorManager::recycleDescriptorPool(
+    const Rc<DxvkDescriptorPool>&     pool) {
+    pool->reset();
+
+    m_pools.returnObject(pool);
+  }
+
+
+  VkDescriptorPool DxvkDescriptorManager::createVulkanDescriptorPool() {
+    auto vk = m_device->vkd();
+
+    { std::lock_guard lock(m_mutex);
+
+      if (m_vkPoolCount)
+        return m_vkPools[--m_vkPoolCount];
+    }
+
     uint32_t maxSets = m_contextType == DxvkContextType::Primary
       ? 8192 : 256;
 
@@ -233,39 +277,23 @@ namespace dxvk {
     if (vk->vkCreateDescriptorPool(vk->device(), &info, nullptr, &pool) != VK_SUCCESS)
       throw DxvkError("DxvkDescriptorPool: Failed to create descriptor pool");
 
-    m_descriptorPools.push_back(pool);
     return pool;
   }
 
   
-  DxvkDescriptorManager::DxvkDescriptorManager(
-          DxvkDevice*                 device,
-          DxvkContextType             contextType)
-  : m_device(device), m_contextType(contextType) {
+  void DxvkDescriptorManager::recycleVulkanDescriptorPool(VkDescriptorPool pool) {
+    auto vk = m_device->vkd();
+    vk->vkResetDescriptorPool(vk->device(), pool, 0);
 
-  }
+    { std::lock_guard lock(m_mutex);
 
+      if (m_vkPoolCount < m_vkPools.size()) {
+        m_vkPools[m_vkPoolCount++] = pool;
+        return;
+      }
+    }
 
-  DxvkDescriptorManager::~DxvkDescriptorManager() {
-
-  }
-
-
-  Rc<DxvkDescriptorPool> DxvkDescriptorManager::getDescriptorPool() {
-    Rc<DxvkDescriptorPool> pool = m_pools.retrieveObject();
-
-    if (pool == nullptr)
-      pool = new DxvkDescriptorPool(m_device, m_contextType);
-
-    return pool;
-  }
-
-
-  void DxvkDescriptorManager::recycleDescriptorPool(
-    const Rc<DxvkDescriptorPool>&     pool) {
-    pool->reset();
-
-    m_pools.returnObject(pool);
+    vk->vkDestroyDescriptorPool(vk->device(), pool, nullptr);
   }
 
 }
\ No newline at end of file
diff --git a/src/dxvk/dxvk_descriptor.h b/src/dxvk/dxvk_descriptor.h
index 346eb3795..d1fc4a6af 100644
--- a/src/dxvk/dxvk_descriptor.h
+++ b/src/dxvk/dxvk_descriptor.h
@@ -9,6 +9,7 @@
 namespace dxvk {
 
   class DxvkDevice;
+  class DxvkDescriptorManager;
   
   /**
    * \brief DXVK context type
@@ -83,7 +84,7 @@ namespace dxvk {
 
     DxvkDescriptorPool(
             DxvkDevice*               device,
-            DxvkContextType           contextType);
+            DxvkDescriptorManager*    manager);
 
     ~DxvkDescriptorPool();
 
@@ -116,7 +117,7 @@ namespace dxvk {
   private:
 
     DxvkDevice*               m_device;
-    DxvkContextType           m_contextType;
+    DxvkDescriptorManager*    m_manager;
 
     std::vector<VkDescriptorPool>                                     m_descriptorPools;
     std::unordered_map<VkDescriptorSetLayout, DxvkDescriptorSetList>  m_setLists;
@@ -176,12 +177,35 @@ namespace dxvk {
     void recycleDescriptorPool(
       const Rc<DxvkDescriptorPool>&     pool);
 
+    /**
+     * \brief Creates a Vulkan descriptor pool
+     *
+     * Returns an existing unused pool or
+     * creates a new one if necessary.
+     * \returns The descriptor pool
+     */
+    VkDescriptorPool createVulkanDescriptorPool();
+
+    /**
+     * \brief Returns unused descriptor pool
+     *
+     * Caches the pool for future use, or destroys
+     * it if there are too many objects in the cache
+     * already.
+     * \param [in] pool Vulkan descriptor pool
+     */
+    void recycleVulkanDescriptorPool(VkDescriptorPool pool);
+
   private:
 
     DxvkDevice*                         m_device;
     DxvkContextType                     m_contextType;
     DxvkRecycler<DxvkDescriptorPool, 8> m_pools;
 
+    dxvk::mutex                         m_mutex;
+    std::array<VkDescriptorPool, 8>     m_vkPools;
+    size_t                              m_vkPoolCount = 0;
+
   };
 
 }
\ No newline at end of file