[dxvk] Move fence object into DxvkCommandList

Reduces command submission overhead by reusing fence objects
instead of creating new ones for each submission. Improves
error reporting in case the submission cannot be complete.

src/dxvk/dxvk_cmdlist.cpp

@@ -1,4 +1,5 @@
 #include "dxvk_cmdlist.h"
+#include "dxvk_device.h"
 
 namespace dxvk {
     
@@ -6,7 +7,17 @@ namespace dxvk {
     const Rc<vk::DeviceFn>& vkd,
           DxvkDevice*       device,
           uint32_t          queueFamily)
-  : m_vkd(vkd), m_descAlloc(vkd), m_stagingAlloc(device) {
+  : m_vkd         (vkd),
+    m_descAlloc   (vkd),
+    m_stagingAlloc(device) {
+    VkFenceCreateInfo fenceInfo;
+    fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+    fenceInfo.pNext = nullptr;
+    fenceInfo.flags = 0;
+    
+    if (m_vkd->vkCreateFence(m_vkd->device(), &fenceInfo, nullptr, &m_fence) != VK_SUCCESS)
+      throw DxvkError("DxvkFence::DxvkFence: Failed to create fence");
+    
     VkCommandPoolCreateInfo poolInfo;
     poolInfo.sType            = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
     poolInfo.pNext            = nullptr;
@@ -29,18 +40,18 @@ namespace dxvk {
   
   
   DxvkCommandList::~DxvkCommandList() {
+    this->synchronize();
     this->reset();
     
-    m_vkd->vkDestroyCommandPool(
-      m_vkd->device(), m_pool, nullptr);
+    m_vkd->vkDestroyCommandPool(m_vkd->device(), m_pool,  nullptr);
+    m_vkd->vkDestroyFence      (m_vkd->device(), m_fence, nullptr);
   }
   
   
-  void DxvkCommandList::submit(
+  VkResult DxvkCommandList::submit(
           VkQueue         queue,
           VkSemaphore     waitSemaphore,
-          VkSemaphore     wakeSemaphore,
-          VkFence         fence) {
+          VkSemaphore     wakeSemaphore) {
     const VkPipelineStageFlags waitStageMask
       = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
     
@@ -55,8 +66,20 @@ namespace dxvk {
     info.signalSemaphoreCount = wakeSemaphore == VK_NULL_HANDLE ? 0 : 1;
     info.pSignalSemaphores    = &wakeSemaphore;
     
-    if (m_vkd->vkQueueSubmit(queue, 1, &info, fence) != VK_SUCCESS)
-      throw DxvkError("DxvkDevice::submitCommandList: Command submission failed");
+    return m_vkd->vkQueueSubmit(queue, 1, &info, m_fence);
+  }
+  
+  
+  VkResult DxvkCommandList::synchronize() {
+    VkResult status = VK_TIMEOUT;
+    
+    while (status == VK_TIMEOUT) {
+      status = m_vkd->vkWaitForFences(
+        m_vkd->device(), 1, &m_fence, VK_FALSE,
+        1'000'000'000ull);
+    }
+    
+    return status;
   }
   
   
@@ -68,16 +91,19 @@ namespace dxvk {
     info.pInheritanceInfo = nullptr;
     
     if (m_vkd->vkResetCommandPool(m_vkd->device(), m_pool, 0) != VK_SUCCESS)
-      throw DxvkError("DxvkCommandList::beginRecording: Failed to reset command pool");
+      Logger::err("DxvkCommandList: Failed to reset command buffer");
     
     if (m_vkd->vkBeginCommandBuffer(m_buffer, &info) != VK_SUCCESS)
-      throw DxvkError("DxvkCommandList::beginRecording: Failed to begin command buffer recording");
+      Logger::err("DxvkCommandList: Failed to begin command buffer");
+    
+    if (m_vkd->vkResetFences(m_vkd->device(), 1, &m_fence) != VK_SUCCESS)
+      Logger::err("DxvkCommandList: Failed to reset fence");
   }
   
   
   void DxvkCommandList::endRecording() {
     if (m_vkd->vkEndCommandBuffer(m_buffer) != VK_SUCCESS)
-      throw DxvkError("DxvkCommandList::endRecording: Failed to record command buffer");
+      Logger::err("DxvkCommandList::endRecording: Failed to record command buffer");
   }
   
   

src/dxvk/dxvk_cmdlist.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <unordered_set>
+#include <limits>
 
 #include "dxvk_binding.h"
 #include "dxvk_buffer.h"
@@ -11,6 +11,7 @@
 #include "dxvk_pipelayout.h"
 #include "dxvk_query_tracker.h"
 #include "dxvk_staging.h"
+#include "dxvk_sync.h"
 
 namespace dxvk {
   
@@ -39,13 +40,21 @@ namespace dxvk {
      * \param [in] queue Device queue
      * \param [in] waitSemaphore Semaphore to wait on
      * \param [in] wakeSemaphore Semaphore to signal
-     * \param [in] fence Fence to signal
+     * \returns Submission status
      */
-    void submit(
+    VkResult submit(
             VkQueue         queue,
             VkSemaphore     waitSemaphore,
-            VkSemaphore     wakeSemaphore,
-            VkFence         fence);
+            VkSemaphore     wakeSemaphore);
+    
+    /**
+     * \brief Synchronizes command buffer execution
+     * 
+     * Waits for the fence associated with
+     * this command buffer to get signaled.
+     * \returns Synchronization status
+     */
+    VkResult synchronize();
     
     /**
      * \brief Begins recording
@@ -143,7 +152,6 @@ namespace dxvk {
      */
     void reset();
     
-    
     VkDescriptorSet allocateDescriptorSet(
             VkDescriptorSetLayout   descriptorLayout) {
       return m_descAlloc.alloc(descriptorLayout);
@@ -517,6 +525,8 @@ namespace dxvk {
     
     Rc<vk::DeviceFn>    m_vkd;
     
+    VkFence             m_fence;
+    
     VkCommandPool       m_pool;
     VkCommandBuffer     m_buffer;
     

src/dxvk/dxvk_device.cpp

@@ -209,12 +209,10 @@ namespace dxvk {
   }
   
   
-  Rc<DxvkFence> DxvkDevice::submitCommandList(
+  void DxvkDevice::submitCommandList(
     const Rc<DxvkCommandList>&      commandList,
     const Rc<DxvkSemaphore>&        waitSync,
     const Rc<DxvkSemaphore>&        wakeSync) {
-    Rc<DxvkFence> fence = new DxvkFence(m_vkd);
-    
     VkSemaphore waitSemaphore = VK_NULL_HANDLE;
     VkSemaphore wakeSemaphore = VK_NULL_HANDLE;
     
@@ -230,13 +228,19 @@ namespace dxvk {
     
     { // Queue submissions are not thread safe
       std::lock_guard<std::mutex> lock(m_submissionLock);
-      commandList->submit(m_graphicsQueue,
-        waitSemaphore, wakeSemaphore, fence->handle());
+      
+      const VkResult status = commandList->submit(
+        m_graphicsQueue, waitSemaphore, wakeSemaphore);
+      
+      if (status != VK_SUCCESS) {
+        Logger::err(str::format(
+          "DxvkDevice: Command buffer submission failed: ",
+          status));
+      }
     }
     
     // Add this to the set of running submissions
-    m_submissionQueue.submit(fence, commandList);
-    return fence;
+    m_submissionQueue.submit(commandList);
   }
   
   

src/dxvk/dxvk_device.h

@@ -313,7 +313,7 @@ namespace dxvk {
      * \param [in] wakeSync (Optional) Semaphore to notify
      * \returns Synchronization fence
      */
-    Rc<DxvkFence> submitCommandList(
+    void submitCommandList(
       const Rc<DxvkCommandList>&      commandList,
       const Rc<DxvkSemaphore>&        waitSync,
       const Rc<DxvkSemaphore>&        wakeSync);

src/dxvk/dxvk_queue.cpp

@@ -20,16 +20,14 @@ namespace dxvk {
   }
   
   
-  void DxvkSubmissionQueue::submit(
-    const Rc<DxvkFence>&        fence,
-    const Rc<DxvkCommandList>&  cmdList) {
+  void DxvkSubmissionQueue::submit(const Rc<DxvkCommandList>& cmdList) {
     { std::unique_lock<std::mutex> lock(m_mutex);
       
       m_condOnTake.wait(lock, [this] {
         return m_entries.size() < MaxNumQueuedCommandBuffers;
       });
       
-      m_entries.push({ fence, cmdList });
+      m_entries.push(cmdList);
       m_condOnAdd.notify_one();
     }
   }
@@ -37,7 +35,7 @@ namespace dxvk {
   
   void DxvkSubmissionQueue::threadFunc() {
     while (!m_stopped.load()) {
-      Entry entry;
+      Rc<DxvkCommandList> cmdList;
       
       { std::unique_lock<std::mutex> lock(m_mutex);
         
@@ -46,22 +44,27 @@ namespace dxvk {
         });
         
         if (m_entries.size() != 0) {
-          entry = std::move(m_entries.front());
+          cmdList = std::move(m_entries.front());
           m_entries.pop();
         }
         
         m_condOnTake.notify_one();
       }
       
-      if (entry.fence != nullptr) {
-        while (!entry.fence->wait(1'000'000'000ull))
-          continue;
+      if (cmdList != nullptr) {
+        VkResult status = cmdList->synchronize();
         
-        entry.cmdList->writeQueryData();
-        entry.cmdList->signalEvents();
-        entry.cmdList->reset();
+        if (status == VK_SUCCESS) {
+          cmdList->writeQueryData();
+          cmdList->signalEvents();
+          cmdList->reset();
           
-        m_device->recycleCommandList(entry.cmdList);
+          m_device->recycleCommandList(cmdList);
+        } else {
+          Logger::err(str::format(
+            "DxvkSubmissionQueue: Failed to sync fence: ",
+            status));
+        }
       }
     }
   }

src/dxvk/dxvk_queue.h

@@ -24,17 +24,10 @@ namespace dxvk {
     DxvkSubmissionQueue(DxvkDevice* device);
     ~DxvkSubmissionQueue();
     
-    void submit(
-      const Rc<DxvkFence>&        fence,
-      const Rc<DxvkCommandList>&  cmdList);
+    void submit(const Rc<DxvkCommandList>& cmdList);
     
   private:
     
-    struct Entry {
-      Rc<DxvkFence>       fence;
-      Rc<DxvkCommandList> cmdList;
-    };
-    
     DxvkDevice*             m_device;
     
     std::atomic<bool>       m_stopped = { false };
@@ -42,7 +35,7 @@ namespace dxvk {
     std::mutex              m_mutex;
     std::condition_variable m_condOnAdd;
     std::condition_variable m_condOnTake;
-    std::queue<Entry>       m_entries;
+    std::queue<Rc<DxvkCommandList>> m_entries;
     std::thread             m_thread;
     
     void threadFunc();