[dxvk] Introduce DxvkDevice::waitForResource

Blocks on the queue thread's condition variable instead of busy-waiting,
and tracks synchronization with new stat counters. Cleanup is rearranged
to minimize delays before signals and resources are notified.

src/dxvk/dxvk_cmdlist.cpp

@@ -169,9 +169,8 @@ namespace dxvk {
   
   
   void DxvkCommandList::reset() {
-    // Signal resources and events to
-    // avoid stalling main thread
-    m_signalTracker.reset();
+    // Free resources and other objects
+    // that are no longer in use
     m_resources.reset();
 
     // Recycle heavy Vulkan objects
@@ -185,6 +184,7 @@ namespace dxvk {
     m_gpuEventTracker.reset();
 
     // Less important stuff
+    m_signalTracker.reset();
     m_statCounters.reset();
   }
 

src/dxvk/dxvk_cmdlist.h

@@ -194,9 +194,10 @@ namespace dxvk {
     }
 
     /**
-     * \brief Notifies signals
+     * \brief Notifies resources and signals
      */
-    void notifySignals() {
+    void notifyObjects() {
+      m_resources.notify();
       m_signalTracker.notify();
     }
     

src/dxvk/dxvk_device.cpp

@@ -248,6 +248,24 @@ namespace dxvk {
   }
 
 
+  void DxvkDevice::waitForResource(const Rc<DxvkResource>& resource, DxvkAccess access) {
+    if (resource->isInUse(access)) {
+      auto t0 = dxvk::high_resolution_clock::now();
+
+      m_submissionQueue.synchronizeUntil([resource, access] {
+        return !resource->isInUse(access);
+      });
+
+      auto t1 = dxvk::high_resolution_clock::now();
+      auto us = std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0);
+
+      std::lock_guard<sync::Spinlock> lock(m_statLock);
+      m_statCounters.addCtr(DxvkStatCounter::GpuSyncCount, 1);
+      m_statCounters.addCtr(DxvkStatCounter::GpuSyncTicks, us.count());
+    }
+  }
+  
+  
   void DxvkDevice::waitForIdle() {
     this->lockSubmission();
     if (m_vkd->vkDeviceWaitIdle(m_vkd->device()) != VK_SUCCESS)

src/dxvk/dxvk_device.h

@@ -475,6 +475,14 @@ namespace dxvk {
      */
     VkResult waitForSubmission(DxvkSubmitStatus* status);
 
+    /**
+     * \brief Waits for resource to become idle
+     *
+     * \param [in] resource Resource to wait for
+     * \param [in] access Access mode to check
+     */
+    void waitForResource(const Rc<DxvkResource>& resource, DxvkAccess access);
+    
     /**
      * \brief Waits until the device becomes idle
      * 

src/dxvk/dxvk_lifetime.cpp

@@ -6,9 +6,20 @@ namespace dxvk {
   DxvkLifetimeTracker::~DxvkLifetimeTracker() { }
   
   
-  void DxvkLifetimeTracker::reset() {
+  void DxvkLifetimeTracker::notify() {
     for (const auto& resource : m_resources)
       resource.first->release(resource.second);
+
+    m_notified = true;
+  }
+
+
+  void DxvkLifetimeTracker::reset() {
+    // If this gets called without ever being submitted then
+    // we should at least report the resources as unused
+    if (!m_notified)
+      this->notify();
+
     m_resources.clear();
   }
   

src/dxvk/dxvk_lifetime.h

@@ -31,6 +31,13 @@ namespace dxvk {
       m_resources.emplace_back(std::move(rc), Access);
     }
 
+    /**
+     * \brief Releases resources
+     *
+     * Marks all tracked resources as unused.
+     */
+    void notify();
+    
     /**
      * \brief Resets the command list
      * 
@@ -42,6 +49,7 @@ namespace dxvk {
   private:
     
     std::vector<std::pair<Rc<DxvkResource>, DxvkAccess>> m_resources;
+    bool m_notified = false;
     
   };
   

src/dxvk/dxvk_queue.cpp

@@ -140,9 +140,9 @@ namespace dxvk {
   void DxvkSubmissionQueue::finishCmdLists() {
     env::setThreadName("dxvk-queue");
 
+    while (!m_stopped.load()) {
       std::unique_lock<dxvk::mutex> lock(m_mutex);
 
-    while (!m_stopped.load()) {
       if (m_finishQueue.empty()) {
         auto t0 = dxvk::high_resolution_clock::now();
 
@@ -171,16 +171,21 @@ namespace dxvk {
         m_device->waitForIdle();
       }
 
-      entry.submit.cmdList->notifySignals();
-      entry.submit.cmdList->reset();
+      // Release resources and signal events, then immediately wake
+      // up any thread that's currently waiting on a resource in
+      // order to reduce delays as much as possible.
+      entry.submit.cmdList->notifyObjects();
 
-      m_device->recycleCommandList(entry.submit.cmdList);
-
-      lock = std::unique_lock<dxvk::mutex>(m_mutex);
+      lock.lock();
       m_pending -= 1;
 
       m_finishQueue.pop();
       m_finishCond.notify_all();
+      lock.unlock();
+
+      // Free the command list and associated objects now
+      entry.submit.cmdList->reset();
+      m_device->recycleCommandList(entry.submit.cmdList);
     }
   }
   

src/dxvk/dxvk_queue.h

@@ -145,6 +145,18 @@ namespace dxvk {
      */
     void synchronize();
 
+    /**
+     * \brief Synchronizes until a given condition becomes true
+     *
+     * Useful to wait for the GPU without busy-waiting.
+     * \param [in] pred Predicate to check
+     */
+    template<typename Pred>
+    void synchronizeUntil(const Pred& pred) {
+      std::unique_lock<dxvk::mutex> lock(m_mutex);
+      m_finishCond.wait(lock, pred);
+    }
+
     /**
      * \brief Locks device queue
      *

src/dxvk/dxvk_stats.h

@@ -20,6 +20,8 @@ namespace dxvk {
     PipeCompilerBusy,         ///< Boolean indicating compiler activity
     QueueSubmitCount,         ///< Number of command buffer submissions
     QueuePresentCount,        ///< Number of present calls / frames
+    GpuSyncCount,             ///< Number of GPU synchronizations
+    GpuSyncTicks,             ///< Time spent waiting for GPU
     GpuIdleTicks,             ///< GPU idle time in microseconds
     CsSyncCount,              ///< CS thread synchronizations
     CsSyncTicks,              ///< Time spent waiting on CS