[dxvk] Use worker to update adapter's memory statistics

Takes a bunch of pointer indirections and atomics off the hot path.

src/dxvk/dxvk_adapter.cpp

@@ -55,14 +55,14 @@ namespace dxvk {
       if (m_hasMemoryBudget) {
         // Handle DXVK's memory allocations separately so that
         // freeing  resources actually is visible to applications.
-        VkDeviceSize allocated = m_memoryAllocated[i].load();
-        VkDeviceSize used = m_memoryUsed[i].load();
+        VkDeviceSize allocated = m_memoryStats[i].allocated.load();
+        VkDeviceSize used = m_memoryStats[i].used.load();
 
         info.heaps[i].memoryBudget    = memBudget.heapBudget[i];
         info.heaps[i].memoryAllocated = std::max(memBudget.heapUsage[i], allocated) - allocated + used;
       } else {
         info.heaps[i].memoryBudget    = memProps.memoryProperties.memoryHeaps[i].size;
-        info.heaps[i].memoryAllocated = m_memoryUsed[i].load();
+        info.heaps[i].memoryAllocated = m_memoryStats[i].used.load();
       }
     }
 
@@ -654,19 +654,14 @@ namespace dxvk {
   }
 
 
-  void DxvkAdapter::notifyMemoryAlloc(
+  void DxvkAdapter::notifyMemoryStats(
           uint32_t            heap,
-          int64_t             bytes) {
-    if (heap < m_memoryAllocated.size())
-      m_memoryAllocated[heap] += bytes;
-  }
-
-
-  void DxvkAdapter::notifyMemoryUse(
-          uint32_t            heap,
-          int64_t             bytes) {
-    if (heap < m_memoryUsed.size())
-      m_memoryUsed[heap] += bytes;
+          int64_t             allocated,
+          int64_t             used) {
+    if (heap < m_memoryStats.size()) {
+      m_memoryStats[heap].allocated += allocated;
+      m_memoryStats[heap].used += used;
+    }
   }
 
 

src/dxvk/dxvk_adapter.h

@@ -57,7 +57,19 @@ namespace dxvk {
     uint32_t transfer;
     uint32_t sparse;
   };
-  
+
+
+  /**
+   * \brief Adapter memory statistics
+   *
+   * Periodically updated by the devices using this adapter.
+   */
+  struct DxvkAdapterMemoryStats {
+    std::atomic<uint64_t> allocated = { 0u };
+    std::atomic<uint64_t> used = { 0u };
+  };
+
+
   /**
    * \brief Device import info
    */
@@ -228,22 +240,13 @@ namespace dxvk {
      * 
      * Updates memory alloc info accordingly.
      * \param [in] heap Memory heap index
-     * \param [in] bytes Allocation size
+     * \param [in] allocated Allocated size delta
+     * \param [in] used Used size delta
      */
-    void notifyMemoryAlloc(
+    void notifyMemoryStats(
             uint32_t            heap,
-            int64_t             bytes);
-    
-    /**
-     * \brief Registers memory suballocation
-     * 
-     * Updates memory alloc info accordingly.
-     * \param [in] heap Memory heap index
-     * \param [in] bytes Allocation size
-     */
-    void notifyMemoryUse(
-            uint32_t            heap,
-            int64_t             bytes);
+            int64_t             allocated,
+            int64_t             used);
     
     /**
      * \brief Tests if the driver matches certain criteria
@@ -328,8 +331,7 @@ namespace dxvk {
 
     std::vector<VkQueueFamilyProperties> m_queueFamilies;
 
-    std::array<std::atomic<uint64_t>, VK_MAX_MEMORY_HEAPS> m_memoryAllocated = { };
-    std::array<std::atomic<uint64_t>, VK_MAX_MEMORY_HEAPS> m_memoryUsed = { };
+    std::array<DxvkAdapterMemoryStats, VK_MAX_MEMORY_HEAPS> m_memoryStats = { };
 
     void queryExtensions();
     void queryDeviceInfo();

src/dxvk/dxvk_device.h

@@ -418,27 +418,17 @@ namespace dxvk {
     uint32_t getCurrentFrameId() const;
     
     /**
-     * \brief Notifies adapter about memory allocation
+     * \brief Notifies adapter about memory allocation changes
      *
      * \param [in] heap Memory heap index
-     * \param [in] bytes Allocation size
+     * \param [in] allocated Allocated size delta
+     * \param [in] used Used size delta
      */
-    void notifyMemoryAlloc(
+    void notifyMemoryStats(
             uint32_t            heap,
-            int64_t             bytes) {
-      m_adapter->notifyMemoryAlloc(heap, bytes);
-    }
-
-    /**
-     * \brief Notifies adapter about memory suballocation
-     *
-     * \param [in] heap Memory heap index
-     * \param [in] bytes Allocation size
-     */
-    void notifyMemoryUse(
-            uint32_t            heap,
-            int64_t             bytes) {
-      m_adapter->notifyMemoryUse(heap, bytes);
+            int64_t             allocated,
+            int64_t             used) {
+      m_adapter->notifyMemoryStats(heap, allocated, used);
     }
 
     /**

src/dxvk/dxvk_memory.cpp

@@ -358,8 +358,6 @@ namespace dxvk {
     }
 
     type.stats.memoryAllocated += size;
-
-    m_device->notifyMemoryAlloc(type.properties.heapIndex, size);
     return result;
   }
 
@@ -408,8 +406,6 @@ namespace dxvk {
           VkDeviceSize          size) {
     type.stats.memoryUsed += size;
 
-    m_device->notifyMemoryUse(type.properties.heapIndex, size);
-
     uint32_t chunkIndex = address >> DxvkPageAllocator::ChunkAddressBits;
 
     auto& chunk = pool.chunks[chunkIndex];
@@ -429,8 +425,6 @@ namespace dxvk {
     const DxvkDeviceMemory&     memory) {
     type.stats.memoryUsed += memory.size;
 
-    m_device->notifyMemoryUse(type.properties.heapIndex, memory.size);
-
     return DxvkMemory(this, &type, memory.buffer, memory.memory,
       DedicatedChunkAddress, memory.size, memory.mapPtr);
   }
@@ -441,8 +435,6 @@ namespace dxvk {
     std::lock_guard<dxvk::mutex> lock(m_mutex);
     memory.m_type->stats.memoryUsed -= memory.m_length;
 
-    m_device->notifyMemoryUse(memory.m_type->properties.heapIndex, -memory.m_length);
-
     if (unlikely(memory.m_address == DedicatedChunkAddress)) {
       DxvkDeviceMemory devMem;
       devMem.buffer   = memory.m_buffer;
@@ -470,7 +462,6 @@ namespace dxvk {
     vk->vkFreeMemory(vk->device(), memory.memory, nullptr);
 
     type.stats.memoryAllocated -= memory.size;
-    m_device->notifyMemoryAlloc(type.properties.heapIndex, memory.size);
   }
 
 
@@ -967,6 +958,9 @@ namespace dxvk {
   void DxvkMemoryAllocator::runWorker() {
     env::setThreadName("dxvk-memory");
 
+    // Local memory statistics that we use to compute stat deltas
+    std::array<DxvkMemoryStats, VK_MAX_MEMORY_HEAPS> heapStats = { };
+
     std::unique_lock lock(m_mutex);
 
     while (true) {
@@ -976,11 +970,29 @@ namespace dxvk {
       if (m_stopWorker)
         break;
 
-      // Periodically free unused memory chunks
+      // Periodically free unused memory chunks and update
+      // memory allocation statistics for the adapter.
       auto currentTime = high_resolution_clock::now();
 
-      for (uint32_t i = 0; i < m_memHeapCount; i++)
+      for (uint32_t i = 0; i < m_memHeapCount; i++) {
         freeEmptyChunksInHeap(m_memHeaps[i], 0, currentTime);
+
+        DxvkMemoryStats stats = getMemoryStats(i);
+
+        m_device->notifyMemoryStats(i,
+          stats.memoryAllocated - heapStats[i].memoryAllocated,
+          stats.memoryUsed - heapStats[i].memoryUsed);
+
+        heapStats[i] = stats;
+      }
+    }
+
+    // Ensure adapter allocation statistics are consistent
+    // when the deivce is being destroyed
+    for (uint32_t i = 0; i < m_memHeapCount; i++) {
+      m_device->notifyMemoryStats(i,
+        -heapStats[i].memoryAllocated,
+        -heapStats[i].memoryUsed);
     }
   }