[dxvk] Add per-chunk allocation list

src/dxvk/dxvk_memory.cpp

@@ -8,6 +8,32 @@
 
 namespace dxvk {
 
+  void DxvkMemoryChunk::addAllocation(DxvkResourceAllocation* allocation) {
+    allocation->m_nextInChunk = allocationList;
+
+    if (allocationList)
+      allocationList->m_prevInChunk = allocation;
+
+    allocationList = allocation;
+  }
+
+
+  void DxvkMemoryChunk::removeAllocation(DxvkResourceAllocation* allocation) {
+    if (allocation->m_nextInChunk)
+      allocation->m_nextInChunk->m_prevInChunk = allocation->m_prevInChunk;
+
+    if (allocation->m_prevInChunk)
+      allocation->m_prevInChunk->m_nextInChunk = allocation->m_nextInChunk;
+    else if (allocationList == allocation)
+      allocationList = allocation->m_nextInChunk;
+
+    allocation->m_prevInChunk = nullptr;
+    allocation->m_nextInChunk = nullptr;
+  }
+
+
+
+
   DxvkResourceBufferViewMap::DxvkResourceBufferViewMap(
           DxvkMemoryAllocator*        allocator,
           VkBuffer                    buffer)
@@ -1116,12 +1142,11 @@ namespace dxvk {
     type.stats.memoryUsed += size;
 
     uint32_t chunkIndex = address >> DxvkPageAllocator::ChunkAddressBits;
+    VkDeviceSize offset = address & DxvkPageAllocator::ChunkAddressMask;
 
     auto& chunk = pool.chunks[chunkIndex];
     chunk.unusedTime = high_resolution_clock::time_point();
 
-    VkDeviceSize offset = address & DxvkPageAllocator::ChunkAddressMask;
-
     auto allocation = m_allocationPool.create(this, &type);
 
     if (!(allocationInfo.properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) && allocationInfo.resourceCookie)
@@ -1142,6 +1167,9 @@ namespace dxvk {
         ? chunk.memory.gpuVa + offset : 0u;
     }
 
+    if (&pool == &type.devicePool)
+      chunk.addAllocation(allocation);
+
     return allocation;
   }
 
@@ -1217,10 +1245,15 @@ namespace dxvk {
           // We free the actual allocation later, just update stats here.
           allocation->m_type->stats.memoryAllocated -= allocation->m_size;
         } else {
-          auto& pool = allocation->m_mapPtr
+          DxvkMemoryPool& pool = allocation->m_mapPtr
             ? allocation->m_type->mappedPool
             : allocation->m_type->devicePool;
 
+          if (!allocation->m_mapPtr) {
+            uint32_t chunkIndex = allocation->m_address >> DxvkPageAllocator::ChunkAddressBits;
+            pool.chunks[chunkIndex].removeAllocation(allocation);
+          }
+
           if (unlikely(pool.free(allocation->m_address, allocation->m_size)))
             freeEmptyChunksInPool(*allocation->m_type, pool, 0, high_resolution_clock::now());
         }

src/dxvk/dxvk_memory.h

@@ -15,6 +15,7 @@ namespace dxvk {
   class DxvkMemoryChunk;
   class DxvkSparsePageTable;
   class DxvkSharedAllocationCache;
+  class DxvkResourceAllocation;
 
   /**
    * \brief Memory stats
@@ -78,6 +79,11 @@ namespace dxvk {
     /// Time when the chunk has been marked as unused. Must
     /// be set to 0 when allocating memory from the chunk
     high_resolution_clock::time_point unusedTime = { };
+    /// Unordered list of resources suballocated from this chunk.
+    DxvkResourceAllocation* allocationList = nullptr;
+
+    void addAllocation(DxvkResourceAllocation* allocation);
+    void removeAllocation(DxvkResourceAllocation* allocation);
   };
 
   
@@ -466,6 +472,7 @@ namespace dxvk {
   class alignas(CACHE_LINE_SIZE) DxvkResourceAllocation {
     friend DxvkMemoryAllocator;
 
+    friend struct DxvkMemoryChunk;
     friend class DxvkLocalAllocationCache;
     friend class DxvkSharedAllocationCache;
   public:
@@ -611,6 +618,9 @@ namespace dxvk {
 
     DxvkResourceAllocation*     m_nextCached = nullptr;
 
+    DxvkResourceAllocation*     m_prevInChunk = nullptr;
+    DxvkResourceAllocation*     m_nextInChunk = nullptr;
+
     void destroyBufferViews();
 
     void free();
@@ -621,8 +631,6 @@ namespace dxvk {
 
   };
 
-  static_assert(sizeof(DxvkResourceAllocation) == 2u * CACHE_LINE_SIZE);
-
 
   /**
    * \brief Resource allocation pool