[dxvk] Implemented new memory allocator with sub-allocation

src/dxvk/dxvk_include.h

@@ -6,6 +6,7 @@
 #include "../util/util_env.h"
 #include "../util/util_error.h"
 #include "../util/util_flags.h"
+#include "../util/util_math.h"
 #include "../util/util_string.h"
 
 #include "../util/rc/util_rc.h"

src/dxvk/dxvk_memory.cpp

@@ -8,38 +8,233 @@ namespace dxvk {
   
   
   DxvkMemory::DxvkMemory(
-    DxvkMemoryAllocator*  alloc,
+          DxvkMemoryChunk*  chunk,
+          DxvkMemoryHeap*   heap,
           VkDeviceMemory    memory,
+          VkDeviceSize      offset,
+          VkDeviceSize      length,
           void*             mapPtr)
-  : m_alloc (alloc),
+  : m_chunk   (chunk),
-    m_memory(memory),
+    m_heap    (heap),
-    m_mapPtr(mapPtr) { }
+    m_memory  (memory),
+    m_offset  (offset),
+    m_length  (length),
+    m_mapPtr  (mapPtr) { }
   
   
   DxvkMemory::DxvkMemory(DxvkMemory&& other)
-  : m_alloc (other.m_alloc),
+  : m_chunk   (std::exchange(other.m_chunk,  nullptr)),
-    m_memory(other.m_memory),
+    m_heap    (std::exchange(other.m_heap,   nullptr)),
-    m_mapPtr(other.m_mapPtr) {
+    m_memory  (std::exchange(other.m_memory, VkDeviceMemory(nullptr))),
-    other.m_alloc  = nullptr;
+    m_offset  (std::exchange(other.m_offset, 0)),
-    other.m_memory = VK_NULL_HANDLE;
+    m_length  (std::exchange(other.m_length, 0)),
-    other.m_mapPtr = nullptr;
+    m_mapPtr  (std::exchange(other.m_mapPtr, nullptr)) { }
-  }
   
   
   DxvkMemory& DxvkMemory::operator = (DxvkMemory&& other) {
-    this->m_alloc  = other.m_alloc;
+    m_chunk   = std::exchange(other.m_chunk,  nullptr);
-    this->m_memory = other.m_memory;
+    m_heap    = std::exchange(other.m_heap,   nullptr);
-    this->m_mapPtr = other.m_mapPtr;
+    m_memory  = std::exchange(other.m_memory, VkDeviceMemory(nullptr));
-    other.m_alloc  = nullptr;
+    m_offset  = std::exchange(other.m_offset, 0);
-    other.m_memory = VK_NULL_HANDLE;
+    m_length  = std::exchange(other.m_length, 0);
-    other.m_mapPtr = nullptr;
+    m_mapPtr  = std::exchange(other.m_mapPtr, nullptr);
     return *this;
   }
   
   
   DxvkMemory::~DxvkMemory() {
-    if (m_memory != VK_NULL_HANDLE)
+    if (m_chunk != nullptr)
-      m_alloc->freeMemory(m_memory);
+      m_heap->free(m_chunk, m_offset, m_length);
+    else if (m_heap != nullptr)
+      m_heap->freeDeviceMemory(m_memory);
+  }
+  
+
+  DxvkMemoryChunk::DxvkMemoryChunk(
+          DxvkMemoryHeap* heap,
+          VkDeviceMemory  memory,
+          void*           mapPtr,
+          VkDeviceSize    size)
+  : m_heap  (heap),
+    m_memory(memory),
+    m_mapPtr(mapPtr),
+    m_size  (size),
+    m_free  (size) {
+    TRACE(this);
+    // Mark the entire chunk as free
+    m_freeList.push_back(FreeSlice { 0, size });
+  }
+  
+  
+  DxvkMemoryChunk::~DxvkMemoryChunk() {
+    TRACE(this);
+    m_heap->freeDeviceMemory(m_memory);
+  }
+  
+  
+  DxvkMemory DxvkMemoryChunk::alloc(VkDeviceSize size, VkDeviceSize align) {
+    // Fast exit if the chunk is full already
+    if (size > m_free)
+      return DxvkMemory();
+    
+    // Select the slice to allocate from in a worst-fit
+    // manner. This may help keep fragmentation low.
+    auto bestSlice = m_freeList.begin();
+    
+    for (auto slice = m_freeList.begin(); slice != m_freeList.end(); slice++) {
+      if (slice->length > bestSlice->length)
+        bestSlice = slice;
+    }
+    
+    // We need to align the allocation to the requested alignment
+    const VkDeviceSize sliceStart = bestSlice->offset;
+    const VkDeviceSize sliceEnd   = bestSlice->offset + bestSlice->length;
+    
+    const VkDeviceSize allocStart = dxvk::align(sliceStart,        align);
+    const VkDeviceSize allocEnd   = dxvk::align(allocStart + size, align);
+    
+    if (allocEnd > sliceEnd)
+      return DxvkMemory();
+    
+    // We can use this slice, but we'll have to add
+    // the unused parts of it back to the free list.
+    m_freeList.erase(bestSlice);
+    m_free -= size;
+    
+    if (allocStart != sliceStart)
+      m_freeList.push_back({ sliceStart, allocStart - sliceStart });
+    
+    if (allocEnd != sliceEnd)
+      m_freeList.push_back({ allocEnd, sliceEnd - allocEnd });
+    
+    // Create the memory object with the aligned slice
+    return DxvkMemory(this, m_heap,
+      m_memory, allocStart, allocEnd - allocStart,
+      reinterpret_cast<char*>(m_mapPtr) + allocStart);
+  }
+  
+  
+  void DxvkMemoryChunk::free(
+          VkDeviceSize  offset,
+          VkDeviceSize  length) {
+    m_free += length;
+    
+    // Remove adjacent entries from the free list and then add
+    // a new slice that covers all those entries. Without doing
+    // so, the slice could not be reused for larger allocations.
+    auto curr = m_freeList.begin();
+    
+    while (curr != m_freeList.end()) {
+      if (curr->offset == offset + length) {
+        length += curr->length;
+        curr = m_freeList.erase(curr);
+      } else if (curr->offset + curr->length == offset) {
+        offset -= curr->length;
+        length += curr->length;
+        curr = m_freeList.erase(curr);
+      } else {
+        curr++;
+      }
+    }
+    
+    m_freeList.push_back({ offset, length });
+  }
+  
+  
+  DxvkMemoryHeap::DxvkMemoryHeap(
+    const Rc<vk::DeviceFn>    vkd,
+          uint32_t            memTypeId,
+          VkMemoryType        memType)
+  : m_vkd       (vkd),
+    m_memTypeId (memTypeId),
+    m_memType   (memType) {
+    
+  }
+  
+  
+  DxvkMemoryHeap::~DxvkMemoryHeap() {
+    
+  }
+  
+  
+  DxvkMemory DxvkMemoryHeap::alloc(VkDeviceSize size, VkDeviceSize align) {
+    // We don't sub-allocate large allocations from one of the
+    // chunks since that might lead to severe fragmentation.
+    if (size >= (m_chunkSize / 4)) {
+      VkDeviceMemory memory = this->allocDeviceMemory(size);
+      void*          mapPtr = this->mapDeviceMemory(memory);
+      
+      return DxvkMemory(nullptr, this, memory, 0, size, mapPtr);
+    } else {
+      std::lock_guard<std::mutex> lock(m_mutex);
+      
+      // Probe chunks in a first-fit manner
+      for (const auto& chunk : m_chunks) {
+        DxvkMemory memory = chunk->alloc(size, align);
+        
+        if (memory.memory() != VK_NULL_HANDLE)
+          return memory;
+      }
+      
+      // None of the existing chunks could satisfy
+      // the request, we need to create a new one
+      VkDeviceMemory chunkMem = this->allocDeviceMemory(m_chunkSize);
+      void*          chunkPtr = this->mapDeviceMemory(chunkMem);
+      
+      Rc<DxvkMemoryChunk> newChunk = new DxvkMemoryChunk(
+        this, chunkMem, chunkPtr, m_chunkSize);
+      DxvkMemory memory = newChunk->alloc(size, align);
+      
+      m_chunks.push_back(std::move(newChunk));
+      return memory;
+    }
+  }
+  
+  
+  VkDeviceMemory DxvkMemoryHeap::allocDeviceMemory(VkDeviceSize memorySize) {
+    VkMemoryAllocateInfo info;
+    info.sType            = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
+    info.pNext            = nullptr;
+    info.allocationSize   = memorySize;
+    info.memoryTypeIndex  = m_memTypeId;
+    
+    VkDeviceMemory memory = VK_NULL_HANDLE;
+    
+    if (m_vkd->vkAllocateMemory(m_vkd->device(),
+        &info, nullptr, &memory) != VK_SUCCESS)
+      return VK_NULL_HANDLE;
+    
+    return memory;
+  }
+  
+  
+  void DxvkMemoryHeap::freeDeviceMemory(VkDeviceMemory memory) {
+    m_vkd->vkFreeMemory(m_vkd->device(), memory, nullptr);
+  }
+  
+  
+  void* DxvkMemoryHeap::mapDeviceMemory(VkDeviceMemory memory) {
+    if ((m_memType.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
+      return nullptr;
+    
+    void* ptr = nullptr;
+    
+    VkResult status = m_vkd->vkMapMemory(m_vkd->device(),
+      memory, 0, VK_WHOLE_SIZE, 0, &ptr);
+    
+    if (status != VK_SUCCESS) {
+      Logger::err("DxvkMemoryHeap: Failed to map memory");
+      return nullptr;
+    } return ptr;
+  }
+  
+  
+  void DxvkMemoryHeap::free(
+          DxvkMemoryChunk*  chunk,
+          VkDeviceSize      offset,
+          VkDeviceSize      length) {
+    std::lock_guard<std::mutex> lock(m_mutex);
+    chunk->free(offset, length);
   }
   
   
@@ -47,7 +242,8 @@ namespace dxvk {
     const Rc<DxvkAdapter>&  adapter,
     const Rc<vk::DeviceFn>& vkd)
   : m_vkd(vkd), m_memProps(adapter->memoryProperties()) {
-    
+    for (uint32_t i = 0; i < m_memProps.memoryTypeCount; i++)
+      m_heaps[i] = new DxvkMemoryHeap(m_vkd, i, m_memProps.memoryTypes[i]);
   }
   
   
@@ -60,52 +256,21 @@ namespace dxvk {
     const VkMemoryRequirements& req,
     const VkMemoryPropertyFlags flags) {
     
-    VkDeviceMemory memory = VK_NULL_HANDLE;
+    for (uint32_t i = 0; i < m_heaps.size(); i++) {
-    void*          mapPtr = nullptr;
-    
-    for (uint32_t i = 0; i < m_memProps.memoryTypeCount; i++) {
       const bool supported = (req.memoryTypeBits & (1u << i)) != 0;
       const bool adequate  = (m_memProps.memoryTypes[i].propertyFlags & flags) == flags;
       
       if (supported && adequate) {
-        memory = this->allocMemory(req.size, i);
+        DxvkMemory memory = m_heaps.at(i)->alloc(req.size, req.alignment);
         
-        if (memory != VK_NULL_HANDLE)
+        if (memory.memory() != VK_NULL_HANDLE)
-          break;
-      }
-    }
-    
-    if (memory == VK_NULL_HANDLE)
-      throw DxvkError("DxvkMemoryAllocator::alloc: Failed to allocate memory");
-    
-    if (flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
-      if (m_vkd->vkMapMemory(m_vkd->device(), memory,
-          0, VK_WHOLE_SIZE, 0, &mapPtr) != VK_SUCCESS)
-        throw DxvkError("DxvkMemoryAllocator::alloc: Failed to map memory");
-    }
-    
-    return DxvkMemory(this, memory, mapPtr);
-  }
-  
-  
-  VkDeviceMemory DxvkMemoryAllocator::allocMemory(
-    VkDeviceSize blockSize, uint32_t memoryType) {
-    VkMemoryAllocateInfo info;
-    info.sType            = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
-    info.pNext            = nullptr;
-    info.allocationSize   = blockSize;
-    info.memoryTypeIndex  = memoryType;
-    
-    VkDeviceMemory memory = VK_NULL_HANDLE;
-    if (m_vkd->vkAllocateMemory(m_vkd->device(),
-        &info, nullptr, &memory) != VK_SUCCESS)
-      return VK_NULL_HANDLE;
           return memory;
       }
+    }
     
-  
+    throw DxvkError(str::format(
-  void DxvkMemoryAllocator::freeMemory(VkDeviceMemory memory) {
+      "DxvkMemoryAllocator: Failed to allocate ",
-    m_vkd->vkFreeMemory(m_vkd->device(), memory, nullptr);
+      req.size, " bytes"));
   }
   
 }

src/dxvk/dxvk_memory.h

@@ -4,11 +4,16 @@
 
 namespace dxvk {
   
+  class DxvkMemoryHeap;
+  class DxvkMemoryChunk;
   class DxvkMemoryAllocator;
   
   
   /**
    * \brief Memory slice
+   * 
+   * Represents a slice of memory that has
+   * been sub-allocated from a bigger chunk.
    */
   class DxvkMemory {
     
@@ -16,8 +21,11 @@ namespace dxvk {
     
     DxvkMemory();
     DxvkMemory(
-      DxvkMemoryAllocator*  alloc,
+      DxvkMemoryChunk*  chunk,
+      DxvkMemoryHeap*   heap,
       VkDeviceMemory    memory,
+      VkDeviceSize      offset,
+      VkDeviceSize      length,
       void*             mapPtr);
     DxvkMemory             (DxvkMemory&& other);
     DxvkMemory& operator = (DxvkMemory&& other);
@@ -35,14 +43,14 @@ namespace dxvk {
     }
     
     /**
-     * \brief Offset from memory object
+     * \brief Offset into device memory
      * 
      * This information is required when
      * binding memory to Vulkan objects.
-     * \returns Offset from memory object
+     * \returns Offset into device memory
      */
     VkDeviceSize offset() const {
-      return 0;
+      return m_offset;
     }
     
     /**
@@ -57,13 +65,141 @@ namespace dxvk {
     
   private:
     
-    DxvkMemoryAllocator*  m_alloc  = nullptr;
+    DxvkMemoryChunk*  m_chunk  = nullptr;
+    DxvkMemoryHeap*   m_heap   = nullptr;
     VkDeviceMemory    m_memory = VK_NULL_HANDLE;
+    VkDeviceSize      m_offset = 0;
+    VkDeviceSize      m_length = 0;
     void*             m_mapPtr = nullptr;
     
   };
   
   
+  /**
+   * \brief Memory chunk
+   * 
+   * A single chunk of memory that provides a
+   * sub-allocator. This is not thread-safe.
+   */
+  class DxvkMemoryChunk : public RcObject {
+    
+  public:
+    
+    DxvkMemoryChunk(
+            DxvkMemoryHeap* heap,
+            VkDeviceMemory  memory,
+            void*           mapPtr,
+            VkDeviceSize    size);
+    
+    ~DxvkMemoryChunk();
+    
+    /**
+     * \brief Allocates memory from the chunk
+     * 
+     * On failure, this returns a slice with
+     * \c VK_NULL_HANDLE as the memory handle.
+     * \param [in] size Number of bytes to allocate
+     * \param [in] align Required alignment
+     * \returns The allocated memory slice
+     */
+    DxvkMemory alloc(
+            VkDeviceSize size,
+            VkDeviceSize align);
+    
+    /**
+     * \brief Frees memory
+     * 
+     * Returns a slice back to the chunk.
+     * Called automatically when a memory
+     * slice runs out of scope.
+     * \param [in] offset Slice offset
+     * \param [in] length Slice length
+     */
+    void free(
+            VkDeviceSize  offset,
+            VkDeviceSize  length);
+    
+  private:
+    
+    struct FreeSlice {
+      VkDeviceSize offset;
+      VkDeviceSize length;
+    };
+    
+    DxvkMemoryHeap* const m_heap;
+    VkDeviceMemory  const m_memory;
+    void*           const m_mapPtr;
+    VkDeviceSize    const m_size;
+    VkDeviceSize          m_free = 0;
+    
+    std::vector<FreeSlice> m_freeList;
+    
+  };
+  
+  
+  /**
+   * \brief Memory heap
+   * 
+   * Implements a memory allocator for a single
+   * memory type. This class is thread-safe.
+   */
+  class DxvkMemoryHeap : public RcObject {
+    friend class DxvkMemory;
+    friend class DxvkMemoryChunk;
+  public:
+    
+    DxvkMemoryHeap(
+      const Rc<vk::DeviceFn>    vkd,
+            uint32_t            memTypeId,
+            VkMemoryType        memType);
+    
+    DxvkMemoryHeap             (DxvkMemoryHeap&&) = delete;
+    DxvkMemoryHeap& operator = (DxvkMemoryHeap&&) = delete;
+    
+    ~DxvkMemoryHeap();
+    
+    /**
+     * \brief Allocates memory from the heap
+     * 
+     * Unless the requested allocation size is big
+     * enough to justify a dedicated device allocation,
+     * this will try to sub-allocate the block from an
+     * existing chunk and create new chunks as necessary.
+     * \param [in] size Amount of memory to allocate
+     * \param [in] align Alignment requirements
+     * \returns The allocated memory slice
+     */
+    DxvkMemory alloc(
+            VkDeviceSize size,
+            VkDeviceSize align);
+    
+  private:
+    
+    const Rc<vk::DeviceFn>           m_vkd;
+    const uint32_t                   m_memTypeId;
+    const VkMemoryType               m_memType;
+    const VkDeviceSize               m_chunkSize = 16 * 1024 * 1024;
+    
+    std::mutex                       m_mutex;
+    std::vector<Rc<DxvkMemoryChunk>> m_chunks;
+    
+    VkDeviceMemory allocDeviceMemory(
+            VkDeviceSize    memorySize);
+    
+    void freeDeviceMemory(
+            VkDeviceMemory  memory);
+    
+    void* mapDeviceMemory(
+            VkDeviceMemory  memory);
+    
+    void free(
+            DxvkMemoryChunk*  chunk,
+            VkDeviceSize      offset,
+            VkDeviceSize      length);
+    
+  };
+  
+  
   /**
    * \brief Memory allocator
    * 
@@ -95,12 +231,7 @@ namespace dxvk {
     const Rc<vk::DeviceFn>                 m_vkd;
     const VkPhysicalDeviceMemoryProperties m_memProps;
     
-    VkDeviceMemory allocMemory(
+    std::array<Rc<DxvkMemoryHeap>, VK_MAX_MEMORY_TYPES> m_heaps;
-      VkDeviceSize    blockSize,
-      uint32_t        memoryType);
-    
-    void freeMemory(
-      VkDeviceMemory  memory);
     
   };