[dxvk] Use DxvkResourceAllocation internally

Changes DxvkMemory to be nothing more than a wrapper.

src/dxvk/dxvk_memory.cpp

@@ -8,62 +8,6 @@
 
 namespace dxvk {
 
-  DxvkMemory::DxvkMemory() { }
-  DxvkMemory::DxvkMemory(
-          DxvkMemoryAllocator*  alloc,
-          DxvkMemoryType*       type,
-          VkBuffer              buffer,
-          VkDeviceMemory        memory,
-          VkDeviceSize          address,
-          VkDeviceSize          length,
-          void*                 mapPtr)
-  : m_alloc   (alloc),
-    m_type    (type),
-    m_buffer  (buffer),
-    m_memory  (memory),
-    m_address (address),
-    m_length  (length),
-    m_mapPtr  (mapPtr) {
-
-  }
-  
-  
-  DxvkMemory::DxvkMemory(DxvkMemory&& other)
-  : m_alloc   (std::exchange(other.m_alloc,  nullptr)),
-    m_type    (std::exchange(other.m_type,   nullptr)),
-    m_buffer  (std::exchange(other.m_buffer, VkBuffer(VK_NULL_HANDLE))),
-    m_memory  (std::exchange(other.m_memory, VkDeviceMemory(VK_NULL_HANDLE))),
-    m_address (std::exchange(other.m_address, 0)),
-    m_length  (std::exchange(other.m_length, 0)),
-    m_mapPtr  (std::exchange(other.m_mapPtr, nullptr)) { }
-  
-  
-  DxvkMemory& DxvkMemory::operator = (DxvkMemory&& other) {
-    this->free();
-    m_alloc   = std::exchange(other.m_alloc,  nullptr);
-    m_type    = std::exchange(other.m_type,   nullptr);
-    m_buffer  = std::exchange(other.m_buffer, VkBuffer(VK_NULL_HANDLE));
-    m_memory  = std::exchange(other.m_memory, VkDeviceMemory(VK_NULL_HANDLE));
-    m_address = std::exchange(other.m_address, 0);
-    m_length  = std::exchange(other.m_length, 0);
-    m_mapPtr  = std::exchange(other.m_mapPtr, nullptr);
-    return *this;
-  }
-  
-  
-  DxvkMemory::~DxvkMemory() {
-    this->free();
-  }
-  
-  
-  void DxvkMemory::free() {
-    if (m_alloc != nullptr)
-      m_alloc->free(*this);
-  }
-
-
-
-
   DxvkResourceBufferViewMap::DxvkResourceBufferViewMap(
           DxvkMemoryAllocator*        allocator,
           VkBuffer                    buffer)
@@ -176,11 +120,6 @@ namespace dxvk {
 
 
 
-  DxvkResourceAllocation::DxvkResourceAllocation() {
-
-  }
-
-
   DxvkResourceAllocation::~DxvkResourceAllocation() {
     if (m_buffer) {
       if (unlikely(m_bufferViews))
@@ -328,31 +267,32 @@ namespace dxvk {
     // fails, we may still fall back to a suballocation unless a
     // dedicated allocation is explicitly required.
     if (unlikely(info.dedicated.buffer || info.dedicated.image)) {
-      DxvkMemory memory = allocateDedicatedMemory(
+      Rc<DxvkResourceAllocation> allocation = allocateDedicatedMemory(
         req.core.memoryRequirements, info.flags, &info.dedicated);
 
-      if (memory)
+      if (allocation)
-        return memory;
+        return DxvkMemory(std::move(allocation));
 
       if (req.dedicated.requiresDedicatedAllocation && (info.flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) {
-        return allocateDedicatedMemory(req.core.memoryRequirements,
+        allocation = allocateDedicatedMemory(req.core.memoryRequirements,
           info.flags & ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, &info.dedicated);
+        return DxvkMemory(std::move(allocation));
       }
     }
 
     // Suballocate memory from an existing chunk
-    DxvkMemory memory = allocateMemory(req.core.memoryRequirements, info.flags);
+    Rc<DxvkResourceAllocation> allocation = allocateMemory(req.core.memoryRequirements, info.flags);
 
-    if (unlikely(!memory) && (info.flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) {
+    if (unlikely(!allocation) && (info.flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) {
-      memory = allocateMemory(req.core.memoryRequirements,
+      allocation = allocateMemory(req.core.memoryRequirements,
         info.flags & ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
     }
 
-    return memory;
+    return DxvkMemory(std::move(allocation));
   }
 
 
-  DxvkMemory DxvkMemoryAllocator::allocateMemory(
+  Rc<DxvkResourceAllocation> DxvkMemoryAllocator::allocateMemory(
     const VkMemoryRequirements&             requirements,
           VkMemoryPropertyFlags             properties) {
     std::lock_guard<dxvk::mutex> lock(m_mutex);
@@ -375,7 +315,7 @@ namespace dxvk {
       int64_t address = selectedPool.alloc(size, requirements.alignment);
 
       if (likely(address >= 0))
-        return createMemory(type, selectedPool, address, size);
+        return createAllocation(type, selectedPool, address, size);
 
       // If the memory type is host-visible, try to find an existing chunk
       // in the other memory pool of the memory type and move over.
@@ -400,7 +340,7 @@ namespace dxvk {
           address = selectedPool.alloc(size, requirements.alignment);
 
           if (likely(address >= 0))
-            return createMemory(type, selectedPool, address, size);
+            return createAllocation(type, selectedPool, address, size);
         }
       }
 
@@ -423,7 +363,7 @@ namespace dxvk {
           continue;
 
         mapDeviceMemory(memory, properties);
-        return createMemory(type, memory);
+        return createAllocation(type, memory);
       }
 
       // Try to allocate a new chunk that is large enough to hold
@@ -435,18 +375,18 @@ namespace dxvk {
 
       if (allocateChunkInPool(type, selectedPool, properties, size, desiredSize)) {
         address = selectedPool.alloc(size, requirements.alignment);
-        return createMemory(type, selectedPool, address, size);
+        return createAllocation(type, selectedPool, address, size);
       }
     }
 
     logMemoryError(requirements);
     logMemoryStats();
 
-    return DxvkMemory();
+    return nullptr;
   }
 
 
-  DxvkMemory DxvkMemoryAllocator::allocateDedicatedMemory(
+  Rc<DxvkResourceAllocation> DxvkMemoryAllocator::allocateDedicatedMemory(
     const VkMemoryRequirements&             requirements,
           VkMemoryPropertyFlags             properties,
     const void*                             next) {
@@ -460,14 +400,14 @@ namespace dxvk {
 
       if (likely(memory.memory != VK_NULL_HANDLE)) {
         mapDeviceMemory(memory, properties);
-        return createMemory(type, memory);
+        return createAllocation(type, memory);
       }
     }
 
     logMemoryError(requirements);
     logMemoryStats();
 
-    return DxvkMemory();
+    return nullptr;
   }
 
 
@@ -612,7 +552,7 @@ namespace dxvk {
   }
 
 
-  DxvkMemory DxvkMemoryAllocator::createMemory(
+  Rc<DxvkResourceAllocation> DxvkMemoryAllocator::createAllocation(
           DxvkMemoryType&       type,
           DxvkMemoryPool&       pool,
           VkDeviceSize          address,
@@ -624,46 +564,43 @@ namespace dxvk {
     auto& chunk = pool.chunks[chunkIndex];
     chunk.unusedTime = high_resolution_clock::time_point();
 
-    void* mapPtr = chunk.memory.mapPtr
+    VkDeviceSize offset = address & DxvkPageAllocator::ChunkAddressMask;
-      ? reinterpret_cast<char*>(chunk.memory.mapPtr) + (address & DxvkPageAllocator::ChunkAddressMask)
-      : nullptr;
 
-    return DxvkMemory(this, &type, chunk.memory.buffer,
+    auto allocation = m_allocationPool.create(this, &type);
-      chunk.memory.memory, address, size, mapPtr);
+    allocation->m_memory = chunk.memory.memory;
+    allocation->m_address = address;
+    allocation->m_size = size;
+
+    if (chunk.memory.mapPtr)
+      allocation->m_mapPtr = reinterpret_cast<char*>(chunk.memory.mapPtr) + offset;
+
+    if (chunk.memory.buffer) {
+      allocation->m_buffer = chunk.memory.buffer;
+      allocation->m_bufferOffset = offset;
+    }
+
+    return allocation;
   }
 
 
-  DxvkMemory DxvkMemoryAllocator::createMemory(
+  Rc<DxvkResourceAllocation> DxvkMemoryAllocator::createAllocation(
           DxvkMemoryType&       type,
     const DxvkDeviceMemory&     memory) {
     type.stats.memoryUsed += memory.size;
 
-    return DxvkMemory(this, &type, memory.buffer, memory.memory,
+    auto allocation = m_allocationPool.create(this, &type);
-      DedicatedChunkAddress, memory.size, memory.mapPtr);
+    allocation->m_flags.set(DxvkAllocationFlag::OwnsMemory);
-  }
 
+    if (memory.buffer)
+      allocation->m_flags.set(DxvkAllocationFlag::OwnsBuffer);
 
-  void DxvkMemoryAllocator::free(
+    allocation->m_memory = memory.memory;
-    const DxvkMemory&           memory) {
+    allocation->m_address = DedicatedChunkAddress;
-    std::lock_guard<dxvk::mutex> lock(m_mutex);
+    allocation->m_size = memory.size;
-    memory.m_type->stats.memoryUsed -= memory.m_length;
+    allocation->m_mapPtr = memory.mapPtr;
 
-    if (unlikely(memory.m_address == DedicatedChunkAddress)) {
+    allocation->m_buffer = memory.buffer;
-      DxvkDeviceMemory devMem;
+    return allocation;
-      devMem.buffer   = memory.m_buffer;
-      devMem.memory   = memory.m_memory;
-      devMem.mapPtr   = nullptr;
-      devMem.size     = memory.m_length;
-
-      this->freeDeviceMemory(*memory.m_type, devMem);
-    } else {
-      DxvkMemoryPool& pool = memory.m_mapPtr
-        ? memory.m_type->mappedPool
-        : memory.m_type->devicePool;
-
-      if (unlikely(pool.free(memory.m_address, memory.m_length)))
-        freeEmptyChunksInPool(*memory.m_type, pool, 0, high_resolution_clock::now());
-    }
   }
 
 

src/dxvk/dxvk_memory.h

@@ -433,9 +433,13 @@ namespace dxvk {
    */
   class alignas(CACHE_LINE_SIZE) DxvkResourceAllocation {
     friend DxvkMemoryAllocator;
+    friend class DxvkMemory;
   public:
 
-    DxvkResourceAllocation();
+    DxvkResourceAllocation(
+            DxvkMemoryAllocator*        allocator,
+            DxvkMemoryType*             type)
+    : m_allocator(allocator), m_type(type) { }
 
     ~DxvkResourceAllocation();
 
@@ -650,22 +654,16 @@ namespace dxvk {
    * Represents a slice of memory that has
    * been sub-allocated from a bigger chunk.
    */
-  class DxvkMemory {
+  struct DxvkMemory {
-    friend class DxvkMemoryAllocator;
+    DxvkMemory() = default;
-  public:
 
-    DxvkMemory();
+    explicit DxvkMemory(Rc<DxvkResourceAllocation>&& allocation_)
-    DxvkMemory(
+    : allocation(std::move(allocation_)) { }
-      DxvkMemoryAllocator*  alloc,
+
-      DxvkMemoryType*       type,
+    DxvkMemory(DxvkMemory&& other) = default;
-      VkBuffer              buffer,
+    DxvkMemory& operator = (DxvkMemory&& other) = default;
-      VkDeviceMemory        memory,
+
-      VkDeviceSize          address,
+    ~DxvkMemory() = default;
-      VkDeviceSize          length,
-      void*                 mapPtr);
-    DxvkMemory             (DxvkMemory&& other);
-    DxvkMemory& operator = (DxvkMemory&& other);
-    ~DxvkMemory();
     
     /**
      * \brief Memory object
@@ -675,7 +673,7 @@ namespace dxvk {
      * \returns Memory object
      */
     VkDeviceMemory memory() const {
-      return m_memory;
+      return allocation ? allocation->m_memory : VK_NULL_HANDLE;
     }
     
     /**
@@ -685,7 +683,7 @@ namespace dxvk {
      * \returns Buffer object
      */
     VkBuffer buffer() const {
-      return m_buffer;
+      return allocation ? allocation->m_buffer : VK_NULL_HANDLE;
     }
 
     /**
@@ -696,7 +694,9 @@ namespace dxvk {
      * \returns Offset into device memory
      */
     VkDeviceSize offset() const {
-      return m_address & DxvkPageAllocator::ChunkAddressMask;
+      return allocation
+        ? allocation->m_address & DxvkPageAllocator::ChunkAddressMask
+        : 0u;
     }
     
     /**
@@ -706,16 +706,17 @@ namespace dxvk {
      * \returns Pointer to mapped data
      */
     void* mapPtr(VkDeviceSize offset) const {
-      return reinterpret_cast<char*>(m_mapPtr) + offset;
+      return allocation && allocation->m_mapPtr
+        ? reinterpret_cast<char*>(allocation->m_mapPtr) + offset
+        : nullptr;
     }
 
     /**
      * \brief Returns length of memory allocated
-     * 
      * \returns Memory size
      */
     VkDeviceSize length() const {
-      return m_length;
+      return allocation ? allocation->m_size : 0u;
     }
 
     /**
@@ -725,7 +726,7 @@ namespace dxvk {
      *          memory, and \c false if it is undefined.
      */
     explicit operator bool () const {
-      return m_memory != VK_NULL_HANDLE;
+      return bool(allocation);
     }
 
     /**
@@ -733,21 +734,12 @@ namespace dxvk {
      * \returns Global buffer usage flags, if any
      */
     VkBufferUsageFlags getBufferUsage() const {
-      return m_buffer ? m_type->bufferUsage : 0u;
+      return allocation && allocation->m_type
+        ? allocation->m_type->bufferUsage
+        : 0u;
     }
 
-  private:
+    Rc<DxvkResourceAllocation> allocation;
-    
-    DxvkMemoryAllocator*  m_alloc  = nullptr;
-    DxvkMemoryType*       m_type   = nullptr;
-    VkBuffer              m_buffer = VK_NULL_HANDLE;
-    VkDeviceMemory        m_memory = VK_NULL_HANDLE;
-    VkDeviceSize          m_address = 0;
-    VkDeviceSize          m_length = 0;
-    void*                 m_mapPtr = nullptr;
-    
-    void free();
-    
   };
 
 
@@ -807,8 +799,9 @@ namespace dxvk {
      * \param [in] requirements Memory requirements
      * \param [in] properties Memory property flags. Some of
      *    these may be ignored in case of memory pressure.
+     * \returns Allocated memory
      */
-    DxvkMemory allocateMemory(
+    Rc<DxvkResourceAllocation> allocateMemory(
       const VkMemoryRequirements&             requirements,
             VkMemoryPropertyFlags             properties);
 
@@ -820,8 +813,9 @@ namespace dxvk {
      * \param [in] properties Memory property flags. Some of
      *    these may be ignored in case of memory pressure.
      * \param [in] next Further memory properties
+     * \returns Allocated memory
      */
-    DxvkMemory allocateDedicatedMemory(
+    Rc<DxvkResourceAllocation> allocateDedicatedMemory(
       const VkMemoryRequirements&             requirements,
             VkMemoryPropertyFlags             properties,
       const void*                             next);
@@ -903,15 +897,6 @@ namespace dxvk {
             VkDeviceSize          requiredSize,
             VkDeviceSize          desiredSize);
 
-    DxvkMemory createMemory(
-            DxvkMemoryType&       type,
-            DxvkMemoryPool&       pool,
-            VkDeviceSize          address,
-            VkDeviceSize          size);
-
-    void free(
-      const DxvkMemory&           memory);
-
     void freeDeviceMemory(
             DxvkMemoryType&       type,
             DxvkDeviceMemory      memory);
@@ -942,13 +927,13 @@ namespace dxvk {
             DxvkDeviceMemory&     memory,
             VkMemoryPropertyFlags properties);
 
-    DxvkMemory createMemory(
+    Rc<DxvkResourceAllocation> createAllocation(
             DxvkMemoryType&       type,
-      const DxvkMemoryPool&       pool,
+            DxvkMemoryPool&       pool,
             VkDeviceSize          address,
             VkDeviceSize          size);
 
-    DxvkMemory createMemory(
+    Rc<DxvkResourceAllocation> createAllocation(
             DxvkMemoryType&       type,
       const DxvkDeviceMemory&     memory);