[dxvk] Create global buffer for allocated memory chunks

src/dxvk/dxvk_memory.cpp

@@ -12,6 +12,7 @@ namespace dxvk {
           DxvkMemoryAllocator*  alloc,
           DxvkMemoryChunk*      chunk,
           DxvkMemoryType*       type,
+          VkBuffer              buffer,
           VkDeviceMemory        memory,
           VkDeviceSize          offset,
           VkDeviceSize          length,
@@ -19,6 +20,7 @@ namespace dxvk {
   : m_alloc   (alloc),
     m_chunk   (chunk),
     m_type    (type),
+    m_buffer  (buffer),
     m_memory  (memory),
     m_offset  (offset),
     m_length  (length),
@@ -29,6 +31,7 @@ namespace dxvk {
   : m_alloc   (std::exchange(other.m_alloc,  nullptr)),
     m_chunk   (std::exchange(other.m_chunk,  nullptr)),
     m_type    (std::exchange(other.m_type,   nullptr)),
+    m_buffer  (std::exchange(other.m_buffer, VkBuffer(VK_NULL_HANDLE))),
     m_memory  (std::exchange(other.m_memory, VkDeviceMemory(VK_NULL_HANDLE))),
     m_offset  (std::exchange(other.m_offset, 0)),
     m_length  (std::exchange(other.m_length, 0)),
@@ -40,6 +43,7 @@ namespace dxvk {
     m_alloc   = std::exchange(other.m_alloc,  nullptr);
     m_chunk   = std::exchange(other.m_chunk,  nullptr);
     m_type    = std::exchange(other.m_type,   nullptr);
+    m_buffer  = std::exchange(other.m_buffer, VkBuffer(VK_NULL_HANDLE));
     m_memory  = std::exchange(other.m_memory, VkDeviceMemory(VK_NULL_HANDLE));
     m_offset  = std::exchange(other.m_offset, 0);
     m_length  = std::exchange(other.m_length, 0);
@@ -126,7 +130,7 @@ namespace dxvk {
     
     // Create the memory object with the aligned slice
     return DxvkMemory(m_alloc, this, m_type,
-      m_memory.memHandle, allocStart, allocEnd - allocStart,
+      m_memory.buffer, m_memory.memHandle, allocStart, allocEnd - allocStart,
       reinterpret_cast<char*>(m_memory.memPointer) + allocStart);
   }
   
@@ -195,10 +199,13 @@ namespace dxvk {
       m_memTypes[i].memType    = m_memProps.memoryTypes[i];
       m_memTypes[i].memTypeId  = i;
       m_memTypes[i].chunkSize  = MinChunkSize;
+      m_memTypes[i].bufferUsage = 0;
     }
 
     if (device->features().core.features.sparseBinding)
       m_sparseMemoryTypes = determineSparseMemoryTypes(device);
+
+    determineBufferUsageFlagsPerMemoryType();
   }
   
   
@@ -366,8 +373,10 @@ namespace dxvk {
 
       DxvkDeviceMemory devMem = this->tryAllocDeviceMemory(type, size, info, hints);
 
-      if (devMem.memHandle != VK_NULL_HANDLE)
+      if (devMem.memHandle != VK_NULL_HANDLE) {
-        memory = DxvkMemory(this, nullptr, type, devMem.memHandle, 0, size, devMem.memPointer);
+        memory = DxvkMemory(this, nullptr, type,
+          devMem.buffer, devMem.memHandle, 0, size, devMem.memPointer);
+      }
     }
 
     if (memory) {
@@ -396,11 +405,16 @@ namespace dxvk {
     if (hints.test(DxvkMemoryFlag::GpuWritable))
       priority = 1.0f;
 
+    bool dedicated = false;
+
     DxvkDeviceMemory result;
     result.memSize  = size;
     result.memFlags = info.flags;
     result.priority = priority;
 
+    VkMemoryAllocateFlagsInfo memoryFlags = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO };
+    memoryFlags.flags |= VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT;
+
     VkMemoryPriorityAllocateInfoEXT priorityInfo = { VK_STRUCTURE_TYPE_MEMORY_PRIORITY_ALLOCATE_INFO_EXT };
     priorityInfo.priority       = priority;
 
@@ -420,6 +434,9 @@ namespace dxvk {
     if (useMemoryPriority)
       priorityInfo.pNext = std::exchange(memoryInfo.pNext, &priorityInfo);
 
+    if (!info.dedicated.image && (type->bufferUsage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT))
+      memoryFlags.pNext = std::exchange(memoryInfo.pNext, &memoryFlags);
+
     if (vk->vkAllocateMemory(vk->device(), &memoryInfo, nullptr, &result.memHandle))
       return DxvkDeviceMemory();
     
@@ -433,6 +450,43 @@ namespace dxvk {
       }
     }
 
+    if (type->bufferUsage && !dedicated) {
+      VkBuffer buffer = VK_NULL_HANDLE;
+
+      VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
+      bufferInfo.size = result.memSize;
+      bufferInfo.usage = type->bufferUsage;
+      bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
+
+      VkResult status = vk->vkCreateBuffer(vk->device(), &bufferInfo, nullptr, &buffer);
+
+      if (status == VK_SUCCESS) {
+        VkBufferMemoryRequirementsInfo2 memInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2 };
+        memInfo.buffer = buffer;
+
+        VkMemoryRequirements2 requirements = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2 };
+        vk->vkGetBufferMemoryRequirements2(vk->device(), &memInfo, &requirements);
+
+        if ((requirements.memoryRequirements.size == size)
+         && (requirements.memoryRequirements.memoryTypeBits & (1u << type->memTypeId))) {
+          status = vk->vkBindBufferMemory(vk->device(), buffer, result.memHandle, 0);
+
+          if (status == VK_SUCCESS)
+            result.buffer = buffer;
+        }
+
+        if (!result.buffer)
+          vk->vkDestroyBuffer(vk->device(), buffer, nullptr);
+      }
+
+      if (!result.buffer) {
+        Logger::warn(str::format("Failed to create global buffer:",
+          "\n  size:  ", std::dec, size,
+          "\n  usage: ", std::hex, type->bufferUsage,
+          "\n  type:  ", std::dec, type->memTypeId));
+      }
+    }
+
     type->heap->stats.memoryAllocated += size;
     m_device->notifyMemoryAlloc(type->heapId, size);
     return result;
@@ -452,6 +506,7 @@ namespace dxvk {
         memory.m_length);
     } else {
       DxvkDeviceMemory devMem;
+      devMem.buffer     = memory.m_buffer;
       devMem.memHandle  = memory.m_memory;
       devMem.memPointer = nullptr;
       devMem.memSize    = memory.m_length;
@@ -487,6 +542,7 @@ namespace dxvk {
           DxvkMemoryType*       type,
           DxvkDeviceMemory      memory) {
     auto vk = m_device->vkd();
+    vk->vkDestroyBuffer(vk->device(), memory.buffer, nullptr);
     vk->vkFreeMemory(vk->device(), memory.memHandle, nullptr);
 
     type->heap->stats.memoryAllocated -= memory.memSize;
@@ -593,7 +649,7 @@ namespace dxvk {
           DxvkDevice*           device) const {
     auto vk = device->vkd();
 
-    VkMemoryRequirements requirements = { };
+    VkMemoryRequirements2 requirements = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2 };
     uint32_t typeMask = ~0u;
 
     // Create sparse dummy buffer to find available memory types
@@ -613,16 +669,8 @@ namespace dxvk {
                             | VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
     bufferInfo.sharingMode  = VK_SHARING_MODE_EXCLUSIVE;
 
-    VkBuffer buffer = VK_NULL_HANDLE;
+    if (getBufferMemoryRequirements(bufferInfo, requirements))
-
+      typeMask &= requirements.memoryRequirements.memoryTypeBits;
-    if (vk->vkCreateBuffer(vk->device(), &bufferInfo, nullptr, &buffer)) {
-      Logger::err("Failed to create dummy buffer to query sparse memory types");
-      return 0;
-    }
-
-    vk->vkGetBufferMemoryRequirements(vk->device(), buffer, &requirements);
-    vk->vkDestroyBuffer(vk->device(), buffer, nullptr);
-    typeMask &= requirements.memoryTypeBits;
 
     // Create sparse dummy image to find available memory types
     VkImageCreateInfo imageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO };
@@ -643,16 +691,8 @@ namespace dxvk {
                             | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
     imageInfo.sharingMode   = VK_SHARING_MODE_EXCLUSIVE;
 
-    VkImage image = VK_NULL_HANDLE;
+    if (getImageMemoryRequirements(imageInfo, requirements))
-
+      typeMask &= requirements.memoryRequirements.memoryTypeBits;
-    if (vk->vkCreateImage(vk->device(), &imageInfo, nullptr, &image)) {
-      Logger::err("Failed to create dummy image to query sparse memory types");
-      return 0;
-    }
-
-    vk->vkGetImageMemoryRequirements(vk->device(), image, &requirements);
-    vk->vkDestroyImage(vk->device(), image, nullptr);
-    typeMask &= requirements.memoryTypeBits;
 
     Logger::log(typeMask ? LogLevel::Info : LogLevel::Error,
       str::format("Memory type mask for sparse resources: 0x", std::hex, typeMask));
@@ -660,6 +700,127 @@ namespace dxvk {
   }
 
 
+  void DxvkMemoryAllocator::determineBufferUsageFlagsPerMemoryType() {
+    VkBufferUsageFlags flags = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT
+                             | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT
+                             | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT
+                             | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT
+                             | VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT;
+
+    // Lock storage texel buffer usage to maintenance5 support since we will
+    // otherwise not be able to legally use formats that support one type of
+    // texel buffer but not the other. Also lock index buffer usage since we
+    // cannot explicitly specify a buffer range otherwise.
+    if (m_device->features().khrMaintenance5.maintenance5) {
+      flags |= VK_BUFFER_USAGE_INDEX_BUFFER_BIT
+            |  VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT;
+    }
+
+    if (m_device->features().extTransformFeedback.transformFeedback) {
+      flags |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT
+            |  VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_COUNTER_BUFFER_BIT_EXT;
+    }
+
+    if (m_device->features().vk12.bufferDeviceAddress)
+      flags |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT;
+
+    // Check which individual flags are supported on each memory type. This is a
+    // bit dodgy since the spec technically does not require a combination of flags
+    // to be supported, but we need to be robust around buffer creation anyway.
+    VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
+    bufferInfo.size = 65536;
+    bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
+
+    VkMemoryRequirements2 requirements = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2 };
+
+    while (flags) {
+      VkBufferCreateFlags flag = flags & -flags;
+
+      bufferInfo.usage = flag
+        | VK_BUFFER_USAGE_TRANSFER_DST_BIT
+        | VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
+
+      if (getBufferMemoryRequirements(bufferInfo, requirements)) {
+        uint32_t typeMask = requirements.memoryRequirements.memoryTypeBits;
+
+        while (typeMask) {
+          uint32_t type = bit::tzcnt(typeMask);
+
+          if (type < m_memProps.memoryTypeCount)
+            m_memTypes.at(type).bufferUsage |= bufferInfo.usage;
+
+          typeMask &= typeMask - 1;
+        }
+      }
+
+      flags &= ~flag;
+    }
+
+    // Only use a minimal set of usage flags for the global buffer if the
+    // full combination of flags is not supported for whatever reason.
+    for (uint32_t i = 0; i < m_memProps.memoryTypeCount; i++) {
+      bufferInfo.usage = m_memTypes[i].bufferUsage;
+
+      if (!getBufferMemoryRequirements(bufferInfo, requirements)
+       || !(requirements.memoryRequirements.memoryTypeBits & (1u << i))) {
+        m_memTypes[i].bufferUsage &= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT
+                                  |  VK_BUFFER_USAGE_TRANSFER_DST_BIT
+                                  |  VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
+      }
+    }
+  }
+
+
+  bool DxvkMemoryAllocator::getBufferMemoryRequirements(
+    const VkBufferCreateInfo&     createInfo,
+          VkMemoryRequirements2&  memoryRequirements) const {
+    auto vk = m_device->vkd();
+
+    if (m_device->features().vk13.maintenance4) {
+      VkDeviceBufferMemoryRequirements info = { VK_STRUCTURE_TYPE_DEVICE_BUFFER_MEMORY_REQUIREMENTS };
+      info.pCreateInfo = &createInfo;
+
+      vk->vkGetDeviceBufferMemoryRequirements(vk->device(), &info, &memoryRequirements);
+      return true;
+    } else {
+      VkBufferMemoryRequirementsInfo2 info = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2 };
+      VkResult vr = vk->vkCreateBuffer(vk->device(), &createInfo, nullptr, &info.buffer);
+
+      if (vr != VK_SUCCESS)
+        return false;
+
+      vk->vkGetBufferMemoryRequirements2(vk->device(), &info, &memoryRequirements);
+      vk->vkDestroyBuffer(vk->device(), info.buffer, nullptr);
+      return true;
+    }
+  }
+
+
+  bool DxvkMemoryAllocator::getImageMemoryRequirements(
+    const VkImageCreateInfo&      createInfo,
+          VkMemoryRequirements2&  memoryRequirements) const {
+    auto vk = m_device->vkd();
+
+    if (m_device->features().vk13.maintenance4) {
+      VkDeviceImageMemoryRequirements info = { VK_STRUCTURE_TYPE_DEVICE_IMAGE_MEMORY_REQUIREMENTS };
+      info.pCreateInfo = &createInfo;
+
+      vk->vkGetDeviceImageMemoryRequirements(vk->device(), &info, &memoryRequirements);
+      return true;
+    } else {
+      VkImageMemoryRequirementsInfo2 info = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2 };
+      VkResult vr = vk->vkCreateImage(vk->device(), &createInfo, nullptr, &info.image);
+
+      if (vr != VK_SUCCESS)
+        return false;
+
+      vk->vkGetImageMemoryRequirements2(vk->device(), &info, &memoryRequirements);
+      vk->vkDestroyImage(vk->device(), info.image, nullptr);
+      return true;
+    }
+  }
+
+
   void DxvkMemoryAllocator::logMemoryError(const VkMemoryRequirements& req) const {
     std::stringstream sstr;
     sstr << "DxvkMemoryAllocator: Memory allocation failed" << std::endl

src/dxvk/dxvk_memory.h

@@ -49,6 +49,7 @@ namespace dxvk {
    * be persistently mapped.
    */
   struct DxvkDeviceMemory {
+    VkBuffer              buffer     = VK_NULL_HANDLE;
     VkDeviceMemory        memHandle  = VK_NULL_HANDLE;
     void*                 memPointer = nullptr;
     VkDeviceSize          memSize    = 0;
@@ -84,6 +85,7 @@ namespace dxvk {
     uint32_t          memTypeId;
 
     VkDeviceSize      chunkSize;
+    VkBufferUsageFlags bufferUsage;
 
     std::vector<Rc<DxvkMemoryChunk>> chunks;
   };
@@ -104,6 +106,7 @@ namespace dxvk {
       DxvkMemoryAllocator*  alloc,
       DxvkMemoryChunk*      chunk,
       DxvkMemoryType*       type,
+      VkBuffer              buffer,
       VkDeviceMemory        memory,
       VkDeviceSize          offset,
       VkDeviceSize          length,
@@ -123,6 +126,16 @@ namespace dxvk {
       return m_memory;
     }
     
+    /**
+     * \brief Buffer object
+     *
+     * Global buffer covering the entire memory allocation.
+     * \returns Buffer object
+     */
+    VkBuffer buffer() const {
+      return m_buffer;
+    }
+
     /**
      * \brief Offset into device memory
      * 
@@ -163,11 +176,20 @@ namespace dxvk {
       return m_memory != VK_NULL_HANDLE;
     }
 
+    /**
+     * \brief Queries global buffer usage flags
+     * \returns Global buffer usage flags, if any
+     */
+    VkBufferUsageFlags getBufferUsage() const {
+      return m_buffer ? m_type->bufferUsage : 0u;
+    }
+
   private:
     
     DxvkMemoryAllocator*  m_alloc  = nullptr;
     DxvkMemoryChunk*      m_chunk  = nullptr;
     DxvkMemoryType*       m_type   = nullptr;
+    VkBuffer              m_buffer = VK_NULL_HANDLE;
     VkDeviceMemory        m_memory = VK_NULL_HANDLE;
     VkDeviceSize          m_offset = 0;
     VkDeviceSize          m_length = 0;
@@ -355,6 +377,30 @@ namespace dxvk {
       return m_memHeaps[heap].stats;
     }
     
+    /**
+     * \brief Queries buffer memory requirements
+     *
+     * Can be used to get memory requirements without having
+     * to create a buffer object first.
+     * \param [in] createInfo Buffer create info
+     * \param [in,out] memoryRequirements Memory requirements
+     */
+    bool getBufferMemoryRequirements(
+      const VkBufferCreateInfo&     createInfo,
+            VkMemoryRequirements2&  memoryRequirements) const;
+
+    /**
+     * \brief Queries image memory requirements
+     *
+     * Can be used to get memory requirements without having
+     * to create an image object first.
+     * \param [in] createInfo Image create info
+     * \param [in,out] memoryRequirements Memory requirements
+     */
+    bool getImageMemoryRequirements(
+      const VkImageCreateInfo&      createInfo,
+            VkMemoryRequirements2&  memoryRequirements) const;
+
   private:
 
     DxvkDevice*                                     m_device;
@@ -421,6 +467,8 @@ namespace dxvk {
     uint32_t determineSparseMemoryTypes(
             DxvkDevice*           device) const;
 
+    void determineBufferUsageFlagsPerMemoryType();
+
     void logMemoryError(
       const VkMemoryRequirements& req) const;
 

src/vulkan/vulkan_loader.h

@@ -198,8 +198,10 @@ namespace dxvk::vk {
     VULKAN_FN(vkBindImageMemory);
     VULKAN_FN(vkGetBufferMemoryRequirements);
     VULKAN_FN(vkGetBufferMemoryRequirements2);
+    VULKAN_FN(vkGetDeviceBufferMemoryRequirements);
     VULKAN_FN(vkGetImageMemoryRequirements);
     VULKAN_FN(vkGetImageMemoryRequirements2);
+    VULKAN_FN(vkGetDeviceImageMemoryRequirements);
     VULKAN_FN(vkGetImageSparseMemoryRequirements);
     VULKAN_FN(vkGetImageSparseMemoryRequirements2);
     VULKAN_FN(vkQueueBindSparse);