[dxvk] Implement sparse memory allocator

src/dxvk/dxvk_device.cpp

@@ -216,6 +216,11 @@ namespace dxvk {
   }
   
   
+  Rc<DxvkSparsePageAllocator> DxvkDevice::createSparsePageAllocator() {
+    return new DxvkSparsePageAllocator(this, m_objects.memoryManager());
+  }
+
+
   DxvkStatCounters DxvkDevice::getStatCounters() {
     DxvkPipelineCount pipe = m_objects.pipelineManager().getPipelineCount();
     

src/dxvk/dxvk_device.h

@@ -22,6 +22,7 @@
 #include "dxvk_renderpass.h"
 #include "dxvk_sampler.h"
 #include "dxvk_shader.h"
+#include "dxvk_sparse.h"
 #include "dxvk_stats.h"
 #include "dxvk_unbound.h"
 #include "dxvk_marker.h"
@@ -380,6 +381,12 @@ namespace dxvk {
     Rc<DxvkSampler> createSampler(
       const DxvkSamplerCreateInfo&  createInfo);
 
+    /**
+     * \brief Creates a sparse page allocator
+     * \returns Sparse page allocator
+     */
+    Rc<DxvkSparsePageAllocator> createSparsePageAllocator();
+
     /**
      * \brief Retrieves stat counters
      * 

src/dxvk/dxvk_memory.cpp

@@ -179,9 +179,8 @@ namespace dxvk {
   }
 
 
-  DxvkMemoryAllocator::DxvkMemoryAllocator(const DxvkDevice* device)
+  DxvkMemoryAllocator::DxvkMemoryAllocator(DxvkDevice* device)
-  : m_vkd             (device->vkd()),
+  : m_device          (device),
-    m_device          (device),
     m_devProps        (device->adapter()->deviceProperties()),
     m_memProps        (device->adapter()->memoryProperties()) {
     for (uint32_t i = 0; i < m_memProps.memoryHeapCount; i++) {
@@ -202,6 +201,9 @@ namespace dxvk {
       m_memTypes[i].memType    = m_memProps.memoryTypes[i];
       m_memTypes[i].memTypeId  = i;
     }
+
+    if (device->features().core.features.sparseBinding)
+      m_sparseMemoryTypes = determineSparseMemoryTypes(device);
   }
   
   
@@ -366,6 +368,8 @@ namespace dxvk {
           VkDeviceSize                      size,
           DxvkMemoryProperties              info,
           DxvkMemoryFlags                   hints) {
+    auto vk = m_device->vkd();
+
     bool useMemoryPriority = (info.flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
                           && (m_device->features().extMemoryPriority.memoryPriority);
     
@@ -403,15 +407,15 @@ namespace dxvk {
     if (useMemoryPriority)
       priorityInfo.pNext = std::exchange(memoryInfo.pNext, &priorityInfo);
 
-    if (m_vkd->vkAllocateMemory(m_vkd->device(), &memoryInfo, nullptr, &result.memHandle) != VK_SUCCESS)
+    if (vk->vkAllocateMemory(vk->device(), &memoryInfo, nullptr, &result.memHandle))
       return DxvkDeviceMemory();
     
     if (info.flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
-      VkResult status = m_vkd->vkMapMemory(m_vkd->device(), result.memHandle, 0, VK_WHOLE_SIZE, 0, &result.memPointer);
+      VkResult status = vk->vkMapMemory(vk->device(), result.memHandle, 0, VK_WHOLE_SIZE, 0, &result.memPointer);
 
-      if (status != VK_SUCCESS) {
+      if (status) {
         Logger::err(str::format("DxvkMemoryAllocator: Mapping memory failed with ", status));
-        m_vkd->vkFreeMemory(m_vkd->device(), result.memHandle, nullptr);
+        vk->vkFreeMemory(vk->device(), result.memHandle, nullptr);
         return DxvkDeviceMemory();
       }
     }
@@ -467,7 +471,9 @@ namespace dxvk {
   void DxvkMemoryAllocator::freeDeviceMemory(
           DxvkMemoryType*       type,
           DxvkDeviceMemory      memory) {
-    m_vkd->vkFreeMemory(m_vkd->device(), memory.memHandle, nullptr);
+    auto vk = m_device->vkd();
+    vk->vkFreeMemory(vk->device(), memory.memHandle, nullptr);
+
     type->heap->stats.memoryAllocated -= memory.memSize;
     m_device->adapter()->notifyHeapMemoryFree(type->heapId, memory.memSize);
   }
@@ -542,4 +548,75 @@ namespace dxvk {
     }
   }
 
+
+  uint32_t DxvkMemoryAllocator::determineSparseMemoryTypes(
+          DxvkDevice*           device) const {
+    auto vk = device->vkd();
+
+    VkMemoryRequirements requirements = { };
+    uint32_t typeMask = ~0u;
+
+    // Create sparse dummy buffer to find available memory types
+    VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
+    bufferInfo.flags        = VK_BUFFER_CREATE_SPARSE_BINDING_BIT
+                            | VK_BUFFER_CREATE_SPARSE_ALIASED_BIT
+                            | VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT;
+    bufferInfo.size         = 65536;
+    bufferInfo.usage        = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT
+                            | VK_BUFFER_USAGE_INDEX_BUFFER_BIT
+                            | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT
+                            | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT
+                            | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT
+                            | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT
+                            | VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT
+                            | VK_BUFFER_USAGE_TRANSFER_DST_BIT
+                            | VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
+    bufferInfo.sharingMode  = VK_SHARING_MODE_EXCLUSIVE;
+
+    VkBuffer buffer = VK_NULL_HANDLE;
+
+    if (vk->vkCreateBuffer(vk->device(), &bufferInfo, nullptr, &buffer)) {
+      Logger::err("Failed to create dummy buffer to query sparse memory types");
+      return 0;
+    }
+
+    vk->vkGetBufferMemoryRequirements(vk->device(), buffer, &requirements);
+    vk->vkDestroyBuffer(vk->device(), buffer, nullptr);
+    typeMask &= requirements.memoryTypeBits;
+
+    // Create sparse dummy image to find available memory types
+    VkImageCreateInfo imageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO };
+    imageInfo.flags         = VK_IMAGE_CREATE_SPARSE_BINDING_BIT
+                            | VK_IMAGE_CREATE_SPARSE_ALIASED_BIT
+                            | VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
+    imageInfo.imageType     = VK_IMAGE_TYPE_2D;
+    imageInfo.format        = VK_FORMAT_R8G8B8A8_UNORM;
+    imageInfo.extent        = { 256, 256, 1 };
+    imageInfo.mipLevels     = 1;
+    imageInfo.arrayLayers   = 1;
+    imageInfo.samples       = VK_SAMPLE_COUNT_1_BIT;
+    imageInfo.tiling        = VK_IMAGE_TILING_OPTIMAL;
+    imageInfo.usage         = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT
+                            | VK_IMAGE_USAGE_SAMPLED_BIT
+                            | VK_IMAGE_USAGE_STORAGE_BIT
+                            | VK_IMAGE_USAGE_TRANSFER_DST_BIT
+                            | VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
+    imageInfo.sharingMode   = VK_SHARING_MODE_EXCLUSIVE;
+
+    VkImage image = VK_NULL_HANDLE;
+
+    if (vk->vkCreateImage(vk->device(), &imageInfo, nullptr, &image)) {
+      Logger::err("Failed to create dummy image to query sparse memory types");
+      return 0;
+    }
+
+    vk->vkGetImageMemoryRequirements(vk->device(), image, &requirements);
+    vk->vkDestroyImage(vk->device(), image, nullptr);
+    typeMask &= requirements.memoryTypeBits;
+
+    Logger::log(typeMask ? LogLevel::Info : LogLevel::Error,
+      str::format("Memory type mask for sparse resources: 0x", std::hex, typeMask));
+    return typeMask;
+  }
+
 }

src/dxvk/dxvk_memory.h

@@ -306,7 +306,7 @@ namespace dxvk {
     constexpr static VkDeviceSize SmallAllocationThreshold = 256 << 10;
   public:
     
-    DxvkMemoryAllocator(const DxvkDevice* device);
+    DxvkMemoryAllocator(DxvkDevice* device);
     ~DxvkMemoryAllocator();
     
     /**
@@ -320,7 +320,15 @@ namespace dxvk {
     VkDeviceSize bufferImageGranularity() const {
       return m_devProps.limits.bufferImageGranularity;
     }
-    
+
+    /**
+     * \brief Memory type mask for sparse resources
+     * \returns Sparse resource memory types
+     */
+    uint32_t getSparseMemoryTypes() const {
+      return m_sparseMemoryTypes;
+    }
+
     /**
      * \brief Allocates device memory
      * 
@@ -348,15 +356,16 @@ namespace dxvk {
     
   private:
 
-    const Rc<vk::DeviceFn>                 m_vkd;
+    DxvkDevice*                                     m_device;
-    const DxvkDevice*                      m_device;
+    VkPhysicalDeviceProperties                      m_devProps;
-    const VkPhysicalDeviceProperties       m_devProps;
+    VkPhysicalDeviceMemoryProperties                m_memProps;
-    const VkPhysicalDeviceMemoryProperties m_memProps;
     
     dxvk::mutex                                     m_mutex;
     std::array<DxvkMemoryHeap, VK_MAX_MEMORY_HEAPS> m_memHeaps;
     std::array<DxvkMemoryType, VK_MAX_MEMORY_TYPES> m_memTypes;
 
+    uint32_t m_sparseMemoryTypes = 0u;
+
     DxvkMemory tryAlloc(
       const DxvkMemoryRequirements&           req,
       const DxvkMemoryProperties&             info,
@@ -403,6 +412,9 @@ namespace dxvk {
     void freeEmptyChunks(
       const DxvkMemoryHeap*       heap);
 
+    uint32_t determineSparseMemoryTypes(
+            DxvkDevice*           device) const;
+
   };
   
 }

src/dxvk/dxvk_sparse.cpp

@@ -5,6 +5,156 @@
 
 namespace dxvk {
 
+  DxvkSparseMapping::DxvkSparseMapping()
+  : m_pool(nullptr),
+    m_page(nullptr) {
+
+  }
+
+
+  DxvkSparseMapping::DxvkSparseMapping(
+          Rc<DxvkSparsePageAllocator> allocator,
+          Rc<DxvkSparsePage>          page)
+  : m_pool(std::move(allocator)),
+    m_page(std::move(page)) {
+
+  }
+
+
+  DxvkSparseMapping::DxvkSparseMapping(
+          DxvkSparseMapping&&         other)
+  : m_pool(std::move(other.m_pool)),
+    m_page(std::move(other.m_page)) {
+    // No need to acquire here. The only place from which
+    // this constructor can be called does this atomically.
+  }
+
+
+  DxvkSparseMapping::DxvkSparseMapping(
+    const DxvkSparseMapping&          other)
+  : m_pool(other.m_pool),
+    m_page(other.m_page) {
+    this->acquire();
+  }
+
+
+  DxvkSparseMapping& DxvkSparseMapping::operator = (
+          DxvkSparseMapping&&         other) {
+    this->release();
+
+    m_pool = std::move(other.m_pool);
+    m_page = std::move(other.m_page);
+    return *this;
+  }
+
+
+  DxvkSparseMapping& DxvkSparseMapping::operator = (
+    const DxvkSparseMapping&          other) {
+    other.acquire();
+    this->release();
+
+    m_pool = other.m_pool;
+    m_page = other.m_page;
+    return *this;
+  }
+
+
+  DxvkSparseMapping::~DxvkSparseMapping() {
+    this->release();
+  }
+
+
+  void DxvkSparseMapping::acquire() const {
+    if (m_page != nullptr)
+      m_pool->acquirePage(m_page);
+  }
+
+
+  void DxvkSparseMapping::release() const {
+    if (m_page != nullptr)
+      m_pool->releasePage(m_page);
+  }
+
+
+  DxvkSparsePageAllocator::DxvkSparsePageAllocator(
+          DxvkDevice*           device,
+          DxvkMemoryAllocator&  memoryAllocator)
+  : m_device(device), m_memory(&memoryAllocator) {
+
+  }
+
+
+  DxvkSparsePageAllocator::~DxvkSparsePageAllocator() {
+
+  }
+
+
+  DxvkSparseMapping DxvkSparsePageAllocator::acquirePage(
+          uint32_t              page) {
+    std::lock_guard lock(m_mutex);
+
+    if (unlikely(page >= m_pageCount))
+      return DxvkSparseMapping();
+
+    m_useCount += 1;
+    return DxvkSparseMapping(this, m_pages[page]);
+  }
+
+
+  void DxvkSparsePageAllocator::setCapacity(
+          uint32_t              pageCount) {
+    std::lock_guard lock(m_mutex);
+
+    if (pageCount < m_pageCount) {
+      if (!m_useCount)
+        m_pages.resize(pageCount);
+    } else if (pageCount > m_pageCount) {
+      while (m_pages.size() < pageCount)
+        m_pages.push_back(allocPage());
+    }
+
+    m_pageCount = pageCount;
+  }
+
+
+  Rc<DxvkSparsePage> DxvkSparsePageAllocator::allocPage() {
+    DxvkMemoryRequirements memoryRequirements = { };
+    memoryRequirements.core = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2 };
+
+    // We don't know what kind of resource the memory6
+    // might be bound to, so just guess the memory types
+    auto& core = memoryRequirements.core.memoryRequirements;
+    core.size           = SparseMemoryPageSize;
+    core.alignment      = SparseMemoryPageSize;
+    core.memoryTypeBits = m_memory->getSparseMemoryTypes();
+
+    DxvkMemoryProperties memoryProperties = { };
+    memoryProperties.flags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
+
+    DxvkMemory memory = m_memory->alloc(memoryRequirements,
+      memoryProperties, DxvkMemoryFlag::GpuReadable);
+
+    return new DxvkSparsePage(std::move(memory));
+  }
+
+
+  void DxvkSparsePageAllocator::acquirePage(
+    const Rc<DxvkSparsePage>&   page) {
+    std::lock_guard lock(m_mutex);
+    m_useCount += 1;
+  }
+
+
+  void DxvkSparsePageAllocator::releasePage(
+    const Rc<DxvkSparsePage>&   page) {
+    std::lock_guard lock(m_mutex);
+    m_useCount -= 1;
+
+    if (!m_useCount)
+      m_pages.resize(m_pageCount);
+  }
+
+
   DxvkSparsePageTable::DxvkSparsePageTable() {
 
   }
@@ -20,6 +170,7 @@ namespace dxvk {
     // and consists of consecutive 64k pages
     size_t pageCount = align(bufferSize, SparseMemoryPageSize) / SparseMemoryPageSize;
     m_metadata.resize(pageCount);
+    m_mappings.resize(pageCount);
 
     for (size_t i = 0; i < pageCount; i++) {
       VkDeviceSize pageOffset = SparseMemoryPageSize * i;
@@ -135,6 +286,7 @@ namespace dxvk {
 
     // Fill in page metadata
     m_metadata.reserve(totalPageCount);
+    m_mappings.resize(totalPageCount);
 
     for (uint32_t l = 0; l < image->info().numLayers; l++) {
       for (uint32_t m = 0; m < m_properties.pagedMipCount; m++) {

src/dxvk/dxvk_sparse.h

@@ -10,7 +10,7 @@ namespace dxvk {
   class DxvkBuffer;
   class DxvkImage;
   class DxvkSparsePage;
-  class DxvkSparsePagePool;
+  class DxvkSparsePageAllocator;
 
   constexpr static VkDeviceSize SparseMemoryPageSize = 1ull << 16;
 
@@ -112,6 +112,152 @@ namespace dxvk {
   };
 
 
+  /**
+   * \brief Sparse memory page
+   *
+   * Stores a single reference-counted page
+   * of memory. The page size is 64k.
+   */
+  class DxvkSparsePage : public DxvkResource {
+
+  public:
+
+    DxvkSparsePage(DxvkMemory&& memory)
+    : m_memory(std::move(memory)) { }
+
+    /**
+     * \brief Queries memory handle
+     * \returns Memory information
+     */
+    DxvkSparsePageHandle getHandle() const {
+      DxvkSparsePageHandle result;
+      result.memory = m_memory.memory();
+      result.offset = m_memory.offset();
+      result.length = m_memory.length();
+      return result;
+    }
+
+  private:
+
+    DxvkMemory  m_memory;
+
+  };
+
+
+  /**
+   * \brief Sparse page mapping
+   *
+   * Stores a reference to a page as well as the pool that the page
+   * was allocated from, and automatically manages the use counter
+   * of the pool as the reference is being moved or copied around.
+   */
+  class DxvkSparseMapping {
+    friend DxvkSparsePageAllocator;
+  public:
+
+    DxvkSparseMapping();
+
+    DxvkSparseMapping(DxvkSparseMapping&& other);
+    DxvkSparseMapping(const DxvkSparseMapping& other);
+
+    DxvkSparseMapping& operator = (DxvkSparseMapping&& other);
+    DxvkSparseMapping& operator = (const DxvkSparseMapping& other);
+
+    ~DxvkSparseMapping();
+
+    Rc<DxvkSparsePage> getPage() const {
+      return m_page;
+    }
+
+    bool operator == (const DxvkSparseMapping& other) const {
+      // Pool is a function of the page, so no need to check both
+      return m_page == other.m_page;
+    }
+
+    bool operator != (const DxvkSparseMapping& other) const {
+      return m_page != other.m_page;
+    }
+
+    operator bool () const {
+      return m_page != nullptr;
+    }
+
+  private:
+
+    Rc<DxvkSparsePageAllocator> m_pool;
+    Rc<DxvkSparsePage>          m_page;
+
+    DxvkSparseMapping(
+            Rc<DxvkSparsePageAllocator> allocator,
+            Rc<DxvkSparsePage>          page);
+
+    void acquire() const;
+
+    void release() const;
+
+  };
+
+
+  /**
+   * \brief Sparse memory allocator
+   *
+   * Provides an allocator for sparse pages with variable capacity.
+   * Pages are use-counted to make sure they are not removed from
+   * the allocator too early.
+   */
+  class DxvkSparsePageAllocator : public RcObject {
+    friend DxvkSparseMapping;
+  public:
+
+    DxvkSparsePageAllocator(
+            DxvkDevice*           device,
+            DxvkMemoryAllocator&  memoryAllocator);
+
+    ~DxvkSparsePageAllocator();
+
+    /**
+     * \brief Acquires page at the given offset
+     *
+     * If the offset is valid, this will atomically
+     * increment the allocator's use count and return
+     * a reference to the page.
+     * \param [in] page Page index
+     * \returns Page mapping object
+     */
+    DxvkSparseMapping acquirePage(
+            uint32_t              page);
+
+    /**
+     * \brief Changes the allocator's maximum capacity
+     *
+     * Allocates new pages as necessary, and frees existing
+     * pages if none of the pages are currently in use.
+     * \param [in] pageCount New capacity, in pages
+     */
+    void setCapacity(
+            uint32_t              pageCount);
+
+  private:
+
+    DxvkDevice*                       m_device;
+    DxvkMemoryAllocator*              m_memory;
+
+    dxvk::mutex                       m_mutex;
+    uint32_t                          m_pageCount = 0u;
+    uint32_t                          m_useCount = 0u;
+    std::vector<Rc<DxvkSparsePage>>   m_pages;
+
+    Rc<DxvkSparsePage> allocPage();
+
+    void acquirePage(
+      const Rc<DxvkSparsePage>&   page);
+
+    void releasePage(
+      const Rc<DxvkSparsePage>&   page);
+
+  };
+
+
   /**
    * \brief Sparse page table
    *
@@ -202,6 +348,7 @@ namespace dxvk {
     DxvkSparseImageProperties                         m_properties    = { };
     std::vector<DxvkSparseImageSubresourceProperties> m_subresources;
     std::vector<DxvkSparsePageInfo>                   m_metadata;
+    std::vector<DxvkSparseMapping>                    m_mappings;
 
   };