[dxvk] Introduce concept of priorities to device-local memory allocations

The priority defines the importance of any given allocation being
resident in device-local memory rather than system memory, with
values between 0 (low) and 1 (high).

Suballocations will only work on chunks with the requested priority.
This ensures that high-priority allocations are reserved solely for
high-priority resources, and that high-priority resources will be
allocated from a high-priority memory chunk.

src/dxvk/dxvk_buffer.cpp

@@ -127,7 +127,7 @@ namespace dxvk {
 
     bool useDedicated = dedicatedRequirements.prefersDedicatedAllocation;
     handle.memory = m_memAlloc->alloc(&memReq.memoryRequirements,
-      useDedicated ? &dedMemoryAllocInfo : nullptr, m_memFlags);
+      useDedicated ? &dedMemoryAllocInfo : nullptr, m_memFlags, 0.5f);
     
     if (vkd->vkBindBufferMemory(vkd->device(), handle.buffer,
         handle.memory.memory(), handle.memory.offset()) != VK_SUCCESS)

src/dxvk/dxvk_image.cpp

@@ -91,7 +91,7 @@ namespace dxvk {
 
     bool useDedicated = dedicatedRequirements.prefersDedicatedAllocation;
     m_memory = memAlloc.alloc(&memReq.memoryRequirements,
-      useDedicated ? &dedMemoryAllocInfo : nullptr, memFlags);
+      useDedicated ? &dedMemoryAllocInfo : nullptr, memFlags, 0.5f);
     
     // Try to bind the allocated memory slice to the image
     if (m_vkd->vkBindImageMemory(m_vkd->device(),

src/dxvk/dxvk_memory.cpp

@@ -75,10 +75,12 @@ namespace dxvk {
   DxvkMemory DxvkMemoryChunk::alloc(
           VkMemoryPropertyFlags flags,
           VkDeviceSize          size,
-          VkDeviceSize          align) {
+          VkDeviceSize          align,
+          float                 priority) {
     // Property flags must be compatible. This could
     // be refined a bit in the future if necessary.
-    if (m_memory.memFlags != flags)
+    if (m_memory.memFlags != flags
+     || m_memory.priority != priority)
       return DxvkMemory();
     
     // If the chunk is full, return
@@ -181,13 +183,14 @@ namespace dxvk {
   DxvkMemory DxvkMemoryAllocator::alloc(
     const VkMemoryRequirements*             req,
     const VkMemoryDedicatedAllocateInfoKHR* dedAllocInfo,
-          VkMemoryPropertyFlags             flags) {
+          VkMemoryPropertyFlags             flags,
+          float                             priority) {
     std::lock_guard<std::mutex> lock(m_mutex);
 
-    DxvkMemory result = this->tryAlloc(req, dedAllocInfo, flags);
+    DxvkMemory result = this->tryAlloc(req, dedAllocInfo, flags, priority);
     
     if (!result && (flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT))
-      result = this->tryAlloc(req, dedAllocInfo, flags & ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
+      result = this->tryAlloc(req, dedAllocInfo, flags & ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, priority);
     
     if (!result) {
       Logger::err(str::format(
@@ -220,7 +223,8 @@ namespace dxvk {
   DxvkMemory DxvkMemoryAllocator::tryAlloc(
     const VkMemoryRequirements*             req,
     const VkMemoryDedicatedAllocateInfoKHR* dedAllocInfo,
-          VkMemoryPropertyFlags             flags) {
+          VkMemoryPropertyFlags             flags,
+          float                             priority) {
     DxvkMemory result;
 
     for (uint32_t i = 0; i < m_memProps.memoryTypeCount && !result; i++) {
@@ -229,7 +233,7 @@ namespace dxvk {
       
       if (supported && adequate) {
         result = this->tryAllocFromType(&m_memTypes[i],
-          flags, req->size, req->alignment, dedAllocInfo);
+          flags, req->size, req->alignment, priority, dedAllocInfo);
       }
     }
     
@@ -242,28 +246,35 @@ namespace dxvk {
           VkMemoryPropertyFlags             flags,
           VkDeviceSize                      size,
           VkDeviceSize                      align,
+          float                             priority,
     const VkMemoryDedicatedAllocateInfoKHR* dedAllocInfo) {
+    // Prevent unnecessary external host memory fragmentation
+    bool isDeviceLocal = (flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0;
+
+    if (!isDeviceLocal)
+      priority = 0.0f;
+
     DxvkMemory memory;
 
     if ((size >= type->heap->chunkSize / 4) || dedAllocInfo) {
       DxvkDeviceMemory devMem = this->tryAllocDeviceMemory(
-        type, flags, size, dedAllocInfo);
+        type, flags, size, priority, dedAllocInfo);
 
       if (devMem.memHandle != VK_NULL_HANDLE)
         memory = DxvkMemory(this, nullptr, type, devMem.memHandle, 0, size, devMem.memPointer);
     } else {
       for (uint32_t i = 0; i < type->chunks.size() && !memory; i++)
-        memory = type->chunks[i]->alloc(flags, size, align);
+        memory = type->chunks[i]->alloc(flags, size, align, priority);
       
       if (!memory) {
         DxvkDeviceMemory devMem = tryAllocDeviceMemory(
-          type, flags, type->heap->chunkSize, nullptr);
+          type, flags, type->heap->chunkSize, priority, nullptr);
 
         if (devMem.memHandle == VK_NULL_HANDLE)
           return DxvkMemory();
         
         Rc<DxvkMemoryChunk> chunk = new DxvkMemoryChunk(this, type, devMem);
-        memory = chunk->alloc(flags, size, align);
+        memory = chunk->alloc(flags, size, align, priority);
 
         type->chunks.push_back(std::move(chunk));
       }
@@ -280,6 +291,7 @@ namespace dxvk {
           DxvkMemoryType*                   type,
           VkMemoryPropertyFlags             flags,
           VkDeviceSize                      size,
+          float                             priority,
     const VkMemoryDedicatedAllocateInfoKHR* dedAllocInfo) {
     if ((type->memType.propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
      && (type->heap->stats.memoryAllocated + size > type->heap->properties.size)
@@ -289,6 +301,7 @@ namespace dxvk {
     DxvkDeviceMemory result;
     result.memSize  = size;
     result.memFlags = flags;
+    result.priority = priority;
 
     VkMemoryAllocateInfo info;
     info.sType            = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;

src/dxvk/dxvk_memory.h

@@ -31,6 +31,7 @@ namespace dxvk {
     void*                 memPointer = nullptr;
     VkDeviceSize          memSize    = 0;
     VkMemoryPropertyFlags memFlags   = 0;
+    float                 priority   = 0.0f;
   };
 
   
@@ -170,12 +171,14 @@ namespace dxvk {
      * \param [in] flags Requested memory flags
      * \param [in] size Number of bytes to allocate
      * \param [in] align Required alignment
+     * \param [in] priority Requested priority
      * \returns The allocated memory slice
      */
     DxvkMemory alloc(
             VkMemoryPropertyFlags flags,
             VkDeviceSize          size,
-            VkDeviceSize          align);
+            VkDeviceSize          align,
+            float                 priority);
     
     /**
      * \brief Frees memory
@@ -236,13 +239,16 @@ namespace dxvk {
      * \brief Allocates device memory
      * 
      * \param [in] req Memory requirements
-     * \param [in] flats Memory type flags
+     * \param [in] dedAllocInfo Dedicated allocation info
+     * \param [in] flags Memory type flags
+     * \param [in] priority Device-local memory priority
      * \returns Allocated memory slice
      */
     DxvkMemory alloc(
       const VkMemoryRequirements*             req,
       const VkMemoryDedicatedAllocateInfoKHR* dedAllocInfo,
-            VkMemoryPropertyFlags             flags);
+            VkMemoryPropertyFlags             flags,
+            float                             priority);
     
     /**
      * \brief Queries memory stats
@@ -268,19 +274,22 @@ namespace dxvk {
     DxvkMemory tryAlloc(
       const VkMemoryRequirements*             req,
       const VkMemoryDedicatedAllocateInfoKHR* dedAllocInfo,
-            VkMemoryPropertyFlags             flags);
+            VkMemoryPropertyFlags             flags,
+            float                             priority);
     
     DxvkMemory tryAllocFromType(
             DxvkMemoryType*                   type,
             VkMemoryPropertyFlags             flags,
             VkDeviceSize                      size,
             VkDeviceSize                      align,
+            float                             priority,
       const VkMemoryDedicatedAllocateInfoKHR* dedAllocInfo);
     
     DxvkDeviceMemory tryAllocDeviceMemory(
             DxvkMemoryType*                   type,
             VkMemoryPropertyFlags             flags,
             VkDeviceSize                      size,
+            float                             priority,
       const VkMemoryDedicatedAllocateInfoKHR* dedAllocInfo);
     
     void free(