diff --git a/src/dxvk/dxvk_buffer.cpp b/src/dxvk/dxvk_buffer.cpp
index 558b623ba..bf718d660 100644
--- a/src/dxvk/dxvk_buffer.cpp
+++ b/src/dxvk/dxvk_buffer.cpp
@@ -1,3 +1,4 @@
+#include "dxvk_barrier.h"
 #include "dxvk_buffer.h"
 #include "dxvk_device.h"
 
@@ -108,6 +109,12 @@ namespace dxvk {
     if (isGpuWritable)
       hints.set(DxvkMemoryFlag::GpuWritable);
 
+    // Staging buffers that can't even be used as a transfer destinations
+    // are likely short-lived, so we should put them on a separate memory
+    // pool in order to avoid fragmentation
+    if (DxvkBarrierSet::getAccessTypes(m_info.access) == DxvkAccess::Read)
+      hints.set(DxvkMemoryFlag::Transient);
+
     // Ask driver whether we should be using a dedicated allocation
     handle.memory = m_memAlloc->alloc(&memReq.memoryRequirements,
       dedicatedRequirements, dedMemoryAllocInfo, m_memFlags, hints);
diff --git a/src/dxvk/dxvk_memory.cpp b/src/dxvk/dxvk_memory.cpp
index 851d79387..a33772c70 100644
--- a/src/dxvk/dxvk_memory.cpp
+++ b/src/dxvk/dxvk_memory.cpp
@@ -169,7 +169,8 @@ namespace dxvk {
     DxvkMemoryFlags mask(
       DxvkMemoryFlag::Small,
       DxvkMemoryFlag::GpuReadable,
-      DxvkMemoryFlag::GpuWritable);
+      DxvkMemoryFlag::GpuWritable,
+      DxvkMemoryFlag::Transient);
 
     if (hints.test(DxvkMemoryFlag::IgnoreConstraints))
       mask = DxvkMemoryFlags();
@@ -255,10 +256,10 @@ namespace dxvk {
       hints.clr(DxvkMemoryFlag::GpuWritable, DxvkMemoryFlag::GpuReadable);
     }
 
-    // Ignore all hints for host-visible allocations since they
+    // Ignore most hints for host-visible allocations since they
     // usually don't make much sense for those resources
     if (flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
-      hints = DxvkMemoryFlags();
+      hints = hints & DxvkMemoryFlag::Transient;
 
     // Try to allocate from a memory type which supports the given flags exactly
     auto dedAllocPtr = dedAllocReq.prefersDedicatedAllocation ? &dedAllocInfo : nullptr;
diff --git a/src/dxvk/dxvk_memory.h b/src/dxvk/dxvk_memory.h
index 4a7ef9600..3d331b6b9 100644
--- a/src/dxvk/dxvk_memory.h
+++ b/src/dxvk/dxvk_memory.h
@@ -165,7 +165,8 @@ namespace dxvk {
     Small             = 0,  ///< Small allocation
     GpuReadable       = 1,  ///< Medium-priority resource
     GpuWritable       = 2,  ///< High-priority resource
-    IgnoreConstraints = 3,  ///< Ignore most allocation flags
+    Transient         = 3,  ///< Resource is short-lived
+    IgnoreConstraints = 4,  ///< Ignore most allocation flags
   };
 
   using DxvkMemoryFlags = Flags<DxvkMemoryFlag>;