mirror of
https://github.com/doitsujin/dxvk.git
synced 2025-03-14 04:29:15 +01:00
[dxvk] Implemented new memory allocator with sub-allocation
This commit is contained in:
Philip Rebohle
parent
85120d2d01
commit
d5a49698b4
@ -6,6 +6,7 @@
|
||||
#include "../util/util_env.h"
|
||||
#include "../util/util_error.h"
|
||||
#include "../util/util_flags.h"
|
||||
#include "../util/util_math.h"
|
||||
#include "../util/util_string.h"
|
||||
|
||||
#include "../util/rc/util_rc.h"
|
||||
|
||||
@ -8,38 +8,233 @@ namespace dxvk {
|
||||
|
||||
|
||||
DxvkMemory::DxvkMemory(
|
||||
DxvkMemoryAllocator* alloc,
|
||||
VkDeviceMemory memory,
|
||||
void* mapPtr)
|
||||
: m_alloc (alloc),
|
||||
m_memory(memory),
|
||||
m_mapPtr(mapPtr) { }
|
||||
DxvkMemoryChunk* chunk,
|
||||
DxvkMemoryHeap* heap,
|
||||
VkDeviceMemory memory,
|
||||
VkDeviceSize offset,
|
||||
VkDeviceSize length,
|
||||
void* mapPtr)
|
||||
: m_chunk (chunk),
|
||||
m_heap (heap),
|
||||
m_memory (memory),
|
||||
m_offset (offset),
|
||||
m_length (length),
|
||||
m_mapPtr (mapPtr) { }
|
||||
|
||||
|
||||
DxvkMemory::DxvkMemory(DxvkMemory&& other)
|
||||
: m_alloc (other.m_alloc),
|
||||
m_memory(other.m_memory),
|
||||
m_mapPtr(other.m_mapPtr) {
|
||||
other.m_alloc = nullptr;
|
||||
other.m_memory = VK_NULL_HANDLE;
|
||||
other.m_mapPtr = nullptr;
|
||||
}
|
||||
: m_chunk (std::exchange(other.m_chunk, nullptr)),
|
||||
m_heap (std::exchange(other.m_heap, nullptr)),
|
||||
m_memory (std::exchange(other.m_memory, VkDeviceMemory(nullptr))),
|
||||
m_offset (std::exchange(other.m_offset, 0)),
|
||||
m_length (std::exchange(other.m_length, 0)),
|
||||
m_mapPtr (std::exchange(other.m_mapPtr, nullptr)) { }
|
||||
|
||||
|
||||
DxvkMemory& DxvkMemory::operator = (DxvkMemory&& other) {
|
||||
this->m_alloc = other.m_alloc;
|
||||
this->m_memory = other.m_memory;
|
||||
this->m_mapPtr = other.m_mapPtr;
|
||||
other.m_alloc = nullptr;
|
||||
other.m_memory = VK_NULL_HANDLE;
|
||||
other.m_mapPtr = nullptr;
|
||||
m_chunk = std::exchange(other.m_chunk, nullptr);
|
||||
m_heap = std::exchange(other.m_heap, nullptr);
|
||||
m_memory = std::exchange(other.m_memory, VkDeviceMemory(nullptr));
|
||||
m_offset = std::exchange(other.m_offset, 0);
|
||||
m_length = std::exchange(other.m_length, 0);
|
||||
m_mapPtr = std::exchange(other.m_mapPtr, nullptr);
|
||||
return *this;
|
||||
}
|
||||
|
||||
|
||||
DxvkMemory::~DxvkMemory() {
|
||||
if (m_memory != VK_NULL_HANDLE)
|
||||
m_alloc->freeMemory(m_memory);
|
||||
if (m_chunk != nullptr)
|
||||
m_heap->free(m_chunk, m_offset, m_length);
|
||||
else if (m_heap != nullptr)
|
||||
m_heap->freeDeviceMemory(m_memory);
|
||||
}
|
||||
|
||||
|
||||
DxvkMemoryChunk::DxvkMemoryChunk(
|
||||
DxvkMemoryHeap* heap,
|
||||
VkDeviceMemory memory,
|
||||
void* mapPtr,
|
||||
VkDeviceSize size)
|
||||
: m_heap (heap),
|
||||
m_memory(memory),
|
||||
m_mapPtr(mapPtr),
|
||||
m_size (size),
|
||||
m_free (size) {
|
||||
TRACE(this);
|
||||
// Mark the entire chunk as free
|
||||
m_freeList.push_back(FreeSlice { 0, size });
|
||||
}
|
||||
|
||||
|
||||
DxvkMemoryChunk::~DxvkMemoryChunk() {
|
||||
TRACE(this);
|
||||
m_heap->freeDeviceMemory(m_memory);
|
||||
}
|
||||
|
||||
|
||||
DxvkMemory DxvkMemoryChunk::alloc(VkDeviceSize size, VkDeviceSize align) {
|
||||
// Fast exit if the chunk is full already
|
||||
if (size > m_free)
|
||||
return DxvkMemory();
|
||||
|
||||
// Select the slice to allocate from in a worst-fit
|
||||
// manner. This may help keep fragmentation low.
|
||||
auto bestSlice = m_freeList.begin();
|
||||
|
||||
for (auto slice = m_freeList.begin(); slice != m_freeList.end(); slice++) {
|
||||
if (slice->length > bestSlice->length)
|
||||
bestSlice = slice;
|
||||
}
|
||||
|
||||
// We need to align the allocation to the requested alignment
|
||||
const VkDeviceSize sliceStart = bestSlice->offset;
|
||||
const VkDeviceSize sliceEnd = bestSlice->offset + bestSlice->length;
|
||||
|
||||
const VkDeviceSize allocStart = dxvk::align(sliceStart, align);
|
||||
const VkDeviceSize allocEnd = dxvk::align(allocStart + size, align);
|
||||
|
||||
if (allocEnd > sliceEnd)
|
||||
return DxvkMemory();
|
||||
|
||||
// We can use this slice, but we'll have to add
|
||||
// the unused parts of it back to the free list.
|
||||
m_freeList.erase(bestSlice);
|
||||
m_free -= size;
|
||||
|
||||
if (allocStart != sliceStart)
|
||||
m_freeList.push_back({ sliceStart, allocStart - sliceStart });
|
||||
|
||||
if (allocEnd != sliceEnd)
|
||||
m_freeList.push_back({ allocEnd, sliceEnd - allocEnd });
|
||||
|
||||
// Create the memory object with the aligned slice
|
||||
return DxvkMemory(this, m_heap,
|
||||
m_memory, allocStart, allocEnd - allocStart,
|
||||
reinterpret_cast<char*>(m_mapPtr) + allocStart);
|
||||
}
|
||||
|
||||
|
||||
void DxvkMemoryChunk::free(
|
||||
VkDeviceSize offset,
|
||||
VkDeviceSize length) {
|
||||
m_free += length;
|
||||
|
||||
// Remove adjacent entries from the free list and then add
|
||||
// a new slice that covers all those entries. Without doing
|
||||
// so, the slice could not be reused for larger allocations.
|
||||
auto curr = m_freeList.begin();
|
||||
|
||||
while (curr != m_freeList.end()) {
|
||||
if (curr->offset == offset + length) {
|
||||
length += curr->length;
|
||||
curr = m_freeList.erase(curr);
|
||||
} else if (curr->offset + curr->length == offset) {
|
||||
offset -= curr->length;
|
||||
length += curr->length;
|
||||
curr = m_freeList.erase(curr);
|
||||
} else {
|
||||
curr++;
|
||||
}
|
||||
}
|
||||
|
||||
m_freeList.push_back({ offset, length });
|
||||
}
|
||||
|
||||
|
||||
DxvkMemoryHeap::DxvkMemoryHeap(
|
||||
const Rc<vk::DeviceFn> vkd,
|
||||
uint32_t memTypeId,
|
||||
VkMemoryType memType)
|
||||
: m_vkd (vkd),
|
||||
m_memTypeId (memTypeId),
|
||||
m_memType (memType) {
|
||||
|
||||
}
|
||||
|
||||
|
||||
DxvkMemoryHeap::~DxvkMemoryHeap() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
DxvkMemory DxvkMemoryHeap::alloc(VkDeviceSize size, VkDeviceSize align) {
|
||||
// We don't sub-allocate large allocations from one of the
|
||||
// chunks since that might lead to severe fragmentation.
|
||||
if (size >= (m_chunkSize / 4)) {
|
||||
VkDeviceMemory memory = this->allocDeviceMemory(size);
|
||||
void* mapPtr = this->mapDeviceMemory(memory);
|
||||
|
||||
return DxvkMemory(nullptr, this, memory, 0, size, mapPtr);
|
||||
} else {
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
|
||||
// Probe chunks in a first-fit manner
|
||||
for (const auto& chunk : m_chunks) {
|
||||
DxvkMemory memory = chunk->alloc(size, align);
|
||||
|
||||
if (memory.memory() != VK_NULL_HANDLE)
|
||||
return memory;
|
||||
}
|
||||
|
||||
// None of the existing chunks could satisfy
|
||||
// the request, we need to create a new one
|
||||
VkDeviceMemory chunkMem = this->allocDeviceMemory(m_chunkSize);
|
||||
void* chunkPtr = this->mapDeviceMemory(chunkMem);
|
||||
|
||||
Rc<DxvkMemoryChunk> newChunk = new DxvkMemoryChunk(
|
||||
this, chunkMem, chunkPtr, m_chunkSize);
|
||||
DxvkMemory memory = newChunk->alloc(size, align);
|
||||
|
||||
m_chunks.push_back(std::move(newChunk));
|
||||
return memory;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
VkDeviceMemory DxvkMemoryHeap::allocDeviceMemory(VkDeviceSize memorySize) {
|
||||
VkMemoryAllocateInfo info;
|
||||
info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
|
||||
info.pNext = nullptr;
|
||||
info.allocationSize = memorySize;
|
||||
info.memoryTypeIndex = m_memTypeId;
|
||||
|
||||
VkDeviceMemory memory = VK_NULL_HANDLE;
|
||||
|
||||
if (m_vkd->vkAllocateMemory(m_vkd->device(),
|
||||
&info, nullptr, &memory) != VK_SUCCESS)
|
||||
return VK_NULL_HANDLE;
|
||||
|
||||
return memory;
|
||||
}
|
||||
|
||||
|
||||
void DxvkMemoryHeap::freeDeviceMemory(VkDeviceMemory memory) {
|
||||
m_vkd->vkFreeMemory(m_vkd->device(), memory, nullptr);
|
||||
}
|
||||
|
||||
|
||||
void* DxvkMemoryHeap::mapDeviceMemory(VkDeviceMemory memory) {
|
||||
if ((m_memType.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
|
||||
return nullptr;
|
||||
|
||||
void* ptr = nullptr;
|
||||
|
||||
VkResult status = m_vkd->vkMapMemory(m_vkd->device(),
|
||||
memory, 0, VK_WHOLE_SIZE, 0, &ptr);
|
||||
|
||||
if (status != VK_SUCCESS) {
|
||||
Logger::err("DxvkMemoryHeap: Failed to map memory");
|
||||
return nullptr;
|
||||
} return ptr;
|
||||
}
|
||||
|
||||
|
||||
void DxvkMemoryHeap::free(
|
||||
DxvkMemoryChunk* chunk,
|
||||
VkDeviceSize offset,
|
||||
VkDeviceSize length) {
|
||||
std::lock_guard<std::mutex> lock(m_mutex);
|
||||
chunk->free(offset, length);
|
||||
}
|
||||
|
||||
|
||||
@ -47,7 +242,8 @@ namespace dxvk {
|
||||
const Rc<DxvkAdapter>& adapter,
|
||||
const Rc<vk::DeviceFn>& vkd)
|
||||
: m_vkd(vkd), m_memProps(adapter->memoryProperties()) {
|
||||
|
||||
for (uint32_t i = 0; i < m_memProps.memoryTypeCount; i++)
|
||||
m_heaps[i] = new DxvkMemoryHeap(m_vkd, i, m_memProps.memoryTypes[i]);
|
||||
}
|
||||
|
||||
|
||||
@ -60,52 +256,21 @@ namespace dxvk {
|
||||
const VkMemoryRequirements& req,
|
||||
const VkMemoryPropertyFlags flags) {
|
||||
|
||||
VkDeviceMemory memory = VK_NULL_HANDLE;
|
||||
void* mapPtr = nullptr;
|
||||
|
||||
for (uint32_t i = 0; i < m_memProps.memoryTypeCount; i++) {
|
||||
for (uint32_t i = 0; i < m_heaps.size(); i++) {
|
||||
const bool supported = (req.memoryTypeBits & (1u << i)) != 0;
|
||||
const bool adequate = (m_memProps.memoryTypes[i].propertyFlags & flags) == flags;
|
||||
|
||||
if (supported && adequate) {
|
||||
memory = this->allocMemory(req.size, i);
|
||||
DxvkMemory memory = m_heaps.at(i)->alloc(req.size, req.alignment);
|
||||
|
||||
if (memory != VK_NULL_HANDLE)
|
||||
break;
|
||||
if (memory.memory() != VK_NULL_HANDLE)
|
||||
return memory;
|
||||
}
|
||||
}
|
||||
|
||||
if (memory == VK_NULL_HANDLE)
|
||||
throw DxvkError("DxvkMemoryAllocator::alloc: Failed to allocate memory");
|
||||
|
||||
if (flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
|
||||
if (m_vkd->vkMapMemory(m_vkd->device(), memory,
|
||||
0, VK_WHOLE_SIZE, 0, &mapPtr) != VK_SUCCESS)
|
||||
throw DxvkError("DxvkMemoryAllocator::alloc: Failed to map memory");
|
||||
}
|
||||
|
||||
return DxvkMemory(this, memory, mapPtr);
|
||||
}
|
||||
|
||||
|
||||
VkDeviceMemory DxvkMemoryAllocator::allocMemory(
|
||||
VkDeviceSize blockSize, uint32_t memoryType) {
|
||||
VkMemoryAllocateInfo info;
|
||||
info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
|
||||
info.pNext = nullptr;
|
||||
info.allocationSize = blockSize;
|
||||
info.memoryTypeIndex = memoryType;
|
||||
|
||||
VkDeviceMemory memory = VK_NULL_HANDLE;
|
||||
if (m_vkd->vkAllocateMemory(m_vkd->device(),
|
||||
&info, nullptr, &memory) != VK_SUCCESS)
|
||||
return VK_NULL_HANDLE;
|
||||
return memory;
|
||||
}
|
||||
|
||||
|
||||
void DxvkMemoryAllocator::freeMemory(VkDeviceMemory memory) {
|
||||
m_vkd->vkFreeMemory(m_vkd->device(), memory, nullptr);
|
||||
throw DxvkError(str::format(
|
||||
"DxvkMemoryAllocator: Failed to allocate ",
|
||||
req.size, " bytes"));
|
||||
}
|
||||
|
||||
}
|
||||
@ -4,11 +4,16 @@
|
||||
|
||||
namespace dxvk {
|
||||
|
||||
class DxvkMemoryHeap;
|
||||
class DxvkMemoryChunk;
|
||||
class DxvkMemoryAllocator;
|
||||
|
||||
|
||||
/**
|
||||
* \brief Memory slice
|
||||
*
|
||||
* Represents a slice of memory that has
|
||||
* been sub-allocated from a bigger chunk.
|
||||
*/
|
||||
class DxvkMemory {
|
||||
|
||||
@ -16,9 +21,12 @@ namespace dxvk {
|
||||
|
||||
DxvkMemory();
|
||||
DxvkMemory(
|
||||
DxvkMemoryAllocator* alloc,
|
||||
VkDeviceMemory memory,
|
||||
void* mapPtr);
|
||||
DxvkMemoryChunk* chunk,
|
||||
DxvkMemoryHeap* heap,
|
||||
VkDeviceMemory memory,
|
||||
VkDeviceSize offset,
|
||||
VkDeviceSize length,
|
||||
void* mapPtr);
|
||||
DxvkMemory (DxvkMemory&& other);
|
||||
DxvkMemory& operator = (DxvkMemory&& other);
|
||||
~DxvkMemory();
|
||||
@ -35,14 +43,14 @@ namespace dxvk {
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Offset from memory object
|
||||
* \brief Offset into device memory
|
||||
*
|
||||
* This information is required when
|
||||
* binding memory to Vulkan objects.
|
||||
* \returns Offset from memory object
|
||||
* \returns Offset into device memory
|
||||
*/
|
||||
VkDeviceSize offset() const {
|
||||
return 0;
|
||||
return m_offset;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -57,9 +65,137 @@ namespace dxvk {
|
||||
|
||||
private:
|
||||
|
||||
DxvkMemoryAllocator* m_alloc = nullptr;
|
||||
VkDeviceMemory m_memory = VK_NULL_HANDLE;
|
||||
void* m_mapPtr = nullptr;
|
||||
DxvkMemoryChunk* m_chunk = nullptr;
|
||||
DxvkMemoryHeap* m_heap = nullptr;
|
||||
VkDeviceMemory m_memory = VK_NULL_HANDLE;
|
||||
VkDeviceSize m_offset = 0;
|
||||
VkDeviceSize m_length = 0;
|
||||
void* m_mapPtr = nullptr;
|
||||
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* \brief Memory chunk
|
||||
*
|
||||
* A single chunk of memory that provides a
|
||||
* sub-allocator. This is not thread-safe.
|
||||
*/
|
||||
class DxvkMemoryChunk : public RcObject {
|
||||
|
||||
public:
|
||||
|
||||
DxvkMemoryChunk(
|
||||
DxvkMemoryHeap* heap,
|
||||
VkDeviceMemory memory,
|
||||
void* mapPtr,
|
||||
VkDeviceSize size);
|
||||
|
||||
~DxvkMemoryChunk();
|
||||
|
||||
/**
|
||||
* \brief Allocates memory from the chunk
|
||||
*
|
||||
* On failure, this returns a slice with
|
||||
* \c VK_NULL_HANDLE as the memory handle.
|
||||
* \param [in] size Number of bytes to allocate
|
||||
* \param [in] align Required alignment
|
||||
* \returns The allocated memory slice
|
||||
*/
|
||||
DxvkMemory alloc(
|
||||
VkDeviceSize size,
|
||||
VkDeviceSize align);
|
||||
|
||||
/**
|
||||
* \brief Frees memory
|
||||
*
|
||||
* Returns a slice back to the chunk.
|
||||
* Called automatically when a memory
|
||||
* slice runs out of scope.
|
||||
* \param [in] offset Slice offset
|
||||
* \param [in] length Slice length
|
||||
*/
|
||||
void free(
|
||||
VkDeviceSize offset,
|
||||
VkDeviceSize length);
|
||||
|
||||
private:
|
||||
|
||||
struct FreeSlice {
|
||||
VkDeviceSize offset;
|
||||
VkDeviceSize length;
|
||||
};
|
||||
|
||||
DxvkMemoryHeap* const m_heap;
|
||||
VkDeviceMemory const m_memory;
|
||||
void* const m_mapPtr;
|
||||
VkDeviceSize const m_size;
|
||||
VkDeviceSize m_free = 0;
|
||||
|
||||
std::vector<FreeSlice> m_freeList;
|
||||
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* \brief Memory heap
|
||||
*
|
||||
* Implements a memory allocator for a single
|
||||
* memory type. This class is thread-safe.
|
||||
*/
|
||||
class DxvkMemoryHeap : public RcObject {
|
||||
friend class DxvkMemory;
|
||||
friend class DxvkMemoryChunk;
|
||||
public:
|
||||
|
||||
DxvkMemoryHeap(
|
||||
const Rc<vk::DeviceFn> vkd,
|
||||
uint32_t memTypeId,
|
||||
VkMemoryType memType);
|
||||
|
||||
DxvkMemoryHeap (DxvkMemoryHeap&&) = delete;
|
||||
DxvkMemoryHeap& operator = (DxvkMemoryHeap&&) = delete;
|
||||
|
||||
~DxvkMemoryHeap();
|
||||
|
||||
/**
|
||||
* \brief Allocates memory from the heap
|
||||
*
|
||||
* Unless the requested allocation size is big
|
||||
* enough to justify a dedicated device allocation,
|
||||
* this will try to sub-allocate the block from an
|
||||
* existing chunk and create new chunks as necessary.
|
||||
* \param [in] size Amount of memory to allocate
|
||||
* \param [in] align Alignment requirements
|
||||
* \returns The allocated memory slice
|
||||
*/
|
||||
DxvkMemory alloc(
|
||||
VkDeviceSize size,
|
||||
VkDeviceSize align);
|
||||
|
||||
private:
|
||||
|
||||
const Rc<vk::DeviceFn> m_vkd;
|
||||
const uint32_t m_memTypeId;
|
||||
const VkMemoryType m_memType;
|
||||
const VkDeviceSize m_chunkSize = 16 * 1024 * 1024;
|
||||
|
||||
std::mutex m_mutex;
|
||||
std::vector<Rc<DxvkMemoryChunk>> m_chunks;
|
||||
|
||||
VkDeviceMemory allocDeviceMemory(
|
||||
VkDeviceSize memorySize);
|
||||
|
||||
void freeDeviceMemory(
|
||||
VkDeviceMemory memory);
|
||||
|
||||
void* mapDeviceMemory(
|
||||
VkDeviceMemory memory);
|
||||
|
||||
void free(
|
||||
DxvkMemoryChunk* chunk,
|
||||
VkDeviceSize offset,
|
||||
VkDeviceSize length);
|
||||
|
||||
};
|
||||
|
||||
@ -95,12 +231,7 @@ namespace dxvk {
|
||||
const Rc<vk::DeviceFn> m_vkd;
|
||||
const VkPhysicalDeviceMemoryProperties m_memProps;
|
||||
|
||||
VkDeviceMemory allocMemory(
|
||||
VkDeviceSize blockSize,
|
||||
uint32_t memoryType);
|
||||
|
||||
void freeMemory(
|
||||
VkDeviceMemory memory);
|
||||
std::array<Rc<DxvkMemoryHeap>, VK_MAX_MEMORY_TYPES> m_heaps;
|
||||
|
||||
};
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user