rooty/dxvk
1
0
Fork 0
mirror of https://github.com/doitsujin/dxvk.git synced 2025-02-20 19:54:19 +01:00
Code Issues Projects Releases Wiki Activity

[dxvk] Implement sparse memory allocator

Browse Source
This commit is contained in:
Philip Rebohle 2022-08-20 21:57:46 +02:00
parent 6f216f9df4
commit f9db4921e0
6 changed files with 415 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/dxvk/dxvk_device.cpp
View File

@ -216,6 +216,11 @@ namespace dxvk {
}
Rc<DxvkSparsePageAllocator> DxvkDevice::createSparsePageAllocator() {
return new DxvkSparsePageAllocator(this, m_objects.memoryManager());
}
DxvkStatCounters DxvkDevice::getStatCounters() {
DxvkPipelineCount pipe = m_objects.pipelineManager().getPipelineCount();

7
src/dxvk/dxvk_device.h
View File

@ -22,6 +22,7 @@
#include "dxvk_renderpass.h"
#include "dxvk_sampler.h"
#include "dxvk_shader.h"
#include "dxvk_sparse.h"
#include "dxvk_stats.h"
#include "dxvk_unbound.h"
#include "dxvk_marker.h"
@ -380,6 +381,12 @@ namespace dxvk {
Rc<DxvkSampler> createSampler(
const DxvkSamplerCreateInfo& createInfo);
/**
* \brief Creates a sparse page allocator
* \returns Sparse page allocator
*/
Rc<DxvkSparsePageAllocator> createSparsePageAllocator();
/**
* \brief Retrieves stat counters
*

93
src/dxvk/dxvk_memory.cpp
View File

@ -179,9 +179,8 @@ namespace dxvk {
}
DxvkMemoryAllocator::DxvkMemoryAllocator(const DxvkDevice* device)
: m_vkd (device->vkd()),
m_device (device),
DxvkMemoryAllocator::DxvkMemoryAllocator(DxvkDevice* device)
: m_device (device),
m_devProps (device->adapter()->deviceProperties()),
m_memProps (device->adapter()->memoryProperties()) {
for (uint32_t i = 0; i < m_memProps.memoryHeapCount; i++) {
@ -202,6 +201,9 @@ namespace dxvk {
m_memTypes[i].memType = m_memProps.memoryTypes[i];
m_memTypes[i].memTypeId = i;
}
if (device->features().core.features.sparseBinding)
m_sparseMemoryTypes = determineSparseMemoryTypes(device);
}
@ -366,6 +368,8 @@ namespace dxvk {
VkDeviceSize size,
DxvkMemoryProperties info,
DxvkMemoryFlags hints) {
auto vk = m_device->vkd();
bool useMemoryPriority = (info.flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
&& (m_device->features().extMemoryPriority.memoryPriority);
@ -403,15 +407,15 @@ namespace dxvk {
if (useMemoryPriority)
priorityInfo.pNext = std::exchange(memoryInfo.pNext, &priorityInfo);
if (m_vkd->vkAllocateMemory(m_vkd->device(), &memoryInfo, nullptr, &result.memHandle) != VK_SUCCESS)
if (vk->vkAllocateMemory(vk->device(), &memoryInfo, nullptr, &result.memHandle))
return DxvkDeviceMemory();
if (info.flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
VkResult status = m_vkd->vkMapMemory(m_vkd->device(), result.memHandle, 0, VK_WHOLE_SIZE, 0, &result.memPointer);
VkResult status = vk->vkMapMemory(vk->device(), result.memHandle, 0, VK_WHOLE_SIZE, 0, &result.memPointer);
if (status != VK_SUCCESS) {
if (status) {
Logger::err(str::format("DxvkMemoryAllocator: Mapping memory failed with ", status));
m_vkd->vkFreeMemory(m_vkd->device(), result.memHandle, nullptr);
vk->vkFreeMemory(vk->device(), result.memHandle, nullptr);
return DxvkDeviceMemory();
}
}
@ -467,7 +471,9 @@ namespace dxvk {
void DxvkMemoryAllocator::freeDeviceMemory(
DxvkMemoryType* type,
DxvkDeviceMemory memory) {
m_vkd->vkFreeMemory(m_vkd->device(), memory.memHandle, nullptr);
auto vk = m_device->vkd();
vk->vkFreeMemory(vk->device(), memory.memHandle, nullptr);
type->heap->stats.memoryAllocated -= memory.memSize;
m_device->adapter()->notifyHeapMemoryFree(type->heapId, memory.memSize);
}
@ -542,4 +548,75 @@ namespace dxvk {
}
}
uint32_t DxvkMemoryAllocator::determineSparseMemoryTypes(
DxvkDevice* device) const {
auto vk = device->vkd();
VkMemoryRequirements requirements = { };
uint32_t typeMask = ~0u;
// Create sparse dummy buffer to find available memory types
VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
bufferInfo.flags = VK_BUFFER_CREATE_SPARSE_BINDING_BIT
| VK_BUFFER_CREATE_SPARSE_ALIASED_BIT
| VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT;
bufferInfo.size = 65536;
bufferInfo.usage = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT
| VK_BUFFER_USAGE_INDEX_BUFFER_BIT
| VK_BUFFER_USAGE_VERTEX_BUFFER_BIT
| VK_BUFFER_USAGE_STORAGE_BUFFER_BIT
| VK_BUFFER_USAGE_STORAGE_BUFFER_BIT
| VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT
| VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT
| VK_BUFFER_USAGE_TRANSFER_DST_BIT
| VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
VkBuffer buffer = VK_NULL_HANDLE;
if (vk->vkCreateBuffer(vk->device(), &bufferInfo, nullptr, &buffer)) {
Logger::err("Failed to create dummy buffer to query sparse memory types");
return 0;
}
vk->vkGetBufferMemoryRequirements(vk->device(), buffer, &requirements);
vk->vkDestroyBuffer(vk->device(), buffer, nullptr);
typeMask &= requirements.memoryTypeBits;
// Create sparse dummy image to find available memory types
VkImageCreateInfo imageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO };
imageInfo.flags = VK_IMAGE_CREATE_SPARSE_BINDING_BIT
| VK_IMAGE_CREATE_SPARSE_ALIASED_BIT
| VK_IMAGE_CREATE_SPARSE_RESIDENCY_BIT;
imageInfo.imageType = VK_IMAGE_TYPE_2D;
imageInfo.format = VK_FORMAT_R8G8B8A8_UNORM;
imageInfo.extent = { 256, 256, 1 };
imageInfo.mipLevels = 1;
imageInfo.arrayLayers = 1;
imageInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageInfo.tiling = VK_IMAGE_TILING_OPTIMAL;
imageInfo.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT
| VK_IMAGE_USAGE_SAMPLED_BIT
| VK_IMAGE_USAGE_STORAGE_BIT
| VK_IMAGE_USAGE_TRANSFER_DST_BIT
| VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
VkImage image = VK_NULL_HANDLE;
if (vk->vkCreateImage(vk->device(), &imageInfo, nullptr, &image)) {
Logger::err("Failed to create dummy image to query sparse memory types");
return 0;
}
vk->vkGetImageMemoryRequirements(vk->device(), image, &requirements);
vk->vkDestroyImage(vk->device(), image, nullptr);
typeMask &= requirements.memoryTypeBits;
Logger::log(typeMask ? LogLevel::Info : LogLevel::Error,
str::format("Memory type mask for sparse resources: 0x", std::hex, typeMask));
return typeMask;
}
}

24
src/dxvk/dxvk_memory.h
View File

@ -306,7 +306,7 @@ namespace dxvk {
constexpr static VkDeviceSize SmallAllocationThreshold = 256 << 10;
public:
DxvkMemoryAllocator(const DxvkDevice* device);
DxvkMemoryAllocator(DxvkDevice* device);
~DxvkMemoryAllocator();
/**
@ -320,7 +320,15 @@ namespace dxvk {
VkDeviceSize bufferImageGranularity() const {
return m_devProps.limits.bufferImageGranularity;
}
/**
* \brief Memory type mask for sparse resources
* \returns Sparse resource memory types
*/
uint32_t getSparseMemoryTypes() const {
return m_sparseMemoryTypes;
}
/**
* \brief Allocates device memory
*
@ -348,15 +356,16 @@ namespace dxvk {
private:
const Rc<vk::DeviceFn> m_vkd;
const DxvkDevice* m_device;
const VkPhysicalDeviceProperties m_devProps;
const VkPhysicalDeviceMemoryProperties m_memProps;
DxvkDevice* m_device;
VkPhysicalDeviceProperties m_devProps;
VkPhysicalDeviceMemoryProperties m_memProps;
dxvk::mutex m_mutex;
std::array<DxvkMemoryHeap, VK_MAX_MEMORY_HEAPS> m_memHeaps;
std::array<DxvkMemoryType, VK_MAX_MEMORY_TYPES> m_memTypes;
uint32_t m_sparseMemoryTypes = 0u;
DxvkMemory tryAlloc(
const DxvkMemoryRequirements& req,
const DxvkMemoryProperties& info,
@ -403,6 +412,9 @@ namespace dxvk {
void freeEmptyChunks(
const DxvkMemoryHeap* heap);
uint32_t determineSparseMemoryTypes(
DxvkDevice* device) const;
};
}

152
src/dxvk/dxvk_sparse.cpp
View File

@ -5,6 +5,156 @@
namespace dxvk {
DxvkSparseMapping::DxvkSparseMapping()
: m_pool(nullptr),
m_page(nullptr) {
}
DxvkSparseMapping::DxvkSparseMapping(
Rc<DxvkSparsePageAllocator> allocator,
Rc<DxvkSparsePage> page)
: m_pool(std::move(allocator)),
m_page(std::move(page)) {
}
DxvkSparseMapping::DxvkSparseMapping(
DxvkSparseMapping&& other)
: m_pool(std::move(other.m_pool)),
m_page(std::move(other.m_page)) {
// No need to acquire here. The only place from which
// this constructor can be called does this atomically.
}
DxvkSparseMapping::DxvkSparseMapping(
const DxvkSparseMapping& other)
: m_pool(other.m_pool),
m_page(other.m_page) {
this->acquire();
}
DxvkSparseMapping& DxvkSparseMapping::operator = (
DxvkSparseMapping&& other) {
this->release();
m_pool = std::move(other.m_pool);
m_page = std::move(other.m_page);
return *this;
}
DxvkSparseMapping& DxvkSparseMapping::operator = (
const DxvkSparseMapping& other) {
other.acquire();
this->release();
m_pool = other.m_pool;
m_page = other.m_page;
return *this;
}
DxvkSparseMapping::~DxvkSparseMapping() {
this->release();
}
void DxvkSparseMapping::acquire() const {
if (m_page != nullptr)
m_pool->acquirePage(m_page);
}
void DxvkSparseMapping::release() const {
if (m_page != nullptr)
m_pool->releasePage(m_page);
}
DxvkSparsePageAllocator::DxvkSparsePageAllocator(
DxvkDevice* device,
DxvkMemoryAllocator& memoryAllocator)
: m_device(device), m_memory(&memoryAllocator) {
}
DxvkSparsePageAllocator::~DxvkSparsePageAllocator() {
}
DxvkSparseMapping DxvkSparsePageAllocator::acquirePage(
uint32_t page) {
std::lock_guard lock(m_mutex);
if (unlikely(page >= m_pageCount))
return DxvkSparseMapping();
m_useCount += 1;
return DxvkSparseMapping(this, m_pages[page]);
}
void DxvkSparsePageAllocator::setCapacity(
uint32_t pageCount) {
std::lock_guard lock(m_mutex);
if (pageCount < m_pageCount) {
if (!m_useCount)
m_pages.resize(pageCount);
} else if (pageCount > m_pageCount) {
while (m_pages.size() < pageCount)
m_pages.push_back(allocPage());
}
m_pageCount = pageCount;
}
Rc<DxvkSparsePage> DxvkSparsePageAllocator::allocPage() {
DxvkMemoryRequirements memoryRequirements = { };
memoryRequirements.core = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2 };
// We don't know what kind of resource the memory6
// might be bound to, so just guess the memory types
auto& core = memoryRequirements.core.memoryRequirements;
core.size = SparseMemoryPageSize;
core.alignment = SparseMemoryPageSize;
core.memoryTypeBits = m_memory->getSparseMemoryTypes();
DxvkMemoryProperties memoryProperties = { };
memoryProperties.flags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
DxvkMemory memory = m_memory->alloc(memoryRequirements,
memoryProperties, DxvkMemoryFlag::GpuReadable);
return new DxvkSparsePage(std::move(memory));
}
void DxvkSparsePageAllocator::acquirePage(
const Rc<DxvkSparsePage>& page) {
std::lock_guard lock(m_mutex);
m_useCount += 1;
}
void DxvkSparsePageAllocator::releasePage(
const Rc<DxvkSparsePage>& page) {
std::lock_guard lock(m_mutex);
m_useCount -= 1;
if (!m_useCount)
m_pages.resize(m_pageCount);
}
DxvkSparsePageTable::DxvkSparsePageTable() {
}
@ -20,6 +170,7 @@ namespace dxvk {
// and consists of consecutive 64k pages
size_t pageCount = align(bufferSize, SparseMemoryPageSize) / SparseMemoryPageSize;
m_metadata.resize(pageCount);
m_mappings.resize(pageCount);
for (size_t i = 0; i < pageCount; i++) {
VkDeviceSize pageOffset = SparseMemoryPageSize * i;
@ -135,6 +286,7 @@ namespace dxvk {
// Fill in page metadata
m_metadata.reserve(totalPageCount);
m_mappings.resize(totalPageCount);
for (uint32_t l = 0; l < image->info().numLayers; l++) {
for (uint32_t m = 0; m < m_properties.pagedMipCount; m++) {

149
src/dxvk/dxvk_sparse.h
View File

@ -10,7 +10,7 @@ namespace dxvk {
class DxvkBuffer;
class DxvkImage;
class DxvkSparsePage;
class DxvkSparsePagePool;
class DxvkSparsePageAllocator;
constexpr static VkDeviceSize SparseMemoryPageSize = 1ull << 16;
@ -112,6 +112,152 @@ namespace dxvk {
};
/**
* \brief Sparse memory page
*
* Stores a single reference-counted page
* of memory. The page size is 64k.
*/
class DxvkSparsePage : public DxvkResource {
public:
DxvkSparsePage(DxvkMemory&& memory)
: m_memory(std::move(memory)) { }
/**
* \brief Queries memory handle
* \returns Memory information
*/
DxvkSparsePageHandle getHandle() const {
DxvkSparsePageHandle result;
result.memory = m_memory.memory();
result.offset = m_memory.offset();
result.length = m_memory.length();
return result;
}
private:
DxvkMemory m_memory;
};
/**
* \brief Sparse page mapping
*
* Stores a reference to a page as well as the pool that the page
* was allocated from, and automatically manages the use counter
* of the pool as the reference is being moved or copied around.
*/
class DxvkSparseMapping {
friend DxvkSparsePageAllocator;
public:
DxvkSparseMapping();
DxvkSparseMapping(DxvkSparseMapping&& other);
DxvkSparseMapping(const DxvkSparseMapping& other);
DxvkSparseMapping& operator = (DxvkSparseMapping&& other);
DxvkSparseMapping& operator = (const DxvkSparseMapping& other);
~DxvkSparseMapping();
Rc<DxvkSparsePage> getPage() const {
return m_page;
}
bool operator == (const DxvkSparseMapping& other) const {
// Pool is a function of the page, so no need to check both
return m_page == other.m_page;
}
bool operator != (const DxvkSparseMapping& other) const {
return m_page != other.m_page;
}
operator bool () const {
return m_page != nullptr;
}
private:
Rc<DxvkSparsePageAllocator> m_pool;
Rc<DxvkSparsePage> m_page;
DxvkSparseMapping(
Rc<DxvkSparsePageAllocator> allocator,
Rc<DxvkSparsePage> page);
void acquire() const;
void release() const;
};
/**
* \brief Sparse memory allocator
*
* Provides an allocator for sparse pages with variable capacity.
* Pages are use-counted to make sure they are not removed from
* the allocator too early.
*/
class DxvkSparsePageAllocator : public RcObject {
friend DxvkSparseMapping;
public:
DxvkSparsePageAllocator(
DxvkDevice* device,
DxvkMemoryAllocator& memoryAllocator);
~DxvkSparsePageAllocator();
/**
* \brief Acquires page at the given offset
*
* If the offset is valid, this will atomically
* increment the allocator's use count and return
* a reference to the page.
* \param [in] page Page index
* \returns Page mapping object
*/
DxvkSparseMapping acquirePage(
uint32_t page);
/**
* \brief Changes the allocator's maximum capacity
*
* Allocates new pages as necessary, and frees existing
* pages if none of the pages are currently in use.
* \param [in] pageCount New capacity, in pages
*/
void setCapacity(
uint32_t pageCount);
private:
DxvkDevice* m_device;
DxvkMemoryAllocator* m_memory;
dxvk::mutex m_mutex;
uint32_t m_pageCount = 0u;
uint32_t m_useCount = 0u;
std::vector<Rc<DxvkSparsePage>> m_pages;
Rc<DxvkSparsePage> allocPage();
void acquirePage(
const Rc<DxvkSparsePage>& page);
void releasePage(
const Rc<DxvkSparsePage>& page);
};
/**
* \brief Sparse page table
*
@ -202,6 +348,7 @@ namespace dxvk {
DxvkSparseImageProperties m_properties = { };
std::vector<DxvkSparseImageSubresourceProperties> m_subresources;
std::vector<DxvkSparsePageInfo> m_metadata;
std::vector<DxvkSparseMapping> m_mappings;
};