rooty/dxvk
1
0
Fork 0
mirror of https://github.com/doitsujin/dxvk.git synced 2025-01-31 14:52:11 +01:00
Code Issues Projects Releases Wiki Activity

[dxvk] Add chunk concept to page allocator

Browse Source 
This allows the allocator to operate on the entire allocated memory pool.
This commit is contained in:
Philip Rebohle 2024-09-20 16:47:55 +02:00 committed by Philip Rebohle
parent 266b99ad8d
commit 3a4dadb528
3 changed files with 163 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
src/dxvk/dxvk_allocator.cpp
View File

@ -8,13 +8,8 @@
namespace dxvk {
DxvkPageAllocator::DxvkPageAllocator(uint64_t capacity)
: m_pageCount(capacity / PageSize), m_freeListLutByPage(m_pageCount, -1) {
PageRange freeRange = { };
freeRange.index = 0u;
freeRange.count = m_pageCount;
DxvkPageAllocator::DxvkPageAllocator() {
insertFreeRange(freeRange, -1);
}
@ -47,7 +42,8 @@ namespace dxvk {
insertFreeRange(entry, index);
m_pagesUsed += count;
uint32_t chunkIndex = pageIndex >> ChunkPageBits;
m_chunks[chunkIndex].pagesUsed += count;
return pageIndex;
} else {
// Apply alignment and skip if the free range is too small.
@ -72,7 +68,9 @@ namespace dxvk {
if (nextRange.count)
insertFreeRange(nextRange, -1);
m_pagesUsed += count;
uint32_t chunkIndex = pageIndex >> ChunkPageBits;
m_chunks[chunkIndex].pagesUsed += count;
return pageIndex;
}
}
@ -81,24 +79,24 @@ namespace dxvk {
}
void DxvkPageAllocator::free(uint64_t address, uint64_t size) {
bool DxvkPageAllocator::free(uint64_t address, uint64_t size) {
uint32_t pageIndex = address / PageSize;
uint32_t pageCount = (size + PageSize - 1u) / PageSize;
freePages(pageIndex, pageCount);
return freePages(pageIndex, pageCount);
}
void DxvkPageAllocator::freePages(uint32_t index, uint32_t count) {
bool DxvkPageAllocator::freePages(uint32_t index, uint32_t count) {
// Use the lookup table to quickly determine which
// free ranges we can actually merge with
int32_t prevRange = -1;
int32_t nextRange = -1;
if (index > 0u)
if (index & ChunkPageMask)
prevRange = m_freeListLutByPage[index - 1];
if (index + count < m_pageCount)
if ((index + count) & ChunkPageMask)
nextRange = m_freeListLutByPage[index + count];
if (prevRange < 0) {
@ -144,14 +142,60 @@ namespace dxvk {
insertFreeRange(mergedRange, std::min(prevRange, nextRange));
}
m_pagesUsed -= count;
uint32_t chunkIndex = index >> ChunkPageBits;
return !(m_chunks[chunkIndex].pagesUsed -= count);
}
void DxvkPageAllocator::getPageAllocationMask(uint32_t* pageMask) const {
uint32_t DxvkPageAllocator::addChunk(uint64_t size) {
int32_t chunkIndex = m_freeChunk;
if (chunkIndex < 0) {
chunkIndex = m_chunks.size();
m_freeListLutByPage.resize((chunkIndex + 1u) << ChunkPageBits, -1);
m_chunks.emplace_back();
}
auto& chunk = m_chunks[chunkIndex];
m_freeChunk = chunk.nextChunk;
chunk.pageCount = size / PageSize;
chunk.pagesUsed = 0u;
chunk.nextChunk = -1;
PageRange pageRange = { };
pageRange.index = uint32_t(chunkIndex) << ChunkPageBits;
pageRange.count = chunk.pageCount;
insertFreeRange(pageRange, -1);
return uint32_t(chunkIndex);
}
void DxvkPageAllocator::removeChunk(uint32_t chunkIndex) {
auto& chunk = m_chunks[chunkIndex];
chunk.pageCount = 0u;
chunk.pagesUsed = 0u;
chunk.nextChunk = std::exchange(m_freeChunk, int32_t(chunkIndex));
uint32_t pageIndex = chunkIndex << ChunkPageBits;
PageRange pageRange = { };
pageRange.index = pageIndex;
pageRange.count = 0;
insertFreeRange(pageRange, m_freeListLutByPage[pageIndex]);
}
void DxvkPageAllocator::getPageAllocationMask(uint32_t chunkIndex, uint32_t* pageMask) const {
// Initialize bit mask with all ones
uint32_t fullCount = m_pageCount / 32u;
uint32_t lastCount = m_pageCount % 32u;
const auto& chunk = m_chunks[chunkIndex];
uint32_t fullCount = chunk.pageCount / 32u;
uint32_t lastCount = chunk.pageCount % 32u;
for (uint32_t i = 0; i < fullCount; i++)
pageMask[i] = ~0u;
@ -159,8 +203,14 @@ namespace dxvk {
if (lastCount)
pageMask[fullCount] = (1u << lastCount) - 1u;
// Iterate over free list and set all included pages to 0.
// Iterate over free list and set all pages included
// in the current chunk to 0.
for (PageRange range : m_freeList) {
if ((range.index >> ChunkPageBits) != chunkIndex)
continue;
range.index &= ChunkPageMask;
uint32_t index = range.index / 32u;
uint32_t shift = range.index % 32u;
@ -276,7 +326,7 @@ namespace dxvk {
DxvkPoolAllocator::DxvkPoolAllocator(DxvkPageAllocator& pageAllocator)
: m_pageAllocator(&pageAllocator), m_pageInfos(m_pageAllocator->pageCount()) {
: m_pageAllocator(&pageAllocator) {
}
@ -349,7 +399,7 @@ namespace dxvk {
}
void DxvkPoolAllocator::free(uint64_t address, uint64_t size) {
bool DxvkPoolAllocator::free(uint64_t address, uint64_t size) {
uint32_t listIndex = computeListIndex(size);
uint32_t pageIndex = computePageIndexFromByteAddress(address);
@ -369,7 +419,9 @@ namespace dxvk {
page.pool |= MaskType(1) << itemIndex;
if (unlikely(bit::tzcnt(page.pool + 1u) >= poolCapacity))
freePage(pageIndex, listIndex);
return freePage(pageIndex, listIndex);
return false;
} else {
PageInfo& page = m_pageInfos[pageIndex];
PagePool& pool = m_pagePools[page.pool];
@ -390,9 +442,11 @@ namespace dxvk {
if (unlikely(!pool.usedMask)) {
freePagePool(page.pool);
freePage(pageIndex, listIndex);
return freePage(pageIndex, listIndex);
}
}
return false;
}
}
@ -403,15 +457,20 @@ namespace dxvk {
if (unlikely(pageIndex < 0))
return -1;
if (unlikely(uint32_t(pageIndex) >= m_pageInfos.size())) {
uint32_t chunkCount = (pageIndex >> DxvkPageAllocator::ChunkPageBits) + 1u;
m_pageInfos.resize(chunkCount << DxvkPageAllocator::ChunkPageBits);
}
addPageToList(pageIndex, listIndex);
return pageIndex;
}
void DxvkPoolAllocator::freePage(uint32_t pageIndex, uint32_t listIndex) {
bool DxvkPoolAllocator::freePage(uint32_t pageIndex, uint32_t listIndex) {
removePageFromList(pageIndex, listIndex);
m_pageAllocator->freePages(pageIndex, 1u);
return m_pageAllocator->freePages(pageIndex, 1u);
}

93
src/dxvk/dxvk_allocator.h
View File

@ -28,24 +28,51 @@ namespace dxvk {
constexpr static uint32_t PageBits = 16;
constexpr static uint64_t PageSize = 1u << PageBits;
DxvkPageAllocator(uint64_t capacity);
/// Maximum number of pages per chunk. Chunks represent contiguous memory
/// allocations whose free regions can be merged.
constexpr static uint32_t ChunkPageBits = 12u;
constexpr static uint32_t ChunkPageMask = (1u << ChunkPageBits) - 1u;
/// Chunk address bits. Can be used to quickly compute the chunk index
/// and allocation offset within the chunk from a raw byte address.
constexpr static uint32_t ChunkAddressBits = ChunkPageBits + PageBits;
constexpr static uint64_t ChunkAddressMask = (1u << ChunkAddressBits) - 1u;
constexpr static uint64_t MaxChunkSize = 1u << ChunkAddressBits;
DxvkPageAllocator();
~DxvkPageAllocator();
/**
* \brief Queries number of available pages
* \returns Total page count
* \brief Queries total number of chunks
*
* This number may include chuks that have already been removed.
* \returns Total chunk count
*/
uint32_t pageCount() const {
return m_pageCount;
uint32_t chunkCount() const {
return uint32_t(m_chunks.size());
}
/**
* \brief Queries number of allocated pages
* \returns \c Used page count
* \brief Queries number of available pages in a chunk
*
* \param [in] chunkIndex Chunk index
* \returns Capacity of the given chunk
*/
uint32_t pagesUsed() const {
return m_pagesUsed;
uint32_t pageCount(uint32_t chunkIndex) const {
return m_chunks.at(chunkIndex).pageCount;
}
/**
* \brief Queries number of allocated pages in a chunk
*
* \param [in] chunkIndex Chunk index
* \returns Used page count in the given chunk
*/
uint32_t pagesUsed(uint32_t chunkIndex) const {
return m_chunks.at(chunkIndex).pagesUsed;
}
/**
@ -73,16 +100,36 @@ namespace dxvk {
*
* \param [in] address Allocated address, in bytes
* \param [in] size Allocation size, in bytes
* \returns \c true if a chunk was freed
*/
void free(uint64_t address, uint64_t size);
bool free(uint64_t address, uint64_t size);
/**
* \brief Frees pages
*
* \param [in] index Index of first page to free
* \param [in] count Number of pages to free
* \returns \c true if a chunk was freed
*/
void freePages(uint32_t index, uint32_t count);
bool freePages(uint32_t index, uint32_t count);
/**
* \brief Adds a chunk to the allocator
*
* Adds the given region to the free list, so
* that subsequent allocations can succeed.
* \param [in] size Total chunk size, in bytes
* \returns Chunk index
*/
uint32_t addChunk(uint64_t size);
/**
* \brief Removes chunk from the allocator
*
* Must only be used if the entire chunk is unused.
* \param [in] chunkIndex Chunk index
*/
void removeChunk(uint32_t chunkIndex);
/**
* \brief Queries page allocation mask
@ -91,23 +138,30 @@ namespace dxvk {
* a bit mask where each set bit represents an allocated page,
* with the page index corresponding to the page index. The
* output array must be sized appropriately.
* \param [out] chunkIndex Chunk index
* \param [out] pageMask Page mask
*/
void getPageAllocationMask(uint32_t* pageMask) const;
void getPageAllocationMask(uint32_t chunkIndex, uint32_t* pageMask) const;
private:
struct PageRange {
uint32_t index = 0u;
uint32_t count = 0u;
struct ChunkInfo {
uint32_t pageCount = 0u;
uint32_t pagesUsed = 0u;
int32_t nextChunk = -1;
};
uint32_t m_pageCount = 0u;
uint32_t m_pagesUsed = 0u;
struct PageRange {
uint32_t index = 0u;
uint32_t count = 0u;
};
std::vector<PageRange> m_freeList;
std::vector<int32_t> m_freeListLutByPage;
std::vector<ChunkInfo> m_chunks;
int32_t m_freeChunk = -1;
int32_t searchFreeList(uint32_t count);
void addLutEntry(const PageRange& range, int32_t index);
@ -163,8 +217,9 @@ namespace dxvk {
*
* \param [in] address Memory address, in bytes
* \param [in] size Allocation size, in bytes
* \returns \c true if a chunk was freed
*/
void free(uint64_t address, uint64_t size);
bool free(uint64_t address, uint64_t size);
private:
@ -197,7 +252,7 @@ namespace dxvk {
int32_t allocPage(uint32_t listIndex);
void freePage(uint32_t pageIndex, uint32_t listIndex);
bool freePage(uint32_t pageIndex, uint32_t listIndex);
void addPageToList(uint32_t pageIndex, uint32_t listIndex);

13
src/dxvk/dxvk_memory.cpp
View File

@ -68,9 +68,8 @@ namespace dxvk {
DxvkMemoryType* type,
DxvkDeviceMemory memory)
: m_alloc(alloc), m_type(type), m_memory(memory),
m_pageAllocator(memory.memSize),
m_poolAllocator(m_pageAllocator) {
m_pageAllocator.addChunk(memory.memSize);
}
@ -131,19 +130,19 @@ namespace dxvk {
bool DxvkMemoryChunk::isEmpty() const {
return m_pageAllocator.pagesUsed() == 0u;
return m_pageAllocator.pagesUsed(0u) == 0u;
}
void DxvkMemoryChunk::getAllocationStats(DxvkMemoryAllocationStats& stats) const {
auto& chunkStats = stats.chunks.emplace_back();
chunkStats.capacity = uint64_t(m_pageAllocator.pageCount()) * DxvkPageAllocator::PageSize;
chunkStats.used = uint64_t(m_pageAllocator.pagesUsed()) * DxvkPageAllocator::PageSize;
chunkStats.capacity = uint64_t(m_pageAllocator.pageCount(0u)) * DxvkPageAllocator::PageSize;
chunkStats.used = uint64_t(m_pageAllocator.pagesUsed(0u)) * DxvkPageAllocator::PageSize;
chunkStats.pageMaskOffset = stats.pageMasks.size();
chunkStats.pageCount = m_pageAllocator.pageCount();
chunkStats.pageCount = m_pageAllocator.pageCount(0u);
stats.pageMasks.resize(chunkStats.pageMaskOffset + (chunkStats.pageCount + 31u) / 32u);
m_pageAllocator.getPageAllocationMask(&stats.pageMasks.at(chunkStats.pageMaskOffset));
m_pageAllocator.getPageAllocationMask(0u, &stats.pageMasks.at(chunkStats.pageMaskOffset));
}