diff --git a/src/dxvk/dxvk_allocator.cpp b/src/dxvk/dxvk_allocator.cpp index 0692cc846..97a0fdebd 100644 --- a/src/dxvk/dxvk_allocator.cpp +++ b/src/dxvk/dxvk_allocator.cpp @@ -8,13 +8,8 @@ namespace dxvk { - DxvkPageAllocator::DxvkPageAllocator(uint64_t capacity) - : m_pageCount(capacity / PageSize), m_freeListLutByPage(m_pageCount, -1) { - PageRange freeRange = { }; - freeRange.index = 0u; - freeRange.count = m_pageCount; + DxvkPageAllocator::DxvkPageAllocator() { - insertFreeRange(freeRange, -1); } @@ -47,7 +42,8 @@ namespace dxvk { insertFreeRange(entry, index); - m_pagesUsed += count; + uint32_t chunkIndex = pageIndex >> ChunkPageBits; + m_chunks[chunkIndex].pagesUsed += count; return pageIndex; } else { // Apply alignment and skip if the free range is too small. @@ -72,7 +68,9 @@ namespace dxvk { if (nextRange.count) insertFreeRange(nextRange, -1); - m_pagesUsed += count; + uint32_t chunkIndex = pageIndex >> ChunkPageBits; + m_chunks[chunkIndex].pagesUsed += count; + return pageIndex; } } @@ -81,24 +79,24 @@ namespace dxvk { } - void DxvkPageAllocator::free(uint64_t address, uint64_t size) { + bool DxvkPageAllocator::free(uint64_t address, uint64_t size) { uint32_t pageIndex = address / PageSize; uint32_t pageCount = (size + PageSize - 1u) / PageSize; - freePages(pageIndex, pageCount); + return freePages(pageIndex, pageCount); } - void DxvkPageAllocator::freePages(uint32_t index, uint32_t count) { + bool DxvkPageAllocator::freePages(uint32_t index, uint32_t count) { // Use the lookup table to quickly determine which // free ranges we can actually merge with int32_t prevRange = -1; int32_t nextRange = -1; - if (index > 0u) + if (index & ChunkPageMask) prevRange = m_freeListLutByPage[index - 1]; - if (index + count < m_pageCount) + if ((index + count) & ChunkPageMask) nextRange = m_freeListLutByPage[index + count]; if (prevRange < 0) { @@ -144,14 +142,60 @@ namespace dxvk { insertFreeRange(mergedRange, std::min(prevRange, nextRange)); } - m_pagesUsed -= count; + uint32_t chunkIndex = index >> ChunkPageBits; + return !(m_chunks[chunkIndex].pagesUsed -= count); } - void DxvkPageAllocator::getPageAllocationMask(uint32_t* pageMask) const { + uint32_t DxvkPageAllocator::addChunk(uint64_t size) { + int32_t chunkIndex = m_freeChunk; + + if (chunkIndex < 0) { + chunkIndex = m_chunks.size(); + + m_freeListLutByPage.resize((chunkIndex + 1u) << ChunkPageBits, -1); + m_chunks.emplace_back(); + } + + auto& chunk = m_chunks[chunkIndex]; + m_freeChunk = chunk.nextChunk; + + chunk.pageCount = size / PageSize; + chunk.pagesUsed = 0u; + chunk.nextChunk = -1; + + PageRange pageRange = { }; + pageRange.index = uint32_t(chunkIndex) << ChunkPageBits; + pageRange.count = chunk.pageCount; + + insertFreeRange(pageRange, -1); + + return uint32_t(chunkIndex); + } + + + void DxvkPageAllocator::removeChunk(uint32_t chunkIndex) { + auto& chunk = m_chunks[chunkIndex]; + chunk.pageCount = 0u; + chunk.pagesUsed = 0u; + chunk.nextChunk = std::exchange(m_freeChunk, int32_t(chunkIndex)); + + uint32_t pageIndex = chunkIndex << ChunkPageBits; + + PageRange pageRange = { }; + pageRange.index = pageIndex; + pageRange.count = 0; + + insertFreeRange(pageRange, m_freeListLutByPage[pageIndex]); + } + + + void DxvkPageAllocator::getPageAllocationMask(uint32_t chunkIndex, uint32_t* pageMask) const { // Initialize bit mask with all ones - uint32_t fullCount = m_pageCount / 32u; - uint32_t lastCount = m_pageCount % 32u; + const auto& chunk = m_chunks[chunkIndex]; + + uint32_t fullCount = chunk.pageCount / 32u; + uint32_t lastCount = chunk.pageCount % 32u; for (uint32_t i = 0; i < fullCount; i++) pageMask[i] = ~0u; @@ -159,8 +203,14 @@ namespace dxvk { if (lastCount) pageMask[fullCount] = (1u << lastCount) - 1u; - // Iterate over free list and set all included pages to 0. + // Iterate over free list and set all pages included + // in the current chunk to 0. for (PageRange range : m_freeList) { + if ((range.index >> ChunkPageBits) != chunkIndex) + continue; + + range.index &= ChunkPageMask; + uint32_t index = range.index / 32u; uint32_t shift = range.index % 32u; @@ -276,7 +326,7 @@ namespace dxvk { DxvkPoolAllocator::DxvkPoolAllocator(DxvkPageAllocator& pageAllocator) - : m_pageAllocator(&pageAllocator), m_pageInfos(m_pageAllocator->pageCount()) { + : m_pageAllocator(&pageAllocator) { } @@ -349,7 +399,7 @@ namespace dxvk { } - void DxvkPoolAllocator::free(uint64_t address, uint64_t size) { + bool DxvkPoolAllocator::free(uint64_t address, uint64_t size) { uint32_t listIndex = computeListIndex(size); uint32_t pageIndex = computePageIndexFromByteAddress(address); @@ -369,7 +419,9 @@ namespace dxvk { page.pool |= MaskType(1) << itemIndex; if (unlikely(bit::tzcnt(page.pool + 1u) >= poolCapacity)) - freePage(pageIndex, listIndex); + return freePage(pageIndex, listIndex); + + return false; } else { PageInfo& page = m_pageInfos[pageIndex]; PagePool& pool = m_pagePools[page.pool]; @@ -390,9 +442,11 @@ namespace dxvk { if (unlikely(!pool.usedMask)) { freePagePool(page.pool); - freePage(pageIndex, listIndex); + return freePage(pageIndex, listIndex); } } + + return false; } } @@ -403,15 +457,20 @@ namespace dxvk { if (unlikely(pageIndex < 0)) return -1; + if (unlikely(uint32_t(pageIndex) >= m_pageInfos.size())) { + uint32_t chunkCount = (pageIndex >> DxvkPageAllocator::ChunkPageBits) + 1u; + m_pageInfos.resize(chunkCount << DxvkPageAllocator::ChunkPageBits); + } + addPageToList(pageIndex, listIndex); return pageIndex; } - void DxvkPoolAllocator::freePage(uint32_t pageIndex, uint32_t listIndex) { + bool DxvkPoolAllocator::freePage(uint32_t pageIndex, uint32_t listIndex) { removePageFromList(pageIndex, listIndex); - m_pageAllocator->freePages(pageIndex, 1u); + return m_pageAllocator->freePages(pageIndex, 1u); } diff --git a/src/dxvk/dxvk_allocator.h b/src/dxvk/dxvk_allocator.h index ba4942ec4..ea1357f83 100644 --- a/src/dxvk/dxvk_allocator.h +++ b/src/dxvk/dxvk_allocator.h @@ -28,24 +28,51 @@ namespace dxvk { constexpr static uint32_t PageBits = 16; constexpr static uint64_t PageSize = 1u << PageBits; - DxvkPageAllocator(uint64_t capacity); + /// Maximum number of pages per chunk. Chunks represent contiguous memory + /// allocations whose free regions can be merged. + constexpr static uint32_t ChunkPageBits = 12u; + constexpr static uint32_t ChunkPageMask = (1u << ChunkPageBits) - 1u; + + /// Chunk address bits. Can be used to quickly compute the chunk index + /// and allocation offset within the chunk from a raw byte address. + constexpr static uint32_t ChunkAddressBits = ChunkPageBits + PageBits; + constexpr static uint64_t ChunkAddressMask = (1u << ChunkAddressBits) - 1u; + + constexpr static uint64_t MaxChunkSize = 1u << ChunkAddressBits; + + + DxvkPageAllocator(); ~DxvkPageAllocator(); /** - * \brief Queries number of available pages - * \returns Total page count + * \brief Queries total number of chunks + * + * This number may include chuks that have already been removed. + * \returns Total chunk count */ - uint32_t pageCount() const { - return m_pageCount; + uint32_t chunkCount() const { + return uint32_t(m_chunks.size()); } /** - * \brief Queries number of allocated pages - * \returns \c Used page count + * \brief Queries number of available pages in a chunk + * + * \param [in] chunkIndex Chunk index + * \returns Capacity of the given chunk */ - uint32_t pagesUsed() const { - return m_pagesUsed; + uint32_t pageCount(uint32_t chunkIndex) const { + return m_chunks.at(chunkIndex).pageCount; + } + + /** + * \brief Queries number of allocated pages in a chunk + * + * \param [in] chunkIndex Chunk index + * \returns Used page count in the given chunk + */ + uint32_t pagesUsed(uint32_t chunkIndex) const { + return m_chunks.at(chunkIndex).pagesUsed; } /** @@ -73,16 +100,36 @@ namespace dxvk { * * \param [in] address Allocated address, in bytes * \param [in] size Allocation size, in bytes + * \returns \c true if a chunk was freed */ - void free(uint64_t address, uint64_t size); + bool free(uint64_t address, uint64_t size); /** * \brief Frees pages * * \param [in] index Index of first page to free * \param [in] count Number of pages to free + * \returns \c true if a chunk was freed */ - void freePages(uint32_t index, uint32_t count); + bool freePages(uint32_t index, uint32_t count); + + /** + * \brief Adds a chunk to the allocator + * + * Adds the given region to the free list, so + * that subsequent allocations can succeed. + * \param [in] size Total chunk size, in bytes + * \returns Chunk index + */ + uint32_t addChunk(uint64_t size); + + /** + * \brief Removes chunk from the allocator + * + * Must only be used if the entire chunk is unused. + * \param [in] chunkIndex Chunk index + */ + void removeChunk(uint32_t chunkIndex); /** * \brief Queries page allocation mask @@ -91,23 +138,30 @@ namespace dxvk { * a bit mask where each set bit represents an allocated page, * with the page index corresponding to the page index. The * output array must be sized appropriately. + * \param [out] chunkIndex Chunk index * \param [out] pageMask Page mask */ - void getPageAllocationMask(uint32_t* pageMask) const; + void getPageAllocationMask(uint32_t chunkIndex, uint32_t* pageMask) const; private: - struct PageRange { - uint32_t index = 0u; - uint32_t count = 0u; + struct ChunkInfo { + uint32_t pageCount = 0u; + uint32_t pagesUsed = 0u; + int32_t nextChunk = -1; }; - uint32_t m_pageCount = 0u; - uint32_t m_pagesUsed = 0u; + struct PageRange { + uint32_t index = 0u; + uint32_t count = 0u; + }; std::vector m_freeList; std::vector m_freeListLutByPage; + std::vector m_chunks; + int32_t m_freeChunk = -1; + int32_t searchFreeList(uint32_t count); void addLutEntry(const PageRange& range, int32_t index); @@ -163,8 +217,9 @@ namespace dxvk { * * \param [in] address Memory address, in bytes * \param [in] size Allocation size, in bytes + * \returns \c true if a chunk was freed */ - void free(uint64_t address, uint64_t size); + bool free(uint64_t address, uint64_t size); private: @@ -197,7 +252,7 @@ namespace dxvk { int32_t allocPage(uint32_t listIndex); - void freePage(uint32_t pageIndex, uint32_t listIndex); + bool freePage(uint32_t pageIndex, uint32_t listIndex); void addPageToList(uint32_t pageIndex, uint32_t listIndex); diff --git a/src/dxvk/dxvk_memory.cpp b/src/dxvk/dxvk_memory.cpp index 152729f84..436ab04fb 100644 --- a/src/dxvk/dxvk_memory.cpp +++ b/src/dxvk/dxvk_memory.cpp @@ -68,9 +68,8 @@ namespace dxvk { DxvkMemoryType* type, DxvkDeviceMemory memory) : m_alloc(alloc), m_type(type), m_memory(memory), - m_pageAllocator(memory.memSize), m_poolAllocator(m_pageAllocator) { - + m_pageAllocator.addChunk(memory.memSize); } @@ -131,19 +130,19 @@ namespace dxvk { bool DxvkMemoryChunk::isEmpty() const { - return m_pageAllocator.pagesUsed() == 0u; + return m_pageAllocator.pagesUsed(0u) == 0u; } void DxvkMemoryChunk::getAllocationStats(DxvkMemoryAllocationStats& stats) const { auto& chunkStats = stats.chunks.emplace_back(); - chunkStats.capacity = uint64_t(m_pageAllocator.pageCount()) * DxvkPageAllocator::PageSize; - chunkStats.used = uint64_t(m_pageAllocator.pagesUsed()) * DxvkPageAllocator::PageSize; + chunkStats.capacity = uint64_t(m_pageAllocator.pageCount(0u)) * DxvkPageAllocator::PageSize; + chunkStats.used = uint64_t(m_pageAllocator.pagesUsed(0u)) * DxvkPageAllocator::PageSize; chunkStats.pageMaskOffset = stats.pageMasks.size(); - chunkStats.pageCount = m_pageAllocator.pageCount(); + chunkStats.pageCount = m_pageAllocator.pageCount(0u); stats.pageMasks.resize(chunkStats.pageMaskOffset + (chunkStats.pageCount + 31u) / 32u); - m_pageAllocator.getPageAllocationMask(&stats.pageMasks.at(chunkStats.pageMaskOffset)); + m_pageAllocator.getPageAllocationMask(0u, &stats.pageMasks.at(chunkStats.pageMaskOffset)); }