diff --git a/src/dxvk/dxvk_sparse.cpp b/src/dxvk/dxvk_sparse.cpp index e521a49f1..c022b8063 100644 --- a/src/dxvk/dxvk_sparse.cpp +++ b/src/dxvk/dxvk_sparse.cpp @@ -1,3 +1,5 @@ +#include + #include "dxvk_buffer.h" #include "dxvk_device.h" #include "dxvk_image.h" @@ -396,4 +398,330 @@ namespace dxvk { } } + + DxvkSparseBindSubmission::DxvkSparseBindSubmission() { + + } + + + DxvkSparseBindSubmission::~DxvkSparseBindSubmission() { + + } + + + void DxvkSparseBindSubmission::waitSemaphore( + VkSemaphore semaphore, + uint64_t value) { + m_waitSemaphores.push_back(semaphore); + m_waitSemaphoreValues.push_back(value); + } + + + void DxvkSparseBindSubmission::signalSemaphore( + VkSemaphore semaphore, + uint64_t value) { + m_signalSemaphores.push_back(semaphore); + m_signalSemaphoreValues.push_back(value); + } + + + void DxvkSparseBindSubmission::bindBufferMemory( + const DxvkSparseBufferBindKey& key, + const DxvkSparsePageHandle& memory) { + m_bufferBinds.insert_or_assign(key, memory); + } + + + void DxvkSparseBindSubmission::bindImageMemory( + const DxvkSparseImageBindKey& key, + const DxvkSparsePageHandle& memory) { + m_imageBinds.insert_or_assign(key, memory); + } + + + void DxvkSparseBindSubmission::bindImageOpaqueMemory( + const DxvkSparseImageOpaqueBindKey& key, + const DxvkSparsePageHandle& memory) { + m_imageOpaqueBinds.insert_or_assign(key, memory); + } + + + VkResult DxvkSparseBindSubmission::submit( + DxvkDevice* device, + VkQueue queue) { + auto vk = device->vkd(); + + DxvkSparseBufferBindArrays buffer; + this->processBufferBinds(buffer); + + DxvkSparseImageBindArrays image; + this->processImageBinds(image); + + DxvkSparseImageOpaqueBindArrays opaque; + this->processOpaqueBinds(opaque); + + // The sparse binding API has never been updated to take the new + // semaphore submission info structs, so we have to do this instead + VkTimelineSemaphoreSubmitInfo timelineInfo = { VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO }; + timelineInfo.waitSemaphoreValueCount = m_waitSemaphoreValues.size(); + timelineInfo.pWaitSemaphoreValues = m_waitSemaphoreValues.data(); + timelineInfo.signalSemaphoreValueCount = m_signalSemaphoreValues.size(); + timelineInfo.pSignalSemaphoreValues = m_signalSemaphoreValues.data(); + + VkBindSparseInfo bindInfo = { VK_STRUCTURE_TYPE_BIND_SPARSE_INFO }; + + if (!m_waitSemaphores.empty()) { + bindInfo.pNext = &timelineInfo; + bindInfo.waitSemaphoreCount = m_waitSemaphores.size(); + bindInfo.pWaitSemaphores = m_waitSemaphores.data(); + } + + if (!buffer.infos.empty()) { + bindInfo.bufferBindCount = buffer.infos.size(); + bindInfo.pBufferBinds = buffer.infos.data(); + } + + if (!opaque.infos.empty()) { + bindInfo.imageOpaqueBindCount = opaque.infos.size(); + bindInfo.pImageOpaqueBinds = opaque.infos.data(); + } + + if (!image.infos.empty()) { + bindInfo.imageBindCount = image.infos.size(); + bindInfo.pImageBinds = image.infos.data(); + } + + if (!m_signalSemaphores.empty()) { + bindInfo.pNext = &timelineInfo; + bindInfo.signalSemaphoreCount = m_signalSemaphores.size(); + bindInfo.pSignalSemaphores = m_signalSemaphores.data(); + } + + VkResult vr = vk->vkQueueBindSparse(queue, 1, &bindInfo, VK_NULL_HANDLE); + + if (vr) { + Logger::err(str::format("Sparse binding failed: ", vr)); + this->logSparseBindingInfo(LogLevel::Error, &bindInfo); + } + + this->reset(); + return vr; + } + + + void DxvkSparseBindSubmission::reset() { + m_waitSemaphoreValues.clear(); + m_waitSemaphores.clear(); + m_signalSemaphoreValues.clear(); + m_signalSemaphores.clear(); + + m_bufferBinds.clear(); + m_imageBinds.clear(); + m_imageOpaqueBinds.clear(); + } + + + bool DxvkSparseBindSubmission::tryMergeMemoryBind( + VkSparseMemoryBind& oldBind, + const VkSparseMemoryBind& newBind) { + if (newBind.memory != oldBind.memory || newBind.flags != oldBind.flags) + return false; + + // The resource region must be consistent + if (newBind.resourceOffset != oldBind.resourceOffset + oldBind.size) + return false; + + // If memory is not null, the memory range must also be consistent + if (newBind.memory && newBind.memoryOffset != oldBind.memoryOffset + oldBind.size) + return false; + + oldBind.size += newBind.size; + return true; + } + + + void DxvkSparseBindSubmission::processBufferBinds( + DxvkSparseBufferBindArrays& buffer) { + std::vector> ranges; + ranges.reserve(m_bufferBinds.size()); + + for (const auto& e : m_bufferBinds) { + const auto& key = e.first; + const auto& handle = e.second; + + VkSparseMemoryBind bind = { }; + bind.resourceOffset = key.offset; + bind.size = key.size; + bind.memory = handle.memory; + bind.memoryOffset = handle.offset; + + bool merged = false; + + if (!ranges.empty() && ranges.back().first == key.buffer) + merged = tryMergeMemoryBind(ranges.back().second, bind); + + if (!merged) + ranges.push_back({ key.buffer, bind }); + } + + populateOutputArrays(buffer.binds, buffer.infos, ranges); + } + + + void DxvkSparseBindSubmission::processImageBinds( + DxvkSparseImageBindArrays& image) { + std::vector> ranges; + ranges.reserve(m_imageBinds.size()); + + for (const auto& e : m_imageBinds) { + const auto& key = e.first; + const auto& handle = e.second; + + VkSparseImageMemoryBind bind = { }; + bind.subresource = key.subresource; + bind.offset = key.offset; + bind.extent = key.extent; + bind.memory = handle.memory; + bind.memoryOffset = handle.offset; + + ranges.push_back({ key.image, bind }); + } + + populateOutputArrays(image.binds, image.infos, ranges); + } + + + void DxvkSparseBindSubmission::processOpaqueBinds( + DxvkSparseImageOpaqueBindArrays& opaque) { + std::vector> ranges; + ranges.reserve(m_imageOpaqueBinds.size()); + + for (const auto& e : m_imageOpaqueBinds) { + const auto& key = e.first; + const auto& handle = e.second; + + VkSparseMemoryBind bind = { }; + bind.resourceOffset = key.offset; + bind.size = key.size; + bind.memory = handle.memory; + bind.memoryOffset = handle.offset; + bind.flags = key.flags; + + bool merged = false; + + if (!ranges.empty() && ranges.back().first == key.image) + merged = tryMergeMemoryBind(ranges.back().second, bind); + + if (!merged) + ranges.push_back({ key.image, bind }); + } + + populateOutputArrays(opaque.binds, opaque.infos, ranges); + } + + + template + void DxvkSparseBindSubmission::populateOutputArrays( + std::vector& binds, + std::vector& infos, + const std::vector>& input) { + HandleType handle = VK_NULL_HANDLE; + + // Resize bind array so that pointers remain + // valid as we iterate over the input array + binds.resize(input.size()); + + for (size_t i = 0; i < input.size(); i++) { + binds[i] = input[i].second; + + if (handle != input[i].first) { + // Create new info entry if the handle + // differs from that of the previous entry + handle = input[i].first; + infos.push_back({ handle, 1u, &binds[i] }); + } else { + // Otherwise just increment the bind count + infos.back().bindCount += 1; + } + } + } + + + void DxvkSparseBindSubmission::logSparseBindingInfo( + LogLevel level, + const VkBindSparseInfo* info) { + std::stringstream str; + str << "VkBindSparseInfo:" << std::endl; + + auto timelineInfo = static_cast(info->pNext); + + if (info->waitSemaphoreCount) { + str << " Wait semaphores (" << std::dec << info->waitSemaphoreCount << "):" << std::endl; + + for (uint32_t i = 0; i < info->waitSemaphoreCount; i++) + str << " " << info->pWaitSemaphores[i] << " (" << timelineInfo->pWaitSemaphoreValues[i] << ")" << std::endl; + } + + if (info->bufferBindCount) { + str << " Buffer binds (" << std::dec << info->bufferBindCount << "):" << std::endl; + + for (uint32_t i = 0; i < info->bufferBindCount; i++) { + const auto* bindInfo = &info->pBufferBinds[i]; + str << " VkBuffer " << bindInfo->buffer << " (" << bindInfo->bindCount << "):" << std::endl; + + for (uint32_t j = 0; j < bindInfo->bindCount; j++) { + const auto* bind = &bindInfo->pBinds[j]; + str << " " << bind->resourceOffset << " -> " << bind->memory + << " (" << bind->memoryOffset << "," << bind->size << ")" << std::endl; + } + } + } + + if (info->imageOpaqueBindCount) { + str << " Opaque image binds (" << std::dec << info->imageOpaqueBindCount << "):" << std::endl; + + for (uint32_t i = 0; i < info->imageOpaqueBindCount; i++) { + const auto* bindInfo = &info->pImageOpaqueBinds[i]; + str << " VkImage " << bindInfo->image << " (" << bindInfo->bindCount << "):" << std::endl; + + for (uint32_t j = 0; j < bindInfo->bindCount; j++) { + const auto* bind = &bindInfo->pBinds[j]; + str << " " << bind->resourceOffset << " -> " << bind->memory + << " (" << bind->memoryOffset << "," << bind->size << ")" << std::endl; + } + } + } + + if (info->imageBindCount) { + str << " Opaque image binds (" << std::dec << info->imageOpaqueBindCount << "):" << std::endl; + + for (uint32_t i = 0; i < info->imageBindCount; i++) { + const auto* bindInfo = &info->pImageBinds[i]; + str << " VkImage " << bindInfo->image << " (" << bindInfo->bindCount << "):" << std::endl; + + for (uint32_t j = 0; j < bindInfo->bindCount; j++) { + const auto* bind = &bindInfo->pBinds[j]; + + str << " Aspect 0x" << std::hex << bind->subresource.aspectMask + << ", Mip " << std::dec << bind->subresource.mipLevel + << ", Layer " << bind->subresource.arrayLayer + << ":" << std::endl; + + str << " " << bind->offset.x << "," << bind->offset.y << "," << bind->offset.z << ":" + << bind->extent.width << "x" << bind->extent.height << "x" << bind->extent.depth + << " -> " << bind->memory << " (" << bind->memoryOffset << ")" << std::endl; + } + } + } + + if (info->signalSemaphoreCount) { + str << " Signal semaphores (" << std::dec << info->signalSemaphoreCount << "):" << std::endl; + + for (uint32_t i = 0; i < info->signalSemaphoreCount; i++) + str << " " << info->pSignalSemaphores[i] << " (" << timelineInfo->pSignalSemaphoreValues[i] << ")" << std::endl; + } + + Logger::log(level, str.str()); + } + } diff --git a/src/dxvk/dxvk_sparse.h b/src/dxvk/dxvk_sparse.h index f68aeb06a..158a2b5a7 100644 --- a/src/dxvk/dxvk_sparse.h +++ b/src/dxvk/dxvk_sparse.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "dxvk_memory.h" #include "dxvk_resource.h" @@ -492,4 +494,264 @@ namespace dxvk { }; -} \ No newline at end of file + + /** + * \brief Key for sparse buffer binding entry + * + * Provides a strong ordering by resource, resource offset, + * and finally, size. The ordering can be used to easily + * merge adjacent ranges. + */ + struct DxvkSparseBufferBindKey { + VkBuffer buffer; + VkDeviceSize offset; + VkDeviceSize size; + + bool operator < (const DxvkSparseBufferBindKey& other) const { + if (buffer < other.buffer) return true; + if (buffer > other.buffer) return false; + + if (offset < other.offset) return true; + if (offset > other.offset) return false; + + return size < other.size; + } + }; + + + /** + * \brief Key for sparse image binding entry + * + * Provides a strong ordering by resource, subresource, + * offset (z -> y -> x), and finally, extent (d -> h -> w). + * The ordering can be used to easily merge adjacent regions. + */ + struct DxvkSparseImageBindKey { + VkImage image; + VkImageSubresource subresource; + VkOffset3D offset; + VkExtent3D extent; + + bool operator < (const DxvkSparseImageBindKey& other) const { + if (image < other.image) return true; + if (image > other.image) return false; + + uint64_t aSubresource = this->encodeSubresource(); + uint64_t bSubresource = other.encodeSubresource(); + + if (aSubresource < bSubresource) return true; + if (aSubresource > bSubresource) return false; + + uint64_t aOffset = this->encodeOffset(); + uint64_t bOffset = other.encodeOffset(); + + if (aOffset < bOffset) return true; + if (aOffset > bOffset) return false; + + uint64_t aExtent = this->encodeExtent(); + uint64_t bExtent = other.encodeExtent(); + + return aExtent < bExtent; + } + + uint64_t encodeSubresource() const { + return uint64_t(subresource.aspectMask) << 48 + | uint64_t(subresource.arrayLayer) << 24 + | uint64_t(subresource.mipLevel); + } + + uint64_t encodeOffset() const { + return uint64_t(offset.z) << 48 + | uint64_t(offset.y) << 24 + | uint64_t(offset.x); + } + + uint64_t encodeExtent() const { + return uint64_t(extent.depth) << 48 + | uint64_t(extent.height) << 24 + | uint64_t(extent.width); + } + }; + + + /** + * \brief Key for sparse opaque image binding entry + * + * Provides a strong ordering by resource, resource offset, + * and finally, size. The ordering can be used to easily + * merge adjacent ranges. + */ + struct DxvkSparseImageOpaqueBindKey { + VkImage image; + VkDeviceSize offset; + VkDeviceSize size; + VkSparseMemoryBindFlags flags; + + bool operator < (const DxvkSparseImageOpaqueBindKey& other) const { + if (image < other.image) return true; + if (image > other.image) return false; + + if (offset < other.offset) return true; + if (offset > other.offset) return false; + + return size < other.size; + } + }; + + + /** + * \brief Arrays required for buffer binds + */ + struct DxvkSparseBufferBindArrays { + std::vector binds; + std::vector infos; + }; + + + /** + * \brief Arrays required for image binds + */ + struct DxvkSparseImageBindArrays { + std::vector binds; + std::vector infos; + }; + + + /** + * \brief Arrays required for opaque image binds + */ + struct DxvkSparseImageOpaqueBindArrays { + std::vector binds; + std::vector infos; + }; + + + /** + * \brief Sparse bind submission + * + * Stores information for a single sparse binding operation, + * and supports submitting that operation to a device queue. + * + * All methods to add bindings assume that the binding range is + * either identical to an existing range, in which case the old + * binding will be overwritten, or otherwise, that the range is + * disjoint from all existing ranges. Overlapping ranges are not + * supported. This condition is trivial to maintain when binding + * only one sparse page at a time. + */ + class DxvkSparseBindSubmission { + + public: + + DxvkSparseBindSubmission(); + + ~DxvkSparseBindSubmission(); + + /** + * \brief Waits for a semaphore + * + * \param [in] semaphore Semaphore to wait for + * \param [in] value Semaphore value to wait on + */ + void waitSemaphore( + VkSemaphore semaphore, + uint64_t value); + + /** + * \brief Signals a semaphore + * + * \param [in] semaphore Semaphore to signal + * \param [in] value Calue to signal semaphore to + */ + void signalSemaphore( + VkSemaphore semaphore, + uint64_t value); + + /** + * \brief Adds a buffer memory bind + * + * \param [in] key Buffer range key + * \param [in] memory Page handle + */ + void bindBufferMemory( + const DxvkSparseBufferBindKey& key, + const DxvkSparsePageHandle& memory); + + /** + * \brief Adds an image memory bind + * + * \param [in] key Image region key + * \param [in] memory Page handle + */ + void bindImageMemory( + const DxvkSparseImageBindKey& key, + const DxvkSparsePageHandle& memory); + + /** + * \brief Adds an opaque image memory bind + * + * \param [in] key Opaque region key + * \param [in] memory Page handle + */ + void bindImageOpaqueMemory( + const DxvkSparseImageOpaqueBindKey& key, + const DxvkSparsePageHandle& memory); + + /** + * \brief Submits sparse binding operation + * + * Generates structures required for the sparse bind, resolving + * any conflicts in the process and merging ranges where possible. + * Note that this operation is slow. Resets object after the call. + * \param [in] device DXVK device + * \param [in] queue Queue to perform the operation on + * \returns Return value of the sparse bind operation + */ + VkResult submit( + DxvkDevice* device, + VkQueue queue); + + /** + * \brief Resets object + * + * Clears all internal structures. + */ + void reset(); + + private: + + std::vector m_waitSemaphoreValues; + std::vector m_waitSemaphores; + std::vector m_signalSemaphoreValues; + std::vector m_signalSemaphores; + + std::map m_bufferBinds; + std::map m_imageBinds; + std::map m_imageOpaqueBinds; + + static bool tryMergeMemoryBind( + VkSparseMemoryBind& oldBind, + const VkSparseMemoryBind& newBind); + + void processBufferBinds( + DxvkSparseBufferBindArrays& buffer); + + void processImageBinds( + DxvkSparseImageBindArrays& image); + + void processOpaqueBinds( + DxvkSparseImageOpaqueBindArrays& opaque); + + template + void populateOutputArrays( + std::vector& binds, + std::vector& infos, + const std::vector>& input); + + void logSparseBindingInfo( + LogLevel level, + const VkBindSparseInfo* info); + + }; + +}