[dxvk] Introduce DxvkSparseBindSubmission

src/dxvk/dxvk_sparse.cpp

@@ -1,3 +1,5 @@
+#include <sstream>
+
 #include "dxvk_buffer.h"
 #include "dxvk_device.h"
 #include "dxvk_image.h"
@@ -396,4 +398,330 @@ namespace dxvk {
     }
   }
 
+
+  DxvkSparseBindSubmission::DxvkSparseBindSubmission() {
+
+  }
+
+
+  DxvkSparseBindSubmission::~DxvkSparseBindSubmission() {
+
+  }
+
+
+  void DxvkSparseBindSubmission::waitSemaphore(
+          VkSemaphore             semaphore,
+          uint64_t                value) {
+    m_waitSemaphores.push_back(semaphore);
+    m_waitSemaphoreValues.push_back(value);
+  }
+
+
+  void DxvkSparseBindSubmission::signalSemaphore(
+          VkSemaphore             semaphore,
+          uint64_t                value) {
+    m_signalSemaphores.push_back(semaphore);
+    m_signalSemaphoreValues.push_back(value);
+  }
+
+
+  void DxvkSparseBindSubmission::bindBufferMemory(
+    const DxvkSparseBufferBindKey& key,
+    const DxvkSparsePageHandle&   memory) {
+    m_bufferBinds.insert_or_assign(key, memory);
+  }
+
+
+  void DxvkSparseBindSubmission::bindImageMemory(
+    const DxvkSparseImageBindKey& key,
+    const DxvkSparsePageHandle&   memory) {
+    m_imageBinds.insert_or_assign(key, memory);
+  }
+
+
+  void DxvkSparseBindSubmission::bindImageOpaqueMemory(
+    const DxvkSparseImageOpaqueBindKey& key,
+    const DxvkSparsePageHandle&   memory) {
+    m_imageOpaqueBinds.insert_or_assign(key, memory);
+  }
+
+
+  VkResult DxvkSparseBindSubmission::submit(
+          DxvkDevice*             device,
+          VkQueue                 queue) {
+    auto vk = device->vkd();
+
+    DxvkSparseBufferBindArrays buffer;
+    this->processBufferBinds(buffer);
+
+    DxvkSparseImageBindArrays image;
+    this->processImageBinds(image);
+
+    DxvkSparseImageOpaqueBindArrays opaque;
+    this->processOpaqueBinds(opaque);
+
+    // The sparse binding API has never been updated to take the new
+    // semaphore submission info structs, so we have to do this instead
+    VkTimelineSemaphoreSubmitInfo timelineInfo = { VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO };
+    timelineInfo.waitSemaphoreValueCount = m_waitSemaphoreValues.size();
+    timelineInfo.pWaitSemaphoreValues = m_waitSemaphoreValues.data();
+    timelineInfo.signalSemaphoreValueCount = m_signalSemaphoreValues.size();
+    timelineInfo.pSignalSemaphoreValues = m_signalSemaphoreValues.data();
+
+    VkBindSparseInfo bindInfo = { VK_STRUCTURE_TYPE_BIND_SPARSE_INFO };
+
+    if (!m_waitSemaphores.empty()) {
+      bindInfo.pNext = &timelineInfo;
+      bindInfo.waitSemaphoreCount = m_waitSemaphores.size();
+      bindInfo.pWaitSemaphores = m_waitSemaphores.data();
+    }
+
+    if (!buffer.infos.empty()) {
+      bindInfo.bufferBindCount = buffer.infos.size();
+      bindInfo.pBufferBinds = buffer.infos.data();
+    }
+
+    if (!opaque.infos.empty()) {
+      bindInfo.imageOpaqueBindCount = opaque.infos.size();
+      bindInfo.pImageOpaqueBinds = opaque.infos.data();
+    }
+
+    if (!image.infos.empty()) {
+      bindInfo.imageBindCount = image.infos.size();
+      bindInfo.pImageBinds = image.infos.data();
+    }
+
+    if (!m_signalSemaphores.empty()) {
+      bindInfo.pNext = &timelineInfo;
+      bindInfo.signalSemaphoreCount = m_signalSemaphores.size();
+      bindInfo.pSignalSemaphores = m_signalSemaphores.data();
+    }
+
+    VkResult vr = vk->vkQueueBindSparse(queue, 1, &bindInfo, VK_NULL_HANDLE);
+
+    if (vr) {
+      Logger::err(str::format("Sparse binding failed: ", vr));
+      this->logSparseBindingInfo(LogLevel::Error, &bindInfo);
+    }
+
+    this->reset();
+    return vr;
+  }
+
+
+  void DxvkSparseBindSubmission::reset() {
+    m_waitSemaphoreValues.clear();
+    m_waitSemaphores.clear();
+    m_signalSemaphoreValues.clear();
+    m_signalSemaphores.clear();
+
+    m_bufferBinds.clear();
+    m_imageBinds.clear();
+    m_imageOpaqueBinds.clear();
+  }
+
+
+  bool DxvkSparseBindSubmission::tryMergeMemoryBind(
+          VkSparseMemoryBind&               oldBind,
+    const VkSparseMemoryBind&               newBind) {
+    if (newBind.memory != oldBind.memory || newBind.flags != oldBind.flags)
+      return false;
+
+    // The resource region must be consistent
+    if (newBind.resourceOffset != oldBind.resourceOffset + oldBind.size)
+      return false;
+
+    // If memory is not null, the memory range must also be consistent
+    if (newBind.memory && newBind.memoryOffset != oldBind.memoryOffset + oldBind.size)
+      return false;
+
+    oldBind.size += newBind.size;
+    return true;
+  }
+
+
+  void DxvkSparseBindSubmission::processBufferBinds(
+          DxvkSparseBufferBindArrays&       buffer) {
+    std::vector<std::pair<VkBuffer, VkSparseMemoryBind>> ranges;
+    ranges.reserve(m_bufferBinds.size());
+
+    for (const auto& e : m_bufferBinds) {
+      const auto& key = e.first;
+      const auto& handle = e.second;
+
+      VkSparseMemoryBind bind = { };
+      bind.resourceOffset = key.offset;
+      bind.size           = key.size;
+      bind.memory         = handle.memory;
+      bind.memoryOffset   = handle.offset;
+
+      bool merged = false;
+
+      if (!ranges.empty() && ranges.back().first == key.buffer)
+        merged = tryMergeMemoryBind(ranges.back().second, bind);
+
+      if (!merged)
+        ranges.push_back({ key.buffer, bind });
+    }
+
+    populateOutputArrays(buffer.binds, buffer.infos, ranges);
+  }
+
+
+  void DxvkSparseBindSubmission::processImageBinds(
+          DxvkSparseImageBindArrays&        image) {
+    std::vector<std::pair<VkImage, VkSparseImageMemoryBind>> ranges;
+    ranges.reserve(m_imageBinds.size());
+
+    for (const auto& e : m_imageBinds) {
+      const auto& key = e.first;
+      const auto& handle = e.second;
+
+      VkSparseImageMemoryBind bind = { };
+      bind.subresource    = key.subresource;
+      bind.offset         = key.offset;
+      bind.extent         = key.extent;
+      bind.memory         = handle.memory;
+      bind.memoryOffset   = handle.offset;
+
+      ranges.push_back({ key.image, bind });
+    }
+
+    populateOutputArrays(image.binds, image.infos, ranges);
+  }
+
+
+  void DxvkSparseBindSubmission::processOpaqueBinds(
+          DxvkSparseImageOpaqueBindArrays&  opaque) {
+    std::vector<std::pair<VkImage, VkSparseMemoryBind>> ranges;
+    ranges.reserve(m_imageOpaqueBinds.size());
+
+    for (const auto& e : m_imageOpaqueBinds) {
+      const auto& key = e.first;
+      const auto& handle = e.second;
+
+      VkSparseMemoryBind bind = { };
+      bind.resourceOffset = key.offset;
+      bind.size           = key.size;
+      bind.memory         = handle.memory;
+      bind.memoryOffset   = handle.offset;
+      bind.flags          = key.flags;
+
+      bool merged = false;
+
+      if (!ranges.empty() && ranges.back().first == key.image)
+        merged = tryMergeMemoryBind(ranges.back().second, bind);
+
+      if (!merged)
+        ranges.push_back({ key.image, bind });
+    }
+
+    populateOutputArrays(opaque.binds, opaque.infos, ranges);
+  }
+
+
+  template<typename HandleType, typename BindType, typename InfoType>
+  void DxvkSparseBindSubmission::populateOutputArrays(
+          std::vector<BindType>&            binds,
+          std::vector<InfoType>&            infos,
+    const std::vector<std::pair<HandleType, BindType>>& input) {
+    HandleType handle = VK_NULL_HANDLE;
+
+    // Resize bind array so that pointers remain
+    // valid as we iterate over the input array
+    binds.resize(input.size());
+
+    for (size_t i = 0; i < input.size(); i++) {
+      binds[i] = input[i].second;
+
+      if (handle != input[i].first) {
+        // Create new info entry if the handle
+        // differs from that of the previous entry
+        handle = input[i].first;
+        infos.push_back({ handle, 1u, &binds[i] });
+      } else {
+        // Otherwise just increment the bind count
+        infos.back().bindCount += 1;
+      }
+    }
+  }
+
+
+  void DxvkSparseBindSubmission::logSparseBindingInfo(
+          LogLevel                          level,
+    const VkBindSparseInfo*                 info) {
+    std::stringstream str;
+    str << "VkBindSparseInfo:" << std::endl;
+
+    auto timelineInfo = static_cast<const VkTimelineSemaphoreSubmitInfo*>(info->pNext);
+
+    if (info->waitSemaphoreCount) {
+      str << "  Wait semaphores (" << std::dec << info->waitSemaphoreCount << "):" << std::endl;
+
+      for (uint32_t i = 0; i < info->waitSemaphoreCount; i++)
+        str << "    " << info->pWaitSemaphores[i] << " (" << timelineInfo->pWaitSemaphoreValues[i] << ")" << std::endl;
+    }
+
+    if (info->bufferBindCount) {
+      str << "  Buffer binds (" << std::dec << info->bufferBindCount << "):" << std::endl;
+
+      for (uint32_t i = 0; i < info->bufferBindCount; i++) {
+        const auto* bindInfo = &info->pBufferBinds[i];
+        str << "    VkBuffer " << bindInfo->buffer << " (" << bindInfo->bindCount << "):" << std::endl;
+
+        for (uint32_t j = 0; j < bindInfo->bindCount; j++) {
+          const auto* bind = &bindInfo->pBinds[j];
+          str << "        " << bind->resourceOffset << " -> " << bind->memory
+              << " (" << bind->memoryOffset << "," << bind->size << ")" << std::endl;
+        }
+      }
+    }
+
+    if (info->imageOpaqueBindCount) {
+      str << "  Opaque image binds (" << std::dec << info->imageOpaqueBindCount << "):" << std::endl;
+
+      for (uint32_t i = 0; i < info->imageOpaqueBindCount; i++) {
+        const auto* bindInfo = &info->pImageOpaqueBinds[i];
+        str << "    VkImage " << bindInfo->image << " (" << bindInfo->bindCount << "):" << std::endl;
+
+        for (uint32_t j = 0; j < bindInfo->bindCount; j++) {
+          const auto* bind = &bindInfo->pBinds[j];
+          str << "        " << bind->resourceOffset << " -> " << bind->memory
+              << " (" << bind->memoryOffset << "," << bind->size << ")" << std::endl;
+        }
+      }
+    }
+
+    if (info->imageBindCount) {
+      str << "  Opaque image binds (" << std::dec << info->imageOpaqueBindCount << "):" << std::endl;
+
+      for (uint32_t i = 0; i < info->imageBindCount; i++) {
+        const auto* bindInfo = &info->pImageBinds[i];
+        str << "    VkImage " << bindInfo->image << " (" << bindInfo->bindCount << "):" << std::endl;
+
+        for (uint32_t j = 0; j < bindInfo->bindCount; j++) {
+          const auto* bind = &bindInfo->pBinds[j];
+
+          str << "        Aspect 0x" << std::hex << bind->subresource.aspectMask
+              << ", Mip " << std::dec << bind->subresource.mipLevel
+              << ", Layer " << bind->subresource.arrayLayer
+              << ":" << std::endl;
+
+          str << "        " << bind->offset.x << "," << bind->offset.y << "," << bind->offset.z << ":"
+              << bind->extent.width << "x" << bind->extent.height << "x" << bind->extent.depth
+              << " -> " << bind->memory << " (" << bind->memoryOffset << ")" << std::endl;
+        }
+      }
+    }
+
+    if (info->signalSemaphoreCount) {
+      str << "  Signal semaphores (" << std::dec << info->signalSemaphoreCount << "):" << std::endl;
+
+      for (uint32_t i = 0; i < info->signalSemaphoreCount; i++)
+        str << "    " << info->pSignalSemaphores[i] << " (" << timelineInfo->pSignalSemaphoreValues[i] << ")" << std::endl;
+    }
+
+    Logger::log(level, str.str());
+  }
+
 }

src/dxvk/dxvk_sparse.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <map>
+
 #include "dxvk_memory.h"
 #include "dxvk_resource.h"
 
@@ -492,4 +494,264 @@ namespace dxvk {
 
   };
 
-}
+
+  /**
+   * \brief Key for sparse buffer binding entry
+   *
+   * Provides a strong ordering by resource, resource offset,
+   * and finally, size. The ordering can be used to easily
+   * merge adjacent ranges.
+   */
+  struct DxvkSparseBufferBindKey {
+    VkBuffer                    buffer;
+    VkDeviceSize                offset;
+    VkDeviceSize                size;
+
+    bool operator < (const DxvkSparseBufferBindKey& other) const {
+      if (buffer < other.buffer) return true;
+      if (buffer > other.buffer) return false;
+
+      if (offset < other.offset) return true;
+      if (offset > other.offset) return false;
+
+      return size < other.size;
+    }
+  };
+
+
+  /**
+   * \brief Key for sparse image binding entry
+   *
+   * Provides a strong ordering by resource, subresource,
+   * offset (z -> y -> x), and finally, extent (d -> h -> w).
+   * The ordering can be used to easily merge adjacent regions.
+   */
+  struct DxvkSparseImageBindKey {
+    VkImage                     image;
+    VkImageSubresource          subresource;
+    VkOffset3D                  offset;
+    VkExtent3D                  extent;
+
+    bool operator < (const DxvkSparseImageBindKey& other) const {
+      if (image < other.image) return true;
+      if (image > other.image) return false;
+
+      uint64_t aSubresource = this->encodeSubresource();
+      uint64_t bSubresource = other.encodeSubresource();
+
+      if (aSubresource < bSubresource) return true;
+      if (aSubresource > bSubresource) return false;
+
+      uint64_t aOffset = this->encodeOffset();
+      uint64_t bOffset = other.encodeOffset();
+
+      if (aOffset < bOffset) return true;
+      if (aOffset > bOffset) return false;
+
+      uint64_t aExtent = this->encodeExtent();
+      uint64_t bExtent = other.encodeExtent();
+
+      return aExtent < bExtent;
+    }
+
+    uint64_t encodeSubresource() const {
+      return uint64_t(subresource.aspectMask) << 48
+           | uint64_t(subresource.arrayLayer) << 24
+           | uint64_t(subresource.mipLevel);
+    }
+
+    uint64_t encodeOffset() const {
+      return uint64_t(offset.z) << 48
+           | uint64_t(offset.y) << 24
+           | uint64_t(offset.x);
+    }
+
+    uint64_t encodeExtent() const {
+      return uint64_t(extent.depth) << 48
+           | uint64_t(extent.height) << 24
+           | uint64_t(extent.width);
+    }
+  };
+
+
+  /**
+   * \brief Key for sparse opaque image binding entry
+   *
+   * Provides a strong ordering by resource, resource offset,
+   * and finally, size. The ordering can be used to easily
+   * merge adjacent ranges.
+   */
+  struct DxvkSparseImageOpaqueBindKey {
+    VkImage                     image;
+    VkDeviceSize                offset;
+    VkDeviceSize                size;
+    VkSparseMemoryBindFlags     flags;
+
+    bool operator < (const DxvkSparseImageOpaqueBindKey& other) const {
+      if (image < other.image) return true;
+      if (image > other.image) return false;
+
+      if (offset < other.offset) return true;
+      if (offset > other.offset) return false;
+
+      return size < other.size;
+    }
+  };
+
+
+  /**
+   * \brief Arrays required for buffer binds
+   */
+  struct DxvkSparseBufferBindArrays {
+    std::vector<VkSparseMemoryBind> binds;
+    std::vector<VkSparseBufferMemoryBindInfo> infos;
+  };
+
+
+  /**
+   * \brief Arrays required for image binds
+   */
+  struct DxvkSparseImageBindArrays {
+    std::vector<VkSparseImageMemoryBind> binds;
+    std::vector<VkSparseImageMemoryBindInfo> infos;
+  };
+
+
+  /**
+   * \brief Arrays required for opaque image binds
+   */
+  struct DxvkSparseImageOpaqueBindArrays {
+    std::vector<VkSparseMemoryBind> binds;
+    std::vector<VkSparseImageOpaqueMemoryBindInfo> infos;
+  };
+
+
+  /**
+   * \brief Sparse bind submission
+   *
+   * Stores information for a single sparse binding operation,
+   * and supports submitting that operation to a device queue.
+   *
+   * All methods to add bindings assume that the binding range is
+   * either identical to an existing range, in which case the old
+   * binding will be overwritten, or otherwise, that the range is
+   * disjoint from all existing ranges. Overlapping ranges are not
+   * supported. This condition is trivial to maintain when binding
+   * only one sparse page at a time.
+   */
+  class DxvkSparseBindSubmission {
+
+  public:
+
+    DxvkSparseBindSubmission();
+
+    ~DxvkSparseBindSubmission();
+
+    /**
+     * \brief Waits for a semaphore
+     *
+     * \param [in] semaphore Semaphore to wait for
+     * \param [in] value Semaphore value to wait on
+     */
+    void waitSemaphore(
+            VkSemaphore             semaphore,
+            uint64_t                value);
+
+    /**
+     * \brief Signals a semaphore
+     *
+     * \param [in] semaphore Semaphore to signal
+     * \param [in] value Calue to signal semaphore to
+     */
+    void signalSemaphore(
+            VkSemaphore             semaphore,
+            uint64_t                value);
+
+    /**
+     * \brief Adds a buffer memory bind
+     *
+     * \param [in] key Buffer range key
+     * \param [in] memory Page handle
+     */
+    void bindBufferMemory(
+      const DxvkSparseBufferBindKey& key,
+      const DxvkSparsePageHandle&   memory);
+
+    /**
+     * \brief Adds an image memory bind
+     *
+     * \param [in] key Image region key
+     * \param [in] memory Page handle
+     */
+    void bindImageMemory(
+      const DxvkSparseImageBindKey& key,
+      const DxvkSparsePageHandle&   memory);
+
+    /**
+     * \brief Adds an opaque image memory bind
+     *
+     * \param [in] key Opaque region key
+     * \param [in] memory Page handle
+     */
+    void bindImageOpaqueMemory(
+      const DxvkSparseImageOpaqueBindKey& key,
+      const DxvkSparsePageHandle&   memory);
+
+    /**
+     * \brief Submits sparse binding operation
+     *
+     * Generates structures required for the sparse bind, resolving
+     * any conflicts in the process and merging ranges where possible.
+     * Note that this operation is slow. Resets object after the call.
+     * \param [in] device DXVK device
+     * \param [in] queue Queue to perform the operation on
+     * \returns Return value of the sparse bind operation
+     */
+    VkResult submit(
+            DxvkDevice*             device,
+            VkQueue                 queue);
+
+    /**
+     * \brief Resets object
+     *
+     * Clears all internal structures.
+     */
+    void reset();
+
+  private:
+
+    std::vector<uint64_t>     m_waitSemaphoreValues;
+    std::vector<VkSemaphore>  m_waitSemaphores;
+    std::vector<uint64_t>     m_signalSemaphoreValues;
+    std::vector<VkSemaphore>  m_signalSemaphores;
+
+    std::map<DxvkSparseBufferBindKey,      DxvkSparsePageHandle> m_bufferBinds;
+    std::map<DxvkSparseImageBindKey,       DxvkSparsePageHandle> m_imageBinds;
+    std::map<DxvkSparseImageOpaqueBindKey, DxvkSparsePageHandle> m_imageOpaqueBinds;
+
+    static bool tryMergeMemoryBind(
+            VkSparseMemoryBind&               oldBind,
+      const VkSparseMemoryBind&               newBind);
+
+    void processBufferBinds(
+            DxvkSparseBufferBindArrays&       buffer);
+
+    void processImageBinds(
+            DxvkSparseImageBindArrays&        image);
+
+    void processOpaqueBinds(
+            DxvkSparseImageOpaqueBindArrays&  opaque);
+
+    template<typename HandleType, typename BindType, typename InfoType>
+    void populateOutputArrays(
+            std::vector<BindType>&            binds,
+            std::vector<InfoType>&            infos,
+      const std::vector<std::pair<HandleType, BindType>>& input);
+
+    void logSparseBindingInfo(
+            LogLevel                          level,
+      const VkBindSparseInfo*                 info);
+
+  };
+
+}