[dxvk] Track order-invariant access ops in barrier tracker

This elides barriers between draws or dispatches if we can prove
order-invariance through atomic operations.

src/dxvk/dxvk_barrier.cpp

@@ -20,20 +20,37 @@ namespace dxvk {
 
   bool DxvkBarrierTracker::findRange(
     const DxvkAddressRange&           range,
-          DxvkAccess                  accessType) const {
+          DxvkAccess                  accessType,
+          DxvkAccessOp                accessOp) const {
     uint32_t rootIndex = computeRootIndex(range, accessType);
-    return findNode(range, rootIndex);
+    uint32_t nodeIndex = findNode(range, rootIndex);
+
+    if (likely(!nodeIndex || accessOp == DxvkAccessOp::None))
+      return nodeIndex;
+
+    // If we are checking for a specific order-invariant store
+    // op, the op must have been the only op used to access the
+    // resource, and the tracked range must cover the requested
+    // range in its entirety so we can rule out that other parts
+    // of the resource have been accessed in a different way.
+    auto& node = m_nodes[nodeIndex];
+
+    return node.payload.accessOps != DxvkAccessOps(accessOp)
+        || !node.addressRange.contains(range);
   }
 
 
   void DxvkBarrierTracker::insertRange(
     const DxvkAddressRange&           range,
-          DxvkAccess                  accessType) {
+          DxvkAccess                  accessType,
-    uint32_t rootIndex = computeRootIndex(range, accessType);
+          DxvkAccessOp                accessOp) {
+    DxvkBarrierPayload payload = { };
+    payload.accessOps.set(accessOp);
 
     // If we can just insert the node with no conflicts,
     // we don't have to do anything.
-    uint32_t nodeIndex = insertNode(range, rootIndex);
+    uint32_t rootIndex = computeRootIndex(range, accessType);
+    uint32_t nodeIndex = insertNode(range, rootIndex, payload);
 
     if (likely(!nodeIndex))
       return;
@@ -41,6 +58,7 @@ namespace dxvk {
     // If there's an existing node and it contains the entire
     // range we want to add already, also don't do anything.
     auto& node = m_nodes[nodeIndex];
+    node.payload.accessOps.set(payload.accessOps);
 
     if (node.addressRange.contains(range))
       return;
@@ -82,12 +100,14 @@ namespace dxvk {
       mergedRange.rangeStart = std::min(mergedRange.rangeStart, node.addressRange.rangeStart);
       mergedRange.rangeEnd = std::max(mergedRange.rangeEnd, node.addressRange.rangeEnd);
 
+      payload.accessOps.set(node.payload.accessOps);
+
       removeNode(nodeIndex, rootIndex);
 
       nodeIndex = findNode(range, rootIndex);
     }
 
-    insertNode(mergedRange, rootIndex);
+    insertNode(mergedRange, rootIndex, payload);
   }
 
 
@@ -166,7 +186,8 @@ namespace dxvk {
 
   uint32_t DxvkBarrierTracker::insertNode(
     const DxvkAddressRange&           range,
-          uint32_t                    rootIndex) {
+          uint32_t                    rootIndex,
+          DxvkBarrierPayload          payload) {
     // Check if the given root is valid at all
     uint64_t rootBit = uint64_t(1u) << (rootIndex - 1u);
 
@@ -178,6 +199,7 @@ namespace dxvk {
       auto& node = m_nodes[rootIndex];
       node.header = 0;
       node.addressRange = range;
+      node.payload = payload;
       return 0;
     } else {
       // Traverse tree and abort if we find any range
@@ -209,6 +231,7 @@ namespace dxvk {
       node.setRed(true);
       node.setParent(parentIndex);
       node.addressRange = range;
+      node.payload = payload;
 
       // Only do the fixup to maintain red-black properties if
       // we haven't marked the root node as red in a deletion.
@@ -238,6 +261,7 @@ namespace dxvk {
         childIndex = m_nodes[childIndex].child(0);
 
       node.addressRange = m_nodes[childIndex].addressRange;
+      node.payload = m_nodes[childIndex].payload;
       removeNode(childIndex, rootIndex);
     } else {
       // Deletion is expected to be exceptionally rare, to the point of
@@ -268,6 +292,7 @@ namespace dxvk {
           node.setRed(child.isRed());
 
         node.addressRange = child.addressRange;
+        node.payload = child.payload;
 
         if (cl) m_nodes[cl].setParent(nodeIndex);
         if (cr) m_nodes[cr].setParent(nodeIndex);
@@ -378,6 +403,7 @@ namespace dxvk {
     node.setChild(1, rr);
 
     std::swap(node.addressRange, m_nodes[r].addressRange);
+    std::swap(node.payload, m_nodes[r].payload);
   }
 
 
@@ -406,6 +432,7 @@ namespace dxvk {
     node.setChild(1, l);
 
     std::swap(node.addressRange, m_nodes[l].addressRange);
+    std::swap(node.payload, m_nodes[l].payload);
   }
 
 
@@ -498,4 +525,4 @@ namespace dxvk {
     flush(list);
   }
 
-}
+}

src/dxvk/dxvk_barrier.h

@@ -42,6 +42,14 @@ namespace dxvk {
   };
 
 
+  /**
+   * \brief Barrier node payload
+   */
+  struct DxvkBarrierPayload {
+    DxvkAccessOps accessOps = 0u;
+  };
+
+
   /**
    * \brief Barrier tree node
    *
@@ -62,6 +70,9 @@ namespace dxvk {
     // Address range of the node
     DxvkAddressRange addressRange = { };
 
+    // Node payload
+    DxvkBarrierPayload payload = { };
+
     void setRed(bool red) {
       header &= ~uint64_t(1u);
       header |= uint64_t(red);
@@ -117,21 +128,25 @@ namespace dxvk {
      *
      * \param [in] range Resource range
      * \param [in] accessType Access type
+     * \param [in] accessOp Access operation
      * \returns \c true if the range has a pending access
      */
     bool findRange(
       const DxvkAddressRange&           range,
-            DxvkAccess                  accessType) const;
+            DxvkAccess                  accessType,
+            DxvkAccessOp                accessOp) const;
 
     /**
      * \brief Inserts address range for a given access type
      *
      * \param [in] range Resource range
      * \param [in] accessType Access type
+     * \param [in] accessOp Access operation
      */
     void insertRange(
       const DxvkAddressRange&           range,
-            DxvkAccess                  accessType);
+            DxvkAccess                  accessType,
+            DxvkAccessOp                accessOp);
 
     /**
      * \brief Clears the entire structure
@@ -166,7 +181,8 @@ namespace dxvk {
 
     uint32_t insertNode(
       const DxvkAddressRange&           range,
-            uint32_t                    rootIndex);
+            uint32_t                    rootIndex,
+            DxvkBarrierPayload          payload);
 
     void removeNode(
             uint32_t                    nodeIndex,
@@ -285,4 +301,4 @@ namespace dxvk {
 
   };
 
-}
+}

src/dxvk/dxvk_context.cpp

@@ -7796,9 +7796,9 @@ namespace dxvk {
                        + (subresources.baseArrayLayer + subresources.layerCount - 1u);
 
         if (hasWrite)
-          m_barrierTracker.insertRange(range, DxvkAccess::Write);
+          m_barrierTracker.insertRange(range, DxvkAccess::Write, accessOp);
         if (hasRead)
-          m_barrierTracker.insertRange(range, DxvkAccess::Read);
+          m_barrierTracker.insertRange(range, DxvkAccess::Read, accessOp);
       } else {
         DxvkAddressRange range;
         range.resource = image.getResourceId();
@@ -7808,9 +7808,9 @@ namespace dxvk {
           range.rangeEnd = range.rangeStart + subresources.layerCount - 1u;
 
           if (hasWrite)
-            m_barrierTracker.insertRange(range, DxvkAccess::Write);
+            m_barrierTracker.insertRange(range, DxvkAccess::Write, accessOp);
           if (hasRead)
-            m_barrierTracker.insertRange(range, DxvkAccess::Read);
+            m_barrierTracker.insertRange(range, DxvkAccess::Read, accessOp);
         }
       }
     }
@@ -7863,9 +7863,9 @@ namespace dxvk {
       range.rangeEnd = offset + size - 1;
 
       if (srcAccess & vk::AccessWriteMask)
-        m_barrierTracker.insertRange(range, DxvkAccess::Write);
+        m_barrierTracker.insertRange(range, DxvkAccess::Write, accessOp);
       if (srcAccess & vk::AccessReadMask)
-        m_barrierTracker.insertRange(range, DxvkAccess::Read);
+        m_barrierTracker.insertRange(range, DxvkAccess::Read, accessOp);
     }
   }
 
@@ -8037,7 +8037,7 @@ namespace dxvk {
     range.rangeStart = offset;
     range.rangeEnd = offset + size - 1;
 
-    return m_barrierTracker.findRange(range, access);
+    return m_barrierTracker.findRange(range, access, accessOp);
   }
 
 
@@ -8071,7 +8071,7 @@ namespace dxvk {
     // Probe all subresources first, only check individual mip levels
     // if there are overlaps and if we are checking a subset of array
     // layers of multiple mips.
-    bool dirty = m_barrierTracker.findRange(range, access);
+    bool dirty = m_barrierTracker.findRange(range, access, accessOp);
 
     if (!dirty || subresources.levelCount == 1u || subresources.layerCount == layerCount)
       return dirty;
@@ -8080,7 +8080,7 @@ namespace dxvk {
       range.rangeStart = i * layerCount + subresources.baseArrayLayer;
       range.rangeEnd = range.rangeStart + subresources.layerCount - 1u;
 
-      dirty = m_barrierTracker.findRange(range, access);
+      dirty = m_barrierTracker.findRange(range, access, accessOp);
     }
 
     return dirty;

src/dxvk/dxvk_pipelayout.h

@@ -29,6 +29,9 @@ namespace dxvk {
     UMax  = 8,
   };
 
+  using DxvkAccessOps = Flags<DxvkAccessOp>;
+
+
   /**
    * \brief Descriptor set indices
    */