[dxvk,d3d9,d3d11] Normalize depth-stencil state in front-end

Reduces some unnecessary pipeline look-ups and also helps with
recognizing read-only depth-stencil attachments in more scenarios.

src/d3d11/d3d11_depth_stencil.cpp

@@ -14,6 +14,8 @@ namespace dxvk {
     m_state.setDepthCompareOp(DecodeCompareOp(desc.DepthFunc));
     m_state.setStencilOpFront(DecodeStencilOpState(desc.FrontFace, desc));
     m_state.setStencilOpBack(DecodeStencilOpState(desc.BackFace, desc));
+
+    m_state.normalize();
   }
   
   

src/d3d9/d3d9_device.cpp

@@ -6950,7 +6950,9 @@ namespace dxvk {
 
     EmitCs([
       cState = state
-    ](DxvkContext* ctx) {
+    ] (DxvkContext* ctx) mutable {
+      cState.normalize();
+
       ctx->setDepthStencilState(cState);
     });
   }

src/dxvk/dxvk_constant_state.cpp

@@ -0,0 +1,68 @@
+#include "dxvk_constant_state.h"
+
+namespace dxvk {
+
+  bool DxvkStencilOp::normalize(VkCompareOp depthOp) {
+    if (writeMask()) {
+      // If the depth test always passes, this is irrelevant
+      if (depthOp == VK_COMPARE_OP_ALWAYS)
+        setDepthFailOp(VK_STENCIL_OP_KEEP);
+
+      // Also mask out unused ops if the stencil test
+      // always pases or always fails
+      if (compareOp() == VK_COMPARE_OP_ALWAYS)
+        setFailOp(VK_STENCIL_OP_KEEP);
+      else if (compareOp() == VK_COMPARE_OP_NEVER)
+        setPassOp(VK_STENCIL_OP_KEEP);
+
+      // If all stencil ops are no-ops, clear write mask
+      if (passOp() == VK_STENCIL_OP_KEEP
+       && failOp() == VK_STENCIL_OP_KEEP
+       && depthFailOp() == VK_STENCIL_OP_KEEP)
+        setWriteMask(0u);
+    } else {
+      // Normalize stencil ops if write mask is 0
+      setPassOp(VK_STENCIL_OP_KEEP);
+      setFailOp(VK_STENCIL_OP_KEEP);
+      setDepthFailOp(VK_STENCIL_OP_KEEP);
+    }
+
+    // Check if the stencil test for this face is a no-op
+    return writeMask() || compareOp() != VK_COMPARE_OP_ALWAYS;
+  }
+
+
+  void DxvkDepthStencilState::normalize() {
+    if (depthTest()) {
+      // If depth func is equal or if the depth test always fails, depth
+      // writes will not have any observable effect so we can skip them.
+      if (depthCompareOp() == VK_COMPARE_OP_EQUAL
+       || depthCompareOp() == VK_COMPARE_OP_NEVER)
+        setDepthWrite(false);
+
+      // If the depth test always passes and no writes are performed, the
+      // depth test as a whole is a no-op and can safely be disabled.
+      if (depthCompareOp() == VK_COMPARE_OP_ALWAYS && !depthWrite())
+        setDepthTest(false);
+    } else {
+      setDepthWrite(false);
+      setDepthCompareOp(VK_COMPARE_OP_ALWAYS);
+    }
+
+    if (stencilTest()) {
+      // Normalize stencil op and disable stencil testing if both are no-ops.
+      bool frontIsNoOp = !m_stencilOpFront.normalize(depthCompareOp());
+      bool backIsNoOp = !m_stencilOpBack.normalize(depthCompareOp());
+
+      if (frontIsNoOp && backIsNoOp)
+        setStencilTest(false);
+    }
+
+    // Normalize stencil ops if stencil test is disabled
+    if (!stencilTest()) {
+      setStencilOpFront(DxvkStencilOp());
+      setStencilOpBack(DxvkStencilOp());
+    }
+  }
+
+}

src/dxvk/dxvk_constant_state.h

@@ -197,6 +197,8 @@ namespace dxvk {
       m_writeMask = mask;
     }
 
+    bool normalize(VkCompareOp depthOp);
+
   private:
 
     uint16_t m_failOp            : 3;
@@ -268,6 +270,8 @@ namespace dxvk {
       m_stencilOpBack = op;
     }
 
+    void normalize();
+
   private:
 
     uint16_t m_enableDepthTest   : 1;

src/dxvk/meson.build

@@ -75,6 +75,7 @@ dxvk_src = [
   'dxvk_buffer.cpp',
   'dxvk_cmdlist.cpp',
   'dxvk_compute.cpp',
+  'dxvk_constant_state.cpp',
   'dxvk_context.cpp',
   'dxvk_cs.cpp',
   'dxvk_descriptor.cpp',