[d3d9] Handle depth stencil hazards

Track depth stencil textures being used and whether we have any active hazards.

Rebind the framebuffer with VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL if we encounter a depth stencil hazard.

Fixes black squares in Mass Effect and validation errors in some titles.

Closes #1484

src/d3d9/d3d9_common_texture.h

@@ -324,12 +324,16 @@ namespace dxvk {
         : VK_IMAGE_LAYOUT_GENERAL;
     }
 
-    VkImageLayout DetermineDepthStencilLayout() const {
-      return m_image != nullptr &&
-             m_image->info().tiling == VK_IMAGE_TILING_OPTIMAL &&
-            !m_hazardous
-        ? VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL
-        : VK_IMAGE_LAYOUT_GENERAL;
+    VkImageLayout DetermineDepthStencilLayout(bool hazardous) const {
+      VkImageLayout layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
+
+      if (unlikely(hazardous))
+        layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL;
+
+      if (unlikely(m_image->info().tiling == VK_IMAGE_TILING_OPTIMAL))
+        layout = VK_IMAGE_LAYOUT_GENERAL;
+
+      return layout;
     }
 
     Rc<DxvkImageView> CreateView(

src/d3d9/d3d9_device.cpp

@@ -1235,6 +1235,8 @@ namespace dxvk {
 
     m_state.depthStencil = ds;
 
+    UpdateActiveHazardsDS();
+
     return D3D_OK;
   }
 
@@ -4718,6 +4720,7 @@ namespace dxvk {
     const uint32_t bit = 1 << index;
 
     m_activeRTTextures       &= ~bit;
+    m_activeDSTextures       &= ~bit;
     m_activeTextures         &= ~bit;
     m_activeTexturesToUpload &= ~bit;
 
@@ -4728,11 +4731,15 @@ namespace dxvk {
       if (unlikely(tex->IsRenderTarget()))
         m_activeRTTextures |= bit;
 
+      if (unlikely(tex->IsDepthStencil()))
+        m_activeDSTextures |= bit;
+
       if (unlikely(tex->NeedsAnyUpload()))
         m_activeTexturesToUpload |= bit;
     }
 
     UpdateActiveHazardsRT();
+    UpdateActiveHazardsDS();
   }
 
 
@@ -4763,6 +4770,23 @@ namespace dxvk {
   }
 
 
+  inline void D3D9DeviceEx::UpdateActiveHazardsDS() {
+    m_activeHazardsDS = 0;
+    if (m_state.depthStencil != nullptr &&
+        m_state.depthStencil->GetBaseTexture() != nullptr) {
+      for (uint32_t sampler = m_activeDSTextures; sampler; sampler &= sampler - 1) {
+        IDirect3DBaseTexture9* dsBase  = m_state.depthStencil->GetBaseTexture();
+        IDirect3DBaseTexture9* texBase = m_state.textures[bit::tzcnt(sampler)];
+
+        if (likely(dsBase != texBase))
+          continue;
+
+        m_activeHazardsDS |= 1 << bit::tzcnt(sampler);
+      }
+    }
+  }
+
+
   void D3D9DeviceEx::MarkRenderHazards() {
     for (uint32_t rt = m_activeHazardsRT; rt; rt &= rt - 1) {
       // Guaranteed to not be nullptr...
@@ -4947,7 +4971,7 @@ namespace dxvk {
       if (likely(sampleCount == VK_SAMPLE_COUNT_FLAG_BITS_MAX_ENUM || sampleCount == dsImageInfo.sampleCount)) {
         attachments.depth = {
           m_state.depthStencil->GetDepthStencilView(),
-          m_state.depthStencil->GetDepthStencilLayout() };
+          m_state.depthStencil->GetDepthStencilLayout(m_activeHazardsDS != 0) };
       }
     }
 
@@ -5444,6 +5468,11 @@ namespace dxvk {
         MarkRenderHazards();
     }
 
+    if (unlikely((m_lastHazardsDS == 0) != (m_activeHazardsDS == 0))) {
+      m_flags.set(D3D9DeviceFlag::DirtyFramebuffer);
+      m_lastHazardsDS = m_activeHazardsDS;
+    }
+
     for (uint32_t i = 0; i < caps::MaxStreams; i++) {
       auto* vbo = GetCommonBuffer(m_state.vertexBuffers[i].vertexBuffer);
       if (vbo != nullptr && vbo->NeedsUpload())

src/d3d9/d3d9_device.h

@@ -744,6 +744,8 @@ namespace dxvk {
 
     void UpdateActiveHazardsRT();
 
+    void UpdateActiveHazardsDS();
+
     void MarkRenderHazards();
 
     void UploadManagedTextures(uint32_t mask);
@@ -1028,11 +1030,15 @@ namespace dxvk {
 
     uint32_t                        m_activeRTs        = 0;
     uint32_t                        m_activeRTTextures = 0;
+    uint32_t                        m_activeDSTextures = 0;
     uint32_t                        m_activeHazardsRT  = 0;
     uint32_t                        m_alphaSwizzleRTs  = 0;
     uint32_t                        m_activeTextures   = 0;
     uint32_t                        m_activeTexturesToUpload = 0;
 
+    uint32_t                        m_activeHazardsDS = 0;
+    uint32_t                        m_lastHazardsDS   = 0;
+
     D3D9ShaderMasks                 m_vsShaderMasks = D3D9ShaderMasks();
     D3D9ShaderMasks                 m_psShaderMasks = FixedFunctionMask;
 

src/d3d9/d3d9_subresource.h

@@ -99,8 +99,8 @@ namespace dxvk {
       return view;
     }
 
-    VkImageLayout GetDepthStencilLayout() const {
-      return m_texture->DetermineDepthStencilLayout();
+    VkImageLayout GetDepthStencilLayout(bool hazardous) const {
+      return m_texture->DetermineDepthStencilLayout(hazardous);
     }
 
     bool IsNull() {