[dxvk] Disable alpha to coverage if sample mask is written

Matches D3D11 behaviour and fixes tree rendering in A Total War Saga: TROY.

src/dxvk/dxvk_context.cpp

@@ -5769,7 +5769,8 @@ namespace dxvk {
       VkSampleMask sampleMask = m_state.gp.state.ms.sampleMask() & ((1u << sampleCount) - 1u);
       m_cmd->cmdSetMultisampleState(sampleCount, sampleMask);
 
-      if (m_device->features().extExtendedDynamicState3.extendedDynamicState3AlphaToCoverageEnable)
+      if (m_device->features().extExtendedDynamicState3.extendedDynamicState3AlphaToCoverageEnable
+       && !m_state.gp.flags.test(DxvkGraphicsPipelineFlag::HasSampleMaskExport))
         m_cmd->cmdSetAlphaToCoverageState(m_state.gp.state.ms.enableAlphaToCoverage());
     }
 

src/dxvk/dxvk_graphics.cpp

@@ -301,9 +301,12 @@ namespace dxvk {
       msInfo.minSampleShading     = 1.0f;
     }
 
+    // Alpha to coverage is not supported with sample mask exports.
+    cbUseDynamicAlphaToCoverage = !fs || !fs->flags().test(DxvkShaderFlag::ExportsSampleMask);
+
     msSampleMask                  = state.ms.sampleMask() & ((1u << msInfo.rasterizationSamples) - 1);
     msInfo.pSampleMask            = &msSampleMask;
-    msInfo.alphaToCoverageEnable  = state.ms.enableAlphaToCoverage();
+    msInfo.alphaToCoverageEnable  = state.ms.enableAlphaToCoverage() && cbUseDynamicAlphaToCoverage;
 
     // We need to be fully consistent with the pipeline state here, and
     // while we could consistently infer it, just don't take any chances
@@ -325,6 +328,7 @@ namespace dxvk {
            && msInfo.alphaToOneEnable         == other.msInfo.alphaToOneEnable
            && msSampleMask                    == other.msSampleMask
            && cbUseDynamicBlendConstants      == other.cbUseDynamicBlendConstants
+           && cbUseDynamicAlphaToCoverage     == other.cbUseDynamicAlphaToCoverage
            && feedbackLoop                    == other.feedbackLoop;
 
     for (uint32_t i = 0; i < rtInfo.colorAttachmentCount && eq; i++)
@@ -364,6 +368,7 @@ namespace dxvk {
     hash.add(uint32_t(msInfo.alphaToOneEnable));
     hash.add(uint32_t(msSampleMask));
     hash.add(uint32_t(cbUseDynamicBlendConstants));
+    hash.add(uint32_t(cbUseDynamicAlphaToCoverage));
     hash.add(uint32_t(feedbackLoop));
 
     for (uint32_t i = 0; i < rtInfo.colorAttachmentCount; i++)
@@ -402,7 +407,8 @@ namespace dxvk {
       dynamicStates[dynamicStateCount++] = VK_DYNAMIC_STATE_RASTERIZATION_SAMPLES_EXT;
       dynamicStates[dynamicStateCount++] = VK_DYNAMIC_STATE_SAMPLE_MASK_EXT;
 
-      if (device->features().extExtendedDynamicState3.extendedDynamicState3AlphaToCoverageEnable)
+      if (device->features().extExtendedDynamicState3.extendedDynamicState3AlphaToCoverageEnable
+       && state.cbUseDynamicAlphaToCoverage)
         dynamicStates[dynamicStateCount++] = VK_DYNAMIC_STATE_ALPHA_TO_COVERAGE_ENABLE_EXT;
     }
 
@@ -944,8 +950,12 @@ namespace dxvk {
     if (m_barrier.access & VK_ACCESS_SHADER_WRITE_BIT)
       m_flags.set(DxvkGraphicsPipelineFlag::HasStorageDescriptors);
 
-    if (m_shaders.fs != nullptr && m_shaders.fs->flags().test(DxvkShaderFlag::HasSampleRateShading))
+    if (m_shaders.fs != nullptr) {
-      m_flags.set(DxvkGraphicsPipelineFlag::HasSampleRateShading);
+      if (m_shaders.fs->flags().test(DxvkShaderFlag::HasSampleRateShading))
+        m_flags.set(DxvkGraphicsPipelineFlag::HasSampleRateShading);
+      if (m_shaders.fs->flags().test(DxvkShaderFlag::ExportsSampleMask))
+        m_flags.set(DxvkGraphicsPipelineFlag::HasSampleMaskExport);
+    }
   }
   
   
@@ -1183,7 +1193,8 @@ namespace dxvk {
           return false;
 
         if (!canUseDynamicAlphaToCoverage
-         && (state.ms.enableAlphaToCoverage()))
+         && (state.ms.enableAlphaToCoverage())
+         && !m_shaders.fs->flags().test(DxvkShaderFlag::ExportsSampleMask))
           return false;
       }
     }

src/dxvk/dxvk_graphics.h

@@ -31,6 +31,7 @@ namespace dxvk {
     HasTransformFeedback,
     HasStorageDescriptors,
     HasSampleRateShading,
+    HasSampleMaskExport,
   };
 
   using DxvkGraphicsPipelineFlags = Flags<DxvkGraphicsPipelineFlag>;
@@ -116,6 +117,7 @@ namespace dxvk {
 
     VkSampleMask                                    msSampleMask               = 0u;
     VkBool32                                        cbUseDynamicBlendConstants = VK_FALSE;
+    VkBool32                                        cbUseDynamicAlphaToCoverage = VK_FALSE;
 
     std::array<VkPipelineColorBlendAttachmentState, MaxNumRenderTargets> cbAttachments  = { };
     std::array<VkFormat,                            MaxNumRenderTargets> rtColorFormats = { };

src/dxvk/dxvk_shader.cpp

@@ -1296,8 +1296,10 @@ namespace dxvk {
       dynamicStates[dynamicStateCount++] = VK_DYNAMIC_STATE_RASTERIZATION_SAMPLES_EXT;
       dynamicStates[dynamicStateCount++] = VK_DYNAMIC_STATE_SAMPLE_MASK_EXT;
 
-      if (m_device->features().extExtendedDynamicState3.extendedDynamicState3AlphaToCoverageEnable)
+      if (!m_shaders.fs || !m_shaders.fs->flags().test(DxvkShaderFlag::ExportsSampleMask)) {
-        dynamicStates[dynamicStateCount++] = VK_DYNAMIC_STATE_ALPHA_TO_COVERAGE_ENABLE_EXT;
+        if (m_device->features().extExtendedDynamicState3.extendedDynamicState3AlphaToCoverageEnable)
+          dynamicStates[dynamicStateCount++] = VK_DYNAMIC_STATE_ALPHA_TO_COVERAGE_ENABLE_EXT;
+      }
     }
 
     VkPipelineDynamicStateCreateInfo dyInfo = { VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO };