[dxvk] Add output component mask state to graphics pipelines

This is required in order to implement swizzled render target views.
We currently use this to remap color write masks as needed.

src/dxvk/dxvk_context.cpp

@@ -2246,6 +2246,14 @@ namespace dxvk {
       m_state.gp.state.msSampleCount = fb->getSampleCount();
       m_state.om.framebuffer = fb;
 
+      for (uint32_t i = 0; i < MaxNumRenderTargets; i++) {
+        Rc<DxvkImageView> attachment = fb->getColorTarget(i).view;
+
+        m_state.gp.state.omComponentMapping[i] = attachment != nullptr
+          ? util::invertComponentMapping(attachment->info().swizzle)
+          : VkComponentMapping();
+      }
+      
       m_flags.set(DxvkContextFlag::GpDirtyPipelineState);
     }
   }

src/dxvk/dxvk_graphics.cpp

@@ -229,6 +229,17 @@ namespace dxvk {
     if (m_gs  != nullptr) stages.push_back(m_gs->stageInfo(&specInfo));
     if (m_fs  != nullptr) stages.push_back(m_fs->stageInfo(&specInfo));
 
+    // Fix up color write masks using the component mappings
+    std::array<VkPipelineColorBlendAttachmentState, MaxNumRenderTargets> omBlendAttachments;
+
+    for (uint32_t i = 0; i < MaxNumRenderTargets; i++) {
+      omBlendAttachments[i] = state.omBlendAttachments[i];
+      omBlendAttachments[i].colorWriteMask = util::remapComponentMask(
+        state.omBlendAttachments[i].colorWriteMask,
+        state.omComponentMapping[i]);
+    }
+
+    // Generate per-instance attribute divisors
     std::array<VkVertexInputBindingDivisorDescriptionEXT, MaxNumVertexBindings> viDivisorDesc;
     uint32_t                                                                    viDivisorCount = 0;
     
@@ -332,7 +343,7 @@ namespace dxvk {
     cbInfo.logicOpEnable          = state.omEnableLogicOp;
     cbInfo.logicOp                = state.omLogicOp;
     cbInfo.attachmentCount        = DxvkLimits::MaxNumRenderTargets;
-    cbInfo.pAttachments           = state.omBlendAttachments;
+    cbInfo.pAttachments           = omBlendAttachments.data();
     
     for (uint32_t i = 0; i < 4; i++)
       cbInfo.blendConstants[i] = 0.0f;

src/dxvk/dxvk_graphics.h

@@ -69,6 +69,7 @@ namespace dxvk {
     VkBool32                            omEnableLogicOp;
     VkLogicOp                           omLogicOp;
     VkPipelineColorBlendAttachmentState omBlendAttachments[MaxNumRenderTargets];
+    VkComponentMapping                  omComponentMapping[MaxNumRenderTargets];
   };
   
   

src/dxvk/dxvk_util.cpp

@@ -75,4 +75,65 @@ namespace dxvk::util {
     return formatInfo->elementSize * flattenImageExtent(computeBlockCount(extent, formatInfo->blockSize));
   }
 
+
+  static VkColorComponentFlags remapComponentFlag(
+          VkColorComponentFlags       mask,
+          VkComponentSwizzle          swizzle,
+          VkColorComponentFlagBits    identity) {
+    VkColorComponentFlags bit;
+
+    switch (swizzle) {
+      case VK_COMPONENT_SWIZZLE_IDENTITY: bit = identity;                 break;
+      case VK_COMPONENT_SWIZZLE_R:        bit = VK_COLOR_COMPONENT_R_BIT; break;
+      case VK_COMPONENT_SWIZZLE_G:        bit = VK_COLOR_COMPONENT_G_BIT; break;
+      case VK_COMPONENT_SWIZZLE_B:        bit = VK_COLOR_COMPONENT_B_BIT; break;
+      case VK_COMPONENT_SWIZZLE_A:        bit = VK_COLOR_COMPONENT_A_BIT; break;
+      default:                            bit = 0; /* SWIZZLE_ZERO, SWIZZLE_ONE */
+    }
+
+    return (mask & bit) ? identity : 0;
+  }
+
+
+  VkColorComponentFlags remapComponentMask(
+          VkColorComponentFlags       mask,
+          VkComponentMapping          mapping) {
+    VkColorComponentFlags result = 0;
+    result |= remapComponentFlag(mask, mapping.r, VK_COLOR_COMPONENT_R_BIT);
+    result |= remapComponentFlag(mask, mapping.g, VK_COLOR_COMPONENT_G_BIT);
+    result |= remapComponentFlag(mask, mapping.b, VK_COLOR_COMPONENT_B_BIT);
+    result |= remapComponentFlag(mask, mapping.a, VK_COLOR_COMPONENT_A_BIT);
+    return result;
+  }
+
+
+  static VkComponentSwizzle findComponentSwizzle(
+          VkComponentSwizzle          swizzle,
+          VkComponentSwizzle          identity,
+          VkComponentMapping          mapping) {
+    if (identity == VK_COMPONENT_SWIZZLE_IDENTITY)
+      return VK_COMPONENT_SWIZZLE_IDENTITY;
+    
+    if (mapping.r == swizzle)
+      return VK_COMPONENT_SWIZZLE_R;
+    if (mapping.g == swizzle)
+      return VK_COMPONENT_SWIZZLE_G;
+    if (mapping.b == swizzle)
+      return VK_COMPONENT_SWIZZLE_B;
+    if (mapping.a == swizzle)
+      return VK_COMPONENT_SWIZZLE_A;
+    
+    return VK_COMPONENT_SWIZZLE_ZERO;
+  }
+
+
+  VkComponentMapping invertComponentMapping(VkComponentMapping mapping) {
+    VkComponentMapping result;
+    result.r = findComponentSwizzle(VK_COMPONENT_SWIZZLE_R, mapping.r, mapping);
+    result.g = findComponentSwizzle(VK_COMPONENT_SWIZZLE_G, mapping.g, mapping);
+    result.b = findComponentSwizzle(VK_COMPONENT_SWIZZLE_B, mapping.b, mapping);
+    result.a = findComponentSwizzle(VK_COMPONENT_SWIZZLE_A, mapping.a, mapping);
+    return result;
+  }
+
 }

src/dxvk/dxvk_util.h

@@ -172,6 +172,35 @@ namespace dxvk::util {
    */
   VkDeviceSize computeImageDataSize(VkFormat format, VkExtent3D extent);
 
+  /**
+   * \brief Applies a component mapping to a component mask
+   * 
+   * For each component, the component specified in the mapping
+   * is used to look up the flag of the original component mask.
+   * If the component mapping is zero or one, the corresponding
+   * mask bit will be set to zero.
+   * \param [in] mask The original component mask
+   * \param [in] mapping Component mapping to apply
+   * \returns Remapped component mask
+   */
+  VkColorComponentFlags remapComponentMask(
+          VkColorComponentFlags       mask,
+          VkComponentMapping          mapping);
+  
+  /**
+   * \brief Inverts a component mapping
+   *
+   * Transforms a component mapping so that components can
+   * be mapped back to their original location. Requires
+   * that each component is used only once.
+   * 
+   * For example. when given a mapping of (0,0,0,R),
+   * this function will return the mapping (A,0,0,0).
+   * \returns Inverted component mapping
+   */
+  VkComponentMapping invertComponentMapping(
+          VkComponentMapping          mapping);
+  
 }