[dxbc] Implement workaround to replace NaN render target outputs by zero

dxvk.conf

@@ -126,6 +126,14 @@
 # d3d11.strictDivision = False
 
 
+# Replaces NaN outputs from fragment shaders with zeroes for floating
+# point render target. Used in some games to prevent artifacting.
+#
+# Supported values: True, False
+
+# d3d11.enableRtOutputNanFixup = False
+
+
 # Clears workgroup memory in compute shaders to zero. Some games don't do
 # this and rely on undefined behaviour. Enabling may reduce performance.
 #

src/d3d11/d3d11_options.cpp

@@ -9,6 +9,7 @@ namespace dxvk {
 
     this->dcSingleUseMode       = config.getOption<bool>("d3d11.dcSingleUseMode", true);
     this->strictDivision           = config.getOption<bool>("d3d11.strictDivision", false);
+    this->enableRtOutputNanFixup   = config.getOption<bool>("d3d11.enableRtOutputNanFixup", false);
     this->zeroInitWorkgroupMemory  = config.getOption<bool>("d3d11.zeroInitWorkgroupMemory", false);
     this->relaxedBarriers       = config.getOption<bool>("d3d11.relaxedBarriers", false);
     this->maxTessFactor         = config.getOption<int32_t>("d3d11.maxTessFactor", 0);

src/d3d11/d3d11_options.h

@@ -25,6 +25,10 @@ namespace dxvk {
     /// games may expect correct behaviour.
     bool strictDivision;
 
+    /// Enables workaround to replace NaN render target
+    /// outputs with zero
+    bool enableRtOutputNanFixup;
+
     /// Enables out-of-bounds access check for constant
     /// buffers. Workaround for a few broken games that
     /// access random data inside their shaders.

src/dxbc/dxbc_compiler.cpp

@@ -5834,10 +5834,21 @@ namespace dxvk {
         scalars[c] = m_module.opVectorExtractDynamic(compTypeId, vector.id, specId);
       }
 
-      vector.id = m_module.opCompositeConstruct(
-        getVectorTypeId(vector.type),
-        vector.type.ccount,
-        scalars.data());
+      uint32_t typeId = getVectorTypeId(vector.type);
+      vector.id = m_module.opCompositeConstruct(typeId, vector.type.ccount, scalars.data());
+
+      // Replace NaN by zero if requested
+      if (m_moduleInfo.options.enableRtOutputNanFixup && vector.type.ctype == DxbcScalarType::Float32) {
+        uint32_t boolType = m_module.defBoolType();
+
+        if (vector.type.ccount > 1)
+          boolType = m_module.defVectorType(boolType, vector.type.ccount);
+
+        uint32_t zero = emitBuildConstVecf32(0.0f, 0.0f, 0.0f, 0.0f,
+          DxbcRegMask((1u << vector.type.ccount) - 1)).id;
+        uint32_t isNan = m_module.opIsNan(boolType, vector.id);
+        vector.id = m_module.opSelect(typeId, isNan, zero, vector.id);
+      }
       
       emitValueStore(m_oRegs[i], vector,
         DxbcRegMask::firstN(vector.type.ccount));

src/dxbc/dxbc_options.cpp

@@ -37,6 +37,7 @@ namespace dxvk {
       case Tristate::False: minSsboAlignment = ~0u; break;
     }
     
+    enableRtOutputNanFixup   = options.enableRtOutputNanFixup;
     strictDivision           = options.strictDivision;
     zeroInitWorkgroupMemory  = options.zeroInitWorkgroupMemory;
     dynamicIndexedConstantBufferAsSsbo = options.constantBufferRangeCheck;

src/dxbc/dxbc_options.h

@@ -32,6 +32,9 @@ namespace dxvk {
     /// dword offsets. Fixes RE2 and DMC5 on Nvidia drivers.
     bool useSdivForBufferIndex = false;
 
+    /// Enables NaN fixup for render target outputs
+    bool enableRtOutputNanFixup = false;
+
     /// Enables sm4-compliant division-by-zero behaviour
     bool strictDivision = false;