[dxso] Implement option for alpha test wiggle room

dxvk.conf

@@ -332,3 +332,15 @@
 # - True/False
 
 # d3d9.longMad = False
+
+# Long Mad
+#
+# Workaround for games using alpha test == 1.0, etc due to wonky interpolation or
+# misc. imprecision on some vendors
+#
+# Defaults to enabled on NVIDIA
+#
+# Supported values:
+# - True/False
+
+# d3d9.alphaTestWiggleRoom = is_nvidia

src/d3d9/d3d9_options.cpp

@@ -71,6 +71,7 @@ namespace dxvk {
     this->enumerateByDisplays   = config.getOption<bool>       ("d3d9.enumerateByDisplays", true);
     this->longMad               = config.getOption<bool>       ("d3d9.longMad",            false);
     this->tearFree              = config.getOption<Tristate>   ("d3d9.tearFree", Tristate::Auto);
+    this->alphaTestWiggleRoom   = config.getOption<bool>       ("d3d9.alphaTestWiggleRoom", false);
 
     // If we are not Nvidia, enable general hazards.
     this->generalHazards = adapter == nullptr || !adapter->matchesDriver(DxvkGpuVendor::Nvidia, VK_DRIVER_ID_NVIDIA_PROPRIETARY_KHR, 0, 0);

src/d3d9/d3d9_options.h

@@ -147,6 +147,10 @@ namespace dxvk {
     /// Tear-free mode if vsync is disabled
     /// Tearing mode if vsync is enabled
     Tristate tearFree;
+
+    /// Workaround for games using alpha test == 1.0, etc due to wonky interpolation or
+    /// misc. imprecision on some vendors
+    bool alphaTestWiggleRoom;
   };
 
 }

src/dxso/dxso_compiler.cpp

@@ -3596,6 +3596,21 @@ void DxsoCompiler::emitControlFlowGenericLoop(
       uint32_t alphaId = m_module.opCompositeExtract(floatType,
         m_module.opLoad(m_module.defVectorType(floatType, 4), oC0.id),
         1, &alphaComponentId);
+
+      if (m_moduleInfo.options.alphaTestWiggleRoom) {
+        // NV has wonky interpolation of all 1's in a VS -> PS going to 0.999999...
+        // This causes garbage-looking graphics on people's clothing in EverQuest 2 as it does alpha == 1.0.
+
+        // My testing shows the alpha test has a precision of 1/256 for all A8 and below formats,
+        // and around 1 / 2048 for A32F formats and 1 / 4096 for A16F formats (It makes no sense to me too)
+        // so anyway, we're just going to round this to a precision of 1 / 4096 and hopefully this should make things happy
+        // everywhere.
+        const uint32_t alphaSizeId = m_module.constf32(4096.0f);
+
+        alphaId = m_module.opFMul(floatType, alphaId, alphaSizeId);
+        alphaId = m_module.opRound(floatType, alphaId);
+        alphaId = m_module.opFDiv(floatType, alphaId, alphaSizeId);
+      }
       
       // Load alpha reference
       uint32_t alphaRefMember = m_module.constu32(uint32_t(D3D9RenderStateItem::AlphaRef));

src/dxso/dxso_options.cpp

@@ -47,6 +47,8 @@ namespace dxvk {
     vertexConstantBufferAsSSBO = pDevice->GetVertexConstantLayout().totalSize() > devInfo.core.properties.limits.maxUniformBufferRange;
 
     longMad = options.longMad;
+
+    alphaTestWiggleRoom = options.alphaTestWiggleRoom;
   }
 
 }

src/dxso/dxso_options.h

@@ -50,6 +50,10 @@ namespace dxvk {
     /// This solves some rendering bugs in games that have z-pass shaders which
     /// don't match entirely to the regular vertex shader in this way.
     bool longMad;
+
+    /// Workaround for games using alpha test == 1.0, etc due to wonky interpolation or
+    /// misc. imprecision on some vendors
+    bool alphaTestWiggleRoom;
   };
 
 }