[d3d8] Validate normals component count for FVF shaders

src/d3d8/d3d8_device.cpp

@@ -1826,34 +1826,42 @@ namespace dxvk {
       }
     }
 
-    D3D8VertexShaderInfo& info = m_vertexShaders.emplace_back();
+    D3D9VertexShaderCode translatedVS;
-
+    HRESULT res = TranslateVertexShader8(pDeclaration, pFunction, m_d3d8Options, translatedVS);
-    // Store D3D8 bytecodes in the shader info
-    for (UINT i = 0; pDeclaration[i] != D3DVSD_END(); i++)
-      info.declaration.push_back(pDeclaration[i]);
-    info.declaration.push_back(D3DVSD_END());
-
-    if (pFunction != nullptr) {
-      for (UINT i = 0; pFunction[i] != D3DVS_END(); i++)
-        info.function.push_back(pFunction[i]);
-      info.function.push_back(D3DVS_END());
-    }
-
-    D3D9VertexShaderCode result = TranslateVertexShader8(pDeclaration, pFunction, m_d3d8Options);
-
-    // Create vertex declaration
-    HRESULT res = GetD3D9()->CreateVertexDeclaration(result.declaration, &(info.pVertexDecl));
     if (unlikely(FAILED(res)))
       return res;
 
+    // Create vertex declaration
+    Com<d3d9::IDirect3DVertexDeclaration9> pVertexDecl;
+    res = GetD3D9()->CreateVertexDeclaration(translatedVS.declaration, &pVertexDecl);
+    if (unlikely(FAILED(res)))
+      return res;
+
+    Com<d3d9::IDirect3DVertexShader9> pVertexShader;
     if (pFunction != nullptr) {
-      res = GetD3D9()->CreateVertexShader(result.function.data(), &(info.pVertexShader));
+      res = GetD3D9()->CreateVertexShader(translatedVS.function.data(), &pVertexShader);
     } else {
       // pFunction is NULL: fixed function pipeline
-      info.pVertexShader = nullptr;
+      pVertexShader = nullptr;
     }
 
     if (likely(SUCCEEDED(res))) {
+      D3D8VertexShaderInfo& info = m_vertexShaders.emplace_back();
+
+      info.pVertexDecl = std::move(pVertexDecl);
+      info.pVertexShader = std::move(pVertexShader);
+
+      // Store D3D8 bytecodes in the shader info
+      for (UINT i = 0; pDeclaration[i] != D3DVSD_END(); i++)
+        info.declaration.push_back(pDeclaration[i]);
+      info.declaration.push_back(D3DVSD_END());
+
+      if (pFunction != nullptr) {
+        for (UINT i = 0; pFunction[i] != D3DVS_END(); i++)
+          info.function.push_back(pFunction[i]);
+        info.function.push_back(D3DVS_END());
+      }
+
       // Set bit to indicate this is not an FVF
       *pHandle = getShaderHandle(m_vertexShaders.size());
     }
@@ -2064,12 +2072,11 @@ namespace dxvk {
       return D3DERR_INVALIDCALL;
     }
 
-    d3d9::IDirect3DPixelShader9* pPixelShader;
+    Com<d3d9::IDirect3DPixelShader9> pPixelShader;
-
     HRESULT res = GetD3D9()->CreatePixelShader(pFunction, &pPixelShader);
 
     if (likely(SUCCEEDED(res))) {
-      m_pixelShaders.push_back(pPixelShader);
+      m_pixelShaders.push_back(std::move(pPixelShader));
       // Still set the shader bit, to prevent conflicts with NULL.
       *pHandle = getShaderHandle(m_pixelShaders.size());
     }

src/d3d8/d3d8_shader.cpp

@@ -110,26 +110,27 @@ namespace dxvk {
   }
 
   /**
-   * Converts a D3D8 vertex shader + declaration
+   * Validates and converts a D3D8 vertex shader
-   * to a D3D9 vertex shader + declaration.
+   * + declaration to a D3D9 vertex shader + declaration.
   */
-  D3D9VertexShaderCode TranslateVertexShader8(
+  HRESULT TranslateVertexShader8(
-      const DWORD*        pDeclaration,
+      const DWORD*          pDeclaration,
-      const DWORD*        pFunction,
+      const DWORD*          pFunction,
-      const D3D8Options&  options) {
+      const D3D8Options&    options,
+      D3D9VertexShaderCode& pTranslatedVS) {
     using d3d9::D3DDECLTYPE;
     using d3d9::D3DDECLTYPE_UNUSED;
 
-    D3D9VertexShaderCode result;
+    HRESULT res = D3D_OK;
 
-    std::vector<DWORD>& tokens = result.function;
+    std::vector<DWORD>& tokens = pTranslatedVS.function;
     std::vector<DWORD> defs; // Constant definitions
 
     // shaderInputRegisters:
     // set bit N to enable input register vN
     DWORD shaderInputRegisters = 0;
 
-    d3d9::D3DVERTEXELEMENT9* vertexElements = result.declaration;
+    d3d9::D3DVERTEXELEMENT9* vertexElements = pTranslatedVS.declaration;
     unsigned int elementIdx = 0;
 
     // These are used for pDeclaration and pFunction
@@ -206,6 +207,12 @@ namespace dxvk {
             D3DVSDT_TYPE     type = D3DVSDT_TYPE(VSD_SHIFT_MASK(token, D3DVSD_DATATYPE));
             D3DVSDE_REGISTER reg  = D3DVSDE_REGISTER(VSD_SHIFT_MASK(token, D3DVSD_VERTEXREG));
 
+            // FVF normals are expected to only have 3 components
+            if (unlikely(pFunction == nullptr && reg == D3DVSDE_NORMAL && type != D3DVSDT_FLOAT3)) {
+              Logger::err("D3D8Device::CreateVertexShader: Invalid FVF declaration: D3DVSDE_NORMAL must use D3DVSDT_FLOAT3");
+              return D3DERR_INVALIDCALL;
+            }
+
             addVertexElement(reg, type);
 
             dbg << "type=" << type << ", register=" << reg;
@@ -332,7 +339,7 @@ namespace dxvk {
       } while (token != D3DVS_END());
     }
 
-    return result;
+    return res;
   }
 
 }

src/d3d8/d3d8_shader.h

@@ -10,9 +10,10 @@ namespace dxvk {
       std::vector<DWORD> function;
     };
 
-    D3D9VertexShaderCode TranslateVertexShader8(
+    HRESULT TranslateVertexShader8(
-      const DWORD*        pDeclaration,
+      const DWORD*          pDeclaration,
-      const DWORD*        pFunction,
+      const DWORD*          pFunction,
-      const D3D8Options&  overrides);
+      const D3D8Options&    overrides,
+      D3D9VertexShaderCode& pTranslatedVS);
 
 }