From 43afe90fcd548c5bcf87c5477e62df0ecf9f5073 Mon Sep 17 00:00:00 2001 From: WinterSnowfall Date: Tue, 4 Mar 2025 19:59:46 +0200 Subject: [PATCH] [d3d8] Validate normals component count for FVF shaders --- src/d3d8/d3d8_device.cpp | 51 +++++++++++++++++++++++----------------- src/d3d8/d3d8_shader.cpp | 27 +++++++++++++-------- src/d3d8/d3d8_shader.h | 9 +++---- 3 files changed, 51 insertions(+), 36 deletions(-) diff --git a/src/d3d8/d3d8_device.cpp b/src/d3d8/d3d8_device.cpp index 88227c2c3..2b1963e39 100644 --- a/src/d3d8/d3d8_device.cpp +++ b/src/d3d8/d3d8_device.cpp @@ -1826,34 +1826,42 @@ namespace dxvk { } } - D3D8VertexShaderInfo& info = m_vertexShaders.emplace_back(); - - // Store D3D8 bytecodes in the shader info - for (UINT i = 0; pDeclaration[i] != D3DVSD_END(); i++) - info.declaration.push_back(pDeclaration[i]); - info.declaration.push_back(D3DVSD_END()); - - if (pFunction != nullptr) { - for (UINT i = 0; pFunction[i] != D3DVS_END(); i++) - info.function.push_back(pFunction[i]); - info.function.push_back(D3DVS_END()); - } - - D3D9VertexShaderCode result = TranslateVertexShader8(pDeclaration, pFunction, m_d3d8Options); - - // Create vertex declaration - HRESULT res = GetD3D9()->CreateVertexDeclaration(result.declaration, &(info.pVertexDecl)); + D3D9VertexShaderCode translatedVS; + HRESULT res = TranslateVertexShader8(pDeclaration, pFunction, m_d3d8Options, translatedVS); if (unlikely(FAILED(res))) return res; + // Create vertex declaration + Com pVertexDecl; + res = GetD3D9()->CreateVertexDeclaration(translatedVS.declaration, &pVertexDecl); + if (unlikely(FAILED(res))) + return res; + + Com pVertexShader; if (pFunction != nullptr) { - res = GetD3D9()->CreateVertexShader(result.function.data(), &(info.pVertexShader)); + res = GetD3D9()->CreateVertexShader(translatedVS.function.data(), &pVertexShader); } else { // pFunction is NULL: fixed function pipeline - info.pVertexShader = nullptr; + pVertexShader = nullptr; } if (likely(SUCCEEDED(res))) { + D3D8VertexShaderInfo& info = m_vertexShaders.emplace_back(); + + info.pVertexDecl = std::move(pVertexDecl); + info.pVertexShader = std::move(pVertexShader); + + // Store D3D8 bytecodes in the shader info + for (UINT i = 0; pDeclaration[i] != D3DVSD_END(); i++) + info.declaration.push_back(pDeclaration[i]); + info.declaration.push_back(D3DVSD_END()); + + if (pFunction != nullptr) { + for (UINT i = 0; pFunction[i] != D3DVS_END(); i++) + info.function.push_back(pFunction[i]); + info.function.push_back(D3DVS_END()); + } + // Set bit to indicate this is not an FVF *pHandle = getShaderHandle(m_vertexShaders.size()); } @@ -2064,12 +2072,11 @@ namespace dxvk { return D3DERR_INVALIDCALL; } - d3d9::IDirect3DPixelShader9* pPixelShader; - + Com pPixelShader; HRESULT res = GetD3D9()->CreatePixelShader(pFunction, &pPixelShader); if (likely(SUCCEEDED(res))) { - m_pixelShaders.push_back(pPixelShader); + m_pixelShaders.push_back(std::move(pPixelShader)); // Still set the shader bit, to prevent conflicts with NULL. *pHandle = getShaderHandle(m_pixelShaders.size()); } diff --git a/src/d3d8/d3d8_shader.cpp b/src/d3d8/d3d8_shader.cpp index 4d48994d2..919957d10 100644 --- a/src/d3d8/d3d8_shader.cpp +++ b/src/d3d8/d3d8_shader.cpp @@ -110,26 +110,27 @@ namespace dxvk { } /** - * Converts a D3D8 vertex shader + declaration - * to a D3D9 vertex shader + declaration. + * Validates and converts a D3D8 vertex shader + * + declaration to a D3D9 vertex shader + declaration. */ - D3D9VertexShaderCode TranslateVertexShader8( - const DWORD* pDeclaration, - const DWORD* pFunction, - const D3D8Options& options) { + HRESULT TranslateVertexShader8( + const DWORD* pDeclaration, + const DWORD* pFunction, + const D3D8Options& options, + D3D9VertexShaderCode& pTranslatedVS) { using d3d9::D3DDECLTYPE; using d3d9::D3DDECLTYPE_UNUSED; - D3D9VertexShaderCode result; + HRESULT res = D3D_OK; - std::vector& tokens = result.function; + std::vector& tokens = pTranslatedVS.function; std::vector defs; // Constant definitions // shaderInputRegisters: // set bit N to enable input register vN DWORD shaderInputRegisters = 0; - d3d9::D3DVERTEXELEMENT9* vertexElements = result.declaration; + d3d9::D3DVERTEXELEMENT9* vertexElements = pTranslatedVS.declaration; unsigned int elementIdx = 0; // These are used for pDeclaration and pFunction @@ -206,6 +207,12 @@ namespace dxvk { D3DVSDT_TYPE type = D3DVSDT_TYPE(VSD_SHIFT_MASK(token, D3DVSD_DATATYPE)); D3DVSDE_REGISTER reg = D3DVSDE_REGISTER(VSD_SHIFT_MASK(token, D3DVSD_VERTEXREG)); + // FVF normals are expected to only have 3 components + if (unlikely(pFunction == nullptr && reg == D3DVSDE_NORMAL && type != D3DVSDT_FLOAT3)) { + Logger::err("D3D8Device::CreateVertexShader: Invalid FVF declaration: D3DVSDE_NORMAL must use D3DVSDT_FLOAT3"); + return D3DERR_INVALIDCALL; + } + addVertexElement(reg, type); dbg << "type=" << type << ", register=" << reg; @@ -332,7 +339,7 @@ namespace dxvk { } while (token != D3DVS_END()); } - return result; + return res; } } diff --git a/src/d3d8/d3d8_shader.h b/src/d3d8/d3d8_shader.h index f0a83ae12..5016c0e6c 100644 --- a/src/d3d8/d3d8_shader.h +++ b/src/d3d8/d3d8_shader.h @@ -10,9 +10,10 @@ namespace dxvk { std::vector function; }; - D3D9VertexShaderCode TranslateVertexShader8( - const DWORD* pDeclaration, - const DWORD* pFunction, - const D3D8Options& overrides); + HRESULT TranslateVertexShader8( + const DWORD* pDeclaration, + const DWORD* pFunction, + const D3D8Options& overrides, + D3D9VertexShaderCode& pTranslatedVS); } \ No newline at end of file