From f140d2de0ddf70ebe064ad0919d20f741f154116 Mon Sep 17 00:00:00 2001
From: Joshua Ashton <joshua@froggi.es>
Date: Wed, 3 May 2023 12:34:37 +0100
Subject: [PATCH] [d3d9] Handle swapchain OOM and other errors more gracefully

Supercedes: #2964
---
 src/d3d9/d3d9_device.cpp    |  5 ++--
 src/d3d9/d3d9_swapchain.cpp | 50 +++++++++++++++++++++++++++++--------
 src/d3d9/d3d9_swapchain.h   |  2 +-
 3 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/src/d3d9/d3d9_device.cpp b/src/d3d9/d3d9_device.cpp
index d66ba349..2ff29674 100644
--- a/src/d3d9/d3d9_device.cpp
+++ b/src/d3d9/d3d9_device.cpp
@@ -7363,8 +7363,9 @@ namespace dxvk {
     }
 
     if (m_implicitSwapchain != nullptr) {
-      if (FAILED(m_implicitSwapchain->Reset(pPresentationParameters, pFullscreenDisplayMode)))
-        return D3DERR_INVALIDCALL;
+      HRESULT hr = m_implicitSwapchain->Reset(pPresentationParameters, pFullscreenDisplayMode);
+      if (FAILED(hr))
+        return hr;
     }
     else
       m_implicitSwapchain = new D3D9SwapChainEx(this, pPresentationParameters, pFullscreenDisplayMode);
diff --git a/src/d3d9/d3d9_swapchain.cpp b/src/d3d9/d3d9_swapchain.cpp
index 8b091f2b..017b562e 100644
--- a/src/d3d9/d3d9_swapchain.cpp
+++ b/src/d3d9/d3d9_swapchain.cpp
@@ -43,7 +43,9 @@ namespace dxvk {
         RecreateSwapChain(false);
     }
 
-    CreateBackBuffers(m_presentParams.BackBufferCount);
+    if (FAILED(CreateBackBuffers(m_presentParams.BackBufferCount)))
+      throw DxvkError("D3D9: Failed to create swapchain backbuffers");
+
     CreateBlitter();
     CreateHud();
 
@@ -101,6 +103,14 @@ namespace dxvk {
           DWORD    dwFlags) {
     D3D9DeviceLock lock = m_parent->LockDevice();
 
+    // If we have no backbuffers, error out.
+    // This handles the case where a ::Reset failed due to OOM
+    // or whatever.
+    // I am not sure what the actual HRESULT returned here is
+    // or should be, but it is better than crashing... probably!
+    if (m_backBuffers.empty())
+      return D3D_OK;
+
     uint32_t presentInterval = m_presentParams.PresentationInterval;
 
     // This is not true directly in d3d9 to to timing differences that don't matter for us.
@@ -515,6 +525,8 @@ namespace dxvk {
           D3DDISPLAYMODEEX*      pFullscreenDisplayMode) {
     D3D9DeviceLock lock = m_parent->LockDevice();
 
+    HRESULT hr = D3D_OK;
+
     this->SynchronizePresent();
     this->NormalizePresentParameters(pPresentParams);
 
@@ -529,15 +541,17 @@ namespace dxvk {
     }
     else {
       if (changeFullscreen) {
-        if (FAILED(this->EnterFullscreenMode(pPresentParams, pFullscreenDisplayMode)))
-          return D3DERR_INVALIDCALL;
+        hr = this->EnterFullscreenMode(pPresentParams, pFullscreenDisplayMode);
+        if (FAILED(hr))
+          return hr;
       }
 
       D3D9WindowMessageFilter filter(m_window);
 
       if (!changeFullscreen) {
-        if (FAILED(ChangeDisplayMode(pPresentParams, pFullscreenDisplayMode)))
-          return D3DERR_INVALIDCALL;
+        hr = ChangeDisplayMode(pPresentParams, pFullscreenDisplayMode);
+        if (FAILED(hr))
+          return hr;
 
         wsi::updateFullscreenWindow(m_monitor, m_window, true);
       }
@@ -548,7 +562,9 @@ namespace dxvk {
     if (changeFullscreen)
       SetGammaRamp(0, &m_ramp);
 
-    CreateBackBuffers(m_presentParams.BackBufferCount);
+    hr = CreateBackBuffers(m_presentParams.BackBufferCount);
+    if (FAILED(hr))
+      return hr;
 
     return D3D_OK;
   }
@@ -921,13 +937,15 @@ namespace dxvk {
   }
 
 
-  void D3D9SwapChainEx::CreateBackBuffers(uint32_t NumBackBuffers) {
+  HRESULT D3D9SwapChainEx::CreateBackBuffers(uint32_t NumBackBuffers) {
     // Explicitly destroy current swap image before
     // creating a new one to free up resources
     DestroyBackBuffers();
 
     int NumFrontBuffer = m_parent->GetOptions()->noExplicitFrontBuffer ? 0 : 1;
-    m_backBuffers.resize(NumBackBuffers + NumFrontBuffer);
+    const uint32_t NumBuffers = NumBackBuffers + NumFrontBuffer;
+
+    m_backBuffers.reserve(NumBuffers);
 
     // Create new back buffer
     D3D9_COMMON_TEXTURE_DESC desc;
@@ -947,8 +965,18 @@ namespace dxvk {
     // Docs: Also note that - unlike textures - swap chain back buffers, render targets [..] can be locked
     desc.IsLockable         = TRUE;
 
-    for (uint32_t i = 0; i < m_backBuffers.size(); i++)
-      m_backBuffers[i] = new D3D9Surface(m_parent, &desc, this, nullptr);
+    for (uint32_t i = 0; i < NumBuffers; i++) {
+      D3D9Surface* surface;
+      try {
+        surface = new D3D9Surface(m_parent, &desc, this, nullptr);
+      } catch (const DxvkError& e) {
+        DestroyBackBuffers();
+        Logger::err(e.message());
+        return D3DERR_OUTOFVIDEOMEMORY;
+      }
+
+      m_backBuffers.emplace_back(surface);
+    }
 
     auto swapImage = m_backBuffers[0]->GetCommonTexture()->GetImage();
 
@@ -973,6 +1001,8 @@ namespace dxvk {
     m_device->submitCommandList(
       m_context->endRecording(),
       nullptr);
+
+    return D3D_OK;
   }
 
 
diff --git a/src/d3d9/d3d9_swapchain.h b/src/d3d9/d3d9_swapchain.h
index 2979ecb4..2313dd60 100644
--- a/src/d3d9/d3d9_swapchain.h
+++ b/src/d3d9/d3d9_swapchain.h
@@ -150,7 +150,7 @@ namespace dxvk {
 
     void DestroyBackBuffers();
 
-    void CreateBackBuffers(
+    HRESULT CreateBackBuffers(
             uint32_t            NumBackBuffers);
 
     void CreateBlitter();