diff --git a/src/d3d9/d3d9_surface.cpp b/src/d3d9/d3d9_surface.cpp
index 46ac75ac9..77e3f555a 100644
--- a/src/d3d9/d3d9_surface.cpp
+++ b/src/d3d9/d3d9_surface.cpp
@@ -30,7 +30,7 @@ namespace dxvk {
         pBaseTexture,
         pBaseTexture) { }
 
-  void D3D9Surface::AddRefPrivate(bool swapchain) {
+  void D3D9Surface::AddRefPrivate() {
     if (m_baseTexture != nullptr) {
       D3DRESOURCETYPE type = m_baseTexture->GetType();
       if (type == D3DRTYPE_TEXTURE)
@@ -40,15 +40,11 @@ namespace dxvk {
 
       return;
     }
-    else if (m_container != nullptr && !swapchain) {
-      // Container must be a swapchain if it isn't a base texture.
-      static_cast<D3D9SwapChainEx*>(m_container)->AddRefPrivate();
-    }
 
     D3D9SurfaceBase::AddRefPrivate();
   }
 
-  void D3D9Surface::ReleasePrivate(bool swapchain) {
+  void D3D9Surface::ReleasePrivate() {
     if (m_baseTexture != nullptr) {
       D3DRESOURCETYPE type = m_baseTexture->GetType();
       if (type == D3DRTYPE_TEXTURE)
@@ -58,10 +54,6 @@ namespace dxvk {
 
       return;
     }
-    else if (m_container != nullptr && !swapchain) {
-      // Container must be a swapchain if it isn't a base texture.
-      static_cast<D3D9SwapChainEx*>(m_container)->ReleasePrivate();
-    }
 
     D3D9SurfaceBase::ReleasePrivate();
   }
@@ -188,4 +180,9 @@ namespace dxvk {
     return D3D_OK;
   }
 
+
+  void D3D9Surface::ClearContainer() {
+    m_container = nullptr;
+  }
+
 }
diff --git a/src/d3d9/d3d9_surface.h b/src/d3d9/d3d9_surface.h
index d921af7c3..10086cace 100644
--- a/src/d3d9/d3d9_surface.h
+++ b/src/d3d9/d3d9_surface.h
@@ -29,9 +29,9 @@ namespace dxvk {
             UINT                      MipLevel,
             IDirect3DBaseTexture9*    pBaseTexture);
 
-    void AddRefPrivate(bool swapchain = false);
+    void AddRefPrivate();
 
-    void ReleasePrivate(bool swapchain = false);
+    void ReleasePrivate();
 
     HRESULT STDMETHODCALLTYPE QueryInterface(REFIID riid, void** ppvObject);
 
@@ -56,6 +56,8 @@ namespace dxvk {
       };
     }
 
+    void ClearContainer();
+
   private:
 
     D3D9GDIDesc m_dcDesc;
diff --git a/src/d3d9/d3d9_swapchain.cpp b/src/d3d9/d3d9_swapchain.cpp
index 5fa24565b..bc416cd82 100644
--- a/src/d3d9/d3d9_swapchain.cpp
+++ b/src/d3d9/d3d9_swapchain.cpp
@@ -177,6 +177,8 @@ namespace dxvk {
 
 
   D3D9SwapChainEx::~D3D9SwapChainEx() {
+    DestroyBackBuffers();
+
     ResetWindowProc(m_window);
     RestoreDisplayMode(m_monitor);
 
@@ -463,7 +465,7 @@ namespace dxvk {
       return D3DERR_INVALIDCALL;
     }
 
-    *ppBackBuffer = ref(m_backBuffers[iBackBuffer]);
+    *ppBackBuffer = ref(m_backBuffers[iBackBuffer].ptr());
     return D3D_OK;
   }
 
@@ -712,7 +714,7 @@ namespace dxvk {
     if (iBackBuffer >= m_presentParams.BackBufferCount)
       return nullptr;
 
-    return m_backBuffers[iBackBuffer];
+    return m_backBuffers[iBackBuffer].ptr();
   }
 
 
@@ -876,7 +878,7 @@ namespace dxvk {
     // Rotate swap chain buffers so that the back
     // buffer at index 0 becomes the front buffer.
     for (uint32_t i = 1; i < m_backBuffers.size(); i++)
-      m_backBuffers[i]->Swap(m_backBuffers[i - 1]);
+      m_backBuffers[i]->Swap(m_backBuffers[i - 1].ptr());
 
     m_parent->m_flags.set(D3D9DeviceFlag::DirtyFramebuffer);
   }
@@ -1011,8 +1013,8 @@ namespace dxvk {
 
 
   void D3D9SwapChainEx::DestroyBackBuffers() {
-    for (auto* backBuffer : m_backBuffers)
-      backBuffer->ReleasePrivate(true);
+    for (auto& backBuffer : m_backBuffers)
+      backBuffer->ClearContainer();
 
     m_backBuffers.clear();
   }
@@ -1043,10 +1045,8 @@ namespace dxvk {
     desc.Usage              = D3DUSAGE_RENDERTARGET;
     desc.Discard            = FALSE;
 
-    for (uint32_t i = 0; i < m_backBuffers.size(); i++) {
+    for (uint32_t i = 0; i < m_backBuffers.size(); i++)
       m_backBuffers[i] = new D3D9Surface(m_parent, &desc, this);
-      m_backBuffers[i]->AddRefPrivate(true);
-    }
 
     auto swapImage = m_backBuffers[0]->GetCommonTexture()->GetImage();
 
diff --git a/src/d3d9/d3d9_swapchain.h b/src/d3d9/d3d9_swapchain.h
index bd5178a3c..70d7f2d6f 100644
--- a/src/d3d9/d3d9_swapchain.h
+++ b/src/d3d9/d3d9_swapchain.h
@@ -129,7 +129,7 @@ namespace dxvk {
     DxvkLogicOpState        m_loState;
     DxvkBlendMode           m_blendMode;
 
-    std::vector<D3D9Surface*> m_backBuffers;
+    std::vector<Com<D3D9Surface, false>> m_backBuffers;
     
     RECT                    m_srcRect;
     RECT                    m_dstRect;