diff --git a/src/d3d11/d3d11_swapchain.cpp b/src/d3d11/d3d11_swapchain.cpp
index f6032841c..7fbebd618 100644
--- a/src/d3d11/d3d11_swapchain.cpp
+++ b/src/d3d11/d3d11_swapchain.cpp
@@ -38,9 +38,6 @@ namespace dxvk {
     m_device->waitForSubmission(&m_presentStatus);
     m_device->waitForIdle();
     
-    if (m_backBuffer)
-      m_backBuffer->ReleasePrivate();
-
     DestroyFrameLatencyEvent();
   }
 
@@ -456,9 +453,6 @@ namespace dxvk {
   void D3D11SwapChain::CreateBackBuffer() {
     // Explicitly destroy current swap image before
     // creating a new one to free up resources
-    if (m_backBuffer)
-      m_backBuffer->ReleasePrivate();
-    
     m_swapImage         = nullptr;
     m_swapImageView     = nullptr;
     m_backBuffer        = nullptr;
@@ -497,9 +491,7 @@ namespace dxvk {
       dxgiUsage |= DXGI_USAGE_DISCARD_ON_PRESENT;
 
     m_backBuffer = new D3D11Texture2D(m_parent, &desc, dxgiUsage, VK_NULL_HANDLE);
-    m_backBuffer->AddRefPrivate();
-
-    m_swapImage = GetCommonTexture(m_backBuffer)->GetImage();
+    m_swapImage = GetCommonTexture(m_backBuffer.ptr())->GetImage();
 
     // Create an image view that allows the
     // image to be bound as a shader resource.
diff --git a/src/d3d11/d3d11_swapchain.h b/src/d3d11/d3d11_swapchain.h
index 8fc9fe10c..f4debea14 100644
--- a/src/d3d11/d3d11_swapchain.h
+++ b/src/d3d11/d3d11_swapchain.h
@@ -98,7 +98,7 @@ namespace dxvk {
 
     Rc<hud::Hud>              m_hud;
 
-    D3D11Texture2D*           m_backBuffer = nullptr;
+    Com<D3D11Texture2D, false> m_backBuffer;
     DxvkSubmitStatus          m_presentStatus;
 
     std::vector<Rc<DxvkImageView>> m_imageViews;