[dxgi] Use recursive locking and report fullscreen state changes early

Fixes lockups in fullscreen mode in a certain weeb game.

src/dxgi/dxgi_swapchain.cpp

@@ -81,7 +81,7 @@ namespace dxvk {
   
   
   HRESULT STDMETHODCALLTYPE DxgiSwapChain::GetBuffer(UINT Buffer, REFIID riid, void** ppSurface) {
-    std::lock_guard<std::mutex> lock(m_mutex);
+    std::lock_guard<std::recursive_mutex> lock(m_mutex);
     
     if (Buffer > 0) {
       Logger::err("DxgiSwapChain::GetBuffer: Buffer > 0 not supported");
@@ -112,7 +112,7 @@ namespace dxvk {
     if (pDesc == nullptr)
       return DXGI_ERROR_INVALID_CALL;
     
-    std::lock_guard<std::mutex> lock(m_mutex);
+    std::lock_guard<std::recursive_mutex> lock(m_mutex);
     *pDesc = m_desc;
     return S_OK;
   }
@@ -122,7 +122,7 @@ namespace dxvk {
     if (pStats == nullptr)
       return DXGI_ERROR_INVALID_CALL;
     
-    std::lock_guard<std::mutex> lock(m_mutex);
+    std::lock_guard<std::recursive_mutex> lock(m_mutex);
     *pStats = m_stats;
     return S_OK;
   }
@@ -131,7 +131,7 @@ namespace dxvk {
   HRESULT STDMETHODCALLTYPE DxgiSwapChain::GetFullscreenState(
           BOOL*         pFullscreen,
           IDXGIOutput** ppTarget) {
-    std::lock_guard<std::mutex> lock(m_mutex);
+    std::lock_guard<std::recursive_mutex> lock(m_mutex);
     
     HRESULT hr = S_OK;
     
@@ -149,14 +149,14 @@ namespace dxvk {
     if (pLastPresentCount == nullptr)
       return DXGI_ERROR_INVALID_CALL;
     
-    std::lock_guard<std::mutex> lock(m_mutex);
+    std::lock_guard<std::recursive_mutex> lock(m_mutex);
     *pLastPresentCount = m_stats.PresentCount;
     return S_OK;
   }
   
   
   HRESULT STDMETHODCALLTYPE DxgiSwapChain::Present(UINT SyncInterval, UINT Flags) {
-    std::lock_guard<std::mutex> lock(m_mutex);
+    std::lock_guard<std::recursive_mutex> lock(m_mutex);
     
     try {
       // Submit pending rendering commands
@@ -191,7 +191,7 @@ namespace dxvk {
           UINT        Height,
           DXGI_FORMAT NewFormat,
           UINT        SwapChainFlags) {
-    std::lock_guard<std::mutex> lock(m_mutex);
+    std::lock_guard<std::recursive_mutex> lock(m_mutex);
     
     const VkExtent2D windowSize = GetWindowSize();
     
@@ -214,7 +214,7 @@ namespace dxvk {
     if (pNewTargetParameters == nullptr)
       return DXGI_ERROR_INVALID_CALL;
     
-    std::lock_guard<std::mutex> lock(m_mutex);
+    std::lock_guard<std::recursive_mutex> lock(m_mutex);
     
     // TODO support fullscreen mode
     RECT newRect = { 0, 0, 0, 0 };
@@ -237,7 +237,7 @@ namespace dxvk {
   HRESULT STDMETHODCALLTYPE DxgiSwapChain::SetFullscreenState(
           BOOL          Fullscreen,
           IDXGIOutput*  pTarget) {
-    std::lock_guard<std::mutex> lock(m_mutex);
+    std::lock_guard<std::recursive_mutex> lock(m_mutex);
     
     if (Fullscreen)
       Logger::warn("DxgiSwapChain: Display mode changes not implemented");
@@ -308,6 +308,9 @@ namespace dxvk {
       }
     }
     
+    // Update swap chain description
+    m_desc.Windowed = FALSE;
+    
     // Change the window flags to remove the decoration etc.
     LONG style   = ::GetWindowLongW(m_desc.OutputWindow, GWL_STYLE);
     LONG exstyle = ::GetWindowLongW(m_desc.OutputWindow, GWL_EXSTYLE);
@@ -334,13 +337,13 @@ namespace dxvk {
       rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top,
       SWP_FRAMECHANGED | SWP_SHOWWINDOW | SWP_NOACTIVATE);
     
-    // Update swap chain description
-    m_desc.Windowed = FALSE;
     return S_OK;
   }
   
   
   HRESULT DxgiSwapChain::LeaveFullscreenMode() {
+    m_desc.Windowed = TRUE;
+    
     // FIXME wine only restores window flags if the application
     // has not modified them in the meantime. Some applications
     // may rely on that behaviour.
@@ -353,7 +356,6 @@ namespace dxvk {
       rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top,
       SWP_FRAMECHANGED | SWP_NOZORDER | SWP_NOACTIVATE);
     
-    m_desc.Windowed = TRUE;
     return S_OK;
   }
   

src/dxgi/dxgi_swapchain.h

@@ -88,7 +88,7 @@ namespace dxvk {
       RECT     rect    = { 0, 0, 0, 0 };
     };
     
-    std::mutex m_mutex;
+    std::recursive_mutex m_mutex;
     
     Com<DxgiFactory>                m_factory;
     Com<DxgiAdapter>                m_adapter;