Shader cache (#188)

* [util] Adds getTempDirectory() function

Will be used by on-disk pipeline caching

* [dxvk] Implement on-disk shader caching

Saving the pipeline cache to disk when the application exits
should be sufficient but the DxvkPipelineCache destructor isn't
reliably called on exit (ref-counting issue?).
As a workaround every frame we check and save the cache if the
size increased by some amount or after one minute elapsed.

* [dxvk] Periodically update shader cache file in separate thread

src/dxvk/dxvk_compute.cpp

@@ -98,6 +98,7 @@ namespace dxvk {
       return VK_NULL_HANDLE;
     }
     
+    m_cache->update();
     return pipeline;
   }
   

src/dxvk/dxvk_device.cpp

@@ -250,4 +250,4 @@ namespace dxvk {
     m_recycledCommandLists.returnObject(cmdList);
   }
   
-}
+}

src/dxvk/dxvk_graphics.cpp

@@ -260,6 +260,7 @@ namespace dxvk {
       return VK_NULL_HANDLE;
     }
     
+    m_cache->update();
     return pipeline;
   }
   

src/dxvk/dxvk_pipecache.cpp

@@ -4,14 +4,19 @@ namespace dxvk {
   
   DxvkPipelineCache::DxvkPipelineCache(
     const Rc<vk::DeviceFn>& vkd)
-  : m_vkd(vkd) {
-    // TODO read pipeline cache from file
+  : m_vkd           (vkd),
+    m_fileName      (getFileName()),
+    m_updateStop    (0),
+    m_updateCounter (0),
+    m_updateThread  ([this] { this->runThread(); }) {
+    auto initialData = this->loadPipelineCache();
+    
     VkPipelineCacheCreateInfo info;
     info.sType            = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO;
     info.pNext            = nullptr;
     info.flags            = 0;
-    info.initialDataSize  = 0;
-    info.pInitialData     = nullptr;
+    info.initialDataSize  = initialData.size();
+    info.pInitialData     = initialData.data();
     
     if (m_vkd->vkCreatePipelineCache(m_vkd->device(),
         &info, nullptr, &m_handle) != VK_SUCCESS)
@@ -20,8 +25,157 @@ namespace dxvk {
   
   
   DxvkPipelineCache::~DxvkPipelineCache() {
+    // Stop the shader cache update thread
+    { std::unique_lock<std::mutex> lock(m_updateMutex);
+      m_updateStop = -1;
+      m_updateCond.notify_one();
+    }
+    
+    m_updateThread.join();
+    
+    // Make sure we store the full cache again
+    this->storePipelineCache(
+      this->getPipelineCache());
+    
     m_vkd->vkDestroyPipelineCache(
       m_vkd->device(), m_handle, nullptr);
   }
   
-}
+  
+  void DxvkPipelineCache::update() {
+    m_updateCounter += 1;
+  }
+  
+  
+  void DxvkPipelineCache::runThread() {
+    uint32_t prevUpdateCounter = 0;
+    uint32_t currUpdateCounter = 0;
+    
+    while (true) {
+      // Periodically check whether we need to update once a minute.
+      // We don't want to write the full pipeline cache with every
+      // single update because that would cause unnecessary load.
+      { std::unique_lock<std::mutex> lock(m_updateMutex);
+        
+        bool exit = m_updateCond.wait_for(lock,
+          std::chrono::seconds(60),
+          [this] { return m_updateStop != 0; });
+        
+        if (exit)
+          return;
+      }
+      
+      // Only update the cache if new pipelines
+      // have been created in the meantime
+      currUpdateCounter = m_updateCounter.load();
+      
+      if (currUpdateCounter != prevUpdateCounter) {
+        prevUpdateCounter = currUpdateCounter;
+        this->storePipelineCache(this->getPipelineCache());
+      }
+    }
+  }
+  
+  
+  std::vector<char> DxvkPipelineCache::getPipelineCache() const {
+    std::vector<char> cacheData;
+    
+    VkResult status = VK_INCOMPLETE;
+    
+    while (status == VK_INCOMPLETE) {
+      size_t cacheSize = 0;
+      
+      if (m_vkd->vkGetPipelineCacheData(m_vkd->device(),
+            m_handle, &cacheSize, nullptr) != VK_SUCCESS) {
+        Logger::warn("DxvkPipelineCache: Failed to retrieve cache size");
+        break;
+      }
+      
+      cacheData.resize(cacheSize);
+      
+      status = m_vkd->vkGetPipelineCacheData(m_vkd->device(),
+        m_handle, &cacheSize, cacheData.data());
+    }
+    
+    if (status != VK_SUCCESS) {
+      Logger::warn("DxvkPipelineCache: Failed to retrieve cache data");
+      cacheData.resize(0);
+    }
+    
+    return cacheData;
+  }
+  
+  
+  std::vector<char> DxvkPipelineCache::loadPipelineCache() const {
+    std::vector<char> cacheData;
+    
+    if (m_fileName.size() == 0) {
+      Logger::warn("DxvkPipelineCache: Failed to locate cache file");
+      return cacheData;
+    }
+    
+    auto cacheFile = std::ifstream(m_fileName,
+      std::ios_base::binary | std::ios_base::ate);
+    
+    if (!cacheFile.good()) {
+      Logger::warn("DxvkPipelineCache: Failed to read cache file");
+      return cacheData;
+    }
+    
+    std::streamsize cacheSize = cacheFile.tellg();
+    cacheFile.seekg(0, std::ios_base::beg);
+    cacheData.resize(cacheSize);
+    
+    if (!cacheFile.read(cacheData.data(), cacheData.size())) {
+      Logger::warn("DxvkPipelineCache: Failed to read cache file");
+      cacheData.resize(0);
+    }
+    
+    Logger::debug(str::format(
+      "DxvkPipelineCache: Read ", cacheData.size(),
+      " bytes from ", m_fileName));
+    return cacheData;
+  }
+  
+  
+  void DxvkPipelineCache::storePipelineCache(const std::vector<char>& cacheData) const {
+    if (m_fileName.size() == 0) {
+      Logger::warn("DxvkPipelineCache: Failed to locate cache file");
+      return;
+    }
+    
+    auto cacheFile = std::ofstream(m_fileName,
+      std::ios_base::binary | std::ios_base::trunc);
+    
+    if (!cacheFile.good()) {
+      Logger::warn("DxvkPipelineCache: Failed to open cache file");
+      return;
+    }
+    
+    cacheFile.write(cacheData.data(), cacheData.size());
+    
+    if (!cacheFile.good()) {
+      Logger::warn("DxvkPipelineCache: Failed to write shader cache file");
+      return;
+    }
+    
+    Logger::debug(str::format(
+      "DxvkPipelineCache: Wrote ", cacheData.size(),
+      " bytes to ", m_fileName));
+  }
+  
+  
+  std::string DxvkPipelineCache::getFileName() {
+    const auto exeName = env::getExeName();
+    const auto filename = Sha1Hash::compute(
+      reinterpret_cast<const uint8_t*>(exeName.c_str()), exeName.size());
+    
+    const auto temp = env::getTempDirectory();
+    
+    if (temp.size() != 0)
+      return str::format(temp, filename.toString(), ".pipecache");
+    
+    return std::string();
+  }
+  
+}

src/dxvk/dxvk_pipecache.h

@@ -1,7 +1,16 @@
 #pragma once
 
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <fstream>
+#include <thread>
+
 #include "dxvk_include.h"
 
+#include "../util/sha1/sha1_util.h"
+#include "../util/util_env.h"
+
 namespace dxvk {
   
   /**
@@ -25,11 +34,39 @@ namespace dxvk {
       return m_handle;
     }
     
+    /**
+     * \brief Sends an update signal
+     * 
+     * Notifies the update thread that the
+     * pipeline cache should be updated.
+     */
+    void update();
+    
   private:
     
-    Rc<vk::DeviceFn> m_vkd;
-    VkPipelineCache  m_handle;
+    Rc<vk::DeviceFn>        m_vkd;
+    VkPipelineCache         m_handle;
+    
+    std::string             m_fileName;
+    
+    std::atomic<uint32_t>   m_updateStop;
+    std::atomic<uint32_t>   m_updateCounter;
+    
+    std::mutex              m_updateMutex;
+    std::condition_variable m_updateCond;
+    std::thread             m_updateThread;
+    
+    void runThread();
+    
+    std::vector<char> getPipelineCache() const;
+    
+    std::vector<char> loadPipelineCache() const;
+    
+    void storePipelineCache(
+      const std::vector<char>& cacheData) const;
+    
+    static std::string getFileName();
     
   };
   
-}
+}

src/util/util_env.cpp

@@ -35,4 +35,20 @@ namespace dxvk::env {
       : fullPath;
   }
   
+  
+  std::string getTempDirectory() {
+    WCHAR windowsTempDir[MAX_PATH] = {0};
+    UINT ret = ::GetTempPathW(MAX_PATH, windowsTempDir);
+    if (ret > MAX_PATH || ret == 0)
+      return std::string();
+    
+    auto dxvkTempDir = std::wstring(windowsTempDir) + L"dxvk\\";
+    if (::CreateDirectoryW(dxvkTempDir.c_str(), 0) == 0) {
+      if (::GetLastError() != ERROR_ALREADY_EXISTS)
+        return std::string();
+    }
+    
+    return str::fromws(dxvkTempDir);
+  }
+  
 }

src/util/util_env.h

@@ -25,4 +25,13 @@ namespace dxvk::env {
    */
   std::string getExeName();
   
-}
+  /**
+   * \brief Gets the path to the dxvk specific temporary directory
+   * 
+   * Returns the path to the temporary directory for dxvk.
+   * If no such directory can be found the string will be empty.
+   * \returns Temporary directory
+   */
+  std::string getTempDirectory();
+  
+}