new storage mechanism

src/common/config.cpp

@@ -71,6 +71,7 @@ static bool readbackLinearImagesEnabled = false;
 static bool directMemoryAccessEnabled = false;
 static bool shouldDumpShaders = false;
 static bool shouldPatchShaders = false;
+static bool shaderCachePreloadEnabled = false;
 static u32 vblankDivider = 1;
 static bool isFullscreen = false;
 static std::string fullscreenMode = "Windowed";
@@ -106,7 +107,7 @@ u32 m_language = 1; // english
 static std::string trophyKey = "";
 
 // Expected number of items in the config file
-static constexpr u64 total_entries = 54;
+static constexpr u64 total_entries = 55;
 
 bool allowHDR() {
     return isHDRAllowed;
@@ -289,6 +290,10 @@ bool patchShaders() {
     return shouldPatchShaders;
 }
 
+bool getShaderCachePreloadEnabled() {
+    return shaderCachePreloadEnabled;
+}
+
 bool isRdocEnabled() {
     return rdocEnable;
 }
@@ -405,6 +410,10 @@ void setDumpShaders(bool enable) {
     shouldDumpShaders = enable;
 }
 
+void setShaderCachePreloadEnabled(bool enable) {
+    shaderCachePreloadEnabled = enable;
+}
+
 void setVkValidation(bool enable) {
     vkValidation = enable;
 }
@@ -664,6 +673,7 @@ void load(const std::filesystem::path& path) {
             toml::find_or<bool>(gpu, "directMemoryAccess", directMemoryAccessEnabled);
         shouldDumpShaders = toml::find_or<bool>(gpu, "dumpShaders", shouldDumpShaders);
         shouldPatchShaders = toml::find_or<bool>(gpu, "patchShaders", shouldPatchShaders);
+        shaderCachePreloadEnabled = toml::find_or<bool>(gpu, "shaderCachePreload", shaderCachePreloadEnabled);
         vblankDivider = toml::find_or<int>(gpu, "vblankDivider", vblankDivider);
         isFullscreen = toml::find_or<bool>(gpu, "Fullscreen", isFullscreen);
         fullscreenMode = toml::find_or<std::string>(gpu, "FullscreenMode", fullscreenMode);
@@ -837,6 +847,7 @@ void save(const std::filesystem::path& path) {
     data["GPU"]["directMemoryAccess"] = directMemoryAccessEnabled;
     data["GPU"]["dumpShaders"] = shouldDumpShaders;
     data["GPU"]["patchShaders"] = shouldPatchShaders;
+    data["GPU"]["shaderCachePreload"] = shaderCachePreloadEnabled;
     data["GPU"]["vblankDivider"] = vblankDivider;
     data["GPU"]["Fullscreen"] = isFullscreen;
     data["GPU"]["FullscreenMode"] = fullscreenMode;
@@ -940,6 +951,7 @@ void setDefaultValues() {
     directMemoryAccessEnabled = false;
     shouldDumpShaders = false;
     shouldPatchShaders = false;
+    shaderCachePreloadEnabled = false;
     vblankDivider = 1;
     isFullscreen = false;
     fullscreenMode = "Windowed";

src/common/config.h

@@ -56,6 +56,8 @@ bool directMemoryAccess();
 void setDirectMemoryAccess(bool enable);
 bool dumpShaders();
 void setDumpShaders(bool enable);
+bool getShaderCachePreloadEnabled();
+void setShaderCachePreloadEnabled();
 u32 vblankDiv();
 void setVblankDiv(u32 value);
 bool getisTrophyPopupDisabled();

src/emulator.cpp

@@ -257,10 +257,14 @@ void Emulator::Run(std::filesystem::path file, const std::vector<std::string> ar
     }
     VideoCore::SetOutputDir(mount_captures_dir, id);
 
+    // Initialize shader cache
     if (!std::filesystem::exists(SHADER_CACHE_DIR)) {
         std::filesystem::create_directories(SHADER_CACHE_DIR);
         LOG_INFO(Loader, "Created shader cache directory: {}", SHADER_CACHE_DIR.string());
     }
+    ShaderCache::InitializeShaderCache();
+    LOG_INFO(Loader, "{} shaders in cache {}", ShaderCache::shader_registry.size(), Config::getShaderCachePreloadEnabled() != 0 ? "(preloaded) " : "");
+
     // Initialize kernel and library facilities.
     Libraries::InitHLELibs(&linker->GetHLESymbols());
 

src/video_core/renderer_vulkan/shader_cache.cpp

@@ -17,6 +17,7 @@
 #include "common/io_file.h"
 #include "common/logging/log.h"
 #include "common/path_util.h"
+#include "common/config.h"
 #include "shader_recompiler/info.h"
 #include "shader_recompiler/ir/type.h"
 #include "shader_recompiler/specialization.h"
@@ -243,86 +244,77 @@ u64 CalculateSpecializationHash(const Shader::StageSpecialization& spec) {
 }
 
 bool CheckShaderCache(std::string shader_id) {
-    std::filesystem::path spirv_cache_file_path =
+    if (Config::getShaderCachePreloadEnabled()) {
-        SHADER_CACHE_DIR / static_cast<std::filesystem::path>(shader_id + ".spv");
+        return shader_cache.contains(shader_id);
-    std::filesystem::path resources_file_path =
-        SHADER_CACHE_DIR / static_cast<std::filesystem::path>(shader_id + ".resources");
-    ;
-
-    if (!std::filesystem::exists(spirv_cache_file_path)) {
-        return false;
     }
 
-    if (!std::filesystem::exists(resources_file_path)) {
+    return shader_registry.contains(shader_id);
-        return false;
+}
+
+void InitializeShaderCache() {
+    if (!std::filesystem::exists(SHADER_CACHE_REGISTRY_PATH) || std::filesystem::file_size(SHADER_CACHE_REGISTRY_PATH) == 0) {
+        return;
     }
-
+    std::ifstream registry_file(SHADER_CACHE_REGISTRY_PATH, std::ios::binary);
-    Common::FS::IOFile spirv_file(spirv_cache_file_path, Common::FS::FileAccessMode::Read);
+    cereal::BinaryInputArchive registry_ar(registry_file);
-    Common::FS::IOFile resources_file(resources_file_path, Common::FS::FileAccessMode::Read);
+    while (registry_file.tellg() < std::filesystem::file_size(SHADER_CACHE_REGISTRY_PATH)) {
-
+        std::string shader_key;
-    const bool spirv_valid = spirv_file.IsOpen() && spirv_file.GetSize() > 0;
+        u64 offset;
-    const bool resources_valid = resources_file.IsOpen() && resources_file.GetSize() > 0;
+        registry_ar(shader_key, offset);
-
+        shader_registry[shader_key] = offset;
-    spirv_file.Close();
+    }
-    resources_file.Close();
+    if (Config::getShaderCachePreloadEnabled()) {
-
+        std::ifstream blob_file(SHADER_CACHE_BLOB_PATH, std::ios::binary);
-    if (!spirv_valid || !resources_valid) {
+        for (auto const& [shader_key, offset] : shader_registry) {
-        LOG_WARNING(Render_Vulkan, "Invalid cache file for shader with ID: {}", shader_id);
+            blob_file.seekg(offset, std::ios::beg);
-        if (std::filesystem::exists(spirv_cache_file_path)) {
+            {
-            std::filesystem::remove(spirv_cache_file_path);
+                cereal::BinaryInputArchive blob_ar(blob_file);
+                std::string resources;
+                std::vector<u32> spv;
+                blob_ar(spv, resources);
+                shader_cache[shader_key] = std::make_pair(spv, resources);
+            }
         }
-        if (std::filesystem::exists(resources_file_path)) {
-            std::filesystem::remove(resources_file_path);
-        }
-        return false;
     }
-
-    LOG_INFO(Render_Vulkan, "Found shader with ID {} in the cache", shader_id);
-    return true;
 }
 
 void GetShader(std::string shader_id, Shader::Info& info, std::vector<u32>& spv) {
-    std::filesystem::path spirv_cache_filename = shader_id + ".spv";
+    std::string resources;
-    std::filesystem::path spirv_cache_file_path = SHADER_CACHE_DIR / spirv_cache_filename;
+    if (Config::getShaderCachePreloadEnabled()) {
-    Common::FS::IOFile spirv_cache_file(spirv_cache_file_path, Common::FS::FileAccessMode::Read);
+        auto& [spv_cached, resources] = shader_cache[shader_id];
-    spv.resize(spirv_cache_file.GetSize() / sizeof(u32));
+        spv = spv_cached;
-    spirv_cache_file.Read(spv);
+    }
-    spirv_cache_file.Close();
+    else {
+        std::ifstream blob_file(SHADER_CACHE_BLOB_PATH, std::ios::binary);
+        blob_file.seekg(shader_registry[shader_id], std::ios::beg);
+        cereal::BinaryInputArchive ar(blob_file);
 
-    std::filesystem::path resource_dump_filename = shader_id + ".resources";
+        ar(spv, resources);
-    std::filesystem::path resources_dump_file_path = SHADER_CACHE_DIR / resource_dump_filename;
-    Common::FS::IOFile resources_dump_file(resources_dump_file_path,
-                                           Common::FS::FileAccessMode::Read);
-
-    std::vector<char> resources_data;
-    resources_data.resize(resources_dump_file.GetSize());
-    resources_dump_file.Read(resources_data);
-    resources_dump_file.Close();
-
-    std::istringstream combined_stream(std::string(resources_data.begin(), resources_data.end()));
-
-    std::istringstream info_stream;
-    info_stream.str(std::string(resources_data.begin(), resources_data.end()));
-}
-
-void AddShader(std::string shader_id, std::vector<u32> spv, std::ostream& info_serialized) {
-    std::filesystem::path spirv_cache_filename = shader_id + ".spv";
-    std::filesystem::path spirv_cache_file_path = SHADER_CACHE_DIR / spirv_cache_filename;
-    Common::FS::IOFile shader_cache_file(spirv_cache_file_path, Common::FS::FileAccessMode::Write);
-    shader_cache_file.WriteSpan(std::span<const u32>(spv));
-    shader_cache_file.Close();
-
-    std::filesystem::path resource_dump_filename = shader_id + ".resources";
-    std::filesystem::path resources_dump_file_path = SHADER_CACHE_DIR / resource_dump_filename;
-    Common::FS::IOFile resources_dump_file(resources_dump_file_path,
-                                           Common::FS::FileAccessMode::Write);
-
-    if (std::ostringstream* info_oss = dynamic_cast<std::ostringstream*>(&info_serialized)) {
-        std::string info_data = info_oss->str();
-        resources_dump_file.WriteSpan(std::span<const char>(info_data.data(), info_data.size()));
     }
 
-    resources_dump_file.Close();
+    std::istringstream info_serialized(resources);
+    DeserializeInfo(info_serialized, info);
+}
+
+void AddShader(std::string shader_id, std::vector<u32> spv, std::ostringstream& info_serialized) {
+    std::ofstream registry_file(SHADER_CACHE_REGISTRY_PATH, std::ios::binary | std::ios::app);
+    registry_file.seekp(0, std::ios::end);
+    cereal::BinaryOutputArchive reg_ar(registry_file);
+
+    std::ofstream blob_file(SHADER_CACHE_BLOB_PATH, std::ios::binary | std::ios::app);
+    blob_file.seekp(0, std::ios::end);
+    cereal::BinaryOutputArchive blob_ar(blob_file);
+
+    u64 offset = static_cast<u64>(blob_file.tellp());
+    reg_ar(shader_id, offset);
+
+    std::string info_blob = info_serialized.str();
+    blob_ar(spv, info_blob);
+
+    shader_registry[shader_id] = offset;
+    if (Config::getShaderCachePreloadEnabled()) {
+        shader_cache[shader_id] = std::make_pair(spv, info_blob);
+    }
 }
 
 void SerializeInfo(std::ostream& info_serialized, Shader::Info& info) {
@@ -334,8 +326,6 @@ void SerializeInfo(std::ostream& info_serialized, Shader::Info& info) {
     ar << info.images;
     ar << info.samplers;
     ar << info.fmasks;
-    // srt info
-    ar << info.flattened_ud_buf;
     ar << info.fs_interpolation;
 }
 
@@ -348,8 +338,6 @@ void DeserializeInfo(std::istream& info_serialized, Shader::Info& info) {
     ar >> info.images;
     ar >> info.samplers;
     ar >> info.fmasks;
-    // srt info
-    ar >> info.flattened_ud_buf;
     ar >> info.fs_interpolation;
 }
 

src/video_core/renderer_vulkan/shader_cache.h

@@ -6,20 +6,28 @@
 #include <iostream>
 #include <string>
 #include <vector>
+
 #include "shader_recompiler/info.h"
-#include <shader_recompiler/specialization.h>
+#include "shader_recompiler/specialization.h"
+#include "common/elf_info.h"
 
 namespace ShaderCache {
 
 #define SHADER_CACHE_DIR (Common::FS::GetUserPath(Common::FS::PathType::ShaderDir) / "cache" / "portable")
+#define SHADER_CACHE_BLOB_PATH (SHADER_CACHE_DIR / (std::string{Common::ElfInfo::Instance().GameSerial()} + "_shaders.bin"))
+#define SHADER_CACHE_REGISTRY_PATH (SHADER_CACHE_DIR / (std::string{Common::ElfInfo::Instance().GameSerial()} + "_shaders_registry.bin"))
+
+inline std::map<std::string, u64> shader_registry; // shader_key:offset
+inline std::map<std::string, std::pair<std::vector<u32>, std::string>> shader_cache; // only used when preload active // shader_key:blob
 
 u64 CalculateSpecializationHash(const Shader::StageSpecialization& spec);
+void InitializeShaderCache();
 void SerializeInfo(
     std::ostream& info_serialized, Shader::Info& info);
 void DeserializeInfo(std::istream& info_serialized, Shader::Info& info);
 
 bool CheckShaderCache(std::string shader_id);
 void GetShader(std::string shader_id, Shader::Info& info, std::vector<u32>& spv);
-void AddShader(std::string shader_id, std::vector<u32> spv, std::ostream& info_serialized);
+void AddShader(std::string shader_id, std::vector<u32> spv, std::ostringstream& info_serialized);
 
 } // namespace ShaderCache

src/video_core/renderer_vulkan/shader_cache_serialization.h

@@ -7,6 +7,7 @@
 #include <cereal/types/utility.hpp>
 #include <cereal/types/variant.hpp>
 #include <cereal/types/vector.hpp>
+#include <cereal/types/string.hpp>
 #include "common/serialization.h"
 #include "shader_recompiler/info.h"