From da818336666f4851b0ac45b3f925e9f534a3c78a Mon Sep 17 00:00:00 2001 From: Fire Cube Date: Fri, 18 Jul 2025 21:20:28 +0200 Subject: [PATCH] new storage mechanism --- src/common/config.cpp | 14 +- src/common/config.h | 2 + src/emulator.cpp | 4 + .../renderer_vulkan/shader_cache.cpp | 134 ++++++++---------- src/video_core/renderer_vulkan/shader_cache.h | 12 +- .../shader_cache_serialization.h | 1 + 6 files changed, 91 insertions(+), 76 deletions(-) diff --git a/src/common/config.cpp b/src/common/config.cpp index 6f8563377..4f298ba54 100644 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -71,6 +71,7 @@ static bool readbackLinearImagesEnabled = false; static bool directMemoryAccessEnabled = false; static bool shouldDumpShaders = false; static bool shouldPatchShaders = false; +static bool shaderCachePreloadEnabled = false; static u32 vblankDivider = 1; static bool isFullscreen = false; static std::string fullscreenMode = "Windowed"; @@ -106,7 +107,7 @@ u32 m_language = 1; // english static std::string trophyKey = ""; // Expected number of items in the config file -static constexpr u64 total_entries = 54; +static constexpr u64 total_entries = 55; bool allowHDR() { return isHDRAllowed; @@ -289,6 +290,10 @@ bool patchShaders() { return shouldPatchShaders; } +bool getShaderCachePreloadEnabled() { + return shaderCachePreloadEnabled; +} + bool isRdocEnabled() { return rdocEnable; } @@ -405,6 +410,10 @@ void setDumpShaders(bool enable) { shouldDumpShaders = enable; } +void setShaderCachePreloadEnabled(bool enable) { + shaderCachePreloadEnabled = enable; +} + void setVkValidation(bool enable) { vkValidation = enable; } @@ -664,6 +673,7 @@ void load(const std::filesystem::path& path) { toml::find_or(gpu, "directMemoryAccess", directMemoryAccessEnabled); shouldDumpShaders = toml::find_or(gpu, "dumpShaders", shouldDumpShaders); shouldPatchShaders = toml::find_or(gpu, "patchShaders", shouldPatchShaders); + shaderCachePreloadEnabled = toml::find_or(gpu, "shaderCachePreload", shaderCachePreloadEnabled); vblankDivider = toml::find_or(gpu, "vblankDivider", vblankDivider); isFullscreen = toml::find_or(gpu, "Fullscreen", isFullscreen); fullscreenMode = toml::find_or(gpu, "FullscreenMode", fullscreenMode); @@ -837,6 +847,7 @@ void save(const std::filesystem::path& path) { data["GPU"]["directMemoryAccess"] = directMemoryAccessEnabled; data["GPU"]["dumpShaders"] = shouldDumpShaders; data["GPU"]["patchShaders"] = shouldPatchShaders; + data["GPU"]["shaderCachePreload"] = shaderCachePreloadEnabled; data["GPU"]["vblankDivider"] = vblankDivider; data["GPU"]["Fullscreen"] = isFullscreen; data["GPU"]["FullscreenMode"] = fullscreenMode; @@ -940,6 +951,7 @@ void setDefaultValues() { directMemoryAccessEnabled = false; shouldDumpShaders = false; shouldPatchShaders = false; + shaderCachePreloadEnabled = false; vblankDivider = 1; isFullscreen = false; fullscreenMode = "Windowed"; diff --git a/src/common/config.h b/src/common/config.h index e54425676..8cbe1b627 100644 --- a/src/common/config.h +++ b/src/common/config.h @@ -56,6 +56,8 @@ bool directMemoryAccess(); void setDirectMemoryAccess(bool enable); bool dumpShaders(); void setDumpShaders(bool enable); +bool getShaderCachePreloadEnabled(); +void setShaderCachePreloadEnabled(); u32 vblankDiv(); void setVblankDiv(u32 value); bool getisTrophyPopupDisabled(); diff --git a/src/emulator.cpp b/src/emulator.cpp index 705060252..8553dba7f 100644 --- a/src/emulator.cpp +++ b/src/emulator.cpp @@ -257,10 +257,14 @@ void Emulator::Run(std::filesystem::path file, const std::vector ar } VideoCore::SetOutputDir(mount_captures_dir, id); + // Initialize shader cache if (!std::filesystem::exists(SHADER_CACHE_DIR)) { std::filesystem::create_directories(SHADER_CACHE_DIR); LOG_INFO(Loader, "Created shader cache directory: {}", SHADER_CACHE_DIR.string()); } + ShaderCache::InitializeShaderCache(); + LOG_INFO(Loader, "{} shaders in cache {}", ShaderCache::shader_registry.size(), Config::getShaderCachePreloadEnabled() != 0 ? "(preloaded) " : ""); + // Initialize kernel and library facilities. Libraries::InitHLELibs(&linker->GetHLESymbols()); diff --git a/src/video_core/renderer_vulkan/shader_cache.cpp b/src/video_core/renderer_vulkan/shader_cache.cpp index 903b5dc33..01edfd9dc 100644 --- a/src/video_core/renderer_vulkan/shader_cache.cpp +++ b/src/video_core/renderer_vulkan/shader_cache.cpp @@ -17,6 +17,7 @@ #include "common/io_file.h" #include "common/logging/log.h" #include "common/path_util.h" +#include "common/config.h" #include "shader_recompiler/info.h" #include "shader_recompiler/ir/type.h" #include "shader_recompiler/specialization.h" @@ -243,86 +244,77 @@ u64 CalculateSpecializationHash(const Shader::StageSpecialization& spec) { } bool CheckShaderCache(std::string shader_id) { - std::filesystem::path spirv_cache_file_path = - SHADER_CACHE_DIR / static_cast(shader_id + ".spv"); - std::filesystem::path resources_file_path = - SHADER_CACHE_DIR / static_cast(shader_id + ".resources"); - ; - - if (!std::filesystem::exists(spirv_cache_file_path)) { - return false; + if (Config::getShaderCachePreloadEnabled()) { + return shader_cache.contains(shader_id); } - if (!std::filesystem::exists(resources_file_path)) { - return false; + return shader_registry.contains(shader_id); +} + +void InitializeShaderCache() { + if (!std::filesystem::exists(SHADER_CACHE_REGISTRY_PATH) || std::filesystem::file_size(SHADER_CACHE_REGISTRY_PATH) == 0) { + return; } - - Common::FS::IOFile spirv_file(spirv_cache_file_path, Common::FS::FileAccessMode::Read); - Common::FS::IOFile resources_file(resources_file_path, Common::FS::FileAccessMode::Read); - - const bool spirv_valid = spirv_file.IsOpen() && spirv_file.GetSize() > 0; - const bool resources_valid = resources_file.IsOpen() && resources_file.GetSize() > 0; - - spirv_file.Close(); - resources_file.Close(); - - if (!spirv_valid || !resources_valid) { - LOG_WARNING(Render_Vulkan, "Invalid cache file for shader with ID: {}", shader_id); - if (std::filesystem::exists(spirv_cache_file_path)) { - std::filesystem::remove(spirv_cache_file_path); + std::ifstream registry_file(SHADER_CACHE_REGISTRY_PATH, std::ios::binary); + cereal::BinaryInputArchive registry_ar(registry_file); + while (registry_file.tellg() < std::filesystem::file_size(SHADER_CACHE_REGISTRY_PATH)) { + std::string shader_key; + u64 offset; + registry_ar(shader_key, offset); + shader_registry[shader_key] = offset; + } + if (Config::getShaderCachePreloadEnabled()) { + std::ifstream blob_file(SHADER_CACHE_BLOB_PATH, std::ios::binary); + for (auto const& [shader_key, offset] : shader_registry) { + blob_file.seekg(offset, std::ios::beg); + { + cereal::BinaryInputArchive blob_ar(blob_file); + std::string resources; + std::vector spv; + blob_ar(spv, resources); + shader_cache[shader_key] = std::make_pair(spv, resources); + } } - if (std::filesystem::exists(resources_file_path)) { - std::filesystem::remove(resources_file_path); - } - return false; } - - LOG_INFO(Render_Vulkan, "Found shader with ID {} in the cache", shader_id); - return true; } void GetShader(std::string shader_id, Shader::Info& info, std::vector& spv) { - std::filesystem::path spirv_cache_filename = shader_id + ".spv"; - std::filesystem::path spirv_cache_file_path = SHADER_CACHE_DIR / spirv_cache_filename; - Common::FS::IOFile spirv_cache_file(spirv_cache_file_path, Common::FS::FileAccessMode::Read); - spv.resize(spirv_cache_file.GetSize() / sizeof(u32)); - spirv_cache_file.Read(spv); - spirv_cache_file.Close(); + std::string resources; + if (Config::getShaderCachePreloadEnabled()) { + auto& [spv_cached, resources] = shader_cache[shader_id]; + spv = spv_cached; + } + else { + std::ifstream blob_file(SHADER_CACHE_BLOB_PATH, std::ios::binary); + blob_file.seekg(shader_registry[shader_id], std::ios::beg); + cereal::BinaryInputArchive ar(blob_file); - std::filesystem::path resource_dump_filename = shader_id + ".resources"; - std::filesystem::path resources_dump_file_path = SHADER_CACHE_DIR / resource_dump_filename; - Common::FS::IOFile resources_dump_file(resources_dump_file_path, - Common::FS::FileAccessMode::Read); - - std::vector resources_data; - resources_data.resize(resources_dump_file.GetSize()); - resources_dump_file.Read(resources_data); - resources_dump_file.Close(); - - std::istringstream combined_stream(std::string(resources_data.begin(), resources_data.end())); - - std::istringstream info_stream; - info_stream.str(std::string(resources_data.begin(), resources_data.end())); -} - -void AddShader(std::string shader_id, std::vector spv, std::ostream& info_serialized) { - std::filesystem::path spirv_cache_filename = shader_id + ".spv"; - std::filesystem::path spirv_cache_file_path = SHADER_CACHE_DIR / spirv_cache_filename; - Common::FS::IOFile shader_cache_file(spirv_cache_file_path, Common::FS::FileAccessMode::Write); - shader_cache_file.WriteSpan(std::span(spv)); - shader_cache_file.Close(); - - std::filesystem::path resource_dump_filename = shader_id + ".resources"; - std::filesystem::path resources_dump_file_path = SHADER_CACHE_DIR / resource_dump_filename; - Common::FS::IOFile resources_dump_file(resources_dump_file_path, - Common::FS::FileAccessMode::Write); - - if (std::ostringstream* info_oss = dynamic_cast(&info_serialized)) { - std::string info_data = info_oss->str(); - resources_dump_file.WriteSpan(std::span(info_data.data(), info_data.size())); + ar(spv, resources); } - resources_dump_file.Close(); + std::istringstream info_serialized(resources); + DeserializeInfo(info_serialized, info); +} + +void AddShader(std::string shader_id, std::vector spv, std::ostringstream& info_serialized) { + std::ofstream registry_file(SHADER_CACHE_REGISTRY_PATH, std::ios::binary | std::ios::app); + registry_file.seekp(0, std::ios::end); + cereal::BinaryOutputArchive reg_ar(registry_file); + + std::ofstream blob_file(SHADER_CACHE_BLOB_PATH, std::ios::binary | std::ios::app); + blob_file.seekp(0, std::ios::end); + cereal::BinaryOutputArchive blob_ar(blob_file); + + u64 offset = static_cast(blob_file.tellp()); + reg_ar(shader_id, offset); + + std::string info_blob = info_serialized.str(); + blob_ar(spv, info_blob); + + shader_registry[shader_id] = offset; + if (Config::getShaderCachePreloadEnabled()) { + shader_cache[shader_id] = std::make_pair(spv, info_blob); + } } void SerializeInfo(std::ostream& info_serialized, Shader::Info& info) { @@ -334,8 +326,6 @@ void SerializeInfo(std::ostream& info_serialized, Shader::Info& info) { ar << info.images; ar << info.samplers; ar << info.fmasks; - // srt info - ar << info.flattened_ud_buf; ar << info.fs_interpolation; } @@ -348,8 +338,6 @@ void DeserializeInfo(std::istream& info_serialized, Shader::Info& info) { ar >> info.images; ar >> info.samplers; ar >> info.fmasks; - // srt info - ar >> info.flattened_ud_buf; ar >> info.fs_interpolation; } diff --git a/src/video_core/renderer_vulkan/shader_cache.h b/src/video_core/renderer_vulkan/shader_cache.h index 51b2b17ed..2d8a3818c 100644 --- a/src/video_core/renderer_vulkan/shader_cache.h +++ b/src/video_core/renderer_vulkan/shader_cache.h @@ -6,20 +6,28 @@ #include #include #include + #include "shader_recompiler/info.h" -#include +#include "shader_recompiler/specialization.h" +#include "common/elf_info.h" namespace ShaderCache { #define SHADER_CACHE_DIR (Common::FS::GetUserPath(Common::FS::PathType::ShaderDir) / "cache" / "portable") +#define SHADER_CACHE_BLOB_PATH (SHADER_CACHE_DIR / (std::string{Common::ElfInfo::Instance().GameSerial()} + "_shaders.bin")) +#define SHADER_CACHE_REGISTRY_PATH (SHADER_CACHE_DIR / (std::string{Common::ElfInfo::Instance().GameSerial()} + "_shaders_registry.bin")) + +inline std::map shader_registry; // shader_key:offset +inline std::map, std::string>> shader_cache; // only used when preload active // shader_key:blob u64 CalculateSpecializationHash(const Shader::StageSpecialization& spec); +void InitializeShaderCache(); void SerializeInfo( std::ostream& info_serialized, Shader::Info& info); void DeserializeInfo(std::istream& info_serialized, Shader::Info& info); bool CheckShaderCache(std::string shader_id); void GetShader(std::string shader_id, Shader::Info& info, std::vector& spv); -void AddShader(std::string shader_id, std::vector spv, std::ostream& info_serialized); +void AddShader(std::string shader_id, std::vector spv, std::ostringstream& info_serialized); } // namespace ShaderCache diff --git a/src/video_core/renderer_vulkan/shader_cache_serialization.h b/src/video_core/renderer_vulkan/shader_cache_serialization.h index b1a5aaf5c..d83230efa 100644 --- a/src/video_core/renderer_vulkan/shader_cache_serialization.h +++ b/src/video_core/renderer_vulkan/shader_cache_serialization.h @@ -7,6 +7,7 @@ #include #include #include +#include #include "common/serialization.h" #include "shader_recompiler/info.h"