rdoc: manual capture trigger

src/common/config.cpp

@@ -25,6 +25,7 @@ static bool shouldDumpPM4 = false;
 static u32 vblankDivider = 1;
 static bool vkValidation = false;
 static bool vkValidationSync = false;
+static bool rdocEnable = false;
 // Gui
 std::string settings_install_dir = "";
 u32 main_window_geometry_x = 400;
@@ -95,6 +96,10 @@ bool dumpPM4() {
     return shouldDumpPM4;
 }
 
+bool isRdocEnabled() {
+    return rdocEnable;
+}
+
 u32 vblankDiv() {
     return vblankDivider;
 }
@@ -238,7 +243,6 @@ void load(const std::filesystem::path& path) {
 
             screenWidth = toml::find_or<toml::integer>(gpu, "screenWidth", screenWidth);
             screenHeight = toml::find_or<toml::integer>(gpu, "screenHeight", screenHeight);
-            gpuId = toml::find_or<toml::integer>(gpu, "gpuId", 0);
             isNullGpu = toml::find_or<toml::boolean>(gpu, "nullGpu", false);
             shouldDumpShaders = toml::find_or<toml::boolean>(gpu, "dumpShaders", false);
             shouldDumpPM4 = toml::find_or<toml::boolean>(gpu, "dumpPM4", false);
@@ -250,8 +254,10 @@ void load(const std::filesystem::path& path) {
         if (vkResult.is_ok()) {
             auto vk = vkResult.unwrap();
 
+            gpuId = toml::find_or<toml::integer>(vk, "gpuId", 0);
             vkValidation = toml::find_or<toml::boolean>(vk, "validation", true);
             vkValidationSync = toml::find_or<toml::boolean>(vk, "validation_sync", true);
+            rdocEnable = toml::find_or<toml::boolean>(vk, "rdocEnable", false);
         }
     }
     if (data.contains("Debug")) {
@@ -318,15 +324,16 @@ void save(const std::filesystem::path& path) {
     data["General"]["logFilter"] = logFilter;
     data["General"]["logType"] = logType;
     data["General"]["showSplash"] = isShowSplash;
-    data["GPU"]["gpuId"] = gpuId;
     data["GPU"]["screenWidth"] = screenWidth;
     data["GPU"]["screenHeight"] = screenHeight;
     data["GPU"]["nullGpu"] = isNullGpu;
     data["GPU"]["dumpShaders"] = shouldDumpShaders;
     data["GPU"]["dumpPM4"] = shouldDumpPM4;
     data["GPU"]["vblankDivider"] = vblankDivider;
+    data["Vulkan"]["gpuId"] = gpuId;
     data["Vulkan"]["validation"] = vkValidation;
     data["Vulkan"]["validation_sync"] = vkValidationSync;
+    data["Vulkan"]["rdocEnable"] = rdocEnable;
     data["Debug"]["DebugDump"] = isDebugDump;
     data["LLE"]["libc"] = isLibc;
     data["GUI"]["theme"] = mw_themes;

src/common/config.h

@@ -26,6 +26,7 @@ bool showSplash();
 bool nullGpu();
 bool dumpShaders();
 bool dumpPM4();
+bool isRdocEnabled();
 u32 vblankDiv();
 
 bool vkValidationEnabled();

src/common/path_util.cpp

@@ -72,6 +72,7 @@ static auto UserPaths = [] {
     create_path(PathType::GameDataDir, user_dir / GAMEDATA_DIR);
     create_path(PathType::TempDataDir, user_dir / TEMPDATA_DIR);
     create_path(PathType::SysModuleDir, user_dir / SYSMODULES_DIR);
+    create_path(PathType::CapturesDir, user_dir / CAPTURES_DIR);
 
     return paths;
 }();

src/common/path_util.h

@@ -18,6 +18,7 @@ enum class PathType {
     TempDataDir,    // Where game temp data is stored.
     GameDataDir,    // Where game data is stored.
     SysModuleDir,   // Where system modules are stored.
+    CapturesDir,    // Where rdoc captures are stored.
 };
 
 constexpr auto PORTABLE_DIR = "user";
@@ -31,6 +32,7 @@ constexpr auto SAVEDATA_DIR = "savedata";
 constexpr auto GAMEDATA_DIR = "data";
 constexpr auto TEMPDATA_DIR = "temp";
 constexpr auto SYSMODULES_DIR = "sys_modules";
+constexpr auto CAPTURES_DIR = "captures";
 
 // Filenames
 constexpr auto LOG_FILE = "shad_log.txt";

src/emulator.cpp

@@ -105,6 +105,12 @@ void Emulator::Run(const std::filesystem::path& file) {
     }
     mnt->Mount(mount_temp_dir, "/temp0"); // called in app_content ==> stat/mkdir
 
+    const auto& mount_captures_dir = Common::FS::GetUserPath(Common::FS::PathType::CapturesDir);
+    if (!std::filesystem::exists(mount_captures_dir)) {
+        std::filesystem::create_directory(mount_captures_dir);
+    }
+    VideoCore::SetOutputDir(mount_captures_dir.generic_string(), id);
+
     // Initialize kernel and library facilities.
     Libraries::Kernel::init_pthreads();
     Libraries::InitHLELibs(&linker->GetHLESymbols());

src/sdl_window.cpp

@@ -11,6 +11,7 @@
 #include "core/libraries/pad/pad.h"
 #include "input/controller.h"
 #include "sdl_window.h"
+#include "video_core/renderdoc.h"
 
 #ifdef __APPLE__
 #include <SDL3/SDL_metal.h>
@@ -179,6 +180,11 @@ void WindowSDL::onKeyPress(const SDL_Event* event) {
         ax = Input::GetAxis(-0x80, 0x80, axisvalue);
         break;
     case SDLK_S:
+        if (event->key.mod == SDL_KMOD_LCTRL) {
+            // Trigger rdoc capture
+            VideoCore::TriggerCapture();
+            break;
+        }
         axis = Input::Axis::LeftY;
         if (event->type == SDL_EVENT_KEY_DOWN) {
             axisvalue += 127;

src/video_core/amdgpu/liverpool.cpp

@@ -41,6 +41,8 @@ void Liverpool::Process(std::stop_token stoken) {
             break;
         }
 
+        VideoCore::StartCapture();
+
         int qid = -1;
 
         while (num_submits) {
@@ -71,6 +73,8 @@ void Liverpool::Process(std::stop_token stoken) {
         }
 
         if (submit_done) {
+            VideoCore::EndCapture();
+
             if (rasterizer) {
                 rasterizer->Flush();
             }

src/video_core/renderdoc.cpp

@@ -1,23 +1,59 @@
 // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include <renderdoc_app.h>
 #include "common/assert.h"
+#include "common/config.h"
 #include "video_core/renderdoc.h"
 
+#include <renderdoc_app.h>
+
 #ifdef _WIN32
 #include <windows.h>
 #else
 #include <dlfcn.h>
 #endif
 
+#include <filesystem>
+
 namespace VideoCore {
 
+enum class CaptureState {
+    Idle,
+    Triggered,
+    InProgress,
+};
+static CaptureState capture_state{CaptureState::Idle};
+
 RENDERDOC_API_1_6_0* rdoc_api{};
 
 void LoadRenderDoc() {
 #ifdef WIN32
-    if (HMODULE mod = GetModuleHandleA("renderdoc.dll")) {
+
+    // Check if we are running by RDoc GUI
+    HMODULE mod = GetModuleHandleA("renderdoc.dll");
+    if (!mod && Config::isRdocEnabled()) {
+        // If enabled in config, try to load RDoc runtime in offline mode
+        HKEY h_reg_key;
+        LONG result = RegOpenKeyExW(HKEY_LOCAL_MACHINE,
+                                    L"SOFTWARE\\Classes\\RenderDoc.RDCCapture.1\\DefaultIcon\\", 0,
+                                    KEY_READ, &h_reg_key);
+        if (result != ERROR_SUCCESS) {
+            return;
+        }
+        std::array<wchar_t, MAX_PATH> key_str{};
+        DWORD str_sz_out{key_str.size()};
+        result = RegQueryValueExW(h_reg_key, L"", 0, NULL, (LPBYTE)key_str.data(), &str_sz_out);
+        if (result != ERROR_SUCCESS) {
+            return;
+        }
+
+        std::filesystem::path path{key_str.cbegin(), key_str.cend()};
+        path = path.parent_path().append("renderdoc.dll");
+        const auto path_to_lib = path.generic_string();
+        mod = LoadLibraryA(path_to_lib.c_str());
+    }
+
+    if (mod) {
         const auto RENDERDOC_GetAPI =
             reinterpret_cast<pRENDERDOC_GetAPI>(GetProcAddress(mod, "RENDERDOC_GetAPI"));
         const s32 ret = RENDERDOC_GetAPI(eRENDERDOC_API_Version_1_6_0, (void**)&rdoc_api);
@@ -36,18 +72,49 @@ void LoadRenderDoc() {
         ASSERT(ret == 1);
     }
 #endif
+    if (rdoc_api) {
+        // Disable default capture keys as they suppose to trigger present-to-present capturing
+        // and it is not what we want
+        rdoc_api->SetCaptureKeys(nullptr, 0);
+
+        // Also remove rdoc crash handler
+        rdoc_api->UnloadCrashHandler();
+    }
 }
 
 void StartCapture() {
-    if (rdoc_api) {
+    if (!rdoc_api) {
+        return;
+    }
+
+    if (capture_state == CaptureState::Triggered) {
         rdoc_api->StartFrameCapture(nullptr, nullptr);
+        capture_state = CaptureState::InProgress;
     }
 }
 
 void EndCapture() {
-    if (rdoc_api) {
+    if (!rdoc_api) {
+        return;
+    }
+
+    if (capture_state == CaptureState::InProgress) {
         rdoc_api->EndFrameCapture(nullptr, nullptr);
+        capture_state = CaptureState::Idle;
     }
 }
 
+void TriggerCapture() {
+    if (capture_state == CaptureState::Idle) {
+        capture_state = CaptureState::Triggered;
+    }
+}
+
+void SetOutputDir(const std::string& path, const std::string& prefix) {
+    if (!rdoc_api) {
+        return;
+    }
+    rdoc_api->SetCaptureFilePathTemplate((path + '\\' + prefix).c_str());
+}
+
 } // namespace VideoCore

src/video_core/renderdoc.h

@@ -3,6 +3,8 @@
 
 #pragma once
 
+#include <string>
+
 namespace VideoCore {
 
 /// Loads renderdoc dynamic library module.
@@ -14,4 +16,10 @@ void StartCapture();
 /// Ends current renderdoc capture.
 void EndCapture();
 
+/// Triggers capturing process.
+void TriggerCapture();
+
+/// Sets output directory for captures
+void SetOutputDir(const std::string& path, const std::string& prefix);
+
 } // namespace VideoCore