VK Wayland RenderDoc log incompatible (#3675)

documents/Debugging/Debugging.md

@@ -11,6 +11,13 @@ This document covers information about debugging, troubleshooting and reporting
 
 This section will guide you through setting up tools for debugging the emulator. This list will likely expand as more tools and platforms receive consistent setups.
 
+<details>
+<summary>Linux</summary>
+
+RenderDoc doesn't work with Wayland, so to use it you have to run the emulator with `SDL_VIDEODRIVER=x11` set.
+
+</details>
+
 <details>
 <summary>Windows and Visual Studio</summary>
 

src/video_core/renderer_vulkan/vk_platform.cpp

@@ -86,6 +86,11 @@ vk::SurfaceKHR CreateSurface(vk::Instance instance, const Frontend::WindowSDL& e
             UNREACHABLE();
         }
     } else if (window_info.type == Frontend::WindowSystemType::Wayland) {
+        if (Config::isRdocEnabled()) {
+            LOG_ERROR(Render_Vulkan,
+                      "RenderDoc is not compatible with Wayland, use an X11 window instead.");
+        }
+
         const vk::WaylandSurfaceCreateInfoKHR wayland_ci = {
             .display = static_cast<wl_display*>(window_info.display_connection),
             .surface = static_cast<wl_surface*>(window_info.render_surface),