From 25d1defb5b6c289cada7450bf3abdf4a5595cf38 Mon Sep 17 00:00:00 2001
From: squidbus <175574877+squidbus@users.noreply.github.com>
Date: Mon, 25 Aug 2025 04:17:20 -0700
Subject: [PATCH] shader_recompiler: Do not emit Layer when emulating primitive
 type with tessellation. (#3457)

Layer must be output from the final pre-rasterization stage, so when tessellation is
being used the vertex shader cannot set it. We could pass it through to the tessellation
shaders in some way, but unless it becomes an issue that's more effort than it's worth.
---
 src/shader_recompiler/ir/position.h             | 17 +++++++++++++++--
 src/shader_recompiler/runtime_info.h            |  2 ++
 .../renderer_vulkan/vk_pipeline_cache.cpp       |  3 +++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/shader_recompiler/ir/position.h b/src/shader_recompiler/ir/position.h
index eb67634f0..41e27c5fe 100644
--- a/src/shader_recompiler/ir/position.h
+++ b/src/shader_recompiler/ir/position.h
@@ -10,8 +10,9 @@
 namespace Shader::IR {
 
 /// Maps special position export to builtin attribute stores
-inline void ExportPosition(IREmitter& ir, const auto& stage, Attribute attribute, u32 comp,
-                           const IR::F32& value) {
+template <typename StageRuntimeInfo>
+inline void ExportPosition(IREmitter& ir, const StageRuntimeInfo& stage, Attribute attribute,
+                           u32 comp, const IR::F32& value) {
     if (attribute == Attribute::Position0) {
         ir.SetAttribute(attribute, value, comp);
         return;
@@ -44,6 +45,18 @@ inline void ExportPosition(IREmitter& ir, const auto& stage, Attribute attribute
         break;
     }
     case Output::GsMrtIndex:
+        if constexpr (std::is_same_v<StageRuntimeInfo, VertexRuntimeInfo>) {
+            // When using tessellation, layer is supposed to be set by the tessellation evaluation
+            // stage. We don't currently have a mechanism for that when emulating rect/quad lists
+            // using tessellation, so just ignore the write for now. Note that this only matters
+            // for vertex shaders, as geometry shaders come last in the pre-rasterization stage.
+            if (stage.tess_emulated_primitive) {
+                LOG_WARNING(Render,
+                            "Exporting Layer from a vertex shader when using tessellation-based "
+                            "primitive emulation is currently unsupported.");
+                return;
+            }
+        }
         ir.SetAttribute(IR::Attribute::RenderTargetId, value);
         break;
     case Output::None:
diff --git a/src/shader_recompiler/runtime_info.h b/src/shader_recompiler/runtime_info.h
index 7ac876c50..6e138888a 100644
--- a/src/shader_recompiler/runtime_info.h
+++ b/src/shader_recompiler/runtime_info.h
@@ -82,6 +82,7 @@ using OutputMap = std::array<Output, 4>;
 struct VertexRuntimeInfo {
     u32 num_outputs;
     std::array<OutputMap, 3> outputs;
+    bool tess_emulated_primitive{};
     bool emulate_depth_negative_one_to_one{};
     bool clip_disable{};
     u32 step_rate_0;
@@ -94,6 +95,7 @@ struct VertexRuntimeInfo {
 
     bool operator==(const VertexRuntimeInfo& other) const noexcept {
         return num_outputs == other.num_outputs && outputs == other.outputs &&
+               tess_emulated_primitive == other.tess_emulated_primitive &&
                emulate_depth_negative_one_to_one == other.emulate_depth_negative_one_to_one &&
                clip_disable == other.clip_disable && tess_type == other.tess_type &&
                tess_topology == other.tess_topology &&
diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
index e95260554..dd7875bcb 100644
--- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
+++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp
@@ -128,6 +128,9 @@ const Shader::RuntimeInfo& PipelineCache::BuildRuntimeInfo(Stage stage, LogicalS
         info.vs_info.emulate_depth_negative_one_to_one =
             !instance.IsDepthClipControlSupported() &&
             regs.clipper_control.clip_space == Liverpool::ClipSpace::MinusWToW;
+        info.vs_info.tess_emulated_primitive =
+            regs.primitive_type == AmdGpu::PrimitiveType::RectList ||
+            regs.primitive_type == AmdGpu::PrimitiveType::QuadList;
         info.vs_info.clip_disable = regs.IsClipDisabled();
         if (l_stage == LogicalStage::TessellationEval) {
             info.vs_info.tess_type = regs.tess_config.type;