renderer_vulkan: a heuristic for blend override when alpha out is masked

src/video_core/amdgpu/liverpool.h

@@ -420,6 +420,13 @@ struct Liverpool {
     };
 
     union ColorBufferMask {
+        enum ColorComponent : u32 {
+            ComponentR = (1u << 0),
+            ComponentG = (1u << 1),
+            ComponentB = (1u << 2),
+            ComponentA = (1u << 3),
+        };
+
         u32 raw;
         BitField<0, 4, u32> output0_mask;
         BitField<4, 4, u32> output1_mask;
@@ -430,7 +437,7 @@ struct Liverpool {
         BitField<24, 4, u32> output6_mask;
         BitField<28, 4, u32> output7_mask;
 
-        [[nodiscard]] u8 GetMask(int buf_id) const {
+        u32 GetMask(int buf_id) const {
             return (raw >> (buf_id * 4)) & 0xfu;
         }
     };

src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp

@@ -196,7 +196,7 @@ GraphicsPipeline::GraphicsPipeline(const Instance& instance_, Scheduler& schedul
         const auto dst_color = LiverpoolToVK::BlendFactor(control.color_dst_factor);
         const auto color_blend = LiverpoolToVK::BlendOp(control.color_func);
         attachments[i] = vk::PipelineColorBlendAttachmentState{
-            .blendEnable = key.blend_controls[i].enable,
+            .blendEnable = control.enable,
             .srcColorBlendFactor = src_color,
             .dstColorBlendFactor = dst_color,
             .colorBlendOp = color_blend,
@@ -215,6 +215,29 @@ GraphicsPipeline::GraphicsPipeline(const Instance& instance_, Scheduler& schedul
                           vk::ColorComponentFlagBits::eB | vk::ColorComponentFlagBits::eA
                     : key.write_masks[i],
         };
+
+        // On GCN GPU there is an additional mask which allows to control color components exported
+        // from a pixel shader. A situation possible, when the game may mask out the alpha channel,
+        // while it is still need to be used in blending ops. For such cases, HW will default alpha
+        // to 1 and perform the blending, while shader normally outputs 0 in the last component.
+        // Unfortunatelly, Vulkan doesn't provide any control on blend inputs, so below we detecting
+        // such cases and override alpha value in order to emulate HW behaviour.
+        const auto has_alpha_masked_out =
+            (key.cb_shader_mask.GetMask(i) & Liverpool::ColorBufferMask::ComponentA) == 0;
+        const auto has_src_alpha_in_src_blend = src_color == vk::BlendFactor::eSrcAlpha ||
+                                                src_color == vk::BlendFactor::eOneMinusSrcAlpha;
+        const auto has_src_alpha_in_dst_blend = dst_color == vk::BlendFactor::eSrcAlpha ||
+                                                dst_color == vk::BlendFactor::eOneMinusSrcAlpha;
+        if (has_alpha_masked_out && has_src_alpha_in_src_blend) {
+            attachments[i].srcColorBlendFactor = src_color == vk::BlendFactor::eSrcAlpha
+                                                     ? vk::BlendFactor::eOne
+                                                     : vk::BlendFactor::eZero; // 1-A
+        }
+        if (has_alpha_masked_out && has_src_alpha_in_dst_blend) {
+            attachments[i].dstColorBlendFactor = dst_color == vk::BlendFactor::eSrcAlpha
+                                                     ? vk::BlendFactor::eOne
+                                                     : vk::BlendFactor::eZero; // 1-A
+        }
     }
 
     const vk::PipelineColorBlendStateCreateInfo color_blending = {

src/video_core/renderer_vulkan/vk_graphics_pipeline.h

@@ -46,6 +46,7 @@ struct GraphicsPipelineKey {
     Liverpool::CullMode cull_mode;
     Liverpool::FrontFace front_face;
     Liverpool::ClipSpace clip_space;
+    Liverpool::ColorBufferMask cb_shader_mask{};
     std::array<Liverpool::BlendControl, Liverpool::NumColorBuffers> blend_controls;
     std::array<vk::ColorComponentFlags, Liverpool::NumColorBuffers> write_masks;
 

src/video_core/renderer_vulkan/vk_pipeline_cache.cpp

@@ -132,6 +132,7 @@ void PipelineCache::RefreshGraphicsKey() {
         key.blend_controls[remapped_cb].enable.Assign(key.blend_controls[remapped_cb].enable &&
                                                       !col_buf.info.blend_bypass);
         key.write_masks[remapped_cb] = vk::ColorComponentFlags{regs.color_target_mask.GetMask(cb)};
+        key.cb_shader_mask = regs.color_shader_mask;
 
         ++remapped_cb;
     }