Base impl (pending 16K pages and getbuffersize)

.gitmodules

@@ -48,7 +48,8 @@
 	shallow = true
 [submodule "externals/sirit"]
 	path = externals/sirit
-	url = https://github.com/shadps4-emu/sirit.git
+	url = https://github.com/LNDF/sirit.git
+	branch = uwu
 	shallow = true
 [submodule "externals/xxhash"]
 	path = externals/xxhash

src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp

@@ -162,15 +162,26 @@ void EmitGetGotoVariable(EmitContext&) {
 
 using PointerType = EmitContext::PointerType;
 
-Id EmitReadConst(EmitContext& ctx, IR::Inst* inst) {
+Id EmitReadConst(EmitContext& ctx, IR::Inst* inst, Id addr, Id offset) {
-    const u32 flatbuf_off_dw = inst->Flags<u32>();
+    const Id base_lo = ctx.OpUConvert(ctx.U64, ctx.OpCompositeExtract(ctx.U32[1], addr, 0));
-    const auto& srt_flatbuf = ctx.buffers[ctx.flatbuf_index];
+    const Id base_hi = ctx.OpUConvert(ctx.U64, ctx.OpCompositeExtract(ctx.U32[1], addr, 1));
-    ASSERT(srt_flatbuf.binding >= 0 && flatbuf_off_dw > 0 &&
+    const Id base_sift = ctx.OpShiftLeftLogical(ctx.U64, base_hi, ctx.ConstU32(32u));
-           srt_flatbuf.buffer_type == BufferType::ReadConstUbo);
+    const Id base = ctx.OpBitwiseOr(ctx.U64, base_lo, base_sift);
-    const auto [id, pointer_type] = srt_flatbuf[PointerType::U32];
+    const Id address = ctx.OpIAdd(ctx.U64, base, ctx.OpUConvert(ctx.U64, offset));
-    const Id ptr{
+    return ctx.EmitMemoryAccess(
-        ctx.OpAccessChain(pointer_type, id, ctx.u32_zero_value, ctx.ConstU32(flatbuf_off_dw))};
+        ctx.U32[1], address, [&]() {
-    return ctx.OpLoad(ctx.U32[1], ptr);
+            const u32 flatbuf_off_dw = inst->Flags<u32>();
+            if (flatbuf_off_dw == 0) {
+                return ctx.u32_zero_value;
+            } else {
+                const auto& srt_flatbuf = ctx.buffers[ctx.flatbuf_index];
+                ASSERT(srt_flatbuf.binding >= 0 > 0 && srt_flatbuf.buffer_type == BufferType::Flatbuf);
+                const auto [id, pointer_type] = srt_flatbuf[PointerType::U32];
+                const Id ptr{
+                    ctx.OpAccessChain(pointer_type, id, ctx.u32_zero_value, ctx.ConstU32(flatbuf_off_dw))};
+                return ctx.OpLoad(ctx.U32[1], ptr);
+            }
+        });
 }
 
 Id EmitReadConstBuffer(EmitContext& ctx, u32 handle, Id index) {

src/shader_recompiler/backend/spirv/emit_spirv_instructions.h

@@ -61,7 +61,7 @@ void EmitSetVectorRegister(EmitContext& ctx);
 void EmitSetGotoVariable(EmitContext& ctx);
 void EmitGetGotoVariable(EmitContext& ctx);
 void EmitSetScc(EmitContext& ctx);
-Id EmitReadConst(EmitContext& ctx, IR::Inst* inst);
+Id EmitReadConst(EmitContext& ctx, IR::Inst* inst, Id addr, Id offset);
 Id EmitReadConstBuffer(EmitContext& ctx, u32 handle, Id index);
 Id EmitLoadBufferU8(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address);
 Id EmitLoadBufferU16(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address);

src/shader_recompiler/backend/spirv/spirv_emit_context.cpp

@@ -7,6 +7,7 @@
 #include "shader_recompiler/frontend/fetch_shader.h"
 #include "shader_recompiler/runtime_info.h"
 #include "video_core/amdgpu/types.h"
+#include "video_core/buffer_cache/buffer_cache.h"
 
 #include <boost/container/static_vector.hpp>
 #include <fmt/format.h>
@@ -146,6 +147,7 @@ void EmitContext::DefineArithmeticTypes() {
     u32_one_value = ConstU32(1U);
     u32_zero_value = ConstU32(0U);
     f32_zero_value = ConstF32(0.0f);
+    u64_zero_value = Constant(U64, 0ULL);
 
     pi_x2 = ConstF32(2.0f * float{std::numbers::pi});
 
@@ -180,12 +182,24 @@ void EmitContext::DefineArithmeticTypes() {
         physical_pointer_types[PointerType::F16] = TypePointer(spv::StorageClass::PhysicalStorageBuffer, F16[1]);
     }
     if (True(info.dma_types & IR::Type::U16)) {
-        physical_pointer_types[PointerType::U32] = TypePointer(spv::StorageClass::PhysicalStorageBuffer, U16);
+        physical_pointer_types[PointerType::U16] = TypePointer(spv::StorageClass::PhysicalStorageBuffer, U16);
+    }
+    if (True(info.dma_types & IR::Type::U8)) {
+        physical_pointer_types[PointerType::U8] = TypePointer(spv::StorageClass::PhysicalStorageBuffer, U8);
     }
     
-    // We allways want U8 if using DMA, for the fault readback buffer
     if (info.dma_types != IR::Type::Void) {
-        physical_pointer_types[PointerType::U32] = TypePointer(spv::StorageClass::PhysicalStorageBuffer, U8);
+        constexpr u64 host_access_mask = 0x1UL;
+        constexpr u64 host_access_inv_mask = ~host_access_mask;
+
+        caching_pagebits_value = Constant(U64, static_cast<u64>(VideoCore::BufferCache::CACHING_PAGEBITS));
+        caching_pagemask_value = Constant(U64, VideoCore::BufferCache::CACHING_PAGESIZE - 1);
+        host_access_mask_value = Constant(U64, host_access_mask);
+        host_access_inv_mask_value = Constant(U64, host_access_inv_mask);
+
+        // Used to calculate fault readback buffer position and mask
+        u32_three_value = ConstU32(3U);
+        u32_seven_value = ConstU32(7U);
     }
 }
 
@@ -227,7 +241,7 @@ EmitContext::SpirvAttribute EmitContext::GetAttributeInfo(AmdGpu::NumberFormat f
 Id EmitContext::GetBufferSize(const u32 sharp_idx) {
     // Can this be done with memory access? Like we do now with ReadConst
     const auto& srt_flatbuf = buffers[flatbuf_index];
-    ASSERT(srt_flatbuf.buffer_type == BufferType::ReadConstUbo);
+    ASSERT(srt_flatbuf.buffer_type == BufferType::Flatbuf);
     const auto [id, pointer_type] = srt_flatbuf[PointerType::U32];
 
     const auto rsrc1{
@@ -721,8 +735,8 @@ EmitContext::BufferSpv EmitContext::DefineBuffer(bool is_storage, bool is_writte
     case Shader::BufferType::GdsBuffer:
         Name(id, "gds_buffer");
         break;
-    case Shader::BufferType::ReadConstUbo:
+    case Shader::BufferType::Flatbuf:
-        Name(id, "srt_flatbuf_ubo");
+        Name(id, "srt_flatbuf");
         break;
     case Shader::BufferType::BdaPagetable:
         Name(id, "bda_pagetable");
@@ -743,12 +757,14 @@ EmitContext::BufferSpv EmitContext::DefineBuffer(bool is_storage, bool is_writte
 
 void EmitContext::DefineBuffers() {
     if (!profile.supports_robust_buffer_access && !info.has_readconst) {
-        // In case ReadConstUbo has not already been bound by IR and is needed
+        // In case Flatbuf has not already been bound by IR and is needed
         // to query buffer sizes, bind it now.
         info.buffers.push_back({
             .used_types = IR::Type::U32,
-            .inline_cbuf = AmdGpu::Buffer::Null(),
+            // We can't guarantee that flatbuf will now grow bast UBO
-            .buffer_type = BufferType::ReadConstUbo,
+            // limit if there are a lot of ReadConsts. (We could specialize)
+            .inline_cbuf = AmdGpu::Buffer::Placeholder(std::numeric_limits<u32>::max()),
+            .buffer_type = BufferType::Flatbuf,
         });
     }
     for (const auto& desc : info.buffers) {
@@ -756,7 +772,7 @@ void EmitContext::DefineBuffers() {
         const bool is_storage = desc.IsStorage(buf_sharp, profile);
 
         // Set indexes for special buffers.
-        if (desc.buffer_type == BufferType::ReadConstUbo) {
+        if (desc.buffer_type == BufferType::Flatbuf) {
             flatbuf_index = buffers.size();
         } else if (desc.buffer_type == BufferType::BdaPagetable) {
             bda_pagetable_index = buffers.size();

src/shader_recompiler/backend/spirv/spirv_emit_context.h

@@ -37,10 +37,22 @@ struct VectorIds {
 
 class EmitContext final : public Sirit::Module {
 public:
+
     explicit EmitContext(const Profile& profile, const RuntimeInfo& runtime_info, Info& info,
                          Bindings& binding);
     ~EmitContext();
 
+    enum class PointerType : u32 {
+        U8,
+        U16,
+        F16,
+        U32,
+        F32,
+        U64,
+        F64,
+        NumAlias,
+    };
+
     Id Def(const IR::Value& value);
 
     void DefineBufferProperties();
@@ -133,16 +145,92 @@ public:
         return ConstantComposite(type, constituents);
     }
 
-    inline Id OpLabel(std::string_view label_name) {
+    inline Id AddLabel() {
-        last_label = Module::OpLabel(label_name);
+        last_label = Module::AddLabel();
         return last_label;
     }
 
-    inline Id OpLabel() {
+    inline Id AddLabel(Id label) {
-        last_label = Module::OpLabel();
+        last_label = Module::AddLabel(label);
         return last_label;
     }
 
+    PointerType PointerTypeFromType(Id type) {
+        if (type.value == U8.value) return PointerType::U8;
+        if (type.value == U16.value) return PointerType::U16;
+        if (type.value == F16[1].value) return PointerType::F16;
+        if (type.value == U32[1].value) return PointerType::U32;
+        if (type.value == F32[1].value) return PointerType::F32;
+        if (type.value == U64.value) return PointerType::U64;
+        if (type.value == F64[1].value) return PointerType::F64;
+        UNREACHABLE_MSG("Unknown type for pointer");
+    }
+
+    template <typename Func>
+    Id EmitMemoryAccess(Id type, Id address, Func&& fallback) {
+        const Id host_access_label = OpLabel();
+        const Id after_host_access_label = OpLabel();
+        const Id fallback_label = OpLabel();
+        const Id available_label = OpLabel();
+        const Id merge_label = OpLabel();
+
+        // Get page BDA
+        const Id page = OpShiftRightLogical(U64, address, caching_pagebits_value);
+        const Id page32 = OpUConvert(U32[1], page);
+        const auto& bda_buffer = buffers[bda_pagetable_index];
+        const auto [bda_buffer_id, bda_pointer_type] = bda_buffer[PointerType::U64];
+        const Id bda_ptr = OpAccessChain(bda_pointer_type, bda_buffer_id, u32_zero_value, page32);
+        const Id bda = OpLoad(U64, bda_ptr);
+
+        // Check if it's a host memory access
+        const Id bda_and_host_access_mask = OpBitwiseAnd(U64, bda, host_access_mask_value);
+        const Id bda_host_access = OpINotEqual(U1[1], bda_and_host_access_mask, host_access_mask_value);
+        OpSelectionMerge(after_host_access_label, spv::SelectionControlMask::MaskNone);
+        OpBranchConditional(bda_host_access, host_access_label, after_host_access_label);
+
+        // Host access, set bit in fault readback buffer
+        AddLabel(host_access_label);
+        const auto& fault_buffer = buffers[fault_readback_index];
+        const auto [fault_buffer_id, fault_pointer_type] = fault_buffer[PointerType::U8];
+        const Id page_div8 = OpShiftRightLogical(U32[1], page32, u32_three_value);
+        const Id page_mod8 = OpBitwiseAnd(U32[1], page32, u32_seven_value);
+        const Id page_mask = OpShiftLeftLogical(U32[1], u32_one_value, page_mod8);
+        const Id fault_ptr = OpAccessChain(fault_pointer_type, fault_buffer_id, u32_zero_value,
+                                           page_div8);
+        const Id fault_value = OpLoad(U8, fault_ptr);
+        const Id page_mask8 = OpUConvert(U8, page_mask);
+        const Id fault_value_masked = OpBitwiseOr(U8, fault_value, page_mask8);
+        OpStore(fault_ptr, fault_value_masked);
+        OpBranch(after_host_access_label);
+
+        // Check if the value is available
+        AddLabel(after_host_access_label);
+        const Id bda_eq_zero = OpIEqual(U1[1], bda, u64_zero_value);
+        OpSelectionMerge(merge_label, spv::SelectionControlMask::MaskNone);
+        OpBranchConditional(bda_eq_zero, fallback_label, available_label);
+
+        // Fallback
+        AddLabel(fallback_label);
+        const Id fallback_result = fallback();
+        OpBranch(merge_label);
+
+        // Get value from memory
+        AddLabel(available_label);
+        const Id untagged_bda = OpBitwiseAnd(U64, bda, host_access_inv_mask_value);
+        const Id offset_in_bda = OpBitwiseAnd(U64, address, caching_pagemask_value);
+        const Id addr = OpIAdd(U64, untagged_bda, offset_in_bda);
+        const PointerType pointer_type = PointerTypeFromType(type);
+        const Id pointer_type_id = physical_pointer_types[pointer_type];
+        const Id addr_ptr = OpConvertUToPtr(pointer_type_id, addr);
+        const Id result = OpLoad(type, addr_ptr, spv::MemoryAccessMask::Aligned, 4u);
+        OpBranch(merge_label);
+
+        // Merge
+        AddLabel(merge_label);
+        const Id final_result = OpPhi(type, fallback_result, fallback_label, result, available_label);
+        return final_result;
+    }
+
     Info& info;
     const RuntimeInfo& runtime_info;
     const Profile& profile;
@@ -173,9 +261,17 @@ public:
 
     Id true_value{};
     Id false_value{};
+    Id u32_seven_value{};
+    Id u32_three_value{};
     Id u32_one_value{};
     Id u32_zero_value{};
     Id f32_zero_value{};
+    Id u64_zero_value{};
+
+    Id caching_pagebits_value{};
+    Id caching_pagemask_value{};
+    Id host_access_mask_value{};
+    Id host_access_inv_mask_value{};
 
     Id shared_u8{};
     Id shared_u16{};
@@ -243,17 +339,6 @@ public:
         bool is_storage = false;
     };
 
-    enum class PointerType : u32 {
-        U8,
-        U16,
-        F16,
-        U32,
-        F32,
-        U64,
-        F64,
-        NumAlias,
-    };
-
     struct BufferSpv {
         Id id;
         Id pointer_type;

src/shader_recompiler/info.h

@@ -41,7 +41,7 @@ constexpr u32 NUM_TEXTURE_TYPES = 7;
 
 enum class BufferType : u32 {
     Guest,
-    ReadConstUbo,
+    Flatbuf,
     BdaPagetable,
     FaultReadback,
     GdsBuffer,

src/shader_recompiler/ir/passes/shader_info_collection_pass.cpp

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #include "shader_recompiler/ir/program.h"
+#include "video_core/buffer_cache/buffer_cache.h"
 
 namespace Shader::Optimization {
 
@@ -82,8 +83,10 @@ void Visit(Info& info, const IR::Inst& inst) {
         if (!info.has_readconst) {
             info.buffers.push_back({
                 .used_types = IR::Type::U32,
-                .inline_cbuf = AmdGpu::Buffer::Null(),
+                // We can't guarantee that flatbuf will now grow bast UBO
-                .buffer_type = BufferType::ReadConstUbo,
+                // limit if there are a lot of ReadConsts. (We could specialize)
+                .inline_cbuf = AmdGpu::Buffer::Placeholder(std::numeric_limits<u32>::max()),
+                .buffer_type = BufferType::Flatbuf,
             });
             info.has_readconst = true;
         }
@@ -110,13 +113,14 @@ void CollectShaderInfoPass(IR::Program& program) {
     if (program.info.dma_types != IR::Type::Void) {
         program.info.buffers.push_back({
             .used_types = IR::Type::U64,
-            .inline_cbuf = AmdGpu::Buffer::Null(),
+            .inline_cbuf = AmdGpu::Buffer::Placeholder(VideoCore::BufferCache::BDA_PAGETABLE_SIZE),
             .buffer_type = BufferType::BdaPagetable,
         });
         program.info.buffers.push_back({
             .used_types = IR::Type::U8,
-            .inline_cbuf = AmdGpu::Buffer::Null(),
+            .inline_cbuf = AmdGpu::Buffer::Placeholder(VideoCore::BufferCache::FAULT_READBACK_SIZE),
             .buffer_type = BufferType::FaultReadback,
+            .is_written = true,
         });
     }
 }

src/video_core/amdgpu/resource.h

@@ -37,6 +37,13 @@ struct Buffer {
         return buffer;
     }
 
+    static constexpr Buffer Placeholder(u32 size) {
+        Buffer buffer{};
+        buffer.base_address = 1;
+        buffer.num_records = size;
+        return buffer;
+    }
+
     bool Valid() const {
         return type == 0u;
     }

src/video_core/buffer_cache/buffer.cpp

@@ -195,6 +195,7 @@ ImportedHostBuffer::ImportedHostBuffer(const Vulkan::Instance& instance_,
                     "Failed to import host memory at {} size {:#x}, Reason: {}",
                     cpu_addr, size_bytes, vk::to_string(device_memory_result.result));
         instance->GetDevice().destroyBuffer(buffer);
+        buffer = VK_NULL_HANDLE;
         has_failed = true;
         return;
     }

src/video_core/buffer_cache/buffer_cache.cpp

@@ -568,20 +568,22 @@ void BufferCache::CreateFaultBuffers() {
     scheduler.Finish();
     std::array<u8, FAULT_READBACK_SIZE> buffer{};
     std::memcpy(buffer.data(), mapped, FAULT_READBACK_SIZE);
-    // Reset the fault readback buffer
-    cmdbuf.fillBuffer(fault_readback_buffer.buffer, 0, FAULT_READBACK_SIZE, 0);
     // Create the fault buffers batched
     boost::icl::interval_set<VAddr> fault_ranges;
-    for (u64 i = 0; i < FAULT_READBACK_SIZE / sizeof(vk::DeviceAddress); ++i) {
+    for (u64 i = 0; i < FAULT_READBACK_SIZE; ++i) {
-        if (buffer[i] != 0) {
+        if (buffer[i] == 0) {
-            // Each byte contains information for 8 pages.
+            continue;
-            // We are oing to create an aligned buffer of
+        }
-            // 8 * 64 KB = 512 KB arround the fault address.
+        // Each bit is a page
-            const VAddr fault_addr = buffer[i] << CACHING_PAGEBITS;
+        const u64 page = i * 8;
-            const u32 fault_end = mapped[i + 1] << CACHING_PAGEBITS;
+        for (u8 j = 0; j < 8; ++j) {
-            auto range = decltype(fault_ranges)::interval_type::right_open(
+            if ((buffer[i] & (1 << j)) == 0) {
-                fault_addr, fault_end);
+                continue;
-            fault_ranges += range;
+            }
+            const VAddr start = (page + j) << CACHING_PAGEBITS;
+            const VAddr end = start + CACHING_PAGESIZE;
+            fault_ranges += boost::icl::interval_set<VAddr>::interval_type::right_open(start, end);
+            LOG_WARNING(Render_Vulkan, "Accessed non GPU-local memory at {:#x}", start);
         }
     }
     for (const auto& range : fault_ranges) {
@@ -591,6 +593,41 @@ void BufferCache::CreateFaultBuffers() {
     }
 }
 
+void BufferCache::ResetFaultReadbackBuffer() {
+    const vk::BufferMemoryBarrier2 pre_barrier = {
+        .srcStageMask = vk::PipelineStageFlagBits2::eAllCommands,
+        .srcAccessMask = vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite,
+        .dstStageMask = vk::PipelineStageFlagBits2::eTransfer,
+        .dstAccessMask = vk::AccessFlagBits2::eTransferWrite,
+        .buffer = fault_readback_buffer.Handle(),
+        .offset = 0,
+        .size = FAULT_READBACK_SIZE,
+    };
+    const vk::BufferMemoryBarrier2 post_barrier = {
+        .srcStageMask = vk::PipelineStageFlagBits2::eTransfer,
+        .srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
+        .dstStageMask = vk::PipelineStageFlagBits2::eAllCommands,
+        .dstAccessMask = vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite,
+        .buffer = fault_readback_buffer.Handle(),
+        .offset = 0,
+        .size = FAULT_READBACK_SIZE,
+    };
+    // Reset the fault readback buffer
+    scheduler.EndRendering();
+    const auto cmdbuf = scheduler.CommandBuffer();
+    cmdbuf.pipelineBarrier2(vk::DependencyInfo{
+        .dependencyFlags = vk::DependencyFlagBits::eByRegion,
+        .bufferMemoryBarrierCount = 1,
+        .pBufferMemoryBarriers = &pre_barrier,
+    });
+    cmdbuf.fillBuffer(fault_readback_buffer.buffer, 0, FAULT_READBACK_SIZE, 0);
+    cmdbuf.pipelineBarrier2(vk::DependencyInfo{
+        .dependencyFlags = vk::DependencyFlagBits::eByRegion,
+        .bufferMemoryBarrierCount = 1,
+        .pBufferMemoryBarriers = &post_barrier,
+    });
+}
+
 void BufferCache::Register(BufferId buffer_id) {
     ChangeRegister<true>(buffer_id);
 }

src/video_core/buffer_cache/buffer_cache.h

@@ -50,7 +50,7 @@ public:
     struct Traits {
         using Entry = BufferId;
         static constexpr size_t AddressSpaceBits = 40;
-        static constexpr size_t FirstLevelBits = 14;
+        static constexpr size_t FirstLevelBits = 18;
         static constexpr size_t PageBits = CACHING_PAGEBITS;
     };
     using PageTable = MultiLevelPageTable<Traits>;
@@ -138,6 +138,9 @@ public:
     /// Creates buffers for "faulted" shader accesses to host memory.
     void CreateFaultBuffers();
 
+    /// Reset the fault readback buffer.
+    void ResetFaultReadbackBuffer();
+
     /// Synchronizes all buffers in the specified range.
     void SynchronizeRange(VAddr device_addr, u32 size);
 

src/video_core/renderer_vulkan/vk_rasterizer.cpp

@@ -475,6 +475,7 @@ bool Rasterizer::BindResources(const Pipeline* pipeline) {
                 buffer_cache.SynchronizeRange(range.lower(), range.upper() - range.lower());
             }
         }
+        buffer_cache.ResetFaultReadbackBuffer();
     }
 
     return true;
@@ -535,7 +536,7 @@ void Rasterizer::BindBuffers(const Shader::Info& stage, Shader::Backend::Binding
             if (desc.buffer_type == Shader::BufferType::GdsBuffer) {
                 const auto* gds_buf = buffer_cache.GetGdsBuffer();
                 buffer_infos.emplace_back(gds_buf->Handle(), 0, gds_buf->SizeBytes());
-            } else if (desc.buffer_type == Shader::BufferType::ReadConstUbo) {
+            } else if (desc.buffer_type == Shader::BufferType::Flatbuf) {
                 auto& vk_buffer = buffer_cache.GetStreamBuffer();
                 const u32 ubo_size = stage.flattened_ud_buf.size() * sizeof(u32);
                 const u64 offset = vk_buffer.Copy(stage.flattened_ud_buf.data(), ubo_size,