diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp index 0653fb02a..57730c782 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp @@ -21,11 +21,19 @@ Id SharedAtomicU32(EmitContext& ctx, Id offset, Id value, return (ctx.*atomic_func)(ctx.U32[1], pointer, scope, semantics, value); } -Id BufferAtomicU32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value, + +Id BufferAtomicU32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value, Id (Sirit::Module::*atomic_func)(Id, Id, Id, Id, Id)) { - const Id pointer{ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, ctx.ConstU32(0U))}; + // Get srsrc buffer + auto& buffer = ctx.buffers[handle]; + // Get address of vdata by vaddr + buffer offset + address = ctx.OpIAdd(ctx.U32[1], address, buffer.offset); + // Get first index of data (4-aligned indices, addr >> 2) + const Id index = ctx.OpShiftRightLogical(ctx.U32[1], address, ctx.ConstU32(2u)); + // Get pointer to first data value in buffer using index + const Id ptr = ctx.OpAccessChain(buffer.pointer_type, buffer.id, ctx.u32_zero_value, index); const auto [scope, semantics]{AtomicArgs(ctx)}; - return (ctx.*atomic_func)(ctx.U32[1], pointer, scope, semantics, value); + return (ctx.*atomic_func)(ctx.U32[1], ptr, scope, semantics, value); } Id ImageAtomicU32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id coords, Id value, @@ -57,50 +65,50 @@ Id EmitSharedAtomicSMin32(EmitContext& ctx, Id offset, Id value) { return SharedAtomicU32(ctx, offset, value, &Sirit::Module::OpAtomicSMin); } -Id EmitBufferAtomicIAdd32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value) { - return BufferAtomicU32(ctx, inst, handle, value, &Sirit::Module::OpAtomicIAdd); +Id EmitBufferAtomicIAdd32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value) { + return BufferAtomicU32(ctx, inst, handle, address, value, &Sirit::Module::OpAtomicIAdd); } -Id EmitBufferAtomicSMin32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value) { - return BufferAtomicU32(ctx, inst, handle, value, &Sirit::Module::OpAtomicSMin); +Id EmitBufferAtomicSMin32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value) { + return BufferAtomicU32(ctx, inst, handle, address, value, &Sirit::Module::OpAtomicSMin); } -Id EmitBufferAtomicUMin32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value) { - return BufferAtomicU32(ctx, inst, handle, value, &Sirit::Module::OpAtomicUMin); +Id EmitBufferAtomicUMin32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value) { + return BufferAtomicU32(ctx, inst, handle, address, value, &Sirit::Module::OpAtomicUMin); } -Id EmitBufferAtomicSMax32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value) { - return BufferAtomicU32(ctx, inst, handle, value, &Sirit::Module::OpAtomicSMax); +Id EmitBufferAtomicSMax32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value) { + return BufferAtomicU32(ctx, inst, handle, address, value, &Sirit::Module::OpAtomicSMax); } -Id EmitBufferAtomicUMax32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value) { - return BufferAtomicU32(ctx, inst, handle, value, &Sirit::Module::OpAtomicUMax); +Id EmitBufferAtomicUMax32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value) { + return BufferAtomicU32(ctx, inst, handle, address, value, &Sirit::Module::OpAtomicUMax); } -Id EmitBufferAtomicInc32(EmitContext&, IR::Inst*, u32, Id) { +Id EmitBufferAtomicInc32(EmitContext&, IR::Inst*, u32, Id, Id) { // TODO UNREACHABLE_MSG("Unsupported BUFFER_ATOMIC opcode: ", IR::Opcode::BufferAtomicInc32); } -Id EmitBufferAtomicDec32(EmitContext&, IR::Inst*, u32, Id) { +Id EmitBufferAtomicDec32(EmitContext&, IR::Inst*, u32, Id, Id) { // TODO UNREACHABLE_MSG("Unsupported BUFFER_ATOMIC opcode: ", IR::Opcode::BufferAtomicDec32); } -Id EmitBufferAtomicAnd32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value) { - return BufferAtomicU32(ctx, inst, handle, value, &Sirit::Module::OpAtomicAnd); +Id EmitBufferAtomicAnd32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value) { + return BufferAtomicU32(ctx, inst, handle, address, value, &Sirit::Module::OpAtomicAnd); } -Id EmitBufferAtomicOr32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value) { - return BufferAtomicU32(ctx, inst, handle, value, &Sirit::Module::OpAtomicOr); +Id EmitBufferAtomicOr32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value) { + return BufferAtomicU32(ctx, inst, handle, address, value, &Sirit::Module::OpAtomicOr); } -Id EmitBufferAtomicXor32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value) { - return BufferAtomicU32(ctx, inst, handle, value, &Sirit::Module::OpAtomicXor); +Id EmitBufferAtomicXor32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value) { + return BufferAtomicU32(ctx, inst, handle, address, value, &Sirit::Module::OpAtomicXor); } -Id EmitBufferAtomicExchange32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value) { - return BufferAtomicU32(ctx, inst, handle, value, &Sirit::Module::OpAtomicExchange); +Id EmitBufferAtomicExchange32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value) { + return BufferAtomicU32(ctx, inst, handle, address, value, &Sirit::Module::OpAtomicExchange); } Id EmitImageAtomicIAdd32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id coords, Id value) { diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp index f933ed3c6..0b02f3a37 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_context_get_set.cpp @@ -495,6 +495,7 @@ static void EmitStoreBufferFormatF32xN(EmitContext& ctx, u32 handle, Id address, case AmdGpu::DataFormat::Format8_8_8_8: case AmdGpu::DataFormat::Format16: case AmdGpu::DataFormat::Format32: + case AmdGpu::DataFormat::Format32_32: case AmdGpu::DataFormat::Format32_32_32_32: { ASSERT(N == AmdGpu::NumComponents(format)); diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h index 676d2553d..bc39bc0f3 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h +++ b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h @@ -81,17 +81,17 @@ void EmitStoreBufferFormatF32x2(EmitContext& ctx, IR::Inst* inst, u32 handle, Id void EmitStoreBufferFormatF32x3(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); void EmitStoreBufferFormatF32x4(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); void EmitStoreBufferU32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); -Id EmitBufferAtomicIAdd32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value); -Id EmitBufferAtomicSMin32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value); -Id EmitBufferAtomicUMin32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value); -Id EmitBufferAtomicSMax32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value); -Id EmitBufferAtomicUMax32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value); -Id EmitBufferAtomicInc32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value); -Id EmitBufferAtomicDec32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value); -Id EmitBufferAtomicAnd32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value); -Id EmitBufferAtomicOr32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value); -Id EmitBufferAtomicXor32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value); -Id EmitBufferAtomicExchange32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id value); +Id EmitBufferAtomicIAdd32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); +Id EmitBufferAtomicSMin32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); +Id EmitBufferAtomicUMin32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); +Id EmitBufferAtomicSMax32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); +Id EmitBufferAtomicUMax32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); +Id EmitBufferAtomicInc32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); +Id EmitBufferAtomicDec32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); +Id EmitBufferAtomicAnd32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); +Id EmitBufferAtomicOr32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); +Id EmitBufferAtomicXor32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); +Id EmitBufferAtomicExchange32(EmitContext& ctx, IR::Inst* inst, u32 handle, Id address, Id value); Id EmitGetAttribute(EmitContext& ctx, IR::Attribute attr, u32 comp); Id EmitGetAttributeU32(EmitContext& ctx, IR::Attribute attr, u32 comp); void EmitSetAttribute(EmitContext& ctx, IR::Attribute attr, Id value, u32 comp); diff --git a/src/shader_recompiler/frontend/translate/vector_memory.cpp b/src/shader_recompiler/frontend/translate/vector_memory.cpp index 28b7885ed..f293a2205 100644 --- a/src/shader_recompiler/frontend/translate/vector_memory.cpp +++ b/src/shader_recompiler/frontend/translate/vector_memory.cpp @@ -440,80 +440,47 @@ void Translator::BUFFER_STORE_FORMAT(u32 num_dwords, bool is_typed, bool is_form } void Translator::BUFFER_ATOMIC(u32 num_dwords, AtomicOp op, const GcnInst& inst) { + // Get controls for mubuf-specific instructions const auto& mubuf = inst.control.mubuf; + // Get vaddr register const IR::VectorReg vaddr{inst.src[0].code}; + // Get vdata register + const IR::VectorReg vdata{inst.src[1].code}; + // Get srsrc register const IR::ScalarReg srsrc{inst.src[2].code * 4}; + // Get offset value from soffset register + const IR::U32 soffset{GetSrc(inst.src[3])}; // TODO: Use this maybe? + ASSERT_MSG(soffset.IsImmediate() && soffset.U32() == 0, "Non immediate offset not supported"); + + // Setup instruction info from controls + IR::BufferInstInfo info{}; + info.index_enable.Assign(mubuf.idxen); + info.inst_offset.Assign(mubuf.offset); + info.offset_enable.Assign(mubuf.offen); + + // Get vdata value(s) + IR::Value vdata_val = ir.GetVectorReg(vdata); + + // Get address of vdata + const IR::U32 address = ir.GetVectorReg(vaddr); + + // Construct srsrc SGPRs const IR::Value handle = ir.CompositeConstruct(ir.GetScalarReg(srsrc), ir.GetScalarReg(srsrc + 1), ir.GetScalarReg(srsrc + 2), ir.GetScalarReg(srsrc + 3)); - /*const IR::Value address = [&] -> IR::Value { - if (mubuf.idxen && mubuf.offen) { - return ir.CompositeConstruct(ir.GetVectorReg(vaddr), ir.GetVectorReg(vaddr + 1)); - } - if (mubuf.idxen || mubuf.offen) { - return ir.GetVectorReg(vaddr); - } - return {}; - }();*/ - const IR::Value soffset{GetSrc(inst.src[3])}; - ASSERT_MSG(soffset.IsImmediate() && soffset.U32() == 0, "Non immediate offset not supported"); - IR::BufferInstInfo info{}; - info.index_enable.Assign(mubuf.idxen); - info.offset_enable.Assign(mubuf.offen); - info.inst_offset.Assign(mubuf.offset); + // Get current srsrc value + IR::U32 prev_val = GetSrc(inst.src[2]); - const IR::Value tst{GetSrc(inst.src[1])}; - - IR::Value value{}; - const IR::VectorReg src_reg{inst.src[1].code}; - switch (num_dwords) { - case 1: - value = ir.GetVectorReg(src_reg); - break; - case 2: - value = ir.CompositeConstruct(ir.GetVectorReg(src_reg), - ir.GetVectorReg(src_reg + 1)); - break; - } - const IR::Value handle{GetSrc(inst.src[2])}; - - const IR::Value result = [&] { - switch (op) { - case AtomicOp::Swap: - return ir.BufferAtomicExchange(handle, value, info); - case AtomicOp::Add: - if (num_dwords == 1) { - return ir.BufferAtomicIAdd(handle, tst, info); - } else if (num_dwords == 2) { - // return ir.BufferAtomicFAdd(handle, final_address, value, info); - } - case AtomicOp::Smin: - return ir.BufferAtomicIMin(handle, value, true, info); - case AtomicOp::Umin: - return ir.BufferAtomicIMin(handle, value, false, info); - case AtomicOp::Smax: - return ir.BufferAtomicIMax(handle, value, true, info); - case AtomicOp::Umax: - return ir.BufferAtomicIMax(handle, value, false, info); - case AtomicOp::And: - return ir.BufferAtomicAnd(handle, value, info); - case AtomicOp::Or: - return ir.BufferAtomicOr(handle, value, info); - case AtomicOp::Xor: - return ir.BufferAtomicXor(handle, value, info); - case AtomicOp::Inc: - return ir.BufferAtomicInc(handle, value, info); - case AtomicOp::Dec: - return ir.BufferAtomicDec(handle, value, info); - default: - UNREACHABLE(); - } - }(); + // Apply atomic op + // derefs srsrc buffer and adds vdata value to it + const IR::U32 new_vdata = IR::U32{ir.BufferAtomicIAdd(handle, address, vdata_val, info)}; if (mubuf.glc) { - ir.SetVectorReg(src_reg, IR::U32{result}); + ir.SetVectorReg(vdata, new_vdata); } + + return; } void Translator::IMAGE_GET_LOD(const GcnInst& inst) { diff --git a/src/shader_recompiler/ir/ir_emitter.cpp b/src/shader_recompiler/ir/ir_emitter.cpp index 1fb63e120..76425723d 100644 --- a/src/shader_recompiler/ir/ir_emitter.cpp +++ b/src/shader_recompiler/ir/ir_emitter.cpp @@ -369,50 +369,52 @@ void IREmitter::StoreBuffer(int num_dwords, const Value& handle, const Value& ad } } -Value IREmitter::BufferAtomicIAdd(const Value& handle, const Value& value, - BufferInstInfo info) { - return Inst(Opcode::BufferAtomicIAdd32, /*Flags{info},*/ handle, value); +Value IREmitter::BufferAtomicIAdd(const Value& handle, const Value& address, + const Value& value, BufferInstInfo info) { + return Inst(Opcode::BufferAtomicIAdd32, Flags{info}, handle, address, value); } -Value IREmitter::BufferAtomicIMin(const Value& handle, const Value& value, +Value IREmitter::BufferAtomicIMin(const Value& handle, const Value& address, + const Value& value, bool is_signed, BufferInstInfo info) { return is_signed ? Inst(Opcode::BufferAtomicSMin32, Flags{info}, handle, value) : Inst(Opcode::BufferAtomicUMin32, Flags{info}, handle, value); } -Value IREmitter::BufferAtomicIMax(const Value& handle, const Value& value, +Value IREmitter::BufferAtomicIMax(const Value& handle, const Value& address, + const Value& value, bool is_signed, BufferInstInfo info) { return is_signed ? Inst(Opcode::BufferAtomicSMax32, Flags{info}, handle, value) : Inst(Opcode::BufferAtomicUMax32, Flags{info}, handle, value); } -Value IREmitter::BufferAtomicInc(const Value& handle, const Value& value, - BufferInstInfo info) { +Value IREmitter::BufferAtomicInc(const Value& handle, const Value& address, + const Value& value, BufferInstInfo info) { return Inst(Opcode::BufferAtomicInc32, Flags{info}, handle, value); } -Value IREmitter::BufferAtomicDec(const Value& handle, const Value& value, - BufferInstInfo info) { +Value IREmitter::BufferAtomicDec(const Value& handle, const Value& address, + const Value& value, BufferInstInfo info) { return Inst(Opcode::BufferAtomicDec32, Flags{info}, handle, value); } -Value IREmitter::BufferAtomicAnd(const Value& handle, const Value& value, - BufferInstInfo info) { +Value IREmitter::BufferAtomicAnd(const Value& handle, const Value& address, + const Value& value, BufferInstInfo info) { return Inst(Opcode::BufferAtomicAnd32, Flags{info}, handle, value); } -Value IREmitter::BufferAtomicOr(const Value& handle, const Value& value, - BufferInstInfo info) { +Value IREmitter::BufferAtomicOr(const Value& handle, const Value& address, + const Value& value, BufferInstInfo info) { return Inst(Opcode::BufferAtomicOr32, Flags{info}, handle, value); } -Value IREmitter::BufferAtomicXor(const Value& handle, const Value& value, - BufferInstInfo info) { +Value IREmitter::BufferAtomicXor(const Value& handle, const Value& address, + const Value& value, BufferInstInfo info) { return Inst(Opcode::BufferAtomicXor32, Flags{info}, handle, value); } -Value IREmitter::BufferAtomicExchange(const Value& handle, const Value& value, - BufferInstInfo info) { +Value IREmitter::BufferAtomicExchange(const Value& handle, const Value& address, + const Value& value, BufferInstInfo info) { return Inst(Opcode::BufferAtomicExchange32, Flags{info}, handle, value); } diff --git a/src/shader_recompiler/ir/ir_emitter.h b/src/shader_recompiler/ir/ir_emitter.h index cd90c3d40..71dc2cc05 100644 --- a/src/shader_recompiler/ir/ir_emitter.h +++ b/src/shader_recompiler/ir/ir_emitter.h @@ -100,23 +100,25 @@ public: void StoreBufferFormat(int num_dwords, const Value& handle, const Value& address, const Value& data, BufferInstInfo info); - [[nodiscard]] Value BufferAtomicIAdd(const Value& handle, const Value& a, - BufferInstInfo info); - [[nodiscard]] Value BufferAtomicIMin(const Value& handle, const Value& a, + [[nodiscard]] Value BufferAtomicIAdd(const Value& handle, const Value& address, + const Value& value, BufferInstInfo info); + [[nodiscard]] Value BufferAtomicIMin(const Value& handle, const Value& address, + const Value& a, bool is_signed, BufferInstInfo info); - [[nodiscard]] Value BufferAtomicIMax(const Value& handle, const Value& a, + [[nodiscard]] Value BufferAtomicIMax(const Value& handle, const Value& address, + const Value& a, bool is_signed, BufferInstInfo info); - [[nodiscard]] Value BufferAtomicInc(const Value& handle, + [[nodiscard]] Value BufferAtomicInc(const Value& handle, const Value& address, const Value& value, BufferInstInfo info); - [[nodiscard]] Value BufferAtomicDec(const Value& handle, + [[nodiscard]] Value BufferAtomicDec(const Value& handle, const Value& address, const Value& value, BufferInstInfo info); - [[nodiscard]] Value BufferAtomicAnd(const Value& handle, + [[nodiscard]] Value BufferAtomicAnd(const Value& handle, const Value& address, const Value& value, BufferInstInfo info); - [[nodiscard]] Value BufferAtomicOr(const Value& handle, + [[nodiscard]] Value BufferAtomicOr(const Value& handle, const Value& address, const Value& value, BufferInstInfo info); - [[nodiscard]] Value BufferAtomicXor(const Value& handle, + [[nodiscard]] Value BufferAtomicXor(const Value& handle, const Value& address, const Value& value, BufferInstInfo info); - [[nodiscard]] Value BufferAtomicExchange(const Value& handle, + [[nodiscard]] Value BufferAtomicExchange(const Value& handle, const Value& address, const Value& value, BufferInstInfo info); [[nodiscard]] U32 LaneId(); diff --git a/src/shader_recompiler/ir/opcodes.inc b/src/shader_recompiler/ir/opcodes.inc index 4c6346dc5..a86a4fd48 100644 --- a/src/shader_recompiler/ir/opcodes.inc +++ b/src/shader_recompiler/ir/opcodes.inc @@ -96,7 +96,7 @@ OPCODE(StoreBufferFormatF32x4, Void, Opaq OPCODE(StoreBufferU32, Void, Opaque, Opaque, U32, ) // Buffer atomic operations -OPCODE(BufferAtomicIAdd32, U32, Opaque, F32, ) +OPCODE(BufferAtomicIAdd32, U32, Opaque, Opaque, Opaque ) OPCODE(BufferAtomicSMin32, U32, U32, U32, ) OPCODE(BufferAtomicUMin32, U32, U32, U32, ) OPCODE(BufferAtomicSMax32, U32, U32, U32, )