mirrors/shadPS4
1
0
Fork 0
mirror of https://github.com/shadps4-emu/shadPS4.git synced 2025-12-10 05:38:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

video_core: Implement DMA. (#2819)

Browse Source 
* Import memory

* 64K pages and fix memory mapping

* Queue coverage

* Buffer syncing, faulted readback adn BDA in Buffer

* Base DMA implementation

* Preparations for implementing SPV DMA access

* Base impl (pending 16K pages and getbuffersize)

* 16K pages and stack overflow fix

* clang-format

* clang-format but for real this time

* Try to fix macOS build

* Correct decltype

* Add testing log

* Fix stride and patch phi node blocks

* No need to check if it is a deleted buffer

* Clang format once more

* Offset in bytes

* Removed host buffers (may do it in another PR)

Also some random barrier fixes

* Add IR dumping from my read-const branch

* clang-format

* Correct size insteed of end

* Fix incorrect assert

* Possible fix for NieR deadlock

* Copy to avoid deadlock

* Use 2 mutexes insteed of copy

* Attempt to range sync error

* Revert "Attempt to range sync error"

This reverts commit dd287b48682b50f215680bb0956e39c2809bf3fe.

* Fix size truncated when syncing range

And memory barrier

* Some fixes (and async testing (doesn't work))

* Use compute to parse fault buffer

* Process faults on submit

* Only sync in the first time we see a readconst

Thsi is partialy wrong. We need to save the state into the submission context itself, not the rasterizer since we can yield and process another sumission (if im not understanding wrong).

* Use spec const and 32 bit atomic

* 32 bit counter

* Fix store_index

* Better sync (WIP, breaks PR now)

* Fixes for better sync

* Better sync

* Remove memory coveragte logic

* Point sirit to upstream

* Less waiting and barriers

* Correctly checkout moltenvk

* Bring back applying pending operations in wait

* Sync the whole buffer insteed of only the range

* Implement recursive shared/scoped locks

* Iterators

* Faster syncing with ranges

* Some alignment fixes

* fixed clang format

* Fix clang-format again

* Port page_manager from readbacks-poc

* clang-format

* Defer memory protect

* Remove RENDERER_TRACE

* Experiment: only sync on first readconst

* Added profiling (will be removed)

* Don't sync entire buffers

* Added logging for testing

* Updated temporary workaround to use 4k pages

* clang.-format

* Cleanup part 1

* Make ReadConst a SPIR-V function

---------

Co-authored-by: georgemoralis <giorgosmrls@gmail.com>
This commit is contained in:
Lander Gallastegi
2025-05-22 20:00:15 +02:00
committed by GitHub
parent 37887e8fde
commit f9bbde9c79
40 changed files with 1641 additions and 311 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/video_core/amdgpu/liverpool.cpp
View File

@@ -133,6 +133,7 @@ void Liverpool::Process(std::stop_token stoken) {
VideoCore::EndCapture();
if (rasterizer) {
rasterizer->ProcessFaults();
rasterizer->Flush();
}
submit_done = false;

7
src/video_core/amdgpu/resource.h
View File

@@ -37,6 +37,13 @@ struct Buffer {
return buffer;
}
static constexpr Buffer Placeholder(u32 size) {
Buffer buffer{};
buffer.base_address = 1;
buffer.num_records = size;
return buffer;
}
bool Valid() const {
return type == 0u;
}

14
src/video_core/buffer_cache/buffer.cpp
View File

@@ -70,8 +70,11 @@ UniqueBuffer::~UniqueBuffer() {
void UniqueBuffer::Create(const vk::BufferCreateInfo& buffer_ci, MemoryUsage usage,
VmaAllocationInfo* out_alloc_info) {
const bool with_bda = bool(buffer_ci.usage & vk::BufferUsageFlagBits::eShaderDeviceAddress);
const VmaAllocationCreateFlags bda_flag =
with_bda ? VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT : 0;
const VmaAllocationCreateInfo alloc_ci = {
.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage),
.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | bda_flag | MemoryUsageVmaFlags(usage),
.usage = MemoryUsageVma(usage),
.requiredFlags = 0,
.preferredFlags = MemoryUsagePreferredVmaFlags(usage),
@@ -86,6 +89,15 @@ void UniqueBuffer::Create(const vk::BufferCreateInfo& buffer_ci, MemoryUsage usa
ASSERT_MSG(result == VK_SUCCESS, "Failed allocating buffer with error {}",
vk::to_string(vk::Result{result}));
buffer = vk::Buffer{unsafe_buffer};
if (with_bda) {
vk::BufferDeviceAddressInfo bda_info{
.buffer = buffer,
};
auto bda_result = device.getBufferAddress(bda_info);
ASSERT_MSG(bda_result != 0, "Failed to get buffer device address");
bda_addr = bda_result;
}
}
Buffer::Buffer(const Vulkan::Instance& instance_, Vulkan::Scheduler& scheduler_, MemoryUsage usage_,

6
src/video_core/buffer_cache/buffer.h
View File

@@ -68,6 +68,7 @@ struct UniqueBuffer {
VmaAllocator allocator;
VmaAllocation allocation;
vk::Buffer buffer{};
vk::DeviceAddress bda_addr = 0;
};
class Buffer {
@@ -115,6 +116,11 @@ public:
return buffer;
}
vk::DeviceAddress BufferDeviceAddress() const noexcept {
ASSERT_MSG(buffer.bda_addr != 0, "Can't get BDA from a non BDA buffer");
return buffer.bda_addr;
}
std::optional<vk::BufferMemoryBarrier2> GetBarrier(
vk::Flags<vk::AccessFlagBits2> dst_acess_mask, vk::PipelineStageFlagBits2 dst_stage,
u32 offset = 0) {

470
src/video_core/buffer_cache/buffer_cache.cpp
View File

@@ -3,13 +3,17 @@
#include <algorithm>
#include "common/alignment.h"
#include "common/debug.h"
#include "common/scope_exit.h"
#include "common/types.h"
#include "video_core/amdgpu/liverpool.h"
#include "video_core/buffer_cache/buffer_cache.h"
#include "video_core/host_shaders/fault_buffer_process_comp.h"
#include "video_core/renderer_vulkan/vk_graphics_pipeline.h"
#include "video_core/renderer_vulkan/vk_instance.h"
#include "video_core/renderer_vulkan/vk_rasterizer.h"
#include "video_core/renderer_vulkan/vk_scheduler.h"
#include "video_core/renderer_vulkan/vk_shader_util.h"
#include "video_core/texture_cache/texture_cache.h"
namespace VideoCore {
@@ -17,17 +21,26 @@ namespace VideoCore {
static constexpr size_t DataShareBufferSize = 64_KB;
static constexpr size_t StagingBufferSize = 512_MB;
static constexpr size_t UboStreamBufferSize = 128_MB;
static constexpr size_t DownloadBufferSize = 128_MB;
static constexpr size_t MaxPageFaults = 1024;
BufferCache::BufferCache(const Vulkan::Instance& instance_, Vulkan::Scheduler& scheduler_,
AmdGpu::Liverpool* liverpool_, TextureCache& texture_cache_,
PageManager& tracker_)
: instance{instance_}, scheduler{scheduler_}, liverpool{liverpool_},
Vulkan::Rasterizer& rasterizer_, AmdGpu::Liverpool* liverpool_,
TextureCache& texture_cache_, PageManager& tracker_)
: instance{instance_}, scheduler{scheduler_}, rasterizer{rasterizer_}, liverpool{liverpool_},
texture_cache{texture_cache_}, tracker{tracker_},
staging_buffer{instance, scheduler, MemoryUsage::Upload, StagingBufferSize},
stream_buffer{instance, scheduler, MemoryUsage::Stream, UboStreamBufferSize},
download_buffer(instance, scheduler, MemoryUsage::Download, DownloadBufferSize),
gds_buffer{instance, scheduler, MemoryUsage::Stream, 0, AllFlags, DataShareBufferSize},
memory_tracker{&tracker} {
bda_pagetable_buffer{instance, scheduler, MemoryUsage::DeviceLocal,
0, AllFlags, BDA_PAGETABLE_SIZE},
fault_buffer(instance, scheduler, MemoryUsage::DeviceLocal, 0, AllFlags, FAULT_BUFFER_SIZE),
memory_tracker{tracker} {
Vulkan::SetObjectName(instance.GetDevice(), gds_buffer.Handle(), "GDS Buffer");
Vulkan::SetObjectName(instance.GetDevice(), bda_pagetable_buffer.Handle(),
"BDA Page Table Buffer");
Vulkan::SetObjectName(instance.GetDevice(), fault_buffer.Handle(), "Fault Buffer");
// Ensure the first slot is used for the null buffer
const auto null_id =
@@ -35,15 +48,93 @@ BufferCache::BufferCache(const Vulkan::Instance& instance_, Vulkan::Scheduler& s
ASSERT(null_id.index == 0);
const vk::Buffer& null_buffer = slot_buffers[null_id].buffer;
Vulkan::SetObjectName(instance.GetDevice(), null_buffer, "Null Buffer");
// Prepare the fault buffer parsing pipeline
boost::container::static_vector<vk::DescriptorSetLayoutBinding, 2> bindings{
{
.binding = 0,
.descriptorType = vk::DescriptorType::eStorageBuffer,
.descriptorCount = 1,
.stageFlags = vk::ShaderStageFlagBits::eCompute,
},
{
.binding = 1,
.descriptorType = vk::DescriptorType::eStorageBuffer,
.descriptorCount = 1,
.stageFlags = vk::ShaderStageFlagBits::eCompute,
},
};
const vk::DescriptorSetLayoutCreateInfo desc_layout_ci = {
.flags = vk::DescriptorSetLayoutCreateFlagBits::ePushDescriptorKHR,
.bindingCount = static_cast<u32>(bindings.size()),
.pBindings = bindings.data(),
};
auto [desc_layout_result, desc_layout] =
instance.GetDevice().createDescriptorSetLayoutUnique(desc_layout_ci);
ASSERT_MSG(desc_layout_result == vk::Result::eSuccess,
"Failed to create descriptor set layout: {}", vk::to_string(desc_layout_result));
fault_process_desc_layout = std::move(desc_layout);
const auto& module = Vulkan::Compile(HostShaders::FAULT_BUFFER_PROCESS_COMP,
vk::ShaderStageFlagBits::eCompute, instance.GetDevice());
Vulkan::SetObjectName(instance.GetDevice(), module, "Fault Buffer Parser");
const vk::SpecializationMapEntry specialization_map_entry = {
.constantID = 0,
.offset = 0,
.size = sizeof(u32),
};
const vk::SpecializationInfo specialization_info = {
.mapEntryCount = 1,
.pMapEntries = &specialization_map_entry,
.dataSize = sizeof(u32),
.pData = &CACHING_PAGEBITS,
};
const vk::PipelineShaderStageCreateInfo shader_ci = {
.stage = vk::ShaderStageFlagBits::eCompute,
.module = module,
.pName = "main",
.pSpecializationInfo = &specialization_info,
};
const vk::PipelineLayoutCreateInfo layout_info = {
.setLayoutCount = 1U,
.pSetLayouts = &(*fault_process_desc_layout),
};
auto [layout_result, layout] = instance.GetDevice().createPipelineLayoutUnique(layout_info);
ASSERT_MSG(layout_result == vk::Result::eSuccess, "Failed to create pipeline layout: {}",
vk::to_string(layout_result));
fault_process_pipeline_layout = std::move(layout);
const vk::ComputePipelineCreateInfo pipeline_info = {
.stage = shader_ci,
.layout = *fault_process_pipeline_layout,
};
auto [pipeline_result, pipeline] =
instance.GetDevice().createComputePipelineUnique({}, pipeline_info);
ASSERT_MSG(pipeline_result == vk::Result::eSuccess, "Failed to create compute pipeline: {}",
vk::to_string(pipeline_result));
fault_process_pipeline = std::move(pipeline);
Vulkan::SetObjectName(instance.GetDevice(), *fault_process_pipeline,
"Fault Buffer Parser Pipeline");
instance.GetDevice().destroyShaderModule(module);
}
BufferCache::~BufferCache() = default;
void BufferCache::InvalidateMemory(VAddr device_addr, u64 size) {
void BufferCache::InvalidateMemory(VAddr device_addr, u64 size, bool unmap) {
const bool is_tracked = IsRegionRegistered(device_addr, size);
if (is_tracked) {
// Mark the page as CPU modified to stop tracking writes.
memory_tracker.MarkRegionAsCpuModified(device_addr, size);
if (unmap) {
return;
}
}
}
@@ -69,20 +160,20 @@ void BufferCache::DownloadBufferMemory(Buffer& buffer, VAddr device_addr, u64 si
if (total_size_bytes == 0) {
return;
}
const auto [staging, offset] = staging_buffer.Map(total_size_bytes);
const auto [download, offset] = download_buffer.Map(total_size_bytes);
for (auto& copy : copies) {
// Modify copies to have the staging offset in mind
copy.dstOffset += offset;
}
staging_buffer.Commit();
download_buffer.Commit();
scheduler.EndRendering();
const auto cmdbuf = scheduler.CommandBuffer();
cmdbuf.copyBuffer(buffer.buffer, staging_buffer.Handle(), copies);
cmdbuf.copyBuffer(buffer.buffer, download_buffer.Handle(), copies);
scheduler.Finish();
for (const auto& copy : copies) {
const VAddr copy_device_addr = buffer.CpuAddr() + copy.srcOffset;
const u64 dst_offset = copy.dstOffset - offset;
std::memcpy(std::bit_cast<u8*>(copy_device_addr), staging + dst_offset, copy.size);
std::memcpy(std::bit_cast<u8*>(copy_device_addr), download + dst_offset, copy.size);
}
}
@@ -206,58 +297,37 @@ void BufferCache::InlineData(VAddr address, const void* value, u32 num_bytes, bo
memcpy(std::bit_cast<void*>(address), value, num_bytes);
return;
}
scheduler.EndRendering();
const Buffer* buffer = [&] {
Buffer* buffer = [&] {
if (is_gds) {
return &gds_buffer;
}
const BufferId buffer_id = FindBuffer(address, num_bytes);
return &slot_buffers[buffer_id];
}();
const auto cmdbuf = scheduler.CommandBuffer();
const vk::BufferMemoryBarrier2 pre_barrier = {
.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands,
.srcAccessMask = vk::AccessFlagBits2::eMemoryRead,
.dstStageMask = vk::PipelineStageFlagBits2::eTransfer,
.dstAccessMask = vk::AccessFlagBits2::eTransferWrite,
.buffer = buffer->Handle(),
.offset = buffer->Offset(address),
.size = num_bytes,
};
const vk::BufferMemoryBarrier2 post_barrier = {
.srcStageMask = vk::PipelineStageFlagBits2::eTransfer,
.srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands,
.dstAccessMask = vk::AccessFlagBits2::eMemoryRead,
.buffer = buffer->Handle(),
.offset = buffer->Offset(address),
.size = num_bytes,
};
cmdbuf.pipelineBarrier2(vk::DependencyInfo{
.dependencyFlags = vk::DependencyFlagBits::eByRegion,
.bufferMemoryBarrierCount = 1,
.pBufferMemoryBarriers = &pre_barrier,
});
// vkCmdUpdateBuffer can only copy up to 65536 bytes at a time.
static constexpr u32 UpdateBufferMaxSize = 65536;
const auto dst_offset = buffer->Offset(address);
for (u32 offset = 0; offset < num_bytes; offset += UpdateBufferMaxSize) {
const auto* update_src = static_cast<const u8*>(value) + offset;
const auto update_dst = dst_offset + offset;
const auto update_size = std::min(num_bytes - offset, UpdateBufferMaxSize);
cmdbuf.updateBuffer(buffer->Handle(), update_dst, update_size, update_src);
InlineDataBuffer(*buffer, address, value, num_bytes);
}
void BufferCache::WriteData(VAddr address, const void* value, u32 num_bytes, bool is_gds) {
ASSERT_MSG(address % 4 == 0, "GDS offset must be dword aligned");
if (!is_gds && !IsRegionRegistered(address, num_bytes)) {
memcpy(std::bit_cast<void*>(address), value, num_bytes);
return;
}
cmdbuf.pipelineBarrier2(vk::DependencyInfo{
.dependencyFlags = vk::DependencyFlagBits::eByRegion,
.bufferMemoryBarrierCount = 1,
.pBufferMemoryBarriers = &post_barrier,
});
Buffer* buffer = [&] {
if (is_gds) {
return &gds_buffer;
}
const BufferId buffer_id = FindBuffer(address, num_bytes);
return &slot_buffers[buffer_id];
}();
WriteDataBuffer(*buffer, address, value, num_bytes);
}
std::pair<Buffer*, u32> BufferCache::ObtainBuffer(VAddr device_addr, u32 size, bool is_written,
bool is_texel_buffer, BufferId buffer_id) {
// For small uniform buffers that have not been modified by gpu
// use device local stream buffer to reduce renderpass breaks.
// Maybe we want to modify the threshold now that the page size is 16KB?
static constexpr u64 StreamThreshold = CACHING_PAGESIZE;
const bool is_gpu_dirty = memory_tracker.IsRegionGpuModified(device_addr, size);
if (!is_written && size <= StreamThreshold && !is_gpu_dirty) {
@@ -280,7 +350,7 @@ std::pair<Buffer*, u32> BufferCache::ObtainBuffer(VAddr device_addr, u32 size, b
std::pair<Buffer*, u32> BufferCache::ObtainViewBuffer(VAddr gpu_addr, u32 size, bool prefer_gpu) {
// Check if any buffer contains the full requested range.
const u64 page = gpu_addr >> CACHING_PAGEBITS;
const BufferId buffer_id = page_table[page];
const BufferId buffer_id = page_table[page].buffer_id;
if (buffer_id) {
Buffer& buffer = slot_buffers[buffer_id];
if (buffer.IsInBounds(gpu_addr, size)) {
@@ -300,24 +370,8 @@ std::pair<Buffer*, u32> BufferCache::ObtainViewBuffer(VAddr gpu_addr, u32 size,
}
bool BufferCache::IsRegionRegistered(VAddr addr, size_t size) {
const VAddr end_addr = addr + size;
const u64 page_end = Common::DivCeil(end_addr, CACHING_PAGESIZE);
for (u64 page = addr >> CACHING_PAGEBITS; page < page_end;) {
const BufferId buffer_id = page_table[page];
if (!buffer_id) {
++page;
continue;
}
std::shared_lock lk{mutex};
Buffer& buffer = slot_buffers[buffer_id];
const VAddr buf_start_addr = buffer.CpuAddr();
const VAddr buf_end_addr = buf_start_addr + buffer.SizeBytes();
if (buf_start_addr < end_addr && addr < buf_end_addr) {
return true;
}
page = Common::DivCeil(buf_end_addr, CACHING_PAGESIZE);
}
return false;
// Check if we are missing some edge case here
return buffer_ranges.Intersects(addr, size);
}
bool BufferCache::IsRegionCpuModified(VAddr addr, size_t size) {
@@ -333,7 +387,7 @@ BufferId BufferCache::FindBuffer(VAddr device_addr, u32 size) {
return NULL_BUFFER_ID;
}
const u64 page = device_addr >> CACHING_PAGEBITS;
const BufferId buffer_id = page_table[page];
const BufferId buffer_id = page_table[page].buffer_id;
if (!buffer_id) {
return CreateBuffer(device_addr, size);
}
@@ -379,7 +433,7 @@ BufferCache::OverlapResult BufferCache::ResolveOverlaps(VAddr device_addr, u32 w
}
for (; device_addr >> CACHING_PAGEBITS < Common::DivCeil(end, CACHING_PAGESIZE);
device_addr += CACHING_PAGESIZE) {
const BufferId overlap_id = page_table[device_addr >> CACHING_PAGEBITS];
const BufferId overlap_id = page_table[device_addr >> CACHING_PAGEBITS].buffer_id;
if (!overlap_id) {
continue;
}
@@ -480,11 +534,21 @@ BufferId BufferCache::CreateBuffer(VAddr device_addr, u32 wanted_size) {
const OverlapResult overlap = ResolveOverlaps(device_addr, wanted_size);
const u32 size = static_cast<u32>(overlap.end - overlap.begin);
const BufferId new_buffer_id = [&] {
std::scoped_lock lk{mutex};
std::scoped_lock lk{slot_buffers_mutex};
return slot_buffers.insert(instance, scheduler, MemoryUsage::DeviceLocal, overlap.begin,
AllFlags, size);
AllFlags | vk::BufferUsageFlagBits::eShaderDeviceAddress, size);
}();
auto& new_buffer = slot_buffers[new_buffer_id];
boost::container::small_vector<vk::DeviceAddress, 128> bda_addrs;
const u64 start_page = overlap.begin >> CACHING_PAGEBITS;
const u64 size_pages = size >> CACHING_PAGEBITS;
bda_addrs.reserve(size_pages);
for (u64 i = 0; i < size_pages; ++i) {
vk::DeviceAddress addr = new_buffer.BufferDeviceAddress() + (i << CACHING_PAGEBITS);
bda_addrs.push_back(addr);
}
WriteDataBuffer(bda_pagetable_buffer, start_page * sizeof(vk::DeviceAddress), bda_addrs.data(),
bda_addrs.size() * sizeof(vk::DeviceAddress));
const size_t size_bytes = new_buffer.SizeBytes();
const auto cmdbuf = scheduler.CommandBuffer();
scheduler.EndRendering();
@@ -496,6 +560,129 @@ BufferId BufferCache::CreateBuffer(VAddr device_addr, u32 wanted_size) {
return new_buffer_id;
}
void BufferCache::ProcessFaultBuffer() {
// Run fault processing shader
const auto [mapped, offset] = download_buffer.Map(MaxPageFaults * sizeof(u64));
vk::BufferMemoryBarrier2 fault_buffer_barrier{
.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands,
.srcAccessMask = vk::AccessFlagBits2::eShaderWrite,
.dstStageMask = vk::PipelineStageFlagBits2::eComputeShader,
.dstAccessMask = vk::AccessFlagBits2::eShaderRead,
.buffer = fault_buffer.Handle(),
.offset = 0,
.size = FAULT_BUFFER_SIZE,
};
vk::BufferMemoryBarrier2 download_barrier{
.srcStageMask = vk::PipelineStageFlagBits2::eTransfer,
.srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
.dstStageMask = vk::PipelineStageFlagBits2::eComputeShader,
.dstAccessMask = vk::AccessFlagBits2::eShaderRead | vk::AccessFlagBits2::eShaderWrite,
.buffer = download_buffer.Handle(),
.offset = offset,
.size = MaxPageFaults * sizeof(u64),
};
std::array<vk::BufferMemoryBarrier2, 2> barriers{fault_buffer_barrier, download_barrier};
vk::DescriptorBufferInfo fault_buffer_info{
.buffer = fault_buffer.Handle(),
.offset = 0,
.range = FAULT_BUFFER_SIZE,
};
vk::DescriptorBufferInfo download_info{
.buffer = download_buffer.Handle(),
.offset = offset,
.range = MaxPageFaults * sizeof(u64),
};
boost::container::small_vector<vk::WriteDescriptorSet, 2> writes{
{
.dstSet = VK_NULL_HANDLE,
.dstBinding = 0,
.dstArrayElement = 0,
.descriptorCount = 1,
.descriptorType = vk::DescriptorType::eStorageBuffer,
.pBufferInfo = &fault_buffer_info,
},
{
.dstSet = VK_NULL_HANDLE,
.dstBinding = 1,
.dstArrayElement = 0,
.descriptorCount = 1,
.descriptorType = vk::DescriptorType::eStorageBuffer,
.pBufferInfo = &download_info,
},
};
download_buffer.Commit();
scheduler.EndRendering();
const auto cmdbuf = scheduler.CommandBuffer();
cmdbuf.fillBuffer(download_buffer.Handle(), offset, MaxPageFaults * sizeof(u64), 0);
cmdbuf.pipelineBarrier2(vk::DependencyInfo{
.dependencyFlags = vk::DependencyFlagBits::eByRegion,
.bufferMemoryBarrierCount = 2,
.pBufferMemoryBarriers = barriers.data(),
});
cmdbuf.bindPipeline(vk::PipelineBindPoint::eCompute, *fault_process_pipeline);
cmdbuf.pushDescriptorSetKHR(vk::PipelineBindPoint::eCompute, *fault_process_pipeline_layout, 0,
writes);
constexpr u32 num_threads = CACHING_NUMPAGES / 32; // 1 bit per page, 32 pages per workgroup
constexpr u32 num_workgroups = Common::DivCeil(num_threads, 64u);
cmdbuf.dispatch(num_workgroups, 1, 1);
// Reset fault buffer
const vk::BufferMemoryBarrier2 reset_pre_barrier = {
.srcStageMask = vk::PipelineStageFlagBits2::eComputeShader,
.srcAccessMask = vk::AccessFlagBits2::eShaderRead,
.dstStageMask = vk::PipelineStageFlagBits2::eTransfer,
.dstAccessMask = vk::AccessFlagBits2::eTransferWrite,
.buffer = fault_buffer.Handle(),
.offset = 0,
.size = FAULT_BUFFER_SIZE,
};
const vk::BufferMemoryBarrier2 reset_post_barrier = {
.srcStageMask = vk::PipelineStageFlagBits2::eTransfer,
.srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands,
.dstAccessMask = vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite,
.buffer = fault_buffer.Handle(),
.offset = 0,
.size = FAULT_BUFFER_SIZE,
};
cmdbuf.pipelineBarrier2(vk::DependencyInfo{
.dependencyFlags = vk::DependencyFlagBits::eByRegion,
.bufferMemoryBarrierCount = 1,
.pBufferMemoryBarriers = &reset_pre_barrier,
});
cmdbuf.fillBuffer(fault_buffer.buffer, 0, FAULT_BUFFER_SIZE, 0);
cmdbuf.pipelineBarrier2(vk::DependencyInfo{
.dependencyFlags = vk::DependencyFlagBits::eByRegion,
.bufferMemoryBarrierCount = 1,
.pBufferMemoryBarriers = &reset_post_barrier,
});
// Defer creating buffers
scheduler.DeferOperation([this, mapped]() {
// Create the fault buffers batched
boost::icl::interval_set<VAddr> fault_ranges;
const u64* fault_ptr = std::bit_cast<const u64*>(mapped);
const u32 fault_count = static_cast<u32>(*(fault_ptr++));
for (u32 i = 0; i < fault_count; ++i) {
const VAddr fault = *(fault_ptr++);
const VAddr fault_end = fault + CACHING_PAGESIZE; // This can be adjusted
fault_ranges +=
boost::icl::interval_set<VAddr>::interval_type::right_open(fault, fault_end);
LOG_INFO(Render_Vulkan, "Accessed non-GPU mapped memory at {:#x}", fault);
}
for (const auto& range : fault_ranges) {
const VAddr start = range.lower();
const VAddr end = range.upper();
const u64 page_start = start >> CACHING_PAGEBITS;
const u64 page_end = Common::DivCeil(end, CACHING_PAGESIZE);
// Buffer size is in 32 bits
ASSERT_MSG((range.upper() - range.lower()) <= std::numeric_limits<u32>::max(),
"Buffer size is too large");
CreateBuffer(start, static_cast<u32>(end - start));
}
});
}
void BufferCache::Register(BufferId buffer_id) {
ChangeRegister<true>(buffer_id);
}
@@ -514,11 +701,16 @@ void BufferCache::ChangeRegister(BufferId buffer_id) {
const u64 page_end = Common::DivCeil(device_addr_end, CACHING_PAGESIZE);
for (u64 page = page_begin; page != page_end; ++page) {
if constexpr (insert) {
page_table[page] = buffer_id;
page_table[page].buffer_id = buffer_id;
} else {
page_table[page] = BufferId{};
page_table[page].buffer_id = BufferId{};
}
}
if constexpr (insert) {
buffer_ranges.Add(buffer.CpuAddr(), buffer.SizeBytes(), buffer_id);
} else {
buffer_ranges.Subtract(buffer.CpuAddr(), buffer.SizeBytes());
}
}
void BufferCache::SynchronizeBuffer(Buffer& buffer, VAddr device_addr, u32 size,
@@ -697,6 +889,138 @@ bool BufferCache::SynchronizeBufferFromImage(Buffer& buffer, VAddr device_addr,
return true;
}
void BufferCache::SynchronizeBuffersInRange(VAddr device_addr, u64 size) {
if (device_addr == 0) {
return;
}
VAddr device_addr_end = device_addr + size;
ForEachBufferInRange(device_addr, size, [&](BufferId buffer_id, Buffer& buffer) {
RENDERER_TRACE;
VAddr start = std::max(buffer.CpuAddr(), device_addr);
VAddr end = std::min(buffer.CpuAddr() + buffer.SizeBytes(), device_addr_end);
u32 size = static_cast<u32>(end - start);
SynchronizeBuffer(buffer, start, size, false);
});
}
void BufferCache::MemoryBarrier() {
// Vulkan doesn't know which buffer we access in a shader if we use
// BufferDeviceAddress. We need a full memory barrier.
// For now, we only read memory using BDA. If we want to write to it,
// we might need to change this.
scheduler.EndRendering();
const auto cmdbuf = scheduler.CommandBuffer();
vk::MemoryBarrier2 barrier = {
.srcStageMask = vk::PipelineStageFlagBits2::eTransfer,
.srcAccessMask = vk::AccessFlagBits2::eMemoryWrite,
.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands,
.dstAccessMask = vk::AccessFlagBits2::eMemoryRead,
};
cmdbuf.pipelineBarrier2(vk::DependencyInfo{
.memoryBarrierCount = 1,
.pMemoryBarriers = &barrier,
});
}
void BufferCache::InlineDataBuffer(Buffer& buffer, VAddr address, const void* value,
u32 num_bytes) {
scheduler.EndRendering();
const auto cmdbuf = scheduler.CommandBuffer();
const vk::BufferMemoryBarrier2 pre_barrier = {
.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands,
.srcAccessMask = vk::AccessFlagBits2::eMemoryRead,
.dstStageMask = vk::PipelineStageFlagBits2::eTransfer,
.dstAccessMask = vk::AccessFlagBits2::eTransferWrite,
.buffer = buffer.Handle(),
.offset = buffer.Offset(address),
.size = num_bytes,
};
const vk::BufferMemoryBarrier2 post_barrier = {
.srcStageMask = vk::PipelineStageFlagBits2::eTransfer,
.srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands,
.dstAccessMask = vk::AccessFlagBits2::eMemoryRead,
.buffer = buffer.Handle(),
.offset = buffer.Offset(address),
.size = num_bytes,
};
cmdbuf.pipelineBarrier2(vk::DependencyInfo{
.dependencyFlags = vk::DependencyFlagBits::eByRegion,
.bufferMemoryBarrierCount = 1,
.pBufferMemoryBarriers = &pre_barrier,
});
// vkCmdUpdateBuffer can only copy up to 65536 bytes at a time.
static constexpr u32 UpdateBufferMaxSize = 65536;
const auto dst_offset = buffer.Offset(address);
for (u32 offset = 0; offset < num_bytes; offset += UpdateBufferMaxSize) {
const auto* update_src = static_cast<const u8*>(value) + offset;
const auto update_dst = dst_offset + offset;
const auto update_size = std::min(num_bytes - offset, UpdateBufferMaxSize);
cmdbuf.updateBuffer(buffer.Handle(), update_dst, update_size, update_src);
}
cmdbuf.pipelineBarrier2(vk::DependencyInfo{
.dependencyFlags = vk::DependencyFlagBits::eByRegion,
.bufferMemoryBarrierCount = 1,
.pBufferMemoryBarriers = &post_barrier,
});
}
void BufferCache::WriteDataBuffer(Buffer& buffer, VAddr address, const void* value, u32 num_bytes) {
vk::BufferCopy copy = {
.srcOffset = 0,
.dstOffset = buffer.Offset(address),
.size = num_bytes,
};
vk::Buffer src_buffer = staging_buffer.Handle();
if (num_bytes < StagingBufferSize) {
const auto [staging, offset] = staging_buffer.Map(num_bytes);
std::memcpy(staging, value, num_bytes);
copy.srcOffset = offset;
staging_buffer.Commit();
} else {
// For large one time transfers use a temporary host buffer.
// RenderDoc can lag quite a bit if the stream buffer is too large.
Buffer temp_buffer{
instance, scheduler, MemoryUsage::Upload, 0, vk::BufferUsageFlagBits::eTransferSrc,
num_bytes};
src_buffer = temp_buffer.Handle();
u8* const staging = temp_buffer.mapped_data.data();
std::memcpy(staging, value, num_bytes);
scheduler.DeferOperation([buffer = std::move(temp_buffer)]() mutable {});
}
scheduler.EndRendering();
const auto cmdbuf = scheduler.CommandBuffer();
const vk::BufferMemoryBarrier2 pre_barrier = {
.srcStageMask = vk::PipelineStageFlagBits2::eAllCommands,
.srcAccessMask = vk::AccessFlagBits2::eMemoryRead,
.dstStageMask = vk::PipelineStageFlagBits2::eTransfer,
.dstAccessMask = vk::AccessFlagBits2::eTransferWrite,
.buffer = buffer.Handle(),
.offset = buffer.Offset(address),
.size = num_bytes,
};
const vk::BufferMemoryBarrier2 post_barrier = {
.srcStageMask = vk::PipelineStageFlagBits2::eTransfer,
.srcAccessMask = vk::AccessFlagBits2::eTransferWrite,
.dstStageMask = vk::PipelineStageFlagBits2::eAllCommands,
.dstAccessMask = vk::AccessFlagBits2::eMemoryRead | vk::AccessFlagBits2::eMemoryWrite,
.buffer = buffer.Handle(),
.offset = buffer.Offset(address),
.size = num_bytes,
};
cmdbuf.pipelineBarrier2(vk::DependencyInfo{
.dependencyFlags = vk::DependencyFlagBits::eByRegion,
.bufferMemoryBarrierCount = 1,
.pBufferMemoryBarriers = &pre_barrier,
});
cmdbuf.copyBuffer(src_buffer, buffer.Handle(), copy);
cmdbuf.pipelineBarrier2(vk::DependencyInfo{
.dependencyFlags = vk::DependencyFlagBits::eByRegion,
.bufferMemoryBarrierCount = 1,
.pBufferMemoryBarriers = &post_barrier,
});
}
void BufferCache::DeleteBuffer(BufferId buffer_id) {
Buffer& buffer = slot_buffers[buffer_id];
Unregister(buffer_id);

86
src/video_core/buffer_cache/buffer_cache.h
View File

@@ -38,14 +38,22 @@ class TextureCache;
class BufferCache {
public:
static constexpr u32 CACHING_PAGEBITS = 12;
static constexpr u32 CACHING_PAGEBITS = 14;
static constexpr u64 CACHING_PAGESIZE = u64{1} << CACHING_PAGEBITS;
static constexpr u64 DEVICE_PAGESIZE = 4_KB;
static constexpr u64 DEVICE_PAGESIZE = 16_KB;
static constexpr u64 CACHING_NUMPAGES = u64{1} << (40 - CACHING_PAGEBITS);
static constexpr u64 BDA_PAGETABLE_SIZE = CACHING_NUMPAGES * sizeof(vk::DeviceAddress);
static constexpr u64 FAULT_BUFFER_SIZE = CACHING_NUMPAGES / 8; // Bit per page
struct PageData {
BufferId buffer_id{};
};
struct Traits {
using Entry = BufferId;
using Entry = PageData;
static constexpr size_t AddressSpaceBits = 40;
static constexpr size_t FirstLevelBits = 14;
static constexpr size_t FirstLevelBits = 16;
static constexpr size_t PageBits = CACHING_PAGEBITS;
};
using PageTable = MultiLevelPageTable<Traits>;
@@ -59,8 +67,8 @@ public:
public:
explicit BufferCache(const Vulkan::Instance& instance, Vulkan::Scheduler& scheduler,
AmdGpu::Liverpool* liverpool, TextureCache& texture_cache,
PageManager& tracker);
Vulkan::Rasterizer& rasterizer_, AmdGpu::Liverpool* liverpool,
TextureCache& texture_cache, PageManager& tracker);
~BufferCache();
/// Returns a pointer to GDS device local buffer.
@@ -73,13 +81,23 @@ public:
return stream_buffer;
}
/// Retrieves the device local DBA page table buffer.
[[nodiscard]] Buffer* GetBdaPageTableBuffer() noexcept {
return &bda_pagetable_buffer;
}
/// Retrieves the fault buffer.
[[nodiscard]] Buffer* GetFaultBuffer() noexcept {
return &fault_buffer;
}
/// Retrieves the buffer with the specified id.
[[nodiscard]] Buffer& GetBuffer(BufferId id) {
return slot_buffers[id];
}
/// Invalidates any buffer in the logical page range.
void InvalidateMemory(VAddr device_addr, u64 size);
void InvalidateMemory(VAddr device_addr, u64 size, bool unmap);
/// Binds host vertex buffers for the current draw.
void BindVertexBuffers(const Vulkan::GraphicsPipeline& pipeline);
@@ -87,9 +105,12 @@ public:
/// Bind host index buffer for the current draw.
void BindIndexBuffer(u32 index_offset);
/// Writes a value to GPU buffer.
/// Writes a value to GPU buffer. (uses command buffer to temporarily store the data)
void InlineData(VAddr address, const void* value, u32 num_bytes, bool is_gds);
/// Writes a value to GPU buffer. (uses staging buffer to temporarily store the data)
void WriteData(VAddr address, const void* value, u32 num_bytes, bool is_gds);
/// Obtains a buffer for the specified region.
[[nodiscard]] std::pair<Buffer*, u32> ObtainBuffer(VAddr gpu_addr, u32 size, bool is_written,
bool is_texel_buffer = false,
@@ -108,24 +129,29 @@ public:
/// Return true when a CPU region is modified from the GPU
[[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size);
[[nodiscard]] BufferId FindBuffer(VAddr device_addr, u32 size);
/// Return buffer id for the specified region
BufferId FindBuffer(VAddr device_addr, u32 size);
/// Processes the fault buffer.
void ProcessFaultBuffer();
/// Synchronizes all buffers in the specified range.
void SynchronizeBuffersInRange(VAddr device_addr, u64 size);
/// Synchronizes all buffers neede for DMA.
void SynchronizeDmaBuffers();
/// Record memory barrier. Used for buffers when accessed via BDA.
void MemoryBarrier();
private:
template <typename Func>
void ForEachBufferInRange(VAddr device_addr, u64 size, Func&& func) {
const u64 page_end = Common::DivCeil(device_addr + size, CACHING_PAGESIZE);
for (u64 page = device_addr >> CACHING_PAGEBITS; page < page_end;) {
const BufferId buffer_id = page_table[page];
if (!buffer_id) {
++page;
continue;
}
Buffer& buffer = slot_buffers[buffer_id];
func(buffer_id, buffer);
const VAddr end_addr = buffer.CpuAddr() + buffer.SizeBytes();
page = Common::DivCeil(end_addr, CACHING_PAGESIZE);
}
buffer_ranges.ForEachInRange(device_addr, size,
[&](u64 page_start, u64 page_end, BufferId id) {
Buffer& buffer = slot_buffers[id];
func(id, buffer);
});
}
void DownloadBufferMemory(Buffer& buffer, VAddr device_addr, u64 size);
@@ -134,7 +160,7 @@ private:
void JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, bool accumulate_stream_score);
[[nodiscard]] BufferId CreateBuffer(VAddr device_addr, u32 wanted_size);
BufferId CreateBuffer(VAddr device_addr, u32 wanted_size);
void Register(BufferId buffer_id);
@@ -147,21 +173,33 @@ private:
bool SynchronizeBufferFromImage(Buffer& buffer, VAddr device_addr, u32 size);
void InlineDataBuffer(Buffer& buffer, VAddr address, const void* value, u32 num_bytes);
void WriteDataBuffer(Buffer& buffer, VAddr address, const void* value, u32 num_bytes);
void DeleteBuffer(BufferId buffer_id);
const Vulkan::Instance& instance;
Vulkan::Scheduler& scheduler;
Vulkan::Rasterizer& rasterizer;
AmdGpu::Liverpool* liverpool;
TextureCache& texture_cache;
PageManager& tracker;
StreamBuffer staging_buffer;
StreamBuffer stream_buffer;
StreamBuffer download_buffer;
Buffer gds_buffer;
std::shared_mutex mutex;
Buffer bda_pagetable_buffer;
Buffer fault_buffer;
std::shared_mutex slot_buffers_mutex;
Common::SlotVector<Buffer> slot_buffers;
RangeSet gpu_modified_ranges;
SplitRangeMap<BufferId> buffer_ranges;
MemoryTracker memory_tracker;
PageTable page_table;
vk::UniqueDescriptorSetLayout fault_process_desc_layout;
vk::UniquePipeline fault_process_pipeline;
vk::UniquePipelineLayout fault_process_pipeline_layout;
};
} // namespace VideoCore

24
src/video_core/buffer_cache/memory_tracker_base.h
View File

@@ -7,6 +7,7 @@
#include <deque>
#include <type_traits>
#include <vector>
#include "common/debug.h"
#include "common/types.h"
#include "video_core/buffer_cache/word_manager.h"
@@ -19,11 +20,11 @@ public:
static constexpr size_t MANAGER_POOL_SIZE = 32;
public:
explicit MemoryTracker(PageManager* tracker_) : tracker{tracker_} {}
explicit MemoryTracker(PageManager& tracker_) : tracker{&tracker_} {}
~MemoryTracker() = default;
/// Returns true if a region has been modified from the CPU
[[nodiscard]] bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
bool IsRegionCpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
return IteratePages<true>(
query_cpu_addr, query_size, [](RegionManager* manager, u64 offset, size_t size) {
return manager->template IsRegionModified<Type::CPU>(offset, size);
@@ -31,7 +32,7 @@ public:
}
/// Returns true if a region has been modified from the GPU
[[nodiscard]] bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
bool IsRegionGpuModified(VAddr query_cpu_addr, u64 query_size) noexcept {
return IteratePages<false>(
query_cpu_addr, query_size, [](RegionManager* manager, u64 offset, size_t size) {
return manager->template IsRegionModified<Type::GPU>(offset, size);
@@ -57,8 +58,7 @@ public:
}
/// Call 'func' for each CPU modified range and unmark those pages as CPU modified
template <typename Func>
void ForEachUploadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
void ForEachUploadRange(VAddr query_cpu_range, u64 query_size, auto&& func) {
IteratePages<true>(query_cpu_range, query_size,
[&func](RegionManager* manager, u64 offset, size_t size) {
manager->template ForEachModifiedRange<Type::CPU, true>(
@@ -67,17 +67,12 @@ public:
}
/// Call 'func' for each GPU modified range and unmark those pages as GPU modified
template <bool clear, typename Func>
void ForEachDownloadRange(VAddr query_cpu_range, u64 query_size, Func&& func) {
template <bool clear>
void ForEachDownloadRange(VAddr query_cpu_range, u64 query_size, auto&& func) {
IteratePages<false>(query_cpu_range, query_size,
[&func](RegionManager* manager, u64 offset, size_t size) {
if constexpr (clear) {
manager->template ForEachModifiedRange<Type::GPU, true>(
manager->GetCpuAddr() + offset, size, func);
} else {
manager->template ForEachModifiedRange<Type::GPU, false>(
manager->GetCpuAddr() + offset, size, func);
}
manager->template ForEachModifiedRange<Type::GPU, clear>(
manager->GetCpuAddr() + offset, size, func);
});
}
@@ -91,6 +86,7 @@ private:
*/
template <bool create_region_on_fail, typename Func>
bool IteratePages(VAddr cpu_address, size_t size, Func&& func) {
RENDERER_TRACE;
using FuncReturn = typename std::invoke_result<Func, RegionManager*, u64, size_t>::type;
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
std::size_t remaining_size{size};

175
src/video_core/buffer_cache/range_set.h
View File

@@ -3,7 +3,10 @@
#pragma once
#include <boost/icl/discrete_interval.hpp>
#include <boost/icl/interval_map.hpp>
#include <boost/icl/split_interval_map.hpp>
#include <boost/icl/split_interval_set.hpp>
#include <boost/pool/pool.hpp>
#include <boost/pool/pool_alloc.hpp>
#include <boost/pool/poolfwd.hpp>
@@ -38,6 +41,22 @@ struct RangeSet {
m_ranges_set.subtract(interval);
}
void Clear() {
m_ranges_set.clear();
}
bool Contains(VAddr base_address, size_t size) const {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
return boost::icl::contains(m_ranges_set, interval);
}
bool Intersects(VAddr base_address, size_t size) const {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
return boost::icl::intersects(m_ranges_set, interval);
}
template <typename Func>
void ForEach(Func&& func) const {
if (m_ranges_set.empty()) {
@@ -77,14 +96,29 @@ struct RangeSet {
}
}
template <typename Func>
void ForEachNotInRange(VAddr base_addr, size_t size, Func&& func) const {
const VAddr end_addr = base_addr + size;
ForEachInRange(base_addr, size, [&](VAddr range_addr, VAddr range_end) {
if (size_t gap_size = range_addr - base_addr; gap_size != 0) {
func(base_addr, gap_size);
}
base_addr = range_end;
});
if (base_addr != end_addr) {
func(base_addr, end_addr - base_addr);
}
}
IntervalSet m_ranges_set;
};
template <typename T>
class RangeMap {
public:
using IntervalMap =
boost::icl::interval_map<VAddr, u64, boost::icl::partial_absorber, std::less,
boost::icl::inplace_plus, boost::icl::inter_section,
boost::icl::interval_map<VAddr, T, boost::icl::total_absorber, std::less,
boost::icl::inplace_identity, boost::icl::inter_section,
ICL_INTERVAL_INSTANCE(ICL_INTERVAL_DEFAULT, VAddr, std::less),
RangeSetsAllocator>;
using IntervalType = typename IntervalMap::interval_type;
@@ -99,7 +133,7 @@ public:
RangeMap(RangeMap&& other);
RangeMap& operator=(RangeMap&& other);
void Add(VAddr base_address, size_t size, u64 value) {
void Add(VAddr base_address, size_t size, const T& value) {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
m_ranges_map.add({interval, value});
@@ -111,6 +145,35 @@ public:
m_ranges_map -= interval;
}
void Clear() {
m_ranges_map.clear();
}
bool Contains(VAddr base_address, size_t size) const {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
return boost::icl::contains(m_ranges_map, interval);
}
bool Intersects(VAddr base_address, size_t size) const {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
return boost::icl::intersects(m_ranges_map, interval);
}
template <typename Func>
void ForEach(Func&& func) const {
if (m_ranges_map.empty()) {
return;
}
for (const auto& [interval, value] : m_ranges_map) {
const VAddr inter_addr_end = interval.upper();
const VAddr inter_addr = interval.lower();
func(inter_addr, inter_addr_end, value);
}
}
template <typename Func>
void ForEachInRange(VAddr base_addr, size_t size, Func&& func) const {
if (m_ranges_map.empty()) {
@@ -140,7 +203,111 @@ public:
template <typename Func>
void ForEachNotInRange(VAddr base_addr, size_t size, Func&& func) const {
const VAddr end_addr = base_addr + size;
ForEachInRange(base_addr, size, [&](VAddr range_addr, VAddr range_end, u64) {
ForEachInRange(base_addr, size, [&](VAddr range_addr, VAddr range_end, const T&) {
if (size_t gap_size = range_addr - base_addr; gap_size != 0) {
func(base_addr, gap_size);
}
base_addr = range_end;
});
if (base_addr != end_addr) {
func(base_addr, end_addr - base_addr);
}
}
private:
IntervalMap m_ranges_map;
};
template <typename T>
class SplitRangeMap {
public:
using IntervalMap = boost::icl::split_interval_map<
VAddr, T, boost::icl::total_absorber, std::less, boost::icl::inplace_identity,
boost::icl::inter_section, ICL_INTERVAL_INSTANCE(ICL_INTERVAL_DEFAULT, VAddr, std::less),
RangeSetsAllocator>;
using IntervalType = typename IntervalMap::interval_type;
public:
SplitRangeMap() = default;
~SplitRangeMap() = default;
SplitRangeMap(SplitRangeMap const&) = delete;
SplitRangeMap& operator=(SplitRangeMap const&) = delete;
SplitRangeMap(SplitRangeMap&& other);
SplitRangeMap& operator=(SplitRangeMap&& other);
void Add(VAddr base_address, size_t size, const T& value) {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
m_ranges_map.add({interval, value});
}
void Subtract(VAddr base_address, size_t size) {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
m_ranges_map -= interval;
}
void Clear() {
m_ranges_map.clear();
}
bool Contains(VAddr base_address, size_t size) const {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
return boost::icl::contains(m_ranges_map, interval);
}
bool Intersects(VAddr base_address, size_t size) const {
const VAddr end_address = base_address + size;
IntervalType interval{base_address, end_address};
return boost::icl::intersects(m_ranges_map, interval);
}
template <typename Func>
void ForEach(Func&& func) const {
if (m_ranges_map.empty()) {
return;
}
for (const auto& [interval, value] : m_ranges_map) {
const VAddr inter_addr_end = interval.upper();
const VAddr inter_addr = interval.lower();
func(inter_addr, inter_addr_end, value);
}
}
template <typename Func>
void ForEachInRange(VAddr base_addr, size_t size, Func&& func) const {
if (m_ranges_map.empty()) {
return;
}
const VAddr start_address = base_addr;
const VAddr end_address = start_address + size;
const IntervalType search_interval{start_address, end_address};
auto it = m_ranges_map.lower_bound(search_interval);
if (it == m_ranges_map.end()) {
return;
}
auto end_it = m_ranges_map.upper_bound(search_interval);
for (; it != end_it; it++) {
VAddr inter_addr_end = it->first.upper();
VAddr inter_addr = it->first.lower();
if (inter_addr_end > end_address) {
inter_addr_end = end_address;
}
if (inter_addr < start_address) {
inter_addr = start_address;
}
func(inter_addr, inter_addr_end, it->second);
}
}
template <typename Func>
void ForEachNotInRange(VAddr base_addr, size_t size, Func&& func) const {
const VAddr end_addr = base_addr + size;
ForEachInRange(base_addr, size, [&](VAddr range_addr, VAddr range_end, const T&) {
if (size_t gap_size = range_addr - base_addr; gap_size != 0) {
func(base_addr, gap_size);
}

32
src/video_core/buffer_cache/word_manager.h
View File

@@ -10,8 +10,10 @@
#ifdef __linux__
#include "common/adaptive_mutex.h"
#endif
#else
#include "common/spin_lock.h"
#endif
#include "common/debug.h"
#include "common/types.h"
#include "video_core/page_manager.h"
@@ -56,7 +58,7 @@ public:
return cpu_addr;
}
static u64 ExtractBits(u64 word, size_t page_start, size_t page_end) {
static constexpr u64 ExtractBits(u64 word, size_t page_start, size_t page_end) {
constexpr size_t number_bits = sizeof(u64) * 8;
const size_t limit_page_end = number_bits - std::min(page_end, number_bits);
u64 bits = (word >> page_start) << page_start;
@@ -64,7 +66,7 @@ public:
return bits;
}
static std::pair<size_t, size_t> GetWordPage(VAddr address) {
static constexpr std::pair<size_t, size_t> GetWordPage(VAddr address) {
const size_t converted_address = static_cast<size_t>(address);
const size_t word_number = converted_address / BYTES_PER_WORD;
const size_t amount_pages = converted_address % BYTES_PER_WORD;
@@ -73,6 +75,7 @@ public:
template <typename Func>
void IterateWords(size_t offset, size_t size, Func&& func) const {
RENDERER_TRACE;
using FuncReturn = std::invoke_result_t<Func, std::size_t, u64>;
static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
const size_t start = static_cast<size_t>(std::max<s64>(static_cast<s64>(offset), 0LL));
@@ -104,13 +107,13 @@ public:
}
}
template <typename Func>
void IteratePages(u64 mask, Func&& func) const {
void IteratePages(u64 mask, auto&& func) const {
RENDERER_TRACE;
size_t offset = 0;
while (mask != 0) {
const size_t empty_bits = std::countr_zero(mask);
offset += empty_bits;
mask = mask >> empty_bits;
mask >>= empty_bits;
const size_t continuous_bits = std::countr_one(mask);
func(offset, continuous_bits);
@@ -155,8 +158,9 @@ public:
* @param size Size in bytes of the CPU range to loop over
* @param func Function to call for each turned off region
*/
template <Type type, bool clear, typename Func>
void ForEachModifiedRange(VAddr query_cpu_range, s64 size, Func&& func) {
template <Type type, bool clear>
void ForEachModifiedRange(VAddr query_cpu_range, s64 size, auto&& func) {
RENDERER_TRACE;
std::scoped_lock lk{lock};
static_assert(type != Type::Untracked);
@@ -170,6 +174,7 @@ public:
(pending_pointer - pending_offset) * BYTES_PER_PAGE);
};
IterateWords(offset, size, [&](size_t index, u64 mask) {
RENDERER_TRACE;
if constexpr (type == Type::GPU) {
mask &= ~untracked[index];
}
@@ -177,14 +182,13 @@ public:
if constexpr (clear) {
if constexpr (type == Type::CPU) {
UpdateProtection<true>(index, untracked[index], mask);
}
state_words[index] &= ~mask;
if constexpr (type == Type::CPU) {
untracked[index] &= ~mask;
}
state_words[index] &= ~mask;
}
const size_t base_offset = index * PAGES_PER_WORD;
IteratePages(word, [&](size_t pages_offset, size_t pages_size) {
RENDERER_TRACE;
const auto reset = [&]() {
pending_offset = base_offset + pages_offset;
pending_pointer = base_offset + pages_offset + pages_size;
@@ -245,11 +249,13 @@ private:
*/
template <bool add_to_tracker>
void UpdateProtection(u64 word_index, u64 current_bits, u64 new_bits) const {
RENDERER_TRACE;
constexpr s32 delta = add_to_tracker ? 1 : -1;
u64 changed_bits = (add_to_tracker ? current_bits : ~current_bits) & new_bits;
VAddr addr = cpu_addr + word_index * BYTES_PER_WORD;
IteratePages(changed_bits, [&](size_t offset, size_t size) {
tracker->UpdatePagesCachedCount(addr + offset * BYTES_PER_PAGE, size * BYTES_PER_PAGE,
add_to_tracker ? 1 : -1);
tracker->UpdatePageWatchers<delta>(addr + offset * BYTES_PER_PAGE,
size * BYTES_PER_PAGE);
});
}

1
src/video_core/host_shaders/CMakeLists.txt
View File

@@ -11,6 +11,7 @@ set(SHADER_FILES
detilers/micro_32bpp.comp
detilers/micro_64bpp.comp
detilers/micro_8bpp.comp
fault_buffer_process.comp
fs_tri.vert
fsr.comp
post_process.frag

42
src/video_core/host_shaders/fault_buffer_process.comp Normal file
View File

@@ -0,0 +1,42 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#version 450
#extension GL_ARB_gpu_shader_int64 : enable
layout(local_size_x = 64, local_size_y = 1, local_size_z = 1) in;
layout(std430, binding = 0) buffer input_buf {
uint fault_buffer[];
};
layout(std430, binding = 1) buffer output_buf {
uint64_t download_buffer[];
};
// Overlap for 32 bit atomics
layout(std430, binding = 1) buffer output_buf32 {
uint download_buffer32[];
};
layout(constant_id = 0) const uint CACHING_PAGEBITS = 0;
void main() {
uint id = gl_GlobalInvocationID.x;
uint word = fault_buffer[id];
if (word == 0u) {
return;
}
// 1 page per bit
uint base_bit = id * 32u;
while (word != 0u) {
uint bit = findLSB(word);
word &= word - 1;
uint page = base_bit + bit;
uint store_index = atomicAdd(download_buffer32[0], 1u) + 1u;
// It is very unlikely, but should we check for overflow?
if (store_index < 1024u) { // only support 1024 page faults
download_buffer[store_index] = uint64_t(page) << CACHING_PAGEBITS;
}
}
}

180
src/video_core/page_manager.cpp
View File

@@ -1,11 +1,9 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include <thread>
#include <boost/icl/interval_set.hpp>
#include "common/alignment.h"
#include <boost/container/small_vector.hpp>
#include "common/assert.h"
#include "common/error.h"
#include "common/debug.h"
#include "common/signal_context.h"
#include "core/memory.h"
#include "core/signals.h"
@@ -15,23 +13,60 @@
#ifndef _WIN64
#include <sys/mman.h>
#ifdef ENABLE_USERFAULTFD
#include <thread>
#include <fcntl.h>
#include <linux/userfaultfd.h>
#include <poll.h>
#include <sys/ioctl.h>
#include "common/error.h"
#endif
#else
#include <windows.h>
#endif
#ifdef __linux__
#include "common/adaptive_mutex.h"
#else
#include "common/spin_lock.h"
#endif
namespace VideoCore {
constexpr size_t PAGESIZE = 4_KB;
constexpr size_t PAGEBITS = 12;
constexpr size_t PAGE_SIZE = 4_KB;
constexpr size_t PAGE_BITS = 12;
#ifdef ENABLE_USERFAULTFD
struct PageManager::Impl {
Impl(Vulkan::Rasterizer* rasterizer_) : rasterizer{rasterizer_} {
struct PageState {
u8 num_watchers{};
Core::MemoryPermission Perm() const noexcept {
return num_watchers == 0 ? Core::MemoryPermission::ReadWrite
: Core::MemoryPermission::Read;
}
template <s32 delta>
u8 AddDelta() {
if constexpr (delta == 1) {
return ++num_watchers;
} else {
ASSERT_MSG(num_watchers > 0, "Not enough watchers");
return --num_watchers;
}
}
};
struct UpdateProtectRange {
VAddr addr;
u64 size;
Core::MemoryPermission perms;
};
static constexpr size_t ADDRESS_BITS = 40;
static constexpr size_t NUM_ADDRESS_PAGES = 1ULL << (40 - PAGE_BITS);
inline static Vulkan::Rasterizer* rasterizer;
#ifdef ENABLE_USERFAULTFD
Impl(Vulkan::Rasterizer* rasterizer_) {
rasterizer = rasterizer_;
uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY);
ASSERT_MSG(uffd != -1, "{}", Common::GetLastErrorMsg());
@@ -63,7 +98,8 @@ struct PageManager::Impl {
ASSERT_MSG(ret != -1, "Uffdio unregister failed");
}
void Protect(VAddr address, size_t size, bool allow_write) {
void Protect(VAddr address, size_t size, Core::MemoryPermission perms) {
bool allow_write = True(perms & Core::MemoryPermission::Write);
uffdio_writeprotect wp;
wp.range.start = address;
wp.range.len = size;
@@ -118,12 +154,9 @@ struct PageManager::Impl {
}
}
Vulkan::Rasterizer* rasterizer;
std::jthread ufd_thread;
int uffd;
};
#else
struct PageManager::Impl {
Impl(Vulkan::Rasterizer* rasterizer_) {
rasterizer = rasterizer_;
@@ -141,12 +174,11 @@ struct PageManager::Impl {
// No-op
}
void Protect(VAddr address, size_t size, bool allow_write) {
void Protect(VAddr address, size_t size, Core::MemoryPermission perms) {
RENDERER_TRACE;
auto* memory = Core::Memory::Instance();
auto& impl = memory->GetAddressSpace();
impl.Protect(address, size,
allow_write ? Core::MemoryPermission::ReadWrite
: Core::MemoryPermission::Read);
impl.Protect(address, size, perms);
}
static bool GuestFaultSignalHandler(void* context, void* fault_address) {
@@ -157,23 +189,76 @@ struct PageManager::Impl {
return false;
}
inline static Vulkan::Rasterizer* rasterizer;
};
#endif
template <s32 delta>
void UpdatePageWatchers(VAddr addr, u64 size) {
RENDERER_TRACE;
boost::container::small_vector<UpdateProtectRange, 16> update_ranges;
{
std::scoped_lock lk(lock);
size_t page = addr >> PAGE_BITS;
auto perms = cached_pages[page].Perm();
u64 range_begin = 0;
u64 range_bytes = 0;
const auto release_pending = [&] {
if (range_bytes > 0) {
RENDERER_TRACE;
// Add pending (un)protect action
update_ranges.push_back({range_begin << PAGE_BITS, range_bytes, perms});
range_bytes = 0;
}
};
// Iterate requested pages
const u64 page_end = Common::DivCeil(addr + size, PAGE_SIZE);
for (; page != page_end; ++page) {
PageState& state = cached_pages[page];
// Apply the change to the page state
const u8 new_count = state.AddDelta<delta>();
// If the protection changed add pending (un)protect action
if (auto new_perms = state.Perm(); new_perms != perms) [[unlikely]] {
release_pending();
perms = new_perms;
}
// If the page must be (un)protected, add it to the pending range
if ((new_count == 0 && delta < 0) || (new_count == 1 && delta > 0)) {
if (range_bytes == 0) {
range_begin = page;
}
range_bytes += PAGE_SIZE;
} else {
release_pending();
}
}
// Add pending (un)protect action
release_pending();
}
// Flush deferred protects
for (const auto& range : update_ranges) {
Protect(range.addr, range.size, range.perms);
}
}
std::array<PageState, NUM_ADDRESS_PAGES> cached_pages{};
#ifdef __linux__
Common::AdaptiveMutex lock;
#else
Common::SpinLock lock;
#endif
};
PageManager::PageManager(Vulkan::Rasterizer* rasterizer_)
: impl{std::make_unique<Impl>(rasterizer_)}, rasterizer{rasterizer_} {}
: impl{std::make_unique<Impl>(rasterizer_)} {}
PageManager::~PageManager() = default;
VAddr PageManager::GetPageAddr(VAddr addr) {
return Common::AlignDown(addr, PAGESIZE);
}
VAddr PageManager::GetNextPageAddr(VAddr addr) {
return Common::AlignUp(addr + 1, PAGESIZE);
}
void PageManager::OnGpuMap(VAddr address, size_t size) {
impl->OnMap(address, size);
}
@@ -182,41 +267,12 @@ void PageManager::OnGpuUnmap(VAddr address, size_t size) {
impl->OnUnmap(address, size);
}
void PageManager::UpdatePagesCachedCount(VAddr addr, u64 size, s32 delta) {
static constexpr u64 PageShift = 12;
std::scoped_lock lk{lock};
const u64 num_pages = ((addr + size - 1) >> PageShift) - (addr >> PageShift) + 1;
const u64 page_start = addr >> PageShift;
const u64 page_end = page_start + num_pages;
const auto pages_interval =
decltype(cached_pages)::interval_type::right_open(page_start, page_end);
if (delta > 0) {
cached_pages.add({pages_interval, delta});
}
const auto& range = cached_pages.equal_range(pages_interval);
for (const auto& [range, count] : boost::make_iterator_range(range)) {
const auto interval = range & pages_interval;
const VAddr interval_start_addr = boost::icl::first(interval) << PageShift;
const VAddr interval_end_addr = boost::icl::last_next(interval) << PageShift;
const u32 interval_size = interval_end_addr - interval_start_addr;
ASSERT_MSG(rasterizer->IsMapped(interval_start_addr, interval_size),
"Attempted to track non-GPU memory at address {:#x}, size {:#x}.",
interval_start_addr, interval_size);
if (delta > 0 && count == delta) {
impl->Protect(interval_start_addr, interval_size, false);
} else if (delta < 0 && count == -delta) {
impl->Protect(interval_start_addr, interval_size, true);
} else {
ASSERT(count >= 0);
}
}
if (delta < 0) {
cached_pages.add({pages_interval, delta});
}
template <s32 delta>
void PageManager::UpdatePageWatchers(VAddr addr, u64 size) const {
impl->UpdatePageWatchers<delta>(addr, size);
}
template void PageManager::UpdatePageWatchers<1>(VAddr addr, u64 size) const;
template void PageManager::UpdatePageWatchers<-1>(VAddr addr, u64 size) const;
} // namespace VideoCore

32
src/video_core/page_manager.h
View File

@@ -4,11 +4,7 @@
#pragma once
#include <memory>
#include <boost/icl/interval_map.hpp>
#ifdef __linux__
#include "common/adaptive_mutex.h"
#endif
#include "common/spin_lock.h"
#include "common/alignment.h"
#include "common/types.h"
namespace Vulkan {
@@ -18,6 +14,9 @@ class Rasterizer;
namespace VideoCore {
class PageManager {
static constexpr size_t PAGE_BITS = 12;
static constexpr size_t PAGE_SIZE = 1ULL << PAGE_BITS;
public:
explicit PageManager(Vulkan::Rasterizer* rasterizer);
~PageManager();
@@ -28,22 +27,23 @@ public:
/// Unregister a range of gpu memory that was unmapped.
void OnGpuUnmap(VAddr address, size_t size);
/// Increase/decrease the number of surface in pages touching the specified region
void UpdatePagesCachedCount(VAddr addr, u64 size, s32 delta);
/// Updates watches in the pages touching the specified region.
template <s32 delta>
void UpdatePageWatchers(VAddr addr, u64 size) const;
static VAddr GetPageAddr(VAddr addr);
static VAddr GetNextPageAddr(VAddr addr);
/// Returns page aligned address.
static constexpr VAddr GetPageAddr(VAddr addr) {
return Common::AlignDown(addr, PAGE_SIZE);
}
/// Returns address of the next page.
static constexpr VAddr GetNextPageAddr(VAddr addr) {
return Common::AlignUp(addr + 1, PAGE_SIZE);
}
private:
struct Impl;
std::unique_ptr<Impl> impl;
Vulkan::Rasterizer* rasterizer;
boost::icl::interval_map<VAddr, s32> cached_pages;
#ifdef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP
Common::AdaptiveMutex lock;
#else
Common::SpinLock lock;
#endif
};
} // namespace VideoCore

1
src/video_core/renderdoc.cpp
View File

@@ -121,6 +121,7 @@ void SetOutputDir(const std::filesystem::path& path, const std::string& prefix)
if (!rdoc_api) {
return;
}
LOG_WARNING(Common, "RenderDoc capture path: {}", (path / prefix).string());
rdoc_api->SetCaptureFilePathTemplate(fmt::UTF((path / prefix).u8string()).data.data());
}

3
src/video_core/renderer_vulkan/vk_instance.cpp
View File

@@ -147,6 +147,7 @@ Instance::Instance(Frontend::WindowSDL& window, s32 physical_device_index,
available_extensions = GetSupportedExtensions(physical_device);
format_properties = GetFormatProperties(physical_device);
properties = physical_device.getProperties();
memory_properties = physical_device.getMemoryProperties();
CollectDeviceParameters();
ASSERT_MSG(properties.apiVersion >= TargetVulkanApiVersion,
"Vulkan {}.{} is required, but only {}.{} is supported by device!",
@@ -375,6 +376,7 @@ bool Instance::CreateDevice() {
.separateDepthStencilLayouts = vk12_features.separateDepthStencilLayouts,
.hostQueryReset = vk12_features.hostQueryReset,
.timelineSemaphore = vk12_features.timelineSemaphore,
.bufferDeviceAddress = vk12_features.bufferDeviceAddress,
},
vk::PhysicalDeviceVulkan13Features{
.robustImageAccess = vk13_features.robustImageAccess,
@@ -505,6 +507,7 @@ void Instance::CreateAllocator() {
};
const VmaAllocatorCreateInfo allocator_info = {
.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT,
.physicalDevice = physical_device,
.device = *device,
.pVulkanFunctions = &functions,

6
src/video_core/renderer_vulkan/vk_instance.h
View File

@@ -286,6 +286,11 @@ public:
return vk12_props;
}
/// Returns the memory properties of the physical device.
const vk::PhysicalDeviceMemoryProperties& GetMemoryProperties() const noexcept {
return memory_properties;
}
/// Returns true if shaders can declare the ClipDistance attribute
bool IsShaderClipDistanceSupported() const {
return features.shaderClipDistance;
@@ -335,6 +340,7 @@ private:
vk::PhysicalDevice physical_device;
vk::UniqueDevice device;
vk::PhysicalDeviceProperties properties;
vk::PhysicalDeviceMemoryProperties memory_properties;
vk::PhysicalDeviceVulkan11Properties vk11_props;
vk::PhysicalDeviceVulkan12Properties vk12_props;
vk::PhysicalDevicePushDescriptorPropertiesKHR push_descriptor_props;

48
src/video_core/renderer_vulkan/vk_rasterizer.cpp
View File

@@ -36,7 +36,7 @@ static Shader::PushData MakeUserData(const AmdGpu::Liverpool::Regs& regs) {
Rasterizer::Rasterizer(const Instance& instance_, Scheduler& scheduler_,
AmdGpu::Liverpool* liverpool_)
: instance{instance_}, scheduler{scheduler_}, page_manager{this},
buffer_cache{instance, scheduler, liverpool_, texture_cache, page_manager},
buffer_cache{instance, scheduler, *this, liverpool_, texture_cache, page_manager},
texture_cache{instance, scheduler, buffer_cache, page_manager}, liverpool{liverpool_},
memory{Core::Memory::Instance()}, pipeline_cache{instance, scheduler, liverpool} {
if (!Config::nullGpu()) {
@@ -439,6 +439,13 @@ void Rasterizer::Finish() {
scheduler.Finish();
}
void Rasterizer::ProcessFaults() {
if (fault_process_pending) {
fault_process_pending = false;
buffer_cache.ProcessFaultBuffer();
}
}
bool Rasterizer::BindResources(const Pipeline* pipeline) {
if (IsComputeMetaClear(pipeline)) {
return false;
@@ -449,6 +456,8 @@ bool Rasterizer::BindResources(const Pipeline* pipeline) {
buffer_infos.clear();
image_infos.clear();
bool uses_dma = false;
// Bind resource buffers and textures.
Shader::Backend::Bindings binding{};
Shader::PushData push_data = MakeUserData(liverpool->regs);
@@ -459,9 +468,28 @@ bool Rasterizer::BindResources(const Pipeline* pipeline) {
stage->PushUd(binding, push_data);
BindBuffers(*stage, binding, push_data);
BindTextures(*stage, binding);
uses_dma |= stage->dma_types != Shader::IR::Type::Void;
}
pipeline->BindResources(set_writes, buffer_barriers, push_data);
if (uses_dma && !fault_process_pending) {
// We only use fault buffer for DMA right now.
{
// TODO: GPU might have written to memory (for example with EVENT_WRITE_EOP)
// we need to account for that and synchronize.
Common::RecursiveSharedLock lock{mapped_ranges_mutex};
for (auto& range : mapped_ranges) {
buffer_cache.SynchronizeBuffersInRange(range.lower(),
range.upper() - range.lower());
}
}
buffer_cache.MemoryBarrier();
}
fault_process_pending |= uses_dma;
return true;
}
@@ -520,12 +548,18 @@ void Rasterizer::BindBuffers(const Shader::Info& stage, Shader::Backend::Binding
if (desc.buffer_type == Shader::BufferType::GdsBuffer) {
const auto* gds_buf = buffer_cache.GetGdsBuffer();
buffer_infos.emplace_back(gds_buf->Handle(), 0, gds_buf->SizeBytes());
} else if (desc.buffer_type == Shader::BufferType::ReadConstUbo) {
} else if (desc.buffer_type == Shader::BufferType::Flatbuf) {
auto& vk_buffer = buffer_cache.GetStreamBuffer();
const u32 ubo_size = stage.flattened_ud_buf.size() * sizeof(u32);
const u64 offset = vk_buffer.Copy(stage.flattened_ud_buf.data(), ubo_size,
instance.UniformMinAlignment());
buffer_infos.emplace_back(vk_buffer.Handle(), offset, ubo_size);
} else if (desc.buffer_type == Shader::BufferType::BdaPagetable) {
const auto* bda_buffer = buffer_cache.GetBdaPageTableBuffer();
buffer_infos.emplace_back(bda_buffer->Handle(), 0, bda_buffer->SizeBytes());
} else if (desc.buffer_type == Shader::BufferType::FaultBuffer) {
const auto* fault_buffer = buffer_cache.GetFaultBuffer();
buffer_infos.emplace_back(fault_buffer->Handle(), 0, fault_buffer->SizeBytes());
} else if (desc.buffer_type == Shader::BufferType::SharedMemory) {
auto& lds_buffer = buffer_cache.GetStreamBuffer();
const auto& cs_program = liverpool->GetCsRegs();
@@ -925,7 +959,7 @@ bool Rasterizer::InvalidateMemory(VAddr addr, u64 size) {
// Not GPU mapped memory, can skip invalidation logic entirely.
return false;
}
buffer_cache.InvalidateMemory(addr, size);
buffer_cache.InvalidateMemory(addr, size, false);
texture_cache.InvalidateMemory(addr, size);
return true;
}
@@ -937,24 +971,24 @@ bool Rasterizer::IsMapped(VAddr addr, u64 size) {
}
const auto range = decltype(mapped_ranges)::interval_type::right_open(addr, addr + size);
std::shared_lock lock{mapped_ranges_mutex};
Common::RecursiveSharedLock lock{mapped_ranges_mutex};
return boost::icl::contains(mapped_ranges, range);
}
void Rasterizer::MapMemory(VAddr addr, u64 size) {
{
std::unique_lock lock{mapped_ranges_mutex};
std::scoped_lock lock{mapped_ranges_mutex};
mapped_ranges += decltype(mapped_ranges)::interval_type::right_open(addr, addr + size);
}
page_manager.OnGpuMap(addr, size);
}
void Rasterizer::UnmapMemory(VAddr addr, u64 size) {
buffer_cache.InvalidateMemory(addr, size);
buffer_cache.InvalidateMemory(addr, size, true);
texture_cache.UnmapMemory(addr, size);
page_manager.OnGpuUnmap(addr, size);
{
std::unique_lock lock{mapped_ranges_mutex};
std::scoped_lock lock{mapped_ranges_mutex};
mapped_ranges -= decltype(mapped_ranges)::interval_type::right_open(addr, addr + size);
}
}

15
src/video_core/renderer_vulkan/vk_rasterizer.h
View File

@@ -4,7 +4,7 @@
#pragma once
#include <shared_mutex>
#include "common/recursive_lock.h"
#include "video_core/buffer_cache/buffer_cache.h"
#include "video_core/page_manager.h"
#include "video_core/renderer_vulkan/vk_pipeline_cache.h"
@@ -65,11 +65,21 @@ public:
void CpSync();
u64 Flush();
void Finish();
void ProcessFaults();
PipelineCache& GetPipelineCache() {
return pipeline_cache;
}
template <typename Func>
void ForEachMappedRangeInRange(VAddr addr, u64 size, Func&& func) {
const auto range = decltype(mapped_ranges)::interval_type::right_open(addr, addr + size);
Common::RecursiveSharedLock lock{mapped_ranges_mutex};
for (const auto& mapped_range : (mapped_ranges & range)) {
func(mapped_range);
}
}
private:
RenderState PrepareRenderState(u32 mrt_mask);
void BeginRendering(const GraphicsPipeline& pipeline, RenderState& state);
@@ -100,6 +110,8 @@ private:
bool IsComputeMetaClear(const Pipeline* pipeline);
private:
friend class VideoCore::BufferCache;
const Instance& instance;
Scheduler& scheduler;
VideoCore::PageManager page_manager;
@@ -126,6 +138,7 @@ private:
boost::container::static_vector<BufferBindingInfo, Shader::NumBuffers> buffer_bindings;
using ImageBindingInfo = std::pair<VideoCore::ImageId, VideoCore::TextureCache::TextureDesc>;
boost::container::static_vector<ImageBindingInfo, Shader::NumImages> image_bindings;
bool fault_process_pending{false};
};
} // namespace Vulkan

14
src/video_core/renderer_vulkan/vk_scheduler.cpp
View File

@@ -70,6 +70,11 @@ void Scheduler::Flush(SubmitInfo& info) {
SubmitExecution(info);
}
void Scheduler::Flush() {
SubmitInfo info{};
Flush(info);
}
void Scheduler::Finish() {
// When finishing, we need to wait for the submission to have executed on the device.
const u64 presubmit_tick = CurrentTick();
@@ -85,6 +90,15 @@ void Scheduler::Wait(u64 tick) {
Flush(info);
}
master_semaphore.Wait(tick);
// CAUTION: This can introduce unexpected variation in the wait time.
// We don't currently sync the GPU, and some games are very sensitive to this.
// If this becomes a problem, it can be commented out.
// Idealy we would implement proper gpu sync.
while (!pending_ops.empty() && pending_ops.front().gpu_tick <= tick) {
pending_ops.front().callback();
pending_ops.pop();
}
}
void Scheduler::AllocateWorkerCommandBuffers() {

4
src/video_core/renderer_vulkan/vk_scheduler.h
View File

@@ -307,6 +307,10 @@ public:
/// and increments the scheduler timeline semaphore.
void Flush(SubmitInfo& info);
/// Sends the current execution context to the GPU
/// and increments the scheduler timeline semaphore.
void Flush();
/// Sends the current execution context to the GPU and waits for it to complete.
void Finish();

12
src/video_core/texture_cache/texture_cache.cpp
View File

@@ -672,7 +672,7 @@ void TextureCache::TrackImage(ImageId image_id) {
// Re-track the whole image
image.track_addr = image_begin;
image.track_addr_end = image_end;
tracker.UpdatePagesCachedCount(image_begin, image.info.guest_size, 1);
tracker.UpdatePageWatchers<1>(image_begin, image.info.guest_size);
} else {
if (image_begin < image.track_addr) {
TrackImageHead(image_id);
@@ -695,7 +695,7 @@ void TextureCache::TrackImageHead(ImageId image_id) {
ASSERT(image.track_addr != 0 && image_begin < image.track_addr);
const auto size = image.track_addr - image_begin;
image.track_addr = image_begin;
tracker.UpdatePagesCachedCount(image_begin, size, 1);
tracker.UpdatePageWatchers<1>(image_begin, size);
}
void TextureCache::TrackImageTail(ImageId image_id) {
@@ -711,7 +711,7 @@ void TextureCache::TrackImageTail(ImageId image_id) {
const auto addr = image.track_addr_end;
const auto size = image_end - image.track_addr_end;
image.track_addr_end = image_end;
tracker.UpdatePagesCachedCount(addr, size, 1);
tracker.UpdatePageWatchers<1>(addr, size);
}
void TextureCache::UntrackImage(ImageId image_id) {
@@ -724,7 +724,7 @@ void TextureCache::UntrackImage(ImageId image_id) {
image.track_addr = 0;
image.track_addr_end = 0;
if (size != 0) {
tracker.UpdatePagesCachedCount(addr, size, -1);
tracker.UpdatePageWatchers<-1>(addr, size);
}
}
@@ -743,7 +743,7 @@ void TextureCache::UntrackImageHead(ImageId image_id) {
// Cehck its hash later.
MarkAsMaybeDirty(image_id, image);
}
tracker.UpdatePagesCachedCount(image_begin, size, -1);
tracker.UpdatePageWatchers<-1>(image_begin, size);
}
void TextureCache::UntrackImageTail(ImageId image_id) {
@@ -762,7 +762,7 @@ void TextureCache::UntrackImageTail(ImageId image_id) {
// Cehck its hash later.
MarkAsMaybeDirty(image_id, image);
}
tracker.UpdatePagesCachedCount(addr, size, -1);
tracker.UpdatePageWatchers<-1>(addr, size);
}
void TextureCache::DeleteImage(ImageId image_id) {