core: Add common signal dispatch system and use for on-demand TCB patches.

This commit is contained in:
squidbus 2024-09-09 13:02:44 -07:00
parent 69b7088120
commit cc8e746808
5 changed files with 305 additions and 278 deletions

View File

@ -437,6 +437,8 @@ set(CORE src/core/aerolib/stubs.cpp
src/core/module.cpp src/core/module.cpp
src/core/module.h src/core/module.h
src/core/platform.h src/core/platform.h
src/core/signals.cpp
src/core/signals.h
src/core/tls.cpp src/core/tls.cpp
src/core/tls.h src/core/tls.h
src/core/virtual_memory.cpp src/core/virtual_memory.cpp

View File

@ -1,15 +1,16 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later // SPDX-License-Identifier: GPL-2.0-or-later
#include <csignal>
#include <map> #include <map>
#include <memory> #include <memory>
#include <mutex> #include <mutex>
#include <set>
#include <Zydis/Zydis.h> #include <Zydis/Zydis.h>
#include <xbyak/xbyak.h> #include <xbyak/xbyak.h>
#include "common/alignment.h" #include "common/alignment.h"
#include "common/assert.h" #include "common/assert.h"
#include "common/types.h" #include "common/types.h"
#include "core/signals.h"
#include "core/tls.h" #include "core/tls.h"
#include "cpu_patches.h" #include "cpu_patches.h"
@ -537,7 +538,7 @@ static bool FilterRosetta2Only(const ZydisDecodedOperand*) {
return ret; return ret;
} }
#endif // __APPLE__ #else // __APPLE__
static bool FilterTcbAccess(const ZydisDecodedOperand* operands) { static bool FilterTcbAccess(const ZydisDecodedOperand* operands) {
const auto& dst_op = operands[0]; const auto& dst_op = operands[0];
@ -583,6 +584,8 @@ static void GenerateTcbAccess(const ZydisDecodedOperand* operands, Xbyak::CodeGe
#endif #endif
} }
#endif // __APPLE__
using PatchFilter = bool (*)(const ZydisDecodedOperand*); using PatchFilter = bool (*)(const ZydisDecodedOperand*);
using InstructionGenerator = void (*)(const ZydisDecodedOperand*, Xbyak::CodeGenerator&); using InstructionGenerator = void (*)(const ZydisDecodedOperand*, Xbyak::CodeGenerator&);
struct PatchInfo { struct PatchInfo {
@ -596,7 +599,14 @@ struct PatchInfo {
bool trampoline; bool trampoline;
}; };
static const std::unordered_map<ZydisMnemonic, PatchInfo> OnDemandPatches = { static const std::unordered_map<ZydisMnemonic, PatchInfo> Patches = {
#if defined(_WIN32)
// Windows needs a trampoline.
{ZYDIS_MNEMONIC_MOV, {FilterTcbAccess, GenerateTcbAccess, true}},
#elif !defined(__APPLE__)
{ZYDIS_MNEMONIC_MOV, {FilterTcbAccess, GenerateTcbAccess, false}},
#endif
#ifdef __APPLE__ #ifdef __APPLE__
// Patches for instruction sets not supported by Rosetta 2. // Patches for instruction sets not supported by Rosetta 2.
// BMI1 // BMI1
@ -611,20 +621,8 @@ static const std::unordered_map<ZydisMnemonic, PatchInfo> OnDemandPatches = {
#endif #endif
}; };
// TODO: Currently only illegal instruction patches are set up to be caught at runtime.
// TODO: These other patches like TCB access should be moved into the same system in the future.
static const std::unordered_map<ZydisMnemonic, PatchInfo> StartupPatches = {
#if defined(_WIN32)
// Windows needs a trampoline.
{ZYDIS_MNEMONIC_MOV, {FilterTcbAccess, GenerateTcbAccess, true}},
#elif !defined(__APPLE__)
{ZYDIS_MNEMONIC_MOV, {FilterTcbAccess, GenerateTcbAccess, false}},
#endif
};
static std::once_flag init_flag; static std::once_flag init_flag;
static ZydisDecoder instr_decoder; static ZydisDecoder instr_decoder;
static ZydisFormatter instr_formatter;
struct PatchModule { struct PatchModule {
/// Mutex controlling access to module code regions. /// Mutex controlling access to module code regions.
@ -636,6 +634,9 @@ struct PatchModule {
/// End of the module. /// End of the module.
u8* end; u8* end;
/// Tracker for patched code locations.
std::set<u8*> patched;
/// Code generator for patching the module. /// Code generator for patching the module.
Xbyak::CodeGenerator patch_gen; Xbyak::CodeGenerator patch_gen;
@ -649,48 +650,47 @@ struct PatchModule {
}; };
static std::map<u64, PatchModule> modules; static std::map<u64, PatchModule> modules;
static PatchModule& GetModule(const void* ptr) { static PatchModule* GetModule(const void* ptr) {
auto upper_bound = modules.upper_bound(reinterpret_cast<u64>(ptr)); auto upper_bound = modules.upper_bound(reinterpret_cast<u64>(ptr));
ASSERT_MSG(upper_bound != modules.begin(), "Unable to find module for code at: {}", if (upper_bound == modules.begin()) {
fmt::ptr(ptr)); return nullptr;
return std::prev(upper_bound)->second; }
return &(std::prev(upper_bound)->second);
} }
static u64 TryPatch(u8* code, PatchModule& module, static bool TryPatch(void* code_address) {
const std::unordered_map<ZydisMnemonic, PatchInfo>& patches, auto* code = static_cast<u8*>(code_address);
bool required = false) { auto* module = GetModule(code);
std::unique_lock lock{module.mutex}; if (module == nullptr) {
return false;
}
std::unique_lock lock{module->mutex};
// Return early if already patched, in case multiple threads signaled at the same time.
if (std::ranges::find(module->patched, code) != module->patched.end()) {
return true;
}
ZydisDecodedInstruction instruction; ZydisDecodedInstruction instruction;
ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT]; ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT];
const auto status = const auto status =
ZydisDecoderDecodeFull(&instr_decoder, code, module.end - code, &instruction, operands); ZydisDecoderDecodeFull(&instr_decoder, code, module->end - code, &instruction, operands);
if (!ZYAN_SUCCESS(status)) { if (!ZYAN_SUCCESS(status)) {
if (required) { return false;
UNREACHABLE_MSG("Unable to decode instruction at {}", fmt::ptr(code));
}
return 1;
} }
// Assume a jmp is an existing patch, in case multiple threads signaled at the same time. if (Patches.contains(instruction.mnemonic)) {
if (instruction.mnemonic == ZYDIS_MNEMONIC_JMP) { const auto& patch_info = Patches.at(instruction.mnemonic);
if (required) {
LOG_INFO(Core, "Skipping already patched instruction at {}", fmt::ptr(code));
}
return instruction.length;
}
if (patches.contains(instruction.mnemonic)) {
const auto& patch_info = patches.at(instruction.mnemonic);
if (patch_info.filter(operands)) { if (patch_info.filter(operands)) {
auto& patch_gen = module.patch_gen; auto& patch_gen = module->patch_gen;
// Reset state and move to current code position. // Reset state and move to current code position.
patch_gen.reset(); patch_gen.reset();
patch_gen.setSize(code - patch_gen.getCode()); patch_gen.setSize(code - patch_gen.getCode());
if (patch_info.trampoline) { if (patch_info.trampoline) {
auto& trampoline_gen = module.trampoline_gen; auto& trampoline_gen = module->trampoline_gen;
const auto trampoline_ptr = trampoline_gen.getCurr(); const auto trampoline_ptr = trampoline_gen.getCurr();
patch_info.generator(operands, trampoline_gen); patch_info.generator(operands, trampoline_gen);
@ -714,59 +714,34 @@ static u64 TryPatch(u8* code, PatchModule& module,
// Fill remaining space with nops. // Fill remaining space with nops.
patch_gen.nop(instruction.length - patch_size); patch_gen.nop(instruction.length - patch_size);
module->patched.insert(code);
LOG_DEBUG(Core, "Patched instruction '{}' at: {}", LOG_DEBUG(Core, "Patched instruction '{}' at: {}",
ZydisMnemonicGetString(instruction.mnemonic), fmt::ptr(code)); ZydisMnemonicGetString(instruction.mnemonic), fmt::ptr(code));
return instruction.length; return true;
} }
} }
} }
if (required) { return false;
char buffer[256];
ZydisFormatterFormatInstruction(&instr_formatter, &instruction, operands,
instruction.operand_count_visible, buffer, sizeof(buffer),
reinterpret_cast<u64>(code), ZYAN_NULL);
UNIMPLEMENTED_MSG("Encountered instruction at {} without patch: {}", fmt::ptr(code),
buffer);
}
return instruction.length;
} }
#if defined(_WIN32) static bool PatchesAccessViolationHandler(void* code_address, void* fault_address, bool is_write) {
static LONG WINAPI SignalHandler(EXCEPTION_POINTERS* pExp) noexcept { return TryPatch(code_address);
const u32 ec = pExp->ExceptionRecord->ExceptionCode;
if (ec == EXCEPTION_ILLEGAL_INSTRUCTION) {
auto* code = reinterpret_cast<u8*>(pExp->ExceptionRecord->ExceptionAddress);
auto& module = GetModule(code);
TryPatch(code, module, OnDemandPatches, true);
return EXCEPTION_CONTINUE_EXECUTION;
}
return EXCEPTION_CONTINUE_SEARCH;
} }
#else
static void SignalHandler(int sig, siginfo_t* info, void* raw_context) { static bool PatchesIllegalInstructionHandler(void* code_address) {
auto* code = static_cast<u8*>(info->si_addr); return TryPatch(code_address);
auto& module = GetModule(code);
TryPatch(code, module, OnDemandPatches, true);
} }
#endif
static void PatchesInit() { static void PatchesInit() {
ZydisDecoderInit(&instr_decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64); ZydisDecoderInit(&instr_decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64);
ZydisFormatterInit(&instr_formatter, ZYDIS_FORMATTER_STYLE_INTEL);
if (!OnDemandPatches.empty()) { if (!Patches.empty()) {
#if defined(_WIN32) auto* signals = Signals::Instance();
ASSERT_MSG(AddVectoredExceptionHandler(0, SignalHandler), // Should be called last.
"Failed to register code patching exception handler."); constexpr auto priority = std::numeric_limits<u32>::max();
#else signals->RegisterAccessViolationHandler(PatchesAccessViolationHandler, priority);
constexpr struct sigaction action { signals->RegisterIllegalInstructionHandler(PatchesIllegalInstructionHandler, priority);
.sa_flags = SA_SIGINFO | SA_ONSTACK, .sa_sigaction = SignalHandler, .sa_mask = 0,
};
ASSERT_MSG(sigaction(SIGILL, &action, nullptr) == 0,
"Failed to register code patching signal handler.");
#endif
} }
} }
@ -782,24 +757,14 @@ void RegisterPatchModule(void* module_ptr, u64 module_size, void* trampoline_are
} }
void PrePatchInstructions(u64 segment_addr, u64 segment_size) { void PrePatchInstructions(u64 segment_addr, u64 segment_size) {
auto& module = GetModule(reinterpret_cast<void*>(segment_addr));
if (!StartupPatches.empty()) {
u8* code = reinterpret_cast<u8*>(segment_addr);
u8* end = code + segment_size;
while (code < end) {
code += TryPatch(code, module, StartupPatches);
}
}
#ifdef __APPLE__ #ifdef __APPLE__
// HACK: For some reason patching in the signal handler at the start of a page does not work // HACK: For some reason patching in the signal handler at the start of a page does not work
// under Rosetta 2. Patch any instructions at the start of a page ahead of time. // under Rosetta 2. Patch any instructions at the start of a page ahead of time.
if (!OnDemandPatches.empty()) { if (!Patches.empty()) {
u8* code_page = reinterpret_cast<u8*>(Common::AlignUp(segment_addr, 0x1000)); auto* code_page = reinterpret_cast<u8*>(Common::AlignUp(segment_addr, 0x1000));
u8* end_page = code_page + Common::AlignUp(segment_size, 0x1000); const auto* end_page = code_page + Common::AlignUp(segment_size, 0x1000);
while (code_page < end_page) { while (code_page < end_page) {
TryPatch(code_page, module, OnDemandPatches); TryPatch(code_page);
code_page += 0x1000; code_page += 0x1000;
} }
} }

168
src/core/signals.cpp Normal file
View File

@ -0,0 +1,168 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#include "common/arch.h"
#include "common/assert.h"
#include "core/signals.h"
#ifdef _WIN32
#include <windows.h>
#else
#include <csignal>
#ifdef ARCH_X86_64
#include <Zydis/Decoder.h>
#include <Zydis/Formatter.h>
#endif
#endif
namespace Core {
#if defined(_WIN32)
static LONG WINAPI SignalHandler(EXCEPTION_POINTERS* pExp) noexcept {
const auto* signals = Signals::Instance();
auto* code_address = reinterpret_cast<void*>(pExp->ContextRecord->Rip);
bool handled = false;
switch (pExp->ExceptionRecord->ExceptionCode) {
case EXCEPTION_ACCESS_VIOLATION:
handled = signals->DispatchAccessViolation(
code_address, reinterpret_cast<void*>(pExp->ExceptionRecord->ExceptionInformation[1]),
pExp->ExceptionRecord->ExceptionInformation[0] == 1);
break;
case EXCEPTION_ILLEGAL_INSTRUCTION:
handled = signals->DispatchIllegalInstruction(code_address);
break;
default:
break;
}
return handled ? EXCEPTION_CONTINUE_EXECUTION : EXCEPTION_CONTINUE_SEARCH;
}
#else
#ifdef __APPLE__
#if defined(ARCH_X86_64)
#define CODE_ADDRESS(ctx) reinterpret_cast<void*>((ctx)->uc_mcontext->__ss.__rip)
#define IS_WRITE_ERROR(ctx) ((ctx)->uc_mcontext->__es.__err & 0x2)
#elif defined(ARCH_ARM64)
#define CODE_ADDRESS(ctx) reinterpret_cast<void*>((ctx)->uc_mcontext->__ss.__pc)
#define IS_WRITE_ERROR(ctx) ((ctx)->uc_mcontext->__es.__esr & 0x40)
#endif
#else
#if defined(ARCH_X86_64)
#define CODE_ADDRESS(ctx) reinterpret_cast<void*>((ctx)->uc_mcontext.gregs[REG_RIP])
#define IS_WRITE_ERROR(ctx) ((ctx)->uc_mcontext.gregs[REG_ERR] & 0x2)
#endif
#endif
#ifndef IS_WRITE_ERROR
#error "Missing IS_WRITE_ERROR() implementation for target OS and CPU architecture.
#endif
static std::string DisassembleInstruction(void* code_address) {
char buffer[256] = "<unable to decode>";
#ifdef ARCH_X86_64
ZydisDecoder decoder;
ZydisDecoderInit(&decoder, ZYDIS_MACHINE_MODE_LONG_64, ZYDIS_STACK_WIDTH_64);
ZydisDecodedInstruction instruction;
ZydisDecodedOperand operands[ZYDIS_MAX_OPERAND_COUNT];
static constexpr u64 max_length = 0x20;
const auto status =
ZydisDecoderDecodeFull(&decoder, code_address, max_length, &instruction, operands);
if (ZYAN_SUCCESS(status)) {
ZydisFormatter formatter;
ZydisFormatterInit(&formatter, ZYDIS_FORMATTER_STYLE_INTEL);
ZydisFormatterFormatInstruction(&formatter, &instruction, operands,
instruction.operand_count_visible, buffer, sizeof(buffer),
reinterpret_cast<u64>(code_address), ZYAN_NULL);
}
#endif
return buffer;
}
static void SignalHandler(int sig, siginfo_t* info, void* raw_context) {
const auto* ctx = static_cast<ucontext_t*>(raw_context);
const auto* signals = Signals::Instance();
auto* code_address = CODE_ADDRESS(ctx);
switch (sig) {
case SIGSEGV:
case SIGBUS:
if (const bool is_write = IS_WRITE_ERROR(ctx);
!signals->DispatchAccessViolation(code_address, info->si_addr, is_write)) {
UNREACHABLE_MSG("Unhandled access violation at code address {}: {} address {}",
fmt::ptr(code_address), is_write ? "Write to" : "Read from",
fmt::ptr(info->si_addr));
}
break;
case SIGILL:
if (!signals->DispatchIllegalInstruction(code_address)) {
UNREACHABLE_MSG("Unhandled illegal instruction at code address {}: {}",
fmt::ptr(code_address), DisassembleInstruction(code_address));
}
break;
default:
break;
}
}
#endif
SignalDispatch::SignalDispatch() {
#if defined(_WIN32)
ASSERT_MSG(handle = AddVectoredExceptionHandler(0, SignalHandler),
"Failed to register exception handler.");
#else
constexpr struct sigaction action {
.sa_flags = SA_SIGINFO | SA_ONSTACK, .sa_sigaction = SignalHandler, .sa_mask = 0,
};
ASSERT_MSG(sigaction(SIGSEGV, &action, nullptr) == 0 &&
sigaction(SIGBUS, &action, nullptr) == 0,
"Failed to register access violation signal handler.");
ASSERT_MSG(sigaction(SIGILL, &action, nullptr) == 0,
"Failed to register illegal instruction signal handler.");
#endif
}
SignalDispatch::~SignalDispatch() {
#if defined(_WIN32)
ASSERT_MSG(RemoveVectoredExceptionHandler(handle), "Failed to remove exception handler.");
#else
constexpr struct sigaction action {
.sa_flags = 0, .sa_handler = SIG_DFL, .sa_mask = 0,
};
ASSERT_MSG(sigaction(SIGSEGV, &action, nullptr) == 0 &&
sigaction(SIGBUS, &action, nullptr) == 0,
"Failed to remove access violation signal handler.");
ASSERT_MSG(sigaction(SIGILL, &action, nullptr) == 0,
"Failed to remove illegal instruction signal handler.");
#endif
}
bool SignalDispatch::DispatchAccessViolation(void* code_address, void* fault_address,
bool is_write) const {
for (const auto& [handler, _] : access_violation_handlers) {
if (handler(code_address, fault_address, is_write)) {
return true;
}
}
return false;
}
bool SignalDispatch::DispatchIllegalInstruction(void* code_address) const {
for (const auto& [handler, _] : illegal_instruction_handlers) {
if (handler(code_address)) {
return true;
}
}
return false;
}
} // namespace Core

56
src/core/signals.h Normal file
View File

@ -0,0 +1,56 @@
// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later
#pragma once
#include <set>
#include "common/singleton.h"
namespace Core {
using AccessViolationHandler = bool (*)(void* code_address, void* fault_address, bool is_write);
using IllegalInstructionHandler = bool (*)(void* code_address);
/// Receives OS signals and dispatches to the appropriate handlers.
class SignalDispatch {
public:
SignalDispatch();
~SignalDispatch();
/// Registers a handler for memory access violation signals.
void RegisterAccessViolationHandler(const AccessViolationHandler& handler, u32 priority) {
access_violation_handlers.emplace(handler, priority);
}
/// Registers a handler for illegal instruction signals.
void RegisterIllegalInstructionHandler(const IllegalInstructionHandler& handler, u32 priority) {
illegal_instruction_handlers.emplace(handler, priority);
}
/// Dispatches an access violation signal, returning whether it was successfully handled.
bool DispatchAccessViolation(void* code_address, void* fault_address, bool is_write) const;
/// Dispatches an illegal instruction signal, returning whether it was successfully handled.
bool DispatchIllegalInstruction(void* code_address) const;
private:
template <typename T>
struct HandlerEntry {
T handler;
u32 priority;
std::strong_ordering operator<=>(const HandlerEntry& right) const {
return priority <=> right.priority;
}
};
std::set<HandlerEntry<AccessViolationHandler>> access_violation_handlers;
std::set<HandlerEntry<IllegalInstructionHandler>> illegal_instruction_handlers;
#ifdef _WIN32
void* handle{};
#endif
};
using Signals = Common::Singleton<SignalDispatch>;
} // namespace Core

View File

@ -2,22 +2,16 @@
// SPDX-License-Identifier: GPL-2.0-or-later // SPDX-License-Identifier: GPL-2.0-or-later
#include <thread> #include <thread>
#include <boost/icl/interval_set.hpp>
#include "common/alignment.h" #include "common/alignment.h"
#include "common/arch.h"
#include "common/assert.h" #include "common/assert.h"
#include "common/error.h" #include "common/error.h"
#include "core/signals.h"
#include "video_core/page_manager.h" #include "video_core/page_manager.h"
#include "video_core/renderer_vulkan/vk_rasterizer.h" #include "video_core/renderer_vulkan/vk_rasterizer.h"
#ifndef _WIN64 #ifndef _WIN64
#include <fcntl.h>
#include <poll.h>
#include <signal.h>
#include <sys/ioctl.h>
#include <sys/mman.h> #include <sys/mman.h>
#ifdef ENABLE_USERFAULTFD
#include <linux/userfaultfd.h>
#endif
#else #else
#include <windows.h> #include <windows.h>
#endif #endif
@ -27,207 +21,49 @@ namespace VideoCore {
constexpr size_t PAGESIZE = 4_KB; constexpr size_t PAGESIZE = 4_KB;
constexpr size_t PAGEBITS = 12; constexpr size_t PAGEBITS = 12;
#ifdef _WIN64
struct PageManager::Impl { struct PageManager::Impl {
Impl(Vulkan::Rasterizer* rasterizer_) { Impl(Vulkan::Rasterizer* rasterizer_) {
rasterizer = rasterizer_; rasterizer = rasterizer_;
veh_handle = AddVectoredExceptionHandler(0, GuestFaultSignalHandler); // Should be called first.
ASSERT_MSG(veh_handle, "Failed to register an exception handler"); constexpr auto priority = std::numeric_limits<u32>::min();
Core::Signals::Instance()->RegisterAccessViolationHandler(GuestFaultSignalHandler,
priority);
} }
void OnMap(VAddr address, size_t size) {} void OnMap(VAddr address, size_t size) {
owned_ranges += boost::icl::interval<VAddr>::right_open(address, address + size);
}
void OnUnmap(VAddr address, size_t size) {} void OnUnmap(VAddr address, size_t size) {
owned_ranges -= boost::icl::interval<VAddr>::right_open(address, address + size);
}
void Protect(VAddr address, size_t size, bool allow_write) { void Protect(VAddr address, size_t size, bool allow_write) {
#ifdef _WIN32
DWORD prot = allow_write ? PAGE_READWRITE : PAGE_READONLY; DWORD prot = allow_write ? PAGE_READWRITE : PAGE_READONLY;
DWORD old_prot{}; DWORD old_prot{};
BOOL result = VirtualProtect(std::bit_cast<LPVOID>(address), size, prot, &old_prot); BOOL result = VirtualProtect(std::bit_cast<LPVOID>(address), size, prot, &old_prot);
ASSERT_MSG(result != 0, "Region protection failed"); ASSERT_MSG(result != 0, "Region protection failed");
}
static LONG WINAPI GuestFaultSignalHandler(EXCEPTION_POINTERS* pExp) noexcept {
const u32 ec = pExp->ExceptionRecord->ExceptionCode;
if (ec == EXCEPTION_ACCESS_VIOLATION) {
const auto info = pExp->ExceptionRecord->ExceptionInformation;
if (info[0] == 1) { // Write violation
const VAddr addr_aligned = Common::AlignDown(info[1], PAGESIZE);
rasterizer->InvalidateMemory(addr_aligned, PAGESIZE);
return EXCEPTION_CONTINUE_EXECUTION;
} /* else {
UNREACHABLE();
}*/
}
return EXCEPTION_CONTINUE_SEARCH; // pass further
}
inline static Vulkan::Rasterizer* rasterizer;
void* veh_handle{};
};
#elif ENABLE_USERFAULTFD
struct PageManager::Impl {
Impl(Vulkan::Rasterizer* rasterizer_) : rasterizer{rasterizer_} {
uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
ASSERT_MSG(uffd != -1, "{}", Common::GetLastErrorMsg());
// Request uffdio features from kernel.
uffdio_api api;
api.api = UFFD_API;
api.features = UFFD_FEATURE_THREAD_ID;
const int ret = ioctl(uffd, UFFDIO_API, &api);
ASSERT(ret == 0 && api.api == UFFD_API);
// Create uffd handler thread
ufd_thread = std::jthread([&](std::stop_token token) { UffdHandler(token); });
}
void OnMap(VAddr address, size_t size) {
uffdio_register reg;
reg.range.start = address;
reg.range.len = size;
reg.mode = UFFDIO_REGISTER_MODE_WP;
const int ret = ioctl(uffd, UFFDIO_REGISTER, &reg);
ASSERT_MSG(ret != -1, "Uffdio register failed");
}
void OnUnmap(VAddr address, size_t size) {
uffdio_range range;
range.start = address;
range.len = size;
const int ret = ioctl(uffd, UFFDIO_UNREGISTER, &range);
ASSERT_MSG(ret != -1, "Uffdio unregister failed");
}
void Protect(VAddr address, size_t size, bool allow_write) {
uffdio_writeprotect wp;
wp.range.start = address;
wp.range.len = size;
wp.mode = allow_write ? 0 : UFFDIO_WRITEPROTECT_MODE_WP;
const int ret = ioctl(uffd, UFFDIO_WRITEPROTECT, &wp);
ASSERT_MSG(ret != -1, "Uffdio writeprotect failed with error: {}",
Common::GetLastErrorMsg());
}
void UffdHandler(std::stop_token token) {
while (!token.stop_requested()) {
pollfd pollfd;
pollfd.fd = uffd;
pollfd.events = POLLIN;
// Block until the descriptor is ready for data reads.
const int pollres = poll(&pollfd, 1, -1);
switch (pollres) {
case -1:
perror("Poll userfaultfd");
continue;
break;
case 0:
continue;
case 1:
break;
default:
UNREACHABLE_MSG("Unexpected number of descriptors {} out of poll", pollres);
}
// We don't want an error condition to have occured.
ASSERT_MSG(!(pollfd.revents & POLLERR), "POLLERR on userfaultfd");
// We waited until there is data to read, we don't care about anything else.
if (!(pollfd.revents & POLLIN)) {
continue;
}
// Read message from kernel.
uffd_msg msg;
const int readret = read(uffd, &msg, sizeof(msg));
ASSERT_MSG(readret != -1 || errno == EAGAIN, "Unexpected result of uffd read");
if (errno == EAGAIN) {
continue;
}
ASSERT_MSG(readret == sizeof(msg), "Unexpected short read, exiting");
ASSERT(msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP);
// Notify rasterizer about the fault.
const VAddr addr = msg.arg.pagefault.address;
const VAddr addr_page = Common::AlignDown(addr, PAGESIZE);
rasterizer->InvalidateMemory(addr_page, PAGESIZE);
}
}
Vulkan::Rasterizer* rasterizer;
std::jthread ufd_thread;
int uffd;
};
#else #else
#if defined(__APPLE__)
#if defined(ARCH_X86_64)
#define IS_WRITE_ERROR(ctx) ((ctx)->uc_mcontext->__es.__err & 0x2)
#elif defined(ARCH_ARM64)
#define IS_WRITE_ERROR(ctx) ((ctx)->uc_mcontext->__es.__esr & 0x40)
#endif
#else
#if defined(ARCH_X86_64)
#define IS_WRITE_ERROR(ctx) ((ctx)->uc_mcontext.gregs[REG_ERR] & 0x2)
#endif
#endif
#ifndef IS_WRITE_ERROR
#error "Missing IS_WRITE_ERROR() implementation for target OS and CPU architecture.
#endif
struct PageManager::Impl {
Impl(Vulkan::Rasterizer* rasterizer_) {
rasterizer = rasterizer_;
#ifdef __APPLE__
// Read-only memory write results in SIGBUS on Apple.
static constexpr int SignalType = SIGBUS;
#else
static constexpr int SignalType = SIGSEGV;
#endif
sigset_t signal_mask;
sigemptyset(&signal_mask);
sigaddset(&signal_mask, SignalType);
using HandlerType = decltype(sigaction::sa_sigaction);
struct sigaction guest_access_fault {};
guest_access_fault.sa_flags = SA_SIGINFO | SA_ONSTACK;
guest_access_fault.sa_sigaction = &GuestFaultSignalHandler;
guest_access_fault.sa_mask = signal_mask;
sigaction(SignalType, &guest_access_fault, nullptr);
}
void OnMap(VAddr address, size_t size) {}
void OnUnmap(VAddr address, size_t size) {}
void Protect(VAddr address, size_t size, bool allow_write) {
mprotect(reinterpret_cast<void*>(address), size, mprotect(reinterpret_cast<void*>(address), size,
PROT_READ | (allow_write ? PROT_WRITE : 0)); PROT_READ | (allow_write ? PROT_WRITE : 0));
#endif
} }
static void GuestFaultSignalHandler(int sig, siginfo_t* info, void* raw_context) { static bool GuestFaultSignalHandler(void* code_address, void* fault_address, bool is_write) {
ucontext_t* ctx = reinterpret_cast<ucontext_t*>(raw_context); const auto addr = reinterpret_cast<VAddr>(fault_address);
const VAddr address = reinterpret_cast<VAddr>(info->si_addr); if (is_write && owned_ranges.find(addr) != owned_ranges.end()) {
if (IS_WRITE_ERROR(ctx)) { const VAddr addr_aligned = Common::AlignDown(addr, PAGESIZE);
const VAddr addr_aligned = Common::AlignDown(address, PAGESIZE);
rasterizer->InvalidateMemory(addr_aligned, PAGESIZE); rasterizer->InvalidateMemory(addr_aligned, PAGESIZE);
} else { return true;
// Read not supported!
UNREACHABLE();
} }
return false;
} }
inline static Vulkan::Rasterizer* rasterizer; inline static Vulkan::Rasterizer* rasterizer;
inline static boost::icl::interval_set<VAddr> owned_ranges;
}; };
#endif
PageManager::PageManager(Vulkan::Rasterizer* rasterizer_) PageManager::PageManager(Vulkan::Rasterizer* rasterizer_)
: impl{std::make_unique<Impl>(rasterizer_)}, rasterizer{rasterizer_} {} : impl{std::make_unique<Impl>(rasterizer_)}, rasterizer{rasterizer_} {}