core: Many things (#194)

* video_core: Add a few missed things

* libkernel: More proper memory mapped files

* memory: Fix tessellation buffer mapping

* Cuphead work

* sceKernelPollSema fix

* clang format

* fixed ngs2 lle loading and rtc lib

* draft pthreads keys implementation

* fixed return codes

* return error code if sceKernelLoadStartModule module is invalid

* re-enabled system modules and disable debug in libs.h

* Improve linux support

* fix windows build

* kernel: Rework keys

---------

Co-authored-by: georgemoralis <giorgosmrls@gmail.com>

src/core/libraries/kernel/threads/kernel_threads.h

@@ -1,53 +0,0 @@
-// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-#pragma once
-
-#include "core/libraries/kernel/thread_management.h"
-
-namespace Core::Loader {
-class SymbolsResolver;
-}
-
-namespace Libraries::Kernel {
-/****
- * rwlock calls
- */
-int PS4_SYSV_ABI posix_pthread_rwlock_destroy(OrbisPthreadRwlock* rwlock);
-int PS4_SYSV_ABI posix_pthread_rwlock_init(OrbisPthreadRwlock* rwlock,
-                                           const OrbisPthreadRwlockattr* attr, const char* name);
-int PS4_SYSV_ABI posix_pthread_rwlock_rdlock(OrbisPthreadRwlock* rwlock);
-int PS4_SYSV_ABI posix_pthread_rwlock_reltimedrdlock_np();
-int PS4_SYSV_ABI posix_pthread_rwlock_reltimedwrlock_np();
-int PS4_SYSV_ABI posix_pthread_rwlock_setname_np();
-int PS4_SYSV_ABI posix_pthread_rwlock_timedrdlock();
-int PS4_SYSV_ABI posix_pthread_rwlock_timedwrlock();
-int PS4_SYSV_ABI posix_pthread_rwlock_tryrdlock(OrbisPthreadRwlock* rwlock);
-int PS4_SYSV_ABI posix_pthread_rwlock_trywrlock(OrbisPthreadRwlock* rwlock);
-int PS4_SYSV_ABI posix_pthread_rwlock_unlock(OrbisPthreadRwlock* rwlock);
-int PS4_SYSV_ABI posix_pthread_rwlock_wrlock(OrbisPthreadRwlock* rwlock);
-int PS4_SYSV_ABI posix_pthread_rwlockattr_destroy(OrbisPthreadRwlockattr* attr);
-int PS4_SYSV_ABI posix_pthread_rwlockattr_getpshared();
-int PS4_SYSV_ABI posix_pthread_rwlockattr_gettype_np();
-int PS4_SYSV_ABI posix_pthread_rwlockattr_init(OrbisPthreadRwlockattr* attr);
-int PS4_SYSV_ABI posix_pthread_rwlockattr_setpshared();
-int PS4_SYSV_ABI posix_pthread_rwlockattr_settype_np();
-int PS4_SYSV_ABI scePthreadRwlockattrDestroy(OrbisPthreadRwlockattr* attr);
-int PS4_SYSV_ABI scePthreadRwlockattrGetpshared();
-int PS4_SYSV_ABI scePthreadRwlockattrGettype();
-int PS4_SYSV_ABI scePthreadRwlockattrInit(OrbisPthreadRwlockattr* attr);
-int PS4_SYSV_ABI scePthreadRwlockattrSetpshared();
-int PS4_SYSV_ABI scePthreadRwlockattrSettype();
-int PS4_SYSV_ABI scePthreadRwlockDestroy(OrbisPthreadRwlock* rwlock);
-int PS4_SYSV_ABI scePthreadRwlockInit(OrbisPthreadRwlock* rwlock,
-                                      const OrbisPthreadRwlockattr* attr, const char* name);
-int PS4_SYSV_ABI scePthreadRwlockRdlock(OrbisPthreadRwlock* rwlock);
-int PS4_SYSV_ABI scePthreadRwlockTimedrdlock();
-int PS4_SYSV_ABI scePthreadRwlockTimedwrlock();
-int PS4_SYSV_ABI scePthreadRwlockTryrdlock(OrbisPthreadRwlock* rwlock);
-int PS4_SYSV_ABI scePthreadRwlockTrywrlock(OrbisPthreadRwlock* rwlock);
-int PS4_SYSV_ABI scePthreadRwlockUnlock(OrbisPthreadRwlock* rwlock);
-int PS4_SYSV_ABI scePthreadRwlockWrlock(OrbisPthreadRwlock* rwlock);
-
-void ThreadsRwlockSymbolsRegister(Core::Loader::SymbolsResolver* sym);
-} // namespace Libraries::Kernel

src/core/libraries/kernel/threads/keys.cpp

@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "core/libraries/error_codes.h"
+#include "core/libraries/kernel/thread_management.h"
+#include "core/libraries/libs.h"
+
+namespace Libraries::Kernel {
+
+int PS4_SYSV_ABI scePthreadKeyCreate(OrbisPthreadKey* key, PthreadKeyDestructor destructor) {
+    if (key == nullptr) {
+        return ORBIS_KERNEL_ERROR_EINVAL;
+    }
+
+    int result = pthread_key_create(key, nullptr);
+    if (destructor) {
+        auto thread = scePthreadSelf();
+        thread->key_destructors.emplace_back(*key, destructor);
+    }
+
+    if (result != 0) {
+        LOG_ERROR(Kernel_Pthread, "scePthreadKeyCreate: error = {}", result);
+        result += ORBIS_KERNEL_ERROR_UNKNOWN;
+    }
+    return result;
+}
+
+void* PS4_SYSV_ABI scePthreadGetspecific(OrbisPthreadKey key) {
+    return pthread_getspecific(key);
+}
+
+int PS4_SYSV_ABI scePthreadSetspecific(OrbisPthreadKey key, /* const*/ void* value) {
+    int result = pthread_setspecific(key, value);
+    if (result != 0) {
+        LOG_ERROR(Kernel_Pthread, "scePthreadSetspecific: error = {}", result);
+        result += ORBIS_KERNEL_ERROR_UNKNOWN;
+    }
+    return result;
+}
+
+void KeySymbolsRegister(Core::Loader::SymbolsResolver* sym) {
+    LIB_FUNCTION("geDaqgH9lTg", "libkernel", 1, "libkernel", 1, 1, scePthreadKeyCreate);
+    LIB_FUNCTION("eoht7mQOCmo", "libkernel", 1, "libkernel", 1, 1, scePthreadGetspecific);
+    LIB_FUNCTION("+BzXYkqYeLE", "libkernel", 1, "libkernel", 1, 1, scePthreadSetspecific);
+}
+
+} // namespace Libraries::Kernel

src/core/libraries/kernel/threads/rwlock.cpp

@@ -4,7 +4,7 @@
 #include "common/logging/log.h"
 #include "core/libraries/error_codes.h"
 #include "core/libraries/libs.h"
-#include "kernel_threads.h"
+#include "threads.h"
 
 namespace Libraries::Kernel {
 
@@ -34,7 +34,23 @@ int PS4_SYSV_ABI posix_pthread_rwlock_init(OrbisPthreadRwlock* rwlock,
     return ORBIS_OK;
 }
 
+OrbisPthreadRwlock* createRwlock(OrbisPthreadRwlock* rwlock) {
+    if (rwlock == nullptr || *rwlock != nullptr) {
+        return rwlock;
+    }
+    static std::mutex mutex;
+    std::scoped_lock lk{mutex};
+    if (*rwlock != nullptr) {
+        return rwlock;
+    }
+    const VAddr addr = std::bit_cast<VAddr>(rwlock);
+    const auto name = fmt::format("rwlock{:#x}", addr);
+    posix_pthread_rwlock_init(rwlock, nullptr, name.c_str());
+    return rwlock;
+}
+
 int PS4_SYSV_ABI posix_pthread_rwlock_rdlock(OrbisPthreadRwlock* rwlock) {
+    rwlock = createRwlock(rwlock);
     int result = pthread_rwlock_rdlock(&(*rwlock)->pth_rwlock);
     if (result != 0) {
         LOG_ERROR(Kernel_Pthread, "posix_pthread_rwlock_rdlock: error = {}", result);
@@ -69,6 +85,10 @@ int PS4_SYSV_ABI posix_pthread_rwlock_timedwrlock() {
 }
 
 int PS4_SYSV_ABI posix_pthread_rwlock_tryrdlock(OrbisPthreadRwlock* rwlock) {
+    rwlock = createRwlock(rwlock);
+    if (rwlock == nullptr) {
+        return ORBIS_KERNEL_ERROR_EINVAL;
+    }
     int result = pthread_rwlock_tryrdlock(&(*rwlock)->pth_rwlock);
     if (result != 0) {
         LOG_ERROR(Kernel_Pthread, "posix_pthread_rwlock_tryrdlock: error = {}", result);
@@ -77,6 +97,10 @@ int PS4_SYSV_ABI posix_pthread_rwlock_tryrdlock(OrbisPthreadRwlock* rwlock) {
 }
 
 int PS4_SYSV_ABI posix_pthread_rwlock_trywrlock(OrbisPthreadRwlock* rwlock) {
+    rwlock = createRwlock(rwlock);
+    if (rwlock == nullptr) {
+        return ORBIS_KERNEL_ERROR_EINVAL;
+    }
     int result = pthread_rwlock_trywrlock(&(*rwlock)->pth_rwlock);
     if (result != 0) {
         LOG_ERROR(Kernel_Pthread, "posix_pthread_rwlock_trywrlock: error = {}", result);
@@ -85,6 +109,10 @@ int PS4_SYSV_ABI posix_pthread_rwlock_trywrlock(OrbisPthreadRwlock* rwlock) {
 }
 
 int PS4_SYSV_ABI posix_pthread_rwlock_unlock(OrbisPthreadRwlock* rwlock) {
+    rwlock = createRwlock(rwlock);
+    if (rwlock == nullptr) {
+        return ORBIS_KERNEL_ERROR_EINVAL;
+    }
     int result = pthread_rwlock_unlock(&(*rwlock)->pth_rwlock);
     if (result != 0) {
         LOG_ERROR(Kernel_Pthread, "posix_pthread_rwlock_unlock: error = {}", result);
@@ -93,6 +121,10 @@ int PS4_SYSV_ABI posix_pthread_rwlock_unlock(OrbisPthreadRwlock* rwlock) {
 }
 
 int PS4_SYSV_ABI posix_pthread_rwlock_wrlock(OrbisPthreadRwlock* rwlock) {
+    rwlock = createRwlock(rwlock);
+    if (rwlock == nullptr) {
+        return ORBIS_KERNEL_ERROR_EINVAL;
+    }
     int result = pthread_rwlock_wrlock(&(*rwlock)->pth_rwlock);
     if (result != 0) {
         LOG_ERROR(Kernel_Pthread, "posix_pthread_rwlock_wrlock: error = {}", result);
@@ -271,9 +303,7 @@ int PS4_SYSV_ABI scePthreadRwlockUnlock(OrbisPthreadRwlock* rwlock) {
 }
 
 int PS4_SYSV_ABI scePthreadRwlockWrlock(OrbisPthreadRwlock* rwlock) {
-    if (rwlock == nullptr || *rwlock == nullptr) {
-        return ORBIS_KERNEL_ERROR_EINVAL;
-    }
+    rwlock = createRwlock(rwlock);
     int result = pthread_rwlock_wrlock(&(*rwlock)->pth_rwlock);
     if (result != 0) {
         LOG_ERROR(Kernel_Pthread, "scePthreadRwlockWrlock: error = {}", result);
@@ -282,7 +312,7 @@ int PS4_SYSV_ABI scePthreadRwlockWrlock(OrbisPthreadRwlock* rwlock) {
     return result;
 }
 
-void ThreadsRwlockSymbolsRegister(Core::Loader::SymbolsResolver* sym) {
+void RwlockSymbolsRegister(Core::Loader::SymbolsResolver* sym) {
     LIB_FUNCTION("1471ajPzxh0", "libkernel", 1, "libkernel", 1, 1, posix_pthread_rwlock_destroy);
     LIB_FUNCTION("ytQULN-nhL4", "libkernel", 1, "libkernel", 1, 1, posix_pthread_rwlock_init);
     LIB_FUNCTION("iGjsr1WAtI0", "libkernel", 1, "libkernel", 1, 1, posix_pthread_rwlock_rdlock);
@@ -350,4 +380,4 @@ void ThreadsRwlockSymbolsRegister(Core::Loader::SymbolsResolver* sym) {
     LIB_FUNCTION("+L98PIbGttk", "libkernel", 1, "libkernel", 1, 1, scePthreadRwlockUnlock);
     LIB_FUNCTION("mqdNorrB+gI", "libkernel", 1, "libkernel", 1, 1, scePthreadRwlockWrlock);
 }
-} // namespace Libraries::Kernel
+} // namespace Libraries::Kernel

src/core/libraries/kernel/threads/semaphore.cpp

@@ -0,0 +1,180 @@
+// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include <condition_variable>
+#include <mutex>
+#include <boost/intrusive/list.hpp>
+#include <pthread.h>
+#include "common/assert.h"
+#include "common/logging/log.h"
+#include "common/scope_exit.h"
+#include "core/libraries/error_codes.h"
+#include "core/libraries/libs.h"
+
+namespace Libraries::Kernel {
+
+using ListBaseHook =
+    boost::intrusive::list_base_hook<boost::intrusive::link_mode<boost::intrusive::normal_link>>;
+
+class Semaphore {
+public:
+    Semaphore(s32 init_count, s32 max_count, const char* name, bool is_fifo)
+        : name{name}, token_count{init_count}, max_count{max_count}, is_fifo{is_fifo} {}
+
+    bool Wait(bool can_block, s32 need_count, u64* timeout) {
+        if (HasAvailableTokens(need_count)) {
+            return true;
+        }
+        if (!can_block) {
+            return false;
+        }
+
+        // Create waiting thread object and add it into the list of waiters.
+        WaitingThread waiter{need_count, is_fifo};
+        AddWaiter(waiter);
+        SCOPE_EXIT {
+            PopWaiter(waiter);
+        };
+
+        // Perform the wait.
+        return waiter.Wait(timeout);
+    }
+
+    bool Signal(s32 signal_count) {
+        std::scoped_lock lk{mutex};
+        if (token_count + signal_count > max_count) {
+            return false;
+        }
+        token_count += signal_count;
+
+        // Wake up threads in order of priority.
+        for (auto& waiter : wait_list) {
+            if (waiter.need_count > token_count) {
+                continue;
+            }
+            token_count -= waiter.need_count;
+            waiter.cv.notify_one();
+        }
+
+        return true;
+    }
+
+private:
+    struct WaitingThread : public ListBaseHook {
+        std::mutex mutex;
+        std::condition_variable cv;
+        u32 priority;
+        s32 need_count;
+
+        explicit WaitingThread(s32 need_count, bool is_fifo) : need_count{need_count} {
+            if (is_fifo) {
+                return;
+            }
+            // Retrieve calling thread priority for sorting into waiting threads list.
+            s32 policy;
+            sched_param param;
+            pthread_getschedparam(pthread_self(), &policy, &param);
+            priority = param.sched_priority;
+        }
+
+        bool Wait(u64* timeout) {
+            std::unique_lock lk{mutex};
+            if (!timeout) {
+                // Wait indefinitely until we are woken up.
+                cv.wait(lk);
+                return true;
+            }
+            // Wait until timeout runs out, recording how much remaining time there was.
+            const auto start = std::chrono::high_resolution_clock::now();
+            const auto status = cv.wait_for(lk, std::chrono::microseconds(*timeout));
+            const auto end = std::chrono::high_resolution_clock::now();
+            const auto time =
+                std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();
+            *timeout -= time;
+            return status != std::cv_status::timeout;
+        }
+
+        bool operator<(const WaitingThread& other) const {
+            return priority < other.priority;
+        }
+    };
+
+    void AddWaiter(WaitingThread& waiter) {
+        std::scoped_lock lk{mutex};
+        // Insert at the end of the list for FIFO order.
+        if (is_fifo) {
+            wait_list.push_back(waiter);
+            return;
+        }
+        // Find the first with priority less then us and insert right before it.
+        auto it = wait_list.begin();
+        while (it != wait_list.end() && it->priority > waiter.priority) {
+            it++;
+        }
+        wait_list.insert(it, waiter);
+    }
+
+    void PopWaiter(WaitingThread& waiter) {
+        std::scoped_lock lk{mutex};
+        wait_list.erase(WaitingThreads::s_iterator_to(waiter));
+    }
+
+    bool HasAvailableTokens(s32 need_count) {
+        std::scoped_lock lk{mutex};
+        if (token_count >= need_count) {
+            token_count -= need_count;
+            return true;
+        }
+        return false;
+    }
+
+    using WaitingThreads =
+        boost::intrusive::list<WaitingThread, boost::intrusive::base_hook<ListBaseHook>,
+                               boost::intrusive::constant_time_size<false>>;
+    WaitingThreads wait_list;
+    std::string name;
+    std::atomic<s32> token_count;
+    std::mutex mutex;
+    s32 max_count;
+    bool is_fifo;
+};
+
+using OrbisKernelSema = Semaphore*;
+
+s32 PS4_SYSV_ABI sceKernelCreateSema(OrbisKernelSema* sem, const char* pName, u32 attr,
+                                     s32 initCount, s32 maxCount, const void* pOptParam) {
+    if (!pName || attr > 2 || initCount < 0 || maxCount <= 0 || initCount > maxCount) {
+        LOG_ERROR(Lib_Kernel, "Semaphore creation parameters are invalid!");
+        return ORBIS_KERNEL_ERROR_EINVAL;
+    }
+    *sem = new Semaphore(initCount, maxCount, pName, attr == 1);
+    return ORBIS_OK;
+}
+
+s32 PS4_SYSV_ABI sceKernelWaitSema(OrbisKernelSema sem, s32 needCount, u64* pTimeout) {
+    ASSERT(sem->Wait(true, needCount, pTimeout));
+    return ORBIS_OK;
+}
+
+s32 PS4_SYSV_ABI sceKernelSignalSema(OrbisKernelSema sem, s32 signalCount) {
+    if (!sem->Signal(signalCount)) {
+        return ORBIS_KERNEL_ERROR_EINVAL;
+    }
+    return ORBIS_OK;
+}
+
+s32 PS4_SYSV_ABI sceKernelPollSema(OrbisKernelSema sem, s32 needCount) {
+    if (!sem->Wait(false, needCount, nullptr)) {
+        return ORBIS_KERNEL_ERROR_EBUSY;
+    }
+    return ORBIS_OK;
+}
+
+void SemaphoreSymbolsRegister(Core::Loader::SymbolsResolver* sym) {
+    LIB_FUNCTION("188x57JYp0g", "libkernel", 1, "libkernel", 1, 1, sceKernelCreateSema);
+    LIB_FUNCTION("Zxa0VhQVTsk", "libkernel", 1, "libkernel", 1, 1, sceKernelWaitSema);
+    LIB_FUNCTION("4czppHBiriw", "libkernel", 1, "libkernel", 1, 1, sceKernelSignalSema);
+    LIB_FUNCTION("12wOHk8ywb0", "libkernel", 1, "libkernel", 1, 1, sceKernelPollSema);
+}
+
+} // namespace Libraries::Kernel

src/core/libraries/kernel/threads/threads.h

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include "core/libraries/kernel/thread_management.h"
+
+namespace Core::Loader {
+class SymbolsResolver;
+}
+
+namespace Libraries::Kernel {
+
+int PS4_SYSV_ABI scePthreadRwlockattrInit(OrbisPthreadRwlockattr* attr);
+
+void SemaphoreSymbolsRegister(Core::Loader::SymbolsResolver* sym);
+void RwlockSymbolsRegister(Core::Loader::SymbolsResolver* sym);
+void KeySymbolsRegister(Core::Loader::SymbolsResolver* sym);
+
+} // namespace Libraries::Kernel