diff --git a/src/core/libraries/system/userservice.cpp b/src/core/libraries/system/userservice.cpp
index 986af1169..5381b4d64 100644
--- a/src/core/libraries/system/userservice.cpp
+++ b/src/core/libraries/system/userservice.cpp
@@ -6,10 +6,12 @@
 #include "common/config.h"
 #include "common/logging/log.h"
 
+#include "common/singleton.h"
 #include "core/libraries/libs.h"
 #include "core/libraries/system/userservice.h"
 #include "core/libraries/system/userservice_error.h"
 #include "core/tls.h"
+#include "input/controller.h"
 
 namespace Libraries::UserService {
 
@@ -584,13 +586,9 @@ s32 PS4_SYSV_ABI sceUserServiceGetLoginUserIdList(OrbisUserServiceLoginUserIdLis
         return ORBIS_USER_SERVICE_ERROR_INVALID_ARGUMENT;
     }
     // TODO only first user, do the others as well
-    int player_count = Config::GetNumberOfPlayers();
     for (int i = 0; i < 4; i++) {
-        if (i < player_count) {
-            userIdList->user_id[i] = i + 1;
-        } else {
-            userIdList->user_id[i] = ORBIS_USER_SERVICE_USER_ID_INVALID;
-        }
+        auto controllers = *Common::Singleton<Input::GameControllers>::Instance();
+        userIdList->user_id[i] = controllers[i]->GetUserId();
     }
     return ORBIS_OK;
 }
diff --git a/src/input/controller.cpp b/src/input/controller.cpp
index 130b9d416..06d97fe95 100644
--- a/src/input/controller.cpp
+++ b/src/input/controller.cpp
@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
+#include <unordered_set>
 #include <SDL3/SDL.h>
 #include "common/config.h"
 #include "common/logging/log.h"
@@ -242,21 +243,52 @@ void GameController::SetTouchpadState(int touchIndex, bool touchDown, float x, f
 void GameControllers::TryOpenSDLControllers(GameControllers& controllers) {
     using namespace Libraries::UserService;
     int controller_count;
-    SDL_JoystickID* joysticks = SDL_GetGamepads(&controller_count);
+    SDL_JoystickID* new_joysticks = SDL_GetGamepads(&controller_count);
+
+    std::unordered_set<SDL_JoystickID> assigned_ids;
+    std::array<bool, 4> slot_taken{false, false, false, false};
+
     for (int i = 0; i < 4; i++) {
-        if (i < controller_count) {
-            SDL_Gamepad** temp = &(controllers[i]->m_sdl_gamepad);
-            controllers[i]->m_sdl_gamepad = SDL_OpenGamepad(joysticks[i]);
-            if (*temp == 0) {
-                AddUserServiceEvent({OrbisUserServiceEventType::Login,
-                                     SDL_GetGamepadPlayerIndex(controllers[i]->m_sdl_gamepad) + 2});
+        SDL_Gamepad* pad = controllers[i]->m_sdl_gamepad;
+        if (pad) {
+            SDL_JoystickID id = SDL_GetGamepadID(pad);
+            bool still_connected = false;
+            for (int j = 0; j < controller_count; j++) {
+                if (new_joysticks[j] == id) {
+                    still_connected = true;
+                    assigned_ids.insert(id);
+                    slot_taken[i] = true;
+                    break;
+                }
             }
-        } else {
-            SDL_Gamepad** temp = &(controllers[i]->m_sdl_gamepad);
-            controllers[i]->m_sdl_gamepad = nullptr;
-            if (*temp != 0) {
+            if (!still_connected) {
                 AddUserServiceEvent(
-                    {OrbisUserServiceEventType::Logout, SDL_GetGamepadPlayerIndex(*temp) + 2});
+                    {OrbisUserServiceEventType::Logout, SDL_GetGamepadPlayerIndex(pad) + 1});
+                SDL_CloseGamepad(pad);
+                controllers[i]->m_sdl_gamepad = nullptr;
+                controllers[i]->user_id = -1;
+            }
+        }
+    }
+
+    // Now, add any new controllers not already assigned
+    for (int j = 0; j < controller_count; j++) {
+        SDL_JoystickID id = new_joysticks[j];
+        if (assigned_ids.contains(id))
+            continue; // already handled
+
+        SDL_Gamepad* pad = SDL_OpenGamepad(id);
+        if (!pad)
+            continue;
+
+        for (int i = 0; i < 4; i++) {
+            if (!slot_taken[i]) {
+                controllers[i]->m_sdl_gamepad = pad;
+                controllers[i]->user_id = i + 1;
+                slot_taken[i] = true;
+                AddUserServiceEvent(
+                    {OrbisUserServiceEventType::Login, SDL_GetGamepadPlayerIndex(pad) + 1});
+                break;
             }
         }
     }
diff --git a/src/input/controller.h b/src/input/controller.h
index a60a83dea..c5706e7fc 100644
--- a/src/input/controller.h
+++ b/src/input/controller.h
@@ -65,6 +65,9 @@ public:
     bool SetVibration(u8 smallMotor, u8 largeMotor);
     void SetTouchpadState(int touchIndex, bool touchDown, float x, float y);
     u32 Poll();
+    u32 GetUserId() {
+        return user_id;
+    }
 
     float gyro_poll_rate;
     float accel_poll_rate;
@@ -86,6 +89,7 @@ private:
     u32 m_first_state = 0;
     std::array<State, MAX_STATES> m_states;
     std::array<StateInternal, MAX_STATES> m_private;
+    u32 user_id = -1; // ORBIS_USER_SERVICE_USER_ID_INVALID
 
     SDL_Gamepad* m_sdl_gamepad = nullptr;
 };
diff --git a/src/sdl_window.cpp b/src/sdl_window.cpp
index 29d078f62..d9ee4774c 100644
--- a/src/sdl_window.cpp
+++ b/src/sdl_window.cpp
@@ -117,7 +117,6 @@ WindowSDL::WindowSDL(s32 width_, s32 height_, Input::GameControllers* controller
     SDL_SetWindowFullscreen(window, Config::getIsFullscreen());
 
     SDL_InitSubSystem(SDL_INIT_GAMEPAD);
-    Input::GameControllers::TryOpenSDLControllers(controllers);
 
 #if defined(SDL_PLATFORM_WIN32)
     window_info.type = WindowSystemType::Windows;
@@ -146,10 +145,10 @@ WindowSDL::WindowSDL(s32 width_, s32 height_, Input::GameControllers* controller
     Input::ParseInputConfig(std::string(Common::ElfInfo::Instance().GameSerial()));
     // default login
     using namespace Libraries::UserService;
-    int player_count = Config::GetNumberOfPlayers();
-    for (int i = 0; i < player_count; i++) {
-        AddUserServiceEvent({OrbisUserServiceEventType::Login, i + 1});
-    }
+    // int player_count = Config::GetNumberOfPlayers();
+    // for (int i = 0; i < player_count; i++) {
+    //     AddUserServiceEvent({OrbisUserServiceEventType::Login, i + 1});
+    // }
 }
 
 WindowSDL::~WindowSDL() = default;