diff --git a/src/qt_gui/game_list_frame.cpp b/src/qt_gui/game_list_frame.cpp
index 468f5afa6..46ea5b5ab 100644
--- a/src/qt_gui/game_list_frame.cpp
+++ b/src/qt_gui/game_list_frame.cpp
@@ -54,10 +54,6 @@ GameListFrame::GameListFrame(std::shared_ptr<gui_settings> gui_settings,
     this->horizontalHeader()->setSectionResizeMode(10, QHeaderView::Fixed);
     PopulateGameList();
 
-    connect(this, &QTableWidget::cellClicked, this, [=, this](int row, int column) {
-        ToggleFavorite(row, column);
-        PopulateGameList(false);
-    });
     connect(this, &QTableWidget::currentCellChanged, this, &GameListFrame::onCurrentCellChanged);
     connect(this->verticalScrollBar(), &QScrollBar::valueChanged, this,
             &GameListFrame::RefreshListBackgroundImage);
@@ -94,6 +90,12 @@ GameListFrame::GameListFrame(std::shared_ptr<gui_settings> gui_settings,
             auto url_issues = "https://github.com/shadps4-emu/shadps4-game-compatibility/issues/";
             QDesktopServices::openUrl(
                 QUrl(url_issues + m_game_info->m_games[row].compatibility.issue_number));
+        } else if (column == 10) {
+            QString serialStr = QString::fromStdString(m_game_info->m_games[row].serial);
+            bool isFavorite = m_gui_settings->GetValue(gui::favorites, serialStr, false).toBool();
+            m_gui_settings->SetValue(gui::favorites, serialStr, !isFavorite, true);
+            this->setCurrentCell(-1, -1);
+            PopulateGameList(false);
         }
     });
 }
@@ -455,27 +457,6 @@ void GameListFrame::SetFavoriteIcon(int row, int column) {
     }
 }
 
-void GameListFrame::ToggleFavorite(int row, int column) {
-    if (column != 10) {
-        return;
-    }
-
-    QWidget* cellWidget = this->cellWidget(row, column);
-    if (!cellWidget) {
-        return;
-    }
-
-    QLabel* label = cellWidget->findChild<QLabel*>("favoriteIcon");
-    if (!label) {
-        return;
-    }
-
-    QString serialStr = QString::fromStdString(m_game_info->m_games[row].serial);
-    bool isFavorite = m_gui_settings->GetValue(gui::favorites, serialStr, false).toBool();
-    m_gui_settings->SetValue(gui::favorites, serialStr, !isFavorite, true);
-    label->setVisible(!isFavorite);
-}
-
 QString GameListFrame::GetPlayTime(const std::string& serial) {
     QString playTime;
     const auto user_dir = Common::FS::GetUserPath(Common::FS::PathType::UserDir);
@@ -508,4 +489,4 @@ QString GameListFrame::GetPlayTime(const std::string& serial) {
 
 QTableWidgetItem* GameListFrame::GetCurrentItem() {
     return m_current_item;
-}
\ No newline at end of file
+}
diff --git a/src/qt_gui/game_list_frame.h b/src/qt_gui/game_list_frame.h
index abdadd722..69ca76e7d 100644
--- a/src/qt_gui/game_list_frame.h
+++ b/src/qt_gui/game_list_frame.h
@@ -38,7 +38,6 @@ public Q_SLOTS:
     void PlayBackgroundMusic(QTableWidgetItem* item);
     void onCurrentCellChanged(int currentRow, int currentColumn, int previousRow,
                               int previousColumn);
-    void ToggleFavorite(int row, int column);
 
 private:
     void SetTableItem(int row, int column, QString itemStr);
diff --git a/src/qt_gui/gui_context_menus.h b/src/qt_gui/gui_context_menus.h
index 2b78b1a94..c6a0994f6 100644
--- a/src/qt_gui/gui_context_menus.h
+++ b/src/qt_gui/gui_context_menus.h
@@ -66,13 +66,22 @@ public:
 
         menu.addMenu(openFolderMenu);
 
-        QAction addToFavorites(tr("Add/Remove Favorite"), widget);
+        QString serialStr = QString::fromStdString(m_games[itemID].serial);
+        bool isFavorite = m_gui_settings->GetValue(gui::favorites, serialStr, false).toBool();
+
+        QAction* toggleFavorite;
+
+        if (isFavorite) {
+            toggleFavorite = new QAction(tr("Remove from Favorites"), widget);
+        } else {
+            toggleFavorite = new QAction(tr("Add to Favorites"), widget);
+        }
         QAction createShortcut(tr("Create Shortcut"), widget);
         QAction openCheats(tr("Cheats / Patches"), widget);
         QAction openSfoViewer(tr("SFO Viewer"), widget);
         QAction openTrophyViewer(tr("Trophy Viewer"), widget);
 
-        menu.addAction(&addToFavorites);
+        menu.addAction(toggleFavorite);
         menu.addAction(&createShortcut);
         menu.addAction(&openCheats);
         menu.addAction(&openSfoViewer);
@@ -306,10 +315,9 @@ public:
             }
         }
 
-        if (selected == &addToFavorites) {
-            QString serialStr = QString::fromStdString(m_games[itemID].serial);
-            bool isFavorite = m_gui_settings->GetValue(gui::favorites, serialStr, false).toBool();
+        if (selected == toggleFavorite) {
             m_gui_settings->SetValue(gui::favorites, serialStr, !isFavorite, true);
+            widget->setCurrentCell(-1, -1);
         }
 
         if (selected == &openCheats) {
diff --git a/src/qt_gui/main_window.cpp b/src/qt_gui/main_window.cpp
index 9ba48b9b0..30f82b470 100644
--- a/src/qt_gui/main_window.cpp
+++ b/src/qt_gui/main_window.cpp
@@ -841,13 +841,6 @@ void MainWindow::CreateConnects() {
 }
 
 void MainWindow::StartGame() {
-    // Ignore favorite column
-    if (m_game_list_frame->currentItem()->column() == 10) {
-        m_game_list_frame->ToggleFavorite(m_game_list_frame->currentItem()->row(),
-                                          m_game_list_frame->currentItem()->column());
-        return;
-    }
-
     BackgroundMusicPlayer::getInstance().stopMusic();
     QString gamePath = "";
     int table_mode = m_gui_settings->GetValue(gui::gl_mode).toInt();