From 33abdfff9bf0a549b0d1d327e914e9a1ab4b799b Mon Sep 17 00:00:00 2001
From: lat9nq <lat9nq@gmail.com>
Date: Sun, 10 Jul 2022 16:10:35 -0400
Subject: [PATCH] yuzu: Simplify broken Vulkan handling

---
 src/yuzu/configuration/config.cpp             |  8 --
 src/yuzu/configuration/configure_graphics.cpp | 37 ++-------
 src/yuzu/configuration/configure_graphics.h   |  2 +-
 src/yuzu/configuration/configure_graphics.ui  |  9 +--
 src/yuzu/main.cpp                             | 30 ++++---
 src/yuzu/main.h                               |  2 +-
 src/yuzu/startup_checks.cpp                   | 78 ++++++++-----------
 src/yuzu/startup_checks.h                     | 12 ++-
 src/yuzu/uisettings.h                         |  2 +-
 9 files changed, 65 insertions(+), 115 deletions(-)

diff --git a/src/yuzu/configuration/config.cpp b/src/yuzu/configuration/config.cpp
index 9686412d0..ca10e9f23 100644
--- a/src/yuzu/configuration/config.cpp
+++ b/src/yuzu/configuration/config.cpp
@@ -682,12 +682,6 @@ void Config::ReadRendererValues() {
     ReadGlobalSetting(Settings::values.bg_green);
     ReadGlobalSetting(Settings::values.bg_blue);
 
-    if (!global && UISettings::values.has_broken_vulkan &&
-        Settings::values.renderer_backend.GetValue() == Settings::RendererBackend::Vulkan &&
-        !Settings::values.renderer_backend.UsingGlobal()) {
-        Settings::values.renderer_backend.SetGlobal(true);
-    }
-
     if (global) {
         ReadBasicSetting(Settings::values.renderer_debug);
         ReadBasicSetting(Settings::values.renderer_shader_feedback);
@@ -807,7 +801,6 @@ void Config::ReadUIValues() {
     ReadBasicSetting(UISettings::values.pause_when_in_background);
     ReadBasicSetting(UISettings::values.mute_when_in_background);
     ReadBasicSetting(UISettings::values.hide_mouse);
-    ReadBasicSetting(UISettings::values.has_broken_vulkan);
     ReadBasicSetting(UISettings::values.disable_web_applet);
 
     qt_config->endGroup();
@@ -1355,7 +1348,6 @@ void Config::SaveUIValues() {
     WriteBasicSetting(UISettings::values.pause_when_in_background);
     WriteBasicSetting(UISettings::values.mute_when_in_background);
     WriteBasicSetting(UISettings::values.hide_mouse);
-    WriteBasicSetting(UISettings::values.has_broken_vulkan);
     WriteBasicSetting(UISettings::values.disable_web_applet);
 
     qt_config->endGroup();
diff --git a/src/yuzu/configuration/configure_graphics.cpp b/src/yuzu/configuration/configure_graphics.cpp
index 85f34dc35..6b33c4535 100644
--- a/src/yuzu/configuration/configure_graphics.cpp
+++ b/src/yuzu/configuration/configure_graphics.cpp
@@ -58,24 +58,9 @@ ConfigureGraphics::ConfigureGraphics(const Core::System& system_, QWidget* paren
         UpdateBackgroundColorButton(new_bg_color);
     });
 
-    connect(ui->button_check_vulkan, &QAbstractButton::clicked, this, [this] {
-        UISettings::values.has_broken_vulkan = false;
-
-        if (RetrieveVulkanDevices()) {
-            ui->api->setEnabled(true);
-            ui->button_check_vulkan->hide();
-
-            for (const auto& device : vulkan_devices) {
-                ui->device->addItem(device);
-            }
-        } else {
-            UISettings::values.has_broken_vulkan = true;
-        }
-    });
-
-    ui->api->setEnabled(!UISettings::values.has_broken_vulkan.GetValue());
-    ui->button_check_vulkan->setVisible(UISettings::values.has_broken_vulkan.GetValue());
-
+    ui->api->setEnabled(!UISettings::values.has_broken_vulkan);
+    ui->api_widget->setEnabled(!UISettings::values.has_broken_vulkan ||
+                               Settings::IsConfiguringGlobal());
     ui->bg_label->setVisible(Settings::IsConfiguringGlobal());
     ui->bg_combobox->setVisible(!Settings::IsConfiguringGlobal());
 }
@@ -315,7 +300,7 @@ void ConfigureGraphics::UpdateAPILayout() {
         vulkan_device = Settings::values.vulkan_device.GetValue(true);
         shader_backend = Settings::values.shader_backend.GetValue(true);
         ui->device_widget->setEnabled(false);
-        ui->backend_widget->setEnabled(UISettings::values.has_broken_vulkan.GetValue());
+        ui->backend_widget->setEnabled(false);
     } else {
         vulkan_device = Settings::values.vulkan_device.GetValue();
         shader_backend = Settings::values.shader_backend.GetValue();
@@ -337,9 +322,9 @@ void ConfigureGraphics::UpdateAPILayout() {
     }
 }
 
-bool ConfigureGraphics::RetrieveVulkanDevices() try {
+void ConfigureGraphics::RetrieveVulkanDevices() try {
     if (UISettings::values.has_broken_vulkan) {
-        return false;
+        return;
     }
 
     using namespace Vulkan;
@@ -355,11 +340,8 @@ bool ConfigureGraphics::RetrieveVulkanDevices() try {
         const std::string name = vk::PhysicalDevice(device, dld).GetProperties().deviceName;
         vulkan_devices.push_back(QString::fromStdString(name));
     }
-
-    return true;
 } catch (const Vulkan::vk::Exception& exception) {
     LOG_ERROR(Frontend, "Failed to enumerate devices with error: {}", exception.what());
-    return false;
 }
 
 Settings::RendererBackend ConfigureGraphics::GetCurrentGraphicsBackend() const {
@@ -440,11 +422,4 @@ void ConfigureGraphics::SetupPerGameUI() {
         ui->api, static_cast<int>(Settings::values.renderer_backend.GetValue(true)));
     ConfigurationShared::InsertGlobalItem(
         ui->nvdec_emulation, static_cast<int>(Settings::values.nvdec_emulation.GetValue(true)));
-
-    if (UISettings::values.has_broken_vulkan) {
-        ui->backend_widget->setEnabled(true);
-        ConfigurationShared::SetColoredComboBox(
-            ui->backend, ui->backend_widget,
-            static_cast<int>(Settings::values.shader_backend.GetValue(true)));
-    }
 }
diff --git a/src/yuzu/configuration/configure_graphics.h b/src/yuzu/configuration/configure_graphics.h
index 8438f0187..1b101c940 100644
--- a/src/yuzu/configuration/configure_graphics.h
+++ b/src/yuzu/configuration/configure_graphics.h
@@ -41,7 +41,7 @@ private:
     void UpdateDeviceSelection(int device);
     void UpdateShaderBackendSelection(int backend);
 
-    bool RetrieveVulkanDevices();
+    void RetrieveVulkanDevices();
 
     void SetupPerGameUI();
 
diff --git a/src/yuzu/configuration/configure_graphics.ui b/src/yuzu/configuration/configure_graphics.ui
index 2f94c94bc..1e4f74704 100644
--- a/src/yuzu/configuration/configure_graphics.ui
+++ b/src/yuzu/configuration/configure_graphics.ui
@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>471</width>
+    <width>541</width>
     <height>759</height>
    </rect>
   </property>
@@ -574,13 +574,6 @@
      </property>
     </spacer>
    </item>
-   <item>
-    <widget class="QPushButton" name="button_check_vulkan">
-     <property name="text">
-      <string>Check for Working Vulkan</string>
-     </property>
-    </widget>
-   </item>
   </layout>
  </widget>
  <resources/>
diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp
index f2e449560..a2b11fdbf 100644
--- a/src/yuzu/main.cpp
+++ b/src/yuzu/main.cpp
@@ -252,7 +252,7 @@ static QString PrettyProductName() {
     return QSysInfo::prettyProductName();
 }
 
-GMainWindow::GMainWindow()
+GMainWindow::GMainWindow(bool has_broken_vulkan)
     : ui{std::make_unique<Ui::MainWindow>()}, system{std::make_unique<Core::System>()},
       input_subsystem{std::make_shared<InputCommon::InputSubsystem>()},
       config{std::make_unique<Config>(*system)},
@@ -352,17 +352,15 @@ GMainWindow::GMainWindow()
 
     MigrateConfigFiles();
 
-    if (!CheckVulkan()) {
-        config->Save();
+    if (has_broken_vulkan) {
+        UISettings::values.has_broken_vulkan = true;
+
+        QMessageBox::warning(this, tr("Broken Vulkan Installation Detected"),
+                             tr("Vulkan initialization failed during boot.<br><br>Click <a "
+                                "href='https://yuzu-emu.org/wiki/faq/"
+                                "#yuzu-starts-with-the-error-broken-vulkan-installation-detected'>"
+                                "here for instructions to fix the issue</a>."));
 
-        QMessageBox::warning(
-            this, tr("Broken Vulkan Installation Detected"),
-            tr("Vulkan initialization failed on the previous boot.<br><br>Click <a "
-               "href='https://yuzu-emu.org/wiki/faq/"
-               "#yuzu-starts-with-the-error-broken-vulkan-installation-detected'>here for "
-               "instructions to fix the issue</a>."));
-    }
-    if (UISettings::values.has_broken_vulkan) {
         Settings::values.renderer_backend = Settings::RendererBackend::OpenGL;
 
         renderer_status_button->setDisabled(true);
@@ -3853,17 +3851,17 @@ void GMainWindow::SetDiscordEnabled([[maybe_unused]] bool state) {
 #endif
 
 int main(int argc, char* argv[]) {
+    bool has_broken_vulkan = false;
 #ifdef _WIN32
     char variable_contents[32];
     const DWORD startup_check_var =
         GetEnvironmentVariable(STARTUP_CHECK_ENV_VAR, variable_contents, 32);
-    if (startup_check_var != 0) {
-        std::fprintf(stderr, "perform statup checks\n");
+    const std::string variable_contents_s{variable_contents};
+    if (startup_check_var > 0 && variable_contents_s == "ON") {
         CheckVulkan();
         return 0;
-    } else {
-        std::fprintf(stderr, "%d\n", StartupChecks());
     }
+    StartupChecks(argv[0], &has_broken_vulkan);
 #elif YUZU_UNIX
 #error "Unimplemented"
 #endif
@@ -3907,7 +3905,7 @@ int main(int argc, char* argv[]) {
     // generating shaders
     setlocale(LC_ALL, "C");
 
-    GMainWindow main_window{};
+    GMainWindow main_window{has_broken_vulkan};
     // After settings have been loaded by GMainWindow, apply the filter
     main_window.show();
 
diff --git a/src/yuzu/main.h b/src/yuzu/main.h
index 8cf224c9c..7df656002 100644
--- a/src/yuzu/main.h
+++ b/src/yuzu/main.h
@@ -118,7 +118,7 @@ class GMainWindow : public QMainWindow {
 public:
     void filterBarSetChecked(bool state);
     void UpdateUITheme();
-    explicit GMainWindow();
+    explicit GMainWindow(bool has_broken_vulkan);
     ~GMainWindow() override;
 
     bool DropAction(QDropEvent* event);
diff --git a/src/yuzu/startup_checks.cpp b/src/yuzu/startup_checks.cpp
index cfd14a6d5..c860f0aa0 100644
--- a/src/yuzu/startup_checks.cpp
+++ b/src/yuzu/startup_checks.cpp
@@ -22,29 +22,7 @@
 #include "yuzu/startup_checks.h"
 #include "yuzu/uisettings.h"
 
-constexpr char TEMP_FILE_NAME[] = "vulkan_check";
-
-bool CheckVulkan() {
-    if (UISettings::values.has_broken_vulkan) {
-        return true;
-    }
-
-    LOG_DEBUG(Frontend, "Checking presence of Vulkan");
-
-    const auto fs_config_loc = Common::FS::GetYuzuPath(Common::FS::YuzuPath::ConfigDir);
-    const auto temp_file_loc = fs_config_loc / TEMP_FILE_NAME;
-
-    if (std::filesystem::exists(temp_file_loc)) {
-        LOG_WARNING(Frontend, "Detected recovery from previous failed Vulkan initialization");
-
-        UISettings::values.has_broken_vulkan = true;
-        std::filesystem::remove(temp_file_loc);
-        return false;
-    }
-
-    std::ofstream temp_file_handle(temp_file_loc);
-    temp_file_handle.close();
-
+void CheckVulkan() {
     try {
         Vulkan::vk::InstanceDispatch dld;
         const Common::DynamicLibrary library = Vulkan::OpenLibrary();
@@ -53,32 +31,48 @@ bool CheckVulkan() {
 
     } catch (const Vulkan::vk::Exception& exception) {
         LOG_ERROR(Frontend, "Failed to initialize Vulkan: {}", exception.what());
-        // Don't set has_broken_vulkan to true here: we care when loading Vulkan crashes the
-        // application, not when we can handle it.
     }
-
-    std::filesystem::remove(temp_file_loc);
-    return true;
 }
 
-bool StartupChecks() {
+bool StartupChecks(const char* arg0, bool* has_broken_vulkan) {
 #ifdef _WIN32
     const bool env_var_set = SetEnvironmentVariableA(STARTUP_CHECK_ENV_VAR, "ON");
     if (!env_var_set) {
-        LOG_ERROR(Frontend, "SetEnvironmentVariableA failed to set {}, {}", STARTUP_CHECK_ENV_VAR,
-                  GetLastError());
+        std::fprintf(stderr, "SetEnvironmentVariableA failed to set %s, %d\n",
+                     STARTUP_CHECK_ENV_VAR, GetLastError());
         return false;
     }
 
-    STARTUPINFOA startup_info;
     PROCESS_INFORMATION process_info;
+    std::memset(&process_info, '\0', sizeof(process_info));
+
+    if (!SpawnChild(arg0, &process_info)) {
+        return false;
+    }
+
+    // wait until the processs exits
+    DWORD exit_code = STILL_ACTIVE;
+    while (exit_code == STILL_ACTIVE) {
+        GetExitCodeProcess(process_info.hProcess, &exit_code);
+    }
+
+    *has_broken_vulkan = (exit_code != 0);
+
+    CloseHandle(process_info.hProcess);
+    CloseHandle(process_info.hThread);
+#endif
+    return true;
+}
+
+#ifdef _WIN32
+bool SpawnChild(const char* arg0, PROCESS_INFORMATION* pi) {
+    STARTUPINFOA startup_info;
 
     std::memset(&startup_info, '\0', sizeof(startup_info));
-    std::memset(&process_info, '\0', sizeof(process_info));
     startup_info.cb = sizeof(startup_info);
 
     char p_name[255];
-    std::strncpy(p_name, "yuzu.exe", 255);
+    std::strncpy(p_name, arg0, 255);
 
     // TODO: use argv[0] instead of yuzu.exe
     const bool process_created = CreateProcessA(nullptr,       // lpApplicationName
@@ -90,23 +84,13 @@ bool StartupChecks() {
                                                 nullptr,       // lpEnvironment
                                                 nullptr,       // lpCurrentDirectory
                                                 &startup_info, // lpStartupInfo
-                                                &process_info  // lpProcessInformation
+                                                pi             // lpProcessInformation
     );
     if (!process_created) {
-        LOG_ERROR(Frontend, "CreateProcessA failed, {}", GetLastError());
+        std::fprintf(stderr, "CreateProcessA failed, %d\n", GetLastError());
         return false;
     }
 
-    // wait until the processs exits
-    DWORD exit_code = STILL_ACTIVE;
-    while (exit_code == STILL_ACTIVE) {
-        GetExitCodeProcess(process_info.hProcess, &exit_code);
-    }
-
-    std::fprintf(stderr, "exit code: %d\n", exit_code);
-
-    CloseHandle(process_info.hProcess);
-    CloseHandle(process_info.hThread);
-#endif
     return true;
 }
+#endif
diff --git a/src/yuzu/startup_checks.h b/src/yuzu/startup_checks.h
index 98bd5f4bf..096dd54a8 100644
--- a/src/yuzu/startup_checks.h
+++ b/src/yuzu/startup_checks.h
@@ -3,7 +3,15 @@
 
 #pragma once
 
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
 constexpr char STARTUP_CHECK_ENV_VAR[] = "YUZU_DO_STARTUP_CHECKS";
 
-bool CheckVulkan();
-bool StartupChecks();
+void CheckVulkan();
+bool StartupChecks(const char* arg0, bool* has_broken_vulkan);
+
+#ifdef _WIN32
+bool SpawnChild(const char* arg0, PROCESS_INFORMATION* pi);
+#endif
diff --git a/src/yuzu/uisettings.h b/src/yuzu/uisettings.h
index 044d88ca6..2f6948243 100644
--- a/src/yuzu/uisettings.h
+++ b/src/yuzu/uisettings.h
@@ -78,7 +78,7 @@ struct Values {
     Settings::Setting<bool> mute_when_in_background{false, "muteWhenInBackground"};
     Settings::Setting<bool> hide_mouse{true, "hideInactiveMouse"};
     // Set when Vulkan is known to crash the application
-    Settings::Setting<bool> has_broken_vulkan{false, "has_broken_vulkan"};
+    bool has_broken_vulkan = false;
 
     Settings::Setting<bool> select_user_on_boot{false, "select_user_on_boot"};