Use proper Windows APIs for checking/setting default app (#5000)

# Description of Changes Use proper Windows APIs for checking/setting default app --------- Co-authored-by: Connor Yoh <con.yoh13@gmail.com> Co-authored-by: ConnorYoh <40631091+ConnorYoh@users.noreply.github.com>
2025-12-18 20:04:17 +01:00 · 2025-11-25 21:31:02 +00:00 · 2025-11-25 21:31:02 +00:00 · d8a99fcb07
commit d8a99fcb07
parent 8016d271aa
7 changed files with 160 additions and 57 deletions
--- a/frontend/public/locales/en-GB/translation.json
+++ b/frontend/public/locales/en-GB/translation.json
@ -63,7 +63,7 @@
    },
    "settingsOpened": {
      "title": "Settings Opened",
-      "message": "Please select Stirling PDF in your system settings"
+      "message": "In Windows Settings, search for 'PDF' and select Stirling PDF as your default app"
    },
    "error": {
      "title": "Error",
--- a/frontend/src-tauri/Cargo.lock
+++ b/frontend/src-tauri/Cargo.lock
@ -4250,6 +4250,7 @@ dependencies = [
 "tokio",
 "url",
 "urlencoding",
+ "windows 0.58.0",
 ]

 [[package]]
@ -4437,7 +4438,7 @@ dependencies = [
 "tao-macros",
 "unicode-segmentation",
 "url",
- "windows",
+ "windows 0.61.3",
 "windows-core 0.61.2",
 "windows-version",
 "x11-dl",
@ -4514,7 +4515,7 @@ dependencies = [
 "webkit2gtk",
 "webview2-com",
 "window-vibrancy",
- "windows",
+ "windows 0.61.3",
 ]

 [[package]]
@ -4683,7 +4684,7 @@ dependencies = [
 "tauri-plugin",
 "thiserror 2.0.17",
 "url",
- "windows",
+ "windows 0.61.3",
 "zbus",
 ]

@ -4761,7 +4762,7 @@ dependencies = [
 "url",
 "webkit2gtk",
 "webview2-com",
- "windows",
+ "windows 0.61.3",
 ]

 [[package]]
@ -4787,7 +4788,7 @@ dependencies = [
 "url",
 "webkit2gtk",
 "webview2-com",
- "windows",
+ "windows 0.61.3",
 "wry",
 ]

@ -5624,10 +5625,10 @@ checksum = "d4ba622a989277ef3886dd5afb3e280e3dd6d974b766118950a08f8f678ad6a4"
 dependencies = [
 "webview2-com-macros",
 "webview2-com-sys",
- "windows",
+ "windows 0.61.3",
 "windows-core 0.61.2",
- "windows-implement",
- "windows-interface",
+ "windows-implement 0.60.2",
+ "windows-interface 0.59.3",
 ]

 [[package]]
@ -5648,7 +5649,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "36695906a1b53a3bf5c4289621efedac12b73eeb0b89e7e1a89b517302d5d75c"
 dependencies = [
 "thiserror 2.0.17",
- "windows",
+ "windows 0.61.3",
 "windows-core 0.61.2",
 ]

@ -5698,6 +5699,16 @@ dependencies = [
 "windows-version",
 ]

+[[package]]
+name = "windows"
+version = "0.58.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6"
+dependencies = [
+ "windows-core 0.58.0",
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows"
 version = "0.61.3"
@ -5720,14 +5731,27 @@ dependencies = [
 "windows-core 0.61.2",
 ]

+[[package]]
+name = "windows-core"
+version = "0.58.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99"
+dependencies = [
+ "windows-implement 0.58.0",
+ "windows-interface 0.58.0",
+ "windows-result 0.2.0",
+ "windows-strings 0.1.0",
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-core"
 version = "0.61.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
 dependencies = [
- "windows-implement",
- "windows-interface",
+ "windows-implement 0.60.2",
+ "windows-interface 0.59.3",
 "windows-link 0.1.3",
 "windows-result 0.3.4",
 "windows-strings 0.4.2",
@ -5739,8 +5763,8 @@ version = "0.62.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
 dependencies = [
- "windows-implement",
- "windows-interface",
+ "windows-implement 0.60.2",
+ "windows-interface 0.59.3",
 "windows-link 0.2.1",
 "windows-result 0.4.1",
 "windows-strings 0.5.1",
@ -5757,6 +5781,17 @@ dependencies = [
 "windows-threading",
 ]

+[[package]]
+name = "windows-implement"
+version = "0.58.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.108",
+]
+
 [[package]]
 name = "windows-implement"
 version = "0.60.2"
@ -5768,6 +5803,17 @@ dependencies = [
 "syn 2.0.108",
 ]

+[[package]]
+name = "windows-interface"
+version = "0.58.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.108",
+]
+
 [[package]]
 name = "windows-interface"
 version = "0.59.3"
@ -5812,6 +5858,15 @@ dependencies = [
 "windows-strings 0.4.2",
 ]

+[[package]]
+name = "windows-result"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-result"
 version = "0.3.4"
@ -5830,6 +5885,16 @@ dependencies = [
 "windows-link 0.2.1",
 ]

+[[package]]
+name = "windows-strings"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
+dependencies = [
+ "windows-result 0.2.0",
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-strings"
 version = "0.4.2"
@ -6252,7 +6317,7 @@ dependencies = [
 "webkit2gtk",
 "webkit2gtk-sys",
 "webview2-com",
- "windows",
+ "windows 0.61.3",
 "windows-core 0.61.2",
 "windows-version",
 "x11-dl",
--- a/frontend/src-tauri/Cargo.toml
+++ b/frontend/src-tauri/Cargo.toml
@ -45,3 +45,11 @@ rand = "0.8"
 [target.'cfg(target_os = "macos")'.dependencies]
 core-foundation = "0.10"
 core-services = "1.0"
+
+[target.'cfg(target_os = "windows")'.dependencies]
+windows = { version = "0.58", features = [
+    "Win32_Foundation",
+    "Win32_System_Com",
+    "Win32_UI_Shell",
+    "Win32_System_ApplicationInstallationAndServicing",
+] }
--- a/frontend/src-tauri/src/commands/default_app.rs
+++ b/frontend/src-tauri/src/commands/default_app.rs
@ -1,8 +1,5 @@
 use crate::utils::add_log;

-#[cfg(any(target_os = "windows", target_os = "linux"))]
-use std::process::Command;
-
 /// Check if Stirling PDF is the default PDF handler
 #[tauri::command]
 pub fn is_default_pdf_handler() -> Result<bool, String> {
@ -51,50 +48,80 @@ pub fn set_as_default_pdf_handler() -> Result<String, String> {

 #[cfg(target_os = "windows")]
 fn check_default_windows() -> Result<bool, String> {
-    use std::os::windows::process::CommandExt;
-    const CREATE_NO_WINDOW: u32 = 0x08000000;
+    use windows::core::HSTRING;
+    use windows::Win32::Foundation::RPC_E_CHANGED_MODE;
+    use windows::Win32::System::Com::{
+        CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER,
+        COINIT_APARTMENTTHREADED,
+    };
+    use windows::Win32::UI::Shell::{
+        IApplicationAssociationRegistration, ApplicationAssociationRegistration,
+        ASSOCIATIONTYPE, ASSOCIATIONLEVEL,
+    };

-    // Query the default handler for .pdf extension
-    let output = Command::new("cmd")
-        .args(["/C", "assoc .pdf"])
-        .creation_flags(CREATE_NO_WINDOW)
-        .output()
-        .map_err(|e| format!("Failed to check default app: {}", e))?;
+    unsafe {
+        // Initialize COM for this thread
+        let hr = CoInitializeEx(None, COINIT_APARTMENTTHREADED);
+        // RPC_E_CHANGED_MODE means COM is already initialized, which is fine
+        if hr.is_err() && hr != RPC_E_CHANGED_MODE {
+            return Err(format!("Failed to initialize COM: {:?}", hr));
+        }

-    let assoc = String::from_utf8_lossy(&output.stdout);
-    add_log(format!("Windows PDF association: {}", assoc.trim()));
+        let result = (|| -> Result<bool, String> {
+            // Create the IApplicationAssociationRegistration instance
+            let reg: IApplicationAssociationRegistration =
+                CoCreateInstance(&ApplicationAssociationRegistration, None, CLSCTX_INPROC_SERVER)
+                    .map_err(|e| format!("Failed to create COM instance: {}", e))?;

-    // Get the ProgID for .pdf files
-    if let Some(prog_id) = assoc.trim().strip_prefix(".pdf=") {
-        // Query what application handles this ProgID
-        let output = Command::new("cmd")
-            .args(["/C", &format!("ftype {}", prog_id)])
-            .creation_flags(CREATE_NO_WINDOW)
-            .output()
-            .map_err(|e| format!("Failed to query file type: {}", e))?;
+            // Query the current default handler for .pdf extension
+            let extension = HSTRING::from(".pdf");

-        let ftype = String::from_utf8_lossy(&output.stdout);
-        add_log(format!("Windows file type: {}", ftype.trim()));
+            let default_app = reg.QueryCurrentDefault(
+                &extension,
+                ASSOCIATIONTYPE(0), // AT_FILEEXTENSION
+                ASSOCIATIONLEVEL(1), // AL_EFFECTIVE - gets the effective default (user or machine level)
+            )
+            .map_err(|e| format!("Failed to query current default: {}", e))?;

-        // Check if it contains "Stirling" or our app name
-        let is_default = ftype.to_lowercase().contains("stirling");
-        Ok(is_default)
-    } else {
-        Ok(false)
+            // Convert PWSTR to String
+            let default_str = default_app.to_string()
+                .map_err(|e| format!("Failed to convert default app string: {}", e))?;
+
+            add_log(format!("Windows PDF handler ProgID: {}", default_str));
+
+            // Check if it contains "Stirling" (case-insensitive)
+            // Note: This checks the ProgID registered by the installer
+            let is_default = default_str.to_lowercase().contains("stirling");
+            Ok(is_default)
+        })();
+
+        // Clean up COM
+        CoUninitialize();
+
+        result
    }
 }

 #[cfg(target_os = "windows")]
 fn set_default_windows() -> Result<String, String> {
-    // On Windows 10+, we need to open the Default Apps settings
-    // as programmatic setting requires a signed installer
-    Command::new("cmd")
-        .args(["/C", "start", "ms-settings:defaultapps"])
-        .spawn()
-        .map_err(|e| format!("Failed to open default apps settings: {}", e))?;
+    use std::process::Command;

-    add_log("Opened Windows Default Apps settings".to_string());
-    Ok("opened_settings".to_string())
+    // Windows 10+ approach: Open Settings app directly to default apps
+    // This is more reliable than COM APIs which require pre-registration
+    // ms-settings:defaultapps opens the default apps settings page
+    let result = Command::new("cmd")
+        .args(["/C", "start", "ms-settings:defaultapps"])
+        .output()
+        .map_err(|e| format!("Failed to open Windows Settings: {}", e))?;
+
+    if result.status.success() {
+        add_log("Opened Windows default apps settings".to_string());
+        Ok("opened_dialog".to_string())
+    } else {
+        let error = String::from_utf8_lossy(&result.stderr);
+        add_log(format!("Failed to open settings: {}", error));
+        Err(format!("Failed to open default apps settings: {}", error))
+    }
 }

 // ============================================================================
@ -184,6 +211,8 @@ fn set_default_macos() -> Result<String, String> {

 #[cfg(target_os = "linux")]
 fn check_default_linux() -> Result<bool, String> {
+    use std::process::Command;
+
    // Use xdg-mime to check the default application for PDF files
    let output = Command::new("xdg-mime")
        .args(["query", "default", "application/pdf"])
@ -200,6 +229,8 @@ fn check_default_linux() -> Result<bool, String> {

 #[cfg(target_os = "linux")]
 fn set_default_linux() -> Result<String, String> {
+    use std::process::Command;
+
    // Use xdg-mime to set the default application for PDF files
    let result = Command::new("xdg-mime")
        .args(["default", "stirling-pdf.desktop", "application/pdf"])
--- a/frontend/src/desktop/hooks/useDefaultApp.ts
+++ b/frontend/src/desktop/hooks/useDefaultApp.ts
@ -33,11 +33,11 @@ export const useDefaultApp = () => {
          body: t('defaultApp.success.message', 'Stirling PDF is now your default PDF editor'),
        });
        setIsDefault(true);
-      } else if (result === 'opened_settings') {
+      } else if (result === 'opened_dialog') {
        alert({
          alertType: 'neutral',
          title: t('defaultApp.settingsOpened.title', 'Settings Opened'),
-          body: t('defaultApp.settingsOpened.message', 'Please select Stirling PDF in your system settings'),
+          body: t('defaultApp.settingsOpened.message', 'Please select Stirling PDF in the file association dialogue'),
        });
      }
    } catch (error) {
--- a/frontend/src/desktop/hooks/useEndpointConfig.ts
+++ b/frontend/src/desktop/hooks/useEndpointConfig.ts
@ -88,8 +88,7 @@ export function useEndpointEnabled(endpoint: string): {
    try {
      setError(null);

-      const response = await apiClient.get<boolean>('/api/v1/config/endpoint-enabled', {
-        params: { endpoint },
+      const response = await apiClient.get<boolean>(`/api/v1/config/endpoint-enabled?endpoint=${encodeURIComponent(endpoint)}`, {
        suppressErrorToast: true,
      });

--- a/frontend/src/desktop/services/defaultAppService.ts
+++ b/frontend/src/desktop/services/defaultAppService.ts
@ -22,10 +22,10 @@ export const defaultAppService = {
   * Set or prompt to set Stirling PDF as default PDF handler
   * Returns a status string indicating what happened
   */
-  async setAsDefaultPdfHandler(): Promise<'set_successfully' | 'opened_settings' | 'error'> {
+  async setAsDefaultPdfHandler(): Promise<'set_successfully' | 'opened_dialog' | 'error'> {
    try {
      const result = await invoke<string>('set_as_default_pdf_handler');
-      return result as 'set_successfully' | 'opened_settings';
+      return result as 'set_successfully' | 'opened_dialog';
    } catch (error) {
      console.error('[DefaultApp] Failed to set default handler:', error);
      return 'error';