fix(desktop): normalize Linux Wayland/X11 backend and decoration policy (#13143)

Co-authored-by: Brendan Allan <brendonovich@outlook.com>

packages/desktop/src-tauri/src/lib.rs

@@ -2,6 +2,8 @@ mod cli;
 mod constants;
 #[cfg(target_os = "linux")]
 pub mod linux_display;
+#[cfg(target_os = "linux")]
+pub mod linux_windowing;
 mod logging;
 mod markdown;
 mod server;

packages/desktop/src-tauri/src/linux_windowing.rs

@@ -0,0 +1,475 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Backend {
+    Auto,
+    Wayland,
+    X11,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct BackendDecision {
+    pub backend: Backend,
+    pub note: String,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct SessionEnv {
+    pub wayland_display: bool,
+    pub xdg_session_type: Option<String>,
+    pub display: bool,
+    pub xdg_current_desktop: Option<String>,
+    pub xdg_session_desktop: Option<String>,
+    pub desktop_session: Option<String>,
+    pub oc_allow_wayland: Option<String>,
+    pub oc_force_x11: Option<String>,
+    pub oc_force_wayland: Option<String>,
+    pub oc_linux_decorations: Option<String>,
+    pub oc_force_decorations: Option<String>,
+    pub oc_no_decorations: Option<String>,
+    pub i3_sock: bool,
+}
+
+impl SessionEnv {
+    pub fn capture() -> Self {
+        Self {
+            wayland_display: std::env::var_os("WAYLAND_DISPLAY").is_some(),
+            xdg_session_type: std::env::var("XDG_SESSION_TYPE").ok(),
+            display: std::env::var_os("DISPLAY").is_some(),
+            xdg_current_desktop: std::env::var("XDG_CURRENT_DESKTOP").ok(),
+            xdg_session_desktop: std::env::var("XDG_SESSION_DESKTOP").ok(),
+            desktop_session: std::env::var("DESKTOP_SESSION").ok(),
+            oc_allow_wayland: std::env::var("OC_ALLOW_WAYLAND").ok(),
+            oc_force_x11: std::env::var("OC_FORCE_X11").ok(),
+            oc_force_wayland: std::env::var("OC_FORCE_WAYLAND").ok(),
+            oc_linux_decorations: std::env::var("OC_LINUX_DECORATIONS").ok(),
+            oc_force_decorations: std::env::var("OC_FORCE_DECORATIONS").ok(),
+            oc_no_decorations: std::env::var("OC_NO_DECORATIONS").ok(),
+            i3_sock: std::env::var_os("I3SOCK").is_some(),
+        }
+    }
+}
+
+pub fn select_backend(env: &SessionEnv, prefer_wayland: bool) -> Option<BackendDecision> {
+    if is_truthy(env.oc_force_x11.as_deref()) {
+        return Some(BackendDecision {
+            backend: Backend::X11,
+            note: "Forcing X11 due to OC_FORCE_X11=1".into(),
+        });
+    }
+
+    if is_truthy(env.oc_force_wayland.as_deref()) {
+        return Some(BackendDecision {
+            backend: Backend::Wayland,
+            note: "Forcing native Wayland due to OC_FORCE_WAYLAND=1".into(),
+        });
+    }
+
+    if !is_wayland_session(env) {
+        return None;
+    }
+
+    if prefer_wayland {
+        return Some(BackendDecision {
+            backend: Backend::Wayland,
+            note: "Wayland session detected; forcing native Wayland from settings".into(),
+        });
+    }
+
+    if is_truthy(env.oc_allow_wayland.as_deref()) {
+        return Some(BackendDecision {
+            backend: Backend::Wayland,
+            note: "Wayland session detected; forcing native Wayland due to OC_ALLOW_WAYLAND=1"
+                .into(),
+        });
+    }
+
+    Some(BackendDecision {
+        backend: Backend::Auto,
+        note: "Wayland session detected; using native Wayland first with X11 fallback (auto backend). Set OC_FORCE_X11=1 to force X11."
+            .into(),
+    })
+}
+
+pub fn use_decorations(env: &SessionEnv) -> bool {
+    if let Some(mode) = decoration_override(env.oc_linux_decorations.as_deref()) {
+        return match mode {
+            DecorationOverride::Native => true,
+            DecorationOverride::None => false,
+            DecorationOverride::Auto => default_use_decorations(env),
+        };
+    }
+
+    if is_truthy(env.oc_force_decorations.as_deref()) {
+        return true;
+    }
+    if is_truthy(env.oc_no_decorations.as_deref()) {
+        return false;
+    }
+
+    default_use_decorations(env)
+}
+
+fn default_use_decorations(env: &SessionEnv) -> bool {
+    if is_known_tiling_session(env) {
+        return false;
+    }
+    if !is_wayland_session(env) {
+        return true;
+    }
+    is_full_desktop_session(env)
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum DecorationOverride {
+    Auto,
+    Native,
+    None,
+}
+
+fn decoration_override(value: Option<&str>) -> Option<DecorationOverride> {
+    let value = value?.trim().to_ascii_lowercase();
+    if matches!(value.as_str(), "auto") {
+        return Some(DecorationOverride::Auto);
+    }
+    if matches!(
+        value.as_str(),
+        "native" | "server" | "de" | "wayland" | "on" | "true" | "1"
+    ) {
+        return Some(DecorationOverride::Native);
+    }
+    if matches!(
+        value.as_str(),
+        "none" | "off" | "false" | "0" | "client" | "csd"
+    ) {
+        return Some(DecorationOverride::None);
+    }
+    None
+}
+
+fn is_truthy(value: Option<&str>) -> bool {
+    matches!(
+        value.map(|v| v.trim().to_ascii_lowercase()),
+        Some(v) if matches!(v.as_str(), "1" | "true" | "yes" | "on")
+    )
+}
+
+fn is_wayland_session(env: &SessionEnv) -> bool {
+    env.wayland_display
+        || matches!(
+            env.xdg_session_type.as_deref(),
+            Some(value) if value.eq_ignore_ascii_case("wayland")
+        )
+}
+
+fn is_full_desktop_session(env: &SessionEnv) -> bool {
+    desktop_tokens(env).any(|value| {
+        matches!(
+            value.as_str(),
+            "gnome"
+                | "kde"
+                | "plasma"
+                | "xfce"
+                | "xfce4"
+                | "x-cinnamon"
+                | "cinnamon"
+                | "mate"
+                | "lxqt"
+                | "budgie"
+                | "pantheon"
+                | "deepin"
+                | "unity"
+                | "cosmic"
+        )
+    })
+}
+
+fn is_known_tiling_session(env: &SessionEnv) -> bool {
+    if env.i3_sock {
+        return true;
+    }
+
+    desktop_tokens(env).any(|value| {
+        matches!(
+            value.as_str(),
+            "niri"
+                | "sway"
+                | "swayfx"
+                | "hyprland"
+                | "river"
+                | "i3"
+                | "i3wm"
+                | "bspwm"
+                | "dwm"
+                | "qtile"
+                | "xmonad"
+                | "leftwm"
+                | "dwl"
+                | "awesome"
+                | "herbstluftwm"
+                | "spectrwm"
+                | "worm"
+                | "i3-gnome"
+        )
+    })
+}
+
+fn desktop_tokens<'a>(env: &'a SessionEnv) -> impl Iterator<Item = String> + 'a {
+    [
+        env.xdg_current_desktop.as_deref(),
+        env.xdg_session_desktop.as_deref(),
+        env.desktop_session.as_deref(),
+    ]
+    .into_iter()
+    .flatten()
+    .flat_map(|desktop| desktop.split(':'))
+    .map(|value| value.trim().to_ascii_lowercase())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn prefers_wayland_first_on_wayland_session() {
+        let env = SessionEnv {
+            wayland_display: true,
+            display: true,
+            ..Default::default()
+        };
+
+        let decision = select_backend(&env, false).expect("missing decision");
+        assert_eq!(decision.backend, Backend::Auto);
+    }
+
+    #[test]
+    fn force_x11_override_wins() {
+        let env = SessionEnv {
+            wayland_display: true,
+            display: true,
+            oc_force_x11: Some("1".into()),
+            oc_allow_wayland: Some("1".into()),
+            oc_force_wayland: Some("1".into()),
+            ..Default::default()
+        };
+
+        let decision = select_backend(&env, true).expect("missing decision");
+        assert_eq!(decision.backend, Backend::X11);
+    }
+
+    #[test]
+    fn prefer_wayland_forces_wayland_backend() {
+        let env = SessionEnv {
+            wayland_display: true,
+            display: true,
+            ..Default::default()
+        };
+
+        let decision = select_backend(&env, true).expect("missing decision");
+        assert_eq!(decision.backend, Backend::Wayland);
+    }
+
+    #[test]
+    fn force_wayland_override_works_outside_wayland_session() {
+        let env = SessionEnv {
+            display: true,
+            oc_force_wayland: Some("1".into()),
+            ..Default::default()
+        };
+
+        let decision = select_backend(&env, false).expect("missing decision");
+        assert_eq!(decision.backend, Backend::Wayland);
+    }
+
+    #[test]
+    fn allow_wayland_forces_wayland_backend() {
+        let env = SessionEnv {
+            wayland_display: true,
+            display: true,
+            oc_allow_wayland: Some("1".into()),
+            ..Default::default()
+        };
+
+        let decision = select_backend(&env, false).expect("missing decision");
+        assert_eq!(decision.backend, Backend::Wayland);
+    }
+
+    #[test]
+    fn xdg_session_type_wayland_is_detected() {
+        let env = SessionEnv {
+            xdg_session_type: Some("wayland".into()),
+            ..Default::default()
+        };
+
+        let decision = select_backend(&env, false).expect("missing decision");
+        assert_eq!(decision.backend, Backend::Auto);
+    }
+
+    #[test]
+    fn returns_none_when_not_wayland_and_no_overrides() {
+        let env = SessionEnv {
+            display: true,
+            xdg_current_desktop: Some("GNOME".into()),
+            ..Default::default()
+        };
+
+        assert!(select_backend(&env, false).is_none());
+    }
+
+    #[test]
+    fn prefer_wayland_setting_does_not_override_x11_session() {
+        let env = SessionEnv {
+            display: true,
+            xdg_current_desktop: Some("GNOME".into()),
+            ..Default::default()
+        };
+
+        assert!(select_backend(&env, true).is_none());
+    }
+
+    #[test]
+    fn disables_decorations_on_niri() {
+        let env = SessionEnv {
+            xdg_current_desktop: Some("niri".into()),
+            wayland_display: true,
+            ..Default::default()
+        };
+
+        assert!(!use_decorations(&env));
+    }
+
+    #[test]
+    fn keeps_decorations_on_gnome() {
+        let env = SessionEnv {
+            xdg_current_desktop: Some("GNOME".into()),
+            wayland_display: true,
+            ..Default::default()
+        };
+
+        assert!(use_decorations(&env));
+    }
+
+    #[test]
+    fn disables_decorations_when_session_desktop_is_tiling() {
+        let env = SessionEnv {
+            xdg_session_desktop: Some("Hyprland".into()),
+            wayland_display: true,
+            ..Default::default()
+        };
+
+        assert!(!use_decorations(&env));
+    }
+
+    #[test]
+    fn disables_decorations_for_unknown_wayland_session() {
+        let env = SessionEnv {
+            xdg_current_desktop: Some("labwc".into()),
+            wayland_display: true,
+            ..Default::default()
+        };
+
+        assert!(!use_decorations(&env));
+    }
+
+    #[test]
+    fn disables_decorations_for_dwm_on_x11() {
+        let env = SessionEnv {
+            xdg_current_desktop: Some("dwm".into()),
+            display: true,
+            ..Default::default()
+        };
+
+        assert!(!use_decorations(&env));
+    }
+
+    #[test]
+    fn disables_decorations_for_i3_on_x11() {
+        let env = SessionEnv {
+            xdg_current_desktop: Some("i3".into()),
+            display: true,
+            ..Default::default()
+        };
+
+        assert!(!use_decorations(&env));
+    }
+
+    #[test]
+    fn disables_decorations_for_i3sock_without_xdg_tokens() {
+        let env = SessionEnv {
+            display: true,
+            i3_sock: true,
+            ..Default::default()
+        };
+
+        assert!(!use_decorations(&env));
+    }
+
+    #[test]
+    fn keeps_decorations_for_gnome_on_x11() {
+        let env = SessionEnv {
+            xdg_current_desktop: Some("GNOME".into()),
+            display: true,
+            ..Default::default()
+        };
+
+        assert!(use_decorations(&env));
+    }
+
+    #[test]
+    fn no_decorations_override_wins() {
+        let env = SessionEnv {
+            xdg_current_desktop: Some("GNOME".into()),
+            oc_no_decorations: Some("1".into()),
+            ..Default::default()
+        };
+
+        assert!(!use_decorations(&env));
+    }
+
+    #[test]
+    fn linux_decorations_native_override_wins() {
+        let env = SessionEnv {
+            xdg_current_desktop: Some("niri".into()),
+            wayland_display: true,
+            oc_linux_decorations: Some("native".into()),
+            ..Default::default()
+        };
+
+        assert!(use_decorations(&env));
+    }
+
+    #[test]
+    fn linux_decorations_none_override_wins() {
+        let env = SessionEnv {
+            xdg_current_desktop: Some("GNOME".into()),
+            wayland_display: true,
+            oc_linux_decorations: Some("none".into()),
+            ..Default::default()
+        };
+
+        assert!(!use_decorations(&env));
+    }
+
+    #[test]
+    fn linux_decorations_auto_uses_default_policy() {
+        let env = SessionEnv {
+            xdg_current_desktop: Some("sway".into()),
+            wayland_display: true,
+            oc_linux_decorations: Some("auto".into()),
+            ..Default::default()
+        };
+
+        assert!(!use_decorations(&env));
+    }
+
+    #[test]
+    fn linux_decorations_override_beats_legacy_overrides() {
+        let env = SessionEnv {
+            xdg_current_desktop: Some("GNOME".into()),
+            wayland_display: true,
+            oc_linux_decorations: Some("none".into()),
+            oc_force_decorations: Some("1".into()),
+            ..Default::default()
+        };
+
+        assert!(!use_decorations(&env));
+    }
+}

packages/desktop/src-tauri/src/main.rs

@@ -4,6 +4,7 @@
 // borrowed from https://github.com/skyline69/balatro-mod-manager
 #[cfg(target_os = "linux")]
 fn configure_display_backend() -> Option<String> {
+    use opencode_lib::linux_windowing::{Backend, SessionEnv, select_backend};
     use std::env;
 
     let set_env_if_absent = |key: &str, value: &str| {
@@ -14,45 +15,28 @@ fn configure_display_backend() -> Option<String> {
         }
     };
 
-    let on_wayland = env::var_os("WAYLAND_DISPLAY").is_some()
-        || matches!(
-            env::var("XDG_SESSION_TYPE"),
-            Ok(v) if v.eq_ignore_ascii_case("wayland")
-        );
-    if !on_wayland {
-        return None;
-    }
-
+    let session = SessionEnv::capture();
     let prefer_wayland = opencode_lib::linux_display::read_wayland().unwrap_or(false);
-    let allow_wayland = prefer_wayland
-        || matches!(
-            env::var("OC_ALLOW_WAYLAND"),
-            Ok(v) if matches!(v.to_ascii_lowercase().as_str(), "1" | "true" | "yes")
-        );
-    if allow_wayland {
-        if prefer_wayland {
-            return Some("Wayland session detected; using native Wayland from settings".into());
+    let decision = select_backend(&session, prefer_wayland)?;
+
+    match decision.backend {
+        Backend::X11 => {
+            set_env_if_absent("WINIT_UNIX_BACKEND", "x11");
+            set_env_if_absent("GDK_BACKEND", "x11");
+            set_env_if_absent("WEBKIT_DISABLE_DMABUF_RENDERER", "1");
+        }
+        Backend::Wayland => {
+            set_env_if_absent("WINIT_UNIX_BACKEND", "wayland");
+            set_env_if_absent("GDK_BACKEND", "wayland");
+            set_env_if_absent("WEBKIT_DISABLE_DMABUF_RENDERER", "1");
+        }
+        Backend::Auto => {
+            set_env_if_absent("GDK_BACKEND", "wayland,x11");
+            set_env_if_absent("WEBKIT_DISABLE_DMABUF_RENDERER", "1");
         }
-        return Some("Wayland session detected; respecting OC_ALLOW_WAYLAND=1".into());
     }
 
-    // Prefer XWayland when available to avoid Wayland protocol errors seen during startup.
-    if env::var_os("DISPLAY").is_some() {
-        set_env_if_absent("WINIT_UNIX_BACKEND", "x11");
-        set_env_if_absent("GDK_BACKEND", "x11");
-        set_env_if_absent("WEBKIT_DISABLE_DMABUF_RENDERER", "1");
-        return Some(
-            "Wayland session detected; forcing X11 backend to avoid compositor protocol errors. \
-                Set OC_ALLOW_WAYLAND=1 to keep native Wayland."
-                .into(),
-        );
-    }
-
-    set_env_if_absent("WEBKIT_DISABLE_DMABUF_RENDERER", "1");
-    Some(
-        "Wayland session detected without X11; leaving Wayland enabled (set WINIT_UNIX_BACKEND/GDK_BACKEND manually if needed)."
-            .into(),
-    )
+    Some(decision.note)
 }
 
 fn main() {

packages/desktop/src-tauri/src/windows.rs

@@ -7,6 +7,22 @@ use tauri::{AppHandle, Manager, Runtime, WebviewUrl, WebviewWindow, WebviewWindo
 use tauri_plugin_window_state::AppHandleExt;
 use tokio::sync::mpsc;
 
+#[cfg(target_os = "linux")]
+use std::sync::OnceLock;
+
+#[cfg(target_os = "linux")]
+fn use_decorations() -> bool {
+    static DECORATIONS: OnceLock<bool> = OnceLock::new();
+    *DECORATIONS.get_or_init(|| {
+        crate::linux_windowing::use_decorations(&crate::linux_windowing::SessionEnv::capture())
+    })
+}
+
+#[cfg(not(target_os = "linux"))]
+fn use_decorations() -> bool {
+    true
+}
+
 pub struct MainWindow(WebviewWindow);
 
 impl Deref for MainWindow {
@@ -31,13 +47,13 @@ impl MainWindow {
             .ok()
             .map(|v| v.enabled)
             .unwrap_or(false);
-
+        let decorations = use_decorations();
         let window_builder = base_window_config(
             WebviewWindowBuilder::new(app, Self::LABEL, WebviewUrl::App("/".into())),
             app,
+            decorations,
         )
         .title("OpenCode")
-        .decorations(true)
         .disable_drag_drop_handler()
         .zoom_hotkeys_enabled(false)
         .visible(true)
@@ -113,9 +129,12 @@ impl LoadingWindow {
     pub const LABEL: &str = "loading";
 
     pub fn create(app: &AppHandle) -> Result<Self, tauri::Error> {
+        let decorations = use_decorations();
+
         let window_builder = base_window_config(
             WebviewWindowBuilder::new(app, Self::LABEL, tauri::WebviewUrl::App("/loading".into())),
             app,
+            decorations,
         )
         .center()
         .resizable(false)
@@ -129,8 +148,9 @@ impl LoadingWindow {
 fn base_window_config<'a, R: Runtime, M: Manager<R>>(
     window_builder: WebviewWindowBuilder<'a, R, M>,
     _app: &AppHandle,
+    decorations: bool,
 ) -> WebviewWindowBuilder<'a, R, M> {
-    let window_builder = window_builder.decorations(true);
+    let window_builder = window_builder.decorations(decorations);
 
     #[cfg(windows)]
     let window_builder = window_builder