Detecting if UTILITY is popup (#323)

Ardour uses UTILITY atom all over the place for toplevel windows:
Plugins, Dialogs etc
However it does not provide any MOTIF_HINTS at all.

WeChat however uses them for popups and also provides MOTIF_HINTS with
flags 0x2 indicating that only decorations are active. MaterialMaker
also follows what WeChat is doing for right click menus.

This fix assigns UTILITY as popup ONLY if MOTIF_HINTS are provided and
functions are not active.

Couple of apps like Godot mark their windows
(_NET_WM_WINDOW_TYPE_UTILITY) with override_redirect which makes them
popup by default. Potentionally is_popup can be overriden in case MOTIF
functions exists with so no_function_motif would be false.

This fix prefers override_redirect in case that scenario comes up.

Closes #294

src/server/event.rs

@@ -140,7 +140,7 @@ impl Event for SurfaceEvents {
                 let needs_server_side_decorations = window_data
                     .attrs
                     .decorations
-                    .is_none_or(|d| d == Decorations::Server);
+                    .is_none_or(|d| d.is_serverside());
 
                 if mode == Mode::ServerSide || !needs_server_side_decorations {
                     let mut role = entity.get::<&mut SurfaceRole>().unwrap();

src/server/mod.rs

@@ -1400,11 +1400,8 @@ impl<S: X11Selection + 'static> InnerServerState<S> {
             .world
             .get::<&client::wl_surface::WlSurface>(entity)
             .unwrap();
-        let needs_satellite_decorations = wl_decoration.is_none()
+        let needs_satellite_decorations =
-            && window
+            wl_decoration.is_none() && window.attrs.decorations.is_none_or(|d| d.is_serverside());
-                .attrs
-                .decorations
-                .is_none_or(|d| d == Decorations::Server);
         let (sat_decoration, buf) = needs_satellite_decorations
             .then(|| {
                 DecorationsDataSatellite::try_new(

src/xstate/mod.rs

@@ -686,7 +686,10 @@ impl XState {
         motif_hints: Option<motif::Hints>,
         has_transient_for: bool,
     ) -> XResult<bool> {
+        let mut motif_popup = false;
         if let Some(hints) = motif_hints {
+            // If MOTIF_WM_HINTS provides no decorations for client assume its a popup
+            motif_popup = hints.decorations.is_some_and(|d| d.is_clientside());
             // If the motif hints indicate the user shouldn't be able to do anything
             // to the window at all, it stands to reason it's probably a popup.
             if hints.functions.is_some_and(|f| f.is_empty()) {
@@ -735,7 +738,10 @@ impl XState {
         for ty in window_types {
             match ty {
                 x if x == self.window_atoms.normal || x == self.window_atoms.dialog => {
-                    is_popup = override_redirect;
+                    is_popup = override_redirect
+                }
+                x if x == self.window_atoms.utility => {
+                    is_popup = override_redirect || motif_popup;
                 }
                 x if [
                     self.window_atoms.menu,
@@ -743,7 +749,6 @@ impl XState {
                     self.window_atoms.dropdown_menu,
                     self.window_atoms.tooltip,
                     self.window_atoms.drag_n_drop,
-                    self.window_atoms.utility,
                 ]
                 .contains(&x) =>
                 {
@@ -1151,6 +1156,19 @@ mod motif {
         }
     }
 
+    bitflags! {
+        #[derive(Debug, PartialEq, Eq, Clone, Copy)]
+        pub struct Decorations: u32 {
+            const All = 1;
+            const Border = 2;
+            const Resizeh = 4;
+            const TitleBar = 8;
+            const Menu = 16;
+            const Minimize = 32;
+            const Maximize = 64;
+        }
+    }
+
     #[derive(Default)]
     pub(super) struct Hints {
         pub(super) functions: Option<Functions>,
@@ -1167,25 +1185,28 @@ mod motif {
                 ret.functions = Some(Functions::from_bits_truncate(value[1]));
             }
             if flags.contains(HintsFlags::Decorations) {
-                ret.decorations = value[2].try_into().ok();
+                ret.decorations = Some(Decorations::from_bits_truncate(value[2]));
             }
 
             ret
         }
     }
+    impl Decorations {
+        pub fn is_clientside(&self) -> bool {
+            self.is_empty()
+        }
 
-    #[derive(Debug, PartialEq, Eq, Clone, Copy, num_enum::TryFromPrimitive)]
+        pub fn is_serverside(&self) -> bool {
-    #[repr(u32)]
+            !self.is_empty()
-    pub enum Decorations {
+        }
-        Client = 0,
-        Server = 1,
     }
 
     impl From<Decorations> for zxdg_toplevel_decoration_v1::Mode {
-        fn from(value: Decorations) -> Self {
+        fn from(decorations: Decorations) -> Self {
-            match value {
+            if decorations.is_empty() {
-                Decorations::Client => zxdg_toplevel_decoration_v1::Mode::ClientSide,
+                zxdg_toplevel_decoration_v1::Mode::ClientSide
-                Decorations::Server => zxdg_toplevel_decoration_v1::Mode::ServerSide,
+            } else {
+                zxdg_toplevel_decoration_v1::Mode::ServerSide
             }
         }
     }

tests/integration.rs

@@ -2048,7 +2048,37 @@ fn popup_heuristics() {
         connection.atoms.win_type,
         &[connection.atoms.win_type_utility],
     );
+    connection.set_property(
+        wechat_popup,
+        connection.atoms.motif_wm_hints,
+        connection.atoms.motif_wm_hints,
+        &[0x2_u32, 0, 0, 0, 0],
+    );
     f.map_as_popup(&mut connection, wechat_popup);
+
+    let godot_popup = connection.new_window(connection.root, 10, 10, 50, 50, true);
+    connection.set_property(
+        godot_popup,
+        x::ATOM_ATOM,
+        connection.atoms.win_type,
+        &[connection.atoms.win_type_utility],
+    );
+    connection.set_property(
+        godot_popup,
+        connection.atoms.motif_wm_hints,
+        connection.atoms.motif_wm_hints,
+        &[0x2_u32, 0, 0, 0, 0],
+    );
+    f.map_as_popup(&mut connection, godot_popup);
+
+    let ardour_toplevel = connection.new_window(connection.root, 10, 10, 50, 50, false);
+    connection.set_property(
+        ardour_toplevel,
+        x::ATOM_ATOM,
+        connection.atoms.win_type,
+        &[connection.atoms.win_type_utility],
+    );
+    f.map_as_toplevel(&mut connection, ardour_toplevel);
 }
 
 #[test]