diff --git a/src/server/event.rs b/src/server/event.rs
index 2e2afc1..ea2277c 100644
--- a/src/server/event.rs
+++ b/src/server/event.rs
@@ -983,7 +983,7 @@ impl Event for client::wl_touch::Event {
     }
 }
 
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, PartialEq, Eq)]
 pub(super) struct OnOutput(pub Entity);
 struct OutputName(String);
 fn get_output_name(output: Option<&OnOutput>, world: &World) -> Option<String> {
diff --git a/src/server/mod.rs b/src/server/mod.rs
index 662d1a9..0496628 100644
--- a/src/server/mod.rs
+++ b/src/server/mod.rs
@@ -795,26 +795,38 @@ impl<S: X11Selection + 'static> InnerServerState<S> {
         handle_new_globals::<S>(&mut self.globals_map, &self.dh, &globals);
 
         let globals = std::mem::take(&mut self.world.removed_globals);
-        if globals.is_empty() {
-            return;
+        for global in globals {
+            let (global_struct, global_id) = self.globals_map.remove(&global).unwrap();
+            self.dh.disable_global::<InnerServerState<S>>(global_id);
+            if global_struct.interface == <WlOutput>::interface().name {
+                self.remove_output(global);
+            }
         }
+    }
+
+    fn remove_output(&mut self, global: GlobalName) {
         let query = self
             .world
             .query_mut::<(&WlOutput, &GlobalName)>()
             .into_iter()
             .map(|(e, (_, name))| (e, *name))
             .collect::<Vec<_>>();
-        for global in globals {
-            let (global_struct, global_id) = self.globals_map.remove(&global).unwrap();
-            self.dh.disable_global::<InnerServerState<S>>(global_id);
-            if global_struct.interface == <WlOutput>::interface().name {
-                for (entity, name) in query.iter() {
-                    if *name == global {
-                        self.updated_outputs.push(*entity);
-                        self.world.despawn(*entity).unwrap();
-                        break;
+        for (entity, name) in query.iter() {
+            if *name == global {
+                self.updated_outputs.push(*entity);
+                self.world.remove_one::<OutputScaleFactor>(*entity).unwrap();
+                let query = self
+                    .world
+                    .query_mut::<&OnOutput>()
+                    .into_iter()
+                    .map(|(e, on_out)| (e, *on_out))
+                    .collect::<Vec<_>>();
+                for (e, on_out) in query.iter() {
+                    if *on_out == OnOutput(*entity) {
+                        self.world.remove_one::<OnOutput>(*e).unwrap();
                     }
                 }
+                break;
             }
         }
     }
diff --git a/src/server/tests.rs b/src/server/tests.rs
index 8f398e6..68495d0 100644
--- a/src/server/tests.rs
+++ b/src/server/tests.rs
@@ -2770,6 +2770,17 @@ fn disconnected_output_rescaling() {
     assert_eq!(f.satellite.inner.new_scale, Some(1.5));
 
     f.remove_output(output_ext);
+    let surface_data = f.testwl.get_surface_data(id).expect("No surface data");
+    let fractional = surface_data
+        .fractional
+        .as_ref()
+        .expect("No fractional scale for surface");
+    fractional.preferred_scale(120); // 1.0 scale
+    f.run();
+    f.run();
+    // An fractional scale change done while the surface is on a removed output is ignored
+    assert_eq!(f.satellite.inner.new_scale, Some(1.5));
+
     f.testwl.move_surface_to_output(id, &output_main);
     let surface_data = f.testwl.get_surface_data(id).expect("No surface data");
     let fractional = surface_data
@@ -2779,7 +2790,7 @@ fn disconnected_output_rescaling() {
     fractional.preferred_scale(240); // 2.0 scale
     f.run();
     f.run();
-    // Afteer the output is disconnected, only the 2x scale output remains, so use that scale
+    // After the output is disconnected, only the 2x scale output remains, so use that scale
     assert_eq!(f.satellite.inner.new_scale, Some(2.0));
 }