diff --git a/crates/bevy_asset/src/server/info.rs b/crates/bevy_asset/src/server/info.rs
index 1297daec49ba8..41bbdf27bf3ec 100644
--- a/crates/bevy_asset/src/server/info.rs
+++ b/crates/bevy_asset/src/server/info.rs
@@ -554,6 +554,35 @@ impl AssetInfos {
         }
     }
 
+    fn remove_dependants_and_labels(
+        info: &AssetInfo,
+        loader_dependants: &mut HashMap<AssetPath<'static>, HashSet<AssetPath<'static>>>,
+        path: &AssetPath<'static>,
+        living_labeled_assets: &mut HashMap<AssetPath<'static>, HashSet<String>>,
+    ) {
+        for loader_dependency in info.loader_dependencies.keys() {
+            if let Some(dependants) = loader_dependants.get_mut(loader_dependency) {
+                dependants.remove(path);
+            }
+        }
+
+        let Some(label) = path.label() else {
+            return;
+        };
+
+        let mut without_label = path.to_owned();
+        without_label.remove_label();
+
+        let Entry::Occupied(mut entry) = living_labeled_assets.entry(without_label) else {
+            return;
+        };
+
+        entry.get_mut().remove(label);
+        if entry.get().is_empty() {
+            entry.remove();
+        }
+    }
+
     fn process_handle_drop_internal(
         infos: &mut HashMap<UntypedAssetId, AssetInfo>,
         path_to_id: &mut HashMap<AssetPath<'static>, UntypedAssetId>,
@@ -562,44 +591,32 @@ impl AssetInfos {
         watching_for_changes: bool,
         id: UntypedAssetId,
     ) -> bool {
-        match infos.entry(id) {
-            Entry::Occupied(mut entry) => {
-                if entry.get_mut().handle_drops_to_skip > 0 {
-                    entry.get_mut().handle_drops_to_skip -= 1;
-                    false
-                } else {
-                    let info = entry.remove();
-                    if let Some(path) = info.path {
-                        if watching_for_changes {
-                            for loader_dependency in info.loader_dependencies.keys() {
-                                if let Some(dependants) =
-                                    loader_dependants.get_mut(loader_dependency)
-                                {
-                                    dependants.remove(&path);
-                                }
-                            }
-                            if let Some(label) = path.label() {
-                                let mut without_label = path.to_owned();
-                                without_label.remove_label();
-                                if let Entry::Occupied(mut entry) =
-                                    living_labeled_assets.entry(without_label)
-                                {
-                                    entry.get_mut().remove(label);
-                                    if entry.get().is_empty() {
-                                        entry.remove();
-                                    }
-                                };
-                            }
-                        }
-                        path_to_id.remove(&path);
-                    }
-                    true
-                }
-            }
+        let Entry::Occupied(mut entry) = infos.entry(id) else {
             // Either the asset was already dropped, it doesn't exist, or it isn't managed by the asset server
             // None of these cases should result in a removal from the Assets collection
-            Entry::Vacant(_) => false,
+            return false;
+        };
+
+        if entry.get_mut().handle_drops_to_skip > 0 {
+            entry.get_mut().handle_drops_to_skip -= 1;
+            return false;
         }
+
+        let info = entry.remove();
+        let Some(path) = &info.path else {
+            return true;
+        };
+
+        if watching_for_changes {
+            Self::remove_dependants_and_labels(
+                &info,
+                loader_dependants,
+                path,
+                living_labeled_assets,
+            );
+        }
+        path_to_id.remove(path);
+        true
     }
 
     /// Consumes all current handle drop events. This will update information in [`AssetInfos`], but it
@@ -625,7 +642,6 @@ impl AssetInfos {
         }
     }
 }
-
 /// Determines how a handle should be initialized
 #[derive(Copy, Clone, PartialEq, Eq)]
 pub(crate) enum HandleLoadingMode {