remove old edges when task recomputation completes

vercel · Aug 13, 2024 · d06d548 · d06d548
1 parent 21e9206
commit d06d548
Show file tree

Hide file tree

Showing 4 changed files with 223 additions and 8 deletions.
diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs
@@ -19,7 +19,7 @@ use anyhow::Result;
 use auto_hash_map::{AutoMap, AutoSet};
 use dashmap::DashMap;
 pub use operation::AnyOperation;
-use operation::ConnectChildOperation;
+use operation::{CleanupOldEdgesOperation, ConnectChildOperation, OutdatedEdge};
 use parking_lot::{Condvar, Mutex};
 use rustc_hash::FxHasher;
 use smallvec::smallvec;
@@ -277,13 +277,16 @@ impl TurboTasksBackend {
                 });
                 drop(task);
                 let mut reader_task = ctx.task(reader);
-                reader_task.add(CachedDataItem::CellDependency {
-                    target: CellRef {
-                        task: task_id,
-                        cell,
-                    },
-                    value: (),
-                });
+                let target = CellRef {
+                    task: task_id,
+                    cell,
+                };
+                if reader_task
+                    .remove(&CachedDataItemKey::OutdatedCellDependency { target })
+                    .is_none()
+                {
+                    reader_task.add(CachedDataItem::CellDependency { target, value: () });
+                }
             }
             return Ok(Ok(CellContent(Some(content)).into_typed(cell.type_id)));
         }
@@ -413,8 +416,95 @@ impl Backend for TurboTasksBackend {
                     done_event,
                 },
             });
+
+            // Make all current children outdated (remove left-over outdated children)
+            enum Child {
+                Current(TaskId),
+                Outdated(TaskId),
+            }
+            let children = task
+                .iter()
+                .filter_map(|(key, _)| match *key {
+                    CachedDataItemKey::Child { task } => Some(Child::Current(task)),
+                    CachedDataItemKey::OutdatedChild { task } => Some(Child::Outdated(task)),
+                    _ => None,
+                })
+                .collect::<Vec<_>>();
+            for child in children {
+                match child {
+                    Child::Current(child) => {
+                        task.add(CachedDataItem::OutdatedChild {
+                            task: child,
+                            value: (),
+                        });
+                    }
+                    Child::Outdated(child) => {
+                        if !task.has_key(&CachedDataItemKey::Child { task: child }) {
+                            task.remove(&CachedDataItemKey::OutdatedChild { task: child });
+                        }
+                    }
+                }
+            }
+
+            // Make all dependencies outdated
+            enum Dep {
+                CurrentCell(CellRef),
+                CurrentOutput(TaskId),
+                OutdatedCell(CellRef),
+                OutdatedOutput(TaskId),
+            }
+            let dependencies = task
+                .iter()
+                .filter_map(|(key, _)| match *key {
+                    CachedDataItemKey::CellDependency { target } => Some(Dep::CurrentCell(target)),
+                    CachedDataItemKey::OutputDependency { target } => {
+                        Some(Dep::CurrentOutput(target))
+                    }
+                    CachedDataItemKey::OutdatedCellDependency { target } => {
+                        Some(Dep::OutdatedCell(target))
+                    }
+                    CachedDataItemKey::OutdatedOutputDependency { target } => {
+                        Some(Dep::OutdatedOutput(target))
+                    }
+                    _ => None,
+                })
+                .collect::<Vec<_>>();
+            for dep in dependencies {
+                match dep {
+                    Dep::CurrentCell(cell) => {
+                        task.add(CachedDataItem::OutdatedCellDependency {
+                            target: cell,
+                            value: (),
+                        });
+                    }
+                    Dep::CurrentOutput(output) => {
+                        task.add(CachedDataItem::OutdatedOutputDependency {
+                            target: output,
+                            value: (),
+                        });
+                    }
+                    Dep::OutdatedCell(cell) => {
+                        if !task.has_key(&CachedDataItemKey::CellDependency { target: cell }) {
+                            task.remove(&CachedDataItemKey::OutdatedCellDependency {
+                                target: cell,
+                            });
+                        }
+                    }
+                    Dep::OutdatedOutput(output) => {
+                        if !task.has_key(&CachedDataItemKey::OutputDependency { target: output }) {
+                            task.remove(&CachedDataItemKey::OutdatedOutputDependency {
+                                target: output,
+                            });
+                        }
+                    }
+                }
+            }
+
+            // TODO: Make all collectibles outdated
+
             start_event.notify(usize::MAX);
         }
+
         let (span, future) = if let Some(task_type) = self.task_cache.lookup_reverse(&task_id) {
             match &*task_type {
                 CachedTaskType::Native { fn_type, this, arg } => (
@@ -536,8 +626,27 @@ impl Backend for TurboTasksBackend {
                     done_event,
                 },
             });
+            drop(task);
+            drop(ctx);
         } else {
+            let old_edges = task
+                .iter()
+                .filter_map(|(key, _)| match *key {
+                    CachedDataItemKey::OutdatedChild { task } => Some(OutdatedEdge::Child(task)),
+                    CachedDataItemKey::OutdatedCellDependency { target } => {
+                        Some(OutdatedEdge::CellDependency(target))
+                    }
+                    CachedDataItemKey::OutdatedOutputDependency { target } => {
+                        Some(OutdatedEdge::OutputDependency(target))
+                    }
+                    _ => None,
+                })
+                .collect::<Vec<_>>();
+
             done_event.notify(usize::MAX);
+            drop(task);
+
+            CleanupOldEdgesOperation::run(task_id, old_edges, ctx);
         }
 
         stale

diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs
@@ -0,0 +1,96 @@
+use serde::{Deserialize, Serialize};
+use turbo_tasks::TaskId;
+
+use super::{ExecuteContext, Operation};
+use crate::data::{CachedDataItemKey, CellRef};
+
+#[derive(Serialize, Deserialize, Clone, Default)]
+pub enum CleanupOldEdgesOperation {
+    RemoveEdges {
+        task_id: TaskId,
+        outdated: Vec<OutdatedEdge>,
+    },
+    #[default]
+    Done,
+    // TODO Add aggregated edge
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+pub enum OutdatedEdge {
+    Child(TaskId),
+    CellDependency(CellRef),
+    OutputDependency(TaskId),
+}
+
+impl CleanupOldEdgesOperation {
+    pub fn run(task_id: TaskId, outdated: Vec<OutdatedEdge>, ctx: ExecuteContext<'_>) {
+        CleanupOldEdgesOperation::RemoveEdges { task_id, outdated }.execute(&ctx);
+    }
+}
+
+impl Operation for CleanupOldEdgesOperation {
+    fn execute(mut self, ctx: &ExecuteContext<'_>) {
+        loop {
+            ctx.operation_suspend_point(&self);
+            match self {
+                CleanupOldEdgesOperation::RemoveEdges {
+                    task_id,
+                    ref mut outdated,
+                } => {
+                    if let Some(edge) = outdated.pop() {
+                        match edge {
+                            OutdatedEdge::Child(child_id) => {
+                                let mut task = ctx.task(task_id);
+                                task.remove(&CachedDataItemKey::Child { task: child_id });
+                                // TODO remove aggregated edge
+                            }
+                            OutdatedEdge::CellDependency(CellRef {
+                                task: cell_task_id,
+                                cell,
+                            }) => {
+                                {
+                                    let mut task = ctx.task(cell_task_id);
+                                    task.remove(&CachedDataItemKey::CellDependent {
+                                        cell,
+                                        task: task_id,
+                                    });
+                                }
+                                {
+                                    let mut task = ctx.task(task_id);
+                                    task.remove(&CachedDataItemKey::CellDependency {
+                                        target: CellRef {
+                                            task: cell_task_id,
+                                            cell,
+                                        },
+                                    });
+                                }
+                            }
+                            OutdatedEdge::OutputDependency(output_task_id) => {
+                                {
+                                    let mut task = ctx.task(output_task_id);
+                                    task.remove(&CachedDataItemKey::OutputDependent {
+                                        task: task_id,
+                                    });
+                                }
+                                {
+                                    let mut task = ctx.task(task_id);
+                                    task.remove(&CachedDataItemKey::OutputDependency {
+                                        target: output_task_id,
+                                    });
+                                }
+                            }
+                        }
+                    }
+
+                    if outdated.is_empty() {
+                        self = CleanupOldEdgesOperation::Done;
+                    }
+                    continue;
+                }
+                CleanupOldEdgesOperation::Done => {
+                    return;
+                }
+            }
+        }
+    }
+}
diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs
@@ -1,3 +1,4 @@
+mod cleanup_old_edges;
 mod connect_child;
 mod invalidate;
 mod update_cell;
@@ -234,11 +235,14 @@ macro_rules! impl_operation {
 pub enum AnyOperation {
     ConnectChild(connect_child::ConnectChildOperation),
     Invalidate(invalidate::InvalidateOperation),
+    CleanupOldEdges(cleanup_old_edges::CleanupOldEdgesOperation),
     Nested(Vec<AnyOperation>),
 }
 
 impl_operation!(ConnectChild connect_child::ConnectChildOperation);
 impl_operation!(Invalidate invalidate::InvalidateOperation);
+impl_operation!(CleanupOldEdges cleanup_old_edges::CleanupOldEdgesOperation);
 
+pub use cleanup_old_edges::OutdatedEdge;
 pub use update_cell::UpdateCellOperation;
 pub use update_output::UpdateOutputOperation;
diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs
@@ -150,6 +150,10 @@ pub enum CachedDataItem {
         target: CellRef,
         value: (),
     },
+    OutdatedChild {
+        task: TaskId,
+        value: (),
+    },
 
     // Transient Error State
     Error {
@@ -183,6 +187,7 @@ impl CachedDataItem {
             CachedDataItem::OutdatedCollectible { .. } => false,
             CachedDataItem::OutdatedOutputDependency { .. } => false,
             CachedDataItem::OutdatedCellDependency { .. } => false,
+            CachedDataItem::OutdatedChild { .. } => false,
             CachedDataItem::Error { .. } => false,
         }
     }
@@ -224,6 +229,7 @@ impl CachedDataItemKey {
             CachedDataItemKey::OutdatedCollectible { .. } => false,
             CachedDataItemKey::OutdatedOutputDependency { .. } => false,
             CachedDataItemKey::OutdatedCellDependency { .. } => false,
+            CachedDataItemKey::OutdatedChild { .. } => false,
             CachedDataItemKey::Error { .. } => false,
         }
     }