From fdf5e86ebd4f6bcc6ad72bade21f21867942270f Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 28 Sep 2023 09:17:36 +0200 Subject: [PATCH] refactor TaskScopes to use an aggregation tree (vercel/turbo#5992) ### Description Remove TaskScope and add a similar tree structure instead. It aggregates Tasks in the task graph (parent-children) to aggregated nodes which summarize important information from all contained tasks. It's a tree-structure and any subgraph can be aggregated to a single aggregated node to query these information in an efficient way. We aggregate the following information from tasks: * is there any unfinished task * and an event when this reaches zero (so one can wait for it) * collectibles emitted * a list of dirty tasks Receiving this information doesn't require to walk the tree since it's eagerly aggregated when it changes on Tasks. Since it's a tree structure updating such information on a Task has to walk the tree upwards which is `O(R + log(R) * log(N))` where N is the number of Tasks in the graph and R the number of roots of the graph. It's also possible to query the aggregation from any Task to get information about the roots. This has to walk the tree to every root which should be `O(R + log(R) * log(N))`, but it's possible to shortcut when any root already contains the information needed. We use that to gather the following information about roots: * is this root active The tree is only connected from bottom to top. It's not possible to walk the tree from top to bottom. The tree is build from two parts, the top tree and the bottom tree. A height 0 bottom tree will aggregate Tasks up to a configured connectivity. The height 1 bottom tree will aggregate height 0 bottom tree up to a configured connectivity. This continues recursively. Since one doesn't know which height of bottom tree is needed to aggregate a certain subgraph, a top tree is used. A depth 0 top tree aggregates Tasks of infinite connectivity, by using a bottom tree of height X and optionally a depth 1 top tree. This continues with depth 1 top tree using a height X + 1 bottom tree and a depth 2 top tree. The connectivity and X are subject of fine tuning. In general a Task can be in multiple bottom trees as inner node, but to ensure tree reuse there is a limitation to that. Once a bottom tree starts at a certain Task, it cannot be an inner node of other bottom tree. So a Task is either non-root inner node in one or more bottom tree or root node of exactly 1 bottom tree. When a task is inner node in multiple bottom tree, the cost of the children of the task will multiply with the number of bottom trees. This can create a performance hit. To avoid that there is a threshold (subject of fine tuning) which converts the Task into a root of a new bottom tree when the multiple reaches the threshold. The same limitations apply on higher level bottom trees. Closes WEB-1621 --- crates/node-file-trace/src/lib.rs | 7 +- crates/turbo-tasks-auto-hash-map/src/lib.rs | 2 + crates/turbo-tasks-auto-hash-map/src/map.rs | 108 + crates/turbo-tasks-auto-hash-map/src/set.rs | 6 + crates/turbo-tasks-memory/Cargo.toml | 3 + .../benches/scope_stress.rs | 2 +- .../src/aggregation_tree/bottom_connection.rs | 306 +++ .../src/aggregation_tree/bottom_tree.rs | 722 ++++++ .../src/aggregation_tree/inner_refs.rs | 79 + .../src/aggregation_tree/leaf.rs | 405 ++++ .../src/aggregation_tree/mod.rs | 146 ++ .../src/aggregation_tree/tests.rs | 592 +++++ .../src/aggregation_tree/top_tree.rs | 180 ++ .../turbo-tasks-memory/src/count_hash_set.rs | 132 +- crates/turbo-tasks-memory/src/gc.rs | 2 - crates/turbo-tasks-memory/src/lib.rs | 4 +- .../turbo-tasks-memory/src/memory_backend.rs | 337 +-- .../src/memory_backend_with_pg.rs | 11 +- .../turbo-tasks-memory/src/priority_pair.rs | 41 - crates/turbo-tasks-memory/src/scope.rs | 775 ------- crates/turbo-tasks-memory/src/stats.rs | 24 +- crates/turbo-tasks-memory/src/task.rs | 2021 +++++------------ .../src/task/aggregation.rs | 527 +++++ .../turbo-tasks-memory/src/task/meta_state.rs | 41 +- crates/turbo-tasks-memory/src/viz/graph.rs | 15 - crates/turbo-tasks-memory/src/viz/mod.rs | 19 - crates/turbo-tasks-memory/src/viz/table.rs | 28 - .../turbo-tasks-memory/tests/collectibles.rs | 62 +- .../turbo-tasks-memory/tests/scope_stress.rs | 51 + crates/turbo-tasks-testing/src/lib.rs | 15 +- crates/turbo-tasks/src/backend.rs | 9 +- crates/turbo-tasks/src/collectibles.rs | 8 +- crates/turbo-tasks/src/lib.rs | 2 +- crates/turbo-tasks/src/manager.rs | 36 +- crates/turbo-tasks/src/raw_vc.rs | 76 +- crates/turbo-tasks/src/vc/mod.rs | 7 +- crates/turbopack-cli-utils/src/issue.rs | 2 +- .../turbopack-cli/src/dev/turbo_tasks_viz.rs | 5 +- crates/turbopack-core/src/diagnostics/mod.rs | 10 +- crates/turbopack-core/src/issue/mod.rs | 41 +- crates/turbopack-dev-server/src/http.rs | 8 +- .../src/introspect/mod.rs | 4 +- crates/turbopack-dev-server/src/lib.rs | 2 +- .../src/source/asset_graph.rs | 162 +- .../src/source/resolve.rs | 2 +- .../src/source/route_tree.rs | 6 +- .../turbopack-dev-server/src/update/stream.rs | 3 +- crates/turbopack-tests/tests/execution.rs | 7 +- crates/turbopack-tests/tests/snapshot.rs | 7 +- 49 files changed, 4087 insertions(+), 2973 deletions(-) create mode 100644 crates/turbo-tasks-memory/src/aggregation_tree/bottom_connection.rs create mode 100644 crates/turbo-tasks-memory/src/aggregation_tree/bottom_tree.rs create mode 100644 crates/turbo-tasks-memory/src/aggregation_tree/inner_refs.rs create mode 100644 crates/turbo-tasks-memory/src/aggregation_tree/leaf.rs create mode 100644 crates/turbo-tasks-memory/src/aggregation_tree/mod.rs create mode 100644 crates/turbo-tasks-memory/src/aggregation_tree/tests.rs create mode 100644 crates/turbo-tasks-memory/src/aggregation_tree/top_tree.rs delete mode 100644 crates/turbo-tasks-memory/src/priority_pair.rs delete mode 100644 crates/turbo-tasks-memory/src/scope.rs create mode 100644 crates/turbo-tasks-memory/src/task/aggregation.rs create mode 100644 crates/turbo-tasks-memory/tests/scope_stress.rs diff --git a/crates/node-file-trace/src/lib.rs b/crates/node-file-trace/src/lib.rs index 5a5d2c53d8d0b..15bb17f4bfe24 100644 --- a/crates/node-file-trace/src/lib.rs +++ b/crates/node-file-trace/src/lib.rs @@ -495,13 +495,10 @@ async fn run>( module_options, resolve_options, ); + let _ = output.resolve_strongly_consistent().await?; let source = TransientValue::new(Vc::into_raw(output)); - let issues = output - .peek_issues_with_path() - .await? - .strongly_consistent() - .await?; + let issues = output.peek_issues_with_path().await?; let console_ui = ConsoleUi::new(log_options); Vc::upcast::>(console_ui) diff --git a/crates/turbo-tasks-auto-hash-map/src/lib.rs b/crates/turbo-tasks-auto-hash-map/src/lib.rs index 62642d4ee96ff..811fd845c99ae 100644 --- a/crates/turbo-tasks-auto-hash-map/src/lib.rs +++ b/crates/turbo-tasks-auto-hash-map/src/lib.rs @@ -1,3 +1,5 @@ +#![feature(hash_raw_entry)] + pub mod map; pub mod set; diff --git a/crates/turbo-tasks-auto-hash-map/src/map.rs b/crates/turbo-tasks-auto-hash-map/src/map.rs index 167c26a00f535..2e92cf23dbcdd 100644 --- a/crates/turbo-tasks-auto-hash-map/src/map.rs +++ b/crates/turbo-tasks-auto-hash-map/src/map.rs @@ -190,6 +190,28 @@ impl AutoMap { } } + pub fn raw_entry_mut(&mut self, key: &Q) -> RawEntry<'_, K, V, H> + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + let this = self as *mut Self; + match self { + AutoMap::List(list) => match list.iter().position(|(k, _)| k.borrow() == key) { + Some(index) => RawEntry::Occupied(OccupiedRawEntry::List { list, index }), + None => RawEntry::Vacant(VacantRawEntry::List { this, list }), + }, + AutoMap::Map(map) => match map.raw_entry_mut().from_key(key) { + std::collections::hash_map::RawEntryMut::Occupied(entry) => { + RawEntry::Occupied(OccupiedRawEntry::Map { this, entry }) + } + std::collections::hash_map::RawEntryMut::Vacant(entry) => { + RawEntry::Vacant(VacantRawEntry::Map(entry)) + } + }, + } + } + /// see [HashMap::shrink_to_fit](https://doc.rust-lang.org/std/collections/struct.HashMap.html#method.shrink_to_fit) pub fn shrink_to_fit(&mut self) { match self { @@ -337,6 +359,15 @@ impl<'a, K, V> Iterator for Iter<'a, K, V> { } } +impl<'a, K, V> Clone for Iter<'a, K, V> { + fn clone(&self) -> Self { + match self { + Iter::List(iter) => Iter::List(iter.clone()), + Iter::Map(iter) => Iter::Map(iter.clone()), + } + } +} + pub enum IterMut<'a, K, V> { List(std::slice::IterMut<'a, (K, V)>), Map(std::collections::hash_map::IterMut<'a, K, V>), @@ -515,6 +546,83 @@ impl<'a, K: Eq + Hash, V, H: BuildHasher + Default + 'a> VacantEntry<'a, K, V, H } } +pub enum RawEntry<'a, K, V, H> { + Occupied(OccupiedRawEntry<'a, K, V, H>), + Vacant(VacantRawEntry<'a, K, V, H>), +} + +pub enum OccupiedRawEntry<'a, K, V, H> { + List { + list: &'a mut Vec<(K, V)>, + index: usize, + }, + Map { + this: *mut AutoMap, + entry: std::collections::hash_map::RawOccupiedEntryMut<'a, K, V, H>, + }, +} + +impl<'a, K: Eq + Hash, V, H: BuildHasher> OccupiedRawEntry<'a, K, V, H> { + /// see [HashMap::RawOccupiedEntryMut::get_mut](https://doc.rust-lang.org/std/collections/hash_map/struct.RawOccupiedEntryMut.html#method.get_mut) + pub fn get_mut(&mut self) -> &mut V { + match self { + OccupiedRawEntry::List { list, index } => &mut list[*index].1, + OccupiedRawEntry::Map { entry, .. } => entry.get_mut(), + } + } + + /// see [HashMap::RawOccupiedEntryMut::into_mut](https://doc.rust-lang.org/std/collections/hash_map/struct.RawOccupiedEntryMut.html#method.into_mut) + pub fn into_mut(self) -> &'a mut V { + match self { + OccupiedRawEntry::List { list, index } => &mut list[index].1, + OccupiedRawEntry::Map { entry, .. } => entry.into_mut(), + } + } +} + +impl<'a, K: Eq + Hash, V, H: BuildHasher + Default> OccupiedRawEntry<'a, K, V, H> { + /// see [HashMap::OccupiedEntry::remove](https://doc.rust-lang.org/std/collections/hash_map/enum.OccupiedEntry.html#method.remove) + pub fn remove(self) -> V { + match self { + OccupiedRawEntry::List { list, index } => list.swap_remove(index).1, + OccupiedRawEntry::Map { entry, this } => { + let v = entry.remove(); + let this = unsafe { &mut *this }; + if this.len() < MIN_HASH_SIZE { + this.convert_to_list(); + } + v + } + } + } +} + +pub enum VacantRawEntry<'a, K, V, H> { + List { + this: *mut AutoMap, + list: &'a mut Vec<(K, V)>, + }, + Map(std::collections::hash_map::RawVacantEntryMut<'a, K, V, H>), +} + +impl<'a, K: Eq + Hash, V, H: BuildHasher + Default + 'a> VacantRawEntry<'a, K, V, H> { + /// see [HashMap::RawVacantEntryMut::insert](https://doc.rust-lang.org/std/collections/hash_map/struct.RawVacantEntryMut.html#method.insert) + pub fn insert(self, key: K, value: V) -> &'a mut V { + match self { + VacantRawEntry::List { this, list } => { + if list.len() >= MAX_LIST_SIZE { + let this = unsafe { &mut *this }; + this.convert_to_map().entry(key).or_insert(value) + } else { + list.push((key, value)); + &mut list.last_mut().unwrap().1 + } + } + VacantRawEntry::Map(entry) => entry.insert(key, value).1, + } + } +} + impl Serialize for AutoMap where K: Eq + Hash + Serialize, diff --git a/crates/turbo-tasks-auto-hash-map/src/set.rs b/crates/turbo-tasks-auto-hash-map/src/set.rs index b6476ae9c45da..e125fce9e762a 100644 --- a/crates/turbo-tasks-auto-hash-map/src/set.rs +++ b/crates/turbo-tasks-auto-hash-map/src/set.rs @@ -137,6 +137,12 @@ impl<'a, K> Iterator for Iter<'a, K> { } } +impl<'a, K> Clone for Iter<'a, K> { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + pub struct IntoIter(super::map::IntoIter); impl Iterator for IntoIter { diff --git a/crates/turbo-tasks-memory/Cargo.toml b/crates/turbo-tasks-memory/Cargo.toml index fe25a7cc9c36e..b4b8f5cb07f91 100644 --- a/crates/turbo-tasks-memory/Cargo.toml +++ b/crates/turbo-tasks-memory/Cargo.toml @@ -19,6 +19,7 @@ num_cpus = "1.13.1" once_cell = { workspace = true } parking_lot = { workspace = true } priority-queue = "1.3.0" +ref-cast = "1.0.20" rustc-hash = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } @@ -38,6 +39,7 @@ turbo-tasks-testing = { workspace = true } turbo-tasks-build = { workspace = true } [features] +track_unfinished = [] unsafe_once_map = [] log_running_tasks = [] log_scheduled_tasks = [] @@ -48,6 +50,7 @@ print_scope_updates = [] print_task_invalidation = [] inline_add_to_scope = [] inline_remove_from_scope = [] +default = [] [[bench]] name = "mod" diff --git a/crates/turbo-tasks-memory/benches/scope_stress.rs b/crates/turbo-tasks-memory/benches/scope_stress.rs index 46ae4fc970dac..3c8d47338bff4 100644 --- a/crates/turbo-tasks-memory/benches/scope_stress.rs +++ b/crates/turbo-tasks-memory/benches/scope_stress.rs @@ -18,7 +18,7 @@ pub fn scope_stress(c: &mut Criterion) { let mut group = c.benchmark_group("turbo_tasks_memory_scope_stress"); group.sample_size(20); - for size in [10, 100, 200, 300] { + for size in [5, 10, 15, 20, 25, 30, 100, 200, 300] { group.throughput(criterion::Throughput::Elements( /* tasks for fib from 0 to size - 1 = */ (size as u64) * (size as u64) + diff --git a/crates/turbo-tasks-memory/src/aggregation_tree/bottom_connection.rs b/crates/turbo-tasks-memory/src/aggregation_tree/bottom_connection.rs new file mode 100644 index 0000000000000..5fb61e0047299 --- /dev/null +++ b/crates/turbo-tasks-memory/src/aggregation_tree/bottom_connection.rs @@ -0,0 +1,306 @@ +use std::{hash::Hash, ops::ControlFlow, sync::Arc}; + +use auto_hash_map::{map::RawEntry, AutoMap}; +use nohash_hasher::{BuildNoHashHasher, IsEnabled}; + +use super::{ + bottom_tree::BottomTree, + inner_refs::{BottomRef, ChildLocation}, + AggregationContext, +}; + +struct BottomRefInfo { + count: isize, + distance: u8, +} + +/// A map that stores references to bottom trees which a specific distance. It +/// stores the minimum distance added to the map. +/// +/// This is used to store uppers of leafs or smaller bottom trees with the +/// current distance. The distance is imporant to keep the correct connectivity. +#[derive(Default)] +pub struct DistanceCountMap { + map: AutoMap>, +} + +impl DistanceCountMap { + pub fn new() -> Self { + Self { + map: AutoMap::with_hasher(), + } + } + + pub fn is_unset(&self) -> bool { + self.map.is_empty() + } + + pub fn iter(&self) -> impl Iterator { + self.map + .iter() + .filter(|(_, info)| info.count > 0) + .map(|(item, &BottomRefInfo { distance, .. })| (item, distance)) + } + + pub fn add_clonable(&mut self, item: &T, distance: u8) -> bool { + match self.map.raw_entry_mut(item) { + RawEntry::Occupied(e) => { + let info = e.into_mut(); + info.count += 1; + if distance < info.distance { + info.distance = distance; + } + false + } + RawEntry::Vacant(e) => { + e.insert(item.clone(), BottomRefInfo { count: 1, distance }); + true + } + } + } + + pub fn remove_clonable(&mut self, item: &T) -> bool { + match self.map.raw_entry_mut(item) { + RawEntry::Occupied(mut e) => { + let info = e.get_mut(); + info.count -= 1; + if info.count == 0 { + e.remove(); + true + } else { + false + } + } + RawEntry::Vacant(e) => { + e.insert( + item.clone(), + BottomRefInfo { + count: -1, + distance: 0, + }, + ); + false + } + } + } + + pub fn into_counts(self) -> impl Iterator { + self.map.into_iter().map(|(item, info)| (item, info.count)) + } + + pub fn len(&self) -> usize { + self.map.len() + } +} + +/// Connection to upper bottom trees. It has two modes: A single bottom tree, +/// where the current left/smaller bottom tree is the left-most child. Or +/// multiple bottom trees, where the current left/smaller bottom tree is an +/// inner child (not left-most). +pub enum BottomConnection { + Left(Arc>), + Inner(DistanceCountMap>), +} + +impl BottomConnection { + pub fn new() -> Self { + Self::Inner(DistanceCountMap::new()) + } + + pub fn is_unset(&self) -> bool { + match self { + Self::Left(_) => false, + Self::Inner(list) => list.is_unset(), + } + } + + pub fn as_cloned_uppers(&self) -> BottomUppers { + match self { + Self::Left(upper) => BottomUppers::Left(upper.clone()), + Self::Inner(upper) => BottomUppers::Inner( + upper + .iter() + .map(|(item, distance)| (item.clone(), distance)) + .collect(), + ), + } + } + + #[must_use] + pub fn set_left_upper( + &mut self, + upper: &Arc>, + ) -> DistanceCountMap> { + match std::mem::replace(self, BottomConnection::Left(upper.clone())) { + BottomConnection::Left(_) => unreachable!("Can't have two left children"), + BottomConnection::Inner(old_inner) => old_inner, + } + } + + pub fn unset_left_upper(&mut self, upper: &Arc>) { + match std::mem::replace(self, BottomConnection::Inner(DistanceCountMap::new())) { + BottomConnection::Left(old_upper) => { + debug_assert!(Arc::ptr_eq(&old_upper, upper)); + } + BottomConnection::Inner(_) => unreachable!("Must that a left child"), + } + } +} + +impl BottomConnection { + pub fn child_change>( + &self, + aggregation_context: &C, + change: &C::ItemChange, + ) { + match self { + BottomConnection::Left(upper) => { + upper.child_change(aggregation_context, change); + } + BottomConnection::Inner(list) => { + for (BottomRef { upper }, _) in list.iter() { + upper.child_change(aggregation_context, change); + } + } + } + } + + pub fn get_root_info>( + &self, + aggregation_context: &C, + root_info_type: &C::RootInfoType, + mut result: C::RootInfo, + ) -> C::RootInfo { + match &self { + BottomConnection::Left(upper) => { + let info = upper.get_root_info(aggregation_context, root_info_type); + if aggregation_context.merge_root_info(&mut result, info) == ControlFlow::Break(()) + { + return result; + } + } + BottomConnection::Inner(list) => { + for (BottomRef { upper }, _) in list.iter() { + let info = upper.get_root_info(aggregation_context, root_info_type); + if aggregation_context.merge_root_info(&mut result, info) + == ControlFlow::Break(()) + { + return result; + } + } + } + } + result + } +} + +pub enum BottomUppers { + Left(Arc>), + Inner(Vec<(BottomRef, u8)>), +} + +impl BottomUppers { + pub fn add_children_of_child<'a, C: AggregationContext>( + &self, + aggregation_context: &C, + children: impl IntoIterator + Clone, + ) where + I: 'a, + { + match self { + BottomUppers::Left(upper) => { + upper.add_children_of_child(aggregation_context, ChildLocation::Left, children, 0); + } + BottomUppers::Inner(list) => { + for &(BottomRef { ref upper }, nesting_level) in list { + upper.add_children_of_child( + aggregation_context, + ChildLocation::Inner, + children.clone(), + nesting_level + 1, + ); + } + } + } + } + + pub fn add_child_of_child>( + &self, + aggregation_context: &C, + child_of_child: &I, + ) { + match self { + BottomUppers::Left(upper) => { + upper.add_child_of_child( + aggregation_context, + ChildLocation::Left, + child_of_child, + 0, + ); + } + BottomUppers::Inner(list) => { + for &(BottomRef { ref upper }, nesting_level) in list.iter() { + upper.add_child_of_child( + aggregation_context, + ChildLocation::Inner, + child_of_child, + nesting_level + 1, + ); + } + } + } + } + + pub fn remove_child_of_child>( + &self, + aggregation_context: &C, + child_of_child: &I, + ) { + match self { + BottomUppers::Left(upper) => { + upper.remove_child_of_child(aggregation_context, child_of_child); + } + BottomUppers::Inner(list) => { + for (BottomRef { upper }, _) in list { + upper.remove_child_of_child(aggregation_context, child_of_child); + } + } + } + } + + pub fn remove_children_of_child<'a, C: AggregationContext>( + &self, + aggregation_context: &C, + children: impl IntoIterator + Clone, + ) where + I: 'a, + { + match self { + BottomUppers::Left(upper) => { + upper.remove_children_of_child(aggregation_context, children); + } + BottomUppers::Inner(list) => { + for (BottomRef { upper }, _) in list { + upper.remove_children_of_child(aggregation_context, children.clone()); + } + } + } + } + + pub fn child_change>( + &self, + aggregation_context: &C, + change: &C::ItemChange, + ) { + match self { + BottomUppers::Left(upper) => { + upper.child_change(aggregation_context, change); + } + BottomUppers::Inner(list) => { + for (BottomRef { upper }, _) in list { + upper.child_change(aggregation_context, change); + } + } + } + } +} diff --git a/crates/turbo-tasks-memory/src/aggregation_tree/bottom_tree.rs b/crates/turbo-tasks-memory/src/aggregation_tree/bottom_tree.rs new file mode 100644 index 0000000000000..36dcba8900203 --- /dev/null +++ b/crates/turbo-tasks-memory/src/aggregation_tree/bottom_tree.rs @@ -0,0 +1,722 @@ +use std::{hash::Hash, ops::ControlFlow, sync::Arc}; + +use nohash_hasher::{BuildNoHashHasher, IsEnabled}; +use parking_lot::{Mutex, MutexGuard}; +use ref_cast::RefCast; + +use super::{ + bottom_connection::BottomConnection, + inner_refs::{BottomRef, ChildLocation, TopRef}, + leaf::{ + add_inner_upper_to_item, bottom_tree, remove_inner_upper_from_item, + remove_left_upper_from_item, + }, + top_tree::TopTree, + AggregationContext, CHILDREN_INNER_THRESHOLD, CONNECTIVITY_LIMIT, +}; +use crate::count_hash_set::{CountHashSet, RemoveIfEntryResult}; + +/// The bottom half of the aggregation tree. It aggregates items up the a +/// certain connectivity depending on the "height". Every level of the tree +/// aggregates the previous level. +pub struct BottomTree { + height: u8, + item: I, + state: Mutex>, +} + +pub struct BottomTreeState { + data: T, + bottom_upper: BottomConnection, + top_upper: CountHashSet, BuildNoHashHasher>>, + // TODO can this become negative? + following: CountHashSet>, +} + +impl BottomTree { + pub fn new(item: I, height: u8) -> Self { + Self { + height, + item, + state: Mutex::new(BottomTreeState { + data: T::default(), + bottom_upper: BottomConnection::new(), + top_upper: CountHashSet::new(), + following: CountHashSet::new(), + }), + } + } +} + +impl BottomTree { + pub fn add_children_of_child<'a, C: AggregationContext>( + self: &Arc, + aggregation_context: &C, + child_location: ChildLocation, + children: impl IntoIterator, + nesting_level: u8, + ) where + I: 'a, + { + match child_location { + ChildLocation::Left => { + // the left child has new children + // this means it's a inner child of this node + // We always want to aggregate over at least connectivity 1 + self.add_children_of_child_inner(aggregation_context, children, nesting_level); + } + ChildLocation::Inner => { + // the inner child has new children + // this means white children are inner children of this node + // and blue children need to propagate up + let mut children = children.into_iter().collect(); + if nesting_level > CONNECTIVITY_LIMIT { + self.add_children_of_child_following(aggregation_context, children); + return; + } + + self.add_children_of_child_if_following(&mut children); + self.add_children_of_child_inner(aggregation_context, children, nesting_level); + } + } + } + + fn add_children_of_child_if_following(&self, children: &mut Vec<&I>) { + let mut state = self.state.lock(); + children.retain(|&child| !state.following.add_if_entry(child)); + } + + fn add_children_of_child_following>( + self: &Arc, + aggregation_context: &C, + mut children: Vec<&I>, + ) { + let mut state = self.state.lock(); + children.retain(|&child| state.following.add_clonable(child)); + if children.is_empty() { + return; + } + let buttom_uppers = state.bottom_upper.as_cloned_uppers(); + let top_upper = state.top_upper.iter().cloned().collect::>(); + drop(state); + for TopRef { upper } in top_upper { + upper.add_children_of_child(aggregation_context, children.iter().copied()); + } + buttom_uppers.add_children_of_child(aggregation_context, children.iter().copied()); + } + + fn add_children_of_child_inner<'a, C: AggregationContext>( + self: &Arc, + aggregation_context: &C, + children: impl IntoIterator, + nesting_level: u8, + ) where + I: 'a, + { + let mut following = Vec::new(); + if self.height == 0 { + for child in children { + let can_be_inner = + add_inner_upper_to_item(aggregation_context, child, self, nesting_level); + if !can_be_inner { + following.push(child); + } + } + } else { + for child in children { + let can_be_inner = bottom_tree(aggregation_context, child, self.height - 1) + .add_inner_bottom_tree_upper(aggregation_context, self, nesting_level); + if !can_be_inner { + following.push(child); + } + } + } + if !following.is_empty() { + self.add_children_of_child_following(aggregation_context, following); + } + } + + pub fn add_child_of_child>( + self: &Arc, + aggregation_context: &C, + child_location: ChildLocation, + child_of_child: &I, + nesting_level: u8, + ) { + debug_assert!(child_of_child != &self.item); + match child_location { + ChildLocation::Left => { + // the left child has a new child + // this means it's a inner child of this node + // We always want to aggregate over at least connectivity 1 + self.add_child_of_child_inner(aggregation_context, child_of_child, nesting_level); + } + ChildLocation::Inner => { + if nesting_level <= CONNECTIVITY_LIMIT { + // the inner child has a new child + // but it's not a blue node and we are not too deep + // this means it's a inner child of this node + // if it's not already a following child + if !self.add_child_of_child_if_following(child_of_child) { + self.add_child_of_child_inner( + aggregation_context, + child_of_child, + nesting_level, + ); + } + } else { + // the inner child has a new child + // this means we need to propagate the change up + // and store them in our own list + self.add_child_of_child_following(aggregation_context, child_of_child); + } + } + } + } + + fn add_child_of_child_if_following(&self, child_of_child: &I) -> bool { + let mut state = self.state.lock(); + state.following.add_if_entry(child_of_child) + } + + fn add_child_of_child_following>( + self: &Arc, + aggregation_context: &C, + child_of_child: &I, + ) { + let mut state = self.state.lock(); + if !state.following.add_clonable(child_of_child) { + // Already connect, nothing more to do + return; + } + + propagate_new_following_to_uppers(state, aggregation_context, child_of_child); + } + + fn add_child_of_child_inner>( + self: &Arc, + aggregation_context: &C, + child_of_child: &I, + nesting_level: u8, + ) { + let can_be_inner = if self.height == 0 { + add_inner_upper_to_item(aggregation_context, child_of_child, self, nesting_level) + } else { + bottom_tree(aggregation_context, child_of_child, self.height - 1) + .add_inner_bottom_tree_upper(aggregation_context, self, nesting_level) + }; + if !can_be_inner { + self.add_child_of_child_following(aggregation_context, child_of_child); + } + } + + pub fn remove_child_of_child>( + self: &Arc, + aggregation_context: &C, + child_of_child: &I, + ) { + if !self.remove_child_of_child_if_following(aggregation_context, child_of_child) { + self.remove_child_of_child_inner(aggregation_context, child_of_child); + } + } + + pub fn remove_children_of_child<'a, C: AggregationContext>( + self: &Arc, + aggregation_context: &C, + children: impl IntoIterator, + ) where + I: 'a, + { + let mut children = children.into_iter().collect(); + self.remove_children_of_child_if_following(aggregation_context, &mut children); + self.remove_children_of_child_inner(aggregation_context, children); + } + + fn remove_child_of_child_if_following>( + self: &Arc, + aggregation_context: &C, + child_of_child: &I, + ) -> bool { + let mut state = self.state.lock(); + match state.following.remove_if_entry(child_of_child) { + RemoveIfEntryResult::PartiallyRemoved => return true, + RemoveIfEntryResult::NotPresent => return false, + RemoveIfEntryResult::Removed => {} + } + propagate_lost_following_to_uppers(state, aggregation_context, child_of_child); + true + } + + fn remove_children_of_child_if_following<'a, C: AggregationContext>( + self: &Arc, + aggregation_context: &C, + children: &mut Vec<&'a I>, + ) { + let mut state = self.state.lock(); + let mut removed = Vec::new(); + children.retain(|&child| match state.following.remove_if_entry(child) { + RemoveIfEntryResult::PartiallyRemoved => false, + RemoveIfEntryResult::NotPresent => true, + RemoveIfEntryResult::Removed => { + removed.push(child); + false + } + }); + if !removed.is_empty() { + propagate_lost_followings_to_uppers(state, aggregation_context, removed); + } + } + + fn remove_child_of_child_following>( + self: &Arc, + aggregation_context: &C, + child_of_child: &I, + ) -> bool { + let mut state = self.state.lock(); + if !state.following.remove_clonable(child_of_child) { + // no present, nothing to do + return false; + } + propagate_lost_following_to_uppers(state, aggregation_context, child_of_child); + true + } + + fn remove_children_of_child_following>( + self: &Arc, + aggregation_context: &C, + mut children: Vec<&I>, + ) { + let mut state = self.state.lock(); + children.retain(|&child| state.following.remove_clonable(child)); + propagate_lost_followings_to_uppers(state, aggregation_context, children); + } + + fn remove_child_of_child_inner>( + self: &Arc, + aggregation_context: &C, + child_of_child: &I, + ) { + let can_remove_inner = if self.height == 0 { + remove_inner_upper_from_item(aggregation_context, child_of_child, self) + } else { + bottom_tree(aggregation_context, child_of_child, self.height - 1) + .remove_inner_bottom_tree_upper(aggregation_context, self) + }; + if !can_remove_inner { + self.remove_child_of_child_following(aggregation_context, child_of_child); + } + } + + fn remove_children_of_child_inner<'a, C: AggregationContext>( + self: &Arc, + aggregation_context: &C, + children: impl IntoIterator, + ) where + I: 'a, + { + let unremoveable = if self.height == 0 { + children + .into_iter() + .filter(|&child| !remove_inner_upper_from_item(aggregation_context, child, self)) + .collect::>() + } else { + children + .into_iter() + .filter(|&child| { + !bottom_tree(aggregation_context, child, self.height - 1) + .remove_inner_bottom_tree_upper(aggregation_context, self) + }) + .collect::>() + }; + if !unremoveable.is_empty() { + self.remove_children_of_child_following(aggregation_context, unremoveable); + } + } + + pub fn add_left_bottom_tree_upper>( + &self, + aggregation_context: &C, + upper: &Arc>, + ) { + let mut state = self.state.lock(); + let old_inner = state.bottom_upper.set_left_upper(upper); + let add_change = aggregation_context.info_to_add_change(&state.data); + let children = state.following.iter().cloned().collect::>(); + + let remove_change = (!old_inner.is_unset()) + .then(|| aggregation_context.info_to_remove_change(&state.data)) + .flatten(); + + drop(state); + if let Some(change) = add_change { + upper.child_change(aggregation_context, &change); + } + if !children.is_empty() { + upper.add_children_of_child( + aggregation_context, + ChildLocation::Left, + children.iter(), + 1, + ); + } + + // Convert this node into a following node for all old (inner) uppers + // + // Old state: + // I1, I2 + // \ + // self + // Adding L as new left upper: + // I1, I2 L + // \ / + // self + // Final state: (I1 and I2 have L as following instead) + // I1, I2 ----> L + // / + // self + // I1 and I2 have "self" change removed since it's now part of L instead. + // L = upper, I1, I2 = old_inner + // + for (BottomRef { upper: old_upper }, count) in old_inner.into_counts() { + let item = &self.item; + old_upper.migrate_old_inner( + aggregation_context, + item, + count, + &remove_change, + &children, + ); + } + } + + pub fn migrate_old_inner>( + self: &Arc, + aggregation_context: &C, + item: &I, + count: isize, + remove_change: &Option, + following: &[I], + ) { + let mut state = self.state.lock(); + if count > 0 { + // add as following + if state.following.add_count(item.clone(), count as usize) { + propagate_new_following_to_uppers(state, aggregation_context, item); + } else { + drop(state); + } + // remove from self + if let Some(change) = remove_change.as_ref() { + self.child_change(aggregation_context, change); + } + self.remove_children_of_child(aggregation_context, following); + } else { + // remove count from following instead + if state.following.remove_count(item.clone(), -count as usize) { + propagate_lost_following_to_uppers(state, aggregation_context, item); + } + } + } + + #[must_use] + pub fn add_inner_bottom_tree_upper>( + &self, + aggregation_context: &C, + upper: &Arc>, + nesting_level: u8, + ) -> bool { + let mut state = self.state.lock(); + let number_of_following = state.following.len(); + let BottomConnection::Inner(inner) = &mut state.bottom_upper else { + return false; + }; + if inner.len() * number_of_following > CHILDREN_INNER_THRESHOLD { + return false; + }; + let new = inner.add_clonable(BottomRef::ref_cast(upper), nesting_level); + if new { + if let Some(change) = aggregation_context.info_to_add_change(&state.data) { + upper.child_change(aggregation_context, &change); + } + let children = state.following.iter().cloned().collect::>(); + drop(state); + if !children.is_empty() { + upper.add_children_of_child( + aggregation_context, + ChildLocation::Inner, + &children, + nesting_level + 1, + ); + } + } + true + } + + pub fn remove_left_bottom_tree_upper>( + self: &Arc, + aggregation_context: &C, + upper: &Arc>, + ) { + let mut state = self.state.lock(); + state.bottom_upper.unset_left_upper(upper); + if let Some(change) = aggregation_context.info_to_remove_change(&state.data) { + upper.child_change(aggregation_context, &change); + } + let following = state.following.iter().cloned().collect::>(); + if state.top_upper.is_empty() { + drop(state); + self.remove_self_from_lower(aggregation_context); + } else { + drop(state); + } + upper.remove_children_of_child(aggregation_context, &following); + } + + #[must_use] + pub fn remove_inner_bottom_tree_upper>( + &self, + aggregation_context: &C, + upper: &Arc>, + ) -> bool { + let mut state = self.state.lock(); + let BottomConnection::Inner(inner) = &mut state.bottom_upper else { + return false; + }; + let removed = inner.remove_clonable(BottomRef::ref_cast(upper)); + if removed { + let remove_change = aggregation_context.info_to_remove_change(&state.data); + let following = state.following.iter().cloned().collect::>(); + drop(state); + if let Some(change) = remove_change { + upper.child_change(aggregation_context, &change); + } + upper.remove_children_of_child(aggregation_context, &following); + } + true + } + + pub fn add_top_tree_upper>( + &self, + aggregation_context: &C, + upper: &Arc>, + ) { + let mut state = self.state.lock(); + let new = state.top_upper.add_clonable(TopRef::ref_cast(upper)); + if new { + if let Some(change) = aggregation_context.info_to_add_change(&state.data) { + upper.child_change(aggregation_context, &change); + } + for following in state.following.iter() { + upper.add_child_of_child(aggregation_context, following); + } + } + } + + #[allow(dead_code)] + pub fn remove_top_tree_upper>( + self: &Arc, + aggregation_context: &C, + upper: &Arc>, + ) { + let mut state = self.state.lock(); + let removed = state.top_upper.remove_clonable(TopRef::ref_cast(upper)); + if removed { + if let Some(change) = aggregation_context.info_to_remove_change(&state.data) { + upper.child_change(aggregation_context, &change); + } + for following in state.following.iter() { + upper.remove_child_of_child(aggregation_context, following); + } + if state.top_upper.is_empty() + && !matches!(state.bottom_upper, BottomConnection::Left(_)) + { + drop(state); + self.remove_self_from_lower(aggregation_context); + } + } + } + + fn remove_self_from_lower( + self: &Arc, + aggregation_context: &impl AggregationContext, + ) { + if self.height == 0 { + remove_left_upper_from_item(aggregation_context, &self.item, self); + } else { + bottom_tree(aggregation_context, &self.item, self.height - 1) + .remove_left_bottom_tree_upper(aggregation_context, self); + } + } + + pub fn child_change>( + &self, + aggregation_context: &C, + change: &C::ItemChange, + ) { + let mut state = self.state.lock(); + let change = aggregation_context.apply_change(&mut state.data, change); + propagate_change_to_upper(&mut state, aggregation_context, change); + } + + pub fn get_root_info>( + &self, + aggregation_context: &C, + root_info_type: &C::RootInfoType, + ) -> C::RootInfo { + let mut result = aggregation_context.new_root_info(root_info_type); + let state = self.state.lock(); + for TopRef { upper } in state.top_upper.iter() { + let info = upper.get_root_info(aggregation_context, root_info_type); + if aggregation_context.merge_root_info(&mut result, info) == ControlFlow::Break(()) { + return result; + } + } + state + .bottom_upper + .get_root_info(aggregation_context, root_info_type, result) + } +} + +fn propagate_lost_following_to_uppers( + state: MutexGuard<'_, BottomTreeState>, + aggregation_context: &C, + child_of_child: &C::ItemRef, +) { + let bottom_uppers = state.bottom_upper.as_cloned_uppers(); + let top_upper = state.top_upper.iter().cloned().collect::>(); + drop(state); + for TopRef { upper } in top_upper { + upper.remove_child_of_child(aggregation_context, child_of_child); + } + bottom_uppers.remove_child_of_child(aggregation_context, child_of_child); +} + +fn propagate_lost_followings_to_uppers<'a, C: AggregationContext>( + state: MutexGuard<'_, BottomTreeState>, + aggregation_context: &C, + children: impl IntoIterator + Clone, +) where + C::ItemRef: 'a, +{ + let bottom_uppers = state.bottom_upper.as_cloned_uppers(); + let top_upper = state.top_upper.iter().cloned().collect::>(); + drop(state); + for TopRef { upper } in top_upper { + upper.remove_children_of_child(aggregation_context, children.clone()); + } + bottom_uppers.remove_children_of_child(aggregation_context, children); +} + +fn propagate_new_following_to_uppers( + state: MutexGuard<'_, BottomTreeState>, + aggregation_context: &C, + child_of_child: &C::ItemRef, +) { + let bottom_uppers = state.bottom_upper.as_cloned_uppers(); + let top_upper = state.top_upper.iter().cloned().collect::>(); + drop(state); + for TopRef { upper } in top_upper { + upper.add_child_of_child(aggregation_context, child_of_child); + } + bottom_uppers.add_child_of_child(aggregation_context, child_of_child); +} + +fn propagate_change_to_upper( + state: &mut MutexGuard>, + aggregation_context: &C, + change: Option, +) { + let Some(change) = change else { + return; + }; + state + .bottom_upper + .child_change(aggregation_context, &change); + for TopRef { upper } in state.top_upper.iter() { + upper.child_change(aggregation_context, &change); + } +} + +#[cfg(test)] +fn visit_graph( + aggregation_context: &C, + entry: &C::ItemRef, + height: u8, +) -> (usize, usize) { + use std::collections::{HashSet, VecDeque}; + let mut queue = VecDeque::new(); + let mut visited = HashSet::new(); + visited.insert(entry.clone()); + queue.push_back(entry.clone()); + let mut edges = 0; + while let Some(item) = queue.pop_front() { + let tree = bottom_tree(aggregation_context, &item, height); + let state = tree.state.lock(); + for next in state.following.iter() { + edges += 1; + if visited.insert(next.clone()) { + queue.push_back(next.clone()); + } + } + } + (visited.len(), edges) +} + +#[cfg(test)] +pub fn print_graph( + aggregation_context: &C, + entry: &C::ItemRef, + height: u8, + color_upper: bool, + name_fn: impl Fn(&C::ItemRef) -> String, +) { + use std::{ + collections::{HashSet, VecDeque}, + fmt::Write, + }; + let (nodes, edges) = visit_graph(aggregation_context, entry, height); + if !color_upper { + print!("subgraph cluster_{} {{", height); + print!( + "label = \"Level {}\\n{} nodes, {} edges\";", + height, nodes, edges + ); + print!("color = \"black\";"); + } + let mut edges = String::new(); + let mut queue = VecDeque::new(); + let mut visited = HashSet::new(); + visited.insert(entry.clone()); + queue.push_back(entry.clone()); + while let Some(item) = queue.pop_front() { + let tree = bottom_tree(aggregation_context, &item, height); + let name = name_fn(&item); + let label = format!("{}", name); + let state = tree.state.lock(); + if color_upper { + print!(r#""{} {}" [color=red];"#, height - 1, name); + } else { + print!(r#""{} {}" [label="{}"];"#, height, name, label); + } + for next in state.following.iter() { + if !color_upper { + write!( + edges, + r#""{} {}" -> "{} {}";"#, + height, + name, + height, + name_fn(next) + ) + .unwrap(); + } + if visited.insert(next.clone()) { + queue.push_back(next.clone()); + } + } + } + if !color_upper { + println!("}}"); + println!("{}", edges); + } +} diff --git a/crates/turbo-tasks-memory/src/aggregation_tree/inner_refs.rs b/crates/turbo-tasks-memory/src/aggregation_tree/inner_refs.rs new file mode 100644 index 0000000000000..92693d65f3a7c --- /dev/null +++ b/crates/turbo-tasks-memory/src/aggregation_tree/inner_refs.rs @@ -0,0 +1,79 @@ +use std::{ + hash::{Hash, Hasher}, + sync::Arc, +}; + +use nohash_hasher::IsEnabled; +use ref_cast::RefCast; + +use super::{bottom_tree::BottomTree, top_tree::TopTree}; + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub enum ChildLocation { + // Left-most child + Left, + // Inner child, not left-most + Inner, +} + +/// A reference to a [TopTree]. +#[derive(RefCast)] +#[repr(transparent)] +pub struct TopRef { + pub upper: Arc>, +} + +impl IsEnabled for TopRef {} + +impl Hash for TopRef { + fn hash(&self, state: &mut H) { + Arc::as_ptr(&self.upper).hash(state); + } +} + +impl PartialEq for TopRef { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.upper, &other.upper) + } +} + +impl Eq for TopRef {} + +impl Clone for TopRef { + fn clone(&self) -> Self { + Self { + upper: self.upper.clone(), + } + } +} + +/// A reference to a [BottomTree]. +#[derive(RefCast)] +#[repr(transparent)] +pub struct BottomRef { + pub upper: Arc>, +} + +impl Hash for BottomRef { + fn hash(&self, state: &mut H) { + Arc::as_ptr(&self.upper).hash(state); + } +} + +impl IsEnabled for BottomRef {} + +impl PartialEq for BottomRef { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.upper, &other.upper) + } +} + +impl Eq for BottomRef {} + +impl Clone for BottomRef { + fn clone(&self) -> Self { + Self { + upper: self.upper.clone(), + } + } +} diff --git a/crates/turbo-tasks-memory/src/aggregation_tree/leaf.rs b/crates/turbo-tasks-memory/src/aggregation_tree/leaf.rs new file mode 100644 index 0000000000000..597c74f26864a --- /dev/null +++ b/crates/turbo-tasks-memory/src/aggregation_tree/leaf.rs @@ -0,0 +1,405 @@ +use std::{hash::Hash, sync::Arc}; + +use auto_hash_map::AutoSet; +use nohash_hasher::IsEnabled; +use ref_cast::RefCast; +use tracing::Level; + +use super::{ + bottom_connection::{BottomConnection, DistanceCountMap}, + bottom_tree::BottomTree, + inner_refs::{BottomRef, ChildLocation}, + top_tree::TopTree, + AggregationContext, AggregationItemLock, CHILDREN_INNER_THRESHOLD, +}; + +/// The leaf of the aggregation tree. It's usually stored inside of the nodes +/// that should be aggregated by the aggregation tree. It caches [TopTree]s and +/// [BottomTree]s created from that node. And it also stores the upper bottom +/// trees. +pub struct AggregationTreeLeaf { + top_trees: Vec>>>, + bottom_trees: Vec>>>, + upper: BottomConnection, +} + +impl AggregationTreeLeaf { + pub fn new() -> Self { + Self { + top_trees: Vec::new(), + bottom_trees: Vec::new(), + upper: BottomConnection::new(), + } + } + + /// Prepares the addition of new children. It returns a closure that should + /// be executed outside of the leaf lock. + #[allow(unused)] + pub fn add_children_job<'a, C: AggregationContext>( + &self, + aggregation_context: &'a C, + children: Vec, + ) -> impl FnOnce() + 'a + where + I: 'a, + T: 'a, + { + let uppers = self.upper.as_cloned_uppers(); + move || { + uppers.add_children_of_child(aggregation_context, &children); + } + } + + /// Prepares the addition of a new child. It returns a closure that should + /// be executed outside of the leaf lock. + pub fn add_child_job<'a, C: AggregationContext>( + &self, + aggregation_context: &'a C, + child: &'a I, + ) -> impl FnOnce() + 'a + where + T: 'a, + { + let uppers = self.upper.as_cloned_uppers(); + move || { + uppers.add_child_of_child(aggregation_context, child); + } + } + + /// Removes a child. + pub fn remove_child>( + &self, + aggregation_context: &C, + child: &I, + ) { + self.upper + .as_cloned_uppers() + .remove_child_of_child(aggregation_context, child); + } + + /// Prepares the removal of a child. It returns a closure that should be + /// executed outside of the leaf lock. + pub fn remove_children_job<'a, C: AggregationContext, H>( + &self, + aggregation_context: &'a C, + children: AutoSet, + ) -> impl FnOnce() + 'a + where + T: 'a, + I: 'a, + H: 'a, + { + let uppers = self.upper.as_cloned_uppers(); + move || uppers.remove_children_of_child(aggregation_context, children.iter()) + } + + /// Communicates a change on the leaf to updated aggregated nodes. Prefer + /// [Self::change_job] to avoid leaf locking. + pub fn change>( + &self, + aggregation_context: &C, + change: &C::ItemChange, + ) { + self.upper.child_change(aggregation_context, change); + } + + /// Prepares the communication of a change on the leaf to updated aggregated + /// nodes. It returns a closure that should be executed outside of the leaf + /// lock. + pub fn change_job<'a, C: AggregationContext>( + &self, + aggregation_context: &'a C, + change: C::ItemChange, + ) -> impl FnOnce() + 'a + where + I: 'a, + T: 'a, + { + let uppers = self.upper.as_cloned_uppers(); + move || { + uppers.child_change(aggregation_context, &change); + } + } + + /// Captures information about the aggregation tree roots. + pub fn get_root_info>( + &self, + aggregation_context: &C, + root_info_type: &C::RootInfoType, + ) -> C::RootInfo { + self.upper.get_root_info( + aggregation_context, + root_info_type, + aggregation_context.new_root_info(root_info_type), + ) + } + + pub fn has_upper(&self) -> bool { + !self.upper.is_unset() + } +} + +fn get_or_create_in_vec( + vec: &mut Vec>, + index: usize, + create: impl FnOnce() -> T, +) -> (&mut T, bool) { + if vec.len() <= index { + vec.resize_with(index + 1, || None); + } + let item = &mut vec[index]; + if item.is_none() { + *item = Some(create()); + (item.as_mut().unwrap(), true) + } else { + (item.as_mut().unwrap(), false) + } +} + +#[tracing::instrument(level = Level::TRACE, skip(aggregation_context, reference))] +pub fn top_tree( + aggregation_context: &C, + reference: &C::ItemRef, + depth: u8, +) -> Arc> { + let new_top_tree = { + let mut item = aggregation_context.item(reference); + let leaf = item.leaf(); + let (tree, new) = get_or_create_in_vec(&mut leaf.top_trees, depth as usize, || { + Arc::new(TopTree::new(depth)) + }); + if !new { + return tree.clone(); + } + tree.clone() + }; + let bottom_tree = bottom_tree(aggregation_context, reference, depth + 4); + bottom_tree.add_top_tree_upper(aggregation_context, &new_top_tree); + new_top_tree +} + +pub fn bottom_tree( + aggregation_context: &C, + reference: &C::ItemRef, + height: u8, +) -> Arc> { + let _span; + let new_bottom_tree; + let mut result = None; + { + let mut item = aggregation_context.item(reference); + let leaf = item.leaf(); + let (tree, new) = get_or_create_in_vec(&mut leaf.bottom_trees, height as usize, || { + Arc::new(BottomTree::new(reference.clone(), height)) + }); + if !new { + return tree.clone(); + } + new_bottom_tree = tree.clone(); + _span = (height > 2).then(|| tracing::trace_span!("bottom_tree", height).entered()); + + if height == 0 { + result = Some(add_left_upper_to_item_step_1::( + &mut item, + &new_bottom_tree, + )); + } + } + if let Some(result) = result { + add_left_upper_to_item_step_2(aggregation_context, reference, &new_bottom_tree, result); + } + if height != 0 { + bottom_tree(aggregation_context, reference, height - 1) + .add_left_bottom_tree_upper(aggregation_context, &new_bottom_tree); + } + new_bottom_tree +} + +#[must_use] +pub fn add_inner_upper_to_item( + aggregation_context: &C, + reference: &C::ItemRef, + upper: &Arc>, + nesting_level: u8, +) -> bool { + let (change, children) = { + let mut item = aggregation_context.item(reference); + let number_of_children = item.number_of_children(); + let leaf = item.leaf(); + let BottomConnection::Inner(inner) = &mut leaf.upper else { + return false; + }; + if inner.len() * number_of_children > CHILDREN_INNER_THRESHOLD { + return false; + } + let new = inner.add_clonable(BottomRef::ref_cast(upper), nesting_level); + if new { + let change = item.get_add_change(); + ( + change, + item.children().map(|r| r.into_owned()).collect::>(), + ) + } else { + return true; + } + }; + if let Some(change) = change { + upper.child_change(aggregation_context, &change); + } + if !children.is_empty() { + upper.add_children_of_child( + aggregation_context, + ChildLocation::Inner, + &children, + nesting_level + 1, + ) + } + true +} + +struct AddLeftUpperIntermediateResult( + Option, + Vec, + DistanceCountMap>, + Option, + Vec, +); + +#[must_use] +fn add_left_upper_to_item_step_1( + item: &mut C::ItemLock<'_>, + upper: &Arc>, +) -> AddLeftUpperIntermediateResult { + let old_inner = item.leaf().upper.set_left_upper(upper); + let remove_change_for_old_inner = (!old_inner.is_unset()) + .then(|| item.get_remove_change()) + .flatten(); + let children_for_old_inner = (!old_inner.is_unset()) + .then(|| { + item.children() + .map(|child| child.into_owned()) + .collect::>() + }) + .unwrap_or_default(); + AddLeftUpperIntermediateResult( + item.get_add_change(), + item.children().map(|r| r.into_owned()).collect(), + old_inner, + remove_change_for_old_inner, + children_for_old_inner, + ) +} + +fn add_left_upper_to_item_step_2( + aggregation_context: &C, + reference: &C::ItemRef, + upper: &Arc>, + step_1_result: AddLeftUpperIntermediateResult, +) { + let AddLeftUpperIntermediateResult( + change, + children, + old_inner, + remove_change_for_old_inner, + following_for_old_uppers, + ) = step_1_result; + if let Some(change) = change { + upper.child_change(aggregation_context, &change); + } + if !children.is_empty() { + upper.add_children_of_child(aggregation_context, ChildLocation::Left, &children, 1) + } + for (BottomRef { upper: old_upper }, count) in old_inner.into_counts() { + old_upper.migrate_old_inner( + aggregation_context, + reference, + count, + &remove_change_for_old_inner, + &following_for_old_uppers, + ); + } +} + +pub fn remove_left_upper_from_item( + aggregation_context: &C, + reference: &C::ItemRef, + upper: &Arc>, +) { + let mut item = aggregation_context.item(reference); + let leaf = &mut item.leaf(); + leaf.upper.unset_left_upper(upper); + let change = item.get_remove_change(); + let children = item.children().map(|r| r.into_owned()).collect::>(); + drop(item); + if let Some(change) = change { + upper.child_change(aggregation_context, &change); + } + for child in children { + upper.remove_child_of_child(aggregation_context, &child) + } +} + +#[must_use] +pub fn remove_inner_upper_from_item( + aggregation_context: &C, + reference: &C::ItemRef, + upper: &Arc>, +) -> bool { + let mut item = aggregation_context.item(reference); + let BottomConnection::Inner(inner) = &mut item.leaf().upper else { + return false; + }; + if !inner.remove_clonable(BottomRef::ref_cast(upper)) { + // Nothing to do + return true; + } + let change = item.get_remove_change(); + let children = item.children().map(|r| r.into_owned()).collect::>(); + drop(item); + + if let Some(change) = change { + upper.child_change(aggregation_context, &change); + } + for child in children { + upper.remove_child_of_child(aggregation_context, &child) + } + true +} + +/// Checks thresholds for an item to ensure the aggregation graph stays +/// well-formed. Run this before added a child to an item. Returns a closure +/// that should be executed outside of the leaf lock. +pub fn ensure_thresholds<'a, C: AggregationContext>( + aggregation_context: &'a C, + item: &mut C::ItemLock<'_>, +) -> impl FnOnce() + 'a { + let mut result = None; + + let number_of_total_children = item.number_of_children(); + let reference = item.reference().clone(); + let leaf = item.leaf(); + if let BottomConnection::Inner(list) = &leaf.upper { + if list.len() * number_of_total_children > CHILDREN_INNER_THRESHOLD { + let (tree, new) = get_or_create_in_vec(&mut leaf.bottom_trees, 0, || { + Arc::new(BottomTree::new(reference.clone(), 0)) + }); + debug_assert!(new); + let new_bottom_tree = tree.clone(); + result = Some(( + add_left_upper_to_item_step_1::(item, &new_bottom_tree), + reference, + new_bottom_tree, + )); + } + } + || { + if let Some((result, reference, new_bottom_tree)) = result { + add_left_upper_to_item_step_2( + aggregation_context, + &reference, + &new_bottom_tree, + result, + ); + } + } +} diff --git a/crates/turbo-tasks-memory/src/aggregation_tree/mod.rs b/crates/turbo-tasks-memory/src/aggregation_tree/mod.rs new file mode 100644 index 0000000000000..83472da064bf6 --- /dev/null +++ b/crates/turbo-tasks-memory/src/aggregation_tree/mod.rs @@ -0,0 +1,146 @@ +//! The module implements a datastructure that aggregates a "forest" into less +//! nodes. For any node one can ask for a single aggregated version of all +//! children on that node. Changes the the forest will propagate up the +//! aggregation tree to keep it up to date. So asking of an aggregated +//! information is cheap and one can even wait for aggregated info to change. +//! +//! The aggregation will try to reuse aggregated nodes on every level to reduce +//! memory and cpu usage of propagating changes. The tree structure is designed +//! for multi-thread usage. +//! +//! The aggregation tree is build out of two halfs. The top tree and the bottom +//! tree. One node of the bottom tree can aggregate items of connectivity +//! 2^height. It will do that by having bottom trees of height - 1 as children. +//! One node of the top tree can aggregate items of any connectivity. It will do +//! that by having a bottom tree of height = depth as a child and top trees of +//! depth + 1 as children. So it's basically a linked list of bottom trees of +//! increasing height. Any top or bottom node can be shared between multiple +//! parents. +//! +//! Notations: +//! - parent/child: Relationship in the original forest resp. the aggregated +//! version of the relationships. +//! - upper: Relationship to a aggregated node in a higher level (more +//! aggregated). Since all communication is strictly upwards there is no down +//! relationship for that. + +mod bottom_connection; +mod bottom_tree; +mod inner_refs; +mod leaf; +#[cfg(test)] +mod tests; +mod top_tree; + +use std::{borrow::Cow, hash::Hash, ops::ControlFlow, sync::Arc}; + +use nohash_hasher::IsEnabled; + +use self::{leaf::top_tree, top_tree::TopTree}; +pub use self::{ + leaf::{ensure_thresholds, AggregationTreeLeaf}, + top_tree::AggregationInfoGuard, +}; + +/// The maximum connectivity of one layer of bottom tree. +const CONNECTIVITY_LIMIT: u8 = 7; + +/// The maximum of number of children muliplied by number of upper bottom trees. +/// When reached the parent of the children will form a new bottom tree. +const CHILDREN_INNER_THRESHOLD: usize = 2000; + +/// The context trait which defines how the aggregation tree should behave. +pub trait AggregationContext { + type ItemLock<'a>: AggregationItemLock< + ItemRef = Self::ItemRef, + Info = Self::Info, + ItemChange = Self::ItemChange, + > + where + Self: 'a; + type Info: Default; + type ItemChange; + type ItemRef: Eq + Hash + Clone + IsEnabled; + type RootInfo; + type RootInfoType; + + /// Gets mutable access to an item. + fn item<'a>(&'a self, reference: &Self::ItemRef) -> Self::ItemLock<'a>; + + /// Apply a changeset to an aggregated info object. Returns a new changeset + /// that should be applied to the next aggregation level. Might return None, + /// if no change should be applied to the next level. + fn apply_change( + &self, + info: &mut Self::Info, + change: &Self::ItemChange, + ) -> Option; + + /// Creates a changeset from an aggregated info object, that represents + /// adding the aggregated node to an aggregated node of the next level. + fn info_to_add_change(&self, info: &Self::Info) -> Option; + /// Creates a changeset from an aggregated info object, that represents + /// removing the aggregated node from an aggregated node of the next level. + fn info_to_remove_change(&self, info: &Self::Info) -> Option; + + /// Initializes a new empty root info object. + fn new_root_info(&self, root_info_type: &Self::RootInfoType) -> Self::RootInfo; + /// Creates a new root info object from an aggregated info object. This is + /// only called on the root of the aggregation tree. + fn info_to_root_info( + &self, + info: &Self::Info, + root_info_type: &Self::RootInfoType, + ) -> Self::RootInfo; + /// Merges two root info objects. Can optionally break the root info + /// gathering which will return this root info object as final result. + fn merge_root_info( + &self, + root_info: &mut Self::RootInfo, + other: Self::RootInfo, + ) -> ControlFlow<()>; +} + +/// A lock on a single item. +pub trait AggregationItemLock { + type Info; + type ItemRef: Clone + IsEnabled; + type ItemChange; + type ChildrenIter<'a>: Iterator> + 'a + where + Self: 'a; + /// Returns a reference to the item. + fn reference(&self) -> &Self::ItemRef; + /// Get mutable access to the leaf info. + fn leaf(&mut self) -> &mut AggregationTreeLeaf; + /// Returns the number of children. + fn number_of_children(&self) -> usize; + /// Returns an iterator over the children. + fn children(&self) -> Self::ChildrenIter<'_>; + /// Returns a changeset that represents the addition of the item. + fn get_add_change(&self) -> Option; + /// Returns a changeset that represents the removal of the item. + fn get_remove_change(&self) -> Option; +} + +/// Gives an reference to the root aggregated info for a given item. +pub fn aggregation_info( + aggregation_context: &C, + reference: &C::ItemRef, +) -> AggregationInfoReference { + AggregationInfoReference { + tree: top_tree(aggregation_context, reference, 0), + } +} + +/// A reference to the root aggregated info of a node. +pub struct AggregationInfoReference { + tree: Arc>, +} + +impl AggregationInfoReference { + /// Locks the info and gives mutable access to it. + pub fn lock(&self) -> AggregationInfoGuard { + self.tree.lock_info() + } +} diff --git a/crates/turbo-tasks-memory/src/aggregation_tree/tests.rs b/crates/turbo-tasks-memory/src/aggregation_tree/tests.rs new file mode 100644 index 0000000000000..f24ca29045344 --- /dev/null +++ b/crates/turbo-tasks-memory/src/aggregation_tree/tests.rs @@ -0,0 +1,592 @@ +use std::{ + borrow::Cow, + hash::Hash, + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, + time::Instant, +}; + +use nohash_hasher::IsEnabled; +use parking_lot::{Mutex, MutexGuard}; +use ref_cast::RefCast; + +use super::{aggregation_info, AggregationContext, AggregationItemLock, AggregationTreeLeaf}; +use crate::aggregation_tree::{bottom_tree::print_graph, leaf::ensure_thresholds}; + +struct Node { + inner: Mutex, +} + +impl Node { + fn incr(&self, aggregation_context: &NodeAggregationContext) { + let mut guard = self.inner.lock(); + guard.value += 10000; + guard + .aggregation_leaf + .change(aggregation_context, &Change { value: 10000 }); + } +} + +#[derive(Copy, Clone)] +struct Change { + value: i32, +} + +impl Change { + fn is_empty(&self) -> bool { + self.value == 0 + } +} + +struct NodeInner { + children: Vec>, + aggregation_leaf: AggregationTreeLeaf, + value: u32, +} + +struct NodeAggregationContext<'a> { + additions: AtomicU32, + #[allow(dead_code)] + something_with_lifetime: &'a u32, + add_value: bool, +} + +#[derive(Clone, RefCast)] +#[repr(transparent)] +struct NodeRef(Arc); + +impl Hash for NodeRef { + fn hash(&self, state: &mut H) { + Arc::as_ptr(&self.0).hash(state); + } +} + +impl IsEnabled for NodeRef {} + +impl PartialEq for NodeRef { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } +} + +impl Eq for NodeRef {} + +struct NodeGuard { + guard: MutexGuard<'static, NodeInner>, + node: Arc, +} + +impl NodeGuard { + unsafe fn new<'a>(guard: MutexGuard<'a, NodeInner>, node: Arc) -> Self { + NodeGuard { + guard: unsafe { std::mem::transmute(guard) }, + node, + } + } +} + +impl AggregationItemLock for NodeGuard { + type Info = Aggregated; + type ItemRef = NodeRef; + type ItemChange = Change; + type ChildrenIter<'a> = impl Iterator> + 'a; + + fn reference(&self) -> &Self::ItemRef { + NodeRef::ref_cast(&self.node) + } + + fn leaf(&mut self) -> &mut AggregationTreeLeaf { + &mut self.guard.aggregation_leaf + } + + fn number_of_children(&self) -> usize { + self.guard.children.len() + } + + fn children(&self) -> Self::ChildrenIter<'_> { + self.guard + .children + .iter() + .map(|child| Cow::Owned(NodeRef(child.clone()))) + } + + fn get_remove_change(&self) -> Option { + let change = Change { + value: -(self.guard.value as i32), + }; + if change.is_empty() { + None + } else { + Some(change) + } + } + + fn get_add_change(&self) -> Option { + let change = Change { + value: self.guard.value as i32, + }; + if change.is_empty() { + None + } else { + Some(change) + } + } +} + +impl<'a> AggregationContext for NodeAggregationContext<'a> { + type ItemLock<'l> = NodeGuard where Self: 'l; + type Info = Aggregated; + type ItemRef = NodeRef; + type ItemChange = Change; + + fn item<'b>(&'b self, reference: &Self::ItemRef) -> Self::ItemLock<'b> { + let r = reference.0.clone(); + let guard = reference.0.inner.lock(); + unsafe { NodeGuard::new(guard, r) } + } + + fn apply_change(&self, info: &mut Aggregated, change: &Change) -> Option { + if info.value != 0 { + self.additions.fetch_add(1, Ordering::SeqCst); + } + if self.add_value { + info.value += change.value; + } + Some(change.clone()) + } + + fn info_to_add_change(&self, info: &Self::Info) -> Option { + let change = Change { + value: info.value as i32, + }; + if change.is_empty() { + None + } else { + Some(change) + } + } + + fn info_to_remove_change(&self, info: &Self::Info) -> Option { + let change = Change { + value: -(info.value as i32), + }; + if change.is_empty() { + None + } else { + Some(change) + } + } + + type RootInfo = bool; + + type RootInfoType = (); + + fn new_root_info(&self, root_info_type: &Self::RootInfoType) -> Self::RootInfo { + match root_info_type { + () => false, + } + } + + fn info_to_root_info( + &self, + info: &Self::Info, + root_info_type: &Self::RootInfoType, + ) -> Self::RootInfo { + match root_info_type { + () => info.active, + } + } + + fn merge_root_info( + &self, + root_info: &mut Self::RootInfo, + other: Self::RootInfo, + ) -> std::ops::ControlFlow<()> { + if other { + *root_info = true; + std::ops::ControlFlow::Break(()) + } else { + std::ops::ControlFlow::Continue(()) + } + } +} + +#[derive(Default)] +struct Aggregated { + value: i32, + active: bool, +} + +#[test] +fn chain() { + let something_with_lifetime = 0; + let ctx = NodeAggregationContext { + additions: AtomicU32::new(0), + something_with_lifetime: &something_with_lifetime, + add_value: true, + }; + let leaf = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: vec![], + aggregation_leaf: AggregationTreeLeaf::new(), + value: 10000, + }), + }); + let mut current = leaf.clone(); + for i in 1..=100 { + current = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: vec![current], + aggregation_leaf: AggregationTreeLeaf::new(), + value: i, + }), + }); + } + let current = NodeRef(current); + + { + let root_info = leaf.inner.lock().aggregation_leaf.get_root_info(&ctx, &()); + assert_eq!(root_info, false); + } + + { + let aggregated = aggregation_info(&ctx, ¤t); + assert_eq!(aggregated.lock().value, 15050); + } + assert_eq!(ctx.additions.load(Ordering::SeqCst), 100); + ctx.additions.store(0, Ordering::SeqCst); + + print(&ctx, ¤t); + + { + let root_info = leaf.inner.lock().aggregation_leaf.get_root_info(&ctx, &()); + assert_eq!(root_info, false); + } + + leaf.incr(&ctx); + // The change need to propagate through 5 top trees and 5 bottom trees + assert_eq!(ctx.additions.load(Ordering::SeqCst), 6); + ctx.additions.store(0, Ordering::SeqCst); + + { + let aggregated = aggregation_info(&ctx, ¤t); + let mut aggregated = aggregated.lock(); + assert_eq!(aggregated.value, 25050); + (*aggregated).active = true; + } + assert_eq!(ctx.additions.load(Ordering::SeqCst), 0); + ctx.additions.store(0, Ordering::SeqCst); + + { + let root_info = leaf.inner.lock().aggregation_leaf.get_root_info(&ctx, &()); + assert_eq!(root_info, true); + } + + let i = 101; + let current = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: vec![current.0], + aggregation_leaf: AggregationTreeLeaf::new(), + value: i, + }), + }); + let current = NodeRef(current); + + { + let aggregated = aggregation_info(&ctx, ¤t); + let aggregated = aggregated.lock(); + assert_eq!(aggregated.value, 25151); + } + // This should be way less the 100 to prove that we are reusing trees + assert_eq!(ctx.additions.load(Ordering::SeqCst), 1); + ctx.additions.store(0, Ordering::SeqCst); + + leaf.incr(&ctx); + // This should be less the 20 to prove that we are reusing trees + assert_eq!(ctx.additions.load(Ordering::SeqCst), 9); + ctx.additions.store(0, Ordering::SeqCst); + + { + let root_info = leaf.inner.lock().aggregation_leaf.get_root_info(&ctx, &()); + assert_eq!(root_info, true); + } +} + +#[test] +fn chain_double_connected() { + let something_with_lifetime = 0; + let ctx = NodeAggregationContext { + additions: AtomicU32::new(0), + something_with_lifetime: &something_with_lifetime, + add_value: true, + }; + let leaf = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: vec![], + aggregation_leaf: AggregationTreeLeaf::new(), + value: 1, + }), + }); + let mut current = leaf.clone(); + let mut current2 = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: vec![leaf.clone()], + aggregation_leaf: AggregationTreeLeaf::new(), + value: 2, + }), + }); + for i in 3..=100 { + let new_node = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: vec![current, current2.clone()], + aggregation_leaf: AggregationTreeLeaf::new(), + value: i, + }), + }); + current = current2; + current2 = new_node; + } + let current = NodeRef(current2); + + print(&ctx, ¤t); + + { + let aggregated = aggregation_info(&ctx, ¤t); + assert_eq!(aggregated.lock().value, 8230); + } + assert_eq!(ctx.additions.load(Ordering::SeqCst), 204); + ctx.additions.store(0, Ordering::SeqCst); +} + +const RECT_SIZE: usize = 100; +const RECT_MULT: usize = 100; + +#[test] +fn rectangle_tree() { + let something_with_lifetime = 0; + let ctx = NodeAggregationContext { + additions: AtomicU32::new(0), + something_with_lifetime: &something_with_lifetime, + add_value: false, + }; + let mut nodes: Vec>> = Vec::new(); + for y in 0..RECT_SIZE { + let mut line: Vec> = Vec::new(); + for x in 0..RECT_SIZE { + let mut children = Vec::new(); + if x > 0 { + children.push(line[x - 1].clone()); + } + if y > 0 { + children.push(nodes[y - 1][x].clone()); + } + let value = (x + y * RECT_MULT) as u32; + let node = Arc::new(Node { + inner: Mutex::new(NodeInner { + children, + aggregation_leaf: AggregationTreeLeaf::new(), + value, + }), + }); + line.push(node.clone()); + } + nodes.push(line); + } + + let root = NodeRef(nodes[RECT_SIZE - 1][RECT_SIZE - 1].clone()); + + print(&ctx, &root); +} + +#[test] +fn rectangle_adding_tree() { + let something_with_lifetime = 0; + let ctx = NodeAggregationContext { + additions: AtomicU32::new(0), + something_with_lifetime: &something_with_lifetime, + add_value: false, + }; + let mut nodes: Vec>> = Vec::new(); + + fn add_child( + parent: &Arc, + node: &Arc, + aggregation_context: &NodeAggregationContext<'_>, + ) { + let node_ref = NodeRef(node.clone()); + let mut state = parent.inner.lock(); + state.children.push(node.clone()); + let job = state + .aggregation_leaf + .add_child_job(aggregation_context, &node_ref); + drop(state); + job(); + } + for y in 0..RECT_SIZE { + let mut line: Vec> = Vec::new(); + for x in 0..RECT_SIZE { + let value = (x + y * RECT_MULT) as u32; + let node = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: Vec::new(), + aggregation_leaf: AggregationTreeLeaf::new(), + value, + }), + }); + line.push(node.clone()); + if x > 0 { + let parent = &line[x - 1]; + add_child(parent, &node, &ctx); + } + if y > 0 { + let parent = &nodes[y - 1][x]; + add_child(parent, &node, &ctx); + } + if x == 0 && y == 0 { + aggregation_info(&ctx, &NodeRef(node.clone())).lock().active = true; + } + } + nodes.push(line); + } + + let root = NodeRef(nodes[0][0].clone()); + + print(&ctx, &root); +} + +#[test] +fn many_children() { + let something_with_lifetime = 0; + let ctx = NodeAggregationContext { + additions: AtomicU32::new(0), + something_with_lifetime: &something_with_lifetime, + add_value: false, + }; + let mut roots: Vec> = Vec::new(); + let mut children: Vec> = Vec::new(); + const CHILDREN: u32 = 5000; + const ROOTS: u32 = 100; + let inner_node = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: Vec::new(), + aggregation_leaf: AggregationTreeLeaf::new(), + value: 0, + }), + }); + let start = Instant::now(); + for i in 0..ROOTS { + let node = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: Vec::new(), + aggregation_leaf: AggregationTreeLeaf::new(), + value: 10000 + i, + }), + }); + roots.push(node.clone()); + aggregation_info(&ctx, &NodeRef(node.clone())).lock().active = true; + connect_child(&ctx, &node, &inner_node); + } + println!("Roots: {:?}", start.elapsed()); + let start = Instant::now(); + for i in 0..CHILDREN { + let node = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: Vec::new(), + aggregation_leaf: AggregationTreeLeaf::new(), + value: 20000 + i, + }), + }); + children.push(node.clone()); + connect_child(&ctx, &inner_node, &node); + } + println!("Children: {:?}", start.elapsed()); + let start = Instant::now(); + for i in 0..ROOTS { + let node = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: Vec::new(), + aggregation_leaf: AggregationTreeLeaf::new(), + value: 30000 + i, + }), + }); + roots.push(node.clone()); + aggregation_info(&ctx, &NodeRef(node.clone())).lock().active = true; + connect_child(&ctx, &node, &inner_node); + } + println!("Roots: {:?}", start.elapsed()); + let start = Instant::now(); + for i in 0..CHILDREN { + let node = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: Vec::new(), + aggregation_leaf: AggregationTreeLeaf::new(), + value: 40000 + i, + }), + }); + children.push(node.clone()); + connect_child(&ctx, &inner_node, &node); + } + let children_duration = start.elapsed(); + println!("Children: {:?}", children_duration); + for _ in 0..10 { + let start = Instant::now(); + for i in 0..CHILDREN { + let node = Arc::new(Node { + inner: Mutex::new(NodeInner { + children: Vec::new(), + aggregation_leaf: AggregationTreeLeaf::new(), + value: 40000 + i, + }), + }); + children.push(node.clone()); + connect_child(&ctx, &inner_node, &node); + } + let dur = start.elapsed(); + println!("Children: {:?}", dur); + assert!(dur < children_duration * 2); + } + + let root = NodeRef(roots[0].clone()); + + print(&ctx, &root); +} + +fn connect_child( + aggregation_context: &NodeAggregationContext<'_>, + parent: &Arc, + child: &Arc, +) { + let state = parent.inner.lock(); + let node_ref = NodeRef(child.clone()); + let mut node_guard = unsafe { NodeGuard::new(state, parent.clone()) }; + let job1 = ensure_thresholds(aggregation_context, &mut node_guard); + let NodeGuard { + guard: mut state, .. + } = node_guard; + state.children.push(child.clone()); + let job2 = state + .aggregation_leaf + .add_child_job(aggregation_context, &node_ref); + drop(state); + job1(); + job2(); +} + +fn print(aggregation_context: &NodeAggregationContext<'_>, current: &NodeRef) { + println!("digraph {{"); + let start = 0; + let end = 3; + for i in start..end { + print_graph(aggregation_context, current, i, false, |item| { + format!("{}", item.0.inner.lock().value) + }); + } + for i in start + 1..end + 1 { + print_graph(aggregation_context, current, i, true, |item| { + format!("{}", item.0.inner.lock().value) + }); + } + println!("\n}}"); +} diff --git a/crates/turbo-tasks-memory/src/aggregation_tree/top_tree.rs b/crates/turbo-tasks-memory/src/aggregation_tree/top_tree.rs new file mode 100644 index 0000000000000..e55684c1a32cf --- /dev/null +++ b/crates/turbo-tasks-memory/src/aggregation_tree/top_tree.rs @@ -0,0 +1,180 @@ +use std::{mem::transmute, ops::ControlFlow, sync::Arc}; + +use parking_lot::{Mutex, MutexGuard}; +use ref_cast::RefCast; + +use super::{inner_refs::TopRef, leaf::top_tree, AggregationContext}; +use crate::count_hash_set::CountHashSet; + +/// The top half of the aggregation tree. It can aggregate all nodes of a +/// subgraph. To do that it used a [BottomTree] of a specific height and, since +/// a bottom tree only aggregates up to a specific connectivity, also another +/// TopTree of the current depth + 1. This continues recursively until all nodes +/// are aggregated. +pub struct TopTree { + pub depth: u8, + state: Mutex>, +} + +struct TopTreeState { + data: T, + upper: CountHashSet>, +} + +impl TopTree { + pub fn new(depth: u8) -> Self { + Self { + depth, + state: Mutex::new(TopTreeState { + data: T::default(), + upper: CountHashSet::new(), + }), + } + } +} + +impl TopTree { + pub fn add_children_of_child<'a, C: AggregationContext>( + self: &Arc, + aggregation_context: &C, + children: impl IntoIterator, + ) where + C::ItemRef: 'a, + { + for child in children { + top_tree(aggregation_context, child, self.depth + 1) + .add_upper(aggregation_context, self); + } + } + + pub fn add_child_of_child>( + self: &Arc, + aggregation_context: &C, + child_of_child: &C::ItemRef, + ) { + top_tree(aggregation_context, child_of_child, self.depth + 1) + .add_upper(aggregation_context, self); + } + + pub fn remove_child_of_child>( + self: &Arc, + aggregation_context: &C, + child_of_child: &C::ItemRef, + ) { + top_tree(aggregation_context, child_of_child, self.depth + 1) + .remove_upper(aggregation_context, self); + } + + pub fn remove_children_of_child<'a, C: AggregationContext>( + self: &Arc, + aggregation_context: &C, + children: impl IntoIterator, + ) where + C::ItemRef: 'a, + { + for child in children { + top_tree(aggregation_context, child, self.depth + 1) + .remove_upper(aggregation_context, self); + } + } + + pub fn add_upper>( + &self, + aggregation_context: &C, + upper: &Arc>, + ) { + let mut state = self.state.lock(); + if state.upper.add_clonable(TopRef::ref_cast(upper)) { + if let Some(change) = aggregation_context.info_to_add_change(&state.data) { + upper.child_change(aggregation_context, &change); + } + } + } + + pub fn remove_upper>( + &self, + aggregation_context: &C, + upper: &Arc>, + ) { + let mut state = self.state.lock(); + if state.upper.remove_clonable(TopRef::ref_cast(upper)) { + if let Some(change) = aggregation_context.info_to_remove_change(&state.data) { + upper.child_change(aggregation_context, &change); + } + } + } + + pub fn child_change>( + &self, + aggregation_context: &C, + change: &C::ItemChange, + ) { + let mut state = self.state.lock(); + let change = aggregation_context.apply_change(&mut state.data, change); + propagate_change_to_upper(&mut state, aggregation_context, change); + } + + pub fn get_root_info>( + &self, + aggregation_context: &C, + root_info_type: &C::RootInfoType, + ) -> C::RootInfo { + let state = self.state.lock(); + if self.depth == 0 { + // This is the root + aggregation_context.info_to_root_info(&state.data, root_info_type) + } else { + let mut result = aggregation_context.new_root_info(root_info_type); + for TopRef { upper } in state.upper.iter() { + let info = upper.get_root_info(aggregation_context, root_info_type); + if aggregation_context.merge_root_info(&mut result, info) == ControlFlow::Break(()) + { + break; + } + } + result + } + } + + pub fn lock_info(self: &Arc) -> AggregationInfoGuard { + AggregationInfoGuard { + // SAFETY: We can cast the lifetime as we keep a strong reference to the tree. + // The order of the field in the struct is important to drop guard before tree. + guard: unsafe { transmute(self.state.lock()) }, + tree: self.clone(), + } + } +} + +fn propagate_change_to_upper( + state: &mut MutexGuard>, + aggregation_context: &C, + change: Option, +) { + let Some(change) = change else { + return; + }; + for TopRef { upper } in state.upper.iter() { + upper.child_change(aggregation_context, &change); + } +} + +pub struct AggregationInfoGuard { + guard: MutexGuard<'static, TopTreeState>, + #[allow(dead_code, reason = "need to stay alive until the guard is dropped")] + tree: Arc>, +} + +impl std::ops::Deref for AggregationInfoGuard { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.guard.data + } +} + +impl std::ops::DerefMut for AggregationInfoGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.guard.data + } +} diff --git a/crates/turbo-tasks-memory/src/count_hash_set.rs b/crates/turbo-tasks-memory/src/count_hash_set.rs index 42c4b8ec345bb..75d84491f6fc8 100644 --- a/crates/turbo-tasks-memory/src/count_hash_set.rs +++ b/crates/turbo-tasks-memory/src/count_hash_set.rs @@ -1,4 +1,5 @@ use std::{ + borrow::Borrow, collections::hash_map::RandomState, fmt::{Debug, Formatter}, hash::{BuildHasher, Hash}, @@ -6,7 +7,7 @@ use std::{ }; use auto_hash_map::{ - map::{Entry, IntoIter, Iter}, + map::{Entry, Iter, RawEntry}, AutoMap, }; @@ -61,12 +62,12 @@ impl CountHashSet { pub fn is_empty(&self) -> bool { self.len() == 0 } +} - /// Checks if this set is equal to a fresh created set, meaning it has no - /// positive but also no negative entries. - pub fn is_unset(&self) -> bool { - self.inner.is_empty() - } +pub enum RemoveIfEntryResult { + PartiallyRemoved, + Removed, + NotPresent, } impl CountHashSet { @@ -107,9 +108,27 @@ impl CountHashSet { self.add_count(item, 1) } - /// Returns the current count of an item - pub fn get(&self, item: &T) -> isize { - *self.inner.get(item).unwrap_or(&0) + /// Returns true, when the value has been added. Returns false, when the + /// value was not part of the set before (positive or negative). The + /// visibility from outside will never change due to this method. + pub fn add_if_entry(&mut self, item: &Q) -> bool + where + T: Borrow, + Q: Hash + Eq + ?Sized, + { + match self.inner.raw_entry_mut(item) { + RawEntry::Occupied(mut e) => { + let value = e.get_mut(); + *value += 1; + if *value == 0 { + // it was negative and has become zero + self.negative_entries -= 1; + e.remove(); + } + true + } + RawEntry::Vacant(_) => false, + } } /// Returns true when the value is no longer visible from outside @@ -144,9 +163,22 @@ impl CountHashSet { } } - /// Returns true, when the value is no longer visible from outside - pub fn remove(&mut self, item: T) -> bool { - self.remove_count(item, 1) + /// Removes an item if it is present. + pub fn remove_if_entry(&mut self, item: &T) -> RemoveIfEntryResult { + match self.inner.raw_entry_mut(item) { + RawEntry::Occupied(mut e) => { + let value = e.get_mut(); + *value -= 1; + if *value == 0 { + // It was positive and has become zero + e.remove(); + RemoveIfEntryResult::Removed + } else { + RemoveIfEntryResult::PartiallyRemoved + } + } + RawEntry::Vacant(_) => RemoveIfEntryResult::NotPresent, + } } pub fn iter(&self) -> CountHashSetIter<'_, T> { @@ -154,13 +186,81 @@ impl CountHashSet { inner: self.inner.iter().filter_map(filter), } } +} - pub fn into_counts(self) -> IntoIter { - self.inner.into_iter() +impl CountHashSet { + /// Returns true, when the value has become visible from outside + pub fn add_clonable_count(&mut self, item: &T, count: usize) -> bool { + match self.inner.raw_entry_mut(item) { + RawEntry::Occupied(mut e) => { + let value = e.get_mut(); + let old = *value; + *value += count as isize; + if old > 0 { + // it was positive before + false + } else if *value > 0 { + // it was negative and has become positive + self.negative_entries -= 1; + true + } else if *value == 0 { + // it was negative and has become zero + self.negative_entries -= 1; + e.remove(); + false + } else { + // it was and still is negative + false + } + } + RawEntry::Vacant(e) => { + // it was zero and is now positive + e.insert(item.clone(), count as isize); + true + } + } } - pub fn counts(&self) -> Iter<'_, T, isize> { - self.inner.iter() + /// Returns true when the value has become visible from outside + pub fn add_clonable(&mut self, item: &T) -> bool { + self.add_clonable_count(item, 1) + } + + /// Returns true when the value is no longer visible from outside + pub fn remove_clonable_count(&mut self, item: &T, count: usize) -> bool { + match self.inner.raw_entry_mut(item) { + RawEntry::Occupied(mut e) => { + let value = e.get_mut(); + let old = *value; + *value -= count as isize; + if *value > 0 { + // It was and still is positive + false + } else if *value == 0 { + // It was positive and has become zero + e.remove(); + true + } else if old > 0 { + // It was positive and is negative now + self.negative_entries += 1; + true + } else { + // It was and still is negative + false + } + } + RawEntry::Vacant(e) => { + // It was zero and is negative now + e.insert(item.clone(), -(count as isize)); + self.negative_entries += 1; + false + } + } + } + + /// Returns true, when the value is no longer visible from outside + pub fn remove_clonable(&mut self, item: &T) -> bool { + self.remove_clonable_count(item, 1) } } diff --git a/crates/turbo-tasks-memory/src/gc.rs b/crates/turbo-tasks-memory/src/gc.rs index 8bc89f8854c06..c3f117346e7ce 100644 --- a/crates/turbo-tasks-memory/src/gc.rs +++ b/crates/turbo-tasks-memory/src/gc.rs @@ -168,7 +168,6 @@ impl GcQueue { // Process through the gc queue. let now = turbo_tasks.program_duration_until(Instant::now()); let mut task_duration_cache = HashMap::with_hasher(BuildNoHashHasher::default()); - let mut scope_active_cache = HashMap::with_hasher(BuildNoHashHasher::default()); let mut stats = GcStats::default(); let result = self.select_tasks(factor, |task_id, _priority, max_priority| { backend.with_task(task_id, |task| { @@ -176,7 +175,6 @@ impl GcQueue { now, max_priority, &mut task_duration_cache, - &mut scope_active_cache, &mut stats, backend, turbo_tasks, diff --git a/crates/turbo-tasks-memory/src/lib.rs b/crates/turbo-tasks-memory/src/lib.rs index 171ce94d53de3..57c491a4554a9 100644 --- a/crates/turbo-tasks-memory/src/lib.rs +++ b/crates/turbo-tasks-memory/src/lib.rs @@ -4,8 +4,10 @@ #![feature(lint_reasons)] #![feature(box_patterns)] #![feature(int_roundings)] +#![feature(impl_trait_in_assoc_type)] #![deny(unsafe_op_in_unsafe_fn)] +mod aggregation_tree; mod cell; mod concurrent_priority_queue; mod count_hash_set; @@ -14,8 +16,6 @@ mod map_guard; mod memory_backend; mod memory_backend_with_pg; mod output; -mod priority_pair; -pub mod scope; pub mod stats; mod task; pub mod viz; diff --git a/crates/turbo-tasks-memory/src/memory_backend.rs b/crates/turbo-tasks-memory/src/memory_backend.rs index dc3d474b56396..de44267259c64 100644 --- a/crates/turbo-tasks-memory/src/memory_backend.rs +++ b/crates/turbo-tasks-memory/src/memory_backend.rs @@ -2,7 +2,6 @@ use std::{ borrow::{Borrow, Cow}, cell::RefCell, cmp::min, - collections::VecDeque, future::Future, hash::{BuildHasher, BuildHasherDefault, Hash}, pin::Pin, @@ -14,12 +13,12 @@ use std::{ }; use anyhow::{bail, Result}; -use auto_hash_map::AutoSet; +use auto_hash_map::{AutoMap, AutoSet}; use dashmap::{mapref::entry::Entry, DashMap}; use nohash_hasher::BuildNoHashHasher; use rustc_hash::FxHasher; use tokio::task::futures::TaskLocalFuture; -use tracing::{trace_span, Instrument}; +use tracing::trace_span; use turbo_tasks::{ backend::{ Backend, BackendJobId, CellContent, PersistentTaskType, TaskExecutionSpec, @@ -27,33 +26,24 @@ use turbo_tasks::{ }, event::EventListener, util::{IdFactory, NoMoveVec}, - CellId, RawVc, TaskId, TraitTypeId, TurboTasksBackendApi, Unused, Vc, + CellId, RawVc, TaskId, TraitTypeId, TurboTasksBackendApi, Unused, }; use crate::{ cell::RecomputingCell, gc::GcQueue, output::Output, - priority_pair::PriorityPair, - scope::{TaskScope, TaskScopeId}, - task::{ - run_add_to_scope_queue, run_remove_from_scope_queue, Task, TaskDependency, - DEPENDENCIES_TO_TRACK, - }, + task::{Task, TaskDependency, DEPENDENCIES_TO_TRACK}, }; pub struct MemoryBackend { memory_tasks: NoMoveVec, - memory_task_scopes: NoMoveVec, - scope_id_factory: IdFactory, - pub(crate) initial_scope: TaskScopeId, backend_jobs: NoMoveVec, backend_job_id_factory: IdFactory, task_cache: DashMap, TaskId, BuildHasherDefault>, memory_limit: usize, gc_queue: Option, idle_gc_active: AtomicBool, - scope_add_remove_priority: PriorityPair, } impl Default for MemoryBackend { @@ -64,24 +54,14 @@ impl Default for MemoryBackend { impl MemoryBackend { pub fn new(memory_limit: usize) -> Self { - let memory_task_scopes = NoMoveVec::new(); - let scope_id_factory = IdFactory::new(); - let initial_scope: TaskScopeId = scope_id_factory.get(); - unsafe { - memory_task_scopes.insert(*initial_scope, TaskScope::new_active(initial_scope, 0, 0)); - } Self { memory_tasks: NoMoveVec::new(), - memory_task_scopes, - scope_id_factory, - initial_scope, backend_jobs: NoMoveVec::new(), backend_job_id_factory: IdFactory::new(), task_cache: DashMap::default(), memory_limit, gc_queue: (memory_limit != usize::MAX).then(GcQueue::new), idle_gc_active: AtomicBool::new(false), - scope_add_remove_priority: PriorityPair::new(), } } @@ -131,98 +111,8 @@ impl MemoryBackend { } #[inline(always)] - pub fn with_scope(&self, id: TaskScopeId, func: impl FnOnce(&TaskScope) -> T) -> T { - func(self.memory_task_scopes.get(*id).unwrap()) - } - - pub fn create_new_scope(&self, tasks: usize) -> TaskScopeId { - let id = self.scope_id_factory.get(); - unsafe { - self.memory_task_scopes - .insert(*id, TaskScope::new(id, tasks)); - } - id - } - - pub fn create_new_no_collectibles_scope(&self, tasks: usize) -> TaskScopeId { - let id = self.scope_id_factory.get(); - unsafe { - self.memory_task_scopes - .insert(*id, TaskScope::new_no_collectibles(id, tasks)); - } - id - } - - fn increase_scope_active_queue( - &self, - mut queue: Vec, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - while let Some(scope) = queue.pop() { - if let Some(tasks) = self.with_scope(scope, |scope| { - scope.state.lock().increment_active(&mut queue) - }) { - turbo_tasks.schedule_backend_foreground_job( - self.create_backend_job(Job::ScheduleWhenDirtyFromScope(tasks)), - ); - } - } - } - - pub(crate) fn increase_scope_active( - &self, - scope: TaskScopeId, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - self.increase_scope_active_queue(vec![scope], turbo_tasks); - } - - pub(crate) fn increase_scope_active_by( - &self, - scope: TaskScopeId, - count: usize, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - let mut queue = Vec::new(); - if let Some(tasks) = self.with_scope(scope, |scope| { - scope.state.lock().increment_active_by(count, &mut queue) - }) { - turbo_tasks.schedule_backend_foreground_job( - self.create_backend_job(Job::ScheduleWhenDirtyFromScope(tasks)), - ); - } - self.increase_scope_active_queue(queue, turbo_tasks); - } - - pub(crate) fn decrease_scope_active( - &self, - scope: TaskScopeId, - task_id: TaskId, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - self.decrease_scope_active_by(scope, task_id, 1, turbo_tasks); - } - - pub(crate) fn decrease_scope_active_by( - &self, - scope_id: TaskScopeId, - task_id: TaskId, - count: usize, - _turbo_tasks: &dyn TurboTasksBackendApi, - ) { - let mut queue = Vec::new(); - self.with_scope(scope_id, |scope| { - if scope.state.lock().decrement_active_by(count, &mut queue) { - if let Some(gc_queue) = &self.gc_queue { - gc_queue.task_might_become_inactive(task_id); - } - } - }); - while let Some(scope) = queue.pop() { - self.with_scope(scope, |scope| { - scope.state.lock().decrement_active_by(count, &mut queue) - }); - } + pub fn task(&self, id: TaskId) -> &Task { + self.memory_tasks.get(*id).unwrap() } pub fn on_task_might_become_inactive(&self, task: TaskId) { @@ -275,67 +165,6 @@ impl MemoryBackend { } } - pub(crate) fn get_or_create_read_task_collectibles_task( - &self, - task_id: TaskId, - trait_type: TraitTypeId, - parent_task: TaskId, - turbo_tasks: &dyn TurboTasksBackendApi, - ) -> TaskId { - self.with_task(task_id, |task| { - let id = task.get_read_collectibles_task(trait_type, || { - let scope = self.create_new_no_collectibles_scope(1); - - let id = turbo_tasks.get_fresh_task_id().into(); - let task = Task::new_read_task_collectibles( - // Safety: That task will hold the value, but we are still in - // control of the task - id, - scope, - task_id, - trait_type, - turbo_tasks.stats_type(), - ); - // Safety: We have a fresh task id that nobody knows about yet - unsafe { self.memory_tasks.insert(*id, task) }; - self.with_scope(scope, |scope| { - scope.state.lock().add_dirty_task(id); - }); - id - }); - self.connect_task_child(parent_task, id, turbo_tasks); - id - }) - } - - pub(crate) fn get_or_create_read_scope_collectibles_task( - &self, - scope_id: TaskScopeId, - trait_type: TraitTypeId, - parent_task: TaskId, - turbo_tasks: &dyn TurboTasksBackendApi, - ) -> TaskId { - self.with_scope(scope_id, |scope| { - let mut state = scope.state.lock(); - let task_id = state.get_read_collectibles_task(trait_type, || { - let id = turbo_tasks.get_fresh_task_id().into(); - let task = Task::new_read_scope_collectibles( - // Safety: That task will hold the value, but we are still in - // control of the task - id, - scope_id, - trait_type, - turbo_tasks.stats_type(), - ); - // Safety: We have a fresh task id that nobody knows about yet - unsafe { self.memory_tasks.insert(*id, task) }; - id - }); - self.connect_task_child(parent_task, task_id, turbo_tasks); - task_id - }) - } - fn insert_and_connect_fresh_task( &self, parent_task: TaskId, @@ -343,15 +172,11 @@ impl MemoryBackend { key: K, new_id: Unused, task: Task, - root_scoped: bool, turbo_tasks: &dyn TurboTasksBackendApi, ) -> TaskId { let new_id = new_id.into(); // Safety: We have a fresh task id that nobody knows about yet - let task = unsafe { self.memory_tasks.insert(*new_id, task) }; - if root_scoped { - task.make_root_scoped(self, turbo_tasks); - } + unsafe { self.memory_tasks.insert(*new_id, task) }; let result_task = match task_cache.entry(key) { Entry::Vacant(entry) => { // This is the most likely case @@ -389,6 +214,18 @@ impl MemoryBackend { *task }) } + + pub(crate) fn schedule_when_dirty_from_aggregation( + &self, + set: AutoSet>, + turbo_tasks: &dyn TurboTasksBackendApi, + ) { + for task in set { + self.with_task(task, |task| { + task.schedule_when_dirty_from_aggregation(self, turbo_tasks) + }); + } + } } impl Backend for MemoryBackend { @@ -506,7 +343,7 @@ impl Backend for MemoryBackend { move || format!("reading task output from {reader}"), turbo_tasks, |output| { - Task::add_dependency_to_current(TaskDependency::TaskOutput(task)); + Task::add_dependency_to_current(TaskDependency::Output(task)); output.read(reader) }, ) @@ -539,7 +376,7 @@ impl Backend for MemoryBackend { task.with_cell(index, |cell| cell.read_own_content_untracked()) }))) } else { - Task::add_dependency_to_current(TaskDependency::TaskCell(task_id, index)); + Task::add_dependency_to_current(TaskDependency::Cell(task_id, index)); self.with_task(task_id, |task| { match task.with_cell_mut(index, |cell| { cell.read_content( @@ -601,10 +438,8 @@ impl Backend for MemoryBackend { trait_id: TraitTypeId, reader: TaskId, turbo_tasks: &dyn TurboTasksBackendApi, - ) -> Vc> { - self.with_task(id, |task| { - task.read_task_collectibles(reader, trait_id, self, turbo_tasks) - }) + ) -> AutoMap { + Task::read_collectibles(id, trait_id, reader, self, turbo_tasks) } fn emit_collectible( @@ -623,11 +458,12 @@ impl Backend for MemoryBackend { &self, trait_type: TraitTypeId, collectible: RawVc, + count: u32, id: TaskId, turbo_tasks: &dyn TurboTasksBackendApi, ) { self.with_task(id, |task| { - task.unemit_collectible(trait_type, collectible, self, turbo_tasks) + task.unemit_collectible(trait_type, collectible, count, self, turbo_tasks); }); } @@ -694,7 +530,6 @@ impl Backend for MemoryBackend { task_type, id, task, - false, turbo_tasks, ) } @@ -712,9 +547,9 @@ impl Backend for MemoryBackend { fn mark_own_task_as_finished( &self, task: TaskId, - _turbo_tasks: &dyn TurboTasksBackendApi, + turbo_tasks: &dyn TurboTasksBackendApi, ) { - self.with_task(task, |task| task.mark_as_finished(self)) + self.with_task(task, |task| task.mark_as_finished(self, turbo_tasks)) } fn create_transient_task( @@ -723,60 +558,37 @@ impl Backend for MemoryBackend { turbo_tasks: &dyn TurboTasksBackendApi, ) -> TaskId { let id = turbo_tasks.get_fresh_task_id(); - // use INITIAL_SCOPE - let scope = self.initial_scope; - self.with_scope(scope, |scope| { - scope.increment_tasks(); - scope.increment_unfinished_tasks(self); - }); let stats_type = turbo_tasks.stats_type(); let id = id.into(); - let task = match task_type { - TransientTaskType::Root(f) => Task::new_root(id, scope, move || f() as _, stats_type), - TransientTaskType::Once(f) => Task::new_once(id, scope, f, stats_type), + match task_type { + TransientTaskType::Root(f) => { + let task = Task::new_root(id, move || f() as _, stats_type); + // SAFETY: We have a fresh task id where nobody knows about yet + unsafe { self.memory_tasks.insert(*id, task) }; + Task::set_root(id, self, turbo_tasks); + } + TransientTaskType::Once(f) => { + let task = Task::new_once(id, f, stats_type); + // SAFETY: We have a fresh task id where nobody knows about yet + unsafe { self.memory_tasks.insert(*id, task) }; + Task::set_once(id, self, turbo_tasks); + } }; - // SAFETY: We have a fresh task id where nobody knows about yet - #[allow(unused_variables)] - let task = unsafe { self.memory_tasks.insert(*id, task) }; - #[cfg(feature = "print_scope_updates")] - println!("new {scope} for {task}"); id } + + fn dispose_root_task(&self, task: TaskId, turbo_tasks: &dyn TurboTasksBackendApi) { + Task::unset_root(task, self, turbo_tasks); + } } pub(crate) enum Job { - RemoveFromScopes(AutoSet>, Vec), - RemoveFromScope(AutoSet>, TaskScopeId), - ScheduleWhenDirtyFromScope(AutoSet>), - /// Add tasks from a scope. Scheduled by `run_add_from_scope_queue` to - /// split off work. - AddToScopeQueue { - queue: VecDeque, - scope: TaskScopeId, - /// Number of scopes that are currently being merged into this scope. - /// This information is only used for optimization. - merging_scopes: usize, - }, - /// Remove tasks from a scope. Scheduled by `run_remove_from_scope_queue` to - /// split off work. - RemoveFromScopeQueue(VecDeque, TaskScopeId), - /// Unloads a previously used root scope after all other foreground tasks - /// are done. - UnloadRootScope(TaskScopeId), GarbageCollection, } impl Job { - fn before_schedule(&self, backend: &MemoryBackend) { - match self { - Job::RemoveFromScopes(..) - | Job::RemoveFromScope(..) - | Job::RemoveFromScopeQueue(..) => { - backend.scope_add_remove_priority.start_high(); - } - _ => {} - } - } + // TODO remove this method + fn before_schedule(&self, _backend: &MemoryBackend) {} async fn run( self, @@ -784,63 +596,6 @@ impl Job { turbo_tasks: &dyn TurboTasksBackendApi, ) { match self { - Job::RemoveFromScopes(tasks, scopes) => { - let _guard = trace_span!("Job::RemoveFromScopes").entered(); - for task in tasks { - backend.with_task(task, |task| { - task.remove_from_scopes(scopes.iter().copied(), backend, turbo_tasks) - }); - } - backend.scope_add_remove_priority.finish_high(); - } - Job::RemoveFromScope(tasks, scope) => { - let _guard = trace_span!("Job::RemoveFromScope").entered(); - for task in tasks { - backend.with_task(task, |task| { - task.remove_from_scope(scope, backend, turbo_tasks) - }); - } - backend.scope_add_remove_priority.finish_high(); - } - Job::ScheduleWhenDirtyFromScope(tasks) => { - let _guard = trace_span!("Job::ScheduleWhenDirtyFromScope").entered(); - for task in tasks.into_iter() { - backend.with_task(task, |task| { - task.schedule_when_dirty_from_scope(backend, turbo_tasks); - }) - } - } - Job::AddToScopeQueue { - queue, - scope, - merging_scopes, - } => { - backend - .scope_add_remove_priority - .run_low(async { - run_add_to_scope_queue(queue, scope, merging_scopes, backend, turbo_tasks); - }) - .instrument(trace_span!("Job::AddToScopeQueue")) - .await; - } - Job::RemoveFromScopeQueue(queue, id) => { - let _guard = trace_span!("Job::AddToScopeQueue").entered(); - run_remove_from_scope_queue(queue, id, backend, turbo_tasks); - backend.scope_add_remove_priority.finish_high(); - } - Job::UnloadRootScope(id) => { - let span = trace_span!("Job::UnloadRootScope"); - if let Some(future) = turbo_tasks.wait_foreground_done_excluding_own() { - future.instrument(span.clone()).await; - } - let _guard = span.entered(); - backend.with_scope(id, |scope| { - scope.assert_unused(); - }); - unsafe { - backend.scope_id_factory.reuse(id); - } - } Job::GarbageCollection => { let _guard = trace_span!("Job::GarbageCollection").entered(); backend.run_gc(true, turbo_tasks); diff --git a/crates/turbo-tasks-memory/src/memory_backend_with_pg.rs b/crates/turbo-tasks-memory/src/memory_backend_with_pg.rs index 7cde3277596f0..cf57039b91965 100644 --- a/crates/turbo-tasks-memory/src/memory_backend_with_pg.rs +++ b/crates/turbo-tasks-memory/src/memory_backend_with_pg.rs @@ -13,7 +13,7 @@ use std::{ }; use anyhow::{anyhow, Result}; -use auto_hash_map::AutoSet; +use auto_hash_map::{AutoMap, AutoSet}; use concurrent_queue::ConcurrentQueue; use dashmap::{mapref::entry::Entry, DashMap, DashSet}; use nohash_hasher::BuildNoHashHasher; @@ -28,7 +28,7 @@ use turbo_tasks::{ PersistedGraphApi, ReadTaskState, TaskCell, TaskData, }, util::{IdFactory, NoMoveVec, SharedError}, - CellId, RawVc, TaskId, TraitTypeId, TurboTasksBackendApi, Unused, Vc, + CellId, RawVc, TaskId, TraitTypeId, TurboTasksBackendApi, Unused, }; type RootTaskFn = @@ -1443,7 +1443,7 @@ impl Backend for MemoryBackendWithPersistedGraph

{ _trait_id: TraitTypeId, _reader: TaskId, _turbo_tasks: &dyn TurboTasksBackendApi>, - ) -> Vc> { + ) -> AutoMap { todo!() } @@ -1461,6 +1461,7 @@ impl Backend for MemoryBackendWithPersistedGraph

{ &self, _trait_id: TraitTypeId, _collectible: RawVc, + _count: u32, _task: TaskId, _turbo_tasks: &dyn TurboTasksBackendApi>, ) { @@ -1603,6 +1604,10 @@ impl Backend for MemoryBackendWithPersistedGraph

{ self.only_known_to_memory_tasks.insert(task); task } + + fn dispose_root_task(&self, _task: TaskId, _turbo_tasks: &dyn TurboTasksBackendApi) { + todo!() + } } struct MemoryBackendPersistedGraphApi<'a, P: PersistedGraph + 'static> { diff --git a/crates/turbo-tasks-memory/src/priority_pair.rs b/crates/turbo-tasks-memory/src/priority_pair.rs deleted file mode 100644 index f9f2a029f14d6..0000000000000 --- a/crates/turbo-tasks-memory/src/priority_pair.rs +++ /dev/null @@ -1,41 +0,0 @@ -use std::{ - future::Future, - sync::atomic::{AtomicUsize, Ordering}, -}; - -use turbo_tasks::event::Event; - -/// A pair of two priorities which allows to run the higher priority task first. -pub struct PriorityPair { - current: AtomicUsize, - event: Event, -} - -impl PriorityPair { - pub fn new() -> Self { - Self { - current: AtomicUsize::new(0), - event: Event::new(|| "PriorityPair::event".to_string()), - } - } - - pub fn start_high(&self) { - self.current.fetch_add(1, Ordering::Release); - } - - pub fn finish_high(&self) { - if self.current.fetch_sub(1, Ordering::Release) == 1 { - self.event.notify(usize::MAX); - } - } - - pub async fn run_low>(&self, f: F) -> T { - while self.current.load(Ordering::Acquire) != 0 { - let listener = self.event.listen(); - if self.current.load(Ordering::Acquire) != 0 { - listener.await; - } - } - f.await - } -} diff --git a/crates/turbo-tasks-memory/src/scope.rs b/crates/turbo-tasks-memory/src/scope.rs deleted file mode 100644 index d26e31327b9f4..0000000000000 --- a/crates/turbo-tasks-memory/src/scope.rs +++ /dev/null @@ -1,775 +0,0 @@ -use std::{ - fmt::{Debug, Display}, - hash::Hash, - mem::take, - ops::Deref, - sync::atomic::{AtomicIsize, AtomicUsize, Ordering}, -}; - -use auto_hash_map::{map::Entry, AutoMap, AutoSet}; -use nohash_hasher::BuildNoHashHasher; -use parking_lot::Mutex; -use turbo_tasks::{ - event::{Event, EventListener}, - RawVc, TaskId, TraitTypeId, -}; - -use crate::{ - count_hash_set::{CountHashSet, CountHashSetIter}, - task::{Task, TaskDependency}, - MemoryBackend, -}; - -macro_rules! log_scope_update { - ($($args:expr),+) => { - #[cfg(feature = "print_scope_updates")] - println!($($args),+); - }; -} - -#[derive(Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct TaskScopeId { - id: usize, -} - -impl Display for TaskScopeId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "TaskScopeId {}", self.id) - } -} - -impl Debug for TaskScopeId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "TaskScopeId {}", self.id) - } -} - -impl Deref for TaskScopeId { - type Target = usize; - - fn deref(&self) -> &Self::Target { - &self.id - } -} - -impl From for TaskScopeId { - fn from(id: usize) -> Self { - Self { id } - } -} - -impl nohash_hasher::IsEnabled for TaskScopeId {} - -#[derive(Clone, Debug)] -pub enum TaskScopes { - Root(TaskScopeId), - /// inner scopes and a counter for changes to start optimized when a - /// threshold is reached - Inner( - CountHashSet>, - usize, - ), -} - -impl Default for TaskScopes { - fn default() -> Self { - TaskScopes::Inner(CountHashSet::default(), 0) - } -} - -impl TaskScopes { - pub fn iter(&self) -> TaskScopesIterator { - match self { - TaskScopes::Root(r) => TaskScopesIterator::Root(*r), - TaskScopes::Inner(set, _) => TaskScopesIterator::Inner(set.iter()), - } - } - - pub fn is_root(&self) -> bool { - matches!(self, TaskScopes::Root(_)) - } -} - -pub enum TaskScopesIterator<'a> { - Done, - Root(TaskScopeId), - Inner(CountHashSetIter<'a, TaskScopeId>), -} - -impl<'a> Iterator for TaskScopesIterator<'a> { - type Item = TaskScopeId; - - fn next(&mut self) -> Option { - match self { - TaskScopesIterator::Done => None, - &mut TaskScopesIterator::Root(id) => { - *self = TaskScopesIterator::Done; - Some(id) - } - TaskScopesIterator::Inner(it) => it.next().copied(), - } - } -} - -#[derive(Debug)] -pub struct TaskScope { - #[cfg(feature = "print_scope_updates")] - pub id: TaskScopeId, - /// If true, this scope will propagate collectibles to parent scopes - propagate_collectibles: bool, - /// Total number of tasks - tasks: AtomicUsize, - /// Number of tasks that are not Done, unfinished child scopes also count as - /// unfinished tasks. This value might temporarly become negative in race - /// conditions. - /// When this value crosses the 0 to 1 boundary, we need to look into - /// potentially updating [TaskScopeState]::has_unfinished_tasks with a mutex - /// lock. [TaskScopeState]::has_unfinished_tasks is the real truth, if a - /// task scope has unfinished tasks. - unfinished_tasks: AtomicIsize, - /// State that requires locking - pub state: Mutex, -} - -#[derive(Debug, Default)] -struct ScopeCollectiblesInfo { - collectibles: CountHashSet, - dependent_tasks: AutoSet>, - read_collectibles_task: Option, -} - -impl ScopeCollectiblesInfo { - fn is_unset(&self) -> bool { - self.collectibles.is_unset() - && self.dependent_tasks.is_empty() - && self.read_collectibles_task.is_none() - } -} - -#[derive(Debug)] -pub struct TaskScopeState { - #[cfg(feature = "print_scope_updates")] - pub id: TaskScopeId, - /// Number of active parents or tasks. Non-zero value means the scope is - /// active - active: isize, - /// When not active, this list contains all dirty tasks. - /// When the scope becomes active, these need to be scheduled. - dirty_tasks: AutoSet>, - /// All child scopes, when the scope becomes active, child scopes need to - /// become active too - children: CountHashSet>, - /// flag if this scope has unfinished tasks - has_unfinished_tasks: bool, - /// Event that will be notified when all unfinished tasks and children are - /// done - event: Event, - /// All parent scopes - pub parents: CountHashSet>, - /// Tasks that have read children - /// When they change these tasks are invalidated - dependent_tasks: AutoSet>, - /// Emitted collectibles with count and dependent_tasks by trait type - collectibles: AutoMap>, -} - -impl TaskScope { - #[allow(unused_variables)] - pub fn new(id: TaskScopeId, tasks: usize) -> Self { - Self { - #[cfg(feature = "print_scope_updates")] - id, - propagate_collectibles: true, - tasks: AtomicUsize::new(tasks), - unfinished_tasks: AtomicIsize::new(0), - state: Mutex::new(TaskScopeState::new( - #[cfg(feature = "print_scope_updates")] - id, - false, - )), - } - } - - #[allow(unused_variables)] - pub fn new_no_collectibles(id: TaskScopeId, tasks: usize) -> Self { - Self { - #[cfg(feature = "print_scope_updates")] - id, - propagate_collectibles: false, - tasks: AtomicUsize::new(tasks), - unfinished_tasks: AtomicIsize::new(tasks as isize), - state: Mutex::new(TaskScopeState::new( - #[cfg(feature = "print_scope_updates")] - id, - tasks > 0, - )), - } - } - - #[allow(unused_variables)] - pub fn new_active(id: TaskScopeId, tasks: usize, unfinished: usize) -> Self { - Self { - #[cfg(feature = "print_scope_updates")] - id, - propagate_collectibles: true, - tasks: AtomicUsize::new(tasks), - unfinished_tasks: AtomicIsize::new(unfinished as isize), - state: Mutex::new(TaskScopeState::new_active( - #[cfg(feature = "print_scope_updates")] - id, - tasks > 0, - )), - } - } - - pub fn increment_tasks(&self) { - self.tasks.fetch_add(1, Ordering::Relaxed); - } - - pub fn decrement_tasks(&self) { - self.tasks.fetch_sub(1, Ordering::Relaxed); - } - - pub fn increment_unfinished_tasks(&self, backend: &MemoryBackend) { - if self.increment_unfinished_tasks_internal() { - self.update_unfinished_state(backend); - } - } - - /// Returns true if the state requires an update - #[must_use] - fn increment_unfinished_tasks_internal(&self) -> bool { - // crossing the 0 to 1 boundary requires an update - // SAFETY: This need to sync with the unfinished_tasks load in - // update_unfinished_state - self.unfinished_tasks.fetch_add(1, Ordering::Release) == 0 - } - - pub fn decrement_unfinished_tasks(&self, backend: &MemoryBackend) { - if self.decrement_unfinished_tasks_internal() { - self.update_unfinished_state(backend); - } - } - - /// Returns true if the state requires an update - #[must_use] - fn decrement_unfinished_tasks_internal(&self) -> bool { - // crossing the 0 to 1 boundary requires an update - // SAFETY: This need to sync with the unfinished_tasks load in - // update_unfinished_state - self.unfinished_tasks.fetch_sub(1, Ordering::Release) == 1 - } - - pub fn add_parent(&self, parent: TaskScopeId, backend: &MemoryBackend) { - { - let mut state = self.state.lock(); - if !state.parents.add(parent) || !state.has_unfinished_tasks { - return; - } - }; - // As we added a parent while having unfinished tasks we need to increment the - // unfinished task count and potentially update the state - backend.with_scope(parent, |parent| { - let update = parent.increment_unfinished_tasks_internal(); - - if update { - parent.update_unfinished_state(backend); - } - }); - } - - /// Removes a parent from this scope, returns true if the scope parents are - /// now empty and unset - pub fn remove_parent(&self, parent: TaskScopeId, backend: &MemoryBackend) -> bool { - let result = { - let mut state = self.state.lock(); - if !state.parents.remove(parent) || !state.has_unfinished_tasks { - return state.parents.is_unset(); - } - state.parents.is_unset() - }; - // As we removed a parent while having unfinished tasks we need to decrement the - // unfinished task count and potentially update the state - backend.with_scope(parent, |parent| { - let update = parent.decrement_unfinished_tasks_internal(); - - if update { - parent.update_unfinished_state(backend); - } - }); - result - } - - fn update_unfinished_state(&self, backend: &MemoryBackend) { - let mut state = self.state.lock(); - // we need to load the atomic under the lock to ensure consistency - let count = self.unfinished_tasks.load(Ordering::SeqCst); - let has_unfinished_tasks = count > 0; - let mut to_update = Vec::new(); - if state.has_unfinished_tasks != has_unfinished_tasks { - state.has_unfinished_tasks = has_unfinished_tasks; - if has_unfinished_tasks { - to_update.extend(state.parents.iter().copied().filter(|parent| { - backend.with_scope(*parent, |scope| scope.increment_unfinished_tasks_internal()) - })); - } else { - state.event.notify(usize::MAX); - to_update.extend(state.parents.iter().copied().filter(|parent| { - backend.with_scope(*parent, |scope| scope.decrement_unfinished_tasks_internal()) - })); - } - } - drop(state); - - for scope in to_update { - backend.with_scope(scope, |scope| scope.update_unfinished_state(backend)); - } - } - - pub fn has_unfinished_tasks(&self) -> Option { - let state = self.state.lock(); - if state.has_unfinished_tasks { - Some(state.event.listen()) - } else { - None - } - } - - pub fn read_collectibles_and_children( - &self, - self_id: TaskScopeId, - trait_id: TraitTypeId, - reader: TaskId, - ) -> Result<(CountHashSet, Vec), EventListener> { - let mut state = self.state.lock(); - if state.has_unfinished_tasks { - return Err(state.event.listen()); - } - let children = state.children.iter().copied().collect::>(); - state.dependent_tasks.insert(reader); - Task::add_dependency_to_current(TaskDependency::ScopeChildren(self_id)); - - let current = { - let ScopeCollectiblesInfo { - collectibles, - dependent_tasks, - .. - } = state.collectibles.entry(trait_id).or_default(); - dependent_tasks.insert(reader); - Task::add_dependency_to_current(TaskDependency::ScopeCollectibles(self_id, trait_id)); - collectibles.clone() - }; - drop(state); - - Ok((current, children)) - } - - pub(crate) fn remove_dependent_task(&self, reader: TaskId) { - let mut state = self.state.lock(); - state.dependent_tasks.remove(&reader); - } - - pub(crate) fn remove_collectible_dependent_task( - &self, - trait_type: TraitTypeId, - reader: TaskId, - ) { - let mut state = self.state.lock(); - if let Entry::Occupied(mut entry) = state.collectibles.entry(trait_type) { - let info = entry.get_mut(); - info.dependent_tasks.remove(&reader); - if info.is_unset() { - entry.remove(); - } - } - } - - pub(crate) fn is_propagating_collectibles(&self) -> bool { - self.propagate_collectibles - } - - pub(crate) fn assert_unused(&self) { - // This method checks if everything was cleaned up correctly - // no more tasks should be attached to this scope in any way - - assert_eq!( - self.tasks.load(Ordering::Acquire), - 0, - "Scope tasks not correctly cleaned up" - ); - assert_eq!( - self.unfinished_tasks.load(Ordering::Acquire), - 0, - "Scope unfinished tasks not correctly cleaned up" - ); - let state = self.state.lock(); - assert!( - state.dependent_tasks.is_empty(), - "Scope dependent tasks not correctly cleaned up: {:?}", - state.dependent_tasks - ); - // TODO(WEB-615) read_collectibles_tasks need to be cleaned up - // assert!( - // state.collectibles.is_empty(), - // "Scope collectibles not correctly cleaned up: {:?}", - // state.collectibles - // ); - // assert!( - // state.dirty_tasks.is_empty(), - // "Scope dirty tasks not correctly cleaned up: {:?}", - // state.dirty_tasks - // ); - // TODO find the bug that causes dirty tasks to remain in the scope - if !state.dirty_tasks.is_empty() { - println!( - "Scope dirty tasks not correctly cleaned up: {:?}", - state.dirty_tasks - ); - } - assert!( - state.children.is_empty(), - "Scope children not correctly cleaned up: {:?}", - state.children - ); - assert!( - state.parents.is_empty(), - "Scope parents not correctly cleaned up: {:?}", - state.parents - ); - assert!( - !state.has_unfinished_tasks, - "Scope has unfinished tasks not correctly cleaned up" - ); - assert_eq!( - state.active, 0, - "Scope active not correctly cleaned up: {}", - state.active - ); - } -} - -pub struct ScopeChildChangeEffect { - pub notify: AutoSet>, - pub active: bool, - /// `true` when the child to parent relationship needs to be updated - pub parent: bool, -} - -pub struct ScopeCollectibleChangeEffect { - pub notify: AutoSet>, -} - -impl TaskScopeState { - /// creates a state that is not active - fn new( - #[cfg(feature = "print_scope_updates")] id: TaskScopeId, - has_unfinished_tasks: bool, - ) -> Self { - Self { - #[cfg(feature = "print_scope_updates")] - id, - active: 0, - dirty_tasks: AutoSet::default(), - children: CountHashSet::new(), - collectibles: AutoMap::default(), - dependent_tasks: AutoSet::default(), - event: Event::new(move || { - #[cfg(feature = "print_scope_updates")] - return format!("TaskScope({id})::event"); - #[cfg(not(feature = "print_scope_updates"))] - return "TaskScope::event".to_owned(); - }), - has_unfinished_tasks, - parents: CountHashSet::new(), - } - } - - /// creates a state that is active - fn new_active( - #[cfg(feature = "print_scope_updates")] id: TaskScopeId, - has_unfinished_tasks: bool, - ) -> Self { - Self { - #[cfg(feature = "print_scope_updates")] - id, - active: 1, - dirty_tasks: AutoSet::default(), - children: CountHashSet::new(), - collectibles: AutoMap::default(), - dependent_tasks: AutoSet::default(), - event: Event::new(move || { - #[cfg(feature = "print_scope_updates")] - return format!("TaskScope({id})::event"); - #[cfg(not(feature = "print_scope_updates"))] - return "TaskScope::event".to_owned(); - }), - has_unfinished_tasks, - parents: CountHashSet::new(), - } - } - - /// returns true if the scope is active - pub fn is_active(&self) -> bool { - self.active > 0 - } - - /// increments the active counter, returns list of tasks that need to be - /// scheduled and list of child scope that need to be incremented after - /// releasing the scope lock - #[must_use] - pub fn increment_active( - &mut self, - more_jobs: &mut Vec, - ) -> Option>> { - self.increment_active_by(1, more_jobs) - } - - /// increments the active counter, returns list of tasks that need to be - /// scheduled and list of child scope that need to be incremented after - /// releasing the scope lock - #[must_use] - pub fn increment_active_by( - &mut self, - count: usize, - more_jobs: &mut Vec, - ) -> Option>> { - let was_zero = self.active <= 0; - self.active += count as isize; - if self.active > 0 && was_zero { - more_jobs.extend(self.children.iter().copied()); - Some(take(&mut self.dirty_tasks)) - } else { - None - } - } - - /// decrement the active counter, returns list of child scopes that need to - /// be decremented after releasing the scope lock - pub fn decrement_active(&mut self, more_jobs: &mut Vec) { - self.decrement_active_by(1, more_jobs); - } - - /// decrement the active counter, returns list of child scopes that need to - /// be decremented after releasing the scope lock. Returns `true` when the - /// scope has become inactive. - pub fn decrement_active_by(&mut self, count: usize, more_jobs: &mut Vec) -> bool { - let was_positive = self.active > 0; - self.active -= count as isize; - if self.active <= 0 && was_positive { - more_jobs.extend(self.children.iter().copied()); - true - } else { - false - } - } - - /// Add a child scope. Returns a [ScopeChildChangeEffect] when the child - /// scope need to have its active counter increased. - #[must_use] - pub fn add_child(&mut self, child: TaskScopeId) -> Option { - self.add_child_count(child, 1) - } - - /// Add a child scope. Returns a [ScopeChildChangeEffect] when the child - /// scope need to have its active counter increased. - #[must_use] - pub fn add_child_count( - &mut self, - child: TaskScopeId, - count: usize, - ) -> Option { - if self.children.add_count(child, count) { - log_scope_update!("add_child {} -> {}", *self.id, *child); - Some(ScopeChildChangeEffect { - notify: self.take_dependent_tasks(), - active: self.active > 0, - parent: true, - }) - } else { - None - } - } - - /// Removes a child scope. Returns true, when the child scope need to have - /// it's active counter decreased. - #[must_use] - pub fn remove_child(&mut self, child: TaskScopeId) -> Option { - self.remove_child_count(child, 1) - } - - /// Removes a child scope. Returns true, when the child scope need to have - /// it's active counter decreased. - #[must_use] - pub fn remove_child_count( - &mut self, - child: TaskScopeId, - count: usize, - ) -> Option { - if self.children.remove_count(child, count) { - log_scope_update!("remove_child {} -> {}", *self.id, *child); - Some(ScopeChildChangeEffect { - notify: self.take_dependent_tasks(), - active: self.active > 0, - parent: true, - }) - } else { - None - } - } - - pub fn add_dirty_task(&mut self, id: TaskId) { - self.dirty_tasks.insert(id); - log_scope_update!("add_dirty_task {} -> {}", *self.id, *id); - } - - pub fn remove_dirty_task(&mut self, id: TaskId) { - self.dirty_tasks.remove(&id); - log_scope_update!("remove_dirty_task {} -> {}", *self.id, *id); - } - - /// Takes all children or collectibles dependent tasks and returns them for - /// notification. - pub fn take_all_dependent_tasks(&mut self) -> AutoSet> { - let mut set = self.take_dependent_tasks(); - self.collectibles = take(&mut self.collectibles) - .into_iter() - .map(|(key, mut info)| { - set.extend(take(&mut info.dependent_tasks)); - (key, info) - }) - .filter(|(_, info)| !info.is_unset()) - .collect(); - set - } - - /// Adds a colletible to the scope. - /// Returns true when it was initially added and dependent_tasks should be - /// notified. - #[must_use] - pub fn add_collectible( - &mut self, - trait_id: TraitTypeId, - collectible: RawVc, - ) -> Option { - self.add_collectible_count(trait_id, collectible, 1) - } - - /// Adds a colletible to the scope. - /// Returns true when it was initially added and dependent_tasks should be - /// notified. - #[must_use] - pub fn add_collectible_count( - &mut self, - trait_id: TraitTypeId, - collectible: RawVc, - count: usize, - ) -> Option { - match self.collectibles.entry(trait_id) { - Entry::Occupied(mut entry) => { - let info = entry.get_mut(); - if info.collectibles.add_count(collectible, count) { - log_scope_update!("add_collectible {} -> {}", *self.id, collectible); - Some(ScopeCollectibleChangeEffect { - notify: take(&mut info.dependent_tasks), - }) - } else { - if info.is_unset() { - entry.remove(); - } - None - } - } - Entry::Vacant(entry) => { - let result = entry - .insert(Default::default()) - .collectibles - .add_count(collectible, count); - debug_assert!(result, "this must be always a new entry"); - log_scope_update!("add_collectible {} -> {}", *self.id, collectible); - Some(ScopeCollectibleChangeEffect { - notify: AutoSet::default(), - }) - } - } - } - - /// Removes a colletible from the scope. - /// Returns true when is was fully removed and dependent_tasks should be - /// notified. - #[must_use] - pub fn remove_collectible( - &mut self, - trait_id: TraitTypeId, - collectible: RawVc, - ) -> Option { - self.remove_collectible_count(trait_id, collectible, 1) - } - - /// Removes a colletible from the scope. - /// Returns true when is was fully removed and dependent_tasks should be - /// notified. - #[must_use] - pub fn remove_collectible_count( - &mut self, - trait_id: TraitTypeId, - collectible: RawVc, - count: usize, - ) -> Option { - match self.collectibles.entry(trait_id) { - Entry::Occupied(mut entry) => { - let info = entry.get_mut(); - let old_value = info.collectibles.get(&collectible); - let new_value = old_value - count as isize; - // NOTE: The read_collectibles need to be invalidated when negative count - // changes. Each negative count will eliminate one child scope emitted - // collectible. So changing from -1 to -2 might affect the visible collectibles. - if info.collectibles.remove_count(collectible, count) || new_value < 0 { - let notify = take(&mut info.dependent_tasks); - if info.is_unset() { - entry.remove(); - } - log_scope_update!( - "remove_collectible {} -> {} ({old_value} -> {new_value})", - *self.id, - collectible - ); - Some(ScopeCollectibleChangeEffect { notify }) - } else { - None - } - } - Entry::Vacant(e) => { - let result = e - .insert(Default::default()) - .collectibles - .remove_count(collectible, count); - - debug_assert!(!result, "this must never be visible from outside"); - None - } - } - } - - pub fn take_dependent_tasks(&mut self) -> AutoSet> { - take(&mut self.dependent_tasks) - } - - pub fn get_read_collectibles_task( - &mut self, - trait_id: TraitTypeId, - create_new: impl FnOnce() -> TaskId, - ) -> TaskId { - let task_id = &mut self - .collectibles - .entry(trait_id) - .or_default() - .read_collectibles_task; - if let Some(task_id) = *task_id { - task_id - } else { - let new_task_id = create_new(); - *task_id = Some(new_task_id); - new_task_id - } - } -} diff --git a/crates/turbo-tasks-memory/src/stats.rs b/crates/turbo-tasks-memory/src/stats.rs index 2072ab989f6e4..83e65d456fd77 100644 --- a/crates/turbo-tasks-memory/src/stats.rs +++ b/crates/turbo-tasks-memory/src/stats.rs @@ -11,24 +11,22 @@ use std::{ use turbo_tasks::{registry, FunctionId, TaskId, TraitTypeId}; use crate::{ - scope::TaskScopeId, task::{Task, TaskStatsInfo}, MemoryBackend, }; pub struct StatsReferences { pub tasks: Vec<(ReferenceType, TaskId)>, - pub scopes: Vec<(ReferenceType, TaskScopeId)>, } #[derive(PartialEq, Eq, Hash, Clone, Debug)] pub enum StatsTaskType { Root(TaskId), Once(TaskId), - ReadCollectibles(TraitTypeId), Native(FunctionId), ResolveNative(FunctionId), ResolveTrait(TraitTypeId, String), + Collectibles(TraitTypeId), } impl Display for StatsTaskType { @@ -36,7 +34,7 @@ impl Display for StatsTaskType { match self { StatsTaskType::Root(_) => write!(f, "root"), StatsTaskType::Once(_) => write!(f, "once"), - StatsTaskType::ReadCollectibles(t) => { + StatsTaskType::Collectibles(t) => { write!(f, "read collectibles {}", registry::get_trait(*t).name) } StatsTaskType::Native(nf) => write!(f, "{}", registry::get_function(*nf).name), @@ -65,11 +63,8 @@ pub enum ReferenceType { #[derive(Clone, Debug)] pub struct ExportedTaskStats { pub count: usize, - pub active_count: usize, pub unloaded_count: usize, pub executions: Option, - pub roots: usize, - pub scopes: usize, pub total_duration: Option, pub total_current_duration: Duration, pub total_update_duration: Duration, @@ -81,11 +76,8 @@ impl Default for ExportedTaskStats { fn default() -> Self { Self { count: 0, - active_count: 0, unloaded_count: 0, executions: None, - roots: 0, - scopes: 0, total_duration: None, total_current_duration: Duration::ZERO, total_update_duration: Duration::ZERO, @@ -131,16 +123,10 @@ impl Stats { total_duration, last_duration, executions, - root_scoped, - child_scopes, - active, unloaded, } = info; let stats = self.tasks.entry(ty).or_default(); stats.count += 1; - if active { - stats.active_count += 1 - } if let Some(total_duration) = total_duration { *stats.total_duration.get_or_insert(Duration::ZERO) += total_duration; } @@ -155,10 +141,6 @@ impl Stats { if let Some(executions) = executions { *stats.executions.get_or_insert(0) += executions; } - if root_scoped { - stats.roots += 1; - } - stats.scopes += child_scopes; let StatsReferences { tasks, .. } = task.get_stats_references(); let set: HashSet<_> = tasks.into_iter().collect(); @@ -193,7 +175,7 @@ impl Stats { StatsTaskType::Root(_) | StatsTaskType::Once(_) | StatsTaskType::Native(_) - | StatsTaskType::ReadCollectibles(..) => false, + | StatsTaskType::Collectibles(..) => false, StatsTaskType::ResolveNative(_) | StatsTaskType::ResolveTrait(_, _) => true, }) } diff --git a/crates/turbo-tasks-memory/src/task.rs b/crates/turbo-tasks-memory/src/task.rs index e0468f7ac9821..3af1ca1c5b22c 100644 --- a/crates/turbo-tasks-memory/src/task.rs +++ b/crates/turbo-tasks-memory/src/task.rs @@ -1,13 +1,14 @@ +mod aggregation; mod meta_state; mod stats; use std::{ borrow::Cow, cell::RefCell, - cmp::{max, Ordering, Reverse}, - collections::{HashMap, HashSet, VecDeque}, + cmp::{max, Reverse}, + collections::{HashMap, HashSet}, fmt::{ - Debug, Display, Formatter, Write, {self}, + Debug, Display, Formatter, {self}, }, future::Future, hash::Hash, @@ -27,56 +28,37 @@ use turbo_tasks::{ backend::{PersistentTaskType, TaskExecutionSpec}, event::{Event, EventListener}, get_invalidator, registry, CellId, Invalidator, RawVc, StatsType, TaskId, TraitTypeId, - TryJoinIterExt, TurboTasksBackendApi, ValueTypeId, Vc, + TurboTasksBackendApi, ValueTypeId, }; use crate::{ + aggregation_tree::{aggregation_info, ensure_thresholds}, cell::Cell, - count_hash_set::CountHashSet, gc::{to_exp_u8, GcPriority, GcStats, GcTaskState}, - memory_backend::Job, output::{Output, OutputContent}, - scope::{ScopeChildChangeEffect, TaskScopeId, TaskScopes}, stats::{ReferenceType, StatsReferences, StatsTaskType}, + task::aggregation::{TaskAggregationContext, TaskChange}, MemoryBackend, }; pub type NativeTaskFuture = Pin> + Send>>; pub type NativeTaskFn = Box NativeTaskFuture + Send + Sync>; -macro_rules! log_scope_update { - ($($args:expr),+) => { - #[cfg(feature = "print_scope_updates")] - println!($($args),+); - }; -} - #[derive(Hash, Copy, Clone, PartialEq, Eq)] pub enum TaskDependency { - TaskOutput(TaskId), - TaskCell(TaskId, CellId), - ScopeChildren(TaskScopeId), - ScopeCollectibles(TaskScopeId, TraitTypeId), + Output(TaskId), + Cell(TaskId, CellId), + Collectibles(TaskId, TraitTypeId), } task_local! { - /// Vc/Scopes that are read during task execution + /// Cells/Outputs/Collectibles that are read during task execution /// These will be stored as dependencies when the execution has finished pub(crate) static DEPENDENCIES_TO_TRACK: RefCell>; } type OnceTaskFn = Mutex> + Send + 'static>>>>; -struct ReadTaskCollectiblesTaskType { - task: TaskId, - trait_type: TraitTypeId, -} - -struct ReadScopeCollectiblesTaskType { - scope: TaskScopeId, - trait_type: TraitTypeId, -} - /// Different Task types enum TaskType { // Note: double boxed to reduce TaskType size @@ -93,26 +75,13 @@ enum TaskType { /// applied. Once(Box), - /// A task that reads all collectibles of a certain trait from a - /// [TaskScope]. It will do that by recursively calling - /// ReadScopeCollectibles on child scopes, so that results by scope are - /// cached. - ReadScopeCollectibles(Box), - - /// A task that reads all collectibles of a certain trait from another task. - /// It will do that by recursively calling ReadScopeCollectibles on child - /// scopes, so that results by task are cached. - ReadTaskCollectibles(Box), - /// A normal persistent task - Persistent(Arc), + Persistent { ty: Arc }, } enum TaskTypeForDescription { Root, Once, - ReadTaskCollectibles(TraitTypeId), - ReadScopeCollectibles(TraitTypeId), Persistent(Arc), } @@ -121,14 +90,7 @@ impl TaskTypeForDescription { match task_type { TaskType::Root(..) => Self::Root, TaskType::Once(..) => Self::Once, - TaskType::ReadTaskCollectibles(box ReadTaskCollectiblesTaskType { - trait_type, .. - }) => Self::ReadTaskCollectibles(*trait_type), - TaskType::ReadScopeCollectibles(box ReadScopeCollectiblesTaskType { - trait_type, - .. - }) => Self::ReadScopeCollectibles(*trait_type), - TaskType::Persistent(ty) => Self::Persistent(ty.clone()), + TaskType::Persistent { ty, .. } => Self::Persistent(ty.clone()), } } } @@ -138,20 +100,7 @@ impl Debug for TaskType { match self { Self::Root(..) => f.debug_tuple("Root").finish(), Self::Once(..) => f.debug_tuple("Once").finish(), - Self::ReadScopeCollectibles(box ReadScopeCollectiblesTaskType { - scope, - trait_type, - }) => f - .debug_tuple("ReadScopeCollectibles") - .field(scope) - .field(®istry::get_trait(*trait_type).name) - .finish(), - Self::ReadTaskCollectibles(box ReadTaskCollectiblesTaskType { task, trait_type }) => f - .debug_tuple("ReadTaskCollectibles") - .field(task) - .field(®istry::get_trait(*trait_type).name) - .finish(), - Self::Persistent(ty) => Debug::fmt(ty, f), + Self::Persistent { ty, .. } => Debug::fmt(ty, f), } } } @@ -161,9 +110,7 @@ impl Display for TaskType { match self { Self::Root(..) => f.debug_tuple("Root").finish(), Self::Once(..) => f.debug_tuple("Once").finish(), - Self::ReadTaskCollectibles(..) => f.debug_tuple("ReadTaskCollectibles").finish(), - Self::ReadScopeCollectibles(..) => f.debug_tuple("ReadScopeCollectibles").finish(), - Self::Persistent(ty) => Display::fmt(ty, f), + Self::Persistent { ty, .. } => Display::fmt(ty, f), } } } @@ -211,7 +158,7 @@ impl Debug for Task { /// The full state of a [Task], it includes all information. struct TaskState { - scopes: TaskScopes, + aggregation_leaf: TaskAggregationTreeLeaf, // TODO using a Atomic might be possible here /// More flags of task state, where not all combinations are possible. @@ -247,7 +194,7 @@ impl TaskState { stats_type: StatsType, ) -> Self { Self { - scopes: Default::default(), + aggregation_leaf: TaskAggregationTreeLeaf::new(), state_type: Dirty { event: Event::new(move || format!("TaskState({})::event", description())), }, @@ -264,13 +211,12 @@ impl TaskState { } } - fn new_scheduled_in_scope( + fn new_scheduled( description: impl Fn() -> String + Send + Sync + 'static, - scope: TaskScopeId, stats_type: StatsType, ) -> Self { Self { - scopes: TaskScopes::Inner(CountHashSet::from([scope]), 0), + aggregation_leaf: TaskAggregationTreeLeaf::new(), state_type: Scheduled { event: Event::new(move || format!("TaskState({})::event", description())), }, @@ -286,45 +232,22 @@ impl TaskState { last_waiting_task: Default::default(), } } - - fn new_root_scoped( - description: impl Fn() -> String + Send + Sync + 'static, - scope: TaskScopeId, - stats_type: StatsType, - ) -> Self { - Self { - scopes: TaskScopes::Root(scope), - state_type: Dirty { - event: Event::new(move || format!("TaskState({})::event", description())), - }, - stateful: false, - children: Default::default(), - collectibles: Default::default(), - output: Default::default(), - prepared_type: PrepareTaskType::None, - cells: Default::default(), - gc: Default::default(), - stats: TaskStats::new(stats_type), - #[cfg(feature = "track_wait_dependencies")] - last_waiting_task: Default::default(), - } - } } /// The partial task state. It's equal to a full TaskState with state = Dirty /// and all other fields empty. It looks like a dirty task that has not been -/// executed yet. The task might still be in some task scopes. +/// executed yet. The task might still referenced by some parents tasks. /// A Task can get into this state when it is unloaded by garbage collection, -/// but is still attached to scopes. +/// but is still attached to parents and aggregated. struct PartialTaskState { stats_type: StatsType, - scopes: TaskScopes, + aggregation_leaf: TaskAggregationTreeLeaf, } impl PartialTaskState { fn into_full(self, description: impl Fn() -> String + Send + Sync + 'static) -> TaskState { TaskState { - scopes: self.scopes, + aggregation_leaf: self.aggregation_leaf, state_type: Dirty { event: Event::new(move || format!("TaskState({})::event", description())), }, @@ -341,7 +264,7 @@ impl PartialTaskState { } /// A fully unloaded task state. It's equal to a partial task state without -/// being attached to any scopes. This state is stored inlined instead of in a +/// being referenced by any parent. This state is stored inlined instead of in a /// [Box] to reduce the memory consumption. Make sure to not add more fields /// than the size of a [Box]. struct UnloadedTaskState { @@ -357,7 +280,7 @@ fn test_unloaded_task_state_size() { impl UnloadedTaskState { fn into_full(self, description: impl Fn() -> String + Send + Sync + 'static) -> TaskState { TaskState { - scopes: Default::default(), + aggregation_leaf: TaskAggregationTreeLeaf::new(), state_type: Dirty { event: Event::new(move || format!("TaskState({})::event", description())), }, @@ -374,12 +297,15 @@ impl UnloadedTaskState { fn into_partial(self) -> PartialTaskState { PartialTaskState { - scopes: TaskScopes::Inner(CountHashSet::new(), 0), + aggregation_leaf: TaskAggregationTreeLeaf::new(), stats_type: self.stats_type, } } } +/// The collectibles of a task. +type Collectibles = AutoMap<(TraitTypeId, RawVc), i32>; + /// Keeps track of emitted and unemitted collectibles and the /// read_collectibles tasks. Defaults to None to avoid allocating memory when no /// collectibles are emitted or read. @@ -388,26 +314,10 @@ struct MaybeCollectibles { inner: Option>, } -/// The collectibles of a task. -#[derive(Default)] -struct Collectibles { - emitted: AutoSet<(TraitTypeId, RawVc)>, - unemitted: AutoSet<(TraitTypeId, RawVc)>, - read_collectibles_tasks: AutoMap, -} - impl MaybeCollectibles { /// Consumes the collectibles (if any) and return them. fn take_collectibles(&mut self) -> Option { - if let Some(inner) = &mut self.inner { - Some(Collectibles { - emitted: take(&mut inner.emitted), - unemitted: take(&mut inner.unemitted), - read_collectibles_tasks: AutoMap::default(), - }) - } else { - None - } + self.inner.as_mut().map(|boxed| take(&mut **boxed)) } /// Consumes the collectibles (if any) and return them. @@ -425,32 +335,23 @@ impl MaybeCollectibles { } /// Emits a collectible. - fn emit(&mut self, trait_type: TraitTypeId, value: RawVc) -> bool { - self.inner + fn emit(&mut self, trait_type: TraitTypeId, value: RawVc) { + let value = self + .inner .get_or_insert_default() - .emitted - .insert((trait_type, value)) + .entry((trait_type, value)) + .or_default(); + *value += 1; } /// Unemits a collectible. - fn unemit(&mut self, trait_type: TraitTypeId, value: RawVc) -> bool { - self.inner - .get_or_insert_default() - .unemitted - .insert((trait_type, value)) - } - - pub fn get_read_collectibles_task( - &mut self, - trait_id: TraitTypeId, - create_new: impl FnOnce() -> TaskId, - ) -> TaskId { - *self + fn unemit(&mut self, trait_type: TraitTypeId, value: RawVc, count: u32) { + let value = self .inner .get_or_insert_default() - .read_collectibles_tasks - .entry(trait_id) - .or_insert_with(create_new) + .entry((trait_type, value)) + .or_default(); + *value -= count as i32; } } @@ -460,7 +361,7 @@ enum TaskStateType { /// on invalidation this will move to Dirty or Scheduled depending on active /// flag Done { - /// Cells/Scopes that the task has read during execution. + /// Cells/Outputs/Collectibles that the task has read during execution. /// The Task will keep these tasks alive as invalidations that happen /// there might affect this task. /// @@ -486,6 +387,9 @@ enum TaskStateType { InProgress { event: Event, count_as_finished: bool, + /// Children that need to be disconnected once leaving this state + outdated_children: AutoSet>, + outdated_collectibles: MaybeCollectibles, }, /// Invalid execution is happening @@ -496,34 +400,20 @@ enum TaskStateType { use TaskStateType::*; -use self::meta_state::{ - FullTaskWriteGuard, TaskMetaState, TaskMetaStateReadGuard, TaskMetaStateWriteGuard, +use self::{ + aggregation::{RootInfoType, RootType, TaskAggregationTreeLeaf, TaskGuard}, + meta_state::{ + FullTaskWriteGuard, TaskMetaState, TaskMetaStateReadGuard, TaskMetaStateWriteGuard, + }, }; -/// Heuristic when a task should switch to root scoped. -/// -/// The `optimization_counter` is a number how often a scope has been added to -/// this task (and therefore to all child tasks as well). We can assume that all -/// scopes might eventually be removed again. We assume that more scopes per -/// task have higher cost, so we want to avoid that. We assume that adding and -/// removing scopes again and again is not great. But having too many root -/// scopes is also not great as it hurts strongly consistent reads and read -/// collectibles. -/// -/// The current implementation uses a heuristic that says that the cost is -/// linear to the number of added scoped and linear to the number of children. -fn should_optimize_to_root_scoped(optimization_counter: usize, children_count: usize) -> bool { - const SCOPE_OPTIMIZATION_THRESHOLD: usize = 255; - optimization_counter * children_count > SCOPE_OPTIMIZATION_THRESHOLD -} - impl Task { pub(crate) fn new_persistent( id: TaskId, task_type: Arc, stats_type: StatsType, ) -> Self { - let ty = TaskType::Persistent(task_type); + let ty = TaskType::Persistent { ty: task_type }; let description = Self::get_event_description_static(id, &ty); Self { id, @@ -537,7 +427,6 @@ impl Task { pub(crate) fn new_root( id: TaskId, - scope: TaskScopeId, functor: impl Fn() -> NativeTaskFuture + Sync + Send + 'static, stats_type: StatsType, ) -> Self { @@ -546,15 +435,15 @@ impl Task { Self { id, ty, - state: RwLock::new(TaskMetaState::Full(Box::new( - TaskState::new_scheduled_in_scope(description, scope, stats_type), - ))), + state: RwLock::new(TaskMetaState::Full(Box::new(TaskState::new_scheduled( + description, + stats_type, + )))), } } pub(crate) fn new_once( id: TaskId, - scope: TaskScopeId, functor: impl Future> + Send + 'static, stats_type: StatsType, ) -> Self { @@ -563,68 +452,59 @@ impl Task { Self { id, ty, - state: RwLock::new(TaskMetaState::Full(Box::new( - TaskState::new_scheduled_in_scope(description, scope, stats_type), - ))), + state: RwLock::new(TaskMetaState::Full(Box::new(TaskState::new_scheduled( + description, + stats_type, + )))), + } + } + + pub(crate) fn is_pure(&self) -> bool { + match &self.ty { + TaskType::Persistent { .. } => true, + TaskType::Root(_) => false, + TaskType::Once(_) => false, } } - pub(crate) fn new_read_scope_collectibles( + pub(crate) fn set_root( id: TaskId, - target_scope: TaskScopeId, - trait_type_id: TraitTypeId, - stats_type: StatsType, - ) -> Self { - let ty = TaskType::ReadScopeCollectibles(Box::new(ReadScopeCollectiblesTaskType { - scope: target_scope, - trait_type: trait_type_id, - })); - let description = Self::get_event_description_static(id, &ty); - Self { - id, - ty, - state: RwLock::new(TaskMetaState::Full(Box::new(TaskState::new( - description, - stats_type, - )))), + backend: &MemoryBackend, + turbo_tasks: &dyn TurboTasksBackendApi, + ) { + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); + { + aggregation_context.aggregation_info(id).lock().root_type = Some(RootType::Root); } + aggregation_context.apply_queued_updates(); } - pub(crate) fn new_read_task_collectibles( + pub(crate) fn set_once( id: TaskId, - scope: TaskScopeId, - target_task: TaskId, - trait_type_id: TraitTypeId, - stats_type: StatsType, - ) -> Self { - let ty = TaskType::ReadTaskCollectibles(Box::new(ReadTaskCollectiblesTaskType { - task: target_task, - trait_type: trait_type_id, - })); - let description = Self::get_event_description_static(id, &ty); - Self { - id, - ty, - state: RwLock::new(TaskMetaState::Full(Box::new(TaskState::new_root_scoped( - description, - scope, - stats_type, - )))), + backend: &MemoryBackend, + turbo_tasks: &dyn TurboTasksBackendApi, + ) { + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); + { + aggregation_context.aggregation_info(id).lock().root_type = Some(RootType::Once); } + aggregation_context.apply_queued_updates(); } - pub(crate) fn is_pure(&self) -> bool { - match &self.ty { - TaskType::Persistent(_) => true, - TaskType::ReadTaskCollectibles(..) => true, - TaskType::ReadScopeCollectibles(..) => true, - TaskType::Root(_) => false, - TaskType::Once(_) => false, + pub(crate) fn unset_root( + id: TaskId, + backend: &MemoryBackend, + turbo_tasks: &dyn TurboTasksBackendApi, + ) { + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); + { + aggregation_context.aggregation_info(id).lock().root_type = None; } + aggregation_context.apply_queued_updates(); } pub(crate) fn get_function_name(&self) -> Option<&'static str> { - if let TaskType::Persistent(ty) = &self.ty { + if let TaskType::Persistent { ty, .. } = &self.ty { match &**ty { PersistentTaskType::Native(native_fn, _) | PersistentTaskType::ResolveNative(native_fn, _) => { @@ -644,16 +524,6 @@ impl Task { match ty { TaskTypeForDescription::Root => format!("[{}] root", id), TaskTypeForDescription::Once => format!("[{}] once", id), - TaskTypeForDescription::ReadTaskCollectibles(trait_type_id) => format!( - "[{}] read task collectibles({})", - id, - registry::get_trait(*trait_type_id).name - ), - TaskTypeForDescription::ReadScopeCollectibles(trait_type_id) => format!( - "[{}] read scope collectibles({})", - id, - registry::get_trait(*trait_type_id).name - ), TaskTypeForDescription::Persistent(ty) => match &**ty { PersistentTaskType::Native(native_fn, _) => { format!("[{}] {}", id, registry::get_function(*native_fn).name) @@ -689,42 +559,56 @@ impl Task { Self::get_event_description_static(self.id, &self.ty) } - pub(crate) fn remove_dependency(dep: TaskDependency, reader: TaskId, backend: &MemoryBackend) { + pub(crate) fn remove_dependency( + dep: TaskDependency, + reader: TaskId, + backend: &MemoryBackend, + turbo_tasks: &dyn TurboTasksBackendApi, + ) { match dep { - TaskDependency::TaskOutput(task) => { + TaskDependency::Output(task) => { backend.with_task(task, |task| { task.with_output_mut_if_available(|output| { output.dependent_tasks.remove(&reader); }); }); } - TaskDependency::TaskCell(task, index) => { + TaskDependency::Cell(task, index) => { backend.with_task(task, |task| { task.with_cell_mut_if_available(index, |cell| { cell.remove_dependent_task(reader); }); }); } - TaskDependency::ScopeChildren(scope) => backend.with_scope(scope, |scope| { - scope.remove_dependent_task(reader); - }), - TaskDependency::ScopeCollectibles(scope, trait_type) => { - backend.with_scope(scope, |scope| { - scope.remove_collectible_dependent_task(trait_type, reader); - }) + TaskDependency::Collectibles(task, trait_type) => { + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); + let aggregation = aggregation_context.aggregation_info(task); + aggregation + .lock() + .remove_collectible_dependent_task(trait_type, reader); } } } #[cfg(not(feature = "report_expensive"))] - fn clear_dependencies(&self, dependencies: AutoSet, backend: &MemoryBackend) { + fn clear_dependencies( + &self, + dependencies: AutoSet, + backend: &MemoryBackend, + turbo_tasks: &dyn TurboTasksBackendApi, + ) { for dep in dependencies.into_iter() { - Task::remove_dependency(dep, self.id, backend); + Task::remove_dependency(dep, self.id, backend, turbo_tasks); } } #[cfg(feature = "report_expensive")] - fn clear_dependencies(&self, dependencies: AutoSet, backend: &MemoryBackend) { + fn clear_dependencies( + &self, + dependencies: AutoSet, + backend: &MemoryBackend, + turbo_tasks: &dyn TurboTasksBackendApi, + ) { use std::time::Instant; use turbo_tasks::util::FormatDuration; @@ -733,7 +617,7 @@ impl Task { let count = dependencies.len(); for dep in dependencies.into_iter() { - Task::remove_dependency(dep, self.id, backend); + Task::remove_dependency(dep, self.id, backend, turbo_tasks); } let elapsed = start.elapsed(); if elapsed.as_millis() >= 10 || count > 10000 { @@ -772,68 +656,42 @@ impl Task { backend: &MemoryBackend, turbo_tasks: &dyn TurboTasksBackendApi, ) -> Option { + let future; let mut state = self.full_state_mut(); - if !self.try_start_execution(&mut state, turbo_tasks, backend) { - return None; - } - let future = self.make_execution_future(state, backend, turbo_tasks); - Some(TaskExecutionSpec { future }) - } - - /// Tries to change the state to InProgress and returns true if it was - /// possible. - fn try_start_execution( - &self, - state: &mut TaskState, - turbo_tasks: &dyn TurboTasksBackendApi, - backend: &MemoryBackend, - ) -> bool { match state.state_type { Done { .. } | InProgress { .. } | InProgressDirty { .. } => { // should not start in this state - return false; + return None; } Scheduled { ref mut event } => { + let event = event.take(); + let outdated_children = take(&mut state.children); + let outdated_collectibles = take(&mut state.collectibles); state.state_type = InProgress { - event: event.take(), + event, count_as_finished: false, + outdated_children, + outdated_collectibles, }; state.stats.increment_executions(); - // TODO we need to reconsider the approach of doing scope changes in background - // since they affect collectibles and need to be computed eagerly to allow - // strongly_consistent to work properly. - // We could move this operation to the point when the task execution is - // finished. - if !state.children.is_empty() { - let set = take(&mut state.children); - remove_from_scopes(set, &state.scopes, backend, turbo_tasks); - } - if let Some(collectibles) = state.collectibles.take_collectibles() { - remove_collectible_from_scopes( - collectibles.emitted, - collectibles.unemitted, - &state.scopes, - backend, - turbo_tasks, - ); - } } Dirty { .. } => { - let state_type = Task::state_string(&*state); + let state_type = Task::state_string(&state); panic!( "{:?} execution started in unexpected state {}", self, state_type ) } }; - true + future = self.make_execution_future(state, backend, turbo_tasks); + Some(TaskExecutionSpec { future }) } /// Prepares task execution and returns a future that will execute the task. fn make_execution_future( self: &Task, mut state: FullTaskWriteGuard<'_>, - backend: &MemoryBackend, + _backend: &MemoryBackend, turbo_tasks: &dyn TurboTasksBackendApi, ) -> Pin> + Send>> { match &self.ty { @@ -845,34 +703,7 @@ impl Task { drop(state); mutex.lock().take().expect("Task can only be executed once") } - &TaskType::ReadTaskCollectibles(box ReadTaskCollectiblesTaskType { - task: task_id, - trait_type, - }) => { - // Connect the task to the current task. This makes strongly consistent behaving - // as expected and we can look up the collectibles in the current scope. - self.connect_child_internal(state, task_id, backend, turbo_tasks); - // state was dropped by previous method - Box::pin(Self::execute_read_task_collectibles( - self.id, - task_id, - trait_type, - turbo_tasks.pin(), - )) - } - &TaskType::ReadScopeCollectibles(box ReadScopeCollectiblesTaskType { - scope, - trait_type, - }) => { - drop(state); - Box::pin(Self::execute_read_scope_collectibles( - self.id, - scope, - trait_type, - turbo_tasks.pin(), - )) - } - TaskType::Persistent(ty) => match &**ty { + TaskType::Persistent { ty, .. } => match &**ty { PersistentTaskType::Native(native_fn, inputs) => { let future = if let PrepareTaskType::Native(bound_fn) = &state.prepared_type { bound_fn() @@ -913,12 +744,18 @@ impl Task { } } - pub(crate) fn mark_as_finished(&self, backend: &MemoryBackend) { + pub(crate) fn mark_as_finished( + &self, + backend: &MemoryBackend, + turbo_tasks: &dyn TurboTasksBackendApi, + ) { let TaskMetaStateWriteGuard::Full(mut state) = self.state_mut() else { return; }; let TaskStateType::InProgress { ref mut count_as_finished, + ref mut outdated_children, + ref mut outdated_collectibles, .. } = state.state_type else { @@ -928,11 +765,33 @@ impl Task { return; } *count_as_finished = true; - for scope in state.scopes.iter() { - backend.with_scope(scope, |scope| { - scope.decrement_unfinished_tasks(backend); - }) + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); + { + let outdated_children = take(outdated_children); + let outdated_collectibles = outdated_collectibles.take_collectibles(); + + let mut change = TaskChange { + unfinished: -1, + #[cfg(feature = "track_unfinished")] + unfinished_tasks_update: vec![(self.id, -1)], + ..Default::default() + }; + if let Some(collectibles) = outdated_collectibles { + for ((trait_type, value), count) in collectibles.into_iter() { + change.collectibles.push((trait_type, value, -count)); + } + } + let change_job = state + .aggregation_leaf + .change_job(&aggregation_context, change); + let remove_job = state + .aggregation_leaf + .remove_children_job(&aggregation_context, outdated_children); + drop(state); + change_job(); + remove_job(); } + aggregation_context.apply_queued_updates(); } pub(crate) fn execution_result( @@ -993,116 +852,98 @@ impl Task { backend: &MemoryBackend, turbo_tasks: &dyn TurboTasksBackendApi, ) -> bool { + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); let mut schedule_task = false; - let mut dependencies = DEPENDENCIES_TO_TRACK.with(|deps| deps.take()); { - let mut state = self.full_state_mut(); - - state - .stats - .register_execution(duration, turbo_tasks.program_duration_until(instant)); - match state.state_type { - InProgress { - ref mut event, - count_as_finished, - } => { - let event = event.take(); - let mut dependencies = take(&mut dependencies); - // This will stay here for longer, so make sure to not consume too much memory - dependencies.shrink_to_fit(); - for cells in state.cells.values_mut() { - cells.shrink_to_fit(); - } - state.cells.shrink_to_fit(); - state.stateful = stateful; - state.state_type = Done { dependencies }; - if !count_as_finished { - for scope in state.scopes.iter() { - backend.with_scope(scope, |scope| { - scope.decrement_unfinished_tasks(backend); - }) + let mut change_job = None; + let mut remove_job = None; + let mut dependencies = DEPENDENCIES_TO_TRACK.with(|deps| deps.take()); + { + let mut state = self.full_state_mut(); + + state + .stats + .register_execution(duration, turbo_tasks.program_duration_until(instant)); + match state.state_type { + InProgress { + ref mut event, + count_as_finished, + ref mut outdated_children, + ref mut outdated_collectibles, + } => { + let event = event.take(); + let outdated_children = take(outdated_children); + let outdated_collectibles = outdated_collectibles.take_collectibles(); + let mut dependencies = take(&mut dependencies); + // This will stay here for longer, so make sure to not consume too much + // memory + dependencies.shrink_to_fit(); + for cells in state.cells.values_mut() { + cells.shrink_to_fit(); + } + state.cells.shrink_to_fit(); + state.stateful = stateful; + state.state_type = Done { dependencies }; + if !count_as_finished { + let mut change = TaskChange { + unfinished: -1, + #[cfg(feature = "track_unfinished")] + unfinished_tasks_update: vec![(self.id, -1)], + ..Default::default() + }; + if let Some(collectibles) = outdated_collectibles { + for ((trait_type, value), count) in collectibles.into_iter() { + change.collectibles.push((trait_type, value, -count)); + } + } + change_job = Some( + state + .aggregation_leaf + .change_job(&aggregation_context, change), + ); + } + if !outdated_children.is_empty() { + remove_job = Some( + state + .aggregation_leaf + .remove_children_job(&aggregation_context, outdated_children), + ); } + event.notify(usize::MAX); } - event.notify(usize::MAX); - } - InProgressDirty { ref mut event } => { - let event = event.take(); - state.state_type = Scheduled { event }; - schedule_task = true; - } - Dirty { .. } | Scheduled { .. } | Done { .. } => { - panic!( - "Task execution completed in unexpected state {}", - Task::state_string(&state) - ) - } - }; - } - if !dependencies.is_empty() { - self.clear_dependencies(dependencies, backend); - } - - if let TaskType::Once(_) = self.ty { - self.remove_root_or_initial_scope(backend, turbo_tasks); - } - - schedule_task - } - - /// When any scope is active it returns true. When no scope is active it - /// returns false and adds the tasks to all scopes as dirty task. - /// When `increment_unfinished` is true it will also increment the - /// unfinished tasks for all scopes, independent of activeness. - fn scopes_dirty_or_active( - &self, - increment_unfinished: bool, - scopes: &TaskScopes, - backend: &MemoryBackend, - ) -> bool { - if increment_unfinished { - // We need to walk all scopes at least once to increment unfinished tasks. - // While doing that we check if any scope is active. - let mut active = false; - for scope in scopes.iter() { - backend.with_scope(scope, |scope| { - scope.increment_unfinished_tasks(backend); - active = active || scope.state.lock().is_active(); - }) + InProgressDirty { ref mut event } => { + let event = event.take(); + state.state_type = Scheduled { event }; + schedule_task = true; + } + Dirty { .. } | Scheduled { .. } | Done { .. } => { + panic!( + "Task execution completed in unexpected state {}", + Task::state_string(&state) + ) + } + }; } - if active { - return true; + if !dependencies.is_empty() { + self.clear_dependencies(dependencies, backend, turbo_tasks); } - } else { - // Without the need to increment unfinished for all scopes we can exit early - if scopes - .iter() - .any(|scope| backend.with_scope(scope, |scope| scope.state.lock().is_active())) - { - return true; + if let Some(job) = change_job { + job(); } - } - for (i, scope) in scopes.iter().enumerate() { - let any_scope_was_active = backend.with_scope(scope, |scope| { - let mut state = scope.state.lock(); - let is_active = state.is_active(); - if !is_active { - state.add_dirty_task(self.id); - } - is_active - }); - if any_scope_was_active { - // A scope is active, revert dirty task changes and return true - for scope in scopes.iter().take(i) { - backend.with_scope(scope, |scope| { - let mut state = scope.state.lock(); - state.remove_dirty_task(self.id); - }) - } - return true; + if let Some(job) = remove_job { + job(); } } - // No scope is active. Task has been added as dirty task to all scopes - false + if let TaskType::Once(_) = self.ty { + // unset the root type, so tasks below are no longer active + aggregation_context + .aggregation_info(self.id) + .lock() + .root_type = None; + } + aggregation_context.apply_queued_updates(); + + schedule_task } fn make_dirty( @@ -1129,6 +970,7 @@ impl Task { } else { self.state_mut() }; + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); if let TaskMetaStateWriteGuard::Full(mut state) = state { let mut clear_dependencies = AutoSet::default(); @@ -1145,6 +987,13 @@ impl Task { format!("TaskState({})::event", description()) }), }; + state.aggregation_leaf.change( + &TaskAggregationContext::new(turbo_tasks, backend), + &TaskChange { + dirty_tasks_update: vec![(self.id, -1)], + ..Default::default() + }, + ); drop(state); turbo_tasks.schedule(self.id); } else { @@ -1155,12 +1004,51 @@ impl Task { Done { ref mut dependencies, } => { + let mut has_set_unfinished = false; clear_dependencies = take(dependencies); // add to dirty lists and potentially schedule let description = self.get_event_description(); - let active = - force_schedule || self.scopes_dirty_or_active(true, &state.scopes, backend); - if active { + let should_schedule = force_schedule + || state + .aggregation_leaf + .get_root_info(&aggregation_context, &RootInfoType::IsActive) + || { + state.aggregation_leaf.change( + &aggregation_context, + &TaskChange { + unfinished: 1, + #[cfg(feature = "track_unfinished")] + unfinished_tasks_update: vec![(self.id, 1)], + dirty_tasks_update: vec![(self.id, 1)], + ..Default::default() + }, + ); + has_set_unfinished = true; + if aggregation_context.take_scheduled_dirty_task(self.id) { + state.aggregation_leaf.change( + &aggregation_context, + &TaskChange { + dirty_tasks_update: vec![(self.id, -1)], + ..Default::default() + }, + ); + true + } else { + false + } + }; + if !has_set_unfinished { + state.aggregation_leaf.change( + &aggregation_context, + &TaskChange { + unfinished: 1, + #[cfg(feature = "track_unfinished")] + unfinished_tasks_update: vec![(self.id, 1)], + ..Default::default() + }, + ); + } + if should_schedule { state.state_type = Scheduled { event: Event::new(move || { format!("TaskState({})::event", description()) @@ -1184,625 +1072,89 @@ impl Task { InProgress { ref mut event, count_as_finished, + ref mut outdated_children, + ref mut outdated_collectibles, } => { let event = event.take(); + let outdated_children = take(outdated_children); + let outdated_collectibles = outdated_collectibles.take_collectibles(); + let mut change_job = None; if count_as_finished { - for scope in state.scopes.iter() { - backend.with_scope(scope, |scope| { - scope.increment_unfinished_tasks(backend); - }) + let mut change = TaskChange { + unfinished: 1, + #[cfg(feature = "track_unfinished")] + unfinished_tasks_update: vec![(self.id, 1)], + ..Default::default() + }; + if let Some(collectibles) = outdated_collectibles { + for ((trait_type, value), count) in collectibles.into_iter() { + change.collectibles.push((trait_type, value, -count)); + } } + change_job = Some( + state + .aggregation_leaf + .change_job(&aggregation_context, change), + ); } + let remove_job = state + .aggregation_leaf + .remove_children_job(&aggregation_context, outdated_children); state.state_type = InProgressDirty { event }; drop(state); + if let Some(job) = change_job { + job(); + } + remove_job(); } } if !clear_dependencies.is_empty() { - self.clear_dependencies(clear_dependencies, backend); + self.clear_dependencies(clear_dependencies, backend, turbo_tasks); } } } - pub(crate) fn schedule_when_dirty_from_scope( + pub(crate) fn schedule_when_dirty_from_aggregation( &self, backend: &MemoryBackend, turbo_tasks: &dyn TurboTasksBackendApi, ) { + let aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); let mut state = self.full_state_mut(); if let TaskStateType::Dirty { ref mut event } = state.state_type { state.state_type = Scheduled { event: event.take(), }; - for scope in state.scopes.iter() { - backend.with_scope(scope, |scope| { - scope.state.lock().remove_dirty_task(self.id); - }) - } + let job = state.aggregation_leaf.change_job( + &aggregation_context, + TaskChange { + dirty_tasks_update: vec![(self.id, -1)], + ..Default::default() + }, + ); drop(state); turbo_tasks.schedule(self.id); + job(); } } - pub(crate) fn add_to_scope_internal_shallow( - &self, - id: TaskScopeId, - merging_scopes: usize, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - queue: &mut VecDeque, - ) { - let mut state = self.full_state_mut(); - let TaskState { - ref mut scopes, - ref children, - .. - } = *state; - match *scopes { - TaskScopes::Root(root) => { - if root == id { - // The task is already in the root scope we're trying to add it to. - return; - } - - if let Some(ScopeChildChangeEffect { - notify, - active, - parent, - }) = backend.with_scope(id, |scope| scope.state.lock().add_child(root)) - { - drop(state); - if !notify.is_empty() { - turbo_tasks.schedule_notify_tasks_set(¬ify); - } - if active { - backend.increase_scope_active(root, turbo_tasks); - } - if parent { - backend.with_scope(root, |child| { - child.add_parent(id, backend); - }) - } - } - } - TaskScopes::Inner(ref mut list, ref mut optimization_counter) => { - if !list.add(id) { - // The task is already in the scope we're trying to add it to. - return; - } - - *optimization_counter += 1; - if merging_scopes > 0 { - *optimization_counter = optimization_counter.saturating_sub(merging_scopes); - } else if should_optimize_to_root_scoped(*optimization_counter, children.len()) { - list.remove(id); - drop(self.make_root_scoped_internal(state, backend, turbo_tasks)); - return self.add_to_scope_internal_shallow( - id, - merging_scopes, - backend, - turbo_tasks, - queue, - ); - } - - if queue.capacity() == 0 { - queue.reserve(max(children.len(), SPLIT_OFF_QUEUE_AT * 2)); - } - queue.extend(children.iter().copied()); - - // add to dirty list of the scope (potentially schedule) - let schedule_self = - self.add_self_to_new_scope(&mut state, id, backend, turbo_tasks); - drop(state); + pub(crate) fn add_dependency_to_current(dep: TaskDependency) { + DEPENDENCIES_TO_TRACK.with(|list| { + let mut list = list.borrow_mut(); + list.insert(dep); + }) + } - if schedule_self { - turbo_tasks.schedule(self.id); - } - } - } + /// Get an [Invalidator] that can be used to invalidate the current [Task] + /// based on external events. + pub fn get_invalidator() -> Invalidator { + get_invalidator() } - pub(crate) fn add_to_scope_internal( - &self, - id: TaskScopeId, - merging_scopes: usize, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - // VecDeque::new() would allocate with 7 items capacity. We don't want that. - let mut queue = VecDeque::with_capacity(0); - self.add_to_scope_internal_shallow(id, merging_scopes, backend, turbo_tasks, &mut queue); - - run_add_to_scope_queue(queue, id, merging_scopes, backend, turbo_tasks); - } - - fn add_self_to_new_scope( - &self, - state: &mut FullTaskWriteGuard<'_>, - id: TaskScopeId, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - ) -> bool { - let mut schedule_self = false; - backend.with_scope(id, |scope| { - scope.increment_tasks(); - if !matches!(state.state_type, TaskStateType::Done { .. }) { - if !matches!( - state.state_type, - TaskStateType::InProgress { - count_as_finished: true, - .. - } - ) { - scope.increment_unfinished_tasks(backend); - } - log_scope_update!("add unfinished task (added): {} -> {}", *scope.id, *self.id); - if let TaskStateType::Dirty { ref mut event } = state.state_type { - let mut scope = scope.state.lock(); - if scope.is_active() { - state.state_type = Scheduled { - event: event.take(), - }; - schedule_self = true; - } else { - scope.add_dirty_task(self.id); - } - } - } - - if let Some(collectibles) = state.collectibles.as_ref() { - let mut tasks = AutoSet::default(); - { - let mut scope_state = scope.state.lock(); - collectibles - .emitted - .iter() - .filter_map(|(trait_id, collectible)| { - scope_state.add_collectible(*trait_id, *collectible) - }) - .for_each(|e| tasks.extend(e.notify)); - collectibles - .unemitted - .iter() - .filter_map(|(trait_id, collectible)| { - scope_state.remove_collectible(*trait_id, *collectible) - }) - .for_each(|e| tasks.extend(e.notify)); - }; - turbo_tasks.schedule_notify_tasks_set(&tasks); - } - }); - schedule_self - } - - fn remove_self_from_scope( - &self, - state: &mut TaskMetaStateWriteGuard<'_>, - id: TaskScopeId, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - match state { - TaskMetaStateWriteGuard::Full(state) => { - self.remove_self_from_scope_full(state, id, backend, turbo_tasks); - } - TaskMetaStateWriteGuard::Partial(_) => backend.with_scope(id, |scope| { - scope.decrement_tasks(); - scope.decrement_unfinished_tasks(backend); - let mut scope = scope.state.lock(); - scope.remove_dirty_task(self.id); - }), - TaskMetaStateWriteGuard::Unloaded(_) => { - unreachable!("remove_self_from_scope must be called with at least a partial state"); - } - } - } - - fn remove_self_from_scope_full( - &self, - state: &mut FullTaskWriteGuard<'_>, - id: TaskScopeId, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - backend.with_scope(id, |scope| { - match state.state_type { - Done { .. } => {} - Dirty { .. } => { - scope.decrement_unfinished_tasks(backend); - let mut scope = scope.state.lock(); - scope.remove_dirty_task(self.id); - } - InProgress { - count_as_finished: true, - .. - } => { - // no need to decrement unfinished tasks - } - _ => { - scope.decrement_unfinished_tasks(backend); - } - } - scope.decrement_tasks(); - - if let Some(collectibles) = state.collectibles.as_ref() { - let mut tasks = AutoSet::default(); - { - let mut scope_state = scope.state.lock(); - collectibles - .emitted - .iter() - .filter_map(|(trait_id, collectible)| { - scope_state.remove_collectible(*trait_id, *collectible) - }) - .for_each(|e| tasks.extend(e.notify)); - collectibles - .unemitted - .iter() - .filter_map(|(trait_id, collectible)| { - scope_state.add_collectible(*trait_id, *collectible) - }) - .for_each(|e| tasks.extend(e.notify)); - }; - turbo_tasks.schedule_notify_tasks_set(&tasks); - } - }); - } - - fn remove_from_scope_internal_shallow( - &self, - id: TaskScopeId, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - queue: &mut VecDeque, - ) { - let mut state = self.partial_state_mut(); - let partial = matches!(state, TaskMetaStateWriteGuard::Partial(_)); - let (scopes, children) = state.scopes_and_children(); - match scopes { - &mut TaskScopes::Root(root) => { - if root != id { - if let Some(ScopeChildChangeEffect { - notify, - active, - parent, - }) = backend.with_scope(id, |scope| scope.state.lock().remove_child(root)) - { - if partial && parent { - // We might be able to drop the root scope now - // Check if this was the last parent that is removed - // (We operate under the task lock to ensure no other thread is adding a - // new parent) - backend.with_scope(root, |child| { - if child.remove_parent(id, backend) { - let stats_type = match &state { - TaskMetaStateWriteGuard::Full(s) => match s.stats { - TaskStats::Essential(_) => StatsType::Essential, - TaskStats::Full(_) => StatsType::Full, - }, - TaskMetaStateWriteGuard::Partial(s) => s.stats_type, - TaskMetaStateWriteGuard::Unloaded(s) => s.stats_type, - }; - let TaskMetaState::Partial(state) = replace( - &mut *state.into_inner(), - TaskMetaState::Unloaded(UnloadedTaskState { stats_type }), - ) else { - unreachable!("partial is set so it must be Partial"); - }; - child.decrement_tasks(); - child.decrement_unfinished_tasks(backend); - let notify = { - // Partial tasks are always dirty - let mut child = child.state.lock(); - child.remove_dirty_task(self.id); - child.take_all_dependent_tasks() - }; - drop(state); - - turbo_tasks.schedule_notify_tasks_set(¬ify); - - // Now this root scope is eventually no longer referenced - // and we can unload it, once all foreground jobs are done - // since there might be ongoing add/remove scopes. - let job = - backend.create_backend_job(Job::UnloadRootScope(root)); - turbo_tasks.schedule_backend_foreground_job(job); - } - }); - } else { - drop(state); - } - if !notify.is_empty() { - turbo_tasks.schedule_notify_tasks_set(¬ify); - } - if active { - backend.decrease_scope_active(root, self.id, turbo_tasks); - } - if !partial && parent { - backend.with_scope(root, |child| { - child.remove_parent(id, backend); - }); - } - } - } - } - TaskScopes::Inner(set, _) => { - if set.remove(id) { - if queue.capacity() == 0 { - queue.reserve(max(children.len(), SPLIT_OFF_QUEUE_AT * 2)); - } - queue.extend(children.iter().copied()); - let unset = set.is_unset(); - self.remove_self_from_scope(&mut state, id, backend, turbo_tasks); - if unset { - if let TaskMetaStateWriteGuard::Partial(state) = state { - let stats_type = state.stats_type; - let mut state = state.into_inner(); - *state = TaskMetaState::Unloaded(UnloadedTaskState { stats_type }); - } else { - drop(state); - } - } else { - drop(state); - } - } - } - } - } - - fn remove_from_scope_internal( - &self, - id: TaskScopeId, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - // VecDeque::new() would allocate with 7 items capacity. We don't want that. - let mut queue = VecDeque::with_capacity(0); - self.remove_from_scope_internal_shallow(id, backend, turbo_tasks, &mut queue); - if !queue.is_empty() { - run_remove_from_scope_queue(queue, id, backend, turbo_tasks); - } - } - - pub(crate) fn remove_from_scope( - &self, - id: TaskScopeId, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - self.remove_from_scope_internal(id, backend, turbo_tasks) - } - - pub(crate) fn remove_from_scopes( - &self, - scopes: impl Iterator, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - for id in scopes { - self.remove_from_scope_internal(id, backend, turbo_tasks) - } - } - - fn remove_root_or_initial_scope( - &self, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - let mut state = self.full_state_mut(); - match state.scopes { - TaskScopes::Root(root) => { - log_scope_update!("removing root scope {root}"); - state.scopes = TaskScopes::default(); - - turbo_tasks.schedule_backend_foreground_job( - backend.create_backend_job(Job::RemoveFromScope(state.children.clone(), root)), - ); - } - TaskScopes::Inner(ref mut set, _) => { - log_scope_update!("removing initial scope"); - let initial = backend.initial_scope; - if set.remove(initial) { - let children = state.children.iter().copied().collect::>(); - self.remove_self_from_scope( - &mut TaskMetaStateWriteGuard::Full(state), - initial, - backend, - turbo_tasks, - ); - // state ends here, as it was passed into `remove_self_from_scope` - - if !children.is_empty() { - run_remove_from_scope_queue(children, initial, backend, turbo_tasks); - } - } - } - } - } - - pub fn make_root_scoped( - &self, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - let state = self.full_state_mut(); - self.make_root_scoped_internal(state, backend, turbo_tasks); - } - - fn make_root_scoped_internal<'a>( - &self, - mut state: FullTaskWriteGuard<'a>, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - ) -> Option> { - if matches!(state.scopes, TaskScopes::Root(_)) { - return Some(state); - } - let root_scope = backend.create_new_scope(0); - // Set the root scope of the current task - if let TaskScopes::Inner(set, _) = replace(&mut state.scopes, TaskScopes::Root(root_scope)) - { - let scopes = set.into_counts().collect::>(); - log_scope_update!( - "new {root_scope} for {:?} as internal root scope (replacing {scopes:?})", - self.ty - ); - let mut active_counter = 0isize; - let mut tasks = AutoSet::default(); - let mut scopes_to_add_as_parent = Vec::new(); - let mut scopes_to_remove_as_parent = Vec::new(); - for (scope_id, count) in scopes.iter() { - backend.with_scope(*scope_id, |scope| { - // add the new root scope as child of old scopes - let mut state = scope.state.lock(); - match count.cmp(&0) { - Ordering::Greater => { - if let Some(ScopeChildChangeEffect { - notify, - active, - parent, - }) = state.add_child_count(root_scope, *count as usize) - { - tasks.extend(notify); - if active { - active_counter += 1; - } - if parent { - scopes_to_add_as_parent.push(*scope_id); - } - } - } - Ordering::Less => { - if let Some(ScopeChildChangeEffect { - notify, - active, - parent, - }) = state.remove_child_count(root_scope, (-*count) as usize) - { - tasks.extend(notify); - if active { - active_counter -= 1; - } - if parent { - scopes_to_remove_as_parent.push(*scope_id); - } - } - } - _ => {} - } - }); - } - if !tasks.is_empty() { - turbo_tasks.schedule_notify_tasks_set(&tasks); - } - backend.with_scope(root_scope, |root_scope| { - for parent in scopes_to_add_as_parent { - root_scope.add_parent(parent, backend); - } - for parent in scopes_to_remove_as_parent { - root_scope.remove_parent(parent, backend); - } - }); - - // We collected how often the new root scope is considered as active by the old - // scopes and increase the active counter by that. - match active_counter.cmp(&0) { - Ordering::Greater => { - backend.increase_scope_active_by( - root_scope, - active_counter as usize, - turbo_tasks, - ); - } - Ordering::Less => { - backend.decrease_scope_active_by( - root_scope, - self.id, - (-active_counter) as usize, - turbo_tasks, - ); - } - _ => {} - } - - // add self to new root scope - let schedule_self = - self.add_self_to_new_scope(&mut state, root_scope, backend, turbo_tasks); - - let mut merging_scopes = Vec::with_capacity(scopes.len()); - // remove self from old scopes - for (scope, count) in scopes.iter() { - if *count > 0 { - merging_scopes.push(*scope); - self.remove_self_from_scope_full(&mut state, *scope, backend, turbo_tasks); - } - } - - if !state.children.is_empty() || schedule_self { - let children = state.children.clone(); - - drop(state); - - // Add children to new root scope - for child in children.iter() { - backend.with_task(*child, |child| { - child.add_to_scope_internal( - root_scope, - merging_scopes.len(), - backend, - turbo_tasks, - ); - }) - } - - // Potentially schedule itself, when root scope is active and task is dirty - // I think that will never happen since it should already be scheduled by the - // old scopes. Anyway let just do it to be safe: - if schedule_self { - turbo_tasks.schedule(self.id); - } - - // Remove children from old scopes - #[cfg(feature = "inline_remove_from_scope")] - for task in children { - backend.with_task(task, |task| { - task.remove_from_scopes( - merging_scopes.iter().copied(), - backend, - turbo_tasks, - ) - }); - } - #[cfg(not(feature = "inline_remove_from_scope"))] - turbo_tasks.schedule_backend_foreground_job( - backend.create_backend_job(Job::RemoveFromScopes(children, merging_scopes)), - ); - None - } else { - Some(state) - } - } else { - unreachable!() - } - } - - pub(crate) fn add_dependency_to_current(dep: TaskDependency) { - DEPENDENCIES_TO_TRACK.with(|list| { - let mut list = list.borrow_mut(); - list.insert(dep); - }) - } - - /// Get an [Invalidator] that can be used to invalidate the current [Task] - /// based on external events. - pub fn get_invalidator() -> Invalidator { - get_invalidator() - } - - /// Called by the [Invalidator]. Invalidate the [Task]. When the task is - /// active it will be scheduled for execution. - pub(crate) fn invalidate( + /// Called by the [Invalidator]. Invalidate the [Task]. When the task is + /// active it will be scheduled for execution. + pub(crate) fn invalidate( &self, backend: &MemoryBackend, turbo_tasks: &dyn TurboTasksBackendApi, @@ -1884,13 +1236,21 @@ impl Task { } } + pub fn is_dirty(&self) -> bool { + if let TaskMetaStateReadGuard::Full(state) = self.state() { + matches!(state.state_type, TaskStateType::Dirty { .. }) + } else { + false + } + } + pub fn reset_stats(&self) { if let TaskMetaStateWriteGuard::Full(mut state) = self.state_mut() { state.stats.reset(); } } - pub fn get_stats_info(&self, backend: &MemoryBackend) -> TaskStatsInfo { + pub fn get_stats_info(&self, _backend: &MemoryBackend) -> TaskStatsInfo { match self.state() { TaskMetaStateReadGuard::Full(state) => { let (total_duration, last_duration, executions) = match &state.stats { @@ -1906,37 +1266,19 @@ impl Task { total_duration, last_duration, executions, - root_scoped: matches!(state.scopes, TaskScopes::Root(_)), - child_scopes: match state.scopes { - TaskScopes::Root(_) => 1, - TaskScopes::Inner(ref list, _) => list.len(), - }, - active: state.scopes.iter().any(|scope| { - backend.with_scope(scope, |scope| scope.state.lock().is_active()) - }), unloaded: false, } } - TaskMetaStateReadGuard::Partial(state) => TaskStatsInfo { + TaskMetaStateReadGuard::Partial(_) => TaskStatsInfo { total_duration: None, last_duration: Duration::ZERO, executions: None, - root_scoped: false, - child_scopes: if let TaskScopes::Inner(ref set, _) = state.scopes { - set.len() - } else { - 0 - }, - active: false, unloaded: true, }, TaskMetaStateReadGuard::Unloaded(_) => TaskStatsInfo { total_duration: None, last_duration: Duration::ZERO, executions: None, - root_scoped: false, - child_scopes: 0, - active: false, unloaded: true, }, } @@ -1946,14 +1288,7 @@ impl Task { match &self.ty { TaskType::Root(_) => StatsTaskType::Root(self.id), TaskType::Once(_) => StatsTaskType::Once(self.id), - TaskType::ReadTaskCollectibles(box ReadTaskCollectiblesTaskType { - trait_type, .. - }) => StatsTaskType::ReadCollectibles(*trait_type), - TaskType::ReadScopeCollectibles(box ReadScopeCollectiblesTaskType { - trait_type, - .. - }) => StatsTaskType::ReadCollectibles(*trait_type), - TaskType::Persistent(ty) => match &**ty { + TaskType::Persistent { ty, .. } => match &**ty { PersistentTaskType::Native(f, _) => StatsTaskType::Native(*f), PersistentTaskType::ResolveNative(f, _) => StatsTaskType::ResolveNative(*f), PersistentTaskType::ResolveTrait(t, n, _) => { @@ -1965,7 +1300,6 @@ impl Task { pub fn get_stats_references(&self) -> StatsReferences { let mut refs = Vec::new(); - let mut scope_refs = Vec::new(); if let TaskMetaStateReadGuard::Full(state) = self.state() { for child in state.children.iter() { refs.push((ReferenceType::Child, *child)); @@ -1973,18 +1307,16 @@ impl Task { if let Done { ref dependencies } = state.state_type { for dep in dependencies.iter() { match dep { - TaskDependency::TaskOutput(task) | TaskDependency::TaskCell(task, _) => { + TaskDependency::Output(task) + | TaskDependency::Cell(task, _) + | TaskDependency::Collectibles(task, _) => { refs.push((ReferenceType::Dependency, *task)) } - TaskDependency::ScopeChildren(scope) - | TaskDependency::ScopeCollectibles(scope, _) => { - scope_refs.push((ReferenceType::Dependency, *scope)) - } } } } } - if let TaskType::Persistent(ty) = &self.ty { + if let TaskType::Persistent { ty, .. } = &self.ty { match &**ty { PersistentTaskType::Native(_, inputs) | PersistentTaskType::ResolveNative(_, inputs) @@ -1997,38 +1329,24 @@ impl Task { } } } - StatsReferences { - tasks: refs, - scopes: scope_refs, - } + StatsReferences { tasks: refs } } - fn state_string(state: &TaskState) -> String { - let mut state_str = match state.state_type { - Scheduled { .. } => "scheduled".to_string(), - InProgress { .. } => "in progress".to_string(), - InProgressDirty { .. } => "in progress (dirty)".to_string(), - Done { .. } => "done".to_string(), - Dirty { .. } => "dirty".to_string(), - }; - match state.scopes { - TaskScopes::Root(root) => { - write!(state_str, " (root scope {})", root).unwrap(); - } - TaskScopes::Inner(ref list, change_counter) => { - if !list.is_empty() || change_counter > 0 { - write!(state_str, " (scopes").unwrap(); - for scope in list.iter() { - write!(state_str, " {}", *scope).unwrap(); - } - if change_counter > 0 { - write!(state_str, " {change_counter} accumulated changes").unwrap(); - } - write!(state_str, ")").unwrap(); - } - } + fn state_string(state: &TaskState) -> &'static str { + match state.state_type { + Scheduled { .. } => "scheduled", + InProgress { + count_as_finished: false, + .. + } => "in progress", + InProgress { + count_as_finished: true, + .. + } => "in progress (marked as finished)", + InProgressDirty { .. } => "in progress (dirty)", + Done { .. } => "done", + Dirty { .. } => "dirty", } - state_str } pub(crate) fn connect_child( @@ -2037,93 +1355,47 @@ impl Task { backend: &MemoryBackend, turbo_tasks: &dyn TurboTasksBackendApi, ) { - let state = self.full_state_mut(); - self.connect_child_internal(state, child_id, backend, turbo_tasks); - } - - fn connect_child_internal( - &self, - mut state: FullTaskWriteGuard<'_>, - child_id: TaskId, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - if state.children.insert(child_id) { - if let TaskScopes::Inner(_, optimization_counter) = &state.scopes { - if should_optimize_to_root_scoped(*optimization_counter, state.children.len()) { - state.children.remove(&child_id); - drop(self.make_root_scoped_internal(state, backend, turbo_tasks)); - return self.connect_child(child_id, backend, turbo_tasks); + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); + { + let thresholds_job; + let mut add_job = None; + { + let mut guard = TaskGuard { + id: self.id, + guard: self.state_mut(), + }; + thresholds_job = ensure_thresholds(&aggregation_context, &mut guard); + let TaskGuard { guard, .. } = guard; + let mut state = TaskMetaStateWriteGuard::full_from(guard.into_inner(), self); + if state.children.insert(child_id) { + add_job = Some( + state + .aggregation_leaf + .add_child_job(&aggregation_context, &child_id), + ); } } - let scopes = state.scopes.clone(); - drop(state); - - backend.with_task(child_id, |child| { - for scope in scopes.iter() { - #[cfg(not(feature = "report_expensive"))] - { - child.add_to_scope_internal(scope, 0, backend, turbo_tasks); - } - #[cfg(feature = "report_expensive")] - { - use std::time::Instant; - - use turbo_tasks::util::FormatDuration; - - let start = Instant::now(); - child.add_to_scope_internal(scope, 0, backend, turbo_tasks); - let elapsed = start.elapsed(); - if elapsed.as_millis() >= 10 { - println!( - "add_to_scope {scope} took {}: {:?}", - FormatDuration(elapsed), - child - ); + thresholds_job(); + if let Some(job) = add_job { + // To avoid bubbling up the dirty tasks into the new parent tree, we make a + // quick check for activeness of the parent when the child is dirty. This is + // purely an optimization and not required for correctness. + // So it's fine to ignore the race condition existing here. + backend.with_task(child_id, |child| { + if child.is_dirty() { + let active = self + .full_state_mut() + .aggregation_leaf + .get_root_info(&aggregation_context, &RootInfoType::IsActive); + if active { + child.schedule_when_dirty_from_aggregation(backend, turbo_tasks); } } - } - }); - } - } - - fn ensure_root_scoped<'a>( - &'a self, - mut state: FullTaskWriteGuard<'a>, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, - ) -> FullTaskWriteGuard<'a> { - while !state.scopes.is_root() { - #[cfg(not(feature = "report_expensive"))] - let result = self.make_root_scoped_internal(state, backend, turbo_tasks); - #[cfg(feature = "report_expensive")] - let result = { - use std::time::Instant; - - use turbo_tasks::util::FormatDuration; - - let start = Instant::now(); - let result = self.make_root_scoped_internal(state, backend, turbo_tasks); - let elapsed = start.elapsed(); - if elapsed.as_millis() >= 10 { - println!( - "make_root_scoped took {}: {:?}", - FormatDuration(elapsed), - self - ); - } - result - }; - if let Some(s) = result { - state = s; - break; - } else { - // We need to acquire a new lock and everything might have changed in between - state = self.full_state_mut(); - continue; + }); + job(); } } - state + aggregation_context.apply_queued_updates(); } pub(crate) fn get_or_wait_output Result>( @@ -2134,29 +1406,24 @@ impl Task { backend: &MemoryBackend, turbo_tasks: &dyn TurboTasksBackendApi, ) -> Result> { + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); + let aggregation_when_strongly_consistent = + strongly_consistent.then(|| aggregation_info(&aggregation_context, &self.id)); let mut state = self.full_state_mut(); - if strongly_consistent { - state = self.ensure_root_scoped(state, backend, turbo_tasks); - // We need to wait for all foreground jobs to be finished as there could be - // ongoing add_to_scope jobs that need to be finished before reading - // from scopes - if let Err(listener) = turbo_tasks.try_foreground_done() { - return Ok(Err(listener)); - } - if let TaskScopes::Root(root) = state.scopes { - if let Some(listener) = backend.with_scope(root, |scope| { - if let Some(listener) = scope.has_unfinished_tasks() { - return Some(listener); - } - None - }) { + if let Some(aggregation) = aggregation_when_strongly_consistent { + { + let aggregation = aggregation.lock(); + if aggregation.unfinished > 0 { + let listener = aggregation.unfinished_event.listen_with_note(note); + drop(aggregation); + drop(state); + aggregation_context.apply_queued_updates(); + return Ok(Err(listener)); } - } else { - unreachable!() } } - match state.state_type { + let result = match state.state_type { Done { .. } => { let result = func(&mut state.output)?; drop(state); @@ -2168,11 +1435,13 @@ impl Task { let event = event.take(); let listener = event.listen_with_note(note); state.state_type = Scheduled { event }; - for scope in state.scopes.iter() { - backend.with_scope(scope, |scope| { - scope.state.lock().remove_dirty_task(self.id); - }) - } + state.aggregation_leaf.change( + &aggregation_context, + &TaskChange { + dirty_tasks_update: vec![(self.id, -1)], + ..Default::default() + }, + ); drop(state); Ok(Err(listener)) } @@ -2183,122 +1452,23 @@ impl Task { drop(state); Ok(Err(listener)) } - } - } - - async fn execute_read_task_collectibles( - read_task_id: TaskId, - task_id: TaskId, - trait_type_id: TraitTypeId, - turbo_tasks: Arc>, - ) -> Result { - Self::execute_read_collectibles( - read_task_id, - |turbo_tasks| { - let backend = turbo_tasks.backend(); - backend.with_task(task_id, |task| { - let state = - task.ensure_root_scoped(task.full_state_mut(), backend, turbo_tasks); - if let TaskScopes::Root(scope_id) = state.scopes { - backend.with_scope(scope_id, |scope| { - scope.read_collectibles_and_children( - scope_id, - trait_type_id, - read_task_id, - ) - }) - } else { - unreachable!(); - } - }) - }, - trait_type_id, - turbo_tasks, - ) - .await - } - - async fn execute_read_scope_collectibles( - read_task_id: TaskId, - scope_id: TaskScopeId, - trait_type_id: TraitTypeId, - turbo_tasks: Arc>, - ) -> Result { - Self::execute_read_collectibles( - read_task_id, - |turbo_tasks| { - let backend = turbo_tasks.backend(); - backend.with_scope(scope_id, |scope| { - scope.read_collectibles_and_children(scope_id, trait_type_id, read_task_id) - }) - }, - trait_type_id, - turbo_tasks, - ) - .await - } - - async fn execute_read_collectibles( - read_task_id: TaskId, - read_collectibles_and_children: impl Fn( - &dyn TurboTasksBackendApi, - ) -> Result< - (CountHashSet, Vec), - EventListener, - >, - trait_type_id: TraitTypeId, - turbo_tasks: Arc>, - ) -> Result { - let (mut current, children) = loop { - // For performance reasons we only want to read collectibles when there are no - // unfinished tasks anymore. - match read_collectibles_and_children(&*turbo_tasks) { - Ok(r) => break r, - Err(listener) => listener.await, - } }; - let backend = turbo_tasks.backend(); - let children = children - .into_iter() - .filter_map(|child| { - backend.with_scope(child, |scope| { - scope.is_propagating_collectibles().then(|| { - let task = backend.get_or_create_read_scope_collectibles_task( - child, - trait_type_id, - read_task_id, - &*turbo_tasks, - ); - unsafe { >>::from_task_id(task) } - }) - }) - }) - .try_join() - .await?; - for child in children { - for v in child.iter() { - current.add(*v); - } - } - Ok(Vc::into_raw(Vc::>::cell( - current.iter().copied().collect(), - ))) + aggregation_context.apply_queued_updates(); + result } - pub(crate) fn read_task_collectibles( - &self, + pub(crate) fn read_collectibles( + id: TaskId, + trait_type: TraitTypeId, reader: TaskId, - trait_id: TraitTypeId, backend: &MemoryBackend, turbo_tasks: &dyn TurboTasksBackendApi, - ) -> Vc> { - let task = backend.get_or_create_read_task_collectibles_task( - self.id, - trait_id, - reader, - turbo_tasks, - ); - RawVc::TaskOutput(task).into() + ) -> AutoMap { + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); + aggregation_context + .aggregation_info(id) + .lock() + .read_collectibles(trait_type, reader) } pub(crate) fn emit_collectible( @@ -2308,47 +1478,40 @@ impl Task { backend: &MemoryBackend, turbo_tasks: &dyn TurboTasksBackendApi, ) { + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); let mut state = self.full_state_mut(); - if state.collectibles.emit(trait_type, collectible) { - let tasks = state - .scopes - .iter() - .flat_map(|id| { - backend.with_scope(id, |scope| { - let mut state = scope.state.lock(); - state.add_collectible(trait_type, collectible) - }) - }) - .flat_map(|e| e.notify) - .collect::>(); - drop(state); - turbo_tasks.schedule_notify_tasks_set(&tasks); - } + state.collectibles.emit(trait_type, collectible); + state.aggregation_leaf.change( + &aggregation_context, + &TaskChange { + collectibles: vec![(trait_type, collectible, 1)], + ..Default::default() + }, + ); + drop(state); + aggregation_context.apply_queued_updates(); } pub(crate) fn unemit_collectible( &self, trait_type: TraitTypeId, collectible: RawVc, + count: u32, backend: &MemoryBackend, turbo_tasks: &dyn TurboTasksBackendApi, ) { + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); let mut state = self.full_state_mut(); - if state.collectibles.unemit(trait_type, collectible) { - let mut tasks = AutoSet::default(); - state - .scopes - .iter() - .flat_map(|id| { - backend.with_scope(id, |scope| { - let mut state = scope.state.lock(); - state.remove_collectible(trait_type, collectible) - }) - }) - .for_each(|e| tasks.extend(e.notify)); - drop(state); - turbo_tasks.schedule_notify_tasks_set(&tasks); - } + state.collectibles.unemit(trait_type, collectible, count); + state.aggregation_leaf.change( + &aggregation_context, + &TaskChange { + collectibles: vec![(trait_type, collectible, -(count as i32))], + ..Default::default() + }, + ); + drop(state); + aggregation_context.apply_queued_updates(); } pub(crate) fn gc_check_inactive(&self, backend: &MemoryBackend) { @@ -2369,7 +1532,6 @@ impl Task { now_relative_to_start: Duration, max_priority: GcPriority, task_duration_cache: &mut HashMap>, - scope_active_cache: &mut HashMap>, stats: &mut GcStats, backend: &MemoryBackend, turbo_tasks: &dyn TurboTasksBackendApi, @@ -2398,21 +1560,6 @@ impl Task { } if let TaskMetaStateWriteGuard::Full(mut state) = self.state_mut() { - fn is_active( - state: &TaskState, - scope_active_cache: &mut HashMap< - TaskScopeId, - bool, - BuildNoHashHasher, - >, - backend: &MemoryBackend, - ) -> bool { - state.scopes.iter().any(|scope| { - *scope_active_cache.entry(scope).or_insert_with(|| { - backend.with_scope(scope, |scope| scope.state.lock().is_active()) - }) - }) - } if state.stateful { stats.no_gc_possible += 1; return None; @@ -2432,7 +1579,11 @@ impl Task { // Check if the task need to be activated again let active = if state.gc.inactive { - if is_active(&state, scope_active_cache, backend) { + let active = state.aggregation_leaf.get_root_info( + &TaskAggregationContext::new(turbo_tasks, backend), + &RootInfoType::IsActive, + ); + if active { state.gc.inactive = false; true } else { @@ -2561,9 +1712,7 @@ impl Task { // new GC priority. if missing_durations.is_empty() { let mut new_priority = GcPriority::Placeholder; - // TODO We currently don't unload root scopes tasks, because of a bug in - // scope unloading. Fix that. - if !active && !state.scopes.is_root() { + if !active { new_priority = GcPriority::InactiveUnload { age: Reverse(age), total_compute_duration: total_compute_duration_u8, @@ -2672,49 +1821,35 @@ impl Task { ) -> bool { let mut clear_dependencies = None; let TaskState { + ref mut aggregation_leaf, ref mut state_type, - ref scopes, .. } = *full_state; match state_type { Done { ref mut dependencies, } => { - for (i, scope) in scopes.iter().enumerate() { - let active = backend.with_scope(scope, |scope| { - scope.increment_unfinished_tasks(backend); - let mut scope_state = scope.state.lock(); - if scope_state.is_active() { - drop(scope_state); - log_scope_update!( - "add unfinished task (unload): {} -> {}", - *scope.id, - *self.id - ); - scope.decrement_unfinished_tasks(backend); - true - } else { - scope_state.add_dirty_task(self.id); - false - } - }); - if active { - // Unloading is only possible for inactive tasks. - // We need to abort the unloading, so revert changes done so far. - for scope in scopes.iter().take(i) { - backend.with_scope(scope, |scope| { - scope.decrement_unfinished_tasks(backend); - log_scope_update!( - "remove unfinished task (undo unload): {} -> {}", - *scope.id, - *self.id - ); - let mut scope = scope.state.lock(); - scope.remove_dirty_task(self.id); - }); - } - return false; - } + let mut aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); + aggregation_leaf.change( + &TaskAggregationContext::new(turbo_tasks, backend), + &TaskChange { + unfinished: 1, + dirty_tasks_update: vec![(self.id, 1)], + ..Default::default() + }, + ); + if aggregation_context.take_scheduled_dirty_task(self.id) { + // Unloading is only possible for inactive tasks. + // We need to abort the unloading, so revert changes done so far. + aggregation_leaf.change( + &TaskAggregationContext::new(turbo_tasks, backend), + &TaskChange { + unfinished: -1, + dirty_tasks_update: vec![(self.id, -1)], + ..Default::default() + }, + ); + return false; } clear_dependencies = Some(take(dependencies)); } @@ -2742,7 +1877,7 @@ impl Task { cells, output, collectibles, - scopes, + aggregation_leaf, stats, // can be dropped as it will be recomputed on next execution stateful: _, @@ -2754,29 +1889,33 @@ impl Task { gc: _, } = old_state.into_full().unwrap(); + let aggregation_context = TaskAggregationContext::new(turbo_tasks, backend); + // Remove all children, as they will be added again when this task is executed // again if !children.is_empty() { - remove_from_scopes(children, &scopes, backend, turbo_tasks); + for child in children { + aggregation_leaf.remove_child(&aggregation_context, &child); + } } // Remove all collectibles, as they will be added again when this task is // executed again. if let Some(collectibles) = collectibles.into_inner() { - remove_collectible_from_scopes( - collectibles.emitted, - collectibles.unemitted, - &scopes, - backend, - turbo_tasks, + aggregation_leaf.change( + &aggregation_context, + &TaskChange { + collectibles: collectibles + .into_iter() + .map(|((t, r), c)| (t, r, -c)) + .collect(), + ..Default::default() + }, ); } - let unset = if let TaskScopes::Inner(ref scopes, _) = scopes { - scopes.is_unset() - } else { - false - }; + // TODO aggregation_leaf + let unset = !aggregation_leaf.has_upper(); let stats_type = match stats { TaskStats::Essential(_) => StatsType::Essential, @@ -2785,7 +1924,10 @@ impl Task { if unset { *state = TaskMetaState::Unloaded(UnloadedTaskState { stats_type }); } else { - *state = TaskMetaState::Partial(Box::new(PartialTaskState { scopes, stats_type })); + *state = TaskMetaState::Partial(Box::new(PartialTaskState { + aggregation_leaf, + stats_type, + })); } drop(state); @@ -2801,131 +1943,11 @@ impl Task { // We can clear the dependencies as we are already marked as dirty if let Some(dependencies) = clear_dependencies { - self.clear_dependencies(dependencies, backend); + self.clear_dependencies(dependencies, backend, turbo_tasks); } true } - - pub fn get_read_collectibles_task( - &self, - trait_id: TraitTypeId, - create_new: impl FnOnce() -> TaskId, - ) -> TaskId { - let mut state = self.full_state_mut(); - state - .collectibles - .get_read_collectibles_task(trait_id, create_new) - } -} - -fn remove_collectible_from_scopes( - emitted: AutoSet<(TraitTypeId, RawVc)>, - unemitted: AutoSet<(TraitTypeId, RawVc)>, - task_scopes: &TaskScopes, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, -) { - task_scopes.iter().for_each(|id| { - backend.with_scope(id, |scope| { - let mut tasks = AutoSet::default(); - { - let mut state = scope.state.lock(); - emitted - .iter() - .filter_map(|(trait_id, collectible)| { - state.remove_collectible(*trait_id, *collectible) - }) - .for_each(|e| tasks.extend(e.notify)); - - unemitted - .iter() - .filter_map(|(trait_id, collectible)| { - state.add_collectible(*trait_id, *collectible) - }) - .for_each(|e| tasks.extend(e.notify)); - }; - turbo_tasks.schedule_notify_tasks_set(&tasks); - }) - }) -} - -fn remove_from_scopes( - tasks: AutoSet>, - task_scopes: &TaskScopes, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, -) { - match task_scopes { - TaskScopes::Root(scope) => { - turbo_tasks.schedule_backend_foreground_job( - backend.create_backend_job(Job::RemoveFromScope(tasks, *scope)), - ); - } - TaskScopes::Inner(ref scopes, _) => { - turbo_tasks.schedule_backend_foreground_job(backend.create_backend_job( - Job::RemoveFromScopes(tasks, scopes.iter().copied().collect()), - )); - } - } -} - -/// Heuristic to decide when to split off work in `run_add_to_scope_queue` and -/// `run_remove_from_scope_queue`. -const SPLIT_OFF_QUEUE_AT: usize = 100; - -/// Adds a list of tasks and their children to a scope, recursively. -pub fn run_add_to_scope_queue( - mut queue: VecDeque, - id: TaskScopeId, - merging_scopes: usize, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, -) { - while let Some(child) = queue.pop_front() { - backend.with_task(child, |child| { - child.add_to_scope_internal_shallow( - id, - merging_scopes, - backend, - turbo_tasks, - &mut queue, - ); - }); - #[cfg(not(feature = "inline_add_to_scope"))] - while queue.len() > SPLIT_OFF_QUEUE_AT { - let split_off_queue = queue.split_off(queue.len() - SPLIT_OFF_QUEUE_AT); - turbo_tasks.schedule_backend_foreground_job(backend.create_backend_job( - Job::AddToScopeQueue { - queue: split_off_queue, - scope: id, - merging_scopes, - }, - )); - } - } -} - -/// Removes a list of tasks and their children from a scope, recursively. -pub fn run_remove_from_scope_queue( - mut queue: VecDeque, - id: TaskScopeId, - backend: &MemoryBackend, - turbo_tasks: &dyn TurboTasksBackendApi, -) { - while let Some(child) = queue.pop_front() { - backend.with_task(child, |child| { - child.remove_from_scope_internal_shallow(id, backend, turbo_tasks, &mut queue); - }); - #[cfg(not(feature = "inline_remove_from_scope"))] - while queue.len() > SPLIT_OFF_QUEUE_AT { - let split_off_queue = queue.split_off(queue.len() - SPLIT_OFF_QUEUE_AT); - - turbo_tasks.schedule_backend_foreground_job( - backend.create_backend_job(Job::RemoveFromScopeQueue(split_off_queue, id)), - ); - } - } } impl Display for Task { @@ -2961,8 +1983,5 @@ pub struct TaskStatsInfo { pub total_duration: Option, pub last_duration: Duration, pub executions: Option, - pub root_scoped: bool, - pub child_scopes: usize, - pub active: bool, pub unloaded: bool, } diff --git a/crates/turbo-tasks-memory/src/task/aggregation.rs b/crates/turbo-tasks-memory/src/task/aggregation.rs new file mode 100644 index 0000000000000..4799347b26a5a --- /dev/null +++ b/crates/turbo-tasks-memory/src/task/aggregation.rs @@ -0,0 +1,527 @@ +use std::{ + borrow::Cow, + hash::{BuildHasher, Hash}, + mem::take, +}; + +use auto_hash_map::{map::Entry, AutoMap, AutoSet}; +use nohash_hasher::BuildNoHashHasher; +use parking_lot::Mutex; +use turbo_tasks::{event::Event, RawVc, TaskId, TraitTypeId, TurboTasksBackendApi}; + +use super::{meta_state::TaskMetaStateWriteGuard, TaskStateType}; +use crate::{ + aggregation_tree::{ + aggregation_info, AggregationContext, AggregationInfoReference, AggregationItemLock, + AggregationTreeLeaf, + }, + MemoryBackend, +}; + +pub enum RootType { + Once, + Root, +} + +#[derive(Debug, Default)] +pub struct CollectiblesInfo { + collectibles: AutoMap, + dependent_tasks: AutoSet>, +} + +impl CollectiblesInfo { + fn is_unset(&self) -> bool { + self.collectibles.is_empty() && self.dependent_tasks.is_empty() + } +} + +pub enum RootInfoType { + IsActive, +} + +pub struct Aggregated { + /// The number of unfinished items in the lower aggregation level. + /// Unfinished means not "Done". + // TODO determine if this can go negative in concurrent situations. + pub unfinished: i32, + /// Event that will be notified when all unfinished tasks are done. + pub unfinished_event: Event, + /// A list of all tasks that are unfinished. Only for debugging. + #[cfg(feature = "track_unfinished")] + pub unfinished_tasks: AutoMap>, + /// A list of all tasks that are dirty. + /// When the it becomes active, these need to be scheduled. + // TODO evaluate a more efficient data structure for this since we are copying the list on + // every level. + pub dirty_tasks: AutoMap>, + /// Emitted collectibles with count and dependent_tasks by trait type + pub collectibles: AutoMap>, + + /// Only used for the aggregation root. Which kind of root is this? + /// [RootType::Once] for OnceTasks or [RootType::Root] for Root Tasks. + /// It's set to None for other tasks, when the once task is done or when the + /// root task is disposed. + pub root_type: Option, +} + +impl Default for Aggregated { + fn default() -> Self { + Self { + unfinished: 0, + unfinished_event: Event::new(|| "Aggregated::unfinished_event".to_string()), + #[cfg(feature = "track_unfinished")] + unfinished_tasks: AutoMap::with_hasher(), + dirty_tasks: AutoMap::with_hasher(), + collectibles: AutoMap::with_hasher(), + root_type: None, + } + } +} + +impl Aggregated { + pub(crate) fn remove_collectible_dependent_task( + &mut self, + trait_type: TraitTypeId, + reader: TaskId, + ) { + if let Entry::Occupied(mut entry) = self.collectibles.entry(trait_type) { + let info = entry.get_mut(); + info.dependent_tasks.remove(&reader); + if info.is_unset() { + entry.remove(); + } + } + } + + pub(crate) fn read_collectibles( + &mut self, + trait_type: TraitTypeId, + reader: TaskId, + ) -> AutoMap { + match self.collectibles.entry(trait_type) { + Entry::Occupied(mut e) => { + let info = e.get_mut(); + info.dependent_tasks.insert(reader); + info.collectibles.clone() + } + Entry::Vacant(e) => { + e.insert(CollectiblesInfo::default()) + .dependent_tasks + .insert(reader); + AutoMap::default() + } + } + } +} + +#[derive(Default, Debug)] +pub struct TaskChange { + pub unfinished: i32, + #[cfg(feature = "track_unfinished")] + pub unfinished_tasks_update: Vec<(TaskId, i32)>, + pub dirty_tasks_update: Vec<(TaskId, i32)>, + pub collectibles: Vec<(TraitTypeId, RawVc, i32)>, +} + +impl TaskChange { + pub fn is_empty(&self) -> bool { + #[allow(unused_mut, reason = "feature flag")] + let mut empty = self.unfinished == 0 + && self.dirty_tasks_update.is_empty() + && self.collectibles.is_empty(); + #[cfg(feature = "track_unfinished")] + if !self.unfinished_tasks_update.is_empty() { + empty = false; + } + empty + } +} + +pub struct TaskAggregationContext<'a> { + pub turbo_tasks: &'a dyn TurboTasksBackendApi, + pub backend: &'a MemoryBackend, + pub dirty_tasks_to_schedule: Mutex>>>, + pub tasks_to_notify: Mutex>>>, +} + +impl<'a> TaskAggregationContext<'a> { + pub fn new( + turbo_tasks: &'a dyn TurboTasksBackendApi, + backend: &'a MemoryBackend, + ) -> Self { + Self { + turbo_tasks, + backend, + dirty_tasks_to_schedule: Mutex::new(None), + tasks_to_notify: Mutex::new(None), + } + } + + pub fn take_scheduled_dirty_task(&mut self, task: TaskId) -> bool { + let dirty_task_to_schedule = self.dirty_tasks_to_schedule.get_mut(); + dirty_task_to_schedule + .as_mut() + .map(|t| t.remove(&task)) + .unwrap_or(false) + } + + pub fn apply_queued_updates(&mut self) { + { + let mut _span = None; + let tasks = self.dirty_tasks_to_schedule.get_mut(); + if let Some(tasks) = tasks.as_mut() { + let tasks = take(tasks); + if !tasks.is_empty() { + _span.get_or_insert_with(|| { + tracing::trace_span!("task aggregation apply_queued_updates").entered() + }); + self.backend + .schedule_when_dirty_from_aggregation(tasks, self.turbo_tasks); + } + } + } + let tasks = self.tasks_to_notify.get_mut(); + if let Some(tasks) = tasks.as_mut() { + let tasks = take(tasks); + if !tasks.is_empty() { + self.turbo_tasks.schedule_notify_tasks_set(&tasks); + } + } + } + + pub fn aggregation_info(&mut self, id: TaskId) -> AggregationInfoReference { + aggregation_info(self, &id) + } +} + +#[cfg(debug_assertions)] +impl<'a> Drop for TaskAggregationContext<'a> { + fn drop(&mut self) { + let tasks_to_schedule = self.dirty_tasks_to_schedule.get_mut(); + if let Some(tasks_to_schedule) = tasks_to_schedule.as_ref() { + if !tasks_to_schedule.is_empty() { + panic!("TaskAggregationContext dropped without scheduling all tasks"); + } + } + let tasks_to_notify = self.tasks_to_notify.get_mut(); + if let Some(tasks_to_notify) = tasks_to_notify.as_ref() { + if !tasks_to_notify.is_empty() { + panic!("TaskAggregationContext dropped without notifying all tasks"); + } + } + } +} + +impl<'a> AggregationContext for TaskAggregationContext<'a> { + type ItemLock<'l> = TaskGuard<'l> where Self: 'l; + type Info = Aggregated; + type ItemChange = TaskChange; + type ItemRef = TaskId; + type RootInfo = bool; + type RootInfoType = RootInfoType; + + fn item<'b>(&'b self, reference: &TaskId) -> Self::ItemLock<'b> { + let task = self.backend.task(*reference); + TaskGuard { + id: *reference, + guard: task.state_mut(), + } + } + + fn apply_change( + &self, + info: &mut Aggregated, + change: &Self::ItemChange, + ) -> Option { + let mut unfinished = 0; + if info.unfinished > 0 { + info.unfinished += change.unfinished; + if info.unfinished == 0 { + info.unfinished_event.notify(usize::MAX); + unfinished = -1; + } + } else { + info.unfinished += change.unfinished; + if info.unfinished > 0 { + unfinished = 1; + } + } + #[cfg(feature = "track_unfinished")] + for &(task, count) in change.unfinished_tasks_update.iter() { + update_count_entry(info.unfinished_tasks.entry(task), count); + } + for &(task, count) in change.dirty_tasks_update.iter() { + let value = update_count_entry(info.dirty_tasks.entry(task), count); + if value > 0 + && value <= count + && matches!(info.root_type, Some(RootType::Root) | Some(RootType::Once)) + { + let mut tasks_to_schedule = self.dirty_tasks_to_schedule.lock(); + tasks_to_schedule.get_or_insert_default().insert(task); + } + } + for &(trait_type_id, collectible, count) in change.collectibles.iter() { + let collectibles_info_entry = info.collectibles.entry(trait_type_id); + match collectibles_info_entry { + Entry::Occupied(mut e) => { + let collectibles_info = e.get_mut(); + let value = update_count_entry( + collectibles_info.collectibles.entry(collectible), + count, + ); + if !collectibles_info.dependent_tasks.is_empty() { + self.tasks_to_notify + .lock() + .get_or_insert_default() + .extend(take(&mut collectibles_info.dependent_tasks).into_iter()); + } + if value == 0 && collectibles_info.is_unset() { + e.remove(); + } + } + Entry::Vacant(e) => { + let mut collectibles_info = CollectiblesInfo::default(); + update_count_entry(collectibles_info.collectibles.entry(collectible), count); + e.insert(collectibles_info); + } + } + } + #[cfg(feature = "track_unfinished")] + if info.unfinished > 0 && info.unfinished_tasks.is_empty() + || info.unfinished == 0 && !info.unfinished_tasks.is_empty() + { + panic!( + "inconsistent state: unfinished {}, unfinished_tasks {:?}, change {:?}", + info.unfinished, info.unfinished_tasks, change + ); + } + let new_change = TaskChange { + unfinished, + #[cfg(feature = "track_unfinished")] + unfinished_tasks_update: change.unfinished_tasks_update.clone(), + dirty_tasks_update: change.dirty_tasks_update.clone(), + collectibles: change.collectibles.clone(), + }; + if new_change.is_empty() { + None + } else { + Some(new_change) + } + } + + fn info_to_add_change(&self, info: &Aggregated) -> Option { + let mut change = TaskChange::default(); + if info.unfinished > 0 { + change.unfinished = 1; + } + #[cfg(feature = "track_unfinished")] + for (&task, &count) in info.unfinished_tasks.iter() { + change.unfinished_tasks_update.push((task, count)); + } + for (&task, &count) in info.dirty_tasks.iter() { + change.dirty_tasks_update.push((task, count)); + } + for (trait_type_id, collectibles_info) in info.collectibles.iter() { + for (collectible, count) in collectibles_info.collectibles.iter() { + change + .collectibles + .push((*trait_type_id, *collectible, *count)); + } + } + if change.is_empty() { + None + } else { + Some(change) + } + } + + fn info_to_remove_change(&self, info: &Aggregated) -> Option { + let mut change = TaskChange::default(); + if info.unfinished > 0 { + change.unfinished = -1; + } + #[cfg(feature = "track_unfinished")] + for (&task, &count) in info.unfinished_tasks.iter() { + change.unfinished_tasks_update.push((task, -count)); + } + for (&task, &count) in info.dirty_tasks.iter() { + change.dirty_tasks_update.push((task, -count)); + } + for (trait_type_id, collectibles_info) in info.collectibles.iter() { + for (collectible, count) in collectibles_info.collectibles.iter() { + change + .collectibles + .push((*trait_type_id, *collectible, -*count)); + } + } + if change.is_empty() { + None + } else { + Some(change) + } + } + + fn new_root_info(&self, _root_info_type: &RootInfoType) -> Self::RootInfo { + false + } + + fn info_to_root_info( + &self, + info: &Aggregated, + root_info_type: &RootInfoType, + ) -> Self::RootInfo { + match root_info_type { + RootInfoType::IsActive => info.root_type.is_some(), + } + } + + fn merge_root_info( + &self, + root_info: &mut Self::RootInfo, + other: Self::RootInfo, + ) -> std::ops::ControlFlow<()> { + if other { + *root_info = true; + std::ops::ControlFlow::Break(()) + } else { + std::ops::ControlFlow::Continue(()) + } + } +} + +pub struct TaskGuard<'l> { + pub(super) id: TaskId, + pub(super) guard: TaskMetaStateWriteGuard<'l>, +} + +impl<'l> AggregationItemLock for TaskGuard<'l> { + type Info = Aggregated; + type ItemRef = TaskId; + type ItemChange = TaskChange; + type ChildrenIter<'a> = impl Iterator> + 'a where Self: 'a; + + fn leaf(&mut self) -> &mut AggregationTreeLeaf { + self.guard.ensure_at_least_partial(); + match self.guard { + TaskMetaStateWriteGuard::Full(ref mut guard) => &mut guard.aggregation_leaf, + TaskMetaStateWriteGuard::Partial(ref mut guard) => &mut guard.aggregation_leaf, + TaskMetaStateWriteGuard::Unloaded(_) => unreachable!(), + } + } + + fn reference(&self) -> &Self::ItemRef { + &self.id + } + + fn number_of_children(&self) -> usize { + match self.guard { + TaskMetaStateWriteGuard::Full(ref guard) => guard.children.len(), + TaskMetaStateWriteGuard::Partial(_) | TaskMetaStateWriteGuard::Unloaded(_) => 0, + } + } + + fn children(&self) -> Self::ChildrenIter<'_> { + match self.guard { + TaskMetaStateWriteGuard::Full(ref guard) => { + Some(guard.children.iter().map(Cow::Borrowed)) + .into_iter() + .flatten() + } + TaskMetaStateWriteGuard::Partial(_) | TaskMetaStateWriteGuard::Unloaded(_) => { + None.into_iter().flatten() + } + } + } + + fn get_add_change(&self) -> Option { + match self.guard { + TaskMetaStateWriteGuard::Full(ref guard) => { + let mut change = TaskChange::default(); + if !matches!( + guard.state_type, + TaskStateType::Done { .. } + | TaskStateType::InProgress { + count_as_finished: true, + .. + } + ) { + change.unfinished = 1; + #[cfg(feature = "track_unfinished")] + change.unfinished_tasks_update.push((self.id, 1)); + } + if matches!(guard.state_type, TaskStateType::Dirty { .. }) { + change.dirty_tasks_update.push((self.id, 1)); + } + if let Some(collectibles) = guard.collectibles.as_ref() { + for (&(trait_type_id, collectible), _) in collectibles.iter() { + change.collectibles.push((trait_type_id, collectible, 1)); + } + } + if change.is_empty() { + None + } else { + Some(change) + } + } + TaskMetaStateWriteGuard::Partial(_) | TaskMetaStateWriteGuard::Unloaded(_) => None, + } + } + + fn get_remove_change(&self) -> Option { + match self.guard { + TaskMetaStateWriteGuard::Full(ref guard) => { + let mut change = TaskChange::default(); + if !matches!( + guard.state_type, + TaskStateType::Done { .. } + | TaskStateType::InProgress { + count_as_finished: true, + .. + } + ) { + change.unfinished = -1; + #[cfg(feature = "track_unfinished")] + change.unfinished_tasks_update.push((self.id, -1)); + } + if matches!(guard.state_type, TaskStateType::Dirty { .. }) { + change.dirty_tasks_update.push((self.id, -1)); + } + if let Some(collectibles) = guard.collectibles.as_ref() { + for (&(trait_type_id, collectible), _) in collectibles.iter() { + change.collectibles.push((trait_type_id, collectible, -1)); + } + } + if change.is_empty() { + None + } else { + Some(change) + } + } + TaskMetaStateWriteGuard::Partial(_) | TaskMetaStateWriteGuard::Unloaded(_) => None, + } + } +} + +pub type TaskAggregationTreeLeaf = AggregationTreeLeaf; + +fn update_count_entry( + entry: Entry<'_, K, i32, H>, + update: i32, +) -> i32 { + match entry { + Entry::Occupied(mut e) => { + let value = e.get_mut(); + *value += update; + if *value == 0 { + e.remove(); + 0 + } else { + *value + } + } + Entry::Vacant(e) => { + e.insert(update); + update + } + } +} diff --git a/crates/turbo-tasks-memory/src/task/meta_state.rs b/crates/turbo-tasks-memory/src/task/meta_state.rs index 349adf333856a..3a62ba060c2c9 100644 --- a/crates/turbo-tasks-memory/src/task/meta_state.rs +++ b/crates/turbo-tasks-memory/src/task/meta_state.rs @@ -1,16 +1,10 @@ use std::mem::replace; -use auto_hash_map::AutoSet; -use nohash_hasher::BuildNoHashHasher; -use once_cell::sync::Lazy; use parking_lot::{RwLockReadGuard, RwLockWriteGuard}; -use turbo_tasks::{StatsType, TaskId}; +use turbo_tasks::StatsType; use super::{PartialTaskState, Task, TaskState, UnloadedTaskState}; -use crate::{ - map_guard::{ReadGuard, WriteGuard}, - scope::TaskScopes, -}; +use crate::map_guard::{ReadGuard, WriteGuard}; pub(super) enum TaskMetaState { Full(Box), @@ -242,31 +236,6 @@ impl<'a> TaskMetaStateWriteGuard<'a> { } } - pub(super) fn scopes_and_children( - &mut self, - ) -> (&mut TaskScopes, &AutoSet>) { - match self { - TaskMetaStateWriteGuard::Full(state) => { - let TaskState { - ref mut scopes, - ref children, - .. - } = **state; - (scopes, children) - } - TaskMetaStateWriteGuard::Partial(state) => { - let PartialTaskState { ref mut scopes, .. } = **state; - static EMPTY: Lazy>> = - Lazy::new(AutoSet::default); - (scopes, &*EMPTY) - } - TaskMetaStateWriteGuard::Unloaded(_) => unreachable!( - "TaskMetaStateWriteGuard::scopes_and_children must be called with at least a \ - partial state" - ), - } - } - pub(super) fn as_full_mut(&mut self) -> Option<&mut TaskState> { match self { TaskMetaStateWriteGuard::Full(state) => Some(&mut **state), @@ -281,4 +250,10 @@ impl<'a> TaskMetaStateWriteGuard<'a> { TaskMetaStateWriteGuard::Unloaded(state) => state.into_inner(), } } + + pub(super) fn ensure_at_least_partial(&mut self) { + if let TaskMetaStateWriteGuard::Unloaded(_state) = self { + todo!() + } + } } diff --git a/crates/turbo-tasks-memory/src/viz/graph.rs b/crates/turbo-tasks-memory/src/viz/graph.rs index e5af0b8e368fd..dfde898c206b9 100644 --- a/crates/turbo-tasks-memory/src/viz/graph.rs +++ b/crates/turbo-tasks-memory/src/viz/graph.rs @@ -248,12 +248,6 @@ fn get_task_label( } else { ("N/A".to_string(), "#ffffff".to_string()) }; - let roots = as_frac(stats.roots, max_values.roots); - let max_scopes = max_values.scopes.saturating_sub(100); - let scopes = as_frac( - (100 * stats.scopes / stats.count).saturating_sub(100), - max_scopes, - ); let full_stats_disclaimer = if stats_type.is_full() { "".to_string() @@ -282,11 +276,6 @@ fn get_task_label( {} {} - - scopes - {} roots - avg {} - {} >", total_color, @@ -299,10 +288,6 @@ fn get_task_label( total_millis, avg_color, avg_label, - as_color(roots), - stats.roots, - as_color(scopes), - (100 * stats.scopes / stats.count) as f32 / 100.0, full_stats_disclaimer ) } diff --git a/crates/turbo-tasks-memory/src/viz/mod.rs b/crates/turbo-tasks-memory/src/viz/mod.rs index eb20275ab4c43..8656c812c4811 100644 --- a/crates/turbo-tasks-memory/src/viz/mod.rs +++ b/crates/turbo-tasks-memory/src/viz/mod.rs @@ -35,12 +35,8 @@ struct MaxValues { pub avg_duration: Option, pub max_duration: Duration, pub count: usize, - pub active_count: usize, pub unloaded_count: usize, pub updates: Option, - pub roots: usize, - /// stored as scopes * 100 - pub scopes: usize, /// stored as dependencies * 100 pub dependencies: usize, /// stored as children * 100 @@ -81,11 +77,8 @@ fn get_max_values_internal(depth: u32, node: &GroupTree) -> MaxValues { let mut max_avg_duration = None; let mut max_max_duration = Duration::ZERO; let mut max_count = 0; - let mut max_active_count = 0; let mut max_unloaded_count = 0; let mut max_updates = None; - let mut max_roots = 0; - let mut max_scopes = 0; let mut max_dependencies = 0; let mut max_children = 0; let mut max_depth = 0; @@ -107,7 +100,6 @@ fn get_max_values_internal(depth: u32, node: &GroupTree) -> MaxValues { } max_max_duration = max(max_max_duration, s.max_duration); max_count = max(max_count, s.count); - max_active_count = max(max_active_count, s.active_count); max_unloaded_count = max(max_unloaded_count, s.unloaded_count); if let Some(executions) = s.executions { let updates = (executions as usize).saturating_sub(s.count); @@ -115,8 +107,6 @@ fn get_max_values_internal(depth: u32, node: &GroupTree) -> MaxValues { .map(|max_updates| max(max_updates, updates)) .or(Some(updates)); } - max_roots = max(max_roots, s.roots); - max_scopes = max(max_scopes, 100 * s.scopes / s.count); max_dependencies = max(max_dependencies, get_avg_dependencies_count_times_100(s)); max_children = max(max_children, get_avg_children_count_times_100(s)); } @@ -136,11 +126,8 @@ fn get_max_values_internal(depth: u32, node: &GroupTree) -> MaxValues { avg_duration, max_duration, count, - active_count, unloaded_count, updates, - roots, - scopes, dependencies, children, depth: inner_depth, @@ -153,11 +140,8 @@ fn get_max_values_internal(depth: u32, node: &GroupTree) -> MaxValues { max_avg_duration = max_avg_duration.zip(avg_duration).map(|(a, b)| max(a, b)); max_max_duration = max(max_max_duration, max_duration); max_count = max(max_count, count); - max_active_count = max(max_active_count, active_count); max_unloaded_count = max(max_unloaded_count, unloaded_count); max_updates = max_updates.zip(updates).map(|(a, b)| max(a, b)); - max_roots = max(max_roots, roots); - max_scopes = max(max_scopes, scopes); max_dependencies = max(max_dependencies, dependencies); max_children = max(max_children, children); max_depth = max(max_depth, inner_depth); @@ -169,11 +153,8 @@ fn get_max_values_internal(depth: u32, node: &GroupTree) -> MaxValues { avg_duration: max_avg_duration, max_duration: max_max_duration, count: max_count, - active_count: max_active_count, unloaded_count: max_unloaded_count, updates: max_updates, - roots: max_roots, - scopes: max_scopes, dependencies: max_dependencies, children: max_children, depth: max_depth, diff --git a/crates/turbo-tasks-memory/src/viz/table.rs b/crates/turbo-tasks-memory/src/viz/table.rs index 3eff32cfa9326..a8de44354471f 100644 --- a/crates/turbo-tasks-memory/src/viz/table.rs +++ b/crates/turbo-tasks-memory/src/viz/table.rs @@ -42,7 +42,6 @@ pub fn create_table(root: GroupTree, stats_type: StatsType) -> String { out += r#""#; out += r#""#; out += r#""#; - out += r#""#; out += r#""#; out += r#""#; out += r#""#; @@ -50,8 +49,6 @@ pub fn create_table(root: GroupTree, stats_type: StatsType) -> String { out += r#""#; out += r#""#; out += r#""#; - out += r#""#; - out += r#""#; out += r#""#; out += r#""#; out += r#""#; @@ -83,13 +80,6 @@ pub fn create_table(root: GroupTree, stats_type: StatsType) -> String { as_frac_color(stats.count, max_values.count), stats.count )?; - // active - write!( - out, - "", - as_frac_color(stats.active_count, max_values.active_count), - stats.active_count - )?; // unloaded write!( out, @@ -189,24 +179,6 @@ pub fn create_table(root: GroupTree, stats_type: StatsType) -> String { stats.max_duration.as_micros(), FormatDuration(stats.max_duration) )?; - // root scopes - write!( - out, - "", - as_frac_color(stats.roots, max_values.roots), - stats.roots - )?; - // avg scopes - let max_scopes = max_values.scopes.saturating_sub(100); - write!( - out, - "", - as_frac_color( - (100 * stats.scopes / stats.count).saturating_sub(100), - max_scopes - ), - (100 * stats.scopes / stats.count) as f32 / 100.0 - )?; // avg dependencies let dependencies = get_avg_dependencies_count_times_100(stats); write!( diff --git a/crates/turbo-tasks-memory/tests/collectibles.rs b/crates/turbo-tasks-memory/tests/collectibles.rs index c8860feee10fd..0a76da21564c5 100644 --- a/crates/turbo-tasks-memory/tests/collectibles.rs +++ b/crates/turbo-tasks-memory/tests/collectibles.rs @@ -4,6 +4,7 @@ use std::{collections::HashSet, time::Duration}; use anyhow::Result; +use auto_hash_map::AutoSet; use tokio::time::sleep; use turbo_tasks::{emit, CollectiblesSource, ValueToString, Vc}; use turbo_tasks_testing::{register, run}; @@ -13,7 +14,8 @@ register!(); async fn transitive_emitting() { run! { let result = my_transitive_emitting_function("".to_string(), "".to_string()); - let list = result.peek_collectibles::>().await?; + result.strongly_consistent().await?; + let list = result.peek_collectibles::>(); assert_eq!(list.len(), 2); let mut expected = ["123", "42"].into_iter().collect::>(); for collectible in list { @@ -23,11 +25,27 @@ async fn transitive_emitting() { } } +#[tokio::test] +async fn transitive_emitting_indirect() { + run! { + let result = my_transitive_emitting_function("".to_string(), "".to_string()); + let collectibles = my_transitive_emitting_function_collectibles("".to_string(), "".to_string()); + let list = collectibles.strongly_consistent().await?; + assert_eq!(list.len(), 2); + let mut expected = ["123", "42"].into_iter().collect::>(); + for collectible in list.iter() { + assert!(expected.remove(collectible.to_string().await?.as_str())) + } + assert_eq!(result.await?.0, 0); + } +} + #[tokio::test] async fn multi_emitting() { run! { let result = my_multi_emitting_function(); - let list = result.peek_collectibles::>().await?; + result.strongly_consistent().await?; + let list = result.peek_collectibles::>(); assert_eq!(list.len(), 2); let mut expected = ["123", "42"].into_iter().collect::>(); for collectible in list { @@ -41,7 +59,7 @@ async fn multi_emitting() { async fn taking_collectibles() { run! { let result = my_collecting_function(); - let list = result.take_collectibles::>().await?; + let list = result.take_collectibles::>(); // my_collecting_function already processed the collectibles so the list should // be empty assert!(list.is_empty()); @@ -53,7 +71,8 @@ async fn taking_collectibles() { async fn taking_collectibles_extra_layer() { run! { let result = my_collecting_function_indirect(); - let list = result.take_collectibles::>().await?; + result.strongly_consistent().await?; + let list = result.take_collectibles::>(); // my_collecting_function already processed the collectibles so the list should // be empty assert!(list.is_empty()); @@ -65,43 +84,52 @@ async fn taking_collectibles_extra_layer() { async fn taking_collectibles_parallel() { run! { let result = my_transitive_emitting_function("".to_string(), "a".to_string()); - let list = result.take_collectibles::>().await?; + result.strongly_consistent().await?; + let list = result.take_collectibles::>(); assert_eq!(list.len(), 2); assert_eq!(result.await?.0, 0); let result = my_transitive_emitting_function("".to_string(), "b".to_string()); - let list = result.take_collectibles::>().await?; + result.strongly_consistent().await?; + let list = result.take_collectibles::>(); assert_eq!(list.len(), 2); assert_eq!(result.await?.0, 0); let result = my_transitive_emitting_function_with_child_scope("".to_string(), "b".to_string(), "1".to_string()); - let list = result.take_collectibles::>().await?; + result.strongly_consistent().await?; + let list = result.take_collectibles::>(); assert_eq!(list.len(), 2); assert_eq!(result.await?.0, 0); let result = my_transitive_emitting_function_with_child_scope("".to_string(), "b".to_string(), "2".to_string()); - let list = result.take_collectibles::>().await?; + result.strongly_consistent().await?; + let list = result.take_collectibles::>(); assert_eq!(list.len(), 2); assert_eq!(result.await?.0, 0); let result = my_transitive_emitting_function_with_child_scope("".to_string(), "c".to_string(), "3".to_string()); - let list = result.take_collectibles::>().await?; + result.strongly_consistent().await?; + let list = result.take_collectibles::>(); assert_eq!(list.len(), 2); assert_eq!(result.await?.0, 0); } } +#[turbo_tasks::value(transparent)] +struct Collectibles(AutoSet>>); + #[turbo_tasks::function] async fn my_collecting_function() -> Result> { let result = my_transitive_emitting_function("".to_string(), "".to_string()); - result.take_collectibles::>().await?; + result.take_collectibles::>(); Ok(result) } #[turbo_tasks::function] async fn my_collecting_function_indirect() -> Result> { let result = my_collecting_function(); - let list = result.peek_collectibles::>().await?; + result.strongly_consistent().await?; + let list = result.peek_collectibles::>(); // my_collecting_function already processed the collectibles so the list should // be empty assert!(list.is_empty()); @@ -122,6 +150,15 @@ async fn my_transitive_emitting_function(key: String, _key2: String) -> Result Vc { + let result = my_transitive_emitting_function(key, key2); + Vc::cell(result.peek_collectibles::>()) +} + #[turbo_tasks::function] async fn my_transitive_emitting_function_with_child_scope( key: String, @@ -129,7 +166,8 @@ async fn my_transitive_emitting_function_with_child_scope( _key3: String, ) -> Result> { let thing = my_transitive_emitting_function(key, key2); - let list = thing.peek_collectibles::>().await?; + thing.strongly_consistent().await?; + let list = thing.peek_collectibles::>(); assert_eq!(list.len(), 2); Ok(thing) } diff --git a/crates/turbo-tasks-memory/tests/scope_stress.rs b/crates/turbo-tasks-memory/tests/scope_stress.rs new file mode 100644 index 0000000000000..dea2b23327ad8 --- /dev/null +++ b/crates/turbo-tasks-memory/tests/scope_stress.rs @@ -0,0 +1,51 @@ +#![feature(arbitrary_self_types)] +#![feature(async_fn_in_trait)] + +use anyhow::Result; +use turbo_tasks::{Completion, TryJoinIterExt, TurboTasks, Vc}; +use turbo_tasks_memory::MemoryBackend; +use turbo_tasks_testing::register; + +register!(); + +#[test] +fn rectangle_stress() { + *REGISTER; + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + let tt = TurboTasks::new(MemoryBackend::default()); + let size = 100; + (0..size) + .map(|a| (a, size - 1)) + .chain((0..size - 1).map(|b| (size - 1, b))) + .map(|(a, b)| { + let tt = &tt; + async move { + let task = tt.spawn_once_task(async move { + rectangle(a, b).strongly_consistent().await?; + Ok(Vc::<()>::default()) + }); + tt.wait_task_completion(task, false).await + } + }) + .try_join() + .await + .unwrap(); + }) +} + +/// This fills a rectagle from (0, 0) to (a, b) by +/// first filling (0, 0) to (a - 1, b) and then (0, 0) to (a, b - 1) recursively +#[turbo_tasks::function] +async fn rectangle(a: u32, b: u32) -> Result> { + if a > 0 { + rectangle(a - 1, b).await?; + } + if b > 0 { + rectangle(a, b - 1).await?; + } + Ok(Completion::new()) +} diff --git a/crates/turbo-tasks-testing/src/lib.rs b/crates/turbo-tasks-testing/src/lib.rs index b03f5c053a9de..88e4f16f442f8 100644 --- a/crates/turbo-tasks-testing/src/lib.rs +++ b/crates/turbo-tasks-testing/src/lib.rs @@ -13,7 +13,7 @@ use std::{ }; use anyhow::{anyhow, Result}; -use auto_hash_map::AutoSet; +use auto_hash_map::AutoMap; use futures::FutureExt; use turbo_tasks::{ backend::CellContent, @@ -21,7 +21,7 @@ use turbo_tasks::{ registry, test_helpers::with_turbo_tasks_for_testing, util::{SharedError, StaticOrArc}, - CellId, InvalidationReason, RawVc, TaskId, TraitTypeId, TurboTasksApi, TurboTasksCallApi, Vc, + CellId, InvalidationReason, RawVc, TaskId, TraitTypeId, TurboTasksApi, TurboTasksCallApi, }; enum Task { @@ -204,19 +204,24 @@ impl TurboTasksApi for VcStorage { unimplemented!() } - fn unemit_collectible(&self, _trait_type: turbo_tasks::TraitTypeId, _collectible: RawVc) { + fn unemit_collectible( + &self, + _trait_type: turbo_tasks::TraitTypeId, + _collectible: RawVc, + _count: u32, + ) { unimplemented!() } fn unemit_collectibles( &self, _trait_type: turbo_tasks::TraitTypeId, - _collectibles: &AutoSet, + _collectibles: &AutoMap, ) { unimplemented!() } - fn read_task_collectibles(&self, _task: TaskId, _trait_id: TraitTypeId) -> Vc> { + fn read_task_collectibles(&self, _task: TaskId, _trait_id: TraitTypeId) -> AutoMap { unimplemented!() } diff --git a/crates/turbo-tasks/src/backend.rs b/crates/turbo-tasks/src/backend.rs index 5d2947afdd5f5..d773e7109539d 100644 --- a/crates/turbo-tasks/src/backend.rs +++ b/crates/turbo-tasks/src/backend.rs @@ -10,7 +10,7 @@ use std::{ }; use anyhow::{anyhow, bail, Result}; -use auto_hash_map::AutoSet; +use auto_hash_map::{AutoMap, AutoSet}; use nohash_hasher::BuildNoHashHasher; use serde::{Deserialize, Serialize}; @@ -18,7 +18,7 @@ pub use crate::id::BackendJobId; use crate::{ event::EventListener, manager::TurboTasksBackendApi, raw_vc::CellId, registry, ConcreteTaskInput, FunctionId, RawVc, ReadRef, SharedReference, TaskId, TaskIdProvider, - TraitRef, TraitTypeId, Vc, VcValueTrait, VcValueType, + TraitRef, TraitTypeId, VcValueTrait, VcValueType, }; pub enum TaskType { @@ -294,7 +294,7 @@ pub trait Backend: Sync + Send { trait_id: TraitTypeId, reader: TaskId, turbo_tasks: &dyn TurboTasksBackendApi, - ) -> Vc>; + ) -> AutoMap; fn emit_collectible( &self, @@ -308,6 +308,7 @@ pub trait Backend: Sync + Send { &self, trait_type: TraitTypeId, collectible: RawVc, + count: u32, task: TaskId, turbo_tasks: &dyn TurboTasksBackendApi, ); @@ -347,6 +348,8 @@ pub trait Backend: Sync + Send { task_type: TransientTaskType, turbo_tasks: &dyn TurboTasksBackendApi, ) -> TaskId; + + fn dispose_root_task(&self, task: TaskId, turbo_tasks: &dyn TurboTasksBackendApi); } impl PersistentTaskType { diff --git a/crates/turbo-tasks/src/collectibles.rs b/crates/turbo-tasks/src/collectibles.rs index cea8f11513fea..e4f9aed0de1ae 100644 --- a/crates/turbo-tasks/src/collectibles.rs +++ b/crates/turbo-tasks/src/collectibles.rs @@ -1,6 +1,8 @@ -use crate::{CollectiblesFuture, VcValueTrait}; +use auto_hash_map::AutoSet; + +use crate::{Vc, VcValueTrait}; pub trait CollectiblesSource { - fn take_collectibles(self) -> CollectiblesFuture; - fn peek_collectibles(self) -> CollectiblesFuture; + fn take_collectibles(self) -> AutoSet>; + fn peek_collectibles(self) -> AutoSet>; } diff --git a/crates/turbo-tasks/src/lib.rs b/crates/turbo-tasks/src/lib.rs index b1c63a7413903..a6e79cef6d6aa 100644 --- a/crates/turbo-tasks/src/lib.rs +++ b/crates/turbo-tasks/src/lib.rs @@ -91,7 +91,7 @@ pub use manager::{ Unused, UpdateInfo, }; pub use native_function::NativeFunction; -pub use raw_vc::{CellId, CollectiblesFuture, RawVc, ReadRawVcFuture, ResolveTypeError}; +pub use raw_vc::{CellId, RawVc, ReadRawVcFuture, ResolveTypeError}; pub use read_ref::ReadRef; pub use state::State; pub use task::{ diff --git a/crates/turbo-tasks/src/manager.rs b/crates/turbo-tasks/src/manager.rs index f4791dc31eb8e..e87f56dbf59c4 100644 --- a/crates/turbo-tasks/src/manager.rs +++ b/crates/turbo-tasks/src/manager.rs @@ -16,7 +16,7 @@ use std::{ }; use anyhow::{anyhow, Result}; -use auto_hash_map::AutoSet; +use auto_hash_map::{AutoMap, AutoSet}; use futures::FutureExt; use nohash_hasher::BuildNoHashHasher; use serde::{de::Visitor, Deserialize, Serialize}; @@ -102,11 +102,11 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send { index: CellId, ) -> Result>; - fn read_task_collectibles(&self, task: TaskId, trait_id: TraitTypeId) -> Vc>; + fn read_task_collectibles(&self, task: TaskId, trait_id: TraitTypeId) -> AutoMap; fn emit_collectible(&self, trait_type: TraitTypeId, collectible: RawVc); - fn unemit_collectible(&self, trait_type: TraitTypeId, collectible: RawVc); - fn unemit_collectibles(&self, trait_type: TraitTypeId, collectibles: &AutoSet); + fn unemit_collectible(&self, trait_type: TraitTypeId, collectible: RawVc, count: u32); + fn unemit_collectibles(&self, trait_type: TraitTypeId, collectibles: &AutoMap); /// INVALIDATION: Be careful with this, it will not track dependencies, so /// using it could break cache invalidation. @@ -356,6 +356,10 @@ impl TurboTasks { id } + pub fn dispose_root_task(&self, task_id: TaskId) { + self.backend.dispose_root_task(task_id, self); + } + // TODO make sure that all dependencies settle before reading them /// Creates a new root task, that is only executed once. /// Dependencies will not invalidate the task. @@ -957,7 +961,7 @@ impl TurboTasksApi for TurboTasks { .try_read_own_task_cell_untracked(current_task, index, self) } - fn read_task_collectibles(&self, task: TaskId, trait_id: TraitTypeId) -> Vc> { + fn read_task_collectibles(&self, task: TaskId, trait_id: TraitTypeId) -> AutoMap { self.backend.read_task_collectibles( task, trait_id, @@ -975,23 +979,27 @@ impl TurboTasksApi for TurboTasks { ); } - fn unemit_collectible(&self, trait_type: TraitTypeId, collectible: RawVc) { + fn unemit_collectible(&self, trait_type: TraitTypeId, collectible: RawVc, count: u32) { self.backend.unemit_collectible( trait_type, collectible, + count, current_task("emitting collectible"), self, ); } - fn unemit_collectibles(&self, trait_type: TraitTypeId, collectibles: &AutoSet) { - for collectible in collectibles { - self.backend.unemit_collectible( - trait_type, - *collectible, - current_task("emitting collectible"), - self, - ); + fn unemit_collectibles(&self, trait_type: TraitTypeId, collectibles: &AutoMap) { + for (&collectible, &count) in collectibles { + if count > 0 { + self.backend.unemit_collectible( + trait_type, + collectible, + count as u32, + current_task("emitting collectible"), + self, + ); + } } } diff --git a/crates/turbo-tasks/src/raw_vc.rs b/crates/turbo-tasks/src/raw_vc.rs index 63d15ac888c37..feb5ab4c961ba 100644 --- a/crates/turbo-tasks/src/raw_vc.rs +++ b/crates/turbo-tasks/src/raw_vc.rs @@ -1,7 +1,7 @@ use std::{ any::Any, fmt::{Debug, Display}, - future::{Future, IntoFuture}, + future::Future, hash::Hash, marker::PhantomData, pin::Pin, @@ -233,28 +233,23 @@ impl RawVc { } impl CollectiblesSource for RawVc { - fn peek_collectibles(self) -> CollectiblesFuture { + fn peek_collectibles(self) -> AutoSet> { let tt = turbo_tasks(); tt.notify_scheduled_tasks(); - let set = tt.read_task_collectibles(self.get_task_id(), T::get_trait_type_id()); - CollectiblesFuture { - turbo_tasks: tt, - inner: set.into_future(), - take: false, - phantom: PhantomData, - } + let map = tt.read_task_collectibles(self.get_task_id(), T::get_trait_type_id()); + map.into_iter() + .filter_map(|(raw, count)| (count > 0).then_some(raw.into())) + .collect() } - fn take_collectibles(self) -> CollectiblesFuture { + fn take_collectibles(self) -> AutoSet> { let tt = turbo_tasks(); tt.notify_scheduled_tasks(); - let set = tt.read_task_collectibles(self.get_task_id(), T::get_trait_type_id()); - CollectiblesFuture { - turbo_tasks: tt, - inner: set.into_future(), - take: true, - phantom: PhantomData, - } + let map = tt.read_task_collectibles(self.get_task_id(), T::get_trait_type_id()); + tt.unemit_collectibles(T::get_trait_type_id(), &map); + map.into_iter() + .filter_map(|(raw, count)| (count > 0).then_some(raw.into())) + .collect() } } @@ -375,6 +370,9 @@ impl Future for ReadRawVcFuture { }; match read_result { Ok(Ok(vc)) => { + // We no longer need to read strongly consistent, as any Vc returned + // from the first task will be inside of the scope of the first task. So + // it's already strongly consistent. this.strongly_consistent = false; this.current = vc; continue 'outer; @@ -415,47 +413,3 @@ unsafe impl Send for ReadRawVcFuture where T: ?Sized {} unsafe impl Sync for ReadRawVcFuture where T: ?Sized {} impl Unpin for ReadRawVcFuture where T: ?Sized {} - -#[derive(Error, Debug)] -#[error("Unable to read collectibles")] -pub struct ReadCollectiblesError { - source: anyhow::Error, -} - -pub struct CollectiblesFuture { - turbo_tasks: Arc, - inner: ReadRawVcFuture>, - take: bool, - phantom: PhantomData T>, -} - -impl CollectiblesFuture { - pub fn strongly_consistent(mut self) -> Self { - self.inner.strongly_consistent = true; - self - } -} - -impl Future for CollectiblesFuture { - type Output = Result>>; - - fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { - // SAFETY: we are not moving `this` - let this = unsafe { self.get_unchecked_mut() }; - // SAFETY: `this` was pinned before - let inner_pin = unsafe { Pin::new_unchecked(&mut this.inner) }; - match inner_pin.poll(cx) { - Poll::Ready(r) => Poll::Ready(match r { - Ok(set) => { - if this.take { - this.turbo_tasks - .unemit_collectibles(T::get_trait_type_id(), &set); - } - Ok(set.iter().map(|raw| (*raw).into()).collect()) - } - Err(e) => Err(ReadCollectiblesError { source: e }.into()), - }), - Poll::Pending => Poll::Pending, - } - } -} diff --git a/crates/turbo-tasks/src/vc/mod.rs b/crates/turbo-tasks/src/vc/mod.rs index bc17ee448f91f..e388c32254ea0 100644 --- a/crates/turbo-tasks/src/vc/mod.rs +++ b/crates/turbo-tasks/src/vc/mod.rs @@ -7,6 +7,7 @@ mod traits; use std::{any::Any, marker::PhantomData, ops::Deref}; use anyhow::Result; +use auto_hash_map::AutoSet; use serde::{Deserialize, Serialize}; use self::cell_mode::VcCellMode; @@ -21,7 +22,7 @@ use crate::{ debug::{ValueDebug, ValueDebugFormat, ValueDebugFormatString}, registry, trace::{TraceRawVcs, TraceRawVcsContext}, - CellId, CollectiblesFuture, CollectiblesSource, RawVc, ReadRawVcFuture, ResolveTypeError, + CellId, CollectiblesSource, RawVc, ReadRawVcFuture, ResolveTypeError, }; /// A Value Cell (`Vc` for short) is a reference to a memoized computation @@ -458,11 +459,11 @@ impl CollectiblesSource for Vc where T: ?Sized + Send, { - fn take_collectibles(self) -> CollectiblesFuture { + fn take_collectibles(self) -> AutoSet> { self.node.take_collectibles() } - fn peek_collectibles(self) -> CollectiblesFuture { + fn peek_collectibles(self) -> AutoSet> { self.node.peek_collectibles() } } diff --git a/crates/turbopack-cli-utils/src/issue.rs b/crates/turbopack-cli-utils/src/issue.rs index 16ee92b034328..34b589b58f900 100644 --- a/crates/turbopack-cli-utils/src/issue.rs +++ b/crates/turbopack-cli-utils/src/issue.rs @@ -349,7 +349,7 @@ impl IssueReporter for ConsoleUi { #[turbo_tasks::function] async fn report_issues( &self, - issues: TransientInstance>, + issues: TransientInstance, source: TransientValue, min_failing_severity: Vc, ) -> Result> { diff --git a/crates/turbopack-cli/src/dev/turbo_tasks_viz.rs b/crates/turbopack-cli/src/dev/turbo_tasks_viz.rs index 9450b59e6d032..bbd80f70b313e 100644 --- a/crates/turbopack-cli/src/dev/turbo_tasks_viz.rs +++ b/crates/turbopack-cli/src/dev/turbo_tasks_viz.rs @@ -117,12 +117,9 @@ impl GetContentSourceContent for TurboTasksSource { }; let mut stats = Stats::new(); let b = tt.backend(); - let active_only = query.contains_key("active"); let include_unloaded = query.contains_key("unloaded"); b.with_all_cached_tasks(|task| { - stats.add_id_conditional(b, task, |_, info| { - (include_unloaded || !info.unloaded) && (!active_only || info.active) - }); + stats.add_id_conditional(b, task, |_, info| include_unloaded || !info.unloaded); }); let tree = stats.treeify(ReferenceType::Dependency); let table = viz::table::create_table(tree, tt.stats_type()); diff --git a/crates/turbopack-core/src/diagnostics/mod.rs b/crates/turbopack-core/src/diagnostics/mod.rs index bdf0324275dcb..4c2a3b247a7d6 100644 --- a/crates/turbopack-core/src/diagnostics/mod.rs +++ b/crates/turbopack-core/src/diagnostics/mod.rs @@ -60,7 +60,7 @@ pub trait DiagnosticContextExt where Self: Sized, { - async fn peek_diagnostics(self) -> Result>; + async fn peek_diagnostics(self) -> Result; } #[async_trait] @@ -68,10 +68,10 @@ impl DiagnosticContextExt for T where T: CollectiblesSource + Copy + Send, { - async fn peek_diagnostics(self) -> Result> { - Ok(CapturedDiagnostics::cell(CapturedDiagnostics { - diagnostics: self.peek_collectibles().strongly_consistent().await?, - })) + async fn peek_diagnostics(self) -> Result { + Ok(CapturedDiagnostics { + diagnostics: self.peek_collectibles(), + }) } } diff --git a/crates/turbopack-core/src/issue/mod.rs b/crates/turbopack-core/src/issue/mod.rs index 146528071a679..4bb8b9fddcd04 100644 --- a/crates/turbopack-core/src/issue/mod.rs +++ b/crates/turbopack-core/src/issue/mod.rs @@ -328,7 +328,7 @@ pub struct Issues(Vec>>); /// A list of issues captured with [`Issue::peek_issues_with_path`] and /// [`Issue::take_issues_with_path`]. -#[turbo_tasks::value] +#[turbo_tasks::value(shared)] #[derive(Debug)] pub struct CapturedIssues { issues: AutoSet>>, @@ -617,7 +617,7 @@ pub trait IssueReporter { /// The minimum issue severity level considered to fatally end the program. fn report_issues( self: Vc, - issues: TransientInstance>, + issues: TransientInstance, source: TransientValue, min_failing_severity: Vc, ) -> Vc; @@ -647,13 +647,13 @@ where /// Returns all issues from `source` in a list with their associated /// processing path. - async fn peek_issues_with_path(self) -> Result>; + async fn peek_issues_with_path(self) -> Result; /// Returns all issues from `source` in a list with their associated /// processing path. /// /// This unemits the issues. They will not propagate up. - async fn take_issues_with_path(self) -> Result>; + async fn take_issues_with_path(self) -> Result; } #[async_trait] @@ -669,7 +669,7 @@ where ) -> Result { #[cfg(feature = "issue_path")] { - let children = self.take_collectibles().await?; + let children = self.take_collectibles(); if !children.is_empty() { emit(Vc::upcast::>( ItemIssueProcessingPath::cell(ItemIssueProcessingPath( @@ -697,7 +697,7 @@ where ) -> Result { #[cfg(feature = "issue_path")] { - let children = self.take_collectibles().await?; + let children = self.take_collectibles(); if !children.is_empty() { emit(Vc::upcast::>( ItemIssueProcessingPath::cell(ItemIssueProcessingPath( @@ -721,26 +721,26 @@ where self.issue_file_path(None, description).await } - async fn peek_issues_with_path(self) -> Result> { - Ok(CapturedIssues::cell(CapturedIssues { - issues: self.peek_collectibles().strongly_consistent().await?, + async fn peek_issues_with_path(self) -> Result { + Ok(CapturedIssues { + issues: self.peek_collectibles(), #[cfg(feature = "issue_path")] processing_path: ItemIssueProcessingPath::cell(ItemIssueProcessingPath( None, - self.peek_collectibles().strongly_consistent().await?, + self.peek_collectibles(), )), - })) + }) } - async fn take_issues_with_path(self) -> Result> { - Ok(CapturedIssues::cell(CapturedIssues { - issues: self.take_collectibles().strongly_consistent().await?, + async fn take_issues_with_path(self) -> Result { + Ok(CapturedIssues { + issues: self.take_collectibles(), #[cfg(feature = "issue_path")] processing_path: ItemIssueProcessingPath::cell(ItemIssueProcessingPath( None, - self.take_collectibles().strongly_consistent().await?, + self.take_collectibles(), )), - })) + }) } } @@ -751,14 +751,11 @@ pub async fn handle_issues( path: Option<&str>, operation: Option<&str>, ) -> Result<()> { - let issues = source - .peek_issues_with_path() - .await? - .strongly_consistent() - .await?; + let _ = source.resolve_strongly_consistent().await?; + let issues = source.peek_issues_with_path().await?; let has_fatal = issue_reporter.report_issues( - TransientInstance::new(issues.clone()), + TransientInstance::new(issues), TransientValue::new(Vc::into_raw(source)), min_failing_severity, ); diff --git a/crates/turbopack-dev-server/src/http.rs b/crates/turbopack-dev-server/src/http.rs index 9caae484387ac..5f6737f12db2b 100644 --- a/crates/turbopack-dev-server/src/http.rs +++ b/crates/turbopack-dev-server/src/http.rs @@ -80,6 +80,8 @@ pub async fn process_request_with_content_source( let original_path = request.uri().path().to_string(); let request = http_request_to_source_request(request).await?; let result = get_from_source(source, TransientInstance::new(request)); + let resolved_result = result.resolve_strongly_consistent().await?; + let side_effects: AutoSet>> = result.peek_collectibles(); handle_issues( result, issue_reporter, @@ -88,9 +90,7 @@ pub async fn process_request_with_content_source( Some("get_from_source"), ) .await?; - let side_effects: AutoSet>> = - result.peek_collectibles().strongly_consistent().await?; - match &*result.strongly_consistent().await? { + match &*resolved_result.await? { GetFromSourceResult::Static { content, status_code, @@ -211,7 +211,7 @@ pub async fn process_request_with_content_source( side_effects, )); } - _ => {} + GetFromSourceResult::NotFound => {} } Ok(( diff --git a/crates/turbopack-dev-server/src/introspect/mod.rs b/crates/turbopack-dev-server/src/introspect/mod.rs index bf5c24736b944..a1f97d72defe9 100644 --- a/crates/turbopack-dev-server/src/introspect/mod.rs +++ b/crates/turbopack-dev-server/src/introspect/mod.rs @@ -92,8 +92,8 @@ impl GetContentSourceContent for IntrospectionSource { path: String, _data: turbo_tasks::Value, ) -> Result> { - // ignore leading slash - let path = &path[1..]; + // get last segment + let path = &path[path.rfind('/').unwrap_or(0) + 1..]; let introspectable = if path.is_empty() { let roots = &self.await?.roots; if roots.len() == 1 { diff --git a/crates/turbopack-dev-server/src/lib.rs b/crates/turbopack-dev-server/src/lib.rs index 898a0056df23d..ca82d75f468e2 100644 --- a/crates/turbopack-dev-server/src/lib.rs +++ b/crates/turbopack-dev-server/src/lib.rs @@ -210,6 +210,7 @@ impl DevServerBuilder { let uri = request.uri(); let path = uri.path().to_string(); let source = source_provider.get_source(); + let resolved_source = source.resolve_strongly_consistent().await?; handle_issues( source, issue_reporter, @@ -218,7 +219,6 @@ impl DevServerBuilder { Some("get source"), ) .await?; - let resolved_source = source.resolve_strongly_consistent().await?; let (response, side_effects) = http::process_request_with_content_source( resolved_source, diff --git a/crates/turbopack-dev-server/src/source/asset_graph.rs b/crates/turbopack-dev-server/src/source/asset_graph.rs index 5109ef90514d8..cb8d174b30f03 100644 --- a/crates/turbopack-dev-server/src/source/asset_graph.rs +++ b/crates/turbopack-dev-server/src/source/asset_graph.rs @@ -1,11 +1,11 @@ use std::{ - collections::{HashMap, HashSet, VecDeque}, + collections::{HashSet, VecDeque}, iter::once, }; use anyhow::Result; -use indexmap::{indexset, IndexSet}; -use turbo_tasks::{Completion, State, Value, ValueToString, Vc}; +use indexmap::{indexset, IndexMap, IndexSet}; +use turbo_tasks::{Completion, State, TryJoinIterExt, Value, ValueToString, Vc}; use turbo_tasks_fs::FileSystemPath; use turbopack_core::{ asset::Asset, @@ -20,9 +20,9 @@ use super::{ }; #[turbo_tasks::value(transparent)] -struct OutputAssetsMap(HashMap>>); +struct OutputAssetsMap(IndexMap>>); -type ExpandedState = State>>>; +type ExpandedState = State>; #[turbo_tasks::value(serialization = "none", eq = "manual", cell = "new")] pub struct AssetGraphContentSource { @@ -105,93 +105,126 @@ async fn expand( root_assets: &IndexSet>>, root_path: &FileSystemPath, expanded: Option<&ExpandedState>, -) -> Result>>> { - let mut map = HashMap::new(); +) -> Result>>> { + let mut map = IndexMap::new(); let mut assets = Vec::new(); let mut queue = VecDeque::with_capacity(32); let mut assets_set = HashSet::new(); + let root_assets_with_path = root_assets + .iter() + .map(|&asset| async move { + let path = asset.ident().path().await?; + Ok((path, asset)) + }) + .try_join() + .await?; + if let Some(expanded) = &expanded { let expanded = expanded.get(); - for root_asset in root_assets.iter() { - let expanded = expanded.contains(root_asset); - assets.push((root_asset.ident().path(), *root_asset)); - assets_set.insert(*root_asset); - if expanded { - queue.push_back(root_asset.references()); + for (path, root_asset) in root_assets_with_path.into_iter() { + if let Some(sub_path) = root_path.get_path_to(&path) { + let (sub_paths_buffer, sub_paths) = get_sub_paths(sub_path); + let expanded = sub_paths_buffer + .iter() + .take(sub_paths) + .any(|sub_path| expanded.contains(sub_path)); + for sub_path in sub_paths_buffer.into_iter().take(sub_paths) { + assets.push((sub_path, root_asset)); + } + assets_set.insert(root_asset); + if expanded { + queue.push_back(root_asset.references()); + } } } } else { - for root_asset in root_assets.iter() { - assets.push((root_asset.ident().path(), *root_asset)); - assets_set.insert(*root_asset); - queue.push_back(root_asset.references()); + for (path, root_asset) in root_assets_with_path.into_iter() { + if let Some(sub_path) = root_path.get_path_to(&path) { + let (sub_paths_buffer, sub_paths) = get_sub_paths(sub_path); + for sub_path in sub_paths_buffer.into_iter().take(sub_paths) { + assets.push((sub_path, root_asset)); + } + queue.push_back(root_asset.references()); + assets_set.insert(root_asset); + } } } while let Some(references) = queue.pop_front() { for asset in references.await?.iter() { if assets_set.insert(*asset) { - let expanded = if let Some(expanded) = &expanded { - // We lookup the unresolved asset in the expanded set. - // We could resolve the asset here, but that would require waiting on the - // computation here and it doesn't seem to be neccessary in this case. We just - // have to be sure that we consistently use the unresolved asset. - expanded.get().contains(asset) - } else { - true - }; - if expanded { - queue.push_back(asset.references()); + let path = asset.ident().path().await?; + if let Some(sub_path) = root_path.get_path_to(&path) { + let (sub_paths_buffer, sub_paths) = get_sub_paths(sub_path); + let expanded = if let Some(expanded) = &expanded { + let expanded = expanded.get(); + sub_paths_buffer + .iter() + .take(sub_paths) + .any(|sub_path| expanded.contains(sub_path)) + } else { + true + }; + if expanded { + queue.push_back(asset.references()); + } + for sub_path in sub_paths_buffer.into_iter().take(sub_paths) { + assets.push((sub_path, *asset)); + } } - assets.push((asset.ident().path(), *asset)); } } } - for (p_vc, asset) in assets { - // For clippy -- This explicit deref is necessary - let p = &*p_vc.await?; - if let Some(sub_path) = root_path.get_path_to(p) { - map.insert(sub_path.to_string(), asset); - if sub_path == "index.html" { - map.insert("".to_string(), asset); - } else if let Some(p) = sub_path.strip_suffix("/index.html") { - map.insert(p.to_string(), asset); - map.insert(format!("{p}/"), asset); - } else if let Some(p) = sub_path.strip_suffix(".html") { - map.insert(p.to_string(), asset); - } + for (sub_path, asset) in assets { + let asset = asset.resolve().await?; + if sub_path == "index.html" { + map.insert("".to_string(), asset); + } else if let Some(p) = sub_path.strip_suffix("/index.html") { + map.insert(p.to_string(), asset); + map.insert(format!("{p}/"), asset); + } else if let Some(p) = sub_path.strip_suffix(".html") { + map.insert(p.to_string(), asset); } + map.insert(sub_path, asset); } Ok(map) } -/// A unresolve asset. We need to have a unresolve Asset here as we need to -/// lookup the Vc identity in the expanded set. -/// -/// This must not be a TaskInput since this would resolve the embedded asset. -#[turbo_tasks::value(serialization = "auto_for_input")] -#[derive(Hash, PartialOrd, Ord, Debug, Clone)] -struct UnresolvedAsset(Vc>); +fn get_sub_paths(sub_path: &str) -> ([String; 3], usize) { + let sub_paths_buffer: [String; 3]; + let n = if sub_path == "index.html" { + sub_paths_buffer = ["".to_string(), sub_path.to_string(), String::new()]; + 2 + } else if let Some(p) = sub_path.strip_suffix("/index.html") { + sub_paths_buffer = [p.to_string(), format!("{p}/"), sub_path.to_string()]; + 3 + } else if let Some(p) = sub_path.strip_suffix(".html") { + sub_paths_buffer = [p.to_string(), sub_path.to_string(), String::new()]; + 2 + } else { + sub_paths_buffer = [sub_path.to_string(), String::new(), String::new()]; + 1 + }; + (sub_paths_buffer, n) +} #[turbo_tasks::value_impl] impl ContentSource for AssetGraphContentSource { #[turbo_tasks::function] async fn get_routes(self: Vc) -> Result> { let assets = self.all_assets_map().strongly_consistent().await?; + let mut paths = Vec::new(); let routes = assets .iter() .map(|(path, asset)| { + paths.push(path.as_str()); RouteTree::new_route( BaseSegment::from_static_pathname(path).collect(), RouteType::Exact, Vc::upcast(AssetGraphGetContentSourceContent::new( - self, /* Passing the asset to a function would normally resolve that - * asset. But */ - // in this special case we want to avoid that and just pass the unresolved - // asset. So to enforce that we need to wrap it in this special value. - // Technically it would be preferable to have some kind of `#[unresolved]` - // attribute on function arguments, but we don't have that yet. - Value::new(UnresolvedAsset(*asset)), + self, + path.to_string(), + *asset, )), ) }) @@ -203,17 +236,22 @@ impl ContentSource for AssetGraphContentSource { #[turbo_tasks::value] struct AssetGraphGetContentSourceContent { source: Vc, - /// The unresolved asset. + path: String, asset: Vc>, } #[turbo_tasks::value_impl] impl AssetGraphGetContentSourceContent { #[turbo_tasks::function] - pub fn new(source: Vc, asset: Value) -> Vc { + pub fn new( + source: Vc, + path: String, + asset: Vc>, + ) -> Vc { Self::cell(AssetGraphGetContentSourceContent { source, - asset: asset.into_value().0, + path, + asset, }) } } @@ -241,11 +279,7 @@ impl ContentSourceSideEffect for AssetGraphGetContentSourceContent { let source = self.source.await?; if let Some(expanded) = &source.expanded { - let asset = self.asset; - expanded.update_conditionally(|expanded| { - // Insert the unresolved asset into the set - expanded.insert(asset) - }); + expanded.update_conditionally(|expanded| expanded.insert(self.path.to_string())); } Ok(Completion::new()) } diff --git a/crates/turbopack-dev-server/src/source/resolve.rs b/crates/turbopack-dev-server/src/source/resolve.rs index 5fd84f8a57695..88a4d3ab86371 100644 --- a/crates/turbopack-dev-server/src/source/resolve.rs +++ b/crates/turbopack-dev-server/src/source/resolve.rs @@ -103,7 +103,7 @@ pub async fn resolve_source_request( } } ContentSourceContent::NotFound => { - return Ok(ResolveSourceRequestResult::NotFound.cell()) + return Ok(ResolveSourceRequestResult::NotFound.cell()); } ContentSourceContent::Static(static_content) => { return Ok(ResolveSourceRequestResult::Static( diff --git a/crates/turbopack-dev-server/src/source/route_tree.rs b/crates/turbopack-dev-server/src/source/route_tree.rs index 2a8d1f97e22cc..b8d22f72d3365 100644 --- a/crates/turbopack-dev-server/src/source/route_tree.rs +++ b/crates/turbopack-dev-server/src/source/route_tree.rs @@ -186,7 +186,7 @@ impl RouteTree { #[turbo_tasks::value_impl] impl ValueToString for RouteTree { #[turbo_tasks::function] - fn to_string(&self) -> Result> { + async fn to_string(&self) -> Result> { let RouteTree { base, sources, @@ -206,8 +206,8 @@ impl ValueToString for RouteTree { if !base.is_empty() { result.push_str(", "); } - for key in static_segments.keys() { - write!(result, "{}, ", key)?; + for (key, tree) in static_segments { + write!(result, "{}: {}, ", key, tree.to_string().await?)?; } if !sources.is_empty() { write!(result, "{} x source, ", sources.len())?; diff --git a/crates/turbopack-dev-server/src/update/stream.rs b/crates/turbopack-dev-server/src/update/stream.rs index 488bbe61bf2f3..19f8729fe6856 100644 --- a/crates/turbopack-dev-server/src/update/stream.rs +++ b/crates/turbopack-dev-server/src/update/stream.rs @@ -24,7 +24,7 @@ use crate::source::{resolve::ResolveSourceRequestResult, ProxyResult}; type GetContentFn = Box Vc + Send + Sync>; async fn peek_issues(source: Vc) -> Result>> { - let captured = source.peek_issues_with_path().await?.await?; + let captured = source.peek_issues_with_path().await?; captured.get_plain_issues().await } @@ -46,6 +46,7 @@ async fn get_update_stream_item( get_content: TransientInstance, ) -> Result> { let content = get_content(); + let _ = content.resolve_strongly_consistent().await?; let mut plain_issues = peek_issues(content).await?; let content_value = match content.await { diff --git a/crates/turbopack-tests/tests/execution.rs b/crates/turbopack-tests/tests/execution.rs index ea05bf5e268fc..f10ce406db721 100644 --- a/crates/turbopack-tests/tests/execution.rs +++ b/crates/turbopack-tests/tests/execution.rs @@ -298,11 +298,8 @@ async fn run_test(resource: String) -> Result> { #[turbo_tasks::function] async fn snapshot_issues(run_result: Vc) -> Result> { - let captured_issues = run_result - .peek_issues_with_path() - .await? - .strongly_consistent() - .await?; + let _ = run_result.resolve_strongly_consistent().await?; + let captured_issues = run_result.peek_issues_with_path().await?; let RunTestResult { js_result: _, path } = *run_result.await?; diff --git a/crates/turbopack-tests/tests/snapshot.rs b/crates/turbopack-tests/tests/snapshot.rs index 0e3ee5ae34fc4..92f0028100344 100644 --- a/crates/turbopack-tests/tests/snapshot.rs +++ b/crates/turbopack-tests/tests/snapshot.rs @@ -143,11 +143,8 @@ async fn run(resource: PathBuf) -> Result<()> { let tt = TurboTasks::new(MemoryBackend::default()); let task = tt.spawn_once_task(async move { let out = run_test(resource.to_str().unwrap().to_string()); - let captured_issues = out - .peek_issues_with_path() - .await? - .strongly_consistent() - .await?; + let _ = out.resolve_strongly_consistent().await?; + let captured_issues = out.peek_issues_with_path().await?; let plain_issues = captured_issues .iter_with_shortest_path()
functioncountactiveunloadedreexecutionstotal durationtotal update durationavg durationmax durationroot scopesavg scopesavg dependenciesavg childrendepth{}{}{}