Merge branch 'canary' into add-statsig-app-router-example

.vscode/settings.json

@@ -96,5 +96,6 @@
     "--ignore-revs-file",
     "${workspaceRoot}/.git-blame-ignore-revs"
   ],
-  "astGrep.serverPath": "node_modules/@ast-grep/cli/ast-grep"
+  "astGrep.serverPath": "node_modules/@ast-grep/cli/ast-grep",
+  "rust-analyzer.imports.prefix": "crate"
 }

Cargo.toml

@@ -45,6 +45,7 @@ swc-ast-explorer = { path = "turbopack/crates/turbopack-swc-ast-explorer" }
 turbo-prehash = { path = "turbopack/crates/turbo-prehash" }
 turbo-tasks-malloc = { path = "turbopack/crates/turbo-tasks-malloc", default-features = false }
 turbo-tasks = { path = "turbopack/crates/turbo-tasks" }
+turbo-tasks-backend = { path = "turbopack/crates/turbo-tasks-backend" }
 turbo-tasks-build = { path = "turbopack/crates/turbo-tasks-build" }
 turbo-tasks-bytes = { path = "turbopack/crates/turbo-tasks-bytes" }
 turbo-tasks-env = { path = "turbopack/crates/turbo-tasks-env" }
@@ -143,6 +144,7 @@ dunce = "1.0.3"
 either = "1.9.0"
 futures = "0.3.26"
 futures-retry = "0.6.0"
+hashbrown = "0.14.5"
 httpmock = { version = "0.6.8", default-features = false }
 image = { version = "0.25.0", default-features = false }
 indexmap = "1.9.2"

crates/napi/Cargo.toml

@@ -37,6 +37,8 @@ __internal_dhat-heap = ["dhat"]
 # effectively does nothing.
 __internal_dhat-ad-hoc = ["dhat"]
 
+new-backend = ["dep:turbo-tasks-backend"]
+
 # Enable specific tls features per-target.
 [target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies]
 next-core = { workspace = true, features = ["native-tls"] }
@@ -105,6 +107,7 @@ lightningcss-napi = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 turbo-tasks = { workspace = true }
 turbo-tasks-memory = { workspace = true }
+turbo-tasks-backend = { workspace = true, optional = true }
 turbo-tasks-fs = { workspace = true }
 next-api = { workspace = true }
 next-build = { workspace = true }

crates/napi/src/next_api/project.rs

@@ -1,4 +1,4 @@
-use std::{io::Write, path::PathBuf, sync::Arc, thread, time::Duration};
+use std::{path::PathBuf, sync::Arc, thread, time::Duration};
 
 use anyhow::{anyhow, bail, Context, Result};
 use napi::{
@@ -24,7 +24,6 @@ use tracing::Instrument;
 use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry};
 use turbo_tasks::{Completion, RcStr, ReadRef, TransientInstance, TurboTasks, UpdateInfo, Vc};
 use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystem, FileSystemPath};
-use turbo_tasks_memory::MemoryBackend;
 use turbopack_core::{
     diagnostics::PlainDiagnostic,
     error::PrettyPrintError,
@@ -44,8 +43,8 @@ use url::Url;
 use super::{
     endpoint::ExternalEndpoint,
     utils::{
-        get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue, RootTask,
-        TurbopackResult, VcArc,
+        create_turbo_tasks, get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue,
+        NextBackend, RootTask, TurbopackResult, VcArc,
     },
 };
 use crate::register;
@@ -99,7 +98,7 @@ pub struct NapiProjectOptions {
 
     /// next.config's distDir. Project initialization occurs eariler than
     /// deserializing next.config, so passing it as separate option.
-    pub dist_dir: Option<String>,
+    pub dist_dir: String,
 
     /// Filesystem watcher options.
     pub watch: NapiWatchOptions,
@@ -273,7 +272,7 @@ impl From<NapiDefineEnv> for DefineEnv {
 }
 
 pub struct ProjectInstance {
-    turbo_tasks: Arc<TurboTasks<MemoryBackend>>,
+    turbo_tasks: Arc<TurboTasks<NextBackend>>,
     container: Vc<ProjectContainer>,
     exit_receiver: tokio::sync::Mutex<Option<ExitReceiver>>,
 }
@@ -309,10 +308,7 @@ pub async fn project_new(
         let subscriber = Registry::default();
 
         let subscriber = subscriber.with(EnvFilter::builder().parse(trace).unwrap());
-        let dist_dir = options
-            .dist_dir
-            .as_ref()
-            .map_or_else(|| ".next".to_string(), |d| d.to_string());
+        let dist_dir = options.dist_dir.clone();
 
         let internal_dir = PathBuf::from(&options.project_path).join(dist_dir);
         std::fs::create_dir_all(&internal_dir)
@@ -338,27 +334,30 @@ pub async fn project_new(
         subscriber.init();
     }
 
-    let turbo_tasks = TurboTasks::new(MemoryBackend::new(
-        turbo_engine_options
-            .memory_limit
-            .map(|m| m as usize)
-            .unwrap_or(usize::MAX),
-    ));
-    let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS");
-    if let Some(stats_path) = stats_path {
-        let task_stats = turbo_tasks.backend().task_statistics().enable().clone();
-        exit.on_exit(async move {
-            tokio::task::spawn_blocking(move || {
-                let mut file = std::fs::File::create(&stats_path)
-                    .with_context(|| format!("failed to create or open {stats_path:?}"))?;
-                serde_json::to_writer(&file, &task_stats)
-                    .context("failed to serialize or write task statistics")?;
-                file.flush().context("failed to flush file")
-            })
-            .await
-            .unwrap()
-            .unwrap();
-        });
+    let memory_limit = turbo_engine_options
+        .memory_limit
+        .map(|m| m as usize)
+        .unwrap_or(usize::MAX);
+    let turbo_tasks = create_turbo_tasks(PathBuf::from(&options.dist_dir), memory_limit)?;
+    #[cfg(not(feature = "new-backend"))]
+    {
+        use std::io::Write;
+        let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS");
+        if let Some(stats_path) = stats_path {
+            let task_stats = turbo_tasks.backend().task_statistics().enable().clone();
+            exit.on_exit(async move {
+                tokio::task::spawn_blocking(move || {
+                    let mut file = std::fs::File::create(&stats_path)
+                        .with_context(|| format!("failed to create or open {stats_path:?}"))?;
+                    serde_json::to_writer(&file, &task_stats)
+                        .context("failed to serialize or write task statistics")?;
+                    file.flush().context("failed to flush file")
+                })
+                .await
+                .unwrap()
+                .unwrap();
+            });
+        }
     }
     let options: ProjectOptions = options.into();
     let container = turbo_tasks
@@ -502,7 +501,7 @@ impl NapiRoute {
     fn from_route(
         pathname: String,
         value: Route,
-        turbo_tasks: &Arc<TurboTasks<MemoryBackend>>,
+        turbo_tasks: &Arc<TurboTasks<NextBackend>>,
     ) -> Self {
         let convert_endpoint = |endpoint: Vc<Box<dyn Endpoint>>| {
             Some(External::new(ExternalEndpoint(VcArc::new(
@@ -569,7 +568,7 @@ struct NapiMiddleware {
 impl NapiMiddleware {
     fn from_middleware(
         value: &Middleware,
-        turbo_tasks: &Arc<TurboTasks<MemoryBackend>>,
+        turbo_tasks: &Arc<TurboTasks<NextBackend>>,
     ) -> Result<Self> {
         Ok(NapiMiddleware {
             endpoint: External::new(ExternalEndpoint(VcArc::new(
@@ -589,7 +588,7 @@ struct NapiInstrumentation {
 impl NapiInstrumentation {
     fn from_instrumentation(
         value: &Instrumentation,
-        turbo_tasks: &Arc<TurboTasks<MemoryBackend>>,
+        turbo_tasks: &Arc<TurboTasks<NextBackend>>,
     ) -> Result<Self> {
         Ok(NapiInstrumentation {
             node_js: External::new(ExternalEndpoint(VcArc::new(

crates/napi/src/next_api/utils.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashMap, future::Future, ops::Deref, sync::Arc};
+use std::{collections::HashMap, future::Future, ops::Deref, path::PathBuf, sync::Arc};
 
 use anyhow::{anyhow, Context, Result};
 use napi::{
@@ -9,7 +9,6 @@ use napi::{
 use serde::Serialize;
 use turbo_tasks::{ReadRef, TaskId, TryJoinIterExt, TurboTasks, Vc};
 use turbo_tasks_fs::FileContent;
-use turbo_tasks_memory::MemoryBackend;
 use turbopack_core::{
     diagnostics::{Diagnostic, DiagnosticContextExt, PlainDiagnostic},
     error::PrettyPrintError,
@@ -19,22 +18,41 @@ use turbopack_core::{
 
 use crate::util::log_internal_error_and_inform;
 
+#[cfg(not(feature = "new-backend"))]
+pub type NextBackend = turbo_tasks_memory::MemoryBackend;
+#[cfg(feature = "new-backend")]
+pub type NextBackend = turbo_tasks_backend::TurboTasksBackend;
+
+#[allow(unused_variables, reason = "feature-gated")]
+pub fn create_turbo_tasks(
+    output_path: PathBuf,
+    memory_limit: usize,
+) -> Result<Arc<TurboTasks<NextBackend>>> {
+    #[cfg(not(feature = "new-backend"))]
+    let backend = TurboTasks::new(turbo_tasks_memory::MemoryBackend::new(memory_limit));
+    #[cfg(feature = "new-backend")]
+    let backend = TurboTasks::new(turbo_tasks_backend::TurboTasksBackend::new(Arc::new(
+        turbo_tasks_backend::LmdbBackingStorage::new(&output_path.join("cache/turbopack"))?,
+    )));
+    Ok(backend)
+}
+
 /// A helper type to hold both a Vc operation and the TurboTasks root process.
 /// Without this, we'd need to pass both individually all over the place
 #[derive(Clone)]
 pub struct VcArc<T> {
-    turbo_tasks: Arc<TurboTasks<MemoryBackend>>,
+    turbo_tasks: Arc<TurboTasks<NextBackend>>,
     /// The Vc. Must be resolved, otherwise you are referencing an inactive
     /// operation.
     vc: T,
 }
 
 impl<T> VcArc<T> {
-    pub fn new(turbo_tasks: Arc<TurboTasks<MemoryBackend>>, vc: T) -> Self {
+    pub fn new(turbo_tasks: Arc<TurboTasks<NextBackend>>, vc: T) -> Self {
         Self { turbo_tasks, vc }
     }
 
-    pub fn turbo_tasks(&self) -> &Arc<TurboTasks<MemoryBackend>> {
+    pub fn turbo_tasks(&self) -> &Arc<TurboTasks<NextBackend>> {
         &self.turbo_tasks
     }
 }
@@ -57,7 +75,7 @@ pub fn serde_enum_to_string<T: Serialize>(value: &T) -> Result<String> {
 /// The root of our turbopack computation.
 pub struct RootTask {
     #[allow(dead_code)]
-    turbo_tasks: Arc<TurboTasks<MemoryBackend>>,
+    turbo_tasks: Arc<TurboTasks<NextBackend>>,
     #[allow(dead_code)]
     task_id: Option<TaskId>,
 }
@@ -301,7 +319,7 @@ impl<T: ToNapiValue> ToNapiValue for TurbopackResult<T> {
 }
 
 pub fn subscribe<T: 'static + Send + Sync, F: Future<Output = Result<T>> + Send, V: ToNapiValue>(
-    turbo_tasks: Arc<TurboTasks<MemoryBackend>>,
+    turbo_tasks: Arc<TurboTasks<NextBackend>>,
     func: JsFunction,
     handler: impl 'static + Sync + Send + Clone + Fn() -> F,
     mapper: impl 'static + Sync + Send + FnMut(ThreadSafeCallContext<T>) -> napi::Result<Vec<V>>,

crates/napi/src/turbotrace.rs

@@ -1,35 +1,35 @@
-use std::sync::Arc;
+use std::{path::PathBuf, sync::Arc};
 
 use napi::bindgen_prelude::*;
 use node_file_trace::{start, Args};
 use turbo_tasks::TurboTasks;
-use turbo_tasks_memory::MemoryBackend;
 use turbopack::{
     module_options::{EcmascriptOptionsContext, ModuleOptionsContext},
     resolve_options_context::ResolveOptionsContext,
 };
 
+use crate::next_api::utils::{self, NextBackend};
+
 #[napi]
-pub fn create_turbo_tasks(memory_limit: Option<i64>) -> External<Arc<TurboTasks<MemoryBackend>>> {
-    let turbo_tasks = TurboTasks::new(MemoryBackend::new(
-        memory_limit.map(|m| m as usize).unwrap_or(usize::MAX),
-    ));
-    External::new_with_size_hint(
-        turbo_tasks,
-        memory_limit.map(|u| u as usize).unwrap_or(usize::MAX),
-    )
+pub fn create_turbo_tasks(
+    output_path: String,
+    memory_limit: Option<i64>,
+) -> External<Arc<TurboTasks<NextBackend>>> {
+    let limit = memory_limit.map(|u| u as usize).unwrap_or(usize::MAX);
+    let turbo_tasks = utils::create_turbo_tasks(PathBuf::from(&output_path), limit)
+        .expect("Failed to create TurboTasks");
+    External::new_with_size_hint(turbo_tasks, limit)
 }
 
 #[napi]
 pub async fn run_turbo_tracing(
     options: Buffer,
-    turbo_tasks: Option<External<Arc<TurboTasks<MemoryBackend>>>>,
+    turbo_tasks: External<Arc<TurboTasks<NextBackend>>>,
 ) -> napi::Result<Vec<String>> {
     let args: Args = serde_json::from_slice(options.as_ref())?;
-    let turbo_tasks = turbo_tasks.map(|t| t.clone());
     let files = start(
         Arc::new(args),
-        turbo_tasks.as_ref(),
+        turbo_tasks.clone(),
         Some(ModuleOptionsContext {
             ecmascript: EcmascriptOptionsContext {
                 enable_types: true,