Skip to content

Commit

Permalink
add new backend feature
Browse files Browse the repository at this point in the history
  • Loading branch information
sokra committed Aug 14, 2024
1 parent a071389 commit 12bee4b
Show file tree
Hide file tree
Showing 13 changed files with 98 additions and 125 deletions.
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ swc-ast-explorer = { path = "turbopack/crates/turbopack-swc-ast-explorer" }
turbo-prehash = { path = "turbopack/crates/turbo-prehash" }
turbo-tasks-malloc = { path = "turbopack/crates/turbo-tasks-malloc", default-features = false }
turbo-tasks = { path = "turbopack/crates/turbo-tasks" }
turbo-tasks-backend = { path = "turbopack/crates/turbo-tasks-backend" }
turbo-tasks-build = { path = "turbopack/crates/turbo-tasks-build" }
turbo-tasks-bytes = { path = "turbopack/crates/turbo-tasks-bytes" }
turbo-tasks-env = { path = "turbopack/crates/turbo-tasks-env" }
Expand Down
3 changes: 3 additions & 0 deletions crates/napi/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ __internal_dhat-heap = ["dhat"]
# effectively does nothing.
__internal_dhat-ad-hoc = ["dhat"]

new-backend = ["dep:turbo-tasks-backend"]

# Enable specific tls features per-target.
[target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies]
next-core = { workspace = true, features = ["native-tls"] }
Expand Down Expand Up @@ -105,6 +107,7 @@ lightningcss-napi = { workspace = true }
tokio = { workspace = true, features = ["full"] }
turbo-tasks = { workspace = true }
turbo-tasks-memory = { workspace = true }
turbo-tasks-backend = { workspace = true, optional = true }
turbo-tasks-fs = { workspace = true }
next-api = { workspace = true }
next-build = { workspace = true }
Expand Down
57 changes: 28 additions & 29 deletions crates/napi/src/next_api/project.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use std::{io::Write, path::PathBuf, sync::Arc, thread, time::Duration};
use std::{path::PathBuf, sync::Arc, thread, time::Duration};

use anyhow::{anyhow, bail, Context, Result};
use napi::{
Expand Down Expand Up @@ -43,8 +43,8 @@ use url::Url;
use super::{
endpoint::ExternalEndpoint,
utils::{
get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue, NextBackend, RootTask,
TurbopackResult, VcArc,
create_turbo_tasks, get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue,
NextBackend, RootTask, TurbopackResult, VcArc,
},
};
use crate::register;
Expand Down Expand Up @@ -88,7 +88,7 @@ pub struct NapiProjectOptions {

/// next.config's distDir. Project initialization occurs eariler than
/// deserializing next.config, so passing it as separate option.
pub dist_dir: Option<String>,
pub dist_dir: String,

/// Whether to watch he filesystem for file changes.
pub watch: bool,
Expand Down Expand Up @@ -279,10 +279,7 @@ pub async fn project_new(
let subscriber = Registry::default();

let subscriber = subscriber.with(EnvFilter::builder().parse(trace).unwrap());
let dist_dir = options
.dist_dir
.as_ref()
.map_or_else(|| ".next".to_string(), |d| d.to_string());
let dist_dir = options.dist_dir.clone();

let internal_dir = PathBuf::from(&options.project_path).join(dist_dir);
std::fs::create_dir_all(&internal_dir)
Expand All @@ -308,27 +305,29 @@ pub async fn project_new(
subscriber.init();
}

let turbo_tasks = TurboTasks::new(NextBackend::new(
turbo_engine_options
.memory_limit
.map(|m| m as usize)
.unwrap_or(usize::MAX),
));
let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS");
if let Some(stats_path) = stats_path {
let task_stats = turbo_tasks.backend().task_statistics().enable().clone();
exit.on_exit(async move {
tokio::task::spawn_blocking(move || {
let mut file = std::fs::File::create(&stats_path)
.with_context(|| format!("failed to create or open {stats_path:?}"))?;
serde_json::to_writer(&file, &task_stats)
.context("failed to serialize or write task statistics")?;
file.flush().context("failed to flush file")
})
.await
.unwrap()
.unwrap();
});
let memory_limit = turbo_engine_options
.memory_limit
.map(|m| m as usize)
.unwrap_or(usize::MAX);
let turbo_tasks = create_turbo_tasks(PathBuf::from(&options.dist_dir), memory_limit)?;
#[cfg(not(feature = "new-backend"))]
{
let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS");
if let Some(stats_path) = stats_path {
let task_stats = turbo_tasks.backend().task_statistics().enable().clone();
exit.on_exit(async move {
tokio::task::spawn_blocking(move || {
let mut file = std::fs::File::create(&stats_path)
.with_context(|| format!("failed to create or open {stats_path:?}"))?;
serde_json::to_writer(&file, &task_stats)
.context("failed to serialize or write task statistics")?;
file.flush().context("failed to flush file")

Check failure on line 324 in crates/napi/src/next_api/project.rs

View workflow job for this annotation

GitHub Actions / rust check / build

the method `flush` exists for struct `File`, but its trait bounds were not satisfied

Check failure on line 324 in crates/napi/src/next_api/project.rs

View workflow job for this annotation

GitHub Actions / build / build

the method `flush` exists for struct `File`, but its trait bounds were not satisfied

Check failure on line 324 in crates/napi/src/next_api/project.rs

View workflow job for this annotation

GitHub Actions / build-native / build

the method `flush` exists for struct `File`, but its trait bounds were not satisfied

Check failure on line 324 in crates/napi/src/next_api/project.rs

View workflow job for this annotation

GitHub Actions / test unit (20) / build

the method `flush` exists for struct `File`, but its trait bounds were not satisfied

Check failure on line 324 in crates/napi/src/next_api/project.rs

View workflow job for this annotation

GitHub Actions / test unit (18) / build

the method `flush` exists for struct `File`, but its trait bounds were not satisfied

Check failure on line 324 in crates/napi/src/next_api/project.rs

View workflow job for this annotation

GitHub Actions / stable - x86_64-unknown-linux-gnu - node@16

the method `flush` exists for struct `File`, but its trait bounds were not satisfied

Check failure on line 324 in crates/napi/src/next_api/project.rs

View workflow job for this annotation

GitHub Actions / test next-swc wasm / build

the method `flush` exists for struct `File`, but its trait bounds were not satisfied

Check failure on line 324 in crates/napi/src/next_api/project.rs

View workflow job for this annotation

GitHub Actions / stable - aarch64-unknown-linux-gnu - node@16

the method `flush` exists for struct `File`, but its trait bounds were not satisfied

Check failure on line 324 in crates/napi/src/next_api/project.rs

View workflow job for this annotation

GitHub Actions / stable - x86_64-pc-windows-msvc - node@16

the method `flush` exists for struct `File`, but its trait bounds were not satisfied

Check failure on line 324 in crates/napi/src/next_api/project.rs

View workflow job for this annotation

GitHub Actions / stable - aarch64-apple-darwin - node@16

the method `flush` exists for struct `File`, but its trait bounds were not satisfied

Check failure on line 324 in crates/napi/src/next_api/project.rs

View workflow job for this annotation

GitHub Actions / stable - x86_64-apple-darwin - node@16

the method `flush` exists for struct `File`, but its trait bounds were not satisfied
})
.await
.unwrap()
.unwrap();
});
}
}
let options = options.into();
let container = turbo_tasks
Expand Down
22 changes: 19 additions & 3 deletions crates/napi/src/next_api/utils.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use std::{collections::HashMap, future::Future, ops::Deref, sync::Arc};
use std::{collections::HashMap, future::Future, ops::Deref, path::PathBuf, sync::Arc};

use anyhow::{anyhow, Context, Result};
use napi::{
Expand All @@ -9,15 +9,31 @@ use napi::{
use serde::Serialize;
use turbo_tasks::{ReadRef, TaskId, TryJoinIterExt, TurboTasks, Vc};
use turbo_tasks_fs::FileContent;
use turbo_tasks_memory::MemoryBackend;
use turbopack_core::{
diagnostics::{Diagnostic, DiagnosticContextExt, PlainDiagnostic},
error::PrettyPrintError,
issue::{IssueDescriptionExt, PlainIssue, PlainIssueSource, PlainSource, StyledString},
source_pos::SourcePos,
};

pub type NextBackend = MemoryBackend;
#[cfg(not(feature = "new-backend"))]
pub type NextBackend = turbo_tasks_memory::MemoryBackend;
#[cfg(feature = "new-backend")]
pub type NextBackend = turbo_tasks_backend::TurboTasksBackend;

#[allow(unused_variables, reason = "feature-gated")]
pub fn create_turbo_tasks(
output_path: PathBuf,
memory_limit: usize,
) -> Result<Arc<TurboTasks<NextBackend>>> {
#[cfg(not(feature = "new-backend"))]
let backend = TurboTasks::new(turbo_tasks_memory::MemoryBackend::new(memory_limit));
#[cfg(feature = "new-backend")]
let backend = TurboTasks::new(turbo_tasks_backend::TurboTasksBackend::new(Arc::new(
turbo_tasks_backend::LmdbBackingStorage::new(&output_path.join("cache/turbopack"))?,
)));
Ok(backend)
}

/// A helper type to hold both a Vc operation and the TurboTasks root process.
/// Without this, we'd need to pass both individually all over the place
Expand Down
25 changes: 12 additions & 13 deletions crates/napi/src/turbotrace.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use std::sync::Arc;
use std::{path::PathBuf, sync::Arc};

use napi::bindgen_prelude::*;
use node_file_trace::{start, Args};
Expand All @@ -8,29 +8,28 @@ use turbopack::{
resolve_options_context::ResolveOptionsContext,
};

use crate::next_api::utils::NextBackend;
use crate::next_api::utils::{self, NextBackend};

#[napi]
pub fn create_turbo_tasks(memory_limit: Option<i64>) -> External<Arc<TurboTasks<NextBackend>>> {
let turbo_tasks = TurboTasks::new(NextBackend::new(
memory_limit.map(|m| m as usize).unwrap_or(usize::MAX),
));
External::new_with_size_hint(
turbo_tasks,
memory_limit.map(|u| u as usize).unwrap_or(usize::MAX),
)
pub fn create_turbo_tasks(
output_path: String,
memory_limit: Option<i64>,
) -> External<Arc<TurboTasks<NextBackend>>> {
let limit = memory_limit.map(|u| u as usize).unwrap_or(usize::MAX);
let turbo_tasks = utils::create_turbo_tasks(PathBuf::from(&output_path), limit)
.expect("Failed to create TurboTasks");
External::new_with_size_hint(turbo_tasks, limit)
}

#[napi]
pub async fn run_turbo_tracing(
options: Buffer,
turbo_tasks: Option<External<Arc<TurboTasks<NextBackend>>>>,
turbo_tasks: External<Arc<TurboTasks<NextBackend>>>,
) -> napi::Result<Vec<String>> {
let args: Args = serde_json::from_slice(options.as_ref())?;
let turbo_tasks = turbo_tasks.map(|t| t.clone());
let files = start(
Arc::new(args),
turbo_tasks.as_ref(),
turbo_tasks.clone(),
Some(ModuleOptionsContext {
ecmascript: EcmascriptOptionsContext {
enable_types: true,
Expand Down
1 change: 1 addition & 0 deletions packages/next/src/build/collect-build-traces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ export async function collectBuildTraces({
let turbotraceOutputPath: string | undefined
let turbotraceFiles: string[] | undefined
turboTasksForTrace = bindings.turbo.createTurboTasks(
distDir,
(config.experimental.turbotrace?.memoryLimit ??
TURBO_TRACE_DEFAULT_MEMORY_LIMIT) *
1024 *
Expand Down
1 change: 1 addition & 0 deletions packages/next/src/build/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1351,6 +1351,7 @@ export default async function build(
{
projectPath: dir,
rootPath: config.outputFileTracingRoot || dir,
distDir,
nextConfig: config,
jsConfig: await getTurbopackJsConfig(dir, config),
watch: false,
Expand Down
14 changes: 11 additions & 3 deletions packages/next/src/build/swc/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,11 @@ export interface ProjectOptions {
*/
projectPath: string

/**
* The path to the .next directory.
*/
distDir: string

/**
* The next.config.js contents.
*/
Expand Down Expand Up @@ -1547,15 +1552,18 @@ function loadNative(importPath?: string) {
initHeapProfiler: bindings.initHeapProfiler,
teardownHeapProfiler: bindings.teardownHeapProfiler,
turbo: {
startTrace: (options = {}, turboTasks: unknown) => {
startTrace: (options = {}, turboTasks: { __napi: 'TurboTasks' }) => {
initHeapProfiler()
return (customBindings ?? bindings).runTurboTracing(
toBuffer({ exact: true, ...options }),
turboTasks
)
},
createTurboTasks: (memoryLimit?: number): unknown =>
bindings.createTurboTasks(memoryLimit),
createTurboTasks: (
outputPath: string,
memoryLimit?: number
): { __napi: 'TurboTasks' } =>
bindings.createTurboTasks(outputPath, memoryLimit),
entrypoints: {
stream: (
turboTasks: any,
Expand Down
1 change: 1 addition & 0 deletions packages/next/src/server/dev/hot-reloader-turbopack.ts
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ export async function createHotReloaderTurbopack(
{
projectPath: dir,
rootPath: opts.nextConfig.outputFileTracingRoot || dir,
distDir,
nextConfig: opts.nextConfig,
jsConfig: await getTurbopackJsConfig(dir, nextConfig),
watch: true,
Expand Down
14 changes: 8 additions & 6 deletions test/development/basic/next-rs-api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -191,13 +191,20 @@ describe('next.rs api', () => {
console.log(next.testDir)
const nextConfig = await loadConfig(PHASE_DEVELOPMENT_SERVER, next.testDir)
const bindings = await loadBindings()
const distDir = path.join(
process.env.NEXT_SKIP_ISOLATE
? path.resolve(__dirname, '../../..')
: next.testDir,
'.next'
)
project = await bindings.turbo.createProject({
env: {},
jsConfig: {
compilerOptions: {},
},
nextConfig: nextConfig,
projectPath: next.testDir,
distDir,
rootPath: process.env.NEXT_SKIP_ISOLATE
? path.resolve(__dirname, '../../..')
: next.testDir,
Expand All @@ -208,12 +215,7 @@ describe('next.rs api', () => {
clientRouterFilters: undefined,
config: nextConfig,
dev: true,
distDir: path.join(
process.env.NEXT_SKIP_ISOLATE
? path.resolve(__dirname, '../../..')
: next.testDir,
'.next'
),
distDir: distDir,
fetchCacheKeyPrefix: undefined,
hasRewrites: false,
middlewareMatchers: undefined,
Expand Down
Loading

0 comments on commit 12bee4b

Please sign in to comment.