App Router - preinitialize chunks during SSR (vercel#54752)

Today when we hydrate an SSR'd RSC response on the client we encounter import chunks which initiate code loading for client components. However we only start fetching these chunks after hydration has begun which is necessarily after the initial chunks for the entrypoint have loaded.

React has upstream changes that need to land which will preinitialize the rendered chunks for all client components used during the SSR pass. This will cause a `<script async="" src... />` tag to be emitted in the head for each chunk we need to load during hydration which allows the browser to start fetching these resources even before the entrypoint has started to execute.

Additionally the implementation for webpack and turbopack is different enough that there will be a new `react-server-dom-turbopack` package in the React repo which should be used when using Turbopack with Next.

This PR also removes a number of patches to React src that proxy loading (`__next_chunk_load__`) and bundler requires (`__next_require__`) through the `globalThis` object. Now the react packages can be fully responsible for implementing chunk loading and all Next needs to do is supply the necessary information such as chunk prefix and crossOrigin attributes necessary for this loading. This information is produced as part of the client-manifest by either a Webpack plugin or Turbopack.

Additionally any modifications to the chunk filename that were previously done at runtime need to be made in the manifest itself now. This means we need to encode the deployment id for skew protection and encode the filename to make it match our static path matching (and resolutions on s3) when using `[` and `]` segment characters.

There are a few followup items to consider in later PRs
1. we currently bundle a node and edge version of react-server-dom-webpack/client. The node version has an implementation for busboy whereas the edge version does not. Next is currently configured to use busboy when handling a fetch action sent as multipart with a node runtime. Ideally we'd only bundle the one platform we are buliding for but some additional refactoring to support better forking is possibly required here

This PR also updates react from 09285d5a7 to d900fadbf.

### React upstream changes

- facebook/react#27439
- facebook/react#26763
- facebook/react#27434
- facebook/react#27433
- facebook/react#27424
- facebook/react#27428
- facebook/react#27427
- facebook/react#27315
- facebook/react#27314
- facebook/react#27400
- facebook/react#27421
- facebook/react#27419
- facebook/react#27418

package.json

@@ -192,14 +192,16 @@
     "random-seed": "0.3.0",
     "react": "18.2.0",
     "react-17": "npm:react@17.0.2",
-    "react-builtin": "npm:react@18.3.0-canary-09285d5a7-20230925",
+    "react-builtin": "npm:react@18.3.0-canary-d900fadbf-20230929",
     "react-dom": "18.2.0",
     "react-dom-17": "npm:react-dom@17.0.2",
-    "react-dom-builtin": "npm:react-dom@18.3.0-canary-09285d5a7-20230925",
-    "react-dom-experimental-builtin": "npm:react-dom@0.0.0-experimental-09285d5a7-20230925",
-    "react-experimental-builtin": "npm:react@0.0.0-experimental-09285d5a7-20230925",
-    "react-server-dom-webpack": "18.3.0-canary-09285d5a7-20230925",
-    "react-server-dom-webpack-experimental": "npm:react-server-dom-webpack@0.0.0-experimental-09285d5a7-20230925",
+    "react-dom-builtin": "npm:react-dom@18.3.0-canary-d900fadbf-20230929",
+    "react-dom-experimental-builtin": "npm:react-dom@0.0.0-experimental-d900fadbf-20230929",
+    "react-experimental-builtin": "npm:react@0.0.0-experimental-d900fadbf-20230929",
+    "react-server-dom-turbopack": "18.3.0-canary-d900fadbf-20230929",
+    "react-server-dom-turbopack-experimental": "npm:react-server-dom-webpack@0.0.0-experimental-d900fadbf-20230929",
+    "react-server-dom-webpack": "18.3.0-canary-d900fadbf-20230929",
+    "react-server-dom-webpack-experimental": "npm:react-server-dom-webpack@0.0.0-experimental-d900fadbf-20230929",
     "react-ssr-prepass": "1.0.8",
     "react-virtualized": "9.22.3",
     "relay-compiler": "13.0.2",
@@ -209,8 +211,8 @@
     "resolve-from": "5.0.0",
     "sass": "1.54.0",
     "satori": "0.10.6",
-    "scheduler-builtin": "npm:scheduler@0.24.0-canary-09285d5a7-20230925",
-    "scheduler-experimental-builtin": "npm:scheduler@0.0.0-experimental-09285d5a7-20230925",
+    "scheduler-builtin": "npm:scheduler@0.24.0-canary-d900fadbf-20230929",
+    "scheduler-experimental-builtin": "npm:scheduler@0.0.0-experimental-d900fadbf-20230929",
     "seedrandom": "3.0.5",
     "selenium-webdriver": "4.0.0-beta.4",
     "semver": "7.3.7",

packages/next-swc/crates/next-api/src/app.rs

@@ -212,6 +212,7 @@ impl AppProject {
                 "next-dynamic".to_string(),
                 Vc::upcast(NextDynamicTransition::new(self.client_transition())),
             ),
+            ("next-ssr".to_string(), Vc::upcast(self.ssr_transition())),
         ]
         .into_iter()
         .collect();
@@ -636,6 +637,10 @@ impl AppEndpoint {
                 client_references_chunks,
                 this.app_project.project().client_chunking_context(),
                 Vc::upcast(this.app_project.project().ssr_chunking_context()),
+                this.app_project
+                    .project()
+                    .next_config()
+                    .computed_asset_prefix(),
             );
             server_assets.push(entry_manifest);
         }

packages/next-swc/crates/next-build/src/next_app/app_entries.rs

@@ -239,6 +239,7 @@ pub async fn get_app_entries(
 /// to `all_chunks`, and the chunking information will be added to the provided
 /// manifests.
 pub async fn compute_app_entries_chunks(
+    next_config: Vc<NextConfig>,
     app_entries: &AppEntries,
     app_client_reference_graph: Vc<ClientReferenceGraph>,
     app_client_references_chunks: Vc<ClientReferencesChunks>,
@@ -340,6 +341,7 @@ pub async fn compute_app_entries_chunks(
             app_client_references_chunks,
             client_chunking_context,
             ssr_chunking_context,
+            next_config.computed_asset_prefix(),
         );
 
         all_chunks.push(entry_manifest);

packages/next-swc/crates/next-build/src/next_build.rs

@@ -331,6 +331,7 @@ pub(crate) async fn next_build(options: TransientInstance<BuildOptions>) -> Resu
     // TODO(alexkirsz) Do some of that in parallel with the above.
 
     compute_app_entries_chunks(
+        next_config,
         &app_entries,
         app_client_references,
         app_client_references_chunks,

packages/next-swc/crates/next-core/src/next_import_map.rs

@@ -89,6 +89,14 @@ pub async fn get_next_client_import_map(
             );
         }
         ClientContextType::App { app_dir } => {
+            import_map.insert_exact_alias(
+                "server-only",
+                request_to_import_mapping(app_dir, "next/dist/compiled/server-only"),
+            );
+            import_map.insert_exact_alias(
+                "client-only",
+                request_to_import_mapping(app_dir, "next/dist/compiled/client-only"),
+            );
             import_map.insert_exact_alias(
                 "react",
                 request_to_import_mapping(app_dir, "next/dist/compiled/react"),
@@ -107,7 +115,10 @@ pub async fn get_next_client_import_map(
             );
             import_map.insert_wildcard_alias(
                 "react-server-dom-webpack/",
-                request_to_import_mapping(app_dir, "next/dist/compiled/react-server-dom-webpack/*"),
+                request_to_import_mapping(
+                    app_dir,
+                    "next/dist/compiled/react-server-dom-turbopack/*",
+                ),
             );
             import_map.insert_exact_alias(
                 "next/head",
@@ -238,7 +249,6 @@ pub async fn get_next_server_import_map(
             import_map.insert_wildcard_alias("react-dom/", external);
             import_map.insert_exact_alias("styled-jsx", external);
             import_map.insert_wildcard_alias("styled-jsx/", external);
-            import_map.insert_wildcard_alias("react-server-dom-webpack/", external);
             // TODO: we should not bundle next/dist/build/utils in the pages renderer at all
             import_map.insert_wildcard_alias("next/dist/build/utils", external);
         }
@@ -253,6 +263,10 @@ pub async fn get_next_server_import_map(
                 "next/dynamic",
                 request_to_import_mapping(project_path, "next/dist/shared/lib/app-dynamic"),
             );
+            import_map.insert_exact_alias(
+                "react-server-dom-webpack/",
+                ImportMapping::External(Some("react-server-dom-turbopack".into())).cell(),
+            );
         }
         ServerContextType::Middleware => {}
     }
@@ -487,11 +501,13 @@ async fn insert_next_server_special_aliases(
                     app_dir,
                     match runtime {
                         NextRuntime::Edge => {
-                            "next/dist/compiled/react-server-dom-webpack/client.edge"
+                            "next/dist/compiled/react-server-dom-turbopack/client.edge"
                         }
+                        // When we access the runtime we still use the webpack name. The runtime
+                        // itself will substitute in the turbopack variant
                         NextRuntime::NodeJs => {
                             "next/dist/server/future/route-modules/app-page/vendored/ssr/\
-                             react-server-dom-webpack-client-edge"
+                             react-server-dom-turbopack-client-edge"
                         }
                     },
                 ),
@@ -505,11 +521,13 @@ async fn insert_next_server_special_aliases(
                     app_dir,
                     match runtime {
                         NextRuntime::Edge => {
-                            "next/dist/compiled/react-server-dom-webpack/client.edge"
+                            "next/dist/compiled/react-server-dom-turbopack/client.edge"
                         }
+                        // When we access the runtime we still use the webpack name. The runtime
+                        // itself will substitute in the turbopack variant
                         NextRuntime::NodeJs => {
                             "next/dist/server/future/route-modules/app-page/vendored/ssr/\
-                             react-server-dom-webpack-client-edge"
+                             react-server-dom-turbopack-client-edge"
                         }
                     },
                 ),
@@ -580,6 +598,14 @@ async fn insert_next_server_special_aliases(
                     },
                 ),
             );
+            import_map.insert_exact_alias(
+                "server-only",
+                request_to_import_mapping(app_dir, "next/dist/compiled/server-only"),
+            );
+            import_map.insert_exact_alias(
+                "client-only",
+                request_to_import_mapping(app_dir, "next/dist/compiled/client-only"),
+            );
             import_map.insert_exact_alias(
                 "react",
                 request_to_import_mapping(
@@ -610,11 +636,13 @@ async fn insert_next_server_special_aliases(
                     app_dir,
                     match runtime {
                         NextRuntime::Edge => {
-                            "next/dist/compiled/react-server-dom-webpack/server.edge"
+                            "next/dist/compiled/react-server-dom-turbopack/server.edge"
                         }
+                        // When we access the runtime we still use the webpack name. The runtime
+                        // itself will substitute in the turbopack variant
                         NextRuntime::NodeJs => {
                             "next/dist/server/future/route-modules/app-page/vendored/rsc/\
-                             react-server-dom-webpack-server-edge"
+                             react-server-dom-turbopack-server-edge"
                         }
                     },
                 ),
@@ -625,11 +653,13 @@ async fn insert_next_server_special_aliases(
                     app_dir,
                     match runtime {
                         NextRuntime::Edge => {
-                            "next/dist/compiled/react-server-dom-webpack/server.node"
+                            "next/dist/compiled/react-server-dom-turbopack/server.node"
                         }
+                        // When we access the runtime we still use the webpack name. The runtime
+                        // itself will substitute in the turbopack variant
                         NextRuntime::NodeJs => {
                             "next/dist/server/future/route-modules/app-page/vendored/rsc/\
-                             react-server-dom-webpack-server-node"
+                             react-server-dom-turbopack-server-node"
                         }
                     },
                 ),

packages/next-swc/crates/next-core/src/next_manifests/client_reference_manifest.rs

@@ -30,8 +30,15 @@ impl ClientReferenceManifest {
         client_references_chunks: Vc<ClientReferencesChunks>,
         client_chunking_context: Vc<Box<dyn EcmascriptChunkingContext>>,
         ssr_chunking_context: Vc<Box<dyn EcmascriptChunkingContext>>,
+        asset_prefix: Vc<Option<String>>,
     ) -> Result<Vc<Box<dyn OutputAsset>>> {
         let mut entry_manifest: ClientReferenceManifest = Default::default();
+        entry_manifest.module_loading.prefix = asset_prefix
+            .await?
+            .as_ref()
+            .map(|p| p.to_owned())
+            .unwrap_or_default();
+        entry_manifest.module_loading.cross_origin = None;
         let client_references_chunks = client_references_chunks.await?;
         let client_relative_path = client_relative_path.await?;
         let node_root_ref = node_root.await?;

packages/next-swc/crates/next-core/src/next_manifests/mod.rs

@@ -178,6 +178,7 @@ pub enum ActionManifestWorkerEntry {
 #[derive(Serialize, Default, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct ClientReferenceManifest {
+    pub module_loading: ModuleLoading,
     /// Mapping of module path and export name to client module ID and required
     /// client chunks.
     pub client_modules: ManifestNode,
@@ -192,6 +193,13 @@ pub struct ClientReferenceManifest {
     pub entry_css_files: HashMap<String, Vec<String>>,
 }
 
+#[derive(Serialize, Default, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct ModuleLoading {
+    pub prefix: String,
+    pub cross_origin: Option<String>,
+}
+
 #[derive(Serialize, Default, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct ManifestNode {

packages/next/src/build/deployment-id.ts

@@ -0,0 +1,6 @@
+export function getDeploymentIdQueryOrEmptyString(): string {
+  if (process.env.NEXT_DEPLOYMENT_ID) {
+    return `?dpl=${process.env.NEXT_DEPLOYMENT_ID}`
+  }
+  return ''
+}