Add handling fo beforeFiles, afterFiles, and fallback rewrites (#23407)

This adds support for returning an object from `rewrites` in `next.config.js` with `beforeFiles`, `afterFiles`, and `fallback` to allow specifying rewrites at different stages of routing. The existing support for returning an array for rewrites is still supported and behaves the same way. The documentation has been updated to include information on these new stages that can be rewritten and removes the outdated note of rewrites not being able to override pages. 



## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR.
- [ ] Related issues linked using `fixes #number`
- [x] Integration tests added
- [x] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.

## Documentation / Examples

- [ ] Make sure the linting passes

docs/api-reference/next.config.js/headers.md

@@ -43,6 +43,11 @@ module.exports = {
 
 - `source` is the incoming request path pattern.
 - `headers` is an array of header objects with the `key` and `value` properties.
+- `basePath`: `false` or `undefined` - if false the basePath won't be included when matching, can be used for external rewrites only.
+- `locale`: `false` or `undefined` - whether the locale should not be included when matching.
+- `has` is an array of [has objects](#header-cookie-and-query-matching) with the `type`, `key` and `value` properties.
+
+Headers are checked before the filesystem which includes pages and `/public` files.
 
 ## Header Overriding Behavior
 

docs/api-reference/next.config.js/redirects.md

@@ -38,6 +38,11 @@ module.exports = {
 - `source` is the incoming request path pattern.
 - `destination` is the path you want to route to.
 - `permanent` if the redirect is permanent or not.
+- `basePath`: `false` or `undefined` - if false the basePath won't be included when matching, can be used for external rewrites only.
+- `locale`: `false` or `undefined` - whether the locale should not be included when matching.
+- `has` is an array of [has objects](#header-cookie-and-query-matching) with the `type`, `key` and `value` properties.
+
+Redirects are checked before the filesystem which includes pages and `/public` files.
 
 ## Path Matching
 

docs/api-reference/next.config.js/rewrites.md

@@ -17,8 +17,6 @@ Rewrites allow you to map an incoming request path to a different destination pa
 
 Rewrites are only available on the Node.js environment and do not affect client-side routing.
 
-Rewrites are not able to override public files or routes in the pages directory as these have higher priority than rewrites. For example, if you have `pages/index.js` you are not able to rewrite `/` to another location unless you rename the `pages/index.js` file.
-
 To use rewrites you can use the `rewrites` key in `next.config.js`:
 
 ```js
@@ -36,8 +34,48 @@ module.exports = {
 
 `rewrites` is an async function that expects an array to be returned holding objects with `source` and `destination` properties:
 
-- `source` is the incoming request path pattern.
-- `destination` is the path you want to route to.
+- `source`: `String` - is the incoming request path pattern.
+- `destination`: `String` is the path you want to route to.
+- `basePath`: `false` or `undefined` - if false the basePath won't be included when matching, can be used for external rewrites only.
+- `locale`: `false` or `undefined` - whether the locale should not be included when matching.
+- `has` is an array of [has objects](#header-cookie-and-query-matching) with the `type`, `key` and `value` properties.
+
+Rewrites are applied after checking the filesystem (pages and `/public` files) and before dynamic routes by default. This behavior can be changed by instead returning an object instead of an array from the `rewrites` function:
+
+```js
+module.exports = {
+  async rewrites() {
+    return {
+      beforeFiles: [
+        // These rewrites are checked after headers/redirects
+        // and before pages/public files which allows overriding
+        // page files
+        {
+          source: '/some-page',
+          destination: '/somewhere-else',
+          has: [{ type: 'query', key: 'overrideMe' }],
+        },
+      ],
+      afterFiles: [
+        // These rewrites are checked after pages/public files
+        // are checked but before dynamic routes
+        {
+          source: '/non-existent',
+          destination: '/somewhere-else',
+        },
+      ],
+      fallback: [
+        // These rewrites are checked after both pages/public files
+        // and dynamic routes are checked
+        {
+          source: '/:path*',
+          destination: 'https://my-old-site.com',
+        },
+      ],
+    }
+  },
+}
+```
 
 ## Rewrite parameters
 

packages/next/build/index.ts

@@ -17,9 +17,11 @@ import {
 import { fileExists } from '../lib/file-exists'
 import { findPagesDir } from '../lib/find-pages-dir'
 import loadCustomRoutes, {
+  CustomRoutes,
   getRedirectStatus,
   normalizeRouteRegex,
   Redirect,
+  Rewrite,
   RouteType,
 } from '../lib/load-custom-routes'
 import { nonNullable } from '../lib/non-nullable'
@@ -51,6 +53,7 @@ import {
 import { __ApiPreviewProps } from '../next-server/server/api-utils'
 import loadConfig, {
   isTargetLikeServerless,
+  NextConfig,
 } from '../next-server/server/config'
 import { BuildManifest } from '../next-server/server/get-page-files'
 import '../next-server/server/node-polyfill-fetch'
@@ -125,19 +128,21 @@ export default async function build(
       .traceChild('load-dotenv')
       .traceFn(() => loadEnvConfig(dir, false, Log))
 
-    const config = await nextBuildSpan
+    const config: NextConfig = await nextBuildSpan
       .traceChild('load-next-config')
       .traceAsyncFn(() => loadConfig(PHASE_PRODUCTION_BUILD, dir, conf))
     const { target } = config
-    const buildId = await nextBuildSpan
+    const buildId: string = await nextBuildSpan
       .traceChild('generate-buildid')
       .traceAsyncFn(() => generateBuildId(config.generateBuildId, nanoid))
     const distDir = path.join(dir, config.distDir)
 
-    const { headers, rewrites, redirects } = await nextBuildSpan
+    const customRoutes: CustomRoutes = await nextBuildSpan
       .traceChild('load-custom-routes')
       .traceAsyncFn(() => loadCustomRoutes(config))
 
+    const { headers, rewrites, redirects } = customRoutes
+
     if (ciEnvironment.isCI && !ciEnvironment.hasNextSupport) {
       const cacheDir = path.join(distDir, 'cache')
       const hasCache = await fileExists(cacheDir)
@@ -354,7 +359,13 @@ export default async function build(
       pages404: boolean
       basePath: string
       redirects: Array<ReturnType<typeof buildCustomRoute>>
-      rewrites: Array<ReturnType<typeof buildCustomRoute>>
+      rewrites:
+        | Array<ReturnType<typeof buildCustomRoute>>
+        | {
+            beforeFiles: Array<ReturnType<typeof buildCustomRoute>>
+            afterFiles: Array<ReturnType<typeof buildCustomRoute>>
+            fallback: Array<ReturnType<typeof buildCustomRoute>>
+          }
       headers: Array<ReturnType<typeof buildCustomRoute>>
       dynamicRoutes: Array<{
         page: string
@@ -384,7 +395,6 @@ export default async function build(
       pages404: true,
       basePath: config.basePath,
       redirects: redirects.map((r: any) => buildCustomRoute(r, 'redirect')),
-      rewrites: rewrites.map((r: any) => buildCustomRoute(r, 'rewrite')),
       headers: headers.map((r: any) => buildCustomRoute(r, 'header')),
       dynamicRoutes: getSortedRoutes(pageKeys)
         .filter(isDynamicRoute)
@@ -401,6 +411,29 @@ export default async function build(
       i18n: config.i18n || undefined,
     }))
 
+    if (rewrites.beforeFiles.length === 0 && rewrites.fallback.length === 0) {
+      routesManifest.rewrites = rewrites.afterFiles.map((r: any) =>
+        buildCustomRoute(r, 'rewrite')
+      )
+    } else {
+      routesManifest.rewrites = {
+        beforeFiles: rewrites.beforeFiles.map((r: any) =>
+          buildCustomRoute(r, 'rewrite')
+        ),
+        afterFiles: rewrites.afterFiles.map((r: any) =>
+          buildCustomRoute(r, 'rewrite')
+        ),
+        fallback: rewrites.fallback.map((r: any) =>
+          buildCustomRoute(r, 'rewrite')
+        ),
+      }
+    }
+    const combinedRewrites: Rewrite[] = [
+      ...rewrites.beforeFiles,
+      ...rewrites.afterFiles,
+      ...rewrites.fallback,
+    ]
+
     const distDirCreated = await nextBuildSpan
       .traceChild('create-dist-dir')
       .traceAsyncFn(async () => {
@@ -1380,11 +1413,12 @@ export default async function build(
           (staticPages.size + ssgPages.size + serverPropsPages.size),
         hasStatic404: useStatic404,
         hasReportWebVitals: namedExports?.includes('reportWebVitals') ?? false,
-        rewritesCount: rewrites.length,
+        rewritesCount: combinedRewrites.length,
         headersCount: headers.length,
         redirectsCount: redirects.length - 1, // reduce one for trailing slash
         headersWithHasCount: headers.filter((r: any) => !!r.has).length,
-        rewritesWithHasCount: rewrites.filter((r: any) => !!r.has).length,
+        rewritesWithHasCount: combinedRewrites.filter((r: any) => !!r.has)
+          .length,
         redirectsWithHasCount: redirects.filter((r: any) => !!r.has).length,
       })
     )

packages/next/build/utils.ts

@@ -351,12 +351,18 @@ export function printCustomRoutes({
   if (redirects.length) {
     printRoutes(redirects, 'Redirects')
   }
-  if (rewrites.length) {
-    printRoutes(rewrites, 'Rewrites')
-  }
   if (headers.length) {
     printRoutes(headers, 'Headers')
   }
+
+  const combinedRewrites = [
+    ...rewrites.beforeFiles,
+    ...rewrites.afterFiles,
+    ...rewrites.fallback,
+  ]
+  if (combinedRewrites.length) {
+    printRoutes(combinedRewrites, 'Rewrites')
+  }
 }
 
 type ComputeManifestShape = {

packages/next/build/webpack-config.ts

@@ -16,7 +16,6 @@ import {
 } from '../lib/constants'
 import { fileExists } from '../lib/file-exists'
 import { getPackageVersion } from '../lib/get-package-version'
-import { Rewrite } from '../lib/load-custom-routes'
 import { getTypeScriptConfiguration } from '../lib/typescript/getTypeScriptConfiguration'
 import {
   CLIENT_STATIC_FILES_RUNTIME_MAIN,
@@ -61,6 +60,7 @@ import WebpackConformancePlugin, {
 import { WellKnownErrorsPlugin } from './webpack/plugins/wellknown-errors-plugin'
 import { NextConfig } from '../next-server/server/config'
 import { relative as relativePath, join as pathJoin } from 'path'
+import { CustomRoutes } from '../lib/load-custom-routes.js'
 
 type ExcludesFalse = <T>(x: T | false) => x is T
 
@@ -201,13 +201,16 @@ export default async function getBaseWebpackConfig(
     target?: string
     reactProductionProfiling?: boolean
     entrypoints: WebpackEntrypoints
-    rewrites: Rewrite[]
+    rewrites: CustomRoutes['rewrites']
   }
 ): Promise<webpack.Configuration> {
   let plugins: PluginMetaData[] = []
   let babelPresetPlugins: { dir: string; config: any }[] = []
 
-  const hasRewrites = rewrites.length > 0
+  const hasRewrites =
+    rewrites.beforeFiles.length > 0 ||
+    rewrites.afterFiles.length > 0 ||
+    rewrites.fallback.length > 0
 
   if (config.experimental.plugins) {
     plugins = await collectPlugins(dir, config.env, config.plugins)

packages/next/build/webpack/loaders/next-serverless-loader/index.ts

@@ -106,9 +106,19 @@ const nextServerlessLoader: webpack.loader.Loader = function () {
 
         import { getApiHandler } from 'next/dist/build/webpack/loaders/next-serverless-loader/api-handler'
 
+        const combinedRewrites = Array.isArray(routesManifest.rewrites)
+          ? routesManifest.rewrites
+          : []
+
+        if (!Array.isArray(routesManifest.rewrites)) {
+          combinedRewrites.push(...routesManifest.rewrites.beforeFiles)
+          combinedRewrites.push(...routesManifest.rewrites.afterFiles)
+          combinedRewrites.push(...routesManifest.rewrites.fallback)
+        }
+
         const apiHandler = getApiHandler({
           pageModule: require("${absolutePagePath}"),
-          rewrites: routesManifest.rewrites,
+          rewrites: combinedRewrites,
           i18n: ${i18n || 'undefined'},
           page: "${page}",
           basePath: "${basePath}",
@@ -160,6 +170,16 @@ const nextServerlessLoader: webpack.loader.Loader = function () {
       export let config = compMod['confi' + 'g'] || (compMod.then && compMod.then(mod => mod['confi' + 'g'])) || {}
       export const _app = App
 
+      const combinedRewrites = Array.isArray(routesManifest.rewrites)
+        ? routesManifest.rewrites
+        : []
+
+      if (!Array.isArray(routesManifest.rewrites)) {
+        combinedRewrites.push(...routesManifest.rewrites.beforeFiles)
+        combinedRewrites.push(...routesManifest.rewrites.afterFiles)
+        combinedRewrites.push(...routesManifest.rewrites.fallback)
+      }
+
       const { renderReqToHTML, render } = getPageHandler({
         pageModule: compMod,
         pageComponent: Component,
@@ -183,7 +203,7 @@ const nextServerlessLoader: webpack.loader.Loader = function () {
         buildManifest,
         reactLoadableManifest,
 
-        rewrites: routesManifest.rewrites,
+        rewrites: combinedRewrites,
         i18n: ${i18n || 'undefined'},
         page: "${page}",
         buildId: "${buildId}",

packages/next/build/webpack/plugins/build-manifest-plugin.ts

@@ -18,6 +18,7 @@ import { ampFirstEntryNamesMap } from './next-drop-client-page-plugin'
 import { Rewrite } from '../../../lib/load-custom-routes'
 import { getSortedRoutes } from '../../../next-server/lib/router/utils'
 import { spans } from './profiling-plugin'
+import { CustomRoutes } from '../../../lib/load-custom-routes'
 
 type DeepMutable<T> = { -readonly [P in keyof T]: DeepMutable<T[P]> }
 
@@ -28,7 +29,7 @@ export type ClientBuildManifest = Record<string, string[]>
 function generateClientManifest(
   compiler: any,
   assetMap: BuildManifest,
-  rewrites: Rewrite[]
+  rewrites: CustomRoutes['rewrites']
 ): string {
   const compilerSpan = spans.get(compiler)
   const genClientManifestSpan = compilerSpan?.traceChild(
@@ -78,25 +79,37 @@ function getEntrypointFiles(entrypoint: any): string[] {
   )
 }
 
+const processRoute = (r: Rewrite) => {
+  const rewrite = { ...r }
+
+  // omit external rewrite destinations since these aren't
+  // handled client-side
+  if (!rewrite.destination.startsWith('/')) {
+    delete (rewrite as any).destination
+  }
+  return rewrite
+}
+
 // This plugin creates a build-manifest.json for all assets that are being output
 // It has a mapping of "entry" filename to real filename. Because the real filename can be hashed in production
 export default class BuildManifestPlugin {
   private buildId: string
-  private rewrites: Rewrite[]
+  private rewrites: CustomRoutes['rewrites']
 
-  constructor(options: { buildId: string; rewrites: Rewrite[] }) {
+  constructor(options: {
+    buildId: string
+    rewrites: CustomRoutes['rewrites']
+  }) {
     this.buildId = options.buildId
 
-    this.rewrites = options.rewrites.map((r) => {
-      const rewrite = { ...r }
-
-      // omit external rewrite destinations since these aren't
-      // handled client-side
-      if (!rewrite.destination.startsWith('/')) {
-        delete (rewrite as any).destination
-      }
-      return rewrite
-    })
+    this.rewrites = {
+      beforeFiles: [],
+      afterFiles: [],
+      fallback: [],
+    }
+    this.rewrites.beforeFiles = options.rewrites.beforeFiles.map(processRoute)
+    this.rewrites.afterFiles = options.rewrites.afterFiles.map(processRoute)
+    this.rewrites.fallback = options.rewrites.fallback.map(processRoute)
   }
 
   createAssets(compiler: any, compilation: any, assets: any) {

packages/next/client/page-loader.ts

@@ -117,11 +117,9 @@ export default class PageLoader {
   }
 
   /**
-   * @param {string} href the route href (file-system path)
+   * @param {string} route - the route (file-system path)
    */
-  _isSsg(href: string): Promise<boolean> {
-    const { pathname: hrefPathname } = parseRelativeUrl(href)
-    const route = normalizeRoute(hrefPathname)
+  _isSsg(route: string): Promise<boolean> {
     return this.promisedSsgManifest!.then((s: ClientSsgManifest) =>
       s.has(route)
     )

packages/next/client/route-loader.ts

@@ -191,7 +191,7 @@ export function getClientBuildManifest(): Promise<ClientBuildManifest> {
     // Mandatory because this is not concurrent safe:
     const cb = self.__BUILD_MANIFEST_CB
     self.__BUILD_MANIFEST_CB = () => {
-      resolve(self.__BUILD_MANIFEST)
+      resolve(self.__BUILD_MANIFEST!)
       cb && cb()
     }
   })