Update to process redirects/rewrites for _next/data with middleware

packages/next/client/page-loader.ts

@@ -117,6 +117,7 @@ export default class PageLoader {
     asPath: string
     href: string
     locale?: string | false
+    skipInterpolation?: boolean
   }): string {
     const { asPath, href, locale } = params
     const { pathname: hrefPathname, query, search } = parseRelativeUrl(href)
@@ -138,7 +139,9 @@ export default class PageLoader {
     }
 
     return getHrefForSlug(
-      isDynamicRoute(route)
+      params.skipInterpolation
+        ? asPathname
+        : isDynamicRoute(route)
         ? interpolateAs(hrefPathname, asPathname, query).result
         : route
     )

packages/next/server/base-server.ts

@@ -439,71 +439,60 @@ export default abstract class Server<ServerOptions extends Options = Options> {
 
       if (
         this.minimalMode &&
-        req.headers['x-matched-path'] &&
         typeof req.headers['x-matched-path'] === 'string'
       ) {
-        const reqUrlIsDataUrl = req.url?.includes('/_next/data')
-        const parsedMatchedPath = parseUrl(req.headers['x-matched-path'] || '')
-        const matchedPathIsDataUrl =
-          parsedMatchedPath.pathname?.includes('/_next/data')
-        const isDataUrl = reqUrlIsDataUrl || matchedPathIsDataUrl
-
-        let parsedPath = parseUrl(
-          isDataUrl ? req.url! : (req.headers['x-matched-path'] as string),
-          true
-        )
-        let matchedPathname = parsedPath.pathname!
-
-        let matchedPathnameNoExt = isDataUrl
-          ? matchedPathname.replace(/\.json$/, '')
-          : matchedPathname
-
-        let srcPathname = isDataUrl
-          ? this.stripNextDataPath(
-              parsedMatchedPath.pathname?.replace(/\.json$/, '') ||
-                matchedPathnameNoExt
-            ) || '/'
-          : matchedPathnameNoExt
-
-        if (this.nextConfig.i18n) {
-          const localePathResult = normalizeLocalePath(
-            matchedPathname || '/',
-            this.nextConfig.i18n.locales
-          )
-
-          if (localePathResult.detectedLocale) {
-            parsedUrl.query.__nextLocale = localePathResult.detectedLocale
+        try {
+          // x-matched-path is the source of truth, it tells what page
+          // should be rendered because we don't process rewrites in minimalMode
+          let matchedPath = new URL(
+            req.headers['x-matched-path'],
+            'http://localhost'
+          ).pathname
+
+          let urlPathname = new URL(req.url, 'http://localhost').pathname
+
+          // For ISR  the URL is normalized to the prerenderPath so if
+          // it's a data request the URL path will be the data URL,
+          // basePath is already stripped by this point
+          if (urlPathname.startsWith(`/_next/data/`)) {
+            parsedUrl.query.__nextDataReq = '1'
           }
-        }
+          matchedPath = this.stripNextDataPath(matchedPath, false)
 
-        if (isDataUrl) {
-          matchedPathname = denormalizePagePath(matchedPathname)
-          matchedPathnameNoExt = denormalizePagePath(matchedPathnameNoExt)
-          srcPathname = denormalizePagePath(srcPathname)
-        }
+          if (this.nextConfig.i18n) {
+            const localeResult = normalizeLocalePath(
+              matchedPath,
+              this.nextConfig.i18n.locales
+            )
+            matchedPath = localeResult.pathname
 
-        if (
-          !isDynamicRoute(srcPathname) &&
-          !(await this.hasPage(srcPathname))
-        ) {
-          for (const dynamicRoute of this.dynamicRoutes || []) {
-            if (dynamicRoute.match(srcPathname)) {
-              srcPathname = dynamicRoute.page
-              break
+            if (localeResult.detectedLocale) {
+              parsedUrl.query.__nextLocale = localeResult.detectedLocale
             }
           }
-        }
+          matchedPath = denormalizePagePath(matchedPath)
+          let srcPathname = matchedPath
 
-        const pageIsDynamic = isDynamicRoute(srcPathname)
-        const utils = getUtils({
-          pageIsDynamic,
-          page: srcPathname,
-          i18n: this.nextConfig.i18n,
-          basePath: this.nextConfig.basePath,
-          rewrites: this.customRoutes.rewrites,
-        })
+          if (
+            !isDynamicRoute(srcPathname) &&
+            !(await this.hasPage(srcPathname))
+          ) {
+            for (const dynamicRoute of this.dynamicRoutes || []) {
+              if (dynamicRoute.match(srcPathname)) {
+                srcPathname = dynamicRoute.page
+                break
+              }
+            }
+          }
 
-        try {
+          const pageIsDynamic = isDynamicRoute(srcPathname)
+          const utils = getUtils({
+            pageIsDynamic,
+            page: srcPathname,
+            i18n: this.nextConfig.i18n,
+            basePath: this.nextConfig.basePath,
+            rewrites: this.customRoutes.rewrites,
+          })
           // ensure parsedUrl.pathname includes URL before processing
           // rewrites or they won't match correctly
           if (defaultLocale && !pathnameInfo.locale) {
@@ -523,7 +512,6 @@ export default abstract class Server<ServerOptions extends Options = Options> {
           if (pageIsDynamic) {
             let params: ParsedUrlQuery | false = {}
 
-            Object.assign(parsedUrl.query, parsedPath.query)
             const paramsResult = utils.normalizeDynamicRouteParams(
               parsedUrl.query
             )
@@ -542,27 +530,17 @@ export default abstract class Server<ServerOptions extends Options = Options> {
                 parsedUrl.query.__nextLocale = opts.locale
               }
             } else {
-              params = utils.dynamicRouteMatcher!(matchedPathnameNoExt) || {}
+              params = utils.dynamicRouteMatcher!(matchedPath) || {}
             }
 
             if (params) {
               if (!paramsResult.hasValidParams) {
                 params = utils.normalizeDynamicRouteParams(params).params
               }
 
-              matchedPathname = utils.interpolateDynamicPath(
-                matchedPathname,
-                params
-              )
+              matchedPath = utils.interpolateDynamicPath(srcPathname, params)
               req.url = utils.interpolateDynamicPath(req.url!, params)
             }
-
-            if (reqUrlIsDataUrl && matchedPathIsDataUrl) {
-              req.url = formatUrl({
-                ...parsedPath,
-                pathname: matchedPathname,
-              })
-            }
             Object.assign(parsedUrl.query, params)
           }
 
@@ -572,20 +550,17 @@ export default abstract class Server<ServerOptions extends Options = Options> {
               ...Object.keys(utils.defaultRouteRegex?.groups || {}),
             ])
           }
+          parsedUrl.pathname = `${this.nextConfig.basePath || ''}${
+            matchedPath === '/' && this.nextConfig.basePath ? '' : matchedPath
+          }`
+          url.pathname = parsedUrl.pathname
         } catch (err) {
           if (err instanceof DecodeError || err instanceof NormalizeError) {
             res.statusCode = 400
             return this.renderError(null, req, res, '/_error', {})
           }
           throw err
         }
-
-        parsedUrl.pathname = `${this.nextConfig.basePath || ''}${
-          matchedPathname === '/' && this.nextConfig.basePath
-            ? ''
-            : matchedPathname
-        }`
-        url.pathname = parsedUrl.pathname
       }
 
       addRequestMeta(req, '__nextHadTrailingSlash', pathnameInfo.trailingSlash)
@@ -773,18 +748,10 @@ export default abstract class Server<ServerOptions extends Options = Options> {
             }
           }
 
-          const parsedUrl = parseUrl(pathname, true)
-
-          await this.render(
-            req,
-            res,
-            pathname,
-            { ..._parsedUrl.query, _nextDataReq: '1' },
-            parsedUrl,
-            true
-          )
           return {
-            finished: true,
+            pathname,
+            query: { ..._parsedUrl.query, __nextDataReq: '1' },
+            finished: false,
           }
         },
       },
@@ -1136,7 +1103,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
     if (
       !internalRender &&
       !this.minimalMode &&
-      !query._nextDataReq &&
+      !query.__nextDataReq &&
       (req.url?.match(/^\/_next\//) ||
         (this.hasStaticDir && req.url!.match(/^\/static\//)))
     ) {
@@ -1208,9 +1175,36 @@ export default abstract class Server<ServerOptions extends Options = Options> {
 
     // Toggle whether or not this is a Data request
     const isDataReq =
-      !!query._nextDataReq && (isSSG || hasServerProps || isServerComponent)
+      !!query.__nextDataReq && (isSSG || hasServerProps || isServerComponent)
 
-    delete query._nextDataReq
+    // normalize req.url for SSG paths as it is not exposed
+    // to getStaticProps and the asPath should not expose /_next/data
+    if (
+      isSSG &&
+      this.minimalMode &&
+      req.headers['x-matched-path'] &&
+      req.url.startsWith('/_next/data')
+    ) {
+      req.url = this.stripNextDataPath(req.url)
+    }
+
+    if (!!query.__nextDataReq) {
+      res.setHeader(
+        'x-nextjs-matched-path',
+        `${query.__nextLocale ? `/${query.__nextLocale}` : ''}${pathname}`
+      )
+      // return empty JSON when not an SSG/SSP page and not an error
+      if (
+        !(isSSG || hasServerProps) &&
+        (!res.statusCode || res.statusCode === 200 || res.statusCode === 404)
+      ) {
+        res.setHeader('content-type', 'application/json')
+        res.body('{}')
+        res.send()
+        return null
+      }
+    }
+    delete query.__nextDataReq
 
     // Don't delete query.__flight__ yet, it still needs to be used in renderToHTML later
     const isFlightRequest = Boolean(
@@ -1710,7 +1704,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
     }
   }
 
-  private stripNextDataPath(path: string) {
+  private stripNextDataPath(path: string, stripLocale = true) {
     if (path.includes(this.buildId)) {
       const splitPath = path.substring(
         path.indexOf(this.buildId) + this.buildId.length
@@ -1719,7 +1713,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
       path = denormalizePagePath(splitPath.replace(/\.json$/, ''))
     }
 
-    if (this.nextConfig.i18n) {
+    if (this.nextConfig.i18n && stripLocale) {
       const { locales } = this.nextConfig.i18n
       return normalizeLocalePath(path, locales).pathname
     }

packages/next/server/dev/next-dev-server.ts

@@ -415,6 +415,9 @@ export default class DevServer extends Server {
             }))
 
           this.router.setDynamicRoutes(this.dynamicRoutes)
+          this.router.setCatchallMiddleware(
+            this.generateCatchAllMiddlewareRoute(true)
+          )
 
           if (!resolved) {
             resolve()

packages/next/server/next-server.ts

@@ -707,7 +707,7 @@ export default class NextNodeServer extends BaseServer {
             ...(components.getStaticProps
               ? ({
                   amp: query.amp,
-                  _nextDataReq: query._nextDataReq,
+                  __nextDataReq: query.__nextDataReq,
                   __nextLocale: query.__nextLocale,
                   __nextDefaultLocale: query.__nextDefaultLocale,
                   __flight__: query.__flight__,
@@ -1116,7 +1116,12 @@ export default class NextNodeServer extends BaseServer {
     const normalizedPathname = removeTrailingSlash(params.parsedUrl.pathname)
 
     // For middleware to "fetch" we must always provide an absolute URL
-    const url = getRequestMeta(params.request, '__NEXT_INIT_URL')!
+    const query = urlQueryToSearchParams(params.parsed.query).toString()
+    const locale = params.parsed.query.__nextLocale
+    const url = `http://${this.hostname}:${this.port}${
+      locale ? `/${locale}` : ''
+    }${params.parsed.pathname}${query ? `?${query}` : ''}`
+
     if (!url.startsWith('http')) {
       throw new Error(
         'To use middleware you must provide a `hostname` and `port` to the Next.js Server'
@@ -1211,9 +1216,15 @@ export default class NextNodeServer extends BaseServer {
     return result
   }
 
-  protected generateCatchAllMiddlewareRoute(): Route | undefined {
+  protected generateCatchAllMiddlewareRoute(
+    devReady?: boolean
+  ): Route | undefined {
     if (this.minimalMode) return undefined
 
+    if ((!this.renderOpts.dev || devReady) && !this.getMiddleware().length) {
+      return undefined
+    }
+
     return {
       match: getPathMatch('/:path*'),
       type: 'route',

packages/next/server/request-meta.ts

@@ -59,7 +59,7 @@ type NextQueryMetadata = {
   __nextLocale?: string
   __nextSsgPath?: string
   _nextBubbleNoFallback?: '1'
-  _nextDataReq?: '1'
+  __nextDataReq?: '1'
 }
 
 export type NextParsedUrlQuery = ParsedUrlQuery &
@@ -80,7 +80,7 @@ export function getNextInternalQuery(
     '__nextLocale',
     '__nextSsgPath',
     '_nextBubbleNoFallback',
-    '_nextDataReq',
+    '__nextDataReq',
   ]
   const nextInternalQuery: NextQueryMetadata = {}
 

packages/next/server/router.ts

@@ -111,6 +111,9 @@ export default class Router {
   setDynamicRoutes(routes: DynamicRoutes = []) {
     this.dynamicRoutes = routes
   }
+  setCatchallMiddleware(route?: Route) {
+    this.catchAllMiddleware = route
+  }
 
   addFsRoute(fsRoute: Route) {
     this.fsRoutes.unshift(fsRoute)
@@ -208,12 +211,15 @@ export default class Router {
       */
 
       const allRoutes = [
+        ...(this.catchAllMiddleware
+          ? this.fsRoutes.filter((r) => r.name === '_next/data catchall')
+          : []),
         ...this.headers,
         ...this.redirects,
-        ...this.rewrites.beforeFiles,
         ...(this.useFileSystemPublicRoutes && this.catchAllMiddleware
           ? [this.catchAllMiddleware]
           : []),
+        ...this.rewrites.beforeFiles,
         ...this.fsRoutes,
         // We only check the catch-all route if public page routes hasn't been
         // disabled