feat(docs): allow to configure noIndex per doc version (#7963)

facebook · Sep 1, 2022 · 5edb652 · 5edb652
1 parent c458f28
commit 5edb652
Show file tree

Hide file tree

Showing 12 changed files with 103 additions and 42 deletions.
diff --git a/packages/docusaurus-plugin-content-docs/src/__tests__/__snapshots__/index.test.ts.snap b/packages/docusaurus-plugin-content-docs/src/__tests__/__snapshots__/index.test.ts.snap
@@ -901,6 +901,7 @@ exports[`simple website content: data 1`] = `
   "label": "Next",
   "banner": null,
   "badge": false,
+  "noIndex": false,
   "className": "docs-version-current",
   "isLast": true,
   "docsSidebars": {
@@ -2608,6 +2609,7 @@ exports[`versioned website (community) content: data 1`] = `
   "label": "1.0.0",
   "banner": null,
   "badge": true,
+  "noIndex": false,
   "className": "docs-version-1.0.0",
   "isLast": true,
   "docsSidebars": {
@@ -2635,6 +2637,7 @@ exports[`versioned website (community) content: data 1`] = `
   "label": "Next",
   "banner": "unreleased",
   "badge": true,
+  "noIndex": false,
   "className": "docs-version-current",
   "isLast": false,
   "docsSidebars": {
@@ -3477,6 +3480,7 @@ exports[`versioned website content: data 1`] = `
   "label": "1.0.0",
   "banner": "unmaintained",
   "badge": true,
+  "noIndex": false,
   "className": "docs-version-1.0.0",
   "isLast": false,
   "docsSidebars": {
@@ -3544,6 +3548,7 @@ exports[`versioned website content: data 1`] = `
   "label": "1.0.1",
   "banner": null,
   "badge": true,
+  "noIndex": true,
   "className": "docs-version-1.0.1",
   "isLast": true,
   "docsSidebars": {
@@ -3599,6 +3604,7 @@ exports[`versioned website content: data 1`] = `
   "label": "Next",
   "banner": "unreleased",
   "badge": true,
+  "noIndex": false,
   "className": "docs-version-current",
   "isLast": false,
   "docsSidebars": {
@@ -3674,6 +3680,7 @@ exports[`versioned website content: data 1`] = `
   "label": "withSlugs",
   "banner": "unmaintained",
   "badge": true,
+  "noIndex": false,
   "className": "docs-version-withSlugs",
   "isLast": false,
   "docsSidebars": {

diff --git a/packages/docusaurus-plugin-content-docs/src/__tests__/index.test.ts b/packages/docusaurus-plugin-content-docs/src/__tests__/index.test.ts
@@ -362,6 +362,11 @@ describe('versioned website', () => {
       options: {
         routeBasePath,
         sidebarPath,
+        versions: {
+          '1.0.1': {
+            noIndex: true,
+          },
+        },
       },
     });
     const plugin = await pluginContentDocs(context, options);

diff --git a/packages/docusaurus-plugin-content-docs/src/__tests__/options.test.ts b/packages/docusaurus-plugin-content-docs/src/__tests__/options.test.ts
@@ -76,6 +76,7 @@ describe('normalizeDocsPluginOptions', () => {
         version1: {
           path: 'hello',
           label: 'world',
+          noIndex: true,
         },
       },
       sidebarCollapsible: false,

diff --git a/packages/docusaurus-plugin-content-docs/src/options.ts b/packages/docusaurus-plugin-content-docs/src/options.ts
@@ -59,6 +59,7 @@ const VersionOptionsSchema = Joi.object({
   banner: Joi.string().equal('none', 'unreleased', 'unmaintained').optional(),
   badge: Joi.boolean().optional(),
   className: Joi.string().optional(),
+  noIndex: Joi.boolean().optional(),
 });
 
 const VersionsOptionsSchema = Joi.object()

diff --git a/packages/docusaurus-plugin-content-docs/src/plugin-content-docs.d.ts b/packages/docusaurus-plugin-content-docs/src/plugin-content-docs.d.ts
@@ -125,6 +125,25 @@ declare module '@docusaurus/plugin-content-docs' {
   // TODO support custom version banner?
   // {type: "error", content: "html content"}
   export type VersionBanner = 'unreleased' | 'unmaintained';
+
+  export type VersionOptions = {
+    /**
+     * The base path of the version, will be appended to `baseUrl` +
+     * `routeBasePath`.
+     */
+    path?: string;
+    /** The label of the version to be used in badges, dropdowns, etc. */
+    label?: string;
+    /** The banner to show at the top of a doc of that version. */
+    banner?: 'none' | VersionBanner;
+    /** Show a badge with the version label at the top of each doc. */
+    badge?: boolean;
+    /** Prevents search engines from indexing this version */
+    noIndex?: boolean;
+    /** Add a custom class name to the <html> element of each doc. */
+    className?: string;
+  };
+
   export type VersionsOptions = {
     /**
      * The version navigated to in priority and displayed by default for docs
@@ -144,23 +163,7 @@ declare module '@docusaurus/plugin-content-docs' {
     /** Include the current version of your docs. */
     includeCurrentVersion: boolean;
     /** Independent customization of each version's properties. */
-    versions: {
-      [versionName: string]: {
-        /**
-         * The base path of the version, will be appended to `baseUrl` +
-         * `routeBasePath`.
-         */
-        path?: string;
-        /** The label of the version to be used in badges, dropdowns, etc. */
-        label?: string;
-        /** The banner to show at the top of a doc of that version. */
-        banner?: 'none' | VersionBanner;
-        /** Show a badge with the version label at the top of each doc. */
-        badge?: boolean;
-        /** Add a custom class name to the <html> element of each doc. */
-        className?: string;
-      };
-    };
+    versions: {[versionName: string]: VersionOptions};
   };
   export type SidebarOptions = {
     /**
@@ -263,6 +266,8 @@ declare module '@docusaurus/plugin-content-docs' {
     banner: VersionBanner | null;
     /** Show a badge with the version label at the top of each doc. */
     badge: boolean;
+    /** Prevents search engines from indexing this version */
+    noIndex: boolean;
     /** Add a custom class name to the <html> element of each doc. */
     className: string;
     /**
@@ -500,7 +505,7 @@ declare module '@docusaurus/plugin-content-docs' {
 
   export type PropVersionMetadata = Pick<
     VersionMetadata,
-    'label' | 'banner' | 'badge' | 'className' | 'isLast'
+    'label' | 'banner' | 'badge' | 'className' | 'isLast' | 'noIndex'
   > & {
     /** ID of the docs plugin this version belongs to. */
     pluginId: string;

diff --git a/packages/docusaurus-plugin-content-docs/src/props.ts b/packages/docusaurus-plugin-content-docs/src/props.ts
@@ -142,6 +142,7 @@ export function toVersionMetadataProp(
     label: loadedVersion.label,
     banner: loadedVersion.banner,
     badge: loadedVersion.badge,
+    noIndex: loadedVersion.noIndex,
     className: loadedVersion.className,
     isLast: loadedVersion.isLast,
     docsSidebars: toSidebarsProp(loadedVersion),

diff --git a/packages/docusaurus-plugin-content-docs/src/versions/__tests__/index.test.ts b/packages/docusaurus-plugin-content-docs/src/versions/__tests__/index.test.ts
@@ -56,6 +56,7 @@ describe('readVersionsMetadata', () => {
         path: '/docs',
         banner: null,
         badge: false,
+        noIndex: false,
         className: 'docs-version-current',
       };
       return {simpleSiteDir, defaultOptions, defaultContext, vCurrent};
@@ -218,6 +219,7 @@ describe('readVersionsMetadata', () => {
         path: '/docs/next',
         banner: 'unreleased',
         badge: true,
+        noIndex: false,
         className: 'docs-version-current',
       };
 
@@ -242,6 +244,7 @@ describe('readVersionsMetadata', () => {
         path: '/docs',
         banner: null,
         badge: true,
+        noIndex: false,
         className: 'docs-version-1.0.1',
       };
 
@@ -266,6 +269,7 @@ describe('readVersionsMetadata', () => {
         path: '/docs/1.0.0',
         banner: 'unmaintained',
         badge: true,
+        noIndex: false,
         className: 'docs-version-1.0.0',
       };
 
@@ -290,6 +294,7 @@ describe('readVersionsMetadata', () => {
         path: '/docs/withSlugs',
         banner: 'unmaintained',
         badge: true,
+        noIndex: false,
         className: 'docs-version-withSlugs',
       };
 
@@ -657,6 +662,7 @@ describe('readVersionsMetadata', () => {
         path: '/communityBasePath/next',
         banner: 'unreleased',
         badge: true,
+        noIndex: false,
         className: 'docs-version-current',
       };
 
@@ -681,6 +687,7 @@ describe('readVersionsMetadata', () => {
         path: '/communityBasePath',
         banner: null,
         badge: true,
+        noIndex: false,
         className: 'docs-version-1.0.0',
       };
 

diff --git a/packages/docusaurus-plugin-content-docs/src/versions/index.ts b/packages/docusaurus-plugin-content-docs/src/versions/index.ts
@@ -122,6 +122,13 @@ export function getVersionBadge({
   return options.versions[versionName]?.badge ?? defaultVersionBadge;
 }
 
+export function getVersionNoIndex({
+  versionName,
+  options,
+}: VersionContext): VersionMetadata['noIndex'] {
+  return options.versions[versionName]?.noIndex ?? false;
+}
+
 function getVersionClassName({
   versionName,
   options,
@@ -179,6 +186,7 @@ async function createVersionMetadata(
     label: getVersionLabel(context),
     banner: getVersionBanner(context),
     badge: getVersionBadge(context),
+    noIndex: getVersionNoIndex(context),
     className: getVersionClassName(context),
     path: routePath,
     tagsPath: normalizeUrl([routePath, options.tagsBasePath]),

diff --git a/packages/docusaurus-theme-classic/src/theme/DocPage/index.tsx b/packages/docusaurus-theme-classic/src/theme/DocPage/index.tsx
@@ -7,7 +7,11 @@
 
 import React from 'react';
 import clsx from 'clsx';
-import {HtmlClassNameProvider, ThemeClassNames} from '@docusaurus/theme-common';
+import {
+  HtmlClassNameProvider,
+  ThemeClassNames,
+  PageMetadata,
+} from '@docusaurus/theme-common';
 import {
   docVersionSearchTag,
   DocsSidebarProvider,
@@ -19,13 +23,8 @@ import NotFound from '@theme/NotFound';
 import SearchMetadata from '@theme/SearchMetadata';
 import type {Props} from '@theme/DocPage';
 
-export default function DocPage(props: Props): JSX.Element {
+function DocPageMetadata(props: Props): JSX.Element {
   const {versionMetadata} = props;
-  const currentDocRouteMetadata = useDocRouteMetadata(props);
-  if (!currentDocRouteMetadata) {
-    return <NotFound />;
-  }
-  const {docElement, sidebarName, sidebarItems} = currentDocRouteMetadata;
   return (
     <>
       <SearchMetadata
@@ -35,6 +34,25 @@ export default function DocPage(props: Props): JSX.Element {
           versionMetadata.version,
         )}
       />
+      <PageMetadata>
+        {versionMetadata.noIndex && (
+          <meta name="robots" content="noindex, nofollow" />
+        )}
+      </PageMetadata>
+    </>
+  );
+}
+
+export default function DocPage(props: Props): JSX.Element {
+  const {versionMetadata} = props;
+  const currentDocRouteMetadata = useDocRouteMetadata(props);
+  if (!currentDocRouteMetadata) {
+    return <NotFound />;
+  }
+  const {docElement, sidebarName, sidebarItems} = currentDocRouteMetadata;
+  return (
+    <>
+      <DocPageMetadata {...props} />
       <HtmlClassNameProvider
         className={clsx(
           // TODO: it should be removed from here

diff --git a/website/_dogfooding/dogfooding.config.js b/website/_dogfooding/dogfooding.config.js
@@ -26,6 +26,11 @@ const dogfoodingPluginInstances = [
       id: 'docs-tests',
       routeBasePath: '/tests/docs',
       sidebarPath: '_dogfooding/docs-tests-sidebars.js',
+      versions: {
+        current: {
+          noIndex: true,
+        },
+      },
 
       // Using a _ prefix to test against an edge case regarding MDX partials: https://github.com/facebook/docusaurus/discussions/5181#discussioncomment-1018079
       path: '_dogfooding/_docs tests',

diff --git a/website/docs/api/plugins/plugin-content-docs.md b/website/docs/api/plugins/plugin-content-docs.md
@@ -142,23 +142,25 @@ type CategoryIndexMatcher = (param: {
 #### `VersionsConfig` {#VersionsConfig}
 
 ```ts
-type VersionsConfig = {
-  [versionName: string]: {
-    /**
-     * The base path of the version, will be appended to `baseUrl` +
-     * `routeBasePath`.
-     */
-    path?: string;
-    /** The label of the version to be used in badges, dropdowns, etc. */
-    label?: string;
-    /** The banner to show at the top of a doc of that version. */
-    banner?: 'none' | 'unreleased' | 'unmaintained';
-    /** Show a badge with the version label at the top of each doc. */
-    badge?: boolean;
-    /** Add a custom class name to the <html> element of each doc */
-    className?: string;
-  };
+type VersionConfig = {
+  /**
+   * The base path of the version, will be appended to `baseUrl` +
+   * `routeBasePath`.
+   */
+  path?: string;
+  /** The label of the version to be used in badges, dropdowns, etc. */
+  label?: string;
+  /** The banner to show at the top of a doc of that version. */
+  banner?: 'none' | 'unreleased' | 'unmaintained';
+  /** Show a badge with the version label at the top of each doc. */
+  badge?: boolean;
+  /** Prevents search engines from indexing this version */
+  noIndex?: boolean;
+  /** Add a custom class name to the <html> element of each doc */
+  className?: string;
 };
+
+type VersionsConfig = {[versionName: string]: VersionConfig};
 ```
 
 ### Example configuration {#ex-config}

diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js
@@ -356,7 +356,8 @@ const config = {
             }
           : undefined,
         sitemap: {
-          ignorePatterns: ['/tests/**'],
+          // Note: /tests/docs already has noIndex: true
+          ignorePatterns: ['/tests/{blog,pages}/**'],
         },
       }),
     ],