⚡ perf: Update robots rule (#4141)

Co-authored-by: canisminor1990 <i@canisminor.cc>
lobehub · Sep 26, 2024 · 03017dd · 03017dd
1 parent 6a12d45
commit 03017dd
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 7 deletions.
diff --git a/next.config.mjs b/next.config.mjs
@@ -112,6 +112,11 @@ const nextConfig = {
       permanent: true,
       source: '/sitemap.xml',
     },
+    {
+      destination: '/sitemap-index.xml',
+      permanent: true,
+      source: '/sitemap-0.xml',
+    },
     {
       destination: '/manifest.webmanifest',
       permanent: true,

diff --git a/src/app/robots.tsx b/src/app/robots.tsx
@@ -3,14 +3,33 @@ import { MetadataRoute } from 'next';
 import { sitemapModule } from '@/server/sitemap';
 import { getCanonicalUrl } from '@/server/utils/url';
 
-export default function robots(): MetadataRoute.Robots {
+const robots = (): MetadataRoute.Robots => {
   return {
     host: getCanonicalUrl(),
-    rules: {
-      allow: ['/'],
-      disallow: ['/api/*'],
-      userAgent: '*',
-    },
+    rules: [
+      {
+        allow: ['/discover/*'],
+        disallow: ['/discover/search/*'],
+        userAgent: ['Facebot', 'facebookexternalhit'],
+      },
+      {
+        allow: ['/discover/*'],
+        disallow: ['/discover/search/*'],
+        userAgent: 'LinkedInBot',
+      },
+      {
+        allow: ['/discover/*'],
+        disallow: ['/discover/search/*'],
+        userAgent: 'Twitterbot',
+      },
+      {
+        allow: ['/'],
+        disallow: ['/api/*', '/login', '/signup', '/files', '/repos/*', '/discover/search/*'],
+        userAgent: '*',
+      },
+    ],
     sitemap: sitemapModule.getRobots(),
   };
-}
+};
+
+export default robots;