fix: use system environment variables for robots (#1077)

.changeset/giant-carrots-peel.md

@@ -0,0 +1,5 @@
+---
+"@bigcommerce/catalyst-core": patch
+---
+
+Uses the deployment URL for the robots.txt sitemap field rather than another environment variable.

core/app/robots.ts

@@ -1,27 +1,30 @@
 import type { MetadataRoute } from 'next';
 
-// Disallow routes that have no SEO value or should not be indexed
-const disallow = ['/cart', '/account'];
-
-// Robots.txt config https://nextjs.org/docs/app/api-reference/file-conventions/metadata/robots#generate-a-robots-file
-const robotsConfig: MetadataRoute.Robots = {
-  rules: [
-    {
-      userAgent: '*',
-      allow: ['/'],
-      disallow,
-    },
-  ],
-};
+function parseUrl(url?: string): URL {
+  let incomingUrl = '';
+  const defaultUrl = new URL('http://localhost:3000/');
 
-// Infer base URL from environment variables
-const baseUrl = process.env.PRODUCTION_BASE_URL || process.env.NEXTAUTH_URL;
+  if (url && !url.startsWith('http')) {
+    incomingUrl = `https://${url}`;
+  }
 
-// Set sitemap URL if base URL is defined, as sitemap URL must be an absolute URL
-if (baseUrl) {
-  robotsConfig.sitemap = `${baseUrl}/sitemap.xml`;
+  return new URL(incomingUrl || defaultUrl);
 }
 
+// Disallow routes that have no SEO value or should not be indexed
+const disallow = ['/cart', '/account'];
+
+// Robots.txt config https://nextjs.org/docs/app/api-reference/file-conventions/metadata/robots#generate-a-robots-file
 export default function robots(): MetadataRoute.Robots {
-  return robotsConfig;
+  return {
+    // Infer base URL from environment variables
+    sitemap: parseUrl(process.env.NEXTAUTH_URL || process.env.VERCEL_URL || '').origin,
+    rules: [
+      {
+        userAgent: '*',
+        allow: ['/'],
+        disallow,
+      },
+    ],
+  };
 }