Disallow several LLM scrapers

Based on https://wandering.shop/@patricksamphire/111147479309728367
offbyone · Sep 29, 2023 · a261152 · a261152
1 parent 404f839
commit a261152
Showing 1 changed file with 15 additions and 1 deletion.
diff --git a/content/extra/robots.txt b/content/extra/robots.txt
@@ -1,9 +1,23 @@
 User-agent: Googlebot
 Disallow: /drafts/
 
+User-agent: CCBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
 User-agent: GPTBot
 Disallow: /
 
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: Omgilibot
+Disallow: /
+
+User-Agent: FacebookBot
+Disallow: /
+
 User-agent: *
 Allow: /
-