Updated robots.txt and sitemap added

2026-06-20 17:05:04 +02:00 · 2026-06-20 17:05:04 +02:00 · 07aa53689e
commit 07aa53689e
parent b9efbf81cc
3 changed files with 186 additions and 0 deletions
--- a/robots.txt
+++ b/robots.txt
@ -0,0 +1,38 @@
+User-agent: facebookexternalhit
+Allow: /
+
+User-agent: Facebot
+Allow: /
+
+User-agent: meta-externalagent
+Allow: /
+
+User-agent: *
+Content-Signal: search=yes,ai-train=no
+Allow: /
+
+User-agent: Amazonbot
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: CloudflareBrowserRenderingCrawler
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+Sitemap: https://bubulescu.org/sitemap.xml