User-agent: *
Allow: /
Disallow: /posts/history/

# General Scrapers/Aggregators
User-agent: Amazonbot
User-agent: Bytespider
User-agent: AhrefsBot
User-agent: CCBot
Disallow: /

# AI Crawlers - Blocked only from /files/
User-agent: Amzn-SearchBot
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: GPTBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: meta-externalagent
User-agent: CloudflareBrowserRenderingCrawler
User-agent: OAI-SearchBot
Disallow: /files/

Sitemap: https://gsbooru.org/sitemap.xml?sitemap=forums

Sitemap: https://gsbooru.org/sitemap.xml?sitemap=posts

Sitemap: https://gsbooru.org/sitemap.xml?sitemap=tags

Sitemap: https://gsbooru.org/sitemap.xml?sitemap=users

Sitemap: https://gsbooru.org/sitemap.xml?sitemap=wiki_pages