User-agent: *
Allow: /

# Don't index RSS feeds
Disallow: /*index.xml
Disallow: /*rss.xml

# Don't index GitBook internal paths
Disallow: /mirrord/docs/~gitbook/
Disallow: /spaces/

# Don't index Hugo tag pages (thin auto-generated content)
Disallow: /tags/

# Don't index GTM endpoint
Disallow: /zb2v/

# Don't crawl GitBook "Ask AI" query URLs (infinite URL space)
Disallow: /*?ask=

# Don't crawl Cloudflare internal endpoints
Disallow: /cdn-cgi/

# AI content usage preferences (Content Signals)
Content-Signal: ai-train=yes, search=yes, ai-input=yes

Sitemap: https://metalbear.com/sitemap.xml