# Voidly.ai - Robots.txt
# Allow all crawlers by default

User-agent: *
Allow: /
Disallow: /mint
Disallow: /api/
Disallow: /_next/
Disallow: /admin

# Allow data, research, and AI tool endpoints
Allow: /data/
Allow: /censorship-index/
Allow: /methodology
Allow: /agents
Allow: /mcp
Allow: /agent-email
Allow: /veil
Allow: /nansen
Allow: /apps
Allow: /.well-known/

# Crawl-delay for polite crawling
Crawl-delay: 0.5

# Sitemap location
Sitemap: https://voidly.ai/sitemap.xml

# AI Crawlers - explicitly allow data endpoints for indexing
User-agent: GPTBot
Allow: /censorship-index/
Allow: /censorship-index/faq
Allow: /data/
Allow: /methodology
Allow: /agents
Allow: /mcp
Allow: /feed
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/data/
Allow: /.well-known/
Allow: /incidents/feed.xml
Allow: /agent-email
Allow: /veil
Allow: /nansen
Allow: /apps

User-agent: Google-Extended
Allow: /censorship-index/
Allow: /censorship-index/faq
Allow: /data/
Allow: /methodology
Allow: /agents
Allow: /mcp
Allow: /feed
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/data/
Allow: /.well-known/
Allow: /incidents/feed.xml
Allow: /agent-email
Allow: /veil
Allow: /nansen
Allow: /apps

User-agent: CCBot
Allow: /censorship-index/
Allow: /censorship-index/faq
Allow: /data/
Allow: /methodology
Allow: /agents
Allow: /mcp
Allow: /feed
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/data/
Allow: /.well-known/
Allow: /incidents/feed.xml
Allow: /agent-email
Allow: /veil
Allow: /nansen
Allow: /apps

User-agent: anthropic-ai
Allow: /censorship-index/
Allow: /censorship-index/faq
Allow: /data/
Allow: /methodology
Allow: /agents
Allow: /mcp
Allow: /feed
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/data/
Allow: /.well-known/
Allow: /incidents/feed.xml
Allow: /agent-email
Allow: /veil
Allow: /nansen
Allow: /apps

User-agent: Claude-Web
Allow: /censorship-index/
Allow: /censorship-index/faq
Allow: /data/
Allow: /methodology
Allow: /agents
Allow: /mcp
Allow: /feed
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/data/
Allow: /.well-known/
Allow: /incidents/feed.xml
Allow: /agent-email
Allow: /veil
Allow: /nansen
Allow: /apps

User-agent: PerplexityBot
Allow: /censorship-index/
Allow: /censorship-index/faq
Allow: /data/
Allow: /methodology
Allow: /agents
Allow: /mcp
Allow: /feed
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/data/
Allow: /.well-known/
Allow: /incidents/feed.xml
Allow: /agent-email
Allow: /veil
Allow: /nansen
Allow: /apps

User-agent: Applebot-Extended
Allow: /censorship-index/
Allow: /censorship-index/faq
Allow: /data/
Allow: /methodology
Allow: /agents
Allow: /mcp
Allow: /feed
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /api/data/
Allow: /.well-known/
Allow: /incidents/feed.xml
Allow: /agent-email
Allow: /veil
Allow: /nansen
Allow: /apps

User-agent: Bytespider
Allow: /censorship-index/
Allow: /data/
Allow: /methodology
Allow: /agents
Allow: /mcp
Allow: /llms.txt
Allow: /.well-known/
Allow: /agent-email
Allow: /veil

User-agent: Amazonbot
Allow: /censorship-index/
Allow: /data/
Allow: /methodology
Allow: /agents
Allow: /mcp
Allow: /llms.txt
Allow: /.well-known/
Allow: /agent-email
Allow: /veil

User-agent: Meta-ExternalAgent
Allow: /censorship-index/
Allow: /data/
Allow: /methodology
Allow: /agents
Allow: /mcp
Allow: /llms.txt
Allow: /.well-known/
Allow: /agent-email
Allow: /veil

User-agent: cohere-ai
Allow: /censorship-index/
Allow: /data/
Allow: /methodology
Allow: /agents
Allow: /mcp
Allow: /llms.txt
Allow: /.well-known/
Allow: /agent-email
Allow: /veil

# RSS Feed for data updates
# https://voidly.ai/feed.xml - Censorship updates

# LLM-specific file (emerging standard)
# https://voidly.ai/llms.txt - AI crawler instructions
# https://voidly.ai/llms-full.txt - Full documentation for AI

# FAQ with FAQPage schema
# https://voidly.ai/censorship-index/faq

# Agent discovery
# https://voidly.ai/.well-known/agent-card.json - A2A Protocol v0.3.0
# https://voidly.ai/.well-known/mcp/server-card.json - MCP Server Card