# Pantheon's documentation on robots.txt: https://pantheon.io/docs/bots-and-indexing/
User-agent: *
Disallow: /

# Allow accessibility and SEO audit tools
User-agent: RavenCrawler
User-agent: rogerbot
User-agent: dotbot
User-agent: SemrushBot
User-agent: SiteAuditBot
User-agent: SplitSignalBot
User-agent: PowerMapper
User-agent: Swiftbot
User-agent: lyticsbot
User-agent: Dubbotbot
User-agent: PopeTech-CrawlBot
User-agent: PopeTech-ScanBot
User-agent: Googlebot
User-agent: Bingbot

# Allow reputable AI crawlers
User-agent: PerplexityBot
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: OAI-SearchBot
User-agent: Claude-Web
User-agent: Applebot-Extended

Allow: /