# EnglishFolio robots.txt
# Search engines and AI search/answer engines are welcome: they link back to
# the pages they use. AI training and dataset crawlers are NOT: this content
# is original work by Bharat Choudhary and may not be used to train AI models
# without written permission. See /terms.html.

User-agent: *
Allow: /

# AI search and answer engines: they cite and link the pages they draw on
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: YouBot
Allow: /

# AI model training and dataset crawlers
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: meta-externalagent
User-agent: FacebookBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Amazonbot
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Timpibot
User-agent: AI2Bot
User-agent: PanguBot
Disallow: /

Sitemap: https://englishfolio.com/sitemap.xml