# Mozilla.ai robots.txt
# Strategy: Open access for all crawlers, including AI training and AI search bots.
# Aligned with Mozilla.ai's mission of trustworthy, transparent, and open AI.

# ============================================
# Default rule: allow all bots access to all content
# ============================================
User-agent: *
Allow: /

# ============================================
# AI training crawlers (explicitly allowed)
# These bots collect content to train AI models.
# ============================================

# OpenAI - trains ChatGPT
User-agent: GPTBot
Allow: /

# Anthropic - trains Claude
User-agent: ClaudeBot
Allow: /

# Common Crawl - open dataset used by many AI projects
User-agent: CCBot
Allow: /

# Google - controls AI training (Gemini, Vertex AI). Separate from Googlebot search indexing.
User-agent: Google-Extended
Allow: /

# Apple - controls AI training (Apple Intelligence). Separate from Applebot search indexing.
User-agent: Applebot-Extended
Allow: /

# Meta - trains Llama and Meta AI products
User-agent: Meta-ExternalAgent
Allow: /

# ByteDance - trains Doubao and other ByteDance AI products
User-agent: Bytespider
Allow: /

# Amazon - trains Alexa AI and other Amazon AI products
User-agent: Amazonbot
Allow: /

# Cohere - trains Cohere models
User-agent: cohere-ai
Allow: /

# ============================================
# AI search and user-initiated fetchers (explicitly allowed)
# These bots fetch content in real-time to answer user queries
# and typically return citation traffic to the source site.
# ============================================

# OpenAI - real-time search for ChatGPT
User-agent: OAI-SearchBot
Allow: /

# OpenAI - fetches URLs requested by ChatGPT users
User-agent: ChatGPT-User
Allow: /

# Anthropic - fetches URLs requested by Claude users
User-agent: Claude-User
Allow: /

# Anthropic - real-time search for Claude
User-agent: Claude-SearchBot
Allow: /

# Perplexity - powers Perplexity answer engine with citations
User-agent: PerplexityBot
Allow: /

# Perplexity - user-initiated fetcher
User-agent: Perplexity-User
Allow: /

# Mistral - user-initiated fetcher for Le Chat
User-agent: MistralAI-User
Allow: /

# DuckDuckGo - powers DuckAssist AI answers
User-agent: DuckAssistBot
Allow: /

# ============================================
# Sitemap
# ============================================
Sitemap: https://www.mozilla.ai/sitemap.xml

Sitemap: https://www.mozilla.ai/sitemap.xml