Crawl-delay: 10
# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow:

Sitemap: https://cadogantate.com/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK

# --- AI / LLM crawlers ---
# Cadogan Tate wants maximum AI visibility: allow both training
# crawlers (so models learn the brand) and search/retrieval bots
# (so we're citable in live AI answers).

# OpenAI
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Google AI (training + grounding for Gemini)
User-agent: Google-Extended
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Common Crawl (feeds many AI datasets)
User-agent: CCBot
Allow: /

# ByteDance — see note below
User-agent: Bytespider
Disallow: /

# Standard SEO bots (Googlebot/Bingbot) remain allowed by default