# robots.txt for veycl.com
# Never block /user: the open content lives there too. Private pages are kept
# out of search with X-Robots-Tag noindex headers (vercel.json) and sign-in,
# which a crawler must be allowed to fetch in order to see.

# Search and AI answer crawlers: allowed.
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: *
# Disallow first: some parsers apply the first matching rule, Google the most specific.
Disallow: /admin
Allow: /

# AI training crawlers: blocked. They copy content into model training and send
# no readers. Blocking them does not affect Google Search, AI Overviews or AI
# search answers.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: meta-externalagent
Disallow: /

Sitemap: https://veycl.com/sitemap.xml