# robots.txt for RxAI. Canonical pages are listed on www.rxai.com.au.

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /private/

# AI Crawler Governance — explicitly allow AI retrieval and answer engines.
# Each named group repeats the /admin/ and /private/ rules on purpose: a crawler
# obeys only its most specific matching group, so a named group that omitted them
# would grant broader access than the "*" group above.

# OpenAI — GPTBot trains/retrieves, OAI-SearchBot powers ChatGPT search citations
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /private/

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /private/

# User-triggered; ignores robots.txt by design, listed for transparency
User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /private/

# Anthropic
User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /private/

User-agent: anthropic-ai
Allow: /
Disallow: /admin/
Disallow: /private/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /private/

# Google — Gemini grounding opt-in, plus site-owner-requested Vertex AI crawls
User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /private/

User-agent: Google-CloudVertexBot
Allow: /
Disallow: /admin/
Disallow: /private/

# Apple
User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /private/

# Sitemap
Sitemap: https://www.rxai.com.au/sitemap.xml

# AI / LLM content index
# Clean Markdown sources (Open Knowledge Format .md pairs) for every canonical
# page are indexed in https://www.rxai.com.au/llms.txt
# All .md files are crawlable under "Allow: /" above — robots.txt lists crawl
# rules, not individual files, so the .md inventory lives in llms.txt.