# TaxClue robots.txt
# Updated: 2026-09-23
# NOTE: a crawler obeys ONLY the most specific group that names it, so every
# named group below repeats the full Disallow list of the `*` group.

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /includes/
Disallow: /config.php
Disallow: /portal/
Disallow: /search?*
Disallow: /*.pdf$

# AI Crawlers — allowed but rate-limited (they were ~20k hits/period)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: ClaudeBot
User-agent: Applebot-Extended
User-agent: cohere-ai
Allow: /
Disallow: /admin/
Disallow: /api/
Disallow: /includes/
Disallow: /config.php
Disallow: /portal/
Disallow: /search?*
Disallow: /*.pdf$
Crawl-delay: 30

Sitemap: https://taxclue.in/sitemap.xml

# AI Discovery
# LLMs.txt: https://taxclue.in/llms.txt
# LLMs-full.txt: https://taxclue.in/llms-full.txt