# Vedara - vedara.me
# Vedic Astrology Insights & Personal Timing

# Every crawler is welcome. Per RFC 9309 the "*" group already covers any bot
# without its own group, but the named groups below are kept because OpenAI
# recommends explicitly allowing OAI-SearchBot for Search inclusion.
#
# IMPORTANT: a crawler obeys ONLY the most specific group that matches it, so
# every named group must repeat the Disallow lines. Add a bare "Allow: /" group
# for a bot and you silently invite it into /internal/ and /api/.

User-agent: *
# The public calculators API (and its OpenAPI spec) is the one /api/ surface
# meant to be crawled and called by agents (PR #534). It must be listed BEFORE
# the /api/ disallow (first-match parsers stop at the first hit) and in EVERY
# group (a bot with its own group never reads the * group's rules).
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /
# Do NOT disallow /_next/static/ — Googlebot must fetch JS/CSS to render
# client-side pages (homepage, tools); blocking it makes them index as empty.

# --- AI crawlers: welcome, same rules as everyone else ---

User-agent: GPTBot
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

User-agent: OAI-SearchBot
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

User-agent: ChatGPT-User
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

User-agent: PerplexityBot
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

User-agent: ClaudeBot
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

User-agent: Claude-User
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

User-agent: Claude-SearchBot
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

# Product tokens are case-insensitive, so one anthropic-ai group covers all
# spellings of it.
User-agent: anthropic-ai
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

User-agent: Google-Extended
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

User-agent: CCBot
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

User-agent: Applebot-Extended
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

User-agent: Meta-ExternalAgent
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

User-agent: FacebookBot
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

User-agent: Amazonbot
Allow: /api/public/tools
Allow: /api/public/agent-tools
Disallow: /internal/
Disallow: /api/
Disallow: /debug-env
Allow: /

# Bytespider now falls through to the "*" group above (allowed), per the
# decision to let every agent crawl.

# Sitemap
Sitemap: https://vedara.me/sitemap.xml

# LLM Context File
# See https://vedara.me/llms.txt for structured information about this site