# MadhuDadi Master Robots.txt — Consolidated (Blog & Portfolio)
# Last Updated: 2026-07-26
# Canonical production source: blog_platform/nginx/robots.txt → /opt/madhudadi-blog/nginx/robots.txt
# Keep in sync with madhu_portfolio/config/robots.production.txt (local dev / CI mirror)
# Policy: allow traditional + AI search/citation crawlers; disallow model-training crawlers.

User-agent: *
Allow: /
# Portfolio Protections
Disallow: /studio/
Disallow: /api/
# Blog Protections (Consolidated)
Disallow: /blog/admin/
Disallow: /blog/api/v1/admin/
# Leaderboard + /blog/profile: do NOT Disallow here.
# Google: robots.txt block is not a de-index signal (pages can still appear from links).
# De-index via X-Robots-Tag: noindex (set in blog_frontend next.config headers).
# Shared/Auth Protections
Disallow: /blog/api/v1/auth/
Disallow: /blog/api/v1/payments/
Disallow: /blog/api/v1/
Disallow: /cdn-cgi/


# ── Bing/Yahoo — keep minimal delay ────────────────────────────────
# A named group REPLACES User-agent: * for that crawler - robots.txt groups do
# not inherit - so every Disallow the * group relies on must be repeated here,
# or Bing/Yahoo are free to crawl the admin and API surfaces.
User-agent: bingbot
User-agent: adidxbot
User-agent: slurp
Crawl-delay: 1
Disallow: /studio/
Disallow: /api/
Disallow: /blog/admin/
Disallow: /blog/api/v1/admin/
Disallow: /blog/api/v1/auth/
Disallow: /blog/api/v1/payments/
Disallow: /blog/api/v1/
Disallow: /cdn-cgi/

# ── Model-training / dataset crawlers — DISALLOW (opt out of training use) ─
# Does NOT block Google Search (Googlebot) or ChatGPT Search (OAI-SearchBot).
# Does NOT block AI Overviews / AI Mode — those follow Googlebot, allowed via *.
# Google-Extended was moved to the ALLOW section (2026-07-27): it is both a
# training AND grounding control. Allowing it keeps Gemini app citations while
# still opting out of training-only crawlers below. Every other token here is
# primarily a training control (GPTBot, ClaudeBot, CCBot, etc.).
# Google confirms none of these affect Search inclusion or ranking.
# Do not list Googlebot-Extended — it is not a real Google crawler token.
# Reviewed against Google's docs 2026-07-27.
User-agent: GPTBot
User-agent: Applebot-Extended
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: CCBot
User-agent: cohere-ai
User-agent: Diffbot
User-agent: Meta-ExternalAgent
User-agent: Bytespider
User-agent: AI2Bot
User-agent: Omgilibot
User-agent: Omgili
User-agent: Timpibot
User-agent: ImagesiftBot
User-agent: PanguBot
User-agent: Webzio-Extended
User-agent: FriendlyCrawler
User-agent: Kangaroo Bot
User-agent: iaskspider/2.0
Disallow: /

# ── AI search, citation, previews, and user-triggered fetchers — ALLOW ─────
# Explicit allow-list (independent of GPTBot/ClaudeBot training opt-out above).
# Blocking Claude-SearchBot / PerplexityBot can reduce citation visibility.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: Applebot
User-agent: BraveBot
User-agent: YouBot
User-agent: facebookexternalhit
User-agent: Google-Extended
User-agent: Meta-ExternalFetcher
Allow: /
Allow: /blog/api/og
Allow: /blog
Allow: /blog/posts
Allow: /blog/series
Allow: /blog/tags
Allow: /blog/ask
Allow: /blog/llms.txt
Allow: /blog/llms-full.txt
Allow: /blog/ai-profile.json
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai-profile.json
Disallow: /blog/admin
Disallow: /blog/api/v1/admin
Disallow: /blog/api/v1/auth
Disallow: /blog/api/v1/payments
Disallow: /blog/api/v1/
Disallow: /studio/
Disallow: /api/
Disallow: /cdn-cgi/

# Sitemaps
# Child sitemaps are listed alongside the index on purpose: the index is a single
# point of failure for discovery, so crawlers keep a working path to each child
# even if it is unreachable.
Sitemap: https://madhudadi.in/sitemap.xml
Sitemap: https://madhudadi.in/sitemap-portfolio.xml
Sitemap: https://madhudadi.in/sitemap-assets.xml
Sitemap: https://madhudadi.in/blog/sitemap.xml