# robots.txt for Sinaptica

# ─── General crawlers ──────────────────────────────────────────────────────
# Allow standard crawlers (Googlebot, Bingbot, Applebot, DuckDuckBot, Yandex,
# Baidu, social-preview crawlers, etc.) to fetch all public content.
User-agent: *
Allow: /

# Disallow API endpoints
Disallow: /api/

# Disallow admin panels
Disallow: /a/
Disallow: /admin/

# Disallow auth pages (no value in indexing)
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify-email
Disallow: /verify-email-pending
Disallow: /account-not-verified

# Disallow user-private pages
Disallow: /settings
Disallow: /messages
Disallow: /notifications
Disallow: /activity-log
Disallow: /create-post
Disallow: /create-course
Disallow: /my-courses
Disallow: /payout-settings

# ─── Google AI grounding / AI Overviews ───────────────────────────────────
# Allow Google-Extended so Sinaptica content appears in AI Overviews / Gemini
# grounding. AI Overviews are a major traffic channel in 2026; the trade-off
# (content used for AI grounding) is acceptable since the content is already
# public. Note: the Cloudflare-managed robots.txt block at the top of the
# served file may override this; flip the toggle in Cloudflare dashboard
# (Security → Bots → Managed robots.txt) to fully enable.
User-agent: Google-Extended
Allow: /

# ─── AI *search* crawlers — these send referral traffic (citations) ────────
# ChatGPT Search, Perplexity and Claude search fetch pages to cite them.
# Distinct from the training crawlers blocked below.
User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

# ─── Block AI training crawlers that don't return traffic ─────────────────
# These crawlers train competitor AI models without sending users back to us.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

# ─── Internal search results are not landing pages ────────────────────────
User-agent: *
Disallow: /?q=
Disallow: /*?q=

# ─── Sitemaps (per-language + news) ───────────────────────────────────────
# /sitemap.xml is the index; it points to the per-language sub-sitemaps.
Sitemap: https://sinapti.ca/sitemap.xml
Sitemap: https://sinapti.ca/sitemap-en.xml
Sitemap: https://sinapti.ca/sitemap-es.xml
Sitemap: https://sinapti.ca/sitemap-news.xml