# Robots.txt for Shotlee
# Public SEO pages are crawlable. App-only surfaces are not.
# Last updated: 2026-08-13

# As a condition of accessing this website, you agree to
# abide by the following content signals:

# (a)  If a content-signal = yes, you may collect content
# for the corresponding use.
# (b)  If a content-signal = no, you may not collect content
# for the corresponding use.
# (c)  If the website operator does not include a content
# signal for a corresponding use, the website operator
# neither grants nor restricts permission via content signal
# with respect to the corresponding use.

# The content signals and their meanings are:

# search: building a search index and providing search
# results (e.g., returning hyperlinks and short excerpts
# from your website's contents).  Search does not include
# providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models
# (e.g., retrieval augmented generation, grounding, or other
# real-time taking of content for generative AI search
# answers).
# ai-train: training or fine-tuning AI models.

# ANY RESTRICTIONS EXPRESSED VIA CONTENT-SIGNALS ARE EXPRESS
# RESERVATIONS OF RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN
# UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED RIGHTS
# IN THE DIGITAL SINGLE MARKET.

# 1. Primary Search Engines — Need JS/CSS assets (_next/static) to render and index pages properly.
User-agent: Googlebot
Allow: /_next/static/
Allow: /_next/image/
# Fonts don't affect rendering/indexing; block to stop per-deploy (?dpl) re-crawl noise
Disallow: /_next/static/media/*.woff2
Disallow: /_next/static/media/*.woff
Allow: /
Allow: /llms.txt
Allow: /rss.xml
Allow: /feed.xml
Allow: /rss
Allow: /feed
Allow: /api/blog/rss
Allow: /api/health
Disallow: /_vercel/
Disallow: /_next/data/
Disallow: /api/
Disallow: /admin/
Disallow: /app/
Disallow: /calendar/
Disallow: /chat/
Disallow: /community/
Allow: /community-guidelines
Disallow: /dashboard/
Disallow: /health/
Disallow: /messages/
Disallow: /profile/
Disallow: /settings/
Disallow: /shots/
Disallow: /summary/
Disallow: /tracker
Disallow: /welcome
Disallow: /auth/
Disallow: /oauth-fix
Disallow: /redirect-handler
Disallow: /reset-password
Disallow: /upgrade
Disallow: /status
Disallow: /system-status

User-agent: Bingbot
Allow: /_next/static/
Allow: /_next/image/
# Fonts don't affect rendering/indexing; block to stop per-deploy (?dpl) re-crawl noise
Disallow: /_next/static/media/*.woff2
Disallow: /_next/static/media/*.woff
Allow: /
Allow: /llms.txt
Allow: /rss.xml
Allow: /feed.xml
Allow: /rss
Allow: /feed
Allow: /api/blog/rss
Allow: /api/health
Disallow: /_vercel/
Disallow: /_next/data/
Disallow: /api/
Disallow: /admin/
Disallow: /app/
Disallow: /calendar/
Disallow: /chat/
Disallow: /community/
Allow: /community-guidelines
Disallow: /dashboard/
Disallow: /health/
Disallow: /messages/
Disallow: /profile/
Disallow: /settings/
Disallow: /shots/
Disallow: /summary/
Disallow: /tracker
Disallow: /welcome
Disallow: /auth/
Disallow: /oauth-fix
Disallow: /redirect-handler
Disallow: /reset-password
Disallow: /upgrade
Disallow: /status
Disallow: /system-status

# 2. AI RETRIEVAL crawlers — allowed. These are what produce citations: they
# fetch a page at answer time and link it. Blocking any of these removes us
# from the index the assistant searches. Training-only crawlers are in §3.
# Blocked from JS/CSS chunks to save budget (they crawl raw HTML only).
#
# Google-Extended is deliberately ALLOWED despite Content-Signal ai-train=no:
# it is a control token that governs Gemini training AND grounding-at-prompt-
# time together, so disallowing it would cost Gemini citations. It does not
# affect Google Search ranking or AI Overviews — those follow Googlebot.
# Amazonbot is allowed as a judgement call: it feeds Alexa and Amazon shopping
# surfaces, and sources disagree on whether it is training or retrieval.
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: Meta-ExternalFetcher
User-agent: Applebot
User-agent: Amazonbot
User-agent: Google-Extended
Allow: /
Allow: /rss.xml
Allow: /feed.xml
Allow: /rss
Allow: /feed
Allow: /api/blog/rss
Allow: /api/health
Disallow: /_next/
Disallow: /_vercel/
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /shots/
Disallow: /profile/
Disallow: /community/
Disallow: /chat/
Disallow: /messages/
Disallow: /health/
Disallow: /calendar/
Disallow: /summary/
Disallow: /tracker
Disallow: /welcome
Disallow: /auth/
Disallow: /oauth-fix
Disallow: /redirect-handler
Disallow: /reset-password
Disallow: /upgrade
Disallow: /status
Disallow: /system-status

User-agent: OAI-SearchBot
Allow: /
Allow: /rss.xml
Allow: /feed.xml
Allow: /rss
Allow: /feed
Allow: /api/blog/rss
Allow: /api/health
Disallow: /_next/
Disallow: /_vercel/
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /shots/
Disallow: /profile/
Disallow: /community/
Disallow: /chat/
Disallow: /messages/
Disallow: /health/
Disallow: /calendar/
Disallow: /summary/
Disallow: /tracker
Disallow: /welcome
Disallow: /auth/
Disallow: /oauth-fix
Disallow: /redirect-handler
Disallow: /reset-password
Disallow: /upgrade
Disallow: /status
Disallow: /system-status

User-agent: ChatGPT-User
Allow: /
Disallow: /_next/
Disallow: /_vercel/

User-agent: PerplexityBot
Allow: /
Allow: /rss.xml
Allow: /feed.xml
Allow: /rss
Allow: /feed
Allow: /api/blog/rss
Allow: /api/health
Disallow: /_next/
Disallow: /_vercel/
Disallow: /api/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /settings/
Disallow: /shots/
Disallow: /profile/
Disallow: /community/
Disallow: /chat/
Disallow: /messages/
Disallow: /health/
Disallow: /calendar/
Disallow: /summary/
Disallow: /tracker
Disallow: /welcome
Disallow: /auth/
Disallow: /oauth-fix
Disallow: /redirect-handler
Disallow: /reset-password
Disallow: /upgrade
Disallow: /status
Disallow: /system-status

# 3. AI TRAINING-only crawlers — blocked, to match Content-Signal ai-train=no.
# Blocking these does NOT cost citations: each vendor uses a separate retrieval
# crawler (§2) for answering, and per Anthropic's own docs "blocking only
# ClaudeBot does not block Claude-SearchBot or Claude-User". The NYT blocked
# GPTBot and still receives ChatGPT traffic for exactly this reason.
User-agent: GPTBot
Disallow: /

# Anthropic training. Claude-SearchBot / Claude-User stay allowed in §2.
User-agent: ClaudeBot
Disallow: /

# Meta AI training. Meta-ExternalFetcher (user-triggered) stays allowed in §2.
User-agent: meta-externalagent
Disallow: /

# Apple foundation-model training only. Plain Applebot (Siri/Spotlight) stays
# allowed in §2 — this token governs generative use, not crawling.
User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: cohere-ai
Disallow: /

# 4. Ad Crawlers
User-agent: Google-adstxt
Disallow:

User-agent: Mediapartners-Google
Disallow:

# 5. Default Rules for All Other Crawlers — Block resource-heavy bundles and app routes
User-agent: *
# ai-train=no   → do not train or fine-tune models on our content
# search=yes    → index us and link back
# ai-input=yes  → cite/quote us in AI answers (this is the traffic we want)
# Key order matches the contentsignals.org example exactly. Unproven whether
# any parser is order-sensitive, but matching the spec costs nothing.
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Allow: /llms.txt
Allow: /rss.xml
Allow: /feed.xml
Allow: /rss
Allow: /feed
Allow: /api/blog/rss
Allow: /api/health
Crawl-delay: 1
Disallow: /_next/
Disallow: /_vercel/
Disallow: /api/
Disallow: /admin/
Disallow: /app/
Disallow: /calendar/
Disallow: /chat/
Disallow: /community/
Allow: /community-guidelines
Disallow: /dashboard/
Disallow: /health/
Disallow: /messages/
Disallow: /profile/
Disallow: /settings/
Disallow: /shots/
Disallow: /summary/
Disallow: /tracker
Disallow: /welcome
Disallow: /auth/
Disallow: /oauth-fix
Disallow: /redirect-handler
Disallow: /reset-password
Disallow: /upgrade
Disallow: /status
Disallow: /system-status

Sitemap: https://www.shotlee.app/sitemap.xml
Sitemap: https://www.shotlee.app/rss.xml