# =============================================================================
# Aurora Lolita - robots.txt
# Last reviewed 2026-07-15.
#
# Layout:
#   1. The default rules apply to ALL crawlers (User-agent: *).
#   2. Explicit Allow blocks for major LLM training / inference crawlers
#      (GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, anthropic-ai,
#      PerplexityBot, Perplexity-User, CCBot, Google-Extended, Applebot-
#      Extended, Amazonbot, Bytespider, YouBot, Diffbot). Without an
#      explicit allow some of these UAs treat the site as opt-out by
#      default; we want our news, blog and product copy to be indexable
#      by AI search.
#   3. The same Disallow list as the default block, repeated per LLM bot,
#      so private endpoints stay private even for AI crawlers.
#
# Notes:
#   - Wildcards in robots.txt are an extension supported by Google,
#     Bing, Yandex and most LLM crawlers but are NOT in the original
#     RFC. We avoid them in this file for portability.
#   - Sitemap is a top-level directive: search engines ignore which
#     User-agent block it sits in.
# =============================================================================

# Global AI Content Usage Preferences (RFC 8288 & Content Signals Initiative)
# Declaring preferences: allow live search referencing, deny core model training.
Content-Signal: ai-train=no, search=yes, ai-input=yes

User-agent: *
Allow: /
Disallow: /leonine.html
Disallow: /ops-portal.html
Disallow: /admin-announcements.html
Disallow: /503.html
Disallow: /login.html
Disallow: /verify-email.html
Disallow: /reset-password.html
Disallow: /account.html
Disallow: /checkout.html
Disallow: /cdn-cgi/
Disallow: /aurora-backend/
Disallow: /aurora-frontend/
Disallow: /deploy/
Disallow: /backup_*/
Disallow: /ai.readme/
Disallow: /.agents/
Disallow: /.vscode/
Disallow: /js/admin.js
Disallow: /js/admin-announcements.js
Disallow: /js/perf.js
Disallow: /error.log
Disallow: /base_structure.html
Disallow: /server_8080.js

# Allow CSS and public JS for rendering
Allow: /css/
Allow: /js/config.js
Allow: /js/i18n.js
Allow: /js/i18n-db.js
Allow: /js/cart.js
Allow: /js/product.js
Allow: /js/index.js
Allow: /js/category.js
Allow: /js/auth.js
Allow: /js/lazy-load.js
Allow: /js/announcements.js
Allow: /js/news-render.js
Allow: /js/blog.js
Allow: /js/blog-post.js

# -----------------------------------------------------------------------------
# AI search & training crawlers - explicit opt-in for the public storefront.
# Each block re-states the same private-area Disallow list so an AI agent
# that reads only its own UA section still respects the boundary.
# -----------------------------------------------------------------------------

# OpenAI - GPTBot is the training crawler, ChatGPT-User is the live agent called by ChatGPT browse, OAI-SearchBot powers SearchGPT.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /
Disallow: /leonine.html
Disallow: /ops-portal.html
Disallow: /admin-announcements.html
Disallow: /503.html
Disallow: /login.html
Disallow: /verify-email.html
Disallow: /reset-password.html
Disallow: /account.html
Disallow: /checkout.html
Disallow: /cdn-cgi/
Disallow: /aurora-backend/
Disallow: /aurora-frontend/
Disallow: /deploy/
Disallow: /backup_*/
Disallow: /ai.readme/
Disallow: /.agents/
Disallow: /.vscode/
Disallow: /js/admin.js
Disallow: /js/admin-announcements.js
Disallow: /js/perf.js
Disallow: /error.log
Disallow: /base_structure.html
Disallow: /server_8080.js

# Anthropic / Claude
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Claude-SearchBot
Allow: /
Disallow: /leonine.html
Disallow: /ops-portal.html
Disallow: /admin-announcements.html
Disallow: /503.html
Disallow: /login.html
Disallow: /verify-email.html
Disallow: /reset-password.html
Disallow: /account.html
Disallow: /checkout.html
Disallow: /cdn-cgi/
Disallow: /aurora-backend/
Disallow: /aurora-frontend/
Disallow: /deploy/
Disallow: /backup_*/
Disallow: /ai.readme/
Disallow: /.agents/
Disallow: /.vscode/
Disallow: /js/admin.js
Disallow: /js/admin-announcements.js
Disallow: /js/perf.js
Disallow: /error.log
Disallow: /base_structure.html
Disallow: /server_8080.js

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /leonine.html
Disallow: /ops-portal.html
Disallow: /admin-announcements.html
Disallow: /503.html
Disallow: /login.html
Disallow: /verify-email.html
Disallow: /reset-password.html
Disallow: /account.html
Disallow: /checkout.html
Disallow: /cdn-cgi/
Disallow: /aurora-backend/
Disallow: /aurora-frontend/
Disallow: /deploy/
Disallow: /backup_*/
Disallow: /ai.readme/
Disallow: /.agents/
Disallow: /.vscode/
Disallow: /js/admin.js
Disallow: /js/admin-announcements.js
Disallow: /js/perf.js
Disallow: /error.log
Disallow: /base_structure.html
Disallow: /server_8080.js

# Common Crawl (training data behind many open LLMs)
User-agent: CCBot
Allow: /
Disallow: /leonine.html
Disallow: /ops-portal.html
Disallow: /admin-announcements.html
Disallow: /503.html
Disallow: /login.html
Disallow: /verify-email.html
Disallow: /reset-password.html
Disallow: /account.html
Disallow: /checkout.html
Disallow: /cdn-cgi/
Disallow: /aurora-backend/
Disallow: /aurora-frontend/
Disallow: /deploy/
Disallow: /backup_*/
Disallow: /ai.readme/
Disallow: /.agents/
Disallow: /.vscode/
Disallow: /js/admin.js
Disallow: /js/admin-announcements.js
Disallow: /js/perf.js
Disallow: /error.log
Disallow: /base_structure.html
Disallow: /server_8080.js

# Google AI overviews / Gemini training
User-agent: Google-Extended
Allow: /
Disallow: /leonine.html
Disallow: /ops-portal.html
Disallow: /admin-announcements.html
Disallow: /503.html
Disallow: /login.html
Disallow: /verify-email.html
Disallow: /reset-password.html
Disallow: /account.html
Disallow: /checkout.html
Disallow: /cdn-cgi/
Disallow: /aurora-backend/
Disallow: /aurora-frontend/
Disallow: /deploy/
Disallow: /backup_*/
Disallow: /ai.readme/
Disallow: /.agents/
Disallow: /.vscode/
Disallow: /js/admin.js
Disallow: /js/admin-announcements.js
Disallow: /js/perf.js
Disallow: /error.log
Disallow: /base_structure.html
Disallow: /server_8080.js

# Apple Intelligence / Siri summaries
User-agent: Applebot-Extended
User-agent: Applebot
Allow: /
Disallow: /leonine.html
Disallow: /ops-portal.html
Disallow: /admin-announcements.html
Disallow: /503.html
Disallow: /login.html
Disallow: /verify-email.html
Disallow: /reset-password.html
Disallow: /account.html
Disallow: /checkout.html
Disallow: /cdn-cgi/
Disallow: /aurora-backend/
Disallow: /aurora-frontend/
Disallow: /deploy/
Disallow: /backup_*/
Disallow: /ai.readme/
Disallow: /.agents/
Disallow: /.vscode/
Disallow: /js/admin.js
Disallow: /js/admin-announcements.js
Disallow: /js/perf.js
Disallow: /error.log
Disallow: /base_structure.html
Disallow: /server_8080.js

# Other AI search agents that have published their UA
User-agent: Amazonbot
User-agent: YouBot
User-agent: Diffbot
User-agent: cohere-ai
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Bytespider
Allow: /
Disallow: /leonine.html
Disallow: /ops-portal.html
Disallow: /admin-announcements.html
Disallow: /503.html
Disallow: /login.html
Disallow: /verify-email.html
Disallow: /reset-password.html
Disallow: /account.html
Disallow: /checkout.html
Disallow: /cdn-cgi/
Disallow: /aurora-backend/
Disallow: /aurora-frontend/
Disallow: /deploy/
Disallow: /backup_*/
Disallow: /ai.readme/
Disallow: /.agents/
Disallow: /.vscode/
Disallow: /js/admin.js
Disallow: /js/admin-announcements.js
Disallow: /js/perf.js
Disallow: /error.log
Disallow: /base_structure.html
Disallow: /server_8080.js

# Public discovery
# AI commerce manifest: https://www.lolitawear.com/.well-known/ai-commerce.json
Sitemap: https://www.lolitawear.com/sitemap.xml
Sitemap: https://www.lolitawear.com/sitemap-pages.xml
Sitemap: https://www.lolitawear.com/sitemap-products.xml
Sitemap: https://www.lolitawear.com/sitemap-categories.xml
Sitemap: https://www.lolitawear.com/sitemap-blog.xml
Sitemap: https://www.lolitawear.com/sitemap-news.xml
Sitemap: https://www.lolitawear.com/sitemap-images.xml
Sitemap: https://www.lolitawear.com/sitemap-ai.xml