# Cyberlete Robots.txt
# https://cyberlete.net
# Social Gaming Platform for Competitive Gamers

User-agent: *
Allow: /

# Landing Pages - High Priority
Allow: /depin
Allow: /token
Allow: /game-server

# Public Content
Allow: /tournament
Allow: /tournament/
Allow: /team
Allow: /team/
Allow: /profile/
Allow: /leagues
Allow: /community
Allow: /streams

# Legal Pages
Allow: /privacy-policy
Allow: /terms-of-service

# AI/LLM Information Files
Allow: /llms.txt
Allow: /ai.txt

# Block authenticated/private routes
Disallow: /settings
Disallow: /notifications
Disallow: /wallet
Disallow: /messages
Disallow: /chat
Disallow: /home

# Block API and internal paths
Disallow: /api/
Disallow: /_/
Disallow: /admin/

# Sitemaps
Sitemap: https://cyberlete.net/sitemap.xml
Sitemap: https://cyberlete.net/api/v1/seo/sitemap.xml

# Crawl-delay for politeness
Crawl-delay: 1

# ============================================
# AI/LLM Crawlers - All Welcome
# ============================================

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /

User-agent: Googlebot
Allow: /

# Anthropic (Claude)
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Common Crawl (used for AI training)
User-agent: CCBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Meta/Facebook AI
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Microsoft/Bing AI
User-agent: bingbot
Allow: /

User-agent: Bingbot
Allow: /

# Apple
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# ByteDance
User-agent: Bytespider
Allow: /

# Brave Search
User-agent: BraveBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Neeva
User-agent: NeevaBot
Allow: /

# AI Search Engines
User-agent: AI2Bot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: omgili
Allow: /

User-agent: omgilibot
Allow: /

# ============================================
# Research & Academic Crawlers
# ============================================

User-agent: ia_archiver
Allow: /

User-agent: archive.org_bot
Allow: /