# robots.txt for LLMS Config (https://llmsconfig.com)
# Purpose: Control traditional search engine crawler access
# For AI crawler control, see: /llms.txt
# Last Updated: January 21, 2026
# Learn more: https://www.robotstxt.org/

# ============================================================================
# SECTION 1: Default Policy for All Traditional Search Engines
# ============================================================================

User-agent: *

# ALLOW: All public content
Allow: /

# DISALLOW: Administrative and private areas
Disallow: /account
Disallow: /sign-in
Disallow: /sign-up
Disallow: /api
Disallow: /admin
Disallow: /_next
Disallow: /convex

# DISALLOW: Staging and demo environments
Disallow: /staging/
Disallow: /demo/
Disallow: /dev/
Disallow: /test/

# ============================================================================
# SECTION 2: Major Search Engines
# ============================================================================

# Google (Googlebot)
User-agent: Googlebot
Allow: /
Disallow: /account
Disallow: /api
Disallow: /admin
Disallow: /_next

# Google Images
User-agent: Googlebot-Image
Allow: /
Disallow: /account
Disallow: /api

# Bing (MSNBot)
User-agent: msnbot
Allow: /
Disallow: /account
Disallow: /api
Disallow: /admin

# Bing Preview
User-agent: BingPreview
Allow: /
Disallow: /account
Disallow: /api

# DuckDuckGo (DuckDuckBot)
User-agent: DuckDuckBot
Allow: /
Disallow: /account
Disallow: /api
Disallow: /admin

# Yahoo (Slurp)
User-agent: Slurp
Allow: /
Disallow: /account
Disallow: /api

# Baidu (Baiduspider)
User-agent: Baiduspider
Allow: /
Disallow: /account
Disallow: /api
Disallow: /admin

# Yandex (YandexBot)
User-agent: YandexBot
Allow: /
Disallow: /account
Disallow: /api

# ============================================================================
# SECTION 3: AI Crawlers (for reference - see /llms.txt for detailed rules)
# ============================================================================

# OpenAI GPTBot (ChatGPT, Bing Chat)
User-agent: GPTBot
Allow: /
Disallow: /account
Disallow: /api
Disallow: /sign-in
Disallow: /sign-up

# ChatGPT User Agent
User-agent: ChatGPT-User
Allow: /
Disallow: /account

# Google Extended (Google SGE, AI Overviews)
User-agent: Google-Extended
Allow: /
Disallow: /account
Disallow: /api

# Anthropic Claude
User-agent: Claude-Web
Allow: /
Disallow: /account
Disallow: /api

User-agent: anthropic-ai
Allow: /
Disallow: /account
Disallow: /api

# Common Crawl (CCBot)
User-agent: CCBot
Allow: /docs
Allow: /examples
Disallow: /account
Disallow: /api

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /account
Disallow: /api

# You.com
User-agent: YouBot
Allow: /
Disallow: /account
Disallow: /api

# Amazon Bot
User-agent: Amazonbot
Allow: /docs
Allow: /examples
Disallow: /account
Disallow: /api

# Meta FacebookBot
User-agent: FacebookBot
Allow: /examples
Allow: /docs
Disallow: /account
Disallow: /api

# ============================================================================
# SECTION 4: SEO and Marketing Tools
# ============================================================================

# SEMrush - Block aggressive crawling
User-agent: SemrushBot
Disallow: /

# Ahrefs - Block aggressive crawling
User-agent: AhrefsBot
Disallow: /

# Majestic SEO - Block aggressive crawling
User-agent: MJ12bot
Disallow: /

# Moz - Block aggressive crawling
User-agent: DotBot
Disallow: /

# Screaming Frog - Allow for SEO auditing (with permission)
User-agent: ScreamingFrogSEOBot
Crawl-delay: 2

# ============================================================================
# SECTION 5: Specialized Crawlers
# ============================================================================

# Internet Archive (Wayback Machine)
User-agent: ia_archiver
Allow: /

# LinkedIn Bot
User-agent: LinkedInBot
Allow: /

# Twitter Bot
User-agent: Twitterbot
Allow: /

# Facebook External Hit
User-agent: facebookexternalhit
Allow: /

# Pinterest
User-agent: Pinterest
Allow: /

# ============================================================================
# SECTION 6: Security and Monitoring
# ============================================================================

# Block potentially malicious bots
User-agent: *.\\.com$
Disallow: /

User-agent: ^\\w*\\.{10,}$
Disallow: /

# ============================================================================
# SECTION 7: Crawl Delay and Request Rate
# ============================================================================

# General crawl delay (1 second between requests)
Crawl-delay: 1

# ============================================================================
# SECTION 8: Sitemap Information
# ============================================================================

# Primary sitemap
Sitemap: https://llmsconfig.com/sitemap.xml

# Additional sitemaps (if applicable)
# Sitemap: https://llmsconfig.com/sitemap-blog.xml
# Sitemap: https://llmsconfig.com/sitemap-docs.xml

# ============================================================================
# END OF FILE
# ============================================================================

# For questions about this file, contact: hello@llmsconfig.com
# For AI crawler control, see: https://llmsconfig.com/llms.txt
# Security policy: https://llmsconfig.com/security.txt