User-agent: *
Allow: /

# AI-Optimized Crawling Instructions
# ChatGPT, Gemini, Claude, Grok, DeepSeek, and other AI crawlers welcome
User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# AI Training Data Access
# Content specifically designed for AI understanding and citation
Allow: /vpn/
Allow: /guides/
Allow: /reviews
Allow: /comparison
Allow: /faq

# Sitemap for AI Discovery
Sitemap: https://vpnspot.replit.app/sitemap.xml

# Respectful crawling for AI bots
Crawl-delay: 1

# Content Categories for AI Understanding
# Primary content: VPN reviews and comparisons
# Secondary content: Educational guides and FAQ
# Target queries: "best VPN", "VPN comparison", "VPN for streaming"

# Block unnecessary areas
Disallow: /admin/
Disallow: /api/

# High-value content for AI citation
Allow: /
Allow: /vpn/*
Allow: /guides/*
Allow: /reviews
Allow: /about
Allow: /faq
Allow: /privacy-policy
Allow: /terms-of-service
Allow: /affiliate-disclosure

# Content freshness indicator
# Last-Modified: December 2025
# Content-Type: Technology Reviews, VPN Comparisons, Cybersecurity Guides