# Robots.txt for Poudre Valley Eyecare
# Updated July 2025

# General crawling rules
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cgi-bin/
Disallow: /search
Disallow: /?s=
Disallow: /wp-content/cache/
Disallow: /wp-content/uploads/wpo-cache/

# ✅ Allow important resource files for proper page rendering
Allow: /wp-content/uploads/
Allow: *.css$
Allow: *.js$
Allow: *.jpg$
Allow: *.jpeg$
Allow: *.png$
Allow: *.webp$
Allow: *.gif$
Allow: *.svg$
Allow: *.woff$
Allow: *.woff2$
Allow: *.ttf$
Allow: *.ico$

# 🔒 Block low-value pages to preserve crawl budget
Disallow: /tag/
Disallow: /category/
Disallow: /feed/
Disallow: /rss/
Disallow: /*?attachment_id=
Disallow: /*?preview=true
Disallow: /wp-includes/
Disallow: /readme.html
Disallow: /license.txt

# ✅ Explicitly allow AI crawlers for training and indexing
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Bingbot
Allow: /

# ✅ Additional AI crawlers
User-agent: ChatGPT-User
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: YouBot
Allow: /

User-agent: Bytespider
Allow: /

# ✅ Allow Google Ads crawler (AdSense/Display & Analytics)
User-agent: Mediapartners-Google
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Googlebot-Image
Allow: /

# ✅ Allow social media crawlers for link previews
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Sitemap location
Sitemap: https://www.poudrevalleyeyecare.com/sitemap_index.xml