# ================================
# Oysten Host – Smart Crawl Control
# ================================

# --- Major Search Engines ---
User-agent: Googlebot

User-agent: Bingbot
Crawl-delay: 10

User-agent: BingPreview
Crawl-delay: 10

User-agent: msnbot
Crawl-delay: 10

User-agent: Yandex
Crawl-delay: 8

User-agent: Baiduspider
Crawl-delay: 10

User-agent: DuckDuckBot
Crawl-delay: 5

# --- SEO Crawlers ---
User-agent: AhrefsBot
Crawl-delay: 20

User-agent: SemrushBot
Crawl-delay: 20

User-agent: MJ12bot
Crawl-delay: 20

User-agent: DotBot
Crawl-delay: 15

User-agent: BLEXBot
Crawl-delay: 20

# --- AI Crawlers ---
User-agent: GPTBot
Crawl-delay: 30

User-agent: CCBot
Crawl-delay: 30

User-agent: ClaudeBot
Crawl-delay: 30

User-agent: PerplexityBot
Crawl-delay: 30

User-agent: Amazonbot
Crawl-delay: 15

# --- Global Rules ---
User-agent: *
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /core/
Disallow: /vendor/
Disallow: /node_modules/
Disallow: /cache/
Disallow: /logs/
Disallow: /includes/
Disallow: /config/
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/

Crawl-delay: 5

# --- Sitemap ---
Sitemap: https://www.oystenhost.com/sitemap.xml