# ==============================================================================
# robots.txt for its.ucla.edu — updated 2026-09-08
# ==============================================================================

# --- Default: all compliant crawlers ---
User-agent: *
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /feed/
Allow: /
Allow: /wp-admin/admin-ajax.php

# --- 1. Allow search-based AI bots (ensures visibility and citations) ---
User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

# --- 2. Block training/scraping AI bots ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

# --- 3. Block dataset/research scrapers (no traffic reciprocity) ---
User-agent: MathPicDatasetCrawler
Disallow: /

User-agent: Reflectionbot
Disallow: /

User-agent: xAI-PDF-Recovery
Disallow: /

User-agent: coccocbot
Disallow: /

User-agent: YisouSpider
Disallow: /

# --- Sitemap ---
Sitemap: https://its.ucla.edu/sitemap_index.xml