# robots.txt for weste.net (Weste.Net)
# Generated: 2026-05-13

# ============================================================
# Default rules for all crawlers
# ============================================================
User-agent: *
Disallow: /css/
Disallow: /js/
Disallow: /assets/

# Allow important paths
Allow: /zh-cn/
Allow: /2026/
Allow: /about/

# Crawl rate limit
Crawl-delay: 1

# Sitemap reference
Sitemap: https://www.weste.net/sitemap.xml

# AI crawler content preference signal (draft-romm-aipref-contentsignals)
Content-Signal: ai-train=yes;search=yes;ai-personalization=no;ai-retrieval=yes

# ============================================================
# AI Crawlers — explicitly allowed (for clarity and future-proofing)
# ============================================================

# OpenAI — ChatGPT search and training
User-agent: GPTBot
Disallow: /css/
Disallow: /js/
Disallow: /assets/
Allow: /

# OpenAI — search only
User-agent: OAI-SearchBot
Disallow: /css/
Disallow: /js/
Disallow: /assets/
Allow: /

# OpenAI — ChatGPT browsing mode
User-agent: ChatGPT-User
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Disallow: /css/
Disallow: /js/
Disallow: /assets/
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Disallow: /css/
Disallow: /js/
Disallow: /assets/
Allow: /

# Google Gemini (training, does not affect Search)
User-agent: Google-Extended
Disallow: /css/
Disallow: /js/
Disallow: /assets/
Allow: /

# Common Crawl (feeds many AI models)
User-agent: CCBot
Disallow: /css/
Disallow: /js/
Disallow: /assets/
Allow: /

# ByteDance / TikTok AI
User-agent: Bytespider
Disallow: /css/
Disallow: /js/
Disallow: /assets/
Allow: /

# Amazon / Alexa AI
User-agent: Amazonbot
Disallow: /css/
Disallow: /js/
Disallow: /assets/
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Disallow: /css/
Disallow: /js/
Disallow: /assets/
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# ============================================================
# Baidu — Chinese search engine (critical for this site)
# ============================================================
User-agent: Baiduspider
Disallow: /rss.php
Disallow: /css/
Disallow: /js/
Disallow: /assets/
Allow: /
Crawl-delay: 2
Sitemap: https://www.weste.net/sitemap.xml

# Baidu mobile crawler
User-agent: Baiduspider-render/2.0
Disallow: /css/
Disallow: /js/
Disallow: /assets/
Allow: /
Sitemap: https://www.weste.net/sitemap.xml

# ============================================================
# Bing
# ============================================================
User-agent: bingbot
Disallow: /css/
Disallow: /js/
Disallow: /assets/
Allow: /
Sitemap: https://www.weste.net/sitemap.xml