# Favly - robots.txt
# https://favly.me

# ==================== 通用爬虫规则 ====================
User-agent: *
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /profile/

# 允许爬取公开 profile 页面
Allow: /u/

# 允许爬取搜索页面
Allow: /search

# 允许爬取静态资源
Allow: /logos/
Allow: /_next/static/

# 爬取延迟（防止服务器过载）
Crawl-delay: 1

# ==================== AI 搜索引擎爬虫 ====================
# 明确允许 AI 爬虫访问，帮助网站在 AI 搜索结果中获得更好展示

# OpenAI ChatGPT
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /profile/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /profile/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /profile/

# Anthropic Claude
User-agent: anthropic-ai
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /profile/

User-agent: Claude-Web
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /profile/

# Google AI (Bard/Gemini)
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /profile/

# Microsoft Bing AI / Copilot
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /profile/

# Common Crawl (用于 AI 训练数据)
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /profile/

# ==================== 搜索引擎爬虫 ====================

# Google
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /profile/

# Bing
User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /auth/
Disallow: /profile/

# ==================== 网站地图 ====================
Sitemap: https://favly.me/sitemap.xml

# ==================== AI 信息文件 ====================
# llms.txt 是专门为 AI 搜索引擎准备的网站说明文件
# 位置: https://favly.me/llms.txt