# AI Skill Market - Robots.txt
# https://aiskill.market

# ==============================================
# AI CRAWLER BOTS (Explicitly Allowed for GEO)
# ==============================================

# OpenAI GPTBot - Powers ChatGPT web browsing
User-agent: GPTBot
Allow: /

# OpenAI ChatGPT-User - ChatGPT's user-triggered browsing
User-agent: ChatGPT-User
Allow: /

# Anthropic Claude-Web - Claude's web browsing
User-agent: Claude-Web
Allow: /

# Anthropic ClaudeBot - Claude's training crawler
User-agent: ClaudeBot
Allow: /

# Anthropic AI crawler
User-agent: anthropic-ai
Allow: /

# Perplexity AI - Perplexity's search crawler
User-agent: PerplexityBot
Allow: /

# Google Gemini / Bard
User-agent: Google-Extended
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# Facebook Sharing Debugger / Open Graph Crawler
User-agent: facebookexternalhit
Allow: /

# Facebook Catalog Crawler
User-agent: Facebot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Twitter/X Card Crawler
User-agent: Twitterbot
Allow: /

# LinkedIn Preview Crawler
User-agent: LinkedInBot
Allow: /

# ==============================================
# SEARCH ENGINE BOTS (Standard SEO)
# ==============================================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

# ==============================================
# BLOCK AGGRESSIVE SEO BOTS (Save CPU)
# ==============================================

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: BomboraBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: ScreamingFrogSEOSpider
Disallow: /

User-agent: seekport
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

# ==============================================
# ALL OTHER BOTS
# ==============================================

User-agent: *
Allow: /

# Disallow administrative and API endpoints
Disallow: /api/
Disallow: /_next/static/

# ==============================================
# SITEMAPS
# ==============================================

Sitemap: https://aiskill.market/sitemap

# ==============================================
# CRAWL-DELAY (Optional, for polite crawling)
# ==============================================
# Most major bots ignore this, but it's good practice
# Crawl-delay: 1