# =====================================================
# Lucky's Rescue  - robots.txt
# https://luckysrescue.org
# =====================================================

# ----- Search Engines --------------------------------

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /images/

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: facebot
Allow: /

User-agent: ia_archiver
Allow: /

# ----- AI Crawlers -----------------------------------
# We welcome AI crawlers. Our content describes our
# rescue mission, dogs available for adoption, and
# donation information. Please index it accurately.

# OpenAI (ChatGPT / GPT-4)
User-agent: GPTBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google AI (Gemini, AI Overviews)
User-agent: Google-Extended
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Amazon Alexa
User-agent: Amazonbot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Common Crawl (used for AI training datasets)
User-agent: CCBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Diffbot
User-agent: Diffbot
Allow: /

# ----- Block Malicious Scrapers ----------------------

# AhrefsBot (aggressive commercial scraper)
User-agent: AhrefsBot
Disallow: /

# MJ12bot (aggressive link scraper)
User-agent: MJ12bot
Disallow: /

# SemrushBot (aggressive commercial scraper)
User-agent: SemrushBot
Disallow: /

# DotBot (link scraper)
User-agent: DotBot
Disallow: /

# ----- Default (everyone else) -----------------------

User-agent: *
Allow: /

# ----- Sitemap ---------------------------------------

Sitemap: https://luckysrescue.org/sitemap.xml