User-agent: *
Allow: /

# Block AI training crawlers
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

# Do not index admin, user account, and payment flow pages
Disallow: /admin
Disallow: /admin/
Disallow: /admin/settings
Disallow: /profile
Disallow: /my-bookings
Disallow: /booking-success
Disallow: /booking-reserved
Disallow: /pay-booking
Disallow: /checkout
Disallow: /review/
Disallow: /cancel-booking/
Disallow: /booking/

# Block demo / test course pages from indexing (TeeOne demo tenants etc.)
Disallow: /club-demo1-desarrollo-links-tee-1-18-hoyos
Disallow: /club-demo2-desarrollo-tee-1-demo2
Disallow: /test-course

# Note: /courses/<slug> is intentionally NOT disallowed — those URLs return
# 301 to /<slug>, and Google needs to crawl them to follow the redirect and
# consolidate the indexing signals. Disallowing would freeze them in the
# index instead of cleaning them up.

# ----------------------------------------------------------------------
# AI / LLM crawlers — explicitly welcomed for the public booking content
# (course catalogue, blog, llms.txt, agents.json, OpenAPI spec).
#
# Per robots.txt spec each User-agent block is independent, so the
# Disallow rules above do NOT inherit. Every AI bot group below
# duplicates the same Disallow set so admin/checkout paths stay private
# regardless of which group the crawler matches.
# ----------------------------------------------------------------------

# OpenAI — training, search, and on-demand ChatGPT browsing
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /admin/settings
Disallow: /profile
Disallow: /my-bookings
Disallow: /booking-success
Disallow: /booking-reserved
Disallow: /pay-booking
Disallow: /checkout
Disallow: /review/
Disallow: /cancel-booking/
Disallow: /booking/

# Anthropic Claude
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /admin/settings
Disallow: /profile
Disallow: /my-bookings
Disallow: /booking-success
Disallow: /booking-reserved
Disallow: /pay-booking
Disallow: /checkout
Disallow: /review/
Disallow: /cancel-booking/
Disallow: /booking/

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /admin/settings
Disallow: /profile
Disallow: /my-bookings
Disallow: /booking-success
Disallow: /booking-reserved
Disallow: /pay-booking
Disallow: /checkout
Disallow: /review/
Disallow: /cancel-booking/
Disallow: /booking/

# Google Gemini / AI Overviews (separate opt-in from regular Googlebot)
User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /admin/settings
Disallow: /profile
Disallow: /my-bookings
Disallow: /booking-success
Disallow: /booking-reserved
Disallow: /pay-booking
Disallow: /checkout
Disallow: /review/
Disallow: /cancel-booking/
Disallow: /booking/

# Common Crawl (used by many LLMs for training corpora)
User-agent: CCBot
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /admin/settings
Disallow: /profile
Disallow: /my-bookings
Disallow: /booking-success
Disallow: /booking-reserved
Disallow: /pay-booking
Disallow: /checkout
Disallow: /review/
Disallow: /cancel-booking/
Disallow: /booking/

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /admin/settings
Disallow: /profile
Disallow: /my-bookings
Disallow: /booking-success
Disallow: /booking-reserved
Disallow: /pay-booking
Disallow: /checkout
Disallow: /review/
Disallow: /cancel-booking/
Disallow: /booking/

# Meta AI
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /admin/settings
Disallow: /profile
Disallow: /my-bookings
Disallow: /booking-success
Disallow: /booking-reserved
Disallow: /pay-booking
Disallow: /checkout
Disallow: /review/
Disallow: /cancel-booking/
Disallow: /booking/

# You.com, Phind, Cohere
User-agent: YouBot
User-agent: PhindBot
User-agent: cohere-ai
Allow: /
Disallow: /admin
Disallow: /admin/
Disallow: /admin/settings
Disallow: /profile
Disallow: /my-bookings
Disallow: /booking-success
Disallow: /booking-reserved
Disallow: /pay-booking
Disallow: /checkout
Disallow: /review/
Disallow: /cancel-booking/
Disallow: /booking/

# Sitemap location (canonical: with www)
Sitemap: https://www.marbellagolftimes.com/sitemap.xml