# =====================================================
# robots.txt for bluegym.it (WooCommerce)
# Updated: 2026-05-21 — Phase 9 bot optimization
# Strategy: Maximum AI visibility + filter crawl control
# Template: Reuse for bluegym.hr, bluegym.si, fitness.bluegym.hr
# =====================================================

Sitemap: https://bluegym.hr/sitemap_index.xml

# ---- Global defaults (User-agent: *) ----
# Note: Some bots (Meta, Google Ads) ignore this section
# and only read their own named section. Those are below.
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cgi-bin/
Disallow: /xmlrpc.php
# Search & pagination & feeds
Disallow: /?s=
Disallow: /search/
Disallow: /feed/
Disallow: /comments/
# WooCommerce non-public endpoints
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wishlist/
Disallow: /compare/
Disallow: /wc-api/
# Faceted navigation — prevents combinatorial crawl explosion
Disallow: /*?add-to-cart=*
Disallow: /*?wc-ajax=*
Disallow: /*?orderby=*
Disallow: /*?filter_*
Disallow: /*?rating=*
Disallow: /*?pa_*
Disallow: /*?attribute_pa_*
# Tracking parameters — no index value
Disallow: /*?replytocom=*
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
# Allow what's needed for rendering
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-json/oembed/1.0/embed
Crawl-delay: 10

# ---- Google ----
User-agent: Googlebot
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wishlist/
Disallow: /compare/
Disallow: /wc-api/
Disallow: /*?add-to-cart=*
Disallow: /*?wc-ajax=*
Disallow: /*?orderby=*
Disallow: /*?filter_*
Disallow: /*?rating=*
Disallow: /*?pa_*
Disallow: /*?attribute_pa_*
Disallow: /*?replytocom=*
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-json/oembed/1.0/embed

# Google AI training — ALLOW for maximum Gemini visibility
User-agent: Google-Extended
Allow: /

User-agent: Bingbot
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wishlist/
Disallow: /compare/
Disallow: /wc-api/
Disallow: /*?add-to-cart=*
Disallow: /*?wc-ajax=*
Disallow: /*?orderby=*
Disallow: /*?filter_*
Disallow: /*?rating=*
Disallow: /*?pa_*
Disallow: /*?attribute_pa_*
Disallow: /*?replytocom=*
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-json/oembed/1.0/embed

# ---- Meta / Facebook ----
# facebookexternalhit — link previews, OG tags, catalog. ESSENTIAL.
# No section needed — follows User-agent: * rules.

# meta-externalagent — Meta AI training + content indexing.
# Ignores User-agent: * — needs its own section.
# ALLOW crawling for AI visibility, but block filter URLs
# that caused 514K-hit crawl loop.
User-agent: meta-externalagent
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wishlist/
Disallow: /compare/
Disallow: /wc-api/
Disallow: /*?add-to-cart=*
Disallow: /*?wc-ajax=*
Disallow: /*?orderby=*
Disallow: /*?filter_*
Disallow: /*?rating=*
Disallow: /*?pa_*
Disallow: /*?attribute_pa_*
Disallow: /*?replytocom=*
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /

# meta-externalads — Ad landing page verification. ESSENTIAL for FB Ads.
# Ignores User-agent: * — needs its own section.
User-agent: meta-externalads
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Allow: /

# Meta-ExternalFetcher — Real-time Meta AI answers. May bypass robots.txt.
# Allowing explicitly for completeness.
User-agent: Meta-ExternalFetcher
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wishlist/
Disallow: /compare/
Disallow: /wc-api/
Disallow: /*?add-to-cart=*
Disallow: /*?wc-ajax=*
Disallow: /*?orderby=*
Disallow: /*?filter_*
Disallow: /*?rating=*
Disallow: /*?pa_*
Disallow: /*?attribute_pa_*
Disallow: /*?replytocom=*
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
Allow: /

# Meta-WebIndexer — Meta AI search index, cites and links to your content.
User-agent: Meta-WebIndexer
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wishlist/
Disallow: /compare/
Disallow: /wc-api/
Disallow: /*?add-to-cart=*
Disallow: /*?wc-ajax=*
Disallow: /*?orderby=*
Disallow: /*?filter_*
Disallow: /*?rating=*
Disallow: /*?pa_*
Disallow: /*?attribute_pa_*
Disallow: /*?replytocom=*
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
Allow: /

# ---- OpenAI ----
# GPTBot — Training + feeds ChatGPT search. ALLOW for visibility.
User-agent: GPTBot
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /*?filter_*
Disallow: /*?orderby=*
Disallow: /*?rating=*
Disallow: /*?pa_*
Disallow: /*?attribute_pa_*
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /

# OAI-SearchBot — ChatGPT real-time search. ALWAYS allow.
User-agent: OAI-SearchBot
Allow: /

# ChatGPT-User — User-initiated browsing from ChatGPT. ALWAYS allow.
User-agent: ChatGPT-User
Allow: /

# ---- Anthropic ----
# ClaudeBot — Training + citation pool. ALLOW for visibility.
User-agent: ClaudeBot
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /*?filter_*
Disallow: /*?orderby=*
Disallow: /*?rating=*
Disallow: /*?pa_*
Disallow: /*?attribute_pa_*
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /

# anthropic-ai — Anthropic search crawler. ALWAYS allow.
User-agent: anthropic-ai
Allow: /

# ---- Perplexity ----
# PerplexityBot — AI search engine, growing shopping use. ALLOW.
User-agent: PerplexityBot
Allow: /

# Perplexity-User — User-initiated fetch. ALWAYS allow.
User-agent: Perplexity-User
Allow: /

# ---- Block: Pure scrapers with zero visibility value ----
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: MJ12bot
Disallow: /