# As a condition of accessing this website, you agree to abide by the following
# content signals:
#
# (a)  If a Content-Signal = yes, you may collect content for the corresponding use.
# (b)  If a Content-Signal = no, you may not collect content for the corresponding use.
# (c)  If the website operator does not include a Content-Signal for a corresponding
#      use, the website operator neither grants nor restricts permission via
#      Content-Signal with respect to the corresponding use.
#
# search:   building a search index and providing search results.
# ai-input: inputting content into AI models for real-time answers (RAG / grounding /
#           generative search answers).
# ai-train: training or fine-tuning AI models.
# use:      how AI systems may consume the content (immediate, reference, or full).
#
# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF RIGHTS
# UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT AND RELATED
# RIGHTS IN THE DIGITAL SINGLE MARKET.

# ============================================================================
# GOAL: Maximise AI visibility (be cited in AI answers) WITHOUT allowing training.
#   - search   = yes  -> appear in search indexes
#   - ai-input = yes  -> allowed to be used in real-time AI answers (visibility)
#   - ai-train = no   -> NOT allowed for model training
#   - use      = reference
# ============================================================================

# --- Default policy for all bots -------------------------------------------
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no, use=reference
Allow: /

# ============================================================================
# ALLOW: AI SEARCH / ANSWER (RETRIEVAL) BOTS  -> these drive AI visibility
# These fetch pages to cite/surface you in AI answers, NOT to train models.
# ============================================================================

# OpenAI / ChatGPT
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic / Claude
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Apple (Siri / Spotlight search surfacing)
User-agent: Applebot
Allow: /

# ============================================================================
# BLOCK: AI TRAINING / DATASET CRAWLERS  -> no model training on our content
# ============================================================================

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

# ============================================================================
# STANDARD SITE RULES (apply to all crawlers via User-agent: *)
# ============================================================================

# Avoid crawling parameterised / faceted URLs
Disallow: *?*=*
Crawl-delay: 10

# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg

# Directories
Disallow: /core/
Disallow: /profiles/

# Files
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt

# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed

# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed