User-Agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /node_modules/
Disallow: /wp-admin/
Disallow: /cgi-bin
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/


User-Agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /node_modules/
Disallow: /wp-admin/
Disallow: /cgi-bin
Disallow: /feed
Disallow: */feed/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/

User-Agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /node_modules/
Disallow: /wp-admin/
Disallow: /cgi-bin
Disallow: /feed
Disallow: */feed/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/

User-Agent: Slurp
Allow: /

User-Agent: DuckDuckBot
Allow: /

User-Agent: facebookexternalhit
Allow: /

User-Agent: Twitterbot
Allow: /

User-Agent: LinkedInBot
Allow: /

User-Agent: WhatsApp
Allow: /

User-Agent: Applebot
Allow: /

# === AI / LLM Crawlers ===

# OpenAI
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
Allow: /

# Google AI
User-agent: Google-Extended
User-agent: Gemini-Deep-Research
User-agent: Google-NotebookLM
User-agent: Google-Agent
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Content signals.
#   search=yes     - index us and return links/excerpts. Core to the business.
#   ai-input=yes   - allow retrieval/grounding for generative answers. This is
#                    the signal that governs whether we can be CITED in AI
#                    answers, which is the whole strategy. Must stay "yes".
#   ai-train=no    - we do not grant permission to train or fine-tune models on
#                    this content. Express reservation of rights under Article 4
#                    of EU Directive 2019/790 (DSM Copyright Directive).
#   use=reference  - index, excerpt and LINK BACK. Deliberately not "full",
#                    which would license reproducing our content without a link
#                    (the zero-click outcome we are trying to avoid).
#
# These are declarations of preference, not access controls. They are not yet
# honoured by every crawler or validator, and no AI operator has committed to
# respecting them. Enforcement, if ever needed, has to happen at the edge
# (Cloudflare WAF / bot rules), not here.
Content-signal: search=yes, ai-train=yes, use=full


Sitemap: https://www.secure.com/sitemap_index.xml