# Super Yacht Interiors — crawler policy.
#
# Every bot is welcome on the public site; only the API is withheld. NOTE for
# anyone editing this file: a crawler obeys exactly ONE group — the most
# specific one matching its token — and ignores every other, including "*". So
# each named group below must repeat the Disallow lines. An earlier version
# listed the AI crawlers with "Allow: /" alone, which quietly opted them into
# routes meant to be withheld. (The /motion-lab and /home-clone rules went with
# those routes: a Disallow for a page that no longer exists only advertises it.)

User-agent: *
Allow: /
Disallow: /api/

# ---------------------------------------------------------------------------
# AI / answer engines — explicitly allowed, so the brand can be cited in AI
# answers. Named individually rather than relying on "*" because several of
# these only honour their own token, and because a few (Google-Extended,
# Applebot-Extended) exist ONLY as an opt-out switch: absent a rule they are
# governed elsewhere, so stating Allow here is the deliberate opt-in.
# ---------------------------------------------------------------------------

# OpenAI — training, live retrieval for ChatGPT, and user-initiated fetches.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Allow: /
Disallow: /api/

# Anthropic — Claude.
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
Allow: /
Disallow: /api/

# Google — Gemini and AI Overviews grounding. (Googlebot itself is covered by
# "*"; Google-Extended governs only AI use and carries no crawling of its own.)
User-agent: Google-Extended
User-agent: GoogleOther
Allow: /
Disallow: /api/

# Microsoft / Bing — Copilot.
User-agent: Bingbot
User-agent: BingPreview
User-agent: msnbot
Allow: /
Disallow: /api/

# Apple — Siri and Spotlight summaries.
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /
Disallow: /api/

# Perplexity.
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /api/

# Other answer engines, assistants and research crawlers.
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: PhindBot
User-agent: Diffbot
User-agent: AI2Bot
User-agent: Kangaroo Bot
User-agent: Timpibot
User-agent: Webzio-Extended
User-agent: omgili
User-agent: omgilibot
User-agent: ImagesiftBot
User-agent: Bytespider
User-agent: PetalBot
User-agent: CCBot
Allow: /
Disallow: /api/

# Plain-language site summary for LLMs (llmstxt.org convention).
# Answer engines that support it read this before crawling HTML.
# https://superyachtinteriors.com/llms.txt

Sitemap: https://superyachtinteriors.com/sitemap-index.xml