# TPEE.app — crawlers welcome on the marketing pages.
# /success and /download carry <meta name="robots" content="noindex">, so they
# must stay crawlable (blocking them here would HIDE the noindex directive).
User-agent: *
Allow: /
# Build 547 (session 0893): three generator scratch files were published at the web
# root and crawlable - a 541-line dump of internal paper titles, an array of every
# URL on the site, and a per-URL status map. They answer no query a buyer types and
# one of them leaks the site's own inventory. Disallowed here AND removed from the
# deploy where possible; /llms.txt stays allowed on purpose, it is the AI-search
# surface. /download and /success must stay crawlable so their noindex is visible.
# Build 548 (0894): the three files above are NO LONGER PUBLISHED at all - a live probe
# found every one of them answering 200 with its contents intact, because Disallow stops
# crawling and not access. They moved to UI_Review/_b548_leak/. The three lines stay as a
# belt-and-braces guard in case a future generator writes them here again.
Disallow: /titles31c.txt
Disallow: /sitemap_urls.json
Disallow: /sm_status.json

Sitemap: https://tpee.app/sitemap.xml

# ---------------------------------------------------------------------------
# Build 557 (session 0902): AI CRAWLERS - named, so the permission is a statement and
# not an inference. Until this build the policy was one wildcard group: permission by
# default, and silence about intent, so a provider deciding whether its crawler may fetch
# a page had to infer the answer from a rule that never mentioned it.
#
# Three jobs, and this site permits all three: TRAINING (reading published pages to
# improve a model), ANSWERING (fetching a page at the moment a person asks) and SEARCH
# (an AI feature answering from its own index). The product is a prompt-engineering editor
# sold as a one-time offline purchase; being quoted accurately by an assistant is how its
# buyer finds it. There is nothing here worth withholding from a model, and a great deal
# worth being quoted correctly.
#
# The corpus for machines: https://tpee.app/llms.txt (short, with links),
# https://tpee.app/llms-full.txt (the product as plain text) and
# https://tpee.app/llms-course.txt (all 134 lesson pages as plain text).
# Terms for use and citation: https://tpee.app/ai.txt
#
# A group matches a crawler whose product token STARTS WITH the name, so "GPTBot" also
# covers GPTBot-Images. Order does not matter; the most specific matching group wins.

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: BingPreview
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: CCBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: MistralAI-User
User-agent: DuckAssistBot
User-agent: DuckDuckBot
User-agent: YouBot
User-agent: Ai2Bot
User-agent: ai2bot-dolma
User-agent: Bytespider
User-agent: Diffbot
User-agent: Timpibot
User-agent: ImagesiftBot
User-agent: PanguBot
User-agent: omgilibot
# Build 560: four crawlers the wildcard group already permitted but never NAMED.
# ExaSearchBot and yandexbot reach an answer engine; SemrushBot and AhrefsBot are the
# SEO tools customers and competitors use to look for us. A named group is a statement
# of intent, and the wildcard 'Allow: /' above never made one.
User-agent: ExaSearchBot
User-agent: yandexbot
User-agent: SemrushBot
User-agent: AhrefsBot
Allow: /
Disallow: /titles31c.txt
Disallow: /sitemap_urls.json
Disallow: /sm_status.json