# https://www.robotstxt.org/robotstxt.html
User-agent: *
Allow: /
# /dict/<lang>/<word> is the in-DB lookup cache surface. Pages are
# served with `X-Robots-Tag: noindex, nofollow` (CloudFront) and the
# static fallback HTML carries `<meta name=robots content=noindex,
# nofollow>`, but disallowing crawl entirely avoids spending crawl
# budget on the unbounded URL space.
Disallow: /dict/

# /conjugate/<lang>/<infinitive> is the dynamic conjugation surface. It
# applies `noindex,nofollow` only at runtime (ConjugatePage), while the
# server fallback HTML is a soft-200 SPA shell. The indexable SEO variant
# lives at the static /verb/<lang>/<slug> pages, so disallow crawling the
# unbounded /conjugate URL space entirely (the indexable /word pages link
# into it with rel="nofollow", but that's a follow hint, not a noindex).
Disallow: /conjugate/

# /my-words is the saved-words app route. The static /word/<lang>/<slug>
# pages link into it as a CTA (/my-words?lookup=<word>&lang=&lemma=1), and
# that React route auto-fires a SmartLookup from the `lookup` param on load.
# A JS-rendering crawler (e.g. meta-externalagent) that follows the CTA boots
# the SPA, triggers a redundant lookup, and mints a throwaway anonymous user
# — inflating analytics for zero SEO value. It's an app surface, not content,
# so disallow crawling it entirely. (The CTA also carries rel="nofollow".)
Disallow: /my-words

# Sitemap
Sitemap: https://wordholder.com/sitemap.xml