Sitemap: https://omophub.com/sitemap.xml

# Machine-readable product summary for agents: https://omophub.com/llms.txt

# CRAWL POLICY, revised 2026-08-11. Keep the groups below in sync with
# BULK_EXTRACTOR in src/middleware.ts, which enforces the same split at 403
# (robots.txt is advisory and only re-read periodically, so a deploy alone does
# not stop an in-flight crawl).
#
# /concepts/{concept_id} covers ~6M concepts and every page links out to its
# parents, children, synonyms and mappings, so the route is an effectively
# unbounded crawl space. It used to be on-demand ISR, which made each
# never-before-seen id a durable, billable cache write; it is now rendered
# dynamically and writes nothing, so bulk crawling it is no longer expensive.
#
# That is why the AI and answer-engine crawlers below are ALLOWED on /concepts
# again after being disallowed 2026-07-23. Being answerable when a developer
# asks an assistant about OMOP vocabularies is distribution for a developer API.
# The thin end of the long tail is kept out of search indexes by per-page
# noindex, not by hiding it from crawlers.
#
# A crawler that matches a named group ignores the `*` group entirely, so the
# disallows have to be repeated in every group.

User-agent: *
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/

# AI and answer-engine crawlers: full access, rate-limited.
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: cohere-ai
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: meta-externalagent
User-agent: meta-webindexer
User-agent: FacebookBot
Crawl-delay: 10
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/

# Bulk extractors with no return path: SEO backlink indexes, contact scrapers,
# content-similarity services, and search engines whose users are never going to
# be looking for an OMOP vocabulary API. Kept off /concepts, which is the only
# part of the site that is expensive to walk at volume.
#
# Bytespider is here on conduct rather than category: it is widely documented as
# ignoring robots.txt, so middleware is the layer that actually stops it.
User-agent: Bytespider
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: PetalBot
User-agent: SeekportBot
User-agent: YisouSpider
User-agent: ZoominfoBot
User-agent: Barkrowler
User-agent: serpstatbot
User-agent: awariobot
User-agent: Diffbot
User-agent: omgili
User-agent: ImagesiftBot
User-agent: Timpibot
User-agent: Turnitin
User-agent: Screaming Frog SEO Spider
Crawl-delay: 10
Disallow: /concepts
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/

# User-initiated fetchers: one request per human question, never bulk.
User-agent: Claude-User
User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/

# Crawl-delay for non-critical bots
User-agent: Googlebot
Crawl-delay: 1
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/

User-agent: Bingbot
Crawl-delay: 2
Allow: /
Disallow: /api/
Disallow: /_next/
Disallow: /admin/
Disallow: /private/