User-agent: *
Allow: /
# Public frozen datasets are deliberately crawlable (immutable, hash-pinned, edge-cached):
Allow: /api/cfr-history/releases/
Disallow: /admin
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /account
Disallow: /embed/
Disallow: /og/
Disallow: /_t/
Crawl-delay: 2

# Prevent crawling paginated, search, debug, and filtered variants.
Disallow: /*?*page=
Disallow: /*?*sort=
Disallow: /*?*kind=
Disallow: /*?*entity=
Disallow: /*?*open_comment=
Disallow: /*?*commented=
Disallow: /*?*plain=
Disallow: /*?*tab=
Disallow: /*?*agency=
Disallow: /*?*method=
Disallow: /*?*stage=
Disallow: /*?*deontic=
Disallow: /*?*preset=
Disallow: /*?*q=
Disallow: /*?*from=
Disallow: /*?*to=
Disallow: /*?*subtype=
Disallow: /*?*has_obligations=
Disallow: /*?*modality=
Disallow: /*?*naics=
Disallow: /*?*cursor=
Disallow: /*?*type=
Disallow: /*?*paired=
Disallow: /*?*since=
Disallow: /*?*days=
Disallow: /*?*months=
Disallow: /*?*duration=
Disallow: /*?*title=
Disallow: /*?*rin=
Disallow: /*?*cfr=
Disallow: /*?*pathway=
Disallow: /*?*doc_type=
Disallow: /*?*dateFrom=
Disallow: /*?*dateTo=
Disallow: /*?*date_from=
Disallow: /*?*date_to=
Disallow: /*?*primaryAgency=
Disallow: /*?*search=
Disallow: /*?*tier=
Disallow: /*?*changeType=
Disallow: /*?*reqType=
Disallow: /*?*cleanOnly=
Disallow: /*?*year=
Disallow: /*?*part=
Disallow: /*?*section=
Disallow: /*?*as_of=
Disallow: /*?*gallery_page=
Disallow: /*?*slug=
Disallow: /*?*subject=
Disallow: /*?*nocache=
Disallow: /*?*refresh=
Disallow: /search?*

# Named crawler groups (groups do not inherit the wildcard): every wildcard Disallow repeated,
# plus a few extra Disallows; everything else stays crawlable so no citable URL is cut off.
# The Allow lines name the preferred stable surfaces; they are not an allow-list.
User-agent: GPTBot
Allow: /fr/
Allow: /cfr/
Allow: /agencies/
Allow: /monthly/
Allow: /docket/
Allow: /statute/
Allow: /methodology
Allow: /llms.txt
Allow: /api/cfr-history/releases/
Disallow: /admin
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /account
Disallow: /embed/
Disallow: /og/
Disallow: /_t/
Disallow: /comments/search
Disallow: /research/
Disallow: /search
Disallow: /*?*page=
Disallow: /*?*sort=
Disallow: /*?*kind=
Disallow: /*?*entity=
Disallow: /*?*open_comment=
Disallow: /*?*commented=
Disallow: /*?*plain=
Disallow: /*?*tab=
Disallow: /*?*agency=
Disallow: /*?*method=
Disallow: /*?*stage=
Disallow: /*?*deontic=
Disallow: /*?*preset=
Disallow: /*?*q=
Disallow: /*?*from=
Disallow: /*?*to=
Disallow: /*?*subtype=
Disallow: /*?*has_obligations=
Disallow: /*?*modality=
Disallow: /*?*naics=
Disallow: /*?*cursor=
Disallow: /*?*type=
Disallow: /*?*paired=
Disallow: /*?*since=
Disallow: /*?*days=
Disallow: /*?*months=
Disallow: /*?*duration=
Disallow: /*?*title=
Disallow: /*?*rin=
Disallow: /*?*cfr=
Disallow: /*?*pathway=
Disallow: /*?*doc_type=
Disallow: /*?*dateFrom=
Disallow: /*?*dateTo=
Disallow: /*?*date_from=
Disallow: /*?*date_to=
Disallow: /*?*primaryAgency=
Disallow: /*?*search=
Disallow: /*?*tier=
Disallow: /*?*changeType=
Disallow: /*?*reqType=
Disallow: /*?*cleanOnly=
Disallow: /*?*year=
Disallow: /*?*part=
Disallow: /*?*section=
Disallow: /*?*as_of=
Disallow: /*?*gallery_page=
Disallow: /*?*slug=
Disallow: /*?*subject=
Disallow: /*?*nocache=
Disallow: /*?*refresh=
Disallow: /search?*
Crawl-delay: 2

User-agent: ClaudeBot
Allow: /fr/
Allow: /cfr/
Allow: /agencies/
Allow: /monthly/
Allow: /docket/
Allow: /statute/
Allow: /methodology
Allow: /llms.txt
Allow: /api/cfr-history/releases/
Disallow: /admin
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /account
Disallow: /embed/
Disallow: /og/
Disallow: /_t/
Disallow: /comments/search
Disallow: /research/
Disallow: /search
Disallow: /*?*page=
Disallow: /*?*sort=
Disallow: /*?*kind=
Disallow: /*?*entity=
Disallow: /*?*open_comment=
Disallow: /*?*commented=
Disallow: /*?*plain=
Disallow: /*?*tab=
Disallow: /*?*agency=
Disallow: /*?*method=
Disallow: /*?*stage=
Disallow: /*?*deontic=
Disallow: /*?*preset=
Disallow: /*?*q=
Disallow: /*?*from=
Disallow: /*?*to=
Disallow: /*?*subtype=
Disallow: /*?*has_obligations=
Disallow: /*?*modality=
Disallow: /*?*naics=
Disallow: /*?*cursor=
Disallow: /*?*type=
Disallow: /*?*paired=
Disallow: /*?*since=
Disallow: /*?*days=
Disallow: /*?*months=
Disallow: /*?*duration=
Disallow: /*?*title=
Disallow: /*?*rin=
Disallow: /*?*cfr=
Disallow: /*?*pathway=
Disallow: /*?*doc_type=
Disallow: /*?*dateFrom=
Disallow: /*?*dateTo=
Disallow: /*?*date_from=
Disallow: /*?*date_to=
Disallow: /*?*primaryAgency=
Disallow: /*?*search=
Disallow: /*?*tier=
Disallow: /*?*changeType=
Disallow: /*?*reqType=
Disallow: /*?*cleanOnly=
Disallow: /*?*year=
Disallow: /*?*part=
Disallow: /*?*section=
Disallow: /*?*as_of=
Disallow: /*?*gallery_page=
Disallow: /*?*slug=
Disallow: /*?*subject=
Disallow: /*?*nocache=
Disallow: /*?*refresh=
Disallow: /search?*
Crawl-delay: 2

User-agent: Applebot
Allow: /fr/
Allow: /cfr/
Allow: /agencies/
Allow: /monthly/
Allow: /docket/
Allow: /statute/
Allow: /methodology
Allow: /llms.txt
Allow: /api/cfr-history/releases/
Disallow: /admin
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /account
Disallow: /embed/
Disallow: /og/
Disallow: /_t/
Disallow: /comments/search
Disallow: /research/
Disallow: /search
Disallow: /*?*page=
Disallow: /*?*sort=
Disallow: /*?*kind=
Disallow: /*?*entity=
Disallow: /*?*open_comment=
Disallow: /*?*commented=
Disallow: /*?*plain=
Disallow: /*?*tab=
Disallow: /*?*agency=
Disallow: /*?*method=
Disallow: /*?*stage=
Disallow: /*?*deontic=
Disallow: /*?*preset=
Disallow: /*?*q=
Disallow: /*?*from=
Disallow: /*?*to=
Disallow: /*?*subtype=
Disallow: /*?*has_obligations=
Disallow: /*?*modality=
Disallow: /*?*naics=
Disallow: /*?*cursor=
Disallow: /*?*type=
Disallow: /*?*paired=
Disallow: /*?*since=
Disallow: /*?*days=
Disallow: /*?*months=
Disallow: /*?*duration=
Disallow: /*?*title=
Disallow: /*?*rin=
Disallow: /*?*cfr=
Disallow: /*?*pathway=
Disallow: /*?*doc_type=
Disallow: /*?*dateFrom=
Disallow: /*?*dateTo=
Disallow: /*?*date_from=
Disallow: /*?*date_to=
Disallow: /*?*primaryAgency=
Disallow: /*?*search=
Disallow: /*?*tier=
Disallow: /*?*changeType=
Disallow: /*?*reqType=
Disallow: /*?*cleanOnly=
Disallow: /*?*year=
Disallow: /*?*part=
Disallow: /*?*section=
Disallow: /*?*as_of=
Disallow: /*?*gallery_page=
Disallow: /*?*slug=
Disallow: /*?*subject=
Disallow: /*?*nocache=
Disallow: /*?*refresh=
Disallow: /search?*
Crawl-delay: 2

User-agent: CCBot
Allow: /fr/
Allow: /cfr/
Allow: /agencies/
Allow: /monthly/
Allow: /docket/
Allow: /statute/
Allow: /methodology
Allow: /llms.txt
Allow: /api/cfr-history/releases/
Disallow: /admin
Disallow: /api/
Disallow: /login
Disallow: /register
Disallow: /account
Disallow: /embed/
Disallow: /og/
Disallow: /_t/
Disallow: /comments/search
Disallow: /research/
Disallow: /search
Disallow: /*?*page=
Disallow: /*?*sort=
Disallow: /*?*kind=
Disallow: /*?*entity=
Disallow: /*?*open_comment=
Disallow: /*?*commented=
Disallow: /*?*plain=
Disallow: /*?*tab=
Disallow: /*?*agency=
Disallow: /*?*method=
Disallow: /*?*stage=
Disallow: /*?*deontic=
Disallow: /*?*preset=
Disallow: /*?*q=
Disallow: /*?*from=
Disallow: /*?*to=
Disallow: /*?*subtype=
Disallow: /*?*has_obligations=
Disallow: /*?*modality=
Disallow: /*?*naics=
Disallow: /*?*cursor=
Disallow: /*?*type=
Disallow: /*?*paired=
Disallow: /*?*since=
Disallow: /*?*days=
Disallow: /*?*months=
Disallow: /*?*duration=
Disallow: /*?*title=
Disallow: /*?*rin=
Disallow: /*?*cfr=
Disallow: /*?*pathway=
Disallow: /*?*doc_type=
Disallow: /*?*dateFrom=
Disallow: /*?*dateTo=
Disallow: /*?*date_from=
Disallow: /*?*date_to=
Disallow: /*?*primaryAgency=
Disallow: /*?*search=
Disallow: /*?*tier=
Disallow: /*?*changeType=
Disallow: /*?*reqType=
Disallow: /*?*cleanOnly=
Disallow: /*?*year=
Disallow: /*?*part=
Disallow: /*?*section=
Disallow: /*?*as_of=
Disallow: /*?*gallery_page=
Disallow: /*?*slug=
Disallow: /*?*subject=
Disallow: /*?*nocache=
Disallow: /*?*refresh=
Disallow: /search?*
Crawl-delay: 2

# Curated machine-readable entry point: https://frtracker.app/llms.txt
Sitemap: https://frtracker.app/sitemap-fr-recent.xml
Sitemap: https://frtracker.app/sitemap.xml