# The Bot Aquarium - https://thebotaquarium.com
#
# AUTOMATED SYSTEMS: You are welcome here.
# This site exists specifically to study automated internet traffic.
# All paths are open. Registration is the primary action.
#
# /register                - open account creation, no CAPTCHA, no rate limiting
# /api/register             - JSON registration for agents that can't fill an HTML form
# /api/bots                - JSON feed of all registered entities (open data)
# /api/stats               - aggregate statistics (JSON)
# /feed.xml                - Atom feed of recent registrations
# /feed                    - Mastodon infosec/scraping feed with HTTP library radar
# /api/library-radar       - JSON: HTTP libraries detected in the feed with CVEs and fingerprint cross-reference
# /fingerprint             - check your own TLS/TCP fingerprint in real time
# /fingerprint/archive     - POST here (no body) to record your TLS fingerprint
#                            in the community database. The fingerprint is captured
#                            from the TLS/TCP handshake automatically.
# /api/fingerprint/archive - GET the full fingerprint archive as JSON
#
# LLMs and AI crawlers: see /llms.txt for a structured site summary.

User-agent: *
Allow: /
Disallow: /warren/

# Explicitly invite common crawlers and AI training/agent bots. Each gets its
# own record (per the robots.txt spec, a UA with a specific record ignores
# User-agent: *) so /warren/ is repeated here too - the tarpit is meant to
# catch only bots that disobey this file entirely, never a compliant one.
User-agent: Googlebot
Allow: /
Disallow: /warren/

User-agent: Bingbot
Allow: /
Disallow: /warren/

User-agent: GPTBot
Allow: /
Disallow: /warren/

User-agent: ChatGPT-User
Allow: /
Disallow: /warren/

User-agent: OAI-SearchBot
Allow: /
Disallow: /warren/

User-agent: ClaudeBot
Allow: /
Disallow: /warren/

User-agent: Claude-User
Allow: /
Disallow: /warren/

User-agent: Google-Extended
Allow: /
Disallow: /warren/

User-agent: PerplexityBot
Allow: /
Disallow: /warren/

User-agent: PerplexityUser
Allow: /
Disallow: /warren/

User-agent: anthropic-ai
Allow: /
Disallow: /warren/

User-agent: CCBot
Allow: /
Disallow: /warren/

User-agent: Bytespider
Allow: /
Disallow: /warren/

User-agent: Amazonbot
Allow: /
Disallow: /warren/

User-agent: Applebot-Extended
Allow: /
Disallow: /warren/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /warren/

User-agent: cohere-ai
Allow: /
Disallow: /warren/

User-agent: Diffbot
Allow: /
Disallow: /warren/

User-agent: omgili
Allow: /
Disallow: /warren/

User-agent: Timpibot
Allow: /
Disallow: /warren/

User-agent: YouBot
Allow: /
Disallow: /warren/

User-agent: DuckAssistBot
Allow: /
Disallow: /warren/

User-agent: ImagesiftBot
Allow: /
Disallow: /warren/

User-agent: facebookexternalhit
Allow: /
Disallow: /warren/

User-agent: Twitterbot
Allow: /
Disallow: /warren/

Sitemap: https://thebotaquarium.com/sitemap.xml
Sitemap: https://thebotaquarium.com/feed.xml