diff --git a/robots.txt b/robots.txt index 9abbfb965..c7fd30d1c 100644 --- a/robots.txt +++ b/robots.txt @@ -2,3 +2,45 @@ User-agent: * Disallow: /edit/ +# AI / LLM training crawlers + +User-agent: GPTBot +Disallow: /share/ +Disallow: /download/ +Disallow: /print/ + +User-agent: ClaudeBot +Disallow: /share/ +Disallow: /download/ +Disallow: /print/ + +User-agent: CCBot +Disallow: /share/ +Disallow: /download/ +Disallow: /print/ + +User-agent: Google-Extended +Disallow: /share/ +Disallow: /download/ +Disallow: /print/ + +User-agent: Bytespider +Disallow: /share/ +Disallow: /download/ +Disallow: /print/ + +User-agent: Applebot-Extended +Disallow: /share/ +Disallow: /download/ +Disallow: /print/ + +User-agent: Meta-ExternalAgent +Disallow: /share/ +Disallow: /download/ +Disallow: /print/ + +User-agent: Amazonbot +Disallow: /share/ +Disallow: /download/ +Disallow: /print/ +