mirror of
https://github.com/naturalcrit/homebrewery.git
synced 2026-11-05 00:48:44 +00:00
47 lines
727 B
Plaintext
47 lines
727 B
Plaintext
# Notes
|
|
User-agent: *
|
|
Disallow: /edit/
|
|
|
|
# AI / LLM training crawlers
|
|
|
|
User-agent: GPTBot
|
|
Disallow: /share/
|
|
Disallow: /download/
|
|
Disallow: /print/
|
|
|
|
User-agent: ClaudeBot
|
|
Disallow: /share/
|
|
Disallow: /download/
|
|
Disallow: /print/
|
|
|
|
User-agent: CCBot
|
|
Disallow: /share/
|
|
Disallow: /download/
|
|
Disallow: /print/
|
|
|
|
User-agent: Google-Extended
|
|
Disallow: /share/
|
|
Disallow: /download/
|
|
Disallow: /print/
|
|
|
|
User-agent: Bytespider
|
|
Disallow: /share/
|
|
Disallow: /download/
|
|
Disallow: /print/
|
|
|
|
User-agent: Applebot-Extended
|
|
Disallow: /share/
|
|
Disallow: /download/
|
|
Disallow: /print/
|
|
|
|
User-agent: Meta-ExternalAgent
|
|
Disallow: /share/
|
|
Disallow: /download/
|
|
Disallow: /print/
|
|
|
|
User-agent: Amazonbot
|
|
Disallow: /share/
|
|
Disallow: /download/
|
|
Disallow: /print/
|
|
|