# https://blogs.bing.com/webmaster/2009/08/10/crawl-delay-and-the-bing-crawler-msnbot
# Slow down bingbot
User-agent: msnbot
Crawl-delay: 10

User-agent: *
Allow: /robots.txt


# too much traffic from these in September 2017
User-agent: MJ12bot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
# too much traffic from this in March 2018
User-agent: BUbiNG
Disallow: /

User-agent: *
Allow: /