# See https://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /

# Global rules
User-agent: *
# Don't crawl pages with hubspot tracking parameter
#Disallow: ?__hst
# Don't crawl our own search results pages
Disallow: */search*
Disallow: /*.aspx*
Disallow: */pdf_productsheet/pdfproduct/*
Sitemap: https://www.crownaudio.com/sitemap

# Yandex tries to pull support page as a jpeg for some reason
User-agent: Yandex
Disallow: /en-US/support.jpg
Disallow: /en/support.jpg

# Majestic12 bot hammers the site with DoS-style traffic
User-agent: MJ12bot
Disallow: /

# Baidu/Chinese search crawlers — Chinese content only, throttled
User-agent: Baiduspider
User-agent: YisouSpider
User-agent: Bytespider
Allow: /zh/
Disallow: /
Crawl-delay: 10

# AI crawlers — allow indexing/training, throttle to match Baidu
User-agent: ClaudeBot
Crawl-delay: 10

User-agent: anthropic-ai
Crawl-delay: 10

User-agent: CCBot
Crawl-delay: 10


User-agent: GPTBot
Crawl-delay: 10

User-agent: ChatGPT-User
Crawl-delay: 10