# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/orig.html
Sitemap: https://www.redsharknews.com/sitemap.xml

User-agent: *
Disallow: /cache/
Disallow: /component/redshark/ad/
Disallow: /component/search/
Disallow: /components/
Disallow: /media/emails/
Disallow: /logs/
Disallow: /modules/
Disallow: /plugins/
Disallow: /tags/
Disallow: /tmp/
Crawl-delay: 60

User-agent: Googlebot
User-agent: Googlebot-Mobile
User-agent: Googlebot-News
Disallow: /cache/
Disallow: /component/redshark/ad/
Disallow: /component/search/
Disallow: /components/
Disallow: /media/emails/
Disallow: /logs/
Disallow: /modules/
Disallow: /plugins/
Allow: /tags/
Disallow: /tmp/

User-agent: bingbot
Disallow: /cache/
Disallow: /component/redshark/ad/
Disallow: /component/search/
Disallow: /components/
Disallow: /media/emails/
Disallow: /logs/
Disallow: /modules/
Disallow: /plugins/
Disallow: /tags/
Disallow: /tmp/
Crawl-delay: 60

User-agent: CrazyWebCrawler-Spider
User-agent: Cliqzbot
User-agent: BUbiNG
User-agent: ia_archiver
User-agent: SemrushBot
Disallow: /