User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php


Sitemap: https://alcester-tc.gov.uk/sitemap.xml

# Deny spbot
User-agent: spbot
Disallow: /

# Deny Soso spider (http://help.soso.com/webspider.htm)
User-agent: Sosospider
Disallow: /

# Deny EC2LinkFinder
User-agent: EC2LinkFinder
Disallow: /

# Deny 80legs.com webcrawler
User-agent: 008
Disallow: /

# Deny MLBot (http://www.metadatalabs.com/mlbot)
User-agent: MLBot
Disallow: /

# Deny Yandex
User-agent: Yandex
Disallow: /
Crawl-delay: 3600

# Deny Sogou spider (http://www.sogou.com/docs/help/webmasters.htm#07)
User-agent: Sogou
Disallow: /

# Deny YoudaoBot (http://www.youdao.com/help/webmaster/spider/)
User-agent: YoudaoBot
Disallow: /

# Deny Yeti (http://help.naver.com/robots/)
User-agent: Yeti
Disallow: /

# Deny Goo Click's Ichiro (http://help.goo.ne.jp/door/crawler.html)
User-agent: ichiro
Disallow: /

# Deny Spinn3r RSS aggregator (http://spinn3r.com/robot)
User-agent: Spinn3r
Disallow: /

# Deny Proximic (https://www.comscore.com/proximic-spider)
User-agent: proximic
Disallow: /

# Deny Baiduspider
User-agent: Baiduspider
Disallow: /

# Deny MJ12Bot
User-Agent: MJ12bot
Disallow: /

# Deny Faraday
User-Agent: Faraday
Disallow: /

# Deny SEMrushBot from crawling your site for web graph of links, add:
User-agent: SemrushBot
Disallow: /

# Deny SEMrushBot from crawling your site for different SEO and technical issues, add:
User-agent: SemrushBot-SA
Disallow: /

# Deny Serpstat Bot
User-agent: serpstatbot
Disallow: /

# General rules
User-agent: *
Disallow: *.bak
Disallow: *.config
Disallow: *.css
Disallow: *.htaccess
Disallow: *.js