# sitemaps
Sitemap: https://www.list.co.uk/sitemap-index.xml.gz


# all
User-agent: *
Disallow:
Disallow: /null/
Disallow: /details/
Disallow: /events/*.xml$
Disallow: /places/*.xml$
# Disallow: /events/*.ics$
Disallow: /*/show:*/
Disallow: /*/sort:*/
Disallow: /articles/*/what:*/
Disallow: /articles/what:*/page:*/
Disallow: /events/*/what:*/
Disallow: /events/what:*/page:*/
Disallow: /listings/*/what:*/
Disallow: /listings/what:*/page:*/
Disallow: /login/
Disallow: /member/
# Disallow: /places/*/what:*/
Disallow: /places/what:*/page:*/
# Disallow: /*/where:*/
# Disallow: /r/
# Disallow: /redirect/
Disallow: /update/*/*/
Disallow: /*/TBC/
Disallow: /*/tbc/
Disallow: /*/trackback/
Disallow: /js/
# Disallow: /*/*/*/*/*/*/*/
# ^ 2017-05-22
Disallow: /*/*/*/*/*/*/
# ^ 2018-04-23


# Bingbot
User-Agent: bingbot
Disallow:
Disallow: /null/
Disallow: /details/
Disallow: /events/*.xml$
Disallow: /places/*.xml$
# Disallow: /events/*.ics$
Disallow: /*/what:*/
Disallow: /*/show:*/
Disallow: /*/sort:*/
Disallow: /*/page:*/
Disallow: /login/
Disallow: /member/
Disallow: /update/*/*/
Disallow: /*/TBC/
Disallow: /*/tbc/
Disallow: /*/trackback/
Disallow: /js/
Disallow: /*/*/*/*/*/
# ^ 2017-07-04


# limited
User-agent: Baiduspider
User-agent: CCBot
User-agent: berlin-fu-cow
User-agent: Flamingo_SearchEngine
User-agent: LCC
User-agent: NewsNow
User-agent: Nutch
User-agent: Scrapy
User-agent: SEOkicks-Robot
User-agent: Spinn3r
User-agent: Yandex
User-agent: YioopBot
User-agent: YisouSpider
Disallow: /details/
Disallow: /events/
Disallow: /listings/
Disallow: /login/
Disallow: /event/
Disallow: /places/
Disallow: /articles/
Disallow: */comments.xml$
Disallow: *.ics$
Disallow: /*/*/*/*/
Crawl-delay: 10


# banned (known to visit)
User-agent: 008
Disallow: /

User-agent: Abonti
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: ConnexunBot
Disallow: /

User-agent: CrystalSemanticsBot
Disallow: /

User-agent: DomainAppender
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: Ezooms
Disallow: /

User-agent: Genieo
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: kalooga
Disallow: /

User-agent: Kraken
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: Mail.Ru
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: netseer
Disallow: /

User-agent: Node/simplecrawler
Disallow: /

User-Agent: panscient.com
Disallow: /

User-agent: Pcore-HTTP
Disallow: /

User-agent: proximic
Disallow: /

User-agent: psbot
Disallow: /

User-agent: R6_CommentReader
Disallow: /

User-agent: SearchmetricsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: SindiceBot
Disallow: /

User-agent: Slurp
Disallow: /

User-agent: sogou spider
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: spbot
Disallow: /

User-agent: trendiction
Disallow: /

User-agent: Vocus
Disallow: /

User-agent: WeSEE
Disallow: /



# banned (just in case)
User-agent: aiHitBot
Disallow: /

User-agent: EC2LinkFinder
Disallow: /

User-agent: InfoPath
Disallow: /

User-agent: InfoPath.2
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: panscient.com
Disallow: /

User-agent: sistrix
Disallow: /

User-agent: sitebot
Disallow: /

User-agent: swebot
Disallow: /

User-agent: taptubot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: TwengaBot
Disallow: /

User-agent: twiceler
Disallow: /

User-agent: vscooter
Disallow: /

User-agent: WBSearchBot
Disallow: /