forked from UsrSpace-Internal/website
281 lines
2.6 KiB
Plaintext
281 lines
2.6 KiB
Plaintext
|
Sitemap: sitemap.xml
|
||
|
|
||
|
User-Agent: *
|
||
|
Crawl-Delay: 600
|
||
|
|
||
|
User-agent: Screaming
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: xovi
|
||
|
Disallow: /
|
||
|
User-agent: XoviBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: Smager
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: SEOkicks
|
||
|
Disallow: /
|
||
|
User-agent: SEOkicks-Robot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: sistrix
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: MajesticSEO
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: BacklinkCrawler
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: MJ12bot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: spbot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: SearchmetricsBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: search17
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: AhrefsBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: ia_archiver
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: Seobility
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: TurnitinBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: SlySearch
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: findlinks
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: magpie-crawler
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: Pixray-Seeker
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: 008
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: Ezooms
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: lb-spider
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: WBSearchBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: psbot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: HuaweiSymantecSpider
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: EC2LinkFinder
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: htdig
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: SemrushBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: discobot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: linkdex.com
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: SeznamBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: EdisterBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: SWEBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: picmole
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: Yeti
|
||
|
Disallow: /
|
||
|
User-agent: Yeti-Mobile
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: PagePeeker
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: CatchBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: yacybot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: netEstateNECrawler
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: SurveyBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: gonzo
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: schrein
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: AfiliasWebMiningTool
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: suggybot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: bdbrandprotect
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: BPImageWalker
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: Updownerbot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: lex
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: ContentCrawler
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: DCPbot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: KaloogaBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: MLBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: iCjobs
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: oBot
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: WebmasterCoffee
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: Qualidator
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: Webinator
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: Scooter
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: thunderstone
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: larbin
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: OpidooBOT
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: ips-agent
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: TinEye
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: UnisterBot
|
||
|
Disallow: /
|
||
|
User-agent: Unister
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: ReverseGet
|
||
|
Disallow: /
|
||
|
|
||
|
|
||
|
User-agent: DotBot
|
||
|
Disallow: /
|
||
|
|
||
|
User-agent: seoscanners.net/1
|
||
|
Disallow: /
|
||
|
|
||
|
User-agent: spider@seoscanners.net
|
||
|
Disallow: /
|