User-agent: gsa-crawler-KNOV Disallow: User-agent: gsa-crawler Disallow: User-agent: * Disallow: /uploads/ Disallow: /cms/ User-agent: panscient.com Disallow: / User-agent: vscooter Disallow: / User-agent: psbot Disallow: / User-agent: ia_archiver Disallow: / User-agent: MJ12bot Disallow: / User-agent: twiceler Disallow: / User-agent: Yandex Disallow: / User-agent: taptubot Disallow: / User-agent: TwengaBot Disallow: / User-agent: sitebot Disallow: / User-agent: Baiduspider Disallow: / User-agent: AhrefsBot Disallow: / User-agent: Ezooms Disallow: / User-agent: sistrix Disallow: / User-agent: aiHitBot Disallow: / User-agent: InfoPath Disallow: / User-agent: InfoPath.2 Disallow: / User-agent: swebot Disallow: / User-agent: EC2LinkFinder Disallow: / User-agent: TurnitinBot Disallow: / User-agent: SearchmetericsBot Disallow: / User-agent: WBSearchBot Disallow: / User-agent: Exabot Disallow: / User-agent: Sosospider Disallow: / User-agent: ip-web-crawler.com Disallow: / User-agent: netEstate NE Crawler Disallow: / User-agent: Aboundexbot Disallow: / User-agent: Aboundex Disallow: / User-agent: meanpathbot Disallow: / User-agent: Mail.Ru Disallow: / User-agent: spbot Disallow: / User-agent: archive.org_bot Disallow: / User-agent: LinkpadBot Disallow: / User-agent: EasouSpider Disallow: / User-agent: SeznamBot Disallow: / User-agent: wotbox Disallow: / User-agent: BLEXBot Disallow: / User-agent: XoviBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: A6-Indexer Disallow: / User-agent: Riddler Disallow: / User-agent: LoadTimeBot Disallow: / User-agent: oBot Disallow: / User-agent: MojeekBot Disallow: / User-agent: memoryBot Disallow: / User-agent: ltx71 Disallow: / User-agent: AdvBot Disallow: / User-agent: SMTBot Disallow: / User-agent: YisouSpider Disallow: / User-agent: LSSRocketCrawler Disallow: / User-agent: Nutch Disallow: / User-agent: tbot-nutch Disallow: / User-agent: thunderstone Disallow: / User-agent: yacybot Disallow: / User-agent: RankSonicBot Disallow: / User-agent: betaBot Disallow: / User-agent: parsijoo-bot Disallow: / User-agent: ltx71 Disallow: / User-agent: NextGenSearchBot Disallow: / User-agent: gocrawl Disallow: / User-agent: plukkie Disallow: / User-agent: Applebot Disallow: / User-agent: Lipperhey Disallow: / User-agent: SafeDNSBot Disallow: / User-agent: RankActiveLinkBot Disallow: / User-agent: Sogou web spider Disallow: / User-agent: Uptimebot Disallow: / User-agent: Slurp Disallow: /*.jpg Disallow: /*.JPG Disallow: /*.png Disallow: /*.PDF Disallow: /*.pdf Disallow: /*.mp3 Disallow: /*.MOV Disallow: /*.mov Disallow: /*.AVI Disallow: /*.avi User-agent: Googlebot Disallow: /*.jpg$ Disallow: /*.JPG$ Disallow: /*.png$ Disallow: /*.PDF$ Disallow: /*.pdf$ Disallow: /*.mp3$ Disallow: /*.MOV$ Disallow: /*.mov$ Disallow: /*.AVI$ Disallow: /*.avi$ User-agent: msnbot Disallow: /*.jpg$ Disallow: /*.JPG$ Disallow: /*.png$ Disallow: /*.PDF$ Disallow: /*.pdf$ Disallow: /*.mp3$ Disallow: /*.MOV$ Disallow: /*.mov$ Disallow: /*.AVI$ Disallow: /*.avi$ User-agent: Qwantify Crawl-delay: 3600 Disallow: /*.jpg Disallow: /*.JPG Disallow: /*.png Disallow: /*.PDF Disallow: /*.pdf Disallow: /*.mp3 Disallow: /*.MOV Disallow: /*.mov Disallow: /*.AVI Disallow: /*.avi User-agent: * Disallow: /*.jpg Disallow: /*.JPG Disallow: /*.png Disallow: /*.PDF Disallow: /*.pdf Disallow: /*.mp3 Disallow: /*.MOV Disallow: /*.mov Disallow: /*.AVI