User-Agent: * Allow: / Disallow: /pdf Sitemap: http://www.autorijschoolmonique.nl/sitemap.xml # Disallow specific bots from indexing or crawling the site at all # Most recent additions first. User-agent: GigaBot Disallow: / User-agent: Voyager Disallow: / User-agent: BaiDuSpider Disallow: / User-agent: BackRub/*.* Disallow: / User-agent: Grub.org Disallow: / User-agent: BotRightHere Disallow: / User-agent: larbin Disallow: / User-agent: psbot Disallow: / User-agent: Walhello appie Disallow: / User-agent: Python-urllib Disallow: / User-agent: Googlebot-Image Disallow: / User-agent: CherryPicker Disallow: / User-agent: EmailCollector Disallow: / User-agent: WebBandit Disallow: / User-agent: EmailWolf Disallow: / User-agent: CopyRightCheck Disallow: / User-agent: Crescent Disallow: / User-agent: yandex bot Disallow: /