User-agent: *
Disallow: /index_files/
Sitemap: http://reference.md/sitemap_index.xml
Crawl-Delay: 4

User-agent: Mediapartners-Google
Disallow: 
Allow: /lwu.exe/lwu/toEng

User-Agent: ASPSeek
Disallow: /

User-Agent: asterias
Disallow: /

User-agent: baiduspider
Disallow: /

User-agent: BravoBrian
Disallow: /

User-agent: dloader(NaverRobot)
Disallow: /

User-agent: Dumbot
Disallow: /

User-agent: EgotoBot
Disallow: /

User-agent: e-SocietyRobot
Disallow: /

User-agent: Gaisbot
Disallow: /

User-agent: Generic
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: almaden.ibm.com/cs/crawler
Disallow: /

User-agent: InfoNaviRobot
Disallow: /

User-agent: Jyxobot
Disallow: /

User-agent: Larbin
Disallow: /

User-agent: msiecrawler
Disallow: /

User-agent: NaverRobot
Disallow: /

User-agent: NexaBot
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: obot
Disallow: /

User-agent: OWR_Crawler
Disallow: /

User-agent: PhpDig
Disallow: /

User-agent: psbot
Disallow: /

User-agent: puf
Disallow: /

User-agent: QuepasaCreep
Disallow: /

User-agent: rabaz
Disallow: /

User-agent: Reaper
Disallow: /

User-agent: RPT-HTTPClient
Disallow: /

User-agent: ScoutAbout
Disallow: /

User-agent: semanticdiscovery
Disallow: /

User-agent: Steeler
Disallow: /

User-agent: Teleport Pro
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: TutorGig
Disallow: /

User-agent: vsecrawler
Disallow: /

User-agent: Wget
Disallow: /

User-agent: Xenu's
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: Slurp
Crawl-delay: 8