User-agent: *
Allow: /


User-agent: *
Disallow: /cache/

User-agent: Slurp 
Disallow: / 

User-agent: dotbot 
Disallow: /

User-agent: yahooSeeker
Disallow: / 

User-agent: MMCrawler
Disallow: / 

User-agent: MJ12bot
Disallow: /


User-agent: Twiceler
Disallow: /

User-agent: twiceler
Disallow: /

User-agent: cuil
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: googlebot 
Crawl-delay: 10