# robots.txt for http://readwrite.com/

User-agent: *
Disallow: /test/ 
Disallow: /new/
Disallow: /42/
Disallow: /cgi-bin/
# CMS
Disallow: /modules/
Disallow: /misc/
Disallow: /themes/
Disallow: /sites/
Disallow: /cron.php
Disallow: /_orion/
Disallow: /admin/
Disallow: /node/
Disallow: /filter/tips/
Disallow: /search
Disallow: /user/
Disallow: /_ajax

User-agent: 008
Disallow: /

User-agent: mxbot
Disallow: /

User-agent: heritrix
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: Sosospider
Disallow: /

User-Agent: Sosoimagespider
Disallow: /

User-Agent: rdfbot
Disallow: /

#User-Agent: Yandex
#Disallow: /

User-Agent: CamontSpider
Disallow: /

User-Agent: twiceler
Disallow: /search/

Sitemap: http://readwrite.com/sitemap_index.xml