### BEGIN FILE - PopSci (July 2011)###
#
# The use of robots or other automated means to access any Bonnier Corporation website
# without the express permission of Bonnier Corporation is strictly prohibited.
# Bonnier Corporation may permit automated access to
# access certain pages but solely for the limited purpose of
# including content in publicly available search engines. Any other
# use of robots or failure to obey the robots exclusion standards set
# forth at http://www.robotstxt.org/wc/exclusion.html is strictly
# prohibited.
#

User-agent: *
#XML Sitemap
Sitemap: http://www.popsci.com/sitemap.xml
Crawl-delay: 10

# Directories
Disallow: /database/
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /sites/
Disallow: /themes/
Disallow: /scripts/
Disallow: /updates/
Disallow: /profiles/
Disallow: /*feed
Disallow: /rss-feeds
Disallow: /customerror/404
Disallow: /terms-of-use
Disallow: /abuse
Disallow: /privacy-policy
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /contact-us
Disallow: /logout/
Disallow: /node/add/
Disallow: /search-results
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /send/
Disallow: /users/
Disallow: /register
Disallow: /uploads/
Disallow: /user-comments/
Disallow: /*most_commented
Disallow: /*top_rated
Disallow: /*most_viewed



# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php


# Hidden tags
Disallow: /category/hidden-tags/lus
Disallow: /category/hidden-tags/lau
Disallow: /category/hidden-tags/dmdmbl
Disallow: /category/hidden-tags/dmd

##Disallow the following spiders##

User-agent: Gigabot
Disallow: /
User-agent: Scrubby
Disallow: /
User-agent: Nutch
Disallow: /
User-agent: baiduspider
Disallow: /
User-agent: naverbot
Disallow: /
User-agent: yeti
Disallow: /
User-agent: asterias
Disallow: /