# robots.txt

# Currently disallow all shop stuff to the Google Image bot
# Mainly image hunters anyway, they eat up bandwidth...
User-agent: Googlebot-Image
Disallow: /cgi-bin/
Disallow: /usage/
Disallow: /ec2/shop/
Disallow: /public_html/cgi-bin
Disallow: /public_html/ap/
Disallow: /public_html/demo/
Disallow: /public_html/images/
Disallow: /images/
Disallow: /ec3/
Disallow: /ec2/ecommerce/

# ALL search engine spiders/crawlers (put at end of file)
User-agent: *
Disallow: /cgi-bin/
Disallow: /public_html/cgi-bin
Disallow: /usage/
Disallow: /ec2/common/
Disallow: /tmp/
Disallow: /access-logs/
Disallow: /etc/
Disallow: /ap/
Disallow: /program_data/
Disallow: /program_data2/


# spiders.txt,v 1.2 2003/05/05 17:58:17 dgw_ Exp $
almaden.ibm.com
appie 1.1
architext
ask jeeves
ask
asterias2.0
augurfind
baiduspider
bannana_bot
bdcindexer
crawler
crawler@fast
docomo
fast-webcrawler
fluffy the spider
frooglebot
geobot
googlebot
gulliver
henrythemiragorobot
ia_archiver
infoseek
kit_fireball
lachesis
lycos_spider
mantraagent
mercator
moget/1.0
muscatferret
nationaldirectory-webspider
naverrobot
ncsa beta
netresearchserver
ng/1.0
osis-project
polybot
pompos
scooter
seventwentyfour
sidewinder
sleek spider
slurp/si
slurp@inktomi.com
steeler/1.3
szukacz
t-h-u-n-d-e-r-s-t-o-n-e
teoma
turnitinbot
ultraseek
vagabondo
voilabot
w3c_validator
zao/0
zyborg/1.0