# /robots.txt file for http://webcrawler.com/
# mod 27feb09
User-agent: *
Disallow: /images/
Disallow: /ttchat/
Disallow: /scgi-bin/
Disallow: /cgi-bin/
Disallow: /webmail/
Disallow: /datafiles/
Disallow: /ic720/
Disallow: /log/
Disallow: /logs/
Disallow: /chat/
Disallow: /online/
Disallow: /Templates/
Disallow: /styles/
Disallow: /geoip/
Disallow: /iplogs/
Disallow: /iplist/
Disallow: /online/
Disallow: /counters/
Disallow: /logcount.php
Disallow: /stats.shtml
Disallow: /iplog.htm
Disallow: /graphcount.php
Disallow: /graphcounts.php
Disallow: /ccounter.php