# Example robots.txt file from TwentyHost
# Upload into your public_html folder when you have made your appropriate changes
# For further help try the following sites. Note: we do not endorse or recommend these sites, nor have any control over their content.
# http://www.robotstxt.org/
# http://www.searchtools.com/robots/robots-txt.html
# http://www.google.com/support/webmasters/bin/topic.py?topic=8475



# stop Webcrawler from indexing/spidering your images folder
User-agent: webcrawler
Disallow: /images/

# stop Google from indexing/spidering your images & logs folders plus 2 specific files
User-agent: google
Disallow: /images/
Disallow: /logs/
Disallow: /main/index.html
Disallow: /main/file.php

# stop all robots from indexing/spidering individual files 
User-agent: *
Disallow: secret.html
Disallow: hidden.php

# stop all robots from indexing/spidering individual folders
User-agent: *
Disallow: /secure/
Disallow: /images/
Disallow: /new_design/