# D:\dh\web\DHD\3\HTM\robots.txt (52 lines) 2007-06-13 04:39 Dean Hannotte #
#----------------------------------------------------------------------#
#        NOTE: Lines in this file must end in LF's, not CRLF's.        #
#                     In KEDIT, issue 'eolout lf'.                     #
#----------------------------------------------------------------------#
#                                                                      #
#      Extracts from 'http://www.robotstxt.org/wc/norobots.html'       #
#                                                                      #
# User-agent                                                           #
#      The value of this field is the name of the robot the record is  #
# describing access policy for.                                        #
#                                                                      #
#      If more than one User-agent field is present the record         #
# describes an identical access policy for more than one robot. At     #
# least one field needs to be present per record.                      #
#                                                                      #
#      The robot should be liberal in interpreting this field. A case  #
# insensitive substring match of the name without version information  #
# is recommended.                                                      #
#                                                                      #
#      If the value is '*', the record describes the default access    #
# policy for any robot that has not matched any of the other records.  #
# It is not allowed to have multiple such records in the "/robots.txt" #
# file.                                                                #
#                                                                      #
# Disallow                                                             #
#      The value of this field specifies a partial URL that is not to  #
# be visited. This can be a full path, or a partial path; any URL that #
# starts with this value will not be retrieved. For example, Disallow: #
# /help disallows both /help.html and /help/index.html, whereas        #
# Disallow: /help/ would disallow /help/index.html but allow           #
# /help.html.                                                          #
#                                                                      #
#      Any empty value, indicates that all URLs can be retrieved. At   #
# least one Disallow field needs to be present in a record.            #
#                                                                      #
#----------------------------------------------------------------------#
User-agent: *

Disallow: /internal/                 # anything in this directory
Disallow: /_                         # private documents
Disallow: /AnneOfGreenGables         # for Rachel
Disallow: /contacts                  # contacts.doc
Disallow: /dhcopa                    # catalog of paleo art
Disallow: /meds                      # meds.doc
Disallow: /Rachel                    # for Rachel
Disallow: /rachel                    # for Rachel
Disallow: /Resume                    # not for search engines
Disallow: /Sites                     # not for search engines
Disallow: /Stat                      # emarketer test files
Disallow: /test                      # test files
Disallow: /ws_ftp.log                # for internal use