# All robots will spider the domain

User-agent: *
Disallow: /rotate_ad/ # this is used to store adverts for rotation
Disallow: /cgi-local/ # scripts
Disallow: /cgi-bin/ # scripts