# All robots will spider the domain User-agent: * Disallow: # Disallow directory /errordocs/ User-agent: * Disallow: /errordocs/ Disallow: /images/ Disallow: /stylesheet/ Disallow: /content/ Disallow: /include/