# robots.txt file for all virtual servers. # Index this site, except for the following files and directories. User-agent: * Disallow: /SystemWidgets User-agent: * Disallow: /include/storage/ User-agent: * Disallow: /authservice User-agent: * Crawl-delay: 10