# www.robotstxt.org/ # www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449 User-agent: meta-externalagent Allow: / Disallow: /stc/ User-agent: meta-webindexer Allow: / Disallow: /stc/ User-agent: meta-externalads Allow: / Disallow: /stc/ User-agent: facebookexternalhit Allow: / Disallow: /stc/ User-agent: meta-externalfetcher Allow: / Disallow: /stc/ User-agent: * Crawl-delay: 10 Disallow: /stc/