# exclude these from indexing crawlers/robots User-agent: * Disallow: /gif/ /cgi-bin/ /Images/ /Themes/ /Private/