User-agent: * # Index the repository summaries but no deeper Disallow: /gitweb/? Allow: /gitweb/?p=*;a=summary # Meta crawlers don't seem to understand the above rules, so block them entirely User-agent: meta-externalagent Disallow: * User-agent: meta-webindexer Disallow: * User-agent: meta-externalads Disallow: * # Thinkbot too User-agent: Thinkbot Disallow: *