# All robots will spider the domain User-agent: * Disallow: /blackhole/ Crawl-delay: 4 # All robots will spider the domain ACAP-crawler: * # User-agent: *