User-agent: Googlebot
Disallow: /nogooglebot/

User-agent: *
Allow: /


User-agent: criteo-crawler
Crawl-delay: 30

User-agent: comscore Web-Crawler
Crawl-delay: 30