User-agent: *
Disallow: *.json$
Disallow: *.xml$
Disallow: /test
Disallow: /admin
Disallow: /expanding_list
Disallow: /top_secret_stuff

User-agent: Slurp
Crawl-delay: 1