User-agent: * # applies to all robots
Disallow: /cgi-bin/ # applies to cgi-bin
Disallow: /tmp/ # applies to tmp folder
Disallow: /admin # applies to admin and contents
Disallow: /pdf # applies to pdf and contents
Disallow: /global # applies to global and contents
Disallow: /dev # applies to dev folder and contents, /dev disallows both /dev.html and /dev/index.html
Disallow: /test # applies to test folder and contents, /test disallows both /test.html and /test/index.html, whereas
Disallow: /test/ # would disallow /test/index.html but allow /test.html
Disallow: /private/ # would disallow /private/index.html but allow /private.html