User-agent: * #allows all search engine spiders.
Disallow: /wp-content/downloads/ #disallow them to crawl secretcontent folder.
Disallow: /wp-content/thankyou/
Disallow: /wp-content/features/