Serve robots.txt as text/plain. Props JeremyVisser. fixes #4037

git-svn-id: http://svn.automattic.com/wordpress/trunk@5117 1a063a9b-81f0-0310-95a4-ce76da25c4cd

wp-includes/functions.php

@@ -982,7 +982,10 @@ function do_feed_atom($for_comments) {
 }
 
 function do_robots() {
+	header('Content-type: text/plain; charset=utf-8');
+
 	do_action('do_robotstxt');
+
 	if ( '0' == get_option('blog_public') ) {
 		echo "User-agent: *\n";
 		echo "Disallow: /\n";