Pastebin launched a little side project called VERYVIRAL.com, check it out ;-) Want more features on Pastebin? Sign Up, it's FREE!
Guest

Bug or Feature?

By: a guest on Feb 10th, 2011  |  syntax: PHP  |  size: 1.93 KB  |  views: 363  |  expires: Never
download  |  raw  |  embed  |  report abuse  |  print
Text below is selected. Please press Ctrl+C to copy to your clipboard. (⌘+C on Mac)
  1. <?php
  2. /*
  3.  Plugin Name: Soupgiant robot.txt extension
  4.  Plugin URI: http://www.soupgiant.com/
  5.  Description: This plugin will exclude crawlers from additionan directories.
  6.  Version: 0.1
  7.  Author: Peter Wilson
  8.  Author URI: http://www.soupgiant.com/
  9.  
  10. */
  11.  
  12.  
  13. function soup_do_robots(){
  14.         if( !defined('WP_CONTENT_URL') )
  15.                 define( 'WP_CONTENT_URL', get_option('siteurl') . '/wp-content');
  16.         if ( !defined('WP_PLUGIN_URL') )
  17.                 define( 'WP_PLUGIN_URL', WP_CONTENT_URL . '/plugins' ); // full url, no trailing slash
  18.         if ( !defined('WPMU_PLUGIN_URL') )
  19.                 define( 'WPMU_PLUGIN_URL', WP_CONTENT_URL . '/mu-plugins' ); // full url, no trailing slash
  20.  
  21.                
  22.         header( 'Content-Type: text/plain; charset=utf-8' );
  23.  
  24.         do_action( 'do_robotstxt' );
  25.  
  26.                
  27.         if ( '0' == get_option( 'blog_public' ) ) {
  28.                 echo "User-agent: *\n";
  29.                 echo "Disallow: /\n";
  30.         } else {
  31.                 echo "User-agent: *\n";
  32.                 echo "Disallow: /wp-admin/\n";
  33.                 echo "Disallow: /wp-includes/\n";
  34.                 echo "Disallow: /wp-login.php\n";
  35.                 echo "Disallow: /wp-register.php\n";
  36.                
  37.                 //Disallow plugin directories, can't just disallow /wp-content/ as uploads and theme images are stored there
  38.                 echo "Disallow: ";
  39.                 echo parse_url(WP_PLUGIN_URL, PHP_URL_PATH) . "/\n";
  40.                 echo "Disallow: ";
  41.                 echo parse_url(WPMU_PLUGIN_URL, PHP_URL_PATH) . "/\n";
  42.  
  43.                 //Block the cache directories if supercache installed.
  44.                 if( function_exists( 'wp_super_cache_text_domain' ) ) {
  45.                         echo "Disallow: ";
  46.                         echo parse_url(WP_PLUGIN_URL, PHP_URL_PATH) . "/cache/\n";
  47.                 }
  48.  
  49.                 //Block blogs.dir if WPMU (rewritten to /files/)
  50.                 if( function_exists( 'is_site_admin' ) ) {
  51.                         echo "Disallow: ";
  52.                         echo parse_url(WP_PLUGIN_URL, PHP_URL_PATH) . "/blogs.dir/\n";
  53.                 }
  54.         }
  55. }
  56.  
  57. function soup_remove_wp_do_robots(){
  58.         remove_action( 'do_robots','do_robots');
  59. }
  60.  
  61. add_action('init', 'soup_remove_wp_do_robots'); //remove default do robots function
  62. add_action('do_robots', 'soup_do_robots'); //get it into the disallow block
  63. ?>
clone this paste RAW Paste Data