1. # robots.txt file for http://www.XXXXXXXXXXXXXXXX.com/
  2. sitemap: http://www.XXXXXXXXXXXXXXXXXXXXX.com/sitemap.xml
  3.  
  4. User-agent: *
  5. # disallow all files in these directories
  6. Disallow: /cgi-bin/
  7. Disallow: /wp-admin/
  8. Disallow: /wp-includes/
  9. Disallow: /wp-content/
  10. Disallow: /go/
  11. Disallow: /archives/
  12. disallow: /*?*
  13. Disallow: /wp-*
  14. Disallow: /author
  15. Disallow: /comments/feed/
  16. Allow: /
  17. Disallow: /search/
  18. Disallow: /tags/
  19. User-agent: Mediapartners-Google*
  20. Allow: /
  21. User-agent: Googlebot-Image
  22. Allow: /wp-content/uploads/
  23. User-agent: Adsbot-Google
  24. Allow: /
  25. User-agent: Googlebot-Mobile
  26. Allow: /