Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #Whitehouse.gov - Robots. \
- User-agent: *
- Crawl-delay: 10
- # Directories
- Disallow: /includes/
- Disallow: /misc/
- Disallow: /modules/
- Disallow: /profiles/
- Disallow: /scripts/
- Disallow: /themes/
- # Files
- Disallow: /CHANGELOG.txt
- Disallow: /cron.php
- Disallow: /INSTALL.mysql.txt
- Disallow: /INSTALL.pgsql.txt
- Disallow: /install.php
- Disallow: /INSTALL.txt
- Disallow: /LICENSE.txt
- Disallow: /MAINTAINERS.txt
- Disallow: /update.php
- Disallow: /UPGRADE.txt
- Disallow: /xmlrpc.php
- # Paths (clean URLs)
- Disallow: /admin/
- Disallow: /comment/reply/
- Disallow: /filter/tips/
- Disallow: /logout/
- Disallow: /node/add/
- Disallow: /search/
- Disallow: /user/register/
- Disallow: /user/password/
- Disallow: /user/login/
- # Paths (no clean URLs)
- Disallow: /?q=admin/
- Disallow: /?q=comment/reply/
- Disallow: /?q=filter/tips/
- Disallow: /?q=logout/
- Disallow: /?q=node/add/
- Disallow: /?q=search/
- Disallow: /?q=user/password/
- Disallow: /?q=user/register/
- Disallow: /?q=user/login/
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
