Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- GET /shop/dresses?clothing_size=250&color=247&dir=desc&limit=15&mode=list&order=position
- GET /shop/dresses?clothing_size=268&color=248&dir=asc&order=name
- GET /shop/sale?clothing_size=252&dir=asc&limit=5&mode=grid&order=name&price=50-100
- User-agent: Googlebot-Image
- Disallow: /
- Allow: /shop/media/catalog/product/
- Allow: /shop/media/wysiwyg/
- # Crawlers Setup
- User-agent: *
- # Directories
- Disallow: /shop/404/
- Disallow: /shop/app/
- Disallow: /shop/cgi-bin/
- Disallow: /shop/downloader/
- Disallow: /shop/errors/
- Disallow: /shop/includes/
- #Disallow: /shop/js/
- #Disallow: /shop/lib/
- Disallow: /shop/magento/
- #Disallow: /shop/media/
- Disallow: /shop/pkginfo/
- Disallow: /shop/report/
- Disallow: /shop/scripts/
- Disallow: /shop/shell/
- Disallow: /shop/skin/
- Disallow: /shop/stats/
- Disallow: /shop/var/
- Disallow: /offline/
- # Paths (clean URLs)
- Disallow: /shop/index.php/
- Disallow: /shop/catalog/product_compare/
- Disallow: /shop/catalog/category/view/
- Disallow: /shop/catalog/product/view/
- Disallow: /shop/catalogsearch/
- #Disallow: /shop/checkout/
- Disallow: /shop/control/
- Disallow: /shop/contacts/
- Disallow: /shop/customer/
- Disallow: /shop/customize/
- Disallow: /shop/newsletter/
- Disallow: /shop/poll/
- Disallow: /shop/review/
- Disallow: /shop/sendfriend/
- Disallow: /shop/tag/
- Disallow: /shop/wishlist/
- Disallow: /shop/catalog/product/gallery/
- # Files
- Disallow: /cron.php
- Disallow: /cron.sh
- Disallow: /error_log
- Disallow: /install.php
- Disallow: /LICENSE.html
- Disallow: /LICENSE.txt
- Disallow: /LICENSE_AFL.txt
- Disallow: /STATUS.txt
- Disallow: /.DS_Store
- Disallow: /api.php
- Disallow: /get.php
- Disallow: /mage
- # Paths (no clean URLs)
- #Disallow: /*.js$
- #Disallow: /*.css$
- Disallow: /*.php$
- Disallow: /*?p=*&
- Disallow: /*?SID=
- RewriteEngine On
- RewriteCond %{HTTP_USER_AGENT} (Googlebot|bingbot|Yahoo) [NC]
- RewriteCond %{QUERY_STRING} ^clothing_size=([0-9]*) [NC]
- RewriteRule .* - [G]
- if ($http_user_agent ~* "Baiduspider|Googlebot|bingbot|Yahoo|YandexBot") { set $layered A; }
- if ($args ~ ^(clothing_size|color|price)=.+) { set "${layered}B"; }
- if ($layered = AB) { return 410; }
Add Comment
Please, Sign In to add comment