Guest User

Untitled

a guest
Aug 15th, 2018
79
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.03 KB | None | 0 0
  1. GET /shop/dresses?clothing_size=250&color=247&dir=desc&limit=15&mode=list&order=position
  2. GET /shop/dresses?clothing_size=268&color=248&dir=asc&order=name
  3. GET /shop/sale?clothing_size=252&dir=asc&limit=5&mode=grid&order=name&price=50-100
  4.  
  5. User-agent: Googlebot-Image
  6. Disallow: /
  7. Allow: /shop/media/catalog/product/
  8. Allow: /shop/media/wysiwyg/
  9.  
  10. # Crawlers Setup
  11. User-agent: *
  12.  
  13. # Directories
  14. Disallow: /shop/404/
  15. Disallow: /shop/app/
  16. Disallow: /shop/cgi-bin/
  17. Disallow: /shop/downloader/
  18. Disallow: /shop/errors/
  19. Disallow: /shop/includes/
  20. #Disallow: /shop/js/
  21. #Disallow: /shop/lib/
  22. Disallow: /shop/magento/
  23. #Disallow: /shop/media/
  24. Disallow: /shop/pkginfo/
  25. Disallow: /shop/report/
  26. Disallow: /shop/scripts/
  27. Disallow: /shop/shell/
  28. Disallow: /shop/skin/
  29. Disallow: /shop/stats/
  30. Disallow: /shop/var/
  31. Disallow: /offline/
  32.  
  33. # Paths (clean URLs)
  34. Disallow: /shop/index.php/
  35. Disallow: /shop/catalog/product_compare/
  36. Disallow: /shop/catalog/category/view/
  37. Disallow: /shop/catalog/product/view/
  38. Disallow: /shop/catalogsearch/
  39. #Disallow: /shop/checkout/
  40. Disallow: /shop/control/
  41. Disallow: /shop/contacts/
  42. Disallow: /shop/customer/
  43. Disallow: /shop/customize/
  44. Disallow: /shop/newsletter/
  45. Disallow: /shop/poll/
  46. Disallow: /shop/review/
  47. Disallow: /shop/sendfriend/
  48. Disallow: /shop/tag/
  49. Disallow: /shop/wishlist/
  50. Disallow: /shop/catalog/product/gallery/
  51.  
  52. # Files
  53. Disallow: /cron.php
  54. Disallow: /cron.sh
  55. Disallow: /error_log
  56. Disallow: /install.php
  57. Disallow: /LICENSE.html
  58. Disallow: /LICENSE.txt
  59. Disallow: /LICENSE_AFL.txt
  60. Disallow: /STATUS.txt
  61. Disallow: /.DS_Store
  62. Disallow: /api.php
  63. Disallow: /get.php
  64. Disallow: /mage
  65.  
  66.  
  67.  
  68. # Paths (no clean URLs)
  69. #Disallow: /*.js$
  70. #Disallow: /*.css$
  71. Disallow: /*.php$
  72. Disallow: /*?p=*&
  73. Disallow: /*?SID=
  74.  
  75. RewriteEngine On
  76. RewriteCond %{HTTP_USER_AGENT} (Googlebot|bingbot|Yahoo) [NC]
  77. RewriteCond %{QUERY_STRING} ^clothing_size=([0-9]*) [NC]
  78. RewriteRule .* - [G]
  79.  
  80. if ($http_user_agent ~* "Baiduspider|Googlebot|bingbot|Yahoo|YandexBot") { set $layered A; }
  81. if ($args ~ ^(clothing_size|color|price)=.+) { set "${layered}B"; }
  82. if ($layered = AB) { return 410; }
Add Comment
Please, Sign In to add comment