daily pastebin goal
10%
SHARE
TWEET

Block Bad Robots - Htaccess

a guest Oct 20th, 2013 7,557 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. RewriteEngine On
  2. RewriteCond %{REQUEST_URI} !/robots.txt$
  3. RewriteCond %{HTTP_USER_AGENT} ^.*BLEXBot.*$ [NC,OR]
  4. RewriteCond %{HTTP_USER_AGENT} ^.*BlackWidow.*$ [NC,OR]
  5. RewriteCond %{HTTP_USER_AGENT} ^.*Nutch.*$ [NC,OR]
  6. RewriteCond %{HTTP_USER_AGENT} ^.*Jetbot.*$ [NC,OR]
  7. RewriteCond %{HTTP_USER_AGENT} ^.*WebVac.*$ [NC,OR]
  8. RewriteCond %{HTTP_USER_AGENT} ^.*Stanford.*$ [NC,OR]
  9. RewriteCond %{HTTP_USER_AGENT} ^.*scooter.*$ [NC,OR]
  10. RewriteCond %{HTTP_USER_AGENT} ^.*naver.*$ [NC,OR]
  11. RewriteCond %{HTTP_USER_AGENT} ^.*dumbot.*$ [NC,OR]
  12. RewriteCond %{HTTP_USER_AGENT} ^.*Hatena\ Antenna.*$ [NC,OR]
  13. RewriteCond %{HTTP_USER_AGENT} ^.*grub.*$ [NC,OR]
  14. RewriteCond %{HTTP_USER_AGENT} ^.*looksmart.*$ [NC,OR]
  15. RewriteCond %{HTTP_USER_AGENT} ^.*WebZip.*$ [NC,OR]
  16. RewriteCond %{HTTP_USER_AGENT} ^.*larbin.*$ [NC,OR]
  17. RewriteCond %{HTTP_USER_AGENT} ^.*b2w/0.1.*$ [NC,OR]
  18. RewriteCond %{HTTP_USER_AGENT} ^.*Copernic.*$ [NC,OR]
  19. RewriteCond %{HTTP_USER_AGENT} ^.*psbot.*$ [NC,OR]
  20. RewriteCond %{HTTP_USER_AGENT} ^.*Python-urllib.*$ [NC,OR]
  21. RewriteCond %{HTTP_USER_AGENT} ^.*NetMechanic.*$ [NC,OR]
  22. RewriteCond %{HTTP_USER_AGENT} ^.*URL_Spider_Pro.*$ [NC,OR]
  23. RewriteCond %{HTTP_USER_AGENT} ^.*CherryPicker.*$ [NC,OR]
  24. RewriteCond %{HTTP_USER_AGENT} ^.*EmailCollector.*$ [NC,OR]
  25. RewriteCond %{HTTP_USER_AGENT} ^.*EmailSiphon.*$ [NC,OR]
  26. RewriteCond %{HTTP_USER_AGENT} ^.*WebBandit.*$ [NC,OR]
  27. RewriteCond %{HTTP_USER_AGENT} ^.*EmailWolf.*$ [NC,OR]
  28. RewriteCond %{HTTP_USER_AGENT} ^.*Email.*$ [NC,OR]
  29. RewriteCond %{HTTP_USER_AGENT} ^.*ExtractorPro.*$ [NC,OR]
  30. RewriteCond %{HTTP_USER_AGENT} ^.*CopyRightCheck.*$ [NC,OR]
  31. RewriteCond %{HTTP_USER_AGENT} ^.*Crescent.*$ [NC,OR]
  32. RewriteCond %{HTTP_USER_AGENT} ^.*SiteSnagger.*$ [NC,OR]
  33. RewriteCond %{HTTP_USER_AGENT} ^.*ProWebWalker.*$ [NC,OR]
  34. RewriteCond %{HTTP_USER_AGENT} ^.*CheeseBot.*$ [NC,OR]
  35. RewriteCond %{HTTP_USER_AGENT} ^.*LNSpiderguy.*$ [NC,OR]
  36. RewriteCond %{HTTP_USER_AGENT} ^.*ia_archiver.*$ [NC,OR]
  37. RewriteCond %{HTTP_USER_AGENT} ^.*Alexibot.*$ [NC,OR]
  38. RewriteCond %{HTTP_USER_AGENT} ^.*Teleport.*$ [NC,OR]
  39. RewriteCond %{HTTP_USER_AGENT} ^.*MIIxpc.*$ [NC,OR]
  40. RewriteCond %{HTTP_USER_AGENT} ^.*Telesoft.*$ [NC,OR]
  41. RewriteCond %{HTTP_USER_AGENT} ^.*Website\ Quester.*$ [NC,OR]
  42. RewriteCond %{HTTP_USER_AGENT} ^.*moget.*$ [NC,OR]
  43. RewriteCond %{HTTP_USER_AGENT} ^.*WebStripper.*$ [NC,OR]
  44. RewriteCond %{HTTP_USER_AGENT} ^.*WebSauger.*$ [NC,OR]
  45. RewriteCond %{HTTP_USER_AGENT} ^.*WebCopier.*$ [NC,OR]
  46. RewriteCond %{HTTP_USER_AGENT} ^.*NetAnts.*$ [NC,OR]
  47. RewriteCond %{HTTP_USER_AGENT} ^.*Mister\ PiX.*$ [NC,OR]
  48. RewriteCond %{HTTP_USER_AGENT} ^.*WebAuto.*$ [NC,OR]
  49. RewriteCond %{HTTP_USER_AGENT} ^.*TheNomad.*$ [NC,OR]
  50. RewriteCond %{HTTP_USER_AGENT} ^.*WWW-Collector-E.*$ [NC,OR]
  51. RewriteCond %{HTTP_USER_AGENT} ^.*RMA.*$ [NC,OR]
  52. RewriteCond %{HTTP_USER_AGENT} ^.*libWeb/clsHTTP.*$ [NC,OR]
  53. RewriteCond %{HTTP_USER_AGENT} ^.*asterias.*$ [NC,OR]
  54. RewriteCond %{HTTP_USER_AGENT} ^.*httplib.*$ [NC,OR]
  55. RewriteCond %{HTTP_USER_AGENT} ^.*turingos.*$ [NC,OR]
  56. RewriteCond %{HTTP_USER_AGENT} ^.*spanner.*$ [NC,OR]
  57. RewriteCond %{HTTP_USER_AGENT} ^.*Harvest.*$ [NC,OR]
  58. RewriteCond %{HTTP_USER_AGENT} ^.*InfoNaviRobot.*$ [NC,OR]
  59. RewriteCond %{HTTP_USER_AGENT} ^.*Bullseye.*$ [NC,OR]
  60. RewriteCond %{HTTP_USER_AGENT} ^.*WebBandit.*$ [NC,OR]
  61. RewriteCond %{HTTP_USER_AGENT} ^.*NICErsPRO.*$ [NC,OR]
  62. RewriteCond %{HTTP_USER_AGENT} ^.*Microsoft\ URL\ Control.*$ [NC,OR]
  63. RewriteCond %{HTTP_USER_AGENT} ^.*DittoSpyder.*$ [NC,OR]
  64. RewriteCond %{HTTP_USER_AGENT} ^.*Foobot.*$ [NC,OR]
  65. RewriteCond %{HTTP_USER_AGENT} ^.*WebmasterWorldForumBot.*$ [NC,OR]
  66. RewriteCond %{HTTP_USER_AGENT} ^.*SpankBot.*$ [NC,OR]
  67. RewriteCond %{HTTP_USER_AGENT} ^.*BotALot.*$ [NC,OR]
  68. RewriteCond %{HTTP_USER_AGENT} ^.*lwp-trivial.*$ [NC,OR]
  69. RewriteCond %{HTTP_USER_AGENT} ^.*WebmasterWorld.*$ [NC,OR]
  70. RewriteCond %{HTTP_USER_AGENT} ^.*BunnySlippers.*$ [NC,OR]
  71. RewriteCond %{HTTP_USER_AGENT} ^.*URLy\ Warning.*$ [NC,OR]
  72. RewriteCond %{HTTP_USER_AGENT} ^.*Wget.*$ [NC,OR]
  73. RewriteCond %{HTTP_USER_AGENT} ^.*LinkWalker.*$ [NC,OR]
  74. RewriteCond %{HTTP_USER_AGENT} ^.*cosmos.*$ [NC,OR]
  75. RewriteCond %{HTTP_USER_AGENT} ^.*hloader.*$ [NC,OR]
  76. RewriteCond %{HTTP_USER_AGENT} ^.*humanlinks.*$ [NC,OR]
  77. RewriteCond %{HTTP_USER_AGENT} ^.*LinkextractorPro.*$ [NC,OR]
  78. RewriteCond %{HTTP_USER_AGENT} ^.*Offline\ Explorer.*$ [NC,OR]
  79. RewriteCond %{HTTP_USER_AGENT} ^.*Mata\ Hari.*$ [NC,OR]
  80. RewriteCond %{HTTP_USER_AGENT} ^.*LexiBot.*$ [NC,OR]
  81. RewriteCond %{HTTP_USER_AGENT} ^.*Web\ Image\ Collector.*$ [NC,OR]
  82. RewriteCond %{HTTP_USER_AGENT} ^.*The\ Intraformant.*$ [NC,OR]
  83. RewriteCond %{HTTP_USER_AGENT} ^.*True_Robot.*$ [NC,OR]
  84. RewriteCond %{HTTP_USER_AGENT} ^.*BlowFish.*$ [NC,OR]
  85. RewriteCond %{HTTP_USER_AGENT} ^.*SearchEngineWorld.*$ [NC,OR]
  86. RewriteCond %{HTTP_USER_AGENT} ^.*JennyBot.*$ [NC,OR]
  87. RewriteCond %{HTTP_USER_AGENT} ^.*MIIxpc.*$ [NC,OR]
  88. RewriteCond %{HTTP_USER_AGENT} ^.*BuiltBotTough.*$ [NC,OR]
  89. RewriteCond %{HTTP_USER_AGENT} ^.*ProPowerBot.*$ [NC,OR]
  90. RewriteCond %{HTTP_USER_AGENT} ^.*BackDoorBot.*$ [NC,OR]
  91. RewriteCond %{HTTP_USER_AGENT} ^.*toCrawl/UrlDispatcher.*$ [NC,OR]
  92. RewriteCond %{HTTP_USER_AGENT} ^.*WebEnhancer.*$ [NC,OR]
  93. RewriteCond %{HTTP_USER_AGENT} ^.*suzuran.*$ [NC,OR]
  94. RewriteCond %{HTTP_USER_AGENT} ^.*WebViewer.*$ [NC,OR]
  95. RewriteCond %{HTTP_USER_AGENT} ^.*VCI.*$ [NC,OR]
  96. RewriteCond %{HTTP_USER_AGENT} ^.*Szukacz.*$ [NC,OR]
  97. RewriteCond %{HTTP_USER_AGENT} ^.*QueryN.*$ [NC,OR]
  98. RewriteCond %{HTTP_USER_AGENT} ^.*Openfind.*$ [NC,OR]
  99. RewriteCond %{HTTP_USER_AGENT} ^.*Openbot.*$ [NC,OR]
  100. RewriteCond %{HTTP_USER_AGENT} ^.*Webster.*$ [NC,OR]
  101. RewriteCond %{HTTP_USER_AGENT} ^.*EroCrawler.*$ [NC,OR]
  102. RewriteCond %{HTTP_USER_AGENT} ^.*LinkScan.*$ [NC,OR]
  103. RewriteCond %{HTTP_USER_AGENT} ^.*Keyword.*$ [NC,OR]
  104. RewriteCond %{HTTP_USER_AGENT} ^.*Kenjin.*$ [NC,OR]
  105. RewriteCond %{HTTP_USER_AGENT} ^.*Iron33.*$ [NC,OR]
  106. RewriteCond %{HTTP_USER_AGENT} ^.*Bookmark\ search\ tool.*$ [NC,OR]
  107. RewriteCond %{HTTP_USER_AGENT} ^.*GetRight.*$ [NC,OR]
  108. RewriteCond %{HTTP_USER_AGENT} ^.*FairAd\ Client.*$ [NC,OR]
  109. RewriteCond %{HTTP_USER_AGENT} ^.*Gaisbot.*$ [NC,OR]
  110. RewriteCond %{HTTP_USER_AGENT} ^.*Aqua_Products.*$ [NC,OR]
  111. RewriteCond %{HTTP_USER_AGENT} ^.*Radiation\ Retriever\ 1.1.*$ [NC,OR]
  112. RewriteCond %{HTTP_USER_AGENT} ^.*Flaming\ AttackBot.*$ [NC,OR]
  113. RewriteCond %{HTTP_USER_AGENT} ^.*Oracle\ Ultra\ Search.*$ [NC,OR]
  114. RewriteCond %{HTTP_USER_AGENT} ^.*MSIECrawler.*$ [NC,OR]
  115. RewriteCond %{HTTP_USER_AGENT} ^.*PerMan.*$ [NC,OR]
  116. RewriteCond %{HTTP_USER_AGENT} ^.*searchpreview.*$ [NC,OR]
  117. RewriteCond %{HTTP_USER_AGENT} ^.*sootle.*$ [NC,OR]
  118. RewriteCond %{HTTP_USER_AGENT} ^.*Enterprise_Search.*$ [NC,OR]
  119. RewriteCond %{HTTP_USER_AGENT} ^.*Bot\ mailto:craftbot@yahoo.com.*$ [NC,OR]
  120. RewriteCond %{HTTP_USER_AGENT} ^.*ChinaClaw.*$ [NC,OR]
  121. RewriteCond %{HTTP_USER_AGENT} ^.*Custo.*$ [NC,OR]
  122. RewriteCond %{HTTP_USER_AGENT} ^.*DISCo.*$ [NC,OR]
  123. RewriteCond %{HTTP_USER_AGENT} ^.*Download\ Demon.*$ [NC,OR]
  124. RewriteCond %{HTTP_USER_AGENT} ^.*eCatch.*$ [NC,OR]
  125. RewriteCond %{HTTP_USER_AGENT} ^.*EirGrabber.*$ [NC,OR]
  126. RewriteCond %{HTTP_USER_AGENT} ^.*EmailSiphon.*$ [NC,OR]
  127. RewriteCond %{HTTP_USER_AGENT} ^.*EmailWolf.*$ [NC,OR]
  128. RewriteCond %{HTTP_USER_AGENT} ^.*Express\ WebPictures.*$ [NC,OR]
  129. RewriteCond %{HTTP_USER_AGENT} ^.*ExtractorPro.*$ [NC,OR]
  130. RewriteCond %{HTTP_USER_AGENT} ^.*EyeNetIE.*$ [NC,OR]
  131. RewriteCond %{HTTP_USER_AGENT} ^.*FlashGet.*$ [NC,OR]
  132. RewriteCond %{HTTP_USER_AGENT} ^.*GetRight.*$ [NC,OR]
  133. RewriteCond %{HTTP_USER_AGENT} ^.*GetWeb!.*$ [NC,OR]
  134. RewriteCond %{HTTP_USER_AGENT} ^.*Go!Zilla.*$ [NC,OR]
  135. RewriteCond %{HTTP_USER_AGENT} ^.*Go-Ahead-Got-It.*$ [NC,OR]
  136. RewriteCond %{HTTP_USER_AGENT} ^.*GrabNet.*$ [NC,OR]
  137. RewriteCond %{HTTP_USER_AGENT} ^.*Grafula.*$ [NC,OR]
  138. RewriteCond %{HTTP_USER_AGENT} ^.*HMView.*$ [NC,OR]
  139. RewriteCond %{HTTP_USER_AGENT} ^.*HTTrack.*$ [NC,OR]
  140. RewriteCond %{HTTP_USER_AGENT} ^.*Image\ Stripper.*$ [NC,OR]
  141. RewriteCond %{HTTP_USER_AGENT} ^.*Image\ Sucker.*$ [NC,OR]
  142. RewriteCond %{HTTP_USER_AGENT} ^.*Indy\ Library.*$ [NC,OR]
  143. RewriteCond %{HTTP_USER_AGENT} ^.*InterGET.*$ [NC,OR]
  144. RewriteCond %{HTTP_USER_AGENT} ^.*Internet\ Ninja.*$ [NC,OR]
  145. RewriteCond %{HTTP_USER_AGENT} ^.*JetCar.*$ [NC,OR]
  146. RewriteCond %{HTTP_USER_AGENT} ^.*JOC\ Web\ Spider.*$ [NC,OR]
  147. RewriteCond %{HTTP_USER_AGENT} ^.*larbin.*$ [NC,OR]
  148. RewriteCond %{HTTP_USER_AGENT} ^.*LeechFTP.*$ [NC,OR]
  149. RewriteCond %{HTTP_USER_AGENT} ^.*Mass\ Downloader.*$ [NC,OR]
  150. RewriteCond %{HTTP_USER_AGENT} ^.*MIDown\ tool.*$ [NC,OR]
  151. RewriteCond %{HTTP_USER_AGENT} ^.*Mister\ PiX.*$ [NC,OR]
  152. RewriteCond %{HTTP_USER_AGENT} ^.*Navroad.*$ [NC,OR]
  153. RewriteCond %{HTTP_USER_AGENT} ^.*NearSite.*$ [NC,OR]
  154. RewriteCond %{HTTP_USER_AGENT} ^.*NetAnts.*$ [NC,OR]
  155. RewriteCond %{HTTP_USER_AGENT} ^.*NetSpider.*$ [NC,OR]
  156. RewriteCond %{HTTP_USER_AGENT} ^.*Net\ Vampire.*$ [NC,OR]
  157. RewriteCond %{HTTP_USER_AGENT} ^.*NetZIP.*$ [NC,OR]
  158. RewriteCond %{HTTP_USER_AGENT} ^.*Octopus.*$ [NC,OR]
  159. RewriteCond %{HTTP_USER_AGENT} ^.*Offline\ Explorer.*$ [NC,OR]
  160. RewriteCond %{HTTP_USER_AGENT} ^.*Offline\ Navigator.*$ [NC,OR]
  161. RewriteCond %{HTTP_USER_AGENT} ^.*PageGrabber.*$ [NC,OR]
  162. RewriteCond %{HTTP_USER_AGENT} ^.*Papa\ Foto.*$ [NC,OR]
  163. RewriteCond %{HTTP_USER_AGENT} ^.*pavuk.*$ [NC,OR]
  164. RewriteCond %{HTTP_USER_AGENT} ^.*pcBrowser.*$ [NC,OR]
  165. RewriteCond %{HTTP_USER_AGENT} ^.*RealDownload.*$ [NC,OR]
  166. RewriteCond %{HTTP_USER_AGENT} ^.*ReGet.*$ [NC,OR]
  167. RewriteCond %{HTTP_USER_AGENT} ^.*SiteSnagger.*$ [NC,OR]
  168. RewriteCond %{HTTP_USER_AGENT} ^.*SmartDownload.*$ [NC,OR]
  169. RewriteCond %{HTTP_USER_AGENT} ^.*SuperBot.*$ [NC,OR]
  170. RewriteCond %{HTTP_USER_AGENT} ^.*SuperHTTP.*$ [NC,OR]
  171. RewriteCond %{HTTP_USER_AGENT} ^.*Surfbot.*$ [NC,OR]
  172. RewriteCond %{HTTP_USER_AGENT} ^.*tAkeOut.*$ [NC,OR]
  173. RewriteCond %{HTTP_USER_AGENT} ^.*Teleport\ Pro.*$ [NC,OR]
  174. RewriteCond %{HTTP_USER_AGENT} ^.*VoidEYE.*$ [NC,OR]
  175. RewriteCond %{HTTP_USER_AGENT} ^.*Web\ Image\ Collector.*$ [NC,OR]
  176. RewriteCond %{HTTP_USER_AGENT} ^.*Web\ Sucker.*$ [NC,OR]
  177. RewriteCond %{HTTP_USER_AGENT} ^.*WebAuto.*$ [NC,OR]
  178. RewriteCond %{HTTP_USER_AGENT} ^.*WebCopier.*$ [NC,OR]
  179. RewriteCond %{HTTP_USER_AGENT} ^.*WebFetch.*$ [NC,OR]
  180. RewriteCond %{HTTP_USER_AGENT} ^.*WebGo\ IS.*$ [NC,OR]
  181. RewriteCond %{HTTP_USER_AGENT} ^.*WebLeacher.*$ [NC,OR]
  182. RewriteCond %{HTTP_USER_AGENT} ^.*WebReaper.*$ [NC,OR]
  183. RewriteCond %{HTTP_USER_AGENT} ^.*WebSauger.*$ [NC,OR]
  184. RewriteCond %{HTTP_USER_AGENT} ^.*Website\ eXtractor.*$ [NC,OR]
  185. RewriteCond %{HTTP_USER_AGENT} ^.*Website\ Quester.*$ [NC,OR]
  186. RewriteCond %{HTTP_USER_AGENT} ^.*WebStripper.*$ [NC,OR]
  187. RewriteCond %{HTTP_USER_AGENT} ^.*WebWhacker.*$ [NC,OR]
  188. RewriteCond %{HTTP_USER_AGENT} ^.*WebZIP.*$ [NC,OR]
  189. RewriteCond %{HTTP_USER_AGENT} ^.*Wget.*$ [NC,OR]
  190. RewriteCond %{HTTP_USER_AGENT} ^.*Widow.*$ [NC,OR]
  191. RewriteCond %{HTTP_USER_AGENT} ^.*WWWOFFLE.*$ [NC,OR]
  192. RewriteCond %{HTTP_USER_AGENT} ^.*Xaldon\ WebSpider.*$ [NC,OR]
  193. RewriteCond %{HTTP_USER_AGENT} ^.*Zeus.*$ [NC,OR]
  194. RewriteCond %{HTTP_USER_AGENT} ^.*Semrush.*$ [NC,OR]
  195. RewriteCond %{HTTP_USER_AGENT} ^.*BecomeBot.*$ [NC,OR]
  196. RewriteCond %{HTTP_USER_AGENT} ^.*AhrefsBot.*$ [NC,OR]
  197. RewriteCond %{HTTP_USER_AGENT} ^.*MJ12bot.*$ [NC,OR]
  198. RewriteCond %{HTTP_USER_AGENT} ^.*rogerbot.*$ [NC,OR]
  199. RewriteCond %{HTTP_USER_AGENT} ^.*exabot.*$ [NC,OR]
  200. RewriteCond %{HTTP_USER_AGENT} ^.*Xenu.*$ [NC,OR]
  201. RewriteCond %{HTTP_USER_AGENT} ^.*dotbot.*$ [NC,OR]
  202. RewriteCond %{HTTP_USER_AGENT} ^.*gigabot.*$ [NC,OR]
  203. RewriteCond %{HTTP_USER_AGENT} ^.*BlekkoBot.*$ [NC]
  204. RewriteRule ^.*.* http://www.webmd.com/ [L]
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top