Advertisement
floqqer

Robots.txt spam

Jul 27th, 2019
695
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.02 KB | None | 0 0
  1. User-agent: MJ12bot
  2. Disallow: /
  3. User-agent: SemrushBot
  4. Disallow: /
  5. User-agent: SemrushBot-SA
  6. Disallow: /
  7. User-agent: dotbot
  8. Disallow:/
  9. User-agent: AhrefsBot
  10. Disallow: /
  11. User-agent: Alexibot
  12. Disallow: /
  13. User-agent: SurveyBot
  14. Disallow: /
  15. User-agent: Xenu’s
  16. Disallow: /
  17. User-agent: Xenu’s Link Sleuth 1.1c
  18. Disallow: /
  19. User-agent: rogerbot
  20. Disallow: /
  21.  
  22. # Block NextGenSearchBot
  23. User-agent: NextGenSearchBot
  24. Disallow: /
  25. # Block ia-archiver from crawling site
  26. User-agent: ia_archiver
  27. Disallow: /
  28. # Block archive.org_bot from crawling site
  29. User-agent: archive.org_bot
  30. Disallow: /
  31. # Block Archive.org Bot from crawling site
  32. User-agent: Archive.org Bot
  33. Disallow: /
  34.  
  35. # Block LinkWalker from crawling site
  36. User-agent: LinkWalker
  37. Disallow: /
  38.  
  39. # Block GigaBlast Spider from crawling site
  40. User-agent: GigaBlast Spider
  41. Disallow: /
  42.  
  43. # Block ia_archiver-web.archive.org_bot from crawling site
  44. User-agent: ia_archiver-web.archive.org
  45. Disallow: /
  46.  
  47. # Block PicScout Crawler from crawling site
  48. User-agent: PicScout
  49. Disallow: /
  50.  
  51. # Block BLEXBot Crawler from crawling site
  52. User-agent: BLEXBot Crawler
  53. Disallow: /
  54.  
  55. # Block TinEye from crawling site
  56. User-agent: TinEye
  57. Disallow: /
  58.  
  59. # Block SEOkicks
  60. User-agent: SEOkicks-Robot
  61. Disallow: /
  62.  
  63. # Block BlexBot
  64. User-agent: BLEXBot
  65. Disallow: /
  66.  
  67. # Block SISTRIX
  68. User-agent: SISTRIX Crawler
  69. Disallow: /
  70.  
  71. # Block Uptime robot
  72. User-agent: UptimeRobot/2.0
  73. Disallow: /
  74.  
  75. # Block Ezooms Robot
  76. User-agent: Ezooms Robot
  77. Disallow: /
  78.  
  79. # Block netEstate NE Crawler (+http://www.website-datenbank.de/)
  80. User-agent: netEstate NE Crawler (+http://www.website-datenbank.de/)
  81. Disallow: /
  82.  
  83. # Block WiseGuys Robot
  84. User-agent: WiseGuys Robot
  85. Disallow: /
  86.  
  87. # Block Turnitin Robot
  88. User-agent: Turnitin Robot
  89. Disallow: /
  90.  
  91. # Block Heritrix
  92. User-agent: Heritrix
  93. Disallow: /
  94.  
  95. # Block pricepi
  96. User-agent: pimonster
  97. Disallow: /
  98. User-agent: Pimonster
  99. Disallow: /
  100. User-agent: Pi-Monster
  101. Disallow: /
  102.  
  103. # Block Eniro
  104. User-agent: ECCP/1.0 (search@eniro.com)
  105. Disallow: /
  106.  
  107. # Block Psbot
  108. User-agent: Psbot
  109. Disallow: /
  110.  
  111. # Block Youdao
  112. User-agent: YoudaoBot
  113. Disallow: /
  114.  
  115. # BLEXBot
  116. User-agent: BLEXBot
  117. Disallow: /
  118.  
  119. # Block NaverBot
  120. User-agent: NaverBot
  121. User-agent: Yeti
  122. Disallow: /
  123.  
  124. # Block ZBot
  125. User-agent: ZBot
  126. Disallow: /
  127.  
  128. # Block Vagabondo
  129. User-agent: Vagabondo
  130. Disallow: /
  131.  
  132. # Block LinkWalker
  133. User-agent: LinkWalker
  134. Disallow: /
  135.  
  136. # Block SimplePie
  137. User-agent: SimplePie
  138. Disallow: /
  139.  
  140. # Block Wget
  141. User-agent: Wget
  142. Disallow: /
  143.  
  144. # Block Pixray-Seeker
  145. User-agent: Pixray-Seeker
  146. Disallow: /
  147.  
  148. # Block BoardReader
  149. User-agent: BoardReader
  150. Disallow: /
  151.  
  152. # Block Quantify
  153. User-agent: Quantify
  154. Disallow: /
  155.  
  156. # Block Plukkie
  157. User-agent: Plukkie
  158. Disallow: /
  159.  
  160. # Block Cuam
  161. User-agent: Cuam
  162. Disallow: /
  163.  
  164. # https://megaindex.com/crawler
  165. User-agent: MegaIndex.ru
  166. Disallow: /
  167.  
  168. User-agent: megaindex.com
  169. Disallow: /
  170.  
  171. User-agent: +http://megaindex.com/crawler
  172. Disallow: /
  173.  
  174. User-agent: MegaIndex.ru/2.0
  175. Disallow: /
  176.  
  177. User-agent: megaIndex.ru
  178. Disallow: /
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement