API tools faq
paste
Advertisement
Guest User

Untitled

a guest
May 24th, 2019
136
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 8.04 KB | None | 0 0
raw download clone embed print report
  1. User-agent: Applebot
  2. Disallow: /ajax/
  3. Disallow: /album.php
  4. Disallow: /checkpoint/
  5. Disallow: /contact_importer/
  6. Disallow: /dialog/
  7. Disallow: /fbml/ajax/dialog/
  8. Disallow: /feeds/
  9. Disallow: /file_download.php
  10. Disallow: /hashtag/
  11. Disallow: /l.php
  12. Disallow: /live/
  13. Disallow: /moments_app/
  14. Disallow: /p.php
  15. Disallow: /photo.php
  16. Disallow: /photos.php
  17. Disallow: /share.php
  18. Disallow: /share/
  19. Disallow: /sharer.php
  20. Disallow: /sharer/
  21.  
  22. User-agent: baiduspider
  23. Disallow: /ajax/
  24. Disallow: /album.php
  25. Disallow: /checkpoint/
  26. Disallow: /contact_importer/
  27. Disallow: /dialog/
  28. Disallow: /fbml/ajax/dialog/
  29. Disallow: /feeds/
  30. Disallow: /file_download.php
  31. Disallow: /hashtag/
  32. Disallow: /l.php
  33. Disallow: /live/
  34. Disallow: /moments_app/
  35. Disallow: /p.php
  36. Disallow: /photo.php
  37. Disallow: /photos.php
  38. Disallow: /share.php
  39. Disallow: /share/
  40. Disallow: /sharer.php
  41. Disallow: /sharer/
  42.  
  43. User-agent: Bingbot
  44. Disallow: /ajax/
  45. Disallow: /album.php
  46. Disallow: /checkpoint/
  47. Disallow: /contact_importer/
  48. Disallow: /dialog/
  49. Disallow: /fbml/ajax/dialog/
  50. Disallow: /feeds/
  51. Disallow: /file_download.php
  52. Disallow: /hashtag/
  53. Disallow: /l.php
  54. Disallow: /live/
  55. Disallow: /moments_app/
  56. Disallow: /p.php
  57. Disallow: /photo.php
  58. Disallow: /photos.php
  59. Disallow: /share.php
  60. Disallow: /share/
  61. Disallow: /sharer.php
  62. Disallow: /sharer/
  63.  
  64. User-agent: Googlebot
  65. Disallow: /ajax/
  66. Disallow: /album.php
  67. Disallow: /checkpoint/
  68. Disallow: /contact_importer/
  69. Disallow: /dialog/
  70. Disallow: /fbml/ajax/dialog/
  71. Disallow: /feeds/
  72. Disallow: /file_download.php
  73. Disallow: /hashtag/
  74. Disallow: /l.php
  75. Disallow: /live/
  76. Disallow: /moments_app/
  77. Disallow: /p.php
  78. Disallow: /photo.php
  79. Disallow: /photos.php
  80. Disallow: /share.php
  81. Disallow: /share/
  82. Disallow: /sharer.php
  83. Disallow: /sharer/
  84.  
  85. User-agent: Googlebot-Image
  86. Disallow: /ajax/
  87. Disallow: /album.php
  88. Disallow: /checkpoint/
  89. Disallow: /contact_importer/
  90. Disallow: /dialog/
  91. Disallow: /fbml/ajax/dialog/
  92. Disallow: /feeds/
  93. Disallow: /file_download.php
  94. Disallow: /hashtag/
  95. Disallow: /l.php
  96. Disallow: /live/
  97. Disallow: /map_tile.php
  98. Disallow: /moments_app/
  99. Disallow: /p.php
  100. Disallow: /photo.php
  101. Disallow: /photos.php
  102. Disallow: /share.php
  103. Disallow: /share/
  104. Disallow: /sharer.php
  105. Disallow: /sharer/
  106. Disallow: /static_map.php
  107.  
  108. User-agent: ia_archiver
  109. Disallow: /
  110. Disallow: /ajax/
  111. Disallow: /album.php
  112. Disallow: /checkpoint/
  113. Disallow: /contact_importer/
  114. Disallow: /dialog/
  115. Disallow: /fbml/ajax/dialog/
  116. Disallow: /feeds/
  117. Disallow: /file_download.php
  118. Disallow: /hashtag/
  119. Disallow: /l.php
  120. Disallow: /live/
  121. Disallow: /moments_app/
  122. Disallow: /p.php
  123. Disallow: /photo.php
  124. Disallow: /photos.php
  125. Disallow: /share.php
  126. Disallow: /share/
  127. Disallow: /sharer.php
  128. Disallow: /sharer/
  129.  
  130. User-agent: msnbot
  131. Disallow: /ajax/
  132. Disallow: /album.php
  133. Disallow: /checkpoint/
  134. Disallow: /contact_importer/
  135. Disallow: /dialog/
  136. Disallow: /fbml/ajax/dialog/
  137. Disallow: /feeds/
  138. Disallow: /file_download.php
  139. Disallow: /hashtag/
  140. Disallow: /l.php
  141. Disallow: /live/
  142. Disallow: /moments_app/
  143. Disallow: /p.php
  144. Disallow: /photo.php
  145. Disallow: /photos.php
  146. Disallow: /share.php
  147. Disallow: /share/
  148. Disallow: /sharer.php
  149. Disallow: /sharer/
  150.  
  151. User-agent: Naverbot
  152. Disallow: /ajax/
  153. Disallow: /album.php
  154. Disallow: /checkpoint/
  155. Disallow: /contact_importer/
  156. Disallow: /dialog/
  157. Disallow: /fbml/ajax/dialog/
  158. Disallow: /feeds/
  159. Disallow: /file_download.php
  160. Disallow: /hashtag/
  161. Disallow: /l.php
  162. Disallow: /live/
  163. Disallow: /moments_app/
  164. Disallow: /p.php
  165. Disallow: /photo.php
  166. Disallow: /photos.php
  167. Disallow: /share.php
  168. Disallow: /share/
  169. Disallow: /sharer.php
  170. Disallow: /sharer/
  171.  
  172. User-agent: seznambot
  173. Disallow: /ajax/
  174. Disallow: /album.php
  175. Disallow: /checkpoint/
  176. Disallow: /contact_importer/
  177. Disallow: /dialog/
  178. Disallow: /fbml/ajax/dialog/
  179. Disallow: /feeds/
  180. Disallow: /file_download.php
  181. Disallow: /hashtag/
  182. Disallow: /l.php
  183. Disallow: /live/
  184. Disallow: /moments_app/
  185. Disallow: /p.php
  186. Disallow: /photo.php
  187. Disallow: /photos.php
  188. Disallow: /share.php
  189. Disallow: /share/
  190. Disallow: /sharer.php
  191. Disallow: /sharer/
  192.  
  193. User-agent: Slurp
  194. Disallow: /ajax/
  195. Disallow: /album.php
  196. Disallow: /checkpoint/
  197. Disallow: /contact_importer/
  198. Disallow: /dialog/
  199. Disallow: /fbml/ajax/dialog/
  200. Disallow: /feeds/
  201. Disallow: /file_download.php
  202. Disallow: /hashtag/
  203. Disallow: /l.php
  204. Disallow: /live/
  205. Disallow: /moments_app/
  206. Disallow: /p.php
  207. Disallow: /photo.php
  208. Disallow: /photos.php
  209. Disallow: /share.php
  210. Disallow: /share/
  211. Disallow: /sharer.php
  212. Disallow: /sharer/
  213.  
  214. User-agent: teoma
  215. Disallow: /ajax/
  216. Disallow: /album.php
  217. Disallow: /checkpoint/
  218. Disallow: /contact_importer/
  219. Disallow: /dialog/
  220. Disallow: /fbml/ajax/dialog/
  221. Disallow: /feeds/
  222. Disallow: /file_download.php
  223. Disallow: /hashtag/
  224. Disallow: /l.php
  225. Disallow: /live/
  226. Disallow: /moments_app/
  227. Disallow: /p.php
  228. Disallow: /photo.php
  229. Disallow: /photos.php
  230. Disallow: /share.php
  231. Disallow: /share/
  232. Disallow: /sharer.php
  233. Disallow: /sharer/
  234.  
  235. User-agent: Twitterbot
  236. Disallow: /ajax/
  237. Disallow: /album.php
  238. Disallow: /checkpoint/
  239. Disallow: /contact_importer/
  240. Disallow: /dialog/
  241. Disallow: /fbml/ajax/dialog/
  242. Disallow: /feeds/
  243. Disallow: /file_download.php
  244. Disallow: /hashtag/
  245. Disallow: /l.php
  246. Disallow: /live/
  247. Disallow: /moments_app/
  248. Disallow: /p.php
  249. Disallow: /photo.php
  250. Disallow: /photos.php
  251. Disallow: /share.php
  252. Disallow: /share/
  253. Disallow: /sharer.php
  254. Disallow: /sharer/
  255.  
  256. User-agent: Yandex
  257. Disallow: /ajax/
  258. Disallow: /album.php
  259. Disallow: /checkpoint/
  260. Disallow: /contact_importer/
  261. Disallow: /dialog/
  262. Disallow: /fbml/ajax/dialog/
  263. Disallow: /feeds/
  264. Disallow: /file_download.php
  265. Disallow: /hashtag/
  266. Disallow: /l.php
  267. Disallow: /live/
  268. Disallow: /moments_app/
  269. Disallow: /p.php
  270. Disallow: /photo.php
  271. Disallow: /photos.php
  272. Disallow: /share.php
  273. Disallow: /share/
  274. Disallow: /sharer.php
  275. Disallow: /sharer/
  276.  
  277. User-agent: Yeti
  278. Disallow: /ajax/
  279. Disallow: /album.php
  280. Disallow: /checkpoint/
  281. Disallow: /contact_importer/
  282. Disallow: /dialog/
  283. Disallow: /fbml/ajax/dialog/
  284. Disallow: /feeds/
  285. Disallow: /file_download.php
  286. Disallow: /hashtag/
  287. Disallow: /l.php
  288. Disallow: /live/
  289. Disallow: /moments_app/
  290. Disallow: /p.php
  291. Disallow: /photo.php
  292. Disallow: /photos.php
  293. Disallow: /share.php
  294. Disallow: /share/
  295. Disallow: /sharer.php
  296. Disallow: /sharer/
  297.  
  298. User-agent: Applebot
  299. Allow: /ajax/bootloader-endpoint/
  300. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  301. Allow: /safetycheck/
  302.  
  303. User-agent: baiduspider
  304. Allow: /ajax/bootloader-endpoint/
  305. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  306. Allow: /safetycheck/
  307.  
  308. User-agent: Bingbot
  309. Allow: /ajax/bootloader-endpoint/
  310. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  311. Allow: /safetycheck/
  312.  
  313. User-agent: Googlebot
  314. Allow: /ajax/bootloader-endpoint/
  315. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  316. Allow: /safetycheck/
  317.  
  318. User-agent: ia_archiver
  319. Allow: /about/privacy
  320. Allow: /ajax/bootloader-endpoint/
  321. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  322. Allow: /full_data_use_policy
  323. Allow: /legal/terms
  324. Allow: /policy.php
  325. Allow: /safetycheck/
  326.  
  327. User-agent: msnbot
  328. Allow: /ajax/bootloader-endpoint/
  329. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  330. Allow: /safetycheck/
  331.  
  332. User-agent: Naverbot
  333. Allow: /ajax/bootloader-endpoint/
  334. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  335. Allow: /safetycheck/
  336.  
  337. User-agent: seznambot
  338. Allow: /ajax/bootloader-endpoint/
  339. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  340. Allow: /safetycheck/
  341.  
  342. User-agent: Slurp
  343. Allow: /ajax/bootloader-endpoint/
  344. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  345. Allow: /safetycheck/
  346.  
  347. User-agent: teoma
  348. Allow: /ajax/bootloader-endpoint/
  349. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  350. Allow: /safetycheck/
  351.  
  352. User-agent: Twitterbot
  353. Allow: /ajax/bootloader-endpoint/
  354. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  355. Allow: /safetycheck/
  356.  
  357. User-agent: Yandex
  358. Allow: /ajax/bootloader-endpoint/
  359. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  360. Allow: /safetycheck/
  361.  
  362. User-agent: Yeti
  363. Allow: /ajax/bootloader-endpoint/
  364. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  365. Allow: /safetycheck/
  366.  
  367. User-agent: *
  368. Disallow: /
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!