API tools faq
paste
Advertisement
nu11secur1ty

facebook_robots.txt

nu11secur1ty
Apr 25th, 2021
1,545
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Bash 11.76 KB | None | 0 0
raw download clone embed print report
  1. # Notice: Collection of data on Facebook through automated means is
  2. # prohibited unless you have express written permission from Facebook
  3. # and may only be conducted for the limited purpose contained in said
  4. # permission.
  5. # See: http://www.facebook.com/apps/site_scraping_tos_terms.php
  6.  
  7. User-agent: Applebot
  8. Disallow: /ajax/
  9. Disallow: /album.php
  10. Disallow: /checkpoint/
  11. Disallow: /contact_importer/
  12. Disallow: /dialog/
  13. Disallow: /fbml/ajax/dialog/
  14. Disallow: /feeds/
  15. Disallow: /file_download.php
  16. Disallow: /hashtag/
  17. Disallow: /job_application/
  18. Disallow: /l.php
  19. Disallow: /moments_app/
  20. Disallow: /p.php
  21. Disallow: /photo.php
  22. Disallow: /photos.php
  23. Disallow: /share.php
  24. Disallow: /share/
  25. Disallow: /sharer.php
  26. Disallow: /sharer/
  27.  
  28. User-agent: baiduspider
  29. Disallow: /ajax/
  30. Disallow: /album.php
  31. Disallow: /checkpoint/
  32. Disallow: /contact_importer/
  33. Disallow: /dialog/
  34. Disallow: /fbml/ajax/dialog/
  35. Disallow: /feeds/
  36. Disallow: /file_download.php
  37. Disallow: /hashtag/
  38. Disallow: /job_application/
  39. Disallow: /l.php
  40. Disallow: /moments_app/
  41. Disallow: /p.php
  42. Disallow: /photo.php
  43. Disallow: /photos.php
  44. Disallow: /share.php
  45. Disallow: /share/
  46. Disallow: /sharer.php
  47. Disallow: /sharer/
  48.  
  49. User-agent: Bingbot
  50. Disallow: /ajax/
  51. Disallow: /album.php
  52. Disallow: /checkpoint/
  53. Disallow: /contact_importer/
  54. Disallow: /dialog/
  55. Disallow: /fbml/ajax/dialog/
  56. Disallow: /feeds/
  57. Disallow: /file_download.php
  58. Disallow: /hashtag/
  59. Disallow: /job_application/
  60. Disallow: /l.php
  61. Disallow: /moments_app/
  62. Disallow: /p.php
  63. Disallow: /photo.php
  64. Disallow: /photos.php
  65. Disallow: /share.php
  66. Disallow: /share/
  67. Disallow: /sharer.php
  68. Disallow: /sharer/
  69.  
  70. User-agent: Discordbot
  71. Disallow: /
  72. Disallow: /ajax/
  73. Disallow: /album.php
  74. Disallow: /checkpoint/
  75. Disallow: /contact_importer/
  76. Disallow: /dialog/
  77. Disallow: /fbml/ajax/dialog/
  78. Disallow: /feeds/
  79. Disallow: /file_download.php
  80. Disallow: /hashtag/
  81. Disallow: /job_application/
  82. Disallow: /l.php
  83. Disallow: /moments_app/
  84. Disallow: /p.php
  85. Disallow: /photo.php
  86. Disallow: /photos.php
  87. Disallow: /share.php
  88. Disallow: /share/
  89. Disallow: /sharer.php
  90. Disallow: /sharer/
  91.  
  92. User-agent: facebookexternalhit
  93. Disallow: /
  94. Disallow: /ajax/
  95. Disallow: /album.php
  96. Disallow: /checkpoint/
  97. Disallow: /contact_importer/
  98. Disallow: /dialog/
  99. Disallow: /fbml/ajax/dialog/
  100. Disallow: /feeds/
  101. Disallow: /file_download.php
  102. Disallow: /hashtag/
  103. Disallow: /job_application/
  104. Disallow: /l.php
  105. Disallow: /moments_app/
  106. Disallow: /p.php
  107. Disallow: /photo.php
  108. Disallow: /photos.php
  109. Disallow: /share.php
  110. Disallow: /share/
  111. Disallow: /sharer.php
  112. Disallow: /sharer/
  113.  
  114. User-agent: Googlebot
  115. Disallow: /ajax/
  116. Disallow: /album.php
  117. Disallow: /checkpoint/
  118. Disallow: /contact_importer/
  119. Disallow: /dialog/
  120. Disallow: /fbml/ajax/dialog/
  121. Disallow: /feeds/
  122. Disallow: /file_download.php
  123. Disallow: /hashtag/
  124. Disallow: /job_application/
  125. Disallow: /l.php
  126. Disallow: /moments_app/
  127. Disallow: /p.php
  128. Disallow: /photo.php
  129. Disallow: /photos.php
  130. Disallow: /share.php
  131. Disallow: /share/
  132. Disallow: /sharer.php
  133. Disallow: /sharer/
  134.  
  135. User-agent: Googlebot-Image
  136. Disallow: /ajax/
  137. Disallow: /album.php
  138. Disallow: /checkpoint/
  139. Disallow: /contact_importer/
  140. Disallow: /dialog/
  141. Disallow: /fbml/ajax/dialog/
  142. Disallow: /feeds/
  143. Disallow: /file_download.php
  144. Disallow: /hashtag/
  145. Disallow: /job_application/
  146. Disallow: /l.php
  147. Disallow: /map_tile.php
  148. Disallow: /moments_app/
  149. Disallow: /p.php
  150. Disallow: /photo.php
  151. Disallow: /photos.php
  152. Disallow: /share.php
  153. Disallow: /share/
  154. Disallow: /sharer.php
  155. Disallow: /sharer/
  156. Disallow: /static_map.php
  157.  
  158. User-agent: ia_archiver
  159. Disallow: /
  160. Disallow: /ajax/
  161. Disallow: /album.php
  162. Disallow: /checkpoint/
  163. Disallow: /contact_importer/
  164. Disallow: /dialog/
  165. Disallow: /fbml/ajax/dialog/
  166. Disallow: /feeds/
  167. Disallow: /file_download.php
  168. Disallow: /hashtag/
  169. Disallow: /job_application/
  170. Disallow: /l.php
  171. Disallow: /moments_app/
  172. Disallow: /p.php
  173. Disallow: /photo.php
  174. Disallow: /photos.php
  175. Disallow: /share.php
  176. Disallow: /share/
  177. Disallow: /sharer.php
  178. Disallow: /sharer/
  179.  
  180. User-agent: LinkedInBot
  181. Disallow: /
  182. Disallow: /ajax/
  183. Disallow: /album.php
  184. Disallow: /checkpoint/
  185. Disallow: /contact_importer/
  186. Disallow: /dialog/
  187. Disallow: /fbml/ajax/dialog/
  188. Disallow: /feeds/
  189. Disallow: /file_download.php
  190. Disallow: /hashtag/
  191. Disallow: /job_application/
  192. Disallow: /l.php
  193. Disallow: /moments_app/
  194. Disallow: /p.php
  195. Disallow: /photo.php
  196. Disallow: /photos.php
  197. Disallow: /share.php
  198. Disallow: /share/
  199. Disallow: /sharer.php
  200. Disallow: /sharer/
  201.  
  202. User-agent: msnbot
  203. Disallow: /ajax/
  204. Disallow: /album.php
  205. Disallow: /checkpoint/
  206. Disallow: /contact_importer/
  207. Disallow: /dialog/
  208. Disallow: /fbml/ajax/dialog/
  209. Disallow: /feeds/
  210. Disallow: /file_download.php
  211. Disallow: /hashtag/
  212. Disallow: /job_application/
  213. Disallow: /l.php
  214. Disallow: /moments_app/
  215. Disallow: /p.php
  216. Disallow: /photo.php
  217. Disallow: /photos.php
  218. Disallow: /share.php
  219. Disallow: /share/
  220. Disallow: /sharer.php
  221. Disallow: /sharer/
  222.  
  223. User-agent: Naverbot
  224. Disallow: /ajax/
  225. Disallow: /album.php
  226. Disallow: /checkpoint/
  227. Disallow: /contact_importer/
  228. Disallow: /dialog/
  229. Disallow: /fbml/ajax/dialog/
  230. Disallow: /feeds/
  231. Disallow: /file_download.php
  232. Disallow: /hashtag/
  233. Disallow: /job_application/
  234. Disallow: /l.php
  235. Disallow: /moments_app/
  236. Disallow: /p.php
  237. Disallow: /photo.php
  238. Disallow: /photos.php
  239. Disallow: /share.php
  240. Disallow: /share/
  241. Disallow: /sharer.php
  242. Disallow: /sharer/
  243.  
  244. User-agent: Pinterestbot
  245. Disallow: /
  246. Disallow: /ajax/
  247. Disallow: /album.php
  248. Disallow: /checkpoint/
  249. Disallow: /contact_importer/
  250. Disallow: /dialog/
  251. Disallow: /fbml/ajax/dialog/
  252. Disallow: /feeds/
  253. Disallow: /file_download.php
  254. Disallow: /hashtag/
  255. Disallow: /job_application/
  256. Disallow: /l.php
  257. Disallow: /moments_app/
  258. Disallow: /p.php
  259. Disallow: /photo.php
  260. Disallow: /photos.php
  261. Disallow: /share.php
  262. Disallow: /share/
  263. Disallow: /sharer.php
  264. Disallow: /sharer/
  265.  
  266. User-agent: seznambot
  267. Disallow: /ajax/
  268. Disallow: /album.php
  269. Disallow: /checkpoint/
  270. Disallow: /contact_importer/
  271. Disallow: /dialog/
  272. Disallow: /fbml/ajax/dialog/
  273. Disallow: /feeds/
  274. Disallow: /file_download.php
  275. Disallow: /hashtag/
  276. Disallow: /job_application/
  277. Disallow: /l.php
  278. Disallow: /moments_app/
  279. Disallow: /p.php
  280. Disallow: /photo.php
  281. Disallow: /photos.php
  282. Disallow: /share.php
  283. Disallow: /share/
  284. Disallow: /sharer.php
  285. Disallow: /sharer/
  286.  
  287. User-agent: Slurp
  288. Disallow: /ajax/
  289. Disallow: /album.php
  290. Disallow: /checkpoint/
  291. Disallow: /contact_importer/
  292. Disallow: /dialog/
  293. Disallow: /fbml/ajax/dialog/
  294. Disallow: /feeds/
  295. Disallow: /file_download.php
  296. Disallow: /hashtag/
  297. Disallow: /job_application/
  298. Disallow: /l.php
  299. Disallow: /moments_app/
  300. Disallow: /p.php
  301. Disallow: /photo.php
  302. Disallow: /photos.php
  303. Disallow: /share.php
  304. Disallow: /share/
  305. Disallow: /sharer.php
  306. Disallow: /sharer/
  307.  
  308. User-agent: teoma
  309. Disallow: /ajax/
  310. Disallow: /album.php
  311. Disallow: /checkpoint/
  312. Disallow: /contact_importer/
  313. Disallow: /dialog/
  314. Disallow: /fbml/ajax/dialog/
  315. Disallow: /feeds/
  316. Disallow: /file_download.php
  317. Disallow: /hashtag/
  318. Disallow: /job_application/
  319. Disallow: /l.php
  320. Disallow: /moments_app/
  321. Disallow: /p.php
  322. Disallow: /photo.php
  323. Disallow: /photos.php
  324. Disallow: /share.php
  325. Disallow: /share/
  326. Disallow: /sharer.php
  327. Disallow: /sharer/
  328.  
  329. User-agent: TelegramBot
  330. Disallow: /
  331. Disallow: /ajax/
  332. Disallow: /album.php
  333. Disallow: /checkpoint/
  334. Disallow: /contact_importer/
  335. Disallow: /dialog/
  336. Disallow: /fbml/ajax/dialog/
  337. Disallow: /feeds/
  338. Disallow: /file_download.php
  339. Disallow: /hashtag/
  340. Disallow: /job_application/
  341. Disallow: /l.php
  342. Disallow: /moments_app/
  343. Disallow: /p.php
  344. Disallow: /photo.php
  345. Disallow: /photos.php
  346. Disallow: /share.php
  347. Disallow: /share/
  348. Disallow: /sharer.php
  349. Disallow: /sharer/
  350.  
  351. User-agent: Twitterbot
  352. Disallow: /ajax/
  353. Disallow: /album.php
  354. Disallow: /checkpoint/
  355. Disallow: /contact_importer/
  356. Disallow: /dialog/
  357. Disallow: /fbml/ajax/dialog/
  358. Disallow: /feeds/
  359. Disallow: /file_download.php
  360. Disallow: /hashtag/
  361. Disallow: /job_application/
  362. Disallow: /l.php
  363. Disallow: /moments_app/
  364. Disallow: /p.php
  365. Disallow: /photo.php
  366. Disallow: /photos.php
  367. Disallow: /share.php
  368. Disallow: /share/
  369. Disallow: /sharer.php
  370. Disallow: /sharer/
  371.  
  372. User-agent: Yandex
  373. Disallow: /ajax/
  374. Disallow: /album.php
  375. Disallow: /checkpoint/
  376. Disallow: /contact_importer/
  377. Disallow: /dialog/
  378. Disallow: /fbml/ajax/dialog/
  379. Disallow: /feeds/
  380. Disallow: /file_download.php
  381. Disallow: /hashtag/
  382. Disallow: /job_application/
  383. Disallow: /l.php
  384. Disallow: /moments_app/
  385. Disallow: /p.php
  386. Disallow: /photo.php
  387. Disallow: /photos.php
  388. Disallow: /share.php
  389. Disallow: /share/
  390. Disallow: /sharer.php
  391. Disallow: /sharer/
  392.  
  393. User-agent: Yeti
  394. Disallow: /ajax/
  395. Disallow: /album.php
  396. Disallow: /checkpoint/
  397. Disallow: /contact_importer/
  398. Disallow: /dialog/
  399. Disallow: /fbml/ajax/dialog/
  400. Disallow: /feeds/
  401. Disallow: /file_download.php
  402. Disallow: /hashtag/
  403. Disallow: /job_application/
  404. Disallow: /l.php
  405. Disallow: /moments_app/
  406. Disallow: /p.php
  407. Disallow: /photo.php
  408. Disallow: /photos.php
  409. Disallow: /share.php
  410. Disallow: /share/
  411. Disallow: /sharer.php
  412. Disallow: /sharer/
  413.  
  414. User-agent: Applebot
  415. Allow: /ajax/bootloader-endpoint/
  416. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  417. Allow: /safetycheck/
  418.  
  419. User-agent: baiduspider
  420. Allow: /ajax/bootloader-endpoint/
  421. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  422. Allow: /safetycheck/
  423.  
  424. User-agent: Bingbot
  425. Allow: /*/videos/
  426. Allow: /ajax/bootloader-endpoint/
  427. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  428. Allow: /safetycheck/
  429. Allow: /watch
  430.  
  431. User-agent: Discordbot
  432. Allow: /*/videos/
  433. Allow: /ajax/bootloader-endpoint/
  434. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  435. Allow: /safetycheck/
  436. Allow: /watch/?v=*
  437.  
  438. User-agent: facebookexternalhit
  439. Allow: /*/videos/
  440. Allow: /ajax/bootloader-endpoint/
  441. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  442. Allow: /safetycheck/
  443. Allow: /watch/?v=*
  444.  
  445. User-agent: Googlebot
  446. Allow: /*/videos/
  447. Allow: /ajax/bootloader-endpoint/
  448. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  449. Allow: /safetycheck/
  450. Allow: /watch
  451.  
  452. User-agent: ia_archiver
  453. Allow: /about/privacy
  454. Allow: /ajax/bootloader-endpoint/
  455. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  456. Allow: /full_data_use_policy
  457. Allow: /legal/terms
  458. Allow: /policy.php
  459. Allow: /safetycheck/
  460.  
  461. User-agent: LinkedInBot
  462. Allow: /*/videos/
  463. Allow: /ajax/bootloader-endpoint/
  464. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  465. Allow: /safetycheck/
  466. Allow: /watch/?v=*
  467.  
  468. User-agent: msnbot
  469. Allow: /ajax/bootloader-endpoint/
  470. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  471. Allow: /safetycheck/
  472.  
  473. User-agent: Naverbot
  474. Allow: /ajax/bootloader-endpoint/
  475. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  476. Allow: /safetycheck/
  477.  
  478. User-agent: Pinterestbot
  479. Allow: /*/videos/
  480. Allow: /ajax/bootloader-endpoint/
  481. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  482. Allow: /safetycheck/
  483. Allow: /watch/?v=*
  484.  
  485. User-agent: seznambot
  486. Allow: /ajax/bootloader-endpoint/
  487. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  488. Allow: /safetycheck/
  489.  
  490. User-agent: Slurp
  491. Allow: /ajax/bootloader-endpoint/
  492. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  493. Allow: /safetycheck/
  494.  
  495. User-agent: teoma
  496. Allow: /ajax/bootloader-endpoint/
  497. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  498. Allow: /safetycheck/
  499.  
  500. User-agent: TelegramBot
  501. Allow: /*/videos/
  502. Allow: /ajax/bootloader-endpoint/
  503. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  504. Allow: /safetycheck/
  505. Allow: /watch/?v=*
  506.  
  507. User-agent: Twitterbot
  508. Allow: /ajax/bootloader-endpoint/
  509. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  510. Allow: /safetycheck/
  511.  
  512. User-agent: Yandex
  513. Allow: /ajax/bootloader-endpoint/
  514. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  515. Allow: /safetycheck/
  516.  
  517. User-agent: Yeti
  518. Allow: /ajax/bootloader-endpoint/
  519. Allow: /ajax/pagelet/generic.php/PagePostsSectionPagelet
  520. Allow: /safetycheck/
  521.  
  522. User-agent: *
  523. Disallow: /
  524.  
  525.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!