SHARE
TWEET

robots.txt

brinsleyparagao Feb 14th, 2020 58 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. Sitemap: https://DOMINIO.com.br/sitemap_index.xml
  2.  
  3. User-agent: *
  4. Allow: /wp-admin/admin-ajax.php
  5. Allow: /*/*.css
  6. Allow: /*/*.js
  7. Disallow: /wp-admin/
  8. Disallow: /wp-includes/
  9. Disallow: /readme.html
  10. Disallow: /license.txt
  11. Disallow: /xmlrpc.php
  12. Disallow: /wp-login.php
  13. Disallow: /wp-register.php
  14. Disallow: */disclaimer/*
  15. Disallow: *?attachment_id=
  16. Disallow: /privacy-policy
  17.  
  18. User-agent: Googlebot
  19. Allow: /
  20.  
  21. User-agent: Googlebot-Image
  22. Allow: /wp-content/uploads/
  23.  
  24. User-agent: Mediapartners-Google
  25. Allow: /
  26.  
  27. User-agent: AdsBot-Google
  28. Allow: /
  29.  
  30. User-agent: AdsBot-Google-Mobile
  31. Allow: /
  32.  
  33. User-agent: Bingbot
  34. Allow: /
  35.  
  36. User-agent: Msnbot
  37. Allow: /
  38.  
  39. User-agent: msnbot-media
  40. Allow: /wp-content/uploads/
  41.  
  42. User-agent: Applebot
  43. Allow: /
  44.  
  45. User-agent: Yandex
  46. Disallow: /
  47.  
  48. User-agent: YandexImages
  49. Disallow: /wp-content/uploads/
  50.  
  51. User-agent: Slurp
  52. Allow: /
  53.  
  54. User-agent: DuckDuckBot
  55. Allow: /
  56.  
  57. User-agent: Qwantify
  58. Disallow: /
  59.  
  60. # Escrotos da china
  61.  
  62. User-agent: Baiduspider
  63. Disallow: /
  64. User-agent: Baiduspider/2.0
  65. Disallow: /
  66. User-agent: Baiduspider-video
  67. Disallow: /
  68. User-agent: Baiduspider-image
  69. Disallow: /
  70. User-agent: Sogou spider
  71. Disallow: /
  72. User-agent: Sogou web spider
  73. Disallow: /
  74. User-agent: Sosospider
  75. Disallow: /
  76. User-agent: Sosospider+
  77. Disallow: /
  78. User-agent: Sosospider/2.0
  79. Disallow: /
  80. User-agent: yodao
  81. Disallow: /
  82. User-agent: youdao
  83. Disallow: /
  84. User-agent: YoudaoBot
  85. Disallow: /
  86. User-agent: YoudaoBot/1.0
  87. Disallow: /
  88.  
  89. # Fazedores de spam nos backlink
  90.  
  91. Disallow: /feed/
  92. Disallow: /feed/$
  93. Disallow: /comments/feed
  94. Disallow: /trackback/
  95. Disallow: */?author=*
  96. Disallow: */author/*
  97. Disallow: /author*
  98. Disallow: /author/
  99. Disallow: */comments$
  100. Disallow: */feed
  101. Disallow: */feed$
  102. Disallow: */trackback
  103. Disallow: */trackback$
  104. Disallow: /?feed=
  105. Disallow: /wp-comments
  106. Disallow: /wp-feed
  107. Disallow: /wp-trackback
  108. Disallow: */replytocom=
  109.  
  110. # Bloquear bots de spam
  111.  
  112. User-agent: DotBot
  113. Disallow: /
  114.  
  115. User-agent: GiftGhostBot
  116. Disallow: /
  117.  
  118. User-agent: Seznam
  119. Disallow: /
  120.  
  121. User-agent: PaperLiBot
  122. Disallow: /
  123.  
  124. User-agent: Genieo
  125. Disallow: /
  126.  
  127. User-agent: Dataprovider/6.101
  128. Disallow: /
  129.  
  130. User-agent: DataproviderSiteExplorer
  131. Disallow: /
  132.  
  133. User-agent: Dazoobot/1.0
  134. Disallow: /
  135.  
  136. User-agent: Diffbot
  137. Disallow: /
  138.  
  139. User-agent: DomainStatsBot/1.0
  140. Disallow: /
  141.  
  142. User-agent: DotBot/1.1
  143. Disallow: /
  144.  
  145. User-agent: dubaiindex
  146. Disallow: /
  147.  
  148. User-agent: eCommerceBot
  149. Disallow: /
  150.  
  151. User-agent: ExpertSearchSpider
  152. Disallow: /
  153.  
  154. User-agent: Feedbin
  155. Disallow: /
  156.  
  157. User-agent: Fetch/2.0a
  158. Disallow: /
  159.  
  160. User-agent: FFbot/1.0
  161. Disallow: /
  162.  
  163. User-agent: focusbot/1.1
  164. Disallow: /
  165.  
  166. User-agent: HuaweiSymantecSpider
  167. Disallow: /
  168.  
  169. User-agent: HuaweiSymantecSpider/1.0
  170. Disallow: /
  171.  
  172. User-agent: JobdiggerSpider
  173. Disallow: /
  174.  
  175. User-agent: LemurWebCrawler
  176. Disallow: /
  177.  
  178. User-agent: LipperheyLinkExplorer
  179. Disallow: /
  180.  
  181. User-agent: LSSRocketCrawler/1.0
  182. Disallow: /
  183.  
  184. User-agent: LYT.SRv1.5
  185. Disallow: /
  186.  
  187. User-agent: MiaDev/0.0.1
  188. Disallow: /
  189.  
  190. User-agent: Najdi.si/3.1
  191. Disallow: /
  192.  
  193. User-agent: BountiiBot
  194. Disallow: /
  195.  
  196. User-agent: Experibot_v1
  197. Disallow: /
  198.  
  199. User-agent: bixocrawler
  200. Disallow: /
  201.  
  202. User-agent: bixocrawler TestCrawler
  203. Disallow: /
  204.  
  205. User-agent: Crawler4j
  206. Disallow: /
  207.  
  208. User-agent: Crowsnest/0.5
  209. Disallow: /
  210.  
  211. User-agent: CukBot
  212. Disallow: /
  213.  
  214. User-agent: Dataprovider/6.92
  215. Disallow: /
  216.  
  217. User-agent: DBLBot/1.0
  218. Disallow: /
  219.  
  220. User-agent: Diffbot/0.1
  221. Disallow: /
  222.  
  223. User-agent: Digg Deeper/v1
  224. Disallow: /
  225.  
  226. User-agent: discobot/1.0
  227. Disallow: /
  228.  
  229. User-agent: discobot/1.1
  230. Disallow: /
  231.  
  232. User-agent: discobot/2.0
  233. Disallow: /
  234.  
  235. User-agent: discoverybot/2.0
  236. Disallow: /
  237.  
  238. User-agent: Dlvr.it/1.0
  239. Disallow: /
  240.  
  241. User-agent: DomainStatsBot/1.0
  242. Disallow: /
  243.  
  244. User-agent: drupact/0.7
  245. Disallow: /
  246.  
  247. User-agent: Ezooms/1.0  
  248. Disallow: /
  249.  
  250. User-agent: fastbot crawler beta 2.0  
  251. Disallow: /
  252.  
  253. User-agent: fastbot crawler beta 4.0  
  254. Disallow: /
  255.  
  256. User-agent: feedly social
  257. Disallow: /
  258.  
  259. User-agent: Feedly/1.0  
  260. Disallow: /
  261.  
  262. User-agent: FeedlyBot/1.0  
  263. Disallow: /
  264.  
  265. User-agent: Feedspot  
  266. Disallow: /
  267.  
  268. User-agent: Feedspotbot/1.0
  269. Disallow: /
  270.  
  271. User-agent: Clickagy Intelligence Bot v2
  272. Disallow: /
  273.  
  274. User-agent: classbot
  275. Disallow: /
  276.  
  277. User-agent: CISPA Vulnerability Notification
  278. Disallow: /
  279.  
  280. User-agent: CirrusExplorer/1.1
  281. Disallow: /
  282.  
  283. User-agent: Checksem/Nutch-1.10
  284. Disallow: /
  285.  
  286. User-agent: CatchBot/5.0
  287. Disallow: /
  288.  
  289. User-agent: CatchBot/3.0
  290. Disallow: /
  291.  
  292. User-agent: CatchBot/2.0
  293. Disallow: /
  294.  
  295. User-agent: CatchBot/1.0
  296. Disallow: /
  297.  
  298. User-agent: CamontSpider/1.0
  299. Disallow: /
  300.  
  301. User-agent: Buzzbot/1.0
  302. Disallow: /
  303.  
  304. User-agent: Buzzbot
  305. Disallow: /
  306.  
  307. User-agent: BusinessSeek.biz_Spider
  308. Disallow: /
  309.  
  310. User-agent: BUbiNG
  311. Disallow: /
  312.  
  313. User-agent: 008/0.85
  314. Disallow: /
  315.  
  316. User-agent: 008/0.83
  317. Disallow: /
  318.  
  319. User-agent: 008/0.71
  320. Disallow: /
  321.  
  322. User-agent: ^Nail
  323. Disallow: /
  324.  
  325. User-agent: FyberSpider/1.3
  326. Disallow: /
  327.  
  328. User-agent: findlinks/1.1.6-beta5
  329. Disallow: /
  330.  
  331. User-agent: g2reader-bot/1.0
  332. Disallow: /
  333.  
  334. User-agent: findlinks/1.1.6-beta6
  335. Disallow: /
  336.  
  337. User-agent: findlinks/2.0
  338. Disallow: /
  339.  
  340. User-agent: findlinks/2.0.1
  341. Disallow: /
  342.  
  343. User-agent: findlinks/2.0.2
  344. Disallow: /
  345.  
  346. User-agent: findlinks/2.0.4
  347. Disallow: /
  348.  
  349. User-agent: findlinks/2.0.5
  350. Disallow: /
  351.  
  352. User-agent: findlinks/2.0.9
  353. Disallow: /
  354.  
  355. User-agent: findlinks/2.1
  356. Disallow: /
  357.  
  358. User-agent: findlinks/2.1.5
  359. Disallow: /
  360.  
  361. User-agent: findlinks/2.1.3
  362. Disallow: /
  363.  
  364. User-agent: findlinks/2.2
  365. Disallow: /
  366.  
  367. User-agent: findlinks/2.5
  368. Disallow: /
  369.  
  370. User-agent: findlinks/2.6
  371. Disallow: /
  372.  
  373. User-agent: FFbot/1.0
  374. Disallow: /
  375.  
  376. User-agent: findlinks/1.0
  377. Disallow: /
  378.  
  379. User-agent: findlinks/1.1.3-beta8
  380. Disallow: /
  381.  
  382. User-agent: findlinks/1.1.3-beta9
  383. Disallow: /
  384.  
  385. User-agent: findlinks/1.1.4-beta7
  386. Disallow: /
  387.  
  388. User-agent: findlinks/1.1.6-beta1
  389. Disallow: /
  390.  
  391. User-agent: findlinks/1.1.6-beta1 Yacy
  392. Disallow: /
  393.  
  394. User-agent: findlinks/1.1.6-beta2
  395. Disallow: /
  396.  
  397. User-agent: findlinks/1.1.6-beta3
  398. Disallow: /
  399.  
  400. User-agent: findlinks/1.1.6-beta4
  401. Disallow: /
  402.  
  403. User-agent: bixo
  404. Disallow: /
  405.  
  406. User-agent: bixolabs/1.0
  407. Disallow: /
  408.  
  409. User-agent: Crawlera/1.10.2
  410. Disallow: /
  411.  
  412. User-agent: Dataprovider Site Explorer
  413. Disallow: /
  414.  
  415. # Backlink Protector. Powered by Better Robots.txt Pro
  416.  
  417. User-agent: AhrefsBot
  418. Disallow: /
  419.  
  420. User-agent: Alexibot
  421. Disallow: /
  422.  
  423. User-agent: MJ12bot
  424. Disallow: /
  425.  
  426. User-agent: SurveyBot
  427. Disallow: /
  428.  
  429. User-agent: Xenu's
  430. Disallow: /
  431.  
  432. User-agent: Xenu's Link Sleuth 1.1c
  433. Disallow: /
  434.  
  435. User-agent: rogerbot
  436. Disallow: /
  437.  
  438. User-agent: SemrushBot
  439. Disallow: /
  440.  
  441. User-agent: SemrushBot-SA
  442. Disallow: /
  443.  
  444. User-agent: SemrushBot-BA
  445. Disallow: /
  446.  
  447. User-agent: SemrushBot-SI
  448. Disallow: /
  449.  
  450. User-agent: SemrushBot-SWA
  451. Disallow: /
  452.  
  453. User-agent: SemrushBot-CT
  454. Disallow: /
  455.  
  456. User-agent: SemrushBot-BM
  457. Disallow: /
  458.  
  459. # Prevenir contra armadilhas do mundo cruel
  460.  
  461. Disallow: /search/
  462. Disallow: *?s=*
  463. Disallow: *?p=*
  464. Disallow: *&p=*
  465. Disallow: *&preview=*
  466. Disallow: /search
  467.  
  468. # Liberar pras rede social, zuquiberqui eh foda
  469.  
  470. User-agent: facebookexternalhit/1.0
  471. Allow: /
  472. User-agent: facebookexternalhit/1.1
  473. Allow: /
  474. User-agent: facebookplatform/1.0
  475. Allow: /
  476. User-agent: Facebot/1.0
  477. Allow: /
  478. User-agent: Visionutils/0.2
  479. Allow: /
  480. User-agent: datagnionbot
  481. Allow: /
  482.  
  483. # Liberar pras rede social
  484.  
  485. User-agent: Twitterbot
  486. Allow: /
  487.  
  488. # Liberar pras rede social
  489.  
  490. User-agent: LinkedInBot/1.0
  491. Allow: /
  492.  
  493. # Liberar pras rede social
  494.  
  495. User-agent: Pinterest/0.1
  496. Allow: /
  497. User-agent: Pinterest/0.2
  498. Allow: /
  499.  
  500. # Permitir Ads.txt
  501.  
  502. Allow: /ads.txt
  503.  
  504. # Permitir App-ads.txt
  505.  
  506. Allow: /app-ads.txt
  507.  
  508. # Vah cum kawmah homi
  509.  
  510. Crawl-delay: 5
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
Top