Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- User-agent: Zeus
- Disallow: /
- User-agent: BredoLab
- Disallow: /
- User-agent: Mariposa
- Disallow: /
- User-agent: Marina Botnet
- Disallow: /
- User-agent: Mega-D
- Disallow: /
- User-agent: Festi
- Disallow: /
- User-agent: Gumblar
- Disallow: /
- User-agent: Akbot
- Disallow: /
- User-agent:google*.com
- Disallow:/
- User-agent:*google.com
- Disallow:/
- User-agent: Kelihos
- Disallow: /
- User-agent: Boatnet
- Disallow: /
- User-agent: Sally
- Disallow: /
- User-agent: BobBot
- Disallow: /
- User-agent: AhrefsBot
- Disallow: /
- User-agent: dotbot
- Crawl-delay: 120
- Disallow: /
- User-agent: DotBot
- Crawl-delay: 120
- Disallow: /
- User-agent: MJ12bot
- Disallow: /
- User-agent: Yandex
- Crawl-delay: 120
- Disallow: /
- User-agent: https://fbstatic-a.akamaihd.net/
- Disallow: /
- User-agent: facebook.*
- Disallow: /
- User-agent: *facebook.*
- Disallow: /
- User-agent: facebook*.*
- Disallow: /
- User-agent: akamaihd
- Disallow: /
- User-agent: *bot
- Disallow: /
- User-agent: bot*
- Disallow: /
- User-agnet: *.gov
- Disallow: /
- User-agent: Police*.*
- Disallow: /
- User-agent: https://akamaihd.net
- Disallow: /
- User-agent: Googlebot
- Disallow: /lst/ng/
- User-agent: Yandex/1.01.001
- Crawl-delay: 120
- Disallow: /
- User-agent: Baiduspider
- Disallow: /
- User-agent: Baiduspider-video
- Disallow: /
- User-agent: Baiduspider-image
- Disallow: /
- User-agent: Exabot
- Disallow: /
- User-agent: Offline Explorer
- Disallow: /
- User-agent: VoilaBot
- Disallow: /
- User-agent: sogou spider
- Crawl-delay: 120
- Disallow: /
- User-agent: Sogou web spider
- Crawl-delay: 120
- Disallow: /
- User-agent: SputnikBot
- Disallow: /
- User-agent: FatBot
- Disallow: /
- User-agent: FatBot 2.0
- Disallow: /
- User-agent: Fetch
- Disallow: /
- User-agent: XoviBot
- Disallow: /
- User-agent: Zao
- Disallow: /
- User-agent: sistrix
- Disallow: /
- User-agent: 007ac9
- Disallow: /
- User-agent: BacklinkCrawler
- Disallow: /
- User-agent: SEOkicks-Robot
- Disallow: /
- User-agent: spbot
- Disallow: /
- User-agent: SiteSnagger
- Disallow: /
- User-agent: magpie-crawler
- Disallow: /
- User-agent: MSIECrawler
- Disallow: /
- User-agent: Teleport
- Disallow: /
- User-agent: UbiCrawler
- Disallow: /
- User-agent: Zealbot
- Disallow: /
- User-agent: w3af
- Disallow: /
- User-agent: WebStripper
- Disallow: /
- User-agent: WebCopier
- Disallow: /
- User-agent: lwp
- Disallow: /
- User-agent: *
- Disallow: /cgi-bin/
- # rest of agents
- User-agent: *
- Disallow:/cgi-bin*
- Disallow: /adminpanel.asp
- Disallow: /administratorlogin.asp
- Disallow: /adminLogin/
- Disallow: /admin-login.asp
- Disallow: /admin/account.asp
- Disallow: /cgi-bin
- Disallow: /wp/wp-admin/
- Disallow: /wp-admin
- Disallow: /wp-login/
- Disallow: /wp-includes
- Disallow: /wp-content/
- Disallow: /*/trackback/
- Disallow: /*/feed/
- Disallow: /*page
- Disallow: /admi
- Disallow: /de/DE/Widget/
- Disallow: /fr/FR/Widget/
- Disallow: /en/GB/Widget/
- Disallow: /nl/BE/Widget/
- Disallow: /fr/BE/Widget/
- Disallow: /dk/DK/Widget/
- Disallow: /es/ES/Widget/
- Disallow: /en/IE/Widget/
- Disallow: /it/IT/Widget/
- Disallow: /nl/NL/Widget/
- Disallow: /no/NO/Widget/
- Disallow: /pl/PL/Widget/
- Disallow: /fi/FI/Widget/
- Disallow: /se/SE/Widget/
- Disallow: /de/AT/Widget/
- Disallow: /en/EU/Widget/
- Disallow: /us/US/Widget/
- Disallow: /us/CA/Widget/
- Disallow: /fr/CA/Widget/
- Disallow: /fr/US/Widget/
- Disallow: /de/DE/Ajax/
- Disallow: /fr/FR/Ajax/
- Disallow: /en/GB/Ajax/
- Disallow: /nl/BE/Ajax/
- Disallow: /fr/BE/Ajax/
- Disallow: /dk/DK/Ajax/
- Disallow: /es/ES/Ajax/
- Disallow: /en/IE/Ajax/
- Disallow: /it/IT/Ajax/
- Disallow: /nl/NL/Ajax/
- Disallow: /no/NO/Ajax/
- Disallow: /pl/PL/Ajax/
- Disallow: /fi/FI/Ajax/
- Disallow: /se/SE/Ajax/
- Disallow: /de/AT/Ajax/
- Disallow: /en/EU/Ajax/
- Disallow: /us/US/Ajax/
- Disallow: /us/CA/Ajax/
- Disallow: /fr/CA/Ajax/
- Disallow: /fr/US/Ajax/
- Disallow: /us-ca/
- Disallow: /fr-ca/
- Disallow: /api/
- Disallow: /log/
- Disallow: /start-C3639
- Disallow: /start-C6220
- Disallow: /*-C6220*
- Disallow: /*-C3639*
- Disallow: /*-C4407*
- Disallow: /*-C3375*
- Disallow: /*articleNotFound*
- Disallow: /*searchQueryNotFound*
- # Disallow: /co.uk-test
- #Allow: /
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement