Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- $botlist = array(
- "!Susie (http://www.sync2it.com/susie)",
- " UnChaos From Chaos To Order Hybrid Web Search Engine.(vadim_gonchar@unchaos.com)",
- " UnChaos Bot Hybrid Web Search Engine. (vadim_gonchar@unchaos.com)",
- " UnChaosBot From Chaos To Order UnChaos Hybrid Web Search Engine at www.unchaos.com (info@unchaos.com)",
- " http://www.sygol.com",
- "( Robots.txt Validator http://www.searchengineworld.com/cgi-bin/robotcheck.cgi )",
- "(DreamPassport/3.0; isao/MyDiGiRabi)",
- "(Privoxy/1.0)",
- "*/Nutch-0.9-dev",
- "+SitiDi.net/SitiDiBot/1.0 (+Have Good Day)",
- "-DIE-KRAEHE- META-SEARCH-ENGINE/1.1 http://www.die-kraehe.de",
- "123spider-Bot (Version: 1.02, powered by www.123spider.de",
- "192.comAgent",
- "1st ZipCommander (Net) - http://www.zipcommander.com/",
- "2Bone_LinkChecker/1.0 libwww-perl/5.64",
- "4anything.com LinkChecker v2.0",
- "8484 Boston Project v 1.0",
- ":robot/1.0 (linux) ( admin e-mail: undefined http://www.neofonie.de/loesungen/search/robot.html )",
- "A-Online Search",
- "A1 Keyword Research/1.0.2 (+http://www.micro-sys.dk/products/keyword-research/) miggibot/2007.03.27",
- "A1 Sitemap Generator/1.0 (+http://www.micro-sys.dk/products/sitemap-generator/) miggibot/2006.01.24",
- "aardvark-crawler",
- "AbachoBOT",
- "AbachoBOT (Mozilla compatible)",
- "ABCdatos BotLink/5.xx.xxx#BBL",
- "Aberja Checkomat",
- "abot/0.1 (abot; http://www.abot.com; abot@abot.com)",
- "About/0.1libwww-perl/5.47",
- "Accelatech RSSCrawler/0.4",
- "accoona",
- "Accoona-AI-Agent/1.1.1 (crawler at accoona dot com)",
- "Accoona-AI-Agent/1.1.2 (aicrawler at accoonabot dot com)",
- "Ace Explorer",
- "Ack (http://www.ackerm.com/)",
- "AcoiRobot",
- "Acoon Robot v1.50.001",
- "Acoon Robot v1.52 (http://www.acoon.de)",
- "Acoon-Robot 4.0.x.[xx] (http://www.acoon.de)",
- "Acoon-Robot v3.xx (http://www.acoon.de and http://www.acoon.com)",
- "Acorn/Nutch-0.9 (Non-Profit Search Engine; acorn.isara.org; acorn at isara dot org)",
- "ActiveBookmark 1.x",
- "Activeworlds",
- "ActiveWorlds/3.xx (xxx)",
- "Ad Muncher v4.xx.x",
- "Ad Muncher v4x Build xxxxx",
- "Adaxas Spider (http://www.adaxas.net/)",
- "Advanced Browser (http://www.avantbrowser.com)",
- "AESOP_com_SpiderMan",
- "agadine/1.x.x (+http://www.agada.de)",
- "Agent-SharewarePlazaFileCheckBot/2.0+(+http://www.SharewarePlaza.com)",
- "AgentName/0.1 libwww-perl/5.48",
- "AIBOT/2.1 By +(www.21seek.com A Real artificial intelligence search engine China)",
- "AideRSS/1.0 (aiderss.com)",
- "aipbot/1.0 (aipbot; http://www.aipbot.com; aipbot@aipbot.com)",
- "aipbot/2-beta (aipbot dev; http://aipbot.com; aipbot@aipbot.com)",
- "Akregator/1.2.9; librss/remnants",
- "Aladin/3.324",
- "Alcatel-BG3/1.0 UP.Browser/5.0.3.1.2",
- "Aleksika Spider/1.0 (+http://www.aleksika.com/)",
- "AlertInfo 2.0 (Powered by Newsbrain)",
- "AlkalineBOT/1.3",
- "AlkalineBOT/1.4 (1.4.0326.0 RTM)",
- "Allesklar/0.1 libwww-perl/5.46",
- "Alligator 1.31 (www.nearsoftware.com)",
- "Allrati/1.1 (+)",
- "AltaVista Intranet V2.0 AVS EVAL search@freeit.com",
- "AltaVista Intranet V2.0 Compaq Altavista Eval sveand@altavista.net",
- "AltaVista Intranet V2.0 evreka.com crawler@evreka.com",
- "AltaVista V2.0B crawler@evreka.com",
- "amaya/x.xx libwww/x.x.x",
- "AmfibiBOT",
- "Amfibibot/0.06 (Amfibi Web Search; http://www.amfibi.com; agent@amfibi.com)",
- "Amfibibot/0.07 (Amfibi Robot; http://www.amfibi.com; agent@amfibi.com)",
- "amibot",
- "Amiga-AWeb/3.4.167SE",
- "AmigaVoyager/3.4.4 (MorphOS/PPC native)",
- "AmiTCP Miami (AmigaOS 2.04)",
- "Amoi 8512/R21.0 NF-Browser/3.3",
- "amzn_assoc",
- "AnnoMille spider 0.1 alpha - http://www.annomille.it",
- "annotate_google; http://ponderer.org/download/annotate_google.user.js",
- "Anonymized by ProxyOS: http://www.megaproxy.com",
- "Anonymizer/1.1",
- "AnswerBus (http://www.answerbus.com/)",
- "AnswerChase PROve x.0",
- "AnswerChase x.0",
- "ANTFresco/x.xx",
- "antibot-V1.1.5/i586-linux-2.2",
- "AnzwersCrawl/2.0 (anzwerscrawl@anzwers.com.au;Engine)",
- "Apexoo Spider 1.x",
- "Aplix HTTP/1.0.1",
- "Aplix_SANYO_browser/1.x (Japanese)",
- "Aplix_SEGASATURN_browser/1.x (Japanese)",
- "Aport",
- "appie 1.1 (www.walhello.com)",
- "Apple iPhone v1.1.4 CoreMedia v1.0.0.4A102",
- "Apple-PubSub/65.1.1",
- "ArabyBot (compatible; Mozilla/5.0; GoogleBot; FAST Crawler 6.4; http://www.araby.com;)",
- "ArachBot",
- "Arachnoidea (arachnoidea@euroseek.com)",
- "aranhabot",
- "ArchitextSpider",
- "archive.org_bot",
- "Argus/1.1 (Nutch; http://www.simpy.com/bot.html; feedback at simpy dot com)",
- "Arikus_Spider",
- "Arquivo-web-crawler (compatible; heritrix/1.12.1 +http://arquivo-web.fccn.pt)",
- "ASAHA Search Engine Turkey V.001 (http://www.asaha.com/)",
- "Asahina-Antenna/1.x",
- "Asahina-Antenna/1.x (libhina.pl/x.x ; libtime.pl/x.x)",
- "ask.24x.info",
- "AskAboutOil/0.06-rcp (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@askaboutoil.com)",
- "asked/Nutch-0.8 (web crawler; http://asked.jp; epicurus at gmail dot com)",
- "ASPSeek/1.2.5",
- "ASPseek/1.2.9d",
- "ASPSeek/1.2.x",
- "ASPSeek/1.2.xa",
- "ASPseek/1.2.xx",
- "ASPSeek/1.2.xxpre",
- "ASSORT/0.10",
- "asterias/2.0",
- "AtlocalBot/1.1 +(http://www.atlocal.com/local-web-site-owner.html)",
- "Atomic_Email_Hunter/4.0",
- "Atomz/1.0",
- "atSpider/1.0",
- "Attentio/Nutch-0.9-dev (Attentio's beta blog crawler; www.attentio.com; info@attentio.com)",
- "AU-MIC/2.0 MMP/2.0",
- "AUDIOVOX-SMT5600",
- "augurfind",
- "augurnfind V-1.x",
- "autoemailspider",
- "autohttp",
- "autowebdir 1.1 (www.autowebdir.com)",
- "AV Fetch 1.0",
- "Avant Browser (http://www.avantbrowser.com)",
- "AVSearch-1.0(peter.turney@nrc.ca)",
- "AVSearch-2.0-fusionIdx-14-CompetitorWebSites",
- "AVSearch-3.0(AltaVista/AVC)",
- "AWeb",
- "axadine/ (Axadine Crawler; http://www.axada.de/; )",
- "AxmoRobot - Crawling your site for better indexing on www.axmo.com search engine.",
- "Azureus 2.x.x.x",
- "BabalooSpider/1.3 (BabalooSpider; http://www.babaloo.si; spider@babaloo.si)",
- "BaboomBot/1.x.x (+http://www.baboom.us)",
- "BackStreet Browser 3.x",
- "BaiduImagespider+(+http://www.baidu.jp/search/s308.html)",
- "BaiDuSpider",
- "Baiduspider+(+http://help.baidu.jp/system/05.html)",
- "Baiduspider+(+http://www.baidu.com/search/spider.htm)",
- "Baiduspider+(+http://www.baidu.com/search/spider_jp.html)",
- "Balihoo/Nutch-1.0-dev (Crawler for Balihoo.com search engine - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)",
- "BanBots/1.2 (spider@banbots.com)",
- "Barca/2.0.xxxx",
- "BarcaPro/1.4.xxxx",
- "BarraHomeCrawler (albertof@barrahome.org)",
- "bCentral Billing Post-Process",
- "bdcindexer_2.6.2 (research@bdc)",
- "BDFetch",
- "BDNcentral Crawler v2.3 [en] (http://www.bdncentral.com/robot.html) (X11; I; Linux 2.0.44 i686)",
- "BeamMachine/0.5 (dead link remover of www.beammachine.net)",
- "beautybot/1.0 (+http://www.uchoose.de/crawler/beautybot/)",
- "BebopBot/2.5.1 ( crawler http://www.apassion4jazz.net/bebopbot.html )",
- "BeebwareDirectory/v0.01",
- "Big Brother (http://pauillac.inria.fr/~fpottier/)",
- "Big Fish v1.0",
- "BigBrother/1.6e",
- "BigCliqueBOT/1.03-dev (bigclicbot; http://www.bigclique.com; bot@bigclique.com)",
- "BIGLOTRON (Beta 2;GNU/Linux)",
- "Bigsearch.ca/Nutch-x.x-dev (Bigsearch.ca Internet Spider; http://www.bigsearch.ca/; info@enhancededge.com)",
- "Bilbo/2.3b-UNIX",
- "BilgiBetaBot/0.8-dev (bilgi.com (Beta) ; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
- "BilgiBot/1.0(beta) (http://www.bilgi.com/; bilgi at bilgi dot com)",
- "billbot wjj@cs.cmu.edu",
- "Bitacle bot/1.1",
- "Bitacle Robot (V:1.0;) (http://www.bitacle.com)",
- "Biyubi/x.x (Sistema Fenix; G11; Familia Toledo; es-mx)",
- "BlackBerry7520/4.0.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/5.0.3.3 UP.Link/5.1.2.12 (Google WAP Proxy/1.0)",
- "BlackWidow",
- "Blaiz-Bee/1.0 (+http://www.blaiz.net)",
- "Blaiz-Bee/2.00.8222 (BE Internet Search Engine http://www.rawgrunt.com)",
- "Blaiz-Bee/2.00.xxxx (+http://www.blaiz.net)",
- "BlitzBOT@tricus.net",
- "BlitzBOT@tricus.net (Mozilla compatible)",
- "BlockNote.Net",
- "BlogBot/1.x",
- "BlogBridge 2.13 (http://www.blogbridge.com/)",
- "Bloglines Title Fetch/1.0 (http://www.bloglines.com)",
- "Bloglines-Images/0.1 (http://www.bloglines.com)",
- "Bloglines/3.1 (http://www.bloglines.com)",
- "BlogMap (http://www.feedmap.net)",
- "Blogpulse (info@blogpulse.com)",
- "BlogPulseLive (support@blogpulse.com)",
- "BlogSearch/1.x +http://www.icerocket.com/",
- "blogsearchbot-pumpkin-3",
- "BlogsNowBot, V 2.01 (+http://www.blogsnow.com/)",
- "BlogVibeBot-v1.1 (spider@blogvibe.nl)",
- "blogWatcher_Spider/0.1 (http://www.lr.pi.titech.ac.jp/blogWatcher/)",
- "BlogzIce/1.0 (+http://icerocket.com; rhodes@icerocket.com)",
- "BlogzIce/1.0 +http://www.icerocket.com/",
- "BloobyBot",
- "Bloodhound/Nutch-0.9 (Testing Crawler for Research - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)",
- "bluefish 0.6 HTML editor",
- "BMCLIENT",
- "BMLAUNCHER",
- "Bobby/4.0.x RPT-HTTPClient/0.3-3E",
- "boitho.com-dc/0.xx (http://www.boitho.com/dcbot.html)",
- "boitho.com-robot/1.x",
- "boitho.com-robot/1.x (http://www.boitho.com/bot.html)",
- "Bookdog/x.x",
- "Bookmark Buddy bookmark checker (http://www.bookmarkbuddy.net/)",
- "Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/]",
- "Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/] (Version 2.0beta)",
- "BookmarkBase(2/;http://bookmarkbase.com)",
- "Bot mailto:craftbot@yahoo.com",
- "BPImageWalker/2.0 (www.bdbrandprotect.com)",
- "BravoBrian bstop.bravobrian.it",
- "BravoBrian SpiderEngine MarcoPolo",
- "BrightCrawler (http://www.brightcloud.com/brightcrawler.asp)",
- "BruinBot (+http://webarchive.cs.ucla.edu/bruinbot.html) ",
- "BSDSeek/1.0",
- "BStop.BravoBrian.it Agent Detector",
- "BTbot/0.x (+http://www.btbot.com/btbot.html)",
- "BTWebClient/180B(9704)",
- "BuildCMS crawler (http://www.buildcms.com/crawler)",
- "Bulkfeeds/r1752 (http://bulkfeeds.net/)",
- "BullsEye",
- "bumblebee@relevare.com",
- "BunnySlippers",
- "BurstFindCrawler/1.1 (crawler.burstfind.com; http://crawler.burstfind.com; crawler@burstfind.com)",
- "Buscaplus Robi/1.0 (http://www.buscaplus.com/robi/)",
- "BW-C-2.0",
- "bwh3_user_agent",
- "Cabot/Nutch-0.9 (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)",
- "Cabot/Nutch-1.0-dev (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)",
- "CamelHttpStream/1.0",
- "Cancer Information and Support International;",
- "carleson/1.0",
- "Carnegie_Mellon_University_Research_WebBOT-->PLEASE READ-->http://www.andrew.cmu.edu/~brgordon/webbot/index.html http://www.andrew.cmu.edu/~brgordon/webbot/index.html",
- "Carnegie_Mellon_University_WebCrawler http://www.andrew.cmu.edu/~brgordon/webbot/index.html",
- "Catall Spider",
- "CazoodleBot/CazoodleBot-0.1 (CazoodleBot Crawler; http://www.cazoodle.com/cazoodlebot; cazoodlebot@cazoodle.com)",
- "CCBot/1.0 (+http://www.commoncrawl.org/bot.html)",
- "ccubee/x.x",
- "CDR/1.7.1 Simulator/0.7(+http://timewe.net) Profile/MIDP-1.0 Configuration/CLDC-1.0",
- "CE-Preload",
- "CentiverseBot",
- "CentiverseBot - investigator",
- "CentiverseBot/3.0 (http://www.centiverse-project.net)",
- "Ceramic Tile Installation Guide (http://www.floorstransformed.com)",
- "CERN-LineMode/2.15",
- "cfetch/1.0",
- "CFNetwork/x.x",
- "cg-eye interactive",
- "Charon/1.x (Amiga)",
- "Chat Catcher/1.0",
- "Checkbot/1.xx LWP/5.xx",
- "CheckLinks/1.x.x",
- "CheckUrl",
- "CheckWeb",
- "Chilkat/1.0.0 (+http://www.chilkatsoft.com/ChilkatHttpUA.asp)",
- "China Local Browse 2.6",
- "Chitika ContentHit 1.0",
- "ChristCRAWLER 2.0",
- "CHttpClient by Open Text Corporation",
- "CipinetBot (http://www.cipinet.com/bot.html)",
- "Cityreview Robot (+http://www.cityreview.org/crawler/)",
- "CJ Spider/",
- "CJB.NET Proxy",
- "ClariaBot/1.0",
- "Claymont.com",
- "CloakDetect/0.9 (+http://fulltext.seznam.cz/)",
- "Clushbot/2.x (+http://www.clush.com/bot.html)",
- "Clushbot/3.x-BinaryFury (+http://www.clush.com/bot.html)",
- "Clushbot/3.xx-Ajax (+http://www.clush.com/bot.html)",
- "Clushbot/3.xx-Hector (+http://www.clush.com/bot.html)",
- "Clushbot/3.xx-Peleus (+http://www.clush.com/bot.html)",
- "COAST WebMaster Pro/4.x.x.xx (Windows NT)",
- "CoBITSProbe",
- "Cocoal.icio.us/1.0 (v36) (Mac OS X; http://www.scifihifi.com/cocoalicious)",
- "Cogentbot/1.X (+http://www.cogentsoftwaresolutions.com/bot.html)",
- "ColdFusion",
- "ColdFusion (BookmarkTracker.com)",
- "collage.cgi/1.xx",
- "combine/0.0",
- "Combine/2.0 http://combine.it.lth.se/",
- "Combine/3 http://combine.it.lth.se/",
- "Combine/x.0",
- "cometrics-bot, http://www.cometrics.de",
- "Commerce Browser Center",
- "complex_network_group/Nutch-0.9-dev (discovering the structure of the world-wide-web; http://cantor.ee.ucla.edu/~networks/crawl; nimakhaj@gmail.com)",
- "Computer_and_Automation_Research_Institute_Crawler crawler@ilab.sztaki.hu",
- "Comrite/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
- "Contact",
- "ContactBot/0.2",
- "ContentSmartz",
- "contype",
- "Convera Internet Spider V6.x",
- "ConveraCrawler/0.2",
- "ConveraCrawler/0.9d (+http://www.authoritativeweb.com/crawl)",
- "ConveraMultiMediaCrawler/0.1 (+http://www.authoritativeweb.com/crawl)",
- "CoolBot",
- "Cooliris/1.5 CFNetwork/459 Darwin/10.0.0d3",
- "CoralWebPrx/0.1.1x (See http://coralcdn.org/)",
- "cosmos/0.8_(robot@xyleme.com)",
- "cosmos/0.9_(robot@xyleme.com)",
- "CoteoNutchCrawler/Nutch-0.9 (info [at] coteo [dot] com)",
- "CougarSearch/0.x (+http://www.cougarsearch.com/faq.shtml)",
- "Covac TexAs Arachbot",
- "CoverScout%203/3.0.1 CFNetwork/339.5 Darwin/9.5.0 (i386) (iMac5,1)",
- "Cowbot-0.1 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)",
- "Cowbot-0.1.x (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)",
- "CrawlConvera0.1 (CrawlConvera@yahoo.com)",
- "Crawler",
- "Crawler (cometsearch@cometsystems.com)",
- "Crawler admin@crawler.de",
- "Crawler V 0.2.x admin@crawler.de",
- "crawler@alexa.com",
- "CrawlerBoy Pinpoint.com",
- "Crawllybot/0.1 (Crawllybot; +http://www.crawlly.com; crawler@crawlly.com)",
- "CreativeCommons/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)",
- "Cricket-A100/1.0 UP.Browser/6.3.0.7 (GUI) MMP/2.0",
- "CrocCrawler vx.3 [en] (http://www.croccrawler.com) (X11; I; Linux 2.0.44 i686)",
- "csci_b659/0.13",
- "CSE HTML Validator Professional (http://www.htmlvalidator.com/)",
- "Cuam Ver0.050bx",
- "Cuasarbot/0.9b http://www.cuasar.com/spider_beta/ ",
- "curl/7.10.x (i386-redhat-linux-gnu) libcurl/7.10.x OpenSSL/0.9.7a ipv6 zlib/1.1.4",
- "curl/7.7.x (i386--freebsd4.3) libcurl 7.7.x (SSL 0.9.6) (ipv6 enabled)",
- "curl/7.8 (i686-pc-linux-gnu) libcurl 7.8 (OpenSSL 0.9.6)",
- "curl/7.9.x (win32) libcurl 7.9.x",
- "CurryGuide SiteScan 1.1",
- "Custo x.x (www.netwu.com)",
- "Custom Spider www.bisnisseek.com /1.0",
- "Cyberdog/2.0 (Macintosh; 68k)",
- "CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)",
- "CyberSpyder Link Test/2.1.12 (admin@mspennyworth.com)",
- "CydralSpider/1.x (Cydral Web Image Search; http://www.cydral.com)",
- "CydralSpider/3.0 (Cydral Image Search; http://www.cydral.com)",
- "DA 3.5 (www.lidan.com)",
- "DA 4.0",
- "DA 4.0 (www.downloadaccelerator.com)",
- "DA 5.0",
- "DA 7.0",
- "DAP x.x",
- "Dart Communications PowerTCP",
- "DataCha0s/2.0",
- "DataFountains/DMOZ Downloader",
- "DataFountains/Dmoz Downloader (http://ivia.ucr.edu/useragents.shtml)",
- "DataFountains/DMOZ Feature Vector Corpus Creator (http://ivia.ucr.edu/useragents.shtml)",
- "DataparkSearch/4.47 (+http://dataparksearch.org/bot)",
- "DataparkSearch/4.xx (http://www.dataparksearch.org/)",
- "DataSpear/1.0 (Spider; http://www.dataspear.com/spider.html; spider@dataspear.com)",
- "DataSpearSpiderBot/0.2 (DataSpear Spider Bot; http://dssb.dataspear.com/bot.html; dssb@dataspear.com)",
- "DatenBot( http://www.sicher-durchs-netz.de/bot.html)",
- "DaviesBot/1.7 (www.wholeweb.net)",
- "daypopbot/0.x",
- "dbDig(http://www.prairielandconsulting.com)",
- "DBrowse 1.4b",
- "DBrowse 1.4d",
- "DC-Sakura/x.xx",
- "dCSbot/1.1",
- "DDD",
- "dds explorer v1.0 beta",
- "de.searchengine.comBot 1.2 (http://de.searchengine.com/spider)",
- "DeadLinkCheck/0.4.0 libwww-perl/5.xx",
- "Deep Link Calculator v1.0",
- "deepak-USC/ISI",
- "DeepIndex",
- "DeepIndex ( http://www.zetbot.com )",
- "DeepIndex (www.en.deepindex.com)",
- "DeepIndexer.ca",
- "del.icio.us-thumbnails/1.0 Mozilla/5.0 (compatible; Konqueror/3.4; FreeBSD) KHTML/3.4.2 (like Gecko)",
- "DeleGate/9.0.5-fix1",
- "Demo Bot DOT 16b",
- "Demo Bot Z 16b",
- "Denmex websearch (http://search.denmex.com)",
- "Der große BilderSauger 2.00u",
- "dev-spider2.searchpsider.com/1.3b",
- "DevComponents.com HtmlDocument Object",
- "DiaGem/1.1 (http://www.skyrocket.gr.jp/diagem.html)",
- "Diamond/x.0",
- "DiamondBot",
- "Digger/1.0 JDK/1.3.0rc3",
- "DigOut4U",
- "DIIbot/1.2",
- "Dillo/0.8.5-i18n-misc",
- "Dillo/0.x.x",
- "disastrous/1.0.5 (running with Python 2.5.1; http://www.bortzmeyer.org/disastrous.html; archangel77@del.icio.us)",
- "DISCo Pump x.x",
- "disco/Nutch-0.9 (experimental crawler; www.discoveryengine.com; disco-crawl@discoveryengine.com)",
- "disco/Nutch-1.0-dev (experimental crawler; www.discoveryengine.com; disco-crawl@discoveryengine.com)",
- "DittoSpyder",
- "dlman",
- "dloader(NaverRobot)/1.0",
- "DNSRight.com WebBot Link Ckeck Tool. Report abuse to: dnsr@dnsright.com",
- "DoCoMo/1.0/Nxxxi/c10",
- "DoCoMo/1.0/Nxxxi/c10/TB",
- "DoCoMo/1.0/P502i/c10 (Google CHTML Proxy/1.0)",
- "DoCoMo/2.0 P900iV(c100;TB;W24H11) ",
- "DoCoMo/2.0 SH901iS(c100;TB;W24H12),gzip(gfe) (via translate.google.com)",
- "DoCoMo/2.0 SH902i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)",
- "DoCoMo/2.0/SO502i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)",
- "DocZilla/1.0 (Windows; U; WinNT4.0; en-US; rv:1.0.0) Gecko/20020804",
- "dodgebot/experimental",
- "DonutP; Windows98SE",
- "Doubanbot/1.0 (bot@douban.com http://www.douban.com)",
- "Download Demon/3.x.x.x",
- "Download Druid 2.x",
- "Download Express 1.0",
- "Download Master",
- "Download Ninja 3.0",
- "Download Wonder",
- "Download-Tipp Linkcheck (http://download-tipp.de/)",
- "Download.exe(1.1) (+http://www.sql-und-xml.de/freeware-tools/)",
- "DownloadDirect.1.0",
- "Dr.Web (R) online scanner: http://online.drweb.com/",
- "Dragonfly File Reader",
- "Drecombot/1.0 (http://career.drecom.jp/bot.html)",
- "Drupal (+http://drupal.org/)",
- "DSurf15a 01",
- "DSurf15a 71",
- "DSurf15a 81",
- "DSurf15a VA",
- "DTAAgent",
- "dtSearchSpider",
- "Dual Proxy",
- "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
- "Dumbot(version 0.1 beta - dumbfind.com)",
- "Dumbot(version 0.1 beta - http://www.dumbfind.com/dumbot.html)",
- "Dumbot(version 0.1 beta)",
- "e-sense 1.0 ea(www.vigiltech.com/esensedisclaim.html)",
- "e-SocietyRobot(http://www.yama.info.waseda.ac.jp/~yamana/es/)",
- "eApolloBot/2.0 (compatible; heritrix/2.0.0-SNAPSHOT-20071024.170148 +http://www.eapollo-opto.com)",
- "EARTHCOM.info/1.x [www.earthcom.info]",
- "EARTHCOM.info/1.xbeta [www.earthcom.info]",
- "EasyDL/3.xx",
- "EasyDL/3.xx http://keywen.com/Encyclopedia/Bot",
- "EBrowse 1.4b",
- "eCatch/3.0",
- "EchO!/2.0",
- "Educate Search VxB",
- "egothor/3.0a (+http://www.xdefine.org/robot.html)",
- "EgotoBot/4.8 (+http://www.egoto.com/about.htm)",
- "ejupiter.com",
- "EldoS TimelyWeb/3.x",
- "elfbot/1.0 (+http://www.uchoose.de/crawler/elfbot/)",
- "ELI/20070402:2.0 (DAUM RSS Robot, Daum Communications Corp.; +http://ws.daum.net/aboutkr.html)",
- "ELinks (0.x.x; Linux 2.4.20 i586; 132x60)",
- "ELinks/0.x.x (textmode; NetBSD 1.6.2 sparc; 132x43)",
- "EmailSiphon",
- "EmailSpider",
- "EmailWolf 1.00",
- "EmeraldShield.com WebBot",
- "EmeraldShield.com WebBot (http://www.emeraldshield.com/webbot.aspx)",
- "EMPAS_ROBOT",
- "EnaBot/1.x (http://www.enaball.com/crawler.html)",
- "endo/1.0 (Mac OS X; ppc i386; http://kula.jp/endo)",
- "Enfish Tracker",
- "Enterprise_Search/1.0",
- "Enterprise_Search/1.0.xxx",
- "Enterprise_Search/1.00.xxx;MSSQL (http://www.innerprise.net/es-spider.asp)",
- "envolk/1.7 (+http://www.envolk.com/envolkspiderinfo.php)",
- "envolk[ITS]spider/1.6(+http://www.envolk.com/envolkspider.html)",
- "EroCrawler",
- "ES.NET_Crawler/2.0 (http://search.innerprise.net/)",
- "eseek-larbin_2.6.2 (crawler@exactseek.com)",
- "ESISmartSpider",
- "eStyleSearch 4 (compatible; MSIE 6.0; Windows NT 5.0)",
- "ESurf15a 15",
- "EuripBot/0.x (+http://www.eurip.com) GetFile",
- "EuripBot/0.x (+http://www.eurip.com) GetRobots",
- "EuripBot/0.x (+http://www.eurip.com) PreCheck",
- "Eurobot/1.0 (http://www.ayell.eu)",
- "EvaalSE - bot@evaal.com",
- "eventax/1.3 (eventax; http://www.eventax.de/; info@eventax.de)",
- "Everest-Vulcan Inc./0.1 (R&D project; host=e-1-24; http://everest.vulcan.com/crawlerhelp)",
- "Everest-Vulcan Inc./0.1 (R&D project; http://everest.vulcan.com/crawlerhelp)",
- "Exabot-Images/1.0",
- "Exabot-Test/1.0",
- "Exabot/2.0",
- "Exabot/3.0",
- "ExactSearch",
- "ExactSeek Crawler/0.1",
- "exactseek-crawler-2.63 (crawler@exactseek.com)",
- "exactseek-pagereaper-2.63 (crawler@exactseek.com)",
- "exactseek.com",
- "Exalead NG/MimeLive Client (convert/http/0.120)",
- "Excalibur Internet Spider V6.5.4",
- "Execrawl/1.0 (Execrawl; http://www.execrawl.com/; bot@execrawl.com)",
- "exooba crawler/exooba crawler (crawler for exooba.com; http://www.exooba.com/; info at exooba dot com)",
- "exooba/exooba crawler (exooba; exooba)",
- "ExperimentalHenrytheMiragoRobot",
- "Expired Domain Sleuth",
- "Express WebPictures (www.express-soft.com)",
- "ExtractorPro",
- "Extreme Picture Finder",
- "EyeCatcher (Download-tipp.de)/1.0",
- "Factbot 1.09 (see http://www.factbites.com/webmasters.php)",
- "factbot : http://www.factbites.com/robots",
- "FaEdit/2.0.x",
- "FairAd Client",
- "FANGCrawl/0.01",
- "FARK.com link verifier",
- "Fast Crawler Gold Edition",
- "FAST Enterprise Crawler 6 (Experimental)",
- "FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/",
- "FAST Enterprise Crawler 6 used by Cobra Development (admin@fastsearch.com)",
- "FAST Enterprise Crawler 6 used by Comperio AS (sts@comperio.no)",
- "FAST Enterprise Crawler 6 used by FAST (FAST)",
- "FAST Enterprise Crawler 6 used by Pages Jaunes (pvincent@pagesjaunes.fr)",
- "FAST Enterprise Crawler 6 used by Sensis.com.au Web Crawler (search_comments\at\sensis\dot\com\dot\au)",
- "FAST Enterprise Crawler 6 used by Singapore Press Holdings (crawler@sphsearch.sg)",
- "FAST Enterprise Crawler 6 used by WWU (wardi@uni-muenster.de)",
- "FAST Enterprise Crawler/6 (www.fastsearch.com)",
- "FAST Enterprise Crawler/6.4 (helpdesk at fast.no)",
- "FAST FirstPage retriever (compatible; MSIE 5.5; Mozilla/4.0)",
- "FAST MetaWeb Crawler (helpdesk at fastsearch dot com)",
- "Fast PartnerSite Crawler",
- "FAST-WebCrawler/2.2.10 (Multimedia Search) (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)",
- "FAST-WebCrawler/2.2.6 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)",
- "FAST-WebCrawler/2.2.7 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no",
- "FAST-WebCrawler/2.2.8 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no",
- "FAST-WebCrawler/3.2 test",
- "FAST-WebCrawler/3.3 (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)",
- "FAST-WebCrawler/3.4/Nirvana (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)",
- "FAST-WebCrawler/3.4/PartnerSite (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)",
- "FAST-WebCrawler/3.5 (atw-crawler at fast dot no; http://fast.no/support.php?c=faqs/crawler)",
- "FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
- "FAST-WebCrawler/3.6/FirstPage (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)",
- "FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
- "FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",
- "FAST-WebCrawler/3.8 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
- "FAST-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
- "FAST-WebCrawler/3.x Multimedia",
- "FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no)",
- "fastbot crawler beta 2.0 (+http://www.fastbot.de)",
- "FastBug http://www.ay-up.com",
- "FastCrawler 3.0.1 (crawler@1klik.dk)",
- "FastSearch Web Crawler for Verizon SuperPages (kevin.watters@fastsearch.com)",
- "Favcollector/2.0 (info@favcollector.com http://www.favcollector.com/)",
- "FavIconizer",
- "favo.eu crawler/0.6 (http://www.favo.eu)",
- "FavOrg",
- "Favorites Checking (http://campulka.net)",
- "Favorites Sweeper v.2.03",
- "Faxobot/1.0",
- "FDM 1.x",
- "FDM 2.x",
- "Feed Seeker Bot (RSS Feed Seeker http://www.MyNewFavoriteThing.com/fsb.php)",
- "Feed24.com",
- "Feed::Find/0.0x",
- "Feedable/0.1 (compatible; MSIE 6.0; Windows NT 5.1)",
- "FeedChecker/0.01",
- "FeedDemon/2.7 (http://www.newsgator.com/; Microsoft Windows XP)",
- "Feedfetcher-Google-iGoogleGadgets; (+http://www.google.com/feedfetcher.html)",
- "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html)",
- "FeedForAll rss2html.php v2",
- "FeedHub FeedDiscovery/1.0 (http://www.feedhub.com)",
- "FeedHub MetaDataFetcher/1.0 (http://www.feedhub.com)",
- "Feedjit Favicon Crawler 1.0",
- "Feedreader 3.xx (Powered by Newsbrain)",
- "Feedshow/x.0 (http://www.feedshow.com; 1 subscriber)",
- "FeedshowOnline (http://www.feedshow.com)",
- "FeedZcollector v1.x (Platinum) http://www.feeds4all.com/feedzcollector",
- "Felix - Mixcat Crawler (+http://mixcat.com)",
- "fetch libfetch/2.0",
- "FFC Trap Door Spider",
- "Filangy/0.01-beta (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)",
- "Filangy/1.0x (Filangy; http://www.filangy.com/filangyinfo.jsp?inc=robots.jsp; filangy-agent@filangy.com)",
- "Filangy/1.0x (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)",
- "fileboost.net/1.0 (+http://www.fileboost.net)",
- "FileHound x.x",
- "Filtrbox/1.0",
- "FindAnISP.com_ISP_Finder_v99a",
- "Findexa Crawler (http://www.findexa.no/gulesider/article26548.ece)",
- "findlinks/x.xxx (+http://wortschatz.uni-leipzig.de/findlinks/) ",
- "FineBot",
- "Finjan-prefetch",
- "Firefly/1.0",
- "Firefly/1.0 (compatible; Mozilla 4.0; MSIE 5.5)",
- "Firefox (kastaneta03@hotmail.com)",
- "Firefox_1.0.6 (kasparek@naparek.cz)",
- "FirstGov.gov Search - POC:firstgov.webmasters@gsa.gov",
- "firstsbot",
- "Flapbot/0.7.2 (Flaptor Crawler; http://www.flaptor.com; crawler at flaptor period com)",
- "FlashGet",
- "FLATARTS_FAVICO",
- "Flexum spider",
- "Flexum/2.0",
- "FlickBot 2.0 RPT-HTTPClient/0.3-3",
- "flunky",
- "fly/6.01 libwww/4.0D",
- "flyindex.net 1.0/http://www.flyindex.net",
- "FnooleBot/2.5.2 (+http://www.fnoole.com/addurl.html)",
- "FocusedSampler/1.0",
- "Folkd.com Spider/0.1 beta 1 (www.folkd.com)",
- "FollowSite Bot ( http://www.followsite.com/bot.html )",
- "FollowSite.com ( http://www.followsite.com/b.html )",
- "Fooky.com/ScorpionBot/ScoutOut; http://www.fooky.com/scorpionbots",
- "Francis/1.0 (francis@neomo.de http://www.neomo.de/)",
- "Franklin Locator 1.8",
- "free-downloads.net download-link validator /0.1",
- "FreeFind.com-SiteSearchEngine/1.0 (http://freefind.com; spiderinfo@freefind.com)",
- "Frelicbot/1.0 +http://www.frelic.com/",
- "FreshDownload/x.xx",
- "FreshNotes crawler< report problems to crawler-at-freshnotes-dot-com",
- "FSurf15a 01",
- "FTB-Bot http://www.findthebest.co.uk/",
- "Full Web Bot 0416B",
- "Full Web Bot 0516B",
- "Full Web Bot 2816B",
- "FuseBulb.Com",
- "FyberSpider (+http://www.fybersearch.com/fyberspider.php)",
- "Gagglebot",
- "GAIS Robot/1.0B2",
- "Gaisbot/3.0 (indexer@gais.cs.ccu.edu.tw; http://gais.cs.ccu.edu.tw/robot.php)",
- "Gaisbot/3.0+(robot06@gais.cs.ccu.edu.tw;+http://gais.cs.ccu.edu.tw/robot.php)",
- "GalaxyBot/1.0 (http://www.galaxy.com/galaxybot.html)",
- "Gallent Search Spider v1.4 Robot 2 (http://robot.GallentSearch.com)",
- "gamekitbot/1.0 (+http://www.uchoose.de/crawler/gamekitbot/)",
- "Gamespy_Arcade",
- "GammaSpider/1.0",
- "gazz/x.x (gazz@nttrd.com)",
- "geckobot",
- "Generic Mobile Phone (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
- "generic_crawler/01.0217/",
- "GenesisBrowser (HTTP 1.1; 0.9; XP SP2; .NET CLR 2.0.50727)",
- "genieBot (http://64.5.245.11/faq/faq.html)",
- "geniebot wgao@genieknows.com",
- "GeoBot/1.0",
- "GeonaBot 1.x; http://www.geona.com/",
- "geourl/2.0b2",
- "GeoURLBot 1.0 (http://geourl.org)",
- "GetBot",
- "GetRight/3.x.x",
- "GetRight/4.5xx",
- "GetRight/4.x",
- "GetRight/4.x[a-e]",
- "GetRight/6.1 (Pro)",
- "GetRightPro/6.0beta2",
- "GetWeb/0.1 libwww-perl/5.16",
- "GhostRouteHunter/20021130 (https://www.sixxs.net/tools/grh/; info@sixxs.net)",
- "gigabaz/3.1x (baz@gigabaz.com; http://gigabaz.com/gigabaz/)",
- "Gigabot/2.0 (gigablast.com)",
- "Gigabot/2.0/gigablast.com/spider.html",
- "Gigabot/2.0; http://www.gigablast.com/spider.html",
- "Gigabot/2.0att",
- "Gigabot/3.0 (http://www.gigablast.com/spider.html)",
- "Gigabot/x.0",
- "GigabotSiteSearch/2.0 (sitesearch.gigablast.com)",
- "GNODSPIDER (www.gnod.net)",
- "Go!Zilla 3.x (www.gozilla.com)",
- "Go!Zilla/4.x.x.xx",
- "Go-Ahead-Got-It/1.1",
- "Goblin/0.9 (http://www.goguides.org/)",
- "Goblin/0.9.x (http://www.goguides.org/goblin-info.html)",
- "GoForIt.com",
- "GOFORITBOT ( http://www.goforit.com/about/ )",
- "GoGuides.Org Link Check",
- "GoldenFeed Spider 1.0 (http://www.goldenfeed.com)",
- "Goldfire Server",
- "gonzo1[P] +http://www.suchen.de/popups/faq.jsp",
- "gonzo2[P] +http://www.suchen.de/faq.html",
- "Goofer/0.2",
- "Google Talk",
- "googlebot (larbin2.6.0@unspecified.mail)",
- "Googlebot-Image/1.0",
- "Googlebot-Image/1.0 ( http://www.googlebot.com/bot.html)",
- "Googlebot/2.1 ( http://www.google.com/bot.html)",
- "Googlebot/2.1 ( http://www.googlebot.com/bot.html)",
- "Googlebot/Test ( http://www.googlebot.com/bot.html)",
- "Gordon's Spider/Nutch-0.9 (http://www.sharethis.com; gordon@sharethis.com)",
- "GrapeFX/0.3 libwww/5.4.0",
- "great-plains-web-spider/flatlandbot (Flatland Industries Web Spider; http://www.flatlandindustries.com/flatlandbot.php; jason@flatlandindustries.com)",
- "GreatNews/1.0",
- "GreenBrowser",
- "gridwell (http://search.gridwell.com)",
- "GrigorBot 0.8 (http://www.grigor.biz/bot.html)",
- "Gromit/1.0",
- "grub crawler(http://www.grub.org)",
- "grub-client",
- "gsa-crawler (Enterprise; GID-01422; jplastiras@google.com)",
- "gsa-crawler (Enterprise; GID-01742;gsatesting@rediffmail.com)",
- "gsa-crawler (Enterprise; GIX-02057; dm@enhesa.com)",
- "gsa-crawler (Enterprise; GIX-03519; cknuetter@stubhub.com)",
- "gsa-crawler (Enterprise; GIX-0xxxx; enterprise-training@google.com)",
- "GSiteCrawler/v1.xx rev. xxx (http://gsitecrawler.com/)",
- "Guestbook Auto Submitter",
- "Gulliver/1.3",
- "Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)",
- "Gungho/0.08004 (http://code.google.com/p/gungho-crawler/wiki/Index)",
- "GurujiBot/1.0 (+http://www.guruji.com/WebmasterFAQ.html)",
- "GurujiImageBot/1.0 (+http://www.guruji.com/en/WebmasterFAQ.html)",
- "Haier-T10C/1.0 iPanel/2.0 WAP2.0 (compatible; UP.Browser/6.2.2.4; UPG1; UP/4.0; Embedded)",
- "HappyFunBot/1.1",
- "Harvest-NG/1.0.2",
- "Haste/0.12 (HOME: http://haste.kytoon.com/)",
- "Hatena Antenna/0.4 (http://a.hatena.ne.jp/help#robot)",
- "Hatena Mobile Gateway/1.0",
- "Hatena Pagetitle Agent/1.0",
- "Hatena RSS/0.3 (http://r.hatena.ne.jp)",
- "HatenaScreenshot/1.0 (checker)",
- "hbtronix.spider.2 -- http://hbtronix.de/spider.php",
- "HeinrichderMiragoRobot",
- "HeinrichderMiragoRobot (http://www.miragorobot.com/scripts/deinfo.asp)",
- "Helix/1.x ( http://www.sitesearch.ca/helix/)",
- "HenriLeRobotMirago (http://www.miragorobot.com/scripts/frinfo.asp)",
- "HenrytheMiragoRobot",
- "HenryTheMiragoRobot (http://www.miragorobot.com/scripts/mrinfo.asp)",
- "hgrepurl/1.0",
- "Hi! I'm CsCrawler my homepage: http://www.kde.cs.uni-kassel.de/lehre/ss2005/googlespam/crawler.html RPT-HTTPClient/0.3-3",
- "HiDownload",
- "Hippias/0.9 Beta",
- "HitList",
- "Hitwise Spider v1.0 http://www.hitwise.com",
- "HLoader",
- "holmes/3.11 (http://morfeo.centrum.cz/bot)",
- "holmes/3.9 (onet.pl)",
- "holmes/3.xx (OnetSzukaj/5.0; +http://szukaj.onet.pl)",
- "holmes/x.x",
- "HolmesBot (http://holmes.ge)",
- "HomePageSearch(hpsearch.uni-trier.de)",
- "Homerbot: www.homerweb.com",
- "Honda-Search/0.7.2 (Nutch; http://lucene.apache.org/nutch/bot.html; search@honda-search.com)",
- "HooWWWer/2.1.3 (debugging run) (+http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-infohiit.fi)",
- "HooWWWer/2.1.x ( http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-infohiit.fi)",
- "HotJava/1.0.1/JRE1.1.x",
- "Hotzonu/x.0",
- "HPL/Nutch-0.9 -",
- "htdig/3.1.6 (http://computerorgs.com)",
- "htdig/3.1.6 (unconfigured@htdig.searchengine.maintainer)",
- "htdig/3.1.x (root@localhost)",
- "Html Link Validator (www.lithopssoft.com)",
- "HTML2JPG Blackbox, http://www.html2jpg.com",
- "HTML2JPG Enterprise",
- "HTMLParser/1.x",
- "HTTP Retriever",
- "http://Anonymouse.org/ (Unix)",
- "http://Ask.24x.Info/ (http://narres.it/)",
- "http://hilfe.acont.de/bot.html ACONTBOT",
- "http://OzySoftware.com/Index.html",
- "http://www.almaden.ibm.com/cs/crawler",
- "http://www.almaden.ibm.com/cs/crawler [rc1.wf.ibm.com]",
- "http://www.almaden.ibm.com/cs/crawler [wf216]",
- "http://www.istarthere.com_spider@istarthere.com",
- "http://www.monogol.de",
- "http://www.trendtech.dk/spider.asp)",
- "HTTP::Lite/2.x.x",
- "HTTPEyes",
- "HTTPResume v. 1.x",
- "httpunit/1.5",
- "httpunit/1.x",
- "Hybrid/1.2 [en] (OS Independent)",
- "HyperEstraier/1.x.xx",
- "i1searchbot/2.0 (i1search web crawler; http://www.i1search.com; crawler@i1search.com)",
- "IAArchiver-1.0",
- "iaskspider",
- "iaskspider2 (iask@staff.sina.com.cn)",
- "ia_archiver",
- "ia_archiver-web.archive.org",
- "ia_archiver/1.6",
- "IBrowse/2.2 (AmigaOS 3.5)",
- "IBrowse/2.2 (Windows 3.1)",
- "iCab/2.5.2 (Macintosh; I; PPC)",
- "ICC-Crawler(Mozilla-compatible; http://kc.nict.go.jp/icc/crawl.html; icc-crawl(at)ml(dot)nict(dot)go(dot)jp)",
- "ICC-Crawler(Mozilla-compatible;http://kc.nict.go.jp/icc/crawl.html;icc-crawl-contact(at)ml(dot)nict(dot)go(dot)jp)",
- "iCCrawler (http://www.iccenter.net)",
- "ICCrawler - ICjobs (http://www.icjobs.de/bot.htm)",
- "ICE Browser/5.05 (Java 1.4.0; Windows 2000 5.0 x86)",
- "ichiro/x.0 (http://help.goo.ne.jp/door/crawler.html)",
- "ichiro/x.0 (ichiro@nttr.co.jp)",
- "IconSurf/2.0 favicon finder (see http://iconsurf.com/robot.html)",
- "IconSurf/2.0 favicon monitor (see http://iconsurf.com/robot.html)",
- "ICOO Loader v.x.x.x",
- "ICRA_label_spider/x.0",
- "icsbot-0.1",
- "IDA",
- "ideare - SignSite/1.x",
- "iearthworm/1.0, iearthworm@yahoo.com.cn",
- "IEFav172Free",
- "iFeed.jp/2.0 (www.psychedelix.com/agents/agents.rss; 0 subscribers)",
- "igdeSpyder (compatible; igde.ru; +http://igde.ru/doc/tech.html)",
- "iGetter/1.x (Macintosh;G;PPC)",
- "iGetter/2 (Macintosh; U; PPC Mac OS X; en)",
- "IIITBOT/1.1 (Indian Language Web Search Engine; http://webkhoj.iiit.net; pvvpr at iiit dot ac dot in)",
- "ilial/Nutch-0.9 (Ilial, Inc. is a Los Angeles based Internet startup company. For more information please visit http://www.ilial.com/crawler; http://www.ilial.com/crawler; crawl@ilial.com)",
- "ilial/Nutch-0.9-dev",
- "IlseBot/1.x",
- "IlTrovatore-Setaccio ( http://www.iltrovatore.it)",
- "Iltrovatore-Setaccio/0.3-dev (Indexing; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)",
- "IlTrovatore-Setaccio/1.2 ( http://www.iltrovatore.it/aiuto/faq.html)",
- "Iltrovatore-Setaccio/1.2 (It-bot; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)",
- "iltrovatore-setaccio/1.2-dev (spidering; http://www.iltrovatore.it/aiuto/.....)",
- "IlTrovatore/1.2 (IlTrovatore; http://www.iltrovatore.it/bot.html; bot@iltrovatore.it)",
- "ImageVisu/v4.x.x",
- "ImageWalker/2.0 (www.bdbrandprotect.com)",
- "Incutio HttpClient v0.x",
- "IncyWincy data gatherer(webmaster@loopimprovements.com",
- "IncyWincy page crawler(webmaster@loopimprovements.com",
- "IncyWincy(http://www.look.com)",
- "IncyWincy(http://www.loopimprovements.com/robot.html)",
- "IncyWincy/2.1(loopimprovements.com/robot.html)",
- "IndexTheWeb.com Crawler7",
- "Industry Program 1.0.x",
- "Inet library",
- "InetURL/1.0",
- "info@pubblisito.com- (http://www.pubblisito.com) il Sud dei Motori di Ricerca",
- "Infoaxe./Nutch-0.9",
- "infoConveraCrawler/0.8 ( http://www.authoritativeweb.com/crawl)",
- "InfoFly/1.0 (http://www.versions-project.org/)",
- "InfoLink/1.x",
- "INFOMINE/8.0 Adders",
- "INFOMINE/8.0 RemoteServices",
- "INFOMINE/8.0 VLCrawler (http://infomine.ucr.edu/useragents)",
- "InfoNaviRobot(F107)",
- "InfoSeek Sidewinder/0.9",
- "InfoSeek Sidewinder/1.0A",
- "InfoSeek Sidewinder/1.1A",
- "Infoseek SideWinder/1.45 (Compatible; MSIE 10.0; UNIX)",
- "Infoseek SideWinder/2.0B (Linux 2.4 i686)",
- "INGRID/3.0 MT (webcrawler@NOSPAMexperimental.net; http://webmaster.ilse.nl/jsp/webmaster.jsp)",
- "Inktomi Search",
- "InnerpriseBot/1.0 (http://www.innerprise.com/)",
- "Insitor.com search and find world wide!",
- "Insitornaut",
- "InstallShield DigitalWizard",
- "integrity/1.6",
- "Intelix/0.x (cs; http://www.microton.cz/intelix/; microton@@microton.cz)",
- "Interarchy/x.x.x (InterarchyCrawler)",
- "Internet Ninja x.0",
- "InternetArchive/0.8-dev(Nutch;http://lucene.apache.org/nutch/bot.html;nutch-agent@lucene.apache",
- "InternetLinkAgent/3.1",
- "InternetSeer.com",
- "intraVnews/1.x",
- "IOI/2.0 (ISC Open Index crawler; http://index.isc.org/; bot@index.isc.org)",
- "IP*Works! V5 HTTP/S Component - by /n software - www.nsoftware.com",
- "http://www.ip2location.com",
- "IP2MapBot/1.1 http://www.ip2map.com",
- "IPiumBot laurion(dot)com",
- "IpselonBot/0.xx-beta (Ipselon; http://www.ipselon.com; ipselonbot@ipselon.com)",
- "Iria/1.xxa",
- "IRLbot/1.0 ( http://irl.cs.tamu.edu/crawler)",
- "IRLbot/3.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler/)",
- "IrssiUrlLog/0.2",
- "Irvine/1.x.x",
- "ISC Systems iRc Search 2.1",
- "iSiloX/4.xx Windows/32",
- "isurf (tszhu@canada.com)",
- "iTunes/x.x.x",
- "IUPUI Research Bot v 1.9a",
- "iVia Page Fetcher (http://ivia.ucr.edu/useragents.shtml)",
- "iVia/4.0 CanonizeUrl (http://infomine.ucr.edu/iVia/useragents.shtml",
- "IWAgent/ 1.0 - www.brandprotect.com",
- "J-PHONE/3.0/J-SH07",
- "Jabot/6.x (http://odin.ingrid.org/)",
- "Jabot/7.x.x (http://odin.ingrid.org/)",
- "Jack",
- "Jakarta Commons-HttpClient/2.0xxx",
- "Jakarta Commons-HttpClient/3.0-rcx",
- "Jambot/0.1.x (Jambot; http://www.jambot.com/blog; crawler@jambot.com)",
- "Jambot/0.2.1 (Jambot; http://www.jambot.com/blog/static.php?page=webmaster-robot; crawler@jambot.com)",
- "Java 1.1",
- "Java/1.4.1_01",
- "Java1.0.21.0",
- "Java1.1.xx.x",
- "Java1.3.0rc1",
- "Java1.3.x",
- "Java1.4.0",
- "Jayde Crawler. http://www.jayde.com",
- "JBH Agent 2.0",
- "jBrowser/J2ME Profile/MIDP-1.0 Configuration/CLDC-1.0 (Google WAP Proxy/1.0)",
- "JCheckLinks/0.1 RPT-HTTPClient/0.3-1",
- "JDK/1.1",
- "Jeode/1.x.x",
- "Jetbot/1.0",
- "JetBrains Omea Reader 1.0.x (http://www.jetbrains.com/omea_reader/)",
- "JetBrains Omea Reader 2.0 Release Candidate 1 (http://www.jetbrains.com/omea_reader/)",
- "JetCar",
- "Jigsaw/2.2.x W3C_CSS_Validator_JFouffa/2.0",
- "JoBo/1.x (http://www.matuschek.net/jobo.html)",
- "JoBo/@JOBO_VERSION@(http://www.matuschek.net/jobo.html)",
- "JobSpider_BA/1.1",
- "JOC Web Spider",
- "JordoMedia/1.0 RSS File Reader (http://www.jordomedia.com)",
- "Journster [alpha] (http://journster.com/)",
- "Journster.com RSS/Atom aggregator 0.5 (http://www.journster.com/bot.phtml)",
- "JRTS Check Favorites Utility",
- "JRTwine Software Check Favorites Utility",
- "Jyxobot/x",
- "K-Meleon/0.6 (Windows; U; Windows NT 5.1; en-US; rv:0.9.5) Gecko/20011011",
- "k2spider",
- "KAIST AITrc Crawler",
- "KakleBot - www.kakle.com/0.1 (KakleBot - www.kakle.com; http:// www.kakle.com/bot.html; support@kakle.com)",
- "kalooga/kalooga-4.0-dev-datahouse (Kalooga; http://www.kalooga.com; info@kalooga.com)",
- "kalooga/KaloogaBot (Kalooga; http://www.kalooga.com/info.html?page=crawler; crawler@kalooga.com)",
- "Kapere (http://www.kapere.com)",
- "Kazehakase/0.x.x.[x]",
- "KDDI-SN22 UP.Browser/6.0.7 (GUI) MMP/1.1 (Google WAP Proxy/1.0)",
- "Kenjin Spider",
- "Kevin http://dznet.com/kevin/",
- "Kevin http://websitealert.net/kevin/",
- "KE_1.0/2.0 libwww/5.2.8",
- "KFSW-Bot (Version: 1.01 powered by KFSW www.kfsw.de)",
- "kinja-imagebot (http://www.kinja.com/)",
- "kinjabot (http://www.kinja.com)",
- "KIT-Fireball/2.0",
- "KIT-Fireball/2.0 (compatible; Mozilla 4.0; MSIE 5.5)",
- "Klondike/1.50 (WSP Win32) (Google WAP Proxy/1.0)",
- "KnowItAll(knowitall@cs.washington.edu)",
- "Knowledge.com/0.x",
- "Kontiki Client x.xx",
- "Krugle/Krugle,Nutch/0.8+ (Krugle web crawler; http://www.krugle.com/crawler/info.html; webcrawler@krugle.com)",
- "KSbot/1.0 (KnowledgeStorm crawler; http://www.knowledgestorm.com/resources/content/crawler/index.html; crawleradmin@knowledgestorm.com)",
- "kuloko-bot/0.x",
- "kulokobot www.kuloko.com kuloko@backweave.com",
- "kulturarw3/0.1",
- "KummHttp/1.1 (compatible; KummClient; Linux rulez)",
- "KWC-KX9/1109 UP.Browser/6.2.3.9.g.1.107 (GUI) MMP/2.0 UP.Link/6.3.0.0.0",
- "Labrador/0.2; http://ir.dcs.gla.ac.uk/labrador; craigm@dcs.gla.ac.uk",
- "Lachesis",
- "lanshanbot/1.0",
- "lanshanbot/1.0 (+http://search.msn.com/msnbot.htm)",
- "LapozzBot/1.4 ( http://robot.lapozz.com)",
- "LapozzBot/1.5 (+http://robot.lapozz.hu)",
- "larbin (samualt9@bigfoot.com)",
- "LARBIN-EXPERIMENTAL (efp@gmx.net)",
- "larbin_2.1.1 larbin2.1.1@somewhere.com",
- "larbin_2.2.0 (crawl@compete.com)",
- "larbin_2.2.1_de_Viennot (Laurent.Viennot@inria.fr)",
- "larbin_2.2.2 (sugayama@lab7.kuis.kyoto-u.ac.jp)",
- "larbin_2.2.2_guillaume (guillaume@liafa.jussieu.fr)",
- "larbin_2.6.0 (larbin2.6.0@unspecified.mail)",
- "larbin_2.6.1 (larbin2.6.1@unspecified.mail)",
- "larbin_2.6.2 (hamasaki@grad.nii.ac.jp)",
- "larbin_2.6.2 (larbin2.6.2@unspecified.mail)",
- "larbin_2.6.2 (listonATccDOTgatechDOTedu)",
- "larbin_2.6.2 (pimenas@systems.tuc.gr)",
- "larbin_2.6.2 (tom@lemurconsulting.com)",
- "larbin_2.6.2 (vitalbox1@hotmail.com)",
- "larbin_2.6.3 (ltaa_web_crawler@groupes.epfl.ch)",
- "larbin_2.6.3 (wgao@genieknows.com)",
- "larbin_2.6.3_for_(http://cosco.hiit.fi/search/) tsilande@hiit.fi",
- "larbin_2.6_basileocaml (basile.starynkevitch@cea.fr)",
- "larbin_devel (http://pauillac.inria.fr/~ailleret/prog/larbin/)",
- "lawinfo-crawler/Nutch-0.9-dev (Crawler for lawinfo.com pages; http://www.lawinfo.com; webmaster@lawinfo.com)",
- 'lc/$ROADS::Version libwww-perl/5.00',
- "lcabotAccept: */*",
- "LeapTag/0.8.1.beta081.r3750 (compatible; Mozilla 4.0; MSIE 5.5; robot@yoriwa.com)",
- "LECodeChecker/3.0 libgetdoc/1.0",
- "LeechGet 200x (www.leechget.de)",
- "LEIA/2.90",
- "LEIA/3.01pr (LEIAcrawler; [SNIP])",
- "LetsCrawl.com/1.0 +http://letscrawl.com/",
- "LexiBot/1.00",
- "LG-LX260 POLARIS-LX260/2.0 MMP/2.0 Profile/MIDP-2.0 Configuration/CLDC-1.1",
- "LG/U8138/v1.0",
- "Libby_1.1/libwww-perl/5.47",
- "libcurl-agent/1.0",
- "LibertyW (+http://www.lw01.com)",
- "libWeb/clsHTTP -- hiongun@kt.co.kr",
- "libwww-perl/5.41",
- "libwww-perl/5.45",
- "libwww-perl/5.48",
- "libwww-perl/5.50",
- "libwww-perl/5.52 FP/2.1",
- "libwww-perl/5.52 FP/4.0",
- "libwww-perl/5.53",
- "libwww-perl/5.63",
- "libwww-perl/5.64",
- "libwww-perl/5.65",
- "libwww-perl/5.800",
- "libwww/5.3.2",
- "Liferea/0.x.x (Linux; en_US.UTF-8; http://liferea.sf.net/)",
- "Liferea/1.x.x (Linux; es_ES.UTF-8; http://liferea.sf.net/)",
- "LightningDownload/1.0beta2",
- "LightningDownload/1.x.x",
- "LightningDownload/1.x.x [Accelerated x]",
- "LijitSpider/Nutch-0.9 (Reports crawler; http://www.lijit.com/; info(a)lijit(d)com)",
- "Lincoln State Web Browser",
- "Link Valet Online 1.x",
- "LinkAlarm/2.x",
- "Linkbot",
- "linkbot",
- "Linkbot x.0",
- "LinkCheck (linkcheck@inter7.com http://www.inter7.com/linkcheck)",
- "LinkLint-checkonly/2.x.x",
- "LinkLint-spider/2.x.x",
- "linknzbot",
- "LinkPimpin v1.0",
- "LinkProver 2.1",
- "Links (0.9x; Linux 2.4.7-10 i686)",
- "Links (0.9xpre12; Linux 2.2.14-5.0 i686; 80x24)",
- "Links (2.xpre7; Linux 2.4.18 i586; x)",
- "Links - http://gossamer-threads.com/scripts/links/",
- "Links 2.0 (http://gossamer-threads.com/scripts/links/)",
- "Links SQL (http://gossamer-threads.com/scripts/links-sql/)",
- "Links4US-Crawler, (+http://links4us.com/)",
- "LinkScan/11.0beta2 UnixShareware robot from Elsop.com (used by Indiafocus/Indiainfo)",
- "LinkScan/9.0g Unix",
- "LinkScan/x.x Unix",
- "LinksManager.com (http://linksmanager.com/linkchecker.html)",
- "LinkSonar/1.35",
- "LinkSweeper/1.x",
- "LinkWalker",
- "link_check3.plx libwww-perl/5.65",
- "ListBidBot (freelance job spider http://listbid.com)Freelance",
- "LiveTrans/Nutch-0.9 (maintainer: cobain at iis dot sinica dot edu dot tw; http://wkd.iis.sinica.edu.tw/LiveTrans/)",
- "Llaut/1.0 (http://mnm.uib.es/~gallir/llaut/bot.html)",
- "LMQueueBot/0.2",
- "lmspider (lmspider@scansoft.com)",
- "LNSpiderguy",
- "LocalBot/1.0 ( http://www.localbot.co.uk/)",
- "LocalcomBot/1.2.x ( http://www.local.com/bot.htm)",
- "Lockstep Spider/1.0",
- "Look.com",
- "Lotus-Notes/4.5 ( Windows-NT )",
- "LotusDiscovery/x.0 (compatible; Mozilla 4.0; MSIE 4.01; Windows NT)",
- "Lovel as 1.0 ( +http://www.everatom.com)",
- "LTI/LemurProject Nutch Spider/Nutch-1.0-dev (lti crawler for CMU; http://www.lti.cs.cmu.edu; changkuk at cmu dot edu)",
- "LTI/LemurProject Nutch Spider/Nutch-1.0-dev (Research spider using Nutch; http://www.lemurproject.org; mhoy@cs.cmu.edu)",
- "luchs.at URL checker",
- "Lunascape",
- "lwp-trivial/1.32",
- "lwp-trivial/1.34",
- "lwp-trivial/1.35",
- "LWP::Simple/5.22",
- "LWP::Simple/5.36",
- "LWP::Simple/5.48",
- "LWP::Simple/5.50",
- "LWP::Simple/5.51",
- "LWP::Simple/5.53",
- "LWP::Simple/5.63",
- "LWP::Simple/5.803",
- "Lycos_Spider_(modspider)",
- "Lycos_Spider_(T-Rex)",
- "Lynx/2-4-2 (Bobcat/0.5 [DOS] Jp Beta04)",
- "Lynx/2.6 libwww-FM/2.14",
- "Lynx/2.8 (;http://seebot.org)",
- "Lynx/2.8.3dev.9 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6",
- "Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6c (human-guided@lerly.net)",
- "Mac Finder 1.0.xx",
- "Mackster( http://www.ukwizz.com )",
- "Mag-Net",
- "MagicWML/1.0 (forcewml)",
- "MagpieRSS/0.7x (+http://magpierss.sf.net)",
- "Mahiti.Com/Mahiti Crawler-1.0 (Mahiti.Com; http://mahiti.com ; mahiti.com)",
- "Mail.Ru/1.0",
- "mailto:webcraft@bea.com",
- "mammoth/1.0 ( http://www.sli-systems.com/)",
- "MantraAgent",
- "MapoftheInternet.com ( http://MapoftheInternet.com)",
- "Mariner/5.1b [de] (Win95; I ;Kolibri gncwebbot)",
- "Marketwave Hit List",
- "Martini",
- "MARTINI",
- "Marvin v0.3",
- "MaSagool/1.0 (MaSagool; http://sagool.jp/; info@sagool.jp)",
- "Mass Downloader 2.x",
- "MasterSeek",
- "Mata Hari/2.00 ",
- "Matrix S.p.A. - FAST Enterprise Crawler 6 (Unknown admin e-mail address)",
- "maxomobot/dev-20051201 (maxomo; http://67.102.134.34:4047/MAXOMO/MAXOMObot.html; maxomobot@maxomo.com)",
- "McBot/5.001 (windows; U; NT4.0; en-us)",
- "MDbot/1.0 (+http://www.megadownload.net/bot.html)",
- "Media Player Classic",
- "MediaCrawler-1.0 (Experimental)",
- "Mediapartners-Google/2.1 ( http://www.googlebot.com/bot.html)",
- "MediaSearch/0.1",
- "MegaSheep v1.0 (www.searchuk.com internet sheep)",
- "Megite2.0 (http://www.megite.com)",
- "Mercator-1.x",
- "Mercator-2.0",
- "Mercator-Scrub-1.1",
- "Metaeuro Web Crawler/0.2 (MetaEuro Web Search Clustering Engine; http://www.metaeuro.com; crawler at metaeuro dot com)",
- "MetaGer-LinkChecker",
- "MetagerBot/0.8-dev (MetagerBot; http://metager.de; )",
- "MetaGer_PreChecker0.1",
- "MetaProducts Download Express/1.x",
- "Metaspinner/0.01 (Metaspinner; http://www.meta-spinner.de/; support@meta-spinner.de/)",
- "metatagsdir/0.7 (+http://metatagsdir.com/directory/)",
- "MFC Foundation Class Library 4.0",
- "MFC_Tear_Sample",
- "MFHttpScan",
- "MicroBaz",
- "Microsoft Data Access Internet Publishing Provider Cache Manager",
- "Microsoft Data Access Internet Publishing Provider DAV",
- "Microsoft Data Access Internet Publishing Provider Protocol Discovery",
- "Microsoft Log Parser 2.2",
- "Microsoft Small Business Indexer",
- "Microsoft URL Control - 6.00.8xxx",
- "MicrosoftPrototypeCrawler (How's my crawling? mailto:newbiecrawler@hotmail.com)",
- "Microsoft_Internet_Explorer_5.00.438 (fjones@isd.net)",
- "MIIxpc/4.2",
- "Mindjet MindManager",
- "minibot",
- "miniRank/1.6 (Website ranking; www.minirank.com; robot)",
- "MiracleAlphaTest",
- "Missauga Locate 1.0.0",
- "Missigua Locator 1.9",
- "Missouri College Browse",
- "Mister Pix II 2.02a",
- "Mister PiX version.dll",
- "Misterbot-Nutch/0.7.1 (Misterbot-Nutch; http://www.misterbot.fr; admin@misterbot.fr)",
- "Miva (AlgoFeedback@miva.com)",
- "Mizzu Labs 2.2",
- "MJ12bot/vx.x.x (http://majestic12.co.uk/bot.php?+)",
- "MJ12bot/vx.x.x (http://www.majestic12.co.uk/projects/dsearch/mj12bot.php)",
- "MJBot (SEO assessment)",
- "MLBot (www.metadatalabs.com)",
- "MnogoSearch/3.2.xx",
- "Mo College 1.9",
- "moget/x.x (moget@goo.ne.jp)",
- "mogimogi/1.0",
- "moiNAG 0.02",
- "MojeekBot/0.x (archi; http://www.mojeek.com/bot.html)",
- "monkeyagent",
- "MoonBrowser (version 0.41 Beta4)",
- "Moreoverbot/x.00 (+http://www.moreover.com)",
- "Morris - Mixcat Crawler ( http://mixcat.com)",
- "Motoricerca-Robots.txt-Checker/1.0 (http://tool.motoricerca.info/robots-checker.phtml)",
- "Motorola-V3m Obigo",
- "Mouse-House/7.4 (spider_monkey spider info at www.mobrien.com/sm.shtml)",
- "MovableType/x.x",
- "mozDex/0.xx-dev (mozDex; http://www.mozdex.com/en/bot.html; spider@mozdex.com)",
- "Mozi!",
- "Mozilla",
- "Mozilla (libwhisker/2.4)",
- "Mozilla (Mozilla@somewhere.com)",
- "Mozilla 4.0(compatible; BotSeer/1.0; +http://botseer.ist.psu.edu)",
- "Mozilla/1.1 (compatible; MSPIE 2.0; Windows CE)",
- "Mozilla/1.10 [en] (Compatible; RISC OS 3.70; Oregano 1.10)",
- "Mozilla/1.22 (compatible; MSIE 2.0d; Windows NT)",
- "Mozilla/1.22 (compatible; MSIE 5.01; PalmOS 3.0) EudoraWeb 2",
- "Mozilla/2.0",
- "Mozilla/2.0 (compatible; AOL 3.0; Mac_PowerPC)",
- "Mozilla/2.0 (Compatible; AOL-IWENG 3.0; Win16)",
- "Mozilla/2.0 (compatible; Ask Jeeves)",
- "Mozilla/2.0 (compatible; Ask Jeeves/Teoma)",
- "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; http://about.ask.com/en/docs/about/webmasters.shtml) ",
- "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; http://sp.ask.com/docs/about/tech_crawling.html)",
- "Mozilla/2.0 (compatible; EZResult -- Internet Search Engine)",
- "Mozilla/2.0 (compatible; MS FrontPage x.0)",
- "Mozilla/2.0 (compatible; MSIE 2.1; Mac_PowerPC)",
- "Mozilla/2.0 (compatible; MSIE 3.02; Update a; AK; Windows NT)",
- "Mozilla/2.0 (compatible; MSIE 3.02; Update a; AOL 3.0; Windows 95)",
- "Mozilla/2.0 (compatible; MSIE 3.0; AK; Windows 95)",
- "Mozilla/2.0 (compatible; MSIE 3.0; Windows 3.1)",
- "Mozilla/2.0 (compatible; MSIE 3.0B; Win32)",
- "Mozilla/2.0 (compatible; NEWT ActiveX; Win32)",
- "Mozilla/2.0 (compatible; T-H-U-N-D-E-R-S-T-O-N-E)",
- "Mozilla/2.0 compatible; Check&Get 1.1x (Windows 98)",
- "Mozilla/2.01 (Win16; I)",
- "Mozilla/2.02Gold (Win95; I)",
- "Mozilla/3.0 (compatible)",
- "Mozilla/3.0 (compatible; AvantGo 3.2)",
- "Mozilla/3.0 (compatible; Fluffy the spider; http://www.searchhippo.com/; info@searchhippo.com)",
- "Mozilla/3.0 (compatible; HP Web PrintSmart 04b0 1.0.1.34)",
- "Mozilla/3.0 (compatible; Indy Library)",
- "Mozilla/3.0 (compatible; Linkman)",
- "Mozilla/3.0 (compatible; MuscatFerret/1.5.4; claude@euroferret.com)",
- "Mozilla/3.0 (compatible; MuscatFerret/1.5; olly@muscat.co.uk)",
- "Mozilla/3.0 (compatible; MuscatFerret/1.6.x; claude@euroferret.com)",
- "Mozilla/3.0 (compatible; netart generator/1.0; libwww-perl/5.64)",
- "Mozilla/3.0 (compatible; NetPositive/2.2)",
- "Mozilla/3.0 (compatible; Opera/3.0; Windows 3.1) v3.1",
- "Mozilla/3.0 (compatible; Opera/3.0; Windows 95/NT4) 3.2",
- "Mozilla/3.0 (compatible; PerMan Surfer 3.0; Win95)",
- "Mozilla/3.0 (compatible; REL Software Web Link Validator 2.x)",
- "Mozilla/3.0 (compatible; scan4mail (advanced version) http://www.peterspages.net/?scan4mail)",
- "Mozilla/3.0 (compatible; ScollSpider; http://www.webwobot.com)",
- "Mozilla/3.0 (compatible; Web Link Validator 2.x)Web Link Validator http://www.relsoftware.com/ link validation software",
- "Mozilla/3.0 (compatible; WebCapture x.x; Auto; Windows)",
- "Mozilla/3.0 (compatible; Webinator-DEV01.home.iprospect.com/2.56)",
- "Mozilla/3.0 (compatible; Webinator-indexer.cyberalert.com/2.56)",
- "Mozilla/3.0 (Compatible;Viking/1.8)",
- "Mozilla/3.0 (DreamPassport/3.0)",
- "Mozilla/3.0 (INGRID/3.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)",
- "Mozilla/3.0 (Liberate DTV 1.1)",
- "Mozilla/3.0 (Planetweb/2.100 JS SSL US; Dreamcast US)",
- "Mozilla/3.0 (Slurp.so/Goo; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
- "Mozilla/3.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
- "Mozilla/3.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
- "Mozilla/3.0 (Vagabondo/1.1 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)",
- "Mozilla/3.0 (Vagabondo/1.x MT; webagent@wise-guys.nl; http://webagent.wise-guys.nl/)",
- "Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)",
- "Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)",
- "Mozilla/3.0 (Win16; I)",
- "Mozilla/3.0 (Win95; I)",
- "Mozilla/3.0 (WinNT; I)",
- "Mozilla/3.0 (WorldGate Gazelle 3.5.1 build 11; FreeBSD2.2.8-STABLE)",
- "Mozilla/3.0 (X11; I; OSF1 V4.0 alpha)",
- "Mozilla/3.0 NAVIO_AOLTV (11; 13; Philips; PH200; 1; R2.0C36_AOL.0110OPTIK; R2.0.0139d_OPTIK)",
- "Mozilla/3.0 WebTV/1.2 (compatible; MSIE 2.0)",
- "Mozilla/3.01 (compatible; AmigaVoyager/2.95; AmigaOS/MC680x0)",
- "Mozilla/3.01 (Compatible; Links2Go Similarity Engine)",
- "Mozilla/3.01 (compatible; Netbox/3.5 R92; Linux 2.2)",
- "Mozilla/3.01-C-MACOS8 (Macintosh; I; PPC)",
- "Mozilla/3.01Gold (X11; I; Linux 2.0.32 i486)",
- "Mozilla/3.01Gold (X11; I; SunOS 5.5.1 sun4m)",
- "Mozilla/3.01SGoldC-SGI (X11; I; IRIX 6.3 IP32)",
- "Mozilla/3.04 (compatible; ANTFresco/2.13; RISC OS 4.02)",
- "Mozilla/3.04 (compatible; NCBrowser/2.35; ANTFresco/2.17; RISC OS-NC 5.13 Laz1UK1309)",
- "Mozilla/3.04 (compatible;QNX Voyager 2.03B ;Photon)",
- "Mozilla/3.x (I-Opener 1.1; Netpliance)",
- "Mozilla/4.0",
- "Mozilla/4.0 (agadine3.0) www.agada.de",
- "Mozilla/4.0 (Compatible); URLBase 6",
- "Mozilla/4.0 (compatible: AstraSpider V.2.1 : astrafind.com)",
- "Mozilla/4.0 (compatible; Vagabondo/2.2; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)",
- "Mozilla/4.0 (compatible; Vagabondo/4.0Beta; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)",
- "Mozilla/4.0 (compatible; ReGet Deluxe 5.1; Windows NT 5.1)",
- "Mozilla/4.0 (compatible; Advanced Email Extractor v2.xx)",
- "Mozilla/4.0 (compatible; Arachmo)",
- "Mozilla/4.0 (compatible; BorderManager 3.0)",
- "Mozilla/4.0 (compatible; BOTW Spider; +http://botw.org)",
- "Mozilla/4.0 (compatible; B_L_I_T_Z_B_O_T)",
- "Mozilla/4.0 (compatible; Cerberian Drtrs Version-3.2-Build-0)",
- "Mozilla/4.0 (compatible; Check&Get 3.0; Windows NT)",
- "Mozilla/4.0 (compatible; ChristCrawler.com ChristCrawler@ChristCENTRAL.com)",
- "Mozilla/4.0 (compatible; crawlx, crawler@trd.overture.com)",
- "Mozilla/4.0 (compatible; DAUMOA-video; +http://ws.daum.net/aboutkr.html)",
- "Mozilla/4.0 (compatible; DepSpid/5.0x; +http://about.depspid.net)",
- "Mozilla/4.0 (compatible; DnloadMage 1.0)",
- "Mozilla/4.0 (compatible; FastCrawler3 support-fastcrawler3@fast.no)",
- "Mozilla/4.0 (compatible; FDSE robot)",
- "Mozilla/4.0 (compatible; GPU p2p crawler http://gpu.sourceforge.net/search_engine.php)",
- "Mozilla/4.0 (compatible; grub-client-0.2.x; Crawl your stuff with http://grub.org)",
- "Mozilla/4.0 (compatible; grub-client-0.3.x; Crawl your own stuff with http://grub.org)",
- "Mozilla/4.0 (compatible; grub-client-2.x)",
- "Mozilla/4.0 (compatible; ibisBrowser)",
- "Mozilla/4.0 (compatible; ICS 1.2.xxx)",
- "Mozilla/4.0 (compatible; IE-Favorites-Check-0.5)",
- "Mozilla/4.0 (compatible; Iplexx Spider/1.0 http://www.iplexx.at)",
- "Mozilla/4.0 (compatible; KeepNI web site monitor)",
- "Mozilla/4.0 (compatible; Link Utility; http://net-promoter.com)",
- "Mozilla/4.0 (compatible; Lotus-Notes/5.0; Windows-NT)",
- "Mozilla/4.0 (compatible; MSIE 4.01; AOL 4.0; Windows 98)",
- "Mozilla/4.0 (compatible; MSIE 4.01; Mac_PowerPC)",
- "Mozilla/4.0 (compatible; MSIE 4.01; MSIECrawler; Windows 95)",
- "Mozilla/4.0 (compatible; MSIE 4.01; Vonna.com b o t)",
- "Mozilla/4.0 (compatible; MSIE 4.01; Windows 95)",
- "Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; MSN Companion 2.0; 800x600; Compaq)",
- "Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; PPC; 240x320; SPV M700; OpVer 19.123.2.733) OrangeBot-Mobile 2008.0 (mobilesearch.support@orange-ftgroup.com)",
- "Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; PPS; 240x320)",
- "Mozilla/4.0 (compatible; MSIE 4.01; Windows NT Windows CE)",
- "Mozilla/4.0 (compatible; MSIE 4.01; Windows NT)",
- "Mozilla/4.0 (compatible; MSIE 4.01; Windows NT; MS Search 4.0 Robot) Microsoft",
- "Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) ACR",
- "Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) Indonesia Interactive",
- "Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) WebQuest Designs",
- "Mozilla/4.0 (compatible; MSIE 5.01; Windows 95) via Avirt Gateway Server v4.0",
- "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0) (samualt9@bigfoot.com)",
- "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; NetCaptor 6.5.0RC1)",
- "Mozilla/4.0 (compatible; MSIE 5.0; AOL 5.0; Windows 95; DigExt; Gateway2000; sureseeker.com)",
- "Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC; AtHome021)",
- "Mozilla/4.0 (compatible; MSIE 5.0; NetNose-Crawler 2.0; A New Search Experience: http://www.netnose.com)",
- "Mozilla/4.0 (compatible; MSIE 5.0; Win32) via proxy gateway CERN-HTTPD/3.0 libwww/2.17",
- "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) TrueRobot; 1.5",
- "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot BETA 1.2 (http://www.voila.com/)",
- "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot; 1.6",
- "Mozilla/4.0 (compatible; MSIE 5.0; Windows ME) Opera 5.11 [en]",
- "Mozilla/4.0 (compatible; MSIE 5.0; Windows ME; Link Checker 2.x.xx http://www.kyosoft.com)",
- "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent",
- "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; http://www.girafa.com)",
- "Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com)",
- "Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com/; http://www.galaxy.com/info/crawler.html)",
- "Mozilla/4.0 (compatible; MSIE 5.0; YANDEX)",
- "Mozilla/4.0 (compatible; MSIE 5.5; AOL 4.0; Windows 98; GoBeez (www.gobeez.com))",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows 95; Transmission Segment; Hotbar 2.0)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Crazy Browser 1.x.x)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; KITV4.7 Wanadoo)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; SAFEXPLORER TL)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; SYMPA; Katiesoft 7; SimulBrowse 3.0)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90; BTinternet V8.1)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90; MSIECrawler)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; obot)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; QXW03018)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Active Cache Request",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Fetch API Request",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AIRF)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AspTear 1.5)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; N_o_k_i_a)",
- "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; T312461) RPT-HTTPClient/0.3-3E",
- "Mozilla/4.0 (compatible; MSIE 6.0 compatible; Asterias Crawler v4; +http://www.singingfish.com/help/spider.html; webmaster@singingfish.com); SpiderThread Revision: 3.10",
- "Mozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows 98; .NET CLR 1.1.4322; MEGAUPLOAD 2.0)",
- "Mozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows NT 5.1; SV1; HbTools 4.7.2)",
- "Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Skampy/0.9.x [en]",
- "Mozilla/4.0 (compatible; MSIE 6.0; TargetSeek/1.0; +http://www.targetgroups.net/TargetSeek.html)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Win32) WebWasher 3.0",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98) REL Software Web Link Validator 2.x)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98) Web Link Validator 2.x)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Net M@nager V3.02 - www.vinn.com.au)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; support@illumit.com; http://www.illumit.com/Products/weblight/)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x 4.90; http://www.Abolimba.de)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; Lunascape 2.1.3)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Google Wireless Transcoder;)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ODP entries t_st; http://tuezilla.de/t_st-odp-entries-agent.html)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ODP links test; http://tuezilla.de/test-odp-links-agent.html)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ZoomSpider.net bot; .NET CLR 1.1.4322)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) (dns_admin@c-a-s-h.com)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Covac UPPS Cathan 1.2.5;)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Crayon Crawler; snprtz|T04056566514940; (R1 1.5))",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Deepnet Explorer)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; heritrix/1.3.0 http://www.cs.washington.edu/research/networking/websys/)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Hotbar 3.0)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iOpus-I-M)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iRider 2.21.1108; FDM)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; KKman3.0)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MathPlayer2.0)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon) ",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; PeoplePal 3.0; MSIECrawler)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461; IOpener Release 1.1.04)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; QihooBot 1.0 qihoobot@qihoo.net)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SIMBAR Enabled; InfoPath.1)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; StumbleUpon.com 1.760; .NET CLR 1.1.4322)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Embedded Web Browser from: http://bsalsa.com/; MSIECrawler)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; DX-Browser 5.0.0.0)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FunWebProducts; ezPeer+ v1.0 Beta (0.4.1.98); ezPeer+ v1.0 (0.5.0.00); .NET CLR 1.1.4322; MSIECrawler)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; IBP; .NET CLR 1.1.4322)",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; MRA 4.3 (build 01218))",
- "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)",
- "Mozilla/4.0 (compatible; MSIE 7.0; Win32) Link Commander 4.0",
- "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; bgft)",
- "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; GTB5; User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://bsalsa.com) ; .NET CLR 2.0.50727)",
- "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Tablet PC 2.0)",
- "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 1.1.4322; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648)",
- "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)",
- "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; Orange 8.0; GTB6.3; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; Embedded Web Browser from: http://bsalsa.com/; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30618; OfficeLiveConnector.1.3; OfficeLivePatch.1.3)",
- "Mozilla/4.0 (compatible; MSIE enviable; DAUMOA 2.0; DAUM Web Robot; Daum Communications Corp., Korea; +http://ws.daum.net/aboutkr.html)",
- "Mozilla/4.0 (compatible; MSIE is not me; DAUMOA/1.0.1; DAUM Web Robot; Daum Communications Corp., Korea)",
- "Mozilla/4.0 (compatible; NaverBot/1.0; http://help.naver.com/delete_main.asp)",
- "Mozilla/4.0 (compatible; Netcraft Web Server Survey)",
- "Mozilla/4.0 (compatible; NetPromoter Spider;http://www.net-promoter.com/)",
- "Mozilla/4.0 (compatible; Opera/3.0; Windows 4.10) 3.51 [en]",
- "Mozilla/4.0 (compatible; Powermarks/3.5; Windows 95/98/2000/NT)",
- "Mozilla/4.0 (compatible; RSS Popper)",
- "Mozilla/4.0 (compatible; SiteKiosk 4.0; MSIE 5.0; Windows 98; SiteCoach 1.0)",
- "Mozilla/4.0 (compatible; SpeedySpider; www.entireweb.com)",
- "Mozilla/4.0 (compatible; SPENG)",
- "Mozilla/4.0 (compatible; SuperCleaner 2.xx; Windows 98)",
- "Mozilla/4.0 (compatible; Synapse)",
- "Mozilla/4.0 (compatible; WebCapture 3.0; Windows)",
- "Mozilla/4.0 (compatible; Win32; WinHttp.WinHttpRequest.5)",
- "Mozilla/4.0 (compatible; WSN Links)",
- "Mozilla/4.0 (compatible; www.euro-directory.com; urlchecker1.0)",
- "Mozilla/4.0 (compatible; www.galaxy.com)",
- "Mozilla/4.0 (compatible; www.linkguard.com Linkguard Online 1.0; Windows NT)",
- "Mozilla/4.0 (compatible; Y!J; for robot study; keyoshid)",
- "Mozilla/4.0 (compatible; Yahoo Japan; for robot study; kasugiya)",
- "Mozilla/4.0 (compatible;MSIE 6.0; Windows NT 5.0; H010818)",
- "Mozilla/4.0 (fantomBrowser)",
- "Mozilla/4.0 (fantomCrew Browser)",
- "Mozilla/4.0 (hhjhj@yahoo.com)",
- "Mozilla/4.0 (JemmaTheTourist;http://www.activtourist.com)",
- "Mozilla/4.0 (MobilePhone PM-8200/US/1.0) NetFront/3.x MMP/2.0",
- "Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)",
- "Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 FAKE (compatible; Googlebot/2.1; http://www.google.com/bot.html)",
- "Mozilla/4.0 (Mozilla; http://www.mozilla.org/docs/en/bot.html; master@mozilla.com)",
- "Mozilla/4.0 (Sleek Spider/1.2)",
- "Mozilla/4.0 compatible FurlBot/Furl Search 2.0 (FurlBot; http://www.furl.net; wn.furlbot@looksmart.net)",
- "Mozilla/4.0 compatible ZyBorg/1.0 (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)",
- "Mozilla/4.0 compatible ZyBorg/1.0 (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)",
- "Mozilla/4.0 compatible ZyBorg/1.0 Dead Link Checker (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)",
- "Mozilla/4.0 compatible ZyBorg/1.0 for Homepage (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)",
- "Mozilla/4.0 efp@gmx.net",
- "Mozilla/4.0 WebTV/2.6 (compatible; MSIE 4.0)",
- "Mozilla/4.0 [en] (Ask Jeeves Corporate Spider)",
- "Mozilla/4.0(compatible; Zealbot 1.0)",
- "Mozilla/4.01 (compatible; NORAD National Defence Network)",
- "Mozilla/4.01 [en](Win95;I)",
- "Mozilla/4.02 [en] (X11; I; SunOS 5.6 sun4u)",
- "Mozilla/4.04 (compatible; Dulance bot; +http://www.dulance.com/bot.jsp)",
- "Mozilla/4.04 [en] (X11; I; HP-UX B.10.20 9000/712)",
- "Mozilla/4.04 [en] (X11; I; IRIX 5.3 IP22)",
- "Mozilla/4.05 (Macintosh; I; 68K Nav)",
- "Mozilla/4.05 (Macintosh; I; PPC Nav)",
- "Mozilla/4.05 [en] (X11; I; SunOS 4.1.4 sun4m)",
- "Mozilla/4.08 [en] (Win98; U ;Nav)",
- "Mozilla/4.08 [en] (WinNT; U)",
- "Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_TrueRobot/1.4 libwww/5.2.8",
- "Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_VoilaBot/1.6 libwww/5.3.2",
- "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)",
- "Mozilla/4.5 (compatible; iCab 2.5.3; Macintosh; I; PPC)",
- "Mozilla/4.5 (compatible; OmniWeb/4.0.5; Mac_PowerPC)",
- "Mozilla/4.5 (compatible; OmniWeb/4.1-beta-1; Mac_PowerPC)",
- "Mozilla/4.5 RPT-HTTPClient/0.3-2",
- "Mozilla/4.5 [en]C-CCK-MCD {RuralNet} (Win98; I)",
- "Mozilla/4.5b1 [en] (X11; I; Linux 2.0.35 i586)",
- "Mozilla/4.6 [en] (http://www.cnet.com/)",
- "Mozilla/4.61 [de] (OS/2; I)",
- "Mozilla/4.61 [en] (X11; U; ) - BrowseX (2.0.0 Windows)",
- "Mozilla/4.7",
- "Mozilla/4.7 (compatible; http://eidetica.com/spider)",
- "Mozilla/4.7 (compatible; Intelliseek; http://www.intelliseek.com)",
- "Mozilla/4.7 (compatible; OffByOne; Windows 98) Webster Pro V3.2",
- "Mozilla/4.7 (compatible; Whizbang)",
- "Mozilla/4.7 (compatible; WhizBang; http://www.whizbang.com/crawler)",
- "Mozilla/4.7 [en](BecomeBot@exava.com)",
- "Mozilla/4.7 [en](Exabot@exava.com)",
- "Mozilla/4.7 [en]C-CCK-MCD {Yahoo;YIP052400} (Win95; I)",
- "Mozilla/4.72 [en] (BACS http://www.ba.be)",
- "Mozilla/4.72C-CCK-MCD Caldera Systems OpenLinux [en] (X11; U; Linux 2.2.14 i686)",
- "Mozilla/4.75C-ja [ja] (X11; U; OSF1 V5.1 alpha)",
- "Mozilla/4.76 (Windows 98; U) Opera 5.12 [en]",
- "Mozilla/4.76 [en] (X11; U; FreeBSD 4.4-STABLE i386)",
- "Mozilla/4.76 [en] (X11; U; SunOS 5.7 sun4u)",
- "Mozilla/4.77C-SGI [en] (X11; U; IRIX 6.5 IP32)",
- "Mozilla/5.0",
- "Mozilla/5.0 (+http://www.eurekster.com/mammoth) Mammoth/0.1",
- "Mozilla/5.0 (+http://www.sli-systems.com/) Mammoth/0.1",
- "Mozilla/5.0 (Clustered-Search-Bot/1.0; support@clush.com; http://www.clush.com/)",
- "Mozilla/5.0 (compatible) GM RSS Panel X",
- "Mozilla/5.0 (compatible; +http://www.evri.com/evrinid)",
- "Mozilla/5.0 (compatible; 008/0.83; http://www.80legs.com/spider.html;) Gecko/2008032620",
- "Mozilla/5.0 (compatible; Abonti/0.8 - http://www.abonti.com)",
- "Mozilla/5.0 (compatible; aiHitBot/1.0; +http://www.aihit.com/)",
- "Mozilla/5.0 (compatible; AnsearchBot/1.x; +http://www.ansearch.com.au/)",
- "Mozilla/5.0 (compatible; archive.org_bot/1.10.0 +http://www.loc.gov/minerva/crawl.html)",
- "Mozilla/5.0 (compatible; archive.org_bot/1.13.1x http://crawler.archive.org)",
- "Mozilla/5.0 (compatible; archive.org_bot/1.5.0-200506132127 http://crawler.archive.org) Hurricane Katrina",
- "Mozilla/5.0 (compatible; Ask Jeeves/Teoma; http://about.ask.com/en/docs/about/webmasters.shtml)",
- "Mozilla/5.0 (compatible; BanBots/2.0b; Fetch; +http://www.banbots.com)",
- "Mozilla/5.0 (compatible; BecomeBot/1.23; http://www.become.com/webmasters.html)",
- "Mozilla/5.0 (compatible; BecomeBot/1.xx; MSIE 6.0 compatible; http://www.become.com/webmasters.html)",
- "Mozilla/5.0 (compatible; BecomeBot/2.0beta; http://www.become.com/webmasters.html)",
- "Mozilla/5.0 (compatible; BecomeBot/2.x; MSIE 6.0 compatible; http://www.become.com/site_owners.html)",
- "Mozilla/5.0 (compatible; BecomeJPBot/2.3; MSIE 6.0 compatible; +http://www.become.co.jp/site_owners.html)",
- "Mozilla/5.0 (compatible; BlogRefsBot/0.1; http://www.blogrefs.com/about/bloggers)",
- "Mozilla/5.0 (compatible; Bot; +http://pressemitteilung.ws/spamfilter",
- "Mozilla/5.0 (compatible; BuzzRankingBot/1.0; +http://www.buzzrankingbot.com/)",
- "Mozilla/5.0 (compatible; Charlotte/1.0b; charlotte@betaspider.com)",
- "Mozilla/5.0 (compatible; Charlotte/1.0b; http://www.searchme.com/support/)",
- "Mozilla/5.0 (compatible; Crawling jpeg; http://www.yama.info.waseda.ac.jp)",
- "Mozilla/5.0 (compatible; Custo 3 (Netwu.com); Windows NT 5.1)",
- "Mozilla/5.0 (compatible; de/1.13.2 +http://www.de.com)",
- "Mozilla/5.0 (compatible; Diffbot/0.1; +http://www.diffbot.com)",
- "Mozilla/5.0 (compatible; DNS-Digger-Explorer/1.0; +http://www.dnsdigger.com)",
- "Mozilla/5.0 (compatible; DNS-Digger/1.0; +http://www.dnsdigger.com)",
- "Mozilla/5.0 (compatible; EARTHCOM.info/2.01; http://www.earthcom.info)",
- "Mozilla/5.0 (compatible; EARTHCOM/2.2; +http://enter4u.eu)",
- "Mozilla/5.0 (compatible; egothor/8.0g; +http://ego.ms.mff.cuni.cz/)",
- "Mozilla/5.0 (compatible; Exabot Test/3.0; +http://www.exabot.com/go/robot)",
- "Mozilla/5.0 (compatible; FatBot 2.0; http://www.thefind.com/main/CrawlerFAQs.fhtml)",
- "Mozilla/5.0 (compatible; Galbot/1.0; +http://www.galbot.com/bot.html)",
- "mozilla/5.0 (compatible; genevabot http://www.healthdash.com)",
- "Mozilla/5.0 (compatible; Google Desktop) Paros/3.2.12",
- "Mozilla/5.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)",
- "mozilla/5.0 (compatible; heritrix/1.0.4 http://innovationblog.com)",
- "Mozilla/5.0 (compatible; heritrix/1.10.2 +http://i.stanford.edu/)",
- "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://newstin.com/)",
- "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.page-store.com)",
- "Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.page-store.com) [email:paul@page-store.com]",
- "mozilla/5.0 (compatible; heritrix/1.3.0 http://archive.crawler.org)",
- "Mozilla/5.0 (compatible; heritrix/1.4.0 +http://www.chepi.net)",
- "Mozilla/5.0 (compatible; heritrix/1.4t http://www.truveo.com/)",
- "Mozilla/5.0 (compatible; heritrix/1.5.0 http://www.l3s.de/~kohlschuetter/projects/crawling/)",
- "Mozilla/5.0 (compatible; heritrix/1.5.0-200506231921 http://pandora.nla.gov.au/crawl.html)",
- "Mozilla/5.0 (compatible; heritrix/1.6.0 http://www.worio.com/)",
- "Mozilla/5.0 (compatible; heritrix/1.7.0 +http://www.greaterera.com/)",
- "Mozilla/5.0 (compatible; Heritrix/1.8.0 http://www.hanzoarchives.com)",
- "Mozilla/5.0 (compatible; heritrix/1.x.x +http://www.accelobot.com)",
- "Mozilla/5.0 (compatible; heritrix/2.0.0-RC1 +http://www.aol.com)",
- "Mozilla/5.0 (compatible; Hermit Search. Com; +http://www.hermitsearch.com)",
- "Mozilla/5.0 (compatible; http://www.IsMySiteUp.Net/bot/ )",
- "Mozilla/5.0 (compatible; http://www.UptimeAuditor.com/bot/ )",
- "Mozilla/5.0 (compatible; HyperixScoop/1.3; +http://www.hyperix.com)",
- "Mozilla/5.0 (compatible; iaskspider/1.0; MSIE 6.0)",
- "Mozilla/5.0 (compatible; IDBot/1.0; +http://www.id-search.org/bot.html)",
- "Mozilla/5.0 (compatible; InterseekWeb/3.x)",
- "Mozilla/5.0 (compatible; Jim +http://www.hanzoarchives.com)",
- "Mozilla/5.0 (compatible; Konqueror/2.0.1; X11); Supports MD5-Digest; Supports gzip encoding",
- "Mozilla/5.0 (compatible; Konqueror/2.1.1; X11)",
- "Mozilla/5.0 (compatible; Konqueror/2.2.2)",
- "Mozilla/5.0 (compatible; Konqueror/2.2.2; Linux 2.4.14-xfs; X11; i686)",
- "Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)",
- "Mozilla/5.0 (compatible; LemSpider 0.1)",
- "Mozilla/5.0 (compatible; LinksManager.com_bot http://linksmanager.com/linkchecker.html)",
- "Mozilla/5.0 (compatible; LinkStash Bookmark Manager; http://www.xrayz.co.uk/)",
- "Mozilla/5.0 (compatible; MojeekBot/2.0; http://www.mojeek.com/bot.html)",
- "Mozilla/5.0 (compatible; MOSBookmarks/v2.6-Plus; Link Checker)",
- "Mozilla/5.0 (compatible; MSIE 6.0; Podtech Network; crawler_admin@podtech.net)",
- "Mozilla/5.0 (compatible; OnetSzukaj/5.0; http://szukaj.onet.pl)",
- "Mozilla/5.0 (compatible; PagestackerBot; http://www.pagestacker.com)",
- "Mozilla/5.0 (compatible; PalmeraBot; http://www.links24h.com/help/palmera) Version 0.001",
- "Mozilla/5.0 (compatible; PEAR HTTP_Request class; http://feed.moo.jp/)",
- "Mozilla/5.0 (compatible; Phonifier; +http://www.phonifier.com)",
- "Mozilla/5.0 (compatible; pmoz.info ODP link checker; +http://pmoz.info/doc/botinfo.htm)",
- "Mozilla/5.0 (compatible; pogodak.ba/3.x)",
- "Mozilla/5.0 (compatible; Pogodak.hr/3.1)",
- "Mozilla/5.0 (compatible; Proximic crawler; +http://www.proximic.com/en/about-us/contact-us.html)",
- "Mozilla/5.0 (compatible; PWeBot/3.1; http://www.programacionweb.net/robot.php)",
- "Mozilla/5.0 (compatible; Quantcastbot/1.0; www.quantcast.com)",
- "Mozilla/5.0 (compatible; robtexbot/1.0; http://www.robtex.com/ )",
- "Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)",
- "Mozilla/5.0 (compatible; Scrubby/2.2; http://www.scrubtheweb.com/)",
- "Mozilla/5.0 (compatible; ShunixBot/1.x.x +http://www.shunix.com/robot.htm)",
- "Mozilla/5.0 (compatible; ShunixBot/1.x; http://www.shunix.com/bot.htm)",
- "Mozilla/5.0 (compatible; SkreemRBot +http://skreemr.com)",
- "Mozilla/5.0 (compatible; SnapPreviewBot; en-US; rv:1.8.0.9) Gecko/20061206 Firefox/1.5.0.9",
- "Mozilla/5.0 (compatible; SpurlBot/0.2) ",
- "Mozilla/5.0 (compatible; SummizeBot +http://www.summize.com)",
- "Mozilla/5.0 (compatible; SYCLIKControl/LinkChecker;)",
- "Mozilla/5.0 (compatible; Synoobot/0.9; http://www.synoo.com/search/bot.html)",
- "Mozilla/5.0 (compatible; Theophrastus/x.x; http://users.cs.cf.ac.uk/N.A.Smith/theophrastus.php)",
- "Mozilla/5.0 (compatible; TridentSpider/3.1)",
- "Mozilla/5.0 (compatible; Vagabondo/2.1; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)",
- "Mozilla/5.0 (compatible; Webduniabot/1.0; +http://search.webdunia.com/bot.aspx)",
- "Mozilla/5.0 (compatible; Windows NT 5.0; phpwebbrainBot/0.1 - http://www.monsterli.ch/phpwebbrain/)",
- "Mozilla/5.0 (compatible; worio bot heritrix/1.10.0 +http://worio.com)",
- "Mozilla/5.0 (compatible; WoW Lemmings Kathune/2.0;http://www.wowlemmings.com/kathune.html)",
- "Mozilla/5.0 (compatible; XTbot/1.0v; +http://www.externaltest.com)",
- "Mozilla/5.0 (compatible; Yahoo! DE Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
- "Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)",
- "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
- "Mozilla/5.0 (compatible; YesupBot/1.0; +http://www.yesup.net/bot.html)",
- "Mozilla/5.0 (compatible; Yoono; http://www.yoono.com/)",
- "Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )",
- "Mozilla/5.0 (compatible; Zenbot/1.3; +http://zen.co.za/webmasters/)",
- "Mozilla/5.0 (compatible; zermelo +http://www.powerset.com) [email:paul@page-store.com,crawl@powerset.com]",
- "Mozilla/5.0 (compatible;archive.org_bot/1.7.1; collectionId=316; Archive-It; +http://www.archive-it.org)",
- "Mozilla/5.0 (compatible;archive.org_bot/heritrix-1.9.0-200608171144 +http://pandora.nla.gov.au/crawl.html)",
- "Mozilla/5.0 (compatible;FindITAnswersbot/1.0;+http://search.it-influentials.com/bot.htm)",
- "Mozilla/5.0 (compatible;MAINSEEK_BOT)",
- "Mozilla/5.0 (Gecko/20070310 Mozshot/0.0.20070628; http://mozshot.nemui.org/)",
- "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9b5) Gecko/2008032619 Firefox/3.0b5",
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20021219 Chimera/0.6 ",
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20030306 Camino/0.7",
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/xx (KHTML like Gecko) OmniWeb/v5xx.xx",
- "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/xxx.x (KHTML like Gecko) Safari/12x.x",
- "Mozilla/5.0 (Macintosh; U; PPC; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1",
- "Mozilla/5.0 (research@mediatrec.com)",
- "Mozilla/5.0 (Sage)",
- "Mozilla/5.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
- "Mozilla/5.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
- "Mozilla/5.0 (SunOS 5.8 sun4u; U) Opera 5.0 [en]",
- "Mozilla/5.0 (Twiceler-0.9 http://www.cuill.com/twiceler/robot.html)",
- "Mozilla/5.0 (Version: xxxx Type:xx)",
- "Mozilla/5.0 (wgao@genieknows.com)",
- "Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1",
- "Mozilla/5.0 (Windows; U; Win98; en-US; rv:x.xx) Gecko/20030423 Firebird Browser/0.6",
- "Mozilla/5.0 (Windows; U; Win9x; en; Stable) Gecko/20020911 Beonex/0.8.1-stable",
- "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.7) NimbleCrawler 1.11 obeys UserAgent NimbleCrawler For problems contact: crawler_at_dataalchemy.com",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.2.153.1 Safari/525.19",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.5) Gecko/20060731 Firefox/1.5.0.5 Flock/0.7.4.1",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.4/Megaupload x.0",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.1) Gecko/2008092215 Firefox/3.0.1 Orca/1.1 beta 3",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.x.x) Gecko/20041107 Firefox/x.x",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xx) Gecko/20030504 Mozilla Firebird/0.6",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xxx) Gecko/20041027 Mnenhy/0.6.0.104",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",
- "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5",
- "Mozilla/5.0 (Windows; U;XMPP Tiscali Communicator v.10.0.1; Windows NT 5.1; it; rv:1.8.1.3) Gecko/20070309 Firefox/2.0.0.3",
- "Mozilla/5.0 (Windows;) NimbleCrawler 1.12 obeys UserAgent NimbleCrawler For problems contact: crawler@health",
- "Mozilla/5.0 (Windows;) NimbleCrawler 1.12 obeys UserAgent NimbleCrawler For problems contact: crawler@healthline.com",
- "Mozilla/5.0 (X11; Linux i686; U;rv: 1.7.13) Gecko/20070322 Kazehakase/0.4.4.1",
- "Mozilla/5.0 (X11; U; Linux 2.4.2-2 i586; en-US; m18) Gecko/20010131 Netscape6/6.01",
- "Mozilla/5.0 (X11; U; Linux i686; de-AT; rv:1.8.0.2) Gecko/20060309 SeaMonkey/1.0",
- "Mozilla/5.0 (X11; U; Linux i686; en-GB; rv:1.7.6) Gecko/20050405 Epiphany/1.6.1 (Ubuntu) (Ubuntu package 1.0.2)",
- "Mozilla/5.0 (X11; U; Linux i686; en-US; Nautilus/1.0Final) Gecko/20020408",
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:0.9.3) Gecko/20010801",
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2.1; aggregator:Spinn3r (Spinn3r 3.1); http://spinn3r.com/robot) Gecko/20021130",
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2b) Gecko/20021007 Phoenix/0.3",
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040413 Epiphany/1.2.1",
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.7) Gecko/20060909 Firefox/1.5.0.7 SnapPreviewBot",
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1) Gecko/20061129 BonEcho/2.0",
- "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.1) Gecko/20061205 Iceweasel/2.0.0.1 (Debian-2.0.0.1+dfsg-2)",
- "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9a8) Gecko/2007100619 GranParadiso/3.0a8",
- "Mozilla/5.0 Galeon/1.0.2 (X11; Linux i686; U;) Gecko/20011224",
- "Mozilla/5.0 gURLChecker/0.x.x (Linux)",
- "Mozilla/5.0 URL-Spider",
- "Mozilla/5.0 usww.com-Spider-for-w8.net",
- "Mozilla/5.0 wgao@genieknows.com",
- "Mozilla/5.0 whoiam [http://www.axxus.de/]",
- "Mozilla/5.0 [en] (compatible; Gulper Web Bot 0.2.4 www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)",
- "MQbot metaquerier.cs.uiuc.edu/crawler",
- "MQBOT/Nutch-0.9-dev (MQBOT Nutch Crawler; http://falcon.cs.uiuc.edu; mqbot@cs.uiuc.edu)",
- "MSFrontPage/4.0",
- "MSIE 4.0 (Win95)",
- "MSIE-5.13 (larbin@unspecified.mail)",
- "msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)",
- "msnbot-Products/1.0 (+http://search.msn.com/msnbot.htm)",
- "MSNBOT/0.xx (http://search.msn.com/msnbot.htm)",
- "msnbot/x.xx ( http://search.msn.com/msnbot.htm)",
- "MSNBOT_Mobile MSMOBOT Mozilla/2.0 (compatible; MSIE 4.02; Windows CE; Default)",
- "MSNPTC/1.0",
- "MSProxy/2.0",
- "MSRBOT",
- "MSRBOT (http://research.microsoft.com/research/sv/msrbot)",
- "Mulder, VCR-1.0",
- "multiBlocker browser",
- "multicrawler ( http://sw.deri.org/2006/04/multicrawler/robots.html)",
- "MultiText/0.1",
- "MusicWalker2.0 ( http://www.somusical.com)",
- "MVAClient",
- "My WinHTTP Connection",
- "myDaemon",
- "MyGetRight/1.0.0",
- "MyGetRight/1.0b",
- "Mylinea.com Crawler 2.0",
- "mylinkcheck/1.02",
- "Naamah 1.0.1/Blogbot (http://blogbot.de/)",
- "Naamah 1.0a/Blogbot (http://blogbot.de/)",
- "NABOT/5.0",
- "nabot_1.0",
- "NameOfAgent (CMS Spider)",
- "naoFavicon4IE/1.xx",
- "NASA Search 1.0",
- "NationalDirectory-WebSpider/1.3",
- "NationalDirectoryAddURL/1.0",
- "NaverBot-1.0 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)",
- "NaverBot_dloader/1.5",
- "NavissoBot",
- "NavissoBot/1.7 (+http://navisso.com/)",
- "NCSA Beta 1 (http://vias.ncsa.uiuc.edu/viasarchivinginformation.html)",
- "Nebullabot/2.2 (http://bot.nebulla.info)",
- "NEC Research Agent -- compuman at research.nj.nec.com",
- "NEC-Hayek/1.0",
- "Net-Seekr Bot/Net-Seekr Bot V1 (http://www.net-seekr.com)",
- "NetAnts/1.2x",
- "NETCOMplete/x.xx",
- "NetinfoBot/1.0 (http://netinfo.bg/netinfobot.html)",
- "NetLookout/2.24",
- "Netluchs/0.8-dev ( ; http://www.netluchs.de/; ___don't___spam_me_@netluchs.de)",
- "NetMechanic Vx.0",
- "NetNewsWire/2.x (Mac OS X; http://ranchero.com/netnewswire/)",
- "NetNoseCrawler/v1.0",
- "Netprospector JavaCrawler",
- "NetPumper/x.xx",
- "NetResearchServer(http://www.look.com)",
- "NetResearchServer/x.x(loopimprovements.com/robot.html)",
- "NetSeer/Nutch-0.9 (NetSeer Crawler; http://www.netseer.com; crawler@netseer.com)",
- "NetSprint -- 2.0",
- "NetWhatCrawler/0.06-dev (NetWhatCrawler from NetWhat.com; http://www.netwhat.com; support@netwhat.com)",
- "NetZippy",
- "NeuralBot/0.2",
- "newsearchengine (ThisUser@unspecified.mail)",
- "NewsGator FetchLinks extension/0.2.0 (http://graemef.com)",
- "NewsGatorOnline/2.0 (http://www.newsgator.com; 1 subscribers)",
- "NextGenSearchBot 1 (for information visit http://www.eliyon.com/NextGenSearchBot)",
- "NextopiaBOT (+http://www.nextopia.com) distributed crawler client beta v0.x",
- "NG-Search/0.90 (NG-SearchBot; http://www.ng-search.com; )",
- "NG/1.0",
- "NG/4.0.1229",
- "nicebot",
- "NICO/1.0",
- "Nikita the Spider (http://NikitaTheSpider.com/)",
- "NITLE Blog Spider/0.01",
- "Nitro Downloader 1.x (www.klsofttools.com)",
- "Noago Spider",
- "Nocilla/1.0",
- "Nokia-WAPToolkit/1.2 googlebot(at)googlebot.com",
- "Nokia6300/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1 (botmobi http://find.mobi/bot.html abuse@mtld.mobi)",
- "Nokia6610/1.0 (3.09) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible;YahooSeeker/M1A1-R2D2; http://help.yahoo.com/help/us/ysearch/crawling/crawling-01.html)",
- "Nokia7110/1.0 (05.01) (Google WAP Proxy/1.0)",
- "NokodoBot/1.x (+http://nokodo.com/bot.htm)",
- "Norbert the Spider(Burf.com)",
- "noxtrumbot/1.0 (crawler@noxtrum.com)",
- "noyona_0_1",
- "NP/0.1 (NP; http://www.nameprotect.com; npbot@nameprotect.com)",
- "NPBot (http://www.nameprotect.com/botinfo.html)",
- "NPBot-1/2.0",
- "Nsauditor/1.x",
- "NSPlayer/10.0.0.xxxx WMFSDK/10.0",
- "nsyght.com/Nutch-1.0-dev (nsyght.com; Nsyght.com)",
- "nsyght.com/Nutch-x.x (nsyght.com; search.nsyght.com)",
- "nttdirectory_robot/0.9 (super-robot@super.navi.ocn.ne.jp)",
- "Nucleus SiteList LinkChecker/1.1",
- "nuSearch Spider www.nusearch.com (compatible; MSIE 4.01)",
- "NuSearch Spider (compatible; MSIE 6.0)",
- "NuSearch Spider www.nusearch.com",
- "Nutch",
- "Nutch crawler/Nutch-0.9 (picapage.com; admin@picapage.com)",
- "Nutch/Nutch-0.9 (Eurobot; http://www.ayell.eu )",
- "NutchCVS/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)",
- "NutchCVS/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)",
- "NutchCVS/0.7.1 (Nutch running at UW; http://www.nutch.org/docs/en/bot.html; sycrawl@cs.washington.edu)",
- "NutchEC2Test/Nutch-0.9-dev (Testing Nutch on Amazon EC2.; http://lucene.apache.org/nutch/bot.html; ec2test at lucene.com)",
- "NutchOrg/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)",
- "nutchsearch/Nutch-0.9 (Nutch Search 1.0; herceg_novi at yahoo dot com)",
- "NutchVinegarCrawl/Nutch-0.8.1 (Vinegar; http://www.cs.washington.edu; eytanadar at gmail dot com)",
- "obidos-bot (just looking for books.)",
- "ObjectsSearch/0.01-dev (ObjectsSearch;http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)",
- "ObjectsSearch/0.0x (ObjectsSearch; http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)",
- "oBot ((compatible;Win32))",
- "Ocelli/1.x (http://www.globalspec.com/Ocelli)",
- "Octopus",
- "Octora Beta - www.octora.com",
- "Octora Beta Bot - www.octora.com",
- "Offline Explorer 1.*",
- "OliverPerry",
- "OmniExplorer_Bot/1.0x (+http://www.omni-explorer.com) Internet CategorizerOmniExplorer http://www.omni-explorer.com/ car & shopping search (64.62.175.xxx)",
- "OmniExplorer_Bot/1.0x (+http://www.omni-explorer.com) Job Crawler",
- "OmniExplorer_Bot/1.1x (+http://www.omni-explorer.com) Torrent Crawler",
- "OmniExplorer_Bot/x.xx (+http://www.omni-explorer.com) WorldIndexer",
- "onCHECK Linkchecker von www.scientec.de fuer www.onsinn.de",
- "onCHECK-Robot, www.onsearch.de",
- "Onet.pl SA- http://szukaj.onet.pl",
- "online link validator (http://www.dead-links.com/)",
- "Online24-Bot (Version: 1.0x, powered by www.online24.de)",
- "OntoSpider/1.0 libwww-perl/5.65",
- "OOZBOT/0.20 ( http://www.setooz.com/oozbot.html ; agentname at setooz dot_com )",
- "OpenAcoon v4.0.x (www.openacoon.de)",
- "Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)",
- "Openfind data gatherer- Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)",
- "Openfind Robot/1.1A2",
- "OpenISearch/1.x (www.openisearch.com)",
- "OpenTaggerBot (http://www.opentagger.com/opentaggerbot.htm)",
- "OpenTextSiteCrawler/2.9.2",
- "OpenWebSpider/0.x.x (http://www.openwebspider.org)",
- "OpenWebSpider/x",
- "Opera/5.0 (Linux 2.0.38 i386; U) [en]",
- "Opera/5.11 (Windows ME; U) [ru]",
- "Opera/5.12 (Windows 98; U) [en]",
- "Opera/6.01 (larbin@unspecified.mail)",
- "Opera/6.x (Linux 2.4.8-26mdk i686; U) [en]",
- "Opera/6.x (Windows NT 4.0; U) [de]",
- "Opera/7.x (Windows NT 5.1; U) [en]",
- "Opera/8.xx (Windows NT 5.1; U; en)",
- "Opera/9.0 (Windows NT 5.1; U; en)",
- "Opera/9.00 (Windows NT 5.1; U; de)",
- "Opera/9.60 (Windows NT 5.1; U; de) Presto/2.1.1",
- "OpidooBOT (larbin2.6.3@unspecified.mail)",
- "OPWV-SDK UP.Browser/7.0.2.3.119 (GUI) MMP/2.0 Push/PO",
- "Oracle Application Server Web Cache 10g",
- "Oracle iMTCrawler",
- "Oracle Ultra Search",
- "OrangeSpider",
- "Orbiter/T-2.0 (+http://www.dailyorbit.com/bot.htm)",
- "Orca Browser (http://www.orcabrowser.com)",
- "OSSProxy 1.3.305.321 (Build 305.321 Win32 en-us)(Dec 21 2005 16:30:54)",
- "OutfoxBot/0.x (For internet experiments; http://; outfox.agent@gmail.com)",
- "OutfoxMelonBot/0.5 (for internet experiments; http://; outfoxbot@gmail.com)",
- "Overture-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
- "OWR_Crawler 0.1",
- "ozelot/2.7.3 (Search engine indexer; www.flying-cat.de/ozelot; ozelot@flying-cat.de)",
- "PADLibrary Spider",
- "PageBitesHyperBot/600 (http://www.pagebites.com/)",
- "Pagebull http://www.pagebull.com/",
- "Pagestacker Bot",
- "page_verifier (http://www.securecomputing.com/goto/pv)",
- "PagmIEDownload",
- "parallelContextFocusCrawler1.1parallelContextFocusCrawler1.1",
- "ParaSite/1.0b (http://www.ianett.com/parasite/)",
- "Patwebbot (http://www.herz-power.de/technik.html)",
- "pavuk/0.9pl29b i686-pc-linux-gnu",
- "PBrowse 1.4b",
- "pd02_1.0.0 pd02_1.0.0@dzimi@post.sk",
- "PEAR HTTP_Request class ( http://pear.php.net/ )",
- "PEERbot www.peerbot.com",
- "PeopleChat/Search_Engine",
- "PEval 1.4b",
- "PHP/3.x.xx",
- "PHP/4.0.4pl1",
- "PHP/4.0.6",
- "PHP/4.1.1",
- "PHP/4.1.2",
- "PicoSearch/1.0",
- "Piffany_Web_Scraper_v0.x",
- "Piffany_Web_Spider_v0.x",
- "PigeonBot1.0 BETA",
- "PingALink Monitoring Services 1.0",
- "PingALink Monitoring Services 1.0 (http://www.pingalink.com)",
- "Pingdom GIGRIB (http://www.pingdom.com)",
- "pipeLiner/0.3a (PipeLine Spider;http://www.pipeline-search.com/webmaster.html; webmaster'at'pipeline-search.com)",
- "pipeLiner/0.xx (PipeLine Spider; http://www.pipeline-search.com/webmaster.html)",
- "Pita",
- "Pizilla++ ver 2.45",
- "PJspider/3.0 (pjspider@portaljuice.com; http://www.portaljuice.com)",
- "Plagger/0.x.xx (http://plagger.org/)",
- "PlagiarBot/1.0",
- "PlantyNet_WebRobot_V1.9 dhkang@plantynet.com",
- "plinki/0.1 (you got plinked! (thats a good thing..); http://www.plinki.com; crawl@plinki.com)",
- "PluckFeedCrawler/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://www.pluck.com; 1 subscribers)",
- "Pluggd/Nutch-0.9 (automated crawler http://www.pluggd.com;support at pluggd dot com)",
- "Pockey-GetHTML/4.12.0 (Win32; GUI; ix86)",
- "Pockey-GetHTML/x.xx",
- "Pockey/x.x.x",
- "Pockey7.x.x(WIN32GUI)",
- "POE-Component-Client-HTTP/0.64 (perl; N; POE; en; rv:0.640000)",
- "Poirot",
- "polybot 1.0 (http://cis.poly.edu/polybot/)",
- "Pompos/1.x http://dir.com/pompos.html",
- "Pompos/1.x pompos@iliad.fr",
- "Popdexter/1.0",
- "Port Huron Labs",
- "PortalBSpider/2.0 (spider@portalb.com)",
- "portalmmm/2.0 S500i(c20;TB)",
- "PostFavorites",
- "potbot 1.0",
- "PRCrawler/Nutch-0.9 (data mining development project; crawler@projectrialto.com)",
- "PrivacyFinder Cache Bot v1.0",
- "PrivacyFinder/1.1",
- "Privoxy/3.0 (Anonymous)",
- "Production Bot 0116B",
- "Production Bot 2016B",
- "Production Bot DOT 3016B",
- "Program Shareware 1.0.2",
- "Progressive Download",
- "Progressive Download HTTP check",
- "Project XP5 [2.03.07-111203]",
- "PROve AnswerBot 4.0",
- "ProWebGuide Link Checker (http://www.prowebguide.com)",
- "psbot/0.1 (+http://www.picsearch.com/bot.html)",
- "PSurf15a 11",
- "PSurf15a 51",
- "PSurf15a VA",
- "psycheclone",
- "PubCrawl (pubcrawl.stanford.edu)",
- "puf/0.91beta6a (Linux 2.2.18; i686)",
- "puf/0.93.2a (Linux 2.4.18; i686)",
- "pulseBot (pulse Web Miner)",
- "PureSight",
- "PuxaRapido v1.0",
- "PWeBot/1.2 Inspector (http://www.programacionweb.net/robot.php)",
- "PycURL",
- "PycURL/7.xx.x",
- "Python-urllib/1.1x",
- "Python-urllib/2.0a1",
- "Qango.com Web Directory (http://www.qango.com/)",
- "QEAVis Agent/Nutch-0.9 (Quantitative Evaluation of Academic Websites Visibility; http://nlp.uned.es/qeavis",
- "QPCreep Test Rig ( We are not indexing- just testing )",
- "QuepasaCreep ( crawler@quepasacorp.com )",
- "QuepasaCreep v0.9.1x",
- "QueryN Metasearch",
- "Quicksilver (Blacktree,MacOSX)",
- "QuickTime�.7.0.4 (qtver=7.0.4;cpu=PPC;os=Mac 10.3.9)",
- "QweeryBot/3.01 ( http://qweerybot.qweery.nl)",
- "Qweery_robot.txt_CheckBot/3.01 (http://qweerybot.qweery.com)",
- "R6_CommentReader_(www.radian6.com/crawler)",
- "R6_FeedFetcher_(www.radian6.com/crawler)",
- "rabaz (rabaz at gigabaz dot com)",
- "RaBot/1.0 Agent-admin/phortse@hanmail.net",
- "Rainbot1.1",
- "ramBot xtreme x.x",
- "RAMPyBot - www.giveRAMP.com/0.1 (RAMPyBot - www.giveRAMP.com; http://www.giveramp.com/bot.html; support@giveRAMP.com)",
- "RAMPyBot/0.8-dev (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
- "Rank Exec (rankexec.com) Reciprocal Link Manager 1.x/bot",
- "Rankivabot/3.2 (www.rankiva.com; 3.2; vzmxikn)",
- "Rational SiteCheck (Windows NT)",
- "RAYSPIDER/Nutch-0.9",
- "ReadABlog Spider (compatible; 1.1; feed update; www.readablog.com)",
- "RealDownload/4.0.0.4x",
- "REAP-crawler Nutch/Nutch-1.0-dev (Reap Project; http://reap.cs.cmu.edu/REAP-crawler/; Reap Project)",
- "Reaper [2.03.10-031204] (http://www.sitesearch.ca/reaper/)",
- "Reaper/2.0x (+http://www.sitesearch.ca/reaper)",
- "REBOL Core 2.x.x.x.x",
- "REBOL View 1.x.x.x.x",
- "RebusnetBot (+http://www.rebusnet.biz)",
- "RebusnetPADBot/1.5x (+http://www.rebusnet.biz)",
- "reciprocal links checker (http://www.recip-links.com/)",
- "RedBot/redbot-1.0 (Rediff.com Crawler; redbot at rediff dot com)",
- "RedCarpet/1.2 (http://www.redcarpet-inc.com/robots.html)",
- "RedCell/0.1 (InfoSec Search Bot (Coming Soon); http://www.telegenetic.net/bot.html; lhall@telegenetic.net)",
- "RedCell/0.1 (RedCell; telegenetic.net/bot.html; lhall_at_telegenetic.net)",
- "RedKernel WWW-Spider 2/0 (+http://www-spider.redkernel-softwares.com/)",
- "REL Link Checker Lite x.x",
- "RepoMonkey Bait & Tackle/v1.01",
- "Rewebber/1.2 libwww-perl/5.41",
- "rico/0.1",
- "RixBot (http://babelserver.org/rix)",
- "RMA/1.0 (compatible; RealMedia)",
- "RoboCrawl (http://www.canadiancontent.net)",
- "RoboCrawl (www.canadiancontent.net)",
- "RoboPal (http://www.findpal.com/)",
- "Robot/www.pj-search.com",
- "Robot: NutchCrawler- Owner: wdavies@acm.org",
- "Robot@SuperSnooper.Com",
- "Robozilla/1.0",
- "Rome Client (http://tinyurl.com/64t5n) Ver: 0.9",
- "Rotondo/3.1 libwww/5.3.1",
- "RPT-HTTPClient/0.3-x",
- "RRC (crawler_admin@bigfoot.com)",
- "RssBandit/1.5.0.10 (.NET CLR 1.1.4322.2407; WinNT 5.1.2600.0; http://www.rssbandit.org) (.NET CLR 1.1.4322.2407; WinNT 5.1.2600.0; )",
- "RSSMicro.com RSS/Atom Feed Robot",
- "RSSOwl/1.2.3 2006-11-26 (Windows; U; zhtw)",
- "RSSOwl/1.2.4 Preview Release 2007-04-15 (Windows; U; zhtw)",
- "RssReader/1.0.xx.x (http://www.rssreader.com) Microsoft Windows NT 5.1.2600.0",
- "RSurf15a 41",
- "RSurf15a 51",
- "RSurf15a 81",
- "Rubbot/1.0 (+http://rubhub.com/)",
- "RufusBot (Rufus Web Miner; http://64.124.122.252/feedback.html)",
- "RufusBot (Rufus Web Miner; http://www.webaroo.com/rooSiteOwners.html)",
- "Rumours-Agent",
- "RX Bar",
- "S&L Spider (http://search.hirners.com/)",
- "S.T.A.L.K.E.R. (http://www.seo-tools.net/en/bot.aspx)",
- "SafariBookmarkChecker (+http://www.coriolis.ch/)",
- "sait/Nutch-0.9 (SAIT Research; http://www.samsung.com)",
- "SandCrawler - Compatibility Testing",
- "SapphireWebCrawler/1.0 (Sapphire Web Crawler using Nutch; http://boston.lti.cs.cmu.edu/crawler/; mhoy@cs.cmu.edu)",
- "SapphireWebCrawler/Nutch-1.0-dev (Sapphire Web Crawler using Nutch; http://boston.lti.cs.cmu.edu/crawler/; mhoy@cs.cmu.edu)",
- "savvybot/0.2",
- "SBIder/0.7 (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html)",
- "SBIder/0.8-dev (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html)",
- "SBL-BOT (http://sbl.net)",
- "ScanWeb",
- "ScholarUniverse/0.8 (Nutch;+http://scholaruniverse.com/bot.jsp; fetch-agent@scholaruniverse.com)",
- "schwarzmann.biz-Spider_for_paddel.org+(http://www.innerprise.net/usp-spider.asp)",
- "Science Traveller International 1X/1.0",
- "ScollSpider/2.0 (+http://www.webwobot.com/ScollSpider.php)",
- "Scooter-3.0.EU",
- "Scooter-3.0.FS",
- "Scooter-3.0.HD",
- "Scooter-3.0.VNS",
- "Scooter-3.0QI",
- "Scooter-3.2",
- "Scooter-3.2.BT",
- "Scooter-3.2.DIL",
- "Scooter-3.2.EX",
- "Scooter-3.2.JT",
- "Scooter-3.2.NIV",
- "Scooter-3.2.SF0",
- "Scooter-3.2.snippet",
- "Scooter-3.3dev",
- "Scooter-ARS-1.1",
- "Scooter-ARS-1.1-ih",
- "scooter-venus-3.0.vns",
- "Scooter-W3-1.0",
- "Scooter-W3.1.2",
- "Scooter/1.0",
- "Scooter/1.0 scooter@pa.dec.com",
- "Scooter/1.1 (custom)",
- "Scooter/2.0 G.R.A.B. V1.1.0",
- "Scooter/2.0 G.R.A.B. X2.0",
- "Scooter/3.3",
- "Scooter/3.3.QA.pczukor",
- "Scooter/3.3.vscooter",
- "Scooter/3.3_SF",
- "Scooter2_Mercator_x-x.0",
- "Scooter_bh0-3.0.3",
- "Scooter_trk3-3.0.3",
- "Scope (Mars+)",
- "ScoutAbout",
- "ScoutAnt/0.1; +http://www.ant.com/what_is_ant.com/",
- "scoutmaster",
- "Scrubby/2.x (http://www.scrubtheweb.com/)",
- "Scrubby/3.0 (+http://www.scrubtheweb.com/help/technology.html)",
- "Search+",
- "Search-Engine-Studio",
- "search.ch V1.4",
- "search.ch V1.4.2 (spiderman@search.ch; http://www.search.ch)",
- "Search/1.0 (http://www.innerprise.net/es-spider.asp)",
- "searchbot admin@google.com",
- "SearchByUsa/2 (SearchByUsa; http://www.SearchByUsa.com/bot.html; info@SearchByUsa.com)",
- "SearchdayBot",
- "SearchExpress Spider0.99",
- "SearchGuild/DMOZ/Experiment (searchguild@gmail.com)",
- "SearchGuild_DMOZ_Experiment (chris@searchguild.com)",
- "Searchit-Now Robot/2.2 (+http://www.searchit-now.co.uk)",
- "Searchmee! Spider v0.98a",
- "SearchSight/2.0 (http://SearchSight.com/)",
- "SearchSpider.com/1.1",
- "Searchspider/1.2 (SearchSpider; http://www.searchspider.com; webmaster@searchspider.com)",
- "SearchTone2.0 - IDEARE",
- "Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/0.3",
- "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.0 (XDF)",
- "Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2",
- "Seeker.lookseek.com",
- "semaforo.net",
- "Semager/1.1 (http://www.semager.de/blog/semager-bots/)",
- "Semager/1.x (http://www.semager.de)",
- "semanticdiscovery/0.x",
- "Sensis Web Crawler (search_comments\at\sensis\dot\com\dot\au)",
- "Sensis.com.au Web Crawler (search_comments\at\sensis\dot\com\dot\au)",
- "SeznamBot/1.0",
- "SeznamBot/1.0 (+http://fulltext.seznam.cz/)",
- "SeznamBot/2.0-test (+http://fulltext.sblog.cz/)",
- "ShablastBot 1.0",
- "Shareaza v1.x.x.xx",
- "SharewarePlazaFileCheckBot/1.0+(+http://www.SharewarePlaza.com)",
- "Shim Crawler",
- "Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)",
- "ShopWiki/1.0 ( +http://www.shopwiki.com/)",
- "ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)",
- "Shoula.com Crawler 2.0",
- "SietsCrawler/1.1 (+http://www.siets.biz)",
- "Sigram/Nutch-1.0-dev (Test agent for Nutch development; http://www.sigram.com/bot.html; bot at sigram dot com)",
- "Siigle Orumcex v.001 Turkey (http://www.siigle.com)",
- "silk/1.0",
- "silk/1.0 (+http://www.slider.com/silk.htm)/3.7",
- "SimpleFavPanel/1.2",
- "Simpy 1.x; http://www.simpy.com/",
- "Simpy/1.x (Simpy; http://www.simpy.com/?ref=bot; feedback at simpy dot com)",
- "Sirketcebot/v.01 (http://www.sirketce.com/bot.html)",
- "SiteBar/3.x.x (Bookmark Server; http://sitebar.org/)",
- "SiteBar/x.x",
- "SiteBar/x.x.x (Bookmark Server; http://sitebar.org/)",
- "sitecheck.internetseer.com",
- "sitecheck.internetseer.com (For more info see: http://sitecheck.internetseer.com)",
- "SiteRecon+(xx)",
- "SiteSnagger",
- "SiteSpider +(http://www.SiteSpider.com/)",
- "SiteSucker/1.x.x",
- "SiteTaggerBot (http://www.sitetagger.com/bot.htm)",
- "SiteTruth.com site rating system",
- "SiteWinder",
- "SiteXpert",
- "Skampy/0.9.x (http://www.skaffe.com/skampy-info.html)",
- "Skimpy/0.x (http://www.skaffe.com/skampy-info.html)",
- "Skywalker/0.1 (Skywalker; anonymous; anonymous)",
- "Slarp/0.1",
- "Sleipnir",
- "Sleipnir Version 1.xx",
- "Sleipnir Version2.x",
- "Sleipnir/2.xx",
- "Slider_Search_v1-de",
- "SlimBrowser",
- "Slurp/2.0 (slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
- "Slurp/2.0-KiteWeekly (slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
- "Slurp/si (slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
- "Slurpy Verifier/1.0",
- "SlySearch (slysearch@slysearch.com)",
- "SlySearch/1.0 http://www.plagiarism.org/crawler/robotinfo.html",
- "SlySearch/1.x http://www.slysearch.com",
- "SmartDownload/1.2.67 (Win32; Jan 12 1999)",
- "SmartDownload/1.2.77 (Win32; Feb 1 2000)",
- "SmartDownload/1.2.77 (Win32; Jun 19 2001)",
- "smartwit.com",
- "SmiffyDCMetaSpider/1.0",
- "sna-0.0.1 (mikemuzio@msn.com)",
- "sna-0.0.1 mikeelliott@hotmail.com",
- "snap.com beta crawler v0",
- "Snapbot/1.0",
- "Snapbot/1.0 (Snap Shots, +http://www.snap.com)",
- "Snappy/1.1 ( http://www.urltrends.com/ )",
- "Snarfer/0.x.x (http://www.snarfware.com/)",
- "SnoopRob/x.x",
- "Snoopy v1.xx",
- "Snoopy v1.xx- : User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2)",
- "Snoopy_v0.xx",
- "SnykeBot/0.6 (http://www.snyke.com)",
- "SocSciBot ()",
- "SoftBank/1.0/812SH/SHJ001 Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1",
- "SoftHypermarketFileCheckBot/1.0+(+http://www.softhypermaket.com)",
- "Softizerbot (http://www.softizer.com)",
- "sogou develop spider",
- "Sogou Orion spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
- "sogou spider",
- "Sogou web spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
- "sohu agent",
- "sohu-search",
- "Sosospider+(+http://help.soso.com/webspider.htm)",
- "Space Bison/0.02 [fu] (Win67; X; SK)",
- "SpeedDownload/1.x",
- "speedfind ramBot xtreme 8.1",
- "Speedy Spider (Beta/x.x; speedy@entireweb.com)",
- "Speedy Spider (Entireweb; Beta/1.0; http://www.entireweb.com/about/search_tech/speedyspider/)",
- "Speedy_Spider (http://www.entireweb.com)",
- "Sphere Scout&v4.0 - scout at sphere dot com",
- "Sphider",
- "Spida/0.1",
- "Spider-Sleek/2.0 (+http://search-info.com/linktous.html)",
- "spider.batsch.com",
- "Spider.TerraNautic.net - v:1.04",
- "spider.yellopet.com - www.yellopet.com",
- "Spider/maxbot.com admin@maxbot.com",
- "SpiderKU/0.x",
- "SpiderMan",
- "SpiderMonkey/7.0x (SpiderMonkey.ca info at http://spidermonkey.ca/sm.shtml)",
- "Spinne/2.0",
- "Spinne/2.0 med",
- "Spinne/2.0 med_AH",
- "Spock Crawler (http://www.spock.com/crawler)",
- "sportsuchmaschine.de-Robot (Version: 1.02- powered by www.sportsuchmaschine.de)",
- "sproose/0.1-alpha (sproose crawler; http://www.sproose.com/bot.html; crawler@sproose.com)",
- "SQ Webscanner",
- "Squid-Prefetch",
- "squidclam",
- "SquidClamAV_Redirector 1.x.x",
- "Sqworm/2.9.81-BETA (beta_release; 20011102-760; i686-pc-linux-gnu)",
- "Sqworm/2.9.85-BETA (beta_release; 20011115-775; i686-pc-linux-gnu)",
- "Sqworm/2.9.89-BETA (beta_release; 20020130-839; i686-pc-linux-gnu) ",
- "SSurf15a 11 ",
- "StackRambler/x.x ",
- "Stamina/1.4",
- "Star Downloader",
- "StarDownloader/1.xx",
- "stat statcrawler@gmail.com",
- "Steeler/1.x (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)",
- "Steeler/3.3 (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)",
- "Strategic Board Bot (+http://www.strategicboard.com)",
- "Submission Spider at surfsafely.com",
- "suchbaer.de",
- "suchbaer.de (CrawlerAgent v0.103)",
- "suchbot",
- "Suchknecht.at-Robot",
- "suchpadbot/1.0 (+http://www.suchpad.de)",
- "Sunrise XP/2.x",
- "Sunrise/0.42g (Windows XP)",
- "SuperBot/x.x (Win32)",
- "SuperBot/x.x.x.xx (Windows XP)",
- "Superdownloads Spiderman",
- "SURF ",
- "SurferF3 1/0",
- "SurfMaster",
- "SurveyBot/2.2 Whois Source",
- "SurveyBot/2.3 (Whois Source)",
- "suzuran",
- "SWB/V1.4 (HP)",
- "swbot/0.9c libwww/5.3.1",
- "Swooglebot/2.0. (+http://swoogle.umbc.edu/swooglebot.htm)",
- "SWSBot-Images/1.2 http://www.smartwaresoft.com/swsbot12.html",
- "SygolBot http://www.sygol.net",
- "Sylera/1.2.x",
- "SyncBot",
- "SyncIT/x.x",
- "Syndirella/0.91pre",
- "SynoBot",
- "Syntryx ANT Scout Chassis Pheromone; Mozilla/4.0 compatible crawler",
- "Szukacz/1.x",
- "Szukacz/1.x (robot; www.szukacz.pl/jakdzialarobot.html; szukacz@proszynski.pl)",
- "T-Online Browser",
- "tags2dir.com/0.8 (+http://tags2dir.com/directory/)",
- "Tagword (http://tagword.com/dmoz_survey.php)",
- "Tagyu Agent/1.0",
- "Talkro Web-Shot/1.0 (E-mail: webshot@daumsoft.com- Home: http://222.122.15.190/webshot)",
- "TAMU_CS_IRL_CRAWLER/1.0",
- "targetblaster.com/0.9k",
- "TargetYourNews.com bot",
- "TCDBOT/Nutch-0.8 (PhD student research;http://www.tcd.ie; mcgettrs at t c d dot IE)",
- "TE",
- "TeamSoft WinInet Component",
- "TECOMAC-Crawler/0.x",
- "Tecomi Bot (http://www.tecomi.com/bot.htm)",
- "Teemer (NetSeer, Inc. is a Los Angeles based Internet startup company.; http://www.netseer.com/crawler.html; crawler@netseer.com)",
- "Teleport Pro/1.2x(.1xxx)",
- "Teoma MP",
- "teomaagent crawler-admin@teoma.com",
- "teomaagent1 [crawler-admin@teoma.com]",
- "teoma_agent1",
- "Teradex Mapper; mapper@teradex.com; http://www.teradex.com",
- "TeragramCrawler",
- "terraminds-bot/1.0 (support@terraminds.de)",
- "TerrawizBot/1.0 (+http://www.terrawiz.com/bot.html)",
- "Test spider",
- "TestCrawler/Nutch-0.9 (Testing Crawler for Research ; http://balihoo.com/index.aspx; tgautier at balihoo dot com)",
- "The Expert HTML Source Viewer (http://www.expert-html.com)",
- "TheRarestParser/0.2a (http://therarestwords.com/)",
- "TheSuBot/0.1 (www.thesubot.de)",
- "thumbshots-de-Bot (Version: 1.02, powered by www.thumbshots.de)",
- "thumbshots-de-Bot (Version: 1.02- powered by www.thumbshots.de)",
- "timboBot/0.9 http://www.breakingblogs.com/timbo_bot.html",
- "http://www.timelyweb.com/",
- "TinEye/1.1 (http://tineye.com/crawler.html)",
- "tivraSpider/1.0 (crawler@tivra.com)",
- "TJG/Spider",
- "TJvMultiHttpGrabber Component",
- "Tkensaku/x.x(http://www.tkensaku.com/q.html)",
- "toCrawl/UrlDispatcher",
- "Topodia/1.2-dev (Topodia - Crawler for HTTP content indexing; http://www.topodia.com/; support@topodia.com)",
- "TOPOS robot/1.1 (http://www.topos.com.ua/)",
- "traazibot/testengine (+http://www.traazi.de)",
- "Trailfire-bot/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
- "Trailfire-bot/0.7.1 (Trailfire page content analyzer; http://trailfire.com; info@trailfire.com)",
- "Trailfire/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
- "Trampelpfad-Spider",
- "Trampelpfad-Spider-v0.1",
- "tricosMetaCheck 1.2216-08-1999 (http://www.tricos.com/metacheck)",
- "TSurf15a 11",
- "TulipChain/5.x (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://java.sun.com/) Linux/2.4.17",
- "TulipChain/5.xx (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://apple.com/) Mac_OS_X/10.2.8",
- "Tumblr/1.0 RSS syndication (+http://www.tumblr.com/) (support@tumblr.com)",
- "TurnitinBot/x.x (http://www.turnitin.com/robot/crawlerinfo.html)",
- "Turnpike Emporium LinkChecker/0.1",
- "TutorGig/1.5 (+http://www.tutorgig.com/crawler)",
- "Tutorial Crawler 1.4 (http://www.tutorgig.com/crawler)",
- "Twiceler www.cuill.com/robots.html",
- "Twiceler-0.9 http://www.cuill.com/twiceler/robot.html",
- "Twisted PageGetter",
- "Twitturly / v0.x",
- "Twotrees Reactive Filter V2.0",
- "Tycoon Agent/Nutch-1.0-dev",
- "TygoBot",
- "TygoProwler",
- "UCmore",
- "UCMore Crawler App",
- "UCWEB5.1",
- "UDM",
- "UdmSearch/3.1.x",
- "UIowaCrawler/1.0",
- "UKWizz/Nutch-0.8.1 (UKWizz Nutch crawler; http://www.ukwizz.com/)",
- "Ultraseek",
- "Under the Rainbow 2.2",
- "unknownght.com Web Server IIS vs Apache Survey. See Results at www.DNSRight.com",
- "UofTDB_experiment (leehyun@cs.toronto.edu)",
- "UP.Browser/3.01-IG01 UP.Link/3.2.3.4",
- "updated/0.1-alpha (updated crawler; http://www.updated.com; crawler@updated.com)",
- "updated/0.1beta (updated.com; http://www.updated.com; crawler@updated.om)",
- "UPG1 UP/4.0 (compatible; Blazer 1.0)",
- "Uptimebot",
- "UptimeBot(www.uptimebot.com)",
- "URI::Fetch/0.06",
- "URL Spider Pro/x.xx (innerprise.net)",
- "URLBase/6.x",
- "URLBlaze",
- "urlfan-bot/1.0; +http://www.urlfan.com/site/bot/350.html",
- "URLGetFile",
- "URL_Spider_Pro/x.x",
- "URL_Spider_Pro/x.x+(http://www.innerprise.net/usp-spider.asp)",
- "User-Agent: BoardReader Favicon Fetcher /1.0 info@boardreader.com",
- "User-Agent: BoardReader Image Fetcher /1.0 info@boardreader.com",
- "User-Agent: LjSEEK Picture-Bot /1.0 contact@ljseek.com",
- "User-Agent: FileHeap! file downloader (http://www.fileheap.com)",
- "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
- "User-Agent: Mozilla/4.0 (SKIZZLE! Distributed Internet Spider v1.0 - www.SKIZZLE.com)",
- "user-agent=Mozilla/3.01Gold",
- "USyd-NLP-Spider (http://www.it.usyd.edu.au/~vinci/bot.html)",
- "UtilMind HTTPGet",
- "Utopia WebWasher 3.0",
- "uTorrent/1500",
- "VadixBot",
- "Vagabondo-WAP/2.0 (webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)/1.0 Profile",
- "Vagabondo/1.x MT (webagent@wise-guys.nl)",
- "Vagabondo/2.0 MT",
- "Vagabondo/2.0 MT (webagent at wise-guys dot nl)",
- "Vagabondo/2.0 MT (webagent@NOSPAMwise-guys.nl)",
- "Vagabondo/3.0 (webagent at wise-guys dot nl)",
- "Vakes/0.01 (Vakes; http://www.vakes.com/; search@vakes.com)",
- "VayalaCreep-v0.0.1 (haploid@haploid.com)",
- "Vayala|Creep-v0.0.1 (codepoet@wildties.com)",
- "vb wininet",
- "versus 0.2 (+http://versus.integis.ch)",
- "versus crawler eda.baykan@epfl.ch",
- "Verticrawlbot",
- "VeryGoodSearch.com.DaddyLongLegs",
- "verzamelgids.nl - Networking4all Bot/x.x",
- "Verzamelgids/2.2 (http://www.verzamelgids.nl)",
- "Vespa Crawler",
- "virus_detector (virus_harvester@securecomputing.com)",
- "VisBot/2.0 (Visvo.com Crawler; http://www.visvo.com/bot.html; bot@visvo.com)",
- "Visicom Toolbar",
- "Vision Research Lab image spider at vision.ece.ucsb.edu",
- "VLC media player - version 0.8.5 Janus - (c) 1996-2006 the VideoLAN team",
- "VMBot/0.x.x (VMBot; http://www.VerticalMatch.com/; vmbot@tradedot.com)",
- "Vortex/2.2 (+http://marty.anstey.ca/robots/vortex/)",
- "voyager-hc/1.0",
- "voyager/1.0",
- "voyager/2.0 (http://www.kosmix.com/html/crawler.html)",
- "VSE/1.0 (testcrawler@hotmail.com)",
- "VSE/1.0 (testcrawler@vivisimo.com)",
- "vspider",
- "vspider/3.x",
- "VWBOT/Nutch-0.9-dev (VWBOT Nutch Crawler; http://vwbot.cs.uiuc.edu;+vwbot@cs.uiuc.edu",
- "W3C-checklink/3.x.x.x libwww-perl/5.xx",
- "W3C-checklink/4.x [4.xx] libwww-perl/5.xxx",
- "W3C-WebCon/5.x.x libwww/5.x.x",
- "W3CLineMode/5.4.0 libwww/5.x.x",
- "W3CRobot/5.4.0 libwww/5.4.0",
- "W3C_Validator/1.xxx libwww-perl/5.xx",
- "w3m/0.x.xx",
- "W3SiteSearch Crawler_v1.1 http://www.w3sitesearch.de",
- "wadaino.jp-crawler 0.2 (http://wadaino.jp/)",
- "WannaBe (Macintosh; PPC)",
- "WapOnWindows 1.0",
- "Watchfire WebXM 1.0",
- "WAVcheck 1.0.x (http://www.webbanalys.se/apps/WAVcheck/)",
- "Wavefire/0.8-dev (Wavefire; http://www.wavefire.com; info@wavefire.com)",
- "Waypath development crawler - info at waypath dot com",
- "Waypath Scout v2.x - info at waypath dot com",
- "WDG_Validator/1.1",
- "Web Image Collector",
- "Web Link Validator 1.5",
- "Web Snooper",
- "web-bekannt (Version: 1.02, powered by www.internetservice-franken.de)",
- "web-bekannt (Version: 1.02, powered by www.web-bekannt.de)",
- "Web-Bot V1.03",
- "Web-Robot/5.0 (en-US; web-robot.com/policy.html) Web-Robot Crawler/2.0.3",
- "web2express.org/Nutch-0.9-dev (leveled playing field; http://web2express.org/; info at web2express.org)",
- "WebAlta Crawler/1.2.1 (http://www.webalta.ru/bot.html)",
- "WebarooBot (Webaroo Bot; http://64.124.122.252/feedback.html)",
- "WebarooBot (Webaroo Bot; http://www.webaroo.com/rooSiteOwners.html)",
- "WebAuto/3.4xxx (WinNT; I)",
- "webbandit/4.xx.0",
- "WebBug/5.x",
- "Webclipping.com",
- "webcollage/1.xx",
- "WebCompass 2.0",
- "WebCopier vx.x",
- "WebCopier vx.xa",
- "WebCorp/1.0",
- "webcrawl.net",
- "WebDownloader for X x.xx",
- "Webdup/0.9",
- "WebFetch",
- "webfetch/5.x.x",
- "WebFilter Robot 1.0",
- "WebFilter Robot 1.x",
- "WebFindBot(http://www.web-find.com)",
- "Webglimpse 2.xx.x (http://webglimpse.net)",
- "webGobbler/1.x.x",
- "webhack",
- "WebImages 0.3 ( http://herbert.groot.jebbink.nl/?app=WebImages )",
- "WebLight/4.x.x (support@illumit.com; http://www.illumit.com/Products/weblight/)",
- "Weblink's checker/",
- "Weblog Attitude Diffusion 1.0",
- "webmeasurement-bot, http://rvs.informatik.uni-leipzig.de",
- "WebMiner/x.x [en] (Win98; I)",
- "WeBoX/0.xx",
- "WebPix 1.0 (www.netwu.com)",
- "WebQL",
- "WebRACE/1.1 (University of Cyprus- Distributed Crawler)",
- "WebRankSpider/1.37 (+http://ulm191.server4you.de/crawler/)",
- "WebReaper vx.x - www.webreaper.net",
- "WebReaper [info@webreaper.net]",
- "WebReaper [webreaper@webreaper.net]",
- "WebSearch.COM.AU/3.0.1 (The Australian Search Engine; http://WebSearch.COM.AU; Search@WebSearch.COM.AU)",
- "WebSearchBench WebCrawler v0.1(Experimental)",
- "WebSearchBench WebCrawler V1.0 (Beta)- Prof. Dr.-Ing. Christoph Lindemann- Universität Dortmund- cl@cs.uni-dortmund.de- http://websearchbench.cs.uni-dortmund.de/",
- "Website Explorer/0.9.x.x",
- "Website eXtractor",
- "WebsiteWorth v1.0",
- "Webspinne/1.0 webmaster@webspinne.de",
- "Websquash.com (Add url robot)",
- "WebStat/1.0 (Unix; beta; 20040314)",
- "Webster v0.3 ( http://webster.healeys.net/ )",
- "webster-internet.de pad browser",
- "WebStripper/2.xx",
- "WebTrafficExpress/x.0",
- "WebTrends/3.0 (WinNT)",
- "WebVac (webmaster@pita.stanford.edu)",
- "WebVal/1.0",
- "Webverzeichnis.de - Telefon: 01908 / 26005",
- "WebVulnCrawl.unknown/1.0 libwww-perl/5.803",
- "WebWatcherMonitor/2.01",
- "WebZIP/x.x (http://www.spidersoft.com)",
- "Wells Search II",
- "WEP Search 00",
- "West Wind Internet Protocols 4.xx",
- "WFARC",
- "Wget/1.x(.x)GNU wget http://www.gnu.org/software/wget/wget.html - file downloader",
- "Wget/1.x+cvs-stable (Red Hat modified)",
- "Wget/1.x.x+cvs",
- "Whatsup/x.x",
- "whatUseek_winona/3.0",
- "WhizBang! Lab",
- "Wildsoft Surfer",
- "Willow Internet Crawler by Twotrees V2.1",
- "WinampMPEG/2.00 (larbin@unspecified.mail)",
- "WincerSong Agent v1.0",
- "Windows-Media-Player/10.00.00.xxxx",
- "WinGet 1.1",
- "WinHTTP Example/1.0",
- "WinkBot/0.06 (Wink.com search engine web crawler; http://www.wink.com/Wink:WinkBot; winkbot@wink.com)",
- "WinPodder (http://winpodder.com)",
- "WinWAP/3.x (3.x.x.xx; Win32) (Google WAP Proxy/1.0)",
- "Wir sind die Borg (Version: 1.03, Sie wurden Assimiliert +http://www.yammba.com/suchmaschine/bot.html)",
- "WIRE/0.11 (Linux; i686; Bot,Robot,Spider,Crawler,aromano@cli.di.unipi.it)",
- "WIRE/0.x (Linux; i686; Bot,Robot,Spider,Crawler)",
- "WISEbot/1.0 (WISEbot@koreawisenut.com; http://wisebot.koreawisenut.com)",
- "WiseWire-Spider2",
- "wish-project (http://wish.slis.tsukuba.ac.jp/)",
- "WordChampBot",
- "WordPress/x.x.x.x PHP/4.x.xx",
- "worio heritrix bot (+http://worio.com/)",
- "woriobot ( http://www.worio.com/)",
- "WorldLight",
- "WorQmada/1.0",
- "Wotbox/alpha0.6 (bot@wotbox.com; http://www.wotbox.com)",
- "Wotbox/alpha0.x.x (bot@wotbox.com; http://www.wotbox.com) Java/1.4.1_02",
- "WSB WebCrawler V1.0 (Beta)- cl@cs.uni-dortmund.de",
- "WSB, http://websearchbench.cs.uni-dortmund.de",
- "wume_crawler/1.1 (http://wume.cse.lehigh.edu/~xiq204/crawler/)",
- "Wusage/x.0@boutell.com",
- "Wwlib/Linux",
- "WWSBOT 1.x [--- http://www.analyzer.nu ---]",
- "WWW-Mechanize/1.1x",
- "www.arianna.it",
- "www.business-socket.com registry verify/1.x",
- "www.doweb.co.uk crawler",
- "www4mail/2.x libwww-FM/2.14 (Unix; I)",
- "WWWC/1.0x",
- "WWWeasel Robot v1.00 (http://wwweasel.de)",
- "WWWOFFLE/2.x",
- "wwwster/1.x (Beta- mailto:gue@cis.uni-muenchen.de)",
- "wxDownload Fast",
- "X-Crawler ",
- "Xaldon WebSpider",
- "Xenu Link Sleuth 1.xx",
- "Xenu's Link Sleuth 1.x[a-z]",
- "Xerka WebBot v1.0.0 [UPVOpenDir]",
- "xine/1.0",
- "xirq/0.1-beta (xirq; http://www.xirq.com; xirq@xirq.com)",
- "XMLSlurp/0.1 libwww-perl/5.805",
- "XRL/2.00b1 (Linux; i686; en-us) (+http://metamark.net/about)",
- "Xylix",
- "xyro_(xcrawler@cosmos.inria.fr)",
- "Y!J-BSC/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",
- "Y!J-SRD/1.0",
- "Y!J/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",
- "Y!OASIS/TEST no-ad Mozilla/4.08 [en] (X11; I; FreeBSD 2.2.8-STABLE i386)",
- "Y!TunnelPro",
- "yacy (www.yacy.net; v20040602; i386 Linux 2.4.26-gentoo-r13; java 1.4.2_06; MET/en)",
- "yacybot (x86 Windows XP 5.1; java 1.5.0_06; Europe/de) yacy.net",
- "Yahoo Pipes 1.0",
- "Yahoo! Mindset",
- "Yahoo-Blogs/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html )",
- "Yahoo-MMAudVid/1.0 (mms dash mmaudvidcrawler dash support at yahoo dash inc dot com)",
- "Yahoo-MMAudVid/2.0(mms dash mm aud vid crawler dash support at yahoo dash inc.com ;Mozilla 4.0 compatible; MSIE 7.0;Windows NT 5.0; .NET CLR 2.0)",
- "Yahoo-MMCrawler/3.x (mm dash crawler at trd dot overture dot com)",
- "Yahoo-Test/4.0",
- "Yahoo-VerticalCrawler-FormerWebCrawler/3.9 crawler at trd dot overture dot com; http://www.alltheweb.com/help/webmaster/crawler",
- "YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide)",
- "YahooSeeker-Testing/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/)",
- "YahooSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/)",
- "YahooSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/yahooseeker.html)",
- "YahooSeeker/1.1 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/)",
- "YahooSeeker/bsv3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html )",
- "YahooSeeker/CafeKelsa-dev (compatible; Konqueror/3.2; FreeBSD ;cafekelsa-dev-webmaster@yahoo-inc.com )",
- "Yandex/1.01.001 (compatible; Win16; I)",
- "Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)",
- "yarienavoir.net/0.2",
- "Yeti",
- "Yeti/0.01 (nhn/1noon, yetibot@naver.com, check robots.txt daily and follows it)",
- "Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/)",
- "yggdrasil/Nutch-0.9 (yggdrasil biorelated search engine; www dot biotec dot tu minus dresden do de slash schroeder; heiko dot dietze at biotec dot tu minus dresden dot de)",
- "YodaoBot/1.0 (http://www.yodao.com/help/webmaster/spider/; )",
- "yoofind/yoofind-0.1-dev (yoono webcrawler; http://www.yoono.com ; MyEmail)",
- "yoogliFetchAgent/0.1",
- "yoono/1.0 web-crawler/1.0",
- "YottaCars_Bot/4.12 (+http://www.yottacars.com) Car Search Engine ",
- "YottaShopping_Bot/4.12 (+http://www.yottashopping.com) Shopping Search Engine",
- "YTunnelPro",
- "Z-Add Link Checker (http://w3.z-add.co.uk/linkcheck/)",
- "Zao-Crawler",
- "Zao-Crawler 0.2b",
- "Zao/0.1 (http://www.kototoi.org/zao/)",
- "ZBot/1.00 (icaulfield@zeus.com)",
- "Zearchit",
- "ZeBot_lseek.net (bot@ze.bz)",
- "ZeBot_www.ze.bz (ze.bz@hotmail.com)",
- "zedzo.digest/0.1 (http://www.zedzo.com/)",
- "Zend_Http_Client",
- "zermelo Mozilla/5.0 compatible; heritrix/1.12.1 (+http://www.powerset.com) [email:crawl@powerset.com,email:paul@page-store.com]",
- "zerxbot/Version 0.6 libwww-perl/5.79",
- "Zeus ThemeSite Viewer Webster Pro V2.9 Win32",
- "Zeus xxxxx Webster Pro V2.9 Win32",
- "Zeusbot/0.07 (Ulysseek's web-crawling robot; http://www.zeusbot.com; agent@zeusbot.com)",
- "Ziggy -- The Clown From Hell!!",
- "ZipppBot/0.xx (ZipppBot; http://www.zippp.net; webmaster@zippp.net)",
- "ZIPPPCVS/0.xx (ZipppBot/.xx;http://www.zippp.net; webmaster@zippp.net)",
- "Zippy v2.0 - Zippyfinder.com",
- "Zoo Tycoon 2 Client -- http://www.zootycoon.com",
- "ZoomSpider - wrensoft.com",
- "Zscho.de Crawler/Nutch-1.0-Zscho.de-semantic_patch (Zscho.de Crawler",
- "zspider/0.9-dev http://feedback.redkolibri.com/",
- "ZyBorg/1.0 (ZyBorg@WISEnut.com; http://www.WISEnut.com)",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 (FM Scene 4.6.1)",
- "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 (.NET CLR 3.5.30729) (Prevx 3.0.5)",
- "ABCdatos BotLink/1.0.2 (test links)",
- "'Ahoy! The Homepage Finder'",
- "AlkalineBOT",
- "AnthillV1.1",
- "appie/1.1",
- "Arachnophilia",
- "Araneo/0.7 (araneo@esperantisto.net; http://esperantisto.net)",
- "AraybOt/1.0 (+http://www.araykoo.com/araybot.html)",
- "arks/1.0",
- "ASpider/0.09",
- "ATN_Worldwide",
- "AURESYS/1.0",
- "BackRub/*.*",
- "BaySpider",
- "bbot/0.100",
- "Big Brother",
- "Die Blinde Kuh",
- "borg-bot/0.9",
- "BoxSeaBot/0.5 (http://boxsea.com/crawler)",
- "Mozilla/3.01 (compatible;)",
- "BSpider/1.0 libwww-perl/0.40",
- "CACTVS Chemistry Spider",
- "Calif/0.6 (kosarev@tnps.net; http://www.tnps.dp.ua)",
- "Digimarc CGIReader/1.0",
- "Checkbot/x.xx LWP/5.x",
- "Mozilla/4.0 (compatible; ChristCrawler.com, ChristCrawler@ChristCENTRAL.com)",
- "cIeNcIaFiCcIoN.nEt Spider (http://www.cienciaficcion.net)",
- "CMC/0.01",
- "LWP",
- "conceptbot/0.3",
- "Confuzzledbot/X.X (+http://www.confuzzled.lu/bot/)",
- "root/0.1",
- "cosmos/0.3",
- "Internet Cruiser Robot/2.1",
- "Cusco/3.2",
- "CyberSpyder/2.1",
- "CydralSpider/X.X (Cydral Web Image Search;",
- "DesertRealm.com; 0.2; [J];",
- "Deweb/1.01",
- "dienstspider/1.0",
- "Digger/1.0 JDK/1.3.0",
- "DIIbot",
- "grabber",
- "DNAbot/1.0",
- "DragonBot/1.0 libwww/5.0",
- "DWCP/2.0",
- "LWP::",
- "EbiNess/0.01a",
- "EIT-Link-Verifier-Robot/0.2",
- "elfinbot",
- "Emacs-w3/v[0-9\.]+",
- "EMC Spider",
- "esculapio/1.1",
- "esther",
- "Evliya Celebi v0.151 - http://ilker.ulak.net.tr",
- "explorersearch",
- "FastCrawler 3.0.X (crawler@1klik.dk) - http://www.1klik.dk",
- "Mozilla/4.0 (compatible: FDSE robot)",
- "FelixIDE/1.0",
- "Hazel's Ferret Web hopper,",
- "ESIRover v1.0",
- "fido/0.9 Harvest/1.4.pl2",
- "Hämähäkki/0.2",
- "KIT-Fireball/2.0 libwww/5.0a",
- "Fish-Search-Robot",
- "Mozilla/2.0 (compatible fouineur v2.0; fouineur.9bit.qc.ca)",
- "Robot du CRIM 1.0a",
- "Freecrawl",
- "FunnelWeb-1.0",
- "gammaSpider xxxxxxx ()/",
- "gazz/1.0",
- "gcreep/1.0",
- "???",
- "GetURL.rexx v1.05",
- "Golem/1.1",
- "Googlebot/2.X (+http://www.googlebot.com/bot.html)",
- "griffon/1.0",
- "Gulliver/1.1",
- "yes",
- "havIndex/X.xx[bxx]",
- "AITCSRobot/1.1",
- "Hometown Spider Pro",
- "wired-digital-newsbot/1.5",
- "htdig/3.1.0b2",
- "HTMLgobble v2.2",
- "no",
- "iajaBot/0.1",
- "IBM_Planetwide,",
- "gestaltIconoclast/1.0 libwww-FM/2.17",
- "INGRID/0.1",
- "Mozilla 3.01 PBWF (Win95)",
- "IncyWincy/1.0b1",
- "Informant",
- "InfoSeek Robot 1.0",
- "Infoseek Sidewinder",
- "InfoSpiders/0.1",
- "inspectorwww/1.0 http://www.greenpac.com/inspectorwww.html",
- "'IAGENT/1.0'",
- "I Robot 0.4 (irobot@chaos.dk)",
- "Iron33/0.0",
- "IsraeliSearch/1.0",
- "JavaBee",
- "JBot (but can be changed by the user)",
- "JCrawler/0.2",
- "JoBo (can be modified by the user)",
- "Jobot/0.1alpha libwww-perl/4.0",
- "JoeBot/x.x,",
- "JubiiRobot/version#",
- "jumpstation",
- "image.kapsi.net/1.0",
- "Katipo/1.0",
- "KDD-Explorer/0.1",
- "KO_Yappo_Robot/1.0.4(http://yappo.com/info/robot.html)",
- "LabelGrab/1.1",
- "larbin (+mail)",
- "legs",
- "Linkidator/0.93",
- "LinkScan Server/5.5 | LinkScan Workstation/5.5",
- "Lockon/xxxxx",
- "logo.gif crawler",
- "Lycos/x.x",
- "Magpie/1.0",
- "marvin/infoseek (marvin-team@webseek.de)",
- "M/3.8",
- "MediaFox/x.y",
- "MerzScope",
- "NEC-MeshExplorer",
- "MindCrawler",
- "UdmSearch",
- "moget/1.0",
- "MOMspider/1.00 libwww-perl/0.40",
- 'Monster/vX.X.X -$TYPE ($OSTYPE)',
- "Motor/0.2",
- "MSNBOT/0.1 (http://search.msn.com/msnbot.htm)",
- "Muninn/0.1 libwww-perl-5.76",
- "MuscatFerret/",
- "MwdSearch/0.1",
- "User-Agent: Mozilla/4.0 (compatible; sharp-info-agent v1.0; )",
- "NDSpider/1.5",
- "NetCarta CyberPilot Pro",
- "NetMechanic",
- "NetScoop/1.0 libwww/5.0a",
- "newscan-online/1.1",
- "NHSEWalker/3.0",
- "Nomad-V2.x",
- "NorthStar",
- "ObjectsSearch/0.01",
- "Occam/1.0",
- "HKU WWW Robot,",
- "Openfind data gatherer, Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)",
- "Orbsearch/1.0",
- "PackRat/1.0",
- "PageBoy/1.0",
- "ParaSite/0.21 (http://www.ianett.com/parasite/)",
- "Patric/0.01a",
- "web robot PEGASUS",
- "Peregrinator-Mathematics/0.7",
- "PerlCrawler/1.0 Xavatoria/2.0",
- "Duppies",
- "phpdig/x.x.x",
- "PiltdownMan/1.0 profitnet@myezmail.com",
- "Mozilla/4.0 (compatible: Pimptrain's robot)",
- "Pioneer",
- "PortalJuice.com/4.0",
- "PGP-KA/1.2",
- "PlumtreeWebAccessor/0.9",
- "Poppi/1.0",
- "PortalBSpider/1.0 (spider@portalb.com)",
- "psbot/0.X (+http://www.picsearch.com/bot.html)",
- "straight FLASH!! GetterroboPlus 1.5",
- "Raven-v2",
- "Resume Robot",
- "RHCS/1.0a",
- "RixBot (http://www.oops-as.no/rix/)",
- "Road Runner: ImageScape Robot (lim@cs.leidenuniv.nl)",
- "Robbie/0.1",
- "ComputingSite Robi/1.0 (robi@computingsite.com)",
- "Robofox v2.0",
- "Roverbot",
- "RuLeS/1.0 libwww/4.0",
- "SafetyNet Robot 0.1,",
- "searchprocess/0.9",
- "Senrigan/xxxxxx",
- "SG-Scout",
- "Shagseeker at http://www.shagseek.com /1.0",
- "Shai'Hulud",
- "libwww-perl-5.41",
- "SimBot/1.0",
- "Site Valet",
- "Open Text Site Crawler V1.0",
- "SiteTech-Rover",
- "aWapClient",
- "SLCrawler",
- "Slurp/2.0",
- "ESISmartSpider/2.0",
- "Snooper/b97_01",
- "Solbot/1.0 LWP/5.07",
- "Spanner/1.0 (Linux 2.0.27 i586)",
- "Speedy Spider ( http://www.entireweb.com/speedy.html )",
- "mouse.house/7.1",
- "SpiderBot/1.0",
- "spiderline/3.1.3",
- "SpiderMan 1.0",
- "Mozilla/4.0 (compatible; SpiderView 1.0;unix)",
- "ssearcher100",
- "suke/*.*",
- "suntek/1.0",
- "http://www.sygol.com",
- "Mozilla/3.0 (Black Widow v1.1.0; Linux 2.0.27; Dec 31 1997 12:25:00",
- "Tarantula/1.0",
- "tarspider",
- "dlw3robot/x.y (in TclX by http://hplyot.obspm.fr/~dl/)",
- "TechBOT",
- "Templeton/{version} for {platform}",
- "teoma_agent1 [teoma_admin@hawkholdings.com]",
- "TitIn/0.2",
- "TITAN/0.1",
- "TLSpider/1.1",
- "UCSD-Crawler",
- "UdmSearch/2.1.1",
- "uptimebot",
- "urlck/1.2.3",
- "URL Spider Pro",
- "Valkyrie/1.0 libwww-perl/0.40",
- "Victoria/1.0",
- "vision-search/3.0'",
- "void-bot/0.1 (bot@void.be; http://www.void.be/)",
- "Voyager/0.0",
- "VWbot_K/4.2",
- "w3index",
- "W3M2/x.xxx",
- "CrawlPaper/n.n.n (Windows n)",
- "WWWWanderer v3.0",
- "w@pSpider/xxx (unix) by wap4.com",
- "WebBandit/1.0",
- "WebCatcher/1.0",
- "WebCopy/(version)",
- "WebFetcher/0.8,",
- "weblayers/0.0",
- "WebLinker/0.0 libwww-perl/0.1",
- "WebMoose/0.0.0000",
- "WebQuest/1.0",
- "Digimarc WebReader/1.2",
- "WebReaper [webreaper@otway.com]",
- "webs@recruit.co.jp",
- "webvac/1.0",
- "webwalk",
- "WebWalker/1.10",
- "WebWatch",
- "Wget/1.4.0",
- "wlm-1.1",
- "w3mir",
- "WOLP/1.0 mda/1.0",
- "WWWC/0.25 (Win95)",
- "XGET/0.7",
- "Nederland.zoek",
- "ltx71 - (http://ltx71.com/)"
- );
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement