<?php
// source: http://www.user-agents.org/
$array = ("!Susie (http://www.sync2it.com/susie)",
"<a href='http://www.unchaos.com/'> UnChaos </a> From Chaos To Order Hybrid Web Search Engine.(vadim_gonchar@unchaos.com)",
"<a href='http://www.unchaos.com/'> UnChaos Bot Hybrid Web Search Engine. </a> (vadim_gonchar@unchaos.com)",
"<b> UnChaosBot From Chaos To Order UnChaos Hybrid Web Search Engine at www.unchaos.com </b> (info@unchaos.com)",
"<http://www.sygol.com/> http://www.sygol.com",
"( Robots.txt Validator http://www.searchengineworld.com/cgi-bin/robotcheck.cgi )",
"(DreamPassport/3.0; isao/MyDiGiRabi)",
"(Privoxy/1.0)",
"*/Nutch-0.9-dev",
"+SitiDi.net/SitiDiBot/1.0 (+Have Good Day)",
"-DIE-KRAEHE- META-SEARCH-ENGINE/1.1 http://www.die-kraehe.de",
"192.comAgent",
"1st ZipCommander (Net) - http://www.zipcommander.com/",
"2Bone_LinkChecker/1.0 libwww-perl/5.64",
"4anything.com LinkChecker v2.0",
"8484 Boston Project v 1.0",
":robot/1.0 (linux) ( admin e-mail: undefined http://www.neofonie.de/loesungen/search/robot.html )",
"A-Online Search",
"A1 Keyword Research/1.0.2 (+http://www.micro-sys.dk/products/keyword-research/) miggibot/2007.03.27",
"A1 Sitemap Generator/1.0 (+http://www.micro-sys.dk/products/sitemap-generator/) miggibot/2006.01.24",
"aardvark-crawler",
"AbachoBOT",
"AbachoBOT (Mozilla compatible)",
"ABCdatos BotLink/5.xx.xxx#BBL",
"Aberja Checkomat",
"abot/0.1 (abot; http://www.abot.com; abot@abot.com)",
"About/0.1libwww-perl/5.47",
"Accelatech RSSCrawler/0.4",
"accoona",
"Accoona-AI-Agent/1.1.1 (crawler at accoona dot com)",
"Accoona-AI-Agent/1.1.2 (aicrawler at accoonabot dot com)",
"Ace Explorer",
"Ack (http://www.ackerm.com/)",
"AcoiRobot",
"Acoon Robot v1.50.001",
"Acoon Robot v1.52 (http://www.acoon.de)",
"Acoon-Robot 4.0.x.[xx] (http://www.acoon.de)",
"Acoon-Robot v3.xx (http://www.acoon.de and http://www.acoon.com)",
"Acorn/Nutch-0.9 (Non-Profit Search Engine; acorn.isara.org; acorn at isara dot org)",
"ActiveBookmark 1.x",
"Activeworlds",
"ActiveWorlds/3.xx (xxx)",
"Ad Muncher v4.xx.x",
"Ad Muncher v4x Build xxxxx",
"Adaxas Spider (http://www.adaxas.net/)",
"Advanced Browser (http://www.avantbrowser.com)",
"AESOP_com_SpiderMan",
"agadine/1.x.x (+http://www.agada.de)",
"Agent-SharewarePlazaFileCheckBot/2.0+(+http://www.SharewarePlaza.com)",
"AgentName/0.1 libwww-perl/5.48",
"AIBOT/2.1 By +(www.21seek.com A Real artificial intelligence search engine China)",
"AideRSS/1.0 (aiderss.com)",
"aipbot/1.0 (aipbot; http://www.aipbot.com; aipbot@aipbot.com)",
"aipbot/2-beta (aipbot dev; http://aipbot.com; aipbot@aipbot.com)",
"Akregator/1.2.9; librss/remnants",
"Aladin/3.324",
"Alcatel-BG3/1.0 UP.Browser/5.0.3.1.2",
"Aleksika Spider/1.0 (+http://www.aleksika.com/)",
"AlertInfo 2.0 (Powered by Newsbrain)",
"AlkalineBOT/1.3",
"AlkalineBOT/1.4 (1.4.0326.0 RTM)",
"Allesklar/0.1 libwww-perl/5.46",
"Alligator 1.31 (www.nearsoftware.com)",
"Allrati/1.1 (+)",
"AltaVista Intranet V2.0 AVS EVAL search@freeit.com",
"AltaVista Intranet V2.0 Compaq Altavista Eval sveand@altavista.net",
"AltaVista Intranet V2.0 evreka.com crawler@evreka.com",
"AltaVista V2.0B crawler@evreka.com",
"AmfibiBOT",
"Amfibibot/0.06 (Amfibi Web Search; http://www.amfibi.com; agent@amfibi.com)",
"Amfibibot/0.07 (Amfibi Robot; http://www.amfibi.com; agent@amfibi.com)",
"amibot",
"Amiga-AWeb/3.4.167SE",
"AmigaVoyager/3.4.4 (MorphOS/PPC native)",
"AmiTCP Miami (AmigaOS 2.04)",
"Amoi 8512/R21.0 NF-Browser/3.3",
"amzn_assoc",
"AnnoMille spider 0.1 alpha - http://www.annomille.it",
"annotate_google; http://ponderer.org/download/annotate_google.user.js",
"Anonymized by ProxyOS: http://www.megaproxy.com",
"Anonymizer/1.1",
"AnswerBus (http://www.answerbus.com/)",
"AnswerChase PROve x.0",
"AnswerChase x.0",
"ANTFresco/x.xx",
"antibot-V1.1.5/i586-linux-2.2",
"AnzwersCrawl/2.0 (anzwerscrawl@anzwers.com.au;Engine)",
"Apexoo Spider 1.x",
"Aplix HTTP/1.0.1",
"Aplix_SANYO_browser/1.x (Japanese)",
"Aplix_SEGASATURN_browser/1.x (Japanese)",
"Aport",
"appie 1.1 (www.walhello.com)",
"Apple iPhone v1.1.4 CoreMedia v1.0.0.4A102",
"ArabyBot (compatible; Mozilla/5.0; GoogleBot; FAST Crawler 6.4; http://www.araby.com;)",
"ArachBot",
"Arachnoidea (arachnoidea@euroseek.com)",
"aranhabot",
"ArchitextSpider",
"archive.org_bot",
"Argus/1.1 (Nutch; http://www.simpy.com/bot.html; feedback at simpy dot com)",
"Arikus_Spider",
"Arquivo-web-crawler (compatible; heritrix/1.12.1 +http://arquivo-web.fccn.pt)",
"ASAHA Search Engine Turkey V.001 (http://www.asaha.com/)",
"Asahina-Antenna/1.x",
"Asahina-Antenna/1.x (libhina.pl/x.x ; libtime.pl/x.x)",
"ask.24x.info",
"AskAboutOil/0.06-rcp (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@askaboutoil.com)",
"asked/Nutch-0.8 (web crawler; http://asked.jp; epicurus at gmail dot com)",
"ASPSeek/1.2.5",
"ASPseek/1.2.9d",
"ASPSeek/1.2.x",
"ASPSeek/1.2.xa",
"ASPseek/1.2.xx",
"ASPSeek/1.2.xxpre",
"ASSORT/0.10",
"asterias/2.0",
"AtlocalBot/1.1 +(http://www.atlocal.com/local-web-site-owner.html)",
"Atomic_Email_Hunter/4.0",
"Atomz/1.0",
"atSpider/1.0",
"Attentio/Nutch-0.9-dev (Attentio's beta blog crawler; www.attentio.com; info@attentio.com)",
"AU-MIC/2.0 MMP/2.0",
"AUDIOVOX-SMT5600",
"augurfind",
"augurnfind V-1.x",
"autoemailspider",
"autohttp",
"autowebdir 1.1 (www.autowebdir.com)",
"AV Fetch 1.0",
"Avant Browser (http://www.avantbrowser.com)",
"AVSearch-1.0(peter.turney@nrc.ca)",
"AVSearch-2.0-fusionIdx-14-CompetitorWebSites",
"AVSearch-3.0(AltaVista/AVC)",
"AWeb",
"axadine/ (Axadine Crawler; http://www.axada.de/; )",
"AxmoRobot - Crawling your site for better indexing on www.axmo.com search engine.",
"******* 2.x.x.x",
"BaboomBot/1.x.x (+http://www.baboom.us)",
"BackStreet Browser 3.x",
"BaiduImagespider+(+http://www.baidu.jp/search/s308.html)",
"BaiDuSpider",
"Baiduspider+(+http://help.baidu.jp/system/05.html)",
"Baiduspider+(+http://www.baidu.com/search/spider.htm)",
"Baiduspider+(+http://www.baidu.com/search/spider_jp.html)",
"Balihoo/Nutch-1.0-dev (Crawler for Balihoo.com search engine - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)",
"BanBots/1.2 (spider@banbots.com)",
"Barca/2.0.xxxx",
"BarcaPro/1.4.xxxx",
"BarraHomeCrawler (albertof@barrahome.org)",
"bCentral Billing Post-Process",
"bdcindexer_2.6.2 (research@bdc)",
"BDFetch",
"BDNcentral Crawler v2.3 [en] (http://www.bdncentral.com/robot.html) (X11; I; Linux 2.0.44 i686)",
"BeamMachine/0.5 (dead link remover of www.beammachine.net)",
"beautybot/1.0 (+http://www.uchoose.de/crawler/beautybot/)",
"BebopBot/2.5.1 ( crawler http://www.apassion4jazz.net/bebopbot.html )",
"BeebwareDirectory/v0.01",
"Big Brother (http://pauillac.inria.fr/~fpottier/)",
"Big Fish v1.0",
"BigBrother/1.6e",
"BigCliqueBOT/1.03-dev (bigclicbot; http://www.bigclique.com; bot@bigclique.com)",
"BIGLOTRON (Beta 2;GNU/Linux)",
"Bigsearch.ca/Nutch-x.x-dev (Bigsearch.ca Internet Spider; http://www.bigsearch.ca/; info@enhancededge.com)",
"Bilbo/2.3b-UNIX",
"BilgiBetaBot/0.8-dev (bilgi.com (Beta) ; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
"BilgiBot/1.0(beta) (http://www.bilgi.com/; bilgi at bilgi dot com)",
"billbot wjj@cs.cmu.edu",
"Bitacle bot/1.1",
"Bitacle Robot (V:1.0;) (http://www.bitacle.com)",
"Biyubi/x.x (Sistema Fenix; G11; Familia Toledo; es-mx)",
"BlackBerry7520/4.0.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/5.0.3.3 UP.Link/5.1.2.12 (Google WAP Proxy/1.0)",
"BlackWidow",
"BlackWidow",
"Blaiz-Bee/1.0 (+http://www.blaiz.net)",
"Blaiz-Bee/2.00.8222 (BE Internet Search Engine http://www.rawgrunt.com)",
"Blaiz-Bee/2.00.xxxx (+http://www.blaiz.net)",
"BlitzBOT@tricus.net",
"BlitzBOT@tricus.net (Mozilla compatible)",
"BlockNote.Net",
"BlogBot/1.x",
"BlogBridge 2.13 (http://www.blogbridge.com/)",
"Bloglines Title Fetch/1.0 (http://www.bloglines.com)",
"Bloglines-Images/0.1 (http://www.bloglines.com)",
"Bloglines/3.1 (http://www.bloglines.com)",
"BlogMap (http://www.feedmap.net)",
"Blogpulse (info@blogpulse.com)",
"BlogPulseLive (support@blogpulse.com)",
"BlogSearch/1.x +http://www.icerocket.com/",
"blogsearchbot-pumpkin-3",
"BlogVibeBot-v1.1 (spider@blogvibe.nl)",
"blogWatcher_Spider/0.1 (http://www.lr.pi.titech.ac.jp/blogWatcher/)",
"BlogzIce/1.0 (+http://icerocket.com; rhodes@icerocket.com)",
"BlogzIce/1.0 +http://www.icerocket.com/",
"BloobyBot",
"Bloodhound/Nutch-0.9 (Testing Crawler for Research - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)",
"bluefish 0.6 HTML editor",
"BMCLIENT",
"BMLAUNCHER",
"Bobby/4.0.x RPT-HTTPClient/0.3-3E",
"boitho.com-dc/0.xx (http://www.boitho.com/dcbot.html)",
"boitho.com-robot/1.x",
"boitho.com-robot/1.x (http://www.boitho.com/bot.html)",
"Bookdog/x.x",
"Bookmark Buddy bookmark checker (http://www.bookmarkbuddy.net/)",
"Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/]",
"Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/] (Version 2.0beta)",
"BookmarkBase(2/;http://bookmarkbase.com)",
"Bot mailto:craftbot@yahoo.com",
"BPImageWalker/2.0 (www.bdbrandprotect.com)",
"BravoBrian bstop.bravobrian.it",
"BravoBrian SpiderEngine MarcoPolo",
"BrightCrawler (http://www.brightcloud.com/brightcrawler.asp)",
"BruinBot (+http://webarchive.cs.ucla.edu/bruinbot.html) ",
"BSDSeek/1.0",
"BStop.BravoBrian.it Agent Detector",
"BTbot/0.x (+http://www.btbot.com/btbot.html)",
"BTWebClient/180B(9704)",
"BuildCMS crawler (http://www.buildcms.com/crawler)",
"Bulkfeeds/r1752 (http://bulkfeeds.net/)",
"BullsEye",
"bumblebee@relevare.com",
"BunnySlippers",
"*****FindCrawler/1.1 (crawler.*****find.com; http://crawler.*****find.com; crawler@*****find.com)",
"Buscaplus Robi/1.0 (http://www.buscaplus.com/robi/)",
"BW-C-2.0",
"bwh3_user_agent",
"Cabot/Nutch-0.9 (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)",
"Cabot/Nutch-1.0-dev (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)",
"CamelHttpStream/1.0",
"Cancer Information and Support International;",
"carleson/1.0",
"Carnegie_Mellon_University_Research_WebBOT-->PLEASE READ-->http://www.andrew.cmu.edu/~brgordon/webbot/index.html http://www.andrew.cmu.edu/~brgordon/webbot/index.html",
"Carnegie_Mellon_University_WebCrawler http://www.andrew.cmu.edu/~brgordon/webbot/index.html",
"Catall Spider",
"CazoodleBot/CazoodleBot-0.1 (CazoodleBot Crawler; http://www.cazoodle.com/cazoodlebot; cazoodlebot@cazoodle.com)",
"CCBot/1.0 (+http://www.commoncrawl.org/bot.html)",
"ccubee/x.x",
"CDR/1.7.1 Simulator/0.7(+http://timewe.net) Profile/MIDP-1.0 Configuration/CLDC-1.0",
"CE-Preload",
"CentiverseBot",
"CentiverseBot - investigator",
"CentiverseBot/3.0 (http://www.centiverse-project.net)",
"Ceramic Tile Installation Guide (http://www.floorstransformed.com)",
"cfetch/1.0",
"CFNetwork/x.x",
"cg-eye interactive",
"Charon/1.x (Amiga)",
"Checkbot/1.xx LWP/5.xx",
"CheckLinks/1.x.x",
"CheckUrl",
"CheckWeb",
"Chilkat/1.0.0 (+http://www.chilkatsoft.com/ChilkatHttpUA.asp)",
"China Local Browse 2.6",
"Chitika ContentHit 1.0",
"ChristCRAWLER 2.0",
"CHttpClient by Open Text Corporation",
"CipinetBot (http://www.cipinet.com/bot.html)",
"Cityreview Robot (+http://www.cityreview.org/crawler/)",
"CJ Spider/",
"CJB.NET Proxy",
"ClariaBot/1.0",
"Claymont.com",
"CloakDetect/0.9 (+http://fulltext.seznam.cz/)",
"Clushbot/2.x (+http://www.clush.com/bot.html)",
"Clushbot/3.x-BinaryFury (+http://www.clush.com/bot.html)",
"Clushbot/3.xx-Ajax (+http://www.clush.com/bot.html)",
"Clushbot/3.xx-Hector (+http://www.clush.com/bot.html)",
"Clushbot/3.xx-Peleus (+http://www.clush.com/bot.html)",
"COAST WebMaster Pro/4.x.x.xx (Windows NT)",
"CoBITSProbe",
"Cocoal.icio.us/1.0 (v36) (Mac OS X; http://www.scifihifi.com/cocoalicious)",
"ColdFusion",
"ColdFusion (BookmarkTracker.com)",
"collage.cgi/1.xx",
"combine/0.0",
"Combine/2.0 http://combine.it.lth.se/",
"Combine/3 http://combine.it.lth.se/",
"Combine/x.0",
"Commerce Browser Center",
"complex_network_group/Nutch-0.9-dev (discovering the structure of the world-wide-web; http://cantor.ee.ucla.edu/~networks/crawl; nimakhaj@gmail.com)",
"Computer_and_Automation_Research_Institute_Crawler crawler@ilab.sztaki.hu",
"Comrite/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
"Contact",
"ContactBot/0.2",
"ContentSmartz",
"contype",
"Convera Internet Spider V6.x",
"ConveraCrawler/0.2",
"ConveraCrawler/0.9d (+http://www.authoritativeweb.com/crawl)",
"ConveraMultiMediaCrawler/0.1 (+http://www.authoritativeweb.com/crawl)",
"CoolBot",
"CoralWebPrx/0.1.1x (See http://coralcdn.org/)",
"cosmos/0.8_(robot@xyleme.com)",
"cosmos/0.9_(robot@xyleme.com)",
"CoteoNutchCrawler/Nutch-0.9 (info [at] coteo [dot] com)",
"CougarSearch/0.x (+http://www.cougarsearch.com/faq.shtml)",
"Covac TexAs Arachbot",
"Cowbot-0.1 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)",
"Cowbot-0.1.x (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)",
"CrawlConvera0.1 (CrawlConvera@yahoo.com)",
"Crawler",
"Crawler (cometsearch@cometsystems.com)",
"Crawler admin@crawler.de",
"Crawler V 0.2.x admin@crawler.de",
"crawler@alexa.com",
"CrawlerBoy Pinpoint.com",
"Crawllybot/0.1 (Crawllybot; +http://www.crawlly.com; crawler@crawlly.com)",
"CreativeCommons/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)",
"CrocCrawler vx.3 [en] (http://www.croccrawler.com) (X11; I; Linux 2.0.44 i686)",
"csci_b659/0.13",
"CSE HTML Validator Professional (http://www.htmlvalidator.com/)",
"Cuam Ver0.050bx",
"Cuasarbot/0.9b http://www.cuasar.com/spider_beta/ ",
"curl/7.10.x (i386-redhat-linux-gnu) libcurl/7.10.x OpenSSL/0.9.7a ipv6 zlib/1.1.4",
"curl/7.7.x (i386--freebsd4.3) libcurl 7.7.x (SSL 0.9.6) (ipv6 enabled)",
"curl/7.8 (i686-pc-linux-gnu) libcurl 7.8 (OpenSSL 0.9.6)",
"curl/7.9.x (win32) libcurl 7.9.x",
"CurryGuide SiteScan 1.1",
"Custo x.x (www.netwu.com)",
"Custom Spider www.bisnisseek.com /1.0",
"Cyberdog/2.0 (Macintosh; 68k)",
"CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)",
"CyberSpyder Link Test/2.1.12 (admin@mspennyworth.com)",
"CydralSpider/1.x (Cydral Web Image Search; http://www.cydral.com)",
"CydralSpider/3.0 (Cydral Image Search; http://www.cydral.com)",
"DA 3.5 (www.lidan.com)",
"DA 4.0",
"DA 4.0 (www.downloadaccelerator.com)",
"DA 5.0",
"DA 7.0",
"Dart Communications PowerTCP",
"DataCha0s/2.0",
"DataCha0s/2.0",
"DataFountains/DMOZ Downloader",
"DataFountains/Dmoz Downloader (http://ivia.ucr.edu/useragents.shtml)",
"DataFountains/DMOZ Feature Vector Corpus Creator (http://ivia.ucr.edu/useragents.shtml)",
"DataparkSearch/4.47 (+http://dataparksearch.org/bot)",
"DataparkSearch/4.xx (http://www.dataparksearch.org/)",
"DataSpear/1.0 (Spider; http://www.dataspear.com/spider.html; spider@dataspear.com)",
"DataSpearSpiderBot/0.2 (DataSpear Spider Bot; http://dssb.dataspear.com/bot.html; dssb@dataspear.com)",
"DatenBot( http://www.sicher-durchs-netz.de/bot.html)",
"DaviesBot/1.7 (www.wholeweb.net)",
"daypopbot/0.x",
"dbDig(http://www.prairielandconsulting.com)",
"DBrowse 1.4b",
"DBrowse 1.4d",
"DC-Sakura/x.xx",
"***bot/1.1",
"DDD",
"dds explorer v1.0 beta",
"de.searchengine.comBot 1.2 (http://de.searchengine.com/spider)",
"DeadLinkCheck/0.4.0 libwww-perl/5.xx",
"Deep Link Calculator v1.0",
"deepak-USC/ISI",
"DeepIndex",
"DeepIndex ( http://www.zetbot.com )",
"DeepIndex (www.en.deepindex.com)",
"DeepIndexer.ca",
"del.icio.us-thumbnails/1.0 Mozilla/5.0 (compatible; Konqueror/3.4; FreeBSD) KHTML/3.4.2 (like Gecko)",
"DeleGate/9.0.5-fix1",
"Demo Bot DOT 16b",
"Demo Bot Z 16b",
"Denmex websearch (http://search.denmex.com)",
"dev-spider2.searchpsider.com/1.3b",
"DevComponents.com HtmlDocument Object",
"DiaGem/1.1 (http://www.skyrocket.gr.jp/diagem.html)",
"Diamond/x.0",
"DiamondBot",
"Digger/1.0 JDK/1.3.0rc3",
"DigOut4U",
"DIIbot/1.2",
"Dillo/0.8.5-i18n-misc",
"Dillo/0.x.x",
"DISCo Pump x.x",
"disco/Nutch-0.9 (experimental crawler; www.discoveryengine.com; disco-crawl@discoveryengine.com)",
"disco/Nutch-1.0-dev (experimental crawler; www.discoveryengine.com; disco-crawl@discoveryengine.com)",
"DittoSpyder",
"dlman",
"dloader(NaverRobot)/1.0",
"DNSRight.com WebBot Link Ckeck Tool. Report abuse to: dnsr@dnsright.com",
"DoCoMo/1.0/Nxxxi/c10",
"DoCoMo/1.0/Nxxxi/c10/TB",
"DoCoMo/1.0/P502i/c10 (Google CHTML Proxy/1.0)",
"DoCoMo/2.0 P900iV(c100;TB;W24H11) ",
"DoCoMo/2.0 SH902i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)",
"DoCoMo/2.0/SO502i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)",
"DocZilla/1.0 (Windows; U; WinNT4.0; en-US; rv:1.0.0) Gecko/20020804",
"*****bot/experimental",
"DonutP; Windows98SE",
"Doubanbot/1.0 (bot@douban.com http://www.douban.com)",
"Download Demon/3.x.x.x",
"Download Druid 2.x",
"Download Express 1.0",
"Download Master",
"Download Ninja 3.0",
"Download Wonder",
"Download-Tipp Linkcheck (http://download-tipp.de/)",
"Download.exe(1.1) (+http://www.sql-und-xml.de/freeware-tools/)",
"DownloadDirect.1.0",
"Dr.Web (R) online scanner: http://online.drweb.com/",
"Dragonfly File Reader",
"Drecombot/1.0 (http://career.drecom.jp/bot.html)",
"Drupal (+http://drupal.org/)",
"DSurf15a 01",
"DSurf15a 71",
"DSurf15a 81",
"DSurf15a VA",
"DTAAgent",
"dtSearchSpider",
"Dual Proxy",
"DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
"Dumbot(version 0.1 beta - dumbfind.com)",
"Dumbot(version 0.1 beta - http://www.dumbfind.com/dumbot.html)",
"Dumbot(version 0.1 beta)",
"e-sense 1.0 ea(www.vigiltech.com/esensedisclaim.html)",
"e-SocietyRobot(http://www.yama.info.waseda.ac.jp/~yamana/es/)",
"eApolloBot/2.0 (compatible; heritrix/2.0.0-SNAPSHOT-20071024.170148 +http://www.eapollo-opto.com)",
"EARTHCOM.info/1.x [www.earthcom.info]",
"EARTHCOM.info/1.xbeta [www.earthcom.info]",
"EasyDL/3.xx",
"EasyDL/3.xx http://keywen.com/Encyclopedia/Bot",
"EBrowse 1.4b",
"eCatch/3.0",
"EchO!/2.0",
"Educate Search VxB",
"egothor/3.0a (+http://www.xdefine.org/robot.html)",
"EgotoBot/4.8 (+http://www.egoto.com/about.htm)",
"ejupiter.com",
"EldoS TimelyWeb/3.x",
"elfbot/1.0 (+http://www.uchoose.de/crawler/elfbot/)",
"ELinks (0.x.x; Linux 2.4.20 i586; 132x60)",
"ELinks/0.x.x (textmode; NetBSD 1.6.2 sparc; 132x43)",
"EmailSiphon",
"EmailSpider",
"EmailWolf 1.00",
"EmeraldShield.com WebBot",
"EmeraldShield.com WebBot (http://www.emeraldshield.com/webbot.aspx)",
"EMPAS_ROBOT",
"EnaBot/1.x (http://www.enaball.com/crawler.html)",
"endo/1.0 (Mac OS X; ppc i386; http://kula.jp/endo)",
"Enfish Tracker",
"Enterprise_Search/1.0",
"Enterprise_Search/1.0.xxx",
"Enterprise_Search/1.00.xxx;MSSQL (http://www.innerprise.net/es-spider.asp)",
"envolk/1.7 (+http://www.envolk.com/envolkspiderinfo.php)",
"envolk[ITS]spider/1.6(+http://www.envolk.com/envolkspider.html)",
"EroCrawler",
"ES.NET_Crawler/2.0 (http://search.innerprise.net/)",
"eseek-larbin_2.6.2 (crawler@exactseek.com)",
"ESISmartSpider",
"eStyleSearch 4 (compatible; MSIE 6.0; Windows NT 5.0)",
"ESurf15a 15",
"EuripBot/0.x (+http://www.eurip.com) GetFile",
"EuripBot/0.x (+http://www.eurip.com) GetRobots",
"EuripBot/0.x (+http://www.eurip.com) PreCheck",
"Eurobot/1.0 (http://www.ayell.eu)",
"EvaalSE - bot@evaal.com",
"eventax/1.3 (eventax; http://www.eventax.de/; info@eventax.de)",
"Everest-Vulcan Inc./0.1 (R&D project; host=e-1-24; http://everest.vulcan.com/crawlerhelp)",
"Everest-Vulcan Inc./0.1 (R&D project; http://everest.vulcan.com/crawlerhelp)",
"Exabot-Images/1.0",
"Exabot-Test/1.0",
"Exabot/2.0",
"Exabot/3.0",
"ExactSearch",
"ExactSeek Crawler/0.1",
"exactseek-crawler-2.63 (crawler@exactseek.com)",
"exactseek-pagereaper-2.63 (crawler@exactseek.com)",
"exactseek.com",
"Exalead NG/MimeLive Client (convert/http/0.120)",
"Excalibur Internet Spider V6.5.4",
"Execrawl/1.0 (Execrawl; http://www.execrawl.com/; bot@execrawl.com)",
"exooba crawler/exooba crawler (crawler for exooba.com; http://www.exooba.com/; info at exooba dot com)",
"exooba/exooba crawler (exooba; exooba)",
"ExperimentalHenrytheMiragoRobot",
"Expired Domain Sleuth",
"Express WebPictures (www.express-soft.com)",
"*********Pro",
"Extreme Picture Finder",
"EyeCatcher (Download-tipp.de)/1.0",
"Factbot 1.09 (see http://www.factbites.com/webmasters.php)",
"factbot : http://www.factbites.com/robots",
"FaEdit/2.0.x",
"FairAd Client",
"FANGCrawl/0.01",
"FARK.com link verifier",
"Fast Crawler Gold Edition",
"FAST Enterprise Crawler 6 (Experimental)",
"FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/",
"FAST Enterprise Crawler 6 used by Cobra Development (admin@fastsearch.com)",
"FAST Enterprise Crawler 6 used by Comperio AS (sts@comperio.no)",
"FAST Enterprise Crawler 6 used by FAST (FAST)",
"FAST Enterprise Crawler 6 used by Pages Jaunes (pvincent@pagesjaunes.fr)",
"FAST Enterprise Crawler 6 used by Sensis.com.au Web Crawler (search_comments\at\sensis\dot\com\dot\au)",
"FAST Enterprise Crawler 6 used by Singapore Press Holdings (crawler@sphsearch.sg)",
"FAST Enterprise Crawler 6 used by WWU (wardi@uni-muenster.de)",
"FAST Enterprise Crawler/6 (www.fastsearch.com)",
"FAST Enterprise Crawler/6.4 (helpdesk at fast.no)",
"FAST FirstPage retriever (compatible; MSIE 5.5; Mozilla/4.0)",
"FAST MetaWeb Crawler (helpdesk at fastsearch dot com)",
"Fast PartnerSite Crawler",
"FAST-WebCrawler/2.2.10 (Multimedia Search) (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)",
"FAST-WebCrawler/2.2.6 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)",
"FAST-WebCrawler/2.2.7 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no",
"FAST-WebCrawler/2.2.8 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no",
"FAST-WebCrawler/3.2 test",
"FAST-WebCrawler/3.3 (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)",
"FAST-WebCrawler/3.4/Nirvana (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)",
"FAST-WebCrawler/3.4/PartnerSite (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)",
"FAST-WebCrawler/3.5 (atw-crawler at fast dot no; http://fast.no/support.php?c=faqs/crawler)",
"FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
"FAST-WebCrawler/3.6/FirstPage (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)",
"FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
"FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",
"FAST-WebCrawler/3.8 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
"FAST-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
"FAST-WebCrawler/3.x Multimedia",
"FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no)",
"fastbot crawler beta 2.0 (+http://www.fastbot.de)",
"FastBug http://www.ay-up.com",
"FastCrawler 3.0.1 (crawler@1klik.dk)",
"FastSearch Web Crawler for Verizon SuperPages (kevin.watters@fastsearch.com)",
"Favcollector/2.0 (info@favcollector.com http://www.favcollector.com/)",
"FavIconizer",
"favo.eu crawler/0.6 (http://www.favo.eu)",
"FavOrg",
"Favorites Checking (http://campulka.net)",
"Favorites Sweeper v.2.03",
"Faxobot/1.0",
"FDM 1.x",
"FDM 2.x",
"Feed Seeker Bot (RSS Feed Seeker http://www.MyNewFavoriteThing.com/fsb.php)",
"Feed24.com",
"Feed::Find/0.0x",
"Feedable/0.1 (compatible; MSIE 6.0; Windows NT 5.1)",
"FeedChecker/0.01",
"FeedDemon/2.7 (http://www.newsgator.com/; Microsoft Windows XP)",
"Feedfetcher-Google-iGoogleGadgets; (+http://www.google.com/feedfetcher.html)",
"Feedfetcher-Google; (+http://www.google.com/feedfetcher.html)",
"FeedForAll rss2html.php v2",
"FeedHub FeedDiscovery/1.0 (http://www.feedhub.com)",
"FeedHub MetaDataFetcher/1.0 (http://www.feedhub.com)",
"Feedjit Favicon Crawler 1.0",
"Feedreader 3.xx (Powered by Newsbrain)",
"Feedshow/x.0 (http://www.feedshow.com; 1 subscriber)",
"FeedshowOnline (http://www.feedshow.com)",
"FeedZcollector v1.x (Platinum) http://www.feeds4all.com/feedzcollector",
"Felix - Mixcat Crawler (+http://mixcat.com)",
"fetch libfetch/2.0",
"FFC Trap Door Spider",
"Filangy/0.01-beta (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)",
"Filangy/1.0x (Filangy; http://www.filangy.com/filangyinfo.jsp?inc=robots.jsp; filangy-agent@filangy.com)",
"Filangy/1.0x (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)",
"fileboost.net/1.0 (+http://www.fileboost.net)",
"FileHound x.x",
"Filtrbox/1.0",
"FindAnISP.com_ISP_Finder_v99a",
"Findexa Crawler (http://www.findexa.no/gulesider/article26548.ece)",
"findlinks/x.xxx (+http://wortschatz.uni-leipzig.de/findlinks/) ",
"FineBot",
"Finjan-prefetch",
"Firefly/1.0",
"Firefly/1.0 (compatible; Mozilla 4.0; MSIE 5.5)",
"Firefox (kastaneta03@hotmail.com)",
"Firefox_1.0.6 (kasparek@naparek.cz)",
"FirstGov.gov Search - POC:firstgov.webmasters@gsa.gov",
"firstsbot",
"Flapbot/0.7.2 (Flaptor Crawler; http://www.flaptor.com; crawler at flaptor period com)",
"FlashGet",
"FLATARTS_FAVICO",
"Flexum spider",
"Flexum/2.0",
"FlickBot 2.0 RPT-HTTPClient/0.3-3",
"flunky",
"fly/6.01 libwww/4.0D",
"flyindex.net 1.0/http://www.flyindex.net",
"FnooleBot/2.5.2 (+http://www.fnoole.com/addurl.html)",
"FocusedSampler/1.0",
"Folkd.com Spider/0.1 beta 1 (www.folkd.com)",
"Fooky.com/ScorpionBot/ScoutOut; http://www.fooky.com/scorpionbots",
"Francis/1.0 (francis@neomo.de http://www.neomo.de/)",
"Franklin Locator 1.8",
"free-downloads.net download-link validator /0.1",
"FreeFind.com-SiteSearchEngine/1.0 (http://freefind.com; spiderinfo@freefind.com)",
"Frelicbot/1.0 +http://www.frelic.com/",
"FreshDownload/x.xx",
"FreshNotes crawler< report problems to crawler-at-freshnotes-dot-com",
"FSurf15a 01",
"Full Web Bot 0416B",
"Full Web Bot 0516B",
"Full Web Bot 2816B",
"FuseBulb.Com",
"FyberSpider (+http://www.fybersearch.com/fyberspider.php)",
"Gagglebot",
"GAIS Robot/1.0B2",
"Gaisbot/3.0 (indexer@gais.cs.ccu.edu.tw; http://gais.cs.ccu.edu.tw/robot.php)",
"Gaisbot/3.0+(robot06@gais.cs.ccu.edu.tw;+http://gais.cs.ccu.edu.tw/robot.php)",
"GalaxyBot/1.0 (http://www.galaxy.com/galaxybot.html)",
"Gallent Search Spider v1.4 Robot 2 (http://robot.GallentSearch.com)",
"gamekitbot/1.0 (+http://www.uchoose.de/crawler/gamekitbot/)",
"Gamespy_Arcade",
"GammaSpider/1.0",
"gazz/x.x (gazz@nttrd.com)",
"geckobot",
"Generic Mobile Phone (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
"generic_crawler/01.0217/",
"genieBot (http://64.5.245.11/faq/faq.html)",
"geniebot wgao@genieknows.com",
"GeoBot/1.0",
"GeonaBot 1.x; http://www.geona.com/",
"geourl/2.0b2",
"GeoURLBot 1.0 (http://geourl.org)",
"GetBot",
"GetRight/3.x.x",
"GetRight/4.5xx",
"GetRight/4.x",
"GetRight/4.x[a-e]",
"GetRight/6.1 (Pro)",
"GetRightPro/6.0beta2",
"GetWeb/0.1 libwww-perl/5.16",
"GhostRouteHunter/20021130 (https://www.sixxs.net/tools/grh/; info@sixxs.net)",
"gigabaz/3.1x (baz@gigabaz.com; http://gigabaz.com/gigabaz/)",
"Gigabot/2.0 (gigablast.com)",
"Gigabot/2.0/gigablast.com/spider.html",
"Gigabot/2.0; http://www.gigablast.com/spider.html",
"Gigabot/2.0att",
"Gigabot/3.0 (http://www.gigablast.com/spider.html)",
"Gigabot/x.0",
"GigabotSiteSearch/2.0 (sitesearch.gigablast.com)",
"GNODSPIDER (www.gnod.net)",
"Go!Zilla 3.x (www.gozilla.com)",
"Go!Zilla/4.x.x.xx",
"Go-Ahead-Got-It/1.1",
"Goblin/0.9 (http://www.goguides.org/)",
"Goblin/0.9.x (http://www.goguides.org/goblin-info.html)",
"GoForIt.com",
"GOFORITBOT ( http://www.goforit.com/about/ )",
"GoGuides.Org Link Check",
"GoldenFeed Spider 1.0 (http://www.goldenfeed.com)",
"Goldfire Server",
"gonzo1[P] +http://www.suchen.de/popups/faq.jsp",
"gonzo2[P] +http://www.suchen.de/faq.html",
"Goofer/0.2",
"Google Talk",
"googlebot (larbin2.6.0@unspecified.mail)",
"Googlebot-Image/1.0",
"Googlebot-Image/1.0 ( http://www.googlebot.com/bot.html)",
"Googlebot/2.1 ( http://www.google.com/bot.html)",
"Googlebot/2.1 ( http://www.googlebot.com/bot.html)",
"Googlebot/Test ( http://www.googlebot.com/bot.html)",
"GrapeFX/0.3 libwww/5.4.0",
"great-plains-web-spider/flatlandbot (Flatland Industries Web Spider; http://www.flatlandindustries.com/flatlandbot.php; jason@flatlandindustries.com)",
"GreatNews/1.0",
"GreenBrowser",
"gridwell (http://search.gridwell.com)",
"GrigorBot 0.8 (http://www.grigor.biz/bot.html)",
"Gromit/1.0",
"grub crawler(http://www.grub.org)",
"grub-client",
"gsa-crawler (Enterprise; GID-01422; jplastiras@google.com)",
"gsa-crawler (Enterprise; GID-01742;gsatesting@rediffmail.com)",
"gsa-crawler (Enterprise; GIX-02057; dm@enhesa.com)",
"gsa-crawler (Enterprise; GIX-03519; cknuetter@stubhub.com)",
"gsa-crawler (Enterprise; GIX-0xxxx; enterprise-training@google.com)",
"GSiteCrawler/v1.xx rev. xxx (http://gsitecrawler.com/)",
"Guestbook Auto Submitter",
"*****ver/1.3",
"Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)",
"Gungho/0.08004 (http://code.google.com/p/gungho-crawler/wiki/Index)",
"GurujiBot/1.0 (+http://www.guruji.com/WebmasterFAQ.html)",
"GurujiImageBot/1.0 (+http://www.guruji.com/en/WebmasterFAQ.html)",
"Haier-T10C/1.0 iPanel/2.0 WAP2.0 (compatible; UP.Browser/6.2.2.4; UPG1; UP/4.0; Embedded)",
"HappyFunBot/1.1",
"Harvest-NG/1.0.2",
"Haste/0.12 (HOME: http://haste.kytoon.com/)",
"Hatena Antenna/0.4 (http://a.hatena.ne.jp/help#robot)",
"Hatena Mobile Gateway/1.0",
"Hatena Pagetitle Agent/1.0",
"Hatena RSS/0.3 (http://r.hatena.ne.jp)",
"HatenaScreenshot/1.0 (checker)",
"hbtronix.spider.2 -- http://hbtronix.de/spider.php",
"HeinrichderMiragoRobot",
"HeinrichderMiragoRobot (http://www.miragorobot.com/scripts/deinfo.asp)",
"Helix/1.x ( http://www.sitesearch.ca/helix/)",
"HenriLeRobotMirago (http://www.miragorobot.com/scripts/frinfo.asp)",
"HenrytheMiragoRobot",
"HenryTheMiragoRobot (http://www.miragorobot.com/scripts/mrinfo.asp)",
"hgrepurl/1.0",
"Hi! I'm CsCrawler my homepage: http://www.kde.cs.uni-kassel.de/lehre/ss2005/googlespam/crawler.html RPT-HTTPClient/0.3-3",
"HiDownload",
"Hippias/0.9 Beta",
"HitList",
"Hitwise Spider v1.0 http://www.hitwise.com",
"HLoader",
"holmes/3.11 (http://morfeo.centrum.cz/bot)",
"holmes/3.9 (onet.pl)",
"holmes/3.xx (OnetSzukaj/5.0; +http://szukaj.onet.pl)",
"holmes/x.x",
"HolmesBot (http://holmes.ge)",
"HomePageSearch(hpsearch.uni-trier.de)",
"Homerbot: www.homerweb.com",
"Honda-Search/0.7.2 (Nutch; http://lucene.apache.org/nutch/bot.html; search@honda-search.com)",
"HooWWWer/2.1.3 (debugging run) (+http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-info<at>hiit.fi)",
"HooWWWer/2.1.x ( http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-info<at>hiit.fi)",
"HotJava/1.0.1/JRE1.1.x",
"Hotzonu/x.0",
"HPL/Nutch-0.9 -",
"htdig/3.1.6 (http://computerorgs.com)",
"htdig/3.1.6 (unconfigured@htdig.searchengine.maintainer)",
"htdig/3.1.x (root@localhost)",
"Html Link Validator (www.lithopssoft.com)",
"HTML2JPG Enterprise",
"HTMLParser/1.x",
"HTTP Retriever",
"http://Anonymouse.org/ (Unix)",
"http://Ask.24x.Info/ (http://narres.it/)",
"http://OzySoftware.com/Index.html",
"http://www.almaden.ibm.com/cs/crawler",
"http://www.almaden.ibm.com/cs/crawler [rc1.wf.ibm.com]",
"http://www.almaden.ibm.com/cs/crawler [wf216]",
"http://www.istarthere.com_spider@istarthere.com",
"http://www.monogol.de",
"http://www.trendtech.dk/spider.asp)",
"HTTP::Lite/2.x.x",
"HTTPEyes",
"HTTPResume v. 1.x",
"httpunit/1.5",
"httpunit/1.x",
"Hybrid/1.2 [en] (OS Independent)",
"HyperEstraier/1.x.xx",
"i1searchbot/2.0 (i1search web crawler; http://www.i1search.com; crawler@i1search.com)",
"IAArchiver-1.0",
"iaskspider",
"iaskspider2 (iask@staff.sina.com.cn)",
"ia_archiver",
"ia_archiver-web.archive.org",
"ia_archiver/1.6",
"IBrowse/2.2 (AmigaOS 3.5)",
"IBrowse/2.2 (Windows 3.1)",
"iCab/2.5.2 (Macintosh; I; PPC)",
"ICC-Crawler(Mozilla-compatible; http://kc.nict.go.jp/icc/crawl.html; icc-crawl(at)ml(dot)nict(dot)go(dot)jp)",
"ICC-Crawler(Mozilla-compatible;http://kc.nict.go.jp/icc/crawl.html;icc-crawl-contact(at)ml(dot)nict(dot)go(dot)jp)",
"iCCrawler (http://www.iccenter.net)",
"ICCrawler - ICjobs (http://www.icjobs.de/bot.htm)",
"ICE Browser/5.05 (Java 1.4.0; Windows 2000 5.0 x86)",
"ichiro/x.0 (http://help.goo.ne.jp/door/crawler.html)",
"ichiro/x.0 (ichiro@nttr.co.jp)",
"IconSurf/2.0 favicon finder (see http://iconsurf.com/robot.html)",
"IconSurf/2.0 favicon monitor (see http://iconsurf.com/robot.html)",
"ICOO Loader v.x.x.x",
"ICRA_label_spider/x.0",
"icsbot-0.1",
"IDA",
"ideare - SignSite/1.x",
"IEFav172Free",
"iFeed.jp/2.0 (www.psychedelix.com/agents/agents.rss; 0 subscribers)",
"igdeSpyder (compatible; igde.ru; +http://igde.ru/doc/tech.html)",
"iGetter/1.x (Macintosh;G;PPC)",
"iGetter/2 (Macintosh; U; PPC Mac OS X; en)",
"IIITBOT/1.1 (Indian Language Web Search Engine; http://webkhoj.iiit.net; pvvpr at iiit dot ac dot in)",
"ilial/Nutch-0.9-dev",
"IlseBot/1.x",
"IlTrovatore-Setaccio ( http://www.iltrovatore.it)",
"Iltrovatore-Setaccio/0.3-dev (Indexing; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)",
"IlTrovatore-Setaccio/1.2 ( http://www.iltrovatore.it/aiuto/faq.html)",
"Iltrovatore-Setaccio/1.2 (It-bot; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)",
"iltrovatore-setaccio/1.2-dev (spidering; http://www.iltrovatore.it/aiuto/.....)",
"IlTrovatore/1.2 (IlTrovatore; http://www.iltrovatore.it/bot.html; bot@iltrovatore.it)",
"ImageVisu/v4.x.x",
"ImageWalker/2.0 (www.bdbrandprotect.com)",
"Incutio HttpClient v0.x",
"IncyWincy data gatherer(webmaster@loopimprovements.com",
"IncyWincy page crawler(webmaster@loopimprovements.com",
"IncyWincy(http://www.look.com)",
"IncyWincy(http://www.loopimprovements.com/robot.html)",
"IncyWincy/2.1(loopimprovements.com/robot.html)",
"IndexTheWeb.com Crawler7",
"Industry Program 1.0.x",
"Inet library",
"InetURL/1.0",
"info@pubblisito.com- (http://www.pubblisito.com) il Sud dei Motori di Ricerca",
"infoConveraCrawler/0.8 ( http://www.authoritativeweb.com/crawl)",
"InfoFly/1.0 (http://www.versions-project.org/)",
"InfoLink/1.x",
"INFOMINE/8.0 Adders",
"INFOMINE/8.0 RemoteServices",
"INFOMINE/8.0 VLCrawler (http://infomine.ucr.edu/useragents)",
"InfoNaviRobot(F107)",
"InfoSeek Sidewinder/0.9",
"InfoSeek Sidewinder/1.0A",
"InfoSeek Sidewinder/1.1A",
"Infoseek SideWinder/1.45 (Compatible; MSIE 10.0; UNIX)",
"Infoseek SideWinder/2.0B (Linux 2.4 i686)",
"INGRID/3.0 MT (webcrawler@NOSPAMexperimental.net; http://webmaster.ilse.nl/jsp/webmaster.jsp)",
"Inktomi Search",
"InnerpriseBot/1.0 (http://www.innerprise.com/)",
"Insitor.com search and find world wide!",
"Insitornaut",
"InstallShield DigitalWizard",
"integrity/1.6",
"Intelix/0.x (cs; http://www.microton.cz/intelix/; microton@@microton.cz)",
"Interarchy/x.x.x (InterarchyCrawler)",
"Internet Ninja x.0",
"InternetArchive/0.8-dev(Nutch;http://lucene.apache.org/nutch/bot.html;nutch-agent@lucene.apache",
"InternetLinkAgent/3.1",
"InternetSeer.com",
"intraVnews/1.x",
"IP*Works! V5 HTTP/S Component - by /n software - www.nsoftware.com",
"http://www.ip2location.com",
"IP2MapBot/1.1 <a href=http://www.ip2map.com>http://www.ip2map.com</a>",
"IPiumBot laurion(dot)com",
"IpselonBot/0.xx-beta (Ipselon; http://www.ipselon.com; ipselonbot@ipselon.com)",
"Iria/1.xxa",
"IRLbot/1.0 ( http://irl.cs.tamu.edu/crawler)",
"IRLbot/3.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler/)",
"IrssiUrlLog/0.2",
"Irvine/1.x.x",
"ISC Systems iRc Search 2.1",
"iSiloX/4.xx Windows/32",
"isurf (tszhu@canada.com)",
"iTunes/x.x.x",
"IUPUI Research Bot v 1.9a",
"iVia Page Fetcher (http://ivia.ucr.edu/useragents.shtml)",
"iVia/4.0 CanonizeUrl (http://infomine.ucr.edu/iVia/useragents.shtml",
"IWAgent/ 1.0 - www.brandprotect.com",
"J-PHONE/3.0/J-SH07",
"Jabot/6.x (http://odin.ingrid.org/)",
"Jabot/7.x.x (http://odin.ingrid.org/)",
"Jack",
"Jakarta Commons-HttpClient/2.0xxx",
"Jakarta Commons-HttpClient/3.0-rcx",
"Jambot/0.1.x (Jambot; http://www.jambot.com/blog; crawler@jambot.com)",
"Jambot/0.2.1 (Jambot; http://www.jambot.com/blog/static.php?page=webmaster-robot; crawler@jambot.com)",
"Java 1.1",
"Java/1.4.1_01",
"Java1.0.21.0",
"Java1.1.xx.x",
"Java1.3.0rc1",
"Java1.3.x",
"Java1.4.0",
"Jayde Crawler. http://www.jayde.com",
"JBH Agent 2.0",
"jBrowser/J2ME Profile/MIDP-1.0 Configuration/CLDC-1.0 (Google WAP Proxy/1.0)",
"JCheckLinks/0.1 RPT-HTTPClient/0.3-1",
"JDK/1.1",
"Jeode/1.x.x",
"Jetbot/1.0",
"JetBrains Omea Reader 1.0.x (http://www.jetbrains.com/omea_reader/)",
"JetBrains Omea Reader 2.0 Release Candidate 1 (http://www.jetbrains.com/omea_reader/)",
"JetCar",
"Jigsaw/2.2.0 W3C_CSS_Validator_JFouffa/2.0",
"JoBo/1.x (http://www.matuschek.net/jobo.html)",
"JoBo/@JOBO_VERSION@(http://www.matuschek.net/jobo.html)",
"JobSpider_BA/1.1",
"JOC Web Spider",
"JordoMedia/1.0 RSS File Reader (http://www.jordomedia.com)",
"Journster [alpha] (http://journster.com/)",
"Journster.com RSS/Atom aggregator 0.5 (http://www.journster.com/bot.phtml)",
"JRTS Check Favorites Utility",
"JRTwine Software Check Favorites Utility",
"Jyxobot/x",
"K-Meleon/0.6 (Windows; U; Windows NT 5.1; en-US; rv:0.9.5) Gecko/20011011",
"k2spider",
"KAIST AITrc Crawler",
"KakleBot - www.kakle.com/0.1 (KakleBot - www.kakle.com; http:// www.kakle.com/bot.html; support@kakle.com)",
"kalooga/kalooga-4.0-dev-datahouse (Kalooga; http://www.kalooga.com; info@kalooga.com)",
"kalooga/KaloogaBot (Kalooga; http://www.kalooga.com/info.html?page=crawler; crawler@kalooga.com)",
"Kapere (http://www.kapere.com)",
"Kazehakase/0.x.x.[x]",
"KDDI-SN22 UP.Browser/6.0.7 (GUI) MMP/1.1 (Google WAP Proxy/1.0)",
"Kenjin Spider",
"Kevin http://dznet.com/kevin/",
"Kevin http://websitealert.net/kevin/",
"KE_1.0/2.0 libwww/5.2.8",
"KFSW-Bot (Version: 1.01 powered by KFSW www.kfsw.de)",
"kinja-imagebot (http://www.kinja.com/)",
"kinjabot (http://www.kinja.com)",
"KIT-Fireball/2.0",
"KIT-Fireball/2.0 (compatible; Mozilla 4.0; MSIE 5.5)",
"Klondike/1.50 (WSP Win32) (Google WAP Proxy/1.0)",
"KnowItAll(knowitall@cs.washington.edu)",
"Knowledge.com/0.x",
"Kontiki Client x.xx",
"KSbot/1.0 (KnowledgeStorm crawler; http://www.knowledgestorm.com/resources/content/crawler/index.html; crawleradmin@knowledgestorm.com)",
"kuloko-bot/0.x",
"kulokobot www.kuloko.com kuloko@backweave.com",
"kulturarw3/0.1",
"KummHttp/1.1 (compatible; KummClient; Linux rulez)",
"Lachesis",
"lanshanbot/1.0",
"lanshanbot/1.0 (+http://search.msn.com/msnbot.htm)",
"LapozzBot/1.4 ( http://robot.lapozz.com)",
"LapozzBot/1.5 (+http://robot.lapozz.hu)",
"larbin (samualt9@bigfoot.com)",
"LARBIN-EXPERIMENTAL (efp@gmx.net)",
"larbin_2.1.1 larbin2.1.1@somewhere.com",
"larbin_2.2.0 (crawl@compete.com)",
"larbin_2.2.1_de_Viennot (Laurent.Viennot@inria.fr)",
"larbin_2.2.2 (sugayama@lab7.kuis.kyoto-u.ac.jp)",
"larbin_2.2.2_guillaume (guillaume@liafa.jussieu.fr)",
"larbin_2.6.0 (larbin2.6.0@unspecified.mail)",
"larbin_2.6.1 (larbin2.6.1@unspecified.mail)",
"larbin_2.6.2 (hamasaki@grad.nii.ac.jp)",
"larbin_2.6.2 (larbin2.6.2@unspecified.mail)",
"larbin_2.6.2 (listonATccDOTgatechDOTedu)",
"larbin_2.6.2 (pimenas@systems.tuc.gr)",
"larbin_2.6.2 (tom@lemurconsulting.com)",
"larbin_2.6.2 (vitalbox1@hotmail.com)",
"larbin_2.6.3 (ltaa_web_crawler@groupes.epfl.ch)",
"larbin_2.6.3 (wgao@genieknows.com)",
"larbin_2.6.3_for_(http://cosco.hiit.fi/search/) tsilande@hiit.fi",
"larbin_2.6_basileocaml (basile.starynkevitch@cea.fr)",
"larbin_devel (http://pauillac.inria.fr/~ailleret/prog/larbin/)",
"lawinfo-crawler/Nutch-0.9-dev (Crawler for lawinfo.com pages; http://www.lawinfo.com; webmaster@lawinfo.com)",
"lc/$ROADS::Version libwww-perl/5.00",
"lcabotAccept: */*",
"LeapTag/0.8.1.beta081.r3750 (compatible; Mozilla 4.0; MSIE 5.5; robot@yoriwa.com)",
"LECodeChecker/3.0 libgetdoc/1.0",
"LeechGet 200x (www.leechget.de)",
"LEIA/2.90",
"LEIA/3.01pr (LEIAcrawler; [SNIP])",
"LetsCrawl.com/1.0 +http://letscrawl.com/",
"LexiBot/1.00",
"LG/U8138/v1.0",
"Libby_1.1/libwww-perl/5.47",
"libcurl-agent/1.0",
"LibertyW (+http://www.lw01.com)",
"libWeb/clsHTTP -- hiongun@kt.co.kr",
"libwww-perl/5.41",
"libwww-perl/5.45",
"libwww-perl/5.48",
"libwww-perl/5.50",
"libwww-perl/5.52 FP/2.1",
"libwww-perl/5.52 FP/4.0",
"libwww-perl/5.53",
"libwww-perl/5.63",
"libwww-perl/5.64",
"libwww-perl/5.65",
"libwww-perl/5.800",
"libwww/5.3.2",
"Liferea/0.x.x (Linux; en_US.UTF-8; http://liferea.sf.net/)",
"Liferea/1.x.x (Linux; es_ES.UTF-8; http://liferea.sf.net/)",
"LightningDownload/1.0beta2",
"LightningDownload/1.x.x",
"LightningDownload/1.x.x [Accelerated x]",
"LijitSpider/Nutch-0.9 (Reports crawler; http://www.lijit.com/; info(a)lijit(d)com)",
"Lincoln State Web Browser",
"Link Valet Online 1.x",
"LinkAlarm/2.x",
"linkbot",
"Linkbot",
"Linkbot x.0",
"LinkCheck (linkcheck@inter7.com http://www.inter7.com/linkcheck)",
"LinkLint-checkonly/2.x.x",
"LinkLint-spider/2.x.x",
"linknzbot",
"LinkPimpin v1.0",
"LinkProver 2.1",
"Links (0.9x; Linux 2.4.7-10 i686)",
"Links (0.9xpre12; Linux 2.2.14-5.0 i686; 80x24)",
"Links (2.xpre7; Linux 2.4.18 i586; x)",
"Links - http://gossamer-threads.com/scripts/links/",
"Links 2.0 (http://gossamer-threads.com/scripts/links/)",
"Links SQL (http://gossamer-threads.com/scripts/links-sql/)",
"LinkScan/11.0beta2 UnixShareware robot from Elsop.com (used by Indiafocus/Indiainfo)",
"LinkScan/9.0g Unix",
"LinkScan/x.x Unix",
"LinksManager.com (http://linksmanager.com/linkchecker.html)",
"LinkSonar/1.35",
"LinkSweeper/1.x",
"LinkWalker",
"link_check3.plx libwww-perl/5.65",
"ListBidBot (freelance job spider http://listbid.com)<a href=http://listbid.com>Freelance</a>",
"LiveTrans/Nutch-0.9 (maintainer: cobain at iis dot sinica dot edu dot tw; http://wkd.iis.sinica.edu.tw/LiveTrans/)",
"Llaut/1.0 (http://mnm.uib.es/~gallir/llaut/bot.html)",
"LMQueueBot/0.2",
"lmspider (lmspider@scansoft.com)",
"LNSpiderguy",
"LocalBot/1.0 ( http://www.localbot.co.uk/)",
"LocalcomBot/1.2.x ( http://www.local.com/bot.htm)",
"Lockstep Spider/1.0",
"Look.com",
"Lotus-Notes/4.5 ( Windows-NT )",
"LotusDiscovery/x.0 (compatible; Mozilla 4.0; MSIE 4.01; Windows NT)",
"Lovel as 1.0 ( +http://www.everatom.com)",
"LTI/LemurProject Nutch Spider/Nutch-1.0-dev (lti crawler for CMU; http://www.lti.cs.cmu.edu; changkuk at cmu dot edu)",
"LTI/LemurProject Nutch Spider/Nutch-1.0-dev (Research spider using Nutch; http://www.lemurproject.org; mhoy@cs.cmu.edu)",
"luchs.at URL checker",
"Lunascape",
"lwp-trivial/1.32",
"lwp-trivial/1.34",
"lwp-trivial/1.34",
"lwp-trivial/1.35",
"lwp-trivial/1.35",
"LWP::Simple/5.22",
"LWP::Simple/5.36",
"LWP::Simple/5.48",
"LWP::Simple/5.50",
"LWP::Simple/5.51",
"LWP::Simple/5.53",
"LWP::Simple/5.63",
"LWP::Simple/5.803",
"Lycos_Spider_(modspider)",
"Lycos_Spider_(T-Rex)",
"Lynx/2-4-2 (Bobcat/0.5 [DOS] Jp Beta04)",
"Lynx/2.6 libwww-FM/2.14",
"Lynx/2.8 (;http://seebot.org)",
"Lynx/2.8.3dev.9 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6",
"Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6c (human-guided@lerly.net)",
"Mac Finder 1.0.xx",
"Mackster( http://www.ukwizz.com )",
"Mag-Net",
"MagicWML/1.0 (forcewml)",
"MagpieRSS/0.7x (+http://magpierss.sf.net)",
"Mahiti.Com/Mahiti Crawler-1.0 (Mahiti.Com; http://mahiti.com ; mahiti.com)",
"Mail.Ru/1.0",
"mailto:webcraft@bea.com",
"*******/1.0 ( http://www.sli-systems.com/)",
"MantraAgent",
"MapoftheInternet.com ( http://MapoftheInternet.com)",
"Mariner/5.1b [de] (Win95; I ;Kolibri gncwebbot)",
"Marketwave Hit List",
"MARTINI",
"Martini",
"Marvin v0.3",
"MaSagool/1.0 (MaSagool; http://sagool.jp/; info@sagool.jp)",
"Mass Downloader 2.x",
"MasterSeek",
"Mata Hari/2.00 ",
"Matrix S.p.A. - FAST Enterprise Crawler 6 (Unknown admin e-mail address)",
"maxomobot/dev-20051201 (maxomo; http://67.102.134.34:4047/MAXOMO/MAXOMObot.html; maxomobot@maxomo.com)",
"McBot/5.001 (windows; U; NT4.0; en-us)",
"MDbot/1.0 (+http://www.megadownload.net/bot.html)",
"Media Player Classic",
"MediaCrawler-1.0 (Experimental)",
"Mediapartners-Google/2.1 ( http://www.googlebot.com/bot.html)",
"MediaSearch/0.1",
"MegaSheep v1.0 (www.searchuk.com internet sheep)",
"Megite2.0 (http://www.megite.com)",
"Mercator-1.x",
"Mercator-2.0",
"Mercator-Scrub-1.1",
"Metaeuro Web Crawler/0.2 (MetaEuro Web Search Clustering Engine; http://www.metaeuro.com; crawler at metaeuro dot com)",
"MetaGer-LinkChecker",
"MetagerBot/0.8-dev (MetagerBot; http://metager.de; )",
"MetaGer_PreChecker0.1",
"MetaProducts Download Express/1.x",
"Metaspinner/0.01 (Metaspinner; http://www.meta-spinner.de/; support@meta-spinner.de/)",
"metatagsdir/0.7 (+http://metatagsdir.com/directory/)",
"MFC Foundation Class Library 4.0",
"MFC_Tear_Sample",
"MFHttpScan",
"MicroBaz",
"Microsoft Data Access Internet Publishing Provider Cache Manager",
"Microsoft Data Access Internet Publishing Provider DAV",
"Microsoft Data Access Internet Publishing Provider Protocol Discovery",
"Microsoft Data Access Internet Publishing Provider Protocol Discovery",
"Microsoft Log Parser 2.2",
"Microsoft Small Business Indexer",
"Microsoft URL Control - 6.00.8xxx",
"MicrosoftPrototypeCrawler (How's my crawling? mailto:newbiecrawler@hotmail.com)",
"Missauga Locate 1.0.0",
"Missigua Locator 1.9",
"Missouri College Browse",
"Mister Pix II 2.02a",
"Mister PiX version.dll",
"Misterbot-Nutch/0.7.1 (Misterbot-Nutch; http://www.misterbot.fr; admin@misterbot.fr)",
"Miva (AlgoFeedback@miva.com)",
"Mizzu Labs 2.2",
"MJ12bot/vx.x.x (http://majestic12.co.uk/bot.php?+)",
"MJ12bot/vx.x.x (http://www.majestic12.co.uk/projects/dsearch/mj12bot.php)",
"MJBot (SEO assessment)",
"MLBot (www.metadatalabs.com)",
"MnogoSearch/3.2.xx",
"Mo College 1.9",
"moget/x.x (moget@goo.ne.jp)",
"mogimogi/1.0",
"moiNAG 0.02",
"MojeekBot/0.x (archi; http://www.mojeek.com/bot.html)",
"Morris - Mixcat Crawler ( http://mixcat.com)",
"Mouse-House/7.4 (spider_monkey spider info at www.mobrien.com/sm.shtml)",
"MovableType/x.x",
"mozDex/0.xx-dev (mozDex; http://www.mozdex.com/en/bot.html; spider@mozdex.com)",
"Mozilla (Mozilla@somewhere.com)",
"Mozilla 4.0(compatible; BotSeer/1.0; +http://botseer.ist.psu.edu)",
"Mozilla/2.0 (compatible; Ask Jeeves)",
"Mozilla/2.0 (compatible; Ask Jeeves/Teoma)",
"Mozilla/2.0 (compatible; Ask Jeeves/Teoma; http://about.ask.com/en/docs/about/webmasters.shtml) ",
"Mozilla/2.0 (compatible; Ask Jeeves/Teoma; http://sp.ask.com/docs/about/tech_crawling.html)",
"Mozilla/2.0 (compatible; EZResult -- Internet Search Engine)",
"Mozilla/2.0 (compatible; NEWT ActiveX; Win32)",
"Mozilla/2.0 (compatible; T-H-U-N-D-E-R-S-T-O-N-E)",
"Mozilla/3.0 (compatible)",
"Mozilla/3.0 (compatible; Fluffy the spider; http://www.searchhippo.com/; info@searchhippo.com)",
"Mozilla/3.0 (compatible; Indy Library)",
"Mozilla/3.0 (compatible; MuscatFerret/1.5.4; claude@euroferret.com)",
"Mozilla/3.0 (compatible; MuscatFerret/1.5; olly@muscat.co.uk)",
"Mozilla/3.0 (compatible; MuscatFerret/1.6.x; claude@euroferret.com)",
"Mozilla/3.0 (compatible; netart generator/1.0; libwww-perl/5.64)",
"Mozilla/3.0 (compatible; scan4mail (advanced version) http://www.peterspages.net/?scan4mail)",
"Mozilla/3.0 (compatible; ScollSpider; http://www.webwobot.com)",
"Mozilla/3.0 (compatible; Webinator-DEV01.home.iprospect.com/2.56)",
"Mozilla/3.0 (compatible; Webinator-indexer.cyberalert.com/2.56)",
"Mozilla/3.0 (INGRID/3.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)",
"Mozilla/3.0 (Slurp.so/Goo; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Mozilla/3.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Mozilla/3.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Mozilla/3.0 (Vagabondo/1.1 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)",
"Mozilla/3.0 (Vagabondo/1.x MT; webagent@wise-guys.nl; http://webagent.wise-guys.nl/)",
"Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)",
"Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)",
"Mozilla/3.01 (Compatible; Links2Go Similarity Engine)",
"Mozilla/4.0",
"Mozilla/4.0 (agadine3.0) www.agada.de",
"Mozilla/4.0 (compatible: AstraSpider V.2.1 : astrafind.com)",
"Mozilla/4.0 (compatible; Vagabondo/2.2; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)",
"Mozilla/4.0 (compatible; Vagabondo/4.0Beta; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)",
"Mozilla/4.0 (compatible; Advanced Email ********* v2.xx)",
"Mozilla/4.0 (compatible; B_L_I_T_Z_B_O_T)",
"Mozilla/4.0 (compatible; ChristCrawler.com ChristCrawler@ChristCENTRAL.com)",
"Mozilla/4.0 (compatible; DAUMOA-video; +http://ws.daum.net/aboutkr.html)",
"Mozilla/4.0 (compatible; FastCrawler3 support-fastcrawler3@fast.no)",
"Mozilla/4.0 (compatible; FDSE robot)",
"Mozilla/4.0 (compatible; GPU *** crawler http://gpu.sourceforge.net/search_engine.php)",
"Mozilla/4.0 (compatible; grub-client-0.2.x; Crawl your stuff with http://grub.org)",
"Mozilla/4.0 (compatible; grub-client-0.3.x; Crawl your own stuff with http://grub.org)",
"Mozilla/4.0 (compatible; grub-client-2.x)",
"Mozilla/4.0 (compatible; Iplexx Spider/1.0 http://www.iplexx.at)",
"Mozilla/4.0 (compatible; MSIE 4.01; Vonna.com b o t)",
"Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) ACR",
"Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) Indonesia Interactive",
"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0) (samualt9@bigfoot.com)",
"Mozilla/4.0 (compatible; MSIE 5.0; NetNose-Crawler 2.0; A New Search Experience: http://www.netnose.com)",
"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) TrueRobot; 1.5",
"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot BETA 1.2 (http://www.voila.com/)",
"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot; 1.6",
"Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent",
"Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; http://www.girafa.com)",
"Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com)",
"Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com/; http://www.galaxy.com/info/crawler.html)",
"Mozilla/4.0 (compatible; MSIE 5.0; YANDEX)",
"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; obot)",
"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; QXW03018)",
"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Fetch API Request",
"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AspTear 1.5)",
"Mozilla/4.0 (compatible; MSIE 6.0 compatible; Asterias Crawler v4; +http://www.singingfish.com/help/spider.html; webmaster@singingfish.com); SpiderThread Revision: 3.10",
"Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Skampy/0.9.x [en]",
"Mozilla/4.0 (compatible; MSIE 6.0; TargetSeek/1.0; +http://www.targetgroups.net/TargetSeek.html)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ODP entries t_st; http://tuezilla.de/t_st-odp-entries-agent.html)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ODP links test; http://tuezilla.de/test-odp-links-agent.html)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ZoomSpider.net bot; .NET CLR 1.1.4322)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; heritrix/1.3.0 http://www.cs.washington.edu/research/networking/websys/)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MathPlayer2.0)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; QihooBot 1.0 qihoobot@qihoo.net)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)",
"Mozilla/4.0 (compatible; NaverBot/1.0; http://help.naver.com/delete_main.asp)",
"Mozilla/4.0 (compatible; Netcraft Web Server Survey)",
"Mozilla/4.0 (compatible; SpeedySpider; www.entireweb.com)",
"Mozilla/4.0 (compatible; SPENG)",
"Mozilla/4.0 (compatible; www.euro-directory.com; urlchecker1.0)",
"Mozilla/4.0 (compatible; www.galaxy.com)",
"Mozilla/4.0 (compatible; Y!J; for robot study; keyoshid)",
"Mozilla/4.0 (compatible; Yahoo Japan; for robot study; kasugiya)",
"Faked IE id string used by DeepTrawl link checking tool",
"Mozilla/4.0 (fantomBrowser)",
"Mozilla/4.0 (fantomCrew Browser)",
"Mozilla/4.0 (hhjhj@yahoo.com)",
"Mozilla/4.0 (JemmaTheTourist;http://www.activtourist.com)",
"Mozilla/4.0 (MobilePhone PM-8200/US/1.0) NetFront/3.x MMP/2.0",
"Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)",
"Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 FAKE (compatible; Googlebot/2.1; http://www.google.com/bot.html)",
"Mozilla/4.0 (Mozilla; http://www.mozilla.org/docs/en/bot.html; master@mozilla.com)",
"Mozilla/4.0 (Sleek Spider/1.2)",
"Mozilla/4.0 compatible FurlBot/Furl Search 2.0 (FurlBot; http://www.furl.net; wn.furlbot@looksmart.net)",
"Mozilla/4.0 compatible ZyBorg/1.0 (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)",
"Mozilla/4.0 compatible ZyBorg/1.0 (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)",
"Mozilla/4.0 compatible ZyBorg/1.0 Dead Link Checker (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)",
"Mozilla/4.0 compatible ZyBorg/1.0 for Homepage (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)",
"Mozilla/4.0 efp@gmx.net",
"Mozilla/4.0 WebTV/2.6 (compatible; MSIE 4.0)",
"Mozilla/4.0 [en] (Ask Jeeves Corporate Spider)",
"Mozilla/4.0(compatible; Zealbot 1.0)",
"Mozilla/4.01 (compatible; NORAD National Defence Network)",
"Mozilla/4.01 [en](Win95;I)",
"Mozilla/4.02 [en] (X11; I; SunOS 5.6 sun4u)",
"Mozilla/4.04 (compatible; Dulance bot; +http://www.dulance.com/bot.jsp)",
"Mozilla/4.04 [en] (X11; I; HP-UX B.10.20 9000/712)",
"Mozilla/4.04 [en] (X11; I; IRIX 5.3 IP22)",
"Mozilla/4.05 (Macintosh; I; 68K Nav)",
"Mozilla/4.05 (Macintosh; I; PPC Nav)",
"Mozilla/4.05 [en] (X11; I; SunOS 4.1.4 sun4m)",
"Mozilla/4.08 [en] (Win98; U ;Nav)",
"Mozilla/4.08 [en] (WinNT; U)",
"Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_TrueRobot/1.4 libwww/5.2.8",
"Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_VoilaBot/1.6 libwww/5.3.2",
"Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)",
"Mozilla/4.5 (compatible; iCab 2.5.3; Macintosh; I; PPC)",
"Mozilla/4.5 (compatible; OmniWeb/4.0.5; Mac_PowerPC)",
"Mozilla/4.5 (compatible; OmniWeb/4.1-beta-1; Mac_PowerPC)",
"Mozilla/4.5 RPT-HTTPClient/0.3-2",
"Mozilla/4.5 [en]C-CCK-MCD {RuralNet} (Win98; I)",
"Mozilla/4.5b1 [en] (X11; I; Linux 2.0.35 i586)",
"Mozilla/4.6 [en] (http://www.cnet.com/)",
"Mozilla/4.61 [de] (OS/2; I)",
"Mozilla/4.61 [en] (X11; U; ) - BrowseX (2.0.0 Windows)",
"Mozilla/4.7",
"Mozilla/4.7 (compatible; http://eidetica.com/spider)",
"Mozilla/4.7 (compatible; Intelliseek; http://www.intelliseek.com)",
"Mozilla/4.7 (compatible; OffByOne; Windows 98) Webster Pro V3.2",
"Mozilla/4.7 (compatible; Whizbang)",
"Mozilla/4.7 (compatible; WhizBang; http://www.whizbang.com/crawler)",
"Mozilla/4.7 [en](BecomeBot@exava.com)",
"Mozilla/4.7 [en](Exabot@exava.com)",
"Mozilla/4.7 [en]C-CCK-MCD {Yahoo;YIP052400} (Win95; I)",
"Mozilla/4.72 [en] (BACS http://www.ba.be)",
"Mozilla/4.72C-CCK-MCD Caldera Systems OpenLinux [en] (X11; U; Linux 2.2.14 i686)",
"Mozilla/4.75C-ja [ja] (X11; U; OSF1 V5.1 alpha)",
"Mozilla/4.76 (Windows 98; U) Opera 5.12 [en]",
"Mozilla/4.76 [en] (X11; U; FreeBSD 4.4-STABLE i386)",
"Mozilla/4.76 [en] (X11; U; SunOS 5.7 sun4u)",
"Mozilla/4.77C-SGI [en] (X11; U; IRIX 6.5 IP32)",
"Mozilla/5.0",
"Mozilla/5.0 (+http://www.eurekster.com/*******) *******/0.1",
"Mozilla/5.0 (+http://www.sli-systems.com/) *******/0.1",
"Mozilla/5.0 (Clustered-Search-Bot/1.0; support@clush.com; http://www.clush.com/)",
"Mozilla/5.0 (compatible) GM RSS Panel X",
"Mozilla/5.0 (compatible; AnsearchBot/1.x; +http://www.ansearch.com.au/)",
"Mozilla/5.0 (compatible; archive.org_bot/1.10.0 +http://www.loc.gov/minerva/crawl.html)",
"Mozilla/5.0 (compatible; archive.org_bot/1.13.1x http://crawler.archive.org)",
"Mozilla/5.0 (compatible; archive.org_bot/1.5.0-200506132127 http://crawler.archive.org) Hurricane Katrina",
"Mozilla/5.0 (compatible; Ask Jeeves/Teoma; http://about.ask.com/en/docs/about/webmasters.shtml)",
"Mozilla/5.0 (compatible; BanBots/2.0b; Fetch; +http://www.banbots.com)",
"Mozilla/5.0 (compatible; BecomeBot/1.23; http://www.become.com/webmasters.html)",
"Mozilla/5.0 (compatible; BecomeBot/1.xx; MSIE 6.0 compatible; http://www.become.com/webmasters.html)",
"Mozilla/5.0 (compatible; BecomeBot/2.0beta; http://www.become.com/webmasters.html)",
"Mozilla/5.0 (compatible; BecomeBot/2.x; MSIE 6.0 compatible; http://www.become.com/site_owners.html)",
"Mozilla/5.0 (compatible; BecomeJPBot/2.3; MSIE 6.0 compatible; +http://www.become.co.jp/site_owners.html)",
"Mozilla/5.0 (compatible; BlogRefsBot/0.1; http://www.blogrefs.com/about/bloggers)",
"Mozilla/5.0 (compatible; Bot; +http://pressemitteilung.ws/spamfilter",
"Mozilla/5.0 (compatible; BuzzRankingBot/1.0; +http://www.buzzrankingbot.com/)",
"Mozilla/5.0 (compatible; Charlotte/1.0b; charlotte@betaspider.com)",
"Mozilla/5.0 (compatible; Charlotte/1.0b; http://www.searchme.com/support/)",
"Mozilla/5.0 (compatible; Crawling jpeg; http://www.yama.info.waseda.ac.jp)",
"Mozilla/5.0 (compatible; Custo 3 (Netwu.com); Windows NT 5.1)",
"Mozilla/5.0 (compatible; de/1.13.2 +http://www.de.com)",
"Mozilla/5.0 (compatible; DNS-Digger-Explorer/1.0; +http://www.dnsdigger.com)",
"Mozilla/5.0 (compatible; DNS-Digger/1.0; +http://www.dnsdigger.com)",
"Mozilla/5.0 (compatible; EARTHCOM.info/2.01; http://www.earthcom.info)",
"Mozilla/5.0 (compatible; EARTHCOM/2.2; +http://enter4u.eu)",
"Mozilla/5.0 (compatible; egothor/8.0g; +http://ego.ms.mff.cuni.cz/)",
"Mozilla/5.0 (compatible; Exabot Test/3.0; +http://www.exabot.com/go/robot)",
"Mozilla/5.0 (compatible; FatBot 2.0; http://www.thefind.com/main/CrawlerFAQs.fhtml)",
"mozilla/5.0 (compatible; genevabot http://www.healthdash.com)",
"Mozilla/5.0 (compatible; Google Desktop) Paros/3.2.12",
"Mozilla/5.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)",
"mozilla/5.0 (compatible; heritrix/1.0.4 http://innovationblog.com)",
"Mozilla/5.0 (compatible; heritrix/1.10.2 +http://i.stanford.edu/)",
"Mozilla/5.0 (compatible; heritrix/1.12.1 +http://newstin.com/)",
"Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.page-store.com)",
"Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.page-store.com) [email:paul@page-store.com]",
"mozilla/5.0 (compatible; heritrix/1.3.0 http://archive.crawler.org)",
"Mozilla/5.0 (compatible; heritrix/1.4.0 +http://www.chepi.net)",
"Mozilla/5.0 (compatible; heritrix/1.4t http://www.truveo.com/)",
"Mozilla/5.0 (compatible; heritrix/1.5.0 http://www.l3s.de/~kohlschuetter/projects/crawling/)",
"Mozilla/5.0 (compatible; heritrix/1.5.0-200506231921 http://pandora.nla.gov.au/crawl.html)",
"Mozilla/5.0 (compatible; heritrix/1.6.0 http://www.worio.com/)",
"Mozilla/5.0 (compatible; heritrix/1.7.0 +http://www.greaterera.com/)",
"Mozilla/5.0 (compatible; Heritrix/1.8.0 http://www.hanzoarchives.com)",
"Mozilla/5.0 (compatible; heritrix/1.x.x +http://www.accelobot.com)",
"Mozilla/5.0 (compatible; heritrix/2.0.0-RC1 +http://www.aol.com)",
"Mozilla/5.0 (compatible; Hermit Search. Com; +http://www.hermitsearch.com)",
"Mozilla/5.0 (compatible; http://www.IsMySiteUp.Net/bot/ )",
"Mozilla/5.0 (compatible; http://www.UptimeAuditor.com/bot/ )",
"Mozilla/5.0 (compatible; HyperixScoop/1.3; +http://www.hyperix.com)",
"Mozilla/5.0 (compatible; iaskspider/1.0; MSIE 6.0)",
"Mozilla/5.0 (compatible; IDBot/1.0; +http://www.id-search.org/bot.html)",
"Mozilla/5.0 (compatible; InterseekWeb/3.x)",
"Mozilla/5.0 (compatible; Jim +http://www.hanzoarchives.com)",
"Mozilla/5.0 (compatible; Konqueror/2.0.1; X11); Supports MD5-Digest; Supports gzip encoding",
"Mozilla/5.0 (compatible; Konqueror/2.1.1; X11)",
"Mozilla/5.0 (compatible; Konqueror/2.2.2)",
"Mozilla/5.0 (compatible; Konqueror/2.2.2; Linux 2.4.14-xfs; X11; i686)",
"Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)",
"Mozilla/5.0 (compatible; LemSpider 0.1)",
"Mozilla/5.0 (compatible; LinksManager.com_bot http://linksmanager.com/linkchecker.html)",
"Mozilla/5.0 (compatible; LinkStash Bookmark Manager; http://www.xrayz.co.uk/)",
"Mozilla/5.0 (compatible; MojeekBot/2.0; http://www.mojeek.com/bot.html)",
"Mozilla/5.0 (compatible; MOSBookmarks/v2.6-Plus; Link Checker)",
"Mozilla/5.0 (compatible; MSIE 6.0; Podtech Network; crawler_admin@podtech.net)",
"Mozilla/5.0 (compatible; OnetSzukaj/5.0; http://szukaj.onet.pl)",
"Mozilla/5.0 (compatible; PagestackerBot; http://www.pagestacker.com)",
"Mozilla/5.0 (compatible; PalmeraBot; http://www.links24h.com/help/palmera) Version 0.001",
"Mozilla/5.0 (compatible; PEAR HTTP_Request class; http://feed.moo.jp/)",
"Mozilla/5.0 (compatible; Phonifier; +http://www.phonifier.com)",
"Mozilla/5.0 (compatible; pmoz.info ODP link checker; +http://pmoz.info/doc/botinfo.htm)",
"Mozilla/5.0 (compatible; pogodak.ba/3.x)",
"Mozilla/5.0 (compatible; Pogodak.hr/3.1)",
"Mozilla/5.0 (compatible; Proximic crawler; +http://www.proximic.com/en/about-us/contact-us.html)",
"Mozilla/5.0 (compatible; PWeBot/3.1; http://www.programacionweb.net/robot.php)",
"Mozilla/5.0 (compatible; Quantcastbot/1.0; www.quantcast.com)",
"Mozilla/5.0 (compatible; robtexbot/1.0; http://www.robtex.com/ )",
"Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)",
"Mozilla/5.0 (compatible; Scrubby/2.2; http://www.scrubtheweb.com/)",
"Mozilla/5.0 (compatible; ShunixBot/1.x.x +http://www.shunix.com/robot.htm)",
"Mozilla/5.0 (compatible; ShunixBot/1.x; http://www.shunix.com/bot.htm)",
"Mozilla/5.0 (compatible; SkreemRBot +http://skreemr.com)",
"Mozilla/5.0 (compatible; SnapPreviewBot; en-US; rv:1.8.0.9) Gecko/20061206 Firefox/1.5.0.9",
"Mozilla/5.0 (compatible; SpurlBot/0.2) ",
"Mozilla/5.0 (compatible; SummizeBot +http://www.summize.com)",
"Mozilla/5.0 (compatible; SYCLIKControl/LinkChecker;)",
"Mozilla/5.0 (compatible; Synoobot/0.9; http://www.synoo.com/search/bot.html)",
"Mozilla/5.0 (compatible; Theophrastus/x.x; http://users.cs.cf.ac.uk/N.A.Smith/theophrastus.php)",
"Mozilla/5.0 (compatible; TridentSpider/3.1)",
"Mozilla/5.0 (compatible; Vagabondo/2.1; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)",
"Mozilla/5.0 (compatible; Windows NT 5.0; phpwebbrainBot/0.1 - http://www.monsterli.ch/phpwebbrain/)",
"Mozilla/5.0 (compatible; worio bot heritrix/1.10.0 +http://worio.com)",
"Mozilla/5.0 (compatible; WoW Lemmings Kathune/2.0;http://www.wowlemmings.com/kathune.html)",
"Mozilla/5.0 (compatible; XTbot/1.0v; +http://www.externaltest.com)",
"Mozilla/5.0 (compatible; Yahoo! DE Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
"Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)",
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
"Mozilla/5.0 (compatible; Yoono; http://www.yoono.com/)",
"Mozilla/5.0 (compatible; Zenbot/1.3; +http://zen.co.za/webmasters/)",
"Mozilla/5.0 (compatible;archive.org_bot/1.7.1; collectionId=316; Archive-It; +http://www.archive-it.org)",
"Mozilla/5.0 (compatible;archive.org_bot/heritrix-1.9.0-200608171144 +http://pandora.nla.gov.au/crawl.html)",
"Mozilla/5.0 (compatible;FindITAnswersbot/1.0;+http://search.it-influentials.com/bot.htm)",
"Mozilla/5.0 (compatible;MAINSEEK_BOT)",
"Mozilla/5.0 (Gecko/20070310 Mozshot/0.0.20070628; http://mozshot.nemui.org/)",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9b5) Gecko/2008032619 Firefox/3.0b5",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20021219 Chimera/0.6 ",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20030306 Camino/0.7",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/xx (KHTML like Gecko) OmniWeb/v5xx.xx",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/xxx.x (KHTML like Gecko) Safari/12x.x",
"Mozilla/5.0 (Macintosh; U; PPC; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1",
"Mozilla/5.0 (research@mediatrec.com)",
"Mozilla/5.0 (Sage)",
"Mozilla/5.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Mozilla/5.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Mozilla/5.0 (SunOS 5.8 sun4u; U) Opera 5.0 [en]",
"Mozilla/5.0 (Twiceler-0.9 http://www.cuill.com/twiceler/robot.html)",
"Mozilla/5.0 (Version: xxxx Type:xx)",
"Mozilla/5.0 (wgao@genieknows.com)",
"Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1",
"Mozilla/5.0 (Windows; U; Win98; en-US; rv:x.xx) Gecko/20030423 Firebird Browser/0.6",
"Mozilla/5.0 (Windows; U; Win9x; en; Stable) Gecko/20020911 Beonex/0.8.1-stable",
"Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.7) NimbleCrawler 1.11 obeys UserAgent NimbleCrawler For problems contact: crawler_at_dataalchemy.com",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.5) Gecko/20060731 Firefox/1.5.0.5 Flock/0.7.4.1",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.4/Megaupload x.0",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.1) Gecko/2008092215 Firefox/3.0.1 Orca/1.1 beta 3",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.x.x) Gecko/20041107 Firefox/x.x",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xx) Gecko/20030504 Mozilla Firebird/0.6",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xxx) Gecko/20041027 Mnenhy/0.6.0.104",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",
"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5",
"Mozilla/5.0 (Windows; U;XMPP Tiscali Communicator v.10.0.1; Windows NT 5.1; it; rv:1.8.1.3) Gecko/20070309 Firefox/2.0.0.3",
"Mozilla/5.0 (Windows;) NimbleCrawler 1.12 obeys UserAgent NimbleCrawler For problems contact: crawler@health",
"Mozilla/5.0 (Windows;) NimbleCrawler 1.12 obeys UserAgent NimbleCrawler For problems contact: crawler@healthline.com",
"Mozilla/5.0 (X11; Linux i686; U;rv: 1.7.13) Gecko/20070322 Kazehakase/0.4.4.1",
"Mozilla/5.0 (X11; U; Linux 2.4.2-2 i586; en-US; m18) Gecko/20010131 Netscape6/6.01",
"Mozilla/5.0 (X11; U; Linux i686; de-AT; rv:1.8.0.2) Gecko/20060309 SeaMonkey/1.0",
"Mozilla/5.0 (X11; U; Linux i686; en-GB; rv:1.7.6) Gecko/20050405 Epiphany/1.6.1 (Ubuntu) (Ubuntu package 1.0.2)",
"Mozilla/5.0 (X11; U; Linux i686; en-US; Nautilus/1.0Final) Gecko/20020408",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:0.9.3) Gecko/20010801",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2b) Gecko/20021007 Phoenix/0.3",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040413 Epiphany/1.2.1",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.7) Gecko/20060909 Firefox/1.5.0.7 SnapPreviewBot",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1) Gecko/20061129 BonEcho/2.0",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.1) Gecko/20061205 Iceweasel/2.0.0.1 (Debian-2.0.0.1+dfsg-2)",
"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9a8) Gecko/2007100619 GranParadiso/3.0a8",
"Mozilla/5.0 Galeon/1.0.2 (X11; Linux i686; U;) Gecko/20011224",
"Mozilla/5.0 gURLChecker/0.x.x (Linux)",
"Mozilla/5.0 URL-Spider",
"Mozilla/5.0 usww.com-Spider-for-w8.net",
"Mozilla/5.0 wgao@genieknows.com",
"Mozilla/5.0 whoiam [http://www.axxus.de/]",
"Mozilla/5.0 [en] (compatible; Gulper Web Bot 0.2.4 www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)",
"MQbot metaquerier.cs.uiuc.edu/crawler",
"MQBOT/Nutch-0.9-dev (MQBOT Nutch Crawler; http://falcon.cs.uiuc.edu; mqbot@cs.uiuc.edu)",
"MSFrontPage/4.0",
"MSIE 4.0 (Win95)",
"MSIE-5.13 (larbin@unspecified.mail)",
"msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)",
"msnbot-Products/1.0 (+http://search.msn.com/msnbot.htm)",
"MSNBOT/0.xx (http://search.msn.com/msnbot.htm)",
"msnbot/x.xx ( http://search.msn.com/msnbot.htm)",
"MSNBOT_Mobile MSMOBOT Mozilla/2.0 (compatible; MSIE 4.02; Windows CE; Default)",
"MSNPTC/1.0",
"MSProxy/2.0",
"MSRBOT",
"MSRBOT (http://research.microsoft.com/research/sv/msrbot)",
"multiBlocker browser",
"multicrawler ( http://sw.deri.org/2006/04/multicrawler/robots.html)",
"MultiText/0.1",
"MusicWalker2.0 ( http://www.somusical.com)",
"MVAClient",
"My WinHTTP Connection",
"myDaemon",
"MyGetRight/1.0.0",
"MyGetRight/1.0b",
"Mylinea.com Crawler 2.0",
"mylinkcheck/1.02",
"Naamah 1.0.1/Blogbot (http://blogbot.de/)",
"Naamah 1.0a/Blogbot (http://blogbot.de/)",
"NABOT/5.0",
"nabot_1.0",
"NameOfAgent (CMS Spider)",
"naoFavicon4IE/1.xx",
"NASA Search 1.0",
"NationalDirectory-WebSpider/1.3",
"NationalDirectoryAddURL/1.0",
"NaverBot-1.0 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)",
"NaverBot_dloader/1.5",
"NavissoBot",
"NavissoBot/1.7 (+http://navisso.com/)",
"NCSA Beta 1 (http://vias.ncsa.uiuc.edu/viasarchivinginformation.html)",
"Nebullabot/2.2 (http://bot.nebulla.info)",
"NEC Research Agent -- compuman at research.nj.nec.com",
"NEC-Hayek/1.0",
"Net-Seekr Bot/Net-Seekr Bot V1 (http://www.net-seekr.com)",
"NetAnts/1.2x",
"NETCOMplete/x.xx",
"NetinfoBot/1.0 (http://netinfo.bg/netinfobot.html)",
"NetLookout/2.24",
"Netluchs/0.8-dev ( ; http://www.netluchs.de/; ___don't___spam_me_@netluchs.de)",
"NetMechanic Vx.0",
"NetNewsWire/2.x (Mac OS X; http://ranchero.com/netnewswire/)",
"NetNoseCrawler/v1.0",
"Netprospector JavaCrawler",
"NetPumper/x.xx",
"NetResearchServer(http://www.look.com)",
"NetResearchServer/x.x(loopimprovements.com/robot.html)",
"NetSprint -- 2.0",
"NetWhatCrawler/0.06-dev (NetWhatCrawler from NetWhat.com; http://www.netwhat.com; support@netwhat.com)",
"NetZippy",
"NeuralBot/0.2",
"newsearchengine (ThisUser@unspecified.mail)",
"NewsGator FetchLinks extension/0.2.0 (http://graemef.com)",
"NewsGatorOnline/2.0 (http://www.newsgator.com; 1 subscribers)",
"NextGenSearchBot 1 (for information visit http://www.eliyon.com/NextGenSearchBot)",
"NextopiaBOT (+http://www.nextopia.com) distributed crawler client beta v0.x",
"NG-Search/0.90 (NG-SearchBot; http://www.ng-search.com; )",
"NG/1.0",
"NG/4.0.1229",
"nicebot",
"NICO/1.0",
"Nikita the Spider (http://NikitaTheSpider.com/)",
"NITLE Blog Spider/0.01",
"Nitro Downloader 1.x (www.klsofttools.com)",
"Noago Spider",
"Nocilla/1.0",
"Nokia-WAPToolkit/1.2 googlebot(at)googlebot.com",
"Nokia6300/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1 (botmobi http://find.mobi/bot.html abuse@mtld.mobi)",
"Nokia6610/1.0 (3.09) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible;YahooSeeker/M1A1-R2D2; http://help.yahoo.com/help/us/ysearch/crawling/crawling-01.html)",
"Nokia7110/1.0 (05.01) (Google WAP Proxy/1.0)",
"NokodoBot/1.x (+http://nokodo.com/bot.htm)",
"Norbert the Spider(Burf.com)",
"noxtrumbot/1.0 (crawler@noxtrum.com)",
"noyona_0_1",
"NP/0.1 (NP; http://www.nameprotect.com; npbot@nameprotect.com)",
"NPBot (http://www.nameprotect.com/botinfo.html)",
"NPBot-1/2.0",
"Nsauditor/1.x",
"NSPlayer/10.0.0.xxxx WMFSDK/10.0",
"nsyght.com/Nutch-1.0-dev (nsyght.com; Nsyght.com)",
"nsyght.com/Nutch-x.x (nsyght.com; search.nsyght.com)",
"nttdirectory_robot/0.9 (super-robot@super.navi.ocn.ne.jp)",
"Nucleus SiteList LinkChecker/1.1",
"nuSearch Spider <a href='http://www.nusearch.com'>www.nusearch.com</a> (compatible; MSIE 4.01)",
"NuSearch Spider (compatible; MSIE 6.0)",
"NuSearch Spider www.nusearch.com",
"Nutch",
"Nutch crawler/Nutch-0.9 (picapage.com; admin@picapage.com)",
"Nutch/Nutch-0.9 (Eurobot; http://www.ayell.eu )",
"NutchCVS/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)",
"NutchCVS/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)",
"NutchCVS/0.7.1 (Nutch running at UW; http://www.nutch.org/docs/en/bot.html; sycrawl@cs.washington.edu)",
"NutchEC2Test/Nutch-0.9-dev (Testing Nutch on Amazon EC2.; http://lucene.apache.org/nutch/bot.html; ec2test at lucene.com)",
"NutchOrg/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)",
"nutchsearch/Nutch-0.9 (Nutch Search 1.0; herceg_novi at yahoo dot com)",
"NutchVinegarCrawl/Nutch-0.8.1 (Vinegar; http://www.cs.washington.edu; eytanadar at gmail dot com)",
"obidos-bot (just looking for books.)",
"ObjectsSearch/0.01-dev (ObjectsSearch;http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)",
"ObjectsSearch/0.0x (ObjectsSearch; http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)",
"oBot ((compatible;Win32))",
"Ocelli/1.x (http://www.globalspec.com/Ocelli)",
"Octopus",
"Octora Beta - www.octora.com",
"Octora Beta Bot - www.octora.com",
"Offline Explorer 1.*",
"OliverPerry",
"OmniExplorer_Bot/1.0x (+http://www.omni-explorer.com) Internet CategorizerOmniExplorer http://www.omni-explorer.com/ car & shopping search (64.62.175.xxx)",
"OmniExplorer_Bot/1.0x (+http://www.omni-explorer.com) Job Crawler",
"OmniExplorer_Bot/1.1x (+http://www.omni-explorer.com) ******* Crawler",
"OmniExplorer_Bot/x.xx (+http://www.omni-explorer.com) WorldIndexer",
"onCHECK Linkchecker von www.scientec.de fuer www.onsinn.de",
"Onet.pl SA- http://szukaj.onet.pl",
"online link validator (http://www.dead-links.com/)",
"OntoSpider/1.0 libwww-perl/5.65",
"OpenAcoon v4.0.x (www.openacoon.de)",
"Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)",
"Openfind data gatherer- Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)",
"Openfind Robot/1.1A2",
"OpenISearch/1.x (www.openisearch.com)",
"OpenTaggerBot (http://www.opentagger.com/opentaggerbot.htm)",
"OpenTextSiteCrawler/2.9.2",
"OpenWebSpider/0.x.x (http://www.openwebspider.org)",
"OpenWebSpider/x",
"Opera/5.0 (Linux 2.0.38 i386; U) [en]",
"Opera/5.11 (Windows ME; U) [ru]",
"Opera/5.12 (Windows 98; U) [en]",
"Opera/6.01 (larbin@unspecified.mail)",
"Opera/6.x (Linux 2.4.8-26mdk i686; U) [en]",
"Opera/6.x (Windows NT 4.0; U) [de]",
"Opera/7.x (Windows NT 5.1; U) [en]",
"Opera/8.xx (Windows NT 5.1; U; en)",
"Opera/9.0 (Windows NT 5.1; U; en)",
"Opera/9.00 (Windows NT 5.1; U; de)",
"Opera/9.60 (Windows NT 5.1; U; de) Presto/2.1.1",
"OpidooBOT (larbin2.6.3@unspecified.mail)",
"OPWV-SDK UP.Browser/7.0.2.3.119 (GUI) MMP/2.0 Push/PO",
"Oracle Application Server Web Cache 10g",
"Oracle iMTCrawler",
"Oracle Ultra Search",
"OrangeSpider",
"Orbiter/T-2.0 (+http://www.dailyorbit.com/bot.htm)",
"Orca Browser (http://www.orcabrowser.com)",
"OSSProxy 1.3.305.321 (Build 305.321 Win32 en-us)(Dec 21 2005 16:30:54)",
"OutfoxBot/0.x (For internet experiments; http://; outfox.agent@gmail.com)",
"OutfoxMelonBot/0.5 (for internet experiments; http://; outfoxbot@gmail.com)",
"Overture-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
"OWR_Crawler 0.1",
"ozelot/2.7.3 (Search engine indexer; www.flying-cat.de/ozelot; ozelot@flying-cat.de)",
"PADLibrary Spider",
"PageBitesHyperBot/600 (http://www.pagebites.com/)",
"Pagebull http://www.pagebull.com/",
"Pagestacker Bot",
"page_verifier (http://www.securecomputing.com/goto/pv)",
"PagmIEDownload",
"parallelContextFocusCrawler1.1parallelContextFocusCrawler1.1",
"ParaSite/1.0b (http://www.ianett.com/parasite/)",
"Patwebbot (http://www.herz-power.de/technik.html)",
"pavuk/0.9pl29b i686-pc-linux-gnu",
"PBrowse 1.4b",
"pd02_1.0.0 pd02_1.0.0@dzimi@post.sk",
"PEAR HTTP_Request class ( http://pear.php.net/ )",
"PEERbot www.peerbot.com",
"PeopleChat/Search_Engine",
"PEval 1.4b",
"PHP/3.x.xx",
"PHP/4.0.4pl1",
"PHP/4.0.6",
"PHP/4.1.1",
"PHP/4.1.2",
"PicoSearch/1.0",
"Piffany_Web_Scraper_v0.x",
"Piffany_Web_Spider_v0.x",
"PigeonBot1.0 BETA",
"PingALink Monitoring Services 1.0",
"PingALink Monitoring Services 1.0 (http://www.pingalink.com)",
"Pingdom GIGRIB (http://www.pingdom.com)",
"pipeLiner/0.3a (PipeLine Spider;http://www.pipeline-search.com/webmaster.html; webmaster'at'pipeline-search.com)",
"pipeLiner/0.xx (PipeLine Spider; http://www.pipeline-search.com/webmaster.html)",
"Pita",
"Pizilla++ ver 2.45",
"PJspider/3.0 (pjspider@portaljuice.com; http://www.portaljuice.com)",
"Plagger/0.x.xx (http://plagger.org/)",
"PlagiarBot/1.0",
"PlantyNet_WebRobot_V1.9 dhkang@plantynet.com",
"plinki/0.1 (you got plinked! (thats a good thing..); http://www.plinki.com; crawl@plinki.com)",
"PluckFeedCrawler/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://www.pluck.com; 1 subscribers)",
"Pluggd/Nutch-0.9 (automated crawler http://www.pluggd.com;support at pluggd dot com)",
"Pockey-GetHTML/4.12.0 (Win32; GUI; ix86)",
"Pockey-GetHTML/x.xx",
"Pockey/x.x.x",
"Pockey7.x.x(WIN32GUI)",
"POE-Component-Client-HTTP/0.64 (perl; N; POE; en; rv:0.640000)",
"Poirot",
"polybot 1.0 (http://cis.poly.edu/polybot/)",
"Pompos/1.x http://dir.com/pompos.html",
"Pompos/1.x pompos@iliad.fr",
"Popdexter/1.0",
"Port Huron Labs",
"PortalBSpider/2.0 (spider@portalb.com)",
"portalmmm/2.0 S500i(c20;TB)",
"PostFavorites",
"potbot 1.0",
"PRCrawler/Nutch-0.9 (data mining development project; crawler@projectrialto.com)",
"PrivacyFinder Cache Bot v1.0",
"PrivacyFinder/1.1",
"Privoxy/3.0 (Anonymous)",
"Production Bot 0116B",
"Production Bot 2016B",
"Production Bot DOT 3016B",
"Program Shareware 1.0.2",
"Progressive Download",
"Progressive Download HTTP check",
"Project XP5 [2.03.07-111203]",
"PROve AnswerBot 4.0",
"ProWebGuide Link Checker (http://www.prowebguide.com)",
"psbot/0.1 (+http://www.picsearch.com/bot.html)",
"PSurf15a 11",
"PSurf15a 51",
"PSurf15a VA",
"psyche*****",
"PubCrawl (pubcrawl.stanford.edu)",
"puf/0.91beta6a (Linux 2.2.18; i686)",
"puf/0.93.2a (Linux 2.4.18; i686)",
"pulseBot (pulse Web Miner)",
"PureSight",
"PuxaRapido v1.0",
"PWeBot/1.2 Inspector (http://www.programacionweb.net/robot.php)",
"PycURL",
"PycURL/7.xx.x",
"Python-urllib/1.1x",
"Python-urllib/2.0a1",
"Qango.com Web Directory (http://www.qango.com/)",
"QEAVis Agent/Nutch-0.9 (Quantitative Evaluation of Academic Websites Visibility; http://nlp.uned.es/qeavis",
"QPCreep Test Rig ( We are not indexing- just testing )",
"QuepasaCreep ( crawler@quepasacorp.com )",
"QuepasaCreep v0.9.1x",
"QueryN Metasearch",
"QuickTime\xaa.7.0.4 (qtver=7.0.4;cpu=PPC;os=Mac 10.3.9)",
"QweeryBot/3.01 ( http://qweerybot.qweery.nl)",
"Qweery_robot.txt_CheckBot/3.01 (http://qweerybot.qweery.com)",
"R6_CommentReader_(www.radian6.com/crawler)",
"R6_FeedFetcher_(www.radian6.com/crawler)",
"rabaz (rabaz at gigabaz dot com)",
"RaBot/1.0 Agent-admin/phortse@hanmail.net",
"Rainbot1.1",
"ramBot xtreme x.x",
"RAMPyBot - www.giveRAMP.com/0.1 (RAMPyBot - www.giveRAMP.com; http://www.giveramp.com/bot.html; support@giveRAMP.com)",
"RAMPyBot/0.8-dev (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
"Rank Exec (rankexec.com) Reciprocal Link Manager 1.x/bot",
"Rankivabot/3.2 (www.rankiva.com; 3.2; vzmxikn)",
"Rational SiteCheck (Windows NT)",
"RAYSPIDER/Nutch-0.9",
"ReadABlog Spider (compatible; 1.1; feed update; www.readablog.com)",
"RealDownload/4.0.0.4x",
"REAP-crawler Nutch/Nutch-1.0-dev (Reap Project; http://reap.cs.cmu.edu/REAP-crawler/; Reap Project)",
"Reaper [2.03.10-031204] (http://www.sitesearch.ca/reaper/)",
"Reaper/2.0x (+http://www.sitesearch.ca/reaper)",
"REBOL Core 2.x.x.x.x",
"REBOL View 1.x.x.x.x",
"RebusnetBot (+http://www.rebusnet.biz)",
"RebusnetPADBot/1.5x (+http://www.rebusnet.biz)",
"reciprocal links checker (http://www.recip-links.com/)",
"RedBot/redbot-1.0 (Rediff.com Crawler; redbot at rediff dot com)",
"RedCarpet/1.2 (http://www.redcarpet-inc.com/robots.html)",
"RedCell/0.1 (InfoSec Search Bot (Coming Soon); http://www.telegenetic.net/bot.html; lhall@telegenetic.net)",
"RedCell/0.1 (RedCell; telegenetic.net/bot.html; lhall_at_telegenetic.net)",
"RedKernel WWW-Spider 2/0 (+http://www-spider.redkernel-softwares.com/)",
"REL Link Checker Lite x.x",
"RepoMonkey Bait & Tackle/v1.01",
"Rewebber/1.2 libwww-perl/5.41",
"rico/0.1",
"RixBot (http://babelserver.org/rix)",
"RMA/1.0 (compatible; RealMedia)",
"RMA/1.0 (compatible; RealMedia)",
"RoboCrawl (http://www.canadiancontent.net)",
"RoboCrawl (www.canadiancontent.net)",
"RoboPal (http://www.findpal.com/)",
"Robot/www.pj-search.com",
"Robot: NutchCrawler- Owner: wdavies@acm.org",
"Robot@SuperSnooper.Com",
"Robozilla/1.0",
"Rome Client (http://tinyurl.com/64t5n) Ver: 0.9",
"Rotondo/3.1 libwww/5.3.1",
"RPT-HTTPClient/0.3-x",
"RRC (crawler_admin@bigfoot.com)",
"RssBandit/1.5.0.10 (.NET CLR 1.1.4322.2407; WinNT 5.1.2600.0; http://www.rssbandit.org) (.NET CLR 1.1.4322.2407; WinNT 5.1.2600.0; )",
"RSSMicro.com RSS/Atom Feed Robot",
"RSSOwl/1.2.3 2006-11-26 (Windows; U; zhtw)",
"RSSOwl/1.2.4 Preview Release 2007-04-15 (Windows; U; zhtw)",
"RssReader/1.0.xx.x (http://www.rssreader.com) Microsoft Windows NT 5.1.2600.0",
"RSurf15a 41",
"RSurf15a 51",
"RSurf15a 81",
"Rubbot/1.0 (+http://rubhub.com/)",
"RufusBot (Rufus Web Miner; http://64.124.122.252/feedback.html)",
"RufusBot (Rufus Web Miner; http://www.webaroo.com/rooSiteOwners.html)",
"Rumours-Agent",
"RX Bar",
"S&L Spider (http://search.hirners.com/)",
"S.T.A.L.K.E.R. (http://www.seo-tools.net/en/bot.aspx)",
"SafariBookmarkChecker (+http://www.coriolis.ch/)",
"sait/Nutch-0.9 (SAIT Research; http://www.samsung.com)",
"SandCrawler - Compatibility Testing",
"savvybot/0.2",
"SBIder/0.7 (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html)",
"SBIder/0.8-dev (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html)",
"SBL-BOT (http://sbl.net)",
"ScanWeb",
"ScholarUniverse/0.8 (Nutch;+http://scholaruniverse.com/bot.jsp; fetch-agent@scholaruniverse.com)",
"schwarzmann.biz-Spider_for_paddel.org+(http://www.innerprise.net/usp-spider.asp)",
"Science Traveller International 1X/1.0",
"ScollSpider/2.0 (+http://www.webwobot.com/ScollSpider.php)",
"Scooter-3.0.EU",
"Scooter-3.0.FS",
"Scooter-3.0.HD",
"Scooter-3.0.VNS",
"Scooter-3.0QI",
"Scooter-3.2",
"Scooter-3.2.BT",
"Scooter-3.2.DIL",
"Scooter-3.2.EX",
"Scooter-3.2.JT",
"Scooter-3.2.NIV",
"Scooter-3.2.SF0",
"Scooter-3.2.snippet",
"Scooter-3.3dev",
"Scooter-ARS-1.1",
"Scooter-ARS-1.1-ih",
"scooter-venus-3.0.vns",
"Scooter-W3-1.0",
"Scooter-W3.1.2",
"Scooter/1.0",
"Scooter/1.0 scooter@pa.dec.com",
"Scooter/1.1 (custom)",
"Scooter/2.0 G.R.A.B. V1.1.0",
"Scooter/2.0 G.R.A.B. X2.0",
"Scooter/3.3",
"Scooter/3.3.QA.pczukor",
"Scooter/3.3.vscooter",
"Scooter/3.3_SF",
"Scooter2_Mercator_x-x.0",
"Scooter_bh0-3.0.3",
"Scooter_trk3-3.0.3",
"Scope (Mars+)",
"ScoutAbout",
"ScoutAnt/0.1; +http://www.ant.com/what_is_ant.com/",
"scoutmaster",
"Scrubby/2.x (http://www.scrubtheweb.com/)",
"Scrubby/3.0 (+http://www.scrubtheweb.com/help/technology.html)",
"Search+",
"Search-Engine-Studio",
"search.ch V1.4",
"search.ch V1.4.2 (spiderman@search.ch; http://www.search.ch)",
"Search/1.0 (http://www.innerprise.net/es-spider.asp)",
"searchbot admin@google.com",
"SearchByUsa/2 (SearchByUsa; http://www.SearchByUsa.com/bot.html; info@SearchByUsa.com)",
"SearchdayBot",
"SearchExpress Spider0.99",
"SearchGuild/DMOZ/Experiment (searchguild@gmail.com)",
"SearchGuild_DMOZ_Experiment (chris@searchguild.com)",
"Searchit-Now Robot/2.2 (+http://www.searchit-now.co.uk)",
"Searchmee! Spider v0.98a",
"SearchSight/2.0 (http://SearchSight.com/)",
"SearchSpider.com/1.1",
"Searchspider/1.2 (SearchSpider; http://www.searchspider.com; webmaster@searchspider.com)",
"SearchTone2.0 - IDEARE",
"Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/0.3",
"Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.0 (XDF)",
"Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2",
"Seeker.lookseek.com",
"semaforo.net",
"Semager/1.1 (http://www.semager.de/blog/semager-bots/)",
"Semager/1.x (http://www.semager.de)",
"semanticdiscovery/0.x",
"Sensis Web Crawler (search_comments\at\sensis\dot\com\dot\au)",
"Sensis.com.au Web Crawler (search_comments\at\sensis\dot\com\dot\au)",
"SeznamBot/1.0",
"SeznamBot/1.0 (+http://fulltext.seznam.cz/)",
"SeznamBot/2.0-test (+http://fulltext.sblog.cz/)",
"ShablastBot 1.0",
"******** v1.x.x.xx",
"SharewarePlazaFileCheckBot/1.0+(+http://www.SharewarePlaza.com)",
"Shim Crawler",
"Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)",
"ShopWiki/1.0 ( +http://www.shopwiki.com/)",
"ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)",
"Shoula.com Crawler 2.0",
"SietsCrawler/1.1 (+http://www.siets.biz)",
"Sigram/Nutch-1.0-dev (Test agent for Nutch development; http://www.sigram.com/bot.html; bot at sigram dot com)",
"Siigle Orumcex v.001 Turkey (http://www.siigle.com)",
"silk/1.0",
"silk/1.0 (+http://www.slider.com/silk.htm)/3.7",
"SimpleFavPanel/1.2",
"Simpy 1.x; http://www.simpy.com/",
"Simpy/1.x (Simpy; http://www.simpy.com/?ref=bot; feedback at simpy dot com)",
"Sirketcebot/v.01 (http://www.sirketce.com/bot.html)",
"SiteBar/3.x.x (Bookmark Server; http://sitebar.org/)",
"SiteBar/x.x",
"SiteBar/x.x.x (Bookmark Server; http://sitebar.org/)",
"sitecheck.internetseer.com",
"sitecheck.internetseer.com (For more info see: http://sitecheck.internetseer.com)",
"SiteRecon+(xx)",
"SiteSnagger",
"SiteSpider +(http://www.SiteSpider.com/)",
"SiteSucker/1.x.x",
"SiteTaggerBot (http://www.sitetagger.com/bot.htm)",
"SiteTruth.com site rating system",
"SiteWinder",
"SiteXpert",
"Skampy/0.9.x (http://www.skaffe.com/skampy-info.html)",
"Skimpy/0.x (http://www.skaffe.com/skampy-info.html)",
"Skywalker/0.1 (Skywalker; anonymous; anonymous)",
"Slarp/0.1",
"Sleipnir",
"Sleipnir Version 1.xx",
"Sleipnir Version2.x",
"Sleipnir/2.xx",
"Slider_Search_v1-de",
"SlimBrowser",
"Slurp/2.0 (slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Slurp/2.0-KiteWeekly (slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Slurp/si (slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Slurpy Verifier/1.0",
"SlySearch (slysearch@slysearch.com)",
"SlySearch/1.0 http://www.plagiarism.org/crawler/robotinfo.html",
"SlySearch/1.x http://www.slysearch.com",
"SmartDownload/1.2.67 (Win32; Jan 12 1999)",
"SmartDownload/1.2.77 (Win32; Feb 1 2000)",
"SmartDownload/1.2.77 (Win32; Jun 19 2001)",
"smartwit.com",
"SmiffyDCMetaSpider/1.0",
"sna-0.0.1 (mikemuzio@msn.com)",
"sna-0.0.1 mikeelliott@hotmail.com",
"snap.com beta crawler v0",
"Snapbot/1.0",
"Snappy/1.1 ( http://www.urltrends.com/ )",
"Snarfer/0.x.x (http://www.snarfware.com/)",
"SnoopRob/x.x",
"Snoopy v1.xx",
"Snoopy_v0.xx",
"SnykeBot/0.6 (http://www.snyke.com)",
"SocSciBot ()",
"SoftBank/1.0/812SH/SHJ001 Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1",
"SoftHypermarketFileCheckBot/1.0+(+http://www.softhypermaket.com)",
"Softizerbot (http://www.softizer.com)",
"sogou develop spider",
"Sogou Orion spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
"sogou spider",
"Sogou web spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
"sohu agent",
"sohu-search",
"Sosospider+(+http://help.soso.com/webspider.htm)",
"Space Bison/0.02 [fu] (Win67; X; SK)",
"SpeedDownload/1.x",
"speedfind ramBot xtreme 8.1",
"Speedy Spider (Beta/x.x; speedy@entireweb.com)",
"Speedy Spider (Entireweb; Beta/1.0; http://www.entireweb.com/about/search_tech/speedyspider/)",
"Speedy_Spider (http://www.entireweb.com)",
"Sphere Scout&v4.0 - scout at sphere dot com",
"Sphider",
"Spida/0.1",
"Spider-Sleek/2.0 (+http://search-info.com/linktous.html)",
"spider.batsch.com",
"Spider.TerraNautic.net - v:1.04",
"spider.yellopet.com - www.yellopet.com",
"Spider/maxbot.com admin@maxbot.com",
"SpiderKU/0.x",
"SpiderMan",
"SpiderMonkey/7.0x (SpiderMonkey.ca info at http://spidermonkey.ca/sm.shtml)",
"Spinne/2.0",
"Spinne/2.0 med",
"Spinne/2.0 med_AH",
"Spock Crawler (http://www.spock.com/crawler)",
"sportsuchmaschine.de-Robot (Version: 1.02- powered by www.sportsuchmaschine.de)",
"sproose/0.1-alpha (sproose crawler; http://www.sproose.com/bot.html; crawler@sproose.com)",
"SQ Webscanner",
"Squid-Prefetch",
"squidclam",
"SquidClamAV_Redirector 1.x.x",
"Sqworm/2.9.81-BETA (beta_release; 20011102-760; i686-pc-linux-gnu)",
"Sqworm/2.9.85-BETA (beta_release; 20011115-775; i686-pc-linux-gnu)",
"Sqworm/2.9.89-BETA (beta_release; 20020130-839; i686-pc-linux-gnu) ",
"SSurf15a 11 ",
"StackRambler/x.x ",
"Stamina/1.4",
"Star Downloader",
"StarDownloader/1.xx",
"stat statcrawler@gmail.com",
"Steeler/1.x (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)",
"Steeler/3.3 (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)",
"Strategic Board Bot (+http://www.strategicboard.com)",
"Strategic Board Bot (+http://www.strategicboard.com)",
"Submission Spider at surfsafely.com",
"suchbaer.de",
"suchbaer.de (CrawlerAgent v0.103)",
"suchbot",
"Suchknecht.at-Robot",
"suchpadbot/1.0 (+http://www.suchpad.de)",
"Sunrise XP/2.x",
"Sunrise/0.42g (Windows XP)",
"SuperBot/x.x (Win32)",
"SuperBot/x.x.x.xx (Windows XP)",
"Superdownloads Spiderman",
"SURF ",
"SurferF3 1/0",
"SurfMaster",
"SurveyBot/2.2 <a href='http://www.whois.sc'>Whois Source</a>",
"SurveyBot/2.3 (Whois Source)",
"suzuran",
"SWB/V1.4 (HP)",
"swbot/0.9c libwww/5.3.1",
"Swooglebot/2.0. (+http://swoogle.umbc.edu/swooglebot.htm)",
"SWSBot-Images/1.2 http://www.smartwaresoft.com/swsbot12.html",
"SygolBot http://www.sygol.net",
"Sylera/1.2.x",
"SyncBot",
"SyncIT/x.x",
"Syndirella/0.91pre",
"SynoBot",
"Syntryx ANT Scout Chassis Pheromone; Mozilla/4.0 compatible crawler",
"Szukacz/1.x",
"Szukacz/1.x (robot; www.szukacz.pl/jakdzialarobot.html; szukacz@proszynski.pl)",
"T-Online Browser",
"tags2dir.com/0.8 (+http://tags2dir.com/directory/)",
"Tagword (http://tagword.com/dmoz_survey.php)",
"Tagyu Agent/1.0",
"Talkro Web-Shot/1.0 (E-mail: webshot@daumsoft.com- Home: http://222.122.15.190/webshot)",
"TAMU_CS_IRL_CRAWLER/1.0",
"targetblaster.com/0.9k",
"TargetYourNews.com bot",
"TCDBOT/Nutch-0.8 (PhD student research;http://www.tcd.ie; mcgettrs at t c d dot IE)",
"TE",
"TeamSoft WinInet Component",
"TECOMAC-Crawler/0.x",
"Tecomi Bot (http://www.tecomi.com/bot.htm)",
"Teleport Pro/1.2x(.1xxx)",
"Teoma MP",
"teomaagent crawler-admin@teoma.com",
"teomaagent1 [crawler-admin@teoma.com]",
"teoma_agent1",
"Teradex Mapper; mapper@teradex.com; http://www.teradex.com",
"TeragramCrawler",
"terraminds-bot/1.0 (support@terraminds.de)",
"TerrawizBot/1.0 (+http://www.terrawiz.com/bot.html)",
"Test spider",
"TestCrawler/Nutch-0.9 (Testing Crawler for Research ; http://balihoo.com/index.aspx; tgautier at balihoo dot com)",
"The Expert HTML Source Viewer (http://www.expert-html.com)",
"TheRarestParser/0.2a (http://therarestwords.com/)",
"TheSuBot/0.1 (www.thesubot.de)",
"thumbshots-de-Bot (Version: 1.02- powered by www.thumbshots.de)",
"timboBot/0.9 http://www.breakingblogs.com/timbo_bot.html",
"http://www.timelyweb.com/",
"TinEye/1.1 (http://tineye.com/crawler.html)",
"tivraSpider/1.0 (crawler@tivra.com)",
"TJG/Spider",
"TJvMultiHttpGrabber Component",
"Tkensaku/x.x(http://www.tkensaku.com/q.html)",
"toCrawl/UrlDispatcher",
"Topodia/1.2-dev (Topodia - Crawler for HTTP content indexing; http://www.topodia.com/; support@topodia.com)",
"TOPOS robot/1.1 (http://www.topos.com.ua/)",
"Toutatis x-xx.x (hoppa.com)",
"Toutatis x.x (hoppa.com)",
"Toutatis x.x-x",
"traazibot/testengine (+http://www.traazi.de)",
"Trailfire-bot/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
"Trailfire-bot/0.7.1 (Trailfire page content analyzer; http://trailfire.com; info@trailfire.com)",
"Trailfire/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
"Trampelpfad-Spider",
"Trampelpfad-Spider-v0.1",
"tricosMetaCheck 1.2216-08-1999 (http://www.tricos.com/metacheck)",
"TSurf15a 11",
"TulipChain/5.x (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://java.sun.com/) Linux/2.4.17",
"TulipChain/5.xx (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://apple.com/) Mac_OS_X/10.2.8",
"Tumblr/1.0 RSS syndication (+http://www.tumblr.com/) (support@tumblr.com)",
"TurnitinBot/x.x (http://www.turnitin.com/robot/crawlerinfo.html)",
"Turnpike Emporium LinkChecker/0.1",
"TutorGig/1.5 (+http://www.tutorgig.com/crawler)",
"Tutorial Crawler 1.4 (http://www.tutorgig.com/crawler)",
"Twiceler www.cuill.com/robots.html",
"Twiceler-0.9 http://www.cuill.com/twiceler/robot.html",
"Twisted PageGetter",
"Twitturly / v0.x",
"Twotrees Reactive Filter V2.0",
"Tycoon Agent/Nutch-1.0-dev",
"TygoBot",
"TygoProwler",
"UCmore",
"UCMore Crawler App",
"UCWEB5.1",
"UDM",
"UdmSearch/3.1.x",
"UIowaCrawler/1.0",
"UKWizz/Nutch-0.8.1 (UKWizz Nutch crawler; http://www.ukwizz.com/)",
"Ultraseek",
"Under the Rainbow 2.2",
"unknownght.com Web Server IIS vs Apache Survey. See Results at www.DNSRight.com",
"UofTDB_experiment (leehyun@cs.toronto.edu)",
"UP.Browser/3.01-IG01 UP.Link/3.2.3.4",
"updated/0.1-alpha (updated crawler; http://www.updated.com; crawler@updated.com)",
"updated/0.1beta (updated.com; http://www.updated.com; crawler@updated.om)",
"UPG1 UP/4.0 (compatible; Blazer 1.0)",
"Uptimebot",
"UptimeBot(www.uptimebot.com)",
"URI::Fetch/0.06",
"URL Spider Pro/x.xx (innerprise.net)",
"URLBase/6.x",
"URLBlaze",
"urlfan-bot/1.0; +http://www.urlfan.com/site/bot/350.html",
"URLGetFile",
"URL_Spider_Pro/x.x",
"URL_Spider_Pro/x.x+(http://www.innerprise.net/usp-spider.asp)",
"User-Agent: BoardReader Favicon Fetcher /1.0 info@boardreader.com",
"User-Agent: BoardReader Image Fetcher /1.0 info@boardreader.com",
"User-Agent: LjSEEK Picture-Bot /1.0 contact@ljseek.com",
"User-Agent: FileHeap! file downloader (http://www.fileheap.com)",
"User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
"User-Agent: Mozilla/4.0 (SKIZZLE! Distributed Internet Spider v1.0 - www.SKIZZLE.com)",
"user-agent=Mozilla/3.01Gold",
"USyd-NLP-Spider (http://www.it.usyd.edu.au/~vinci/bot.html)",
"UtilMind HTTPGet",
"Utopia WebWasher 3.0",
"u*******/1500",
"VadixBot",
"Vagabondo-WAP/2.0 (webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)/1.0 Profile",
"Vagabondo/1.x MT (webagent@wise-guys.nl)",
"Vagabondo/2.0 MT",
"Vagabondo/2.0 MT (webagent at wise-guys dot nl)",
"Vagabondo/2.0 MT (webagent@NOSPAMwise-guys.nl)",
"Vagabondo/3.0 (webagent at wise-guys dot nl)",
"Vakes/0.01 (Vakes; http://www.vakes.com/; search@vakes.com)",
"VayalaCreep-v0.0.1 (haploid@haploid.com)",
"Vayala|Creep-v0.0.1 (codepoet@wildties.com)",
"vb wininet",
"versus 0.2 (+http://versus.integis.ch)",
"versus crawler eda.baykan@epfl.ch",
"Verticrawlbot",
"VeryGoodSearch.com.DaddyLongLegs",
"verzamelgids.nl - Networking4all Bot/x.x",
"Verzamelgids/2.2 (http://www.verzamelgids.nl)",
"Vespa Crawler",
"virus_detector (virus_harvester@securecomputing.com)",
"VisBot/2.0 (Visvo.com Crawler; http://www.visvo.com/bot.html; bot@visvo.com)",
"Visicom Toolbar",
"Vision Research Lab image spider at vision.ece.ucsb.edu",
"VLC media player - version 0.8.5 Janus - (c) 1996-2006 the VideoLAN team",
"VMBot/0.x.x (VMBot; http://www.VerticalMatch.com/; vmbot@tradedot.com)",
"Vortex/2.2 (+http://marty.anstey.ca/robots/vortex/)",
"voyager-hc/1.0",
"voyager/1.0",
"VSE/1.0 (testcrawler@hotmail.com)",
"VSE/1.0 (testcrawler@vivisimo.com)",
"vspider",
"vspider/3.x",
"VWBOT/Nutch-0.9-dev (VWBOT Nutch Crawler; http://vwbot.cs.uiuc.edu;+vwbot@cs.uiuc.edu",
"W3C-checklink/3.x.x.x libwww-perl/5.xx",
"W3C-checklink/4.x [4.xx] libwww-perl/5.xxx",
"W3C-WebCon/5.x.x libwww/5.x.x",
"W3CLineMode/5.4.0 libwww/5.x.x",
"W3CRobot/5.4.0 libwww/5.4.0",
"W3C_Validator/1.xxx libwww-perl/5.xx",
"w3m/0.x.xx",
"W3SiteSearch Crawler_v1.1 http://www.w3sitesearch.de",
"wadaino.jp-crawler 0.2 (http://wadaino.jp/)",
"WannaBe (Macintosh; PPC)",
"WapOnWindows 1.0",
"Watchfire WebXM 1.0",
"WAVcheck 1.0.x (http://www.webbanalys.se/apps/WAVcheck/)",
"Wavefire/0.8-dev (Wavefire; http://www.wavefire.com; info@wavefire.com)",
"Waypath development crawler - info at waypath dot com",
"Waypath Scout v2.x - info at waypath dot com",
"WDG_Validator/1.1",
"Web Image Collector",
"Web Link Validator 1.5",
"Web Snooper",
"Web-Bot V1.03",
"Web-Robot/5.0 (en-US; web-robot.com/policy.html) Web-Robot Crawler/2.0.3",
"web2express.org/Nutch-0.9-dev (leveled playing field; http://web2express.org/; info at web2express.org)",
"WebAlta Crawler/1.2.1 (http://www.webalta.ru/bot.html)",
"WebarooBot (Webaroo Bot; http://64.124.122.252/feedback.html)",
"WebarooBot (Webaroo Bot; http://www.webaroo.com/rooSiteOwners.html)",
"WebAuto/3.4xxx (WinNT; I)",
"webbandit/4.xx.0",
"WebBug/5.x",
"Webclipping.com",
"webcollage/1.xx",
"WebCompass 2.0",
"WebCopier vx.x",
"WebCopier vx.xa",
"WebCorp/1.0",
"webcrawl.net",
"WebDownloader for X x.xx",
"Webdup/0.9",
"WebFetch",
"webfetch/5.x.x",
"WebFilter Robot 1.0",
"WebFilter Robot 1.x",
"WebFindBot(http://www.web-find.com)",
"Webglimpse 2.xx.x (http://webglimpse.net)",
"webGobbler/1.x.x",
"webhack",
"WebImages 0.3 ( http://herbert.groot.jebbink.nl/?app=WebImages )",
"WebLight/4.x.x (support@illumit.com; http://www.illumit.com/Products/weblight/)",
"Weblink's checker/",
"Weblog Attitude Diffusion 1.0",
"WebMiner/x.x [en] (Win98; I)",
"WeBoX/0.xx",
"WebPix 1.0 (www.netwu.com)",
"WebQL",
"WebRACE/1.1 (University of Cyprus- Distributed Crawler)",
"WebRankSpider/1.37 (+http://ulm191.server4you.de/crawler/)",
"WebReaper vx.x - www.webreaper.net",
"WebReaper [info@webreaper.net]",
"WebReaper [webreaper@webreaper.net]",
"WebSearch.COM.AU/3.0.1 (The Australian Search Engine; http://WebSearch.COM.AU; Search@WebSearch.COM.AU)",
"WebSearchBench WebCrawler v0.1(Experimental)",
"WebSearchBench WebCrawler V1.0 (Beta)- Prof. Dr.-Ing. Christoph Lindemann- Universität Dortmund- cl@cs.uni-dortmund.de- http://websearchbench.cs.uni-dortmund.de/",
"Website Explorer/0.9.x.x",
"Website *********",
"WebsiteWorth v1.0",
"Webspinne/1.0 webmaster@webspinne.de",
"Websquash.com (Add url robot)",
"WebStat/1.0 (Unix; beta; 20040314)",
"Webster v0.3 ( http://webster.healeys.net/ )",
"webster-internet.de pad browser",
"WebStripper/2.xx",
"WebTrafficExpress/x.0",
"WebTrends/3.0 (WinNT)",
"WebVac (webmaster@pita.stanford.edu)",
"WebVal/1.0",
"Webverzeichnis.de - Telefon: 01908 / 26005",
"WebVulnCrawl.blogspot.com/1.0 libwww-perl/5.803",
"WebWatcherMonitor/2.01",
"WebZIP/x.x (http://www.spidersoft.com)",
"Wells Search II",
"WEP Search 00",
"West Wind Internet Protocols 4.xx",
"WFARC",
"Wget/1.x(.x)GNU wget http://www.gnu.org/software/wget/wget.html - file downloader",
"Wget/1.x+cvs-stable (Red Hat modified)",
"Wget/1.x.x+cvs",
"Whatsup/x.x",
"whatUseek_winona/3.0",
"WhizBang! Lab",
"Wildsoft Surfer",
"Willow Internet Crawler by Twotrees V2.1",
"WinampMPEG/2.00 (larbin@unspecified.mail)",
"WincerSong Agent v1.0",
"Windows-Media-Player/10.00.00.xxxx",
"WinGet 1.1",
"WinHTTP Example/1.0",
"WinkBot/0.06 (Wink.com search engine web crawler; http://www.wink.com/Wink:WinkBot; winkbot@wink.com)",
"WinPodder (http://winpodder.com)",
"WinWAP/3.x (3.x.x.xx; Win32) (Google WAP Proxy/1.0)",
"WISEbot/1.0 (WISEbot@koreawisenut.com; http://wisebot.koreawisenut.com)",
"WiseWire-Spider2",
"wish-project (http://wish.slis.tsukuba.ac.jp/)",
"WordChampBot",
"WordPress/x.x.x.x PHP/4.x.xx",
"worio heritrix bot (+http://worio.com/)",
"woriobot ( http://www.worio.com/)",
"WorldLight",
"WorQmada/1.0",
"Wotbox/alpha0.6 (bot@wotbox.com; http://www.wotbox.com)",
"Wotbox/alpha0.x.x (bot@wotbox.com; http://www.wotbox.com) Java/1.4.1_02",
"WSB WebCrawler V1.0 (Beta)- cl@cs.uni-dortmund.de",
"wume_crawler/1.1 (http://wume.cse.lehigh.edu/~xiq204/crawler/)",
"Wusage/x.0@boutell.com",
"Wwlib/Linux",
"WWSBOT 1.x [--- http://www.analyzer.nu ---]",
"WWW-Mechanize/1.1x",
"www.arianna.it",
"www.business-socket.com registry verify/1.x",
"www.doweb.co.uk crawler",
"www4mail/2.x libwww-FM/2.14 (Unix; I)",
"WWWC/1.0x",
"WWWeasel Robot v1.00 (http://wwweasel.de)",
"WWWOFFLE/2.x",
"wwwster/1.x (Beta- mailto:gue@cis.uni-muenchen.de)",
"wxDownload Fast",
"X-Crawler ",
"Xaldon WebSpider",
"Xenu Link Sleuth 1.xx",
"Xenu's Link Sleuth 1.x[a-z]",
"Xerka WebBot v1.0.0 [UPVOpenDir]",
"xine/1.0",
"xirq/0.1-beta (xirq; http://www.xirq.com; xirq@xirq.com)",
"XMLSlurp/0.1 libwww-perl/5.805",
"XRL/2.00b1 (Linux; i686; en-us) (+http://metamark.net/about)",
"Xylix",
"xyro_(xcrawler@cosmos.inria.fr)",
"Y!J-BSC/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",
"Y!J-SRD/1.0",
"Y!J/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",
"Y!OASIS/TEST no-ad Mozilla/4.08 [en] (X11; I; FreeBSD 2.2.8-STABLE i386)",
"Y!TunnelPro",
"yacy (www.yacy.net; v20040602; i386 Linux 2.4.26-gentoo-r13; java 1.4.2_06; MET/en)",
"yacybot (x86 Windows XP 5.1; java 1.5.0_06; Europe/de) yacy.net",
"Yahoo Pipes 1.0",
"Yahoo! Mindset",
"Yahoo-Blogs/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html )",
"Yahoo-MMAudVid/1.0 (mms dash mmaudvidcrawler dash support at yahoo dash inc dot com)",
"Yahoo-MMAudVid/2.0(mms dash mm aud vid crawler dash support at yahoo dash inc.com ;Mozilla 4.0 compatible; MSIE 7.0;Windows NT 5.0; .NET CLR 2.0)",
"Yahoo-MMCrawler/3.x (mm dash crawler at trd dot overture dot com)",
"Yahoo-Test/4.0",
"Yahoo-VerticalCrawler-FormerWebCrawler/3.9 crawler at trd dot overture dot com; http://www.alltheweb.com/help/webmaster/crawler",
"YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide)",
"YahooSeeker-Testing/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/)",
"YahooSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/)",
"YahooSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/yahooseeker.html)",
"YahooSeeker/1.1 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/)",
"YahooSeeker/bsv3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html )",
"YahooSeeker/CafeKelsa-dev (compatible; Konqueror/3.2; FreeBSD ;cafekelsa-dev-webmaster@yahoo-inc.com )",
"Yandex/1.01.001 (compatible; Win16; I)",
"yarienavoir.net/0.2",
"Yeti",
"yggdrasil/Nutch-0.9 (yggdrasil biorelated search engine; www dot biotec dot tu minus dresden do de slash schroeder; heiko dot dietze at biotec dot tu minus dresden dot de)",
"YodaoBot/1.0 (http://www.yodao.com/help/webmaster/spider/; )",
"yoofind/yoofind-0.1-dev (yoono webcrawler; http://www.yoono.com ; MyEmail)",
"yoogliFetchAgent/0.1",
"yoono/1.0 web-crawler/1.0",
"YottaCars_Bot/4.12 (+http://www.yottacars.com) Car Search Engine ",
"YottaShopping_Bot/4.12 (+http://www.yottashopping.com) Shopping Search Engine",
"YTunnelPro",
"Z-Add Link Checker (http://w3.z-add.co.uk/linkcheck/)",
"Zao-Crawler",
"Zao-Crawler 0.2b",
"Zao/0.1 (http://www.kototoi.org/zao/)",
"ZBot/1.00 (icaulfield@zeus.com)",
"Zearchit",
"ZeBot_lseek.net (bot@ze.bz)",
"ZeBot_www.ze.bz (ze.bz@hotmail.com)",
"zedzo.digest/0.1 (http://www.zedzo.com/)",
"Zend_Http_Client",
"zerxbot/Version 0.6 libwww-perl/5.79",
"Zeus ThemeSite Viewer Webster Pro V2.9 Win32",
"Zeus xxxxx Webster Pro V2.9 Win32",
"Zeusbot/0.07 (Ulysseek's web-crawling robot; http://www.zeusbot.com; agent@zeusbot.com)",
"Ziggy -- The Clown From Hell!!",
"ZipppBot/0.xx (ZipppBot; http://www.zippp.net; webmaster@zippp.net)",
"ZIPPPCVS/0.xx (ZipppBot/.xx;http://www.zippp.net; webmaster@zippp.net)",
"Zippy v2.0 - Zippyfinder.com",
"Zoo Tycoon 2 Client -- http://www.zootycoon.com",
"ZoomSpider - wrensoft.com",
"zspider/0.9-dev http://feedback.redkolibri.com/",
"ZyBorg/1.0 (ZyBorg@WISEnut.com; http://www.WISEnut.com)",
"Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/)",
"Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)",
"voyager/2.0 (http://www.kosmix.com/html/crawler.html)",
",
"!Susie (http://www.sync2it.com/susie)",
"<a href='http://www.unchaos.com/'> UnChaos </a> From Chaos To Order Hybrid Web Search Engine.(vadim_gonchar@unchaos.com)",
"<a href='http://www.unchaos.com/'> UnChaos Bot Hybrid Web Search Engine. </a> (vadim_gonchar@unchaos.com)",
"<b> UnChaosBot From Chaos To Order UnChaos Hybrid Web Search Engine at www.unchaos.com </b> (info@unchaos.com)",
"<http://www.sygol.com/> http://www.sygol.com",
"( Robots.txt Validator http://www.searchengineworld.com/cgi-bin/robotcheck.cgi )",
"(DreamPassport/3.0; isao/MyDiGiRabi)",
"(Privoxy/1.0)",
"*/Nutch-0.9-dev",
"+SitiDi.net/SitiDiBot/1.0 (+Have Good Day)",
"-DIE-KRAEHE- META-SEARCH-ENGINE/1.1 http://www.die-kraehe.de",
"192.comAgent",
"1st ZipCommander (Net) - http://www.zipcommander.com/",
"2Bone_LinkChecker/1.0 libwww-perl/5.64",
"4anything.com LinkChecker v2.0",
"8484 Boston Project v 1.0",
":robot/1.0 (linux) ( admin e-mail: undefined http://www.neofonie.de/loesungen/search/robot.html )",
"A-Online Search",
"A1 Keyword Research/1.0.2 (+http://www.micro-sys.dk/products/keyword-research/) miggibot/2007.03.27",
"A1 Sitemap Generator/1.0 (+http://www.micro-sys.dk/products/sitemap-generator/) miggibot/2006.01.24",
"aardvark-crawler",
"AbachoBOT",
"AbachoBOT (Mozilla compatible)",
"ABCdatos BotLink/5.xx.xxx#BBL",
"Aberja Checkomat",
"abot/0.1 (abot; http://www.abot.com; abot@abot.com)",
"About/0.1libwww-perl/5.47",
"Accelatech RSSCrawler/0.4",
"accoona",
"Accoona-AI-Agent/1.1.1 (crawler at accoona dot com)",
"Accoona-AI-Agent/1.1.2 (aicrawler at accoonabot dot com)",
"Ace Explorer",
"Ack (http://www.ackerm.com/)",
"AcoiRobot",
"Acoon Robot v1.50.001",
"Acoon Robot v1.52 (http://www.acoon.de)",
"Acoon-Robot 4.0.x.[xx] (http://www.acoon.de)",
"Acoon-Robot v3.xx (http://www.acoon.de and http://www.acoon.com)",
"Acorn/Nutch-0.9 (Non-Profit Search Engine; acorn.isara.org; acorn at isara dot org)",
"ActiveBookmark 1.x",
"Activeworlds",
"ActiveWorlds/3.xx (xxx)",
"Ad Muncher v4.xx.x",
"Ad Muncher v4x Build xxxxx",
"Adaxas Spider (http://www.adaxas.net/)",
"Advanced Browser (http://www.avantbrowser.com)",
"AESOP_com_SpiderMan",
"agadine/1.x.x (+http://www.agada.de)",
"Agent-SharewarePlazaFileCheckBot/2.0+(+http://www.SharewarePlaza.com)",
"AgentName/0.1 libwww-perl/5.48",
"AIBOT/2.1 By +(www.21seek.com A Real artificial intelligence search engine China)",
"AideRSS/1.0 (aiderss.com)",
"aipbot/1.0 (aipbot; http://www.aipbot.com; aipbot@aipbot.com)",
"aipbot/2-beta (aipbot dev; http://aipbot.com; aipbot@aipbot.com)",
"Akregator/1.2.9; librss/remnants",
"Aladin/3.324",
"Alcatel-BG3/1.0 UP.Browser/5.0.3.1.2",
"Aleksika Spider/1.0 (+http://www.aleksika.com/)",
"AlertInfo 2.0 (Powered by Newsbrain)",
"AlkalineBOT/1.3",
"AlkalineBOT/1.4 (1.4.0326.0 RTM)",
"Allesklar/0.1 libwww-perl/5.46",
"Alligator 1.31 (www.nearsoftware.com)",
"Allrati/1.1 (+)",
"AltaVista Intranet V2.0 AVS EVAL search@freeit.com",
"AltaVista Intranet V2.0 Compaq Altavista Eval sveand@altavista.net",
"AltaVista Intranet V2.0 evreka.com crawler@evreka.com",
"AltaVista V2.0B crawler@evreka.com",
"AmfibiBOT",
"Amfibibot/0.06 (Amfibi Web Search; http://www.amfibi.com; agent@amfibi.com)",
"Amfibibot/0.07 (Amfibi Robot; http://www.amfibi.com; agent@amfibi.com)",
"amibot",
"Amiga-AWeb/3.4.167SE",
"AmigaVoyager/3.4.4 (MorphOS/PPC native)",
"AmiTCP Miami (AmigaOS 2.04)",
"Amoi 8512/R21.0 NF-Browser/3.3",
"amzn_assoc",
"AnnoMille spider 0.1 alpha - http://www.annomille.it",
"annotate_google; http://ponderer.org/download/annotate_google.user.js",
"Anonymized by ProxyOS: http://www.megaproxy.com",
"Anonymizer/1.1",
"AnswerBus (http://www.answerbus.com/)",
"AnswerChase PROve x.0",
"AnswerChase x.0",
"ANTFresco/x.xx",
"antibot-V1.1.5/i586-linux-2.2",
"AnzwersCrawl/2.0 (anzwerscrawl@anzwers.com.au;Engine)",
"Apexoo Spider 1.x",
"Aplix HTTP/1.0.1",
"Aplix_SANYO_browser/1.x (Japanese)",
"Aplix_SEGASATURN_browser/1.x (Japanese)",
"Aport",
"appie 1.1 (www.walhello.com)",
"Apple iPhone v1.1.4 CoreMedia v1.0.0.4A102",
"ArabyBot (compatible; Mozilla/5.0; GoogleBot; FAST Crawler 6.4; http://www.araby.com;)",
"ArachBot",
"Arachnoidea (arachnoidea@euroseek.com)",
"aranhabot",
"ArchitextSpider",
"archive.org_bot",
"Argus/1.1 (Nutch; http://www.simpy.com/bot.html; feedback at simpy dot com)",
"Arikus_Spider",
"Arquivo-web-crawler (compatible; heritrix/1.12.1 +http://arquivo-web.fccn.pt)",
"ASAHA Search Engine Turkey V.001 (http://www.asaha.com/)",
"Asahina-Antenna/1.x",
"Asahina-Antenna/1.x (libhina.pl/x.x ; libtime.pl/x.x)",
"ask.24x.info",
"AskAboutOil/0.06-rcp (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@askaboutoil.com)",
"asked/Nutch-0.8 (web crawler; http://asked.jp; epicurus at gmail dot com)",
"ASPSeek/1.2.5",
"ASPseek/1.2.9d",
"ASPSeek/1.2.x",
"ASPSeek/1.2.xa",
"ASPseek/1.2.xx",
"ASPSeek/1.2.xxpre",
"ASSORT/0.10",
"asterias/2.0",
"AtlocalBot/1.1 +(http://www.atlocal.com/local-web-site-owner.html)",
"Atomic_Email_Hunter/4.0",
"Atomz/1.0",
"atSpider/1.0",
"Attentio/Nutch-0.9-dev (Attentio's beta blog crawler; www.attentio.com; info@attentio.com)",
"AU-MIC/2.0 MMP/2.0",
"AUDIOVOX-SMT5600",
"augurfind",
"augurnfind V-1.x",
"autoemailspider",
"autohttp",
"autowebdir 1.1 (www.autowebdir.com)",
"AV Fetch 1.0",
"Avant Browser (http://www.avantbrowser.com)",
"AVSearch-1.0(peter.turney@nrc.ca)",
"AVSearch-2.0-fusionIdx-14-CompetitorWebSites",
"AVSearch-3.0(AltaVista/AVC)",
"AWeb",
"axadine/ (Axadine Crawler; http://www.axada.de/; )",
"AxmoRobot - Crawling your site for better indexing on www.axmo.com search engine.",
"******* 2.x.x.x",
"BaboomBot/1.x.x (+http://www.baboom.us)",
"BackStreet Browser 3.x",
"BaiduImagespider+(+http://www.baidu.jp/search/s308.html)",
"BaiDuSpider",
"Baiduspider+(+http://help.baidu.jp/system/05.html)",
"Baiduspider+(+http://www.baidu.com/search/spider.htm)",
"Baiduspider+(+http://www.baidu.com/search/spider_jp.html)",
"Balihoo/Nutch-1.0-dev (Crawler for Balihoo.com search engine - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)",
"BanBots/1.2 (spider@banbots.com)",
"Barca/2.0.xxxx",
"BarcaPro/1.4.xxxx",
"BarraHomeCrawler (albertof@barrahome.org)",
"bCentral Billing Post-Process",
"bdcindexer_2.6.2 (research@bdc)",
"BDFetch",
"BDNcentral Crawler v2.3 [en] (http://www.bdncentral.com/robot.html) (X11; I; Linux 2.0.44 i686)",
"BeamMachine/0.5 (dead link remover of www.beammachine.net)",
"beautybot/1.0 (+http://www.uchoose.de/crawler/beautybot/)",
"BebopBot/2.5.1 ( crawler http://www.apassion4jazz.net/bebopbot.html )",
"BeebwareDirectory/v0.01",
"Big Brother (http://pauillac.inria.fr/~fpottier/)",
"Big Fish v1.0",
"BigBrother/1.6e",
"BigCliqueBOT/1.03-dev (bigclicbot; http://www.bigclique.com; bot@bigclique.com)",
"BIGLOTRON (Beta 2;GNU/Linux)",
"Bigsearch.ca/Nutch-x.x-dev (Bigsearch.ca Internet Spider; http://www.bigsearch.ca/; info@enhancededge.com)",
"Bilbo/2.3b-UNIX",
"BilgiBetaBot/0.8-dev (bilgi.com (Beta) ; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
"BilgiBot/1.0(beta) (http://www.bilgi.com/; bilgi at bilgi dot com)",
"billbot wjj@cs.cmu.edu",
"Bitacle bot/1.1",
"Bitacle Robot (V:1.0;) (http://www.bitacle.com)",
"Biyubi/x.x (Sistema Fenix; G11; Familia Toledo; es-mx)",
"BlackBerry7520/4.0.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/5.0.3.3 UP.Link/5.1.2.12 (Google WAP Proxy/1.0)",
"BlackWidow",
"BlackWidow",
"Blaiz-Bee/1.0 (+http://www.blaiz.net)",
"Blaiz-Bee/2.00.8222 (BE Internet Search Engine http://www.rawgrunt.com)",
"Blaiz-Bee/2.00.xxxx (+http://www.blaiz.net)",
"BlitzBOT@tricus.net",
"BlitzBOT@tricus.net (Mozilla compatible)",
"BlockNote.Net",
"BlogBot/1.x",
"BlogBridge 2.13 (http://www.blogbridge.com/)",
"Bloglines Title Fetch/1.0 (http://www.bloglines.com)",
"Bloglines-Images/0.1 (http://www.bloglines.com)",
"Bloglines/3.1 (http://www.bloglines.com)",
"BlogMap (http://www.feedmap.net)",
"Blogpulse (info@blogpulse.com)",
"BlogPulseLive (support@blogpulse.com)",
"BlogSearch/1.x +http://www.icerocket.com/",
"blogsearchbot-pumpkin-3",
"BlogVibeBot-v1.1 (spider@blogvibe.nl)",
"blogWatcher_Spider/0.1 (http://www.lr.pi.titech.ac.jp/blogWatcher/)",
"BlogzIce/1.0 (+http://icerocket.com; rhodes@icerocket.com)",
"BlogzIce/1.0 +http://www.icerocket.com/",
"BloobyBot",
"Bloodhound/Nutch-0.9 (Testing Crawler for Research - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)",
"bluefish 0.6 HTML editor",
"BMCLIENT",
"BMLAUNCHER",
"Bobby/4.0.x RPT-HTTPClient/0.3-3E",
"boitho.com-dc/0.xx (http://www.boitho.com/dcbot.html)",
"boitho.com-robot/1.x",
"boitho.com-robot/1.x (http://www.boitho.com/bot.html)",
"Bookdog/x.x",
"Bookmark Buddy bookmark checker (http://www.bookmarkbuddy.net/)",
"Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/]",
"Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/] (Version 2.0beta)",
"BookmarkBase(2/;http://bookmarkbase.com)",
"Bot mailto:craftbot@yahoo.com",
"BPImageWalker/2.0 (www.bdbrandprotect.com)",
"BravoBrian bstop.bravobrian.it",
"BravoBrian SpiderEngine MarcoPolo",
"BrightCrawler (http://www.brightcloud.com/brightcrawler.asp)",
"BruinBot (+http://webarchive.cs.ucla.edu/bruinbot.html) ",
"BSDSeek/1.0",
"BStop.BravoBrian.it Agent Detector",
"BTbot/0.x (+http://www.btbot.com/btbot.html)",
"BTWebClient/180B(9704)",
"BuildCMS crawler (http://www.buildcms.com/crawler)",
"Bulkfeeds/r1752 (http://bulkfeeds.net/)",
"BullsEye",
"bumblebee@relevare.com",
"BunnySlippers",
"*****FindCrawler/1.1 (crawler.*****find.com; http://crawler.*****find.com; crawler@*****find.com)",
"Buscaplus Robi/1.0 (http://www.buscaplus.com/robi/)",
"BW-C-2.0",
"bwh3_user_agent",
"Cabot/Nutch-0.9 (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)",
"Cabot/Nutch-1.0-dev (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)",
"CamelHttpStream/1.0",
"Cancer Information and Support International;",
"carleson/1.0",
"Carnegie_Mellon_University_Research_WebBOT-->PLEASE READ-->http://www.andrew.cmu.edu/~brgordon/webbot/index.html http://www.andrew.cmu.edu/~brgordon/webbot/index.html",
"Carnegie_Mellon_University_WebCrawler http://www.andrew.cmu.edu/~brgordon/webbot/index.html",
"Catall Spider",
"CazoodleBot/CazoodleBot-0.1 (CazoodleBot Crawler; http://www.cazoodle.com/cazoodlebot; cazoodlebot@cazoodle.com)",
"CCBot/1.0 (+http://www.commoncrawl.org/bot.html)",
"ccubee/x.x",
"CDR/1.7.1 Simulator/0.7(+http://timewe.net) Profile/MIDP-1.0 Configuration/CLDC-1.0",
"CE-Preload",
"CentiverseBot",
"CentiverseBot - investigator",
"CentiverseBot/3.0 (http://www.centiverse-project.net)",
"Ceramic Tile Installation Guide (http://www.floorstransformed.com)",
"cfetch/1.0",
"CFNetwork/x.x",
"cg-eye interactive",
"Charon/1.x (Amiga)",
"Checkbot/1.xx LWP/5.xx",
"CheckLinks/1.x.x",
"CheckUrl",
"CheckWeb",
"Chilkat/1.0.0 (+http://www.chilkatsoft.com/ChilkatHttpUA.asp)",
"China Local Browse 2.6",
"Chitika ContentHit 1.0",
"ChristCRAWLER 2.0",
"CHttpClient by Open Text Corporation",
"CipinetBot (http://www.cipinet.com/bot.html)",
"Cityreview Robot (+http://www.cityreview.org/crawler/)",
"CJ Spider/",
"CJB.NET Proxy",
"ClariaBot/1.0",
"Claymont.com",
"CloakDetect/0.9 (+http://fulltext.seznam.cz/)",
"Clushbot/2.x (+http://www.clush.com/bot.html)",
"Clushbot/3.x-BinaryFury (+http://www.clush.com/bot.html)",
"Clushbot/3.xx-Ajax (+http://www.clush.com/bot.html)",
"Clushbot/3.xx-Hector (+http://www.clush.com/bot.html)",
"Clushbot/3.xx-Peleus (+http://www.clush.com/bot.html)",
"COAST WebMaster Pro/4.x.x.xx (Windows NT)",
"CoBITSProbe",
"Cocoal.icio.us/1.0 (v36) (Mac OS X; http://www.scifihifi.com/cocoalicious)",
"ColdFusion",
"ColdFusion (BookmarkTracker.com)",
"collage.cgi/1.xx",
"combine/0.0",
"Combine/2.0 http://combine.it.lth.se/",
"Combine/3 http://combine.it.lth.se/",
"Combine/x.0",
"Commerce Browser Center",
"complex_network_group/Nutch-0.9-dev (discovering the structure of the world-wide-web; http://cantor.ee.ucla.edu/~networks/crawl; nimakhaj@gmail.com)",
"Computer_and_Automation_Research_Institute_Crawler crawler@ilab.sztaki.hu",
"Comrite/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
"Contact",
"ContactBot/0.2",
"ContentSmartz",
"contype",
"Convera Internet Spider V6.x",
"ConveraCrawler/0.2",
"ConveraCrawler/0.9d (+http://www.authoritativeweb.com/crawl)",
"ConveraMultiMediaCrawler/0.1 (+http://www.authoritativeweb.com/crawl)",
"CoolBot",
"CoralWebPrx/0.1.1x (See http://coralcdn.org/)",
"cosmos/0.8_(robot@xyleme.com)",
"cosmos/0.9_(robot@xyleme.com)",
"CoteoNutchCrawler/Nutch-0.9 (info [at] coteo [dot] com)",
"CougarSearch/0.x (+http://www.cougarsearch.com/faq.shtml)",
"Covac TexAs Arachbot",
"Cowbot-0.1 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)",
"Cowbot-0.1.x (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)",
"CrawlConvera0.1 (CrawlConvera@yahoo.com)",
"Crawler",
"Crawler (cometsearch@cometsystems.com)",
"Crawler admin@crawler.de",
"Crawler V 0.2.x admin@crawler.de",
"crawler@alexa.com",
"CrawlerBoy Pinpoint.com",
"Crawllybot/0.1 (Crawllybot; +http://www.crawlly.com; crawler@crawlly.com)",
"CreativeCommons/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)",
"CrocCrawler vx.3 [en] (http://www.croccrawler.com) (X11; I; Linux 2.0.44 i686)",
"csci_b659/0.13",
"CSE HTML Validator Professional (http://www.htmlvalidator.com/)",
"Cuam Ver0.050bx",
"Cuasarbot/0.9b http://www.cuasar.com/spider_beta/ ",
"curl/7.10.x (i386-redhat-linux-gnu) libcurl/7.10.x OpenSSL/0.9.7a ipv6 zlib/1.1.4",
"curl/7.7.x (i386--freebsd4.3) libcurl 7.7.x (SSL 0.9.6) (ipv6 enabled)",
"curl/7.8 (i686-pc-linux-gnu) libcurl 7.8 (OpenSSL 0.9.6)",
"curl/7.9.x (win32) libcurl 7.9.x",
"CurryGuide SiteScan 1.1",
"Custo x.x (www.netwu.com)",
"Custom Spider www.bisnisseek.com /1.0",
"Cyberdog/2.0 (Macintosh; 68k)",
"CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)",
"CyberSpyder Link Test/2.1.12 (admin@mspennyworth.com)",
"CydralSpider/1.x (Cydral Web Image Search; http://www.cydral.com)",
"CydralSpider/3.0 (Cydral Image Search; http://www.cydral.com)",
"DA 3.5 (www.lidan.com)",
"DA 4.0",
"DA 4.0 (www.downloadaccelerator.com)",
"DA 5.0",
"DA 7.0",
"Dart Communications PowerTCP",
"DataCha0s/2.0",
"DataCha0s/2.0",
"DataFountains/DMOZ Downloader",
"DataFountains/Dmoz Downloader (http://ivia.ucr.edu/useragents.shtml)",
"DataFountains/DMOZ Feature Vector Corpus Creator (http://ivia.ucr.edu/useragents.shtml)",
"DataparkSearch/4.47 (+http://dataparksearch.org/bot)",
"DataparkSearch/4.xx (http://www.dataparksearch.org/)",
"DataSpear/1.0 (Spider; http://www.dataspear.com/spider.html; spider@dataspear.com)",
"DataSpearSpiderBot/0.2 (DataSpear Spider Bot; http://dssb.dataspear.com/bot.html; dssb@dataspear.com)",
"DatenBot( http://www.sicher-durchs-netz.de/bot.html)",
"DaviesBot/1.7 (www.wholeweb.net)",
"daypopbot/0.x",
"dbDig(http://www.prairielandconsulting.com)",
"DBrowse 1.4b",
"DBrowse 1.4d",
"DC-Sakura/x.xx",
"***bot/1.1",
"DDD",
"dds explorer v1.0 beta",
"de.searchengine.comBot 1.2 (http://de.searchengine.com/spider)",
"DeadLinkCheck/0.4.0 libwww-perl/5.xx",
"Deep Link Calculator v1.0",
"deepak-USC/ISI",
"DeepIndex",
"DeepIndex ( http://www.zetbot.com )",
"DeepIndex (www.en.deepindex.com)",
"DeepIndexer.ca",
"del.icio.us-thumbnails/1.0 Mozilla/5.0 (compatible; Konqueror/3.4; FreeBSD) KHTML/3.4.2 (like Gecko)",
"DeleGate/9.0.5-fix1",
"Demo Bot DOT 16b",
"Demo Bot Z 16b",
"Denmex websearch (http://search.denmex.com)",
"Der große BilderSauger 2.00u",
"dev-spider2.searchpsider.com/1.3b",
"DevComponents.com HtmlDocument Object",
"DiaGem/1.1 (http://www.skyrocket.gr.jp/diagem.html)",
"Diamond/x.0",
"DiamondBot",
"Digger/1.0 JDK/1.3.0rc3",
"DigOut4U",
"DIIbot/1.2",
"Dillo/0.8.5-i18n-misc",
"Dillo/0.x.x",
"DISCo Pump x.x",
"disco/Nutch-0.9 (experimental crawler; www.discoveryengine.com; disco-crawl@discoveryengine.com)",
"disco/Nutch-1.0-dev (experimental crawler; www.discoveryengine.com; disco-crawl@discoveryengine.com)",
"DittoSpyder",
"dlman",
"dloader(NaverRobot)/1.0",
"DNSRight.com WebBot Link Ckeck Tool. Report abuse to: dnsr@dnsright.com",
"DoCoMo/1.0/Nxxxi/c10",
"DoCoMo/1.0/Nxxxi/c10/TB",
"DoCoMo/1.0/P502i/c10 (Google CHTML Proxy/1.0)",
"DoCoMo/2.0 P900iV(c100;TB;W24H11) ",
"DoCoMo/2.0 SH902i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)",
"DoCoMo/2.0/SO502i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)",
"DocZilla/1.0 (Windows; U; WinNT4.0; en-US; rv:1.0.0) Gecko/20020804",
"*****bot/experimental",
"DonutP; Windows98SE",
"Doubanbot/1.0 (bot@douban.com http://www.douban.com)",
"Download Demon/3.x.x.x",
"Download Druid 2.x",
"Download Express 1.0",
"Download Master",
"Download Ninja 3.0",
"Download Wonder",
"Download-Tipp Linkcheck (http://download-tipp.de/)",
"Download.exe(1.1) (+http://www.sql-und-xml.de/freeware-tools/)",
"DownloadDirect.1.0",
"Dr.Web (R) online scanner: http://online.drweb.com/",
"Dragonfly File Reader",
"Drecombot/1.0 (http://career.drecom.jp/bot.html)",
"Drupal (+http://drupal.org/)",
"DSurf15a 01",
"DSurf15a 71",
"DSurf15a 81",
"DSurf15a VA",
"DTAAgent",
"dtSearchSpider",
"Dual Proxy",
"DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
"Dumbot(version 0.1 beta - dumbfind.com)",
"Dumbot(version 0.1 beta - http://www.dumbfind.com/dumbot.html)",
"Dumbot(version 0.1 beta)",
"e-sense 1.0 ea(www.vigiltech.com/esensedisclaim.html)",
"e-SocietyRobot(http://www.yama.info.waseda.ac.jp/~yamana/es/)",
"eApolloBot/2.0 (compatible; heritrix/2.0.0-SNAPSHOT-20071024.170148 +http://www.eapollo-opto.com)",
"EARTHCOM.info/1.x [www.earthcom.info]",
"EARTHCOM.info/1.xbeta [www.earthcom.info]",
"EasyDL/3.xx",
"EasyDL/3.xx http://keywen.com/Encyclopedia/Bot",
"EBrowse 1.4b",
"eCatch/3.0",
"EchO!/2.0",
"Educate Search VxB",
"egothor/3.0a (+http://www.xdefine.org/robot.html)",
"EgotoBot/4.8 (+http://www.egoto.com/about.htm)",
"ejupiter.com",
"EldoS TimelyWeb/3.x",
"elfbot/1.0 (+http://www.uchoose.de/crawler/elfbot/)",
"ELinks (0.x.x; Linux 2.4.20 i586; 132x60)",
"ELinks/0.x.x (textmode; NetBSD 1.6.2 sparc; 132x43)",
"EmailSiphon",
"EmailSpider",
"EmailWolf 1.00",
"EmeraldShield.com WebBot",
"EmeraldShield.com WebBot (http://www.emeraldshield.com/webbot.aspx)",
"EMPAS_ROBOT",
"EnaBot/1.x (http://www.enaball.com/crawler.html)",
"endo/1.0 (Mac OS X; ppc i386; http://kula.jp/endo)",
"Enfish Tracker",
"Enterprise_Search/1.0",
"Enterprise_Search/1.0.xxx",
"Enterprise_Search/1.00.xxx;MSSQL (http://www.innerprise.net/es-spider.asp)",
"envolk/1.7 (+http://www.envolk.com/envolkspiderinfo.php)",
"envolk[ITS]spider/1.6(+http://www.envolk.com/envolkspider.html)",
"EroCrawler",
"ES.NET_Crawler/2.0 (http://search.innerprise.net/)",
"eseek-larbin_2.6.2 (crawler@exactseek.com)",
"ESISmartSpider",
"eStyleSearch 4 (compatible; MSIE 6.0; Windows NT 5.0)",
"ESurf15a 15",
"EuripBot/0.x (+http://www.eurip.com) GetFile",
"EuripBot/0.x (+http://www.eurip.com) GetRobots",
"EuripBot/0.x (+http://www.eurip.com) PreCheck",
"Eurobot/1.0 (http://www.ayell.eu)",
"EvaalSE - bot@evaal.com",
"eventax/1.3 (eventax; http://www.eventax.de/; info@eventax.de)",
"Everest-Vulcan Inc./0.1 (R&D project; host=e-1-24; http://everest.vulcan.com/crawlerhelp)",
"Everest-Vulcan Inc./0.1 (R&D project; http://everest.vulcan.com/crawlerhelp)",
"Exabot-Images/1.0",
"Exabot-Test/1.0",
"Exabot/2.0",
"Exabot/3.0",
"ExactSearch",
"ExactSeek Crawler/0.1",
"exactseek-crawler-2.63 (crawler@exactseek.com)",
"exactseek-pagereaper-2.63 (crawler@exactseek.com)",
"exactseek.com",
"Exalead NG/MimeLive Client (convert/http/0.120)",
"Excalibur Internet Spider V6.5.4",
"Execrawl/1.0 (Execrawl; http://www.execrawl.com/; bot@execrawl.com)",
"exooba crawler/exooba crawler (crawler for exooba.com; http://www.exooba.com/; info at exooba dot com)",
"exooba/exooba crawler (exooba; exooba)",
"ExperimentalHenrytheMiragoRobot",
"Expired Domain Sleuth",
"Express WebPictures (www.express-soft.com)",
"*********Pro",
"Extreme Picture Finder",
"EyeCatcher (Download-tipp.de)/1.0",
"Factbot 1.09 (see http://www.factbites.com/webmasters.php)",
"factbot : http://www.factbites.com/robots",
"FaEdit/2.0.x",
"FairAd Client",
"FANGCrawl/0.01",
"FARK.com link verifier",
"Fast Crawler Gold Edition",
"FAST Enterprise Crawler 6 (Experimental)",
"FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/",
"FAST Enterprise Crawler 6 used by Cobra Development (admin@fastsearch.com)",
"FAST Enterprise Crawler 6 used by Comperio AS (sts@comperio.no)",
"FAST Enterprise Crawler 6 used by FAST (FAST)",
"FAST Enterprise Crawler 6 used by Pages Jaunes (pvincent@pagesjaunes.fr)",
"FAST Enterprise Crawler 6 used by Sensis.com.au Web Crawler (search_comments\at\sensis\dot\com\dot\au)",
"FAST Enterprise Crawler 6 used by Singapore Press Holdings (crawler@sphsearch.sg)",
"FAST Enterprise Crawler 6 used by WWU (wardi@uni-muenster.de)",
"FAST Enterprise Crawler/6 (www.fastsearch.com)",
"FAST Enterprise Crawler/6.4 (helpdesk at fast.no)",
"FAST FirstPage retriever (compatible; MSIE 5.5; Mozilla/4.0)",
"FAST MetaWeb Crawler (helpdesk at fastsearch dot com)",
"Fast PartnerSite Crawler",
"FAST-WebCrawler/2.2.10 (Multimedia Search) (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)",
"FAST-WebCrawler/2.2.6 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)",
"FAST-WebCrawler/2.2.7 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no",
"FAST-WebCrawler/2.2.8 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no",
"FAST-WebCrawler/3.2 test",
"FAST-WebCrawler/3.3 (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)",
"FAST-WebCrawler/3.4/Nirvana (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)",
"FAST-WebCrawler/3.4/PartnerSite (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)",
"FAST-WebCrawler/3.5 (atw-crawler at fast dot no; http://fast.no/support.php?c=faqs/crawler)",
"FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
"FAST-WebCrawler/3.6/FirstPage (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)",
"FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
"FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)",
"FAST-WebCrawler/3.8 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
"FAST-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
"FAST-WebCrawler/3.x Multimedia",
"FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no)",
"fastbot crawler beta 2.0 (+http://www.fastbot.de)",
"FastBug http://www.ay-up.com",
"FastCrawler 3.0.1 (crawler@1klik.dk)",
"FastSearch Web Crawler for Verizon SuperPages (kevin.watters@fastsearch.com)",
"Favcollector/2.0 (info@favcollector.com http://www.favcollector.com/)",
"FavIconizer",
"favo.eu crawler/0.6 (http://www.favo.eu)",
"FavOrg",
"Favorites Checking (http://campulka.net)",
"Favorites Sweeper v.2.03",
"Faxobot/1.0",
"FDM 1.x",
"FDM 2.x",
"Feed Seeker Bot (RSS Feed Seeker http://www.MyNewFavoriteThing.com/fsb.php)",
"Feed24.com",
"Feed::Find/0.0x",
"Feedable/0.1 (compatible; MSIE 6.0; Windows NT 5.1)",
"FeedChecker/0.01",
"FeedDemon/2.7 (http://www.newsgator.com/; Microsoft Windows XP)",
"Feedfetcher-Google-iGoogleGadgets; (+http://www.google.com/feedfetcher.html)",
"Feedfetcher-Google; (+http://www.google.com/feedfetcher.html)",
"FeedForAll rss2html.php v2",
"FeedHub FeedDiscovery/1.0 (http://www.feedhub.com)",
"FeedHub MetaDataFetcher/1.0 (http://www.feedhub.com)",
"Feedjit Favicon Crawler 1.0",
"Feedreader 3.xx (Powered by Newsbrain)",
"Feedshow/x.0 (http://www.feedshow.com; 1 subscriber)",
"FeedshowOnline (http://www.feedshow.com)",
"FeedZcollector v1.x (Platinum) http://www.feeds4all.com/feedzcollector",
"Felix - Mixcat Crawler (+http://mixcat.com)",
"fetch libfetch/2.0",
"FFC Trap Door Spider",
"Filangy/0.01-beta (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)",
"Filangy/1.0x (Filangy; http://www.filangy.com/filangyinfo.jsp?inc=robots.jsp; filangy-agent@filangy.com)",
"Filangy/1.0x (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)",
"fileboost.net/1.0 (+http://www.fileboost.net)",
"FileHound x.x",
"Filtrbox/1.0",
"FindAnISP.com_ISP_Finder_v99a",
"Findexa Crawler (http://www.findexa.no/gulesider/article26548.ece)",
"findlinks/x.xxx (+http://wortschatz.uni-leipzig.de/findlinks/) ",
"FineBot",
"Finjan-prefetch",
"Firefly/1.0",
"Firefly/1.0 (compatible; Mozilla 4.0; MSIE 5.5)",
"Firefox (kastaneta03@hotmail.com)",
"Firefox_1.0.6 (kasparek@naparek.cz)",
"FirstGov.gov Search - POC:firstgov.webmasters@gsa.gov",
"firstsbot",
"Flapbot/0.7.2 (Flaptor Crawler; http://www.flaptor.com; crawler at flaptor period com)",
"FlashGet",
"FLATARTS_FAVICO",
"Flexum spider",
"Flexum/2.0",
"FlickBot 2.0 RPT-HTTPClient/0.3-3",
"flunky",
"fly/6.01 libwww/4.0D",
"flyindex.net 1.0/http://www.flyindex.net",
"FnooleBot/2.5.2 (+http://www.fnoole.com/addurl.html)",
"FocusedSampler/1.0",
"Folkd.com Spider/0.1 beta 1 (www.folkd.com)",
"Fooky.com/ScorpionBot/ScoutOut; http://www.fooky.com/scorpionbots",
"Francis/1.0 (francis@neomo.de http://www.neomo.de/)",
"Franklin Locator 1.8",
"free-downloads.net download-link validator /0.1",
"FreeFind.com-SiteSearchEngine/1.0 (http://freefind.com; spiderinfo@freefind.com)",
"Frelicbot/1.0 +http://www.frelic.com/",
"FreshDownload/x.xx",
"FreshNotes crawler< report problems to crawler-at-freshnotes-dot-com",
"FSurf15a 01",
"Full Web Bot 0416B",
"Full Web Bot 0516B",
"Full Web Bot 2816B",
"FuseBulb.Com",
"FyberSpider (+http://www.fybersearch.com/fyberspider.php)",
"Gagglebot",
"GAIS Robot/1.0B2",
"Gaisbot/3.0 (indexer@gais.cs.ccu.edu.tw; http://gais.cs.ccu.edu.tw/robot.php)",
"Gaisbot/3.0+(robot06@gais.cs.ccu.edu.tw;+http://gais.cs.ccu.edu.tw/robot.php)",
"GalaxyBot/1.0 (http://www.galaxy.com/galaxybot.html)",
"Gallent Search Spider v1.4 Robot 2 (http://robot.GallentSearch.com)",
"gamekitbot/1.0 (+http://www.uchoose.de/crawler/gamekitbot/)",
"Gamespy_Arcade",
"GammaSpider/1.0",
"gazz/x.x (gazz@nttrd.com)",
"geckobot",
"Generic Mobile Phone (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
"generic_crawler/01.0217/",
"genieBot (http://64.5.245.11/faq/faq.html)",
"geniebot wgao@genieknows.com",
"GeoBot/1.0",
"GeonaBot 1.x; http://www.geona.com/",
"geourl/2.0b2",
"GeoURLBot 1.0 (http://geourl.org)",
"GetBot",
"GetRight/3.x.x",
"GetRight/4.5xx",
"GetRight/4.x",
"GetRight/4.x[a-e]",
"GetRight/6.1 (Pro)",
"GetRightPro/6.0beta2",
"GetWeb/0.1 libwww-perl/5.16",
"GhostRouteHunter/20021130 (https://www.sixxs.net/tools/grh/; info@sixxs.net)",
"gigabaz/3.1x (baz@gigabaz.com; http://gigabaz.com/gigabaz/)",
"Gigabot/2.0 (gigablast.com)",
"Gigabot/2.0/gigablast.com/spider.html",
"Gigabot/2.0; http://www.gigablast.com/spider.html",
"Gigabot/2.0att",
"Gigabot/3.0 (http://www.gigablast.com/spider.html)",
"Gigabot/x.0",
"GigabotSiteSearch/2.0 (sitesearch.gigablast.com)",
"GNODSPIDER (www.gnod.net)",
"Go!Zilla 3.x (www.gozilla.com)",
"Go!Zilla/4.x.x.xx",
"Go-Ahead-Got-It/1.1",
"Goblin/0.9 (http://www.goguides.org/)",
"Goblin/0.9.x (http://www.goguides.org/goblin-info.html)",
"GoForIt.com",
"GOFORITBOT ( http://www.goforit.com/about/ )",
"GoGuides.Org Link Check",
"GoldenFeed Spider 1.0 (http://www.goldenfeed.com)",
"Goldfire Server",
"gonzo1[P] +http://www.suchen.de/popups/faq.jsp",
"gonzo2[P] +http://www.suchen.de/faq.html",
"Goofer/0.2",
"Google Talk",
"googlebot (larbin2.6.0@unspecified.mail)",
"Googlebot-Image/1.0",
"Googlebot-Image/1.0 ( http://www.googlebot.com/bot.html)",
"Googlebot/2.1 ( http://www.google.com/bot.html)",
"Googlebot/2.1 ( http://www.googlebot.com/bot.html)",
"Googlebot/Test ( http://www.googlebot.com/bot.html)",
"GrapeFX/0.3 libwww/5.4.0",
"great-plains-web-spider/flatlandbot (Flatland Industries Web Spider; http://www.flatlandindustries.com/flatlandbot.php; jason@flatlandindustries.com)",
"GreatNews/1.0",
"GreenBrowser",
"gridwell (http://search.gridwell.com)",
"GrigorBot 0.8 (http://www.grigor.biz/bot.html)",
"Gromit/1.0",
"grub crawler(http://www.grub.org)",
"grub-client",
"gsa-crawler (Enterprise; GID-01422; jplastiras@google.com)",
"gsa-crawler (Enterprise; GID-01742;gsatesting@rediffmail.com)",
"gsa-crawler (Enterprise; GIX-02057; dm@enhesa.com)",
"gsa-crawler (Enterprise; GIX-03519; cknuetter@stubhub.com)",
"gsa-crawler (Enterprise; GIX-0xxxx; enterprise-training@google.com)",
"GSiteCrawler/v1.xx rev. xxx (http://gsitecrawler.com/)",
"Guestbook Auto Submitter",
"*****ver/1.3",
"Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)",
"Gungho/0.08004 (http://code.google.com/p/gungho-crawler/wiki/Index)",
"GurujiBot/1.0 (+http://www.guruji.com/WebmasterFAQ.html)",
"GurujiImageBot/1.0 (+http://www.guruji.com/en/WebmasterFAQ.html)",
"Haier-T10C/1.0 iPanel/2.0 WAP2.0 (compatible; UP.Browser/6.2.2.4; UPG1; UP/4.0; Embedded)",
"HappyFunBot/1.1",
"Harvest-NG/1.0.2",
"Haste/0.12 (HOME: http://haste.kytoon.com/)",
"Hatena Antenna/0.4 (http://a.hatena.ne.jp/help#robot)",
"Hatena Mobile Gateway/1.0",
"Hatena Pagetitle Agent/1.0",
"Hatena RSS/0.3 (http://r.hatena.ne.jp)",
"HatenaScreenshot/1.0 (checker)",
"hbtronix.spider.2 -- http://hbtronix.de/spider.php",
"HeinrichderMiragoRobot",
"HeinrichderMiragoRobot (http://www.miragorobot.com/scripts/deinfo.asp)",
"Helix/1.x ( http://www.sitesearch.ca/helix/)",
"HenriLeRobotMirago (http://www.miragorobot.com/scripts/frinfo.asp)",
"HenrytheMiragoRobot",
"HenryTheMiragoRobot (http://www.miragorobot.com/scripts/mrinfo.asp)",
"hgrepurl/1.0",
"Hi! I'm CsCrawler my homepage: http://www.kde.cs.uni-kassel.de/lehre/ss2005/googlespam/crawler.html RPT-HTTPClient/0.3-3",
"HiDownload",
"Hippias/0.9 Beta",
"HitList",
"Hitwise Spider v1.0 http://www.hitwise.com",
"HLoader",
"holmes/3.11 (http://morfeo.centrum.cz/bot)",
"holmes/3.9 (onet.pl)",
"holmes/3.xx (OnetSzukaj/5.0; +http://szukaj.onet.pl)",
"holmes/x.x",
"HolmesBot (http://holmes.ge)",
"HomePageSearch(hpsearch.uni-trier.de)",
"Homerbot: www.homerweb.com",
"Honda-Search/0.7.2 (Nutch; http://lucene.apache.org/nutch/bot.html; search@honda-search.com)",
"HooWWWer/2.1.3 (debugging run) (+http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-info<at>hiit.fi)",
"HooWWWer/2.1.x ( http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-info<at>hiit.fi)",
"HotJava/1.0.1/JRE1.1.x",
"Hotzonu/x.0",
"HPL/Nutch-0.9 -",
"htdig/3.1.6 (http://computerorgs.com)",
"htdig/3.1.6 (unconfigured@htdig.searchengine.maintainer)",
"htdig/3.1.x (root@localhost)",
"Html Link Validator (www.lithopssoft.com)",
"HTML2JPG Enterprise",
"HTMLParser/1.x",
"HTTP Retriever",
"http://Anonymouse.org/ (Unix)",
"http://Ask.24x.Info/ (http://narres.it/)",
"http://OzySoftware.com/Index.html",
"http://www.almaden.ibm.com/cs/crawler",
"http://www.almaden.ibm.com/cs/crawler [rc1.wf.ibm.com]",
"http://www.almaden.ibm.com/cs/crawler [wf216]",
"http://www.istarthere.com_spider@istarthere.com",
"http://www.monogol.de",
"http://www.trendtech.dk/spider.asp)",
"HTTP::Lite/2.x.x",
"HTTPEyes",
"HTTPResume v. 1.x",
"httpunit/1.5",
"httpunit/1.x",
"Hybrid/1.2 [en] (OS Independent)",
"HyperEstraier/1.x.xx",
"i1searchbot/2.0 (i1search web crawler; http://www.i1search.com; crawler@i1search.com)",
"IAArchiver-1.0",
"iaskspider",
"iaskspider2 (iask@staff.sina.com.cn)",
"ia_archiver",
"ia_archiver-web.archive.org",
"ia_archiver/1.6",
"IBrowse/2.2 (AmigaOS 3.5)",
"IBrowse/2.2 (Windows 3.1)",
"iCab/2.5.2 (Macintosh; I; PPC)",
"ICC-Crawler(Mozilla-compatible; http://kc.nict.go.jp/icc/crawl.html; icc-crawl(at)ml(dot)nict(dot)go(dot)jp)",
"ICC-Crawler(Mozilla-compatible;http://kc.nict.go.jp/icc/crawl.html;icc-crawl-contact(at)ml(dot)nict(dot)go(dot)jp)",
"iCCrawler (http://www.iccenter.net)",
"ICCrawler - ICjobs (http://www.icjobs.de/bot.htm)",
"ICE Browser/5.05 (Java 1.4.0; Windows 2000 5.0 x86)",
"ichiro/x.0 (http://help.goo.ne.jp/door/crawler.html)",
"ichiro/x.0 (ichiro@nttr.co.jp)",
"IconSurf/2.0 favicon finder (see http://iconsurf.com/robot.html)",
"IconSurf/2.0 favicon monitor (see http://iconsurf.com/robot.html)",
"ICOO Loader v.x.x.x",
"ICRA_label_spider/x.0",
"icsbot-0.1",
"IDA",
"ideare - SignSite/1.x",
"IEFav172Free",
"iFeed.jp/2.0 (www.psychedelix.com/agents/agents.rss; 0 subscribers)",
"igdeSpyder (compatible; igde.ru; +http://igde.ru/doc/tech.html)",
"iGetter/1.x (Macintosh;G;PPC)",
"iGetter/2 (Macintosh; U; PPC Mac OS X; en)",
"IIITBOT/1.1 (Indian Language Web Search Engine; http://webkhoj.iiit.net; pvvpr at iiit dot ac dot in)",
"ilial/Nutch-0.9-dev",
"IlseBot/1.x",
"IlTrovatore-Setaccio ( http://www.iltrovatore.it)",
"Iltrovatore-Setaccio/0.3-dev (Indexing; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)",
"IlTrovatore-Setaccio/1.2 ( http://www.iltrovatore.it/aiuto/faq.html)",
"Iltrovatore-Setaccio/1.2 (It-bot; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)",
"iltrovatore-setaccio/1.2-dev (spidering; http://www.iltrovatore.it/aiuto/.....)",
"IlTrovatore/1.2 (IlTrovatore; http://www.iltrovatore.it/bot.html; bot@iltrovatore.it)",
"ImageVisu/v4.x.x",
"ImageWalker/2.0 (www.bdbrandprotect.com)",
"Incutio HttpClient v0.x",
"IncyWincy data gatherer(webmaster@loopimprovements.com",
"IncyWincy page crawler(webmaster@loopimprovements.com",
"IncyWincy(http://www.look.com)",
"IncyWincy(http://www.loopimprovements.com/robot.html)",
"IncyWincy/2.1(loopimprovements.com/robot.html)",
"IndexTheWeb.com Crawler7",
"Industry Program 1.0.x",
"Inet library",
"InetURL/1.0",
"info@pubblisito.com- (http://www.pubblisito.com) il Sud dei Motori di Ricerca",
"infoConveraCrawler/0.8 ( http://www.authoritativeweb.com/crawl)",
"InfoFly/1.0 (http://www.versions-project.org/)",
"InfoLink/1.x",
"INFOMINE/8.0 Adders",
"INFOMINE/8.0 RemoteServices",
"INFOMINE/8.0 VLCrawler (http://infomine.ucr.edu/useragents)",
"InfoNaviRobot(F107)",
"InfoSeek Sidewinder/0.9",
"InfoSeek Sidewinder/1.0A",
"InfoSeek Sidewinder/1.1A",
"Infoseek SideWinder/1.45 (Compatible; MSIE 10.0; UNIX)",
"Infoseek SideWinder/2.0B (Linux 2.4 i686)",
"INGRID/3.0 MT (webcrawler@NOSPAMexperimental.net; http://webmaster.ilse.nl/jsp/webmaster.jsp)",
"Inktomi Search",
"InnerpriseBot/1.0 (http://www.innerprise.com/)",
"Insitor.com search and find world wide!",
"Insitornaut",
"InstallShield DigitalWizard",
"integrity/1.6",
"Intelix/0.x (cs; http://www.microton.cz/intelix/; microton@@microton.cz)",
"Interarchy/x.x.x (InterarchyCrawler)",
"Internet Ninja x.0",
"InternetArchive/0.8-dev(Nutch;http://lucene.apache.org/nutch/bot.html;nutch-agent@lucene.apache",
"InternetLinkAgent/3.1",
"InternetSeer.com",
"intraVnews/1.x",
"IP*Works! V5 HTTP/S Component - by /n software - www.nsoftware.com",
"http://www.ip2location.com",
"IP2MapBot/1.1 <a href=http://www.ip2map.com>http://www.ip2map.com</a>",
"IPiumBot laurion(dot)com",
"IpselonBot/0.xx-beta (Ipselon; http://www.ipselon.com; ipselonbot@ipselon.com)",
"Iria/1.xxa",
"IRLbot/1.0 ( http://irl.cs.tamu.edu/crawler)",
"IRLbot/3.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler/)",
"IrssiUrlLog/0.2",
"Irvine/1.x.x",
"ISC Systems iRc Search 2.1",
"iSiloX/4.xx Windows/32",
"isurf (tszhu@canada.com)",
"iTunes/x.x.x",
"IUPUI Research Bot v 1.9a",
"iVia Page Fetcher (http://ivia.ucr.edu/useragents.shtml)",
"iVia/4.0 CanonizeUrl (http://infomine.ucr.edu/iVia/useragents.shtml",
"IWAgent/ 1.0 - www.brandprotect.com",
"J-PHONE/3.0/J-SH07",
"Jabot/6.x (http://odin.ingrid.org/)",
"Jabot/7.x.x (http://odin.ingrid.org/)",
"Jack",
"Jakarta Commons-HttpClient/2.0xxx",
"Jakarta Commons-HttpClient/3.0-rcx",
"Jambot/0.1.x (Jambot; http://www.jambot.com/blog; crawler@jambot.com)",
"Jambot/0.2.1 (Jambot; http://www.jambot.com/blog/static.php?page=webmaster-robot; crawler@jambot.com)",
"Java 1.1",
"Java/1.4.1_01",
"Java1.0.21.0",
"Java1.1.xx.x",
"Java1.3.0rc1",
"Java1.3.x",
"Java1.4.0",
"Jayde Crawler. http://www.jayde.com",
"JBH Agent 2.0",
"jBrowser/J2ME Profile/MIDP-1.0 Configuration/CLDC-1.0 (Google WAP Proxy/1.0)",
"JCheckLinks/0.1 RPT-HTTPClient/0.3-1",
"JDK/1.1",
"Jeode/1.x.x",
"Jetbot/1.0",
"JetBrains Omea Reader 1.0.x (http://www.jetbrains.com/omea_reader/)",
"JetBrains Omea Reader 2.0 Release Candidate 1 (http://www.jetbrains.com/omea_reader/)",
"JetCar",
"Jigsaw/2.2.0 W3C_CSS_Validator_JFouffa/2.0",
"JoBo/1.x (http://www.matuschek.net/jobo.html)",
"JoBo/@JOBO_VERSION@(http://www.matuschek.net/jobo.html)",
"JobSpider_BA/1.1",
"JOC Web Spider",
"JordoMedia/1.0 RSS File Reader (http://www.jordomedia.com)",
"Journster [alpha] (http://journster.com/)",
"Journster.com RSS/Atom aggregator 0.5 (http://www.journster.com/bot.phtml)",
"JRTS Check Favorites Utility",
"JRTwine Software Check Favorites Utility",
"Jyxobot/x",
"K-Meleon/0.6 (Windows; U; Windows NT 5.1; en-US; rv:0.9.5) Gecko/20011011",
"k2spider",
"KAIST AITrc Crawler",
"KakleBot - www.kakle.com/0.1 (KakleBot - www.kakle.com; http:// www.kakle.com/bot.html; support@kakle.com)",
"kalooga/kalooga-4.0-dev-datahouse (Kalooga; http://www.kalooga.com; info@kalooga.com)",
"kalooga/KaloogaBot (Kalooga; http://www.kalooga.com/info.html?page=crawler; crawler@kalooga.com)",
"Kapere (http://www.kapere.com)",
"Kazehakase/0.x.x.[x]",
"KDDI-SN22 UP.Browser/6.0.7 (GUI) MMP/1.1 (Google WAP Proxy/1.0)",
"Kenjin Spider",
"Kevin http://dznet.com/kevin/",
"Kevin http://websitealert.net/kevin/",
"KE_1.0/2.0 libwww/5.2.8",
"KFSW-Bot (Version: 1.01 powered by KFSW www.kfsw.de)",
"kinja-imagebot (http://www.kinja.com/)",
"kinjabot (http://www.kinja.com)",
"KIT-Fireball/2.0",
"KIT-Fireball/2.0 (compatible; Mozilla 4.0; MSIE 5.5)",
"Klondike/1.50 (WSP Win32) (Google WAP Proxy/1.0)",
"KnowItAll(knowitall@cs.washington.edu)",
"Knowledge.com/0.x",
"Kontiki Client x.xx",
"KSbot/1.0 (KnowledgeStorm crawler; http://www.knowledgestorm.com/resources/content/crawler/index.html; crawleradmin@knowledgestorm.com)",
"kuloko-bot/0.x",
"kulokobot www.kuloko.com kuloko@backweave.com",
"kulturarw3/0.1",
"KummHttp/1.1 (compatible; KummClient; Linux rulez)",
"Lachesis",
"lanshanbot/1.0",
"lanshanbot/1.0 (+http://search.msn.com/msnbot.htm)",
"LapozzBot/1.4 ( http://robot.lapozz.com)",
"LapozzBot/1.5 (+http://robot.lapozz.hu)",
"larbin (samualt9@bigfoot.com)",
"LARBIN-EXPERIMENTAL (efp@gmx.net)",
"larbin_2.1.1 larbin2.1.1@somewhere.com",
"larbin_2.2.0 (crawl@compete.com)",
"larbin_2.2.1_de_Viennot (Laurent.Viennot@inria.fr)",
"larbin_2.2.2 (sugayama@lab7.kuis.kyoto-u.ac.jp)",
"larbin_2.2.2_guillaume (guillaume@liafa.jussieu.fr)",
"larbin_2.6.0 (larbin2.6.0@unspecified.mail)",
"larbin_2.6.1 (larbin2.6.1@unspecified.mail)",
"larbin_2.6.2 (hamasaki@grad.nii.ac.jp)",
"larbin_2.6.2 (larbin2.6.2@unspecified.mail)",
"larbin_2.6.2 (listonATccDOTgatechDOTedu)",
"larbin_2.6.2 (pimenas@systems.tuc.gr)",
"larbin_2.6.2 (tom@lemurconsulting.com)",
"larbin_2.6.2 (vitalbox1@hotmail.com)",
"larbin_2.6.3 (ltaa_web_crawler@groupes.epfl.ch)",
"larbin_2.6.3 (wgao@genieknows.com)",
"larbin_2.6.3_for_(http://cosco.hiit.fi/search/) tsilande@hiit.fi",
"larbin_2.6_basileocaml (basile.starynkevitch@cea.fr)",
"larbin_devel (http://pauillac.inria.fr/~ailleret/prog/larbin/)",
"lawinfo-crawler/Nutch-0.9-dev (Crawler for lawinfo.com pages; http://www.lawinfo.com; webmaster@lawinfo.com)",
"lc/$ROADS::Version libwww-perl/5.00",
"lcabotAccept: */*",
"LeapTag/0.8.1.beta081.r3750 (compatible; Mozilla 4.0; MSIE 5.5; robot@yoriwa.com)",
"LECodeChecker/3.0 libgetdoc/1.0",
"LeechGet 200x (www.leechget.de)",
"LEIA/2.90",
"LEIA/3.01pr (LEIAcrawler; [SNIP])",
"LetsCrawl.com/1.0 +http://letscrawl.com/",
"LexiBot/1.00",
"LG/U8138/v1.0",
"Libby_1.1/libwww-perl/5.47",
"libcurl-agent/1.0",
"LibertyW (+http://www.lw01.com)",
"libWeb/clsHTTP -- hiongun@kt.co.kr",
"libwww-perl/5.41",
"libwww-perl/5.45",
"libwww-perl/5.48",
"libwww-perl/5.50",
"libwww-perl/5.52 FP/2.1",
"libwww-perl/5.52 FP/4.0",
"libwww-perl/5.53",
"libwww-perl/5.63",
"libwww-perl/5.64",
"libwww-perl/5.65",
"libwww-perl/5.800",
"libwww/5.3.2",
"Liferea/0.x.x (Linux; en_US.UTF-8; http://liferea.sf.net/)",
"Liferea/1.x.x (Linux; es_ES.UTF-8; http://liferea.sf.net/)",
"LightningDownload/1.0beta2",
"LightningDownload/1.x.x",
"LightningDownload/1.x.x [Accelerated x]",
"LijitSpider/Nutch-0.9 (Reports crawler; http://www.lijit.com/; info(a)lijit(d)com)",
"Lincoln State Web Browser",
"Link Valet Online 1.x",
"LinkAlarm/2.x",
"linkbot",
"Linkbot",
"Linkbot x.0",
"LinkCheck (linkcheck@inter7.com http://www.inter7.com/linkcheck)",
"LinkLint-checkonly/2.x.x",
"LinkLint-spider/2.x.x",
"linknzbot",
"LinkPimpin v1.0",
"LinkProver 2.1",
"Links (0.9x; Linux 2.4.7-10 i686)",
"Links (0.9xpre12; Linux 2.2.14-5.0 i686; 80x24)",
"Links (2.xpre7; Linux 2.4.18 i586; x)",
"Links - http://gossamer-threads.com/scripts/links/",
"Links 2.0 (http://gossamer-threads.com/scripts/links/)",
"Links SQL (http://gossamer-threads.com/scripts/links-sql/)",
"LinkScan/11.0beta2 UnixShareware robot from Elsop.com (used by Indiafocus/Indiainfo)",
"LinkScan/9.0g Unix",
"LinkScan/x.x Unix",
"LinksManager.com (http://linksmanager.com/linkchecker.html)",
"LinkSonar/1.35",
"LinkSweeper/1.x",
"LinkWalker",
"link_check3.plx libwww-perl/5.65",
"ListBidBot (freelance job spider http://listbid.com)<a href=http://listbid.com>Freelance</a>",
"LiveTrans/Nutch-0.9 (maintainer: cobain at iis dot sinica dot edu dot tw; http://wkd.iis.sinica.edu.tw/LiveTrans/)",
"Llaut/1.0 (http://mnm.uib.es/~gallir/llaut/bot.html)",
"LMQueueBot/0.2",
"lmspider (lmspider@scansoft.com)",
"LNSpiderguy",
"LocalBot/1.0 ( http://www.localbot.co.uk/)",
"LocalcomBot/1.2.x ( http://www.local.com/bot.htm)",
"Lockstep Spider/1.0",
"Look.com",
"Lotus-Notes/4.5 ( Windows-NT )",
"LotusDiscovery/x.0 (compatible; Mozilla 4.0; MSIE 4.01; Windows NT)",
"Lovel as 1.0 ( +http://www.everatom.com)",
"LTI/LemurProject Nutch Spider/Nutch-1.0-dev (lti crawler for CMU; http://www.lti.cs.cmu.edu; changkuk at cmu dot edu)",
"LTI/LemurProject Nutch Spider/Nutch-1.0-dev (Research spider using Nutch; http://www.lemurproject.org; mhoy@cs.cmu.edu)",
"luchs.at URL checker",
"Lunascape",
"lwp-trivial/1.32",
"lwp-trivial/1.34",
"lwp-trivial/1.34",
"lwp-trivial/1.35",
"lwp-trivial/1.35",
"LWP::Simple/5.22",
"LWP::Simple/5.36",
"LWP::Simple/5.48",
"LWP::Simple/5.50",
"LWP::Simple/5.51",
"LWP::Simple/5.53",
"LWP::Simple/5.63",
"LWP::Simple/5.803",
"Lycos_Spider_(modspider)",
"Lycos_Spider_(T-Rex)",
"Lynx/2-4-2 (Bobcat/0.5 [DOS] Jp Beta04)",
"Lynx/2.6 libwww-FM/2.14",
"Lynx/2.8 (;http://seebot.org)",
"Lynx/2.8.3dev.9 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6",
"Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6c (human-guided@lerly.net)",
"Mac Finder 1.0.xx",
"Mackster( http://www.ukwizz.com )",
"Mag-Net",
"MagicWML/1.0 (forcewml)",
"MagpieRSS/0.7x (+http://magpierss.sf.net)",
"Mahiti.Com/Mahiti Crawler-1.0 (Mahiti.Com; http://mahiti.com ; mahiti.com)",
"Mail.Ru/1.0",
"mailto:webcraft@bea.com",
"*******/1.0 ( http://www.sli-systems.com/)",
"MantraAgent",
"MapoftheInternet.com ( http://MapoftheInternet.com)",
"Mariner/5.1b [de] (Win95; I ;Kolibri gncwebbot)",
"Marketwave Hit List",
"MARTINI",
"Martini",
"Marvin v0.3",
"MaSagool/1.0 (MaSagool; http://sagool.jp/; info@sagool.jp)",
"Mass Downloader 2.x",
"MasterSeek",
"Mata Hari/2.00 ",
"Matrix S.p.A. - FAST Enterprise Crawler 6 (Unknown admin e-mail address)",
"maxomobot/dev-20051201 (maxomo; http://67.102.134.34:4047/MAXOMO/MAXOMObot.html; maxomobot@maxomo.com)",
"McBot/5.001 (windows; U; NT4.0; en-us)",
"MDbot/1.0 (+http://www.megadownload.net/bot.html)",
"Media Player Classic",
"MediaCrawler-1.0 (Experimental)",
"Mediapartners-Google/2.1 ( http://www.googlebot.com/bot.html)",
"MediaSearch/0.1",
"MegaSheep v1.0 (www.searchuk.com internet sheep)",
"Megite2.0 (http://www.megite.com)",
"Mercator-1.x",
"Mercator-2.0",
"Mercator-Scrub-1.1",
"Metaeuro Web Crawler/0.2 (MetaEuro Web Search Clustering Engine; http://www.metaeuro.com; crawler at metaeuro dot com)",
"MetaGer-LinkChecker",
"MetagerBot/0.8-dev (MetagerBot; http://metager.de; )",
"MetaGer_PreChecker0.1",
"MetaProducts Download Express/1.x",
"Metaspinner/0.01 (Metaspinner; http://www.meta-spinner.de/; support@meta-spinner.de/)",
"metatagsdir/0.7 (+http://metatagsdir.com/directory/)",
"MFC Foundation Class Library 4.0",
"MFC_Tear_Sample",
"MFHttpScan",
"MicroBaz",
"Microsoft Data Access Internet Publishing Provider Cache Manager",
"Microsoft Data Access Internet Publishing Provider DAV",
"Microsoft Data Access Internet Publishing Provider Protocol Discovery",
"Microsoft Data Access Internet Publishing Provider Protocol Discovery",
"Microsoft Log Parser 2.2",
"Microsoft Small Business Indexer",
"Microsoft URL Control - 6.00.8xxx",
"MicrosoftPrototypeCrawler (How's my crawling? mailto:newbiecrawler@hotmail.com)",
"Mister Pix II 2.02a",
"Mister PiX version.dll",
"Misterbot-Nutch/0.7.1 (Misterbot-Nutch; http://www.misterbot.fr; admin@misterbot.fr)",
"Miva (AlgoFeedback@miva.com)",
"MJ12bot/vx.x.x (http://majestic12.co.uk/bot.php?+)",
"MJ12bot/vx.x.x (http://www.majestic12.co.uk/projects/dsearch/mj12bot.php)",
"MJBot (SEO assessment)",
"MLBot (www.metadatalabs.com)",
"MnogoSearch/3.2.xx",
"moget/x.x (moget@goo.ne.jp)",
"mogimogi/1.0",
"moiNAG 0.02",
"MojeekBot/0.x (archi; http://www.mojeek.com/bot.html)",
"Morris - Mixcat Crawler ( http://mixcat.com)",
"Mouse-House/7.4 (spider_monkey spider info at www.mobrien.com/sm.shtml)",
"MovableType/x.x",
"mozDex/0.xx-dev (mozDex; http://www.mozdex.com/en/bot.html; spider@mozdex.com)",
"Mozilla (Mozilla@somewhere.com)",
"Mozilla 4.0(compatible; BotSeer/1.0; +http://botseer.ist.psu.edu)",
"Mozilla/2.0 (compatible; Ask Jeeves)",
"Mozilla/2.0 (compatible; Ask Jeeves/Teoma)",
"Mozilla/2.0 (compatible; Ask Jeeves/Teoma; http://about.ask.com/en/docs/about/webmasters.shtml) ",
"Mozilla/2.0 (compatible; Ask Jeeves/Teoma; http://sp.ask.com/docs/about/tech_crawling.html)",
"Mozilla/2.0 (compatible; EZResult -- Internet Search Engine)",
"Mozilla/2.0 (compatible; T-H-U-N-D-E-R-S-T-O-N-E)",
"Mozilla/3.0 (compatible; Fluffy the spider; http://www.searchhippo.com/; info@searchhippo.com)",
"Mozilla/3.0 (compatible; MuscatFerret/1.5.4; claude@euroferret.com)",
"Mozilla/3.0 (compatible; MuscatFerret/1.5; olly@muscat.co.uk)",
"Mozilla/3.0 (compatible; MuscatFerret/1.6.x; claude@euroferret.com)",
"Mozilla/3.0 (compatible; netart generator/1.0; libwww-perl/5.64)",
"Mozilla/3.0 (compatible; ScollSpider; http://www.webwobot.com)",
"Mozilla/3.0 (compatible; Webinator-DEV01.home.iprospect.com/2.56)",
"Mozilla/3.0 (compatible; Webinator-indexer.cyberalert.com/2.56)",
"Mozilla/3.0 (INGRID/3.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)",
"Mozilla/3.0 (Slurp.so/Goo; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Mozilla/3.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Mozilla/3.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Mozilla/3.0 (Vagabondo/1.1 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)",
"Mozilla/3.0 (Vagabondo/1.x MT; webagent@wise-guys.nl; http://webagent.wise-guys.nl/)",
"Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)",
"Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)",
"Mozilla/3.01 (Compatible; Links2Go Similarity Engine)",
"Mozilla/4.0",
"Mozilla/4.0 (agadine3.0) www.agada.de",
"Mozilla/4.0 (compatible: AstraSpider V.2.1 : astrafind.com)",
"Mozilla/4.0 (compatible; Vagabondo/2.2; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)",
"Mozilla/4.0 (compatible; Vagabondo/4.0Beta; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)",
"Mozilla/4.0 (compatible; B_L_I_T_Z_B_O_T)",
"Mozilla/4.0 (compatible; ChristCrawler.com ChristCrawler@ChristCENTRAL.com)",
"Mozilla/4.0 (compatible; DAUMOA-video; +http://ws.daum.net/aboutkr.html)",
"Mozilla/4.0 (compatible; FastCrawler3 support-fastcrawler3@fast.no)",
"Mozilla/4.0 (compatible; FDSE robot)",
"Mozilla/4.0 (compatible; GPU *** crawler http://gpu.sourceforge.net/search_engine.php)",
"Mozilla/4.0 (compatible; grub-client-0.2.x; Crawl your stuff with http://grub.org)",
"Mozilla/4.0 (compatible; grub-client-0.3.x; Crawl your own stuff with http://grub.org)",
"Mozilla/4.0 (compatible; grub-client-2.x)",
"Mozilla/4.0 (compatible; MSIE 4.01; Vonna.com b o t)",
"Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) ACR",
"Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) Indonesia Interactive",
"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0) (samualt9@bigfoot.com)",
"Mozilla/4.0 (compatible; MSIE 5.0; NetNose-Crawler 2.0; A New Search Experience: http://www.netnose.com)",
"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) TrueRobot; 1.5",
"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot BETA 1.2 (http://www.voila.com/)",
"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot; 1.6",
"Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; http://www.girafa.com)",
"Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com)",
"Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com/; http://www.galaxy.com/info/crawler.html)",
"Mozilla/4.0 (compatible; MSIE 5.0; YANDEX)",
"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; obot)",
"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; QXW03018)",
"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AspTear 1.5)",
"Mozilla/4.0 (compatible; MSIE 6.0 compatible; Asterias Crawler v4; +http://www.singingfish.com/help/spider.html; webmaster@singingfish.com); SpiderThread Revision: 3.10",
"Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Skampy/0.9.x [en]",
"Mozilla/4.0 (compatible; MSIE 6.0; TargetSeek/1.0; +http://www.targetgroups.net/TargetSeek.html)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ODP entries t_st; http://tuezilla.de/t_st-odp-entries-agent.html)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ODP links test; http://tuezilla.de/test-odp-links-agent.html)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ZoomSpider.net bot; .NET CLR 1.1.4322)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; heritrix/1.3.0 http://www.cs.washington.edu/research/networking/websys/)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MathPlayer2.0)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; QihooBot 1.0 qihoobot@qihoo.net)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)",
"Mozilla/4.0 (compatible; NaverBot/1.0; http://help.naver.com/delete_main.asp)",
"Mozilla/4.0 (compatible; Netcraft Web Server Survey)",
"Mozilla/4.0 (compatible; SpeedySpider; www.entireweb.com)",
"Mozilla/4.0 (compatible; SPENG)",
"Mozilla/4.0 (compatible; www.euro-directory.com; urlchecker1.0)",
"Mozilla/4.0 (compatible; www.galaxy.com)",
"Mozilla/4.0 (compatible; Y!J; for robot study; keyoshid)",
"Mozilla/4.0 (compatible; Yahoo Japan; for robot study; kasugiya)",
"Faked IE id string used by DeepTrawl link checking tool",
"Mozilla/4.0 (fantomBrowser)",
"Mozilla/4.0 (fantomCrew Browser)",
"Mozilla/4.0 (hhjhj@yahoo.com)",
"Mozilla/4.0 (JemmaTheTourist;http://www.activtourist.com)",
"Mozilla/4.0 (MobilePhone PM-8200/US/1.0) NetFront/3.x MMP/2.0",
"Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)",
"Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 FAKE (compatible; Googlebot/2.1; http://www.google.com/bot.html)",
"Mozilla/4.0 (Mozilla; http://www.mozilla.org/docs/en/bot.html; master@mozilla.com)",
"Mozilla/4.0 (Sleek Spider/1.2)",
"Mozilla/4.0 compatible FurlBot/Furl Search 2.0 (FurlBot; http://www.furl.net; wn.furlbot@looksmart.net)",
"Mozilla/4.0 compatible ZyBorg/1.0 (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)",
"Mozilla/4.0 compatible ZyBorg/1.0 (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)",
"Mozilla/4.0 compatible ZyBorg/1.0 Dead Link Checker (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)",
"Mozilla/4.0 compatible ZyBorg/1.0 for Homepage (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)",
"Mozilla/4.0 efp@gmx.net",
"Mozilla/4.0 WebTV/2.6 (compatible; MSIE 4.0)",
"Mozilla/4.0 [en] (Ask Jeeves Corporate Spider)",
"Mozilla/4.0(compatible; Zealbot 1.0)",
"Mozilla/4.01 (compatible; NORAD National Defence Network)",
"Mozilla/4.01 [en](Win95;I)",
"Mozilla/4.02 [en] (X11; I; SunOS 5.6 sun4u)",
"Mozilla/4.04 (compatible; Dulance bot; +http://www.dulance.com/bot.jsp)",
"Mozilla/4.04 [en] (X11; I; HP-UX B.10.20 9000/712)",
"Mozilla/4.04 [en] (X11; I; IRIX 5.3 IP22)",
"Mozilla/4.05 (Macintosh; I; 68K Nav)",
"Mozilla/4.05 (Macintosh; I; PPC Nav)",
"Mozilla/4.05 [en] (X11; I; SunOS 4.1.4 sun4m)",
"Mozilla/4.08 [en] (Win98; U ;Nav)",
"Mozilla/4.08 [en] (WinNT; U)",
"Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_TrueRobot/1.4 libwww/5.2.8",
"Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_VoilaBot/1.6 libwww/5.3.2",
"Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)",
"Mozilla/4.5 (compatible; iCab 2.5.3; Macintosh; I; PPC)",
"Mozilla/4.5 (compatible; OmniWeb/4.0.5; Mac_PowerPC)",
"Mozilla/4.5 (compatible; OmniWeb/4.1-beta-1; Mac_PowerPC)",
"Mozilla/4.5 RPT-HTTPClient/0.3-2",
"Mozilla/4.5 [en]C-CCK-MCD {RuralNet} (Win98; I)",
"Mozilla/4.5b1 [en] (X11; I; Linux 2.0.35 i586)",
"Mozilla/4.6 [en] (http://www.cnet.com/)",
"Mozilla/4.61 [de] (OS/2; I)",
"Mozilla/4.61 [en] (X11; U; ) - BrowseX (2.0.0 Windows)",
"Mozilla/4.7",
"Mozilla/4.7 (compatible; http://eidetica.com/spider)",
"Mozilla/4.7 (compatible; Intelliseek; http://www.intelliseek.com)",
"Mozilla/4.7 (compatible; OffByOne; Windows 98) Webster Pro V3.2",
"Mozilla/4.7 (compatible; Whizbang)",
"Mozilla/4.7 (compatible; WhizBang; http://www.whizbang.com/crawler)",
"Mozilla/4.7 [en](BecomeBot@exava.com)",
"Mozilla/4.7 [en](Exabot@exava.com)",
"Mozilla/4.7 [en]C-CCK-MCD {Yahoo;YIP052400} (Win95; I)",
"Mozilla/4.72 [en] (BACS http://www.ba.be)",
"Mozilla/4.72C-CCK-MCD Caldera Systems OpenLinux [en] (X11; U; Linux 2.2.14 i686)",
"Mozilla/4.75C-ja [ja] (X11; U; OSF1 V5.1 alpha)",
"Mozilla/4.76 (Windows 98; U) Opera 5.12 [en]",
"Mozilla/4.76 [en] (X11; U; FreeBSD 4.4-STABLE i386)",
"Mozilla/4.76 [en] (X11; U; SunOS 5.7 sun4u)",
"Mozilla/4.77C-SGI [en] (X11; U; IRIX 6.5 IP32)",
"Mozilla/5.0",
"Mozilla/5.0 (+http://www.eurekster.com/*******) *******/0.1",
"Mozilla/5.0 (+http://www.sli-systems.com/) *******/0.1",
"Mozilla/5.0 (Clustered-Search-Bot/1.0; support@clush.com; http://www.clush.com/)",
"Mozilla/5.0 (compatible) GM RSS Panel X",
"Mozilla/5.0 (compatible; AnsearchBot/1.x; +http://www.ansearch.com.au/)",
"Mozilla/5.0 (compatible; archive.org_bot/1.10.0 +http://www.loc.gov/minerva/crawl.html)",
"Mozilla/5.0 (compatible; archive.org_bot/1.13.1x http://crawler.archive.org)",
"Mozilla/5.0 (compatible; archive.org_bot/1.5.0-200506132127 http://crawler.archive.org) Hurricane Katrina",
"Mozilla/5.0 (compatible; Ask Jeeves/Teoma; http://about.ask.com/en/docs/about/webmasters.shtml)",
"Mozilla/5.0 (compatible; BanBots/2.0b; Fetch; +http://www.banbots.com)",
"Mozilla/5.0 (compatible; BecomeBot/1.23; http://www.become.com/webmasters.html)",
"Mozilla/5.0 (compatible; BecomeBot/1.xx; MSIE 6.0 compatible; http://www.become.com/webmasters.html)",
"Mozilla/5.0 (compatible; BecomeBot/2.0beta; http://www.become.com/webmasters.html)",
"Mozilla/5.0 (compatible; BecomeBot/2.x; MSIE 6.0 compatible; http://www.become.com/site_owners.html)",
"Mozilla/5.0 (compatible; BecomeJPBot/2.3; MSIE 6.0 compatible; +http://www.become.co.jp/site_owners.html)",
"Mozilla/5.0 (compatible; BlogRefsBot/0.1; http://www.blogrefs.com/about/bloggers)",
"Mozilla/5.0 (compatible; Bot; +http://pressemitteilung.ws/spamfilter",
"Mozilla/5.0 (compatible; BuzzRankingBot/1.0; +http://www.buzzrankingbot.com/)",
"Mozilla/5.0 (compatible; Charlotte/1.0b; charlotte@betaspider.com)",
"Mozilla/5.0 (compatible; Charlotte/1.0b; http://www.searchme.com/support/)",
"Mozilla/5.0 (compatible; Crawling jpeg; http://www.yama.info.waseda.ac.jp)",
"Mozilla/5.0 (compatible; Custo 3 (Netwu.com); Windows NT 5.1)",
"Mozilla/5.0 (compatible; de/1.13.2 +http://www.de.com)",
"Mozilla/5.0 (compatible; DNS-Digger-Explorer/1.0; +http://www.dnsdigger.com)",
"Mozilla/5.0 (compatible; DNS-Digger/1.0; +http://www.dnsdigger.com)",
"Mozilla/5.0 (compatible; EARTHCOM.info/2.01; http://www.earthcom.info)",
"Mozilla/5.0 (compatible; EARTHCOM/2.2; +http://enter4u.eu)",
"Mozilla/5.0 (compatible; egothor/8.0g; +http://ego.ms.mff.cuni.cz/)",
"Mozilla/5.0 (compatible; Exabot Test/3.0; +http://www.exabot.com/go/robot)",
"Mozilla/5.0 (compatible; FatBot 2.0; http://www.thefind.com/main/CrawlerFAQs.fhtml)",
"mozilla/5.0 (compatible; genevabot http://www.healthdash.com)",
"Mozilla/5.0 (compatible; Google Desktop) Paros/3.2.12",
"Mozilla/5.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)",
"mozilla/5.0 (compatible; heritrix/1.0.4 http://innovationblog.com)",
"Mozilla/5.0 (compatible; heritrix/1.10.2 +http://i.stanford.edu/)",
"Mozilla/5.0 (compatible; heritrix/1.12.1 +http://newstin.com/)",
"Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.page-store.com)",
"Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.page-store.com) [email:paul@page-store.com]",
"mozilla/5.0 (compatible; heritrix/1.3.0 http://archive.crawler.org)",
"Mozilla/5.0 (compatible; heritrix/1.4.0 +http://www.chepi.net)",
"Mozilla/5.0 (compatible; heritrix/1.4t http://www.truveo.com/)",
"Mozilla/5.0 (compatible; heritrix/1.5.0 http://www.l3s.de/~kohlschuetter/projects/crawling/)",
"Mozilla/5.0 (compatible; heritrix/1.5.0-200506231921 http://pandora.nla.gov.au/crawl.html)",
"Mozilla/5.0 (compatible; heritrix/1.6.0 http://www.worio.com/)",
"Mozilla/5.0 (compatible; heritrix/1.7.0 +http://www.greaterera.com/)",
"Mozilla/5.0 (compatible; Heritrix/1.8.0 http://www.hanzoarchives.com)",
"Mozilla/5.0 (compatible; heritrix/1.x.x +http://www.accelobot.com)",
"Mozilla/5.0 (compatible; heritrix/2.0.0-RC1 +http://www.aol.com)",
"Mozilla/5.0 (compatible; Hermit Search. Com; +http://www.hermitsearch.com)",
"Mozilla/5.0 (compatible; http://www.IsMySiteUp.Net/bot/ )",
"Mozilla/5.0 (compatible; http://www.UptimeAuditor.com/bot/ )",
"Mozilla/5.0 (compatible; HyperixScoop/1.3; +http://www.hyperix.com)",
"Mozilla/5.0 (compatible; iaskspider/1.0; MSIE 6.0)",
"Mozilla/5.0 (compatible; IDBot/1.0; +http://www.id-search.org/bot.html)",
"Mozilla/5.0 (compatible; InterseekWeb/3.x)",
"Mozilla/5.0 (compatible; Jim +http://www.hanzoarchives.com)",
"Mozilla/5.0 (compatible; Konqueror/2.0.1; X11); Supports MD5-Digest; Supports gzip encoding",
"Mozilla/5.0 (compatible; Konqueror/2.1.1; X11)",
"Mozilla/5.0 (compatible; Konqueror/2.2.2)",
"Mozilla/5.0 (compatible; Konqueror/2.2.2; Linux 2.4.14-xfs; X11; i686)",
"Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)",
"Mozilla/5.0 (compatible; LemSpider 0.1)",
"Mozilla/5.0 (compatible; LinksManager.com_bot http://linksmanager.com/linkchecker.html)",
"Mozilla/5.0 (compatible; LinkStash Bookmark Manager; http://www.xrayz.co.uk/)",
"Mozilla/5.0 (compatible; MojeekBot/2.0; http://www.mojeek.com/bot.html)",
"Mozilla/5.0 (compatible; MOSBookmarks/v2.6-Plus; Link Checker)",
"Mozilla/5.0 (compatible; MSIE 6.0; Podtech Network; crawler_admin@podtech.net)",
"Mozilla/5.0 (compatible; OnetSzukaj/5.0; http://szukaj.onet.pl)",
"Mozilla/5.0 (compatible; PagestackerBot; http://www.pagestacker.com)",
"Mozilla/5.0 (compatible; PalmeraBot; http://www.links24h.com/help/palmera) Version 0.001",
"Mozilla/5.0 (compatible; PEAR HTTP_Request class; http://feed.moo.jp/)",
"Mozilla/5.0 (compatible; Phonifier; +http://www.phonifier.com)",
"Mozilla/5.0 (compatible; pmoz.info ODP link checker; +http://pmoz.info/doc/botinfo.htm)",
"Mozilla/5.0 (compatible; pogodak.ba/3.x)",
"Mozilla/5.0 (compatible; Pogodak.hr/3.1)",
"Mozilla/5.0 (compatible; Proximic crawler; +http://www.proximic.com/en/about-us/contact-us.html)",
"Mozilla/5.0 (compatible; PWeBot/3.1; http://www.programacionweb.net/robot.php)",
"Mozilla/5.0 (compatible; Quantcastbot/1.0; www.quantcast.com)",
"Mozilla/5.0 (compatible; robtexbot/1.0; http://www.robtex.com/ )",
"Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)",
"Mozilla/5.0 (compatible; Scrubby/2.2; http://www.scrubtheweb.com/)",
"Mozilla/5.0 (compatible; ShunixBot/1.x.x +http://www.shunix.com/robot.htm)",
"Mozilla/5.0 (compatible; ShunixBot/1.x; http://www.shunix.com/bot.htm)",
"Mozilla/5.0 (compatible; SkreemRBot +http://skreemr.com)",
"Mozilla/5.0 (compatible; SnapPreviewBot; en-US; rv:1.8.0.9) Gecko/20061206 Firefox/1.5.0.9",
"Mozilla/5.0 (compatible; SpurlBot/0.2) ",
"Mozilla/5.0 (compatible; SummizeBot +http://www.summize.com)",
"Mozilla/5.0 (compatible; SYCLIKControl/LinkChecker;)",
"Mozilla/5.0 (compatible; Synoobot/0.9; http://www.synoo.com/search/bot.html)",
"Mozilla/5.0 (compatible; Theophrastus/x.x; http://users.cs.cf.ac.uk/N.A.Smith/theophrastus.php)",
"Mozilla/5.0 (compatible; TridentSpider/3.1)",
"Mozilla/5.0 (compatible; Vagabondo/2.1; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)",
"Mozilla/5.0 (compatible; Windows NT 5.0; phpwebbrainBot/0.1 - http://www.monsterli.ch/phpwebbrain/)",
"Mozilla/5.0 (compatible; worio bot heritrix/1.10.0 +http://worio.com)",
"Mozilla/5.0 (compatible; WoW Lemmings Kathune/2.0;http://www.wowlemmings.com/kathune.html)",
"Mozilla/5.0 (compatible; XTbot/1.0v; +http://www.externaltest.com)",
"Mozilla/5.0 (compatible; Yahoo! DE Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
"Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)",
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
"Mozilla/5.0 (compatible; Yoono; http://www.yoono.com/)",
"Mozilla/5.0 (compatible; Zenbot/1.3; +http://zen.co.za/webmasters/)",
"Mozilla/5.0 (compatible;archive.org_bot/1.7.1; collectionId=316; Archive-It; +http://www.archive-it.org)",
"Mozilla/5.0 (compatible;archive.org_bot/heritrix-1.9.0-200608171144 +http://pandora.nla.gov.au/crawl.html)",
"Mozilla/5.0 (compatible;FindITAnswersbot/1.0;+http://search.it-influentials.com/bot.htm)",
"Mozilla/5.0 (compatible;MAINSEEK_BOT)",
"Mozilla/5.0 (Gecko/20070310 Mozshot/0.0.20070628; http://mozshot.nemui.org/)",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9b5) Gecko/2008032619 Firefox/3.0b5",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20021219 Chimera/0.6 ",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20030306 Camino/0.7",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/xx (KHTML like Gecko) OmniWeb/v5xx.xx",
"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/xxx.x (KHTML like Gecko) Safari/12x.x",
"Mozilla/5.0 (Macintosh; U; PPC; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1",
"Mozilla/5.0 (research@mediatrec.com)",
"Mozilla/5.0 (Sage)",
"Mozilla/5.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Mozilla/5.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Mozilla/5.0 (SunOS 5.8 sun4u; U) Opera 5.0 [en]",
"Mozilla/5.0 (Twiceler-0.9 http://www.cuill.com/twiceler/robot.html)",
"Mozilla/5.0 (Version: xxxx Type:xx)",
"Mozilla/5.0 (wgao@genieknows.com)",
"Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1",
"Mozilla/5.0 (Windows; U; Win98; en-US; rv:x.xx) Gecko/20030423 Firebird Browser/0.6",
"Mozilla/5.0 (Windows; U; Win9x; en; Stable) Gecko/20020911 Beonex/0.8.1-stable",
"Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.7) NimbleCrawler 1.11 obeys UserAgent NimbleCrawler For problems contact: crawler_at_dataalchemy.com",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.5) Gecko/20060731 Firefox/1.5.0.5 Flock/0.7.4.1",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.4/Megaupload x.0",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.1) Gecko/2008092215 Firefox/3.0.1 Orca/1.1 beta 3",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.x.x) Gecko/20041107 Firefox/x.x",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xx) Gecko/20030504 Mozilla Firebird/0.6",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xxx) Gecko/20041027 Mnenhy/0.6.0.104",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)",
"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5",
"Mozilla/5.0 (Windows; U;XMPP Tiscali Communicator v.10.0.1; Windows NT 5.1; it; rv:1.8.1.3) Gecko/20070309 Firefox/2.0.0.3",
"Mozilla/5.0 (Windows;) NimbleCrawler 1.12 obeys UserAgent NimbleCrawler For problems contact: crawler@health",
"Mozilla/5.0 (Windows;) NimbleCrawler 1.12 obeys UserAgent NimbleCrawler For problems contact: crawler@healthline.com",
"Mozilla/5.0 (X11; Linux i686; U;rv: 1.7.13) Gecko/20070322 Kazehakase/0.4.4.1",
"Mozilla/5.0 (X11; U; Linux 2.4.2-2 i586; en-US; m18) Gecko/20010131 Netscape6/6.01",
"Mozilla/5.0 (X11; U; Linux i686; de-AT; rv:1.8.0.2) Gecko/20060309 SeaMonkey/1.0",
"Mozilla/5.0 (X11; U; Linux i686; en-GB; rv:1.7.6) Gecko/20050405 Epiphany/1.6.1 (Ubuntu) (Ubuntu package 1.0.2)",
"Mozilla/5.0 (X11; U; Linux i686; en-US; Nautilus/1.0Final) Gecko/20020408",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:0.9.3) Gecko/20010801",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2b) Gecko/20021007 Phoenix/0.3",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040413 Epiphany/1.2.1",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.7) Gecko/20060909 Firefox/1.5.0.7 SnapPreviewBot",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1) Gecko/20061129 BonEcho/2.0",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.1) Gecko/20061205 Iceweasel/2.0.0.1 (Debian-2.0.0.1+dfsg-2)",
"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9a8) Gecko/2007100619 GranParadiso/3.0a8",
"Mozilla/5.0 Galeon/1.0.2 (X11; Linux i686; U;) Gecko/20011224",
"Mozilla/5.0 gURLChecker/0.x.x (Linux)",
"Mozilla/5.0 URL-Spider",
"Mozilla/5.0 usww.com-Spider-for-w8.net",
"Mozilla/5.0 wgao@genieknows.com",
"Mozilla/5.0 whoiam [http://www.axxus.de/]",
"Mozilla/5.0 [en] (compatible; Gulper Web Bot 0.2.4 www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)",
"MQbot metaquerier.cs.uiuc.edu/crawler",
"MQBOT/Nutch-0.9-dev (MQBOT Nutch Crawler; http://falcon.cs.uiuc.edu; mqbot@cs.uiuc.edu)",
"MSFrontPage/4.0",
"MSIE 4.0 (Win95)",
"MSIE-5.13 (larbin@unspecified.mail)",
"msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)",
"msnbot-Products/1.0 (+http://search.msn.com/msnbot.htm)",
"MSNBOT/0.xx (http://search.msn.com/msnbot.htm)",
"msnbot/x.xx ( http://search.msn.com/msnbot.htm)",
"MSNBOT_Mobile MSMOBOT Mozilla/2.0 (compatible; MSIE 4.02; Windows CE; Default)",
"MSNPTC/1.0",
"MSProxy/2.0",
"MSRBOT",
"MSRBOT (http://research.microsoft.com/research/sv/msrbot)",
"multiBlocker browser",
"multicrawler ( http://sw.deri.org/2006/04/multicrawler/robots.html)",
"MultiText/0.1",
"MusicWalker2.0 ( http://www.somusical.com)",
"MVAClient",
"My WinHTTP Connection",
"myDaemon",
"MyGetRight/1.0.0",
"MyGetRight/1.0b",
"Mylinea.com Crawler 2.0",
"mylinkcheck/1.02",
"Naamah 1.0.1/Blogbot (http://blogbot.de/)",
"Naamah 1.0a/Blogbot (http://blogbot.de/)",
"NABOT/5.0",
"nabot_1.0",
"NameOfAgent (CMS Spider)",
"naoFavicon4IE/1.xx",
"NASA Search 1.0",
"NationalDirectory-WebSpider/1.3",
"NationalDirectoryAddURL/1.0",
"NaverBot-1.0 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)",
"NaverBot_dloader/1.5",
"NavissoBot",
"NavissoBot/1.7 (+http://navisso.com/)",
"NCSA Beta 1 (http://vias.ncsa.uiuc.edu/viasarchivinginformation.html)",
"Nebullabot/2.2 (http://bot.nebulla.info)",
"NEC Research Agent -- compuman at research.nj.nec.com",
"NEC-Hayek/1.0",
"Net-Seekr Bot/Net-Seekr Bot V1 (http://www.net-seekr.com)",
"NetAnts/1.2x",
"NETCOMplete/x.xx",
"NetinfoBot/1.0 (http://netinfo.bg/netinfobot.html)",
"NetLookout/2.24",
"Netluchs/0.8-dev ( ; http://www.netluchs.de/; ___don't___spam_me_@netluchs.de)",
"NetMechanic Vx.0",
"NetNewsWire/2.x (Mac OS X; http://ranchero.com/netnewswire/)",
"NetNoseCrawler/v1.0",
"Netprospector JavaCrawler",
"NetPumper/x.xx",
"NetResearchServer(http://www.look.com)",
"NetResearchServer/x.x(loopimprovements.com/robot.html)",
"NetSprint -- 2.0",
"NetWhatCrawler/0.06-dev (NetWhatCrawler from NetWhat.com; http://www.netwhat.com; support@netwhat.com)",
"NetZippy",
"NeuralBot/0.2",
"newsearchengine (ThisUser@unspecified.mail)",
"NewsGator FetchLinks extension/0.2.0 (http://graemef.com)",
"NewsGatorOnline/2.0 (http://www.newsgator.com; 1 subscribers)",
"NextGenSearchBot 1 (for information visit http://www.eliyon.com/NextGenSearchBot)",
"NextopiaBOT (+http://www.nextopia.com) distributed crawler client beta v0.x",
"NG-Search/0.90 (NG-SearchBot; http://www.ng-search.com; )",
"NG/1.0",
"NG/4.0.1229",
"nicebot",
"NICO/1.0",
"Nikita the Spider (http://NikitaTheSpider.com/)",
"NITLE Blog Spider/0.01",
"Nitro Downloader 1.x (www.klsofttools.com)",
"Noago Spider",
"Nocilla/1.0",
"Nokia-WAPToolkit/1.2 googlebot(at)googlebot.com",
"Nokia6300/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1 (botmobi http://find.mobi/bot.html abuse@mtld.mobi)",
"Nokia6610/1.0 (3.09) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible;YahooSeeker/M1A1-R2D2; http://help.yahoo.com/help/us/ysearch/crawling/crawling-01.html)",
"Nokia7110/1.0 (05.01) (Google WAP Proxy/1.0)",
"NokodoBot/1.x (+http://nokodo.com/bot.htm)",
"Norbert the Spider(Burf.com)",
"noxtrumbot/1.0 (crawler@noxtrum.com)",
"noyona_0_1",
"NP/0.1 (NP; http://www.nameprotect.com; npbot@nameprotect.com)",
"NPBot (http://www.nameprotect.com/botinfo.html)",
"NPBot-1/2.0",
"Nsauditor/1.x",
"NSPlayer/10.0.0.xxxx WMFSDK/10.0",
"nsyght.com/Nutch-1.0-dev (nsyght.com; Nsyght.com)",
"nsyght.com/Nutch-x.x (nsyght.com; search.nsyght.com)",
"nttdirectory_robot/0.9 (super-robot@super.navi.ocn.ne.jp)",
"Nucleus SiteList LinkChecker/1.1",
"nuSearch Spider <a href='http://www.nusearch.com'>www.nusearch.com</a> (compatible; MSIE 4.01)",
"NuSearch Spider (compatible; MSIE 6.0)",
"NuSearch Spider www.nusearch.com",
"Nutch",
"Nutch crawler/Nutch-0.9 (picapage.com; admin@picapage.com)",
"Nutch/Nutch-0.9 (Eurobot; http://www.ayell.eu )",
"NutchCVS/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)",
"NutchCVS/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)",
"NutchCVS/0.7.1 (Nutch running at UW; http://www.nutch.org/docs/en/bot.html; sycrawl@cs.washington.edu)",
"NutchEC2Test/Nutch-0.9-dev (Testing Nutch on Amazon EC2.; http://lucene.apache.org/nutch/bot.html; ec2test at lucene.com)",
"NutchOrg/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)",
"nutchsearch/Nutch-0.9 (Nutch Search 1.0; herceg_novi at yahoo dot com)",
"NutchVinegarCrawl/Nutch-0.8.1 (Vinegar; http://www.cs.washington.edu; eytanadar at gmail dot com)",
"obidos-bot (just looking for books.)",
"ObjectsSearch/0.01-dev (ObjectsSearch;http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)",
"ObjectsSearch/0.0x (ObjectsSearch; http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)",
"oBot ((compatible;Win32))",
"Ocelli/1.x (http://www.globalspec.com/Ocelli)",
"Octopus",
"Octora Beta - www.octora.com",
"Octora Beta Bot - www.octora.com",
"Offline Explorer 1.*",
"OliverPerry",
"OmniExplorer_Bot/1.0x (+http://www.omni-explorer.com) Internet CategorizerOmniExplorer http://www.omni-explorer.com/ car & shopping search (64.62.175.xxx)",
"OmniExplorer_Bot/1.0x (+http://www.omni-explorer.com) Job Crawler",
"OmniExplorer_Bot/1.1x (+http://www.omni-explorer.com) ******* Crawler",
"OmniExplorer_Bot/x.xx (+http://www.omni-explorer.com) WorldIndexer",
"onCHECK Linkchecker von www.scientec.de fuer www.onsinn.de",
"Onet.pl SA- http://szukaj.onet.pl",
"online link validator (http://www.dead-links.com/)",
"OntoSpider/1.0 libwww-perl/5.65",
"OpenAcoon v4.0.x (www.openacoon.de)",
"Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)",
"Openfind data gatherer- Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)",
"Openfind Robot/1.1A2",
"OpenISearch/1.x (www.openisearch.com)",
"OpenTaggerBot (http://www.opentagger.com/opentaggerbot.htm)",
"OpenTextSiteCrawler/2.9.2",
"OpenWebSpider/0.x.x (http://www.openwebspider.org)",
"OpenWebSpider/x",
"Opera/5.0 (Linux 2.0.38 i386; U) [en]",
"Opera/5.11 (Windows ME; U) [ru]",
"Opera/5.12 (Windows 98; U) [en]",
"Opera/6.01 (larbin@unspecified.mail)",
"Opera/6.x (Linux 2.4.8-26mdk i686; U) [en]",
"Opera/6.x (Windows NT 4.0; U) [de]",
"Opera/7.x (Windows NT 5.1; U) [en]",
"Opera/8.xx (Windows NT 5.1; U; en)",
"Opera/9.0 (Windows NT 5.1; U; en)",
"Opera/9.00 (Windows NT 5.1; U; de)",
"Opera/9.60 (Windows NT 5.1; U; de) Presto/2.1.1",
"OpidooBOT (larbin2.6.3@unspecified.mail)",
"OPWV-SDK UP.Browser/7.0.2.3.119 (GUI) MMP/2.0 Push/PO",
"Oracle Application Server Web Cache 10g",
"Oracle iMTCrawler",
"Oracle Ultra Search",
"OrangeSpider",
"Orbiter/T-2.0 (+http://www.dailyorbit.com/bot.htm)",
"Orca Browser (http://www.orcabrowser.com)",
"OSSProxy 1.3.305.321 (Build 305.321 Win32 en-us)(Dec 21 2005 16:30:54)",
"OutfoxBot/0.x (For internet experiments; http://; outfox.agent@gmail.com)",
"OutfoxMelonBot/0.5 (for internet experiments; http://; outfoxbot@gmail.com)",
"Overture-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)",
"OWR_Crawler 0.1",
"ozelot/2.7.3 (Search engine indexer; www.flying-cat.de/ozelot; ozelot@flying-cat.de)",
"PADLibrary Spider",
"PageBitesHyperBot/600 (http://www.pagebites.com/)",
"Pagebull http://www.pagebull.com/",
"Pagestacker Bot",
"page_verifier (http://www.securecomputing.com/goto/pv)",
"PagmIEDownload",
"parallelContextFocusCrawler1.1parallelContextFocusCrawler1.1",
"ParaSite/1.0b (http://www.ianett.com/parasite/)",
"Patwebbot (http://www.herz-power.de/technik.html)",
"pavuk/0.9pl29b i686-pc-linux-gnu",
"PBrowse 1.4b",
"pd02_1.0.0 pd02_1.0.0@dzimi@post.sk",
"PEAR HTTP_Request class ( http://pear.php.net/ )",
"PEERbot www.peerbot.com",
"PeopleChat/Search_Engine",
"PEval 1.4b",
"PHP/3.x.xx",
"PHP/4.0.4pl1",
"PHP/4.0.6",
"PHP/4.1.1",
"PHP/4.1.2",
"PicoSearch/1.0",
"Piffany_Web_Scraper_v0.x",
"Piffany_Web_Spider_v0.x",
"PigeonBot1.0 BETA",
"PingALink Monitoring Services 1.0",
"PingALink Monitoring Services 1.0 (http://www.pingalink.com)",
"Pingdom GIGRIB (http://www.pingdom.com)",
"pipeLiner/0.3a (PipeLine Spider;http://www.pipeline-search.com/webmaster.html; webmaster'at'pipeline-search.com)",
"pipeLiner/0.xx (PipeLine Spider; http://www.pipeline-search.com/webmaster.html)",
"Pita",
"Pizilla++ ver 2.45",
"PJspider/3.0 (pjspider@portaljuice.com; http://www.portaljuice.com)",
"Plagger/0.x.xx (http://plagger.org/)",
"PlagiarBot/1.0",
"PlantyNet_WebRobot_V1.9 dhkang@plantynet.com",
"plinki/0.1 (you got plinked! (thats a good thing..); http://www.plinki.com; crawl@plinki.com)",
"PluckFeedCrawler/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://www.pluck.com; 1 subscribers)",
"Pluggd/Nutch-0.9 (automated crawler http://www.pluggd.com;support at pluggd dot com)",
"Pockey-GetHTML/4.12.0 (Win32; GUI; ix86)",
"Pockey-GetHTML/x.xx",
"Pockey/x.x.x",
"Pockey7.x.x(WIN32GUI)",
"POE-Component-Client-HTTP/0.64 (perl; N; POE; en; rv:0.640000)",
"Poirot",
"polybot 1.0 (http://cis.poly.edu/polybot/)",
"Pompos/1.x http://dir.com/pompos.html",
"Pompos/1.x pompos@iliad.fr",
"Popdexter/1.0",
"Port Huron Labs",
"PortalBSpider/2.0 (spider@portalb.com)",
"portalmmm/2.0 S500i(c20;TB)",
"PostFavorites",
"potbot 1.0",
"PRCrawler/Nutch-0.9 (data mining development project; crawler@projectrialto.com)",
"PrivacyFinder Cache Bot v1.0",
"PrivacyFinder/1.1",
"Privoxy/3.0 (Anonymous)",
"Production Bot 0116B",
"Production Bot 2016B",
"Production Bot DOT 3016B",
"Program Shareware 1.0.2",
"Progressive Download",
"Progressive Download HTTP check",
"Project XP5 [2.03.07-111203]",
"PROve AnswerBot 4.0",
"ProWebGuide Link Checker (http://www.prowebguide.com)",
"psbot/0.1 (+http://www.picsearch.com/bot.html)",
"PSurf15a 11",
"PSurf15a 51",
"PSurf15a VA",
"psyche*****",
"PubCrawl (pubcrawl.stanford.edu)",
"puf/0.91beta6a (Linux 2.2.18; i686)",
"puf/0.93.2a (Linux 2.4.18; i686)",
"pulseBot (pulse Web Miner)",
"PureSight",
"PuxaRapido v1.0",
"PWeBot/1.2 Inspector (http://www.programacionweb.net/robot.php)",
"PycURL",
"PycURL/7.xx.x",
"Python-urllib/1.1x",
"Python-urllib/2.0a1",
"Qango.com Web Directory (http://www.qango.com/)",
"QEAVis Agent/Nutch-0.9 (Quantitative Evaluation of Academic Websites Visibility; http://nlp.uned.es/qeavis",
"QPCreep Test Rig ( We are not indexing- just testing )",
"QuepasaCreep ( crawler@quepasacorp.com )",
"QuepasaCreep v0.9.1x",
"QueryN Metasearch",
"QuickTime\xaa.7.0.4 (qtver=7.0.4;cpu=PPC;os=Mac 10.3.9)",
"QweeryBot/3.01 ( http://qweerybot.qweery.nl)",
"Qweery_robot.txt_CheckBot/3.01 (http://qweerybot.qweery.com)",
"R6_CommentReader_(www.radian6.com/crawler)",
"R6_FeedFetcher_(www.radian6.com/crawler)",
"rabaz (rabaz at gigabaz dot com)",
"RaBot/1.0 Agent-admin/phortse@hanmail.net",
"Rainbot1.1",
"ramBot xtreme x.x",
"RAMPyBot - www.giveRAMP.com/0.1 (RAMPyBot - www.giveRAMP.com; http://www.giveramp.com/bot.html; support@giveRAMP.com)",
"RAMPyBot/0.8-dev (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
"Rank Exec (rankexec.com) Reciprocal Link Manager 1.x/bot",
"Rankivabot/3.2 (www.rankiva.com; 3.2; vzmxikn)",
"Rational SiteCheck (Windows NT)",
"RAYSPIDER/Nutch-0.9",
"ReadABlog Spider (compatible; 1.1; feed update; www.readablog.com)",
"RealDownload/4.0.0.4x",
"REAP-crawler Nutch/Nutch-1.0-dev (Reap Project; http://reap.cs.cmu.edu/REAP-crawler/; Reap Project)",
"Reaper [2.03.10-031204] (http://www.sitesearch.ca/reaper/)",
"Reaper/2.0x (+http://www.sitesearch.ca/reaper)",
"REBOL Core 2.x.x.x.x",
"REBOL View 1.x.x.x.x",
"RebusnetBot (+http://www.rebusnet.biz)",
"RebusnetPADBot/1.5x (+http://www.rebusnet.biz)",
"reciprocal links checker (http://www.recip-links.com/)",
"RedBot/redbot-1.0 (Rediff.com Crawler; redbot at rediff dot com)",
"RedCarpet/1.2 (http://www.redcarpet-inc.com/robots.html)",
"RedCell/0.1 (InfoSec Search Bot (Coming Soon); http://www.telegenetic.net/bot.html; lhall@telegenetic.net)",
"RedCell/0.1 (RedCell; telegenetic.net/bot.html; lhall_at_telegenetic.net)",
"RedKernel WWW-Spider 2/0 (+http://www-spider.redkernel-softwares.com/)",
"REL Link Checker Lite x.x",
"RepoMonkey Bait & Tackle/v1.01",
"Rewebber/1.2 libwww-perl/5.41",
"rico/0.1",
"RixBot (http://babelserver.org/rix)",
"RMA/1.0 (compatible; RealMedia)",
"RMA/1.0 (compatible; RealMedia)",
"RoboCrawl (http://www.canadiancontent.net)",
"RoboCrawl (www.canadiancontent.net)",
"RoboPal (http://www.findpal.com/)",
"Robot/www.pj-search.com",
"Robot: NutchCrawler- Owner: wdavies@acm.org",
"Robot@SuperSnooper.Com",
"Robozilla/1.0",
"Rome Client (http://tinyurl.com/64t5n) Ver: 0.9",
"Rotondo/3.1 libwww/5.3.1",
"RPT-HTTPClient/0.3-x",
"RRC (crawler_admin@bigfoot.com)",
"RssBandit/1.5.0.10 (.NET CLR 1.1.4322.2407; WinNT 5.1.2600.0; http://www.rssbandit.org) (.NET CLR 1.1.4322.2407; WinNT 5.1.2600.0; )",
"RSSMicro.com RSS/Atom Feed Robot",
"RSSOwl/1.2.3 2006-11-26 (Windows; U; zhtw)",
"RSSOwl/1.2.4 Preview Release 2007-04-15 (Windows; U; zhtw)",
"RssReader/1.0.xx.x (http://www.rssreader.com) Microsoft Windows NT 5.1.2600.0",
"RSurf15a 41",
"RSurf15a 51",
"RSurf15a 81",
"Rubbot/1.0 (+http://rubhub.com/)",
"RufusBot (Rufus Web Miner; http://64.124.122.252/feedback.html)",
"RufusBot (Rufus Web Miner; http://www.webaroo.com/rooSiteOwners.html)",
"Rumours-Agent",
"RX Bar",
"S&L Spider (http://search.hirners.com/)",
"S.T.A.L.K.E.R. (http://www.seo-tools.net/en/bot.aspx)",
"SafariBookmarkChecker (+http://www.coriolis.ch/)",
"sait/Nutch-0.9 (SAIT Research; http://www.samsung.com)",
"SandCrawler - Compatibility Testing",
"savvybot/0.2",
"SBIder/0.7 (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html)",
"SBIder/0.8-dev (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html)",
"SBL-BOT (http://sbl.net)",
"ScanWeb",
"ScholarUniverse/0.8 (Nutch;+http://scholaruniverse.com/bot.jsp; fetch-agent@scholaruniverse.com)",
"schwarzmann.biz-Spider_for_paddel.org+(http://www.innerprise.net/usp-spider.asp)",
"Science Traveller International 1X/1.0",
"ScollSpider/2.0 (+http://www.webwobot.com/ScollSpider.php)",
"Scooter-3.0.EU",
"Scooter-3.0.FS",
"Scooter-3.0.HD",
"Scooter-3.0.VNS",
"Scooter-3.0QI",
"Scooter-3.2",
"Scooter-3.2.BT",
"Scooter-3.2.DIL",
"Scooter-3.2.EX",
"Scooter-3.2.JT",
"Scooter-3.2.NIV",
"Scooter-3.2.SF0",
"Scooter-3.2.snippet",
"Scooter-3.3dev",
"Scooter-ARS-1.1",
"Scooter-ARS-1.1-ih",
"scooter-venus-3.0.vns",
"Scooter-W3-1.0",
"Scooter-W3.1.2",
"Scooter/1.0",
"Scooter/1.0 scooter@pa.dec.com",
"Scooter/1.1 (custom)",
"Scooter/2.0 G.R.A.B. V1.1.0",
"Scooter/2.0 G.R.A.B. X2.0",
"Scooter/3.3",
"Scooter/3.3.QA.pczukor",
"Scooter/3.3.vscooter",
"Scooter/3.3_SF",
"Scooter2_Mercator_x-x.0",
"Scooter_bh0-3.0.3",
"Scooter_trk3-3.0.3",
"Scope (Mars+)",
"ScoutAbout",
"ScoutAnt/0.1; +http://www.ant.com/what_is_ant.com/",
"scoutmaster",
"Scrubby/2.x (http://www.scrubtheweb.com/)",
"Scrubby/3.0 (+http://www.scrubtheweb.com/help/technology.html)",
"Search+",
"Search-Engine-Studio",
"search.ch V1.4",
"search.ch V1.4.2 (spiderman@search.ch; http://www.search.ch)",
"Search/1.0 (http://www.innerprise.net/es-spider.asp)",
"searchbot admin@google.com",
"SearchByUsa/2 (SearchByUsa; http://www.SearchByUsa.com/bot.html; info@SearchByUsa.com)",
"SearchdayBot",
"SearchExpress Spider0.99",
"SearchGuild/DMOZ/Experiment (searchguild@gmail.com)",
"SearchGuild_DMOZ_Experiment (chris@searchguild.com)",
"Searchit-Now Robot/2.2 (+http://www.searchit-now.co.uk)",
"Searchmee! Spider v0.98a",
"SearchSight/2.0 (http://SearchSight.com/)",
"SearchSpider.com/1.1",
"Searchspider/1.2 (SearchSpider; http://www.searchspider.com; webmaster@searchspider.com)",
"SearchTone2.0 - IDEARE",
"Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/0.3",
"Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.0 (XDF)",
"Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2",
"Seeker.lookseek.com",
"semaforo.net",
"Semager/1.1 (http://www.semager.de/blog/semager-bots/)",
"Semager/1.x (http://www.semager.de)",
"semanticdiscovery/0.x",
"Sensis Web Crawler (search_comments\at\sensis\dot\com\dot\au)",
"Sensis.com.au Web Crawler (search_comments\at\sensis\dot\com\dot\au)",
"SeznamBot/1.0",
"SeznamBot/1.0 (+http://fulltext.seznam.cz/)",
"SeznamBot/2.0-test (+http://fulltext.sblog.cz/)",
"ShablastBot 1.0",
"******** v1.x.x.xx",
"SharewarePlazaFileCheckBot/1.0+(+http://www.SharewarePlaza.com)",
"Shim Crawler",
"Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)",
"ShopWiki/1.0 ( +http://www.shopwiki.com/)",
"ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)",
"Shoula.com Crawler 2.0",
"SietsCrawler/1.1 (+http://www.siets.biz)",
"Sigram/Nutch-1.0-dev (Test agent for Nutch development; http://www.sigram.com/bot.html; bot at sigram dot com)",
"Siigle Orumcex v.001 Turkey (http://www.siigle.com)",
"silk/1.0",
"silk/1.0 (+http://www.slider.com/silk.htm)/3.7",
"SimpleFavPanel/1.2",
"Simpy 1.x; http://www.simpy.com/",
"Simpy/1.x (Simpy; http://www.simpy.com/?ref=bot; feedback at simpy dot com)",
"Sirketcebot/v.01 (http://www.sirketce.com/bot.html)",
"SiteBar/3.x.x (Bookmark Server; http://sitebar.org/)",
"SiteBar/x.x",
"SiteBar/x.x.x (Bookmark Server; http://sitebar.org/)",
"sitecheck.internetseer.com",
"sitecheck.internetseer.com (For more info see: http://sitecheck.internetseer.com)",
"SiteRecon+(xx)",
"SiteSnagger",
"SiteSpider +(http://www.SiteSpider.com/)",
"SiteSucker/1.x.x",
"SiteTaggerBot (http://www.sitetagger.com/bot.htm)",
"SiteTruth.com site rating system",
"SiteWinder",
"SiteXpert",
"Skampy/0.9.x (http://www.skaffe.com/skampy-info.html)",
"Skimpy/0.x (http://www.skaffe.com/skampy-info.html)",
"Skywalker/0.1 (Skywalker; anonymous; anonymous)",
"Slarp/0.1",
"Sleipnir",
"Sleipnir Version 1.xx",
"Sleipnir Version2.x",
"Sleipnir/2.xx",
"Slider_Search_v1-de",
"SlimBrowser",
"Slurp/2.0 (slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Slurp/2.0-KiteWeekly (slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Slurp/si (slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
"Slurpy Verifier/1.0",
"SlySearch (slysearch@slysearch.com)",
"SlySearch/1.0 http://www.plagiarism.org/crawler/robotinfo.html",
"SlySearch/1.x http://www.slysearch.com",
"SmartDownload/1.2.67 (Win32; Jan 12 1999)",
"SmartDownload/1.2.77 (Win32; Feb 1 2000)",
"SmartDownload/1.2.77 (Win32; Jun 19 2001)",
"smartwit.com",
"SmiffyDCMetaSpider/1.0",
"sna-0.0.1 (mikemuzio@msn.com)",
"sna-0.0.1 mikeelliott@hotmail.com",
"snap.com beta crawler v0",
"Snapbot/1.0",
"Snappy/1.1 ( http://www.urltrends.com/ )",
"Snarfer/0.x.x (http://www.snarfware.com/)",
"SnoopRob/x.x",
"Snoopy v1.xx",
"Snoopy v1.xx- : User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2)",
"Snoopy_v0.xx",
"SnykeBot/0.6 (http://www.snyke.com)",
"SocSciBot ()",
"SoftBank/1.0/812SH/SHJ001 Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1",
"SoftHypermarketFileCheckBot/1.0+(+http://www.softhypermaket.com)",
"Softizerbot (http://www.softizer.com)",
"sogou develop spider",
"Sogou Orion spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
"sogou spider",
"Sogou web spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
"sohu agent",
"sohu-search",
"Sosospider+(+http://help.soso.com/webspider.htm)",
"Space Bison/0.02 [fu] (Win67; X; SK)",
"SpeedDownload/1.x",
"speedfind ramBot xtreme 8.1",
"Speedy Spider (Beta/x.x; speedy@entireweb.com)",
"Speedy Spider (Entireweb; Beta/1.0; http://www.entireweb.com/about/search_tech/speedyspider/)",
"Speedy_Spider (http://www.entireweb.com)",
"Sphere Scout&v4.0 - scout at sphere dot com",
"Sphider",
"Spida/0.1",
"Spider-Sleek/2.0 (+http://search-info.com/linktous.html)",
"spider.batsch.com",
"Spider.TerraNautic.net - v:1.04",
"spider.yellopet.com - www.yellopet.com",
"Spider/maxbot.com admin@maxbot.com",
"SpiderKU/0.x",
"SpiderMan",
"SpiderMonkey/7.0x (SpiderMonkey.ca info at http://spidermonkey.ca/sm.shtml)",
"Spinne/2.0",
"Spinne/2.0 med",
"Spinne/2.0 med_AH",
"Spock Crawler (http://www.spock.com/crawler)",
"sportsuchmaschine.de-Robot (Version: 1.02- powered by www.sportsuchmaschine.de)",
"sproose/0.1-alpha (sproose crawler; http://www.sproose.com/bot.html; crawler@sproose.com)",
"SQ Webscanner",
"Squid-Prefetch",
"squidclam",
"SquidClamAV_Redirector 1.x.x",
"Sqworm/2.9.81-BETA (beta_release; 20011102-760; i686-pc-linux-gnu)",
"Sqworm/2.9.85-BETA (beta_release; 20011115-775; i686-pc-linux-gnu)",
"Sqworm/2.9.89-BETA (beta_release; 20020130-839; i686-pc-linux-gnu) ",
"SSurf15a 11 ",
"StackRambler/x.x ",
"Stamina/1.4",
"Star Downloader",
"StarDownloader/1.xx",
"stat statcrawler@gmail.com",
"Steeler/1.x (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)",
"Steeler/3.3 (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)",
"Strategic Board Bot (+http://www.strategicboard.com)",
"Strategic Board Bot (+http://www.strategicboard.com)",
"Submission Spider at surfsafely.com",
"suchbaer.de",
"suchbaer.de (CrawlerAgent v0.103)",
"suchbot",
"Suchknecht.at-Robot",
"suchpadbot/1.0 (+http://www.suchpad.de)",
"Sunrise XP/2.x",
"Sunrise/0.42g (Windows XP)",
"SuperBot/x.x (Win32)",
"SuperBot/x.x.x.xx (Windows XP)",
"Superdownloads Spiderman",
"SURF ",
"SurferF3 1/0",
"SurfMaster",
"SurveyBot/2.2 <a href='http://www.whois.sc'>Whois Source</a>",
"SurveyBot/2.3 (Whois Source)",
"suzuran",
"SWB/V1.4 (HP)",
"swbot/0.9c libwww/5.3.1",
"Swooglebot/2.0. (+http://swoogle.umbc.edu/swooglebot.htm)",
"SWSBot-Images/1.2 http://www.smartwaresoft.com/swsbot12.html",
"SygolBot http://www.sygol.net",
"Sylera/1.2.x",
"SyncBot",
"SyncIT/x.x",
"Syndirella/0.91pre",
"SynoBot",
"Syntryx ANT Scout Chassis Pheromone; Mozilla/4.0 compatible crawler",
"Szukacz/1.x",
"Szukacz/1.x (robot; www.szukacz.pl/jakdzialarobot.html; szukacz@proszynski.pl)",
"T-Online Browser",
"tags2dir.com/0.8 (+http://tags2dir.com/directory/)",
"Tagword (http://tagword.com/dmoz_survey.php)",
"Tagyu Agent/1.0",
"Talkro Web-Shot/1.0 (E-mail: webshot@daumsoft.com- Home: http://222.122.15.190/webshot)",
"TAMU_CS_IRL_CRAWLER/1.0",
"targetblaster.com/0.9k",
"TargetYourNews.com bot",
"TCDBOT/Nutch-0.8 (PhD student research;http://www.tcd.ie; mcgettrs at t c d dot IE)",
"TE",
"TeamSoft WinInet Component",
"TECOMAC-Crawler/0.x",
"Tecomi Bot (http://www.tecomi.com/bot.htm)",
"Teleport Pro/1.2x(.1xxx)",
"Teoma MP",
"teomaagent crawler-admin@teoma.com",
"teomaagent1 [crawler-admin@teoma.com]",
"teoma_agent1",
"Teradex Mapper; mapper@teradex.com; http://www.teradex.com",
"TeragramCrawler",
"terraminds-bot/1.0 (support@terraminds.de)",
"TerrawizBot/1.0 (+http://www.terrawiz.com/bot.html)",
"Test spider",
"TestCrawler/Nutch-0.9 (Testing Crawler for Research ; http://balihoo.com/index.aspx; tgautier at balihoo dot com)",
"The Expert HTML Source Viewer (http://www.expert-html.com)",
"TheRarestParser/0.2a (http://therarestwords.com/)",
"TheSuBot/0.1 (www.thesubot.de)",
"thumbshots-de-Bot (Version: 1.02- powered by www.thumbshots.de)",
"timboBot/0.9 http://www.breakingblogs.com/timbo_bot.html",
"http://www.timelyweb.com/",
"TinEye/1.1 (http://tineye.com/crawler.html)",
"tivraSpider/1.0 (crawler@tivra.com)",
"TJG/Spider",
"TJvMultiHttpGrabber Component",
"Tkensaku/x.x(http://www.tkensaku.com/q.html)",
"toCrawl/UrlDispatcher",
"Topodia/1.2-dev (Topodia - Crawler for HTTP content indexing; http://www.topodia.com/; support@topodia.com)",
"TOPOS robot/1.1 (http://www.topos.com.ua/)",
"Toutatis x-xx.x (hoppa.com)",
"Toutatis x.x (hoppa.com)",
"Toutatis x.x-x",
"traazibot/testengine (+http://www.traazi.de)",
"Trailfire-bot/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
"Trailfire-bot/0.7.1 (Trailfire page content analyzer; http://trailfire.com; info@trailfire.com)",
"Trailfire/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)",
"Trampelpfad-Spider",
"Trampelpfad-Spider-v0.1",
"tricosMetaCheck 1.2216-08-1999 (http://www.tricos.com/metacheck)",
"TSurf15a 11",
"TulipChain/5.x (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://java.sun.com/) Linux/2.4.17",
"TulipChain/5.xx (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://apple.com/) Mac_OS_X/10.2.8",
"Tumblr/1.0 RSS syndication (+http://www.tumblr.com/) (support@tumblr.com)",
"TurnitinBot/x.x (http://www.turnitin.com/robot/crawlerinfo.html)",
"Turnpike Emporium LinkChecker/0.1",
"TutorGig/1.5 (+http://www.tutorgig.com/crawler)",
"Tutorial Crawler 1.4 (http://www.tutorgig.com/crawler)",
"Twiceler www.cuill.com/robots.html",
"Twiceler-0.9 http://www.cuill.com/twiceler/robot.html",
"Twisted PageGetter",
"Twitturly / v0.x",
"Twotrees Reactive Filter V2.0",
"Tycoon Agent/Nutch-1.0-dev",
"TygoBot",
"TygoProwler",
"UCmore",
"UCMore Crawler App",
"UCWEB5.1",
"UDM",
"UdmSearch/3.1.x",
"UIowaCrawler/1.0",
"UKWizz/Nutch-0.8.1 (UKWizz Nutch crawler; http://www.ukwizz.com/)",
"Ultraseek",
"Under the Rainbow 2.2",
"unknownght.com Web Server IIS vs Apache Survey. See Results at www.DNSRight.com",
"UofTDB_experiment (leehyun@cs.toronto.edu)",
"UP.Browser/3.01-IG01 UP.Link/3.2.3.4",
"updated/0.1-alpha (updated crawler; http://www.updated.com; crawler@updated.com)",
"updated/0.1beta (updated.com; http://www.updated.com; crawler@updated.om)",
"UPG1 UP/4.0 (compatible; Blazer 1.0)",
"Uptimebot",
"UptimeBot(www.uptimebot.com)",
"URI::Fetch/0.06",
"URL Spider Pro/x.xx (innerprise.net)",
"URLBase/6.x",
"URLBlaze",
"urlfan-bot/1.0; +http://www.urlfan.com/site/bot/350.html",
"URLGetFile",
"URL_Spider_Pro/x.x",
"URL_Spider_Pro/x.x+(http://www.innerprise.net/usp-spider.asp)",
"User-Agent: BoardReader Favicon Fetcher /1.0 info@boardreader.com",
"User-Agent: BoardReader Image Fetcher /1.0 info@boardreader.com",
"User-Agent: LjSEEK Picture-Bot /1.0 contact@ljseek.com",
"User-Agent: FileHeap! file downloader (http://www.fileheap.com)",
"User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
"User-Agent: Mozilla/4.0 (SKIZZLE! Distributed Internet Spider v1.0 - www.SKIZZLE.com)",
"user-agent=Mozilla/3.01Gold",
"USyd-NLP-Spider (http://www.it.usyd.edu.au/~vinci/bot.html)",
"UtilMind HTTPGet",
"Utopia WebWasher 3.0",
"u*******/1500",
"VadixBot",
"Vagabondo-WAP/2.0 (webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)/1.0 Profile",
"Vagabondo/1.x MT (webagent@wise-guys.nl)",
"Vagabondo/2.0 MT",
"Vagabondo/2.0 MT (webagent at wise-guys dot nl)",
"Vagabondo/2.0 MT (webagent@NOSPAMwise-guys.nl)",
"Vagabondo/3.0 (webagent at wise-guys dot nl)",
"Vakes/0.01 (Vakes; http://www.vakes.com/; search@vakes.com)",
"VayalaCreep-v0.0.1 (haploid@haploid.com)",
"Vayala|Creep-v0.0.1 (codepoet@wildties.com)",
"vb wininet",
"versus 0.2 (+http://versus.integis.ch)",
"versus crawler eda.baykan@epfl.ch",
"Verticrawlbot",
"VeryGoodSearch.com.DaddyLongLegs",
"verzamelgids.nl - Networking4all Bot/x.x",
"Verzamelgids/2.2 (http://www.verzamelgids.nl)",
"Vespa Crawler",
"virus_detector (virus_harvester@securecomputing.com)",
"VisBot/2.0 (Visvo.com Crawler; http://www.visvo.com/bot.html; bot@visvo.com)",
"Visicom Toolbar",
"Vision Research Lab image spider at vision.ece.ucsb.edu",
"VLC media player - version 0.8.5 Janus - (c) 1996-2006 the VideoLAN team",
"VMBot/0.x.x (VMBot; http://www.VerticalMatch.com/; vmbot@tradedot.com)",
"Vortex/2.2 (+http://marty.anstey.ca/robots/vortex/)",
"voyager-hc/1.0",
"voyager/1.0",
"VSE/1.0 (testcrawler@hotmail.com)",
"VSE/1.0 (testcrawler@vivisimo.com)",
"vspider",
"vspider/3.x",
"VWBOT/Nutch-0.9-dev (VWBOT Nutch Crawler; http://vwbot.cs.uiuc.edu;+vwbot@cs.uiuc.edu",
"W3C-checklink/3.x.x.x libwww-perl/5.xx",
"W3C-checklink/4.x [4.xx] libwww-perl/5.xxx",
"W3C-WebCon/5.x.x libwww/5.x.x",
"W3CLineMode/5.4.0 libwww/5.x.x",
"W3CRobot/5.4.0 libwww/5.4.0",
"W3C_Validator/1.xxx libwww-perl/5.xx",
"w3m/0.x.xx",
"W3SiteSearch Crawler_v1.1 http://www.w3sitesearch.de",
"wadaino.jp-crawler 0.2 (http://wadaino.jp/)",
"WannaBe (Macintosh; PPC)",
"WapOnWindows 1.0",
"Watchfire WebXM 1.0",
"WAVcheck 1.0.x (http://www.webbanalys.se/apps/WAVcheck/)",
"Wavefire/0.8-dev (Wavefire; http://www.wavefire.com; info@wavefire.com)",
"Waypath development crawler - info at waypath dot com",
"Waypath Scout v2.x - info at waypath dot com",
"WDG_Validator/1.1",
"Web Image Collector",
"Web Link Validator 1.5",
"Web Snooper",
"Web-Bot V1.03",
"Web-Robot/5.0 (en-US; web-robot.com/policy.html) Web-Robot Crawler/2.0.3",
"web2express.org/Nutch-0.9-dev (leveled playing field; http://web2express.org/; info at web2express.org)",
"WebAlta Crawler/1.2.1 (http://www.webalta.ru/bot.html)",
"WebarooBot (Webaroo Bot; http://64.124.122.252/feedback.html)",
"WebarooBot (Webaroo Bot; http://www.webaroo.com/rooSiteOwners.html)",
"WebAuto/3.4xxx (WinNT; I)",
"webbandit/4.xx.0",
"WebBug/5.x",
"Webclipping.com",
"webcollage/1.xx",
"WebCompass 2.0",
"WebCopier vx.x",
"WebCopier vx.xa",
"WebCorp/1.0",
"webcrawl.net",
"WebDownloader for X x.xx",
"Webdup/0.9",
"WebFetch",
"webfetch/5.x.x",
"WebFilter Robot 1.0",
"WebFilter Robot 1.x",
"WebFindBot(http://www.web-find.com)",
"Webglimpse 2.xx.x (http://webglimpse.net)",
"webGobbler/1.x.x",
"webhack",
"WebImages 0.3 ( http://herbert.groot.jebbink.nl/?app=WebImages )",
"WebLight/4.x.x (support@illumit.com; http://www.illumit.com/Products/weblight/)",
"Weblink's checker/",
"Weblog Attitude Diffusion 1.0",
"WebMiner/x.x [en] (Win98; I)",
"WeBoX/0.xx",
"WebPix 1.0 (www.netwu.com)",
"WebQL",
"WebRACE/1.1 (University of Cyprus- Distributed Crawler)",
"WebRankSpider/1.37 (+http://ulm191.server4you.de/crawler/)",
"WebReaper vx.x - www.webreaper.net",
"WebReaper [info@webreaper.net]",
"WebReaper [webreaper@webreaper.net]",
"WebSearch.COM.AU/3.0.1 (The Australian Search Engine; http://WebSearch.COM.AU; Search@WebSearch.COM.AU)",
"WebSearchBench WebCrawler v0.1(Experimental)",
"WebSearchBench WebCrawler V1.0 (Beta)- Prof. Dr.-Ing. Christoph Lindemann- Universität Dortmund- cl@cs.uni-dortmund.de- http://websearchbench.cs.uni-dortmund.de/",
"Website Explorer/0.9.x.x",
"Website *********",
"WebsiteWorth v1.0",
"Webspinne/1.0 webmaster@webspinne.de",
"Websquash.com (Add url robot)",
"WebStat/1.0 (Unix; beta; 20040314)",
"Webster v0.3 ( http://webster.healeys.net/ )",
"webster-internet.de pad browser",
"WebStripper/2.xx",
"WebTrafficExpress/x.0",
"WebTrends/3.0 (WinNT)",
"WebVac (webmaster@pita.stanford.edu)",
"WebVal/1.0",
"Webverzeichnis.de - Telefon: 01908 / 26005",
"WebVulnCrawl.blogspot.com/1.0 libwww-perl/5.803",
"WebWatcherMonitor/2.01",
"WebZIP/x.x (http://www.spidersoft.com)",
"Wells Search II",
"WEP Search 00",
"West Wind Internet Protocols 4.xx",
"WFARC",
"Wget/1.x(.x)GNU wget http://www.gnu.org/software/wget/wget.html - file downloader",
"Wget/1.x+cvs-stable (Red Hat modified)",
"Wget/1.x.x+cvs",
"Whatsup/x.x",
"whatUseek_winona/3.0",
"WhizBang! Lab",
"Wildsoft Surfer",
"Willow Internet Crawler by Twotrees V2.1",
"WinampMPEG/2.00 (larbin@unspecified.mail)",
"WincerSong Agent v1.0",
"Windows-Media-Player/10.00.00.xxxx",
"WinGet 1.1",
"WinHTTP Example/1.0",
"WinkBot/0.06 (Wink.com search engine web crawler; http://www.wink.com/Wink:WinkBot; winkbot@wink.com)",
"WinPodder (http://winpodder.com)",
"WinWAP/3.x (3.x.x.xx; Win32) (Google WAP Proxy/1.0)",
"WISEbot/1.0 (WISEbot@koreawisenut.com; http://wisebot.koreawisenut.com)",
"WiseWire-Spider2",
"wish-project (http://wish.slis.tsukuba.ac.jp/)",
"WordChampBot",
"WordPress/x.x.x.x PHP/4.x.xx",
"worio heritrix bot (+http://worio.com/)",
"woriobot ( http://www.worio.com/)",
"WorldLight",
"WorQmada/1.0",
"Wotbox/alpha0.6 (bot@wotbox.com; http://www.wotbox.com)",
"Wotbox/alpha0.x.x (bot@wotbox.com; http://www.wotbox.com) Java/1.4.1_02",
"WSB WebCrawler V1.0 (Beta)- cl@cs.uni-dortmund.de",
"wume_crawler/1.1 (http://wume.cse.lehigh.edu/~xiq204/crawler/)",
"Wusage/x.0@boutell.com",
"Wwlib/Linux",
"WWSBOT 1.x [--- http://www.analyzer.nu ---]",
"WWW-Mechanize/1.1x",
"www.arianna.it",
"www.business-socket.com registry verify/1.x",
"www.doweb.co.uk crawler",
"www4mail/2.x libwww-FM/2.14 (Unix; I)",
"WWWC/1.0x",
"WWWeasel Robot v1.00 (http://wwweasel.de)",
"WWWOFFLE/2.x",
"wwwster/1.x (Beta- mailto:gue@cis.uni-muenchen.de)",
"wxDownload Fast",
"X-Crawler ",
"Xaldon WebSpider",
"Xenu Link Sleuth 1.xx",
"Xenu's Link Sleuth 1.x[a-z]",
"Xerka WebBot v1.0.0 [UPVOpenDir]",
"xine/1.0",
"xirq/0.1-beta (xirq; http://www.xirq.com; xirq@xirq.com)",
"XMLSlurp/0.1 libwww-perl/5.805",
"XRL/2.00b1 (Linux; i686; en-us) (+http://metamark.net/about)",
"Xylix",
"xyro_(xcrawler@cosmos.inria.fr)",
"Y!J-BSC/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",
"Y!J-SRD/1.0",
"Y!J/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)",
"Y!OASIS/TEST no-ad Mozilla/4.08 [en] (X11; I; FreeBSD 2.2.8-STABLE i386)",
"Y!TunnelPro",
"yacy (www.yacy.net; v20040602; i386 Linux 2.4.26-gentoo-r13; java 1.4.2_06; MET/en)",
"yacybot (x86 Windows XP 5.1; java 1.5.0_06; Europe/de) yacy.net",
"Yahoo Pipes 1.0",
"Yahoo! Mindset",
"Yahoo-Blogs/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html )",
"Yahoo-MMAudVid/1.0 (mms dash mmaudvidcrawler dash support at yahoo dash inc dot com)",
"Yahoo-MMAudVid/2.0(mms dash mm aud vid crawler dash support at yahoo dash inc.com ;Mozilla 4.0 compatible; MSIE 7.0;Windows NT 5.0; .NET CLR 2.0)",
"Yahoo-MMCrawler/3.x (mm dash crawler at trd dot overture dot com)",
"Yahoo-Test/4.0",
"Yahoo-VerticalCrawler-FormerWebCrawler/3.9 crawler at trd dot overture dot com; http://www.alltheweb.com/help/webmaster/crawler",
"YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide)",
"YahooSeeker-Testing/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/)",
"YahooSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/)",
"YahooSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/yahooseeker.html)",
"YahooSeeker/1.1 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/)",
"YahooSeeker/bsv3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html )",
"YahooSeeker/CafeKelsa-dev (compatible; Konqueror/3.2; FreeBSD ;cafekelsa-dev-webmaster@yahoo-inc.com )",
"Yandex/1.01.001 (compatible; Win16; I)",
"yarienavoir.net/0.2",
"Yeti",
"yggdrasil/Nutch-0.9 (yggdrasil biorelated search engine; www dot biotec dot tu minus dresden do de slash schroeder; heiko dot dietze at biotec dot tu minus dresden dot de)",
"YodaoBot/1.0 (http://www.yodao.com/help/webmaster/spider/; )",
"yoofind/yoofind-0.1-dev (yoono webcrawler; http://www.yoono.com ; MyEmail)",
"yoogliFetchAgent/0.1",
"yoono/1.0 web-crawler/1.0",
"YottaCars_Bot/4.12 (+http://www.yottacars.com) Car Search Engine ",
"YottaShopping_Bot/4.12 (+http://www.yottashopping.com) Shopping Search Engine",
"YTunnelPro",
"Z-Add Link Checker (http://w3.z-add.co.uk/linkcheck/)",
"Zao-Crawler",
"Zao-Crawler 0.2b",
"Zao/0.1 (http://www.kototoi.org/zao/)",
"ZBot/1.00 (icaulfield@zeus.com)",
"Zearchit",
"ZeBot_lseek.net (bot@ze.bz)",
"ZeBot_www.ze.bz (ze.bz@hotmail.com)",
"zedzo.digest/0.1 (http://www.zedzo.com/)",
"Zend_Http_Client",
"zerxbot/Version 0.6 libwww-perl/5.79",
"Zeus ThemeSite Viewer Webster Pro V2.9 Win32",
"Zeus xxxxx Webster Pro V2.9 Win32",
"Zeusbot/0.07 (Ulysseek's web-crawling robot; http://www.zeusbot.com; agent@zeusbot.com)",
"Ziggy -- The Clown From Hell!!",
"ZipppBot/0.xx (ZipppBot; http://www.zippp.net; webmaster@zippp.net)",
"ZIPPPCVS/0.xx (ZipppBot/.xx;http://www.zippp.net; webmaster@zippp.net)",
"Zippy v2.0 - Zippyfinder.com",
"Zoo Tycoon 2 Client -- http://www.zootycoon.com",
"ZoomSpider - wrensoft.com",
"zspider/0.9-dev http://feedback.redkolibri.com/",
"ZyBorg/1.0 (ZyBorg@WISEnut.com; http://www.WISEnut.com)",
"Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/)",
"Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)",
"voyager/2.0 (http://www.kosmix.com/html/crawler.html)");
?>
