| 1 | {"!Susie (http://www.sync2it.com/susie)":1,"<a href='http://www.unchaos.com/'> UnChaos </a> From Chaos To Order Hybrid Web Search Engine.(vadim_gonchar@unchaos.com)":1,"<a href='http://www.unchaos.com/'> UnChaos Bot Hybrid Web Search Engine. </a> (vadim_gonchar@unchaos.com)":1,"<b> UnChaosBot From Chaos To Order UnChaos Hybrid Web Search Engine at www.unchaos.com </b> (info@unchaos.com)":1,"<http://www.sygol.com/> http://www.sygol.com":1,"( Robots.txt Validator http://www.searchengineworld.com/cgi-bin/robotcheck.cgi )":1,"(DreamPassport/3.0; isao/MyDiGiRabi)":1,"(Privoxy/1.0)":1,"*/Nutch-0.9-dev":1,"+SitiDi.net/SitiDiBot/1.0 (+Have Good Day)":1,"-DIE-KRAEHE- META-SEARCH-ENGINE/1.1 http://www.die-kraehe.de":1,"123spider-Bot (Version: 1.02, powered by www.123spider.de":1,"192.comAgent":1,"1st ZipCommander (Net) - http://www.zipcommander.com/":1,"2Bone_LinkChecker/1.0 libwww-perl/5.64":1,"4anything.com LinkChecker v2.0":1,"8484 Boston Project v 1.0":1,":robot/1.0 (linux) ( admin e-mail: undefined http://www.neofonie.de/loesungen/search/robot.html )":1,"A-Online Search":1,"A1 Keyword Research/1.0.2 (+http://www.micro-sys.dk/products/keyword-research/) miggibot/2007.03.27":1,"A1 Sitemap Generator/1.0 (+http://www.micro-sys.dk/products/sitemap-generator/) miggibot/2006.01.24":1,"aardvark-crawler":1,"AbachoBOT":1,"AbachoBOT (Mozilla compatible)":1,"ABCdatos BotLink/5.xx.xxx#BBL":1,"Aberja Checkomat":1,"abot/0.1 (abot; http://www.abot.com; abot@abot.com)":1,"About/0.1libwww-perl/5.47":1,"Accelatech RSSCrawler/0.4":1,"accoona":1,"Accoona-AI-Agent/1.1.1 (crawler at accoona dot com)":1,"Accoona-AI-Agent/1.1.2 (aicrawler at accoonabot dot com)":1,"Ace Explorer":1,"Ack (http://www.ackerm.com/)":1,"AcoiRobot":1,"Acoon Robot v1.50.001":1,"Acoon Robot v1.52 (http://www.acoon.de)":1,"Acoon-Robot 4.0.x.[xx] (http://www.acoon.de)":1,"Acoon-Robot v3.xx (http://www.acoon.de and http://www.acoon.com)":1,"Acorn/Nutch-0.9 (Non-Profit Search Engine; acorn.isara.org; acorn at isara dot org)":1,"ActiveBookmark 1.x":1,"Activeworlds":1,"ActiveWorlds/3.xx (xxx)":1,"Ad Muncher v4.xx.x":1,"Ad Muncher v4x Build xxxxx":1,"Adaxas Spider (http://www.adaxas.net/)":1,"Advanced Browser (http://www.avantbrowser.com)":1,"AESOP_com_SpiderMan":1,"agadine/1.x.x (+http://www.agada.de)":1,"Agent-SharewarePlazaFileCheckBot/2.0+(+http://www.SharewarePlaza.com)":1,"AgentName/0.1 libwww-perl/5.48":1,"AIBOT/2.1 By +(www.21seek.com A Real artificial intelligence search engine China)":1,"AideRSS/1.0 (aiderss.com)":1,"aipbot/1.0 (aipbot; http://www.aipbot.com; aipbot@aipbot.com)":1,"aipbot/2-beta (aipbot dev; http://aipbot.com; aipbot@aipbot.com)":1,"Akregator/1.2.9; librss/remnants":1,"Aladin/3.324":1,"Alcatel-BG3/1.0 UP.Browser/5.0.3.1.2":1,"Aleksika Spider/1.0 (+http://www.aleksika.com/)":1,"AlertInfo 2.0 (Powered by Newsbrain)":1,"AlkalineBOT/1.3":1,"AlkalineBOT/1.4 (1.4.0326.0 RTM)":1,"Allesklar/0.1 libwww-perl/5.46":1,"Alligator 1.31 (www.nearsoftware.com)":1,"Allrati/1.1 (+)":1,"AltaVista Intranet V2.0 AVS EVAL search@freeit.com":1,"AltaVista Intranet V2.0 Compaq Altavista Eval sveand@altavista.net":1,"AltaVista Intranet V2.0 evreka.com crawler@evreka.com":1,"AltaVista V2.0B crawler@evreka.com":1,"amaya/x.xx libwww/x.x.x":1,"AmfibiBOT":1,"Amfibibot/0.06 (Amfibi Web Search; http://www.amfibi.com; agent@amfibi.com)":1,"Amfibibot/0.07 (Amfibi Robot; http://www.amfibi.com; agent@amfibi.com)":1,"amibot":1,"Amiga-AWeb/3.4.167SE":1,"AmigaVoyager/3.4.4 (MorphOS/PPC native)":1,"AmiTCP Miami (AmigaOS 2.04)":1,"Amoi 8512/R21.0 NF-Browser/3.3":1,"amzn_assoc":1,"AnnoMille spider 0.1 alpha - http://www.annomille.it":1,"annotate_google; http://ponderer.org/download/annotate_google.user.js":1,"Anonymized by ProxyOS: http://www.megaproxy.com":1,"Anonymizer/1.1":1,"AnswerBus (http://www.answerbus.com/)":1,"AnswerChase PROve x.0":1,"AnswerChase x.0":1,"ANTFresco/x.xx":1,"antibot-V1.1.5/i586-linux-2.2":1,"AnzwersCrawl/2.0 (anzwerscrawl@anzwers.com.au;Engine)":1,"Apexoo Spider 1.x":1,"Aplix HTTP/1.0.1":1,"Aplix_SANYO_browser/1.x (Japanese)":1,"Aplix_SEGASATURN_browser/1.x (Japanese)":1,"Aport":1,"appie 1.1 (www.walhello.com)":1,"Apple iPhone v1.1.4 CoreMedia v1.0.0.4A102":1,"Apple-PubSub/65.1.1":1,"ArabyBot (compatible; Mozilla/5.0; GoogleBot; FAST Crawler 6.4; http://www.araby.com;)":1,"ArachBot":1,"Arachnoidea (arachnoidea@euroseek.com)":1,"aranhabot":1,"ArchitextSpider":1,"archive.org_bot":1,"Argus/1.1 (Nutch; http://www.simpy.com/bot.html; feedback at simpy dot com)":1,"Arikus_Spider":1,"Arquivo-web-crawler (compatible; heritrix/1.12.1 +http://arquivo-web.fccn.pt)":1,"ASAHA Search Engine Turkey V.001 (http://www.asaha.com/)":1,"Asahina-Antenna/1.x":1,"Asahina-Antenna/1.x (libhina.pl/x.x ; libtime.pl/x.x)":1,"ask.24x.info":1,"AskAboutOil/0.06-rcp (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@askaboutoil.com)":1,"asked/Nutch-0.8 (web crawler; http://asked.jp; epicurus at gmail dot com)":1,"ASPSeek/1.2.5":1,"ASPseek/1.2.9d":1,"ASPSeek/1.2.x":1,"ASPSeek/1.2.xa":1,"ASPseek/1.2.xx":1,"ASPSeek/1.2.xxpre":1,"ASSORT/0.10":1,"asterias/2.0":1,"AtlocalBot/1.1 +(http://www.atlocal.com/local-web-site-owner.html)":1,"Atomic_Email_Hunter/4.0":1,"Atomz/1.0":1,"atSpider/1.0":1,"Attentio/Nutch-0.9-dev (Attentio's beta blog crawler; www.attentio.com; info@attentio.com)":1,"AU-MIC/2.0 MMP/2.0":1,"AUDIOVOX-SMT5600":1,"augurfind":1,"augurnfind V-1.x":1,"autoemailspider":1,"autohttp":1,"autowebdir 1.1 (www.autowebdir.com)":1,"AV Fetch 1.0":1,"Avant Browser (http://www.avantbrowser.com)":1,"AVSearch-1.0(peter.turney@nrc.ca)":1,"AVSearch-2.0-fusionIdx-14-CompetitorWebSites":1,"AVSearch-3.0(AltaVista/AVC)":1,"AWeb":1,"axadine/ (Axadine Crawler; http://www.axada.de/; )":1,"AxmoRobot - Crawling your site for better indexing on www.axmo.com search engine.":1,"Azureus 2.x.x.x":1,"BabalooSpider/1.3 (BabalooSpider; http://www.babaloo.si; spider@babaloo.si)":1,"BaboomBot/1.x.x (+http://www.baboom.us)":1,"BackStreet Browser 3.x":1,"BaiduImagespider+(+http://www.baidu.jp/search/s308.html)":1,"BaiDuSpider":1,"Baiduspider+(+http://help.baidu.jp/system/05.html)":1,"Baiduspider+(+http://www.baidu.com/search/spider.htm)":1,"Baiduspider+(+http://www.baidu.com/search/spider_jp.html)":1,"Balihoo/Nutch-1.0-dev (Crawler for Balihoo.com search engine - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)":1,"BanBots/1.2 (spider@banbots.com)":1,"Barca/2.0.xxxx":1,"BarcaPro/1.4.xxxx":1,"BarraHomeCrawler (albertof@barrahome.org)":1,"bCentral Billing Post-Process":1,"bdcindexer_2.6.2 (research@bdc)":1,"BDFetch":1,"BDNcentral Crawler v2.3 [en] (http://www.bdncentral.com/robot.html) (X11; I; Linux 2.0.44 i686)":1,"BeamMachine/0.5 (dead link remover of www.beammachine.net)":1,"beautybot/1.0 (+http://www.uchoose.de/crawler/beautybot/)":1,"BebopBot/2.5.1 ( crawler http://www.apassion4jazz.net/bebopbot.html )":1,"BeebwareDirectory/v0.01":1,"Big Brother (http://pauillac.inria.fr/~fpottier/)":1,"Big Fish v1.0":1,"BigBrother/1.6e":1,"BigCliqueBOT/1.03-dev (bigclicbot; http://www.bigclique.com; bot@bigclique.com)":1,"BIGLOTRON (Beta 2;GNU/Linux)":1,"Bigsearch.ca/Nutch-x.x-dev (Bigsearch.ca Internet Spider; http://www.bigsearch.ca/; info@enhancededge.com)":1,"Bilbo/2.3b-UNIX":1,"BilgiBetaBot/0.8-dev (bilgi.com (Beta) ; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)":1,"BilgiBot/1.0(beta) (http://www.bilgi.com/; bilgi at bilgi dot com)":1,"billbot wjj@cs.cmu.edu":1,"Bitacle bot/1.1":1,"Bitacle Robot (V:1.0;) (http://www.bitacle.com)":1,"Biyubi/x.x (Sistema Fenix; G11; Familia Toledo; es-mx)":1,"BlackBerry7520/4.0.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/5.0.3.3 UP.Link/5.1.2.12 (Google WAP Proxy/1.0)":1,"BlackWidow":1,"Blaiz-Bee/1.0 (+http://www.blaiz.net)":1,"Blaiz-Bee/2.00.8222 (BE Internet Search Engine http://www.rawgrunt.com)":1,"Blaiz-Bee/2.00.xxxx (+http://www.blaiz.net)":1,"BlitzBOT@tricus.net":1,"BlitzBOT@tricus.net (Mozilla compatible)":1,"BlockNote.Net":1,"BlogBot/1.x":1,"BlogBridge 2.13 (http://www.blogbridge.com/)":1,"Bloglines Title Fetch/1.0 (http://www.bloglines.com)":1,"Bloglines-Images/0.1 (http://www.bloglines.com)":1,"Bloglines/3.1 (http://www.bloglines.com)":1,"BlogMap (http://www.feedmap.net)":1,"Blogpulse (info@blogpulse.com)":1,"BlogPulseLive (support@blogpulse.com)":1,"BlogSearch/1.x +http://www.icerocket.com/":1,"blogsearchbot-pumpkin-3":1,"BlogsNowBot, V 2.01 (+http://www.blogsnow.com/)":1,"BlogVibeBot-v1.1 (spider@blogvibe.nl)":1,"blogWatcher_Spider/0.1 (http://www.lr.pi.titech.ac.jp/blogWatcher/)":1,"BlogzIce/1.0 (+http://icerocket.com; rhodes@icerocket.com)":1,"BlogzIce/1.0 +http://www.icerocket.com/":1,"BloobyBot":1,"Bloodhound/Nutch-0.9 (Testing Crawler for Research - obeys robots.txt and robots meta tags ; http://balihoo.com/index.aspx; robot at balihoo dot com)":1,"bluefish 0.6 HTML editor":1,"BMCLIENT":1,"BMLAUNCHER":1,"Bobby/4.0.x RPT-HTTPClient/0.3-3E":1,"boitho.com-dc/0.xx (http://www.boitho.com/dcbot.html)":1,"boitho.com-robot/1.x":1,"boitho.com-robot/1.x (http://www.boitho.com/bot.html)":1,"Bookdog/x.x":1,"Bookmark Buddy bookmark checker (http://www.bookmarkbuddy.net/)":1,"Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/]":1,"Bookmark Renewal Check Agent [http://www.bookmark.ne.jp/] (Version 2.0beta)":1,"BookmarkBase(2/;http://bookmarkbase.com)":1,"Bot mailto:craftbot@yahoo.com":1,"BPImageWalker/2.0 (www.bdbrandprotect.com)":1,"BravoBrian bstop.bravobrian.it":1,"BravoBrian SpiderEngine MarcoPolo":1,"BrightCrawler (http://www.brightcloud.com/brightcrawler.asp)":1,"BruinBot (+http://webarchive.cs.ucla.edu/bruinbot.html) ":1,"BSDSeek/1.0":1,"BStop.BravoBrian.it Agent Detector":1,"BTbot/0.x (+http://www.btbot.com/btbot.html)":1,"BTWebClient/180B(9704)":1,"BuildCMS crawler (http://www.buildcms.com/crawler)":1,"Bulkfeeds/r1752 (http://bulkfeeds.net/)":1,"BullsEye":1,"bumblebee@relevare.com":1,"BunnySlippers":1,"BurstFindCrawler/1.1 (crawler.burstfind.com; http://crawler.burstfind.com; crawler@burstfind.com)":1,"Buscaplus Robi/1.0 (http://www.buscaplus.com/robi/)":1,"BW-C-2.0":1,"bwh3_user_agent":1,"Cabot/Nutch-0.9 (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)":1,"Cabot/Nutch-1.0-dev (Amfibi's web-crawling robot; http://www.amfibi.com/cabot/; agent@amfibi.com)":1,"CamelHttpStream/1.0":1,"Cancer Information and Support International;":1,"carleson/1.0":1,"Carnegie_Mellon_University_Research_WebBOT-->PLEASE READ-->http://www.andrew.cmu.edu/~brgordon/webbot/index.html http://www.andrew.cmu.edu/~brgordon/webbot/index.html":1,"Carnegie_Mellon_University_WebCrawler http://www.andrew.cmu.edu/~brgordon/webbot/index.html":1,"Catall Spider":1,"CazoodleBot/CazoodleBot-0.1 (CazoodleBot Crawler; http://www.cazoodle.com/cazoodlebot; cazoodlebot@cazoodle.com)":1,"CCBot/1.0 (+http://www.commoncrawl.org/bot.html)":1,"ccubee/x.x":1,"CDR/1.7.1 Simulator/0.7(+http://timewe.net) Profile/MIDP-1.0 Configuration/CLDC-1.0":1,"CE-Preload":1,"CentiverseBot":1,"CentiverseBot - investigator":1,"CentiverseBot/3.0 (http://www.centiverse-project.net)":1,"Ceramic Tile Installation Guide (http://www.floorstransformed.com)":1,"CERN-LineMode/2.15":1,"cfetch/1.0":1,"CFNetwork/x.x":1,"cg-eye interactive":1,"Charon/1.x (Amiga)":1,"Chat Catcher/1.0":1,"Checkbot/1.xx LWP/5.xx":1,"CheckLinks/1.x.x":1,"CheckUrl":1,"CheckWeb":1,"Chilkat/1.0.0 (+http://www.chilkatsoft.com/ChilkatHttpUA.asp)":1,"China Local Browse 2.6":1,"Chitika ContentHit 1.0":1,"ChristCRAWLER 2.0":1,"CHttpClient by Open Text Corporation":1,"CipinetBot (http://www.cipinet.com/bot.html)":1,"Cityreview Robot (+http://www.cityreview.org/crawler/)":1,"CJ Spider/":1,"CJB.NET Proxy":1,"ClariaBot/1.0":1,"Claymont.com":1,"CloakDetect/0.9 (+http://fulltext.seznam.cz/)":1,"Clushbot/2.x (+http://www.clush.com/bot.html)":1,"Clushbot/3.x-BinaryFury (+http://www.clush.com/bot.html)":1,"Clushbot/3.xx-Ajax (+http://www.clush.com/bot.html)":1,"Clushbot/3.xx-Hector (+http://www.clush.com/bot.html)":1,"Clushbot/3.xx-Peleus (+http://www.clush.com/bot.html)":1,"COAST WebMaster Pro/4.x.x.xx (Windows NT)":1,"CoBITSProbe":1,"Cocoal.icio.us/1.0 (v36) (Mac OS X; http://www.scifihifi.com/cocoalicious)":1,"Cogentbot/1.X (+http://www.cogentsoftwaresolutions.com/bot.html)":1,"ColdFusion":1,"ColdFusion (BookmarkTracker.com)":1,"collage.cgi/1.xx":1,"combine/0.0":1,"Combine/2.0 http://combine.it.lth.se/":1,"Combine/3 http://combine.it.lth.se/":1,"Combine/x.0":1,"cometrics-bot, http://www.cometrics.de":1,"Commerce Browser Center":1,"complex_network_group/Nutch-0.9-dev (discovering the structure of the world-wide-web; http://cantor.ee.ucla.edu/~networks/crawl; nimakhaj@gmail.com)":1,"Computer_and_Automation_Research_Institute_Crawler crawler@ilab.sztaki.hu":1,"Comrite/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)":1,"Contact":1,"ContactBot/0.2":1,"ContentSmartz":1,"contype":1,"Convera Internet Spider V6.x":1,"ConveraCrawler/0.2":1,"ConveraCrawler/0.9d (+http://www.authoritativeweb.com/crawl)":1,"ConveraMultiMediaCrawler/0.1 (+http://www.authoritativeweb.com/crawl)":1,"CoolBot":1,"Cooliris/1.5 CFNetwork/459 Darwin/10.0.0d3":1,"CoralWebPrx/0.1.1x (See http://coralcdn.org/)":1,"cosmos/0.8_(robot@xyleme.com)":1,"cosmos/0.9_(robot@xyleme.com)":1,"CoteoNutchCrawler/Nutch-0.9 (info [at] coteo [dot] com)":1,"CougarSearch/0.x (+http://www.cougarsearch.com/faq.shtml)":1,"Covac TexAs Arachbot":1,"CoverScout%203/3.0.1 CFNetwork/339.5 Darwin/9.5.0 (i386) (iMac5,1)":1,"Cowbot-0.1 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)":1,"Cowbot-0.1.x (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)":1,"CrawlConvera0.1 (CrawlConvera@yahoo.com)":1,"Crawler":1,"Crawler (cometsearch@cometsystems.com)":1,"Crawler admin@crawler.de":1,"Crawler V 0.2.x admin@crawler.de":1,"crawler@alexa.com":1,"CrawlerBoy Pinpoint.com":1,"Crawllybot/0.1 (Crawllybot; +http://www.crawlly.com; crawler@crawlly.com)":1,"CreativeCommons/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)":1,"Cricket-A100/1.0 UP.Browser/6.3.0.7 (GUI) MMP/2.0":1,"CrocCrawler vx.3 [en] (http://www.croccrawler.com) (X11; I; Linux 2.0.44 i686)":1,"csci_b659/0.13":1,"CSE HTML Validator Professional (http://www.htmlvalidator.com/)":1,"Cuam Ver0.050bx":1,"Cuasarbot/0.9b http://www.cuasar.com/spider_beta/ ":1,"curl/7.10.x (i386-redhat-linux-gnu) libcurl/7.10.x OpenSSL/0.9.7a ipv6 zlib/1.1.4":1,"curl/7.7.x (i386--freebsd4.3) libcurl 7.7.x (SSL 0.9.6) (ipv6 enabled)":1,"curl/7.8 (i686-pc-linux-gnu) libcurl 7.8 (OpenSSL 0.9.6)":1,"curl/7.9.x (win32) libcurl 7.9.x":1,"CurryGuide SiteScan 1.1":1,"Custo x.x (www.netwu.com)":1,"Custom Spider www.bisnisseek.com /1.0":1,"Cyberdog/2.0 (Macintosh; 68k)":1,"CyberPatrol SiteCat Webbot (http://www.cyberpatrol.com/cyberpatrolcrawler.asp)":1,"CyberSpyder Link Test/2.1.12 (admin@mspennyworth.com)":1,"CydralSpider/1.x (Cydral Web Image Search; http://www.cydral.com)":1,"CydralSpider/3.0 (Cydral Image Search; http://www.cydral.com)":1,"DA 3.5 (www.lidan.com)":1,"DA 4.0":1,"DA 4.0 (www.downloadaccelerator.com)":1,"DA 5.0":1,"DA 7.0":1,"DAP x.x":1,"Dart Communications PowerTCP":1,"DataCha0s/2.0":1,"DataFountains/DMOZ Downloader":1,"DataFountains/Dmoz Downloader (http://ivia.ucr.edu/useragents.shtml)":1,"DataFountains/DMOZ Feature Vector Corpus Creator (http://ivia.ucr.edu/useragents.shtml)":1,"DataparkSearch/4.47 (+http://dataparksearch.org/bot)":1,"DataparkSearch/4.xx (http://www.dataparksearch.org/)":1,"DataSpear/1.0 (Spider; http://www.dataspear.com/spider.html; spider@dataspear.com)":1,"DataSpearSpiderBot/0.2 (DataSpear Spider Bot; http://dssb.dataspear.com/bot.html; dssb@dataspear.com)":1,"DatenBot( http://www.sicher-durchs-netz.de/bot.html)":1,"DaviesBot/1.7 (www.wholeweb.net)":1,"daypopbot/0.x":1,"dbDig(http://www.prairielandconsulting.com)":1,"DBrowse 1.4b":1,"DBrowse 1.4d":1,"DC-Sakura/x.xx":1,"dCSbot/1.1":1,"DDD":1,"dds explorer v1.0 beta":1,"de.searchengine.comBot 1.2 (http://de.searchengine.com/spider)":1,"DeadLinkCheck/0.4.0 libwww-perl/5.xx":1,"Deep Link Calculator v1.0":1,"deepak-USC/ISI":1,"DeepIndex":1,"DeepIndex ( http://www.zetbot.com )":1,"DeepIndex (www.en.deepindex.com)":1,"DeepIndexer.ca":1,"del.icio.us-thumbnails/1.0 Mozilla/5.0 (compatible; Konqueror/3.4; FreeBSD) KHTML/3.4.2 (like Gecko)":1,"DeleGate/9.0.5-fix1":1,"Demo Bot DOT 16b":1,"Demo Bot Z 16b":1,"Denmex websearch (http://search.denmex.com)":1,"Der große BilderSauger 2.00u":1,"dev-spider2.searchpsider.com/1.3b":1,"DevComponents.com HtmlDocument Object":1,"DiaGem/1.1 (http://www.skyrocket.gr.jp/diagem.html)":1,"Diamond/x.0":1,"DiamondBot":1,"Digger/1.0 JDK/1.3.0rc3":1,"DigOut4U":1,"DIIbot/1.2":1,"Dillo/0.8.5-i18n-misc":1,"Dillo/0.x.x":1,"disastrous/1.0.5 (running with Python 2.5.1; http://www.bortzmeyer.org/disastrous.html; archangel77@del.icio.us)":1,"DISCo Pump x.x":1,"disco/Nutch-0.9 (experimental crawler; www.discoveryengine.com; disco-crawl@discoveryengine.com)":1,"disco/Nutch-1.0-dev (experimental crawler; www.discoveryengine.com; disco-crawl@discoveryengine.com)":1,"DittoSpyder":1,"dlman":1,"dloader(NaverRobot)/1.0":1,"DNSRight.com WebBot Link Ckeck Tool. Report abuse to: dnsr@dnsright.com":1,"DoCoMo/1.0/Nxxxi/c10":1,"DoCoMo/1.0/Nxxxi/c10/TB":1,"DoCoMo/1.0/P502i/c10 (Google CHTML Proxy/1.0)":1,"DoCoMo/2.0 P900iV(c100;TB;W24H11) ":1,"DoCoMo/2.0 SH901iS(c100;TB;W24H12),gzip(gfe) (via translate.google.com)":1,"DoCoMo/2.0 SH902i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)":1,"DoCoMo/2.0/SO502i (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)":1,"DocZilla/1.0 (Windows; U; WinNT4.0; en-US; rv:1.0.0) Gecko/20020804":1,"dodgebot/experimental":1,"DonutP; Windows98SE":1,"Doubanbot/1.0 (bot@douban.com http://www.douban.com)":1,"Download Demon/3.x.x.x":1,"Download Druid 2.x":1,"Download Express 1.0":1,"Download Master":1,"Download Ninja 3.0":1,"Download Wonder":1,"Download-Tipp Linkcheck (http://download-tipp.de/)":1,"Download.exe(1.1) (+http://www.sql-und-xml.de/freeware-tools/)":1,"DownloadDirect.1.0":1,"Dr.Web (R) online scanner: http://online.drweb.com/":1,"Dragonfly File Reader":1,"Drecombot/1.0 (http://career.drecom.jp/bot.html)":1,"Drupal (+http://drupal.org/)":1,"DSurf15a 01":1,"DSurf15a 71":1,"DSurf15a 81":1,"DSurf15a VA":1,"DTAAgent":1,"dtSearchSpider":1,"Dual Proxy":1,"DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)":1,"Dumbot(version 0.1 beta - dumbfind.com)":1,"Dumbot(version 0.1 beta - http://www.dumbfind.com/dumbot.html)":1,"Dumbot(version 0.1 beta)":1,"e-sense 1.0 ea(www.vigiltech.com/esensedisclaim.html)":1,"e-SocietyRobot(http://www.yama.info.waseda.ac.jp/~yamana/es/)":1,"eApolloBot/2.0 (compatible; heritrix/2.0.0-SNAPSHOT-20071024.170148 +http://www.eapollo-opto.com)":1,"EARTHCOM.info/1.x [www.earthcom.info]":1,"EARTHCOM.info/1.xbeta [www.earthcom.info]":1,"EasyDL/3.xx":1,"EasyDL/3.xx http://keywen.com/Encyclopedia/Bot":1,"EBrowse 1.4b":1,"eCatch/3.0":1,"EchO!/2.0":1,"Educate Search VxB":1,"egothor/3.0a (+http://www.xdefine.org/robot.html)":1,"EgotoBot/4.8 (+http://www.egoto.com/about.htm)":1,"ejupiter.com":1,"EldoS TimelyWeb/3.x":1,"elfbot/1.0 (+http://www.uchoose.de/crawler/elfbot/)":1,"ELI/20070402:2.0 (DAUM RSS Robot, Daum Communications Corp.; +http://ws.daum.net/aboutkr.html)":1,"ELinks (0.x.x; Linux 2.4.20 i586; 132x60)":1,"ELinks/0.x.x (textmode; NetBSD 1.6.2 sparc; 132x43)":1,"EmailSiphon":1,"EmailSpider":1,"EmailWolf 1.00":1,"EmeraldShield.com WebBot":1,"EmeraldShield.com WebBot (http://www.emeraldshield.com/webbot.aspx)":1,"EMPAS_ROBOT":1,"EnaBot/1.x (http://www.enaball.com/crawler.html)":1,"endo/1.0 (Mac OS X; ppc i386; http://kula.jp/endo)":1,"Enfish Tracker":1,"Enterprise_Search/1.0":1,"Enterprise_Search/1.0.xxx":1,"Enterprise_Search/1.00.xxx;MSSQL (http://www.innerprise.net/es-spider.asp)":1,"envolk/1.7 (+http://www.envolk.com/envolkspiderinfo.php)":1,"envolk[ITS]spider/1.6(+http://www.envolk.com/envolkspider.html)":1,"EroCrawler":1,"ES.NET_Crawler/2.0 (http://search.innerprise.net/)":1,"eseek-larbin_2.6.2 (crawler@exactseek.com)":1,"ESISmartSpider":1,"eStyleSearch 4 (compatible; MSIE 6.0; Windows NT 5.0)":1,"ESurf15a 15":1,"EuripBot/0.x (+http://www.eurip.com) GetFile":1,"EuripBot/0.x (+http://www.eurip.com) GetRobots":1,"EuripBot/0.x (+http://www.eurip.com) PreCheck":1,"Eurobot/1.0 (http://www.ayell.eu)":1,"EvaalSE - bot@evaal.com":1,"eventax/1.3 (eventax; http://www.eventax.de/; info@eventax.de)":1,"Everest-Vulcan Inc./0.1 (R&D project; host=e-1-24; http://everest.vulcan.com/crawlerhelp)":1,"Everest-Vulcan Inc./0.1 (R&D project; http://everest.vulcan.com/crawlerhelp)":1,"Exabot-Images/1.0":1,"Exabot-Test/1.0":1,"Exabot/2.0":1,"Exabot/3.0":1,"ExactSearch":1,"ExactSeek Crawler/0.1":1,"exactseek-crawler-2.63 (crawler@exactseek.com)":1,"exactseek-pagereaper-2.63 (crawler@exactseek.com)":1,"exactseek.com":1,"Exalead NG/MimeLive Client (convert/http/0.120)":1,"Excalibur Internet Spider V6.5.4":1,"Execrawl/1.0 (Execrawl; http://www.execrawl.com/; bot@execrawl.com)":1,"exooba crawler/exooba crawler (crawler for exooba.com; http://www.exooba.com/; info at exooba dot com)":1,"exooba/exooba crawler (exooba; exooba)":1,"ExperimentalHenrytheMiragoRobot":1,"Expired Domain Sleuth":1,"Express WebPictures (www.express-soft.com)":1,"ExtractorPro":1,"Extreme Picture Finder":1,"EyeCatcher (Download-tipp.de)/1.0":1,"Factbot 1.09 (see http://www.factbites.com/webmasters.php)":1,"factbot : http://www.factbites.com/robots":1,"FaEdit/2.0.x":1,"FairAd Client":1,"FANGCrawl/0.01":1,"FARK.com link verifier":1,"Fast Crawler Gold Edition":1,"FAST Enterprise Crawler 6 (Experimental)":1,"FAST Enterprise Crawler 6 / Scirus scirus-crawler@fast.no; http://www.scirus.com/srsapp/contactus/":1,"FAST Enterprise Crawler 6 used by Cobra Development (admin@fastsearch.com)":1,"FAST Enterprise Crawler 6 used by Comperio AS (sts@comperio.no)":1,"FAST Enterprise Crawler 6 used by FAST (FAST)":1,"FAST Enterprise Crawler 6 used by Pages Jaunes (pvincent@pagesjaunes.fr)":1,"FAST Enterprise Crawler 6 used by Sensis.com.au Web Crawler (search_comments\\at\\sensis\\dot\\com\\dot\\au)":1,"FAST Enterprise Crawler 6 used by Singapore Press Holdings (crawler@sphsearch.sg)":1,"FAST Enterprise Crawler 6 used by WWU (wardi@uni-muenster.de)":1,"FAST Enterprise Crawler/6 (www.fastsearch.com)":1,"FAST Enterprise Crawler/6.4 (helpdesk at fast.no)":1,"FAST FirstPage retriever (compatible; MSIE 5.5; Mozilla/4.0)":1,"FAST MetaWeb Crawler (helpdesk at fastsearch dot com)":1,"Fast PartnerSite Crawler":1,"FAST-WebCrawler/2.2.10 (Multimedia Search) (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)":1,"FAST-WebCrawler/2.2.6 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)":1,"FAST-WebCrawler/2.2.7 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no":1,"FAST-WebCrawler/2.2.8 (crawler@fast.no; http://www.fast.no/faq/faqfastwebsearch/faqfastwebcrawler.html)http://www.fast.no":1,"FAST-WebCrawler/3.2 test":1,"FAST-WebCrawler/3.3 (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)":1,"FAST-WebCrawler/3.4/Nirvana (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)":1,"FAST-WebCrawler/3.4/PartnerSite (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)":1,"FAST-WebCrawler/3.5 (atw-crawler at fast dot no; http://fast.no/support.php?c=faqs/crawler)":1,"FAST-WebCrawler/3.6 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)":1,"FAST-WebCrawler/3.6/FirstPage (crawler@fast.no; http://fast.no/support.php?c=faqs/crawler)":1,"FAST-WebCrawler/3.7 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)":1,"FAST-WebCrawler/3.7/FirstPage (atw-crawler at fast dot no;http://fast.no/support/crawler.asp)":1,"FAST-WebCrawler/3.8 (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)":1,"FAST-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)":1,"FAST-WebCrawler/3.x Multimedia":1,"FAST-WebCrawler/3.x Multimedia (mm dash crawler at fast dot no)":1,"fastbot crawler beta 2.0 (+http://www.fastbot.de)":1,"FastBug http://www.ay-up.com":1,"FastCrawler 3.0.1 (crawler@1klik.dk)":1,"FastSearch Web Crawler for Verizon SuperPages (kevin.watters@fastsearch.com)":1,"Favcollector/2.0 (info@favcollector.com http://www.favcollector.com/)":1,"FavIconizer":1,"favo.eu crawler/0.6 (http://www.favo.eu)":1,"FavOrg":1,"Favorites Checking (http://campulka.net)":1,"Favorites Sweeper v.2.03":1,"Faxobot/1.0":1,"FDM 1.x":1,"FDM 2.x":1,"Feed Seeker Bot (RSS Feed Seeker http://www.MyNewFavoriteThing.com/fsb.php)":1,"Feed24.com":1,"Feed::Find/0.0x":1,"Feedable/0.1 (compatible; MSIE 6.0; Windows NT 5.1)":1,"FeedChecker/0.01":1,"FeedDemon/2.7 (http://www.newsgator.com/; Microsoft Windows XP)":1,"Feedfetcher-Google-iGoogleGadgets; (+http://www.google.com/feedfetcher.html)":1,"Feedfetcher-Google; (+http://www.google.com/feedfetcher.html)":1,"FeedForAll rss2html.php v2":1,"FeedHub FeedDiscovery/1.0 (http://www.feedhub.com)":1,"FeedHub MetaDataFetcher/1.0 (http://www.feedhub.com)":1,"Feedjit Favicon Crawler 1.0":1,"Feedreader 3.xx (Powered by Newsbrain)":1,"Feedshow/x.0 (http://www.feedshow.com; 1 subscriber)":1,"FeedshowOnline (http://www.feedshow.com)":1,"FeedZcollector v1.x (Platinum) http://www.feeds4all.com/feedzcollector":1,"Felix - Mixcat Crawler (+http://mixcat.com)":1,"fetch libfetch/2.0":1,"FFC Trap Door Spider":1,"Filangy/0.01-beta (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)":1,"Filangy/1.0x (Filangy; http://www.filangy.com/filangyinfo.jsp?inc=robots.jsp; filangy-agent@filangy.com)":1,"Filangy/1.0x (Filangy; http://www.nutch.org/docs/en/bot.html; filangy-agent@filangy.com)":1,"fileboost.net/1.0 (+http://www.fileboost.net)":1,"FileHound x.x":1,"Filtrbox/1.0":1,"FindAnISP.com_ISP_Finder_v99a":1,"Findexa Crawler (http://www.findexa.no/gulesider/article26548.ece)":1,"findlinks/x.xxx (+http://wortschatz.uni-leipzig.de/findlinks/) ":1,"FineBot":1,"Finjan-prefetch":1,"Firefly/1.0":1,"Firefly/1.0 (compatible; Mozilla 4.0; MSIE 5.5)":1,"Firefox (kastaneta03@hotmail.com)":1,"Firefox_1.0.6 (kasparek@naparek.cz)":1,"FirstGov.gov Search - POC:firstgov.webmasters@gsa.gov":1,"firstsbot":1,"Flapbot/0.7.2 (Flaptor Crawler; http://www.flaptor.com; crawler at flaptor period com)":1,"FlashGet":1,"FLATARTS_FAVICO":1,"Flexum spider":1,"Flexum/2.0":1,"FlickBot 2.0 RPT-HTTPClient/0.3-3":1,"flunky":1,"fly/6.01 libwww/4.0D":1,"flyindex.net 1.0/http://www.flyindex.net":1,"FnooleBot/2.5.2 (+http://www.fnoole.com/addurl.html)":1,"FocusedSampler/1.0":1,"Folkd.com Spider/0.1 beta 1 (www.folkd.com)":1,"FollowSite Bot ( http://www.followsite.com/bot.html )":1,"FollowSite.com ( http://www.followsite.com/b.html )":1,"Fooky.com/ScorpionBot/ScoutOut; http://www.fooky.com/scorpionbots":1,"Francis/1.0 (francis@neomo.de http://www.neomo.de/)":1,"Franklin Locator 1.8":1,"free-downloads.net download-link validator /0.1":1,"FreeFind.com-SiteSearchEngine/1.0 (http://freefind.com; spiderinfo@freefind.com)":1,"Frelicbot/1.0 +http://www.frelic.com/":1,"FreshDownload/x.xx":1,"FreshNotes crawler< report problems to crawler-at-freshnotes-dot-com":1,"FSurf15a 01":1,"FTB-Bot http://www.findthebest.co.uk/":1,"Full Web Bot 0416B":1,"Full Web Bot 0516B":1,"Full Web Bot 2816B":1,"FuseBulb.Com":1,"FyberSpider (+http://www.fybersearch.com/fyberspider.php)":1,"Gagglebot":1,"GAIS Robot/1.0B2":1,"Gaisbot/3.0 (indexer@gais.cs.ccu.edu.tw; http://gais.cs.ccu.edu.tw/robot.php)":1,"Gaisbot/3.0+(robot06@gais.cs.ccu.edu.tw;+http://gais.cs.ccu.edu.tw/robot.php)":1,"GalaxyBot/1.0 (http://www.galaxy.com/galaxybot.html)":1,"Gallent Search Spider v1.4 Robot 2 (http://robot.GallentSearch.com)":1,"gamekitbot/1.0 (+http://www.uchoose.de/crawler/gamekitbot/)":1,"Gamespy_Arcade":1,"GammaSpider/1.0":1,"gazz/x.x (gazz@nttrd.com)":1,"geckobot":1,"Generic Mobile Phone (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)":1,"generic_crawler/01.0217/":1,"GenesisBrowser (HTTP 1.1; 0.9; XP SP2; .NET CLR 2.0.50727)":1,"genieBot (http://64.5.245.11/faq/faq.html)":1,"geniebot wgao@genieknows.com":1,"GeoBot/1.0":1,"GeonaBot 1.x; http://www.geona.com/":1,"geourl/2.0b2":1,"GeoURLBot 1.0 (http://geourl.org)":1,"GetBot":1,"GetRight/3.x.x":1,"GetRight/4.5xx":1,"GetRight/4.x":1,"GetRight/4.x[a-e]":1,"GetRight/6.1 (Pro)":1,"GetRightPro/6.0beta2":1,"GetWeb/0.1 libwww-perl/5.16":1,"GhostRouteHunter/20021130 (https://www.sixxs.net/tools/grh/; info@sixxs.net)":1,"gigabaz/3.1x (baz@gigabaz.com; http://gigabaz.com/gigabaz/)":1,"Gigabot/2.0 (gigablast.com)":1,"Gigabot/2.0/gigablast.com/spider.html":1,"Gigabot/2.0; http://www.gigablast.com/spider.html":1,"Gigabot/2.0att":1,"Gigabot/3.0 (http://www.gigablast.com/spider.html)":1,"Gigabot/x.0":1,"GigabotSiteSearch/2.0 (sitesearch.gigablast.com)":1,"GNODSPIDER (www.gnod.net)":1,"Go!Zilla 3.x (www.gozilla.com)":1,"Go!Zilla/4.x.x.xx":1,"Go-Ahead-Got-It/1.1":1,"Goblin/0.9 (http://www.goguides.org/)":1,"Goblin/0.9.x (http://www.goguides.org/goblin-info.html)":1,"GoForIt.com":1,"GOFORITBOT ( http://www.goforit.com/about/ )":1,"GoGuides.Org Link Check":1,"GoldenFeed Spider 1.0 (http://www.goldenfeed.com)":1,"Goldfire Server":1,"gonzo1[P] +http://www.suchen.de/popups/faq.jsp":1,"gonzo2[P] +http://www.suchen.de/faq.html":1,"Goofer/0.2":1,"Google Talk":1,"googlebot (larbin2.6.0@unspecified.mail)":1,"Googlebot-Image/1.0":1,"Googlebot-Image/1.0 ( http://www.googlebot.com/bot.html)":1,"Googlebot/2.1 ( http://www.google.com/bot.html)":1,"Googlebot/2.1 ( http://www.googlebot.com/bot.html)":1,"Googlebot/Test ( http://www.googlebot.com/bot.html)":1,"Gordon's Spider/Nutch-0.9 (http://www.sharethis.com; gordon@sharethis.com)":1,"GrapeFX/0.3 libwww/5.4.0":1,"great-plains-web-spider/flatlandbot (Flatland Industries Web Spider; http://www.flatlandindustries.com/flatlandbot.php; jason@flatlandindustries.com)":1,"GreatNews/1.0":1,"GreenBrowser":1,"gridwell (http://search.gridwell.com)":1,"GrigorBot 0.8 (http://www.grigor.biz/bot.html)":1,"Gromit/1.0":1,"grub crawler(http://www.grub.org)":1,"grub-client":1,"gsa-crawler (Enterprise; GID-01422; jplastiras@google.com)":1,"gsa-crawler (Enterprise; GID-01742;gsatesting@rediffmail.com)":1,"gsa-crawler (Enterprise; GIX-02057; dm@enhesa.com)":1,"gsa-crawler (Enterprise; GIX-03519; cknuetter@stubhub.com)":1,"gsa-crawler (Enterprise; GIX-0xxxx; enterprise-training@google.com)":1,"GSiteCrawler/v1.xx rev. xxx (http://gsitecrawler.com/)":1,"Guestbook Auto Submitter":1,"Gulliver/1.3":1,"Gulper Web Bot 0.2.4 (www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)":1,"Gungho/0.08004 (http://code.google.com/p/gungho-crawler/wiki/Index)":1,"GurujiBot/1.0 (+http://www.guruji.com/WebmasterFAQ.html)":1,"GurujiImageBot/1.0 (+http://www.guruji.com/en/WebmasterFAQ.html)":1,"Haier-T10C/1.0 iPanel/2.0 WAP2.0 (compatible; UP.Browser/6.2.2.4; UPG1; UP/4.0; Embedded)":1,"HappyFunBot/1.1":1,"Harvest-NG/1.0.2":1,"Haste/0.12 (HOME: http://haste.kytoon.com/)":1,"Hatena Antenna/0.4 (http://a.hatena.ne.jp/help#robot)":1,"Hatena Mobile Gateway/1.0":1,"Hatena Pagetitle Agent/1.0":1,"Hatena RSS/0.3 (http://r.hatena.ne.jp)":1,"HatenaScreenshot/1.0 (checker)":1,"hbtronix.spider.2 -- http://hbtronix.de/spider.php":1,"HeinrichderMiragoRobot":1,"HeinrichderMiragoRobot (http://www.miragorobot.com/scripts/deinfo.asp)":1,"Helix/1.x ( http://www.sitesearch.ca/helix/)":1,"HenriLeRobotMirago (http://www.miragorobot.com/scripts/frinfo.asp)":1,"HenrytheMiragoRobot":1,"HenryTheMiragoRobot (http://www.miragorobot.com/scripts/mrinfo.asp)":1,"hgrepurl/1.0":1,"Hi! I'm CsCrawler my homepage: http://www.kde.cs.uni-kassel.de/lehre/ss2005/googlespam/crawler.html RPT-HTTPClient/0.3-3":1,"HiDownload":1,"Hippias/0.9 Beta":1,"HitList":1,"Hitwise Spider v1.0 http://www.hitwise.com":1,"HLoader":1,"holmes/3.11 (http://morfeo.centrum.cz/bot)":1,"holmes/3.9 (onet.pl)":1,"holmes/3.xx (OnetSzukaj/5.0; +http://szukaj.onet.pl)":1,"holmes/x.x":1,"HolmesBot (http://holmes.ge)":1,"HomePageSearch(hpsearch.uni-trier.de)":1,"Homerbot: www.homerweb.com":1,"Honda-Search/0.7.2 (Nutch; http://lucene.apache.org/nutch/bot.html; search@honda-search.com)":1,"HooWWWer/2.1.3 (debugging run) (+http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-info<at>hiit.fi)":1,"HooWWWer/2.1.x ( http://cosco.hiit.fi/search/hoowwwer/ | mailto:crawler-info<at>hiit.fi)":1,"HotJava/1.0.1/JRE1.1.x":1,"Hotzonu/x.0":1,"HPL/Nutch-0.9 -":1,"htdig/3.1.6 (http://computerorgs.com)":1,"htdig/3.1.6 (unconfigured@htdig.searchengine.maintainer)":1,"htdig/3.1.x (root@localhost)":1,"Html Link Validator (www.lithopssoft.com)":1,"HTML2JPG Blackbox, http://www.html2jpg.com":1,"HTML2JPG Enterprise":1,"HTMLParser/1.x":1,"HTTP Retriever":1,"http://Anonymouse.org/ (Unix)":1,"http://Ask.24x.Info/ (http://narres.it/)":1,"http://hilfe.acont.de/bot.html ACONTBOT":1,"http://OzySoftware.com/Index.html":1,"http://www.almaden.ibm.com/cs/crawler":1,"http://www.almaden.ibm.com/cs/crawler [rc1.wf.ibm.com]":1,"http://www.almaden.ibm.com/cs/crawler [wf216]":1,"http://www.istarthere.com_spider@istarthere.com":1,"http://www.monogol.de":1,"http://www.trendtech.dk/spider.asp)":1,"HTTP::Lite/2.x.x":1,"HTTPEyes":1,"HTTPResume v. 1.x":1,"httpunit/1.5":1,"httpunit/1.x":1,"Hybrid/1.2 [en] (OS Independent)":1,"HyperEstraier/1.x.xx":1,"i1searchbot/2.0 (i1search web crawler; http://www.i1search.com; crawler@i1search.com)":1,"IAArchiver-1.0":1,"iaskspider":1,"iaskspider2 (iask@staff.sina.com.cn)":1,"ia_archiver":1,"ia_archiver-web.archive.org":1,"ia_archiver/1.6":1,"IBrowse/2.2 (AmigaOS 3.5)":1,"IBrowse/2.2 (Windows 3.1)":1,"iCab/2.5.2 (Macintosh; I; PPC)":1,"ICC-Crawler(Mozilla-compatible; http://kc.nict.go.jp/icc/crawl.html; icc-crawl(at)ml(dot)nict(dot)go(dot)jp)":1,"ICC-Crawler(Mozilla-compatible;http://kc.nict.go.jp/icc/crawl.html;icc-crawl-contact(at)ml(dot)nict(dot)go(dot)jp)":1,"iCCrawler (http://www.iccenter.net)":1,"ICCrawler - ICjobs (http://www.icjobs.de/bot.htm)":1,"ICE Browser/5.05 (Java 1.4.0; Windows 2000 5.0 x86)":1,"ichiro/x.0 (http://help.goo.ne.jp/door/crawler.html)":1,"ichiro/x.0 (ichiro@nttr.co.jp)":1,"IconSurf/2.0 favicon finder (see http://iconsurf.com/robot.html)":1,"IconSurf/2.0 favicon monitor (see http://iconsurf.com/robot.html)":1,"ICOO Loader v.x.x.x":1,"ICRA_label_spider/x.0":1,"icsbot-0.1":1,"IDA":1,"ideare - SignSite/1.x":1,"iearthworm/1.0, iearthworm@yahoo.com.cn":1,"IEFav172Free":1,"iFeed.jp/2.0 (www.psychedelix.com/agents/agents.rss; 0 subscribers)":1,"igdeSpyder (compatible; igde.ru; +http://igde.ru/doc/tech.html)":1,"iGetter/1.x (Macintosh;G;PPC)":1,"iGetter/2 (Macintosh; U; PPC Mac OS X; en)":1,"IIITBOT/1.1 (Indian Language Web Search Engine; http://webkhoj.iiit.net; pvvpr at iiit dot ac dot in)":1,"ilial/Nutch-0.9 (Ilial, Inc. is a Los Angeles based Internet startup company. For more information please visit http://www.ilial.com/crawler; http://www.ilial.com/crawler; crawl@ilial.com)":1,"ilial/Nutch-0.9-dev":1,"IlseBot/1.x":1,"IlTrovatore-Setaccio ( http://www.iltrovatore.it)":1,"Iltrovatore-Setaccio/0.3-dev (Indexing; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)":1,"IlTrovatore-Setaccio/1.2 ( http://www.iltrovatore.it/aiuto/faq.html)":1,"Iltrovatore-Setaccio/1.2 (It-bot; http://www.iltrovatore.it/bot.html; info@iltrovatore.it)":1,"iltrovatore-setaccio/1.2-dev (spidering; http://www.iltrovatore.it/aiuto/.....)":1,"IlTrovatore/1.2 (IlTrovatore; http://www.iltrovatore.it/bot.html; bot@iltrovatore.it)":1,"ImageVisu/v4.x.x":1,"ImageWalker/2.0 (www.bdbrandprotect.com)":1,"Incutio HttpClient v0.x":1,"IncyWincy data gatherer(webmaster@loopimprovements.com":1,"IncyWincy page crawler(webmaster@loopimprovements.com":1,"IncyWincy(http://www.look.com)":1,"IncyWincy(http://www.loopimprovements.com/robot.html)":1,"IncyWincy/2.1(loopimprovements.com/robot.html)":1,"IndexTheWeb.com Crawler7":1,"Industry Program 1.0.x":1,"Inet library":1,"InetURL/1.0":1,"info@pubblisito.com- (http://www.pubblisito.com) il Sud dei Motori di Ricerca":1,"Infoaxe./Nutch-0.9":1,"infoConveraCrawler/0.8 ( http://www.authoritativeweb.com/crawl)":1,"InfoFly/1.0 (http://www.versions-project.org/)":1,"InfoLink/1.x":1,"INFOMINE/8.0 Adders":1,"INFOMINE/8.0 RemoteServices":1,"INFOMINE/8.0 VLCrawler (http://infomine.ucr.edu/useragents)":1,"InfoNaviRobot(F107)":1,"InfoSeek Sidewinder/0.9":1,"InfoSeek Sidewinder/1.0A":1,"InfoSeek Sidewinder/1.1A":1,"Infoseek SideWinder/1.45 (Compatible; MSIE 10.0; UNIX)":1,"Infoseek SideWinder/2.0B (Linux 2.4 i686)":1,"INGRID/3.0 MT (webcrawler@NOSPAMexperimental.net; http://webmaster.ilse.nl/jsp/webmaster.jsp)":1,"Inktomi Search":1,"InnerpriseBot/1.0 (http://www.innerprise.com/)":1,"Insitor.com search and find world wide!":1,"Insitornaut":1,"InstallShield DigitalWizard":1,"integrity/1.6":1,"Intelix/0.x (cs; http://www.microton.cz/intelix/; microton@@microton.cz)":1,"Interarchy/x.x.x (InterarchyCrawler)":1,"Internet Ninja x.0":1,"InternetArchive/0.8-dev(Nutch;http://lucene.apache.org/nutch/bot.html;nutch-agent@lucene.apache":1,"InternetLinkAgent/3.1":1,"InternetSeer.com":1,"intraVnews/1.x":1,"IOI/2.0 (ISC Open Index crawler; http://index.isc.org/; bot@index.isc.org)":1,"IP*Works! V5 HTTP/S Component - by /n software - www.nsoftware.com":1,"http://www.ip2location.com":1,"IP2MapBot/1.1 <a href=http://www.ip2map.com>http://www.ip2map.com</a>":1,"IPiumBot laurion(dot)com":1,"IpselonBot/0.xx-beta (Ipselon; http://www.ipselon.com; ipselonbot@ipselon.com)":1,"Iria/1.xxa":1,"IRLbot/1.0 ( http://irl.cs.tamu.edu/crawler)":1,"IRLbot/3.0 (compatible; MSIE 6.0; http://irl.cs.tamu.edu/crawler/)":1,"IrssiUrlLog/0.2":1,"Irvine/1.x.x":1,"ISC Systems iRc Search 2.1":1,"iSiloX/4.xx Windows/32":1,"isurf (tszhu@canada.com)":1,"iTunes/x.x.x":1,"IUPUI Research Bot v 1.9a":1,"iVia Page Fetcher (http://ivia.ucr.edu/useragents.shtml)":1,"iVia/4.0 CanonizeUrl (http://infomine.ucr.edu/iVia/useragents.shtml":1,"IWAgent/ 1.0 - www.brandprotect.com":1,"J-PHONE/3.0/J-SH07":1,"Jabot/6.x (http://odin.ingrid.org/)":1,"Jabot/7.x.x (http://odin.ingrid.org/)":1,"Jack":1,"Jakarta Commons-HttpClient/2.0xxx":1,"Jakarta Commons-HttpClient/3.0-rcx":1,"Jambot/0.1.x (Jambot; http://www.jambot.com/blog; crawler@jambot.com)":1,"Jambot/0.2.1 (Jambot; http://www.jambot.com/blog/static.php?page=webmaster-robot; crawler@jambot.com)":1,"Java 1.1":1,"Java/1.4.1_01":1,"Java1.0.21.0":1,"Java1.1.xx.x":1,"Java1.3.0rc1":1,"Java1.3.x":1,"Java1.4.0":1,"Jayde Crawler. http://www.jayde.com":1,"JBH Agent 2.0":1,"jBrowser/J2ME Profile/MIDP-1.0 Configuration/CLDC-1.0 (Google WAP Proxy/1.0)":1,"JCheckLinks/0.1 RPT-HTTPClient/0.3-1":1,"JDK/1.1":1,"Jeode/1.x.x":1,"Jetbot/1.0":1,"JetBrains Omea Reader 1.0.x (http://www.jetbrains.com/omea_reader/)":1,"JetBrains Omea Reader 2.0 Release Candidate 1 (http://www.jetbrains.com/omea_reader/)":1,"JetCar":1,"Jigsaw/2.2.x W3C_CSS_Validator_JFouffa/2.0":1,"JoBo/1.x (http://www.matuschek.net/jobo.html)":1,"JoBo/@JOBO_VERSION@(http://www.matuschek.net/jobo.html)":1,"JobSpider_BA/1.1":1,"JOC Web Spider":1,"JordoMedia/1.0 RSS File Reader (http://www.jordomedia.com)":1,"Journster [alpha] (http://journster.com/)":1,"Journster.com RSS/Atom aggregator 0.5 (http://www.journster.com/bot.phtml)":1,"JRTS Check Favorites Utility":1,"JRTwine Software Check Favorites Utility":1,"Jyxobot/x":1,"K-Meleon/0.6 (Windows; U; Windows NT 5.1; en-US; rv:0.9.5) Gecko/20011011":1,"k2spider":1,"KAIST AITrc Crawler":1,"KakleBot - www.kakle.com/0.1 (KakleBot - www.kakle.com; http:// www.kakle.com/bot.html; support@kakle.com)":1,"kalooga/kalooga-4.0-dev-datahouse (Kalooga; http://www.kalooga.com; info@kalooga.com)":1,"kalooga/KaloogaBot (Kalooga; http://www.kalooga.com/info.html?page=crawler; crawler@kalooga.com)":1,"Kapere (http://www.kapere.com)":1,"Kazehakase/0.x.x.[x]":1,"KDDI-SN22 UP.Browser/6.0.7 (GUI) MMP/1.1 (Google WAP Proxy/1.0)":1,"Kenjin Spider":1,"Kevin http://dznet.com/kevin/":1,"Kevin http://websitealert.net/kevin/":1,"KE_1.0/2.0 libwww/5.2.8":1,"KFSW-Bot (Version: 1.01 powered by KFSW www.kfsw.de)":1,"kinja-imagebot (http://www.kinja.com/)":1,"kinjabot (http://www.kinja.com)":1,"KIT-Fireball/2.0":1,"KIT-Fireball/2.0 (compatible; Mozilla 4.0; MSIE 5.5)":1,"Klondike/1.50 (WSP Win32) (Google WAP Proxy/1.0)":1,"KnowItAll(knowitall@cs.washington.edu)":1,"Knowledge.com/0.x":1,"Kontiki Client x.xx":1,"Krugle/Krugle,Nutch/0.8+ (Krugle web crawler; http://www.krugle.com/crawler/info.html; webcrawler@krugle.com)":1,"KSbot/1.0 (KnowledgeStorm crawler; http://www.knowledgestorm.com/resources/content/crawler/index.html; crawleradmin@knowledgestorm.com)":1,"kuloko-bot/0.x":1,"kulokobot www.kuloko.com kuloko@backweave.com":1,"kulturarw3/0.1":1,"KummHttp/1.1 (compatible; KummClient; Linux rulez)":1,"KWC-KX9/1109 UP.Browser/6.2.3.9.g.1.107 (GUI) MMP/2.0 UP.Link/6.3.0.0.0":1,"Labrador/0.2; http://ir.dcs.gla.ac.uk/labrador; craigm@dcs.gla.ac.uk":1,"Lachesis":1,"lanshanbot/1.0":1,"lanshanbot/1.0 (+http://search.msn.com/msnbot.htm)":1,"LapozzBot/1.4 ( http://robot.lapozz.com)":1,"LapozzBot/1.5 (+http://robot.lapozz.hu)":1,"larbin (samualt9@bigfoot.com)":1,"LARBIN-EXPERIMENTAL (efp@gmx.net)":1,"larbin_2.1.1 larbin2.1.1@somewhere.com":1,"larbin_2.2.0 (crawl@compete.com)":1,"larbin_2.2.1_de_Viennot (Laurent.Viennot@inria.fr)":1,"larbin_2.2.2 (sugayama@lab7.kuis.kyoto-u.ac.jp)":1,"larbin_2.2.2_guillaume (guillaume@liafa.jussieu.fr)":1,"larbin_2.6.0 (larbin2.6.0@unspecified.mail)":1,"larbin_2.6.1 (larbin2.6.1@unspecified.mail)":1,"larbin_2.6.2 (hamasaki@grad.nii.ac.jp)":1,"larbin_2.6.2 (larbin2.6.2@unspecified.mail)":1,"larbin_2.6.2 (listonATccDOTgatechDOTedu)":1,"larbin_2.6.2 (pimenas@systems.tuc.gr)":1,"larbin_2.6.2 (tom@lemurconsulting.com)":1,"larbin_2.6.2 (vitalbox1@hotmail.com)":1,"larbin_2.6.3 (ltaa_web_crawler@groupes.epfl.ch)":1,"larbin_2.6.3 (wgao@genieknows.com)":1,"larbin_2.6.3_for_(http://cosco.hiit.fi/search/) tsilande@hiit.fi":1,"larbin_2.6_basileocaml (basile.starynkevitch@cea.fr)":1,"larbin_devel (http://pauillac.inria.fr/~ailleret/prog/larbin/)":1,"lawinfo-crawler/Nutch-0.9-dev (Crawler for lawinfo.com pages; http://www.lawinfo.com; webmaster@lawinfo.com)":1,"lc/$ROADS::Version libwww-perl/5.00":1,"lcabotAccept: */*":1,"LeapTag/0.8.1.beta081.r3750 (compatible; Mozilla 4.0; MSIE 5.5; robot@yoriwa.com)":1,"LECodeChecker/3.0 libgetdoc/1.0":1,"LeechGet 200x (www.leechget.de)":1,"LEIA/2.90":1,"LEIA/3.01pr (LEIAcrawler; [SNIP])":1,"LetsCrawl.com/1.0 +http://letscrawl.com/":1,"LexiBot/1.00":1,"LG-LX260 POLARIS-LX260/2.0 MMP/2.0 Profile/MIDP-2.0 Configuration/CLDC-1.1":1,"LG/U8138/v1.0":1,"Libby_1.1/libwww-perl/5.47":1,"libcurl-agent/1.0":1,"LibertyW (+http://www.lw01.com)":1,"libWeb/clsHTTP -- hiongun@kt.co.kr":1,"libwww-perl/5.41":1,"libwww-perl/5.45":1,"libwww-perl/5.48":1,"libwww-perl/5.50":1,"libwww-perl/5.52 FP/2.1":1,"libwww-perl/5.52 FP/4.0":1,"libwww-perl/5.53":1,"libwww-perl/5.63":1,"libwww-perl/5.64":1,"libwww-perl/5.65":1,"libwww-perl/5.800":1,"libwww/5.3.2":1,"Liferea/0.x.x (Linux; en_US.UTF-8; http://liferea.sf.net/)":1,"Liferea/1.x.x (Linux; es_ES.UTF-8; http://liferea.sf.net/)":1,"LightningDownload/1.0beta2":1,"LightningDownload/1.x.x":1,"LightningDownload/1.x.x [Accelerated x]":1,"LijitSpider/Nutch-0.9 (Reports crawler; http://www.lijit.com/; info(a)lijit(d)com)":1,"Lincoln State Web Browser":1,"Link Valet Online 1.x":1,"LinkAlarm/2.x":1,"Linkbot":1,"linkbot":1,"Linkbot x.0":1,"LinkCheck (linkcheck@inter7.com http://www.inter7.com/linkcheck)":1,"LinkLint-checkonly/2.x.x":1,"LinkLint-spider/2.x.x":1,"linknzbot":1,"LinkPimpin v1.0":1,"LinkProver 2.1":1,"Links (0.9x; Linux 2.4.7-10 i686)":1,"Links (0.9xpre12; Linux 2.2.14-5.0 i686; 80x24)":1,"Links (2.xpre7; Linux 2.4.18 i586; x)":1,"Links - http://gossamer-threads.com/scripts/links/":1,"Links 2.0 (http://gossamer-threads.com/scripts/links/)":1,"Links SQL (http://gossamer-threads.com/scripts/links-sql/)":1,"Links4US-Crawler, (+http://links4us.com/)":1,"LinkScan/11.0beta2 UnixShareware robot from Elsop.com (used by Indiafocus/Indiainfo)":1,"LinkScan/9.0g Unix":1,"LinkScan/x.x Unix":1,"LinksManager.com (http://linksmanager.com/linkchecker.html)":1,"LinkSonar/1.35":1,"LinkSweeper/1.x":1,"LinkWalker":1,"link_check3.plx libwww-perl/5.65":1,"ListBidBot (freelance job spider http://listbid.com)<a href=http://listbid.com>Freelance</a>":1,"LiveTrans/Nutch-0.9 (maintainer: cobain at iis dot sinica dot edu dot tw; http://wkd.iis.sinica.edu.tw/LiveTrans/)":1,"Llaut/1.0 (http://mnm.uib.es/~gallir/llaut/bot.html)":1,"LMQueueBot/0.2":1,"lmspider (lmspider@scansoft.com)":1,"LNSpiderguy":1,"LocalBot/1.0 ( http://www.localbot.co.uk/)":1,"LocalcomBot/1.2.x ( http://www.local.com/bot.htm)":1,"Lockstep Spider/1.0":1,"Look.com":1,"Lotus-Notes/4.5 ( Windows-NT )":1,"LotusDiscovery/x.0 (compatible; Mozilla 4.0; MSIE 4.01; Windows NT)":1,"Lovel as 1.0 ( +http://www.everatom.com)":1,"LTI/LemurProject Nutch Spider/Nutch-1.0-dev (lti crawler for CMU; http://www.lti.cs.cmu.edu; changkuk at cmu dot edu)":1,"LTI/LemurProject Nutch Spider/Nutch-1.0-dev (Research spider using Nutch; http://www.lemurproject.org; mhoy@cs.cmu.edu)":1,"luchs.at URL checker":1,"Lunascape":1,"lwp-trivial/1.32":1,"lwp-trivial/1.34":1,"lwp-trivial/1.35":1,"LWP::Simple/5.22":1,"LWP::Simple/5.36":1,"LWP::Simple/5.48":1,"LWP::Simple/5.50":1,"LWP::Simple/5.51":1,"LWP::Simple/5.53":1,"LWP::Simple/5.63":1,"LWP::Simple/5.803":1,"Lycos_Spider_(modspider)":1,"Lycos_Spider_(T-Rex)":1,"Lynx/2-4-2 (Bobcat/0.5 [DOS] Jp Beta04)":1,"Lynx/2.6 libwww-FM/2.14":1,"Lynx/2.8 (;http://seebot.org)":1,"Lynx/2.8.3dev.9 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6":1,"Lynx/2.8.4rel.1 libwww-FM/2.14 SSL-MM/1.4.1 OpenSSL/0.9.6c (human-guided@lerly.net)":1,"Mac Finder 1.0.xx":1,"Mackster( http://www.ukwizz.com )":1,"Mag-Net":1,"MagicWML/1.0 (forcewml)":1,"MagpieRSS/0.7x (+http://magpierss.sf.net)":1,"Mahiti.Com/Mahiti Crawler-1.0 (Mahiti.Com; http://mahiti.com ; mahiti.com)":1,"Mail.Ru/1.0":1,"mailto:webcraft@bea.com":1,"mammoth/1.0 ( http://www.sli-systems.com/)":1,"MantraAgent":1,"MapoftheInternet.com ( http://MapoftheInternet.com)":1,"Mariner/5.1b [de] (Win95; I ;Kolibri gncwebbot)":1,"Marketwave Hit List":1,"Martini":1,"MARTINI":1,"Marvin v0.3":1,"MaSagool/1.0 (MaSagool; http://sagool.jp/; info@sagool.jp)":1,"Mass Downloader 2.x":1,"MasterSeek":1,"Mata Hari/2.00 ":1,"Matrix S.p.A. - FAST Enterprise Crawler 6 (Unknown admin e-mail address)":1,"maxomobot/dev-20051201 (maxomo; http://67.102.134.34:4047/MAXOMO/MAXOMObot.html; maxomobot@maxomo.com)":1,"McBot/5.001 (windows; U; NT4.0; en-us)":1,"MDbot/1.0 (+http://www.megadownload.net/bot.html)":1,"Media Player Classic":1,"MediaCrawler-1.0 (Experimental)":1,"Mediapartners-Google/2.1 ( http://www.googlebot.com/bot.html)":1,"MediaSearch/0.1":1,"MegaSheep v1.0 (www.searchuk.com internet sheep)":1,"Megite2.0 (http://www.megite.com)":1,"Mercator-1.x":1,"Mercator-2.0":1,"Mercator-Scrub-1.1":1,"Metaeuro Web Crawler/0.2 (MetaEuro Web Search Clustering Engine; http://www.metaeuro.com; crawler at metaeuro dot com)":1,"MetaGer-LinkChecker":1,"MetagerBot/0.8-dev (MetagerBot; http://metager.de; )":1,"MetaGer_PreChecker0.1":1,"MetaProducts Download Express/1.x":1,"Metaspinner/0.01 (Metaspinner; http://www.meta-spinner.de/; support@meta-spinner.de/)":1,"metatagsdir/0.7 (+http://metatagsdir.com/directory/)":1,"MFC Foundation Class Library 4.0":1,"MFC_Tear_Sample":1,"MFHttpScan":1,"MicroBaz":1,"Microsoft Data Access Internet Publishing Provider Cache Manager":1,"Microsoft Data Access Internet Publishing Provider DAV":1,"Microsoft Data Access Internet Publishing Provider Protocol Discovery":1,"Microsoft Log Parser 2.2":1,"Microsoft Small Business Indexer":1,"Microsoft URL Control - 6.00.8xxx":1,"MicrosoftPrototypeCrawler (How's my crawling? mailto:newbiecrawler@hotmail.com)":1,"Microsoft_Internet_Explorer_5.00.438 (fjones@isd.net)":1,"MIIxpc/4.2":1,"Mindjet MindManager":1,"minibot":1,"miniRank/1.6 (Website ranking; www.minirank.com; robot)":1,"MiracleAlphaTest":1,"Missauga Locate 1.0.0":1,"Missigua Locator 1.9":1,"Missouri College Browse":1,"Mister Pix II 2.02a":1,"Mister PiX version.dll":1,"Misterbot-Nutch/0.7.1 (Misterbot-Nutch; http://www.misterbot.fr; admin@misterbot.fr)":1,"Miva (AlgoFeedback@miva.com)":1,"Mizzu Labs 2.2":1,"MJ12bot/vx.x.x (http://majestic12.co.uk/bot.php?+)":1,"MJ12bot/vx.x.x (http://www.majestic12.co.uk/projects/dsearch/mj12bot.php)":1,"MJBot (SEO assessment)":1,"MLBot (www.metadatalabs.com)":1,"MnogoSearch/3.2.xx":1,"Mo College 1.9":1,"moget/x.x (moget@goo.ne.jp)":1,"mogimogi/1.0":1,"moiNAG 0.02":1,"MojeekBot/0.x (archi; http://www.mojeek.com/bot.html)":1,"monkeyagent":1,"MoonBrowser (version 0.41 Beta4)":1,"Moreoverbot/x.00 (+http://www.moreover.com)":1,"Morris - Mixcat Crawler ( http://mixcat.com)":1,"Motoricerca-Robots.txt-Checker/1.0 (http://tool.motoricerca.info/robots-checker.phtml)":1,"Motorola-V3m Obigo":1,"Mouse-House/7.4 (spider_monkey spider info at www.mobrien.com/sm.shtml)":1,"MovableType/x.x":1,"mozDex/0.xx-dev (mozDex; http://www.mozdex.com/en/bot.html; spider@mozdex.com)":1,"Mozi!":1,"Mozilla":1,"Mozilla (libwhisker/2.4)":1,"Mozilla (Mozilla@somewhere.com)":1,"Mozilla 4.0(compatible; BotSeer/1.0; +http://botseer.ist.psu.edu)":1,"Mozilla/1.1 (compatible; MSPIE 2.0; Windows CE)":1,"Mozilla/1.10 [en] (Compatible; RISC OS 3.70; Oregano 1.10)":1,"Mozilla/1.22 (compatible; MSIE 2.0d; Windows NT)":1,"Mozilla/1.22 (compatible; MSIE 5.01; PalmOS 3.0) EudoraWeb 2":1,"Mozilla/2.0":1,"Mozilla/2.0 (compatible; AOL 3.0; Mac_PowerPC)":1,"Mozilla/2.0 (Compatible; AOL-IWENG 3.0; Win16)":1,"Mozilla/2.0 (compatible; Ask Jeeves)":1,"Mozilla/2.0 (compatible; Ask Jeeves/Teoma)":1,"Mozilla/2.0 (compatible; Ask Jeeves/Teoma; http://about.ask.com/en/docs/about/webmasters.shtml) ":1,"Mozilla/2.0 (compatible; Ask Jeeves/Teoma; http://sp.ask.com/docs/about/tech_crawling.html)":1,"Mozilla/2.0 (compatible; EZResult -- Internet Search Engine)":1,"Mozilla/2.0 (compatible; MS FrontPage x.0)":1,"Mozilla/2.0 (compatible; MSIE 2.1; Mac_PowerPC)":1,"Mozilla/2.0 (compatible; MSIE 3.02; Update a; AK; Windows NT)":1,"Mozilla/2.0 (compatible; MSIE 3.02; Update a; AOL 3.0; Windows 95)":1,"Mozilla/2.0 (compatible; MSIE 3.0; AK; Windows 95)":1,"Mozilla/2.0 (compatible; MSIE 3.0; Windows 3.1)":1,"Mozilla/2.0 (compatible; MSIE 3.0B; Win32)":1,"Mozilla/2.0 (compatible; NEWT ActiveX; Win32)":1,"Mozilla/2.0 (compatible; T-H-U-N-D-E-R-S-T-O-N-E)":1,"Mozilla/2.0 compatible; Check&Get 1.1x (Windows 98)":1,"Mozilla/2.01 (Win16; I)":1,"Mozilla/2.02Gold (Win95; I)":1,"Mozilla/3.0 (compatible)":1,"Mozilla/3.0 (compatible; AvantGo 3.2)":1,"Mozilla/3.0 (compatible; Fluffy the spider; http://www.searchhippo.com/; info@searchhippo.com)":1,"Mozilla/3.0 (compatible; HP Web PrintSmart 04b0 1.0.1.34)":1,"Mozilla/3.0 (compatible; Indy Library)":1,"Mozilla/3.0 (compatible; Linkman)":1,"Mozilla/3.0 (compatible; MuscatFerret/1.5.4; claude@euroferret.com)":1,"Mozilla/3.0 (compatible; MuscatFerret/1.5; olly@muscat.co.uk)":1,"Mozilla/3.0 (compatible; MuscatFerret/1.6.x; claude@euroferret.com)":1,"Mozilla/3.0 (compatible; netart generator/1.0; libwww-perl/5.64)":1,"Mozilla/3.0 (compatible; NetPositive/2.2)":1,"Mozilla/3.0 (compatible; Opera/3.0; Windows 3.1) v3.1":1,"Mozilla/3.0 (compatible; Opera/3.0; Windows 95/NT4) 3.2":1,"Mozilla/3.0 (compatible; PerMan Surfer 3.0; Win95)":1,"Mozilla/3.0 (compatible; REL Software Web Link Validator 2.x)":1,"Mozilla/3.0 (compatible; scan4mail (advanced version) http://www.peterspages.net/?scan4mail)":1,"Mozilla/3.0 (compatible; ScollSpider; http://www.webwobot.com)":1,"Mozilla/3.0 (compatible; Web Link Validator 2.x)Web Link Validator http://www.relsoftware.com/ link validation software":1,"Mozilla/3.0 (compatible; WebCapture x.x; Auto; Windows)":1,"Mozilla/3.0 (compatible; Webinator-DEV01.home.iprospect.com/2.56)":1,"Mozilla/3.0 (compatible; Webinator-indexer.cyberalert.com/2.56)":1,"Mozilla/3.0 (Compatible;Viking/1.8)":1,"Mozilla/3.0 (DreamPassport/3.0)":1,"Mozilla/3.0 (INGRID/3.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)":1,"Mozilla/3.0 (Liberate DTV 1.1)":1,"Mozilla/3.0 (Planetweb/2.100 JS SSL US; Dreamcast US)":1,"Mozilla/3.0 (Slurp.so/Goo; slurp@inktomi.com; http://www.inktomi.com/slurp.html)":1,"Mozilla/3.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)":1,"Mozilla/3.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)":1,"Mozilla/3.0 (Vagabondo/1.1 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)":1,"Mozilla/3.0 (Vagabondo/1.x MT; webagent@wise-guys.nl; http://webagent.wise-guys.nl/)":1,"Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMexperimental.net; http://aanmelden.ilse.nl/?aanmeld_mode=webhints)":1,"Mozilla/3.0 (Vagabondo/2.0 MT; webcrawler@NOSPAMwise-guys.nl; http://webagent.wise-guys.nl/)":1,"Mozilla/3.0 (Win16; I)":1,"Mozilla/3.0 (Win95; I)":1,"Mozilla/3.0 (WinNT; I)":1,"Mozilla/3.0 (WorldGate Gazelle 3.5.1 build 11; FreeBSD2.2.8-STABLE)":1,"Mozilla/3.0 (X11; I; OSF1 V4.0 alpha)":1,"Mozilla/3.0 NAVIO_AOLTV (11; 13; Philips; PH200; 1; R2.0C36_AOL.0110OPTIK; R2.0.0139d_OPTIK)":1,"Mozilla/3.0 WebTV/1.2 (compatible; MSIE 2.0)":1,"Mozilla/3.01 (compatible; AmigaVoyager/2.95; AmigaOS/MC680x0)":1,"Mozilla/3.01 (Compatible; Links2Go Similarity Engine)":1,"Mozilla/3.01 (compatible; Netbox/3.5 R92; Linux 2.2)":1,"Mozilla/3.01-C-MACOS8 (Macintosh; I; PPC)":1,"Mozilla/3.01Gold (X11; I; Linux 2.0.32 i486)":1,"Mozilla/3.01Gold (X11; I; SunOS 5.5.1 sun4m)":1,"Mozilla/3.01SGoldC-SGI (X11; I; IRIX 6.3 IP32)":1,"Mozilla/3.04 (compatible; ANTFresco/2.13; RISC OS 4.02)":1,"Mozilla/3.04 (compatible; NCBrowser/2.35; ANTFresco/2.17; RISC OS-NC 5.13 Laz1UK1309)":1,"Mozilla/3.04 (compatible;QNX Voyager 2.03B ;Photon)":1,"Mozilla/3.x (I-Opener 1.1; Netpliance)":1,"Mozilla/4.0":1,"Mozilla/4.0 (agadine3.0) www.agada.de":1,"Mozilla/4.0 (Compatible); URLBase 6":1,"Mozilla/4.0 (compatible: AstraSpider V.2.1 : astrafind.com)":1,"Mozilla/4.0 (compatible; Vagabondo/2.2; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)":1,"Mozilla/4.0 (compatible; Vagabondo/4.0Beta; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)":1,"Mozilla/4.0 (compatible; <a href=http://www.reget.com>ReGet Deluxe 5.1</a>; Windows NT 5.1)":1,"Mozilla/4.0 (compatible; Advanced Email Extractor v2.xx)":1,"Mozilla/4.0 (compatible; Arachmo)":1,"Mozilla/4.0 (compatible; BorderManager 3.0)":1,"Mozilla/4.0 (compatible; BOTW Spider; +http://botw.org)":1,"Mozilla/4.0 (compatible; B_L_I_T_Z_B_O_T)":1,"Mozilla/4.0 (compatible; Cerberian Drtrs Version-3.2-Build-0)":1,"Mozilla/4.0 (compatible; Check&Get 3.0; Windows NT)":1,"Mozilla/4.0 (compatible; ChristCrawler.com ChristCrawler@ChristCENTRAL.com)":1,"Mozilla/4.0 (compatible; crawlx, crawler@trd.overture.com)":1,"Mozilla/4.0 (compatible; DAUMOA-video; +http://ws.daum.net/aboutkr.html)":1,"Mozilla/4.0 (compatible; DepSpid/5.0x; +http://about.depspid.net)":1,"Mozilla/4.0 (compatible; DnloadMage 1.0)":1,"Mozilla/4.0 (compatible; FastCrawler3 support-fastcrawler3@fast.no)":1,"Mozilla/4.0 (compatible; FDSE robot)":1,"Mozilla/4.0 (compatible; GPU p2p crawler http://gpu.sourceforge.net/search_engine.php)":1,"Mozilla/4.0 (compatible; grub-client-0.2.x; Crawl your stuff with http://grub.org)":1,"Mozilla/4.0 (compatible; grub-client-0.3.x; Crawl your own stuff with http://grub.org)":1,"Mozilla/4.0 (compatible; grub-client-2.x)":1,"Mozilla/4.0 (compatible; ibisBrowser)":1,"Mozilla/4.0 (compatible; ICS 1.2.xxx)":1,"Mozilla/4.0 (compatible; IE-Favorites-Check-0.5)":1,"Mozilla/4.0 (compatible; Iplexx Spider/1.0 http://www.iplexx.at)":1,"Mozilla/4.0 (compatible; KeepNI web site monitor)":1,"Mozilla/4.0 (compatible; Link Utility; http://net-promoter.com)":1,"Mozilla/4.0 (compatible; Lotus-Notes/5.0; Windows-NT)":1,"Mozilla/4.0 (compatible; MSIE 4.01; AOL 4.0; Windows 98)":1,"Mozilla/4.0 (compatible; MSIE 4.01; Mac_PowerPC)":1,"Mozilla/4.0 (compatible; MSIE 4.01; MSIECrawler; Windows 95)":1,"Mozilla/4.0 (compatible; MSIE 4.01; Vonna.com b o t)":1,"Mozilla/4.0 (compatible; MSIE 4.01; Windows 95)":1,"Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; MSN Companion 2.0; 800x600; Compaq)":1,"Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; PPC; 240x320; SPV M700; OpVer 19.123.2.733) OrangeBot-Mobile 2008.0 (mobilesearch.support@orange-ftgroup.com)":1,"Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; PPS; 240x320)":1,"Mozilla/4.0 (compatible; MSIE 4.01; Windows NT Windows CE)":1,"Mozilla/4.0 (compatible; MSIE 4.01; Windows NT)":1,"Mozilla/4.0 (compatible; MSIE 4.01; Windows NT; MS Search 4.0 Robot) Microsoft":1,"Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) ACR":1,"Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) Indonesia Interactive":1,"Mozilla/4.0 (compatible; MSIE 4.0; Windows NT; Site Server 3.0 Robot) WebQuest Designs":1,"Mozilla/4.0 (compatible; MSIE 5.01; Windows 95) via <B>Avirt Gateway Server</B> v4.0":1,"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0) (samualt9@bigfoot.com)":1,"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; NetCaptor 6.5.0RC1)":1,"Mozilla/4.0 (compatible; MSIE 5.0; AOL 5.0; Windows 95; DigExt; Gateway2000; sureseeker.com)":1,"Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC; AtHome021)":1,"Mozilla/4.0 (compatible; MSIE 5.0; NetNose-Crawler 2.0; A New Search Experience: http://www.netnose.com)":1,"Mozilla/4.0 (compatible; MSIE 5.0; Win32) via proxy gateway CERN-HTTPD/3.0 libwww/2.17":1,"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) TrueRobot; 1.5":1,"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot BETA 1.2 (http://www.voila.com/)":1,"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) VoilaBot; 1.6":1,"Mozilla/4.0 (compatible; MSIE 5.0; Windows ME) Opera 5.11 [en]":1,"Mozilla/4.0 (compatible; MSIE 5.0; Windows ME; Link Checker 2.x.xx http://www.kyosoft.com)":1,"Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent":1,"Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Girafabot; girafabot at girafa dot com; http://www.girafa.com)":1,"Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com)":1,"Mozilla/4.0 (compatible; MSIE 5.0; www.galaxy.com; www.psychedelix.com/; http://www.galaxy.com/info/crawler.html)":1,"Mozilla/4.0 (compatible; MSIE 5.0; YANDEX)":1,"Mozilla/4.0 (compatible; MSIE 5.5; AOL 4.0; Windows 98; GoBeez (www.gobeez.com))":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows 95; Transmission Segment; Hotbar 2.0)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Crazy Browser 1.x.x)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; KITV4.7 Wanadoo)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; SAFEXPLORER TL)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; SYMPA; Katiesoft 7; SimulBrowse 3.0)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90; BTinternet V8.1)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows 98; Win 9x 4.90; MSIECrawler)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; obot)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 4.0; QXW03018)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Active Cache Request":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Fetch API Request":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AIRF)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; AspTear 1.5)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; N_o_k_i_a)":1,"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; T312461) RPT-HTTPClient/0.3-3E":1,"Mozilla/4.0 (compatible; MSIE 6.0 compatible; Asterias Crawler v4; +http://www.singingfish.com/help/spider.html; webmaster@singingfish.com); SpiderThread Revision: 3.10":1,"Mozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows 98; .NET CLR 1.1.4322; MEGAUPLOAD 2.0)":1,"Mozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows NT 5.1; SV1; HbTools 4.7.2)":1,"Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Skampy/0.9.x [en]":1,"Mozilla/4.0 (compatible; MSIE 6.0; TargetSeek/1.0; +http://www.targetgroups.net/TargetSeek.html)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Win32) WebWasher 3.0":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98) REL Software Web Link Validator 2.x)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98) Web Link Validator 2.x)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Net M@nager V3.02 - www.vinn.com.au)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; support@illumit.com; http://www.illumit.com/Products/weblight/)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x 4.90; http://www.Abolimba.de)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; Lunascape 2.1.3)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Google Wireless Transcoder;)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ODP entries t_st; http://tuezilla.de/t_st-odp-entries-agent.html)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ODP links test; http://tuezilla.de/test-odp-links-agent.html)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; ZoomSpider.net bot; .NET CLR 1.1.4322)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) (dns_admin@c-a-s-h.com)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Covac UPPS Cathan 1.2.5;)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Crayon Crawler; snprtz|T04056566514940; (R1 1.5))":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Deepnet Explorer)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; heritrix/1.3.0 http://www.cs.washington.edu/research/networking/websys/)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Hotbar 3.0)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iOpus-I-M)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iRider 2.21.1108; FDM)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; KKman3.0)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MathPlayer2.0)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon) ":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; PeoplePal 3.0; MSIECrawler)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461; IOpener Release 1.1.04)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; QihooBot 1.0 qihoobot@qihoo.net)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SIMBAR Enabled; InfoPath.1)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; StumbleUpon.com 1.760; .NET CLR 1.1.4322)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Embedded Web Browser from: http://bsalsa.com/; MSIECrawler)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.changedetection.com/bot.html )":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; DX-Browser 5.0.0.0)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FunWebProducts; ezPeer+ v1.0 Beta (0.4.1.98); ezPeer+ v1.0 (0.5.0.00); .NET CLR 1.1.4322; MSIECrawler)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; IBP; .NET CLR 1.1.4322)":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; MRA 4.3 (build 01218))":1,"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)":1,"Mozilla/4.0 (compatible; MSIE 7.0; Win32) Link Commander 4.0":1,"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; bgft)":1,"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; GTB5; User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://bsalsa.com) ; .NET CLR 2.0.50727)":1,"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Tablet PC 2.0)":1,"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 1.1.4322; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648)":1,"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)":1,"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; Orange 8.0; GTB6.3; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; Embedded Web Browser from: http://bsalsa.com/; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30618; OfficeLiveConnector.1.3; OfficeLivePatch.1.3)":1,"Mozilla/4.0 (compatible; MSIE enviable; DAUMOA 2.0; DAUM Web Robot; Daum Communications Corp., Korea; +http://ws.daum.net/aboutkr.html)":1,"Mozilla/4.0 (compatible; MSIE is not me; DAUMOA/1.0.1; DAUM Web Robot; Daum Communications Corp., Korea)":1,"Mozilla/4.0 (compatible; NaverBot/1.0; http://help.naver.com/delete_main.asp)":1,"Mozilla/4.0 (compatible; Netcraft Web Server Survey)":1,"Mozilla/4.0 (compatible; NetPromoter Spider;http://www.net-promoter.com/)":1,"Mozilla/4.0 (compatible; Opera/3.0; Windows 4.10) 3.51 [en]":1,"Mozilla/4.0 (compatible; Powermarks/3.5; Windows 95/98/2000/NT)":1,"Mozilla/4.0 (compatible; RSS Popper)":1,"Mozilla/4.0 (compatible; SiteKiosk 4.0; MSIE 5.0; Windows 98; SiteCoach 1.0)":1,"Mozilla/4.0 (compatible; SpeedySpider; www.entireweb.com)":1,"Mozilla/4.0 (compatible; SPENG)":1,"Mozilla/4.0 (compatible; SuperCleaner 2.xx; Windows 98)":1,"Mozilla/4.0 (compatible; Synapse)":1,"Mozilla/4.0 (compatible; WebCapture 3.0; Windows)":1,"Mozilla/4.0 (compatible; Win32; WinHttp.WinHttpRequest.5)":1,"Mozilla/4.0 (compatible; WSN Links)":1,"Mozilla/4.0 (compatible; www.euro-directory.com; urlchecker1.0)":1,"Mozilla/4.0 (compatible; www.galaxy.com)":1,"Mozilla/4.0 (compatible; www.linkguard.com Linkguard Online 1.0; Windows NT)":1,"Mozilla/4.0 (compatible; Y!J; for robot study; keyoshid)":1,"Mozilla/4.0 (compatible; Yahoo Japan; for robot study; kasugiya)":1,"Mozilla/4.0 (compatible;MSIE 6.0; Windows NT 5.0; H010818)":1,"Mozilla/4.0 (fantomBrowser)":1,"Mozilla/4.0 (fantomCrew Browser)":1,"Mozilla/4.0 (hhjhj@yahoo.com)":1,"Mozilla/4.0 (JemmaTheTourist;http://www.activtourist.com)":1,"Mozilla/4.0 (MobilePhone PM-8200/US/1.0) NetFront/3.x MMP/2.0":1,"Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)":1,"Mozilla/4.0 (MobilePhone SCP-5500/US/1.0) NetFront/3.0 MMP/2.0 FAKE (compatible; Googlebot/2.1; http://www.google.com/bot.html)":1,"Mozilla/4.0 (Mozilla; http://www.mozilla.org/docs/en/bot.html; master@mozilla.com)":1,"Mozilla/4.0 (Sleek Spider/1.2)":1,"Mozilla/4.0 compatible FurlBot/Furl Search 2.0 (FurlBot; http://www.furl.net; wn.furlbot@looksmart.net)":1,"Mozilla/4.0 compatible ZyBorg/1.0 (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)":1,"Mozilla/4.0 compatible ZyBorg/1.0 (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)":1,"Mozilla/4.0 compatible ZyBorg/1.0 Dead Link Checker (wn.zyborg@looksmart.net; http://www.WISEnutbot.com)":1,"Mozilla/4.0 compatible ZyBorg/1.0 for Homepage (ZyBorg@WISEnutbot.com; http://www.WISEnutbot.com)":1,"Mozilla/4.0 efp@gmx.net":1,"Mozilla/4.0 WebTV/2.6 (compatible; MSIE 4.0)":1,"Mozilla/4.0 [en] (Ask Jeeves Corporate Spider)":1,"Mozilla/4.0(compatible; Zealbot 1.0)":1,"Mozilla/4.01 (compatible; NORAD National Defence Network)":1,"Mozilla/4.01 [en](Win95;I)":1,"Mozilla/4.02 [en] (X11; I; SunOS 5.6 sun4u)":1,"Mozilla/4.04 (compatible; Dulance bot; +http://www.dulance.com/bot.jsp)":1,"Mozilla/4.04 [en] (X11; I; HP-UX B.10.20 9000/712)":1,"Mozilla/4.04 [en] (X11; I; IRIX 5.3 IP22)":1,"Mozilla/4.05 (Macintosh; I; 68K Nav)":1,"Mozilla/4.05 (Macintosh; I; PPC Nav)":1,"Mozilla/4.05 [en] (X11; I; SunOS 4.1.4 sun4m)":1,"Mozilla/4.08 [en] (Win98; U ;Nav)":1,"Mozilla/4.08 [en] (WinNT; U)":1,"Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_TrueRobot/1.4 libwww/5.2.8":1,"Mozilla/4.0_(compatible;_MSIE_5.0;_Windows_95)_VoilaBot/1.6 libwww/5.3.2":1,"Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)":1,"Mozilla/4.5 (compatible; iCab 2.5.3; Macintosh; I; PPC)":1,"Mozilla/4.5 (compatible; OmniWeb/4.0.5; Mac_PowerPC)":1,"Mozilla/4.5 (compatible; OmniWeb/4.1-beta-1; Mac_PowerPC)":1,"Mozilla/4.5 RPT-HTTPClient/0.3-2":1,"Mozilla/4.5 [en]C-CCK-MCD {RuralNet} (Win98; I)":1,"Mozilla/4.5b1 [en] (X11; I; Linux 2.0.35 i586)":1,"Mozilla/4.6 [en] (http://www.cnet.com/)":1,"Mozilla/4.61 [de] (OS/2; I)":1,"Mozilla/4.61 [en] (X11; U; ) - BrowseX (2.0.0 Windows)":1,"Mozilla/4.7":1,"Mozilla/4.7 (compatible; http://eidetica.com/spider)":1,"Mozilla/4.7 (compatible; Intelliseek; http://www.intelliseek.com)":1,"Mozilla/4.7 (compatible; OffByOne; Windows 98) Webster Pro V3.2":1,"Mozilla/4.7 (compatible; Whizbang)":1,"Mozilla/4.7 (compatible; WhizBang; http://www.whizbang.com/crawler)":1,"Mozilla/4.7 [en](BecomeBot@exava.com)":1,"Mozilla/4.7 [en](Exabot@exava.com)":1,"Mozilla/4.7 [en]C-CCK-MCD {Yahoo;YIP052400} (Win95; I)":1,"Mozilla/4.72 [en] (BACS http://www.ba.be)":1,"Mozilla/4.72C-CCK-MCD Caldera Systems OpenLinux [en] (X11; U; Linux 2.2.14 i686)":1,"Mozilla/4.75C-ja [ja] (X11; U; OSF1 V5.1 alpha)":1,"Mozilla/4.76 (Windows 98; U) Opera 5.12 [en]":1,"Mozilla/4.76 [en] (X11; U; FreeBSD 4.4-STABLE i386)":1,"Mozilla/4.76 [en] (X11; U; SunOS 5.7 sun4u)":1,"Mozilla/4.77C-SGI [en] (X11; U; IRIX 6.5 IP32)":1,"Mozilla/5.0":1,"Mozilla/5.0 (+http://www.eurekster.com/mammoth) Mammoth/0.1":1,"Mozilla/5.0 (+http://www.sli-systems.com/) Mammoth/0.1":1,"Mozilla/5.0 (Clustered-Search-Bot/1.0; support@clush.com; http://www.clush.com/)":1,"Mozilla/5.0 (compatible) GM RSS Panel X":1,"Mozilla/5.0 (compatible; +http://www.evri.com/evrinid)":1,"Mozilla/5.0 (compatible; 008/0.83; http://www.80legs.com/spider.html;) Gecko/2008032620":1,"Mozilla/5.0 (compatible; Abonti/0.8 - http://www.abonti.com)":1,"Mozilla/5.0 (compatible; aiHitBot/1.0; +http://www.aihit.com/)":1,"Mozilla/5.0 (compatible; AnsearchBot/1.x; +http://www.ansearch.com.au/)":1,"Mozilla/5.0 (compatible; archive.org_bot/1.10.0 +http://www.loc.gov/minerva/crawl.html)":1,"Mozilla/5.0 (compatible; archive.org_bot/1.13.1x http://crawler.archive.org)":1,"Mozilla/5.0 (compatible; archive.org_bot/1.5.0-200506132127 http://crawler.archive.org) Hurricane Katrina":1,"Mozilla/5.0 (compatible; Ask Jeeves/Teoma; http://about.ask.com/en/docs/about/webmasters.shtml)":1,"Mozilla/5.0 (compatible; BanBots/2.0b; Fetch; +http://www.banbots.com)":1,"Mozilla/5.0 (compatible; BecomeBot/1.23; http://www.become.com/webmasters.html)":1,"Mozilla/5.0 (compatible; BecomeBot/1.xx; MSIE 6.0 compatible; http://www.become.com/webmasters.html)":1,"Mozilla/5.0 (compatible; BecomeBot/2.0beta; http://www.become.com/webmasters.html)":1,"Mozilla/5.0 (compatible; BecomeBot/2.x; MSIE 6.0 compatible; http://www.become.com/site_owners.html)":1,"Mozilla/5.0 (compatible; BecomeJPBot/2.3; MSIE 6.0 compatible; +http://www.become.co.jp/site_owners.html)":1,"Mozilla/5.0 (compatible; BlogRefsBot/0.1; http://www.blogrefs.com/about/bloggers)":1,"Mozilla/5.0 (compatible; Bot; +http://pressemitteilung.ws/spamfilter":1,"Mozilla/5.0 (compatible; BuzzRankingBot/1.0; +http://www.buzzrankingbot.com/)":1,"Mozilla/5.0 (compatible; Charlotte/1.0b; charlotte@betaspider.com)":1,"Mozilla/5.0 (compatible; Charlotte/1.0b; http://www.searchme.com/support/)":1,"Mozilla/5.0 (compatible; Crawling jpeg; http://www.yama.info.waseda.ac.jp)":1,"Mozilla/5.0 (compatible; Custo 3 (Netwu.com); Windows NT 5.1)":1,"Mozilla/5.0 (compatible; de/1.13.2 +http://www.de.com)":1,"Mozilla/5.0 (compatible; Diffbot/0.1; +http://www.diffbot.com)":1,"Mozilla/5.0 (compatible; DNS-Digger-Explorer/1.0; +http://www.dnsdigger.com)":1,"Mozilla/5.0 (compatible; DNS-Digger/1.0; +http://www.dnsdigger.com)":1,"Mozilla/5.0 (compatible; EARTHCOM.info/2.01; http://www.earthcom.info)":1,"Mozilla/5.0 (compatible; EARTHCOM/2.2; +http://enter4u.eu)":1,"Mozilla/5.0 (compatible; egothor/8.0g; +http://ego.ms.mff.cuni.cz/)":1,"Mozilla/5.0 (compatible; Exabot Test/3.0; +http://www.exabot.com/go/robot)":1,"Mozilla/5.0 (compatible; FatBot 2.0; http://www.thefind.com/main/CrawlerFAQs.fhtml)":1,"Mozilla/5.0 (compatible; Galbot/1.0; +http://www.galbot.com/bot.html)":1,"mozilla/5.0 (compatible; genevabot http://www.healthdash.com)":1,"Mozilla/5.0 (compatible; Google Desktop) Paros/3.2.12":1,"Mozilla/5.0 (compatible; Googlebot/2.1; http://www.google.com/bot.html)":1,"mozilla/5.0 (compatible; heritrix/1.0.4 http://innovationblog.com)":1,"Mozilla/5.0 (compatible; heritrix/1.10.2 +http://i.stanford.edu/)":1,"Mozilla/5.0 (compatible; heritrix/1.12.1 +http://newstin.com/)":1,"Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.page-store.com)":1,"Mozilla/5.0 (compatible; heritrix/1.12.1 +http://www.page-store.com) [email:paul@page-store.com]":1,"mozilla/5.0 (compatible; heritrix/1.3.0 http://archive.crawler.org)":1,"Mozilla/5.0 (compatible; heritrix/1.4.0 +http://www.chepi.net)":1,"Mozilla/5.0 (compatible; heritrix/1.4t http://www.truveo.com/)":1,"Mozilla/5.0 (compatible; heritrix/1.5.0 http://www.l3s.de/~kohlschuetter/projects/crawling/)":1,"Mozilla/5.0 (compatible; heritrix/1.5.0-200506231921 http://pandora.nla.gov.au/crawl.html)":1,"Mozilla/5.0 (compatible; heritrix/1.6.0 http://www.worio.com/)":1,"Mozilla/5.0 (compatible; heritrix/1.7.0 +http://www.greaterera.com/)":1,"Mozilla/5.0 (compatible; Heritrix/1.8.0 http://www.hanzoarchives.com)":1,"Mozilla/5.0 (compatible; heritrix/1.x.x +http://www.accelobot.com)":1,"Mozilla/5.0 (compatible; heritrix/2.0.0-RC1 +http://www.aol.com)":1,"Mozilla/5.0 (compatible; Hermit Search. Com; +http://www.hermitsearch.com)":1,"Mozilla/5.0 (compatible; http://www.IsMySiteUp.Net/bot/ )":1,"Mozilla/5.0 (compatible; http://www.UptimeAuditor.com/bot/ )":1,"Mozilla/5.0 (compatible; HyperixScoop/1.3; +http://www.hyperix.com)":1,"Mozilla/5.0 (compatible; iaskspider/1.0; MSIE 6.0)":1,"Mozilla/5.0 (compatible; IDBot/1.0; +http://www.id-search.org/bot.html)":1,"Mozilla/5.0 (compatible; InterseekWeb/3.x)":1,"Mozilla/5.0 (compatible; Jim +http://www.hanzoarchives.com)":1,"Mozilla/5.0 (compatible; Konqueror/2.0.1; X11); Supports MD5-Digest; Supports gzip encoding":1,"Mozilla/5.0 (compatible; Konqueror/2.1.1; X11)":1,"Mozilla/5.0 (compatible; Konqueror/2.2.2)":1,"Mozilla/5.0 (compatible; Konqueror/2.2.2; Linux 2.4.14-xfs; X11; i686)":1,"Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)":1,"Mozilla/5.0 (compatible; LemSpider 0.1)":1,"Mozilla/5.0 (compatible; LinksManager.com_bot http://linksmanager.com/linkchecker.html)":1,"Mozilla/5.0 (compatible; LinkStash Bookmark Manager; http://www.xrayz.co.uk/)":1,"Mozilla/5.0 (compatible; MojeekBot/2.0; http://www.mojeek.com/bot.html)":1,"Mozilla/5.0 (compatible; MOSBookmarks/v2.6-Plus; Link Checker)":1,"Mozilla/5.0 (compatible; MSIE 6.0; Podtech Network; crawler_admin@podtech.net)":1,"Mozilla/5.0 (compatible; OnetSzukaj/5.0; http://szukaj.onet.pl)":1,"Mozilla/5.0 (compatible; PagestackerBot; http://www.pagestacker.com)":1,"Mozilla/5.0 (compatible; PalmeraBot; http://www.links24h.com/help/palmera) Version 0.001":1,"Mozilla/5.0 (compatible; PEAR HTTP_Request class; http://feed.moo.jp/)":1,"Mozilla/5.0 (compatible; Phonifier; +http://www.phonifier.com)":1,"Mozilla/5.0 (compatible; pmoz.info ODP link checker; +http://pmoz.info/doc/botinfo.htm)":1,"Mozilla/5.0 (compatible; pogodak.ba/3.x)":1,"Mozilla/5.0 (compatible; Pogodak.hr/3.1)":1,"Mozilla/5.0 (compatible; Proximic crawler; +http://www.proximic.com/en/about-us/contact-us.html)":1,"Mozilla/5.0 (compatible; PWeBot/3.1; http://www.programacionweb.net/robot.php)":1,"Mozilla/5.0 (compatible; Quantcastbot/1.0; www.quantcast.com)":1,"Mozilla/5.0 (compatible; robtexbot/1.0; http://www.robtex.com/ )":1,"Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)":1,"Mozilla/5.0 (compatible; Scrubby/2.2; http://www.scrubtheweb.com/)":1,"Mozilla/5.0 (compatible; ShunixBot/1.x.x +http://www.shunix.com/robot.htm)":1,"Mozilla/5.0 (compatible; ShunixBot/1.x; http://www.shunix.com/bot.htm)":1,"Mozilla/5.0 (compatible; SkreemRBot +http://skreemr.com)":1,"Mozilla/5.0 (compatible; SnapPreviewBot; en-US; rv:1.8.0.9) Gecko/20061206 Firefox/1.5.0.9":1,"Mozilla/5.0 (compatible; SpurlBot/0.2) ":1,"Mozilla/5.0 (compatible; SummizeBot +http://www.summize.com)":1,"Mozilla/5.0 (compatible; SYCLIKControl/LinkChecker;)":1,"Mozilla/5.0 (compatible; Synoobot/0.9; http://www.synoo.com/search/bot.html)":1,"Mozilla/5.0 (compatible; Theophrastus/x.x; http://users.cs.cf.ac.uk/N.A.Smith/theophrastus.php)":1,"Mozilla/5.0 (compatible; TridentSpider/3.1)":1,"Mozilla/5.0 (compatible; Vagabondo/2.1; webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)":1,"Mozilla/5.0 (compatible; Webduniabot/1.0; +http://search.webdunia.com/bot.aspx)":1,"Mozilla/5.0 (compatible; Windows NT 5.0; phpwebbrainBot/0.1 - http://www.monsterli.ch/phpwebbrain/)":1,"Mozilla/5.0 (compatible; worio bot heritrix/1.10.0 +http://worio.com)":1,"Mozilla/5.0 (compatible; WoW Lemmings Kathune/2.0;http://www.wowlemmings.com/kathune.html)":1,"Mozilla/5.0 (compatible; XTbot/1.0v; +http://www.externaltest.com)":1,"Mozilla/5.0 (compatible; Yahoo! DE Slurp; http://help.yahoo.com/help/us/ysearch/slurp)":1,"Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)":1,"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)":1,"Mozilla/5.0 (compatible; YesupBot/1.0; +http://www.yesup.net/bot.html)":1,"Mozilla/5.0 (compatible; Yoono; http://www.yoono.com/)":1,"Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )":1,"Mozilla/5.0 (compatible; Zenbot/1.3; +http://zen.co.za/webmasters/)":1,"Mozilla/5.0 (compatible; zermelo +http://www.powerset.com) [email:paul@page-store.com,crawl@powerset.com]":1,"Mozilla/5.0 (compatible;archive.org_bot/1.7.1; collectionId=316; Archive-It; +http://www.archive-it.org)":1,"Mozilla/5.0 (compatible;archive.org_bot/heritrix-1.9.0-200608171144 +http://pandora.nla.gov.au/crawl.html)":1,"Mozilla/5.0 (compatible;FindITAnswersbot/1.0;+http://search.it-influentials.com/bot.htm)":1,"Mozilla/5.0 (compatible;MAINSEEK_BOT)":1,"Mozilla/5.0 (Gecko/20070310 Mozshot/0.0.20070628; http://mozshot.nemui.org/)":1,"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9b5) Gecko/2008032619 Firefox/3.0b5":1,"Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20021219 Chimera/0.6 ":1,"Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.0.1) Gecko/20030306 Camino/0.7":1,"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US) AppleWebKit/xx (KHTML like Gecko) OmniWeb/v5xx.xx":1,"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/xxx.x (KHTML like Gecko) Safari/12x.x":1,"Mozilla/5.0 (Macintosh; U; PPC; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1":1,"Mozilla/5.0 (research@mediatrec.com)":1,"Mozilla/5.0 (Sage)":1,"Mozilla/5.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)":1,"Mozilla/5.0 (Slurp/si; slurp@inktomi.com; http://www.inktomi.com/slurp.html)":1,"Mozilla/5.0 (SunOS 5.8 sun4u; U) Opera 5.0 [en]":1,"Mozilla/5.0 (Twiceler-0.9 http://www.cuill.com/twiceler/robot.html)":1,"Mozilla/5.0 (Version: xxxx Type:xx)":1,"Mozilla/5.0 (wgao@genieknows.com)":1,"Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1":1,"Mozilla/5.0 (Windows; U; Win98; en-US; rv:x.xx) Gecko/20030423 Firebird Browser/0.6":1,"Mozilla/5.0 (Windows; U; Win9x; en; Stable) Gecko/20020911 Beonex/0.8.1-stable":1,"Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.7) NimbleCrawler 1.11 obeys UserAgent NimbleCrawler For problems contact: crawler_at_dataalchemy.com":1,"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.2.153.1 Safari/525.19":1,"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.5) Gecko/20060731 Firefox/1.5.0.5 Flock/0.7.4.1":1,"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.4/Megaupload x.0":1,"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.1) Gecko/2008092215 Firefox/3.0.1 Orca/1.1 beta 3":1,"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.x.x) Gecko/20041107 Firefox/x.x":1,"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xx) Gecko/20030504 Mozilla Firebird/0.6":1,"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:x.xxx) Gecko/20041027 Mnenhy/0.6.0.104":1,"Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.8.1) VoilaBot BETA 1.2 (support.voilabot@orange-ftgroup.com)":1,"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5":1,"Mozilla/5.0 (Windows; U;XMPP Tiscali Communicator v.10.0.1; Windows NT 5.1; it; rv:1.8.1.3) Gecko/20070309 Firefox/2.0.0.3":1,"Mozilla/5.0 (Windows;) NimbleCrawler 1.12 obeys UserAgent NimbleCrawler For problems contact: crawler@health":1,"Mozilla/5.0 (Windows;) NimbleCrawler 1.12 obeys UserAgent NimbleCrawler For problems contact: crawler@healthline.com":1,"Mozilla/5.0 (X11; Linux i686; U;rv: 1.7.13) Gecko/20070322 Kazehakase/0.4.4.1":1,"Mozilla/5.0 (X11; U; Linux 2.4.2-2 i586; en-US; m18) Gecko/20010131 Netscape6/6.01":1,"Mozilla/5.0 (X11; U; Linux i686; de-AT; rv:1.8.0.2) Gecko/20060309 SeaMonkey/1.0":1,"Mozilla/5.0 (X11; U; Linux i686; en-GB; rv:1.7.6) Gecko/20050405 Epiphany/1.6.1 (Ubuntu) (Ubuntu package 1.0.2)":1,"Mozilla/5.0 (X11; U; Linux i686; en-US; Nautilus/1.0Final) Gecko/20020408":1,"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:0.9.3) Gecko/20010801":1,"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2.1; aggregator:Spinn3r (Spinn3r 3.1); http://spinn3r.com/robot) Gecko/20021130":1,"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.2b) Gecko/20021007 Phoenix/0.3":1,"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040413 Epiphany/1.2.1":1,"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.7) Gecko/20060909 Firefox/1.5.0.7 SnapPreviewBot":1,"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1) Gecko/20061129 BonEcho/2.0":1,"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.1) Gecko/20061205 Iceweasel/2.0.0.1 (Debian-2.0.0.1+dfsg-2)":1,"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9a8) Gecko/2007100619 GranParadiso/3.0a8":1,"Mozilla/5.0 Galeon/1.0.2 (X11; Linux i686; U;) Gecko/20011224":1,"Mozilla/5.0 gURLChecker/0.x.x (Linux)":1,"Mozilla/5.0 URL-Spider":1,"Mozilla/5.0 usww.com-Spider-for-w8.net":1,"Mozilla/5.0 wgao@genieknows.com":1,"Mozilla/5.0 whoiam [http://www.axxus.de/]":1,"Mozilla/5.0 [en] (compatible; Gulper Web Bot 0.2.4 www.ecsl.cs.sunysb.edu/~maxim/cgi-bin/Link/GulperBot)":1,"MQbot metaquerier.cs.uiuc.edu/crawler":1,"MQBOT/Nutch-0.9-dev (MQBOT Nutch Crawler; http://falcon.cs.uiuc.edu; mqbot@cs.uiuc.edu)":1,"MSFrontPage/4.0":1,"MSIE 4.0 (Win95)":1,"MSIE-5.13 (larbin@unspecified.mail)":1,"msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)":1,"msnbot-Products/1.0 (+http://search.msn.com/msnbot.htm)":1,"MSNBOT/0.xx (http://search.msn.com/msnbot.htm)":1,"msnbot/x.xx ( http://search.msn.com/msnbot.htm)":1,"MSNBOT_Mobile MSMOBOT Mozilla/2.0 (compatible; MSIE 4.02; Windows CE; Default)":1,"MSNPTC/1.0":1,"MSProxy/2.0":1,"MSRBOT":1,"MSRBOT (http://research.microsoft.com/research/sv/msrbot)":1,"Mulder, VCR-1.0":1,"multiBlocker browser":1,"multicrawler ( http://sw.deri.org/2006/04/multicrawler/robots.html)":1,"MultiText/0.1":1,"MusicWalker2.0 ( http://www.somusical.com)":1,"MVAClient":1,"My WinHTTP Connection":1,"myDaemon":1,"MyGetRight/1.0.0":1,"MyGetRight/1.0b":1,"Mylinea.com Crawler 2.0":1,"mylinkcheck/1.02":1,"Naamah 1.0.1/Blogbot (http://blogbot.de/)":1,"Naamah 1.0a/Blogbot (http://blogbot.de/)":1,"NABOT/5.0":1,"nabot_1.0":1,"NameOfAgent (CMS Spider)":1,"naoFavicon4IE/1.xx":1,"NASA Search 1.0":1,"NationalDirectory-WebSpider/1.3":1,"NationalDirectoryAddURL/1.0":1,"NaverBot-1.0 (NHN Corp. / +82-2-3011-1954 / nhnbot@naver.com)":1,"NaverBot_dloader/1.5":1,"NavissoBot":1,"NavissoBot/1.7 (+http://navisso.com/)":1,"NCSA Beta 1 (http://vias.ncsa.uiuc.edu/viasarchivinginformation.html)":1,"Nebullabot/2.2 (http://bot.nebulla.info)":1,"NEC Research Agent -- compuman at research.nj.nec.com":1,"NEC-Hayek/1.0":1,"Net-Seekr Bot/Net-Seekr Bot V1 (http://www.net-seekr.com)":1,"NetAnts/1.2x":1,"NETCOMplete/x.xx":1,"NetinfoBot/1.0 (http://netinfo.bg/netinfobot.html)":1,"NetLookout/2.24":1,"Netluchs/0.8-dev ( ; http://www.netluchs.de/; ___don't___spam_me_@netluchs.de)":1,"NetMechanic Vx.0":1,"NetNewsWire/2.x (Mac OS X; http://ranchero.com/netnewswire/)":1,"NetNoseCrawler/v1.0":1,"Netprospector JavaCrawler":1,"NetPumper/x.xx":1,"NetResearchServer(http://www.look.com)":1,"NetResearchServer/x.x(loopimprovements.com/robot.html)":1,"NetSeer/Nutch-0.9 (NetSeer Crawler; http://www.netseer.com; crawler@netseer.com)":1,"NetSprint -- 2.0":1,"NetWhatCrawler/0.06-dev (NetWhatCrawler from NetWhat.com; http://www.netwhat.com; support@netwhat.com)":1,"NetZippy":1,"NeuralBot/0.2":1,"newsearchengine (ThisUser@unspecified.mail)":1,"NewsGator FetchLinks extension/0.2.0 (http://graemef.com)":1,"NewsGatorOnline/2.0 (http://www.newsgator.com; 1 subscribers)":1,"NextGenSearchBot 1 (for information visit http://www.eliyon.com/NextGenSearchBot)":1,"NextopiaBOT (+http://www.nextopia.com) distributed crawler client beta v0.x":1,"NG-Search/0.90 (NG-SearchBot; http://www.ng-search.com; )":1,"NG/1.0":1,"NG/4.0.1229":1,"nicebot":1,"NICO/1.0":1,"Nikita the Spider (http://NikitaTheSpider.com/)":1,"NITLE Blog Spider/0.01":1,"Nitro Downloader 1.x (www.klsofttools.com)":1,"Noago Spider":1,"Nocilla/1.0":1,"Nokia-WAPToolkit/1.2 googlebot(at)googlebot.com":1,"Nokia6300/2.0 (05.50) Profile/MIDP-2.0 Configuration/CLDC-1.1 (botmobi http://find.mobi/bot.html abuse@mtld.mobi)":1,"Nokia6610/1.0 (3.09) Profile/MIDP-1.0 Configuration/CLDC-1.0 (compatible;YahooSeeker/M1A1-R2D2; http://help.yahoo.com/help/us/ysearch/crawling/crawling-01.html)":1,"Nokia7110/1.0 (05.01) (Google WAP Proxy/1.0)":1,"NokodoBot/1.x (+http://nokodo.com/bot.htm)":1,"Norbert the Spider(Burf.com)":1,"noxtrumbot/1.0 (crawler@noxtrum.com)":1,"noyona_0_1":1,"NP/0.1 (NP; http://www.nameprotect.com; npbot@nameprotect.com)":1,"NPBot (http://www.nameprotect.com/botinfo.html)":1,"NPBot-1/2.0":1,"Nsauditor/1.x":1,"NSPlayer/10.0.0.xxxx WMFSDK/10.0":1,"nsyght.com/Nutch-1.0-dev (nsyght.com; Nsyght.com)":1,"nsyght.com/Nutch-x.x (nsyght.com; search.nsyght.com)":1,"nttdirectory_robot/0.9 (super-robot@super.navi.ocn.ne.jp)":1,"Nucleus SiteList LinkChecker/1.1":1,"nuSearch Spider <a href='http://www.nusearch.com'>www.nusearch.com</a> (compatible; MSIE 4.01)":1,"NuSearch Spider (compatible; MSIE 6.0)":1,"NuSearch Spider www.nusearch.com":1,"Nutch":1,"Nutch crawler/Nutch-0.9 (picapage.com; admin@picapage.com)":1,"Nutch/Nutch-0.9 (Eurobot; http://www.ayell.eu )":1,"NutchCVS/0.06-dev (Nutch; http://www.nutch.org/docs/en/bot.html; nutch-agent@lists.sourceforge.net)":1,"NutchCVS/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)":1,"NutchCVS/0.7.1 (Nutch running at UW; http://www.nutch.org/docs/en/bot.html; sycrawl@cs.washington.edu)":1,"NutchEC2Test/Nutch-0.9-dev (Testing Nutch on Amazon EC2.; http://lucene.apache.org/nutch/bot.html; ec2test at lucene.com)":1,"NutchOrg/0.0x-dev (Nutch; http://www.nutch.org/docs/bot.html; nutch-agent@lists.sourceforge.net)":1,"nutchsearch/Nutch-0.9 (Nutch Search 1.0; herceg_novi at yahoo dot com)":1,"NutchVinegarCrawl/Nutch-0.8.1 (Vinegar; http://www.cs.washington.edu; eytanadar at gmail dot com)":1,"obidos-bot (just looking for books.)":1,"ObjectsSearch/0.01-dev (ObjectsSearch;http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)":1,"ObjectsSearch/0.0x (ObjectsSearch; http://www.ObjectsSearch.com/bot.html; support@thesoftwareobjects.com)":1,"oBot ((compatible;Win32))":1,"Ocelli/1.x (http://www.globalspec.com/Ocelli)":1,"Octopus":1,"Octora Beta - www.octora.com":1,"Octora Beta Bot - www.octora.com":1,"Offline Explorer 1.*":1,"OliverPerry":1,"OmniExplorer_Bot/1.0x (+http://www.omni-explorer.com) Internet CategorizerOmniExplorer http://www.omni-explorer.com/ car & shopping search (64.62.175.xxx)":1,"OmniExplorer_Bot/1.0x (+http://www.omni-explorer.com) Job Crawler":1,"OmniExplorer_Bot/1.1x (+http://www.omni-explorer.com) Torrent Crawler":1,"OmniExplorer_Bot/x.xx (+http://www.omni-explorer.com) WorldIndexer":1,"onCHECK Linkchecker von www.scientec.de fuer www.onsinn.de":1,"onCHECK-Robot, www.onsearch.de":1,"Onet.pl SA- http://szukaj.onet.pl":1,"online link validator (http://www.dead-links.com/)":1,"Online24-Bot (Version: 1.0x, powered by www.online24.de)":1,"OntoSpider/1.0 libwww-perl/5.65":1,"OOZBOT/0.20 ( http://www.setooz.com/oozbot.html ; agentname at setooz dot_com )":1,"OpenAcoon v4.0.x (www.openacoon.de)":1,"Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)":1,"Openfind data gatherer- Openbot/3.0+(robot-response@openfind.com.tw;+http://www.openfind.com.tw/robot.html)":1,"Openfind Robot/1.1A2":1,"OpenISearch/1.x (www.openisearch.com)":1,"OpenTaggerBot (http://www.opentagger.com/opentaggerbot.htm)":1,"OpenTextSiteCrawler/2.9.2":1,"OpenWebSpider/0.x.x (http://www.openwebspider.org)":1,"OpenWebSpider/x":1,"Opera/5.0 (Linux 2.0.38 i386; U) [en]":1,"Opera/5.11 (Windows ME; U) [ru]":1,"Opera/5.12 (Windows 98; U) [en]":1,"Opera/6.01 (larbin@unspecified.mail)":1,"Opera/6.x (Linux 2.4.8-26mdk i686; U) [en]":1,"Opera/6.x (Windows NT 4.0; U) [de]":1,"Opera/7.x (Windows NT 5.1; U) [en]":1,"Opera/8.xx (Windows NT 5.1; U; en)":1,"Opera/9.0 (Windows NT 5.1; U; en)":1,"Opera/9.00 (Windows NT 5.1; U; de)":1,"Opera/9.60 (Windows NT 5.1; U; de) Presto/2.1.1":1,"OpidooBOT (larbin2.6.3@unspecified.mail)":1,"OPWV-SDK UP.Browser/7.0.2.3.119 (GUI) MMP/2.0 Push/PO":1,"Oracle Application Server Web Cache 10g":1,"Oracle iMTCrawler":1,"Oracle Ultra Search":1,"OrangeSpider":1,"Orbiter/T-2.0 (+http://www.dailyorbit.com/bot.htm)":1,"Orca Browser (http://www.orcabrowser.com)":1,"OSSProxy 1.3.305.321 (Build 305.321 Win32 en-us)(Dec 21 2005 16:30:54)":1,"OutfoxBot/0.x (For internet experiments; http://; outfox.agent@gmail.com)":1,"OutfoxMelonBot/0.5 (for internet experiments; http://; outfoxbot@gmail.com)":1,"Overture-WebCrawler/3.8/Fresh (atw-crawler at fast dot no; http://fast.no/support/crawler.asp)":1,"OWR_Crawler 0.1":1,"ozelot/2.7.3 (Search engine indexer; www.flying-cat.de/ozelot; ozelot@flying-cat.de)":1,"PADLibrary Spider":1,"PageBitesHyperBot/600 (http://www.pagebites.com/)":1,"Pagebull http://www.pagebull.com/":1,"Pagestacker Bot":1,"page_verifier (http://www.securecomputing.com/goto/pv)":1,"PagmIEDownload":1,"parallelContextFocusCrawler1.1parallelContextFocusCrawler1.1":1,"ParaSite/1.0b (http://www.ianett.com/parasite/)":1,"Patwebbot (http://www.herz-power.de/technik.html)":1,"pavuk/0.9pl29b i686-pc-linux-gnu":1,"PBrowse 1.4b":1,"pd02_1.0.0 pd02_1.0.0@dzimi@post.sk":1,"PEAR HTTP_Request class ( http://pear.php.net/ )":1,"PEERbot www.peerbot.com":1,"PeopleChat/Search_Engine":1,"PEval 1.4b":1,"PHP/3.x.xx":1,"PHP/4.0.4pl1":1,"PHP/4.0.6":1,"PHP/4.1.1":1,"PHP/4.1.2":1,"PicoSearch/1.0":1,"Piffany_Web_Scraper_v0.x":1,"Piffany_Web_Spider_v0.x":1,"PigeonBot1.0 BETA":1,"PingALink Monitoring Services 1.0":1,"PingALink Monitoring Services 1.0 (http://www.pingalink.com)":1,"Pingdom GIGRIB (http://www.pingdom.com)":1,"pipeLiner/0.3a (PipeLine Spider;http://www.pipeline-search.com/webmaster.html; webmaster'at'pipeline-search.com)":1,"pipeLiner/0.xx (PipeLine Spider; http://www.pipeline-search.com/webmaster.html)":1,"Pita":1,"Pizilla++ ver 2.45":1,"PJspider/3.0 (pjspider@portaljuice.com; http://www.portaljuice.com)":1,"Plagger/0.x.xx (http://plagger.org/)":1,"PlagiarBot/1.0":1,"PlantyNet_WebRobot_V1.9 dhkang@plantynet.com":1,"plinki/0.1 (you got plinked! (thats a good thing..); http://www.plinki.com; crawl@plinki.com)":1,"PluckFeedCrawler/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://www.pluck.com; 1 subscribers)":1,"Pluggd/Nutch-0.9 (automated crawler http://www.pluggd.com;support at pluggd dot com)":1,"Pockey-GetHTML/4.12.0 (Win32; GUI; ix86)":1,"Pockey-GetHTML/x.xx":1,"Pockey/x.x.x":1,"Pockey7.x.x(WIN32GUI)":1,"POE-Component-Client-HTTP/0.64 (perl; N; POE; en; rv:0.640000)":1,"Poirot":1,"polybot 1.0 (http://cis.poly.edu/polybot/)":1,"Pompos/1.x http://dir.com/pompos.html":1,"Pompos/1.x pompos@iliad.fr":1,"Popdexter/1.0":1,"Port Huron Labs":1,"PortalBSpider/2.0 (spider@portalb.com)":1,"portalmmm/2.0 S500i(c20;TB)":1,"PostFavorites":1,"potbot 1.0":1,"PRCrawler/Nutch-0.9 (data mining development project; crawler@projectrialto.com)":1,"PrivacyFinder Cache Bot v1.0":1,"PrivacyFinder/1.1":1,"Privoxy/3.0 (Anonymous)":1,"Production Bot 0116B":1,"Production Bot 2016B":1,"Production Bot DOT 3016B":1,"Program Shareware 1.0.2":1,"Progressive Download":1,"Progressive Download HTTP check":1,"Project XP5 [2.03.07-111203]":1,"PROve AnswerBot 4.0":1,"ProWebGuide Link Checker (http://www.prowebguide.com)":1,"psbot/0.1 (+http://www.picsearch.com/bot.html)":1,"PSurf15a 11":1,"PSurf15a 51":1,"PSurf15a VA":1,"psycheclone":1,"PubCrawl (pubcrawl.stanford.edu)":1,"puf/0.91beta6a (Linux 2.2.18; i686)":1,"puf/0.93.2a (Linux 2.4.18; i686)":1,"pulseBot (pulse Web Miner)":1,"PureSight":1,"PuxaRapido v1.0":1,"PWeBot/1.2 Inspector (http://www.programacionweb.net/robot.php)":1,"PycURL":1,"PycURL/7.xx.x":1,"Python-urllib/1.1x":1,"Python-urllib/2.0a1":1,"Qango.com Web Directory (http://www.qango.com/)":1,"QEAVis Agent/Nutch-0.9 (Quantitative Evaluation of Academic Websites Visibility; http://nlp.uned.es/qeavis":1,"QPCreep Test Rig ( We are not indexing- just testing )":1,"QuepasaCreep ( crawler@quepasacorp.com )":1,"QuepasaCreep v0.9.1x":1,"QueryN Metasearch":1,"Quicksilver (Blacktree,MacOSX)":1,"QuickTime\\xaa.7.0.4 (qtver=7.0.4;cpu=PPC;os=Mac 10.3.9)":1,"QweeryBot/3.01 ( http://qweerybot.qweery.nl)":1,"Qweery_robot.txt_CheckBot/3.01 (http://qweerybot.qweery.com)":1,"R6_CommentReader_(www.radian6.com/crawler)":1,"R6_FeedFetcher_(www.radian6.com/crawler)":1,"rabaz (rabaz at gigabaz dot com)":1,"RaBot/1.0 Agent-admin/phortse@hanmail.net":1,"Rainbot1.1":1,"ramBot xtreme x.x":1,"RAMPyBot - www.giveRAMP.com/0.1 (RAMPyBot - www.giveRAMP.com; http://www.giveramp.com/bot.html; support@giveRAMP.com)":1,"RAMPyBot/0.8-dev (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)":1,"Rank Exec (rankexec.com) Reciprocal Link Manager 1.x/bot":1,"Rankivabot/3.2 (www.rankiva.com; 3.2; vzmxikn)":1,"Rational SiteCheck (Windows NT)":1,"RAYSPIDER/Nutch-0.9":1,"ReadABlog Spider (compatible; 1.1; feed update; www.readablog.com)":1,"RealDownload/4.0.0.4x":1,"REAP-crawler Nutch/Nutch-1.0-dev (Reap Project; http://reap.cs.cmu.edu/REAP-crawler/; Reap Project)":1,"Reaper [2.03.10-031204] (http://www.sitesearch.ca/reaper/)":1,"Reaper/2.0x (+http://www.sitesearch.ca/reaper)":1,"REBOL Core 2.x.x.x.x":1,"REBOL View 1.x.x.x.x":1,"RebusnetBot (+http://www.rebusnet.biz)":1,"RebusnetPADBot/1.5x (+http://www.rebusnet.biz)":1,"reciprocal links checker (http://www.recip-links.com/)":1,"RedBot/redbot-1.0 (Rediff.com Crawler; redbot at rediff dot com)":1,"RedCarpet/1.2 (http://www.redcarpet-inc.com/robots.html)":1,"RedCell/0.1 (InfoSec Search Bot (Coming Soon); http://www.telegenetic.net/bot.html; lhall@telegenetic.net)":1,"RedCell/0.1 (RedCell; telegenetic.net/bot.html; lhall_at_telegenetic.net)":1,"RedKernel WWW-Spider 2/0 (+http://www-spider.redkernel-softwares.com/)":1,"REL Link Checker Lite x.x":1,"RepoMonkey Bait & Tackle/v1.01":1,"Rewebber/1.2 libwww-perl/5.41":1,"rico/0.1":1,"RixBot (http://babelserver.org/rix)":1,"RMA/1.0 (compatible; RealMedia)":1,"RoboCrawl (http://www.canadiancontent.net)":1,"RoboCrawl (www.canadiancontent.net)":1,"RoboPal (http://www.findpal.com/)":1,"Robot/www.pj-search.com":1,"Robot: NutchCrawler- Owner: wdavies@acm.org":1,"Robot@SuperSnooper.Com":1,"Robozilla/1.0":1,"Rome Client (http://tinyurl.com/64t5n) Ver: 0.9":1,"Rotondo/3.1 libwww/5.3.1":1,"RPT-HTTPClient/0.3-x":1,"RRC (crawler_admin@bigfoot.com)":1,"RssBandit/1.5.0.10 (.NET CLR 1.1.4322.2407; WinNT 5.1.2600.0; http://www.rssbandit.org) (.NET CLR 1.1.4322.2407; WinNT 5.1.2600.0; )":1,"RSSMicro.com RSS/Atom Feed Robot":1,"RSSOwl/1.2.3 2006-11-26 (Windows; U; zhtw)":1,"RSSOwl/1.2.4 Preview Release 2007-04-15 (Windows; U; zhtw)":1,"RssReader/1.0.xx.x (http://www.rssreader.com) Microsoft Windows NT 5.1.2600.0":1,"RSurf15a 41":1,"RSurf15a 51":1,"RSurf15a 81":1,"Rubbot/1.0 (+http://rubhub.com/)":1,"RufusBot (Rufus Web Miner; http://64.124.122.252/feedback.html)":1,"RufusBot (Rufus Web Miner; http://www.webaroo.com/rooSiteOwners.html)":1,"Rumours-Agent":1,"RX Bar":1,"S&L Spider (http://search.hirners.com/)":1,"S.T.A.L.K.E.R. (http://www.seo-tools.net/en/bot.aspx)":1,"SafariBookmarkChecker (+http://www.coriolis.ch/)":1,"sait/Nutch-0.9 (SAIT Research; http://www.samsung.com)":1,"SandCrawler - Compatibility Testing":1,"SapphireWebCrawler/1.0 (Sapphire Web Crawler using Nutch; http://boston.lti.cs.cmu.edu/crawler/; mhoy@cs.cmu.edu)":1,"SapphireWebCrawler/Nutch-1.0-dev (Sapphire Web Crawler using Nutch; http://boston.lti.cs.cmu.edu/crawler/; mhoy@cs.cmu.edu)":1,"savvybot/0.2":1,"SBIder/0.7 (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html)":1,"SBIder/0.8-dev (SBIder; http://www.sitesell.com/sbider.html; http://support.sitesell.com/contact-support.html)":1,"SBL-BOT (http://sbl.net)":1,"ScanWeb":1,"ScholarUniverse/0.8 (Nutch;+http://scholaruniverse.com/bot.jsp; fetch-agent@scholaruniverse.com)":1,"schwarzmann.biz-Spider_for_paddel.org+(http://www.innerprise.net/usp-spider.asp)":1,"Science Traveller International 1X/1.0":1,"ScollSpider/2.0 (+http://www.webwobot.com/ScollSpider.php)":1,"Scooter-3.0.EU":1,"Scooter-3.0.FS":1,"Scooter-3.0.HD":1,"Scooter-3.0.VNS":1,"Scooter-3.0QI":1,"Scooter-3.2":1,"Scooter-3.2.BT":1,"Scooter-3.2.DIL":1,"Scooter-3.2.EX":1,"Scooter-3.2.JT":1,"Scooter-3.2.NIV":1,"Scooter-3.2.SF0":1,"Scooter-3.2.snippet":1,"Scooter-3.3dev":1,"Scooter-ARS-1.1":1,"Scooter-ARS-1.1-ih":1,"scooter-venus-3.0.vns":1,"Scooter-W3-1.0":1,"Scooter-W3.1.2":1,"Scooter/1.0":1,"Scooter/1.0 scooter@pa.dec.com":1,"Scooter/1.1 (custom)":1,"Scooter/2.0 G.R.A.B. V1.1.0":1,"Scooter/2.0 G.R.A.B. X2.0":1,"Scooter/3.3":1,"Scooter/3.3.QA.pczukor":1,"Scooter/3.3.vscooter":1,"Scooter/3.3_SF":1,"Scooter2_Mercator_x-x.0":1,"Scooter_bh0-3.0.3":1,"Scooter_trk3-3.0.3":1,"Scope (Mars+)":1,"ScoutAbout":1,"ScoutAnt/0.1; +http://www.ant.com/what_is_ant.com/":1,"scoutmaster":1,"Scrubby/2.x (http://www.scrubtheweb.com/)":1,"Scrubby/3.0 (+http://www.scrubtheweb.com/help/technology.html)":1,"Search+":1,"Search-Engine-Studio":1,"search.ch V1.4":1,"search.ch V1.4.2 (spiderman@search.ch; http://www.search.ch)":1,"Search/1.0 (http://www.innerprise.net/es-spider.asp)":1,"searchbot admin@google.com":1,"SearchByUsa/2 (SearchByUsa; http://www.SearchByUsa.com/bot.html; info@SearchByUsa.com)":1,"SearchdayBot":1,"SearchExpress Spider0.99":1,"SearchGuild/DMOZ/Experiment (searchguild@gmail.com)":1,"SearchGuild_DMOZ_Experiment (chris@searchguild.com)":1,"Searchit-Now Robot/2.2 (+http://www.searchit-now.co.uk)":1,"Searchmee! Spider v0.98a":1,"SearchSight/2.0 (http://SearchSight.com/)":1,"SearchSpider.com/1.1":1,"Searchspider/1.2 (SearchSpider; http://www.searchspider.com; webmaster@searchspider.com)":1,"SearchTone2.0 - IDEARE":1,"Seekbot/1.0 (http://www.seekbot.net/bot.html) HTTPFetcher/0.3":1,"Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.0 (XDF)":1,"Seekbot/1.0 (http://www.seekbot.net/bot.html) RobotsTxtFetcher/1.2":1,"Seeker.lookseek.com":1,"semaforo.net":1,"Semager/1.1 (http://www.semager.de/blog/semager-bots/)":1,"Semager/1.x (http://www.semager.de)":1,"semanticdiscovery/0.x":1,"Sensis Web Crawler (search_comments\\at\\sensis\\dot\\com\\dot\\au)":1,"Sensis.com.au Web Crawler (search_comments\\at\\sensis\\dot\\com\\dot\\au)":1,"SeznamBot/1.0":1,"SeznamBot/1.0 (+http://fulltext.seznam.cz/)":1,"SeznamBot/2.0-test (+http://fulltext.sblog.cz/)":1,"ShablastBot 1.0":1,"Shareaza v1.x.x.xx":1,"SharewarePlazaFileCheckBot/1.0+(+http://www.SharewarePlaza.com)":1,"Shim Crawler":1,"Shim-Crawler(Mozilla-compatible; http://www.logos.ic.i.u-tokyo.ac.jp/crawler/; crawl@logos.ic.i.u-tokyo.ac.jp)":1,"ShopWiki/1.0 ( +http://www.shopwiki.com/)":1,"ShopWiki/1.0 ( +http://www.shopwiki.com/wiki/Help:Bot)":1,"Shoula.com Crawler 2.0":1,"SietsCrawler/1.1 (+http://www.siets.biz)":1,"Sigram/Nutch-1.0-dev (Test agent for Nutch development; http://www.sigram.com/bot.html; bot at sigram dot com)":1,"Siigle Orumcex v.001 Turkey (http://www.siigle.com)":1,"silk/1.0":1,"silk/1.0 (+http://www.slider.com/silk.htm)/3.7":1,"SimpleFavPanel/1.2":1,"Simpy 1.x; http://www.simpy.com/":1,"Simpy/1.x (Simpy; http://www.simpy.com/?ref=bot; feedback at simpy dot com)":1,"Sirketcebot/v.01 (http://www.sirketce.com/bot.html)":1,"SiteBar/3.x.x (Bookmark Server; http://sitebar.org/)":1,"SiteBar/x.x":1,"SiteBar/x.x.x (Bookmark Server; http://sitebar.org/)":1,"sitecheck.internetseer.com":1,"sitecheck.internetseer.com (For more info see: http://sitecheck.internetseer.com)":1,"SiteRecon+(xx)":1,"SiteSnagger":1,"SiteSpider +(http://www.SiteSpider.com/)":1,"SiteSucker/1.x.x":1,"SiteTaggerBot (http://www.sitetagger.com/bot.htm)":1,"SiteTruth.com site rating system":1,"SiteWinder":1,"SiteXpert":1,"Skampy/0.9.x (http://www.skaffe.com/skampy-info.html)":1,"Skimpy/0.x (http://www.skaffe.com/skampy-info.html)":1,"Skywalker/0.1 (Skywalker; anonymous; anonymous)":1,"Slarp/0.1":1,"Sleipnir":1,"Sleipnir Version 1.xx":1,"Sleipnir Version2.x":1,"Sleipnir/2.xx":1,"Slider_Search_v1-de":1,"SlimBrowser":1,"Slurp/2.0 (slurp@inktomi.com; http://www.inktomi.com/slurp.html)":1,"Slurp/2.0-KiteWeekly (slurp@inktomi.com; http://www.inktomi.com/slurp.html)":1,"Slurp/si (slurp@inktomi.com; http://www.inktomi.com/slurp.html)":1,"Slurpy Verifier/1.0":1,"SlySearch (slysearch@slysearch.com)":1,"SlySearch/1.0 http://www.plagiarism.org/crawler/robotinfo.html":1,"SlySearch/1.x http://www.slysearch.com":1,"SmartDownload/1.2.67 (Win32; Jan 12 1999)":1,"SmartDownload/1.2.77 (Win32; Feb 1 2000)":1,"SmartDownload/1.2.77 (Win32; Jun 19 2001)":1,"smartwit.com":1,"SmiffyDCMetaSpider/1.0":1,"sna-0.0.1 (mikemuzio@msn.com)":1,"sna-0.0.1 mikeelliott@hotmail.com":1,"snap.com beta crawler v0":1,"Snapbot/1.0":1,"Snapbot/1.0 (Snap Shots, +http://www.snap.com)":1,"Snappy/1.1 ( http://www.urltrends.com/ )":1,"Snarfer/0.x.x (http://www.snarfware.com/)":1,"SnoopRob/x.x":1,"Snoopy v1.xx":1,"Snoopy v1.xx- : User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2)":1,"Snoopy_v0.xx":1,"SnykeBot/0.6 (http://www.snyke.com)":1,"SocSciBot ()":1,"SoftBank/1.0/812SH/SHJ001 Browser/NetFront/3.3 Profile/MIDP-2.0 Configuration/CLDC-1.1":1,"SoftHypermarketFileCheckBot/1.0+(+http://www.softhypermaket.com)":1,"Softizerbot (http://www.softizer.com)":1,"sogou develop spider":1,"Sogou Orion spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)":1,"sogou spider":1,"Sogou web spider/3.0(+http://www.sogou.com/docs/help/webmasters.htm#07)":1,"sohu agent":1,"sohu-search":1,"Sosospider+(+http://help.soso.com/webspider.htm)":1,"Space Bison/0.02 [fu] (Win67; X; SK)":1,"SpeedDownload/1.x":1,"speedfind ramBot xtreme 8.1":1,"Speedy Spider (Beta/x.x; speedy@entireweb.com)":1,"Speedy Spider (Entireweb; Beta/1.0; http://www.entireweb.com/about/search_tech/speedyspider/)":1,"Speedy_Spider (http://www.entireweb.com)":1,"Sphere Scout&v4.0 - scout at sphere dot com":1,"Sphider":1,"Spida/0.1":1,"Spider-Sleek/2.0 (+http://search-info.com/linktous.html)":1,"spider.batsch.com":1,"Spider.TerraNautic.net - v:1.04":1,"spider.yellopet.com - www.yellopet.com":1,"Spider/maxbot.com admin@maxbot.com":1,"SpiderKU/0.x":1,"SpiderMan":1,"SpiderMonkey/7.0x (SpiderMonkey.ca info at http://spidermonkey.ca/sm.shtml)":1,"Spinne/2.0":1,"Spinne/2.0 med":1,"Spinne/2.0 med_AH":1,"Spock Crawler (http://www.spock.com/crawler)":1,"sportsuchmaschine.de-Robot (Version: 1.02- powered by www.sportsuchmaschine.de)":1,"sproose/0.1-alpha (sproose crawler; http://www.sproose.com/bot.html; crawler@sproose.com)":1,"SQ Webscanner":1,"Squid-Prefetch":1,"squidclam":1,"SquidClamAV_Redirector 1.x.x":1,"Sqworm/2.9.81-BETA (beta_release; 20011102-760; i686-pc-linux-gnu)":1,"Sqworm/2.9.85-BETA (beta_release; 20011115-775; i686-pc-linux-gnu)":1,"Sqworm/2.9.89-BETA (beta_release; 20020130-839; i686-pc-linux-gnu) ":1,"SSurf15a 11 ":1,"StackRambler/x.x ":1,"Stamina/1.4":1,"Star Downloader":1,"StarDownloader/1.xx":1,"stat statcrawler@gmail.com":1,"Steeler/1.x (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)":1,"Steeler/3.3 (http://www.tkl.iis.u-tokyo.ac.jp/~crawler/)":1,"Strategic Board Bot (+http://www.strategicboard.com)":1,"Submission Spider at surfsafely.com":1,"suchbaer.de":1,"suchbaer.de (CrawlerAgent v0.103)":1,"suchbot":1,"Suchknecht.at-Robot":1,"suchpadbot/1.0 (+http://www.suchpad.de)":1,"Sunrise XP/2.x":1,"Sunrise/0.42g (Windows XP)":1,"SuperBot/x.x (Win32)":1,"SuperBot/x.x.x.xx (Windows XP)":1,"Superdownloads Spiderman":1,"SURF ":1,"SurferF3 1/0":1,"SurfMaster":1,"SurveyBot/2.2 <a href='http://www.whois.sc'>Whois Source</a>":1,"SurveyBot/2.3 (Whois Source)":1,"suzuran":1,"SWB/V1.4 (HP)":1,"swbot/0.9c libwww/5.3.1":1,"Swooglebot/2.0. (+http://swoogle.umbc.edu/swooglebot.htm)":1,"SWSBot-Images/1.2 http://www.smartwaresoft.com/swsbot12.html":1,"SygolBot http://www.sygol.net":1,"Sylera/1.2.x":1,"SyncBot":1,"SyncIT/x.x":1,"Syndirella/0.91pre":1,"SynoBot":1,"Syntryx ANT Scout Chassis Pheromone; Mozilla/4.0 compatible crawler":1,"Szukacz/1.x":1,"Szukacz/1.x (robot; www.szukacz.pl/jakdzialarobot.html; szukacz@proszynski.pl)":1,"T-Online Browser":1,"tags2dir.com/0.8 (+http://tags2dir.com/directory/)":1,"Tagword (http://tagword.com/dmoz_survey.php)":1,"Tagyu Agent/1.0":1,"Talkro Web-Shot/1.0 (E-mail: webshot@daumsoft.com- Home: http://222.122.15.190/webshot)":1,"TAMU_CS_IRL_CRAWLER/1.0":1,"targetblaster.com/0.9k":1,"TargetYourNews.com bot":1,"TCDBOT/Nutch-0.8 (PhD student research;http://www.tcd.ie; mcgettrs at t c d dot IE)":1,"TE":1,"TeamSoft WinInet Component":1,"TECOMAC-Crawler/0.x":1,"Tecomi Bot (http://www.tecomi.com/bot.htm)":1,"Teemer (NetSeer, Inc. is a Los Angeles based Internet startup company.; http://www.netseer.com/crawler.html; crawler@netseer.com)":1,"Teleport Pro/1.2x(.1xxx)":1,"Teoma MP":1,"teomaagent crawler-admin@teoma.com":1,"teomaagent1 [crawler-admin@teoma.com]":1,"teoma_agent1":1,"Teradex Mapper; mapper@teradex.com; http://www.teradex.com":1,"TeragramCrawler":1,"terraminds-bot/1.0 (support@terraminds.de)":1,"TerrawizBot/1.0 (+http://www.terrawiz.com/bot.html)":1,"Test spider":1,"TestCrawler/Nutch-0.9 (Testing Crawler for Research ; http://balihoo.com/index.aspx; tgautier at balihoo dot com)":1,"The Expert HTML Source Viewer (http://www.expert-html.com)":1,"TheRarestParser/0.2a (http://therarestwords.com/)":1,"TheSuBot/0.1 (www.thesubot.de)":1,"thumbshots-de-Bot (Version: 1.02, powered by www.thumbshots.de)":1,"thumbshots-de-Bot (Version: 1.02- powered by www.thumbshots.de)":1,"timboBot/0.9 http://www.breakingblogs.com/timbo_bot.html":1,"http://www.timelyweb.com/":1,"TinEye/1.1 (http://tineye.com/crawler.html)":1,"tivraSpider/1.0 (crawler@tivra.com)":1,"TJG/Spider":1,"TJvMultiHttpGrabber Component":1,"Tkensaku/x.x(http://www.tkensaku.com/q.html)":1,"toCrawl/UrlDispatcher":1,"Topodia/1.2-dev (Topodia - Crawler for HTTP content indexing; http://www.topodia.com/; support@topodia.com)":1,"TOPOS robot/1.1 (http://www.topos.com.ua/)":1,"traazibot/testengine (+http://www.traazi.de)":1,"Trailfire-bot/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)":1,"Trailfire-bot/0.7.1 (Trailfire page content analyzer; http://trailfire.com; info@trailfire.com)":1,"Trailfire/0.7.1 (Nutch; http://lucene.apache.org/nutch/bot.html; nutch-agent@lucene.apache.org)":1,"Trampelpfad-Spider":1,"Trampelpfad-Spider-v0.1":1,"tricosMetaCheck 1.2216-08-1999 (http://www.tricos.com/metacheck)":1,"TSurf15a 11":1,"TulipChain/5.x (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://java.sun.com/) Linux/2.4.17":1,"TulipChain/5.xx (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://apple.com/) Mac_OS_X/10.2.8":1,"Tumblr/1.0 RSS syndication (+http://www.tumblr.com/) (support@tumblr.com)":1,"TurnitinBot/x.x (http://www.turnitin.com/robot/crawlerinfo.html)":1,"Turnpike Emporium LinkChecker/0.1":1,"TutorGig/1.5 (+http://www.tutorgig.com/crawler)":1,"Tutorial Crawler 1.4 (http://www.tutorgig.com/crawler)":1,"Twiceler www.cuill.com/robots.html":1,"Twiceler-0.9 http://www.cuill.com/twiceler/robot.html":1,"Twisted PageGetter":1,"Twitturly / v0.x":1,"Twotrees Reactive Filter V2.0":1,"Tycoon Agent/Nutch-1.0-dev":1,"TygoBot":1,"TygoProwler":1,"UCmore":1,"UCMore Crawler App":1,"UCWEB5.1":1,"UDM":1,"UdmSearch/3.1.x":1,"UIowaCrawler/1.0":1,"UKWizz/Nutch-0.8.1 (UKWizz Nutch crawler; http://www.ukwizz.com/)":1,"Ultraseek":1,"Under the Rainbow 2.2":1,"unknownght.com Web Server IIS vs Apache Survey. See Results at www.DNSRight.com":1,"UofTDB_experiment (leehyun@cs.toronto.edu)":1,"UP.Browser/3.01-IG01 UP.Link/3.2.3.4":1,"updated/0.1-alpha (updated crawler; http://www.updated.com; crawler@updated.com)":1,"updated/0.1beta (updated.com; http://www.updated.com; crawler@updated.om)":1,"UPG1 UP/4.0 (compatible; Blazer 1.0)":1,"Uptimebot":1,"UptimeBot(www.uptimebot.com)":1,"URI::Fetch/0.06":1,"URL Spider Pro/x.xx (innerprise.net)":1,"URLBase/6.x":1,"URLBlaze":1,"urlfan-bot/1.0; +http://www.urlfan.com/site/bot/350.html":1,"URLGetFile":1,"URL_Spider_Pro/x.x":1,"URL_Spider_Pro/x.x+(http://www.innerprise.net/usp-spider.asp)":1,"User-Agent: BoardReader Favicon Fetcher /1.0 info@boardreader.com":1,"User-Agent: BoardReader Image Fetcher /1.0 info@boardreader.com":1,"User-Agent: LjSEEK Picture-Bot /1.0 contact@ljseek.com":1,"User-Agent: FileHeap! file downloader (http://www.fileheap.com)":1,"User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)":1,"User-Agent: Mozilla/4.0 (SKIZZLE! Distributed Internet Spider v1.0 - www.SKIZZLE.com)":1,"user-agent=Mozilla/3.01Gold":1,"USyd-NLP-Spider (http://www.it.usyd.edu.au/~vinci/bot.html)":1,"UtilMind HTTPGet":1,"Utopia WebWasher 3.0":1,"uTorrent/1500":1,"VadixBot":1,"Vagabondo-WAP/2.0 (webcrawler at wise-guys dot nl; http://webagent.wise-guys.nl/)/1.0 Profile":1,"Vagabondo/1.x MT (webagent@wise-guys.nl)":1,"Vagabondo/2.0 MT":1,"Vagabondo/2.0 MT (webagent at wise-guys dot nl)":1,"Vagabondo/2.0 MT (webagent@NOSPAMwise-guys.nl)":1,"Vagabondo/3.0 (webagent at wise-guys dot nl)":1,"Vakes/0.01 (Vakes; http://www.vakes.com/; search@vakes.com)":1,"VayalaCreep-v0.0.1 (haploid@haploid.com)":1,"Vayala|Creep-v0.0.1 (codepoet@wildties.com)":1,"vb wininet":1,"versus 0.2 (+http://versus.integis.ch)":1,"versus crawler eda.baykan@epfl.ch":1,"Verticrawlbot":1,"VeryGoodSearch.com.DaddyLongLegs":1,"verzamelgids.nl - Networking4all Bot/x.x":1,"Verzamelgids/2.2 (http://www.verzamelgids.nl)":1,"Vespa Crawler":1,"virus_detector (virus_harvester@securecomputing.com)":1,"VisBot/2.0 (Visvo.com Crawler; http://www.visvo.com/bot.html; bot@visvo.com)":1,"Visicom Toolbar":1,"Vision Research Lab image spider at vision.ece.ucsb.edu":1,"VLC media player - version 0.8.5 Janus - (c) 1996-2006 the VideoLAN team":1,"VMBot/0.x.x (VMBot; http://www.VerticalMatch.com/; vmbot@tradedot.com)":1,"Vortex/2.2 (+http://marty.anstey.ca/robots/vortex/)":1,"voyager-hc/1.0":1,"voyager/1.0":1,"voyager/2.0 (http://www.kosmix.com/html/crawler.html)":1,"VSE/1.0 (testcrawler@hotmail.com)":1,"VSE/1.0 (testcrawler@vivisimo.com)":1,"vspider":1,"vspider/3.x":1,"VWBOT/Nutch-0.9-dev (VWBOT Nutch Crawler; http://vwbot.cs.uiuc.edu;+vwbot@cs.uiuc.edu":1,"W3C-checklink/3.x.x.x libwww-perl/5.xx":1,"W3C-checklink/4.x [4.xx] libwww-perl/5.xxx":1,"W3C-WebCon/5.x.x libwww/5.x.x":1,"W3CLineMode/5.4.0 libwww/5.x.x":1,"W3CRobot/5.4.0 libwww/5.4.0":1,"W3C_Validator/1.xxx libwww-perl/5.xx":1,"w3m/0.x.xx":1,"W3SiteSearch Crawler_v1.1 http://www.w3sitesearch.de":1,"wadaino.jp-crawler 0.2 (http://wadaino.jp/)":1,"WannaBe (Macintosh; PPC)":1,"WapOnWindows 1.0":1,"Watchfire WebXM 1.0":1,"WAVcheck 1.0.x (http://www.webbanalys.se/apps/WAVcheck/)":1,"Wavefire/0.8-dev (Wavefire; http://www.wavefire.com; info@wavefire.com)":1,"Waypath development crawler - info at waypath dot com":1,"Waypath Scout v2.x - info at waypath dot com":1,"WDG_Validator/1.1":1,"Web Image Collector":1,"Web Link Validator 1.5":1,"Web Snooper":1,"web-bekannt (Version: 1.02, powered by www.internetservice-franken.de)":1,"web-bekannt (Version: 1.02, powered by www.web-bekannt.de)":1,"Web-Bot V1.03":1,"Web-Robot/5.0 (en-US; web-robot.com/policy.html) Web-Robot Crawler/2.0.3":1,"web2express.org/Nutch-0.9-dev (leveled playing field; http://web2express.org/; info at web2express.org)":1,"WebAlta Crawler/1.2.1 (http://www.webalta.ru/bot.html)":1,"WebarooBot (Webaroo Bot; http://64.124.122.252/feedback.html)":1,"WebarooBot (Webaroo Bot; http://www.webaroo.com/rooSiteOwners.html)":1,"WebAuto/3.4xxx (WinNT; I)":1,"webbandit/4.xx.0":1,"WebBug/5.x":1,"Webclipping.com":1,"webcollage/1.xx":1,"WebCompass 2.0":1,"WebCopier vx.x":1,"WebCopier vx.xa":1,"WebCorp/1.0":1,"webcrawl.net":1,"WebDownloader for X x.xx":1,"Webdup/0.9":1,"WebFetch":1,"webfetch/5.x.x":1,"WebFilter Robot 1.0":1,"WebFilter Robot 1.x":1,"WebFindBot(http://www.web-find.com)":1,"Webglimpse 2.xx.x (http://webglimpse.net)":1,"webGobbler/1.x.x":1,"webhack":1,"WebImages 0.3 ( http://herbert.groot.jebbink.nl/?app=WebImages )":1,"WebLight/4.x.x (support@illumit.com; http://www.illumit.com/Products/weblight/)":1,"Weblink's checker/":1,"Weblog Attitude Diffusion 1.0":1,"webmeasurement-bot, http://rvs.informatik.uni-leipzig.de":1,"WebMiner/x.x [en] (Win98; I)":1,"WeBoX/0.xx":1,"WebPix 1.0 (www.netwu.com)":1,"WebQL":1,"WebRACE/1.1 (University of Cyprus- Distributed Crawler)":1,"WebRankSpider/1.37 (+http://ulm191.server4you.de/crawler/)":1,"WebReaper vx.x - www.webreaper.net":1,"WebReaper [info@webreaper.net]":1,"WebReaper [webreaper@webreaper.net]":1,"WebSearch.COM.AU/3.0.1 (The Australian Search Engine; http://WebSearch.COM.AU; Search@WebSearch.COM.AU)":1,"WebSearchBench WebCrawler v0.1(Experimental)":1,"WebSearchBench WebCrawler V1.0 (Beta)- Prof. Dr.-Ing. Christoph Lindemann- Universität Dortmund- cl@cs.uni-dortmund.de- http://websearchbench.cs.uni-dortmund.de/":1,"Website Explorer/0.9.x.x":1,"Website eXtractor":1,"WebsiteWorth v1.0":1,"Webspinne/1.0 webmaster@webspinne.de":1,"Websquash.com (Add url robot)":1,"WebStat/1.0 (Unix; beta; 20040314)":1,"Webster v0.3 ( http://webster.healeys.net/ )":1,"webster-internet.de pad browser":1,"WebStripper/2.xx":1,"WebTrafficExpress/x.0":1,"WebTrends/3.0 (WinNT)":1,"WebVac (webmaster@pita.stanford.edu)":1,"WebVal/1.0":1,"Webverzeichnis.de - Telefon: 01908 / 26005":1,"WebVulnCrawl.unknown/1.0 libwww-perl/5.803":1,"WebWatcherMonitor/2.01":1,"WebZIP/x.x (http://www.spidersoft.com)":1,"Wells Search II":1,"WEP Search 00":1,"West Wind Internet Protocols 4.xx":1,"WFARC":1,"Wget/1.x(.x)GNU wget http://www.gnu.org/software/wget/wget.html - file downloader":1,"Wget/1.x+cvs-stable (Red Hat modified)":1,"Wget/1.x.x+cvs":1,"Whatsup/x.x":1,"whatUseek_winona/3.0":1,"WhizBang! Lab":1,"Wildsoft Surfer":1,"Willow Internet Crawler by Twotrees V2.1":1,"WinampMPEG/2.00 (larbin@unspecified.mail)":1,"WincerSong Agent v1.0":1,"Windows-Media-Player/10.00.00.xxxx":1,"WinGet 1.1":1,"WinHTTP Example/1.0":1,"WinkBot/0.06 (Wink.com search engine web crawler; http://www.wink.com/Wink:WinkBot; winkbot@wink.com)":1,"WinPodder (http://winpodder.com)":1,"WinWAP/3.x (3.x.x.xx; Win32) (Google WAP Proxy/1.0)":1,"Wir sind die Borg (Version: 1.03, Sie wurden Assimiliert +http://www.yammba.com/suchmaschine/bot.html)":1,"WIRE/0.11 (Linux; i686; Bot,Robot,Spider,Crawler,aromano@cli.di.unipi.it)":1,"WIRE/0.x (Linux; i686; Bot,Robot,Spider,Crawler)":1,"WISEbot/1.0 (WISEbot@koreawisenut.com; http://wisebot.koreawisenut.com)":1,"WiseWire-Spider2":1,"wish-project (http://wish.slis.tsukuba.ac.jp/)":1,"WordChampBot":1,"WordPress/x.x.x.x PHP/4.x.xx":1,"worio heritrix bot (+http://worio.com/)":1,"woriobot ( http://www.worio.com/)":1,"WorldLight":1,"WorQmada/1.0":1,"Wotbox/alpha0.6 (bot@wotbox.com; http://www.wotbox.com)":1,"Wotbox/alpha0.x.x (bot@wotbox.com; http://www.wotbox.com) Java/1.4.1_02":1,"WSB WebCrawler V1.0 (Beta)- cl@cs.uni-dortmund.de":1,"WSB, http://websearchbench.cs.uni-dortmund.de":1,"wume_crawler/1.1 (http://wume.cse.lehigh.edu/~xiq204/crawler/)":1,"Wusage/x.0@boutell.com":1,"Wwlib/Linux":1,"WWSBOT 1.x [--- http://www.analyzer.nu ---]":1,"WWW-Mechanize/1.1x":1,"www.arianna.it":1,"www.business-socket.com registry verify/1.x":1,"www.doweb.co.uk crawler":1,"www4mail/2.x libwww-FM/2.14 (Unix; I)":1,"WWWC/1.0x":1,"WWWeasel Robot v1.00 (http://wwweasel.de)":1,"WWWOFFLE/2.x":1,"wwwster/1.x (Beta- mailto:gue@cis.uni-muenchen.de)":1,"wxDownload Fast":1,"X-Crawler ":1,"Xaldon WebSpider":1,"Xenu Link Sleuth 1.xx":1,"Xenu's Link Sleuth 1.x[a-z]":1,"Xerka WebBot v1.0.0 [UPVOpenDir]":1,"xine/1.0":1,"xirq/0.1-beta (xirq; http://www.xirq.com; xirq@xirq.com)":1,"XMLSlurp/0.1 libwww-perl/5.805":1,"XRL/2.00b1 (Linux; i686; en-us) (+http://metamark.net/about)":1,"Xylix":1,"xyro_(xcrawler@cosmos.inria.fr)":1,"Y!J-BSC/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)":1,"Y!J-SRD/1.0":1,"Y!J/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)":1,"Y!OASIS/TEST no-ad Mozilla/4.08 [en] (X11; I; FreeBSD 2.2.8-STABLE i386)":1,"Y!TunnelPro":1,"yacy (www.yacy.net; v20040602; i386 Linux 2.4.26-gentoo-r13; java 1.4.2_06; MET/en)":1,"yacybot (x86 Windows XP 5.1; java 1.5.0_06; Europe/de) yacy.net":1,"Yahoo Pipes 1.0":1,"Yahoo! Mindset":1,"Yahoo-Blogs/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html )":1,"Yahoo-MMAudVid/1.0 (mms dash mmaudvidcrawler dash support at yahoo dash inc dot com)":1,"Yahoo-MMAudVid/2.0(mms dash mm aud vid crawler dash support at yahoo dash inc.com ;Mozilla 4.0 compatible; MSIE 7.0;Windows NT 5.0; .NET CLR 2.0)":1,"Yahoo-MMCrawler/3.x (mm dash crawler at trd dot overture dot com)":1,"Yahoo-Test/4.0":1,"Yahoo-VerticalCrawler-FormerWebCrawler/3.9 crawler at trd dot overture dot com; http://www.alltheweb.com/help/webmaster/crawler":1,"YahooFeedSeeker/2.0 (compatible; Mozilla 4.0; MSIE 5.5; http://publisher.yahoo.com/rssguide)":1,"YahooSeeker-Testing/v3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/)":1,"YahooSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/)":1,"YahooSeeker/1.0 (compatible; Mozilla 4.0; MSIE 5.5; http://search.yahoo.com/yahooseeker.html)":1,"YahooSeeker/1.1 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/shop/merchant/)":1,"YahooSeeker/bsv3.9 (compatible; Mozilla 4.0; MSIE 5.5; http://help.yahoo.com/help/us/ysearch/crawling/crawling-02.html )":1,"YahooSeeker/CafeKelsa-dev (compatible; Konqueror/3.2; FreeBSD ;cafekelsa-dev-webmaster@yahoo-inc.com )":1,"Yandex/1.01.001 (compatible; Win16; I)":1,"Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)":1,"yarienavoir.net/0.2":1,"Yeti":1,"Yeti/0.01 (nhn/1noon, yetibot@naver.com, check robots.txt daily and follows it)":1,"Yeti/1.0 (NHN Corp.; http://help.naver.com/robots/)":1,"yggdrasil/Nutch-0.9 (yggdrasil biorelated search engine; www dot biotec dot tu minus dresden do de slash schroeder; heiko dot dietze at biotec dot tu minus dresden dot de)":1,"YodaoBot/1.0 (http://www.yodao.com/help/webmaster/spider/; )":1,"yoofind/yoofind-0.1-dev (yoono webcrawler; http://www.yoono.com ; MyEmail)":1,"yoogliFetchAgent/0.1":1,"yoono/1.0 web-crawler/1.0":1,"YottaCars_Bot/4.12 (+http://www.yottacars.com) Car Search Engine ":1,"YottaShopping_Bot/4.12 (+http://www.yottashopping.com) Shopping Search Engine":1,"YTunnelPro":1,"Z-Add Link Checker (http://w3.z-add.co.uk/linkcheck/)":1,"Zao-Crawler":1,"Zao-Crawler 0.2b":1,"Zao/0.1 (http://www.kototoi.org/zao/)":1,"ZBot/1.00 (icaulfield@zeus.com)":1,"Zearchit":1,"ZeBot_lseek.net (bot@ze.bz)":1,"ZeBot_www.ze.bz (ze.bz@hotmail.com)":1,"zedzo.digest/0.1 (http://www.zedzo.com/)":1,"Zend_Http_Client":1,"zermelo Mozilla/5.0 compatible; heritrix/1.12.1 (+http://www.powerset.com) [email:crawl@powerset.com,email:paul@page-store.com]":1,"zerxbot/Version 0.6 libwww-perl/5.79":1,"Zeus ThemeSite Viewer Webster Pro V2.9 Win32":1,"Zeus xxxxx Webster Pro V2.9 Win32":1,"Zeusbot/0.07 (Ulysseek's web-crawling robot; http://www.zeusbot.com; agent@zeusbot.com)":1,"Ziggy -- The Clown From Hell!!":1,"ZipppBot/0.xx (ZipppBot; http://www.zippp.net; webmaster@zippp.net)":1,"ZIPPPCVS/0.xx (ZipppBot/.xx;http://www.zippp.net; webmaster@zippp.net)":1,"Zippy v2.0 - Zippyfinder.com":1,"Zoo Tycoon 2 Client -- http://www.zootycoon.com":1,"ZoomSpider - wrensoft.com":1,"Zscho.de Crawler/Nutch-1.0-Zscho.de-semantic_patch (Zscho.de Crawler":1,"zspider/0.9-dev http://feedback.redkolibri.com/":1,"ZyBorg/1.0 (ZyBorg@WISEnut.com; http://www.WISEnut.com)":1,"Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 (FM Scene 4.6.1)":1,"Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 (.NET CLR 3.5.30729) (Prevx 3.0.5)":1} |