13932 lines
538 KiB
XML
13932 lines
538 KiB
XML
<?xml version="1.0" encoding="utf-8"?>
|
|
<browsercapitems>
|
|
<comments>
|
|
<comment><![CDATA[nopCommerce uses a short version of the "browscap.xml" file. This short version contains crawlers only. If you want to keep the crawlers list up to date, please download the full version of the original file from the official browscap site (http://browscap.org/). Please save it in the \App_Data folder (The file name should be "browscap.xml"), delete "browscap.crawlersonly.xml", and restart the website.]]></comment>
|
|
<comment><![CDATA[Provided courtesy of http://browscap.org/]]></comment>
|
|
<comment><![CDATA[Created on Sunday, June 16, 2024 at 11:25 AM GMT+0000]]></comment>
|
|
<comment><![CDATA[Keep up with the latest goings-on with the project:]]></comment>
|
|
<comment><![CDATA[Follow us on Twitter <https://twitter.com/browscap>, or...]]></comment>
|
|
<comment><![CDATA[Like us on Facebook <https://facebook.com/browscap>, or...]]></comment>
|
|
<comment><![CDATA[Collaborate on GitHub <https://github.com/browscap>.]]></comment>
|
|
</comments>
|
|
<browscapitem name="StumbleUpon">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) Gecko/* Mozilla/5.0 StumbleUpon/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*stumbleupon.com) Gecko/* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*stumbleupon.com) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE ?.0; *Windows NT 5.1*; StumbleUpon.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ask">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/2.0 (compatible; Ask Jeeves/Teoma*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/2.0 (compatible; Ask Jeeves*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Mozilla/2.0 (compatible; Ask Jeeves/Teoma*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Mozilla/2.0 (compatible; Ask Jeeves*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="360Spider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE 8.0*; *Windows NT 6.2*Trident/4.0*)* 360Spider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE 8.0*; *Windows NT 6.1*Trident/4.0*)* 360Spider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE 8.0*; *Windows NT 6.0*Trident/4.0*)* 360Spider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE 8.0*; *Windows NT 5.2*Trident/4.0*)* 360Spider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE 8.0*; *Windows NT 5.1*Trident/4.0*)* 360Spider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 9.0*; *Windows NT 6.2*Trident/5.0* 360Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 9.0*; *Windows NT 6.1*Trident/5.0* 360Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 9.0*; *Windows NT 6.0*Trident/5.0* 360Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 9.0*; *Windows NT 5.2*Trident/5.0* 360Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 9.0*; *Windows NT 5.1*Trident/5.0* 360Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) *Firefox/* 360Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/*; 360Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/*; 360Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/*; 360Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/*; 360Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/*; 360Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="80Legs">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; *; http://www.80legs.com/*) Gecko*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AhrefsBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AhrefsBot/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AhrefsBot/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AhrefsBot/5.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AhrefsBot/5.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AhrefsBot/5.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AhrefsBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Adbeat">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) adbeat.com* Gecko* Firefox*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?12*) adbeat.com* Version* Safari*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) adbeat.com* Version* Safari*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) adbeat.com* Version* Safari*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?11*) adbeat.com* applewebkit* (*khtml*like*gecko*) Chrome* Safari*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) adbeat.com* applewebkit* (*khtml*like*gecko*) Chrome* Safari*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?11*) adbeat.com* Gecko* Firefox*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) adbeat.com* applewebkit* (*khtml*like*gecko*) Chrome* Safari*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) adbeat.com* applewebkit* (*khtml*like*gecko*) Chrome* Safari*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) adbeat.com* Gecko* Firefox*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) adbeat.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="adbeat_bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NikiBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="niki-bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GrapeshotCrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; GrapeshotCrawler/*; +http://www.grapeshot.co.uk/crawler.php)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; grapeFX/0.9; crawler@grapeshot.co.uk">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; grapeFX/*; crawler@grapeshot.co.uk">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Anonymizied">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Anonymisiert durch AlMiSoft Browser-Maulkorb (Anonymisier*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Anonymisiert*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Anonymizer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Anonymizied*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Anonymous*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Anonymous/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="http://Anonymouse.org/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Randomized by FreeSafeIP*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yandex">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexAddurl/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexBlogs/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexBot/*; MirrorDetector*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexCatalog/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexDirect/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexFavicons/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexImageResizer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexImages/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexMedia/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexMetrika/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexNews/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexNewslinks/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexVideo/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexVideoParser/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexWebmaster/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexZakladki/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yandex/1.01.001 (compatible; Win16; *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (*compatible*;*MSIE 5.0; YANDEX)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexBot/*; MirrorDetector)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexZakladki/*; Dyatel; +http://yandex.com/bots)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YaDirectBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yandex/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YandexSomething/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*http://yandex.com/bots*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows; ?; Windows NT 5.2; en-US; rv:1.9) Gecko VisualParser/3.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; YandexScreenshotBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexAccessibilityBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexDirectDyn/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexPagechecker/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YaDirectFetcher/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexCalendar/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexSitelinks*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexVertis/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Yandex.Translate">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Apache Bench">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ApacheBench/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YandexMobileBot 3.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* (compatible; YandexMobileBot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU*OS* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* (compatible; YandexMobileBot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YandexMobileBot Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU OS 8_1* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* (compatible; YandexMobileBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU*OS* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* (compatible; YandexMobileBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YandexBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexBot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name=": Mozilla/5.0 (compatible; YandexBot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPhone*like Mac OS X) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/* (compatible; YandexBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Goldfire Server">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Goldfire Server*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ArchitextSpider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ArchitextSpider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Become">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BecomeBot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BecomeBot/2.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BecomeBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*BecomeBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*BecomeBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MonkeyCrawl/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BecomeJPBot/2.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BecomeJPBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Convera">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ConveraMultiMediaCrawler/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ConveraMultiMediaCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CrawlConvera*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ConveraCrawler/0.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ConveraCrawler/0.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ConveraCrawler/0.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ConveraCrawler/0.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ConveraCrawler/0.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ConveraCrawler/0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ConveraCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Best of the Web">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; BOTW Feed Grabber*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; BOTW Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ContextAd Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ContextAd Bot 1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ContextAd Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java Standard Library">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java/1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java/1.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java/1.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java/1.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java/1.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java/1.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java/1.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java/1.17*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java1.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java1.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java1.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java1.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java1.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java1.17*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Java/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AntiSpamBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Interdose AntiSpamBot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Interdose AntiSpamBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bitlybot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bitlybot/3.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bitlybot/2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bitlybot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DotBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DotBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DotBot/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DotBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Entireweb">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Speedy Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) Speedy Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Speedy Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BlitzBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BlitzBOT@tricus.com (Mozilla compatible)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BlitzBOT*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Envolk">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="envolk?ITS?spider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="envolk/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Blue Coat Systems">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (*compatible*;*MSIE 6.0; Bluecoat DRTR)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BlueCoat ProxySG">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CerberianDrtrs/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Inne: Mozilla/4.0 (compatible; Cerberian Drtrs*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Cerberian Drtrs*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE 6.0; Bluecoat DRTR)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.35">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.35.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.34">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.34.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.33">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.33.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.32">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.32.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.31">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.31.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.30">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.30.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.29">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.29.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.28">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.28.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.27">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.27.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.26">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.26.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.25">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.25.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.24">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.24.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.23">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.23.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.22">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.22.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.21">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.21.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.20">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.20.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.19">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.19.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.18">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.18.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.17">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.17.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.16">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.16.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.15">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.15.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.14">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.14.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.13">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.13.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.10">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.10.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.9">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.9.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.8">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.8.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.7">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.7.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.6">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.6.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.5">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.5.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.4">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.4.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.3">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.3.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/1.0.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 0.99">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/0.99.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 0.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/0.2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs 0.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/0.1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Exalead">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Konqueror/*; Linux) KHTML/* (like Gecko) (Exabot-Thumbnails)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Exabot-Test/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Exabot/2.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Exabot/3.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Exabot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Exalead NG/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Exabot/3.0*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Exabot/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; NGBot/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NG-Search/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ng/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Exabot@exava.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Exalead CloudView">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ExaleadCloudView/5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ExaleadCloudView/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Exabot-Images">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Exabot-Images/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Exabot-Images/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Exabot-Images/3.0*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Exabot-Images/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Boitho">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="boitho.com-dc/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="boitho.com-robot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="boitho.com-*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gvfs/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Facebot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Facebot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Facebot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) applewebkit* (*khtml*like*gecko*) facebot/* Chrome/* Electron/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*) applewebkit* (*khtml*like*gecko*) facebot/* Chrome/* Electron/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko*) facebot/* Chrome/* Electron/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) applewebkit* (*khtml*like*gecko*) facebot/* Chrome/* Electron/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*) facebot/* Chrome/* Electron/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Facebook">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* facebookexternalhit/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="facebookexternalhit/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="facebookexternalhit/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="facebookexternalhit/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Facebook share follower">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="facebookplatform/1.0 *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="facebookplatform/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Facebook API PHP5 Client 1.1 (curl)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="facebookscraper/1.0( http*://www.facebook.com/sharescraper_help.php)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="facebookscraper/*( http*://www.facebook.com/sharescraper_help.php)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Xing">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="XING-contenttabreceiver/2.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="XING-contenttabreceiver/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Fast/AllTheWeb">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*FAST Enterprise Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FAST Data Search Document Retriever/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FAST MetaWeb Crawler (helpdesk at fastsearch dot com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Fast PartnerSite Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FAST-WebCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FAST-WebCrawler/*/FirstPage*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FAST-WebCrawler/*/Fresh*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FAST-WebCrawler/*/PartnerSite*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FAST-WebCrawler/*?Multimedia*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FastSearch Web Crawler for*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Fast Crawler v X(compatible; Konqueror/*; FreeBSD) (*khtml*like*gecko*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Gigabot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GigabotSiteSearch/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Gigabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Website Thumbnail Generator">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WebThumbnail/3.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WebThumbnail/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Thumbnail Generators">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) Gecko* ThumbShotsBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="thumbshots-de-bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ThumbnailAgent">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ThumbSniper *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gosquared-thumbnailer/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gosquared-thumbnailer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ruby, link_thumbnailer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Thumbor/3.14*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Thumbor/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; FTT2 Toplist Thumb Grabber*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) applewebkit* (*khtml*like*gecko*) khtml2png Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Easy-Thumb*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) AppleWebKit/* (KHTML, like Gecko) WordPress.com mShots Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux* rv:*) Gecko/20100101 Firefox/* WordPress.com mShots">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Macintosh*) AppleWebKit/* (*khtml*like*gecko*) Like-Version/* Safari/* webkit2png/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (*shrinktheweb.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux* rv:*) Gecko/* HeartRails_Capture/* (+http://capture.heartrails.com/) *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64) applewebkit* (*khtml*like*gecko*) web-capture.net/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64) applewebkit* (*khtml*like*gecko*) Chromium/* Site-Shot/* (http://www.site-shot.com/) *Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64) applewebkit* (*khtml*like*gecko*) *Site-Shot/* (http://www.site-shot.com/) *Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64) applewebkit* (*khtml*like*gecko*) www.HomepageScreenshot.de - Free ScreenShotService/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows; rv:*) Gecko/* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BCKLINKS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BCKLINKS 1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BCKLINKS*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MonitorBacklinksWP*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MBCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Daumoa">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;*) Daumoa?4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;*) Daumoa?3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AntBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AntBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AntBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ant bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="lua http client">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="lua-resty-http/0.10 *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="lua-resty-http/0.07 *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="lua-resty-http/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LCC">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LCC *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Wayback Archive Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; waybackarchive.org/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; waybackarchive.org/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FCCN Crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="arquivo-web-crawler (compatible; heritrix/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ilse">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IlseBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="INGRID/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/3.0 (INGRID/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZumBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZumBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ZumBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT 10.0; ZumBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Windows NT 6.1?; ZumBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZmEu">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZmEu*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Made by ZmEu @ WhiteHat v0.* (www.WhiteHat.ro)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Made by ZmEu @ WhiteHat*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="XML Sitemaps Generator">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="XML Sitemaps Generator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; XML Sitemaps Generator; *) Gecko XML-Sitemaps/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; XML Sitemaps Generator; *) Gecko XML-Sitemaps/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WSR-Agent">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="wsr-agent/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="wsr-agent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WeSEE:Search">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WeSEE:Search/0.1 (Alpha*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WeSEE:Search/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebCrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebCrawler_1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebCrawler_*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Web Glance">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko,*safari*) *Webglance/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko,*safari*) *Webglance/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WBSearchBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WBSearchBot/1.1*; +http://www.warebay.com/bot.html)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WBSearchBot/*; +http://www.warebay.com/bot.html)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Wappalyzer">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Wappalyzer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Wappalyzer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="visionutils">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="visionutils/0.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="visionutils/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="URLAppendBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; URLAppendBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; URLAppendBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Unity Web Player">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UnityPlayer/4.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UnityPlayer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tweetmeme Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (TweetmemeBot/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (TweetmemeBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; TweetmemeBot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; TweetmemeBot/2.11*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; TweetmemeBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TweetedTimes Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; TweetedTimes Bot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; TweetedTimes Bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; +http://tweetedtimes*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tagyu">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tagyu Agent/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tagyu Agent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SynapticSearch">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SynapticSearch/AI Crawler 1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SynapticSearch/AI Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SurveyBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SurveyBot/2.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SurveyBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) Gecko* Firefox/* (*) SurveyBot/2.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) Gecko* Firefox/* (*) SurveyBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sReleaseBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; sReleaseBot/6.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; sReleaseBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Spiderlytics">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Spiderlytics/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Spiderlytics/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SoftListBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SoftListBot/2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SoftListBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ShowXML">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ShowXML/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ShowXML/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ShopWiki">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ShopWiki/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ShopWiki/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="shelob">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Shelob (shelob@gmx.net)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="shelob v1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="shelob*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SetLinks.ru Crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SetLinks bot 1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SetLinks bot *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SetLinks bot 1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SetLinks bot *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ScreenerBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ScreenerBot Crawler Beta 2.0 *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ScreenerBot Crawler Beta *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Scrapy">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Scrapy/0.16*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Scrapy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="restify">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="restify/2.6* (*darwin*) node/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="restify/2.6* (*) node/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="restify/* (*darwin*) node/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="restify/* (*) node/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="reddit pic scraper">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla / reddit pic scraper v0.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla / reddit pic scraper*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="publiclibraryarchive Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; publiclibraryarchive.org/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; publiclibraryarchive.org/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Proxy Gear Pro">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Proxy Gear Pro/2.1 (*Windows NT 6.1*)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Proxy Gear Pro/* (*Windows NT 6.1*)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Photon">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Photon/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Photon/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Peeplo Screenshot Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Peeplo Screenshot Bot/0.20*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Peeplo Screenshot Bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Paper.li Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; PaperLiBot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; PaperLiBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OpenWebSpider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OpenWebSpider v0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OpenWebSpider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NSO_Debugger_User">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NSO_Debugger_User/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NSO_Debugger_User/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetzCheckBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetzCheckBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetzCheckBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetSeer Crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; NetSeer crawler/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; NetSeer crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="nbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; nbot/2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; nbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Moreover">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 Moreover/5.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 Moreover/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Metaspinner">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Metaspinner/0.01*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Metaspinner/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MetaGeneratorCrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MetaGeneratorCrawler/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MetaGeneratorCrawler/1.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MetaGeneratorCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="meanpathbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; meanpathbot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; meanpathbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LoadTimeBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; LoadTimeBot/0.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; LoadTimeBot/0.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; LoadTimeBot/0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; LoadTimeBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Livelap Crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LivelapBot/0.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LivelapBot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LivelapBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkpadBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; LinkpadBot/1.06*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; LinkpadBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Lightspeed Systems RocketCrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE 7.0;*Windows NT 5.1*) Lightspeedsystems*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (*Windows NT 5.1*) Lightspeedsystems*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LightspeedSystemsCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LSSRocketCrawler/1.0 LightspeedSystems*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LSSRocketCrawler/* LightspeedSystems*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="lbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (lbot/2.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (lbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="KomodiaBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows NT 5.2*) KomodiaBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows NT 5.1*) KomodiaBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows*) KomodiaBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows NT 5.2*) KomodiaBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows NT 5.1*) KomodiaBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows*) KomodiaBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) KomodiaBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) KomodiaBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IstellaBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; IstellaBot/1.18*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; IstellaBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Infohelfer">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Infohelfer/1.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Infohelfer/1.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Infohelfer/1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Infohelfer/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Infohelfer/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Infohelfer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="infohelfer/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="infohelfer/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="infohelfer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ImplisenseBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ImplisenseBot 1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ImplisenseBot *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="http_requester">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="http_requester/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="http_requester/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ht://Dig">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="htdig/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="htdig/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GroupHigh Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; GroupHigh/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; GroupHigh/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GPTBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 AppleWebKit/* (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gooblog">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gooblog/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gooblog/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GG PeekBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GG PeekBot 1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GG PeekBot 2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GG PeekBot *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GetintentCrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GetintentCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GetIntent Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="General Crawlers">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*altervista.org*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*SqwidgeBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="A .NET Web Crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BilgiBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bot/* (bot; *bot@bot.bot)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="cisco-IOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Covario-IDS/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CyberPatrol*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="cz32ts">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ddetailsbot (http://www.displaydetails.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainsDB.net MetaCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainWatcher Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Drupal (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Dumbot (version *)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EuripBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="eventax/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FANGCrawl/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="favorstarbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FollowSite.com (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Gaisbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gosospider Mozilla/5.0 (compatible; GosoSpider*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Healthbot/Health_and_Longevity_Project_(HealthHaven.com) ">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="hitcrawler_0.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="http://hilfe.acont.de/bot.html ACONTBOT">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="http://www.yellowpages.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="yp-crawl@attinteractive.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HuaweiSymantecSpider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="JetBrains*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="JS-Kit URL Resolver, http://js-kit.com/">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="KakleBot - www.kakle.com/0.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="KBeeBot/0.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Keyword Density/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Lincoln State Web Browser">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Links4US-Crawler,*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Lorkyll *.* -- lorkyll@444.net">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Lsearch/sondeur">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MapoftheInternet.com?(?http://MapoftheInternet.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="metatagsdir/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft Windows Network Diagnostics">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Miva (AlgoFeedback@miva.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="moget/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (compatible; WebCapture*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*redditbot/*http://www.reddit.com/feedback*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; DepSpid/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE 4.01; Vonna.com b o t)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE 4.01; Vonna.com b o t)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE 4.01; Windows95)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MyFamilyBot/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; N-Stealth)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Scumbot/*; Linux/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Spider; Linux)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AdHitz; http://adhitz.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; aiHitBot*/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; aiHitBot*/*; +http://www.aihit.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BuzzRankingBot/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ClixSense; http://www.clixsense.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="mozilla/5.0 (compatible; genevabot +http://www.healthdash.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; JadynAveBot; *http://www.jadynave.com/robot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Kyluka crawl; http://www.kyluka.com/crawl.html; crawl@kyluka.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; LegalAnalysisAgent/1.*; http://www.legalx.net)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 7.0 ?http://www.europarchive.org)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*; MSIE 7.0 ?http://www.europarchive.org)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 7.0; MSIE 6.0; ScanAlert; +http://www.scanalert.com/bot.jsp) Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*; MSIE 7.0; MSIE 6.0; ScanAlert; +http://www.scanalert.com/bot.jsp) Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Nigma.ru/*; crawler@nigma.ru)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Plukkie/1.?; http://www.botje.com/plukkie.htm)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SuchbaerBot/0.*; +http://bot.suchbaer.de/info.html)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Twingly Recon; http://www.twingly.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; unwrapbot/2.*; +http://www.unwrap.jp*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Vermut*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Viralheat Bot/*) ">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Webbot/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="n4p_bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="nabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetCarta_WebMapper/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Netchart Adv Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetID.com Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="neTVision AG andreas.heidoetting@thomson-webcast.net">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NextopiaBOT*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="nicebot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="niXXieBot?Foster*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Nozilla/P.N (Just for IDS woring)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Nudelsalat/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NV32ts">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ocelli/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OpenTaggerBot (http://www.opentagger.com/opentaggerbot.htm)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Oracle Enterprise Search">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Oracle Ultra Search">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pajaczek/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="panscient.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Patwebbot (http://www.herz-power.de/technik.html)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PDFBot (crawler@pdfind.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PhpDig/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PlantyNet_WebRobot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PluckItCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PMAFind">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Poodle_predictor_1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="QuickFinder Crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Radiation Retriever*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RixBot (http://babelserver.org/rix)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="roboobot/1.* (roboo; http://wap.roboo.com; winter.pi@roboo.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SBIder/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ScollSpider/2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Search Fst">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="searchbot admin@google.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Seeker.lookseek.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="semanticdiscovery/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sitecheck.internetseer.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SMBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sohu*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SpankBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="spider (tspyyp@tom.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Superpages URL Verification Engine">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Surf Knight">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SyncMgr">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Talkro Web-Shot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tasap-image-robot/0.* (http://www.tasap.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tecomi Bot (http://www.tecomi.com/bot.htm)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TencentTraveler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TheInformant*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Toata dragostea*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tutorial Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TelegramBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Twitterbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UbiCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCmore">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="USER_AGENT">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="VadixBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="VengaBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Visicom Toolbar">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Visited by http://tools.geek-tools.org">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Webclipping.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="webcollage*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebFilter Robot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WeBoX/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebTrends/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="West Wind Internet Protocols*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WhizBang">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Willow Internet Crawler by Twotrees V*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WIRE/* (Linux*Bot,Robot,Spider,Crawler)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="www.fi crawler, contact crawler@www.fi">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Xerka WebBot v1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="XSpider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YooW!/* (?http://www.yoow.eu)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="QuerySeekerSpider ( http://queryseeker.com/bot.html )">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; proximic; *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE *; AugustBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="awesomebot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cooby Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE 6.0;*Windows NT 5.1*; CrystalSemanticsBot *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux*) applewebkit* (*khtml*like*gecko*)*Chrome/13.0*Safari/* botname/cpan@dataminr.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko*)*Chrome/13.0*Safari/* botname/cpan@dataminr.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GSLFbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Macintosh; *Mac OS X *) applewebkit* (*khtml*like*gecko*) Version/*Safari/* Powered by Spider-Pig*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/*Safari/* Powered by Spider-Pig*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="netEstate NE Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="securepoint cf">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="www.integromedb.org/Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CheckSite Verification Agent*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Squzer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="InternetSeer.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ADmantX Platform Semantic Analyzer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WeSEE:Ads/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Apercite*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; PiplBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Blog Search*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YisouSpider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="hivaBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PagesInventory*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (CMS Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MetaHeadersBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Twingly Recon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="revolt">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkStats Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="stq_bot *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SNK Screenshot Bot/0.20*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SNK Screenshot Bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Phantom.js bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Optimizer)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bot.AraTurka.com/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*java*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HRCrawler/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HRCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="isc header collector handlers*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Forum Poster V3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Forum Poster*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WebMasterAid/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WebMasterAid/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Muenster University of Applied Sciences; *itsscan*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MetaURI API/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MetaURI API*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IPv4Scan*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Zookabot/2.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Zookabot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; discoverybot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; discoverybot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AskPeterBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RED/1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RED/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BUbiNG*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; compatible; semantic-visions.com crawler; HTTPClient 3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; compatible; semantic-visions.com crawler; HTTPClient*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="JoobleBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MixrankBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; memoryBot/1.20*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; memoryBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Icarus6j*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla crawl/5.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla crawl/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RamblerMail/6.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RamblerMail/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (URLfilterDB-crawler/1.1)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (URLfilterDB-crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; OpenHoseBot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; OpenHoseBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DomainSigmaCrawler/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DomainSigmaCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BigBozz/BigBozz-2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BigBozz/BigBozz*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ECCP/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ECCP/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebIndex*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ArchiveTeam ArchiveBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Crawler/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Crawler/0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YaK/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Arachnida Web Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bandscraper*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="cars-app-browser*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HggH PhantomJS Screenshoter*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IC OpenGraph Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ICC-Crawler/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ICC-Crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="InAGist URL Resolver*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jeode/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jeode/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="kraken/0.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="kraken/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PayPal IPN*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Locubot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MailChimp.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MixBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; focuseekbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla,BuSecurityProject*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; kgbody/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; kgbody/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; kulturarw3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MerchantCentricBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MerchantCentricBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE ?.0*; *windows*; MarketwireBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*; MarketwireBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Nett.io bot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Nett.io bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;acapbot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;acapbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PageFreezer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ShowyouBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ltx71 *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CookieReports.com/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CookieReports.com/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Iframely/0.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Iframely/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MetaInspector/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MetaInspector/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; CrawlRobot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Owler/0.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Owler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; pub-crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0+(compatible; RevIP.info site analyzer v4.00*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0+(compatible; RevIP.info site analyzer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SMRF URL Expander*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Superarama.com - BOT/v.0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Superarama.com - BOT/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WNMbot/0.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WNMbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Website Explorer/0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Website Explorer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="city-map screenshot service*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="optivo(R) NetHelper*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (*Genieo*/1.0*webfilter*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (*Genieo*/*webfilter*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BegunAdvertising/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BegunAdvertising/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MegaIndex.ru/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MegaIndex.ru/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; DAWINCI ANTIPLAG SPIDER*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AccServer?HOGANMIDSCHOOL?/5.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AccServer?HOGANMIDSCHOOL?/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SafeSearch microdata crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AdvBot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AdvBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 compatible ZyBorg/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 compatible ZyBorg/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Macintosh; Butterfly/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Macintosh; Butterfly/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; worldwebheritage.org/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; worldwebheritage.org/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="uipbot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="uipbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="uipcrawler/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="uipcrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="mbot v.1.16*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="mbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetLyzer FastProbe*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; TestCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*robots*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Digincore*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NoteTextView*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ShortLinkTranslate*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Grammarly/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Grammarly/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gettor/0.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gettor/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="infoX-WISG*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Melvil/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Melvil/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Melvil Favicon/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Melvil Favicon/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Quora Link Preview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Quora Link Preview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Quora-Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ROR Sitemap Generator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="XmlSitemapGenerator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Stratagems Kumo*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Virtuoso*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SPIP-3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SPIP-*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="webnumbrFetcher/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="webnumbrFetcher/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ARC Reader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Vinti/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Vinti/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EBSCO EJS Content Server*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTTP Service*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Globel*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BlueDragon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Stewdio Windmaker 2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Stewdio Windmaker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="aboutthedomain*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="A1 Sitemap Generator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Sitemap Generator/1.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Sitemap Generator/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HubPages V0.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HubPages*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Casterly*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UserAgent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="wonderbot/JS 1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="wonderbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DMCA.com Page Protection Crawling Service*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Gluten Free Crawler/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Gluten Free Crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="abot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="abot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AdnormCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; FatBot 2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; FatBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; InfuzApp/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; InfuzApp/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="wume_crawler/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="wume_crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0, VM-Crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OpenLink Virtuoso RDF crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SRCCN!Spider/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SRCCN!Spider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZoomInfo::Beehive Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WinWebBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WinWebBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; TeeRaidBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ToutiaoSpider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Domain Re-Animator Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AdlesseBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AndersPinkBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Greplerbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YesupBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; LinkisBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; houzzbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Faveeo/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bumble Bee/* (WikiApiary; *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PocketParser/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ruby, Twurly v*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SafeDNSBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Web-sniffer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IAS crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YOTPO*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GermCrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Fyrebot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="istellabot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Sysomos/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SocialRankIOBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="datagnionbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WebHistoryBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) AppleWebKit/* (*khtml*like*gecko* compatible; MailDBBot/*; +https://maildb.io/bot) Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MuckRack/*; +http://muckrack.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla Metadata Service *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* XaxisSemanticsClassifier/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Macintosh; Intel Mac OS X *) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* moatbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?4.4*Galaxy Nexus Build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* moatbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Discordbot/*https://discordapp.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; Windows NT *; RetrevoPageAnalyzer; +http://www.retrevo.com/content/about-us)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; Windows NT *; Trident/*) CrawlerProcess (http://www.PowerMapper.com)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; Windows NT *; Trident/*) CrawlerProcess (http://www.PowerMapper.com)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; Windows NT *; DomainDB-*; http://domaindb.com/crawler/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; Windows NT *; Clarsentia)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Linux; Android * Build/*) applewebkit* (*khtml*like*gecko*) Version/* Chrome/* Mobile Safari/* DareBoost">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Linux; Android * Build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Mobile Safari/* DareBoost">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* DareBoost">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64; rv:*) Gecko/20100101 Firefox/* DareBoost">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPhone; CPU iPhone OS * like Mac OS X) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/* DareBoost">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* http://spyonweb.com spyonweb@gmail.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *) applewebkit/* (*khtml*like*gecko*) YLT Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *; compatible; BDCbot/*; +http://*.bigdatacorp.com.br/faq.aspx) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; aylienbot/*; +http://www.aylien.com/bot.html)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *; rv:*) Gecko/20100101 Firefox/* Evidon (lab@evidon.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows; U; Windows NT *; en-US; rv:*) Gecko/* Firefox/* (BrowserSpyBot/*) (http://browserspy.dk/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Macintosh; Intel Mac OS X *; rv:*) Gecko/20100101 Firefox/* glindahl-cocrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Macintosh; Intel Mac OS X *) applewebkit* (*khtml*like*gecko*) Version/* Instapaper/* (+http://www.instapaper.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (* like Mac OS X) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/* mixed-content-scan">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; IDwhois/*; Linux; en-US; +http://idwhois.info/about)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Taboolabot/*http://www.taboola.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; adscanner/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jersey/* (HttpUrlConnection*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Toweya.com bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mixmax-LinkPreview 1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mixmax-LinkPreview *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MauiBot *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MailPoet Cron/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MailPoet Cron/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Coveobot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Coveobot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BLP_bbot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BLP_bbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AngleSharp/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BUFF library*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Companybook-Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinqiaMetadataDownloaderBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Mappy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Nimbostratus-Bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; tracemyfile/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NinjaBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SuperPagesUrlVerifyBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="The Knowledge AI*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Upflow/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="adstxtlab.com Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bidswitchbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="http://www.nominet.org.uk/privacypolicy*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="nanagogo-bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="e.ventures Investment Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MAZBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 Founderspider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Shamrock*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="fuelbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="theEmployeeApp*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CreateSend Screenshot Generator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainStatsBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AutoApprovalBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CyberCycloneBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DemandbaseTagMonitor*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DistributorDataSolutionsSpider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Archive.St*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="KeyweeScraper*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AdCrawlerBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DnyzBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; lw-fusion-crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; NewsUSA*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; wKioskBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;contxbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;DS Prospectus bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;Impact Radius Compliance Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; KosmioBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) Gecko* Firefox/* (IndeedBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*x86_64*) applewebkit* (*khtml*like*gecko*) screenshot_crawler Safari*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Opendi Screenshot Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Portrayal*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="pricebot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SAP/SearchCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SmarshBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Truefitbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*x86_64*) Gecko* Firefox/* OPBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PostmanRuntime/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="pyspider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Squirrly crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; iSec_v2_Bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; OpenindexSpider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; OnCrawl/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; 2GDPR/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IVW-Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Autn-WKOOP*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Alignabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AdsTxtCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Cincraw/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="lotayabot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NativeAIBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SirdataBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="beBee bot (*@bebee.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Onespot-ScraperBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Seekport Crawler; *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SeekportBot; *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BrightEdge Crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BublupBot (+https://www.bublup.com/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ISS_CRAWLER_V*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="KareBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *; Win64; x64) AppleWebKit/* (*khtml*like*gecko*) Chrome/* Safari/* MicroAdBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Rely Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WizenozeBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="weborama-fetcher (+http://www.weborama.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Lore Document Collector *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; tracking-quality-spider/*; https://www.awin.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; RyowlEngine/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ghost Inspector BS-20*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PWACrawler +@SamsungInternet">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Narrativ-Crawler (+https://narrativ.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NinjBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yewno-Open*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="deepcrawl*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Magnetbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Caliperbot/* (*)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ISP_TESTING_SUITE*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Asana/* WebsiteMetadataRetriever*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bitdiscovery*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Doximity-Pipeline*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MVision/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Charlotte: The Financial Institution Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CEPCrawler (*)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; startmebot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Googlebot/*; startmebot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Linespider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 AppleWebKit/* (*khtml*like*gecko*compatible; Linespider/*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT 6.1; Win64; x64) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Linespider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ThinkBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Genderanalyzer">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Genderanalyzer/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Genderanalyzer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GarlikCrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GarlikCrawler/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GarlikCrawler/1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GarlikCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="fr-crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; fr-crawler/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; fr-crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X*; *Linux*x86_64*; rv:*) Gecko/* Firefox/* fr_crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Experibot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Experibot_v1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Experibot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DueDil Crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; electricmonk/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; electricmonk/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Dorkbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UT-Dorkbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainScan Server Monitoring">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainSCAN*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DomainSCAN/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DomainSCAN/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainAppender Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DomainAppender /1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DomainAppender /*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Digg Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Digg/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Digg/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Digg Deeper/v1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Digg Deeper/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Crazywebcrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CRAZYWEBCRAWLER 0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CRAZYWEBCRAWLER*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Comodo SSL Checker">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Comodo SSL Checker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Comodo Spider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Comodo Spider 1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Comodo Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cliqzbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Cliqzbot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Cliqzbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cliqzbot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cliqzbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CCBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CCBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CCBot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CCBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ca-crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ca-crawler/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ca-crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BuiBui-Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BuiBui-Bot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BuiBui-Bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BLEXBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BLEXBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BLEXBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BlekkoBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Blekkobot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Barkrowler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Barkrowler/0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Barkrowler/0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Barkrowler/0.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Barkrowler/0.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Barkrowler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Barkrowler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BabalooSpider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BabalooSpider/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BabalooSpider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AOLbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AOLbot/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AOLbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Another Web Mining Tool">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Another Web Mining Tool 1.0; +none; awmt)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Another Web Mining Tool *; +none; awmt)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Amazon CloudFront">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Amazon CloudFront*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Amazonbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* (Amazonbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* (Amazonbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Add Catalog">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Add Catalog/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Add Catalog/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AboutUs Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AboutUsBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AboutUsBot Johnny5/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AboutUsBot Johnny5/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AboutUsBot/* (Website Analysis*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Aboundexbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Aboundex/0.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Aboundex/0.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Aboundex/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Abonti WebSearch">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Abonti/0.91*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Abonti/0.92*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Abonti/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="A6-Indexer">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="A6-Indexer/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="A6-Indexer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="200PleaseBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; 200PleaseBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; 200PleaseBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TurnitinBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TurnitinBot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TurnitinBot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TurnitinBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RedCarpet">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RedCarpet/* CFNetwork/672.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RedCarpet/* CFNetwork/672.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RedCarpet/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TinyURL">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Rome Client (http://tinyurl.com/64t5n) Ver: 0.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Internet Archive Special Archiver">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; special_archiver/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; special_archiver/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; special_archiver/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; special_archiver/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*x86_64*special_archiver*) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SMTBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*) (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS 8?1* like Mac OS X*) applewebkit* (*khtml*like*gecko*)*Version/*Safari/* (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS 8?* like Mac OS X*) applewebkit* (*khtml*like*gecko*)*Version/*Safari/* (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS 7?1* like Mac OS X*) applewebkit* (*khtml*like*gecko*)*Version/*Safari/* (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS 7?* like Mac OS X*) applewebkit* (*khtml*like*gecko*)*Version/*Safari/* (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS * like Mac OS X*) applewebkit* (*khtml*like*gecko*)*Version/*Safari/* (compatible; SMTBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="profiller">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CrOS*) applewebkit* (*khtml*like*gecko*) profiller Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CentOS*) applewebkit* (*khtml*like*gecko*) profiller Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko*) profiller Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko*) profiller Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) applewebkit* (*khtml*like*gecko*) profiller Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*) profiller Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) applewebkit* (*khtml*like*gecko*) profiller Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) applewebkit* (*khtml*like*gecko*) profiller Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) applewebkit* (*khtml*like*gecko*) profiller Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) profiller Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) profiller Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) profiller Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Diffbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Diffbot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Diffbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Diffbot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Diffbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CrOS*) applewebkit* (*khtml*like*gecko; +http://www.diffbot.com) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko; +http://www.diffbot.com) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3* rv:*) Gecko/* Firefox/* (*Diffbot/*; +http://www.diffbot.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2* rv:*) Gecko/* Firefox/* (*Diffbot/*; +http://www.diffbot.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1* rv:*) Gecko/* Firefox/* (*Diffbot/*; +http://www.diffbot.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0* rv:*) Gecko/* Firefox/* (*Diffbot/*; +http://www.diffbot.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2* rv:*) Gecko/* Firefox/* (*Diffbot/*; +http://www.diffbot.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1* rv:*) Gecko/* Firefox/* (*Diffbot/*; +http://www.diffbot.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows* rv:*) Gecko/* Firefox/* (*Diffbot/*; +http://www.diffbot.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko; +http://www.diffbot.com) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) applewebkit* (*khtml*like*gecko; +http://www.diffbot.com) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko; +http://www.diffbot.com) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) applewebkit* (*khtml*like*gecko; +http://www.diffbot.com) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) applewebkit* (*khtml*like*gecko; +http://www.diffbot.com) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) applewebkit* (*khtml*like*gecko; +http://www.diffbot.com) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko; +http://www.diffbot.com) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko; +http://www.diffbot.com) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko; +http://www.diffbot.com) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*diffbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Criteo Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Pix/* (Criteo; x64-criteo)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Pix/* (Criteo; x64-criteo)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="iVia Project">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DataFountains/DMOZ Downloader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DataFountains/DMOZ Feature Vector Corpus Creator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Crawly">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Crawly/1.*; +http://*/crawler.html)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jayde Online">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ExactSeek Crawler/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ExactSeek Crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ExactSeek Crawler (nutch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="exactseek-pagereaper-* (crawler@exactseek.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="exactseek.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jayde Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cynthia">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cynthia 1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cynthia *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Data Mining">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; imrbot/1.10*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; imrbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; LucidWorks*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 CommonCrawler Node*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Dataprovider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SkyGrid/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Clickagy Intelligence Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Datanyze; *Linux*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Eyeotabot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Lycos">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Lycos*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Lycos-Proxy">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Lycos-Spider_(modspider)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Lycos-Spider_(T-Rex)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DCPbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DCPbot/1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DCPbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Zollard Worm">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Zollard; Linux)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snap">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (SnapPreviewBot) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sogou">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="shaboyi spider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sogou develop spider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sogou head spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sogou js robot(*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sogou Orion spider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sogou Pic Agent">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sogou Pic Spider/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sogou Pic Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sogou-Test-Spider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sogou Push Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sogou spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sogou web spider/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sogou web spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="New-Sogou-Spider/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="New-Sogou-Spider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainCrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainCrawler/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainCrawler/2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CipaCrawler/3.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CipaCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YodaoBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YodaoBot/1.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;YodaoBot-Image/1.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WAP_Browser/5.0 (compatible; YodaoBot/1.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YodaoBot/1.* (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Best Whois (http://www.bestwhois.net/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DNSGroup/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TouchStone">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainsBotBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainsBotBot/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainsBotBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebmasterCoffee">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WebmasterCoffee/0.7; +http://webmastercoffee.com/about)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WebmasterCoffee/*; +http://webmastercoffee.com/about)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DomainsBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="domainsbot *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="spbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/4.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/4.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/4.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/4.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/5.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*spbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sp auditbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*sp_auditbot/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*sp_auditbot/4.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*sp_auditbot/4.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*sp_auditbot/4.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*sp_auditbot/4.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*sp_auditbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sistrix Crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SISTRIX Crawler*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; 007ac9 Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sistrix*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="XoviBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; XoviBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; XoviBot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; XoviBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="XoviBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="XoviBot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="XoviBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="webceo Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; online-webceo-bot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; online-webceo-bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SeoSiteCheckup">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPhone*CPU iPhone OS 9?1* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/9.0*Safari/*SeoSiteCheckup*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*x86_64*) applewebkit* (*khtml*like*gecko*) HeadlessChrome/* Safari/* SeoSiteCheckup*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SEOdiver Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SEOdiver/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SEOdiver/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider 2.50">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2.50">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2,50">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider 2.40">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2.40">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2,40">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider 2.30">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2.30">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2,30">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider 2.22">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2.22">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2,22">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider 2.21">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2.21">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2,21">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider 2.20">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2.20">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2,20">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider 2.11">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2.11">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2,11">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider 2.01">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2.01">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2,01">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider 2.00">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2.00">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/2,00">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider 1.90">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/1.90">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/1,90">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SBooksNet">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPhone*CPU iPhone OS 8?3* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/8.0*Safari/*SBooksNet*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; SBooksNet*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="netvigie Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) Gecko* Firefox/* NETVIGIE*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BrandVerity Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPhone*CPU iPhone OS 7?* like Mac OS X*) applewebkit* (*khtml*like*gecko*) Mobile/* Twitter for iPhone*BrandVerity*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) Gecko* Firefox/* BrandVerity*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SEO & Analytics">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SEODat/0.* http://crawler.seodat.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SearchmetricsBot; http://www.searchmetrics.com/en/searchmetrics-bot/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SEOkicks-Robot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebTarantula.com Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SiteExplorer/1.0b;*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SiteExplorer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Prlog/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Prlog/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Browsershots">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ssearch_bot (sSearch Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FreeWebMonitoring SiteChecker/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FreeWebMonitoring SiteChecker/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GIDBot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GIDBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (vBSEO;*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="vBulletin via PHP*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemanticBot/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemanticBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bot-pge.chlooe.com/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bot-pge.chlooe.com/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="wscheck.com/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="wscheck.com/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AjaxSnapBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; HyperCrawl/0.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; HyperCrawl/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SEOstats 2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SEOstats*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Athena Site Analyzer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AuditMyPC Webmaster Tool*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SPDYCheck SPDY Protocol Tester*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WeSEE">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WeSEE_Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Page Analyzer v4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Page Analyzer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; http://alyze.info*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT 6.3;compatible; Leikibot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; seoscanners.net/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SeobilityBot (SEO-Check; *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="cuwhois/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ganarvisitas/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetTrack Anonymous Web Statistics*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TA SEO Crawler v* URLData.pm (smcquaker@tripadvisor.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AlphaBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OnPageBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RyteBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RankingBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Seobility*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SEOmonitorBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Cocolyzebot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Dubbotbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="pr-cy.ru Screenshot Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; PR-CY.RU*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) cXensebot* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Sitebulb*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*x86_64*) applewebkit* (*khtml*like*gecko*) HeadlessChrome/* Safari/* Prerender*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*x86_64*HubSpot Single Page link check*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*x86_64*HubSpot Single Page link check*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Trackuity *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="serpstatbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BacklinkCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DataForSeoBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Adsbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AwarioRssBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AwarioSmartBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SemrushBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemrushBot-SA/0.96*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemrushBot-SA/0.97*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemrushBot-SA/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemrushBot-SI/0.96*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemrushBot-SI/0.97*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemrushBot-SI/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemrushBot/0.96*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemrushBot/0.97*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemrushBot/2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemrushBot/1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemrushBot/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SemrushBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*SEMrushBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Scrubby">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Scrubby/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Scrubby/2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Scrubby/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Scrubby/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Scrubby/2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Scrubby/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Scrubby/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider General">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Screaming Frog SEO Spider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Lipperhey SEO Service">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Lipperhey SEO Service*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Lipperhey Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Lipperhey*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Alexabot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Alexabot/1.0; *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Alexabot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Alexa Site Audit">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; alexa site audit/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; alexa site audit/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TagInspector Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CrOS*) applewebkit* (*khtml*like*gecko*) TagInspector/* Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko*) TagInspector/* Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko*) TagInspector/* Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) applewebkit* (*khtml*like*gecko*) TagInspector/* Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*) TagInspector/* Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) applewebkit* (*khtml*like*gecko*) TagInspector/* Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) applewebkit* (*khtml*like*gecko*) TagInspector/* Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) applewebkit* (*khtml*like*gecko*) TagInspector/* Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) TagInspector/* Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) TagInspector/* Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) TagInspector/* Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*FreeBSD*) applewebkit* (*khtml*like*gecko*) TagInspector/* Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="JamesBOT">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) Gecko* Firefox/* - James BOT - WebCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) Gecko* Firefox/* - James BOT - WebCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) Gecko* Firefox/* - James BOT - WebCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (*Windows NT 5.1*) Gecko* Firefox/* - James BOT - WebCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Vagabondo">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/?.0 (compatible; *Vagabondo/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/?.0 (compatible; *Vagabondo/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/?.0 (compatible; *Vagabondo/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/?.0 (Vagabondo/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/?.0 (Vagabondo/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sosospider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sosospider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Sosospider/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Sosospider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Search Engines">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*FDSE robot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/3.0 (compatible; Fluffy the spider; http://www.searchhippo.com/; info@searchhippo.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Fluffy the spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Abacho*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ah-ha.com crawler (crawler@ah-ha.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AIBOT/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ALeadSoftbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Amfibibot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AnswerBus (http://www.answerbus.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="antibot-V*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="appie*(www.walhello.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ASPSeek/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Atrax Solutions atraxbot/0.*; http://www.atraxsolutions.com/atraxbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BigCliqueBOT/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Blaiz-Bee/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="btbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Busiversebot/v1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Busiversebot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CatchBot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CatchBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CatchBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CipinetBot (http://www.cipinet.com/bot.html)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cogentbot/1.?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cogentbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="compatible; Mozilla 4.0; MSIE 5.5; (SqwidgeBot v1.01*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="compatible; Mozilla 4.0; MSIE 5.5; (SqwidgeBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="cosmos/0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="cosmos*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Deepindex">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DiamondBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Dumbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Eule?Robot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Faxobot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Filangy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="flatlandbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Fooky.com/ScorpionBot/ScoutOut;*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FyberSpider/1.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FyberSpider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Gaisbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gazz/*(gazz@nttr.co.jp)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="geniebot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GOFORITBOT (?http://www.goforit.com/about/?)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GoGuidesBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GroschoBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HenryTheMiragoRobot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HolmesBot (http://holmes.ge)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hotzonu/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HyperEstraier/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="i1searchbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IIITBOT/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Iltrovatore-?etaccio/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="InfociousBot (?http://corp.infocious.com/tech_crawler.php)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Infoseek SideWinder/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="iSEEKbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Knight/0.? (Zook Knight; http://knight.zook.in/; knight@zook.in)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Kolinka Forum Search (www.kolinka.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="KRetrieve/">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LapozzBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Linguee Bot (http://www.linguee.com/bot; bot@linguee.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Linknzbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LocalcomBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MaSagool/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="miniRank/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mnogosearch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/0.9* no dos :) (Linux*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Arachmo)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; http://search.thunderstone.com/texis/websearch/about.html)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; T-H-U-N-D-E-R-S-T-O-N-E)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0(?compatible; MSIE 6.0; Qihoo *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.7 (compatible; WhizBang; http://www.whizbang.com/crawler)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ActiveTouristBot*; http://www.activetourist.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ayna-crawler*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Butterfly/1.0; *)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Charlotte/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 *Charlotte/*searchme.com/support/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; CXL-FatAssANT*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DBLBot/1.0; ?http://www.dontbuylists.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; EARTHCOM.info/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NL-Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; NLCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; OsO;*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ParchBot/1.0;*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pogodak.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Quantcastbot/1.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YoudaoBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YoudaoBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Twiceler*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 CostaCider Search*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NavissoBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NextGenSearchBot*(for information visit *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Norbert the Spider(Burf.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NuSearch Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ObjectsSearch/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OOZBOT/0.20 ( http://www.setooz.com/oozbot.html ; agentname at setooz dot_com )">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OpenISearch/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pagebull http://www.pagebull.com/">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PEERbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pompos/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Popdexter/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Qweery*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RedCell/* (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SaladSpoon/ShopSalad 1.* (Search Engine crawler for ShopSalad.com; *; crawler@shopsalad.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Search-10/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="search.ch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Searchmee! Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Seekbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SiteSpider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Spinne/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sproose/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sqeobot/0.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SquigglebotBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="StackRambler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SygolBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SynoBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Szukacz/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tarantula/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TerrawizBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tkensaku/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TMCrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TwengaBot-Discover (http://www.twenga.fr/bot-discover.html)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Twingly Recon">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="updated/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="URL Spider Pro/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="URL Spider SQL*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="VMBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="voyager/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="voyager/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="voyager/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="wadaino.jp-crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebAlta Crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebCorp/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="webcrawl.net">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WISEbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Wotbox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="www.zatka.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WWWeasel Robot v*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YadowsCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YodaoBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZeBot_www.ze.bz*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="zibber-v*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZipppBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0(?compatible*; MSIE 6.0; Qihoo *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NerdyBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; coccoc/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; coccoc/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; coccocbot-web/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; coccocbot-web/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Wada.vn*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Wada.vn*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AcoonBot/4.11*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AcoonBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Acoon* (*www.acoon.de)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.1 (compatible; MonoBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.1 (compatible; MonoBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.1 (compatible; GLBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.1 (compatible; GLBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.1 (compatible; PWBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.1 (compatible; PWBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.1 (compatible; +5Bot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.1 (compatible; +5Bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WASALive-Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GigablastOpenSource/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GigablastOpenSource/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YioopBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="iZSearch.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DeuSu/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DeuSu/5.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DeuSu/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WbSrch/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WbSrch/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Exploratodo/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Exploratodo/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WebCorp/5.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WebCorp/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; vebidoobot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Findxbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Flamingo_SearchEngine (+http://www.flamingosearch.com/bot)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Seeker v.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="omgili/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; IDBot/*; +http://www.id-search.xyz/bot.html)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT *; rv:* trendictionbot*; *) Gecko/* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; LetsearchBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yooo! Search Engine Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="zoomRank/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; localsearch-web/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; localsearch-web/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AddSearchBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; AddSearchBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*when.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Shopit*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Neevabot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ScoutJet">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ScoutJet; +http://www.scoutjet.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MojeekBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MojeekBot/0.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MojeekBot/0.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MojeekBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GurujiBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GurujiBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GurujiBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 GurujiBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 GurujiBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; GurujiBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; GurujiBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Girafabot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE 7.0;*Windows NT 6.0*Trident/4.0*Mozilla/4.0 (compatible*; MSIE 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE 7.0;*Windows NT 5.2*Trident/4.0*Mozilla/4.0 (compatible*; MSIE 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE 7.0;*Windows NT 5.1*Trident/4.0*Mozilla/4.0 (compatible*; MSIE 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 7.0;*Windows NT 6.0*Trident/4.0*Mozilla/4.0 (compatible; MSIE 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 7.0;*Windows NT 5.2*Trident/4.0*Mozilla/4.0 (compatible; MSIE 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 7.0;*Windows NT 5.1*Trident/4.0*Mozilla/4.0 (compatible; MSIE 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0*; *Windows NT 6.0*Mozilla/4.0 (compatible*; MSIE 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0*; *Windows NT 5.2*Mozilla/4.0 (compatible*; MSIE 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0*; *Windows NT 5.1*Mozilla/4.0 (compatible*; MSIE 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE ?.0*; *Windows NT 6.0*Mozilla/4.0 (compatible; MSIE 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE ?.0*; *Windows NT 5.2*Mozilla/4.0 (compatible; MSIE 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE ?.0*; *Windows NT 5.1*Mozilla/4.0 (compatible; MSIE 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE ?.0*; *Windows NT 6.0*Trident/5.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE ?.0*; *Windows NT 5.2*Trident/5.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE ?.0*; *Windows NT 5.1*Trident/5.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE ?.0*; *Windows NT 6.0*Trident/5.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE ?.0*; *Windows NT 5.2*Trident/5.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE ?.0*; *Windows NT 5.1*Trident/5.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE ?.0*; *Windows NT 6.0*Trident/4.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE ?.0*; *Windows NT 5.2*Trident/4.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE ?.0*; *Windows NT 5.1*Trident/4.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE ?.0*; *Windows NT 6.0*Trident/4.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE ?.0*; *Windows NT 5.2*Trident/4.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE ?.0*; *Windows NT 5.1*Trident/4.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE ?.0; *Windows NT 6.0*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE ?.0; *Windows NT 5.2*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE ?.0; *Windows NT 5.1*Girafabot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EasouSpider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; EasouSpider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS * like Mac OS X*; EasouSpider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="dbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.1 (compatible; DBot/7.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.1 (compatible; DBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; dbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Daum Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) AppleWebKit* (*khtml*like*gecko*) Safari/* Daum/4.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) AppleWebKit* (*khtml*like*gecko*) Safari/* Daum/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Daum/4.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Daum/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DuckDuckBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DuckDuckBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DuckDuckBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*duckduckgo*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BitTorrent Client">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Azureus*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BitComet/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BitTornado/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BitTorrent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BitTorrentMac/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BTSP/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Deluge*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FDM*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="KTorrent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="libtorrent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MediaGet*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="rtorrent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Transmission/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="uTorrent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="uTorrentMac/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Larbin">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="larbin_2.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="larbin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="binlar_2.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="binlar*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Larbin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feed::Find/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena Antenna/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena Bookmark/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena::Bookmark/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HatenaBookmark/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HatenaBookmark/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena RSS/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena::Crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena::Russia::Crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HatenaScreenshot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="URI::Fetch/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DoCoMo/2.0 D903i(c100;TB;W28H20) (compatible; Hatena-Mobile-Gateway/1.2; +http://mgw.hatena.ne.jp/help)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena::Bookmark/2.00*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena Pagetitle Agent/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena Pagetitle Agent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena Star UserAgent/2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena Star UserAgent*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena-Favicon2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena-Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena::Fetcher*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena::Scissors*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Hatena::UserAgent*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Internet Archive">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ia_archiver*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="InternetArchive/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/2.0 (compatible; archive.org*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; archive.org*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*x86_64*archive.org*) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*x86_64*archives.gov*)*applewebkit* (*khtml*like*gecko*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Nutch">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Nutch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CazoodleBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LOOQ/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Nutch/0.? (OpenX Spider)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="linguatools-bot/Nutch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MaxPointCrawler/Nutch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MR Crawler/Nutch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="My Nutch Spider/Nutch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="fetch/Nutch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Nutch?1.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Nutch?1.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Nutch?1.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mobile*) Gecko* Firefox/* commoncrawl.org/research//Nutch-1.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Android*Mobile*) Gecko* Firefox/* commoncrawl.org/research//Nutch-1.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; Windows NT *; Trident/*) commoncrawl.org/research//Nutch-1.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Nutch/* (OpenX Spider)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="The Crawler/Nutch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Domnutch-Bot/Nutch-1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DiscoverEd/Nutch-1.7 (OER search crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Elefent 1.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="elefent/Elefent 1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="elefent/1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Nutch test crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Nutch test crawler/Nutch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Webaroo">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE 7.0;*Windows NT 6.0*; Webaroo/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows; U; Windows *; *; rv:*) Gecko* Firefox/* webaroo/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 4.8">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/4.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 4.8* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 4.7">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/4.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 4.7* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 4.4">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/4.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 4.4* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 4.3">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/4.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 4.3* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 4.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/4.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 4.2* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 4.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 4.0* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.9">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/3.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.9* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.8">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/3.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.8* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.7">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/3.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.7* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.6">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/3.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.6* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.5">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/3.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.5* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.4">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/3.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.4* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.3">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.3* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.2* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.1* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress 3.0* Feed Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress-B-/2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress-Do-P-/2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Incutio XML-RPC -- WordPress/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="The Incutio XML-RPC PHP Library -- WordPress*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WordPress*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Automattic Analytics Crawler/*; http://wordpress.com/crawler/">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Linux; Android * Build/*) applewebkit* (*khtml*like*gecko* Google-Publisher-Plugin) Chrome/* Mobile Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Macintosh; Intel Mac OS X *) applewebkit* (*khtml*like*gecko* Google-Publisher-Plugin) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Esribot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Esribot/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Esribot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Esribot/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Esribot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="oBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="oBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; oBot/2.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; oBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Copyright/Plagiarism">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BDFetch">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="copyright sheriff (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CopyRightCheck*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FairAd Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="iCopyright Conductor*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IPiumBot laurion(dot)com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IWAgent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DKIMRepBot/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SlySearch/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TutorGigBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Blackboard Safeassign">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Blackboard Safeassign*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EuripBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EuripBot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; EuripBot/2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; EuripBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DNS Tools">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Domain Dossier utility*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DNS-Digger/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OpenDNS Domain Crawler noc@opendns.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Download Managers">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="A1 Website Download/1.* (*) miggibot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AutoMate5">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Beamer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BitBeamer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DA *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DAP *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Download Demon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Download Express*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Download Master*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Download Ninja*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Download Wonder*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DownloadSession*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EasyDL/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FDM 1.x">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FlashGet">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FreshDownload/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GetRight/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GetRightPro/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GetSmart/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Go!Zilla*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Gozilla/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Internet Ninja*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Kontiki Client*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LightningDownload/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LMQueueBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MetaProducts Download Express/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Getleft*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Myzilla">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Net Vampire/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Net_Vampire*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetAnts*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetPumper*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetSucker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetZip Downloader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NexTools WebAgent*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Offline Downloader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="P3P Client">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PageDown*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PicaLoader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Prozilla*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RealDownload/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sEasyDL/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="shareaza*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SmartDownload/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SpeedDownload/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Star*Downloader/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="STEROID Download">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SuperBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Vegas95/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebZIP*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WinTools">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Xaldon WebSpider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) applewebkit* (*khtml*like*gecko*) Downloader/* MyComGameCenter/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; Windows NT *; Trident/*; QQDownload *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; Windows NT *; QQDownload *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Scopia Crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Scopia Crawler 1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Scopia Crawler 1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Scopia Crawler 1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Scopia Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="fastbot crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="fastbot crawler beta 4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="fastbot crawler beta 2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="fastbot crawler beta*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ezooms">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Ezooms/1.0; *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Ezooms/1.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Ezooms/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="E-Mail Harvesters">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*E-Mail Address Extractor*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*www4mail/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="8484 Boston Project*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Atomic_Email">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Atomic_Email_Hunter/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CherryPicker*/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Chilkat/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ContactBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="eCatch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EmailCollector*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EMAILsearcher">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EmailSiphon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EmailWolf*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Epsilon SoftWorks? MailMunky">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ExtractorPro*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Franklin Locator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Missigua Locator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Advanced Email Extractor*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Netprospector*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ProWebWalker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sna-0.0.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebEnhancer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebMiner*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZIBB Crawler (email address / WWW address)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feeds Blogs">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bloglines Title Fetch/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bloglines/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bloglines/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bloglines/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BlogPulse (ISSpider-3.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BlogPulseLive (support@blogpulse.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="blogsearchbot-pumpkin-2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Irish Blogs Aggregator/*1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="kinjabot (http://www.kinja.com; *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Net::Trackback/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Reblog*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bloglovin/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bloglovin/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="wp.com feedbot/* (+https://wp.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Spinn3r RSS Aggregator">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*; aggregator:Tailrank (Spinn3r 2.3); *) Gecko*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*; aggregator:Spinn3r (Spinn3r 3.0); *) Gecko*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*; aggregator:Spinn3r (Spinn3r 3.1); *) Gecko*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*; aggregator:Spinn3r (Spinn3r *); *) Gecko*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Spinn3r*http://spinn3r.com/robot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feeds Syndicators">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*LinkLint*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*NetVisualize*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AideRSS/1.0 (aiderss.com)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AideRSS 2.* (postrank.com)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AideRSS/2.0 (aiderss.com)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AppleSyndication/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cocoal.icio.us/* (*)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feed43 Proxy/* (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedBurner/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedDemon/* (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedDigest/* (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedGhost/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedOnFeeds/0.1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feedreader * (Powered by Newsbrain)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feedshow/* (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feedster Crawler/?.0; Feedster, Inc.">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GreatNews/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GreatNews/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Gregarius/0.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Gregarius/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="intraVnews/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="JetBrains Omea Reader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Liferea/0.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Liferea/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Liferea/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="livedoor FeedFetcher/0.01*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="livedoor FeedFetcher/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MagpieRSS/0.72*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MagpieRSS/0.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MagpieRSS/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mobitype * (compatible; Mozilla/*; MSIE *.*; Windows *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*; Rojo *; http://www.rojo.com/corporate/help/agg; *)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 6.0; Podtech Network; crawler_admin@podtech.net)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Newz Crawler *; http://www.newzcrawler.com/?)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; RSSMicro.com RSS/Atom Feed Robot)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;*newstin.com;*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (RSS Reader Panel)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*; aggregator:FeedParser; *) Gecko*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*; aggregator:NewsMonster; *) Gecko*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*; aggregator:Rojo; *) Gecko*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*aggregator:TailRank; http://tailrank.com/robot)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 NewsFox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Netvibes (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewsAlloy/* (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Omnipelagos*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Particls">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Protopage/* (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PubSub-RSS-Reader/* (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RSS Menu/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RssBandit/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RssBar/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SharpReader/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Strategic Board Bot (?http://www.strategicboard.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TargetYourNews.com bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Technoratibot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tumblr/* RSS syndication ( http://www.tumblr.com/) (support@tumblr.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Wizz RSS News Reader">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (*Genieo*/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (*Genieo*/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedBlitz/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedBlitz/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedForAll rss2html.php v2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedForAll rss2html.php*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedlyApp/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedlyApp/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feedly/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feedly/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedlyBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedlyBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; theoldreader.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewsBlur Favicon Fetcher*Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) Version/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewsBlur Favicon Fetcher*Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) Version/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewsBlur Feed Fetcher*Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) Version/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewsBlur Feed Fetcher*Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) Version/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewsBlur Page Fetcher*Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) Version/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewsBlur Page Fetcher*Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) Version/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Superfeedr bot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Superfeedr bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Blogshares Spiders (Renewed V1.65*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Blogshares Spiders*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="R6_FeedFetcher*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="newsme/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="newsme/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sfFeedReader/0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sfFeedReader/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feed Wrangler/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feed Wrangler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feedfetcher-feedraider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Custom-Feed Reader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CITA RSS Aggregator/2.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CITA RSS Aggregator/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedBucket/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="everyfeed-spider/* (http://www.everyfeed.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="feedjira http://feedjira.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TrapitAgent/0.1 (feed processor; +http://trapit.com/about)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Nuzzel">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="newsbeuter/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Specificfeeds- http://www.specificfeeds.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; U; Linux*; en-US; rv:*) Gecko/* Firefox/* Specificfeeds- http://www.specificfeeds.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SerendeputyBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SimplePie">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SimplePie/1.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SimplePie/1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SimplePie/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="General RSS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BlijbolReallySimpleAggregator/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CC Metadata Scaper http://wiki.creativecommons.org/Metadata_Scraper">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible) GM RSS Panel">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 http://www.inclue.com; graeme@inclue.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Runnk online rss reader : http://www.runnk.com/ : RSS favorites : RSS ranking : RSS aggregator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UniversalFeedParser/5.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UniversalFeedParser/4.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UniversalFeedParser/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UniversalFeedParser/5.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UniversalFeedParser/5.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RSSOwl/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HiddenMarket-*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Digg Feed Fetcher 1.0 (Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Digg Feed Fetcher 1.0 (Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Digg Feed Fetcher 1.0 (Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Fever">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Fever/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedBin">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedBin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTML Validators">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="(HTML Validator http://www.searchengineworld.com/validator/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedValidator/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Search Engine World Robots.txt Validator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Weblide/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebmasterWorld StickyMail Server Header Checker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WWWC/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WDG_Validator/1.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WDG_Validator/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Qualidator.com Bot 1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Qualidator.com Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CSE HTML Validator Online*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Let's Encrypt validation server*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DomainSONOCrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TinEye Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* TinEye/* (via http://www.tineye.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TinEye-bot/0.02*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TinEye-bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TinEye/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TinEye*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Image Crawlers">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*PhotoStickies/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Camcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CydralSpider/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CydralSpider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Der gro?xdfe BilderSauger*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Extreme Picture Finder">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FLATARTS_FAVICO">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTML2JPG Blackbox, http://www.html2jpg.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IconSurf/2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mister PIX*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; KaloogaBot; http://www.kalooga.com/info.html?page=crawler)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="naoFavicon4IE*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="pixfinder/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="rssImagesBot/0.1 (*http://herbert.groot.jebbink.nl/?app=rssImages)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Web Image Collector*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebImages * (?http://herbert.groot.jebbink.nl/?app=WebImages?)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebPix*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FOTOCHECKER">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Elmer, the Thinglink ImageBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZnajdzFoto/Image 2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZnajdzFoto/Image*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Caroline/*; +http://justpicsplease.com/caroline.php)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Twiends Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="myTaste image cacher*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Taringa UGC www.taringa.net/shared-link.php*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Picsearch Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="psbot-image*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="psbot-page*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="psbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GWPImages">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; GWPImages/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; GWPImages/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Evernote Clip Resolver">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Evernote Clip Resolver*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Xenus Link Sleuth">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Xenu* Link Sleuth?1.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Xenu* Link Sleuth?1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Xenu* Link Sleuth*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Link Checkers">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="!Susie (http://www.sync2it.com/susie)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*AgentName/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Linkman*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*LinksManager.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Web Link Validator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Zeus*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ActiveBookmark *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bookdog/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bookmark Buddy*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bookmark Renewal Check Agent*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bookmark search tool*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bookmark-Manager">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Checkbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CheckLinks/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CyberSpyder Link Test/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DLC/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DocWeb Link Crawler (http://doc.php.net)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FavOrg">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Favorites Sweeper v.3.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Funnel Web Profiler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Html Link Validator (www.lithopssoft.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IECheck">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="integrity/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="JCheckLinks/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="JRTwine Software Check Favorites Utility">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Link Valet Online*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkAlarm/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Linkbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkextractorPro*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkLint-checkonly/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkScan/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkSweeper/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkWalker/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkWalker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MetaGer-LinkChecker">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; metager2-verification-bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (compatible; linktiger/*; *http://www.linktiger.com*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* LinkTiger *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (Compatible); URLBase*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Link Utility; http://net-promoter.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; smartBot/1.*; checking links; *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; SuperCleaner*;*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 gURLChecker/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Newsgroupreporter LinkCheck">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="onCHECK Linkchecker von www.scientec.de fuer www.onsinn.de">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="online link validator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="REL Link Checker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RLinkCheker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Robozilla/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SafariBookmarkChecker*(?http://www.coriolis.ch/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Simpy/* (Simpy; http://www.simpy.com/?ref=bot; feedback at simpy dot com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SiteBar/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Susie (http://www.sync2it.com/bms/susie.php">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="URLBase/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="VSE/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebTrends Link Analyzer">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WorQmada/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Z-Add Link Checker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 TYPO3-linkvalidator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; pmoz.info ODP link checker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="eZ Publish Link Validator">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinksCrawler 0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinksCrawler *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Sophora Linkchecker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MOSBookmarks/v2.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MOSBookmarks/v2.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MOSBookmarks/v2.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MOSBookmarks/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="URLChecker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Short URL Checker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Raven Link Checker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) BrokenLinkCheck.com/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) BrokenLinkCheck.com/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE * *Windows NT 5.1*) BrokenLinkCheck.com/1.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE * *Windows NT 5.1*) BrokenLinkCheck.com/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LongURL API*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (GetLinkInfo*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Slinky v1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Slinky v*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BMC Link Validator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Premium URL Shortener 4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Premium URL Shortener*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; TURLValidator/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; TURLValidator/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*deadlinkchecker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="inboundli link checker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; TrialPay URL checker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AWeber*Link Validator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PerfectMail/* (http://perfectmail.com/kb/web_probe)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SurdotlyBot/*) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SurdotlyBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WP Engine Domain Verification*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OrgProbe/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; Windows NT *) LinkCheck by Siteimprove.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; Windows NT *; Trident/*) LinkCheck by Siteimprove.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; Windows NT *) Match by Siteimprove.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; Windows NT *; Trident/*) SiteCheck-sitecrawl by Siteimprove.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *; Trident/*; rv *) like Gecko (compatible; Zombiebot/*; +http://www.zombiedomain.net/robot/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *; Trident/*; rv:*; topster.de Linkchecker *) like Gecko *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (ClickMagick Link Monitor v*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT 10.0; Win64; x64) applewebkit/* (*khtml*like*gecko*) (CMLINKCHECK) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Powermarks">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Powermarks/3.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Powermarks/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Powermarks/3.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Powermarks/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Live (http://www.live.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MFC Foundation Class Library*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MFHttpScan">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft BITS/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft Data Access Internet Publishing Provider Cache Manager">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft Data Access Internet Publishing Provider DAV*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft Data Access Internet Publishing Provider Protocol Discovery">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft Office Existence Discovery">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft Office Protocol Discovery">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft Office/* (*Picture Manager*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft URL Control*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft Visio MSIE">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSN Feed Manager">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="iisbot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="iisbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MS Web Services Client Protocol 1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MS Web Services Client Protocol *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZDM/4.0; Windows Mobile 8.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZDM/4.0; Windows Mobile 8.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft CryptoAPI">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft-CryptoAPI/6.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft-CryptoAPI/6.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft-CryptoAPI/6.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft-CryptoAPI/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft-WebDAV">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft-WebDAV-MiniRedir/6.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft-WebDAV-MiniRedir/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DavClnt*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetNewsWire">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetNewsWire/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetNewsWire/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetNewsWire/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetNewsWire/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetNewsWire*/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) NetNewsWire/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) NetNewsWire/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?11*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?11*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?11*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) NetNewsWire/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?11*) applewebkit* (*khtml*like*gecko*) *NetNewsWire/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) *NetNewsWire/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) *NetNewsWire/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) *NetNewsWire/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?11*) applewebkit* (*khtml*like*gecko*) *NetNewsWire*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) *NetNewsWire*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) *NetNewsWire*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) *NetNewsWire*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Miscellaneous Browsers">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Amiga*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="12345*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="1st ZipCommander (Net) - http://www.zipcommander.com/">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ace Explorer">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Enigma Browser*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EVE-minibrowser/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Godzilla/* (Basic*; *; Commodore C=64; *; rv:1.*)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GreenBrowser">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Kopiczek/* (WyderOS*; *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (*) - BrowseX (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; ibisBrowser)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Macintosh; ?; PPC Mac OS X;*) applewebkit* (*) HistoryHound/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetRecorder*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetSurf*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ogeb browser , Version 1.1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SCEJ PSP BROWSER 0102pspNavigator">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WEB.DE MailCheck/2.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WEB.DE MailCheck/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GMX MailCheck/2.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GMX MailCheck/2.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GMX MailCheck/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="T-Online Browser">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="findlinks">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="findlinks/2.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="findlinks/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FindLinks/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Online Scanners">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="JoeDog/* (X11; I; Siege *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Morfeus Fucking Scanner">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Trend Micro tmdr 1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Titanium 2005 (4.02.01)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="virus_detector*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WhatWeb/0.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WhatWeb/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Nmap Scripting Engine*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FirstSearchBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; FirstSearchBot/1.0; *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; FirstSearchBot/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CloudFlare Bots">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Cloudflare-AMP/*; +https://amp.cloudflare.com/*) applewebkit*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Linux; Android *) applewebkit* (*khtml*like*gecko*) Chrome/* Mobile Safari/* (compatible; Cloudflare-AMPHTML)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Proxy Servers">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*squid*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CE-Preload">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IE/6.01 (CP/M; 8-bit*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (TuringOS; Turing Machine; 0.0)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE ?.0; SaferSurf*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Nutscrape">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Nutscrape/* (CP/M; 8-bit*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Privoxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ProxyTester*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SilentSurf*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SmallProxy*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Space*Bison/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sqworm/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SurfControl">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CoralWebPrx*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; SaferSurf*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Synapse)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; Windows NT *; mowser; http://www.mowser.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YahooMailProxy*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Proxy">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AppEngine-Google; (+http://code.google.com/appengine; appid: s~snapchat-proxy*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FlipboardProxy">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CrOS*) Gecko* Firefox/* (FlipboardProxy/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) Gecko* Firefox/* (FlipboardProxy/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) Gecko* Firefox/* (FlipboardProxy/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) Gecko* Firefox/* (FlipboardProxy/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) Gecko* Firefox/* (FlipboardProxy/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) Gecko* Firefox/* (FlipboardProxy/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) Gecko* Firefox/* (FlipboardProxy/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) Gecko* Firefox/* (FlipboardProxy/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) Gecko* Firefox/* (FlipboardBrowserProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) Gecko* Firefox/* (FlipboardBrowserProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) Gecko* Firefox/* (FlipboardBrowserProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) Gecko* Firefox/* (FlipboardProxy/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) Gecko* Firefox/* (FlipboardProxy/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) Gecko* Firefox/* (FlipboardProxy/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CrOS*) Gecko* Firefox/* (FlipboardProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) Gecko* Firefox/* (FlipboardProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) Gecko* Firefox/* (FlipboardProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) Gecko* Firefox/* (FlipboardProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) Gecko* Firefox/* (FlipboardProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) Gecko* Firefox/* (FlipboardProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) Gecko* Firefox/* (FlipboardProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) Gecko* Firefox/* (FlipboardProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) Gecko* Firefox/* (FlipboardProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) Gecko* Firefox/* (FlipboardProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) Gecko* Firefox/* (FlipboardProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Camo Asset Proxy">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Camo Asset Proxy 2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Camo Asset Proxy*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IRLbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IRLbot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IRLbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IRLbot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IRLbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Steeler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Steeler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Steeler/3.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Steeler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Riddler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Riddler *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib/2.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib/2.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib/2.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib/2.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib/1.17*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib/1.16*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib/1.15*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python-urllib/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Vulnerability Scanners">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BOT/0.1* (BOT for JCE)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BOT/* (BOT for JCE)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (FHScan Core 1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (FHScan Core *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.75 *OpenVAS*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OWASP_SECRET_BROWSER/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OWASP_SECRET_BROWSER/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="test certificate info*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="asafaweb.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CATExplorador/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CATExplorador/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebFuck V2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebFuck*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT *; rv:*) Gecko/* Firefox/* (NetShelter ContentScan*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; *Linux*; rv:*; ips-agent) Gecko/* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="masscan/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="masscan/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; DNS SSL/TLS HTTP HTML Website Security Scanner/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="btcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="muhstik*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="struts-pwn*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; WebCookies*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="urlgrabber">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="urlgrabber/3.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="urlgrabber/3.10*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="urlgrabber/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RMSnapKit">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) RMSnapKit/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) RMSnapKit/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) RMSnapKit/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) RMSnapKit/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) RMSnapKit/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) RMSnapKit/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Rippers">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*grub*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*ickHTTP*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ruby">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*WebGrabber*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="3D-FTP/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="3wGet/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ActiveRefresh*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ad Muncher*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Artera (Version *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AutoHotkey">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="b2w/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BasicHTTP/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BlockNote.Net">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CAST">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CFSCHEDULE*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CobWeb/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ColdFusion*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Crawl_Application">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CTerm/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Custo*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DataCha0s/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DeepIndexer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DISCo Pump *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ezic.com http agent *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="fetch libfetch/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FGet*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="findfiles.net/* (Robot;test_robot@gmx-topmail.de)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Flaming AttackBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Foobot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GameSpyHTTP/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gnome-vfs/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Harvest/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="hcat/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HLoader">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Holmes/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="http generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="http://arachnode.net*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="httpclient/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="httpclient*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SynHttpClient*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="httperf/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTTPFetch/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTTPGrab">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HttpSession">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="httpunit/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ICE_GetFile">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="iexplore.exe">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Inet - Eureka App">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="INetURL/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="InetURL:/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Internet Exploiter/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZiggsBot (*compatible*;*MSIE 6.0;*Windows NT 5.2*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IP*Works!*/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="IrssiUrlLog/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="JPluck/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Kapere (http://www.kapere.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LeechFTP">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LeechGet*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="libWeb/clsHTTP*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MFC_Tear_Sample">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Moozilla">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MovableType/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (compatible; OffByOne; Windows*) Webster Pro V3.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/3.0 (compatible; Indy Library)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; BorderManager*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; IPCheck Server Monitor*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OCN-SOC/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Offline Explorer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Open Web Analytics Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pageload*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PageNest/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="pavuk/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PEAR HTTP_Request*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PigBlock (Windows NT 5.1; U)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pockey*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="POE-Component-Client-HTTP/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PycURL/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RepoMonkey*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SBL-BOT*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ScoutAbout*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sherlock/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SiteParser/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SiteSnagger*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SiteSucker/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SiteWinder*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snoopy*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SOFTWING_TEAR_AGENT*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SuperHTTP/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tcl http client package*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Twisted PageGetter">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="URL2File/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UtilMind HTTPGet">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="VCI WebViewer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Web Downloader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Web Downloader/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Web Magnet*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebAuto/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="webbandit/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebCopier*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebDownloader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebFetch">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="webfetch/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebGatherer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebGet">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebReaper*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebRipper">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebSauger*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Website Downloader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Website eXtractor*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Website Quester">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebsiteExtractor*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebSnatcher*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Webster Pro*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebStripper*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebWhacker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WinScripter iNet Tools">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WWW-Mechanize/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mechanize/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="A1 Keyword Research*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="A1 Website Analyzer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="A1 Website Download*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AA bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AACrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Access Browser*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ACCESS robot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Acme.Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AcroForms*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Adaxas Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Advanced URL Catalog*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Advista Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="aipbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AISearchBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BookMacster/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BookmarkSync/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bot.ara.com.tr/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BrowserCrasherChecker *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Chordie.com: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="crawler4j*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Crawllybot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Crawllybot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Crawl*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Dropcatcher.de Domain Bot +http://www.dropcatcher.de/bot.html">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ESigil Request *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="eStyleSearch *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTTP Fetcher/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="http_get">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTTPing *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Echoping/6.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Echoping*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/2.0 (compatible; NEWT ActiveX; *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (compatible; OffByOne; *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PigBlock*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Internet (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/3.0 (compatible)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/3.01 (compatible;)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible;)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/6.0 (compatible)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/6.0 (compatible;)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/?.0 (compatible;*User-agent*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Shockwave Flash">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SiteTruth.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SVN/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TLSProber/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-Live-Social-Object-Extractor-Engine*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WinSysClean*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Dolphin http client/10.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Dolphin http client/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Python*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Smartsite HTTPClient*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RPT-HTTPClient/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EventMachine HttpClient*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="node-superagent/1.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="node-superagent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Goose/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Goose/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AHC/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AHC/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="newspaper/0.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="newspaper/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="EasyBib AutoCite*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Dispatch/0.11*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Dispatch/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="http-kit/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="http-kit/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="node-fetch/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="node-fetch/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Faraday v0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Faraday*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ruby/Curb*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Embed PHP Library*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OpmlViewer network fetcher version 1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OpmlViewer network fetcher*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="raynette_httprequest/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="raynette_httprequest/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ProspectB2B/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; ProspectB2B/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CakePHP*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gpEasy RemoteGet*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="asynchttp*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; U; Pcore-HTTP-UA/v0.12*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; U; Pcore-HTTP-UA/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebClient/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebClient/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="greasemonkey &#8211; google images relinker (v 2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="greasemonkey &#8211; google images relinker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SoftMaker Fetch/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SoftMaker Fetch/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="livedoor HttpClient*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="scrapy-redis*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; U; AnyEvent-HTTP/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Client/* (org.mozilla.ios.Firefox; build:*; iOS *) Alamofire/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SimpleScraper)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="fasthttp* mozilla/5.0 (x11; linux x86_64) applewebkit* (*khtml*like*gecko*) chrome/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="fasthttp*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Snacktory; +*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*x86_64*) AppleWebKit/* (KHTML, like Gecko) splash Version/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Serf/1.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Serf/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="axios/0.18*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="axios/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 zgrab/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Manzama Fetcher/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTTPie/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="okhttp">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="okhttp/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="okhttp/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="okhttp/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="okhttp/2.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="okhttp/2.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="okhttp/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CKHttpGenerator">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="<*>*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CKHttpGenerator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Apache HttpClient">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Apache-HttpClient/4.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Apache-HttpClient/4.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Apache-HttpClient/4.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Apache-HttpClient/4.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Apache-HttpClient/4.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Apache-HttpClient/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Apache-HttpClient/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ActiveWorlds Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ActiveWorlds*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Accoona Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Accoona-AI-Agent/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Accoona-AI-Agent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="accoona*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform 1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/1.0 (MSIE *; *Windows NT 6.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/1.0 (MSIE *; *Windows NT 5.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/1.0 (MSIE *; Windows NT*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.21">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.21.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.21.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.21.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.21.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.20">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.20.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.20.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.20.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.20.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.19">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.19.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.19.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.19.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.19.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.18">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.18.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.18.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.18.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.18.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.17">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.17.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.17.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.17.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.17.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.16">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.16.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.16.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.16.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.16.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.15">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.15.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.15.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.15.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.15.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.14">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.14.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.14.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.14.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.14.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.13">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.13.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.13.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.13.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.13.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.12">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.12.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.12.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.12.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.12.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.11">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.11.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.11.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.11.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.11.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.10">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.10.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.10.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.10.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.10.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.9">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.9.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.9.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.9.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.9.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.8">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.8.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.8.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.8.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.8.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.7">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.7.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.7.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.7.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.7.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.6">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.6.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.6.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.6.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.6.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.5">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.5.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.5.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.5.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.5.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.4">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.4.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.4.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.4.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.4.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.3">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.3.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.3.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.3.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.3.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.2.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.2.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.2.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.1.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.1.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.1.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 2.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.0.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.0.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.0.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/2.0.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 1.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/1.2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/1.2.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/1.2.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/1.2.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests 1.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/1.1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/1.1.* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/1.1.* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/1.1.* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="libcurl">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="libcurl-agent/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="libcurl-agent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform 2.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/2.0 (MSIE *; *Windows NT 6.3*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/2.0 (MSIE *; *Windows NT 6.2*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/2.0 (MSIE *; *Windows NT 6.1*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/2.0 (MSIE *; *Windows NT 6.0*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/2.0 (MSIE *; *Windows NT 5.1*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/2.0 (MSIE *; Windows NT*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/2.0 (IE *; *Windows NT 6.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/2.0 (IE *; *Windows NT 6.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/2.0 (IE *; *Windows NT 6.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-RSS-Platform/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/* CPython/*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/* CPython/*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="python-requests/* CPython/*Darwin*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Site Monitors">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*EasyRider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*maxamine.com--robot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*WebMon ?.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Kenjin Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Kevin http://*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; ChangeDetection/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; Windows NT *; http://www.changedetection.com/bot.html*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Chirp/1.0; +http://www.binarycanary.com/chirp.cfm)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0?(compatible; UptimeRobot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0?(compatible; UptimeRobot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0?(compatible; UptimeRobot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Myst Monitor Service v*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Net Probe">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetMechanic*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NetReality*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Site Valet Online*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SITECHECKER">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="sitemonitor@dnsvr.com/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UpTime Checker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="URL Control*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="URL_Access/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="URLCHECK">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="URLy Warning*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Webcheck *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebPatrol/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="websitepulse checker/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Chirp/*; +http://www.binarycanary.com/chirp.cfm)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows*) applewebkit* (*khtml*like*gecko*) Bromium Safari/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows*) applewebkit* (*khtml*like*gecko*) Bromium Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Astute SRM*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; PRTG Network Monitor*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewRelicPinger/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="montastic-monitor http://www.montastic.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; PingAdmin.Ru/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit/* (*khtml*like*gecko*) Chrome/* Safari/* http://notifyninja.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Testomatobot/* (* +http://www.testomato.com/testomatobot) minicrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="404 Checker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="agentslug.com - website monitoring tool">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; Windows *; www.alertra.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; Windows *; www.alertra.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* YottaaMonitor">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows; U; Windows NT *; en-US; rv:*) Gecko/* Firefox/* YottaaMonitor">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64; GTmetrix http*://gtmetrix.com/) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Linux; Android *; * Build/*; GTmetrix http*://gtmetrix.com/) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux *x86_64; rv:*; GTmetrix http*://gtmetrix.com/) Gecko/20100101 Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (StatusCake*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *; Neustar WPM) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Linux; Android *; * Build/*; Neustar WPM) applewebkit* (*khtml*like*gecko*) Chrome/* Mobile Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows NT *; rv:*; Neustar WPM) Gecko/20100101 Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (* like Mac OS X; Neustar WPM) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64; rv:*) Gecko/20100101 Firefox/* Ghost Inspector">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64) applewebkit* (*khtml*like*gecko*) QtWebEngine/* Safari/* Ghost Inspector">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Macintosh; Intel Mac OS X *) applewebkit* (*khtml*like*gecko*) Version/* Safari/*; Shoppimon Analyzer (http://www.shoppimon.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/3.0 (compatible; WebMon *; Windows XP)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Zabbix*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Statusdroid/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Statusdroid/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pingdom">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pingdom.com_bot_version_1.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pingdom.com_bot_version_*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pingdom GIGRIB*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Unknown; Linux x86_64) applewebkit* (*khtml*like*gecko*) PingdomTMS/0.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Unknown; Linux x86_64) applewebkit* (*khtml*like*gecko*) PingdomTMS/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64) applewebkit* (*khtml*like*gecko*) Ubuntu Chromium/* Chrome/* Safari/* PingdomPageSpeed/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Pingdom*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CloudEndure Scanner">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CloudEndure Scanner*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Catchpoint Monitoring Service">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*Catchpoint)*applewebkit* (*khtml*like*gecko*) Chrome/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*Catchpoint)*applewebkit* (*khtml*like*gecko*) Chrome/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*Catchpoint)*applewebkit* (*khtml*like*gecko*) Chrome/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*Catchpoint)*applewebkit* (*khtml*like*gecko*) Chrome/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*Catchpoint)*applewebkit* (*khtml*like*gecko*) Chrome/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*Catchpoint)*applewebkit* (*khtml*like*gecko*) Chrome/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Uptimebot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Uptimebot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Uptimebot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SpeedCurve WebPage Test">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0* rv:*) Gecko/20100101 Firefox/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4* rv:*) Gecko/20100101 Firefox/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3* rv:*) Gecko/20100101 Firefox/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2* rv:*) Gecko/20100101 Firefox/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1* rv:*) Gecko/20100101 Firefox/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0* rv:*) Gecko/20100101 Firefox/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2* rv:*) Gecko/20100101 Firefox/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1* rv:*) Gecko/20100101 Firefox/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows* rv:*) Gecko/20100101 Firefox/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0* Trident/?.0* rv:*) like gecko PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4* Trident/?.0* rv:*) like gecko PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3* Trident/?.0* rv:*) like gecko PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2* Trident/?.0* rv:*) like gecko PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1* Trident/?.0* rv:*) like gecko PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0* Trident/?.0* rv:*) like gecko PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2* Trident/?.0* rv:*) like gecko PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1* Trident/?.0* rv:*) like gecko PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows* Trident/?.0* rv:*) like gecko PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3* Trident/?.0* rv:* PTST?* like gecko">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2* Trident/?.0* rv:* PTST?* like gecko">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1* Trident/?.0* rv:* PTST?* like gecko">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0* Trident/?.0* rv:* PTST?* like gecko">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows* Trident/?.0* rv:* PTST?* like gecko">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows NT 10.0* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows NT 6.4* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows NT 6.3* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows NT 6.2* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows NT 6.1* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows NT 6.0* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows NT 5.2* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows NT 5.1* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE *; *Windows* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; *Windows NT 10.0* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; *Windows NT 6.4* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; *Windows NT 6.3* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; *Windows NT 6.2* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; *Windows NT 6.1* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; *Windows NT 6.0* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; *Windows NT 5.2* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; *Windows NT 5.1* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; *Windows* Trident/?.0* PTST?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?7.1*) applewebkit* (*khtml*like*gecko*) Chrome/* Mobile Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?7.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Mobile Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?6.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Mobile Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?5.1*) applewebkit* (*khtml*like*gecko*) Chrome/* Mobile Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?5.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Mobile Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?4.4*) applewebkit* (*khtml*like*gecko*) Chrome/* Mobile Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android*) applewebkit* (*khtml*like*gecko*) Chrome/* Mobile Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?7.1*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?7.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?6.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?5.1*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?5.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?4.4*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* PTST/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pingdom Pagespeed">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Social Bookmarkers">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BookmarkBase(2/;http://bookmarkbase.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cocoal.icio.us/1.0 (v43) (Mac OS X; http://www.scifihifi.com/cocoalicious)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) Gecko* Firefox/2.0 OneRiot/1.0 (http://www.oneriot.com) ">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; FriendFeedBot/0.*; +Http://friendfeed.com/about/bot)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Twitturls; +http://twitturls.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SocialSpider-Finder/0.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Twitturly*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WinkBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; FriendFeedBot/*; +Http://friendfeed.com/about/bot)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Gecko* Firefox/2.0 OneRiot/1.0 (http://www.oneriot.com) ">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Slackbot-LinkExpanding*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Slackbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Slack-ImgProxy*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Embedly/0.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Embedly/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Linux; Socialradarbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Linux; Socialradarbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/*; ?collection@infegy.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; EveryoneSocialBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; EveryoneSocialBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="R6_CommentReader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="viralvideochart.unrulymedia.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Linux; InfegyAtlas/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Linux; InfegyAtlas/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="QuickiWiki/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="QuickiWiki/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; vkShare*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="seebot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="seebot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Kraken/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Kraken/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NING/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NING/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sprinklr 2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sprinklr*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Traackr*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Kimengi/nineconnections*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Twikle/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Twikle/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 Tweetminster*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Trove*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pearltrees*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Svven-Summarizer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Leadify/1.01*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Leadify/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="inbound.li parser*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 ( compatible ; Veooz/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 ( compatible ; Veooz/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SentiBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Macintosh; *Mac OS X*) AppleWebKit/* (*khtml*like*gecko*) ShopSpring SpringBot Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DocBase Crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="twieve/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 AppleWebKit/* (*khtml*like*gecko*) Chrome/* Safari/* (Refindbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MagentaNews*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MeltwaterNews*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; PressRush/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; EchoboxBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CheckMarkNetwork*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Qwantify">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Qwantify/2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Qwantify/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Qwantify/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mediatoolkit Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mediatoolkitbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AddThis.com robot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AddThis.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YOURLS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YOURLS v1.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YOURLS v1.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YOURLS v1.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YOURLS v1.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YOURLS v1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YOURLS v1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YOURLS *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Friendica">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Friendica '*' 3.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Friendica '*' 3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Friendica*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Friendica*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Translators">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Seram Server">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TeragramWebcrawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebIndexer/* (Web Indexer*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebIndexer/* (Web Indexer; *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebTrans">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ATA-Translation-Service">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Gnomit">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*http://gnomit.com/) Gecko* Gnomit/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*http://gnomit.com/) Gecko* Gnomit/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Version Checkers">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Automated Browscap.ini Updater. Gary contact me at serge@skycomp.ca with problems.">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Automated Browscap.ini Updater. To report issues contact us at+http://www.skycomp.ca">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Browscap Mirror System/1.* (browscap.giantrealm.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Browscap Mirror v1.30">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Browscap updater">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="browscap updater; interval:weekly; server:rohan.doppy.nl; questions:support@doppy.nl;">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BrowscapUpdater1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Browser Capabilities Project - PHP Browscap*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Browser Capabilities Project AutoDownloader; created by Tom Kelleher Consulting, Inc. (tkelleher.com); used with special permission from Gary Joel Keith; uses Microsoft?s WinHTTP component">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Decode Framework 0.* browscap library">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Desktop Sidebar*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mono Browser Capabilities Updater*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP Browser Capabilities Project/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UpdateBrowscap*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WCC Browscap Updater/0.* (PHP: file_get_contents)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Automated Browscap.ini Updater. To report issues contact us at*http://www.skycomp.ca">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Browser Capabilities Project - PHP Browscap/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Browscap Mirror System/* (browscap.giantrealm.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BrowscapUpdater*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Code Sample Web Client">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Subtext Version 1.9* - http://subtextproject.com/ (Microsoft Windows NT 5.2.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TherapeuticResearch">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Rewmi/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="browsers.garykeith.com browscap.ini bot BETA">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="www.substancia.com AutoHTTPAgent (ver *)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebRingChecker/1.71*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WebRingChecker/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jupdate">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="jupdate*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SophosUpdateManager">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SophosUpdateManager/1.5.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SophosUpdateManager/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SophosAgent">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SophosAgent/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SophosAgent/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="W3C">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*W3C-checklink/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jigsaw/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jigsaw/* W3C_CSS_Validator*/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="P3P Validator">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Unicorn/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="W3C-mobileOK/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="W3C-mobileOK/DDC-*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="W3C-WebCon/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="W3C_Validator/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="W3CLineMode/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="W3CRobot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Validator.nu/LV*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="W3C_I18n-Checker/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="W3C_Unicorn/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GomezAgent 3.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; *Windows NT 6.0*; *GomezAgent 3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; *Windows NT 6.1*; Trident/4.0; *GomezAgent 3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; *Windows NT 5.2*; Trident/4.0; *GomezAgent 3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; *Windows NT 5.1*; Trident/4.0; *GomezAgent 3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*; GomezAgent 3.1*) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*; GomezAgent 3.1*) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*; GomezAgent 3.1*) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*; GomezAgent 3.1*) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) applewebkit* (*khtml*like*gecko* GomezAgent 3.1*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*) applewebkit* (*khtml*like*gecko* GomezAgent 3.1*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko* GomezAgent 3.1*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) applewebkit* (*khtml*like*gecko* GomezAgent 3.1*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GomezAgent 3.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; *Windows NT 6.0*; *GomezAgent 3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; *Windows NT 6.1*; Trident/4.0; *GomezAgent 3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; *Windows NT 5.2*; Trident/4.0; *GomezAgent 3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; *Windows NT 5.1*; Trident/4.0; *GomezAgent 3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*; GomezAgent 3.0*) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*; GomezAgent 3.0*) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*; GomezAgent 3.0*) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*; GomezAgent 3.0*) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) applewebkit* (*khtml*like*gecko* GomezAgent 3.0*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*) applewebkit* (*khtml*like*gecko* GomezAgent 3.0*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko* GomezAgent 3.0*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) applewebkit* (*khtml*like*gecko* GomezAgent 3.0*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GomezAgent 2.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GomezAgent">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; *Windows NT 6.0*; *GomezAgent *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; *Windows NT 6.1*; Trident/4.0; *GomezAgent*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; *Windows NT 5.2*; Trident/4.0; *GomezAgent*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible*; MSIE ?.0; *Windows NT 5.1*; Trident/4.0; *GomezAgent*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*; GomezAgent *) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*; GomezAgent *) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*; GomezAgent *) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) applewebkit* (*khtml*like*gecko* GomezAgent *) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*) applewebkit* (*khtml*like*gecko* GomezAgent *) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) applewebkit* (*khtml*like*gecko* GomezAgent *) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Heritrix">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*; heritrix/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*; heritrix/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible*; heritrix/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*heritrix*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bl.uk_lddc_bot/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="bl.uk_lddc_bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (bl.uk_lddc_bot; *Linux*)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; bnf.fr_bot;*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedHub">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedHub FeedDiscovery/1.0 (http://www.feedhub.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedHub FeedFetcher/1.0 (http://www.feedhub.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="FeedHub MetaDataFetcher/1.0 (http://www.feedhub.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BoardReader">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BoardReader Favicon Fetcher /1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BoardReader Favicon Fetcher /*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BoardReader Blog Indexer*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS 1.13">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS/1.13*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS 1.12">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS/1.12*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS 1.11">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS/1.11*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS 1.10">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS/1.10*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS 1.9">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS/1.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS 1.8">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS/1.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS 1.7">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS/1.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tiny Tiny RSS/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Internet Content Rating Association">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ICRA_label_generator/1.?">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ICRA_Semantic_spider/0.?">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Nagios">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="check_http/* (nagios-plugins 1.1.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="check_http/* (nagios-plugins 1.2.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="check_http/* (nagios-plugins 1.3.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="check_http/* (nagios-plugins 1.4.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="check_http/* (nagios-plugins 1.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="check_http/* (nagios-plugins 2.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ichiro Mobile Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DoCoMo/2.0 P900i(c100;TB;W24H11) (compatible; ichiro/mobile goo*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="* (compatible; ichiro/mobile goo*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ichiro Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ichiro/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ichiro/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NameProtect">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NP/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NPBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Netcraft">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Netcraft Web Server Survey*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Netcraft Webserver Survey*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Netcraft SSL Server Survey*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; *info@netcraft.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0*info@netcraft.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; MSIE *; Windows NT *; Netcraft SSL Server Survey - contact info@netcraft.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jobs">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="JobRoboter/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="JobRoboter/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MetaJobBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; JobdiggerSpider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*compatible; Jobboerse*) Gecko* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64) applewebkit* (*khtml*like*gecko*) JobBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="iCjobs Crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*iCjobs Stellenangebote Jobs*)*iCjobs/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*iCjobs Stellenangebote Jobs*)*iCjobs/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CareerBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; CareerBot/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; CareerBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewsGator">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MarsEdit*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewsGator FetchLinks extension/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewsGator/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewsGatorOnline/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NewsGatorOnline/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LetsCrawl.com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LetsCrawl.com/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LetsCrawl.com/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WooRank">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="woobot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="woobot/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="woobot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; woorankreview/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; woorankreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="uMBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; uMBot-LN/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; uMBot-LN/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; uMBot-FC/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; uMBot-FC/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; um-LN/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; um-LN/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkChecker">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkChecker/7.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkChecker/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="linkCheck">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; linkCheckV3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; linkCheckV*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkedInBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkedInBot/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LinkedInBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LIAuthLibrary:0.0.2 com.linkedin.android:3.4.2 TCL_*android_4.4.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LucidMedia ClickSense">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LucidMedia ClickSense/4.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LucidMedia ClickSense/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="m65bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DoCoMo/2.0 P904i( m65bot/0.1; c; http://m65.jp/bot.html )">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="magpie-crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="magpie-crawler/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="magpie-crawler/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mahalobot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mahalobot/1.0 (?http://www.mahalo.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mahalobot/* (?http://www.mahalo.com/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Marvin">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Marvin v0.3">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Marvin v*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="masidani_bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="masidani_bot_v0.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="masidani_bot_v*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MJ12bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MJ12bot/v0.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MJ12bot/v0.5*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MJ12bot/v0.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MJ12bot/v0.6*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MJ12bot/v0.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MJ12bot/v0.7*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MJ12bot/v0.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MJ12bot/v0.8*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MJ12bot/v0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MJ12bot/v0.9*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MJ12bot/v1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MJ12bot/v1.1*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MJ12bot/v1.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MJ12bot/v1.2*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MJ12bot/v1.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MJ12bot/v1.3*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MJ12bot/v1.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MJ12bot/v1.4*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MJ12bot/v1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MJ12bot/v1.*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MJ12bot/v*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MJ12bot/v*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozdex">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozdex/0.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozdex/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Naver">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cowbot-* (NHN Corp*naver.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; NaverBot/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; NaverBot/*; nhnbot@naver.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NaverBot-* (NHN Corp*naver.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yeti/1.0*http://help.naver.com/robots/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yeti/*http://help.naver.com/robots/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Yeti/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Yeti/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*naver*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AdsBot-Naver">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPhone*CPU iPhone OS 9?1* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/9.0*Safari/*AdsBot-Naver*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="NaverBot Mobile">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DoCoMo/2.0?P-04A *(compatible; Yeti-Mobile/0.1; ?http://help.naver.com/robots/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPhone; CPU iPhone OS * like Mac OS X) (compatible; Yeti-Mobile/*; +http://help.naver.com/robots/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PagePeeker">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko*)*Chrome/* Safari/* PagePeeker/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko*)*Chrome/* Safari/* PagePeeker/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; PPBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pete-Spider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pete-Spider/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pete-Spider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Rival IQ Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* (Rival IQ, rivaliq.com)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* (Rival IQ, rivaliq.com)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="httpclient (http://www.phpclasses.org/httpclient $Revision:*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PECL::HTTP/1.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PECL::HTTP/1.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PECL::HTTP/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="?Ogg.class.php*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Ogg.class.php*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="xmlrpc-epi-php/* (PHP)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP-SOAP/5.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP-SOAP/5.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP-SOAP/5.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP-SOAP/5.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP-SOAP/5.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP-SOAP/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="XML-RPC for PHP *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Zend_XmlRpc_Client">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Zend_Http_Client">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP/5.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP/5.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP/5.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP/5.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP/5.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHP*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GuzzleHttp/6.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GuzzleHttp/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Guzzle/4.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Guzzle/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Taproot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PHPCrawl*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pixray-Seeker">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pixray-Seeker/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pixray-Seeker/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pixray-Seeker/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pixray-Seeker/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="cURL">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="curl/7.54*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="curl/7.35*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="curl/7.29*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="curl/7.19*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="curl/7.19">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="curl/7.15*Linux*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="curl/7.15">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="curl*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*WinHttpRequest*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WinHTTP*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WinHttp*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="TrendWinHttp*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CaptiveNetworkAgent">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CaptiveNetworkSupport*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AppleNewsBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AppleNewsBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Applebot 0.3">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* (*Applebot/0.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS * like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* (*Applebot/0.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Applebot 0.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* (*Applebot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS * like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* (*Applebot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Applebot Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* (*Applebot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS * like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Safari/* (*Applebot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTMLParser">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTMLParser/1.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTMLParser/2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTMLParser/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="libwww">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ShowXML/* libwww/5.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="libwww-perl/5.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (windows; U; *) Gecko* libwww-perl/6.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="libwww-perl/6.03*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="libwww-perl/6.04*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="libwww-perl/6.05*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="libwww-perl/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*libwww*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="lwp">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="lwp*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="LWP::Simple*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="lwp-download*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="lwp-request*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="lwp-rget*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="lwp-trivial*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OSSProxy">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OSSProxy 1.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OSSProxy*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Linkdex Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; linkdexbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; linkdexbot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; linkdexbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="picmole Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;picmole/1.* +http://www.picmole.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;picmole/* +http://www.picmole.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZoomBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="ZoomBot *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="rogerbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="rogerbot/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="rogerbot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="rogerbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RankFlex">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="RankFlex.com Webspider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Baidu">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS 9?1* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* (compatible; Baiduspider-render/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;*Baiduspider/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;*Baiduspider/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Baiduspider-image*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Baiduspider-ads*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Baiduspider-cpro*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Baiduspider-favo*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Baiduspider-image*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Baiduspider-news*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Baiduspider-video*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Baiduspider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Baiduspider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="baidu bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AC-BaiduBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BaiduImageSpider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Baidu Mobile">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Baiduspider-mobile*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DoCoMo/2.0 P05A(c100;TB;W24H15) (compatible; BaiduMobaider/1.0; ?http://www.baidu.jp/spider/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android*) applewebkit* (*khtml*like*gecko*) Version/4.0 Mobile Safari/* (compatible; +http://www.baidu.com/search/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GoogleFriendConnect*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Kml-Google; (+http://code.google.com/apis/kml)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Google-Youtube-Links)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; GoogleToolbar 7.5*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; GoogleToolbar 7.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; GoogleToolbar*Windows*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; GoogleToolbar*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) applewebkit* (*khtml*like*gecko*) Version/3.1 Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="google (*Enterprise*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google OpenSocial agent*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google-Site-Verification*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Google-Site-Verification/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Google-Site-Verification/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feedfetcher-Google-iGoogleGadgets*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) applewebkit*(*khtml*like*gecko; Google-Adwords-DisplayAds-WebRender;)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64) applewebkit* (*khtml*like*gecko* Google Search Console) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPhone; CPU iPhone OS * like Mac OS X) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/* (compatible; Google Search Console)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (X11; Linux x86_64) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Google-HotelAdsVerifier/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows; U; Windows NT *; rv:*; Google-SearchByImage) Gecko/* Firefox/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google-AdWords-Express*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 applewebkit*(*khtml*like*gecko; GoogleAdwordsExpress*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Wireless Transcoder">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Google Wireless Transcoder*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) applewebkit* (*khtml*like*gecko; Google Wireless Transcoder) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Sitemaps">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google-Sitemaps*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google-Shopping-Quality">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 applewebkit*(*khtml*like*gecko*) Chrome/* Safari/* (compatible; Google-Shopping-Quality*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google-Read-Aloud">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 applewebkit*(*khtml*like*gecko*) Chrome/* Safari/* (compatible; Google-Read-Aloud*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Partner Monitoring">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* google_partner_monitoring*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Chrome Lighthouse">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) applewebkit*(*khtml*like*gecko*) Chrome/* Safari/* Chrome-Lighthouse">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="baidu Transcoder">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*baidu Transcoder*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*baidu Transcoder) Gecko* Firefox/3.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google DMCA Takedown Tool Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) applewebkit* (*khtml*like*gecko; Google-Assess) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AdSense Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mediapartners-Google*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Adsense-Snapshot-Google*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS 8?1* like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/*Safari/* (compatible; Mediapartners-Google/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS 8?* like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/*Safari/* (compatible; Mediapartners-Google/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS 7?1* like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/*Safari/* (compatible; Mediapartners-Google/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS 7?* like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/*Safari/* (compatible; Mediapartners-Google/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS * like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/*Safari/* (compatible; Mediapartners-Google/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS * like Mac OS X*) applewebkit* (*khtml*like*gecko*Mediapartners-Google) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Search Appliance">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="gsa-crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google PageSpeed Insights Mobile">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?6.0*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Mobile Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Mobile Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPad*CPU OS 8_1* like Mac OS X*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPad*CPU OS 8* like Mac OS X*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPad*CPU OS 7_1* like Mac OS X*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPad*CPU OS 7* like Mac OS X*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPad*CPU*OS* like Mac OS X*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS * like Mac OS X*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU*OS* like Mac OS X*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google PageSpeed Insights">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CrOS*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko; Google Page Speed Insights) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Favicon">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CrOS*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1* rv:*) Gecko/* Firefox/* Google favicon">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0* rv:*) Gecko/* Firefox/* Google favicon">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2* rv:*) Gecko/* Firefox/* Google favicon">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1* rv:*) Gecko/* Firefox/* Google favicon">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?11*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google Favicon*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Web Snippet">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* Google (+https://developers.google.com/+/web/snippet/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CrOS*) Gecko* Firefox/* Google (+https://developers.google.com/+/web/snippet/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) Gecko* Firefox/* Google (+https://developers.google.com/+/web/snippet/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) Gecko* Firefox/* Google (+https://developers.google.com/+/web/snippet/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) Gecko* Firefox/* Google (+https://developers.google.com/+/web/snippet/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) Gecko* Firefox/* Google (+https://developers.google.com/+/web/snippet/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) Gecko* Firefox/* Google (+https://developers.google.com/+/web/snippet/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) Gecko* Firefox/* Google (+https://developers.google.com/+/web/snippet/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) Gecko* Firefox/* Google (+https://developers.google.com/+/web/snippet/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) Gecko* Firefox/* Google (+https://developers.google.com/+/web/snippet/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) Gecko* Firefox/* Google (+https://developers.google.com/+/web/snippet/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) Gecko* Firefox/* Google (+https://developers.google.com/+/web/snippet/)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Structured-Data Testing Tool">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Google-StructuredDataTestingTool*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google-Structured-Data-Testing-Tool*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AdsBot Google-Mobile for iOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AdsBot?Google?Mobile (?http://www.google.com/mobile/adsbot.html) Mozilla (*CPU iPhone OS * like Mac OS X*) AppleWebKit (*khtml*like*gecko*)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AdsBot?Google?Mobile (?http://www.google.com/mobile/adsbot.html) Mozilla (*CPU*OS* like Mac OS X*) AppleWebKit (*khtml*like*gecko*)*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS 9?1* like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version* Mobile* Safari* (compatible; AdsBot-Google-Mobile; *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS 9?* like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version* Mobile* Safari* (compatible; AdsBot-Google-Mobile; *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS * like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version* Mobile* Safari* (compatible; AdsBot-Google-Mobile; *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AdsBot Google-Mobile for Android">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android?5.0*) applewebkit* (*khtml*like*gecko*) Chrome* Mobile* Safari* (compatible; AdsBot-Google-Mobile; *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android*) applewebkit* (*khtml*like*gecko*) Chrome* Mobile* Safari* (compatible; AdsBot-Google-Mobile; *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AdsBot Google-Mobile">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AdsBot?Google?Mobile*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="APIs Google">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="APIs-Google *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AdsBot Google">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AdsBot?Google*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google App Engine">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AppEngine-Google*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 AppEngine-Google*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCWEB/*(Linux; U; Opera Mini/*) U2/* UCBrowser/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCWEB/*(*; U; Adr *) U2/* UCBrowser/* U2/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCWEB/*(*SymbianOS*) U2/* UCBrowser/* U2/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCWEB/*(*) U2/* UCBrowser/* U2/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (*Windows NT 5.1*)* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (*Windows*)* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*)* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) Gecko/* Firefox/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*) Gecko/* Firefox/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) Gecko/* Firefox/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) Gecko/* Firefox/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) Gecko/* Firefox/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) Gecko/* Firefox/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) Gecko/* Firefox/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?11*) Gecko/* Firefox/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) Gecko/* Firefox/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Android*Mobile*) Gecko/* Firefox/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) Gecko/* Firefox/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) Opera * AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*) Opera * AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) Opera * AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) Opera * AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) Opera * AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) Opera * AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) Opera * AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?12*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?11*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) applewebkit* (*khtml*like*gecko*)* Safari/* AppEngine-Google; (?http://code.google.com/appengine; appid: *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HubSpot Webcrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CrOS*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/* HubSpot Webcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/* HubSpot Webcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/* HubSpot Webcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/* HubSpot Webcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/* HubSpot Webcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/* HubSpot Webcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/* HubSpot Webcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/* HubSpot Webcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/* HubSpot Webcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?9*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/* HubSpot Webcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/* HubSpot Webcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*FreeBSD*) applewebkit* (*khtml*like*gecko*)*Chrome/*Safari/* HubSpot Webcrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HubSpot Links Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; HubSpot Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HubSpot Connect*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HubSpot Website Grader">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPhone*CPU iPhone OS 5?1* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*CriOS/*Safari/*HubSpot Website Grader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Mac OS X 10?10*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* HubSpot Website Grader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HubSpot Grader">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HubSpot Marketing Grader*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* HubSpot Marketing Grader">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Feedfetcher">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Feedfetcher-Google*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible) Feedfetcher-Google; ( http://www.google.com/feedfetcher.html)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Feedfetcher-Google*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Keyword Tool">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Google Keyword Tool;*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Google Keyword Tool*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Keyword Suggestion">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko, Google Keyword Suggestion) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Google Keyword Suggestion*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Google Keyword Suggestion*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Widget Server">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*) applewebkit* (*khtml*like*gecko? Widget Server*)*Chrome/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Web Preview">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko? Google Web Preview*)*Chrome/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPad*CPU OS 8_1* like Mac OS X*) applewebkit* (*khtml*like*gecko? Google Web Preview*)*Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPad*CPU OS 8* like Mac OS X*) applewebkit* (*khtml*like*gecko? Google Web Preview*)*Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPad*CPU OS 7_1* like Mac OS X*) applewebkit* (*khtml*like*gecko? Google Web Preview*)*Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPad*CPU OS 7* like Mac OS X*) applewebkit* (*khtml*like*gecko? Google Web Preview*)*Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPad*CPU*OS* like Mac OS X*) applewebkit* (*khtml*like*gecko? Google Web Preview*)*Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU OS 8_1* like Mac OS X*) applewebkit* (*khtml*like*gecko? Google Web Preview*)*Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU OS 8* like Mac OS X*) applewebkit* (*khtml*like*gecko? Google Web Preview*)*Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU OS 7_1* like Mac OS X*) applewebkit* (*khtml*like*gecko? Google Web Preview*)*Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU OS 7* like Mac OS X*) applewebkit* (*khtml*like*gecko? Google Web Preview*)*Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU*OS* like Mac OS X*) applewebkit* (*khtml*like*gecko? Google Web Preview*)*Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android*) applewebkit* (*khtml*like*gecko? Google Web Preview*)*Version/*Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (en-us) applewebkit* (*khtml*like*gecko; Google Web Preview) Version/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Google Web Preview*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google Apps Script">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Google-Apps-Script*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BingBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;*bingbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;*bingbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone 8.1* Trident/7.0* IEMobile/*) like Gecko (compatible; bingbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone* Trident/7.0* IEMobile/*) like Gecko (compatible; bingbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS 7?* like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/* (compatible; bingbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS * like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/* (compatible; bingbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS 7?* like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/* (compatible; bingbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*CPU iPhone OS * like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/* (compatible; bingbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BingAdsAdInsight">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 applewebkit* (*khtml*like*gecko*compatible; bingbot/2.0*) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Bing Local Search">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Univention Corporate Server 4.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCS (ESX) - 4.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Univention Corporate Server 4.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCS (ESX) - 4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Univention Corporate Server 3.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCS (ESX) - 3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Univention Corporate Server 3.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCS (ESX) - 3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Univention Corporate Server 3.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCS (ESX) - 3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Univention Corporate Server 2.4">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCS (ESX) - 2.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Univention Corporate Server 2.3">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCS (ESX) - 2.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Univention Corporate Server 2.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCS (ESX) - 2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Univention Corporate Server 2.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCS (ESX) - 2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Univention Corporate Server 2.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCS (ESX) - 2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Univention Corporate Server Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UCS (ESX) - *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot 2.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Googlebot/2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows; U*Gecko* Googlebot/2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows; U*Gecko* Googlebot?2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) Gecko* Googlebot?2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GoogleBot/2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Googlebot/2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot 2.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot v2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS 8?3* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* (compatible; Googlebot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* (compatible; Googlebot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*Linux*Android?6.0*)*applewebkit*(*khtml*like*gecko*) Chrome/* Safari/* (compatible; Googlebot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*Linux*Android*)*applewebkit*(*khtml*like*gecko*) Chrome/* Safari/* (compatible; Googlebot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Googlebot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows; U*Gecko* Googlebot?2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.0*) Gecko* Googlebot?2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.2*) Gecko* Googlebot?2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) Gecko* Googlebot?2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GoogleBot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Googlebot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot 2.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Googlebot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Windows; U*Gecko* Googlebot?2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 5.1*) Gecko* Googlebot?2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GoogleBot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Googlebot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot 1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GoogleBot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Googlebot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot/Test*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GoogleBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="googlebot-urlconsole">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot-Image*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot-News*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot-Video/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot-Video*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (en-us) applewebkit* (*khtml*like*gecko* Google PP Default) Chrome/* Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="googlebot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GOOG*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot-Mobile 2.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot-Mobile 2.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* (compatible; Googlebot-Mobile/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DoCoMo/1.0/N505i/c20/TB/W20H10 (compatible; Googlebot-Mobile/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Googlebot-Mobile/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Googlebot-Mobile">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="KDDI-Googlebot-Mobile">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="*Googlebot-Mobile/2.*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo Slurp">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/3.0 (Slurp*; slurp@inktomi.com;*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Slurp*; slurp@inktomi.com;*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Yahoo! *Slurp*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Yahoo! Slurp China*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Yahoo! DE Slurp; http://help.yahoo.com/help/us/ysearch/slurp)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Yahoo! Slurp China*;*http://misc.yahoo.com.cn/help.html)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Yahoo! Slurp*;*http://help.yahoo.com/help/us/ysearch/slurp*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo! Slurp/Site Explorer">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo! Slurp China">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="slurp">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*) Mobile (compatible; Yahoo! Slurp*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU*OS* like Mac OS X*)*applewebkit*(*khtml*like*gecko*) Mobile (compatible; Yahoo! Slurp*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google HTTP Client Library for Java">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google-HTTP-Java-Client/1.17*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Google-HTTP-Java-Client/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mail.Ru Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mail.Ru/1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Mail.RU_Bot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; *Linux* Mail.RU_Bot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; *Linux* Mail.RU_Bot/Fast/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; *Linux* Mail.RU_Bot/Img/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; *Linux* Mail.RU_Bot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Mail.RU/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSN-Media">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot-media/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot-media/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot-media/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) applewebkit* (*khtml*like*gecko*) MsnBot-Media /1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) applewebkit* (*khtml*like*gecko*) MsnBot-Media /1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*) MsnBot-Media /1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot/0.11*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot/2.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot/2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot/0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot-Products/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSN">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Danger hiptop 3.*; U; rv:1.7.*Gecko*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-Live-Social-Object-Extractor-Engine/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Vancouver*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="librabot/1.0 (*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="llssbot/1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Microsoft Bing Mobile SocialStreams Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Danger hiptop 3.*; U; rv:1.7.*) Gecko*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSMOBOT/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSNBot-Academic/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSNBot-News/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSNBot-NewsBlogs/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot-NewsBlogs/2.* (+http://search.msn.com/msnbot.htm)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot-products">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot-webmaster/1.0 (*http://search.msn.com/msnbot.htm)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSR-ISRCCrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSRBOT*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="renlifangbot/1.0 (?http://search.msn.com/msnbot.htm)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows-Live-Social-Object-Extractor-Engine/1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="T-Mobile Dash Mozilla/4.0 (*MSNBOT-MOBILE/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot-UDiscovery/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSNBot-News/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSNBot-NewsBlogs/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="msnbot-webmaster/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSN-Media for iOS 1.0b">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPad*CPU iPhone OS 7?* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* MsnBot-Media /1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPad*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* MsnBot-Media /1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPod*CPU iPhone OS 7?* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* MsnBot-Media /1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPod*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* MsnBot-Media /1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS 7?* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* MsnBot-Media /1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* MsnBot-Media /1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="adidxbot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone 8.1* Trident/7.* IEMobile/*) like Gecko (compatible; adidxbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone* Trident/7.* IEMobile/*) like Gecko (compatible; adidxbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPad*CPU iPhone OS 7?* like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/* (compatible; adidxbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPad*CPU iPhone OS * like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/* (compatible; adidxbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPod*CPU iPhone OS 7?* like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/* (compatible; adidxbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (iPod*CPU iPhone OS * like Mac OS X*) applewebkit* (*khtml*like*gecko*) Version/* Mobile/* Safari/* (compatible; adidxbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; adidxbot/2.0; ?http://www.bing.com/bingbot.htm)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="adidxbot/1.1 (?http://search.msn.com/msnbot.htm)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="adidxbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; adidxbot/*; ?http://www.bing.com/bingbot.htm)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="adidxbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SkypeUriPreview 0.5">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*) SkypeUriPreview Preview/0.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*) SkypeUriPreview Preview/0.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*) SkypeUriPreview Preview/0.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*) SkypeUriPreview Preview/0.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) SkypeUriPreview Preview/0.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BingPreview 1.0b">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*) *BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 9.0;*Windows NT 6.1* Trident/5.0; *BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 10.0* Trident/6.0* BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 6.4* Trident/6.0* BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 6.3* Trident/6.0* BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 6.2* Trident/6.0* BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 6.1* Trident/6.0* BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*Trident/7.0*rv:11.0; BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*Trident/7.0*rv:11.0; BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*Trident/7.0*rv:11.0; BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*Trident/7.0*rv:11.0; BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*Trident/7.0*rv:11.0; BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BingPreview for Windows Phone 1.0b">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone 8.1* Trident/7.* IEMobile/*) like Gecko BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone* Trident/7.* IEMobile/*) like Gecko BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone 8.1* Trident/* IEMobile/*) like Gecko BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone* Trident/* IEMobile/*) like Gecko BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BingPreview for iOS 1.0b">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPad*CPU iPhone OS 7?* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPad*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPod*CPU iPhone OS 7?* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPod*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS 7?* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* BingPreview/1.0b*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BingPreview 1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*) *BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 9.0;*Windows NT 6.1* Trident/5.0; *BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 10.0* Trident/6.0*)*BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 6.4* Trident/6.0*)*BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 6.3* Trident/6.0*)*BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 6.2* Trident/6.0*)*BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 6.1* Trident/6.0*)*BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*Trident/7.0*rv:11.0; BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*Trident/7.0*rv:11.0; BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*Trident/7.0*rv:11.0; BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*Trident/7.0*rv:11.0; BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*Trident/7.0*rv:11.0; BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BingPreview for Windows Phone 1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone 8.1* Trident/7.* IEMobile/*) like Gecko BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone* Trident/7.* IEMobile/*) like Gecko BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone 8.1* Trident/* IEMobile/*) like Gecko BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone* Trident/* IEMobile/*) like Gecko BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BingPreview for iOS 1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPad*CPU iPhone OS 7?* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPad*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPod*CPU iPhone OS 7?* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPod*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS 7?* like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS * like Mac OS X*)*applewebkit*(*khtml*like*gecko*)*Version/*Safari/* BingPreview/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BingPreview Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*) applewebkit* (*khtml*like*gecko*) *BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 9.0;*Windows NT 6.1* Trident/5.0; *BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 10.0* Trident/6.0*)*BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 6.4* Trident/6.0*)*BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 6.3* Trident/6.0*)*BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 6.2* Trident/6.0*)*BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; MSIE 10.0;*Windows NT 6.1* Trident/6.0*)*BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 10.0*Trident/7.0*rv:11.0; BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.4*Trident/7.0*rv:11.0; BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.3*Trident/7.0*rv:11.0; BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.2*Trident/7.0*rv:11.0; BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows NT 6.1*Trident/7.0*rv:11.0; BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (*) applewebkit* (*khtml*like*gecko*) BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BingPreview for Windows Phone Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone 8.1* Trident/7.* IEMobile/*) like Gecko BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone* Trident/7.* IEMobile/*) like Gecko BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone 8.1* Trident/* IEMobile/*) like Gecko BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Windows Phone* Trident/* IEMobile/*) like Gecko BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="BingPreview for iOS Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPad*CPU iPhone OS 7?* like Mac OS X*)*applewebkit*(*KHTML? like Gecko*)*Version/*Safari/* BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPad*CPU iPhone OS * like Mac OS X*)*applewebkit*(*KHTML? like Gecko*)*Version/*Safari/* BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPod*CPU iPhone OS 7?* like Mac OS X*)*applewebkit*(*KHTML? like Gecko*)*Version/*Safari/* BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(iPod*CPU iPhone OS * like Mac OS X*)*applewebkit*(*KHTML? like Gecko*)*Version/*Safari/* BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS 7?* like Mac OS X*)*applewebkit*(*KHTML? like Gecko*)*Version/*Safari/* BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0*(*CPU iPhone OS * like Mac OS X*)*applewebkit*(*KHTML? like Gecko*)*Version/*Safari/* BingPreview/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MSN Mobile Proxy">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/* (compatible*; MSIE *; Windows*; MSN Mobile Proxy)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows Phone Search">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Windows Phone Search (*Windows Phone*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="yoozBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; yoozBot-2.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; yoozBot-*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="VoilaBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*VoilaBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Orangebot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Orangebot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Orangebot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YaCy">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="yacybot (sciencenet?global; *Linux*http://yacy.net/bot.html">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="yacybot (freeworld?global; i386 Linux *http://yacy.net/bot.html">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="yacybot (freeworld?global; *Linux*http://yacy.net/bot.html">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="yacybot (freeworld?global; x86 Windows 7 6.1;*http://yacy.net/bot.html">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="yacybot (freeworld?global; x86 Windows XP 5.1;*http://yacy.net/bot.html">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="yacybot (webportal-global; x86 Windows XP 5.1;*http://yacy.net/bot.html">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="yacybot (webportal-global; *Windows*http://yacy.net/bot.html">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="yacybot (i386 linux *http://yacy.net/bot.html">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="yacybot (* linux *http://yacy.net/bot.html">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="yacybot *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Y!J; for robot study*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="KDDI-HI21 UP.Browser/6.0* (compatible; Y!J-SRD/1.0; http://help.yahoo.co.jp/help/jp/search/indexing/indexing-27.html)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; del.icio.us-thumbnails/*KHTML/* (like Gecko)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; del.icio.us-thumbnails/*; *) KHTML/* (like Gecko)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YahooCacheSystem*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Yahoo-MMCrawler/4.*; mailto:vertical-crawl-support@yahoo-inc.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Yahoo-MMCrawler/*; mailto:vertical-crawl-support@yahoo-inc.com)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (Yahoo-Test/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (YahooYSMcm*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="mp3Spider cn-search-devel at yahoo-inc dot com">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="My Browser">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Scooter/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Scooter*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!OASIS*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo Mindset">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo Pipes*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo! Mindset">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo-Blogs*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo-MMAudVid*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo-MMCrawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YahooExternalCache">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YahooVideoSearch*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YahooYSMcm*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YRL_ODP_CRAWLER">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo:LinkExpander:Slingstone">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Yahoo Ad monitoring*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*OS * like Mac OS X) * (compatible; Yahoo Ad monitoring; https://help.yahoo.com*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-AGENT">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BMC/* (Y!J-AGENT))">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BMF/* (Y!J-AGENT))">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; BMI/* (Y!J-AGENT; 1.0))">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J SearchMonkey*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRE*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRG/GSC*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRI*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRO/YFSJ*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRP/YFSBJ*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRQ/DLCK*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BSC*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-DSC*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-NSC*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-PSC*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-SRD*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-VSC/ViSe*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRT*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRU/VSIDX*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-IER*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRW*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-RIE*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-APP*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRH*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRS*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRD/YSML*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BUZZ*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-ASR*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-SSF*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-MMP/dscv*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRJ/YATS*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRL/YATSS*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRM/YATSD*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRN/YATSA*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRY/YATSH*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRZ/YATSHA*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-DLC*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-BRX/DLCK*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Y!J-PUF*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo AdCrawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Yahoo!-AdCrawler;*http://help.yahoo.com/yahoo_adcrawler)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo Link Preview">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Yahoo Link Preview*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo RobotStudy">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/4.0 (compatible; Yahoo Japan; for robot study; kasugiya)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo Search Monkey">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Yahoo! SearchMonkey*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Yahoo Verifier">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Yahoo! Verifier/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YahooSeeker">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YahooSeeker/CafeKelsa* (compatible; Konqueror/*; FreeBSD*(*khtml*like*gecko*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YahooSeeker/CafeKelsa*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YahooSeeker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YahooSeeker-Mobile">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YahooSeeker/M1A1-R2D2*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YahooFeedSeeker">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YahooFeedSeeker*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Seznam Screenshot Generator">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Seznam screenshot-generator 2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Seznam screenshot-generator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 PhantomJS (compatible; Seznam screenshot-generator 2.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 PhantomJS (compatible; Seznam screenshot-generator*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SeznamBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SeznamBot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SeznamBot/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SeznamBot/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SeznamBot/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SeznamBot/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SeznamBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SeznamBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SklikBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SklikBot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SklikBot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SklikBot/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SklikBot/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SklikBot/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SklikBot/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SklikBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SklikBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Sindice Fetcher">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; sindice-fetcher/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; sindice-fetcher/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Research Projects">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AcadiaUniversityWebCensusClient">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Amico Alpha * (*) Gecko* AmicoAlpha/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="annotate_google; http://ponderer.org/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="CMS crawler (?http://buytaert.net/crawler/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="e-SocietyRobot(http://www.yama.info.waseda.ac.jp/~yamana/es/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Forschungsportal/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Gulper Web *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HooWWWer/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="inetbot/* (?http://www.inetbot.com/bot.html)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="JUST-CRAWLER(*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Lachesis">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; nextthing.org/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Theophrastus/*)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Webscan v0.*; +http://otc.dyndns.org/webscan/)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="MQbot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="OutfoxBot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="polybot?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Shim?Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Taiga web spider">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Theme Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UofTDB_experiment* (leehyun@cs.toronto.edu)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="USyd-NLP-Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="woriobot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; woriobot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; zitebot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="wwwster/* (Beta, mailto:gue@cis.uni-muenchen.de)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Zao-Crawler">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Zao/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Amico Alpha * (*Gecko* AmicoAlpha/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Cloud mapping experiment. Contact research@pdrlabs.net">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Finderlein Research Crawler 1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Finderlein Research Crawler*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="SpellCheck Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SpellCheck Bot 0.9*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; SpellCheck Bot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="AvsDeviceSdk/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Webmaster Tips Bot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; wmtips.com/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; wmtips.com/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WI Job Roboter">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;WI Job Roboter Spider Version 3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;WI Job Roboter Spider*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Xerka WebBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Xerka WebBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YandexBlog">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; YandexBlogs/*; robot)">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="YandexBlog/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Seznam Email Proxy">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 SeznamEmailProxy/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 SeznamEmailProxy/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 SeznamEmailProxy/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jakarta Commons HttpClient">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Wepbot/*(*)*Jakarta Commons-HttpClient/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jakarta Commons-HttpClient/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jakarta Commons-HttpClient/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Jakarta Commons-HttpClient/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="GO HttpClient">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Go 1.1 package http">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Go * package http">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Go-http-client/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Go-http-client/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Go-http-client/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTTP_Request2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTTP_Request2/0.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="HTTP_Request2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tumblr Agent 14.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tumblr/14.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tumblr Agent Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tumblr/* CFNetwork/* Darwin/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Tumblr/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.12 for iOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?12.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?12.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?12.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?12.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?11.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?11.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?11.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?11.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?11.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?10.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?10.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?10.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?10.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?9.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (* iOS?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.12 for Android">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12* (*Android*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.12">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.12*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.11 for iOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?12.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?12.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?12.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?12.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?11.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?11.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?11.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?11.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?11.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?10.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?10.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?10.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?10.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?9.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (* iOS?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.11 for Android">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11* (*Android*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.11">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.11*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.7 for iOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?12.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?12.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?12.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?12.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?11.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?11.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?11.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?11.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?11.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?10.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?10.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?10.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?10.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?9.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (* iOS?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.7 for Android">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7* (*Android*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.7">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.7*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.4 for iOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?12.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?12.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?12.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?12.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?11.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?11.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?11.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?11.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?11.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?10.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?10.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?10.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?10.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?9.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (* iOS?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.4 for Android">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4* (*Android*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.4">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.3 for iOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?12.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?12.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?12.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?12.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?11.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?11.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?11.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?11.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?11.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?10.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?10.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?10.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?10.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?9.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (* iOS?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.3 for Android">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3* (*Android*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.3">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.2 for iOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?12.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?12.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?12.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?12.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?11.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?11.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?11.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?11.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?11.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?10.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?10.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?10.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?10.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?9.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (* iOS?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.2 for Android">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2* (*Android*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 10.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/10.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 9.45 for iOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?12.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?12.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?12.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?12.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?11.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?11.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?11.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?11.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?11.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?10.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?10.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?10.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?10.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?9.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (* iOS?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 9.45 for Android">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45* (*Android*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 9.45">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.45*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 9.44 for iOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?12.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?12.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?12.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?12.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?11.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?11.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?11.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?11.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?11.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?10.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?10.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?10.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?10.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?9.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (* iOS?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 9.44 for Android">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44* (*Android*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 9.44">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.44*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 9.39 for iOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?12.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?12.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?12.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?12.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?11.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?11.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?11.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?11.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?11.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?10.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?10.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?10.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?10.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?9.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (* iOS?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 9.39 for Android">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39* (*Android*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 9.39">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.39*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 9.34 for iOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?12.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?12.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?12.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?12.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?11.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?11.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?11.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?11.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?11.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?10.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?10.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?10.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?10.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?9.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (* iOS?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 9.34 for Android">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34* (*Android*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent 9.34">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/9.34*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent Generic for iOS">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?12.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?12.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?12.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?12.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?11.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?11.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?11.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?11.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?11.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?10.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?10.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?10.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?10.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?9.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (* iOS?*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent Generic for Android">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/* (*Android*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat Agent Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Snapchat/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WhatsApp Bot 2.18">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WhatsApp/2.18*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WhatsApp Bot 2.17">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WhatsApp/2.17*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WhatsApp Bot 2.12">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WhatsApp/2.12*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 4.3">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/4.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/4.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/4.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 4.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/4.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/4.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/4.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 4.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/4.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/4.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/4.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 4.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/4.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 3.6">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/3.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/3.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/3.6*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 3.5">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/3.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/3.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/3.5*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 3.4">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/3.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/3.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/3.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 3.3">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/3.3*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 3.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/3.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 3.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/3.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 3.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/3.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 2.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/2.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 1.8">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/1.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/1.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/1.8*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 1.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/1.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 1.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/1.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 0.2">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/0.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/0.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/0.2*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot 0.1">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/0.1*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WhatsApp Bot Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="WhatsApp/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest Bot Generic">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Pinterest/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; Pinterestbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*linux*android?6.0*nexus 5x build/*) applewebkit* (*khtml*like*gecko*) Chrome/* Safari/* (compatible; Pinterestbot/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Typhoeus">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Typhoeus*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="cURL PHP">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Curl/PHP 5.4*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Curl/PHP*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PetalBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; PetalBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*Android*) AppleWebKit* (KHTML*like*Gecko) Mobile Safari* (compatible; PetalBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible;PetalBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PAD-bot 9.0">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; PAD-bot/9.0*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="PagePeeker Screenshot Maker">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) applewebkit* (*khtml*like*gecko*) domain.com Safari/*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UnisterBot">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (compatible; UnisterBot*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UnisterBot *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="unister-bot *">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UnisterTesting">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="unister-test-zabbix*">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="unister-test-acceptance">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Mozilla/5.0 (*Linux*) Gecko* Firefox/* Iceweasel/* unister-test-acceptance">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="UnisterPortale">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="www.geld.de">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="www.versicherungen.de">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="www.kredit.de">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="www.auto.de">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="auto.de">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="insurance.preisvergleich.de">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
<browscapitem name="Core_Class_HttpClient_Cached">
|
|
<item name="Crawler" value="true" />
|
|
</browscapitem>
|
|
</browsercapitems> |