ABAKUS

Ihr Partner in Sachen SEO und Online Marketing
https://archiv.abakus-internet-marketing.de/foren/

Welche Robots doch nicht aussperren ?

https://archiv.abakus-internet-marketing.de/foren/viewtopic.php?f=52&t=4594

Seite 1 von 1

Welche Robots doch nicht aussperren ?

Verfasst: 12.04.2004, 21:44
von Downer
Hier mal eine Liste der Bots, wo ich mir nicht sicher bin ob ich sie alle sperren soll. Welche von denen sollte man nicht sperren, sind von wichtigen Suchmaschinen?

Code: Alles auswählen

RewriteCond %{HTTP_USER_AGENT} ^.*antibot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*appie.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*architext.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bjaaland.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*digout4u.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*echo.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*fast-webcrawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ferret.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gulliver.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*harvest.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*htdig.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ia_archiver.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jeeves.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jennybot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*linkwalker.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mercator.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*moget.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*muscatferret.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*myweb.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*netcraft.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*nomad.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*petersnews.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*scooter.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*unlost_web_crawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*voila.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*voyager.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webbase.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*weblayers.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wisenutbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*acme.spider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ahoythehomepagefinder.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*alkaline.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*arachnophilia.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*aretha.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ariadne.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*arks.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*aspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*atn.txt.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*atomz.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*auresys.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*backrub.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bigbrother.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*blindekuh.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bloodhound.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*brightnet.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cactvschemistryspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cassandra.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cgireader.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*checkbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*churl.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cmc.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*collective.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*combine.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*conceptbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*coolbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*core.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cosmos.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cruiser.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cusco.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cyberspyder.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*deweb.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*dienstspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*digger.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*diibot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*directhit.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*dnabot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*download_express.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*dragonbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*dwcp.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*e-collector.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ebiness.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*eit.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*elfinbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*emacs.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*emcspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*esther.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*evliyacelebi.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*nzexplorer.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*fdse.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*felix.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*fetchrover.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*fido.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*finnish.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*fireball.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*fouineur.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*francoroute.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*freecrawl.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*funnelweb.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gama.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gazz.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gcreep.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*getbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*geturl.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*golem.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*grapnel.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*griffon.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gromit.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*hambot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*havindex.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*hometown.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*htmlgobble.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*hyperdecontextualizer.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*iajabot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ibm.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*iconoclast.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ilse.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*imagelock.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*incywincy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*informant.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*infoseek.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*infoseeksidewinder.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*infospider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*inspectorwww.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*intelliagent.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*irobot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*iron33.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*israelisearch.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*javabee.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jcrawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jobo.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jobot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*joebot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jubii.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jumpstation.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*katipo.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*kdd.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*kilroy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ko_yappo_robot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*labelgrabber.txt.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*larbin.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*legs.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*linkidator.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*linkscan.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*lockon.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*logo_gif.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*macworm.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*magpie.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*marvin.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mattie.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mediafox.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*merzscope.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*meshexplorer.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mindcrawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*momspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*monster.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*motor.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mwdsearch.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*netcarta.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*netmechanic.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*netscoop.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*newscan-online.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*nhse.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*northstar.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*occam.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*octopus.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*openfind.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*orb_search.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*packrat.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pageboy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*parasite.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*patric.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pegasus.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*perignator.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*perlcrawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*phantom.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*piltdownman.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pimptrain.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pioneer.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pitkow.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pjspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pka.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*plumtreewebaccessor.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*poppi.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*portalb.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*puu.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*python.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*raven.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*rbse.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*resumerobot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*rhcs.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*roadrunner.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*robbie.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*robi.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*robofox.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*robozilla.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*roverbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*rules.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*safetynetrobot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*search_au.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*searchprocess.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*senrigan.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*sgscout.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*shaggy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*shaihulud.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*sift.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*simbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*site-valet.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*sitegrabber.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*sitetech.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*slcrawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*smartspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*snooper.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*solbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spanner.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*speedy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spider_monkey.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spiderbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spiderline.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spiderman.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spiderview.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spry.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ssearcher.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*suke.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*suntek.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*sven.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*tach_bw.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*tarantula.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*tarspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*techbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*templeton.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*teoma_agent1.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*titin.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*titan.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*tkwww.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*tlspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ucsd.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*udmsearch.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*urlck.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*valkyrie.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*victoria.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*visionsearch.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*vwbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*w3index.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*w3m2.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wallpaper.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wanderer.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wapspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webbandit.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webcatcher.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webcopy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webfetcher.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webfoot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*weblinker.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webmirror.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webmoose.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webquest.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webreader.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*websnarf.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webvac.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webwalk.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webwalker.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webwatch.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*whatuseek.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*whowhere.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wired-digital.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wmir.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wolp.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wombat.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*worm.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wwwc.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wz101.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*xget.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*awbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bobby.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*boris.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bumblebee.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cscrawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*daviesbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ezresult.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gigabot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gnodspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*internetseer.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*justview.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*linkbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*linkchecker.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*nederland.zoek.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*perman.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pompos.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pooodle.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*redalert.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*shoutcast.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*slysearch.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ultraseek.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webcompass.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*yandex.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*robot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*psbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*crawl.*$ [OR]

Verfasst:
von

Verfasst: 12.04.2004, 22:34
von Halli
:oops: Mann, sind das viele ...
Etliche von denen kenne ich auch nicht ....

Folgende würde ich nicht aussperren:
ia_archiver ist vom Internet-Archiv
jeeves von der Suchmaschine ask-jeeves
atomz ist eine klasse (gratis) Volltextsuche für die eigene Homepage
fireball kennst Du nicht?
ibm ist wirklich von denen (für Testzwecke)
infoseek auch eine bekannte Suchmaschine
monster ein großes Jobportal
netmechanic hat viele hilfreiche Tools
teoma_agent1 ist wohl von wem?
internetseer überprüft die Erreichbarkeit Deines Servers
linkchecker :idea:

Verfasst: 13.04.2004, 09:43
von viggen
Scooter ist von Altavista (gibt es angeblich ja nicht mehr, crawlt aber trotztem meine Seiten als wenn nichts wer)
Fast Webcrawler ist von Fast und wird z.B von Alltheweb verwendet (obwohl das gleiche gilt wie für AltaVista)

mfg
viggen

Verfasst:
von

Verfasst: 14.04.2004, 11:48
von katerina
Der dritte von unten (*bot.*) - wird damit nicht auch der Googlebot ausgesperrt?

Gruß
K.

Verfasst: 15.04.2004, 08:52
von ErwinRommel
Wieso willst du die Bots überhaupt aussperren?

Verfasst: 17.04.2004, 00:42
von Hulot
ErwinRommel hat geschrieben:Wieso willst du die Bots überhaupt aussperren?
Hallo,
er sollte besser die aufzählen, die auf seine Seite dürfen. :lol:

Gruß Hulot

Verfasst: 24.04.2004, 20:45
von Downer
Ich habe die LIste mal optimiert, hier der gesamte code:

Code: Alles auswählen

RewriteCond %{HTTP_USER_AGENT} ^.*bBot.*$   [OR]
RewriteCond %{HTTP_USER_AGENT} ^EmailSiphon   [OR]
RewriteCond %{HTTP_USER_AGENT} ^EmailWolf     [OR]
RewriteCond %{HTTP_USER_AGENT} ^ExtractorPro  [OR]
RewriteCond %{HTTP_USER_AGENT} ^CherryPicker  [OR]
RewriteCond %{HTTP_USER_AGENT} ^NICErsPRO     [OR]
RewriteCond %{HTTP_USER_AGENT} ^EmailCollector
RewriteRule ^.*$ emailsforyou.php  [L]

# Unerwünschte Robots Umleitung auf Fehlerseite 403 Forbidden
RewriteCond %{HTTP_USER_AGENT} ^.*BackWeb.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Bandit.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*BlackWidow.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Buddy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Collector.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Copier.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Downloader.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*FileHound.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Grabber.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*HTTPClient.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*HTTrack.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Mirror.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Offline.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Recorder.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Sauger.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Snagger.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Snake.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Stripper.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Sucker.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*SuperBot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Surfbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Teleport.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebReaper.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebSpider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*WebZIP.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*Wget.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^Bot\ mailto:craftbot@yahoo.com [OR]
RewriteCond %{HTTP_USER_AGENT} ^ChinaClaw [OR]
RewriteCond %{HTTP_USER_AGENT} ^DISCo [OR]
RewriteCond %{HTTP_USER_AGENT} ^Download\ Demon [OR]
RewriteCond %{HTTP_USER_AGENT} ^eCatch [OR]
RewriteCond %{HTTP_USER_AGENT} ^EirGrabber [OR]
RewriteCond %{HTTP_USER_AGENT} ^Express\ WebPictures [OR]
RewriteCond %{HTTP_USER_AGENT} ^EyeNetIE [OR]
RewriteCond %{HTTP_USER_AGENT} ^FlashGet [OR]
RewriteCond %{HTTP_USER_AGENT} ^GetRight [OR]
RewriteCond %{HTTP_USER_AGENT} ^Go!Zilla [OR]
RewriteCond %{HTTP_USER_AGENT} ^Go-Ahead-Got-It [OR]
RewriteCond %{HTTP_USER_AGENT} ^GrabNet [OR]
RewriteCond %{HTTP_USER_AGENT} ^Grafula [OR]
RewriteCond %{HTTP_USER_AGENT} ^JOC\ Web\ Spider [OR]
RewriteCond %{HTTP_USER_AGENT} ^larbin [OR]
RewriteCond %{HTTP_USER_AGENT} ^LeechFTP [OR]
RewriteCond %{HTTP_USER_AGENT} ^Mass\ Downloader [OR]
RewriteCond %{HTTP_USER_AGENT} ^NetAnts [OR]
RewriteCond %{HTTP_USER_AGENT} ^NetSpider [OR]
RewriteCond %{HTTP_USER_AGENT} ^Net\ Vampire [OR]
RewriteCond %{HTTP_USER_AGENT} ^NetZIP [OR]
RewriteCond %{HTTP_USER_AGENT} ^Octopus [OR]
RewriteCond %{HTTP_USER_AGENT} ^WebStripper [OR]
RewriteCond %{HTTP_USER_AGENT} ^WebWhacker [OR]
RewriteCond %{HTTP_USER_AGENT} ^Widow [OR]
RewriteCond %{HTTP_USER_AGENT} ^Xaldon\ WebSpider [OR]
RewriteCond %{HTTP_USER_AGENT} ^Zeus
RewriteRule ^.*$ - [F]
Ich möchte eigentlich nur die größten Spider (Google) zulassen und vor allem Mailbots aussperren. Belastet nur unnötig den Server.
Alle Zeiten sind UTC+01:00
Seite 1 von 1

Powered by phpBB® Forum Software © phpBB Limited

Deutsche Übersetzung durch phpBB.de