Du befindest Dich im Archiv vom ABAKUS Online Marketing Forum. Hier kannst Du Dich für das Forum mit den aktuellen Beiträgen registrieren.

Welche Robots doch nicht aussperren ?

Alles zum Thema: Robots, Spider, Logfile-Auswertung und Reports
Neues Thema Antworten

 Xaldon WebSpider 2.7.b6

bravobrian www.bravobrian.it

 
Downer
PostRank 1
PostRank 1
Beiträge: 19
Registriert: 11.01.2003, 03:23

Beitrag von Downer » 12.04.2004, 21:44

Hier mal eine Liste der Bots, wo ich mir nicht sicher bin ob ich sie alle sperren soll. Welche von denen sollte man nicht sperren, sind von wichtigen Suchmaschinen?

Code: Alles auswählen

RewriteCond %{HTTP_USER_AGENT} ^.*antibot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*appie.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*architext.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bjaaland.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*digout4u.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*echo.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*fast-webcrawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ferret.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gulliver.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*harvest.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*htdig.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ia_archiver.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jeeves.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jennybot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*linkwalker.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mercator.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*moget.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*muscatferret.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*myweb.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*netcraft.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*nomad.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*petersnews.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*scooter.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*unlost_web_crawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*voila.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*voyager.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webbase.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*weblayers.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wisenutbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*acme.spider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ahoythehomepagefinder.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*alkaline.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*arachnophilia.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*aretha.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ariadne.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*arks.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*aspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*atn.txt.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*atomz.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*auresys.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*backrub.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bigbrother.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*blindekuh.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bloodhound.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*brightnet.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cactvschemistryspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cassandra.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cgireader.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*checkbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*churl.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cmc.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*collective.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*combine.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*conceptbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*coolbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*core.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cosmos.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cruiser.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cusco.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cyberspyder.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*deweb.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*dienstspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*digger.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*diibot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*directhit.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*dnabot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*download_express.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*dragonbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*dwcp.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*e-collector.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ebiness.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*eit.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*elfinbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*emacs.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*emcspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*esther.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*evliyacelebi.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*nzexplorer.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*fdse.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*felix.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*fetchrover.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*fido.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*finnish.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*fireball.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*fouineur.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*francoroute.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*freecrawl.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*funnelweb.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gama.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gazz.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gcreep.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*getbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*geturl.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*golem.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*grapnel.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*griffon.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gromit.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*hambot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*havindex.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*hometown.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*htmlgobble.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*hyperdecontextualizer.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*iajabot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ibm.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*iconoclast.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ilse.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*imagelock.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*incywincy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*informant.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*infoseek.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*infoseeksidewinder.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*infospider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*inspectorwww.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*intelliagent.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*irobot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*iron33.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*israelisearch.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*javabee.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jcrawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jobo.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jobot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*joebot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jubii.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*jumpstation.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*katipo.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*kdd.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*kilroy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ko_yappo_robot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*labelgrabber.txt.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*larbin.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*legs.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*linkidator.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*linkscan.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*lockon.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*logo_gif.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*macworm.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*magpie.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*marvin.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mattie.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mediafox.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*merzscope.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*meshexplorer.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mindcrawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*momspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*monster.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*motor.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*mwdsearch.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*netcarta.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*netmechanic.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*netscoop.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*newscan-online.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*nhse.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*northstar.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*occam.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*octopus.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*openfind.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*orb_search.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*packrat.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pageboy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*parasite.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*patric.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pegasus.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*perignator.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*perlcrawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*phantom.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*piltdownman.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pimptrain.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pioneer.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pitkow.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pjspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pka.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*plumtreewebaccessor.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*poppi.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*portalb.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*puu.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*python.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*raven.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*rbse.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*resumerobot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*rhcs.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*roadrunner.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*robbie.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*robi.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*robofox.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*robozilla.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*roverbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*rules.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*safetynetrobot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*search_au.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*searchprocess.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*senrigan.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*sgscout.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*shaggy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*shaihulud.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*sift.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*simbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*site-valet.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*sitegrabber.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*sitetech.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*slcrawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*smartspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*snooper.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*solbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spanner.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*speedy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spider_monkey.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spiderbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spiderline.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spiderman.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spiderview.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*spry.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ssearcher.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*suke.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*suntek.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*sven.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*tach_bw.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*tarantula.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*tarspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*techbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*templeton.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*teoma_agent1.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*titin.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*titan.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*tkwww.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*tlspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ucsd.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*udmsearch.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*urlck.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*valkyrie.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*victoria.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*visionsearch.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*vwbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*w3index.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*w3m2.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wallpaper.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wanderer.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wapspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webbandit.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webcatcher.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webcopy.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webfetcher.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webfoot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*weblinker.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webmirror.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webmoose.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webquest.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webreader.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*websnarf.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webvac.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webwalk.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webwalker.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webwatch.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*whatuseek.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*whowhere.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wired-digital.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wmir.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wolp.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wombat.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*worm.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wwwc.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*wz101.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*xget.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*awbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bobby.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*boris.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bumblebee.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*cscrawler.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*daviesbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ezresult.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gigabot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*gnodspider.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*internetseer.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*justview.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*linkbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*linkchecker.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*nederland.zoek.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*perman.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pompos.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*pooodle.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*redalert.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*shoutcast.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*slysearch.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*ultraseek.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*webcompass.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*yandex.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*robot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*bot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*psbot.*$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^.*crawl.*$ [OR]
Zuletzt geändert von Downer am 21.05.2004, 12:10, insgesamt 1-mal geändert.
Nach oben
Anzeige von ABAKUS

von Anzeige von ABAKUS »


Hochwertiger Linkaufbau bei ABAKUS:
  • Google-konformer Linkaufbau
  • nachhaltiges Ranking
  • Linkbuilding Angebote zu fairen Preisen
  • internationale Backlinks
Wir bieten Beratung und Umsetzung.
Jetzt anfragen: 0511 / 300325-0
Nach oben
Halli
PostRank 7
PostRank 7
Beiträge: 584
Registriert: 21.08.2003, 00:40
Wohnort: Hannover

Beitrag von Halli » 12.04.2004, 22:34

:oops: Mann, sind das viele ...
Etliche von denen kenne ich auch nicht ....

Folgende würde ich nicht aussperren:
ia_archiver ist vom Internet-Archiv
jeeves von der Suchmaschine ask-jeeves
atomz ist eine klasse (gratis) Volltextsuche für die eigene Homepage
fireball kennst Du nicht?
ibm ist wirklich von denen (für Testzwecke)
infoseek auch eine bekannte Suchmaschine
monster ein großes Jobportal
netmechanic hat viele hilfreiche Tools
teoma_agent1 ist wohl von wem?
internetseer überprüft die Erreichbarkeit Deines Servers
linkchecker :idea:
wer besoffen noch geradeaus fahren kann ist bald reif fürs www.mpu-forum.de
Nach oben
viggen
PostRank 10
PostRank 10
Beiträge: 6249
Registriert: 12.05.2003, 14:44

Beitrag von viggen » 13.04.2004, 09:43

Scooter ist von Altavista (gibt es angeblich ja nicht mehr, crawlt aber trotztem meine Seiten als wenn nichts wer)
Fast Webcrawler ist von Fast und wird z.B von Alltheweb verwendet (obwohl das gleiche gilt wie für AltaVista)

mfg
viggen
Nach oben
Anzeige von ABAKUS

von Anzeige von ABAKUS »

SEO Consulting bei ABAKUS Internet Marketing
Erfahrung seit 2002
  • persönliche Betreuung
  • individuelle Beratung
  • kompetente Umsetzung

Jetzt anfragen: 0511 / 300325-0.

Nach oben
katerina
PostRank 4
PostRank 4
Beiträge: 155
Registriert: 08.04.2004, 08:37
Wohnort: Nürnberg

Beitrag von katerina » 14.04.2004, 11:48

Der dritte von unten (*bot.*) - wird damit nicht auch der Googlebot ausgesperrt?

Gruß
K.
Nach oben
ErwinRommel
PostRank 10
PostRank 10
Beiträge: 3868
Registriert: 14.09.2003, 20:46

Beitrag von ErwinRommel » 15.04.2004, 08:52

Wieso willst du die Bots überhaupt aussperren?
Social Media Beratung nicht nur für SEOs :)
<br>
<br>Onlineshops erfolgreich aufbauen
<br>
<br>Gute Links im Angebot.
Nach oben
Hulot
PostRank 2
PostRank 2
Beiträge: 55
Registriert: 07.03.2004, 02:43

Beitrag von Hulot » 17.04.2004, 00:42

ErwinRommel hat geschrieben:Wieso willst du die Bots überhaupt aussperren?
Hallo,
er sollte besser die aufzählen, die auf seine Seite dürfen. :lol:

Gruß Hulot
Nach oben
Downer
PostRank 1
PostRank 1
Beiträge: 19
Registriert: 11.01.2003, 03:23

Beitrag von Downer » 24.04.2004, 20:45

Ich habe die LIste mal optimiert, hier der gesamte code:

Code: Alles auswählen

RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*bBot.*$   &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^EmailSiphon   &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^EmailWolf     &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^ExtractorPro  &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^CherryPicker  &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^NICErsPRO     &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^EmailCollector
RewriteRule ^.*$ emailsforyou.php  &#91;L&#93;

# Unerwünschte Robots Umleitung auf Fehlerseite 403 Forbidden
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*BackWeb.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Bandit.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*BlackWidow.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Buddy.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Collector.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Copier.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Downloader.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*FileHound.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Grabber.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*HTTPClient.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*HTTrack.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Mirror.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Offline.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Recorder.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Sauger.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Snagger.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Snake.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Stripper.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Sucker.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*SuperBot.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Surfbot.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Teleport.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*WebReaper.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*WebSpider.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*WebZIP.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^.*Wget.*$ &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^Bot\ mailto&#58;craftbot@yahoo.com &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^ChinaClaw &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^DISCo &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^Download\ Demon &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^eCatch &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^EirGrabber &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^Express\ WebPictures &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^EyeNetIE &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^FlashGet &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^GetRight &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^Go!Zilla &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^Go-Ahead-Got-It &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^GrabNet &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^Grafula &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^JOC\ Web\ Spider &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^larbin &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^LeechFTP &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^Mass\ Downloader &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^NetAnts &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^NetSpider &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^Net\ Vampire &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^NetZIP &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^Octopus &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^WebStripper &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^WebWhacker &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^Widow &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^Xaldon\ WebSpider &#91;OR&#93;
RewriteCond %&#123;HTTP_USER_AGENT&#125; ^Zeus
RewriteRule ^.*$ - &#91;F&#93;
Ich möchte eigentlich nur die größten Spider (Google) zulassen und vor allem Mailbots aussperren. Belastet nur unnötig den Server.
Nach oben
Antworten

Zurück zu „Spiders / Robots Themen“

  • Vergleichbare Themen
    Antworten
    Zugriffe
    Letzter Beitrag