# 
# robots.txt for oldenbourg-klick.de
# Rev. 2009-10-15
#


# bots not feeding any known search engine:

User-agent: DOC
Disallow: /

User-agent: UbiCrawler
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: Zao
Disallow: /


# bots forcing problems, requiring to much bandwidth
# or trying to copy the entire site:

User-agent: Download Ninja
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: k2spider
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: linko
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: MS Search 4.0 Robot
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: wget
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: ZyBORG
Disallow: /


# crawl delay for Yahoo and Bing:

User-agent: Slurp
Crawl-delay: 10

User-agent: msn
Crawl-delay: 10

User-agent: msnbot
Crawl-delay: 10


# general exclusions:
User-agent: *
Disallow: /backend/
Disallow: /download/artikel/
Disallow: /extensions/backend/
Disallow: /fileadmin/toolbox/
Disallow: /mein-konto/
Disallow: /typo3/
Disallow: /typo3_src/
Disallow: /typo3conf/
Disallow: /typo3temp/
Disallow: /uploads/
Disallow: /zeitschriften/deutschmagazin/mein-konto/
Disallow: /zeitschriften/foerdermagazin/mein-konto/
Disallow: /zeitschriften/grundschulmagazin-englisch/mein-konto/
Disallow: /zeitschriften/grundschulmagazin/mein-konto/
Disallow: /zeitschriften/grundschulunterricht/mein-konto/
Disallow: /zeitschriften/htw-praxis/mein-konto/
Disallow: /zeitschriften/kleingross/mein-konto/
Disallow: /zeitschriften/praxis-fremdsprachenunterricht/mein-konto/
Disallow: /zeitschriften/schul-management/mein-konto/
Disallow: /zeitschriften/schulmagazin-5-10/mein-konto/