# robots.txt for http://acornitgroup.com/ and related sites
#
# Please note: There are a lot of pages on this site, and there are some misbehaved spiders out there
# that go _way_ too fast. If you're irresponsible, your access to the site may be blocked.
# Persistent unauthorized bots may be banned by host and/or IP.

# Inktomi's "Slurp" can read a minimum delay between hits; if your
# bot supports such a thing using the 'Crawl-delay' or another
# instruction, please let us know.

## *At least* a 1 second please, preferably more :)
Crawl-delay: 1

# Advertising-related bots:
User-agent: Mediapartners-Google*
Disallow: /
User-agent: CJNetworkQuality
Disallow: /

# Image-harvesting bots:
User-agent: Googlebot-Image
Disallow: /
User-agent: IconSurf
Disallow: /

# Wikipedia work bots:
User-agent: IsraBot
Disallow:
User-agent: Orthogaffe
Disallow:

# Crawlers that are kind enough to obey, but which we'd rather not have unless they're feeding search engines.
User-agent: UbiCrawler
Disallow: /
User-agent: DOC
Disallow: /
User-agent: Zao
Disallow: /

# Some bots are known to be trouble, particularly those designed to copy entire sites. Please obey robots.txt.
User-agent: Download Ninja
Disallow: /
User-agent: Fetch
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: larbin
Disallow: /
User-agent: libwww
Disallow: /
User-agent: linko
Disallow: /
User-agent: Microsoft.URL.Control
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: sitecheck.internetseer.com
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: Xenu
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: ZyBORG
Disallow: /

# Sorry, wget in its recursive mode is a frequent problem. Please read the man page and use it properly; there is a
# --wait option you can use to set the delay between hits, for instance.
User-agent: wget
Disallow: /

# The 'grub' distributed client has been *very* poorly behaved.
User-agent: grub-client
Disallow: /

# Doesn't follow robots.txt anyway, but...
User-agent: k2spider
Disallow: /

# Hits many times per second, not acceptable (http://www.nameprotect.com/botinfo.html)
User-agent: NPBot
Disallow: /

# A capture bot, downloads gazillions of pages with no public benefit (http://www.webreaper.net/)
User-agent: WebReaper
Disallow: /

# Don't allow the wayback-maschine to index user-pages
#User-agent: ia_archiver
#Disallow: /wiki/User

# Friendly, low-speed bots are welcome viewing article pages, but not dynamically-generated pages please.
User-agent: *				# Apply to all robots
Disallow: /w/				# Dynamically-generated Wiki scripts (not the content)

# Static non-wiki folders and files (mostly older stuff):
Disallow: /css/				# Stylesheets
Disallow: /errors/			# Error pages
Disallow: /files/			# File library
Disallow: /hr/				# Human resources
Disallow: /icons/			# Shared icons
Disallow: /images/			# Images
Disallow: /public_notices/	# Public notices
Disallow: /trap/			# Badly-behaved bot trap

# Block namespaces we don't want spidered
Disallow: /wiki/Media
Disallow: /wiki/Image
Disallow: /wiki/Template
Disallow: /wiki/Staff
Disallow: /wiki/Forms

# Block public talk namespaces to avoid indexing spam comments
Disallow: /wiki/Talk
Disallow: /wiki/User_talk
Disallow: /wiki/User%20talk
Disallow: /wiki/AcornWiki_talk
Disallow: /wiki/AcornWiki%20talk
Disallow: /wiki/Image_talk
Disallow: /wiki/Image%20talk
Disallow: /wiki/MediaWiki_talk
Disallow: /wiki/MediaWiki%20talk
Disallow: /wiki/Template_talk
Disallow: /wiki/Template%20talk
Disallow: /wiki/Help_talk
Disallow: /wiki/Help%20talk
Disallow: /wiki/Category_talk
Disallow: /wiki/Category%20talk
Disallow: /wiki/Staff_talk
Disallow: /wiki/Staff%20talk
Disallow: /wiki/HR_talk
Disallow: /wiki/HR%20talk
Disallow: /wiki/Forms_talk
Disallow: /wiki/Forms%20talk

# Block search and random pages
Disallow: /wiki/Special:Random
Disallow: /wiki/Special%3ARandom
Disallow: /wiki/Special:Search
Disallow: /wiki/Special%3ASearch

# Block deletion discussions
Disallow: /wiki/AcornWiki:Articles_for_deletion/
Disallow: /wiki/AcornWiki%3AArticles_for_deletion/
Disallow: /wiki/AcornWiki:Votes_for_deletion/
Disallow: /wiki/AcornWiki%3AVotes_for_deletion/
Disallow: /wiki/AcornWiki:Pages_for_deletion/
Disallow: /wiki/AcornWiki%3APages_for_deletion/
Disallow: /wiki/AcornWiki:Miscellany_for_deletion/
Disallow: /wiki/AcornWiki%3AMiscellany_for_deletion/
Disallow: /wiki/AcornWiki:Miscellaneous_deletion/
Disallow: /wiki/AcornWiki%3AMiscellaneous_deletion/
Disallow: /wiki/AcornWiki:Copyright_problems
Disallow: /wiki/AcornWiki%3ACopyright_problems
Disallow: /wiki/AcornWiki:Protected_titles/