# small robots.txt # more information about this file can be found at # http://www.robotstxt.org/wc/robots.html # lines beginning with the pund ("#") sign are comments and can be deleted. # if case your drupal site is in a directory # lower than your docroot (e.g. /drupal) # please add this before the /-es below # to stop a polite robot indexing an exampledir # add a line like (delte the #'s) # user-agent: polite-bot # Disallow: /exampledir/ # a list of know bots can be found at # http://www.robotstxt.org/wc/active/html/index.html # see http://www.sxw.org.uk/computing/robots/check.html # for syntax checking User-agent: * # Crawl-Delay: 10 Disallow: /*size= Disallow: /cms/*size= Disallow: /GeneratedItems Disallow: /cms/GeneratedItems Disallow: /cms/misc Disallow: /cms/dbbackup Disallow: /cms/modules Disallow: /cms/scripts Disallow: /cms/aggregator Disallow: /cms/tracker Disallow: /cms/comment/reply Disallow: /cms/node/add Disallow: /cms/user # Disallow: /cms/files Disallow: /cms/search Disallow: /cms/book/print Disallow: /cms/book/export # Directories Disallow: /cms/book/ Disallow: /cms/database/ Disallow: /cms/includes/ Disallow: /cms/misc/ Disallow: /cms/modules/ Disallow: /cms/sites/ Disallow: /cms/themes/ Disallow: /cms/scripts/ Disallow: /cms/updates/ Disallow: /cms/profiles/ # Files Disallow: /cms/xmlrpc.php Disallow: /cms/cron.php Disallow: /cms/update.php Disallow: /cms/install.php Disallow: /cms/INSTALL.txt Disallow: /cms/INSTALL.mysql.txt Disallow: /cms/INSTALL.pgsql.txt Disallow: /cms/CHANGELOG.txt Disallow: /cms/MAINTAINERS.txt Disallow: /cms/LICENSE.txt Disallow: /cms/UPGRADE.txt # Paths (clean URLs) Disallow: /cms/admin/ Disallow: /cms/admin/logs Disallow: /cms/admin/settings Disallow: /cms/aggregator/ Disallow: /cms/comment/reply/ Disallow: /cms/contact/ Disallow: /cms/logout/ Disallow: /cms/node/add/ Disallow: /cms/search/ Disallow: /cms/user/register/ Disallow: /cms/user/password/ Disallow: /cms/user/login/ # Paths (no clean URLs) Disallow: /cms/?q=admin/ Disallow: /cms/?q=aggregator/ Disallow: /cms/?q=comment/reply/ Disallow: /cms/?q=contact/ Disallow: /cms/?q=logout/ Disallow: /cms/?q=node/add/ Disallow: /cms/?q=search/ Disallow:/cms /?q=user/password/ Disallow: /cms/?q=user/register/ Disallow:/cms /?q=user/login/ #old caddpower blog folders to not index Disallow: /B742661069 Disallow: /B946881262 Disallow: /B1163032662 Disallow: /B1647905227 Disallow: /B1946120057 Disallow: /B2123803424 Disallow: /translations Disallow: /search