User-agent: * # default robots.txt file #User-agent: * #Disallow: /admin/ # Don't spider the CMS #Disallow: /aspnet_client/ # or ASP.net clientside files #Disallow: /bin/ # or assemblies #Disallow: /Controls/ # or controls #Disallow: /csi/ # or client side includes #Disallow: /dic/ # or dictionary files used by FreeTextBox