# robots.txt must-knows:  (Re: https://moz.com/learn/seo/robotstxt)
# In order to be found, a robots.txt file must be placed in a website's top-level directory.
# Robots.txt is case sensitive: the file must be named "robots.txt"� (not Robots.txt, robots.TXT, or otherwise).
# Some user agents (robots) may choose to ignore your robots.txt file. This is especially common with more nefarious crawlers like malware robots or email address scrapers.
# The /robots.txt file is a publicly available: just add /robots.txt to the end of any root domain to see that website's directives (if that site has a robots.txt file!). 
#  This means that anyone can see what pages you do or don't want to be crawled, so don't use them to hide private user information.
# Each subdomain on a root domain uses separate robots.txt files. This means that both blog.example.com and example.com should have their own robots.txt files (at blog.example.com/robots.txt and example.com/robots.txt).
# It's generally a best practice to indicate the location of any sitemaps associated with this domain at the bottom of the robots.txt file. 
#
User-Agent: *
#
# How many milliseconds a crawler should wait before loading and crawling page content. 
# Note that Googlebot does not acknowledge this command, but crawl rate can be set in Google Search Console.
#
Crawl-Delay: 100
#
Disallow: /_/
Disallow: /Admin/
Disallow: /admin/
Disallow: /ClientTools/
Disallow: /fsSite/
Disallow: /Resources.axd
Disallow: /Search/
Disallow: /testing/
Disallow: /content/quarterly-strategy-update.fs
#
# Disallow for fsTools from being indexed
# 
Disallow: /fssite*/
#
# Disallow for WebResource.axd caching issues. Several instances below to cover all search engines.
# 
# To specify matching the end of a URL, use $
#
Disallow: /*.axd$
#
# However, WebResource.axd and ScriptResource.axd always include a query string parameter the URL does 
# not end with .axd thus, the correct robots.txt record for Google would be:
#
Disallow: /*.axd
#
# Not all crawlers recognize the wildcard '*' syntax. To comply with the robots.txt draft RFC
# Note that the records are case sensitive, and error page is showing the requests to be in lower case
# so let's include both cases below:
#
Disallow: /ScriptResource.axd
Disallow: /WebResource.axd
Disallow: /scriptresource.axd
Disallow: /webresource.axd
#
# Used to call out the location of any XML sitemap(s) associated with this URL. Note this command is only supported by Google, Ask, Bing, and Yahoo.
#
Sitemap: https://us.rbcgam.com/sitemap.xml