#  robots.txt Updated by Vickie 7/3/2020
#    - Added enabled code so that only allows crawling from site domain name
#    = added 2024 01 - for audit tool
#  - Added User-agent: AdsBot-Google on 1/21/2025

# robots.txt - Global Generic Template
# Updated: 2026-06-09
#
# SETUP INSTRUCTIONS:
#   Place this file at the root of your domain: https://www.yoursite.com/robots.txt
#   No site-specific changes required.
#
#   NOTE: Sitemap is intentionally omitted so this file can be applied globally.
#   Submit your sitemap directly via Google Search Console or Bing Webmaster Tools.
#
# This file prevents crawling and indexing of certain parts of your site
# by web crawlers and spiders run by sites like Yahoo! and Google.
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html


# Block all unrecognized bots
User-agent: *
Sitemap: https://www.childrensclinicws.com/sitemap.xml
Disallow: /
Crawl-delay: 600

# Allow only trusted crawlers
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Mediapartners-Google
User-agent: AdsBot-Google
User-agent: msnbot
User-agent: msnbot-media
User-agent: Slurp
User-agent: Yahoo-Blogs
User-agent: Yahoo-MMCrawler
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: BrokenLinkCheck
User-agent: RSiteAuditor
User-agent: online-webceo-bot
User-agent: OAI-SearchBot
Allow: /
Crawl-delay: 10

# Directories
Disallow: /app_code/
Disallow: /css/
Disallow: /js/
Disallow: /CMS_Modules/
Disallow: /app_data/
Disallow: /bin/
Disallow: /img/
Disallow: /wp/
Disallow: /app/

# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /app.aspx

# Paths (clean URLs)
Disallow: /admin/
Disallow: /review/
Disallow: /review
Disallow: /comment/reply/
Disallow: /contact/
Disallow: /logout/
Disallow: /node/add/
Disallow: /search/
Disallow: /opensearch/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /wp-admin/
Disallow: /Resources/Kid-Site-Videos/
Disallow: /Medical-Comprehensive/Kid-Site-Videos/
Disallow: /Kid-Site-Videos/

# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=contact/
Disallow: /?q=logout/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/




# Should be Restricting Development or secondary domain names
# These 2 must match
# www.childrensclinicws.com
# https://www.childrensclinicws.com/robotstxt
# 
# True 
# and
# True
# -1 cannot include server2.remedyconnect.com
# true