#
# robots.txt
#
# This file controls crawler access to selected areas of the site.
# It must live at the root of the host to be effective.
#
# More info:
# http://www.robotstxt.org/robotstxt.html
#

# Block selected aggressive crawlers that respect robots.txt.
User-agent: Scrapy
Disallow: /

User-agent: Scrapy/2.11.2
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: Linguee
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: ia_archiver
Disallow: /

# Default rules for all other crawlers.
User-agent: *

# Allow core assets needed for rendering.
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg

Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg

# Allow the main events landing page.
Allow: /events
Allow: /events/

# Core and profile directories.
Disallow: /core/
Disallow: /profiles/

# Documentation and non-public files.
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt
Disallow: /web.config

# Prevent crawling of faceted event and article listing variants.
Disallow: /events/type/
Disallow: /events/category/
Disallow: /articles/type/
Disallow: /articles/category/
Disallow: /events/location/
Disallow: /events/date/
Disallow: /events?*
Disallow: /events/*?*
Disallow: /*f[*
Disallow: /*facet*
Disallow: /*filter*
Disallow: /*sort*
Disallow: /*?calendar_timestamp=
Disallow: /*?keywords=
Disallow: /*?title=
Disallow: /*?page=
Disallow: /*&page=
Disallow: /*&keywords=
Disallow: /*&title=

# Prevent crawling of faceted academics listing variants.
Disallow: /*program_type=
Disallow: /*career_interest=
Disallow: /*license=
Disallow: /*area_of_study=

# Prevent crawling of low-value system/content routes.
Disallow: /taxonomy/term/
Disallow: /media/
Disallow: /media/*/download
Disallow: /media/*/download?*

# Clean URL paths that should not be crawled.
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed

# Non-clean URL equivalents.
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed

# General crawl-delay for bots that support it.
Crawl-delay: 10

# Major crawler overrides.
User-agent: Googlebot
Disallow: /events/type/
Disallow: /events/category/
Disallow: /articles/type/
Disallow: /articles/category/
Disallow: /taxonomy/term/
Disallow: /media/
Disallow: /media/*/download
Disallow: /*?keywords=
Disallow: /*?title=
Disallow: /*?page=
Disallow: /*program_type=
Disallow: /*career_interest=
Disallow: /*license=
Disallow: /*area_of_study=

User-agent: Bingbot
Disallow: /events/type/
Disallow: /events/category/
Disallow: /articles/type/
Disallow: /articles/category/
Disallow: /taxonomy/term/
Disallow: /media/
Disallow: /media/*/download
Disallow: /*?keywords=
Disallow: /*?title=
Disallow: /*?page=

User-agent: Slurp
Disallow: /events/
Disallow: /articles/type/
Disallow: /articles/category/
Disallow: /taxonomy/term/
Disallow: /media/

User-agent: DuckDuckBot
Disallow: /events/
Disallow: /articles/type/
Disallow: /articles/category/
Disallow: /taxonomy/term/
Disallow: /media/

User-agent: Baiduspider
Disallow: /events/
Disallow: /articles/type/
Disallow: /articles/category/
Disallow: /taxonomy/term/
Disallow: /media/

User-agent: YandexBot
Disallow: /events/
Disallow: /articles/type/
Disallow: /articles/category/
Disallow: /taxonomy/term/
Disallow: /media/

User-agent: facebookexternalhit
Disallow: /events/type/
Disallow: /events/category/
Disallow: /articles/type/
Disallow: /articles/category/
Disallow: /taxonomy/term/
Disallow: /media/