User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /media/oembed
Disallow: /*/media/oembed
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed


# Added 2022-03-10
Disallow: /user/register?destination=*
Disallow: /user/login?destination=*
Disallow: /comments/*
Disallow: /comment/*

Disallow: /yikes/pixel.gif*
Disallow: /login?*

# MI-5493
Disallow: /login*
Disallow: /register*
Disallow: /user/login*
Disallow: /user/register*
Disallow: /bruger/*
Disallow: /cookiedeklaration*
Disallow: /kontakt*

# disallow bots
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

# --- Disallow ai crawlers ---

# Common crawl
User-agent: CCBot
Disallow: /

# OpenAI (ChatGPT)
User-agent: GPTBot
Disallow: /

# OpenAI (ChatGPT realtime search)
User-agent: ChatGPT-User
Disallow: /

# Anthropic
User-agent: anthropic-ai
Disallow: /

# https://www.turnitin.com/robot/crawlerinfo.html, not following ai.txt rules
User-agent: TurnitinBot
Disallow: /