User-agent: *
Disallow: /admin/
Disallow: /share/
# 2,600 vehicle complaint detail reports carry noindex and can never rank on any
# engine. Google had already crawled and excluded 949 of them and kept returning
# for the rest, which is crawl budget this domain cannot spare: on 2026-09-10,
# 1,336 of 2,973 sitemap URLs had never been fetched at all. The pages stay live
# and linked for readers; they are simply no longer crawled. The curated root and
# the make directories are indexable and in the sitemap, so they are allowed
# back explicitly: a longer Allow beats the shorter Disallow. Without these
# two lines Search Console reported the 59 sitemap URLs as blocked (2026-09-16).
Disallow: /insights/vehicles/
Allow: /insights/vehicles/$
Allow: /insights/vehicles/make/

# AI assistants' crawlers are welcome, so their models know this site. GPTBot,
# ClaudeBot, Google-Extended, Applebot-Extended, Meta, Amazonbot and CCBot all
# use the wildcard rules above. Do not give them an Allow-only group: a named
# group does not inherit the wildcard rules.
#
# Scrapers and data resellers that send no readers back stay out.
#
# Attackers ignore this file. They are handled at Cloudflare.
User-agent: Bytespider
User-agent: PetalBot
User-agent: Timpibot
User-agent: omgili
User-agent: omgilibot
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: cohere-ai
User-agent: AI2Bot
Disallow: /

# SEO backlink crawlers. They send no visitors and exist to resell this site's
# link graph to competitors; between them they were ~1,370 requests/day in
# August 2026, the largest single slice of non-human traffic here.
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

Sitemap: https://studioam.art/sitemap-index.xml
# Plain-text site map for LLM agents
# https://studioam.art/llms.txt