# The Direct Index welcomes every crawler, AI crawlers included.
User-agent: *
Allow: /
# One city highlighted on a topic page is the same page as its kind view (the canonical); no need to fetch each.
Disallow: /stats/*city=
# The dispute and rescan links on every report are for people: the about page with a domain filled in, and a fresh scan (crawlers already get the latest report instead).
Disallow: /about?dispute=
Disallow: /scan?*force=
# The deep report and owner plan offers under every report are for the business; the report is the page to read.
Disallow: /r/*/deep
Disallow: /r/*/plan
Disallow: /scan/*/deep
Disallow: /scan/*/plan

# Content Signals (contentsignals.org): this content may be searched, used as AI input and used for training.
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Sitemap: https://directindex.org/sitemap.xml