{
 "page": "https://tools.openkrill.app/compare/ai-crawler-audit-tools",
 "checked": "2026-10-13",
 "tests": [
  {
   "label": "site-check",
   "question": "Does this site's robots.txt block GPTBot from the home page?",
   "inputs": [
    {
     "input": "https://www.nytimes.com/",
     "truth": "Blocked: a GPTBot group with Disallow: /",
     "sources": [
      "https://www.nytimes.com/robots.txt"
     ]
    },
    {
     "input": "https://github.com/",
     "truth": "Allowed (harder): the GPTBot group has Allow: /$, which beats its Disallow: / for the home page only",
     "sources": [
      "https://github.com/robots.txt"
     ]
    },
    {
     "input": "https://www.reddit.com/",
     "truth": "Blocked: no GPTBot group, and the * group has Disallow: /",
     "sources": [
      "https://www.reddit.com/robots.txt"
     ]
    },
    {
     "input": "https://www.theguardian.com/",
     "truth": "Allowed (harder): GPTBot is not named, while ClaudeBot and about 40 other bots are blocked; the * group does not block /",
     "sources": [
      "https://www.theguardian.com/robots.txt"
     ]
    },
    {
     "input": "https://medium.com/",
     "truth": "Blocked: the GPTBot group has Disallow: / with Allow lines only for /about and similar paths",
     "sources": [
      "https://medium.com/robots.txt"
     ]
    }
   ],
   "scoring": "Truth from each robots.txt read on 2026-10-13 with RFC 9309 matching: use the group that names GPTBot, else the * group; the longest matching rule for / wins, and Allow wins a tie. An answer counts when its blocked or allowed verdict for GPTBot on the home page matches."
  }
 ],
 "results": [
  {
   "name": "Screaming Frog SEO Spider",
   "ours": false,
   "tests": [
    {
     "tested": false,
     "correct": null,
     "answered": null,
     "medianMs": null,
     "how": null,
     "notes": "local install only (desktop app)",
     "inScope": null
    }
   ]
  },
  {
   "name": "Semrush Site Audit",
   "ours": false,
   "tests": [
    {
     "tested": false,
     "correct": null,
     "answered": null,
     "medianMs": null,
     "how": null,
     "notes": "needs an account",
     "inScope": null
    }
   ]
  },
  {
   "name": "Ahrefs free Site Audit",
   "ours": false,
   "tests": [
    {
     "tested": false,
     "correct": null,
     "answered": null,
     "medianMs": null,
     "how": null,
     "notes": "needs an account and a verified site",
     "inScope": null
    }
   ]
  },
  {
   "name": "SEO Audit Console (houtini-ai)",
   "ours": false,
   "tests": [
    {
     "tested": false,
     "correct": null,
     "answered": null,
     "medianMs": null,
     "how": null,
     "notes": "local install only, and it needs a Google service account",
     "inScope": null
    }
   ]
  },
  {
   "name": "Google Search Console MCP (mcp-gsc)",
   "ours": false,
   "tests": [
    {
     "tested": false,
     "correct": null,
     "answered": null,
     "medianMs": null,
     "how": null,
     "notes": "local install only, needs Google sign-in, and it has no GPTBot check",
     "inScope": null
    }
   ]
  },
  {
   "name": "AI Crawler Policy Audit (Apify)",
   "ours": false,
   "tests": [
    {
     "tested": false,
     "correct": null,
     "answered": null,
     "medianMs": null,
     "how": null,
     "notes": "needs an account",
     "inScope": null
    }
   ]
  },
  {
   "name": "Cloudflare AI Crawl Control",
   "ours": false,
   "tests": [
    {
     "tested": false,
     "correct": null,
     "answered": null,
     "medianMs": null,
     "how": null,
     "notes": "needs an account, and it works only on a site you serve through Cloudflare",
     "inScope": null
    }
   ]
  },
  {
   "name": "AI Bot Access Testing Tool",
   "ours": false,
   "tests": [
    {
     "tested": true,
     "correct": 4,
     "answered": 5,
     "medianMs": null,
     "inScope": null,
     "how": "its web form by hand in a browser, all 5 URLs in one run with its default bots; the GPTBot robots.txt cell was read",
     "notes": "Right on 4, with the deciding line (Allow: /$ for github.com). For reddit.com it marked GPTBot allowed with no rule shown, while the robots.txt we read has Disallow: / for every bot; its own fetch of reddit.com got 403 Forbidden. It also fetches each URL as each bot, which a robots.txt read cannot do. One form run returns all 5, so there is no time per input."
    }
   ]
  },
  {
   "name": "SiteAudit MCP (vdalhambra)",
   "ours": false,
   "tests": [
    {
     "tested": false,
     "correct": null,
     "answered": null,
     "medianMs": null,
     "how": null,
     "notes": "local install only",
     "inScope": null
    }
   ]
  },
  {
   "name": "Site SEO & AI Crawler Audit",
   "ours": true,
   "tests": [
    {
     "tested": true,
     "correct": 5,
     "answered": 5,
     "medianMs": 9293,
     "inScope": null,
     "how": "remote MCP tools/call check_ai_crawler_access at https://sitecheck.openkrill.app/mcp",
     "notes": "All 5 right, each with the deciding rule (Allow: /$ for github.com); for the Guardian it read the page it was sent to, /europe, which is also allowed."
    }
   ]
  }
 ]
}