Merge pull request 'fix(verdict): findings resolved by the change under review never block' (#5) from fix/resolved-findings-dont-block into main

This commit is contained in:
Nirav Patel
2026-08-22 14:44:49 -04:00
2 changed files with 13 additions and 2 deletions
+2
View File
@@ -19,12 +19,14 @@ Output schema (every key required):
"findings": [
{"title": "...", "file": "path", "line": 123, "severity": "critical"|"high"|"medium"|"low",
"confidence": "confirmed"|"plausible", "axis": "correctness"|"security"|"tests_coverage"|"design_simplicity"|"performance"|"style"|"breaking_changes"|"docs",
"resolved_by_this_change": true|false,
"detail": "what is wrong, concrete failure scenario, and the fix"}
],
"verdict": "pass"|"warn"|"fail"
}
Rules:
- A finding must describe a defect that EXISTS IN THE RESULTING CODE after this diff is applied. The pre-existing bug that this diff FIXES is not a finding — acknowledge it in the correctness rationale instead. If you still list it (e.g. for the record), you MUST set "resolved_by_this_change": true on that finding; findings the diff introduces or leaves unfixed get "resolved_by_this_change": false.
- "confirmed" means you can trace a concrete failure from the diff alone; anything needing unseen context is "plausible".
- severity critical/high is reserved for real correctness bugs, security holes, data loss, or breaking API changes.
- verdict "fail" ONLY when a confirmed critical/high correctness or security finding exists; "warn" for notable but non-blocking issues; otherwise "pass".
+11 -2
View File
@@ -7,6 +7,10 @@ const BLOCKING_AXES = new Set(['correctness', 'security']);
export function isBlockingFinding(f) {
if (!f || typeof f !== 'object') return false;
// A finding the diff itself resolves describes the PRE-change state; it
// cannot block the change that fixes it (models routinely narrate the fixed
// bug as a critical/confirmed finding, which failed correct fixes).
if (f.resolved_by_this_change === true) return false;
const sev = String(f.severity || '').toLowerCase();
const conf = String(f.confidence || '').toLowerCase();
if (!BLOCKING_SEVERITIES.has(sev)) return false;
@@ -21,12 +25,16 @@ export function deriveVerdict(review) {
const findings = Array.isArray(review?.findings) ? review.findings : [];
if (findings.some(isBlockingFinding)) return 'fail';
const hasWarn = findings.some(f => {
if (f?.resolved_by_this_change === true) return false;
const sev = String(f?.severity || '').toLowerCase();
return sev === 'critical' || sev === 'high' || sev === 'medium';
});
if (hasWarn) return 'warn';
// trust the model's verdict if it said warn without matching findings
if (review?.verdict === 'warn') return 'warn';
// Trust the model's verdict if it said warn without matching findings
// unless every finding it gave is resolved_by_this_change, in which case its
// warn is about the pre-change state and the resulting code is clean.
const allResolved = findings.length > 0 && findings.every(f => f?.resolved_by_this_change === true);
if (review?.verdict === 'warn' && !allResolved) return 'warn';
return 'pass';
}
@@ -49,6 +57,7 @@ export function normalizeReview(raw) {
severity: pick(String(f?.severity || '').toLowerCase(), ['critical', 'high', 'medium', 'low'], 'low'),
confidence: pick(String(f?.confidence || '').toLowerCase(), ['confirmed', 'plausible'], 'plausible'),
axis: f?.axis ? String(f.axis).toLowerCase() : null,
resolved_by_this_change: f?.resolved_by_this_change === true,
detail: String(f?.detail ?? '').slice(0, 2000)
}));
const review = {