feat(corpus): require retrievable visual review evidence

2026-07-11 16:13:02 +02:00
parent 8b9f948c6b
commit 3e2ab61c97
14 changed files with 325 additions and 73 deletions
@@ -23,6 +23,43 @@ function simplifyId(value) {
    .slice(0, 80) || "unknown";
 }

+function safeCaptureReference(value) {
+  const reference = typeof value === "string" ? value.trim() : "";
+  if (!reference || reference.length > 160 || /[\r\n]/.test(reference)) return null;
+  if (path.isAbsolute(reference) || /^[A-Za-z]:[\\/]/.test(reference) || reference.startsWith("\\\\")) return null;
+  return reference;
+}
+
+function safeCapturedAt(value) {
+  const capturedAt = typeof value === "string" ? value.trim() : "";
+  if (!capturedAt || capturedAt.length > 80 || /[\r\n]/.test(capturedAt)) return null;
+  return capturedAt;
+}
+
+function hasRetrievableLocalPng(value) {
+  if (typeof value !== "string" || path.extname(value).toLowerCase() !== ".png") return false;
+  try {
+    return fs.statSync(value).isFile();
+  } catch {
+    return false;
+  }
+}
+
+function visualEvidenceForCapture(capture) {
+  const captureId = safeCaptureReference(capture?.id);
+  const capturedAt = safeCapturedAt(capture?.capturedAt);
+  const available = hasRetrievableLocalPng(capture?.name);
+  return {
+    captureId,
+    capturedAt,
+    visualEvidence: {
+      status: available ? "available" : "unavailable",
+      source: available ? "local-crop" : "none",
+      reference: captureId,
+    },
+  };
+}
+
 function parsedSummary(parsed) {
  if (!parsed || typeof parsed !== "object") return {};
  return {
@@ -66,6 +103,7 @@ function candidateFromRecord(record, index) {
  const reason = record?.sample?.reason || "missing-reason";
  const savedAt = record?.savedAt || "unknown";
  const capture = record?.sample?.capture || {};
+  const visualEvidence = visualEvidenceForCapture(capture);
  const ocrFieldIds = Object.keys(ocr).sort();
  const missingFastFields = REQUIRED_FAST_OCR_FIELDS.filter((field) => !ocr[field]);
  const hasEquippedFooterOcr = Boolean(ocr["artifact-footer"]);
@@ -76,6 +114,9 @@ function candidateFromRecord(record, index) {
    reason,
    confirmed: false,
    resolution: capture.width && capture.height ? `${capture.width}x${capture.height}` : "",
+    captureId: visualEvidence.captureId,
+    capturedAt: visualEvidence.capturedAt,
+    visualEvidence: visualEvidence.visualEvidence,
    ocrFieldIds,
    missingFastFields,
    hasEquippedFooterOcr,
@@ -98,6 +139,9 @@ function candidateFromRecord(record, index) {
      validationChecks: {
        equipped: hasEquippedFooterOcr ? "Confirm character name or mark Not detected." : "No footer OCR in this sample.",
        locked: locked === null ? "No lock-state payload in this sample." : `Confirm locked=${locked}.`,
+        visualEvidence: visualEvidence.visualEvidence.status === "available"
+          ? "A local PNG was retrievable at export. Inspect it before confirming labels."
+          : "No retrievable local PNG. Do not promote this case to the confirmed corpus.",
      },
    },
  };
@@ -128,9 +172,19 @@ function candidatePriority(candidate) {
  return 5;
 }

+function candidateComparator(left, right) {
+  const visualEvidencePriority = Number(left.visualEvidence?.status !== "available")
+    - Number(right.visualEvidence?.status !== "available");
+  if (visualEvidencePriority) return visualEvidencePriority;
+  const priority = candidatePriority(left) - candidatePriority(right);
+  if (priority) return priority;
+  const missingDiff = (left.missingFastFields?.length || 0) - (right.missingFastFields?.length || 0);
+  if (missingDiff) return missingDiff;
+  return String(right.savedAt).localeCompare(String(left.savedAt));
+}
+
 async function readCandidates(inputPath, limit) {
-  const candidates = [];
-  const seen = new Set();
+  const candidatesByKey = new Map();
  let total = 0;
  let invalid = 0;
  const rl = readline.createInterface({ input: fs.createReadStream(inputPath, { encoding: "utf8" }) });
@@ -147,16 +201,14 @@ async function readCandidates(inputPath, limit) {
    const candidate = candidateFromRecord(record, total);
    if (!candidate) continue;
    const key = candidateKey(candidate);
-    if (seen.has(key)) continue;
-    seen.add(key);
-    candidates.push(candidate);
+    const existing = candidatesByKey.get(key);
+    if (!existing || candidateComparator(candidate, existing) < 0) {
+      candidatesByKey.set(key, candidate);
+    }
  }
+  const candidates = [...candidatesByKey.values()];
  candidates.sort((left, right) => {
-    const priority = candidatePriority(left) - candidatePriority(right);
-    if (priority) return priority;
-    const missingDiff = (left.missingFastFields?.length || 0) - (right.missingFastFields?.length || 0);
-    if (missingDiff) return missingDiff;
-    return String(right.savedAt).localeCompare(String(left.savedAt));
+    return candidateComparator(left, right);
  });
  return { total, invalid, candidates: candidates.slice(0, limit), uniqueCandidates: candidates.length };
 }
@@ -174,12 +226,16 @@ function buildExportStats(candidates) {
  let equippedFooterCandidates = 0;
  let lockedTrueCandidates = 0;
  let lockedFalseCandidates = 0;
+  let visualEvidenceAvailable = 0;
+  let visualEvidenceUnavailable = 0;
  for (const candidate of candidates) {
    increment(reasonCounts, candidate.reason);
    if (candidate.likelyStaleCapture) likelyStaleCaptures++;
    if (candidate.hasEquippedFooterOcr) equippedFooterCandidates++;
    if (candidate.locked === true) lockedTrueCandidates++;
    if (candidate.locked === false) lockedFalseCandidates++;
+    if (candidate.visualEvidence?.status === "available") visualEvidenceAvailable++;
+    else visualEvidenceUnavailable++;
    const missingFields = candidate.missingFastFields || [];
    if (missingFields.length === 0) completeFastFields++;
    for (const field of missingFields) increment(missingFastFieldCounts, field);
@@ -190,6 +246,8 @@ function buildExportStats(candidates) {
    equippedFooterCandidates,
    lockedTrueCandidates,
    lockedFalseCandidates,
+    visualEvidenceAvailable,
+    visualEvidenceUnavailable,
    reasonCounts,
    missingFastFieldCounts,
  };
@@ -209,8 +267,10 @@ function markdownFor(summary, candidates) {
    `Equipped footer candidates: ${summary.exportStats.equippedFooterCandidates}`,
    `Locked=true candidates: ${summary.exportStats.lockedTrueCandidates}`,
    `Locked=false candidates: ${summary.exportStats.lockedFalseCandidates}`,
+    `Visual evidence available: ${summary.exportStats.visualEvidenceAvailable}`,
+    `Visual evidence unavailable: ${summary.exportStats.visualEvidenceUnavailable}`,
    "",
-    "These cases are not ground truth yet. Confirm or correct the expected fields before committing any case into `src/eval/corpus/`.",
+    "These cases are not ground truth yet. Only manually label candidates with `visualEvidence: available`; then confirm or correct the expected fields before committing any case into `src/eval/corpus/`.",
    "",
    "## Export stats",
    "",
@@ -228,6 +288,9 @@ function markdownFor(summary, candidates) {
    lines.push("");
    lines.push(`- savedAt: ${candidate.savedAt}`);
    lines.push(`- reason: ${candidate.reason}`);
+    lines.push(`- captureId: ${candidate.captureId || "unavailable"}`);
+    lines.push(`- capturedAt: ${candidate.capturedAt || "unavailable"}`);
+    lines.push(`- visualEvidence: ${candidate.visualEvidence?.status || "unavailable"} (${candidate.visualEvidence?.source || "none"})`);
    lines.push(`- missingFastFields: ${candidate.missingFastFields.join(", ") || "none"}`);
    lines.push(`- likelyStaleCapture: ${candidate.likelyStaleCapture ? "yes" : "no"}`);
    lines.push(`- hasEquippedFooterOcr: ${candidate.hasEquippedFooterOcr ? "yes" : "no"}`);
@@ -281,8 +344,11 @@ if (require.main === module) {
 module.exports = {
  buildExportStats,
  candidateFromRecord,
+  candidateComparator,
  candidatePriority,
  defaultReviewSamplesPath,
+  hasRetrievableLocalPng,
  markdownFor,
  readCandidates,
+  visualEvidenceForCapture,
 };
@@ -60,12 +60,33 @@ function stableId(value) {
    .slice(0, 80) || "review-case";
 }

+function safeProvenanceValue(value, fallback = "unknown") {
+  const text = typeof value === "string" ? value.trim() : "";
+  if (!text || text.length > 200 || /[\r\n]/.test(text)) return fallback;
+  if (path.isAbsolute(text) || /[A-Za-z]:[\\/]/.test(text) || text.startsWith("\\\\")) return fallback;
+  return text;
+}
+
+function visualEvidenceNote(candidate) {
+  const status = candidate?.visualEvidence?.status === "available" ? "available" : "unavailable";
+  const parts = [`visualEvidence=${status}`];
+  const captureId = safeProvenanceValue(candidate?.captureId, "");
+  const capturedAt = safeProvenanceValue(candidate?.capturedAt, "");
+  if (captureId) parts.push(`captureId=${captureId}`);
+  if (capturedAt) parts.push(`capturedAt=${capturedAt}`);
+  return parts.join(" | ");
+}
+
 function objectLiteral(value, indent = 2) {
  return JSON.stringify(value, null, indent).replace(/"([A-Za-z_$][0-9A-Za-z_$]*)":/g, "$1:");
 }

 function snippetFor(candidate, expect) {
  const id = stableId(argValue("id", `confirmed-${candidate.id}`));
+  const reason = safeProvenanceValue(candidate.reason, "review-sample");
+  const savedAt = safeProvenanceValue(candidate.savedAt, "unknown");
+  const resolution = safeProvenanceValue(candidate.resolution, "");
+  const sourceCandidate = stableId(candidate.id);
  const entry = {
    id,
    confirmed: true,
@@ -73,8 +94,8 @@ function snippetFor(candidate, expect) {
    expect,
    meta: {
      source: "review-sample",
-      resolution: candidate.resolution || undefined,
-      note: `${candidate.reason || "review-sample"} | savedAt=${candidate.savedAt || "unknown"} | sourceCandidate=${candidate.id}`,
+      resolution: resolution || undefined,
+      note: `${reason} | savedAt=${savedAt} | sourceCandidate=${sourceCandidate} | ${visualEvidenceNote(candidate)}`,
    },
  };
  return `${objectLiteral(entry, 2)},\n`;
@@ -90,6 +111,9 @@ function main() {
  if (!candidateId) throw new Error("Missing candidate id. Pass --candidate=<id>.");
  const outputPath = path.resolve(argValue("out", path.join(process.cwd(), "outputs", "review-eval-candidates", `${stableId(candidateId)}.confirmed.ts`)));
  const candidate = loadCandidate(inputPath, candidateId);
+  if (candidate?.visualEvidence?.status !== "available") {
+    throw new Error("Candidate has no retrievable visual evidence. Re-export after retaining a local PNG before preparing a confirmed corpus case.");
+  }
  const expect = parseExpectedFields();
  fs.mkdirSync(path.dirname(outputPath), { recursive: true });
  const snippet = snippetFor(candidate, expect);
@@ -106,4 +130,5 @@ module.exports = {
  parseExpectedFields,
  snippetFor,
  stableId,
+  visualEvidenceNote,
 };