feat(eval): add field-level OCR accuracy harness + seed corpus
Adds a measurement gate for the artifact OCR parser (ADR-007), the prerequisite for the layout-profile and preprocessing rework. runOcrEval feeds labeled OCR text through the real parseArtifactCandidate and scores per-field / per-case accuracy. - src/eval/ocrEvalHarness.ts: pure metrics (per-field, critical-field, exact). - src/eval/corpus/seedCorpus.ts: 23 cases transcribed from the verified parser test assertions; runs at 100%. - src/eval/reviewSampleCorpus.ts: converts review samples into label *candidates* (never ground truth) so the review queue can grow the corpus. - src/eval/ocrEval.test.ts + reviewSampleCorpus.test.ts: gate (must stay 1.0) and converter unit tests. - npm run eval script; docs/ocr-eval.md; ADR-007/008/009. Also records the agreed rework direction: C# input/capture sidecar (ADR-008) and resolution-anchored layout profiles + OCR preprocessing (ADR-009). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
import type { ReviewSampleRecord } from "../types/global";
|
||||
import type { OcrEvalCase } from "./ocrEvalHarness";
|
||||
|
||||
// Turns saved review samples into eval cases so the review queue becomes a
|
||||
// growing labeled corpus (see docs/ocr-eval.md).
|
||||
//
|
||||
// IMPORTANT: a review sample's `parsed` block is the *auto*-parse that was
|
||||
// flagged for review - it is a label CANDIDATE, not verified ground truth.
|
||||
// Using it directly as an expectation would be circular (the parser grading
|
||||
// itself). The intended flow is:
|
||||
// 1. reviewSampleToEvalCase() extracts the OCR + the parser's current guess.
|
||||
// 2. A human confirms or corrects `expect` in the produced case.
|
||||
// 3. The corrected case is committed into src/eval/corpus/.
|
||||
// The `confirmed` flag records whether step 2 happened.
|
||||
|
||||
export interface ReviewSampleEvalCase extends OcrEvalCase {
|
||||
/** False until a human has verified/corrected the `expect` values. */
|
||||
confirmed: boolean;
|
||||
}
|
||||
|
||||
function ocrMapFromRecord(record: ReviewSampleRecord): Record<string, string> | null {
|
||||
const entries = record.sample?.capture?.ocr;
|
||||
if (!entries?.length) return null;
|
||||
const map: Record<string, string> = {};
|
||||
for (const entry of entries) {
|
||||
if (typeof entry?.id === "string" && typeof entry?.text === "string") {
|
||||
map[entry.id] = entry.text;
|
||||
}
|
||||
}
|
||||
return Object.keys(map).length > 0 ? map : null;
|
||||
}
|
||||
|
||||
function expectFromParsed(parsed: unknown): OcrEvalCase["expect"] {
|
||||
if (!parsed || typeof parsed !== "object") return {};
|
||||
const candidate = parsed as Record<string, unknown>;
|
||||
const expect: OcrEvalCase["expect"] = {};
|
||||
|
||||
const stringField = (key: "name" | "slot" | "mainStat" | "mainValue" | "setName" | "equipped") => {
|
||||
const value = candidate[key];
|
||||
// Skip the parser's own "Unknown ..." / "Not detected" / "?" sentinels -
|
||||
// those are non-answers, not labels a human would confirm.
|
||||
if (typeof value === "string" && value && !value.startsWith("Unknown") && value !== "Not detected" && value !== "?") {
|
||||
expect[key] = value;
|
||||
}
|
||||
};
|
||||
|
||||
stringField("name");
|
||||
stringField("slot");
|
||||
stringField("mainStat");
|
||||
stringField("mainValue");
|
||||
stringField("setName");
|
||||
stringField("equipped");
|
||||
|
||||
if (typeof candidate.level === "number" && Number.isFinite(candidate.level) && candidate.level > 0) {
|
||||
expect.level = candidate.level;
|
||||
}
|
||||
if (Array.isArray(candidate.substats) && candidate.substats.every((entry) => typeof entry === "string")) {
|
||||
expect.substats = candidate.substats as string[];
|
||||
}
|
||||
|
||||
return expect;
|
||||
}
|
||||
|
||||
export function reviewSampleToEvalCase(record: ReviewSampleRecord, index = 0): ReviewSampleEvalCase | null {
|
||||
const ocr = ocrMapFromRecord(record);
|
||||
if (!ocr) return null;
|
||||
|
||||
const savedAt = record.savedAt ?? "unknown";
|
||||
const idSuffix = savedAt.replace(/[^0-9A-Za-z]/g, "").slice(0, 14) || String(index);
|
||||
const capture = record.sample?.capture;
|
||||
|
||||
return {
|
||||
id: `review-${idSuffix}-${index}`,
|
||||
ocr,
|
||||
expect: expectFromParsed(record.sample?.parsed),
|
||||
confirmed: false,
|
||||
meta: {
|
||||
source: "review-sample",
|
||||
resolution: capture?.width && capture?.height ? `${capture.width}x${capture.height}` : undefined,
|
||||
note: record.sample?.reason,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function reviewSamplesToEvalCases(records: readonly ReviewSampleRecord[] | null | undefined) {
|
||||
const cases: ReviewSampleEvalCase[] = [];
|
||||
(records ?? []).forEach((record, index) => {
|
||||
const evalCase = reviewSampleToEvalCase(record, index);
|
||||
if (evalCase) cases.push(evalCase);
|
||||
});
|
||||
return cases;
|
||||
}
|
||||
Reference in New Issue
Block a user