feat(eval): add field-level OCR accuracy harness + seed corpus
Adds a measurement gate for the artifact OCR parser (ADR-007), the prerequisite for the layout-profile and preprocessing rework. runOcrEval feeds labeled OCR text through the real parseArtifactCandidate and scores per-field / per-case accuracy. - src/eval/ocrEvalHarness.ts: pure metrics (per-field, critical-field, exact). - src/eval/corpus/seedCorpus.ts: 23 cases transcribed from the verified parser test assertions; runs at 100%. - src/eval/reviewSampleCorpus.ts: converts review samples into label *candidates* (never ground truth) so the review queue can grow the corpus. - src/eval/ocrEval.test.ts + reviewSampleCorpus.test.ts: gate (must stay 1.0) and converter unit tests. - npm run eval script; docs/ocr-eval.md; ADR-007/008/009. Also records the agreed rework direction: C# input/capture sidecar (ADR-008) and resolution-anchored layout profiles + OCR preprocessing (ADR-009). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { formatReport, runOcrEval } from "./ocrEvalHarness";
|
||||
import { seedCorpus } from "./corpus/seedCorpus";
|
||||
|
||||
// Regression gate: the seed corpus is verified ground truth, so the parser must
|
||||
// read every labeled field correctly. A drop here means an OCR/parser change
|
||||
// regressed a previously-correct read - look at the printed failures. If a
|
||||
// change intentionally alters a correct output, update the corpus label in the
|
||||
// same commit (the label is the source of truth, not the code).
|
||||
|
||||
describe("OCR eval harness", () => {
|
||||
const report = runOcrEval(seedCorpus);
|
||||
|
||||
it("prints the accuracy report", () => {
|
||||
// Surfaced in test output for humans; not an assertion.
|
||||
// eslint-disable-next-line no-console
|
||||
console.log("\n" + formatReport(report) + "\n");
|
||||
expect(report.totalCases).toBe(seedCorpus.length);
|
||||
});
|
||||
|
||||
it("reads every labeled field on the seed corpus correctly", () => {
|
||||
const failureSummary = report.failures
|
||||
.map((failure) => {
|
||||
const wrong = failure.fields
|
||||
.filter((entry) => !entry.correct)
|
||||
.map((entry) => `${entry.field}: expected "${entry.expected}" got "${entry.actual}"`)
|
||||
.join("; ");
|
||||
return `${failure.id} -> ${wrong}`;
|
||||
})
|
||||
.join("\n");
|
||||
|
||||
expect(report.fieldAccuracy, `field regressions:\n${failureSummary}`).toBe(1);
|
||||
expect(report.criticalFieldAccuracy).toBe(1);
|
||||
expect(report.exactRate).toBe(1);
|
||||
});
|
||||
|
||||
it("labels every critical field at least once across the corpus", () => {
|
||||
const criticalCoverage = report.perField.filter(
|
||||
(entry) => ["name", "slot", "mainStat", "mainValue", "setName"].includes(entry.field) && entry.evaluated > 0,
|
||||
);
|
||||
// name is derived indirectly on most cases; slot/mainStat/mainValue/setName must be exercised.
|
||||
expect(criticalCoverage.length).toBeGreaterThanOrEqual(4);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user