feat(eval): add field-level OCR accuracy harness + seed corpus

Adds a measurement gate for the artifact OCR parser (ADR-007), the prerequisite for the layout-profile and preprocessing rework. runOcrEval feeds labeled OCR text through the real parseArtifactCandidate and scores per-field / per-case accuracy. - src/eval/ocrEvalHarness.ts: pure metrics (per-field, critical-field, exact). - src/eval/corpus/seedCorpus.ts: 23 cases transcribed from the verified parser test assertions; runs at 100%. - src/eval/reviewSampleCorpus.ts: converts review samples into label *candidates* (never ground truth) so the review queue can grow the corpus. - src/eval/ocrEval.test.ts + reviewSampleCorpus.test.ts: gate (must stay 1.0) and converter unit tests. - npm run eval script; docs/ocr-eval.md; ADR-007/008/009. Also records the agreed rework direction: C# input/capture sidecar (ADR-008) and resolution-anchored layout profiles + OCR preprocessing (ADR-009). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-05 20:41:30 +02:00
parent e76d88e0c7
commit b7dbc618b3
8 changed files with 933 additions and 0 deletions
@@ -0,0 +1,311 @@
+import type { OcrEvalCase } from "../ocrEvalHarness";
+
+// Seed corpus transcribed from src/lib/artifactOcrParser.test.ts. Every label
+// here is an exact assertion that already passes, so a fresh run starts at (or
+// very near) 100% - which is what makes the accuracy thresholds meaningful as a
+// regression gate. Cases that deliberately expect conservative "Unknown ..."
+// behavior contribute only their positively-identified fields (level, value)
+// so field accuracy stays a measure of correct reads, not correct refusals.
+//
+// Grow this corpus from human-confirmed review samples via reviewSampleCorpus.ts.
+
+const SOURCE = "test-seed" as const;
+
+export const seedCorpus: OcrEvalCase[] = [
+  {
+    id: "spring-sands-em",
+    ocr: {
+      "artifact-title": "A Note in Spring's Lei\n| Sands of Eon Vi",
+      "artifact-main-stat": "Elemental Mastery\n187",
+      "artifact-substats": "+20\n+ ATK+29\n+ CRIT DMG+15.5%\n+ CRIT Rate+2.7%\nATK + 15",
+      "artifact-set-effects": "A Day Carved From Rising Winds\n2-Piece Set: ATK +18%.",
+      "artifact-footer": "WV Equipped: Citlali\naR 0",
+    },
+    expect: {
+      slot: "Sands of Eon",
+      level: 20,
+      mainStat: "Elemental Mastery",
+      setName: "A Day Carved From Rising Winds",
+      equipped: "Citlali",
+    },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "moonlit-flower-hp",
+    ocr: {
+      "artifact-title": "Moonlit Offering's Opulent Dr\n| Flower of Life 2",
+      "artifact-main-stat": "4,780\nAhhh",
+      "artifact-substats": "+ Elemental Mastery+54\n+ CRIT Rate+7.4%\n+ ATK+4.7%",
+      "artifact-set-effects": "Aubade of Morningstar and Moor\n2-Piece Set: Increases Elemental Mastery by 80.",
+      "artifact-footer": "? Equipped: Ineffa\naR 0",
+    },
+    expect: {
+      slot: "Flower of Life",
+      mainStat: "HP",
+      mainValue: "4,780",
+      setName: "Aubade of Morningstar and Moon",
+      equipped: "Ineffa",
+    },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "vv-determination-sands-er",
+    ocr: {
+      "artifact-title": "Viridescent Venerer's Determination\nSands of Eon Vi",
+      "artifact-main-stat": "Energy Recharge\n51.8%",
+      "artifact-substats": "+ ATK+5.8%\n+ Elemental Mastery+37\n+ HP+11.7%\n+ ATK+54",
+      "artifact-set-effects": "Viridescent Venerer:\n2-Piece Set: Anemo DMG Bonus +15%",
+      "artifact-footer": "Equipped: Sucrose",
+    },
+    expect: { slot: "Sands of Eon", mainStat: "Energy Recharge" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "pristine-plume-atk",
+    ocr: {
+      "artifact-title": "Pristine Plume of the Bles\n| Plume of Death",
+      "artifact-main-stat": "31 !\npr",
+      "artifact-substats": "+20\n+ CRIT DMG+7.0%\n+ DEF+30.6%\n+ Elemental Mastery+40\nATK +5.8%",
+      "artifact-set-effects": "Silken Moon's Serenade:\n2-Piece Set: Energy Recharge +20%.",
+      "artifact-footer": "Equipped: Aino\nBR",
+    },
+    expect: {
+      slot: "Plume of Death",
+      mainStat: "ATK",
+      mainValue: "311",
+      setName: "Silken Moon's Serenade",
+      equipped: "Aino",
+    },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "deep-gallery-goblet-cryo",
+    ocr: {
+      "artifact-title": "Deep Gallery's Bestowed Banquet\nGoblet of Eonothem",
+      "artifact-main-stat": "Cryo DMG Bonus\n46.6%",
+      "artifact-substats": "+ CRIT Rate+6.6%\n+ CRIT DMG+12.4%\n+ HP+269\n+ ATK+16.3%",
+      "artifact-set-effects": "Finale of the Deep Galleries:\n2-Piece Set: Cryo DMG Bonus +15%",
+      "artifact-footer": "Equipped: Skirk",
+    },
+    expect: { slot: "Goblet of Eonothem", mainStat: "Cryo DMG Bonus", mainValue: "46.6%" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "holy-crown-circlet-crit",
+    ocr: {
+      "artifact-title": "Holy Crown of the Believer\nCirclet of Logos",
+      "artifact-main-stat": "CRIT Rate\n31.1%",
+      "artifact-substats": "+ ATK+4.7%\n+ Elemental Mastery+56\n+ DEF+37\n+ Energy Recharge+11.7%",
+      "artifact-set-effects": "Silken Moon's Serenade:\n2-Piece Set: Energy Recharge +20%.",
+      "artifact-footer": "Equipped: Chongyun",
+    },
+    expect: { slot: "Circlet of Logos", mainStat: "CRIT Rate", mainValue: "31.1%" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "beast-tamer-flower-set",
+    ocr: {
+      "artifact-title": "Beast Tamer's Talisman\nFlower of Life",
+      "artifact-main-stat": "HP\n4,780",
+      "artifact-substats": "+ HP+16.3%\n+ CRIT DMG+7.0%\n+ Energy Recharge+6.5%\n+ Elemental Mastery+68",
+      "artifact-set-effects": "Scroll of the Hero of Cinder City:\n2-Piece Set: When a nearby party member triggers a Nightsoul Burst",
+      "artifact-footer": "Equipped: Citlali",
+    },
+    expect: { setName: "Scroll of the Hero of Cinder City" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "piece-name-to-set-fallback",
+    ocr: {
+      "artifact-title": "Holy Crown of the Believer\nCirclet of Logos",
+      "artifact-main-stat": "CRIT Rate\n31.1%",
+      "artifact-substats": "+ ATK+4.7%\n+ Elemental Mastery+56\n+ DEF+37",
+      "artifact-set-effects": "unreadable noisy set text",
+      "artifact-footer": "Equipped: Chongyun",
+    },
+    expect: { setName: "Silken Moon's Serenade" },
+    meta: { source: SOURCE, note: "set derived from piece name when set-effect OCR is noise" },
+  },
+  {
+    id: "piece-name-to-slot-fallback",
+    ocr: {
+      "artifact-title": "Viridescent Venerer's Determination",
+      "artifact-main-stat": "Energy Recharge\n51.8%",
+      "artifact-substats": "+ ATK+5.8%\n+ Elemental Mastery+37\n+ HP+11.7%\n+ ATK+54",
+      "artifact-set-effects": "Viridescent Venerer:\n2-Piece Set: Anemo DMG Bonus +15%",
+      "artifact-footer": "Equipped: Sucrose",
+    },
+    expect: { slot: "Sands of Eon", setName: "Viridescent Venerer" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "hourglass-atk-percent",
+    ocr: {
+      "artifact-title": "Hourglass of Thunder\nSands of Eon",
+      "artifact-main-stat": "ATK\n40.7%",
+      "artifact-substats": "+17\n+ CRIT DMG+14.8%\n+ Elemental Mastery+21\n+ ATK+53\n+ DEF+19",
+      "artifact-set-effects": "Thundering Fury:\n2-Piece Set: Electro DMG Bonus +15%",
+      "artifact-footer": "Equipped: Fischl",
+    },
+    expect: { level: 17, slot: "Sands of Eon", mainStat: "ATK%", mainValue: "40.7%" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "night-realm-sands-ambiguous",
+    ocr: {
+      "artifact-title": "Myths of the Night Realm\nSands of Eon",
+      "artifact-main-stat": "30.8%",
+      "artifact-substats": "+12\n+ Elemental Mastery+16\n+ CRIT DMG+6.2%\n+ DEF+42\n+ HP+9.9%",
+      "artifact-set-effects": "Obsidian Codex:\n2-Piece Set: While the equipping character is in Nightsoul's Blessing",
+      "artifact-footer": "Equipped: Sandrone",
+    },
+    expect: { level: 12, mainValue: "30.8%" },
+    meta: { source: SOURCE, note: "mainStat intentionally stays Unknown (ambiguous family)" },
+  },
+  {
+    id: "footer-noise-equipped",
+    ocr: {
+      "artifact-title": "Gladiator's Nostalgia\nFlower of Life",
+      "artifact-main-stat": "HP\n4,780",
+      "artifact-substats": "+ Energy Recharge+11.0%\n+ ATK+9.9%\n+ HP+14.6%\n+ CRIT DMG+12.4%",
+      "artifact-set-effects": "Gladiator's Finale:\n2-Piece Set: ATK +18%",
+      "artifact-footer": "1 gv 0RY If tha anuninnina\nJl Equipped: Bennett\nCEE",
+    },
+    expect: { equipped: "Bennett" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "atk-percent-sands",
+    ocr: {
+      "artifact-title": "Myths of the Night Realm\nSands of Eon",
+      "artifact-main-stat": "ATK\n30.8%",
+      "artifact-substats": "+ Elemental Mastery+16\n+ CRIT DMG+6.2%\n+ DEF+42\n+ HP+9.9%",
+      "artifact-set-effects": "Obsidian Codex:\n2-Piece Set: While the equipping character is in Nightsoul's Blessing",
+      "artifact-footer": "Equipped: Sandrone",
+    },
+    expect: { mainStat: "ATK%", mainValue: "30.8%" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "atk-percent-circlet",
+    ocr: {
+      "artifact-title": "Maiden's Fading Beauty\nCirclet of Logos",
+      "artifact-main-stat": "ATK\n46.6%",
+      "artifact-substats": "+ ATK+29\n+ CRIT DMG+10.9%\n+ CRIT Rate+7.0%",
+      "artifact-set-effects": "Maiden Beloved:\n2-Piece Set: Character Healing Effectiveness +15%",
+      "artifact-footer": "Equipped: Qiqi",
+    },
+    expect: { mainStat: "ATK%", mainValue: "46.6%" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "atk-percent-goblet",
+    ocr: {
+      "artifact-title": "Viridescent Venerer's Vessel\nGoblet of Eonothem",
+      "artifact-main-stat": "ATK\n46.6%",
+      "artifact-substats": "+ HP+209\n+ CRIT DMG+25.6%\n+ Elemental Mastery+37",
+      "artifact-set-effects": "Viridescent Venerer:\n2-Piece Set: Anemo DMG Bonus +15%",
+      "artifact-footer": "Equipped: Ganyu",
+    },
+    expect: { mainStat: "ATK%", mainValue: "46.6%" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "garbled-percent-punctuation",
+    ocr: {
+      "artifact-title": "Pristine Circlet of the Bles\n| Circlet of Logos",
+      "artifact-main-stat": "ATK\n46",
+      "artifact-substats": `+ Elemental Mastery+20·5%\n+ CRIT DMG+6.3%\n+ ATK+19\n+ DEF+12`,
+      "artifact-set-effects": "Gladiator's Finale:\n2-Piece Set: ATK +18%",
+      "artifact-footer": "Equipped: Aino",
+    },
+    expect: { slot: "Circlet of Logos" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "garbled-quotes",
+    ocr: {
+      "artifact-title": "Aloy's “Gift”\n| Circlet of Logos",
+      "artifact-main-stat": "Elemental Mastery\n46.6%",
+      "artifact-substats": "+ ATK+4.7\n+ CRIT DMG+12.4%\n+ Energy Recharge+8.1%\n+ HP+11",
+      "artifact-set-effects": "Maiden’s Beloved:\n2-Piece Set: Energy Recharge +16%",
+      "artifact-footer": "Equipped: Shenhe",
+    },
+    expect: { slot: "Circlet of Logos", mainStat: "Elemental Mastery", setName: "Maiden Beloved" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "substats-overflow-into-set-crop",
+    ocr: {
+      "artifact-title": "Moonlit Offering's Opulent Dr\nFlower of Life",
+      "artifact-main-stat": "HP\n4,780",
+      "artifact-substats": "+ CRIT DMG+13.2%",
+      "artifact-set-effects": "+ HP+15.7%\n+ DEF+12.4%\n+ Energy Recharge+5.2%\nAubade of Morningstar and Moon\n2-Piece Set: Increases Elemental Mastery by 80.",
+      "artifact-footer": "Equipped: Venti",
+    },
+    expect: {
+      substats: ["CRIT DMG+13.2%", "HP%+15.7%", "DEF%+12.4%", "Energy Recharge+5.2%"],
+    },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "percent-main-value-no-label",
+    ocr: {
+      "artifact-title": "Moonlit Offering's Final\nSands of Eon",
+      "artifact-main-stat": "46.6%\n1S 2.5.8 J\nSe",
+      "artifact-substats": "+ ATK+19\n+ Energy Recharge+6.5%\n+ CRIT DMG+18.7%",
+      "artifact-set-effects": "Aubade of Morningstar and Moon\n2-Piece Set: Increases Elemental Mastery by 80.",
+      "artifact-footer": "",
+    },
+    expect: { mainValue: "46.6%" },
+    meta: { source: SOURCE, note: "mainStat stays Unknown; value must survive" },
+  },
+  {
+    id: "derive-main-from-value-sands-def",
+    ocr: {
+      "artifact-title": "Revelation's Toll\nSands of Eon",
+      "artifact-main-stat": "58.3%\n1S 2.5.8 J\nSe",
+      "artifact-substats": "+ CRIT Rate+14.0%\n+ Elemental Mastery+33\n+ Energy Recharge+6.5%",
+      "artifact-set-effects": "Night of the Sky's Unveiling:\n2-Piece Set: Increases Elemental Mastery by 80.",
+      "artifact-footer": "Equipped: Zibai",
+    },
+    expect: { mainValue: "58.3%", mainStat: "DEF%" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "derive-main-from-value-circlet-critdmg",
+    ocr: {
+      "artifact-title": "Crown of the Saints\nCirclet of Logos",
+      "artifact-main-stat": "62.2%",
+      "artifact-substats": "+ ATK+18\n+ ATK+10.5%\n+ DEF+17.5%",
+      "artifact-set-effects": "Obsidian Codex:\n2-Piece Set: While the equipping character is in Nightsoul's Blessing",
+      "artifact-footer": "Equipped: Aino",
+    },
+    expect: { mainValue: "62.2%", mainStat: "CRIT DMG" },
+    meta: { source: SOURCE },
+  },
+  {
+    id: "noisy-fragment-circlet-critdmg",
+    ocr: {
+      "artifact-title": "Moonlit Offering's Silver Crown\nCirclet of Logos",
+      "artifact-main-stat": "6\n2 D",
+      "artifact-substats": "+ ATK+29\n+ ATK+5.8%\n+ Elemental Mastery+77\n- DEF+5.1%",
+      "artifact-set-effects": "Aubade of Morningstar and Moon\n2-Piece Set: Increases Elemental Mastery by 80.",
+      "artifact-footer": "",
+    },
+    expect: { mainValue: "62.2%", mainStat: "CRIT DMG" },
+    meta: { source: SOURCE, note: "recovers unique max-value main from noisy digit fragment" },
+  },
+  {
+    id: "noisy-fragment-sands-conservative",
+    ocr: {
+      "artifact-title": "Moonlit Offering's Final Hour\nSands of Eon",
+      "artifact-main-stat": "46.6% |\nPEE",
+      "artifact-substats": "+ ATK+19\n+ Energy Recharge+6.5%\n+ CRIT DMG+18.7%\n- DEF+53",
+      "artifact-set-effects": "Aubade of Morningstar and Moon\n2-Piece Set: Increases Elemental Mastery by 80.",
+      "artifact-footer": "",
+    },
+    expect: { mainValue: "46.6%" },
+    meta: { source: SOURCE, note: "ambiguous 46.6 value stays conservative on mainStat" },
+  },
+];
@@ -0,0 +1,44 @@
+import { describe, expect, it } from "vitest";
+import { formatReport, runOcrEval } from "./ocrEvalHarness";
+import { seedCorpus } from "./corpus/seedCorpus";
+
+// Regression gate: the seed corpus is verified ground truth, so the parser must
+// read every labeled field correctly. A drop here means an OCR/parser change
+// regressed a previously-correct read - look at the printed failures. If a
+// change intentionally alters a correct output, update the corpus label in the
+// same commit (the label is the source of truth, not the code).
+
+describe("OCR eval harness", () => {
+  const report = runOcrEval(seedCorpus);
+
+  it("prints the accuracy report", () => {
+    // Surfaced in test output for humans; not an assertion.
+    // eslint-disable-next-line no-console
+    console.log("\n" + formatReport(report) + "\n");
+    expect(report.totalCases).toBe(seedCorpus.length);
+  });
+
+  it("reads every labeled field on the seed corpus correctly", () => {
+    const failureSummary = report.failures
+      .map((failure) => {
+        const wrong = failure.fields
+          .filter((entry) => !entry.correct)
+          .map((entry) => `${entry.field}: expected "${entry.expected}" got "${entry.actual}"`)
+          .join("; ");
+        return `${failure.id} -> ${wrong}`;
+      })
+      .join("\n");
+
+    expect(report.fieldAccuracy, `field regressions:\n${failureSummary}`).toBe(1);
+    expect(report.criticalFieldAccuracy).toBe(1);
+    expect(report.exactRate).toBe(1);
+  });
+
+  it("labels every critical field at least once across the corpus", () => {
+    const criticalCoverage = report.perField.filter(
+      (entry) => ["name", "slot", "mainStat", "mainValue", "setName"].includes(entry.field) && entry.evaluated > 0,
+    );
+    // name is derived indirectly on most cases; slot/mainStat/mainValue/setName must be exercised.
+    expect(criticalCoverage.length).toBeGreaterThanOrEqual(4);
+  });
+});
@@ -0,0 +1,257 @@
+import { parseArtifactCandidate } from "../lib/artifactOcrParser";
+import type { CaptureResult, OcrResult } from "../types/global";
+
+// Field-level OCR accuracy harness. It runs the real parseArtifactCandidate
+// over a labeled corpus and reports per-field accuracy, so OCR/layout/parser
+// changes can be measured instead of guessed at. The corpus is seeded from the
+// parser test cases and grows from human-confirmed review samples
+// (see reviewSampleCorpus.ts). This is the gate every OCR change runs against.
+
+export type EvalField =
+  | "name"
+  | "slot"
+  | "level"
+  | "mainStat"
+  | "mainValue"
+  | "setName"
+  | "equipped"
+  | "substats";
+
+export const EVAL_FIELDS: EvalField[] = [
+  "name",
+  "slot",
+  "level",
+  "mainStat",
+  "mainValue",
+  "setName",
+  "equipped",
+  "substats",
+];
+
+// Fields that, if wrong, make an artifact record unusable. Reported separately
+// so a regression in "equipped" (nice-to-have) does not read the same as a
+// regression in "mainStat" (critical).
+export const CRITICAL_FIELDS: EvalField[] = ["name", "slot", "mainStat", "mainValue", "setName"];
+
+export interface OcrEvalCase {
+  /** Stable identifier, unique within a corpus. */
+  id: string;
+  /** Crop id -> raw OCR text, exactly as the capture pipeline would produce it. */
+  ocr: Record<string, string>;
+  /** Ground-truth values. Only the provided fields are scored. */
+  expect: Partial<{
+    name: string;
+    slot: string;
+    level: number;
+    mainStat: string;
+    mainValue: string;
+    setName: string;
+    equipped: string;
+    substats: string[];
+  }>;
+  meta?: {
+    resolution?: string;
+    source?: "test-seed" | "review-sample" | "manual";
+    note?: string;
+  };
+}
+
+export interface FieldResult {
+  field: EvalField;
+  expected: string;
+  actual: string;
+  correct: boolean;
+}
+
+export interface CaseResult {
+  id: string;
+  fields: FieldResult[];
+  evaluatedFields: number;
+  correctFields: number;
+  /** True when every evaluated field on the case matched. */
+  exact: boolean;
+  meta?: OcrEvalCase["meta"];
+}
+
+export interface FieldAccuracy {
+  field: EvalField;
+  evaluated: number;
+  correct: number;
+  /** 0..1, or null when the corpus never labels this field. */
+  accuracy: number | null;
+}
+
+export interface EvalReport {
+  totalCases: number;
+  exactCases: number;
+  exactRate: number;
+  evaluatedFields: number;
+  correctFields: number;
+  fieldAccuracy: number;
+  /** Accuracy restricted to CRITICAL_FIELDS. */
+  criticalFieldAccuracy: number;
+  perField: FieldAccuracy[];
+  cases: CaseResult[];
+  failures: CaseResult[];
+}
+
+export function captureFromOcr(textById: Record<string, string>): CaptureResult {
+  const ocr: OcrResult[] = Object.entries(textById).map(([id, text]) => ({
+    id,
+    label: id,
+    text,
+    confidence: 80,
+  }));
+  return {
+    id: "eval",
+    name: "eval capture",
+    width: 1920,
+    height: 1080,
+    dataUrl: "",
+    capturedAt: new Date(0).toISOString(),
+    ocr,
+  };
+}
+
+function normalizeSubstats(values: readonly string[]) {
+  return [...values].map((value) => value.replace(/\s+/g, "").toLowerCase()).sort();
+}
+
+function substatsEqual(expected: readonly string[], actual: readonly string[]) {
+  const left = normalizeSubstats(expected);
+  const right = normalizeSubstats(actual);
+  if (left.length !== right.length) return false;
+  return left.every((value, index) => value === right[index]);
+}
+
+function evaluateCase(evalCase: OcrEvalCase): CaseResult {
+  const parsed = parseArtifactCandidate(captureFromOcr(evalCase.ocr));
+  const fields: FieldResult[] = [];
+  const stringValues: Record<string, string> = parsed
+    ? {
+        name: parsed.name,
+        slot: parsed.slot,
+        mainStat: parsed.mainStat,
+        mainValue: parsed.mainValue,
+        setName: parsed.setName,
+        equipped: parsed.equipped,
+      }
+    : {};
+
+  for (const field of EVAL_FIELDS) {
+    if (!(field in evalCase.expect)) continue;
+
+    if (field === "substats") {
+      const expected = evalCase.expect.substats ?? [];
+      const actual = parsed?.substats ?? [];
+      fields.push({
+        field,
+        expected: expected.join(", "),
+        actual: actual.join(", "),
+        correct: substatsEqual(expected, actual),
+      });
+      continue;
+    }
+
+    if (field === "level") {
+      const expected = evalCase.expect.level;
+      const actual = parsed?.level;
+      fields.push({
+        field,
+        expected: String(expected),
+        actual: parsed ? String(actual) : "<no-parse>",
+        correct: parsed != null && actual === expected,
+      });
+      continue;
+    }
+
+    const expected = String(evalCase.expect[field] ?? "");
+    const actual = parsed ? stringValues[field] ?? "" : "<no-parse>";
+    fields.push({ field, expected, actual, correct: parsed != null && actual === expected });
+  }
+
+  const correctFields = fields.filter((entry) => entry.correct).length;
+  return {
+    id: evalCase.id,
+    fields,
+    evaluatedFields: fields.length,
+    correctFields,
+    exact: fields.length > 0 && correctFields === fields.length,
+    meta: evalCase.meta,
+  };
+}
+
+export function runOcrEval(cases: readonly OcrEvalCase[]): EvalReport {
+  const caseResults = cases.map(evaluateCase);
+
+  const perField: FieldAccuracy[] = EVAL_FIELDS.map((field) => {
+    let evaluated = 0;
+    let correct = 0;
+    for (const caseResult of caseResults) {
+      const entry = caseResult.fields.find((candidate) => candidate.field === field);
+      if (!entry) continue;
+      evaluated++;
+      if (entry.correct) correct++;
+    }
+    return { field, evaluated, correct, accuracy: evaluated === 0 ? null : correct / evaluated };
+  });
+
+  const evaluatedFields = caseResults.reduce((sum, entry) => sum + entry.evaluatedFields, 0);
+  const correctFields = caseResults.reduce((sum, entry) => sum + entry.correctFields, 0);
+  const exactCases = caseResults.filter((entry) => entry.exact).length;
+
+  const criticalEvaluated = perField
+    .filter((entry) => CRITICAL_FIELDS.includes(entry.field))
+    .reduce((sum, entry) => sum + entry.evaluated, 0);
+  const criticalCorrect = perField
+    .filter((entry) => CRITICAL_FIELDS.includes(entry.field))
+    .reduce((sum, entry) => sum + entry.correct, 0);
+
+  return {
+    totalCases: caseResults.length,
+    exactCases,
+    exactRate: caseResults.length === 0 ? 1 : exactCases / caseResults.length,
+    evaluatedFields,
+    correctFields,
+    fieldAccuracy: evaluatedFields === 0 ? 1 : correctFields / evaluatedFields,
+    criticalFieldAccuracy: criticalEvaluated === 0 ? 1 : criticalCorrect / criticalEvaluated,
+    perField,
+    cases: caseResults,
+    failures: caseResults.filter((entry) => !entry.exact),
+  };
+}
+
+function pct(value: number | null) {
+  if (value === null) return "  n/a";
+  return `${(value * 100).toFixed(1)}%`.padStart(6, " ");
+}
+
+export function formatReport(report: EvalReport): string {
+  const lines: string[] = [];
+  lines.push("OCR eval report");
+  lines.push("=".repeat(48));
+  lines.push(`Cases:              ${report.totalCases}`);
+  lines.push(`Exact-match cases:  ${report.exactCases}/${report.totalCases} (${pct(report.exactRate).trim()})`);
+  lines.push(`Field accuracy:     ${pct(report.fieldAccuracy).trim()} (${report.correctFields}/${report.evaluatedFields})`);
+  lines.push(`Critical fields:    ${pct(report.criticalFieldAccuracy).trim()}`);
+  lines.push("");
+  lines.push("Per field:");
+  for (const entry of report.perField) {
+    const critical = CRITICAL_FIELDS.includes(entry.field) ? "*" : " ";
+    lines.push(`  ${critical} ${entry.field.padEnd(11)} ${pct(entry.accuracy)}  (${entry.correct}/${entry.evaluated})`);
+  }
+
+  if (report.failures.length > 0) {
+    lines.push("");
+    lines.push(`Failures (${report.failures.length}):`);
+    for (const failure of report.failures) {
+      const wrong = failure.fields.filter((entry) => !entry.correct);
+      lines.push(`  - ${failure.id}`);
+      for (const entry of wrong) {
+        lines.push(`      ${entry.field}: expected "${entry.expected}" got "${entry.actual}"`);
+      }
+    }
+  }
+
+  return lines.join("\n");
+}
@@ -0,0 +1,77 @@
+import { describe, expect, it } from "vitest";
+import type { ReviewSampleRecord } from "../types/global";
+import { reviewSampleToEvalCase, reviewSamplesToEvalCases } from "./reviewSampleCorpus";
+
+function record(overrides: Partial<ReviewSampleRecord["sample"]> = {}, savedAt = "2026-07-05T10:00:00.000Z"): ReviewSampleRecord {
+  return {
+    savedAt,
+    sample: {
+      reason: "automatic:low-confidence",
+      parsed: {
+        name: "Gladiator's Nostalgia",
+        slot: "Flower of Life",
+        level: 20,
+        mainStat: "HP",
+        mainValue: "4,780",
+        setName: "Gladiator's Finale",
+        equipped: "Bennett",
+        substats: ["CRIT DMG+12.4%", "ATK+9.9%"],
+      },
+      capture: {
+        width: 2560,
+        height: 1440,
+        ocr: [
+          { id: "artifact-title", label: "t", text: "Gladiator's Nostalgia\nFlower of Life", confidence: 80 },
+          { id: "artifact-main-stat", label: "m", text: "HP\n4,780", confidence: 80 },
+        ],
+      },
+      ...overrides,
+    },
+  };
+}
+
+describe("reviewSampleToEvalCase", () => {
+  it("extracts ocr text and parser guesses as an unconfirmed case", () => {
+    const evalCase = reviewSampleToEvalCase(record());
+    expect(evalCase).not.toBeNull();
+    expect(evalCase?.confirmed).toBe(false);
+    expect(evalCase?.ocr["artifact-title"]).toContain("Gladiator's Nostalgia");
+    expect(evalCase?.expect.slot).toBe("Flower of Life");
+    expect(evalCase?.expect.level).toBe(20);
+    expect(evalCase?.meta?.source).toBe("review-sample");
+    expect(evalCase?.meta?.resolution).toBe("2560x1440");
+  });
+
+  it("drops Unknown/Not detected sentinels so they are not treated as labels", () => {
+    const evalCase = reviewSampleToEvalCase(
+      record({
+        parsed: {
+          name: "Unknown artifact",
+          slot: "Circlet of Logos",
+          mainStat: "Unknown main stat",
+          mainValue: "?",
+          setName: "Unknown set",
+          equipped: "Not detected",
+        },
+      }),
+    );
+    expect(evalCase?.expect.name).toBeUndefined();
+    expect(evalCase?.expect.mainStat).toBeUndefined();
+    expect(evalCase?.expect.setName).toBeUndefined();
+    expect(evalCase?.expect.equipped).toBeUndefined();
+    // "?" mainValue is a sentinel non-answer, must not leak in as a label.
+    expect(evalCase?.expect.mainValue).toBeUndefined();
+    expect(evalCase?.expect.slot).toBe("Circlet of Logos");
+  });
+
+  it("returns null when the sample has no OCR", () => {
+    expect(reviewSampleToEvalCase({ savedAt: "x", sample: { capture: { ocr: [] } } })).toBeNull();
+    expect(reviewSampleToEvalCase({ savedAt: "x" })).toBeNull();
+  });
+
+  it("maps a list of records and skips empty ones", () => {
+    const cases = reviewSamplesToEvalCases([record(), { savedAt: "y" }, record({}, "2026-07-05T11:00:00.000Z")]);
+    expect(cases).toHaveLength(2);
+    expect(new Set(cases.map((entry) => entry.id)).size).toBe(2);
+  });
+});
@@ -0,0 +1,92 @@
+import type { ReviewSampleRecord } from "../types/global";
+import type { OcrEvalCase } from "./ocrEvalHarness";
+
+// Turns saved review samples into eval cases so the review queue becomes a
+// growing labeled corpus (see docs/ocr-eval.md).
+//
+// IMPORTANT: a review sample's `parsed` block is the *auto*-parse that was
+// flagged for review - it is a label CANDIDATE, not verified ground truth.
+// Using it directly as an expectation would be circular (the parser grading
+// itself). The intended flow is:
+//   1. reviewSampleToEvalCase() extracts the OCR + the parser's current guess.
+//   2. A human confirms or corrects `expect` in the produced case.
+//   3. The corrected case is committed into src/eval/corpus/.
+// The `confirmed` flag records whether step 2 happened.
+
+export interface ReviewSampleEvalCase extends OcrEvalCase {
+  /** False until a human has verified/corrected the `expect` values. */
+  confirmed: boolean;
+}
+
+function ocrMapFromRecord(record: ReviewSampleRecord): Record<string, string> | null {
+  const entries = record.sample?.capture?.ocr;
+  if (!entries?.length) return null;
+  const map: Record<string, string> = {};
+  for (const entry of entries) {
+    if (typeof entry?.id === "string" && typeof entry?.text === "string") {
+      map[entry.id] = entry.text;
+    }
+  }
+  return Object.keys(map).length > 0 ? map : null;
+}
+
+function expectFromParsed(parsed: unknown): OcrEvalCase["expect"] {
+  if (!parsed || typeof parsed !== "object") return {};
+  const candidate = parsed as Record<string, unknown>;
+  const expect: OcrEvalCase["expect"] = {};
+
+  const stringField = (key: "name" | "slot" | "mainStat" | "mainValue" | "setName" | "equipped") => {
+    const value = candidate[key];
+    // Skip the parser's own "Unknown ..." / "Not detected" / "?" sentinels -
+    // those are non-answers, not labels a human would confirm.
+    if (typeof value === "string" && value && !value.startsWith("Unknown") && value !== "Not detected" && value !== "?") {
+      expect[key] = value;
+    }
+  };
+
+  stringField("name");
+  stringField("slot");
+  stringField("mainStat");
+  stringField("mainValue");
+  stringField("setName");
+  stringField("equipped");
+
+  if (typeof candidate.level === "number" && Number.isFinite(candidate.level) && candidate.level > 0) {
+    expect.level = candidate.level;
+  }
+  if (Array.isArray(candidate.substats) && candidate.substats.every((entry) => typeof entry === "string")) {
+    expect.substats = candidate.substats as string[];
+  }
+
+  return expect;
+}
+
+export function reviewSampleToEvalCase(record: ReviewSampleRecord, index = 0): ReviewSampleEvalCase | null {
+  const ocr = ocrMapFromRecord(record);
+  if (!ocr) return null;
+
+  const savedAt = record.savedAt ?? "unknown";
+  const idSuffix = savedAt.replace(/[^0-9A-Za-z]/g, "").slice(0, 14) || String(index);
+  const capture = record.sample?.capture;
+
+  return {
+    id: `review-${idSuffix}-${index}`,
+    ocr,
+    expect: expectFromParsed(record.sample?.parsed),
+    confirmed: false,
+    meta: {
+      source: "review-sample",
+      resolution: capture?.width && capture?.height ? `${capture.width}x${capture.height}` : undefined,
+      note: record.sample?.reason,
+    },
+  };
+}
+
+export function reviewSamplesToEvalCases(records: readonly ReviewSampleRecord[] | null | undefined) {
+  const cases: ReviewSampleEvalCase[] = [];
+  (records ?? []).forEach((record, index) => {
+    const evalCase = reviewSampleToEvalCase(record, index);
+    if (evalCase) cases.push(evalCase);
+  });
+  return cases;
+}