From b7dbc618b351dc72f1aa6ff97eeb366784dfaf6a Mon Sep 17 00:00:00 2001 From: AzuTear Date: Sun, 5 Jul 2026 20:41:30 +0200 Subject: [PATCH] feat(eval): add field-level OCR accuracy harness + seed corpus Adds a measurement gate for the artifact OCR parser (ADR-007), the prerequisite for the layout-profile and preprocessing rework. runOcrEval feeds labeled OCR text through the real parseArtifactCandidate and scores per-field / per-case accuracy. - src/eval/ocrEvalHarness.ts: pure metrics (per-field, critical-field, exact). - src/eval/corpus/seedCorpus.ts: 23 cases transcribed from the verified parser test assertions; runs at 100%. - src/eval/reviewSampleCorpus.ts: converts review samples into label *candidates* (never ground truth) so the review queue can grow the corpus. - src/eval/ocrEval.test.ts + reviewSampleCorpus.test.ts: gate (must stay 1.0) and converter unit tests. - npm run eval script; docs/ocr-eval.md; ADR-007/008/009. Also records the agreed rework direction: C# input/capture sidecar (ADR-008) and resolution-anchored layout profiles + OCR preprocessing (ADR-009). Co-Authored-By: Claude Opus 4.8 --- docs/DECISIONS.md | 102 +++++++++ docs/ocr-eval.md | 49 +++++ package.json | 1 + src/eval/corpus/seedCorpus.ts | 311 ++++++++++++++++++++++++++++ src/eval/ocrEval.test.ts | 44 ++++ src/eval/ocrEvalHarness.ts | 257 +++++++++++++++++++++++ src/eval/reviewSampleCorpus.test.ts | 77 +++++++ src/eval/reviewSampleCorpus.ts | 92 ++++++++ 8 files changed, 933 insertions(+) create mode 100644 docs/ocr-eval.md create mode 100644 src/eval/corpus/seedCorpus.ts create mode 100644 src/eval/ocrEval.test.ts create mode 100644 src/eval/ocrEvalHarness.ts create mode 100644 src/eval/reviewSampleCorpus.test.ts create mode 100644 src/eval/reviewSampleCorpus.ts diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 43c00f5..c0c41e1 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -12,6 +12,9 @@ This document contains Architecture Decision Records. | ADR-004 | Treat in-game marking as a later opt-in feature | Accepted | 2026-07-04 | | ADR-005 | Use a generated Genshin data package for OCR matching | Accepted | 2026-07-04 | | ADR-006 | Persistent input helper and JSON artifact store before SQLite | Accepted | 2026-07-04 | +| ADR-007 | Measure OCR accuracy with a labeled eval harness before reworking the scanner | Accepted | 2026-07-05 | +| ADR-008 | Replace the PowerShell input/capture helper with a C# sidecar | Accepted | 2026-07-05 | +| ADR-009 | Resolution-anchored layout profiles and OCR preprocessing over color detection | Accepted | 2026-07-05 | ## ADR-001: Build A Local Electron App First @@ -133,3 +136,102 @@ Run one persistent PowerShell helper process (compiled once, JSON protocol over - Automated clicks are verified by checking that the parsed detail signature changed; repeated failures abort with a diagnosis hint instead of clicking blindly. - Leveling an artifact changes its signature and creates a new record; rescan-merge is an open follow-up. - If the helper process dies it is respawned on the next request; pending requests fail loudly instead of hanging. + +## ADR-007: Measure OCR Accuracy With A Labeled Eval Harness Before Reworking The Scanner + +### Status + +Accepted + +### Context + +OCR and crop reliability are the core product risk (ADR-002), but there was no +way to measure field-level accuracy. Every OCR, crop, or parser change was a +blind change - regressions could only be caught by a human re-testing against the +live game, and there was no baseline to compare a new engine or preprocessing +step against. + +### Decision + +Add a field-level eval harness (`src/eval/`) that runs the real +`parseArtifactCandidate` over a labeled corpus and reports per-field and +per-case accuracy. Seed the corpus from the existing parser test cases (verified +labels), and grow it from human-confirmed review samples via +`reviewSampleToEvalCase`. Run it as a gate under `npm test` (must stay 100% on +the verified seed) and as a full report via `npm run eval`. + +### Consequences + +- OCR/layout/preprocessing changes are measured, not guessed at; a new engine or + a preprocessing step has to beat a recorded baseline. +- The review queue does double duty: it flags artifacts for human correction and + feeds the eval corpus. Review-sample `parsed` blocks are label *candidates*, + never ground truth, to avoid the parser grading itself. +- The corpus label is the source of truth. If a code change intentionally alters + a correct output, the label is updated in the same commit. + +## ADR-008: Replace The PowerShell Input/Capture Helper With A C# Sidecar + +### Status + +Accepted + +### Context + +The persistent PowerShell helper (ADR-006) still carries Windows PowerShell 5.1 +quirks (the `Marshal::SizeOf` interop bug), compiles Win32 interop at startup, +and captures each frame by writing a PNG to the temp directory and reading it +back. Inventory Kamera - the proven reference for automated Genshin scanning - +uses a C#/.NET stack with InputSimulator (SendInput) and direct GDI/BitBlt +capture. + +### Decision + +Replace the PowerShell helper with a self-contained .NET (C#) sidecar that speaks +the same JSON-over-stdin/stdout protocol, so the Electron-side `InputHelperService` +interface stays stable. The sidecar does per-monitor DPI-aware SendInput +click/scroll, BitBlt client-rect capture returning bytes without a temp file, and +elevation detection. + +### Consequences + +- No PS 5.1 marshalling bugs, no per-call interop compile, no temp-PNG churn; + lower latency makes batch scans and the ESC/mouse failsafe polling cheaper. +- Adds a .NET build/publish step and ships a compiled exe with the app. +- The migration is behind the existing service interface, so the renderer and + scan loop do not change. + +## ADR-009: Resolution-Anchored Layout Profiles And OCR Preprocessing Over Color Detection + +### Status + +Accepted + +### Context + +The current pipeline finds the artifact detail panel with hardcoded orange/green +color thresholds (`inferDetailRect`) and then crops fixed percentages of that +guessed rectangle. This is brittle against HDR, color profiles, UI scale, aspect +ratio, and game UI updates. Inventory Kamera instead requires borderless 16:9 and +scales fixed crop coordinates from a reference resolution, then feeds Tesseract +preprocessed (grayscale, upscaled, thresholded) crops - which is why general +Tesseract is accurate enough for them. + +### Decision + +Adopt the same approach: require borderless 16:9, drive crops from +resolution-anchored layout profiles scaled from a reference resolution (color +detection only as a fallback), and add a per-region preprocessing pass +(grayscale, upscale, threshold, invert) plus a digit-whitelist mode for numeric +fields. The set name stays derived from the static piece-to-set data package (no +dedicated set-effect crop). English-only OCR is accepted. Every change is +validated against the ADR-007 eval harness. + +### Consequences + +- Crop positions become deterministic per resolution instead of per-frame guesses. +- Preprocessing is expected to lift accuracy enough that a custom OCR engine is + only pursued if the eval harness shows Tesseract-plus-preprocessing is + insufficient. +- Non-16:9 or non-borderless setups are explicitly unsupported for the auto + scanner; the app should detect and warn rather than silently misread. diff --git a/docs/ocr-eval.md b/docs/ocr-eval.md new file mode 100644 index 0000000..5273adb --- /dev/null +++ b/docs/ocr-eval.md @@ -0,0 +1,49 @@ +# OCR Eval Harness + +Field-level accuracy measurement for the artifact OCR parser. This is the gate +every OCR, crop, layout, or parser change runs against (see ADR-007). + +## Run it + +```powershell +npm run eval # full accuracy report for the seed corpus +npm test # runs the eval gate alongside the rest of the suite +``` + +The report prints exact-match rate, overall field accuracy, a per-field +breakdown (critical fields marked with `*`), and every failing case with an +`expected "..." got "..."` diff. + +## How it works + +- `src/eval/ocrEvalHarness.ts` - pure metric functions. `runOcrEval(cases)` + feeds each case's OCR text through the real `parseArtifactCandidate` and scores + the produced fields against the labels. Order-independent for substats. +- `src/eval/corpus/seedCorpus.ts` - the seed corpus, transcribed from the + verified assertions in `src/lib/artifactOcrParser.test.ts`. Must stay at 100%. +- `src/eval/ocrEval.test.ts` - the gate: seed field accuracy, critical-field + accuracy, and exact-match rate must all be 1.0. + +## Growing the corpus from review samples + +The review queue is the corpus source. A saved review sample carries the OCR +text plus the parser's *guess* - `reviewSampleToEvalCase` extracts both. + +The parser's guess is a label **candidate, not ground truth** (using it directly +would be the parser grading itself). To add a real case: + +1. Convert review samples with `reviewSamplesToEvalCases(records)`. +2. Open each produced case and confirm or correct the `expect` values against + what the artifact actually is in-game. Set `confirmed: true`. +3. Move the corrected case into a file under `src/eval/corpus/` and add it to the + corpus array. + +Prefer cases that cover new failure modes: unseen resolutions, new sets or +characters, and OCR noise the current corpus does not exercise. + +## When a change moves a number + +- Accuracy **drops**: a regression. Read the printed failures; fix the parser or + revert. Do not lower the threshold to make it pass. +- A change **intentionally** alters a previously-correct output: update the + corpus label in the same commit. The label is the source of truth, not the code. diff --git a/package.json b/package.json index 295de94..8f5b356 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "start": "electron .", "lint": "tsc --noEmit", "test": "vitest run", + "eval": "vitest run src/eval/ocrEval.test.ts", "data:genshin": "node scripts/generate-genshin-data.cjs" }, "dependencies": { diff --git a/src/eval/corpus/seedCorpus.ts b/src/eval/corpus/seedCorpus.ts new file mode 100644 index 0000000..6ebfce4 --- /dev/null +++ b/src/eval/corpus/seedCorpus.ts @@ -0,0 +1,311 @@ +import type { OcrEvalCase } from "../ocrEvalHarness"; + +// Seed corpus transcribed from src/lib/artifactOcrParser.test.ts. Every label +// here is an exact assertion that already passes, so a fresh run starts at (or +// very near) 100% - which is what makes the accuracy thresholds meaningful as a +// regression gate. Cases that deliberately expect conservative "Unknown ..." +// behavior contribute only their positively-identified fields (level, value) +// so field accuracy stays a measure of correct reads, not correct refusals. +// +// Grow this corpus from human-confirmed review samples via reviewSampleCorpus.ts. + +const SOURCE = "test-seed" as const; + +export const seedCorpus: OcrEvalCase[] = [ + { + id: "spring-sands-em", + ocr: { + "artifact-title": "A Note in Spring's Lei\n| Sands of Eon Vi", + "artifact-main-stat": "Elemental Mastery\n187", + "artifact-substats": "+20\n+ ATK+29\n+ CRIT DMG+15.5%\n+ CRIT Rate+2.7%\nATK + 15", + "artifact-set-effects": "A Day Carved From Rising Winds\n2-Piece Set: ATK +18%.", + "artifact-footer": "WV Equipped: Citlali\naR 0", + }, + expect: { + slot: "Sands of Eon", + level: 20, + mainStat: "Elemental Mastery", + setName: "A Day Carved From Rising Winds", + equipped: "Citlali", + }, + meta: { source: SOURCE }, + }, + { + id: "moonlit-flower-hp", + ocr: { + "artifact-title": "Moonlit Offering's Opulent Dr\n| Flower of Life 2", + "artifact-main-stat": "4,780\nAhhh", + "artifact-substats": "+ Elemental Mastery+54\n+ CRIT Rate+7.4%\n+ ATK+4.7%", + "artifact-set-effects": "Aubade of Morningstar and Moor\n2-Piece Set: Increases Elemental Mastery by 80.", + "artifact-footer": "? Equipped: Ineffa\naR 0", + }, + expect: { + slot: "Flower of Life", + mainStat: "HP", + mainValue: "4,780", + setName: "Aubade of Morningstar and Moon", + equipped: "Ineffa", + }, + meta: { source: SOURCE }, + }, + { + id: "vv-determination-sands-er", + ocr: { + "artifact-title": "Viridescent Venerer's Determination\nSands of Eon Vi", + "artifact-main-stat": "Energy Recharge\n51.8%", + "artifact-substats": "+ ATK+5.8%\n+ Elemental Mastery+37\n+ HP+11.7%\n+ ATK+54", + "artifact-set-effects": "Viridescent Venerer:\n2-Piece Set: Anemo DMG Bonus +15%", + "artifact-footer": "Equipped: Sucrose", + }, + expect: { slot: "Sands of Eon", mainStat: "Energy Recharge" }, + meta: { source: SOURCE }, + }, + { + id: "pristine-plume-atk", + ocr: { + "artifact-title": "Pristine Plume of the Bles\n| Plume of Death", + "artifact-main-stat": "31 !\npr", + "artifact-substats": "+20\n+ CRIT DMG+7.0%\n+ DEF+30.6%\n+ Elemental Mastery+40\nATK +5.8%", + "artifact-set-effects": "Silken Moon's Serenade:\n2-Piece Set: Energy Recharge +20%.", + "artifact-footer": "Equipped: Aino\nBR", + }, + expect: { + slot: "Plume of Death", + mainStat: "ATK", + mainValue: "311", + setName: "Silken Moon's Serenade", + equipped: "Aino", + }, + meta: { source: SOURCE }, + }, + { + id: "deep-gallery-goblet-cryo", + ocr: { + "artifact-title": "Deep Gallery's Bestowed Banquet\nGoblet of Eonothem", + "artifact-main-stat": "Cryo DMG Bonus\n46.6%", + "artifact-substats": "+ CRIT Rate+6.6%\n+ CRIT DMG+12.4%\n+ HP+269\n+ ATK+16.3%", + "artifact-set-effects": "Finale of the Deep Galleries:\n2-Piece Set: Cryo DMG Bonus +15%", + "artifact-footer": "Equipped: Skirk", + }, + expect: { slot: "Goblet of Eonothem", mainStat: "Cryo DMG Bonus", mainValue: "46.6%" }, + meta: { source: SOURCE }, + }, + { + id: "holy-crown-circlet-crit", + ocr: { + "artifact-title": "Holy Crown of the Believer\nCirclet of Logos", + "artifact-main-stat": "CRIT Rate\n31.1%", + "artifact-substats": "+ ATK+4.7%\n+ Elemental Mastery+56\n+ DEF+37\n+ Energy Recharge+11.7%", + "artifact-set-effects": "Silken Moon's Serenade:\n2-Piece Set: Energy Recharge +20%.", + "artifact-footer": "Equipped: Chongyun", + }, + expect: { slot: "Circlet of Logos", mainStat: "CRIT Rate", mainValue: "31.1%" }, + meta: { source: SOURCE }, + }, + { + id: "beast-tamer-flower-set", + ocr: { + "artifact-title": "Beast Tamer's Talisman\nFlower of Life", + "artifact-main-stat": "HP\n4,780", + "artifact-substats": "+ HP+16.3%\n+ CRIT DMG+7.0%\n+ Energy Recharge+6.5%\n+ Elemental Mastery+68", + "artifact-set-effects": "Scroll of the Hero of Cinder City:\n2-Piece Set: When a nearby party member triggers a Nightsoul Burst", + "artifact-footer": "Equipped: Citlali", + }, + expect: { setName: "Scroll of the Hero of Cinder City" }, + meta: { source: SOURCE }, + }, + { + id: "piece-name-to-set-fallback", + ocr: { + "artifact-title": "Holy Crown of the Believer\nCirclet of Logos", + "artifact-main-stat": "CRIT Rate\n31.1%", + "artifact-substats": "+ ATK+4.7%\n+ Elemental Mastery+56\n+ DEF+37", + "artifact-set-effects": "unreadable noisy set text", + "artifact-footer": "Equipped: Chongyun", + }, + expect: { setName: "Silken Moon's Serenade" }, + meta: { source: SOURCE, note: "set derived from piece name when set-effect OCR is noise" }, + }, + { + id: "piece-name-to-slot-fallback", + ocr: { + "artifact-title": "Viridescent Venerer's Determination", + "artifact-main-stat": "Energy Recharge\n51.8%", + "artifact-substats": "+ ATK+5.8%\n+ Elemental Mastery+37\n+ HP+11.7%\n+ ATK+54", + "artifact-set-effects": "Viridescent Venerer:\n2-Piece Set: Anemo DMG Bonus +15%", + "artifact-footer": "Equipped: Sucrose", + }, + expect: { slot: "Sands of Eon", setName: "Viridescent Venerer" }, + meta: { source: SOURCE }, + }, + { + id: "hourglass-atk-percent", + ocr: { + "artifact-title": "Hourglass of Thunder\nSands of Eon", + "artifact-main-stat": "ATK\n40.7%", + "artifact-substats": "+17\n+ CRIT DMG+14.8%\n+ Elemental Mastery+21\n+ ATK+53\n+ DEF+19", + "artifact-set-effects": "Thundering Fury:\n2-Piece Set: Electro DMG Bonus +15%", + "artifact-footer": "Equipped: Fischl", + }, + expect: { level: 17, slot: "Sands of Eon", mainStat: "ATK%", mainValue: "40.7%" }, + meta: { source: SOURCE }, + }, + { + id: "night-realm-sands-ambiguous", + ocr: { + "artifact-title": "Myths of the Night Realm\nSands of Eon", + "artifact-main-stat": "30.8%", + "artifact-substats": "+12\n+ Elemental Mastery+16\n+ CRIT DMG+6.2%\n+ DEF+42\n+ HP+9.9%", + "artifact-set-effects": "Obsidian Codex:\n2-Piece Set: While the equipping character is in Nightsoul's Blessing", + "artifact-footer": "Equipped: Sandrone", + }, + expect: { level: 12, mainValue: "30.8%" }, + meta: { source: SOURCE, note: "mainStat intentionally stays Unknown (ambiguous family)" }, + }, + { + id: "footer-noise-equipped", + ocr: { + "artifact-title": "Gladiator's Nostalgia\nFlower of Life", + "artifact-main-stat": "HP\n4,780", + "artifact-substats": "+ Energy Recharge+11.0%\n+ ATK+9.9%\n+ HP+14.6%\n+ CRIT DMG+12.4%", + "artifact-set-effects": "Gladiator's Finale:\n2-Piece Set: ATK +18%", + "artifact-footer": "1 gv 0RY If tha anuninnina\nJl Equipped: Bennett\nCEE", + }, + expect: { equipped: "Bennett" }, + meta: { source: SOURCE }, + }, + { + id: "atk-percent-sands", + ocr: { + "artifact-title": "Myths of the Night Realm\nSands of Eon", + "artifact-main-stat": "ATK\n30.8%", + "artifact-substats": "+ Elemental Mastery+16\n+ CRIT DMG+6.2%\n+ DEF+42\n+ HP+9.9%", + "artifact-set-effects": "Obsidian Codex:\n2-Piece Set: While the equipping character is in Nightsoul's Blessing", + "artifact-footer": "Equipped: Sandrone", + }, + expect: { mainStat: "ATK%", mainValue: "30.8%" }, + meta: { source: SOURCE }, + }, + { + id: "atk-percent-circlet", + ocr: { + "artifact-title": "Maiden's Fading Beauty\nCirclet of Logos", + "artifact-main-stat": "ATK\n46.6%", + "artifact-substats": "+ ATK+29\n+ CRIT DMG+10.9%\n+ CRIT Rate+7.0%", + "artifact-set-effects": "Maiden Beloved:\n2-Piece Set: Character Healing Effectiveness +15%", + "artifact-footer": "Equipped: Qiqi", + }, + expect: { mainStat: "ATK%", mainValue: "46.6%" }, + meta: { source: SOURCE }, + }, + { + id: "atk-percent-goblet", + ocr: { + "artifact-title": "Viridescent Venerer's Vessel\nGoblet of Eonothem", + "artifact-main-stat": "ATK\n46.6%", + "artifact-substats": "+ HP+209\n+ CRIT DMG+25.6%\n+ Elemental Mastery+37", + "artifact-set-effects": "Viridescent Venerer:\n2-Piece Set: Anemo DMG Bonus +15%", + "artifact-footer": "Equipped: Ganyu", + }, + expect: { mainStat: "ATK%", mainValue: "46.6%" }, + meta: { source: SOURCE }, + }, + { + id: "garbled-percent-punctuation", + ocr: { + "artifact-title": "Pristine Circlet of the Bles\n| Circlet of Logos", + "artifact-main-stat": "ATK\n46", + "artifact-substats": `+ Elemental Mastery+20·5%\n+ CRIT DMG+6.3%\n+ ATK+19\n+ DEF+12`, + "artifact-set-effects": "Gladiator's Finale:\n2-Piece Set: ATK +18%", + "artifact-footer": "Equipped: Aino", + }, + expect: { slot: "Circlet of Logos" }, + meta: { source: SOURCE }, + }, + { + id: "garbled-quotes", + ocr: { + "artifact-title": "Aloy's “Gift”\n| Circlet of Logos", + "artifact-main-stat": "Elemental Mastery\n46.6%", + "artifact-substats": "+ ATK+4.7\n+ CRIT DMG+12.4%\n+ Energy Recharge+8.1%\n+ HP+11", + "artifact-set-effects": "Maiden’s Beloved:\n2-Piece Set: Energy Recharge +16%", + "artifact-footer": "Equipped: Shenhe", + }, + expect: { slot: "Circlet of Logos", mainStat: "Elemental Mastery", setName: "Maiden Beloved" }, + meta: { source: SOURCE }, + }, + { + id: "substats-overflow-into-set-crop", + ocr: { + "artifact-title": "Moonlit Offering's Opulent Dr\nFlower of Life", + "artifact-main-stat": "HP\n4,780", + "artifact-substats": "+ CRIT DMG+13.2%", + "artifact-set-effects": "+ HP+15.7%\n+ DEF+12.4%\n+ Energy Recharge+5.2%\nAubade of Morningstar and Moon\n2-Piece Set: Increases Elemental Mastery by 80.", + "artifact-footer": "Equipped: Venti", + }, + expect: { + substats: ["CRIT DMG+13.2%", "HP%+15.7%", "DEF%+12.4%", "Energy Recharge+5.2%"], + }, + meta: { source: SOURCE }, + }, + { + id: "percent-main-value-no-label", + ocr: { + "artifact-title": "Moonlit Offering's Final\nSands of Eon", + "artifact-main-stat": "46.6%\n1S 2.5.8 J\nSe", + "artifact-substats": "+ ATK+19\n+ Energy Recharge+6.5%\n+ CRIT DMG+18.7%", + "artifact-set-effects": "Aubade of Morningstar and Moon\n2-Piece Set: Increases Elemental Mastery by 80.", + "artifact-footer": "", + }, + expect: { mainValue: "46.6%" }, + meta: { source: SOURCE, note: "mainStat stays Unknown; value must survive" }, + }, + { + id: "derive-main-from-value-sands-def", + ocr: { + "artifact-title": "Revelation's Toll\nSands of Eon", + "artifact-main-stat": "58.3%\n1S 2.5.8 J\nSe", + "artifact-substats": "+ CRIT Rate+14.0%\n+ Elemental Mastery+33\n+ Energy Recharge+6.5%", + "artifact-set-effects": "Night of the Sky's Unveiling:\n2-Piece Set: Increases Elemental Mastery by 80.", + "artifact-footer": "Equipped: Zibai", + }, + expect: { mainValue: "58.3%", mainStat: "DEF%" }, + meta: { source: SOURCE }, + }, + { + id: "derive-main-from-value-circlet-critdmg", + ocr: { + "artifact-title": "Crown of the Saints\nCirclet of Logos", + "artifact-main-stat": "62.2%", + "artifact-substats": "+ ATK+18\n+ ATK+10.5%\n+ DEF+17.5%", + "artifact-set-effects": "Obsidian Codex:\n2-Piece Set: While the equipping character is in Nightsoul's Blessing", + "artifact-footer": "Equipped: Aino", + }, + expect: { mainValue: "62.2%", mainStat: "CRIT DMG" }, + meta: { source: SOURCE }, + }, + { + id: "noisy-fragment-circlet-critdmg", + ocr: { + "artifact-title": "Moonlit Offering's Silver Crown\nCirclet of Logos", + "artifact-main-stat": "6\n2 D", + "artifact-substats": "+ ATK+29\n+ ATK+5.8%\n+ Elemental Mastery+77\n- DEF+5.1%", + "artifact-set-effects": "Aubade of Morningstar and Moon\n2-Piece Set: Increases Elemental Mastery by 80.", + "artifact-footer": "", + }, + expect: { mainValue: "62.2%", mainStat: "CRIT DMG" }, + meta: { source: SOURCE, note: "recovers unique max-value main from noisy digit fragment" }, + }, + { + id: "noisy-fragment-sands-conservative", + ocr: { + "artifact-title": "Moonlit Offering's Final Hour\nSands of Eon", + "artifact-main-stat": "46.6% |\nPEE", + "artifact-substats": "+ ATK+19\n+ Energy Recharge+6.5%\n+ CRIT DMG+18.7%\n- DEF+53", + "artifact-set-effects": "Aubade of Morningstar and Moon\n2-Piece Set: Increases Elemental Mastery by 80.", + "artifact-footer": "", + }, + expect: { mainValue: "46.6%" }, + meta: { source: SOURCE, note: "ambiguous 46.6 value stays conservative on mainStat" }, + }, +]; diff --git a/src/eval/ocrEval.test.ts b/src/eval/ocrEval.test.ts new file mode 100644 index 0000000..cb3710d --- /dev/null +++ b/src/eval/ocrEval.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { formatReport, runOcrEval } from "./ocrEvalHarness"; +import { seedCorpus } from "./corpus/seedCorpus"; + +// Regression gate: the seed corpus is verified ground truth, so the parser must +// read every labeled field correctly. A drop here means an OCR/parser change +// regressed a previously-correct read - look at the printed failures. If a +// change intentionally alters a correct output, update the corpus label in the +// same commit (the label is the source of truth, not the code). + +describe("OCR eval harness", () => { + const report = runOcrEval(seedCorpus); + + it("prints the accuracy report", () => { + // Surfaced in test output for humans; not an assertion. + // eslint-disable-next-line no-console + console.log("\n" + formatReport(report) + "\n"); + expect(report.totalCases).toBe(seedCorpus.length); + }); + + it("reads every labeled field on the seed corpus correctly", () => { + const failureSummary = report.failures + .map((failure) => { + const wrong = failure.fields + .filter((entry) => !entry.correct) + .map((entry) => `${entry.field}: expected "${entry.expected}" got "${entry.actual}"`) + .join("; "); + return `${failure.id} -> ${wrong}`; + }) + .join("\n"); + + expect(report.fieldAccuracy, `field regressions:\n${failureSummary}`).toBe(1); + expect(report.criticalFieldAccuracy).toBe(1); + expect(report.exactRate).toBe(1); + }); + + it("labels every critical field at least once across the corpus", () => { + const criticalCoverage = report.perField.filter( + (entry) => ["name", "slot", "mainStat", "mainValue", "setName"].includes(entry.field) && entry.evaluated > 0, + ); + // name is derived indirectly on most cases; slot/mainStat/mainValue/setName must be exercised. + expect(criticalCoverage.length).toBeGreaterThanOrEqual(4); + }); +}); diff --git a/src/eval/ocrEvalHarness.ts b/src/eval/ocrEvalHarness.ts new file mode 100644 index 0000000..46a0bfb --- /dev/null +++ b/src/eval/ocrEvalHarness.ts @@ -0,0 +1,257 @@ +import { parseArtifactCandidate } from "../lib/artifactOcrParser"; +import type { CaptureResult, OcrResult } from "../types/global"; + +// Field-level OCR accuracy harness. It runs the real parseArtifactCandidate +// over a labeled corpus and reports per-field accuracy, so OCR/layout/parser +// changes can be measured instead of guessed at. The corpus is seeded from the +// parser test cases and grows from human-confirmed review samples +// (see reviewSampleCorpus.ts). This is the gate every OCR change runs against. + +export type EvalField = + | "name" + | "slot" + | "level" + | "mainStat" + | "mainValue" + | "setName" + | "equipped" + | "substats"; + +export const EVAL_FIELDS: EvalField[] = [ + "name", + "slot", + "level", + "mainStat", + "mainValue", + "setName", + "equipped", + "substats", +]; + +// Fields that, if wrong, make an artifact record unusable. Reported separately +// so a regression in "equipped" (nice-to-have) does not read the same as a +// regression in "mainStat" (critical). +export const CRITICAL_FIELDS: EvalField[] = ["name", "slot", "mainStat", "mainValue", "setName"]; + +export interface OcrEvalCase { + /** Stable identifier, unique within a corpus. */ + id: string; + /** Crop id -> raw OCR text, exactly as the capture pipeline would produce it. */ + ocr: Record; + /** Ground-truth values. Only the provided fields are scored. */ + expect: Partial<{ + name: string; + slot: string; + level: number; + mainStat: string; + mainValue: string; + setName: string; + equipped: string; + substats: string[]; + }>; + meta?: { + resolution?: string; + source?: "test-seed" | "review-sample" | "manual"; + note?: string; + }; +} + +export interface FieldResult { + field: EvalField; + expected: string; + actual: string; + correct: boolean; +} + +export interface CaseResult { + id: string; + fields: FieldResult[]; + evaluatedFields: number; + correctFields: number; + /** True when every evaluated field on the case matched. */ + exact: boolean; + meta?: OcrEvalCase["meta"]; +} + +export interface FieldAccuracy { + field: EvalField; + evaluated: number; + correct: number; + /** 0..1, or null when the corpus never labels this field. */ + accuracy: number | null; +} + +export interface EvalReport { + totalCases: number; + exactCases: number; + exactRate: number; + evaluatedFields: number; + correctFields: number; + fieldAccuracy: number; + /** Accuracy restricted to CRITICAL_FIELDS. */ + criticalFieldAccuracy: number; + perField: FieldAccuracy[]; + cases: CaseResult[]; + failures: CaseResult[]; +} + +export function captureFromOcr(textById: Record): CaptureResult { + const ocr: OcrResult[] = Object.entries(textById).map(([id, text]) => ({ + id, + label: id, + text, + confidence: 80, + })); + return { + id: "eval", + name: "eval capture", + width: 1920, + height: 1080, + dataUrl: "", + capturedAt: new Date(0).toISOString(), + ocr, + }; +} + +function normalizeSubstats(values: readonly string[]) { + return [...values].map((value) => value.replace(/\s+/g, "").toLowerCase()).sort(); +} + +function substatsEqual(expected: readonly string[], actual: readonly string[]) { + const left = normalizeSubstats(expected); + const right = normalizeSubstats(actual); + if (left.length !== right.length) return false; + return left.every((value, index) => value === right[index]); +} + +function evaluateCase(evalCase: OcrEvalCase): CaseResult { + const parsed = parseArtifactCandidate(captureFromOcr(evalCase.ocr)); + const fields: FieldResult[] = []; + const stringValues: Record = parsed + ? { + name: parsed.name, + slot: parsed.slot, + mainStat: parsed.mainStat, + mainValue: parsed.mainValue, + setName: parsed.setName, + equipped: parsed.equipped, + } + : {}; + + for (const field of EVAL_FIELDS) { + if (!(field in evalCase.expect)) continue; + + if (field === "substats") { + const expected = evalCase.expect.substats ?? []; + const actual = parsed?.substats ?? []; + fields.push({ + field, + expected: expected.join(", "), + actual: actual.join(", "), + correct: substatsEqual(expected, actual), + }); + continue; + } + + if (field === "level") { + const expected = evalCase.expect.level; + const actual = parsed?.level; + fields.push({ + field, + expected: String(expected), + actual: parsed ? String(actual) : "", + correct: parsed != null && actual === expected, + }); + continue; + } + + const expected = String(evalCase.expect[field] ?? ""); + const actual = parsed ? stringValues[field] ?? "" : ""; + fields.push({ field, expected, actual, correct: parsed != null && actual === expected }); + } + + const correctFields = fields.filter((entry) => entry.correct).length; + return { + id: evalCase.id, + fields, + evaluatedFields: fields.length, + correctFields, + exact: fields.length > 0 && correctFields === fields.length, + meta: evalCase.meta, + }; +} + +export function runOcrEval(cases: readonly OcrEvalCase[]): EvalReport { + const caseResults = cases.map(evaluateCase); + + const perField: FieldAccuracy[] = EVAL_FIELDS.map((field) => { + let evaluated = 0; + let correct = 0; + for (const caseResult of caseResults) { + const entry = caseResult.fields.find((candidate) => candidate.field === field); + if (!entry) continue; + evaluated++; + if (entry.correct) correct++; + } + return { field, evaluated, correct, accuracy: evaluated === 0 ? null : correct / evaluated }; + }); + + const evaluatedFields = caseResults.reduce((sum, entry) => sum + entry.evaluatedFields, 0); + const correctFields = caseResults.reduce((sum, entry) => sum + entry.correctFields, 0); + const exactCases = caseResults.filter((entry) => entry.exact).length; + + const criticalEvaluated = perField + .filter((entry) => CRITICAL_FIELDS.includes(entry.field)) + .reduce((sum, entry) => sum + entry.evaluated, 0); + const criticalCorrect = perField + .filter((entry) => CRITICAL_FIELDS.includes(entry.field)) + .reduce((sum, entry) => sum + entry.correct, 0); + + return { + totalCases: caseResults.length, + exactCases, + exactRate: caseResults.length === 0 ? 1 : exactCases / caseResults.length, + evaluatedFields, + correctFields, + fieldAccuracy: evaluatedFields === 0 ? 1 : correctFields / evaluatedFields, + criticalFieldAccuracy: criticalEvaluated === 0 ? 1 : criticalCorrect / criticalEvaluated, + perField, + cases: caseResults, + failures: caseResults.filter((entry) => !entry.exact), + }; +} + +function pct(value: number | null) { + if (value === null) return " n/a"; + return `${(value * 100).toFixed(1)}%`.padStart(6, " "); +} + +export function formatReport(report: EvalReport): string { + const lines: string[] = []; + lines.push("OCR eval report"); + lines.push("=".repeat(48)); + lines.push(`Cases: ${report.totalCases}`); + lines.push(`Exact-match cases: ${report.exactCases}/${report.totalCases} (${pct(report.exactRate).trim()})`); + lines.push(`Field accuracy: ${pct(report.fieldAccuracy).trim()} (${report.correctFields}/${report.evaluatedFields})`); + lines.push(`Critical fields: ${pct(report.criticalFieldAccuracy).trim()}`); + lines.push(""); + lines.push("Per field:"); + for (const entry of report.perField) { + const critical = CRITICAL_FIELDS.includes(entry.field) ? "*" : " "; + lines.push(` ${critical} ${entry.field.padEnd(11)} ${pct(entry.accuracy)} (${entry.correct}/${entry.evaluated})`); + } + + if (report.failures.length > 0) { + lines.push(""); + lines.push(`Failures (${report.failures.length}):`); + for (const failure of report.failures) { + const wrong = failure.fields.filter((entry) => !entry.correct); + lines.push(` - ${failure.id}`); + for (const entry of wrong) { + lines.push(` ${entry.field}: expected "${entry.expected}" got "${entry.actual}"`); + } + } + } + + return lines.join("\n"); +} diff --git a/src/eval/reviewSampleCorpus.test.ts b/src/eval/reviewSampleCorpus.test.ts new file mode 100644 index 0000000..9415c93 --- /dev/null +++ b/src/eval/reviewSampleCorpus.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; +import type { ReviewSampleRecord } from "../types/global"; +import { reviewSampleToEvalCase, reviewSamplesToEvalCases } from "./reviewSampleCorpus"; + +function record(overrides: Partial = {}, savedAt = "2026-07-05T10:00:00.000Z"): ReviewSampleRecord { + return { + savedAt, + sample: { + reason: "automatic:low-confidence", + parsed: { + name: "Gladiator's Nostalgia", + slot: "Flower of Life", + level: 20, + mainStat: "HP", + mainValue: "4,780", + setName: "Gladiator's Finale", + equipped: "Bennett", + substats: ["CRIT DMG+12.4%", "ATK+9.9%"], + }, + capture: { + width: 2560, + height: 1440, + ocr: [ + { id: "artifact-title", label: "t", text: "Gladiator's Nostalgia\nFlower of Life", confidence: 80 }, + { id: "artifact-main-stat", label: "m", text: "HP\n4,780", confidence: 80 }, + ], + }, + ...overrides, + }, + }; +} + +describe("reviewSampleToEvalCase", () => { + it("extracts ocr text and parser guesses as an unconfirmed case", () => { + const evalCase = reviewSampleToEvalCase(record()); + expect(evalCase).not.toBeNull(); + expect(evalCase?.confirmed).toBe(false); + expect(evalCase?.ocr["artifact-title"]).toContain("Gladiator's Nostalgia"); + expect(evalCase?.expect.slot).toBe("Flower of Life"); + expect(evalCase?.expect.level).toBe(20); + expect(evalCase?.meta?.source).toBe("review-sample"); + expect(evalCase?.meta?.resolution).toBe("2560x1440"); + }); + + it("drops Unknown/Not detected sentinels so they are not treated as labels", () => { + const evalCase = reviewSampleToEvalCase( + record({ + parsed: { + name: "Unknown artifact", + slot: "Circlet of Logos", + mainStat: "Unknown main stat", + mainValue: "?", + setName: "Unknown set", + equipped: "Not detected", + }, + }), + ); + expect(evalCase?.expect.name).toBeUndefined(); + expect(evalCase?.expect.mainStat).toBeUndefined(); + expect(evalCase?.expect.setName).toBeUndefined(); + expect(evalCase?.expect.equipped).toBeUndefined(); + // "?" mainValue is a sentinel non-answer, must not leak in as a label. + expect(evalCase?.expect.mainValue).toBeUndefined(); + expect(evalCase?.expect.slot).toBe("Circlet of Logos"); + }); + + it("returns null when the sample has no OCR", () => { + expect(reviewSampleToEvalCase({ savedAt: "x", sample: { capture: { ocr: [] } } })).toBeNull(); + expect(reviewSampleToEvalCase({ savedAt: "x" })).toBeNull(); + }); + + it("maps a list of records and skips empty ones", () => { + const cases = reviewSamplesToEvalCases([record(), { savedAt: "y" }, record({}, "2026-07-05T11:00:00.000Z")]); + expect(cases).toHaveLength(2); + expect(new Set(cases.map((entry) => entry.id)).size).toBe(2); + }); +}); diff --git a/src/eval/reviewSampleCorpus.ts b/src/eval/reviewSampleCorpus.ts new file mode 100644 index 0000000..4f17ee5 --- /dev/null +++ b/src/eval/reviewSampleCorpus.ts @@ -0,0 +1,92 @@ +import type { ReviewSampleRecord } from "../types/global"; +import type { OcrEvalCase } from "./ocrEvalHarness"; + +// Turns saved review samples into eval cases so the review queue becomes a +// growing labeled corpus (see docs/ocr-eval.md). +// +// IMPORTANT: a review sample's `parsed` block is the *auto*-parse that was +// flagged for review - it is a label CANDIDATE, not verified ground truth. +// Using it directly as an expectation would be circular (the parser grading +// itself). The intended flow is: +// 1. reviewSampleToEvalCase() extracts the OCR + the parser's current guess. +// 2. A human confirms or corrects `expect` in the produced case. +// 3. The corrected case is committed into src/eval/corpus/. +// The `confirmed` flag records whether step 2 happened. + +export interface ReviewSampleEvalCase extends OcrEvalCase { + /** False until a human has verified/corrected the `expect` values. */ + confirmed: boolean; +} + +function ocrMapFromRecord(record: ReviewSampleRecord): Record | null { + const entries = record.sample?.capture?.ocr; + if (!entries?.length) return null; + const map: Record = {}; + for (const entry of entries) { + if (typeof entry?.id === "string" && typeof entry?.text === "string") { + map[entry.id] = entry.text; + } + } + return Object.keys(map).length > 0 ? map : null; +} + +function expectFromParsed(parsed: unknown): OcrEvalCase["expect"] { + if (!parsed || typeof parsed !== "object") return {}; + const candidate = parsed as Record; + const expect: OcrEvalCase["expect"] = {}; + + const stringField = (key: "name" | "slot" | "mainStat" | "mainValue" | "setName" | "equipped") => { + const value = candidate[key]; + // Skip the parser's own "Unknown ..." / "Not detected" / "?" sentinels - + // those are non-answers, not labels a human would confirm. + if (typeof value === "string" && value && !value.startsWith("Unknown") && value !== "Not detected" && value !== "?") { + expect[key] = value; + } + }; + + stringField("name"); + stringField("slot"); + stringField("mainStat"); + stringField("mainValue"); + stringField("setName"); + stringField("equipped"); + + if (typeof candidate.level === "number" && Number.isFinite(candidate.level) && candidate.level > 0) { + expect.level = candidate.level; + } + if (Array.isArray(candidate.substats) && candidate.substats.every((entry) => typeof entry === "string")) { + expect.substats = candidate.substats as string[]; + } + + return expect; +} + +export function reviewSampleToEvalCase(record: ReviewSampleRecord, index = 0): ReviewSampleEvalCase | null { + const ocr = ocrMapFromRecord(record); + if (!ocr) return null; + + const savedAt = record.savedAt ?? "unknown"; + const idSuffix = savedAt.replace(/[^0-9A-Za-z]/g, "").slice(0, 14) || String(index); + const capture = record.sample?.capture; + + return { + id: `review-${idSuffix}-${index}`, + ocr, + expect: expectFromParsed(record.sample?.parsed), + confirmed: false, + meta: { + source: "review-sample", + resolution: capture?.width && capture?.height ? `${capture.width}x${capture.height}` : undefined, + note: record.sample?.reason, + }, + }; +} + +export function reviewSamplesToEvalCases(records: readonly ReviewSampleRecord[] | null | undefined) { + const cases: ReviewSampleEvalCase[] = []; + (records ?? []).forEach((record, index) => { + const evalCase = reviewSampleToEvalCase(record, index); + if (evalCase) cases.push(evalCase); + }); + return cases; +}