From b7dbc618b351dc72f1aa6ff97eeb366784dfaf6a Mon Sep 17 00:00:00 2001 From: AzuTear Date: Sun, 5 Jul 2026 20:41:30 +0200 Subject: [PATCH 01/14] feat(eval): add field-level OCR accuracy harness + seed corpus Adds a measurement gate for the artifact OCR parser (ADR-007), the prerequisite for the layout-profile and preprocessing rework. runOcrEval feeds labeled OCR text through the real parseArtifactCandidate and scores per-field / per-case accuracy. - src/eval/ocrEvalHarness.ts: pure metrics (per-field, critical-field, exact). - src/eval/corpus/seedCorpus.ts: 23 cases transcribed from the verified parser test assertions; runs at 100%. - src/eval/reviewSampleCorpus.ts: converts review samples into label *candidates* (never ground truth) so the review queue can grow the corpus. - src/eval/ocrEval.test.ts + reviewSampleCorpus.test.ts: gate (must stay 1.0) and converter unit tests. - npm run eval script; docs/ocr-eval.md; ADR-007/008/009. Also records the agreed rework direction: C# input/capture sidecar (ADR-008) and resolution-anchored layout profiles + OCR preprocessing (ADR-009). Co-Authored-By: Claude Opus 4.8 --- docs/DECISIONS.md | 102 +++++++++ docs/ocr-eval.md | 49 +++++ package.json | 1 + src/eval/corpus/seedCorpus.ts | 311 ++++++++++++++++++++++++++++ src/eval/ocrEval.test.ts | 44 ++++ src/eval/ocrEvalHarness.ts | 257 +++++++++++++++++++++++ src/eval/reviewSampleCorpus.test.ts | 77 +++++++ src/eval/reviewSampleCorpus.ts | 92 ++++++++ 8 files changed, 933 insertions(+) create mode 100644 docs/ocr-eval.md create mode 100644 src/eval/corpus/seedCorpus.ts create mode 100644 src/eval/ocrEval.test.ts create mode 100644 src/eval/ocrEvalHarness.ts create mode 100644 src/eval/reviewSampleCorpus.test.ts create mode 100644 src/eval/reviewSampleCorpus.ts diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 43c00f5..c0c41e1 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -12,6 +12,9 @@ This document contains Architecture Decision Records. | ADR-004 | Treat in-game marking as a later opt-in feature | Accepted | 2026-07-04 | | ADR-005 | Use a generated Genshin data package for OCR matching | Accepted | 2026-07-04 | | ADR-006 | Persistent input helper and JSON artifact store before SQLite | Accepted | 2026-07-04 | +| ADR-007 | Measure OCR accuracy with a labeled eval harness before reworking the scanner | Accepted | 2026-07-05 | +| ADR-008 | Replace the PowerShell input/capture helper with a C# sidecar | Accepted | 2026-07-05 | +| ADR-009 | Resolution-anchored layout profiles and OCR preprocessing over color detection | Accepted | 2026-07-05 | ## ADR-001: Build A Local Electron App First @@ -133,3 +136,102 @@ Run one persistent PowerShell helper process (compiled once, JSON protocol over - Automated clicks are verified by checking that the parsed detail signature changed; repeated failures abort with a diagnosis hint instead of clicking blindly. - Leveling an artifact changes its signature and creates a new record; rescan-merge is an open follow-up. - If the helper process dies it is respawned on the next request; pending requests fail loudly instead of hanging. + +## ADR-007: Measure OCR Accuracy With A Labeled Eval Harness Before Reworking The Scanner + +### Status + +Accepted + +### Context + +OCR and crop reliability are the core product risk (ADR-002), but there was no +way to measure field-level accuracy. Every OCR, crop, or parser change was a +blind change - regressions could only be caught by a human re-testing against the +live game, and there was no baseline to compare a new engine or preprocessing +step against. + +### Decision + +Add a field-level eval harness (`src/eval/`) that runs the real +`parseArtifactCandidate` over a labeled corpus and reports per-field and +per-case accuracy. Seed the corpus from the existing parser test cases (verified +labels), and grow it from human-confirmed review samples via +`reviewSampleToEvalCase`. Run it as a gate under `npm test` (must stay 100% on +the verified seed) and as a full report via `npm run eval`. + +### Consequences + +- OCR/layout/preprocessing changes are measured, not guessed at; a new engine or + a preprocessing step has to beat a recorded baseline. +- The review queue does double duty: it flags artifacts for human correction and + feeds the eval corpus. Review-sample `parsed` blocks are label *candidates*, + never ground truth, to avoid the parser grading itself. +- The corpus label is the source of truth. If a code change intentionally alters + a correct output, the label is updated in the same commit. + +## ADR-008: Replace The PowerShell Input/Capture Helper With A C# Sidecar + +### Status + +Accepted + +### Context + +The persistent PowerShell helper (ADR-006) still carries Windows PowerShell 5.1 +quirks (the `Marshal::SizeOf` interop bug), compiles Win32 interop at startup, +and captures each frame by writing a PNG to the temp directory and reading it +back. Inventory Kamera - the proven reference for automated Genshin scanning - +uses a C#/.NET stack with InputSimulator (SendInput) and direct GDI/BitBlt +capture. + +### Decision + +Replace the PowerShell helper with a self-contained .NET (C#) sidecar that speaks +the same JSON-over-stdin/stdout protocol, so the Electron-side `InputHelperService` +interface stays stable. The sidecar does per-monitor DPI-aware SendInput +click/scroll, BitBlt client-rect capture returning bytes without a temp file, and +elevation detection. + +### Consequences + +- No PS 5.1 marshalling bugs, no per-call interop compile, no temp-PNG churn; + lower latency makes batch scans and the ESC/mouse failsafe polling cheaper. +- Adds a .NET build/publish step and ships a compiled exe with the app. +- The migration is behind the existing service interface, so the renderer and + scan loop do not change. + +## ADR-009: Resolution-Anchored Layout Profiles And OCR Preprocessing Over Color Detection + +### Status + +Accepted + +### Context + +The current pipeline finds the artifact detail panel with hardcoded orange/green +color thresholds (`inferDetailRect`) and then crops fixed percentages of that +guessed rectangle. This is brittle against HDR, color profiles, UI scale, aspect +ratio, and game UI updates. Inventory Kamera instead requires borderless 16:9 and +scales fixed crop coordinates from a reference resolution, then feeds Tesseract +preprocessed (grayscale, upscaled, thresholded) crops - which is why general +Tesseract is accurate enough for them. + +### Decision + +Adopt the same approach: require borderless 16:9, drive crops from +resolution-anchored layout profiles scaled from a reference resolution (color +detection only as a fallback), and add a per-region preprocessing pass +(grayscale, upscale, threshold, invert) plus a digit-whitelist mode for numeric +fields. The set name stays derived from the static piece-to-set data package (no +dedicated set-effect crop). English-only OCR is accepted. Every change is +validated against the ADR-007 eval harness. + +### Consequences + +- Crop positions become deterministic per resolution instead of per-frame guesses. +- Preprocessing is expected to lift accuracy enough that a custom OCR engine is + only pursued if the eval harness shows Tesseract-plus-preprocessing is + insufficient. +- Non-16:9 or non-borderless setups are explicitly unsupported for the auto + scanner; the app should detect and warn rather than silently misread. diff --git a/docs/ocr-eval.md b/docs/ocr-eval.md new file mode 100644 index 0000000..5273adb --- /dev/null +++ b/docs/ocr-eval.md @@ -0,0 +1,49 @@ +# OCR Eval Harness + +Field-level accuracy measurement for the artifact OCR parser. This is the gate +every OCR, crop, layout, or parser change runs against (see ADR-007). + +## Run it + +```powershell +npm run eval # full accuracy report for the seed corpus +npm test # runs the eval gate alongside the rest of the suite +``` + +The report prints exact-match rate, overall field accuracy, a per-field +breakdown (critical fields marked with `*`), and every failing case with an +`expected "..." got "..."` diff. + +## How it works + +- `src/eval/ocrEvalHarness.ts` - pure metric functions. `runOcrEval(cases)` + feeds each case's OCR text through the real `parseArtifactCandidate` and scores + the produced fields against the labels. Order-independent for substats. +- `src/eval/corpus/seedCorpus.ts` - the seed corpus, transcribed from the + verified assertions in `src/lib/artifactOcrParser.test.ts`. Must stay at 100%. +- `src/eval/ocrEval.test.ts` - the gate: seed field accuracy, critical-field + accuracy, and exact-match rate must all be 1.0. + +## Growing the corpus from review samples + +The review queue is the corpus source. A saved review sample carries the OCR +text plus the parser's *guess* - `reviewSampleToEvalCase` extracts both. + +The parser's guess is a label **candidate, not ground truth** (using it directly +would be the parser grading itself). To add a real case: + +1. Convert review samples with `reviewSamplesToEvalCases(records)`. +2. Open each produced case and confirm or correct the `expect` values against + what the artifact actually is in-game. Set `confirmed: true`. +3. Move the corrected case into a file under `src/eval/corpus/` and add it to the + corpus array. + +Prefer cases that cover new failure modes: unseen resolutions, new sets or +characters, and OCR noise the current corpus does not exercise. + +## When a change moves a number + +- Accuracy **drops**: a regression. Read the printed failures; fix the parser or + revert. Do not lower the threshold to make it pass. +- A change **intentionally** alters a previously-correct output: update the + corpus label in the same commit. The label is the source of truth, not the code. diff --git a/package.json b/package.json index 295de94..8f5b356 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "start": "electron .", "lint": "tsc --noEmit", "test": "vitest run", + "eval": "vitest run src/eval/ocrEval.test.ts", "data:genshin": "node scripts/generate-genshin-data.cjs" }, "dependencies": { diff --git a/src/eval/corpus/seedCorpus.ts b/src/eval/corpus/seedCorpus.ts new file mode 100644 index 0000000..6ebfce4 --- /dev/null +++ b/src/eval/corpus/seedCorpus.ts @@ -0,0 +1,311 @@ +import type { OcrEvalCase } from "../ocrEvalHarness"; + +// Seed corpus transcribed from src/lib/artifactOcrParser.test.ts. Every label +// here is an exact assertion that already passes, so a fresh run starts at (or +// very near) 100% - which is what makes the accuracy thresholds meaningful as a +// regression gate. Cases that deliberately expect conservative "Unknown ..." +// behavior contribute only their positively-identified fields (level, value) +// so field accuracy stays a measure of correct reads, not correct refusals. +// +// Grow this corpus from human-confirmed review samples via reviewSampleCorpus.ts. + +const SOURCE = "test-seed" as const; + +export const seedCorpus: OcrEvalCase[] = [ + { + id: "spring-sands-em", + ocr: { + "artifact-title": "A Note in Spring's Lei\n| Sands of Eon Vi", + "artifact-main-stat": "Elemental Mastery\n187", + "artifact-substats": "+20\n+ ATK+29\n+ CRIT DMG+15.5%\n+ CRIT Rate+2.7%\nATK + 15", + "artifact-set-effects": "A Day Carved From Rising Winds\n2-Piece Set: ATK +18%.", + "artifact-footer": "WV Equipped: Citlali\naR 0", + }, + expect: { + slot: "Sands of Eon", + level: 20, + mainStat: "Elemental Mastery", + setName: "A Day Carved From Rising Winds", + equipped: "Citlali", + }, + meta: { source: SOURCE }, + }, + { + id: "moonlit-flower-hp", + ocr: { + "artifact-title": "Moonlit Offering's Opulent Dr\n| Flower of Life 2", + "artifact-main-stat": "4,780\nAhhh", + "artifact-substats": "+ Elemental Mastery+54\n+ CRIT Rate+7.4%\n+ ATK+4.7%", + "artifact-set-effects": "Aubade of Morningstar and Moor\n2-Piece Set: Increases Elemental Mastery by 80.", + "artifact-footer": "? Equipped: Ineffa\naR 0", + }, + expect: { + slot: "Flower of Life", + mainStat: "HP", + mainValue: "4,780", + setName: "Aubade of Morningstar and Moon", + equipped: "Ineffa", + }, + meta: { source: SOURCE }, + }, + { + id: "vv-determination-sands-er", + ocr: { + "artifact-title": "Viridescent Venerer's Determination\nSands of Eon Vi", + "artifact-main-stat": "Energy Recharge\n51.8%", + "artifact-substats": "+ ATK+5.8%\n+ Elemental Mastery+37\n+ HP+11.7%\n+ ATK+54", + "artifact-set-effects": "Viridescent Venerer:\n2-Piece Set: Anemo DMG Bonus +15%", + "artifact-footer": "Equipped: Sucrose", + }, + expect: { slot: "Sands of Eon", mainStat: "Energy Recharge" }, + meta: { source: SOURCE }, + }, + { + id: "pristine-plume-atk", + ocr: { + "artifact-title": "Pristine Plume of the Bles\n| Plume of Death", + "artifact-main-stat": "31 !\npr", + "artifact-substats": "+20\n+ CRIT DMG+7.0%\n+ DEF+30.6%\n+ Elemental Mastery+40\nATK +5.8%", + "artifact-set-effects": "Silken Moon's Serenade:\n2-Piece Set: Energy Recharge +20%.", + "artifact-footer": "Equipped: Aino\nBR", + }, + expect: { + slot: "Plume of Death", + mainStat: "ATK", + mainValue: "311", + setName: "Silken Moon's Serenade", + equipped: "Aino", + }, + meta: { source: SOURCE }, + }, + { + id: "deep-gallery-goblet-cryo", + ocr: { + "artifact-title": "Deep Gallery's Bestowed Banquet\nGoblet of Eonothem", + "artifact-main-stat": "Cryo DMG Bonus\n46.6%", + "artifact-substats": "+ CRIT Rate+6.6%\n+ CRIT DMG+12.4%\n+ HP+269\n+ ATK+16.3%", + "artifact-set-effects": "Finale of the Deep Galleries:\n2-Piece Set: Cryo DMG Bonus +15%", + "artifact-footer": "Equipped: Skirk", + }, + expect: { slot: "Goblet of Eonothem", mainStat: "Cryo DMG Bonus", mainValue: "46.6%" }, + meta: { source: SOURCE }, + }, + { + id: "holy-crown-circlet-crit", + ocr: { + "artifact-title": "Holy Crown of the Believer\nCirclet of Logos", + "artifact-main-stat": "CRIT Rate\n31.1%", + "artifact-substats": "+ ATK+4.7%\n+ Elemental Mastery+56\n+ DEF+37\n+ Energy Recharge+11.7%", + "artifact-set-effects": "Silken Moon's Serenade:\n2-Piece Set: Energy Recharge +20%.", + "artifact-footer": "Equipped: Chongyun", + }, + expect: { slot: "Circlet of Logos", mainStat: "CRIT Rate", mainValue: "31.1%" }, + meta: { source: SOURCE }, + }, + { + id: "beast-tamer-flower-set", + ocr: { + "artifact-title": "Beast Tamer's Talisman\nFlower of Life", + "artifact-main-stat": "HP\n4,780", + "artifact-substats": "+ HP+16.3%\n+ CRIT DMG+7.0%\n+ Energy Recharge+6.5%\n+ Elemental Mastery+68", + "artifact-set-effects": "Scroll of the Hero of Cinder City:\n2-Piece Set: When a nearby party member triggers a Nightsoul Burst", + "artifact-footer": "Equipped: Citlali", + }, + expect: { setName: "Scroll of the Hero of Cinder City" }, + meta: { source: SOURCE }, + }, + { + id: "piece-name-to-set-fallback", + ocr: { + "artifact-title": "Holy Crown of the Believer\nCirclet of Logos", + "artifact-main-stat": "CRIT Rate\n31.1%", + "artifact-substats": "+ ATK+4.7%\n+ Elemental Mastery+56\n+ DEF+37", + "artifact-set-effects": "unreadable noisy set text", + "artifact-footer": "Equipped: Chongyun", + }, + expect: { setName: "Silken Moon's Serenade" }, + meta: { source: SOURCE, note: "set derived from piece name when set-effect OCR is noise" }, + }, + { + id: "piece-name-to-slot-fallback", + ocr: { + "artifact-title": "Viridescent Venerer's Determination", + "artifact-main-stat": "Energy Recharge\n51.8%", + "artifact-substats": "+ ATK+5.8%\n+ Elemental Mastery+37\n+ HP+11.7%\n+ ATK+54", + "artifact-set-effects": "Viridescent Venerer:\n2-Piece Set: Anemo DMG Bonus +15%", + "artifact-footer": "Equipped: Sucrose", + }, + expect: { slot: "Sands of Eon", setName: "Viridescent Venerer" }, + meta: { source: SOURCE }, + }, + { + id: "hourglass-atk-percent", + ocr: { + "artifact-title": "Hourglass of Thunder\nSands of Eon", + "artifact-main-stat": "ATK\n40.7%", + "artifact-substats": "+17\n+ CRIT DMG+14.8%\n+ Elemental Mastery+21\n+ ATK+53\n+ DEF+19", + "artifact-set-effects": "Thundering Fury:\n2-Piece Set: Electro DMG Bonus +15%", + "artifact-footer": "Equipped: Fischl", + }, + expect: { level: 17, slot: "Sands of Eon", mainStat: "ATK%", mainValue: "40.7%" }, + meta: { source: SOURCE }, + }, + { + id: "night-realm-sands-ambiguous", + ocr: { + "artifact-title": "Myths of the Night Realm\nSands of Eon", + "artifact-main-stat": "30.8%", + "artifact-substats": "+12\n+ Elemental Mastery+16\n+ CRIT DMG+6.2%\n+ DEF+42\n+ HP+9.9%", + "artifact-set-effects": "Obsidian Codex:\n2-Piece Set: While the equipping character is in Nightsoul's Blessing", + "artifact-footer": "Equipped: Sandrone", + }, + expect: { level: 12, mainValue: "30.8%" }, + meta: { source: SOURCE, note: "mainStat intentionally stays Unknown (ambiguous family)" }, + }, + { + id: "footer-noise-equipped", + ocr: { + "artifact-title": "Gladiator's Nostalgia\nFlower of Life", + "artifact-main-stat": "HP\n4,780", + "artifact-substats": "+ Energy Recharge+11.0%\n+ ATK+9.9%\n+ HP+14.6%\n+ CRIT DMG+12.4%", + "artifact-set-effects": "Gladiator's Finale:\n2-Piece Set: ATK +18%", + "artifact-footer": "1 gv 0RY If tha anuninnina\nJl Equipped: Bennett\nCEE", + }, + expect: { equipped: "Bennett" }, + meta: { source: SOURCE }, + }, + { + id: "atk-percent-sands", + ocr: { + "artifact-title": "Myths of the Night Realm\nSands of Eon", + "artifact-main-stat": "ATK\n30.8%", + "artifact-substats": "+ Elemental Mastery+16\n+ CRIT DMG+6.2%\n+ DEF+42\n+ HP+9.9%", + "artifact-set-effects": "Obsidian Codex:\n2-Piece Set: While the equipping character is in Nightsoul's Blessing", + "artifact-footer": "Equipped: Sandrone", + }, + expect: { mainStat: "ATK%", mainValue: "30.8%" }, + meta: { source: SOURCE }, + }, + { + id: "atk-percent-circlet", + ocr: { + "artifact-title": "Maiden's Fading Beauty\nCirclet of Logos", + "artifact-main-stat": "ATK\n46.6%", + "artifact-substats": "+ ATK+29\n+ CRIT DMG+10.9%\n+ CRIT Rate+7.0%", + "artifact-set-effects": "Maiden Beloved:\n2-Piece Set: Character Healing Effectiveness +15%", + "artifact-footer": "Equipped: Qiqi", + }, + expect: { mainStat: "ATK%", mainValue: "46.6%" }, + meta: { source: SOURCE }, + }, + { + id: "atk-percent-goblet", + ocr: { + "artifact-title": "Viridescent Venerer's Vessel\nGoblet of Eonothem", + "artifact-main-stat": "ATK\n46.6%", + "artifact-substats": "+ HP+209\n+ CRIT DMG+25.6%\n+ Elemental Mastery+37", + "artifact-set-effects": "Viridescent Venerer:\n2-Piece Set: Anemo DMG Bonus +15%", + "artifact-footer": "Equipped: Ganyu", + }, + expect: { mainStat: "ATK%", mainValue: "46.6%" }, + meta: { source: SOURCE }, + }, + { + id: "garbled-percent-punctuation", + ocr: { + "artifact-title": "Pristine Circlet of the Bles\n| Circlet of Logos", + "artifact-main-stat": "ATK\n46", + "artifact-substats": `+ Elemental Mastery+20·5%\n+ CRIT DMG+6.3%\n+ ATK+19\n+ DEF+12`, + "artifact-set-effects": "Gladiator's Finale:\n2-Piece Set: ATK +18%", + "artifact-footer": "Equipped: Aino", + }, + expect: { slot: "Circlet of Logos" }, + meta: { source: SOURCE }, + }, + { + id: "garbled-quotes", + ocr: { + "artifact-title": "Aloy's “Gift”\n| Circlet of Logos", + "artifact-main-stat": "Elemental Mastery\n46.6%", + "artifact-substats": "+ ATK+4.7\n+ CRIT DMG+12.4%\n+ Energy Recharge+8.1%\n+ HP+11", + "artifact-set-effects": "Maiden’s Beloved:\n2-Piece Set: Energy Recharge +16%", + "artifact-footer": "Equipped: Shenhe", + }, + expect: { slot: "Circlet of Logos", mainStat: "Elemental Mastery", setName: "Maiden Beloved" }, + meta: { source: SOURCE }, + }, + { + id: "substats-overflow-into-set-crop", + ocr: { + "artifact-title": "Moonlit Offering's Opulent Dr\nFlower of Life", + "artifact-main-stat": "HP\n4,780", + "artifact-substats": "+ CRIT DMG+13.2%", + "artifact-set-effects": "+ HP+15.7%\n+ DEF+12.4%\n+ Energy Recharge+5.2%\nAubade of Morningstar and Moon\n2-Piece Set: Increases Elemental Mastery by 80.", + "artifact-footer": "Equipped: Venti", + }, + expect: { + substats: ["CRIT DMG+13.2%", "HP%+15.7%", "DEF%+12.4%", "Energy Recharge+5.2%"], + }, + meta: { source: SOURCE }, + }, + { + id: "percent-main-value-no-label", + ocr: { + "artifact-title": "Moonlit Offering's Final\nSands of Eon", + "artifact-main-stat": "46.6%\n1S 2.5.8 J\nSe", + "artifact-substats": "+ ATK+19\n+ Energy Recharge+6.5%\n+ CRIT DMG+18.7%", + "artifact-set-effects": "Aubade of Morningstar and Moon\n2-Piece Set: Increases Elemental Mastery by 80.", + "artifact-footer": "", + }, + expect: { mainValue: "46.6%" }, + meta: { source: SOURCE, note: "mainStat stays Unknown; value must survive" }, + }, + { + id: "derive-main-from-value-sands-def", + ocr: { + "artifact-title": "Revelation's Toll\nSands of Eon", + "artifact-main-stat": "58.3%\n1S 2.5.8 J\nSe", + "artifact-substats": "+ CRIT Rate+14.0%\n+ Elemental Mastery+33\n+ Energy Recharge+6.5%", + "artifact-set-effects": "Night of the Sky's Unveiling:\n2-Piece Set: Increases Elemental Mastery by 80.", + "artifact-footer": "Equipped: Zibai", + }, + expect: { mainValue: "58.3%", mainStat: "DEF%" }, + meta: { source: SOURCE }, + }, + { + id: "derive-main-from-value-circlet-critdmg", + ocr: { + "artifact-title": "Crown of the Saints\nCirclet of Logos", + "artifact-main-stat": "62.2%", + "artifact-substats": "+ ATK+18\n+ ATK+10.5%\n+ DEF+17.5%", + "artifact-set-effects": "Obsidian Codex:\n2-Piece Set: While the equipping character is in Nightsoul's Blessing", + "artifact-footer": "Equipped: Aino", + }, + expect: { mainValue: "62.2%", mainStat: "CRIT DMG" }, + meta: { source: SOURCE }, + }, + { + id: "noisy-fragment-circlet-critdmg", + ocr: { + "artifact-title": "Moonlit Offering's Silver Crown\nCirclet of Logos", + "artifact-main-stat": "6\n2 D", + "artifact-substats": "+ ATK+29\n+ ATK+5.8%\n+ Elemental Mastery+77\n- DEF+5.1%", + "artifact-set-effects": "Aubade of Morningstar and Moon\n2-Piece Set: Increases Elemental Mastery by 80.", + "artifact-footer": "", + }, + expect: { mainValue: "62.2%", mainStat: "CRIT DMG" }, + meta: { source: SOURCE, note: "recovers unique max-value main from noisy digit fragment" }, + }, + { + id: "noisy-fragment-sands-conservative", + ocr: { + "artifact-title": "Moonlit Offering's Final Hour\nSands of Eon", + "artifact-main-stat": "46.6% |\nPEE", + "artifact-substats": "+ ATK+19\n+ Energy Recharge+6.5%\n+ CRIT DMG+18.7%\n- DEF+53", + "artifact-set-effects": "Aubade of Morningstar and Moon\n2-Piece Set: Increases Elemental Mastery by 80.", + "artifact-footer": "", + }, + expect: { mainValue: "46.6%" }, + meta: { source: SOURCE, note: "ambiguous 46.6 value stays conservative on mainStat" }, + }, +]; diff --git a/src/eval/ocrEval.test.ts b/src/eval/ocrEval.test.ts new file mode 100644 index 0000000..cb3710d --- /dev/null +++ b/src/eval/ocrEval.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { formatReport, runOcrEval } from "./ocrEvalHarness"; +import { seedCorpus } from "./corpus/seedCorpus"; + +// Regression gate: the seed corpus is verified ground truth, so the parser must +// read every labeled field correctly. A drop here means an OCR/parser change +// regressed a previously-correct read - look at the printed failures. If a +// change intentionally alters a correct output, update the corpus label in the +// same commit (the label is the source of truth, not the code). + +describe("OCR eval harness", () => { + const report = runOcrEval(seedCorpus); + + it("prints the accuracy report", () => { + // Surfaced in test output for humans; not an assertion. + // eslint-disable-next-line no-console + console.log("\n" + formatReport(report) + "\n"); + expect(report.totalCases).toBe(seedCorpus.length); + }); + + it("reads every labeled field on the seed corpus correctly", () => { + const failureSummary = report.failures + .map((failure) => { + const wrong = failure.fields + .filter((entry) => !entry.correct) + .map((entry) => `${entry.field}: expected "${entry.expected}" got "${entry.actual}"`) + .join("; "); + return `${failure.id} -> ${wrong}`; + }) + .join("\n"); + + expect(report.fieldAccuracy, `field regressions:\n${failureSummary}`).toBe(1); + expect(report.criticalFieldAccuracy).toBe(1); + expect(report.exactRate).toBe(1); + }); + + it("labels every critical field at least once across the corpus", () => { + const criticalCoverage = report.perField.filter( + (entry) => ["name", "slot", "mainStat", "mainValue", "setName"].includes(entry.field) && entry.evaluated > 0, + ); + // name is derived indirectly on most cases; slot/mainStat/mainValue/setName must be exercised. + expect(criticalCoverage.length).toBeGreaterThanOrEqual(4); + }); +}); diff --git a/src/eval/ocrEvalHarness.ts b/src/eval/ocrEvalHarness.ts new file mode 100644 index 0000000..46a0bfb --- /dev/null +++ b/src/eval/ocrEvalHarness.ts @@ -0,0 +1,257 @@ +import { parseArtifactCandidate } from "../lib/artifactOcrParser"; +import type { CaptureResult, OcrResult } from "../types/global"; + +// Field-level OCR accuracy harness. It runs the real parseArtifactCandidate +// over a labeled corpus and reports per-field accuracy, so OCR/layout/parser +// changes can be measured instead of guessed at. The corpus is seeded from the +// parser test cases and grows from human-confirmed review samples +// (see reviewSampleCorpus.ts). This is the gate every OCR change runs against. + +export type EvalField = + | "name" + | "slot" + | "level" + | "mainStat" + | "mainValue" + | "setName" + | "equipped" + | "substats"; + +export const EVAL_FIELDS: EvalField[] = [ + "name", + "slot", + "level", + "mainStat", + "mainValue", + "setName", + "equipped", + "substats", +]; + +// Fields that, if wrong, make an artifact record unusable. Reported separately +// so a regression in "equipped" (nice-to-have) does not read the same as a +// regression in "mainStat" (critical). +export const CRITICAL_FIELDS: EvalField[] = ["name", "slot", "mainStat", "mainValue", "setName"]; + +export interface OcrEvalCase { + /** Stable identifier, unique within a corpus. */ + id: string; + /** Crop id -> raw OCR text, exactly as the capture pipeline would produce it. */ + ocr: Record; + /** Ground-truth values. Only the provided fields are scored. */ + expect: Partial<{ + name: string; + slot: string; + level: number; + mainStat: string; + mainValue: string; + setName: string; + equipped: string; + substats: string[]; + }>; + meta?: { + resolution?: string; + source?: "test-seed" | "review-sample" | "manual"; + note?: string; + }; +} + +export interface FieldResult { + field: EvalField; + expected: string; + actual: string; + correct: boolean; +} + +export interface CaseResult { + id: string; + fields: FieldResult[]; + evaluatedFields: number; + correctFields: number; + /** True when every evaluated field on the case matched. */ + exact: boolean; + meta?: OcrEvalCase["meta"]; +} + +export interface FieldAccuracy { + field: EvalField; + evaluated: number; + correct: number; + /** 0..1, or null when the corpus never labels this field. */ + accuracy: number | null; +} + +export interface EvalReport { + totalCases: number; + exactCases: number; + exactRate: number; + evaluatedFields: number; + correctFields: number; + fieldAccuracy: number; + /** Accuracy restricted to CRITICAL_FIELDS. */ + criticalFieldAccuracy: number; + perField: FieldAccuracy[]; + cases: CaseResult[]; + failures: CaseResult[]; +} + +export function captureFromOcr(textById: Record): CaptureResult { + const ocr: OcrResult[] = Object.entries(textById).map(([id, text]) => ({ + id, + label: id, + text, + confidence: 80, + })); + return { + id: "eval", + name: "eval capture", + width: 1920, + height: 1080, + dataUrl: "", + capturedAt: new Date(0).toISOString(), + ocr, + }; +} + +function normalizeSubstats(values: readonly string[]) { + return [...values].map((value) => value.replace(/\s+/g, "").toLowerCase()).sort(); +} + +function substatsEqual(expected: readonly string[], actual: readonly string[]) { + const left = normalizeSubstats(expected); + const right = normalizeSubstats(actual); + if (left.length !== right.length) return false; + return left.every((value, index) => value === right[index]); +} + +function evaluateCase(evalCase: OcrEvalCase): CaseResult { + const parsed = parseArtifactCandidate(captureFromOcr(evalCase.ocr)); + const fields: FieldResult[] = []; + const stringValues: Record = parsed + ? { + name: parsed.name, + slot: parsed.slot, + mainStat: parsed.mainStat, + mainValue: parsed.mainValue, + setName: parsed.setName, + equipped: parsed.equipped, + } + : {}; + + for (const field of EVAL_FIELDS) { + if (!(field in evalCase.expect)) continue; + + if (field === "substats") { + const expected = evalCase.expect.substats ?? []; + const actual = parsed?.substats ?? []; + fields.push({ + field, + expected: expected.join(", "), + actual: actual.join(", "), + correct: substatsEqual(expected, actual), + }); + continue; + } + + if (field === "level") { + const expected = evalCase.expect.level; + const actual = parsed?.level; + fields.push({ + field, + expected: String(expected), + actual: parsed ? String(actual) : "", + correct: parsed != null && actual === expected, + }); + continue; + } + + const expected = String(evalCase.expect[field] ?? ""); + const actual = parsed ? stringValues[field] ?? "" : ""; + fields.push({ field, expected, actual, correct: parsed != null && actual === expected }); + } + + const correctFields = fields.filter((entry) => entry.correct).length; + return { + id: evalCase.id, + fields, + evaluatedFields: fields.length, + correctFields, + exact: fields.length > 0 && correctFields === fields.length, + meta: evalCase.meta, + }; +} + +export function runOcrEval(cases: readonly OcrEvalCase[]): EvalReport { + const caseResults = cases.map(evaluateCase); + + const perField: FieldAccuracy[] = EVAL_FIELDS.map((field) => { + let evaluated = 0; + let correct = 0; + for (const caseResult of caseResults) { + const entry = caseResult.fields.find((candidate) => candidate.field === field); + if (!entry) continue; + evaluated++; + if (entry.correct) correct++; + } + return { field, evaluated, correct, accuracy: evaluated === 0 ? null : correct / evaluated }; + }); + + const evaluatedFields = caseResults.reduce((sum, entry) => sum + entry.evaluatedFields, 0); + const correctFields = caseResults.reduce((sum, entry) => sum + entry.correctFields, 0); + const exactCases = caseResults.filter((entry) => entry.exact).length; + + const criticalEvaluated = perField + .filter((entry) => CRITICAL_FIELDS.includes(entry.field)) + .reduce((sum, entry) => sum + entry.evaluated, 0); + const criticalCorrect = perField + .filter((entry) => CRITICAL_FIELDS.includes(entry.field)) + .reduce((sum, entry) => sum + entry.correct, 0); + + return { + totalCases: caseResults.length, + exactCases, + exactRate: caseResults.length === 0 ? 1 : exactCases / caseResults.length, + evaluatedFields, + correctFields, + fieldAccuracy: evaluatedFields === 0 ? 1 : correctFields / evaluatedFields, + criticalFieldAccuracy: criticalEvaluated === 0 ? 1 : criticalCorrect / criticalEvaluated, + perField, + cases: caseResults, + failures: caseResults.filter((entry) => !entry.exact), + }; +} + +function pct(value: number | null) { + if (value === null) return " n/a"; + return `${(value * 100).toFixed(1)}%`.padStart(6, " "); +} + +export function formatReport(report: EvalReport): string { + const lines: string[] = []; + lines.push("OCR eval report"); + lines.push("=".repeat(48)); + lines.push(`Cases: ${report.totalCases}`); + lines.push(`Exact-match cases: ${report.exactCases}/${report.totalCases} (${pct(report.exactRate).trim()})`); + lines.push(`Field accuracy: ${pct(report.fieldAccuracy).trim()} (${report.correctFields}/${report.evaluatedFields})`); + lines.push(`Critical fields: ${pct(report.criticalFieldAccuracy).trim()}`); + lines.push(""); + lines.push("Per field:"); + for (const entry of report.perField) { + const critical = CRITICAL_FIELDS.includes(entry.field) ? "*" : " "; + lines.push(` ${critical} ${entry.field.padEnd(11)} ${pct(entry.accuracy)} (${entry.correct}/${entry.evaluated})`); + } + + if (report.failures.length > 0) { + lines.push(""); + lines.push(`Failures (${report.failures.length}):`); + for (const failure of report.failures) { + const wrong = failure.fields.filter((entry) => !entry.correct); + lines.push(` - ${failure.id}`); + for (const entry of wrong) { + lines.push(` ${entry.field}: expected "${entry.expected}" got "${entry.actual}"`); + } + } + } + + return lines.join("\n"); +} diff --git a/src/eval/reviewSampleCorpus.test.ts b/src/eval/reviewSampleCorpus.test.ts new file mode 100644 index 0000000..9415c93 --- /dev/null +++ b/src/eval/reviewSampleCorpus.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; +import type { ReviewSampleRecord } from "../types/global"; +import { reviewSampleToEvalCase, reviewSamplesToEvalCases } from "./reviewSampleCorpus"; + +function record(overrides: Partial = {}, savedAt = "2026-07-05T10:00:00.000Z"): ReviewSampleRecord { + return { + savedAt, + sample: { + reason: "automatic:low-confidence", + parsed: { + name: "Gladiator's Nostalgia", + slot: "Flower of Life", + level: 20, + mainStat: "HP", + mainValue: "4,780", + setName: "Gladiator's Finale", + equipped: "Bennett", + substats: ["CRIT DMG+12.4%", "ATK+9.9%"], + }, + capture: { + width: 2560, + height: 1440, + ocr: [ + { id: "artifact-title", label: "t", text: "Gladiator's Nostalgia\nFlower of Life", confidence: 80 }, + { id: "artifact-main-stat", label: "m", text: "HP\n4,780", confidence: 80 }, + ], + }, + ...overrides, + }, + }; +} + +describe("reviewSampleToEvalCase", () => { + it("extracts ocr text and parser guesses as an unconfirmed case", () => { + const evalCase = reviewSampleToEvalCase(record()); + expect(evalCase).not.toBeNull(); + expect(evalCase?.confirmed).toBe(false); + expect(evalCase?.ocr["artifact-title"]).toContain("Gladiator's Nostalgia"); + expect(evalCase?.expect.slot).toBe("Flower of Life"); + expect(evalCase?.expect.level).toBe(20); + expect(evalCase?.meta?.source).toBe("review-sample"); + expect(evalCase?.meta?.resolution).toBe("2560x1440"); + }); + + it("drops Unknown/Not detected sentinels so they are not treated as labels", () => { + const evalCase = reviewSampleToEvalCase( + record({ + parsed: { + name: "Unknown artifact", + slot: "Circlet of Logos", + mainStat: "Unknown main stat", + mainValue: "?", + setName: "Unknown set", + equipped: "Not detected", + }, + }), + ); + expect(evalCase?.expect.name).toBeUndefined(); + expect(evalCase?.expect.mainStat).toBeUndefined(); + expect(evalCase?.expect.setName).toBeUndefined(); + expect(evalCase?.expect.equipped).toBeUndefined(); + // "?" mainValue is a sentinel non-answer, must not leak in as a label. + expect(evalCase?.expect.mainValue).toBeUndefined(); + expect(evalCase?.expect.slot).toBe("Circlet of Logos"); + }); + + it("returns null when the sample has no OCR", () => { + expect(reviewSampleToEvalCase({ savedAt: "x", sample: { capture: { ocr: [] } } })).toBeNull(); + expect(reviewSampleToEvalCase({ savedAt: "x" })).toBeNull(); + }); + + it("maps a list of records and skips empty ones", () => { + const cases = reviewSamplesToEvalCases([record(), { savedAt: "y" }, record({}, "2026-07-05T11:00:00.000Z")]); + expect(cases).toHaveLength(2); + expect(new Set(cases.map((entry) => entry.id)).size).toBe(2); + }); +}); diff --git a/src/eval/reviewSampleCorpus.ts b/src/eval/reviewSampleCorpus.ts new file mode 100644 index 0000000..4f17ee5 --- /dev/null +++ b/src/eval/reviewSampleCorpus.ts @@ -0,0 +1,92 @@ +import type { ReviewSampleRecord } from "../types/global"; +import type { OcrEvalCase } from "./ocrEvalHarness"; + +// Turns saved review samples into eval cases so the review queue becomes a +// growing labeled corpus (see docs/ocr-eval.md). +// +// IMPORTANT: a review sample's `parsed` block is the *auto*-parse that was +// flagged for review - it is a label CANDIDATE, not verified ground truth. +// Using it directly as an expectation would be circular (the parser grading +// itself). The intended flow is: +// 1. reviewSampleToEvalCase() extracts the OCR + the parser's current guess. +// 2. A human confirms or corrects `expect` in the produced case. +// 3. The corrected case is committed into src/eval/corpus/. +// The `confirmed` flag records whether step 2 happened. + +export interface ReviewSampleEvalCase extends OcrEvalCase { + /** False until a human has verified/corrected the `expect` values. */ + confirmed: boolean; +} + +function ocrMapFromRecord(record: ReviewSampleRecord): Record | null { + const entries = record.sample?.capture?.ocr; + if (!entries?.length) return null; + const map: Record = {}; + for (const entry of entries) { + if (typeof entry?.id === "string" && typeof entry?.text === "string") { + map[entry.id] = entry.text; + } + } + return Object.keys(map).length > 0 ? map : null; +} + +function expectFromParsed(parsed: unknown): OcrEvalCase["expect"] { + if (!parsed || typeof parsed !== "object") return {}; + const candidate = parsed as Record; + const expect: OcrEvalCase["expect"] = {}; + + const stringField = (key: "name" | "slot" | "mainStat" | "mainValue" | "setName" | "equipped") => { + const value = candidate[key]; + // Skip the parser's own "Unknown ..." / "Not detected" / "?" sentinels - + // those are non-answers, not labels a human would confirm. + if (typeof value === "string" && value && !value.startsWith("Unknown") && value !== "Not detected" && value !== "?") { + expect[key] = value; + } + }; + + stringField("name"); + stringField("slot"); + stringField("mainStat"); + stringField("mainValue"); + stringField("setName"); + stringField("equipped"); + + if (typeof candidate.level === "number" && Number.isFinite(candidate.level) && candidate.level > 0) { + expect.level = candidate.level; + } + if (Array.isArray(candidate.substats) && candidate.substats.every((entry) => typeof entry === "string")) { + expect.substats = candidate.substats as string[]; + } + + return expect; +} + +export function reviewSampleToEvalCase(record: ReviewSampleRecord, index = 0): ReviewSampleEvalCase | null { + const ocr = ocrMapFromRecord(record); + if (!ocr) return null; + + const savedAt = record.savedAt ?? "unknown"; + const idSuffix = savedAt.replace(/[^0-9A-Za-z]/g, "").slice(0, 14) || String(index); + const capture = record.sample?.capture; + + return { + id: `review-${idSuffix}-${index}`, + ocr, + expect: expectFromParsed(record.sample?.parsed), + confirmed: false, + meta: { + source: "review-sample", + resolution: capture?.width && capture?.height ? `${capture.width}x${capture.height}` : undefined, + note: record.sample?.reason, + }, + }; +} + +export function reviewSamplesToEvalCases(records: readonly ReviewSampleRecord[] | null | undefined) { + const cases: ReviewSampleEvalCase[] = []; + (records ?? []).forEach((record, index) => { + const evalCase = reviewSampleToEvalCase(record, index); + if (evalCase) cases.push(evalCase); + }); + return cases; +} From 92345ef51aad1dc331cd88c1e017b04167ecacb9 Mon Sep 17 00:00:00 2001 From: AzuTear Date: Sun, 5 Jul 2026 21:25:34 +0200 Subject: [PATCH 02/14] fix(scan): repair pre-existing tsc errors so the build is green The scan feature had 16 tsc errors from refactor drift; npm run build failed. - modals/hooks/* are five levels deep but imported ../../../../lib (four); corrected to ../../../../../lib. - useScanDiagnosticsModalModel / useScanReviewQueueModalModel picked saveReviewSample / canSaveReviewSample / loadReviewQueue from the modal Props, but the components wire those through from controller.*; source them from the controller type instead (fixes the downstream unknown-type errors). - useScanResultCardModel let its field-row tuple array widen to (string | ParsedField)[][]; annotate it Array<[string, ParsedField]> like the sibling hook. - ScanTopControlsModel was missing autoScanRunning (destructured by the component); add it. useScanTopControlsModel does not use refreshCaptureSources, so its input is Omit<...,"refreshCaptureSources">. tsc, vite build, and the electron build all pass; 74 tests green. Co-Authored-By: Claude Opus 4.8 --- .../hooks/useScanResultCardsModel.ts | 19 ++++++++++--------- .../hooks/useScanTopControlsModel.ts | 4 +++- .../modals/hooks/useScanDetailsModalModel.ts | 2 +- .../hooks/useScanDiagnosticsModalModel.ts | 12 ++++++++---- .../hooks/useScanReviewQueueModalModel.ts | 6 ++++-- .../modals/hooks/useScanSettingsModalModel.ts | 2 +- 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/features/scan/components/hooks/useScanResultCardsModel.ts b/src/features/scan/components/hooks/useScanResultCardsModel.ts index 95a0a10..41f1c44 100644 --- a/src/features/scan/components/hooks/useScanResultCardsModel.ts +++ b/src/features/scan/components/hooks/useScanResultCardsModel.ts @@ -60,19 +60,20 @@ export function useScanResultCardModel({ parsed, }: Pick): ScanResultCardModel { const substats = parsed.substats; + const rows: Array<[string, ParsedField]> = [ + ["Name", parsed.fields.name], + ["Slot", parsed.fields.slot], + ["Level", getLevelField(parsed)], + ["Main", mergeField(parsed.fields.mainStat, parsed.fields.mainValue)], + ["Set", parsed.fields.setName], + ["Equipped", parsed.fields.equipped], + ["Substats", parsed.fields.substats], + ]; return { levelField: getLevelField(parsed), quality: resolveQuality(parsed.confidence), - fieldRows: [ - ["Name", parsed.fields.name], - ["Slot", parsed.fields.slot], - ["Level", getLevelField(parsed)], - ["Main", mergeField(parsed.fields.mainStat, parsed.fields.mainValue)], - ["Set", parsed.fields.setName], - ["Equipped", parsed.fields.equipped], - ["Substats", parsed.fields.substats], - ].map(([label, field]) => ({ + fieldRows: rows.map(([label, field]) => ({ label, field, confidenceClassName: resolveFieldConfidenceClass(field), diff --git a/src/features/scan/components/hooks/useScanTopControlsModel.ts b/src/features/scan/components/hooks/useScanTopControlsModel.ts index c4e8a6a..e5a386a 100644 --- a/src/features/scan/components/hooks/useScanTopControlsModel.ts +++ b/src/features/scan/components/hooks/useScanTopControlsModel.ts @@ -8,6 +8,7 @@ export interface ScanTopControlsModel { canStartAutoScan: boolean; canStartManualScan: boolean; canCaptureSingle: boolean; + autoScanRunning: boolean; handleSourceChange: (event: ChangeEvent) => void; selectGenshinSource: () => void; openSettings: () => void; @@ -41,7 +42,7 @@ export function useScanTopControlsModel({ bridgeReady, isScanning, controller, -}: ScanTopControlsSectionProps): ScanTopControlsModel { +}: Omit): ScanTopControlsModel { const { setSettingsOpen, setDiagnosticsOpen, @@ -116,6 +117,7 @@ export function useScanTopControlsModel({ canStartAutoScan: hasSourceSelected && !isScanning && !autoScanRunning && canAutoScan && !requiresAdminForAutoScan, canStartManualScan: hasSourceSelected && !isScanning && !autoScanRunning && canCaptureSource, canCaptureSingle: hasSourceSelected && !isScanning && !autoScanRunning && canCaptureSource, + autoScanRunning, handleSourceChange, selectGenshinSource, openSettings, diff --git a/src/features/scan/components/modals/hooks/useScanDetailsModalModel.ts b/src/features/scan/components/modals/hooks/useScanDetailsModalModel.ts index c705690..705c90e 100644 --- a/src/features/scan/components/modals/hooks/useScanDetailsModalModel.ts +++ b/src/features/scan/components/modals/hooks/useScanDetailsModalModel.ts @@ -1,6 +1,6 @@ import { useCallback, type MouseEvent } from "react"; import type { ScanDetailsModalProps } from "../types"; -import type { ParsedArtifactCandidate } from "../../../../lib/artifactOcrParser"; +import type { ParsedArtifactCandidate } from "../../../../../lib/artifactOcrParser"; export interface ScanDetailsModalModel { closeDetails: () => void; diff --git a/src/features/scan/components/modals/hooks/useScanDiagnosticsModalModel.ts b/src/features/scan/components/modals/hooks/useScanDiagnosticsModalModel.ts index 9534871..15d7cf2 100644 --- a/src/features/scan/components/modals/hooks/useScanDiagnosticsModalModel.ts +++ b/src/features/scan/components/modals/hooks/useScanDiagnosticsModalModel.ts @@ -1,5 +1,5 @@ -import { detailFingerprint } from "../../../../lib/autoScanLoop"; -import { sourceVersion } from "../../../../lib/genshinData"; +import { detailFingerprint } from "../../../../../lib/autoScanLoop"; +import { sourceVersion } from "../../../../../lib/genshinData"; import { useCallback, useMemo, type MouseEvent } from "react"; import type { ScanDiagnosticsModalProps } from "../types"; @@ -39,16 +39,20 @@ export interface ScanDiagnosticsModalModel { canSaveReviewSample: boolean; } +type ScanDiagnosticsController = ScanDiagnosticsModalProps["controller"]; + interface UseScanDiagnosticsModalModelInput extends Pick< ScanDiagnosticsModalProps, | "setDetailsOpen" | "setDiagnosticsOpen" - | "saveReviewSample" - | "canSaveReviewSample" | "latestCapture" | "controller" > { captureStatus: string; + // saveReviewSample / canSaveReviewSample live on the controller, not the modal + // props; the component wires them through from controller.* . + saveReviewSample: ScanDiagnosticsController["saveReviewSample"]; + canSaveReviewSample: ScanDiagnosticsController["canSaveReviewSample"]; } export function useScanDiagnosticsModalModel({ diff --git a/src/features/scan/components/modals/hooks/useScanReviewQueueModalModel.ts b/src/features/scan/components/modals/hooks/useScanReviewQueueModalModel.ts index 0d850ef..a14f096 100644 --- a/src/features/scan/components/modals/hooks/useScanReviewQueueModalModel.ts +++ b/src/features/scan/components/modals/hooks/useScanReviewQueueModalModel.ts @@ -1,5 +1,5 @@ import { useCallback, useMemo, type MouseEvent } from "react"; -import type { ReviewSampleAnalysis } from "../../../../lib/reviewSampleAnalysis"; +import type { ReviewSampleAnalysis } from "../../../../../lib/reviewSampleAnalysis"; import type { ScanReviewQueueModalProps } from "../types"; export interface ScanReviewQueueRow { @@ -18,8 +18,10 @@ export interface ScanReviewQueueModalModel { interface UseScanReviewQueueModalModelInput extends Pick< ScanReviewQueueModalProps, - "setReviewQueueOpen" | "loadReviewQueue" + "setReviewQueueOpen" > { + // loadReviewQueue is wired through from controller.* by the component. + loadReviewQueue: ScanReviewQueueModalProps["controller"]["loadReviewQueue"]; reviewAnalysis: ReviewSampleAnalysis; reviewSampleTotal: number; } diff --git a/src/features/scan/components/modals/hooks/useScanSettingsModalModel.ts b/src/features/scan/components/modals/hooks/useScanSettingsModalModel.ts index 6793c80..7744eca 100644 --- a/src/features/scan/components/modals/hooks/useScanSettingsModalModel.ts +++ b/src/features/scan/components/modals/hooks/useScanSettingsModalModel.ts @@ -1,6 +1,6 @@ import { useCallback, type ChangeEvent, type MouseEvent } from "react"; import type { CaptureResult, RuntimeInfo } from "../../../../../types/global"; -import { clampScanLimit, clampSkipRows } from "../../../../lib/scannerSession"; +import { clampScanLimit, clampSkipRows } from "../../../../../lib/scannerSession"; export interface ScanSettingsModalModel { closeSettings: () => void; From c7138b541d33f356339f7de115fca227c18670c5 Mon Sep 17 00:00:00 2001 From: AzuTear Date: Sun, 5 Jul 2026 21:33:26 +0200 Subject: [PATCH 03/14] feat(native): C# input/capture sidecar replacing the PowerShell helper Implements ADR-008. native/input-helper is a self-contained .NET 9 console exe speaking the identical JSON-over-stdin/stdout protocol as the old PowerShell helper (ping/cursor/runtime/focus/click/scroll/bounds/capture), so the InputHelperService interface is unchanged. - Win32 interop compiled once (native exe), not per call. - PerMonitorV2 DPI via manifest so click/capture coordinates stay correct on mixed-DPI multi-monitor setups. - capture returns base64 PNG bytes inline (imageBase64) instead of writing a temp file per frame; the client handles both base64 and the PowerShell path. - InputHelperClient prefers the exe and falls back to the embedded PowerShell helper when the exe is absent, so the app still runs without the .NET build. - main.ts resolves the exe (INPUT_HELPER_EXE env -> packaged resources/input-helper -> native/input-helper/bin/publish). electron-builder ships it via extraResources. - npm run helper:build; README documents the build + fallback. Verified end-to-end through the compiled client: sidecar spawns, runtime info and a base64 primary-screen capture return correctly. Build stays green. Co-Authored-By: Claude Opus 4.8 --- README.md | 14 + electron/main.ts | 22 +- electron/services/inputHelper.ts | 59 +++- native/input-helper/.gitignore | 4 + native/input-helper/InputHelper.csproj | 23 ++ native/input-helper/Program.cs | 466 +++++++++++++++++++++++++ native/input-helper/app.manifest | 11 + package.json | 10 + 8 files changed, 597 insertions(+), 12 deletions(-) create mode 100644 native/input-helper/.gitignore create mode 100644 native/input-helper/InputHelper.csproj create mode 100644 native/input-helper/Program.cs create mode 100644 native/input-helper/app.manifest diff --git a/README.md b/README.md index 983aee3..af7da9a 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,20 @@ npm run dev Use the Electron app window for scanner work. The browser preview does not expose the local capture bridge. +### Input/Capture helper (C# sidecar) + +Input automation and screen capture run through a compiled C# sidecar +(`native/input-helper`, see ADR-008). Build it once: + +```powershell +npm run helper:build # requires the .NET SDK; produces a self-contained exe +``` + +The app auto-detects the exe (`INPUT_HELPER_EXE` env override → packaged +`resources/input-helper` → `native/input-helper/bin/publish`). If the exe is not +present it falls back to the embedded PowerShell helper, so the app still runs +without the .NET build - just slower and with the old per-frame temp-file capture. + ### Automatischer Scan: als Administrator starten Genshin läuft erhöht (Administrator). Windows (UIPI) verwirft dann alle simulierten Maus-Eingaben aus einer nicht-erhöhten App - SendInput meldet dabei trotzdem Erfolg. Für den automatischen Scan muss die App deshalb ebenfalls erhöht laufen: diff --git a/electron/main.ts b/electron/main.ts index 2f7749e..f8cbc11 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -1,5 +1,6 @@ import { app, BrowserWindow, Menu, desktopCapturer, globalShortcut, nativeImage, screen, type NativeImage } from "electron"; import fs from "node:fs/promises"; +import { existsSync } from "node:fs"; import http, { type Server } from "node:http"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -70,6 +71,25 @@ function getInputHelperService() { return inputHelperService; } +// Locate the compiled C# input/capture sidecar (ADR-008). Falls back to null so +// the service uses the embedded PowerShell helper when the exe was never built. +function resolveInputHelperExePath(): string | null { + const candidates = [ + process.env.INPUT_HELPER_EXE, + path.join(process.resourcesPath, "input-helper", "InputHelper.exe"), + path.join(app.getAppPath(), "native", "input-helper", "bin", "publish", "InputHelper.exe"), + ].filter((candidate): candidate is string => Boolean(candidate)); + + for (const candidate of candidates) { + try { + if (existsSync(candidate)) return candidate; + } catch { + // Unreadable path; try the next candidate. + } + } + return null; +} + function getRepositoryContext() { if (!repositoryContext) { throw new Error("Repository context has not been initialized."); @@ -1032,7 +1052,7 @@ function initializeAppLifecycle() { artifactStoreRepository = repositoryContext.artifactStoreRepository; reviewSamplesRepository = repositoryContext.reviewSamplesRepository; scannerLearningRepository = repositoryContext.scannerLearningRepository; - inputHelperService = createInputHelperService({ userDataPath }); + inputHelperService = createInputHelperService({ userDataPath, exePath: resolveInputHelperExePath() }); registerIpcHandlers({ focusMainWindow: () => focusMainWindow(), diff --git a/electron/services/inputHelper.ts b/electron/services/inputHelper.ts index 80e46dd..7b11462 100644 --- a/electron/services/inputHelper.ts +++ b/electron/services/inputHelper.ts @@ -382,7 +382,7 @@ class InputHelperClient { private starting: Promise | null = null; private disposed = false; - constructor(private readonly scriptUserDataPath: string) {} + constructor(private readonly options: { scriptUserDataPath: string; exePath?: string | null }) {} private async ensureStarted() { if (this.child) return; @@ -396,14 +396,31 @@ class InputHelperClient { } private async start() { - const scriptPath = path.join(this.scriptUserDataPath, "input-helper.ps1"); + // Prefer the compiled C# sidecar (ADR-008). If it is missing or fails to + // start, fall back to the embedded PowerShell helper so the app keeps working + // on machines where the native exe was never built. + if (this.options.exePath) { + try { + await this.startWith(spawn(this.options.exePath, [], { windowsHide: true, stdio: ["pipe", "pipe", "pipe"] })); + return; + } catch { + this.teardownChild(); + } + } + await this.startWith(await this.spawnPowershell()); + } + + private async spawnPowershell() { + const scriptPath = path.join(this.options.scriptUserDataPath, "input-helper.ps1"); await fs.mkdir(path.dirname(scriptPath), { recursive: true }); await fs.writeFile(scriptPath, INPUT_HELPER_SCRIPT, "utf8"); - - const child = spawn("powershell.exe", ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", scriptPath], { + return spawn("powershell.exe", ["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", scriptPath], { windowsHide: true, stdio: ["pipe", "pipe", "pipe"], }); + } + + private async startWith(child: ChildProcessWithoutNullStreams) { child.stdout.setEncoding("utf8"); child.stdout.on("data", (chunk: string) => this.handleStdout(chunk)); child.stderr.setEncoding("utf8"); @@ -419,10 +436,22 @@ class InputHelperClient { }); this.child = child; - // First request compiles the Win32 interop; give it extra time. + // The C# sidecar answers ping immediately; the PowerShell fallback compiles + // Win32 interop on the first request, so give it extra time. await this.send("ping", {}, 20000); } + private teardownChild() { + const child = this.child; + this.child = null; + this.buffer = ""; + try { + child?.kill(); + } catch { + // Child was never spawned or already gone. + } + } + private handleStdout(chunk: string) { this.buffer += chunk; let newlineIndex = this.buffer.indexOf("\n"); @@ -485,8 +514,8 @@ export interface InputHelperService { dispose(): void; } -export function createInputHelperService(options: { userDataPath: string }): InputHelperService { - const inputHelper = new InputHelperClient(options.userDataPath); +export function createInputHelperService(options: { userDataPath: string; exePath?: string | null }): InputHelperService { + const inputHelper = new InputHelperClient({ scriptUserDataPath: options.userDataPath, exePath: options.exePath ?? null }); async function request(op: string, params: Record = {}, timeoutMs = 8000) { return inputHelper.request(op, params, timeoutMs); @@ -590,16 +619,24 @@ export function createInputHelperService(options: { userDataPath: string }): Inp async function capturePrimaryScreenViaGdi() { const result = await request("capture", {}, 15000); - const capturePath = String(result.path); - const buffer = await fs.readFile(capturePath); - await fs.unlink(capturePath).catch(() => undefined); + // The C# sidecar returns PNG bytes inline (no temp file). The PowerShell + // fallback writes a temp PNG and returns its path. + let base64: string; + if (typeof result.imageBase64 === "string" && result.imageBase64) { + base64 = result.imageBase64; + } else { + const capturePath = String(result.path); + const buffer = await fs.readFile(capturePath); + await fs.unlink(capturePath).catch(() => undefined); + base64 = buffer.toString("base64"); + } const captureTargetRaw = typeof result.captureTarget === "string" ? result.captureTarget : ""; const captureTarget: "primary-screen" | "genshin-client" = captureTargetRaw === "primary-screen" || captureTargetRaw === "genshin-client" ? captureTargetRaw : "primary-screen"; return { - dataUrl: `data:image/png;base64,${buffer.toString("base64")}`, + dataUrl: `data:image/png;base64,${base64}`, width: Number(result.width), height: Number(result.height), originX: Number(result.originX), diff --git a/native/input-helper/.gitignore b/native/input-helper/.gitignore new file mode 100644 index 0000000..fa06666 --- /dev/null +++ b/native/input-helper/.gitignore @@ -0,0 +1,4 @@ +# .NET build outputs — the self-contained exe is built via `npm run helper:build`, +# not committed (it is ~100 MB). +bin/ +obj/ diff --git a/native/input-helper/InputHelper.csproj b/native/input-helper/InputHelper.csproj new file mode 100644 index 0000000..e8f52ad --- /dev/null +++ b/native/input-helper/InputHelper.csproj @@ -0,0 +1,23 @@ + + + + Exe + net9.0-windows + enable + enable + InputHelper + GenshinAssistant.InputHelper + + true + + win-x64 + true + true + true + true + + app.manifest + + + diff --git a/native/input-helper/Program.cs b/native/input-helper/Program.cs new file mode 100644 index 0000000..26f5943 --- /dev/null +++ b/native/input-helper/Program.cs @@ -0,0 +1,466 @@ +using System.Diagnostics; +using System.Drawing; +using System.Drawing.Imaging; +using System.Runtime.InteropServices; +using System.Security.Principal; +using System.Text; +using System.Text.Json; +using System.Windows.Forms; + +// Long-lived input/capture sidecar for the Genshin Artifact Assistant. +// Drop-in replacement for the old PowerShell helper (see ADR-008): identical +// JSON-over-stdin/stdout protocol - one JSON request per line, one JSON response +// per line - so the Electron-side InputHelperService is unchanged. Win32 interop +// is compiled once (this is a native exe), and capture returns base64 PNG bytes +// directly instead of writing a temp file per frame. + +namespace GenshinAssistant.InputHelper; + +internal static class Program +{ + private static IntPtr _genshinHwnd = IntPtr.Zero; + + private static int Main() + { + // Manifest already declares PerMonitorV2; this is a belt-and-suspenders + // call for hosts that ignore the manifest. + try { Native.SetProcessDpiAwarenessContext(Native.DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2); } + catch { try { Native.SetProcessDpiAwareness(2); } catch { /* oldest fallback */ Native.SetProcessDPIAware(); } } + + Console.OutputEncoding = Encoding.UTF8; + var stdout = Console.Out; + + string? line; + while ((line = Console.In.ReadLine()) != null) + { + if (line.Trim().Length == 0) continue; + + var response = new Dictionary { ["id"] = "", ["ok"] = true }; + try + { + using var doc = JsonDocument.Parse(line); + var root = doc.RootElement; + response["id"] = GetString(root, "id"); + var op = GetString(root, "op"); + Handle(op, root, response); + } + catch (Exception ex) + { + response["ok"] = false; + response["error"] = ex.Message; + } + + stdout.WriteLine(JsonSerializer.Serialize(response)); + stdout.Flush(); + } + + return 0; + } + + private static void Handle(string op, JsonElement root, Dictionary response) + { + switch (op) + { + case "ping": + response["pong"] = true; + break; + + case "cursor": + { + var state = GetCursorState(); + response["cursorX"] = state.X; + response["cursorY"] = state.Y; + response["escapePressed"] = state.Escape; + response["enterPressed"] = state.Enter; + response["f9Pressed"] = state.F9; + break; + } + + case "runtime": + { + var hwnd = FindGenshinWindow(); + var fgHwnd = Native.GetForegroundWindow(); + response["isElevated"] = IsElevated(); + response["genshinFound"] = hwnd != IntPtr.Zero; + response["genshinHwnd"] = hwnd.ToInt64(); + response["targetProcess"] = ProcessNameFromHwnd(hwnd); + response["foregroundProcess"] = ProcessNameFromHwnd(fgHwnd); + response["foregroundHwnd"] = fgHwnd.ToInt64(); + response["helperPid"] = Environment.ProcessId; + break; + } + + case "focus": + { + var info = FocusGenshinWindow(); + response["focused"] = info.Focused; + response["alreadyForeground"] = info.AlreadyForeground; + response["foregroundProcess"] = info.ForegroundProcess; + response["targetProcess"] = info.TargetProcess; + response["genshinFound"] = info.Hwnd != IntPtr.Zero; + response["setForegroundResult"] = info.SetForegroundResult; + break; + } + + case "click": + { + var info = FocusGenshinWindow(); + if (info.Focused && !info.AlreadyForeground) Thread.Sleep(120); + + var targetX = GetInt(root, "x"); + var targetY = GetInt(root, "y"); + // Bare SetCursorPos then a batched down+up click, matching the + // verified Inventory Kamera sequence: no extra move event, no + // gap between move and click. + Native.SetCursorPos(targetX, targetY); + Native.GetCursorPos(out var pt); + var onTarget = Math.Abs(targetX - pt.X) <= 2 && Math.Abs(targetY - pt.Y) <= 2; + var clickEventsSent = onTarget ? SendMouseClickBatch() : 0u; + + var state = GetCursorState(); + response["cursorX"] = state.X; + response["cursorY"] = state.Y; + response["escapePressed"] = state.Escape; + response["enterPressed"] = state.Enter; + response["f9Pressed"] = state.F9; + response["moved"] = onTarget; + response["focused"] = info.Focused; + response["alreadyForeground"] = info.AlreadyForeground; + response["foregroundProcess"] = info.ForegroundProcess; + response["targetProcess"] = info.TargetProcess; + response["isElevated"] = IsElevated(); + // Only report a click when the cursor is verifiably on target and + // SendInput injected both events; real acceptance is proven later + // by the detail-panel fingerprint. + response["clicked"] = onTarget && clickEventsSent >= 2; + response["inputBlocked"] = onTarget && clickEventsSent < 2; + break; + } + + case "scroll": + { + var info = FocusGenshinWindow(); + if (info.Focused && !info.AlreadyForeground) Thread.Sleep(120); + + if (TryGetInt(root, "x", out var ax) && TryGetInt(root, "y", out var ay)) + { + Native.SetCursorPos(ax, ay); + Thread.Sleep(30); + } + + Native.GetCursorPos(out var pt); + response["cursorX"] = pt.X; + response["cursorY"] = pt.Y; + response["focused"] = info.Focused; + response["foregroundProcess"] = info.ForegroundProcess; + response["isElevated"] = IsElevated(); + + var notches = GetInt(root, "notches"); + var stepDelta = notches < 0 ? -120 : 120; + var count = Math.Min(60, Math.Abs(notches)); + uint sentTotal = 0; + for (var i = 0; i < count; i++) + { + sentTotal += SendMouseWheel(stepDelta); + Thread.Sleep(45); + } + response["notchesSent"] = sentTotal; + response["inputBlocked"] = count > 0 && sentTotal == 0; + break; + } + + case "bounds": + { + var bounds = GetGenshinClientBounds(); + if (bounds == null) + { + response["found"] = false; + } + else + { + response["found"] = true; + response["left"] = bounds.Value.Left; + response["top"] = bounds.Value.Top; + response["width"] = bounds.Value.Width; + response["height"] = bounds.Value.Height; + } + break; + } + + case "capture": + { + var bounds = GetGenshinClientBounds(); + string captureTarget; + Rect area; + if (bounds == null) + { + var screen = Screen.PrimaryScreen!.Bounds; + area = new Rect { Left = screen.Left, Top = screen.Top, Width = screen.Width, Height = screen.Height }; + captureTarget = "primary-screen"; + } + else + { + area = bounds.Value; + captureTarget = "genshin-client"; + } + + using var bitmap = new Bitmap(area.Width, area.Height, PixelFormat.Format32bppArgb); + using (var graphics = Graphics.FromImage(bitmap)) + { + graphics.CopyFromScreen(area.Left, area.Top, 0, 0, bitmap.Size, CopyPixelOperation.SourceCopy); + } + + using var stream = new MemoryStream(); + bitmap.Save(stream, ImageFormat.Png); + response["imageBase64"] = Convert.ToBase64String(stream.ToArray()); + response["width"] = area.Width; + response["height"] = area.Height; + response["originX"] = area.Left; + response["originY"] = area.Top; + response["captureTarget"] = captureTarget; + break; + } + + default: + response["ok"] = false; + response["error"] = "unknown op"; + break; + } + } + + private readonly struct Rect + { + public int Left { get; init; } + public int Top { get; init; } + public int Width { get; init; } + public int Height { get; init; } + } + + private struct CursorState + { + public int X; + public int Y; + public bool Escape; + public bool Enter; + public bool F9; + } + + private struct FocusInfo + { + public IntPtr Hwnd; + public bool Focused; + public bool AlreadyForeground; + public string ForegroundProcess; + public string TargetProcess; + public bool SetForegroundResult; + } + + private static CursorState GetCursorState() + { + Native.GetCursorPos(out var pt); + // Only 0x8000 (held right now). The 0x0001 "pressed since last call" bit + // is unreliable and fires for ESC presses used to navigate Genshin menus. + var esc = (Native.GetAsyncKeyState(0x1B) & 0x8000) != 0; + var enter = (Native.GetAsyncKeyState(0x0D) & 0x8000) != 0; + var f9 = (Native.GetAsyncKeyState(0x78) & 0x8000) != 0; + return new CursorState { X = pt.X, Y = pt.Y, Escape = esc, Enter = enter, F9 = f9 }; + } + + private static FocusInfo FocusGenshinWindow() + { + var hwnd = FindGenshinWindow(); + var info = new FocusInfo + { + Hwnd = hwnd, + ForegroundProcess = "", + TargetProcess = ProcessNameFromHwnd(hwnd), + }; + if (hwnd == IntPtr.Zero) return info; + + info.AlreadyForeground = Native.GetForegroundWindow() == hwnd; + if (!info.AlreadyForeground) + { + Native.ShowWindowAsync(hwnd, 9); // SW_RESTORE + // No ALT tap: this app and Genshin run at the same (elevated) + // integrity level, so SetForegroundWindow succeeds on its own. An ALT + // tap would toggle menu-mnemonic mode and swallow the next inputs. + info.SetForegroundResult = Native.SetForegroundWindow(hwnd); + Thread.Sleep(140); + } + + var foreground = Native.GetForegroundWindow(); + info.Focused = foreground == hwnd; + info.ForegroundProcess = ProcessNameFromHwnd(foreground); + return info; + } + + private static IntPtr FindGenshinWindow() + { + if (_genshinHwnd != IntPtr.Zero && Native.IsWindow(_genshinHwnd)) return _genshinHwnd; + + foreach (var proc in Process.GetProcesses()) + { + try + { + var name = proc.ProcessName; + if ((name.Contains("GenshinImpact", StringComparison.OrdinalIgnoreCase) + || name.Contains("YuanShen", StringComparison.OrdinalIgnoreCase) + || name.Contains("Genshin", StringComparison.OrdinalIgnoreCase)) + && proc.MainWindowHandle != IntPtr.Zero) + { + _genshinHwnd = proc.MainWindowHandle; + return _genshinHwnd; + } + } + catch + { + // Process exited between enumeration and inspection; ignore. + } + } + + _genshinHwnd = IntPtr.Zero; + return _genshinHwnd; + } + + private static Rect? GetGenshinClientBounds() + { + var hwnd = FindGenshinWindow(); + if (hwnd == IntPtr.Zero) return null; + if (!Native.GetClientRect(hwnd, out var rect)) return null; + + var topLeft = new Native.POINT { X = 0, Y = 0 }; + if (!Native.ClientToScreen(hwnd, ref topLeft)) return null; + + var width = rect.Right - rect.Left; + var height = rect.Bottom - rect.Top; + if (width <= 0 || height <= 0) return null; + + return new Rect { Left = topLeft.X, Top = topLeft.Y, Width = width, Height = height }; + } + + private static string ProcessNameFromHwnd(IntPtr hwnd) + { + if (hwnd == IntPtr.Zero) return ""; + Native.GetWindowThreadProcessId(hwnd, out var pid); + if (pid == 0) return ""; + try { return Process.GetProcessById((int)pid).ProcessName; } + catch { return ""; } + } + + private static bool IsElevated() + { + using var identity = WindowsIdentity.GetCurrent(); + var principal = new WindowsPrincipal(identity); + return principal.IsInRole(WindowsBuiltInRole.Administrator); + } + + private static uint SendMouseClickBatch() + { + var inputs = new Native.INPUT[2]; + inputs[0].type = 0; // INPUT_MOUSE + inputs[0].mi.dwFlags = Native.MOUSEEVENTF_LEFTDOWN; + inputs[1].type = 0; + inputs[1].mi.dwFlags = Native.MOUSEEVENTF_LEFTUP; + return Native.SendInput(2, inputs, Marshal.SizeOf()); + } + + private static uint SendMouseWheel(int wheelData) + { + var inputs = new Native.INPUT[1]; + inputs[0].type = 0; + inputs[0].mi.mouseData = unchecked((uint)wheelData); + inputs[0].mi.dwFlags = Native.MOUSEEVENTF_WHEEL; + return Native.SendInput(1, inputs, Marshal.SizeOf()); + } + + private static string GetString(JsonElement root, string name) + => root.TryGetProperty(name, out var value) ? value.ToString() : ""; + + private static int GetInt(JsonElement root, string name) + => TryGetInt(root, name, out var value) ? value : 0; + + private static bool TryGetInt(JsonElement root, string name, out int value) + { + value = 0; + if (!root.TryGetProperty(name, out var element)) return false; + if (element.ValueKind == JsonValueKind.Number && element.TryGetInt32(out value)) return true; + if (element.ValueKind == JsonValueKind.String && int.TryParse(element.GetString(), out value)) return true; + return false; + } +} + +internal static class Native +{ + public const uint MOUSEEVENTF_LEFTDOWN = 0x0002; + public const uint MOUSEEVENTF_LEFTUP = 0x0004; + public const uint MOUSEEVENTF_WHEEL = 0x0800; + public static readonly IntPtr DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2 = new(-4); + + [StructLayout(LayoutKind.Sequential)] + public struct POINT { public int X; public int Y; } + + [StructLayout(LayoutKind.Sequential)] + public struct RECT { public int Left; public int Top; public int Right; public int Bottom; } + + // Binary-compatible with the Win32 INPUT for mouse-only use on x64: + // type(4) + 4 pad + MOUSEINPUT(32) = 40 bytes = sizeof(INPUT). + [StructLayout(LayoutKind.Sequential)] + public struct MOUSEINPUT + { + public int dx; + public int dy; + public uint mouseData; + public uint dwFlags; + public uint time; + public UIntPtr dwExtraInfo; + } + + [StructLayout(LayoutKind.Sequential)] + public struct INPUT + { + public int type; + public MOUSEINPUT mi; + } + + [DllImport("user32.dll")] + public static extern bool SetProcessDPIAware(); + + [DllImport("shcore.dll")] + public static extern int SetProcessDpiAwareness(int value); + + [DllImport("user32.dll")] + public static extern bool SetProcessDpiAwarenessContext(IntPtr value); + + [DllImport("user32.dll")] + public static extern bool SetCursorPos(int x, int y); + + [DllImport("user32.dll")] + public static extern bool GetCursorPos(out POINT lpPoint); + + [DllImport("user32.dll", SetLastError = true)] + public static extern bool GetClientRect(IntPtr hWnd, out RECT lpRect); + + [DllImport("user32.dll", SetLastError = true)] + public static extern bool ClientToScreen(IntPtr hWnd, ref POINT lpPoint); + + [DllImport("user32.dll")] + public static extern short GetAsyncKeyState(int vKey); + + [DllImport("user32.dll", SetLastError = true)] + public static extern uint SendInput(uint nInputs, INPUT[] pInputs, int cbSize); + + [DllImport("user32.dll")] + public static extern bool SetForegroundWindow(IntPtr hWnd); + + [DllImport("user32.dll")] + public static extern bool ShowWindowAsync(IntPtr hWnd, int nCmdShow); + + [DllImport("user32.dll")] + public static extern IntPtr GetForegroundWindow(); + + [DllImport("user32.dll")] + public static extern bool IsWindow(IntPtr hWnd); + + [DllImport("user32.dll")] + public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint lpdwProcessId); +} diff --git a/native/input-helper/app.manifest b/native/input-helper/app.manifest new file mode 100644 index 0000000..0895eea --- /dev/null +++ b/native/input-helper/app.manifest @@ -0,0 +1,11 @@ + + + + + + + PerMonitorV2 + true/pm + + + diff --git a/package.json b/package.json index 8f5b356..c19282a 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "lint": "tsc --noEmit", "test": "vitest run", "eval": "vitest run src/eval/ocrEval.test.ts", + "helper:build": "dotnet publish native/input-helper/InputHelper.csproj -c Release -o native/input-helper/bin/publish", "data:genshin": "node scripts/generate-genshin-data.cjs" }, "dependencies": { @@ -48,6 +49,15 @@ "dist-electron/**/*", "package.json" ], + "extraResources": [ + { + "from": "native/input-helper/bin/publish", + "to": "input-helper", + "filter": [ + "**/*" + ] + } + ], "win": { "target": "nsis", "requestedExecutionLevel": "requireAdministrator" From 2c6c1a8b3135f1da6b9a359565881740d366a4dd Mon Sep 17 00:00:00 2001 From: AzuTear Date: Sun, 5 Jul 2026 21:42:16 +0200 Subject: [PATCH 04/14] feat(ocr): resolution-anchored layout module + crop preprocessing Implements ADR-009 (structure + preprocessing; exact IK fixed coordinates still need calibration against a reference 16:9 screenshot). - src/lib/layoutProfile.ts: pure, unit-tested geometry for the artifact screen - detail rect, the four detail crops, inventory rect/count crop, 5-col grid, 16:9 detection, aspect label, and an off-16:9 support warning. Single source of truth; electron/main.ts now delegates all crop/grid geometry to it and keeps colour detection only as the detail-rect fallback. - src/lib/ocrPreprocess.ts: pure, unit-tested Otsu binarization with inversion (artifact text is the bright foreground) over a BGRA bitmap. - main.ts: OCR now reads an upscaled + binarized copy of each crop; the original crop is retained for the diagnostics UI. CaptureResult carries layout info { aspect, isSixteenNine, warning }. NOTE: image preprocessing changes the OCR input and cannot be validated by the text-level eval harness; it needs a live Genshin 16:9 capture to confirm/tune (threshold, invert, upscale factor). 88 tests + build green. Co-Authored-By: Claude Opus 4.8 --- electron/main.ts | 182 +++++++++--------------------- src/lib/layoutProfile.test.ts | 87 +++++++++++++++ src/lib/layoutProfile.ts | 205 ++++++++++++++++++++++++++++++++++ src/lib/ocrPreprocess.test.ts | 74 ++++++++++++ src/lib/ocrPreprocess.ts | 101 +++++++++++++++++ src/types/global.d.ts | 5 + 6 files changed, 523 insertions(+), 131 deletions(-) create mode 100644 src/lib/layoutProfile.test.ts create mode 100644 src/lib/layoutProfile.ts create mode 100644 src/lib/ocrPreprocess.test.ts create mode 100644 src/lib/ocrPreprocess.ts diff --git a/electron/main.ts b/electron/main.ts index f8cbc11..3196bb6 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -22,6 +22,17 @@ import type { ReviewSamplesRepositoryPort, ScannerLearningRepositoryPort, } from "./repositories/index.js"; +import { + aspectRatioLabel, + detailCropRects, + inventoryCountCropRect, + inventoryGrid as layoutInventoryGrid, + inventoryRect as layoutInventoryRect, + isSixteenNine, + layoutSupportWarning, + profileDetailRect, +} from "../src/lib/layoutProfile.js"; +import { binarizeForOcr } from "../src/lib/ocrPreprocess.js"; // Chromium's renderer sandbox can refuse to fully initialize (or silently // crash the GPU/renderer process) when the hosting process runs with a full @@ -746,84 +757,48 @@ function imageCropDataUrl(sourceImage: NativeImage, rect: Electron.Rectangle, im return sourceImage.crop(safeRect).toDataURL(); } +// Preprocessed copy of a crop for OCR (ADR-009): upscale for more pixels, then +// grayscale + Otsu-binarize with inversion (artifact text is the bright +// foreground). The original crop is kept separately for the diagnostics UI. +function preprocessedCropDataUrl(sourceImage: NativeImage, rect: Electron.Rectangle, imageSize: { width: number; height: number }) { + const safeRect = clampCaptureRect(rect, imageSize); + const upscaled = sourceImage.crop(safeRect).resize({ width: Math.max(1, safeRect.width * 2), quality: "best" }); + const size = upscaled.getSize(); + if (!size.width || !size.height) return upscaled.toDataURL(); + const binarized = binarizeForOcr({ data: upscaled.getBitmap(), width: size.width, height: size.height }); + return nativeImage + .createFromBitmap(Buffer.from(binarized.data), { width: binarized.width, height: binarized.height }) + .toDataURL(); +} + function createCrops( sourceImage: NativeImage, imageSize: { width: number; height: number }, detailRect: Electron.Rectangle, inventoryRect: Electron.Rectangle, ) { - const templates: CropTemplate[] = [ - { - id: "artifact-title", - label: "Artifact title", - rect: { - x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.05), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.16), - }, - }, - { - id: "artifact-main-stat", - label: "Main stat", - rect: { - x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.20), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.18), - }, - }, - { - id: "artifact-substats", - label: "Substats", - rect: { - x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.41), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.25), - }, - }, - { - id: "artifact-footer", - label: "Footer", - rect: { - x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.78), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.16), - }, - }, - ]; + const templates: CropTemplate[] = detailCropRects(detailRect, imageSize); if (inventoryRect.width > 120 && inventoryRect.height > 80) { templates.push({ id: "inventory-count", label: "Inventory count", - rect: { - x: Math.round(inventoryRect.x + inventoryRect.width * 0.62), - y: Math.round(inventoryRect.y + inventoryRect.height * 0.02), - width: Math.round(inventoryRect.width * 0.34), - height: Math.round(inventoryRect.height * 0.09), - }, + rect: inventoryCountCropRect(inventoryRect, imageSize), }); } return templates - .map((template) => ({ - ...template, - rect: clampCaptureRect(template.rect, imageSize), - dataUrl: imageCropDataUrl(sourceImage, template.rect, imageSize), - })) - .filter((crop) => crop.rect.width > 0 && crop.rect.height > 0) - .map((crop) => ({ - ...crop, - rect: { - x: crop.rect.x, - y: crop.rect.y, - width: crop.rect.width, - height: crop.rect.height, - }, - })); + .map((template) => { + const rect = clampCaptureRect(template.rect, imageSize); + return { + id: template.id, + label: template.label, + rect, + dataUrl: imageCropDataUrl(sourceImage, rect, imageSize), + ocrDataUrl: preprocessedCropDataUrl(sourceImage, rect, imageSize), + }; + }) + .filter((crop) => crop.rect.width > 0 && crop.rect.height > 0); } function inferDetailRect(bitmap: Buffer, imageSize: { width: number; height: number }) { @@ -865,79 +840,17 @@ function inferDetailRect(bitmap: Buffer, imageSize: { width: number; height: num return clampCaptureRect({ x: left, y: top, width: widthGuess, height: heightGuess }, imageSize); } - if (width > 0 && height > 0) { - return clampCaptureRect( - { - x: Math.round(width * 0.50), - y: Math.round(height * 0.08), - width: Math.round(width * 0.46), - height: Math.round(height * 0.74), - }, - imageSize, - ); - } - - return { x: 0, y: 0, width, height }; + // Colour detection found nothing usable; fall back to the resolution-anchored + // profile rect (single source of truth in layoutProfile). + return profileDetailRect(imageSize); } function inferInventoryRect(imageSize: { width: number; height: number }, detailRect: Electron.Rectangle) { - const { width, height } = imageSize; - const preferredWidth = Math.max(140, Math.round(width * 0.48)); - const x = Math.round(width * 0.03); - const y = Math.round(detailRect.y + detailRect.height * 0.09); - const availableWidth = Math.max(100, detailRect.x - Math.round(width * 0.04)); - const panelWidth = Math.max(100, Math.min(preferredWidth, availableWidth)); - const safeWidth = panelWidth > width * 0.85 ? Math.round(width * 0.55) : panelWidth; - return clampCaptureRect( - { - x, - y, - width: Math.min(safeWidth, Math.max(width - x - Math.round(width * 0.02), 100)), - height: Math.max(140, Math.round(height * 0.70)), - }, - imageSize, - ); + return layoutInventoryRect(imageSize, detailRect); } function inferInventoryGrid(imageSize: { width: number; height: number }, detailRect: Electron.Rectangle) { - const inventoryRect = inferInventoryRect(imageSize, detailRect); - const cols = 5; - if (inventoryRect.width < 160 || inventoryRect.height < 140) { - return { - centers: [], - rows: 0, - cols: 0, - confidence: 0, - source: "missing" as const, - }; - } - - const cellWidth = Math.max(56, Math.round(inventoryRect.width / cols)); - const stepX = Math.round(cellWidth * 0.96); - const stepY = Math.round(cellWidth * 1.03); - const visibleRows = Math.max(2, Math.min(6, Math.round(inventoryRect.height / Math.max(stepY, 1)))); - - const startX = inventoryRect.x + Math.max(6, Math.round(stepX * 0.45)); - const startY = inventoryRect.y + Math.max(6, Math.round(stepY * 0.45)); - const centers = []; - for (let row = 0; row < visibleRows; row++) { - for (let col = 0; col < cols; col++) { - const x = startX + col * stepX; - const y = startY + row * stepY; - if (x < imageSize.width && y < imageSize.height) { - centers.push({ x, y, row, col }); - } - } - } - - const trimmed = centers.filter((center) => center.x > 0 && center.y > 0); - return { - centers: trimmed, - rows: visibleRows, - cols, - confidence: trimmed.length >= cols * 2 ? 76 : trimmed.length >= cols ? 58 : 36, - source: "detected" as const, - }; + return layoutInventoryGrid(imageSize, detailRect); } async function buildCaptureResult( @@ -959,7 +872,9 @@ async function buildCaptureResult( const croppedPayload = crops.map((crop) => ({ id: crop.id, label: crop.label, - dataUrl: crop.dataUrl, + // OCR reads the preprocessed (upscaled + binarized) crop; the original is + // kept below for the diagnostics UI. + dataUrl: crop.ocrDataUrl ?? crop.dataUrl, })); const recognized = options.skipOcr ? { ocr: [], timedOut: false } : await runOcrOnCropsWithTimeout(croppedPayload); @@ -990,6 +905,11 @@ async function buildCaptureResult( })), inventoryGrid: inferInventoryGrid(size, detailRect), inventoryCount: count, + layout: { + aspect: aspectRatioLabel(size), + isSixteenNine: isSixteenNine(size), + warning: layoutSupportWarning(size), + }, }; } diff --git a/src/lib/layoutProfile.test.ts b/src/lib/layoutProfile.test.ts new file mode 100644 index 0000000..b3a0d03 --- /dev/null +++ b/src/lib/layoutProfile.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { + aspectRatioLabel, + detailCropRects, + inventoryCountCropRect, + inventoryGrid, + inventoryRect, + isSixteenNine, + layoutSupportWarning, + profileDetailRect, +} from "./layoutProfile"; + +const HD = { width: 1920, height: 1080 }; +const QHD = { width: 2560, height: 1440 }; +const ULTRAWIDE = { width: 3440, height: 1440 }; + +describe("layoutProfile", () => { + it("detects 16:9 across common resolutions and rejects ultrawide", () => { + expect(isSixteenNine(HD)).toBe(true); + expect(isSixteenNine(QHD)).toBe(true); + expect(isSixteenNine({ width: 3840, height: 2160 })).toBe(true); + expect(isSixteenNine(ULTRAWIDE)).toBe(false); + expect(isSixteenNine({ width: 1920, height: 1200 })).toBe(false); // 16:10 + }); + + it("labels the aspect ratio", () => { + expect(aspectRatioLabel(HD)).toBe("1.78:1"); + expect(aspectRatioLabel({ width: 0, height: 0 })).toBe("unknown"); + }); + + it("warns only for non-16:9 resolutions", () => { + expect(layoutSupportWarning(HD)).toBe(""); + expect(layoutSupportWarning(QHD)).toBe(""); + expect(layoutSupportWarning(ULTRAWIDE)).toContain("nicht 16:9"); + expect(layoutSupportWarning({ width: 0, height: 0 })).toBe(""); + }); + + it("keeps the detail rect inside the image and on the right half", () => { + const rect = profileDetailRect(QHD); + expect(rect.x).toBeGreaterThanOrEqual(QHD.width * 0.45); + expect(rect.x + rect.width).toBeLessThanOrEqual(QHD.width); + expect(rect.y + rect.height).toBeLessThanOrEqual(QHD.height); + }); + + it("produces the four artifact crops in top-to-bottom order, all clamped", () => { + const detail = profileDetailRect(QHD); + const crops = detailCropRects(detail, QHD); + expect(crops.map((crop) => crop.id)).toEqual([ + "artifact-title", + "artifact-main-stat", + "artifact-substats", + "artifact-footer", + ]); + let previousY = -1; + for (const crop of crops) { + expect(crop.rect.x).toBeGreaterThanOrEqual(0); + expect(crop.rect.y).toBeGreaterThan(previousY); + expect(crop.rect.x + crop.rect.width).toBeLessThanOrEqual(QHD.width); + expect(crop.rect.y + crop.rect.height).toBeLessThanOrEqual(QHD.height); + previousY = crop.rect.y; + } + }); + + it("places the inventory count crop inside the inventory panel", () => { + const detail = profileDetailRect(QHD); + const inv = inventoryRect(QHD, detail); + const count = inventoryCountCropRect(inv, QHD); + expect(count.x).toBeGreaterThanOrEqual(inv.x); + expect(count.x + count.width).toBeLessThanOrEqual(QHD.width); + }); + + it("builds a 5-column inventory grid on the left", () => { + const detail = profileDetailRect(QHD); + const grid = inventoryGrid(QHD, detail); + expect(grid.cols).toBe(5); + expect(grid.source).toBe("detected"); + expect(grid.centers.length).toBeGreaterThanOrEqual(10); + expect(grid.centers.every((center) => center.x < detail.x)).toBe(true); + }); + + it("reports a missing grid when the inventory panel is too small", () => { + const tiny = { width: 320, height: 180 }; + const grid = inventoryGrid(tiny, profileDetailRect(tiny)); + expect(grid.source).toBe("missing"); + expect(grid.centers).toHaveLength(0); + }); +}); diff --git a/src/lib/layoutProfile.ts b/src/lib/layoutProfile.ts new file mode 100644 index 0000000..9244420 --- /dev/null +++ b/src/lib/layoutProfile.ts @@ -0,0 +1,205 @@ +// Resolution-anchored layout geometry for the artifact inventory screen +// (ADR-009). Inventory Kamera's proven approach is to require borderless 16:9 and +// derive crop/grid coordinates from the client rectangle instead of detecting the +// panel by colour each frame. This module is the single, pure, unit-tested source +// of that geometry; electron/main.ts consumes it for cropping and keeps a +// colour-based detail-rect detector only as a fallback for off-profile setups. +// +// NOTE: the per-field detail crop fractions below are the current working values. +// True IK-style fixed coordinates need calibration against a reference 16:9 +// screenshot; the structure here is what those calibrated numbers slot into. + +export interface LayoutRect { + x: number; + y: number; + width: number; + height: number; +} + +export interface CropTemplateRect { + id: string; + label: string; + rect: LayoutRect; +} + +export interface InventoryGridLayout { + centers: Array<{ x: number; y: number; row: number; col: number }>; + rows: number; + cols: number; + confidence: number; + source: "detected" | "missing"; +} + +const SIXTEEN_NINE = 16 / 9; + +export function aspectRatio(size: { width: number; height: number }): number { + if (!size.height) return 0; + return size.width / size.height; +} + +export function aspectRatioLabel(size: { width: number; height: number }): string { + const ratio = aspectRatio(size); + if (ratio === 0) return "unknown"; + return `${ratio.toFixed(2)}:1`; +} + +// Genshin's UI is authored for 16:9; other aspect ratios letterbox or reflow and +// the anchored crops no longer line up. Allow a small tolerance for rounding. +export function isSixteenNine(size: { width: number; height: number }, tolerance = 0.02): boolean { + const ratio = aspectRatio(size); + if (ratio === 0) return false; + return Math.abs(ratio - SIXTEEN_NINE) <= SIXTEEN_NINE * tolerance; +} + +// Empty when the client is a supported 16:9; otherwise a warning explaining that +// the anchored crops are unreliable off-profile (ADR-009: non-16:9 is explicitly +// unsupported for the auto scanner). +export function layoutSupportWarning(size: { width: number; height: number }): string { + if (size.width <= 0 || size.height <= 0) return ""; + if (isSixteenNine(size)) return ""; + return `Aufloesung ${size.width}x${size.height} ist nicht 16:9 (${aspectRatioLabel(size)}). Der Auto-Scan ist auf 16:9 im randlosen Fenstermodus ausgelegt; die Erkennung kann daneben liegen.`; +} + +export function clampRect(rect: LayoutRect, imageSize: { width: number; height: number }): LayoutRect { + const x = Math.max(0, Math.min(imageSize.width - 1, rect.x)); + const y = Math.max(0, Math.min(imageSize.height - 1, rect.y)); + const maxWidth = Math.max(1, imageSize.width - x); + const maxHeight = Math.max(1, imageSize.height - y); + return { + x, + y, + width: Math.max(1, Math.min(maxWidth, rect.width)), + height: Math.max(1, Math.min(maxHeight, rect.height)), + }; +} + +// Anchored guess for the artifact detail panel on the right of the screen. Used +// as the primary rect for a clean 16:9 client and as the fallback when colour +// detection cannot find the panel. +export function profileDetailRect(imageSize: { width: number; height: number }): LayoutRect { + const { width, height } = imageSize; + if (width <= 0 || height <= 0) return { x: 0, y: 0, width: Math.max(1, width), height: Math.max(1, height) }; + return clampRect( + { + x: Math.round(width * 0.5), + y: Math.round(height * 0.08), + width: Math.round(width * 0.46), + height: Math.round(height * 0.74), + }, + imageSize, + ); +} + +// The four OCR crops inside the detail panel, as fractions of the detail rect. +export function detailCropRects(detailRect: LayoutRect, imageSize: { width: number; height: number }): CropTemplateRect[] { + const templates: CropTemplateRect[] = [ + { + id: "artifact-title", + label: "Artifact title", + rect: { + x: Math.round(detailRect.x + detailRect.width * 0.055), + y: Math.round(detailRect.y + detailRect.height * 0.05), + width: Math.round(detailRect.width * 0.82), + height: Math.round(detailRect.height * 0.16), + }, + }, + { + id: "artifact-main-stat", + label: "Main stat", + rect: { + x: Math.round(detailRect.x + detailRect.width * 0.055), + y: Math.round(detailRect.y + detailRect.height * 0.2), + width: Math.round(detailRect.width * 0.82), + height: Math.round(detailRect.height * 0.18), + }, + }, + { + id: "artifact-substats", + label: "Substats", + rect: { + x: Math.round(detailRect.x + detailRect.width * 0.055), + y: Math.round(detailRect.y + detailRect.height * 0.41), + width: Math.round(detailRect.width * 0.82), + height: Math.round(detailRect.height * 0.25), + }, + }, + { + id: "artifact-footer", + label: "Footer", + rect: { + x: Math.round(detailRect.x + detailRect.width * 0.055), + y: Math.round(detailRect.y + detailRect.height * 0.78), + width: Math.round(detailRect.width * 0.82), + height: Math.round(detailRect.height * 0.16), + }, + }, + ]; + + return templates.map((template) => ({ ...template, rect: clampRect(template.rect, imageSize) })); +} + +export function inventoryCountCropRect(inventoryRect: LayoutRect, imageSize: { width: number; height: number }): LayoutRect { + return clampRect( + { + x: Math.round(inventoryRect.x + inventoryRect.width * 0.62), + y: Math.round(inventoryRect.y + inventoryRect.height * 0.02), + width: Math.round(inventoryRect.width * 0.34), + height: Math.round(inventoryRect.height * 0.09), + }, + imageSize, + ); +} + +export function inventoryRect(imageSize: { width: number; height: number }, detailRect: LayoutRect): LayoutRect { + const { width, height } = imageSize; + const preferredWidth = Math.max(140, Math.round(width * 0.48)); + const x = Math.round(width * 0.03); + const y = Math.round(detailRect.y + detailRect.height * 0.09); + const availableWidth = Math.max(100, detailRect.x - Math.round(width * 0.04)); + const panelWidth = Math.max(100, Math.min(preferredWidth, availableWidth)); + const safeWidth = panelWidth > width * 0.85 ? Math.round(width * 0.55) : panelWidth; + return clampRect( + { + x, + y, + width: Math.min(safeWidth, Math.max(width - x - Math.round(width * 0.02), 100)), + height: Math.max(140, Math.round(height * 0.7)), + }, + imageSize, + ); +} + +export function inventoryGrid(imageSize: { width: number; height: number }, detailRect: LayoutRect): InventoryGridLayout { + const rect = inventoryRect(imageSize, detailRect); + const cols = 5; + if (rect.width < 160 || rect.height < 140) { + return { centers: [], rows: 0, cols: 0, confidence: 0, source: "missing" }; + } + + const cellWidth = Math.max(56, Math.round(rect.width / cols)); + const stepX = Math.round(cellWidth * 0.96); + const stepY = Math.round(cellWidth * 1.03); + const visibleRows = Math.max(2, Math.min(6, Math.round(rect.height / Math.max(stepY, 1)))); + + const startX = rect.x + Math.max(6, Math.round(stepX * 0.45)); + const startY = rect.y + Math.max(6, Math.round(stepY * 0.45)); + const centers: InventoryGridLayout["centers"] = []; + for (let row = 0; row < visibleRows; row++) { + for (let col = 0; col < cols; col++) { + const x = startX + col * stepX; + const y = startY + row * stepY; + if (x < imageSize.width && y < imageSize.height) { + centers.push({ x, y, row, col }); + } + } + } + + const trimmed = centers.filter((center) => center.x > 0 && center.y > 0); + return { + centers: trimmed, + rows: visibleRows, + cols, + confidence: trimmed.length >= cols * 2 ? 76 : trimmed.length >= cols ? 58 : 36, + source: "detected", + }; +} diff --git a/src/lib/ocrPreprocess.test.ts b/src/lib/ocrPreprocess.test.ts new file mode 100644 index 0000000..eaa97f2 --- /dev/null +++ b/src/lib/ocrPreprocess.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { binarizeForOcr, computeLuminanceHistogram, otsuThreshold, type Bitmap } from "./ocrPreprocess"; + +// Build a BGRA bitmap from a grid of [b,g,r] pixels. +function bitmapFrom(pixels: Array<[number, number, number]>, width: number, height: number): Bitmap { + const data = Buffer.alloc(width * height * 4); + pixels.forEach(([b, g, r], index) => { + data[index * 4] = b; + data[index * 4 + 1] = g; + data[index * 4 + 2] = r; + data[index * 4 + 3] = 255; + }); + return { data, width, height }; +} + +describe("ocrPreprocess", () => { + it("computes a luminance histogram over all pixels", () => { + const bitmap = bitmapFrom([ + [0, 0, 0], + [255, 255, 255], + [0, 0, 0], + [255, 255, 255], + ], 2, 2); + const histogram = computeLuminanceHistogram(bitmap); + expect(histogram[0]).toBe(2); + expect(histogram[255]).toBe(2); + expect(histogram.reduce((sum, count) => sum + count, 0)).toBe(4); + }); + + it("otsu splits a clean bimodal image between the two peaks", () => { + const histogram = new Array(256).fill(0); + histogram[20] = 50; + histogram[220] = 50; + const threshold = otsuThreshold(histogram); + expect(threshold).toBeGreaterThanOrEqual(20); + expect(threshold).toBeLessThan(220); + }); + + it("otsu is safe on an empty histogram", () => { + expect(otsuThreshold(new Array(256).fill(0))).toBe(127); + }); + + it("inverts bright foreground to black-on-white by default", () => { + // Bright text pixel + dark background pixel. + const bitmap = bitmapFrom([ + [255, 255, 255], // bright -> should become black + [0, 0, 0], // dark -> should become white + ], 2, 1); + const out = binarizeForOcr(bitmap, { threshold: 128 }); + expect([out.data[0], out.data[1], out.data[2]]).toEqual([0, 0, 0]); + expect([out.data[4], out.data[5], out.data[6]]).toEqual([255, 255, 255]); + expect(out.data[3]).toBe(255); + }); + + it("keeps bright foreground white when inversion is disabled", () => { + const bitmap = bitmapFrom([ + [255, 255, 255], + [0, 0, 0], + ], 2, 1); + const out = binarizeForOcr(bitmap, { threshold: 128, invertBrightForeground: false }); + expect(out.data[0]).toBe(255); + expect(out.data[4]).toBe(0); + }); + + it("preserves dimensions and always emits opaque pixels", () => { + const bitmap = bitmapFrom(Array.from({ length: 9 }, () => [100, 100, 100] as [number, number, number]), 3, 3); + const out = binarizeForOcr(bitmap); + expect(out.width).toBe(3); + expect(out.height).toBe(3); + for (let pixel = 0; pixel < 9; pixel++) { + expect(out.data[pixel * 4 + 3]).toBe(255); + } + }); +}); diff --git a/src/lib/ocrPreprocess.ts b/src/lib/ocrPreprocess.ts new file mode 100644 index 0000000..18c08c3 --- /dev/null +++ b/src/lib/ocrPreprocess.ts @@ -0,0 +1,101 @@ +// OCR preprocessing for artifact crops (ADR-009). Tesseract reads a clean, high +// contrast, dark-text-on-light image far more reliably than Genshin's native +// bright-text-on-dark UI. This binarizes a crop with Otsu thresholding and (by +// default) inverts, because artifact text is the bright foreground. +// +// Works on a raw BGRA bitmap (Electron NativeImage.getBitmap() layout on +// Windows). Kept pure and channel-order-agnostic for luminance so it is unit +// testable without Electron. Upscaling is done separately via NativeImage.resize +// before this runs - interpolated upscaling of small crops is a big Tesseract win +// and NativeImage does it better than hand-rolled JS. + +export interface Bitmap { + data: Uint8Array | Buffer; + width: number; + height: number; +} + +export interface BinarizeOptions { + /** Artifact text is the bright foreground, so invert to dark-on-light. */ + invertBrightForeground?: boolean; + /** Override Otsu with a fixed 0-255 luminance threshold. */ + threshold?: number; +} + +const BYTES_PER_PIXEL = 4; + +// Rec. 601 luma. Channel order does not matter for a weighted sum as long as we +// read the same three bytes; BGRA and RGBA give the same luminance here because +// we weight by position-independent coefficients applied to the actual R/G/B. +function luminanceAt(data: Uint8Array | Buffer, index: number): number { + // NativeImage on Windows is BGRA: byte0=B, byte1=G, byte2=R. + const b = data[index]; + const g = data[index + 1]; + const r = data[index + 2]; + return 0.299 * r + 0.587 * g + 0.114 * b; +} + +export function computeLuminanceHistogram(bitmap: Bitmap): number[] { + const histogram = new Array(256).fill(0); + const { data, width, height } = bitmap; + const pixels = width * height; + for (let pixel = 0; pixel < pixels; pixel++) { + const value = Math.round(luminanceAt(data, pixel * BYTES_PER_PIXEL)); + histogram[Math.max(0, Math.min(255, value))]++; + } + return histogram; +} + +// Otsu's method: pick the threshold that maximizes between-class variance. +export function otsuThreshold(histogram: readonly number[]): number { + const total = histogram.reduce((sum, count) => sum + count, 0); + if (total === 0) return 127; + + let sumAll = 0; + for (let level = 0; level < 256; level++) sumAll += level * histogram[level]; + + let sumBackground = 0; + let weightBackground = 0; + let maxVariance = -1; + let threshold = 127; + + for (let level = 0; level < 256; level++) { + weightBackground += histogram[level]; + if (weightBackground === 0) continue; + const weightForeground = total - weightBackground; + if (weightForeground === 0) break; + + sumBackground += level * histogram[level]; + const meanBackground = sumBackground / weightBackground; + const meanForeground = (sumAll - sumBackground) / weightForeground; + const betweenVariance = weightBackground * weightForeground * (meanBackground - meanForeground) ** 2; + + if (betweenVariance > maxVariance) { + maxVariance = betweenVariance; + threshold = level; + } + } + + return threshold; +} + +export function binarizeForOcr(bitmap: Bitmap, options: BinarizeOptions = {}): Bitmap { + const { data, width, height } = bitmap; + const invert = options.invertBrightForeground ?? true; + const threshold = options.threshold ?? otsuThreshold(computeLuminanceHistogram(bitmap)); + + const output = Buffer.alloc(width * height * BYTES_PER_PIXEL); + const pixels = width * height; + for (let pixel = 0; pixel < pixels; pixel++) { + const index = pixel * BYTES_PER_PIXEL; + const isBright = luminanceAt(data, index) > threshold; + // Bright foreground text -> black; dark background -> white (inverted). + const value = invert ? (isBright ? 0 : 255) : (isBright ? 255 : 0); + output[index] = value; + output[index + 1] = value; + output[index + 2] = value; + output[index + 3] = 255; + } + + return { data: output, width, height }; +} diff --git a/src/types/global.d.ts b/src/types/global.d.ts index 0c8437e..7252905 100644 --- a/src/types/global.d.ts +++ b/src/types/global.d.ts @@ -56,6 +56,11 @@ export interface CaptureResult { source: "ocr" | "missing"; text: string; }; + layout?: { + aspect: string; + isSixteenNine: boolean; + warning: string; + }; } export interface WindowBounds { From 6ea3d9e7143cc4efd1bd990097e8f0e3e70c549c Mon Sep 17 00:00:00 2001 From: AzuTear Date: Sun, 5 Jul 2026 21:47:03 +0200 Subject: [PATCH 05/14] feat(scan): card-ready gating replaces the fixed settle delay Adds src/lib/cardReadyGate.ts: after a tile click, poll the detail fingerprint until it has both changed from the previous artifact and stabilized across consecutive samples, instead of waiting a hardcoded 280ms and hoping. - Faster on quick machines (proceeds as soon as the card is stable), correct on slow ones (waits up to the budget). - Robust to particle/hover-glow animation: requiring two consecutive equal samples ignores single-frame noise, and if the card never fully stabilizes it still proceeds once the content has changed rather than looping on an animated frame. - ESC/stop abort is honored between polls via checkAbort. autoScanLoop now uses waitForCardReady for both the initial read and the one retry; CLICK_SETTLE_MS removed. 6 new unit tests; 94 total green. Co-Authored-By: Claude Opus 4.8 --- src/lib/autoScanLoop.ts | 45 +++++++++++------ src/lib/cardReadyGate.test.ts | 70 ++++++++++++++++++++++++++ src/lib/cardReadyGate.ts | 92 +++++++++++++++++++++++++++++++++++ 3 files changed, 191 insertions(+), 16 deletions(-) create mode 100644 src/lib/cardReadyGate.test.ts create mode 100644 src/lib/cardReadyGate.ts diff --git a/src/lib/autoScanLoop.ts b/src/lib/autoScanLoop.ts index 30ebfdf..10c29a4 100644 --- a/src/lib/autoScanLoop.ts +++ b/src/lib/autoScanLoop.ts @@ -3,6 +3,7 @@ import type { ParsedArtifactCandidate } from "./artifactOcrParser"; import { sessionSignature } from "./artifactStore"; import { buildGridModel, buildInventoryPagePlan, type GridTarget } from "./automationPlanner"; import { classifyAutoScanCapture, shouldAbortAfterConsecutiveMisses } from "./autoScanController"; +import { waitForCardReady } from "./cardReadyGate"; import { captureRejectionReason, captureSourceRejectionReason, shouldPersistParsedArtifact } from "./scannerCaptureQuality"; import type { AutoScanStats, ScanSummary } from "./scannerSession"; import { clampSkipRows, emptyAutoScanStats, resolveScanTargetCount } from "./scannerSession"; @@ -59,7 +60,11 @@ export type AutoScanLoopResult = { targetCount: number; }; -const CLICK_SETTLE_MS = 280; +// Card-ready gating replaces a fixed settle delay: poll the detail fingerprint +// until it has changed and stabilized (or the budget is spent). See cardReadyGate. +const CARD_READY_MAX_MS = 900; +const CARD_READY_POLL_MS = 90; +const CARD_READY_STABLE_SAMPLES = 2; const MISS_ABORT_THRESHOLD = 3; const UNREADABLE_ABORT_THRESHOLD = 5; @@ -162,6 +167,19 @@ export async function runAutoScanLoop( if (initialParsed) lastDetailSignature = sessionSignature(initialParsed); let lastDetailViewFingerprint = detailFingerprint(currentCapture); + async function awaitCardReady() { + return waitForCardReady( + { + sampleFingerprint: async () => detailFingerprint(await captureFastSelectedSource(0, true)), + wait, + now: () => Date.now(), + checkAbort: checkGuard, + }, + lastDetailViewFingerprint, + { minStableSamples: CARD_READY_STABLE_SAMPLES, maxWaitMs: CARD_READY_MAX_MS, pollIntervalMs: CARD_READY_POLL_MS }, + ); + } + try { while (!blockedReason && !shouldStop() && stats.clicked < maxTargets) { page++; @@ -224,16 +242,13 @@ export async function runAutoScanLoop( break; } - let waitStop = await waitDuringScan(CLICK_SETTLE_MS); - if (waitStop) { - blockedReason = waitStop; + let ready = await awaitCardReady(); + if (ready.abortReason) { + blockedReason = ready.abortReason; aborted = true; break; } - - let previewCapture = await captureFastSelectedSource(0, true); - let previewFingerprint = detailFingerprint(previewCapture); - let changedDetail = Boolean(previewFingerprint && previewFingerprint !== lastDetailViewFingerprint); + let changedDetail = ready.changed; if (!changedDetail) { appendAutomationLog(`retry r${target.row} c${target.col}: Detailansicht unveraendert`); @@ -244,15 +259,13 @@ export async function runAutoScanLoop( aborted = true; break; } - waitStop = await waitDuringScan(CLICK_SETTLE_MS); - if (waitStop) { - blockedReason = waitStop; + ready = await awaitCardReady(); + if (ready.abortReason) { + blockedReason = ready.abortReason; aborted = true; break; } - previewCapture = await captureFastSelectedSource(0, true); - previewFingerprint = detailFingerprint(previewCapture); - changedDetail = Boolean(previewFingerprint && previewFingerprint !== lastDetailViewFingerprint); + changedDetail = ready.changed; } if (!changedDetail) { @@ -482,6 +495,6 @@ export function fingerprintDataUrl(dataUrl: string) { return `${dataUrl.length.toString(16)}:${(hash >>> 0).toString(16)}`; } -function wait(ms: number) { - return new Promise((resolve) => window.setTimeout(resolve, ms)); +function wait(ms: number): Promise { + return new Promise((resolve) => window.setTimeout(() => resolve(), ms)); } diff --git a/src/lib/cardReadyGate.test.ts b/src/lib/cardReadyGate.test.ts new file mode 100644 index 0000000..e24fcf2 --- /dev/null +++ b/src/lib/cardReadyGate.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import { waitForCardReady, type CardReadyDeps } from "./cardReadyGate"; + +// Deterministic clock: each wait advances virtual time, each sample pops the +// next scripted fingerprint. +function harness(samples: string[], step = 90, checkAbort?: () => string) { + let clock = 0; + let index = 0; + const deps: CardReadyDeps = { + sampleFingerprint: async () => samples[Math.min(index++, samples.length - 1)], + wait: async (ms) => { + clock += ms; + }, + now: () => clock, + checkAbort, + }; + return { deps, sampleCount: () => index, step }; +} + +describe("waitForCardReady", () => { + it("returns ready once the card changed and stabilized", async () => { + const { deps } = harness(["old", "new", "new"]); + const result = await waitForCardReady(deps, "old", { minStableSamples: 2, maxWaitMs: 900, pollIntervalMs: 90 }); + expect(result.ready).toBe(true); + expect(result.changed).toBe(true); + expect(result.stable).toBe(true); + expect(result.fingerprint).toBe("new"); + expect(result.polls).toBe(3); + }); + + it("keeps polling while the detail still shows the previous artifact", async () => { + const { deps } = harness(["old", "old", "new", "new"]); + const result = await waitForCardReady(deps, "old", { minStableSamples: 2 }); + expect(result.ready).toBe(true); + expect(result.fingerprint).toBe("new"); + expect(result.polls).toBe(4); + }); + + it("proceeds after the budget when content changed but never stabilizes (animation)", async () => { + // Always different (animated glow): changed but never two-in-a-row equal. + const animated = ["a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8", "a9", "a10"]; + const { deps } = harness(animated, 90); + const result = await waitForCardReady(deps, "old", { minStableSamples: 2, maxWaitMs: 300, pollIntervalMs: 90 }); + expect(result.changed).toBe(true); + expect(result.stable).toBe(false); + expect(result.ready).toBe(true); // budget spent, but content did change + }); + + it("reports not-ready when the detail never changes within budget", async () => { + const { deps } = harness(["old", "old", "old", "old", "old", "old"], 90); + const result = await waitForCardReady(deps, "old", { minStableSamples: 2, maxWaitMs: 200, pollIntervalMs: 90 }); + expect(result.changed).toBe(false); + expect(result.ready).toBe(false); + }); + + it("aborts immediately when checkAbort returns a reason", async () => { + const { deps } = harness(["new", "new"], 90, () => "ESC gehalten"); + const result = await waitForCardReady(deps, "old", {}); + expect(result.abortReason).toBe("ESC gehalten"); + expect(result.ready).toBe(false); + expect(result.polls).toBe(0); + }); + + it("ignores empty fingerprints for stability", async () => { + const { deps } = harness(["", "", "new", "new"], 90); + const result = await waitForCardReady(deps, "old", { minStableSamples: 2, maxWaitMs: 900 }); + expect(result.ready).toBe(true); + expect(result.fingerprint).toBe("new"); + }); +}); diff --git a/src/lib/cardReadyGate.ts b/src/lib/cardReadyGate.ts new file mode 100644 index 0000000..6dad824 --- /dev/null +++ b/src/lib/cardReadyGate.ts @@ -0,0 +1,92 @@ +// Card-ready gating for the auto-scan loop (ADR replaces the fixed 280ms settle +// delay). After clicking a tile, instead of waiting a hardcoded interval and +// hoping the detail card has rendered, poll a cheap detail fingerprint until it +// has both (a) changed from the previously-read artifact and (b) stabilized +// across consecutive samples. This is faster on quick machines and correct on +// slow ones, and it is robust to particle/hover-glow animation: if the card +// never fully stabilizes within the budget it still proceeds once the content +// has changed, rather than looping forever on an animated frame. +// +// Pure except for the injected async sampler/clock, so it is unit testable. + +export interface CardReadyOptions { + /** Consecutive equal samples required to call the card stable. */ + minStableSamples?: number; + /** Total time budget before giving up on full stability. */ + maxWaitMs?: number; + /** Delay between samples. */ + pollIntervalMs?: number; +} + +export interface CardReadyDeps { + /** Fast capture -> detail fingerprint. */ + sampleFingerprint: () => Promise; + wait: (ms: number) => Promise; + now: () => number; + /** Returns a non-empty reason to abort (ESC held, stop pressed, ...). */ + checkAbort?: () => Promise | string; +} + +export interface CardReadyResult { + /** Safe to read the full artifact: content changed (and stabilized or budget spent). */ + ready: boolean; + /** The detail differs from the previously-read artifact. */ + changed: boolean; + /** Reached the required number of consecutive equal samples. */ + stable: boolean; + /** Latest sampled fingerprint. */ + fingerprint: string; + /** Non-empty when aborted via checkAbort. */ + abortReason: string; + polls: number; +} + +const DEFAULT_MIN_STABLE = 2; +const DEFAULT_MAX_WAIT_MS = 900; +const DEFAULT_POLL_MS = 90; + +export async function waitForCardReady( + deps: CardReadyDeps, + previousFingerprint: string, + options: CardReadyOptions = {}, +): Promise { + const minStable = Math.max(1, options.minStableSamples ?? DEFAULT_MIN_STABLE); + const maxWaitMs = Math.max(0, options.maxWaitMs ?? DEFAULT_MAX_WAIT_MS); + const pollIntervalMs = Math.max(1, options.pollIntervalMs ?? DEFAULT_POLL_MS); + + const start = deps.now(); + let previousSample = ""; + let stableCount = 0; + let latest = ""; + let polls = 0; + + for (;;) { + if (deps.checkAbort) { + const abortReason = await deps.checkAbort(); + if (abortReason) { + return { ready: false, changed: false, stable: false, fingerprint: latest, abortReason, polls }; + } + } + + latest = await deps.sampleFingerprint(); + polls++; + + stableCount = latest && latest === previousSample ? stableCount + 1 : 1; + previousSample = latest; + + const changed = Boolean(latest) && latest !== previousFingerprint; + const stable = stableCount >= minStable; + + if (changed && stable) { + return { ready: true, changed: true, stable: true, fingerprint: latest, abortReason: "", polls }; + } + + if (deps.now() - start >= maxWaitMs) { + // Budget spent. Proceed if the content has at least changed, even if it is + // still animating (never fully stabilizes). + return { ready: changed, changed, stable, fingerprint: latest, abortReason: "", polls }; + } + + await deps.wait(pollIntervalMs); + } +} From dcac155887c706082a3ec975c3dbd68a841d91ea Mon Sep 17 00:00:00 2001 From: AzuTear Date: Sun, 5 Jul 2026 21:55:20 +0200 Subject: [PATCH 06/14] feat(store): GOOD interop, rescan-merge, data staleness, lock detection Task #5 building blocks, each a pure + unit-tested module: - goodInterop.ts: GOOD (Genshin Optimizer / Inventory Kamera / Akasha) import and export for scanned StoredArtifactRecords - slot/stat/set key maps both ways, substat string <-> { key, value }, main-value reconstruction on import (ADR-003). Export is lossless; import is best-effort (GOOD lacks piece names). - artifactMerge.ts: rescan-merge (ADR-006 follow-up). Level-independent identity (set + slot + main + substat NAME set) collapses leveled re-scan duplicates, keeping the higher-level/stronger record and summing timesSeen. Conservative: differing substat lineups never merge. - dataPackageStatus.ts: warns when the genshin-db package is older than ~45 days (a patch cycle) so new sets/characters aren't silently missed; surfaced in the Scanner Diagnose data-package line. Adds dataGeneratedAt to genshinData. - lockDetection.ts: EXPERIMENTAL read-only lock-status heuristic (gold-pixel ratio in a top-right icon crop). Pure + tested but not wired into capture; crop position and threshold need calibration against a reference 16:9 screenshot. 120 tests + build green. Remaining wiring (needs UI / live calibration): GOOD import/export buttons and live lock detection. Co-Authored-By: Claude Opus 4.8 --- .../hooks/useScanDiagnosticsModalModel.ts | 6 +- src/lib/artifactMerge.test.ts | 74 +++++++ src/lib/artifactMerge.ts | 84 ++++++++ src/lib/dataPackageStatus.test.ts | 41 ++++ src/lib/dataPackageStatus.ts | 47 ++++ src/lib/genshinData.ts | 1 + src/lib/goodInterop.test.ts | 111 ++++++++++ src/lib/goodInterop.ts | 201 ++++++++++++++++++ src/lib/lockDetection.test.ts | 43 ++++ src/lib/lockDetection.ts | 52 +++++ 10 files changed, 658 insertions(+), 2 deletions(-) create mode 100644 src/lib/artifactMerge.test.ts create mode 100644 src/lib/artifactMerge.ts create mode 100644 src/lib/dataPackageStatus.test.ts create mode 100644 src/lib/dataPackageStatus.ts create mode 100644 src/lib/goodInterop.test.ts create mode 100644 src/lib/goodInterop.ts create mode 100644 src/lib/lockDetection.test.ts create mode 100644 src/lib/lockDetection.ts diff --git a/src/features/scan/components/modals/hooks/useScanDiagnosticsModalModel.ts b/src/features/scan/components/modals/hooks/useScanDiagnosticsModalModel.ts index 15d7cf2..62cc9aa 100644 --- a/src/features/scan/components/modals/hooks/useScanDiagnosticsModalModel.ts +++ b/src/features/scan/components/modals/hooks/useScanDiagnosticsModalModel.ts @@ -1,5 +1,6 @@ import { detailFingerprint } from "../../../../../lib/autoScanLoop"; -import { sourceVersion } from "../../../../../lib/genshinData"; +import { dataGeneratedAt, sourceVersion } from "../../../../../lib/genshinData"; +import { dataPackageStatus } from "../../../../../lib/dataPackageStatus"; import { useCallback, useMemo, type MouseEvent } from "react"; import type { ScanDiagnosticsModalProps } from "../types"; @@ -96,7 +97,8 @@ export function useScanDiagnosticsModalModel({ : "Run a capture once while the artifact inventory is visible."; const learningRulesText = controller.learningRulesLoaded ? `${controller.learningRuleCount} local rules` : "loading"; - const learningRulesSubtext = `Review samples feed deterministic OCR fixes before each new parse. Data package: ${sourceVersion}`; + const dataStaleness = dataPackageStatus(dataGeneratedAt, sourceVersion); + const learningRulesSubtext = `Review samples feed deterministic OCR fixes before each new parse. Data package: ${sourceVersion}${dataStaleness.warning ? ` - ${dataStaleness.warning}` : ""}`; const playerProgress = useMemo(() => { const width = Math.min( diff --git a/src/lib/artifactMerge.test.ts b/src/lib/artifactMerge.test.ts new file mode 100644 index 0000000..dfde2ad --- /dev/null +++ b/src/lib/artifactMerge.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import type { StoredArtifactRecord } from "../types/storage"; +import { mergeIdentity, mergeRescannedArtifacts, substatName } from "./artifactMerge"; + +function record(overrides: Partial = {}): StoredArtifactRecord { + return { + id: "x", + name: "Gladiator's Nostalgia", + slot: "Flower of Life", + level: 0, + setName: "Gladiator's Finale", + mainStat: "HP", + mainValue: "717", + substats: ["CRIT DMG+5.4%", "ATK+19", "HP%+4.1%", "Energy Recharge+5.2%"], + equipped: "Not detected", + confidence: 80, + needsReview: false, + source: "auto-scan", + ...overrides, + }; +} + +describe("artifactMerge", () => { + it("strips values to get the substat name", () => { + expect(substatName("CRIT DMG+13.2%")).toBe("CRIT DMG"); + expect(substatName("ATK+19")).toBe("ATK"); + }); + + it("identity ignores level and substat values", () => { + const low = record({ level: 0, substats: ["CRIT DMG+5.4%", "ATK+19"] }); + const high = record({ level: 20, substats: ["CRIT DMG+13.2%", "ATK+37"] }); + expect(mergeIdentity(low)).toBe(mergeIdentity(high)); + }); + + it("collapses a leveled re-scan into one record, keeping the higher level", () => { + const low = record({ id: "a", level: 0, timesSeen: 1 }); + const high = record({ + id: "b", + level: 20, + timesSeen: 1, + substats: ["CRIT DMG+13.2%", "ATK+37", "HP%+15.7%", "Energy Recharge+11.7%"], + equipped: "Bennett", + }); + const { merged, collapsed } = mergeRescannedArtifacts([low, high]); + expect(collapsed).toBe(1); + expect(merged).toHaveLength(1); + expect(merged[0].level).toBe(20); + expect(merged[0].equipped).toBe("Bennett"); + expect(merged[0].timesSeen).toBe(2); + }); + + it("does not merge pieces with different substat lineups", () => { + const threeLine = record({ id: "a", substats: ["CRIT DMG+5.4%", "ATK+19", "HP%+4.1%"] }); + const fourLine = record({ id: "b", substats: ["CRIT DMG+5.4%", "ATK+19", "HP%+4.1%", "DEF+16"] }); + const { merged, collapsed } = mergeRescannedArtifacts([threeLine, fourLine]); + expect(collapsed).toBe(0); + expect(merged).toHaveLength(2); + }); + + it("keeps distinct sets/slots/mains apart", () => { + const flower = record({ slot: "Flower of Life", mainStat: "HP" }); + const plume = record({ slot: "Plume of Death", mainStat: "ATK" }); + const { merged } = mergeRescannedArtifacts([flower, plume]); + expect(merged).toHaveLength(2); + }); + + it("preserves the earliest firstSeenAt and latest lastSeenAt", () => { + const older = record({ id: "a", level: 0, firstSeenAt: "2026-01-01", lastSeenAt: "2026-01-02" }); + const newer = record({ id: "b", level: 20, firstSeenAt: "2026-06-01", lastSeenAt: "2026-06-10" }); + const { merged } = mergeRescannedArtifacts([older, newer]); + expect(merged[0].firstSeenAt).toBe("2026-01-01"); + expect(merged[0].lastSeenAt).toBe("2026-06-10"); + }); +}); diff --git a/src/lib/artifactMerge.ts b/src/lib/artifactMerge.ts new file mode 100644 index 0000000..3089c71 --- /dev/null +++ b/src/lib/artifactMerge.ts @@ -0,0 +1,84 @@ +import type { StoredArtifactRecord } from "../types/storage"; +import { storedArtifactStrength } from "./artifactStore"; + +// Rescan-merge (ADR-006 open follow-up): leveling an artifact changes its store +// signature (level + substat values), so re-scanning a leveled piece creates a +// duplicate record. This collapses those duplicates using a level-independent +// identity: set + slot + main stat + the SET OF SUBSTAT NAMES (values and level +// excluded). Substat names do not change with leveling, so two scans of the same +// 5-star piece at different levels share an identity and merge; two genuinely +// different pieces with an identical substat lineup can still be merged, which is +// an accepted, low-stakes risk for a triage helper (hence an explicit +// reconciliation pass, not a change to the per-save signature). + +export function substatName(substat: string): string { + const plusIndex = substat.indexOf("+"); + return (plusIndex >= 0 ? substat.slice(0, plusIndex) : substat).trim(); +} + +export function mergeIdentity(record: StoredArtifactRecord): string { + const substatNames = record.substats.map(substatName).filter(Boolean).sort().join(","); + return [record.setName, record.slot, record.mainStat, substatNames].join("::"); +} + +// The more-progressed / stronger record wins: higher level first, then strength. +function preferred(a: StoredArtifactRecord, b: StoredArtifactRecord): StoredArtifactRecord { + const levelA = a.level ?? 0; + const levelB = b.level ?? 0; + if (levelA !== levelB) return levelA > levelB ? a : b; + return storedArtifactStrength(a) >= storedArtifactStrength(b) ? a : b; +} + +function minDate(a: string | undefined, b: string | undefined): string | undefined { + if (!a) return b; + if (!b) return a; + return a <= b ? a : b; +} + +function maxDate(a: string | undefined, b: string | undefined): string | undefined { + if (!a) return b; + if (!b) return a; + return a >= b ? a : b; +} + +function mergePair(winner: StoredArtifactRecord, other: StoredArtifactRecord): StoredArtifactRecord { + return { + ...winner, + // The winner needs review only if it did on its own; a confident higher-level + // scan should clear a stale low-confidence duplicate. + needsReview: winner.needsReview, + timesSeen: (winner.timesSeen ?? 1) + (other.timesSeen ?? 1), + firstSeenAt: minDate(winner.firstSeenAt, other.firstSeenAt), + lastSeenAt: maxDate(winner.lastSeenAt, other.lastSeenAt), + // Keep an equipped character if either scan detected one. + equipped: + winner.equipped && !/not detected/i.test(winner.equipped) + ? winner.equipped + : other.equipped, + }; +} + +export interface MergeResult { + merged: StoredArtifactRecord[]; + collapsed: number; +} + +export function mergeRescannedArtifacts(records: readonly StoredArtifactRecord[]): MergeResult { + const byIdentity = new Map(); + let collapsed = 0; + + for (const record of records) { + const identity = mergeIdentity(record); + const existing = byIdentity.get(identity); + if (!existing) { + byIdentity.set(identity, record); + continue; + } + const winner = preferred(existing, record); + const loser = winner === existing ? record : existing; + byIdentity.set(identity, mergePair(winner, loser)); + collapsed++; + } + + return { merged: [...byIdentity.values()], collapsed }; +} diff --git a/src/lib/dataPackageStatus.test.ts b/src/lib/dataPackageStatus.test.ts new file mode 100644 index 0000000..5912dda --- /dev/null +++ b/src/lib/dataPackageStatus.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { dataPackageAgeDays, dataPackageStatus } from "./dataPackageStatus"; + +const NOW = Date.parse("2026-07-05T00:00:00.000Z"); + +describe("dataPackageStatus", () => { + it("computes age in whole days", () => { + expect(dataPackageAgeDays("2026-07-01T00:00:00.000Z", NOW)).toBe(4); + expect(dataPackageAgeDays("2026-07-05T00:00:00.000Z", NOW)).toBe(0); + }); + + it("returns null for missing or invalid timestamps", () => { + expect(dataPackageAgeDays("", NOW)).toBeNull(); + expect(dataPackageAgeDays("not-a-date", NOW)).toBeNull(); + }); + + it("clamps future timestamps to zero", () => { + expect(dataPackageAgeDays("2026-08-01T00:00:00.000Z", NOW)).toBe(0); + }); + + it("does not warn for a fresh package", () => { + const status = dataPackageStatus("2026-06-20T00:00:00.000Z", "genshin-db@5.2.12", NOW); + expect(status.stale).toBe(false); + expect(status.warning).toBe(""); + expect(status.ageDays).toBe(15); + }); + + it("warns for a package older than the max age", () => { + const status = dataPackageStatus("2026-04-01T00:00:00.000Z", "genshin-db@5.2.12", NOW, 45); + expect(status.stale).toBe(true); + expect(status.warning).toContain("Datenpaket"); + expect(status.warning).toContain("aktualisieren"); + }); + + it("stays quiet when the generation date is unknown", () => { + const status = dataPackageStatus("", "unknown", NOW); + expect(status.stale).toBe(false); + expect(status.warning).toBe(""); + expect(status.ageDays).toBeNull(); + }); +}); diff --git a/src/lib/dataPackageStatus.ts b/src/lib/dataPackageStatus.ts new file mode 100644 index 0000000..3f66134 --- /dev/null +++ b/src/lib/dataPackageStatus.ts @@ -0,0 +1,47 @@ +// Data-package staleness (ADR-005 follow-up). The genshin-db data package is a +// local snapshot; when a new Genshin version ships new sets/characters, an old +// package silently fails to recognize them. We cannot query the live game version +// offline, so staleness is based on the package's generation age: Genshin patches +// land roughly every six weeks, so a package older than ~45 days likely predates +// a content patch and should be regenerated with `npm run data:genshin`. + +const DAY_MS = 24 * 60 * 60 * 1000; +export const DEFAULT_MAX_AGE_DAYS = 45; + +export function dataPackageAgeDays(generatedAt: string, now: number = Date.now()): number | null { + if (!generatedAt) return null; + const generated = Date.parse(generatedAt); + if (!Number.isFinite(generated)) return null; + const age = (now - generated) / DAY_MS; + return age < 0 ? 0 : Math.floor(age); +} + +export interface DataPackageStatus { + ageDays: number | null; + stale: boolean; + warning: string; +} + +export function dataPackageStatus( + generatedAt: string, + sourceVersion: string, + now: number = Date.now(), + maxAgeDays: number = DEFAULT_MAX_AGE_DAYS, +): DataPackageStatus { + const ageDays = dataPackageAgeDays(generatedAt, now); + if (ageDays === null) { + return { + ageDays: null, + stale: false, + warning: "", + }; + } + const stale = ageDays > maxAgeDays; + return { + ageDays, + stale, + warning: stale + ? `Datenpaket (${sourceVersion}) ist ${ageDays} Tage alt. Neue Sets/Charaktere fehlen evtl. - mit "npm run data:genshin" aktualisieren.` + : "", + }; +} diff --git a/src/lib/genshinData.ts b/src/lib/genshinData.ts index f2029f9..37e7446 100644 --- a/src/lib/genshinData.ts +++ b/src/lib/genshinData.ts @@ -44,6 +44,7 @@ export const characterAliases = genshinGameData.aliases?.characterAliases ?? {}; export const knownSets = genshinGameData.artifactSets.map((set) => set.name); export const knownCharacters = (genshinGameData.characters ?? []).map((character) => character.name); export const sourceVersion = genshinGameData.sourceVersion ?? "unknown"; +export const dataGeneratedAt = (genshinGameData as { generatedAt?: string }).generatedAt ?? ""; export const fixedMainStatBySlot: Record = { "Flower of Life": "HP", diff --git a/src/lib/goodInterop.test.ts b/src/lib/goodInterop.test.ts new file mode 100644 index 0000000..434a06e --- /dev/null +++ b/src/lib/goodInterop.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; +import type { StoredArtifactRecord } from "../types/storage"; +import { + goodDatabaseToStoredArtifacts, + goodSubstatToString, + goodToStoredArtifact, + setKeyToName, + setNameToKey, + storedArtifactToGood, + storedArtifactsToGood, + substatStringToGood, +} from "./goodInterop"; + +const record: StoredArtifactRecord = { + id: "x", + name: "Gladiator's Nostalgia", + slot: "Flower of Life", + level: 20, + setName: "Gladiator's Finale", + mainStat: "HP", + mainValue: "4,780", + substats: ["CRIT DMG+13.2%", "ATK+19", "HP%+15.7%", "Energy Recharge+5.2%"], + equipped: "Bennett", + confidence: 90, + needsReview: false, + source: "auto-scan", +}; + +describe("goodInterop set keys", () => { + it("converts set names to GOOD PascalCase keys", () => { + expect(setNameToKey("Gladiator's Finale")).toBe("GladiatorsFinale"); + expect(setNameToKey("Viridescent Venerer")).toBe("ViridescentVenerer"); + expect(setNameToKey("Emblem of Severed Fate")).toBe("EmblemOfSeveredFate"); + }); + + it("round-trips known set keys back to names", () => { + for (const name of ["Gladiator's Finale", "Viridescent Venerer"]) { + expect(setKeyToName(setNameToKey(name))).toBe(name); + } + }); +}); + +describe("goodInterop substats", () => { + it("parses percent and flat substat strings", () => { + expect(substatStringToGood("CRIT DMG+13.2%")).toEqual({ key: "critDMG_", value: 13.2 }); + expect(substatStringToGood("ATK+19")).toEqual({ key: "atk", value: 19 }); + expect(substatStringToGood("HP%+15.7%")).toEqual({ key: "hp_", value: 15.7 }); + }); + + it("returns null for unparseable substats", () => { + expect(substatStringToGood("nonsense")).toBeNull(); + expect(substatStringToGood("Unknown+5")).toBeNull(); + }); + + it("round-trips substat strings", () => { + for (const entry of record.substats) { + const good = substatStringToGood(entry)!; + expect(goodSubstatToString(good)).toBe(entry); + } + }); +}); + +describe("goodInterop export", () => { + it("exports a stored artifact to GOOD", () => { + const good = storedArtifactToGood(record); + expect(good.setKey).toBe("GladiatorsFinale"); + expect(good.slotKey).toBe("flower"); + expect(good.mainStatKey).toBe("hp"); + expect(good.level).toBe(20); + expect(good.rarity).toBe(5); + expect(good.substats).toContainEqual({ key: "critDMG_", value: 13.2 }); + }); + + it("wraps records in a GOOD database envelope", () => { + const db = storedArtifactsToGood([record]); + expect(db.format).toBe("GOOD"); + expect(db.artifacts).toHaveLength(1); + }); +}); + +describe("goodInterop import", () => { + it("imports a GOOD artifact back to a stored record", () => { + const good = storedArtifactToGood(record); + const back = goodToStoredArtifact({ ...good, location: "Bennett" })!; + expect(back.slot).toBe("Flower of Life"); + expect(back.setName).toBe("Gladiator's Finale"); + expect(back.mainStat).toBe("HP"); + expect(back.equipped).toBe("Bennett"); + expect(back.substats).toContain("CRIT DMG+13.2%"); + expect(back.source).toBe("good-import"); + }); + + it("computes a main value from slot + main stat + level on import", () => { + const good = storedArtifactToGood(record); + const back = goodToStoredArtifact(good)!; + // Flower HP main at +20 is the reference max; just assert it is populated. + expect(back.mainValue).not.toBe(""); + }); + + it("skips artifacts with an unknown slot or main stat", () => { + expect(goodToStoredArtifact({ setKey: "GladiatorsFinale", slotKey: "bogus", mainStatKey: "hp" })).toBeNull(); + expect(goodToStoredArtifact({ setKey: "GladiatorsFinale", slotKey: "flower", mainStatKey: "bogus" })).toBeNull(); + }); + + it("maps a full GOOD database", () => { + const db = storedArtifactsToGood([record, { ...record, slot: "Plume of Death", mainStat: "ATK", mainValue: "311" }]); + const imported = goodDatabaseToStoredArtifacts(db); + expect(imported).toHaveLength(2); + expect(imported.map((entry) => entry.slot)).toEqual(["Flower of Life", "Plume of Death"]); + }); +}); diff --git a/src/lib/goodInterop.ts b/src/lib/goodInterop.ts new file mode 100644 index 0000000..1a4a492 --- /dev/null +++ b/src/lib/goodInterop.ts @@ -0,0 +1,201 @@ +import type { GoodExportArtifact } from "../types/global"; +import type { StoredArtifactRecord } from "../types/storage"; +import { knownSets, mainStatValueReferences, pieceToSet, pieceToSlot } from "./genshinData"; +import { simplifyForMatch } from "./fuzzyMatch"; + +// GOOD (Genshin Open Object Description) interop for scanned artifacts, so the +// local store can round-trip with Genshin Optimizer / Inventory Kamera / Akasha +// (ADR-003). Export is lossless for the fields GOOD carries; import is +// best-effort because GOOD does not store piece names or main-stat values. + +export interface GoodImportArtifact { + setKey: string; + slotKey: string; + rarity?: number; + level?: number; + mainStatKey: string; + substats?: Array<{ key: string; value: number }>; + location?: string; + lock?: boolean; +} + +export interface GoodImportDatabase { + format?: string; + version?: number; + source?: string; + artifacts?: GoodImportArtifact[]; +} + +const SLOT_TO_GOOD: Record = { + "Flower of Life": "flower", + "Plume of Death": "plume", + "Sands of Eon": "sands", + "Goblet of Eonothem": "goblet", + "Circlet of Logos": "circlet", +}; + +const GOOD_TO_SLOT: Record = Object.fromEntries( + Object.entries(SLOT_TO_GOOD).map(([display, key]) => [key, display]), +); + +// display name (as used across the app, including the HP%/ATK%/DEF% variants) -> +// GOOD stat key + whether it is a percent stat. +interface StatEntry { + display: string; + key: string; + percent: boolean; +} + +const STAT_ENTRIES: StatEntry[] = [ + { display: "HP", key: "hp", percent: false }, + { display: "HP%", key: "hp_", percent: true }, + { display: "ATK", key: "atk", percent: false }, + { display: "ATK%", key: "atk_", percent: true }, + { display: "DEF", key: "def", percent: false }, + { display: "DEF%", key: "def_", percent: true }, + { display: "Elemental Mastery", key: "eleMas", percent: false }, + { display: "Energy Recharge", key: "enerRech_", percent: true }, + { display: "CRIT Rate", key: "critRate_", percent: true }, + { display: "CRIT DMG", key: "critDMG_", percent: true }, + { display: "Healing Bonus", key: "heal_", percent: true }, + { display: "Physical DMG Bonus", key: "physical_dmg_", percent: true }, + { display: "Pyro DMG Bonus", key: "pyro_dmg_", percent: true }, + { display: "Hydro DMG Bonus", key: "hydro_dmg_", percent: true }, + { display: "Electro DMG Bonus", key: "electro_dmg_", percent: true }, + { display: "Cryo DMG Bonus", key: "cryo_dmg_", percent: true }, + { display: "Anemo DMG Bonus", key: "anemo_dmg_", percent: true }, + { display: "Geo DMG Bonus", key: "geo_dmg_", percent: true }, + { display: "Dendro DMG Bonus", key: "dendro_dmg_", percent: true }, +]; + +const DISPLAY_TO_STAT = new Map(STAT_ENTRIES.map((entry) => [entry.display, entry])); +const KEY_TO_STAT = new Map(STAT_ENTRIES.map((entry) => [entry.key, entry])); + +export function statDisplayToGoodKey(display: string): string { + return DISPLAY_TO_STAT.get(display)?.key ?? ""; +} + +export function goodKeyToStatDisplay(key: string): string { + return KEY_TO_STAT.get(key)?.display ?? ""; +} + +export function setNameToKey(name: string): string { + // GOOD removes apostrophes without re-capitalizing ("Gladiator's" -> + // "Gladiators"), then PascalCases the remaining whitespace/hyphen words. + return name + .replace(/['’]/g, "") + .split(/[\s-]+/) + .map((word) => word.replace(/[^A-Za-z0-9]/g, "")) + .filter(Boolean) + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(""); +} + +const SET_KEY_TO_NAME = new Map(knownSets.map((name) => [setNameToKey(name), name])); + +export function setKeyToName(key: string): string { + const direct = SET_KEY_TO_NAME.get(key); + if (direct) return direct; + const simplifiedKey = simplifyForMatch(key); + const match = knownSets.find((name) => simplifyForMatch(setNameToKey(name)) === simplifiedKey); + return match ?? key; +} + +// "CRIT DMG+13.2%" / "ATK+19" -> GOOD { key, value }. +export function substatStringToGood(entry: string): { key: string; value: number } | null { + const plusIndex = entry.indexOf("+"); + if (plusIndex <= 0) return null; + const display = entry.slice(0, plusIndex).trim(); + const key = statDisplayToGoodKey(display); + if (!key) return null; + const value = Number.parseFloat(entry.slice(plusIndex + 1).replace(/[%,\s]/g, "")); + if (!Number.isFinite(value)) return null; + return { key, value }; +} + +export function goodSubstatToString(substat: { key: string; value: number }): string { + const entry = KEY_TO_STAT.get(substat.key); + if (!entry) return ""; + return entry.percent ? `${entry.display}+${substat.value}%` : `${entry.display}+${substat.value}`; +} + +export function storedArtifactToGood(record: StoredArtifactRecord): GoodExportArtifact { + const substats = record.substats + .map((entry) => substatStringToGood(entry)) + .filter((entry): entry is { key: string; value: number } => entry !== null); + + return { + setKey: setNameToKey(record.setName), + slotKey: SLOT_TO_GOOD[record.slot] ?? "", + rarity: 5, + level: record.level ?? 0, + mainStatKey: statDisplayToGoodKey(record.mainStat), + substats, + lock: Boolean((record as { locked?: boolean }).locked), + }; +} + +export function storedArtifactsToGood(records: readonly StoredArtifactRecord[], source = "Genshin Artifact Assistant") { + return { + format: "GOOD" as const, + version: 2, + source, + artifacts: records.map(storedArtifactToGood), + }; +} + +function reversePieceLookup(setName: string, slotDisplay: string): string { + for (const [piece, set] of pieceToSet.entries()) { + if (set === setName && pieceToSlot.get(piece) === slotDisplay) return piece; + } + return ""; +} + +function computeMainValue(slotDisplay: string, mainStatDisplay: string, level: number): string { + const references = mainStatValueReferences[slotDisplay]; + const reference = Array.isArray(references) + ? (references as Array<{ stat: string; base: number; max: number }>).find((entry) => entry.stat === mainStatDisplay) + : undefined; + if (!reference) return ""; + const clamped = Math.max(0, Math.min(20, level)); + const value = reference.base + (reference.max - reference.base) * (clamped / 20); + const isPercent = DISPLAY_TO_STAT.get(mainStatDisplay)?.percent ?? false; + return isPercent ? `${(Math.round(value * 10) / 10).toFixed(1)}%` : Math.round(value).toLocaleString("en-US"); +} + +export function goodToStoredArtifact(good: GoodImportArtifact, index = 0): StoredArtifactRecord | null { + const slot = GOOD_TO_SLOT[good.slotKey]; + const mainStat = goodKeyToStatDisplay(good.mainStatKey); + if (!slot || !mainStat) return null; + + const setName = setKeyToName(good.setKey); + const level = typeof good.level === "number" ? good.level : 0; + const substats = (good.substats ?? []) + .map((substat) => goodSubstatToString(substat)) + .filter(Boolean); + const name = reversePieceLookup(setName, slot) || setName; + + return { + id: `good-${good.setKey}-${good.slotKey}-${index}`, + name, + slot, + level, + setName, + mainStat, + mainValue: computeMainValue(slot, mainStat, level), + substats, + equipped: good.location || "Not detected", + confidence: 100, + needsReview: false, + source: "good-import", + }; +} + +export function goodDatabaseToStoredArtifacts(database: GoodImportDatabase | null | undefined): StoredArtifactRecord[] { + const records: StoredArtifactRecord[] = []; + (database?.artifacts ?? []).forEach((artifact, index) => { + const record = goodToStoredArtifact(artifact, index); + if (record) records.push(record); + }); + return records; +} diff --git a/src/lib/lockDetection.test.ts b/src/lib/lockDetection.test.ts new file mode 100644 index 0000000..a727811 --- /dev/null +++ b/src/lib/lockDetection.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { detectLockState, isLocked, lockIconCropRect, lockSignalRatio } from "./lockDetection"; +import type { Bitmap } from "./ocrPreprocess"; +import { profileDetailRect } from "./layoutProfile"; + +// Build a BGRA bitmap where `goldPixels` of the pixels are lock-gold and the rest dark. +function bitmap(goldPixels: number, total: number): Bitmap { + const data = Buffer.alloc(total * 4); + for (let pixel = 0; pixel < total; pixel++) { + const index = pixel * 4; + if (pixel < goldPixels) { + data[index] = 40; // B + data[index + 1] = 170; // G + data[index + 2] = 230; // R -> gold + } + data[index + 3] = 255; + } + return { data, width: total, height: 1 }; +} + +describe("lockDetection", () => { + it("places the lock crop in the top-right of the detail card", () => { + const size = { width: 2560, height: 1440 }; + const detail = profileDetailRect(size); + const rect = lockIconCropRect(detail, size); + expect(rect.x).toBeGreaterThan(detail.x + detail.width * 0.5); + expect(rect.x + rect.width).toBeLessThanOrEqual(size.width); + expect(rect.y).toBeLessThan(detail.y + detail.height * 0.5); + }); + + it("measures the gold-pixel ratio", () => { + expect(lockSignalRatio(bitmap(0, 100))).toBe(0); + expect(lockSignalRatio(bitmap(50, 100))).toBeCloseTo(0.5, 5); + expect(lockSignalRatio(bitmap(100, 100))).toBe(1); + }); + + it("thresholds the ratio into a locked flag", () => { + expect(isLocked(0.02)).toBe(false); + expect(isLocked(0.2)).toBe(true); + expect(detectLockState(bitmap(20, 100))).toBe(true); + expect(detectLockState(bitmap(1, 100))).toBe(false); + }); +}); diff --git a/src/lib/lockDetection.ts b/src/lib/lockDetection.ts new file mode 100644 index 0000000..20e3b7c --- /dev/null +++ b/src/lib/lockDetection.ts @@ -0,0 +1,52 @@ +import { clampRect, type LayoutRect } from "./layoutProfile"; +import type { Bitmap } from "./ocrPreprocess"; + +// EXPERIMENTAL, read-only lock-status detection (nice-to-have). Genshin shows a +// padlock at the top-right of the artifact detail card: a bright gold fill when +// locked, a dim outline when not. This estimates that icon region and measures +// the fraction of bright "lock-gold" pixels; above a threshold the piece is +// considered locked. +// +// The crop position and threshold need calibration against a reference 16:9 +// screenshot before this is wired into the capture pipeline, so it ships pure and +// unit-tested but unused by main.ts. It never drives any in-game action - it only +// reads state for triage. + +export function lockIconCropRect(detailRect: LayoutRect, imageSize: { width: number; height: number }): LayoutRect { + return clampRect( + { + x: Math.round(detailRect.x + detailRect.width * 0.8), + y: Math.round(detailRect.y + detailRect.height * 0.03), + width: Math.round(detailRect.width * 0.16), + height: Math.round(detailRect.height * 0.09), + }, + imageSize, + ); +} + +// A gold/highlighted lock pixel: red high, green mid-high, blue low. +function isLockGold(b: number, g: number, r: number): boolean { + return r >= 180 && g >= 140 && b <= 120 && r > b + 40 && g > b + 20; +} + +export function lockSignalRatio(bitmap: Bitmap): number { + const { data, width, height } = bitmap; + const pixels = width * height; + if (pixels === 0) return 0; + let gold = 0; + for (let pixel = 0; pixel < pixels; pixel++) { + const index = pixel * 4; + if (isLockGold(data[index], data[index + 1], data[index + 2])) gold++; + } + return gold / pixels; +} + +export const DEFAULT_LOCK_THRESHOLD = 0.06; + +export function isLocked(signalRatio: number, threshold: number = DEFAULT_LOCK_THRESHOLD): boolean { + return signalRatio >= threshold; +} + +export function detectLockState(bitmap: Bitmap, threshold: number = DEFAULT_LOCK_THRESHOLD): boolean { + return isLocked(lockSignalRatio(bitmap), threshold); +} From b8a38ba5476dfa571f030a89fe058e1eb1d00e90 Mon Sep 17 00:00:00 2001 From: AzuTear Date: Sun, 5 Jul 2026 21:57:23 +0200 Subject: [PATCH 07/14] docs: scanner rework status (done vs remaining live-calibration items) Co-Authored-By: Claude Opus 4.8 --- docs/scanner-rework-status.md | 49 +++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/scanner-rework-status.md diff --git a/docs/scanner-rework-status.md b/docs/scanner-rework-status.md new file mode 100644 index 0000000..fbe51b4 --- /dev/null +++ b/docs/scanner-rework-status.md @@ -0,0 +1,49 @@ +# Scanner rework status + +Progress on the approved scanner/OCR rework. See ADR-007/008/009 in +[DECISIONS.md](DECISIONS.md) for the decisions behind these. + +## Done (implemented, unit-tested, build green) + +- **OCR eval harness** — `src/eval/`, `npm run eval`, gate in `npm test`. See + [ocr-eval.md](ocr-eval.md). +- **C# input/capture sidecar** — `native/input-helper/`, `npm run helper:build`. + Replaces the PowerShell helper on the same JSON protocol; PowerShell remains a + fallback. Verified end-to-end (spawn, runtime, base64 capture). +- **Layout profiles + OCR preprocessing** — `src/lib/layoutProfile.ts` (pure + geometry, 16:9 detection), `src/lib/ocrPreprocess.ts` (grayscale + Otsu + binarize). main.ts crops via the profile and OCRs an upscaled + binarized copy. +- **Card-ready gating** — `src/lib/cardReadyGate.ts` replaces the fixed 280 ms + settle with change+stability polling; robust to animation. +- **GOOD interop** — `src/lib/goodInterop.ts` (export + best-effort import for + scanned records). +- **Rescan-merge** — `src/lib/artifactMerge.ts` collapses leveled re-scan + duplicates by a level-independent identity. +- **Data staleness warning** — `src/lib/dataPackageStatus.ts`, surfaced in the + Scanner Diagnose data-package line. +- **Lock detection (experimental)** — `src/lib/lockDetection.ts`, pure heuristic, + not yet wired into capture. + +## Remaining — needs the live environment or a UI pass + +These cannot be finished/validated without Genshin running at the user's +resolution or without UI work best tested live: + +1. **Calibrate IK-style fixed crop coordinates** (ADR-009). The layout module is + the structure; the exact per-field fractions still come from a + colour-detected/fallback detail rect. A reference 16:9 screenshot of the + artifact screen lets us pin exact client-relative crop coordinates. +2. **Validate/tune OCR preprocessing** on real captures — confirm invert + + threshold + upscale factor help (not hurt) actual Tesseract reads. The + text-level eval harness cannot measure image preprocessing. +3. **Wire GOOD import** — file-picker IPC + merge imported records into the store + (the conversion engine is done and tested). +4. **Wire live lock detection** — calibrate crop position/threshold against a + reference screenshot, then populate a `locked` flag during capture. + +## Grow the eval corpus + +Every low-confidence review sample already stores its crops + OCR. Confirm/correct +those via `reviewSampleToEvalCase` and commit them into `src/eval/corpus/` so the +harness keeps measuring real-world accuracy across patches. See +[ocr-eval.md](ocr-eval.md). From 8d4d24f0bb4a2bcd3fa874dd84ebd1669746c6c9 Mon Sep 17 00:00:00 2001 From: AzuTear Date: Mon, 6 Jul 2026 07:46:31 +0200 Subject: [PATCH 08/14] feat(ui): separate dev info into a Diagnose view, declutter the scan workspace Reworks the UI so the Scan tab shows only core actions and all developer / diagnostic surfaces live in one dedicated view. - New "Diagnose" nav view. ScanView stays mounted for scan|diagnose (the scan controller state persists across the switch) and renders either the clean workspace or the new DiagnosticsView by mode. - DiagnosticsView consolidates runtime/rights, grid detection, learning + data staleness, fingerprint, auto-scan counters, the automation log, the dev-output toggle, the Demo-Daten action, and the raw crops/OCR/confidence dump. Reuses the existing diagnostics/details model hooks. - Scan workspace decluttered: removed the Scanner Diagnose button and the Details button, dropped the rules/grid/mode dev fields from the result brief and capture meta, shortened the topbar headline, and moved Demo-Daten out of the topbar. - Removed the now-dead ScanDiagnosticsModal / ScanDetailsModal components (their content moved into the view); metrics grid hidden on the Diagnose view. - Added dev:web script + .claude/launch.json for browser preview. Verified in the browser preview (both views render, no console errors); 120 tests + build green. Co-Authored-By: Claude Opus 4.8 --- .claude/launch.json | 11 + package.json | 1 + src/features/layout/AppLayout.tsx | 11 +- src/features/layout/navigation.ts | 3 +- src/features/layout/types.ts | 2 +- src/features/scan/ScanView.tsx | 24 ++- .../scan/components/DiagnosticsView.tsx | 203 ++++++++++++++++++ .../scan/components/ScanMainSection.tsx | 13 +- .../scan/components/ScanModalsSection.tsx | 23 +- .../components/ScanTopControlsSection.tsx | 12 +- .../components/modals/ScanDetailsModal.tsx | 83 ------- .../modals/ScanDiagnosticsModal.tsx | 172 --------------- src/pages/app/AppPageLayout.tsx | 7 +- src/styles/global.css | 54 +++++ 14 files changed, 304 insertions(+), 315 deletions(-) create mode 100644 .claude/launch.json create mode 100644 src/features/scan/components/DiagnosticsView.tsx delete mode 100644 src/features/scan/components/modals/ScanDetailsModal.tsx delete mode 100644 src/features/scan/components/modals/ScanDiagnosticsModal.tsx diff --git a/.claude/launch.json b/.claude/launch.json new file mode 100644 index 0000000..4926eec --- /dev/null +++ b/.claude/launch.json @@ -0,0 +1,11 @@ +{ + "version": "0.0.1", + "configurations": [ + { + "name": "vite", + "runtimeExecutable": "npm", + "runtimeArgs": ["run", "dev:web"], + "port": 5173 + } + ] +} diff --git a/package.json b/package.json index c19282a..ddf1deb 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "scripts": { "predev": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\kill-stale-instances.ps1 && tsc -p tsconfig.electron.json && copy electron\\preload.cjs dist-electron\\preload.cjs", "dev": "concurrently -k \"vite --host 127.0.0.1\" \"wait-on tcp:5173 && cross-env VITE_DEV_SERVER_URL=http://127.0.0.1:5173 electron .\"", + "dev:web": "vite --host 127.0.0.1", "dev:admin": ".\\dev-admin.cmd", "build": "tsc && vite build && tsc -p tsconfig.electron.json && copy electron\\\\preload.cjs dist-electron\\\\preload.cjs", "preview": "vite preview --host 127.0.0.1", diff --git a/src/features/layout/AppLayout.tsx b/src/features/layout/AppLayout.tsx index 170c776..6d33249 100644 --- a/src/features/layout/AppLayout.tsx +++ b/src/features/layout/AppLayout.tsx @@ -90,7 +90,7 @@ export function AppTopbar({

Lokaler Windows-Assistent

-

Scanne dein Inventar, triff einfache Artifact-Entscheidungen.

+

Inventar scannen, Artifacts entscheiden.

{topbarStatus && {topbarStatus}} @@ -107,15 +107,6 @@ export function AppTopbar({ {overlayIcon} {overlayButtonLabel} -
); diff --git a/src/features/layout/navigation.ts b/src/features/layout/navigation.ts index 9d1854e..8880a6d 100644 --- a/src/features/layout/navigation.ts +++ b/src/features/layout/navigation.ts @@ -1,4 +1,4 @@ -import { Layers3, Radar, Wand2, Eye } from "lucide-react"; +import { Layers3, Radar, Wand2, Eye, Wrench } from "lucide-react"; import type { AppNavigationItem } from "./types"; export const appNavigationItems: AppNavigationItem[] = [ @@ -6,5 +6,6 @@ export const appNavigationItems: AppNavigationItem[] = [ { id: "triage", label: "Triage", icon: Layers3 }, { id: "builds", label: "Builds", icon: Wand2 }, { id: "overlay", label: "Overlay", icon: Eye }, + { id: "diagnose", label: "Diagnose", icon: Wrench }, ]; diff --git a/src/features/layout/types.ts b/src/features/layout/types.ts index 74ce470..5b6e564 100644 --- a/src/features/layout/types.ts +++ b/src/features/layout/types.ts @@ -1,7 +1,7 @@ import { type ComponentType, type ReactNode } from "react"; import type { LucideProps } from "lucide-react"; -export type NavigationId = "scan" | "triage" | "builds" | "overlay"; +export type NavigationId = "scan" | "triage" | "builds" | "overlay" | "diagnose"; export interface AppNavigationItem { id: NavigationId; diff --git a/src/features/scan/ScanView.tsx b/src/features/scan/ScanView.tsx index 26b391f..54eca1e 100644 --- a/src/features/scan/ScanView.tsx +++ b/src/features/scan/ScanView.tsx @@ -1,9 +1,29 @@ -import { ScanViewLayout } from "./components/ScanViewLayout"; +import { ScanViewLayout } from "./components/ScanViewLayout"; +import { DiagnosticsView } from "./components/DiagnosticsView"; import { useScanViewController } from "./hooks/useScanViewController"; import type { ScanViewProps } from "./types"; -export function ScanView(props: ScanViewProps) { +interface ScanViewExtraProps { + /** "workspace" shows the clean scan surface; "diagnose" shows all dev info. */ + mode?: "workspace" | "diagnose"; + onDemoScan?: () => void; + canDemoScan?: boolean; +} + +export function ScanView({ mode = "workspace", onDemoScan, canDemoScan, ...props }: ScanViewProps & ScanViewExtraProps) { const controller = useScanViewController(props); + if (mode === "diagnose") { + return ( + + ); + } + return ; } diff --git a/src/features/scan/components/DiagnosticsView.tsx b/src/features/scan/components/DiagnosticsView.tsx new file mode 100644 index 0000000..ebbb992 --- /dev/null +++ b/src/features/scan/components/DiagnosticsView.tsx @@ -0,0 +1,203 @@ +import { AlertTriangle, Play, Wrench } from "lucide-react"; +import type { CaptureResult } from "../../../types/global"; +import type { ScanViewControllerResult } from "../types"; +import { FieldConfidenceList } from "./ScanResultCards"; +import { useScanDiagnosticsModalModel } from "./modals/hooks/useScanDiagnosticsModalModel"; +import { useScanDetailsModalModel } from "./modals/hooks/useScanDetailsModalModel"; + +interface DiagnosticsViewProps { + controller: ScanViewControllerResult; + latestCapture: CaptureResult | null; + captureStatus: string; + onDemoScan?: () => void; + canDemoScan?: boolean; +} + +// All developer / diagnostic surfaces live here, separated from the Scan +// workspace: runtime + rights, grid detection, learning + data-package status, +// fingerprint, auto-scan counters, the automation log, and the raw crop/OCR/ +// confidence dump. Read-only; it drives no scan action except the demo snapshot. +export function DiagnosticsView({ controller, latestCapture, captureStatus, onDemoScan, canDemoScan }: DiagnosticsViewProps) { + const { + statusTitle, + rightsClassName, + rightsValue, + genshinClassName, + genshinValue, + shouldShowAdminBanner, + gridSourceClass, + gridMainValue, + gridMetaValue, + learningRulesText, + learningRulesSubtext, + autoScanModeLabel, + fingerprintText, + runtimeRows, + scanLimitText, + autoScanStatsLines, + playerProgress, + reviewStatus, + automationLogLines, + canSaveReviewSample, + handleSaveReviewSample, + } = useScanDiagnosticsModalModel({ + setDetailsOpen: controller.setDetailsOpen, + setDiagnosticsOpen: controller.setDiagnosticsOpen, + saveReviewSample: controller.saveReviewSample, + canSaveReviewSample: controller.canSaveReviewSample, + latestCapture, + controller, + captureStatus, + }); + + const { parsedNotes, showParsedNotes, cropRows, ocrRows, debugText, showCrops, showOcr } = useScanDetailsModalModel({ + setDetailsOpen: controller.setDetailsOpen, + parsedArtifact: controller.parsedArtifact, + latestCapture, + }); + + return ( +
+
+
+

Diagnose & Dev

+

Laufzeit, Erkennung & Rohdaten

+
+
+ + {onDemoScan && ( + + )} +
+
+ +
+
+

Status

+
+
+ App-Rechte + {rightsValue} +
+
+ Genshin + {genshinValue} +
+
+ {shouldShowAdminBanner && ( +

+ App laeuft im Standard-Modus. Auto-Scan braucht Administrator-Rechte: App schliessen und als Administrator neu starten. +

+ )} +
+ Tile grid + {gridMainValue} + {gridMetaValue} +
+
+ Learning & Daten + {learningRulesText} + {learningRulesSubtext} +
+
+ Fingerprint + {fingerprintText} + Aktiver Capture-Fingerprint fuer die Duplikat-Erkennung. +
+
+ +
+

{autoScanModeLabel}

+ {playerProgress.show ? ( +
+ {autoScanStatsLines.map((entry) => ( + + {entry.value} + {entry.label} + + ))} +
+ ) : ( +

Noch keine Scan-Aktivitaet.

+ )} +

{scanLimitText}

+
+ {runtimeRows.map((row) => ( + {row} + ))} +
+ {reviewStatus &&

{reviewStatus}

} +
+
+ +
+

Automation log

+
+ {automationLogLines.length > 0 ? ( + automationLogLines.map((line, index) => {line}) + ) : ( + Keine Scan-Aktivitaet. + )} +
+
+ +
+
+

Crops, OCR & Confidence

+ +
+ {controller.parsedArtifact ? ( + <> + + {showParsedNotes && ( +
+ {parsedNotes.map((note) => ( + {note} + ))} +
+ )} + + ) : ( +

Noch kein Artifact gelesen. Lies ein Artifact im Scan-Tab, um Crops und OCR zu sehen.

+ )} + {showCrops && ( +
+ {cropRows.map((crop) => ( +
+ {crop.label} +
+ {crop.label} + {crop.x},{crop.y} - {crop.width}x{crop.height} +
+
+ ))} +
+ )} + {showOcr && ( +
+ OCR candidates + {ocrRows.map((entry) => ( +
+
+ {entry.label} + {entry.confidence}% confidence +
+ 
{entry.text}
+
+ ))} +
+ )} + {latestCapture &&

{debugText}

} +
+
+ ); +} diff --git a/src/features/scan/components/ScanMainSection.tsx b/src/features/scan/components/ScanMainSection.tsx index 600f9b4..0246328 100644 --- a/src/features/scan/components/ScanMainSection.tsx +++ b/src/features/scan/components/ScanMainSection.tsx @@ -1,4 +1,4 @@ -import { AlertTriangle, Camera, Eye } from "lucide-react"; +import { AlertTriangle, Camera } from "lucide-react"; import { ArtifactResultCard } from "./ScanResultCards"; import { useScanMainSectionModel } from "./hooks/useScanMainSectionModel"; import type { ScanMainSectionProps } from "./types"; @@ -63,9 +63,7 @@ export function ScanMainSection({
Quelle{sourceLabel}
-
Grid{gridLabel}
Inventar{inventoryLabel}
-
Modus{captureModeText}
@@ -81,17 +79,8 @@ export function ScanMainSection({ {targetLabel} {dbLabel} {reviewLabel} - {rulesLabel}
-
{bridgeStatusText} @@ -118,15 +117,6 @@ export function ScanTopControlsSection({ Scan-Setup -
diff --git a/src/features/scan/components/modals/ScanDetailsModal.tsx b/src/features/scan/components/modals/ScanDetailsModal.tsx deleted file mode 100644 index 30b2d53..0000000 --- a/src/features/scan/components/modals/ScanDetailsModal.tsx +++ /dev/null @@ -1,83 +0,0 @@ -import { FieldConfidenceList } from "../ScanResultCards"; -import type { ScanDetailsModalProps } from "./types"; -import { useScanDetailsModalModel } from "./hooks/useScanDetailsModalModel"; - -export function ScanDetailsModal({ - open, - latestCapture, - controller, - setDetailsOpen, -}: ScanDetailsModalProps) { - const { parsedArtifact } = controller; - const { - closeDetails, - stopPropagation, - parsedNotes, - showParsedNotes, - cropRows, - ocrRows, - debugText, - showCrops, - showOcr, - } = useScanDetailsModalModel({ - setDetailsOpen, - parsedArtifact, - latestCapture, - }); - - if (!open || !latestCapture) return null; - - return ( -
-
-
-
-

Dev

-

Crops, OCR & Confidence

-
- -
-
- {parsedArtifact && ( - <> - - {showParsedNotes && ( -
- {parsedNotes.map((note) => {note})} -
- )} - - )} - {showCrops && ( -
- {cropRows.map((crop) => ( -
- {crop.label} -
- {crop.label} - {crop.x},{crop.y} - {crop.width}x{crop.height} -
-
- ))} -
- )} - {showOcr && ( -
- OCR candidates - {ocrRows.map((entry) => ( -
-
- {entry.label} - {entry.confidence}% confidence -
- 
{entry.text}
-
- ))} -
- )} -

{debugText}

-
-
-
- ); -} diff --git a/src/features/scan/components/modals/ScanDiagnosticsModal.tsx b/src/features/scan/components/modals/ScanDiagnosticsModal.tsx deleted file mode 100644 index 45bfe6b..0000000 --- a/src/features/scan/components/modals/ScanDiagnosticsModal.tsx +++ /dev/null @@ -1,172 +0,0 @@ -import { AlertTriangle, Eye, Wrench } from "lucide-react"; -import type { ScanDiagnosticsModalProps } from "./types"; -import { useScanDiagnosticsModalModel } from "./hooks/useScanDiagnosticsModalModel"; - -export function ScanDiagnosticsModal({ - open, - captureStatus, - latestCapture, - controller, - setDetailsOpen, - setDiagnosticsOpen, -}: ScanDiagnosticsModalProps) { - const { - toggleDevMode, - } = controller; - - const { - closeDiagnostics, - openDetails, - handleSaveReviewSample, - stopPropagation, - canOpenDetails, - statusTitle, - rightsClassName, - rightsValue, - genshinClassName, - genshinValue, - shouldShowAdminBanner, - gridSourceClass, - gridMainValue, - gridMetaValue, - learningRulesText, - learningRulesSubtext, - autoScanModeLabel, - autoScanRunning, - fingerprintText, - runtimeRows, - scanLimitText, - scanTipText, - autoScanStatsLines, - playerProgress, - showDevRows, - reviewStatus, - automationLogLines, - canSaveReviewSample, - } = useScanDiagnosticsModalModel({ - setDetailsOpen, - setDiagnosticsOpen, - saveReviewSample: controller.saveReviewSample, - canSaveReviewSample: controller.canSaveReviewSample, - latestCapture, - controller, - captureStatus, - }); - - if (!open) return null; - - return ( -
-
-
-
-

Scanner Diagnose

-

Input, Grid & Lernstatus

-
- -
-
-
- -
- -
-
- App-Rechte - {rightsValue} -
-
- Genshin - {genshinValue} -
-
- {shouldShowAdminBanner && ( -

- App laeuft im Standard-Modus. Auto-Scan braucht Administrator-Rechte: App schliessen und als Administrator neu starten (z.B. Terminal per Rechtsklick "Als Administrator ausfuehren" und darin "npm run dev"). -

- )} - -
- Tile grid - {gridMainValue} - {gridMetaValue} -
- -
- Learning - {learningRulesText} - {learningRulesSubtext} -
- - {playerProgress.show && ( -
- {autoScanModeLabel} - {autoScanStatsLines.map((entry) => ( - - {entry.value} - {entry.label} - - ))} -
- )} - -
- Fingerprint - {fingerprintText} - Active capture fingerprint used for deterministic duplicate guard checks. -
- - {showDevRows && ( -
- {runtimeRows.map((row) => ( - {row} - ))} -
- )} - - {autoScanRunning && playerProgress.show && ( -
-
-
-
-
- )} - - {reviewStatus && ( -

{reviewStatus}

- )} - -
- Automation -
- {automationLogLines.length > 0 ? ( - automationLogLines.map((line, index) => ( - {line} - )) - ) : ( - No scan activity yet. - )} -
-
- -
- - -
- -

{scanLimitText}

-

{scanTipText}

-
-
-
- ); -} diff --git a/src/pages/app/AppPageLayout.tsx b/src/pages/app/AppPageLayout.tsx index e9648bd..483add4 100644 --- a/src/pages/app/AppPageLayout.tsx +++ b/src/pages/app/AppPageLayout.tsx @@ -53,9 +53,12 @@ export function AppPageLayout({ controller }: AppPageLayoutProps) { overlayIcon={} demoIcon={} /> - - {activeView === "scan" && ( + {activeView !== "diagnose" && } + {(activeView === "scan" || activeView === "diagnose") && ( Date: Mon, 6 Jul 2026 07:56:31 +0200 Subject: [PATCH 09/14] feat(good): wire GOOD import/export UI into the Diagnose view Completes the GOOD interop point end-to-end (the conversion engine landed earlier in goodInterop.ts). No new IPC needed - reuses the existing artifacts:load / artifacts:saveMany / good:export bridge. - Scan controller gains exportGoodFromStore (loadArtifacts -> storedArtifactsToGood -> exportGood) and importGoodArtifacts (saveMany + snapshot refresh), plus a canGoodInterop flag. - DiagnosticsView adds a GOOD Interop card: export the scan store as GOOD, or import a GOOD file. The file is read in the renderer via a file input + goodDatabaseToStoredArtifacts, so no file-dialog IPC is required. Verified in the browser preview after a clean restart: the Diagnose view renders all cards (Status, Last scan, GOOD Interop, Automation log, Crops/OCR), the Scan<->Diagnose switch works with no console errors (the earlier hook-order warnings were stale-HMR artifacts from deleting files mid-session). 120 tests + build green. Co-Authored-By: Claude Opus 4.8 --- .../scan/components/DiagnosticsView.tsx | 61 ++++++++++++++++++- .../scan/hooks/useScanViewController.ts | 25 ++++++++ src/features/scan/types.ts | 4 ++ 3 files changed, 89 insertions(+), 1 deletion(-) diff --git a/src/features/scan/components/DiagnosticsView.tsx b/src/features/scan/components/DiagnosticsView.tsx index ebbb992..bd59eea 100644 --- a/src/features/scan/components/DiagnosticsView.tsx +++ b/src/features/scan/components/DiagnosticsView.tsx @@ -1,6 +1,8 @@ -import { AlertTriangle, Play, Wrench } from "lucide-react"; +import { useRef, useState, type ChangeEvent } from "react"; +import { AlertTriangle, Download, Play, Upload, Wrench } from "lucide-react"; import type { CaptureResult } from "../../../types/global"; import type { ScanViewControllerResult } from "../types"; +import { goodDatabaseToStoredArtifacts, type GoodImportDatabase } from "../../../lib/goodInterop"; import { FieldConfidenceList } from "./ScanResultCards"; import { useScanDiagnosticsModalModel } from "./modals/hooks/useScanDiagnosticsModalModel"; import { useScanDetailsModalModel } from "./modals/hooks/useScanDetailsModalModel"; @@ -56,6 +58,42 @@ export function DiagnosticsView({ controller, latestCapture, captureStatus, onDe latestCapture, }); + const [interopStatus, setInteropStatus] = useState(""); + const fileInputRef = useRef(null); + + const handleExportGood = async () => { + setInteropStatus("Exportiere GOOD..."); + const result = await controller.exportGoodFromStore(); + setInteropStatus( + result.ok + ? `GOOD exportiert: ${result.count} Artifacts${result.path ? ` -> ${result.path}` : ""}` + : "GOOD-Export fehlgeschlagen (App im Electron-Fenster oeffnen).", + ); + }; + + const handleImportGood = async (event: ChangeEvent) => { + const file = event.target.files?.[0]; + event.target.value = ""; + if (!file) return; + try { + const database = JSON.parse(await file.text()) as GoodImportDatabase; + const records = goodDatabaseToStoredArtifacts(database); + if (records.length === 0) { + setInteropStatus("Keine gueltigen Artifacts in der Datei gefunden."); + return; + } + setInteropStatus(`Importiere ${records.length} Artifacts...`); + const result = await controller.importGoodArtifacts(records); + setInteropStatus( + result.ok + ? `Importiert: ${result.added} neu, ${result.updated} aktualisiert.` + : "Import fehlgeschlagen (App im Electron-Fenster oeffnen).", + ); + } catch { + setInteropStatus("Datei ist kein gueltiges GOOD/JSON."); + } + }; + return (
@@ -136,6 +174,27 @@ export function DiagnosticsView({ controller, latestCapture, captureStatus, onDe
+
+
+

GOOD Interop

+
+ + + +
+
+

+ Exportiert den Scan-Store als GOOD (Genshin Optimizer / Inventory Kamera / Akasha) oder importiert eine GOOD-Datei in den Store. +

+ {interopStatus &&

{interopStatus}

} +
+

Automation log

diff --git a/src/features/scan/hooks/useScanViewController.ts b/src/features/scan/hooks/useScanViewController.ts index 78e3998..2a51761 100644 --- a/src/features/scan/hooks/useScanViewController.ts +++ b/src/features/scan/hooks/useScanViewController.ts @@ -16,6 +16,8 @@ import { useScanViewStateSync } from "./useScanViewStateSync"; import { emptyAutoScanStats, resolveScanTargetCount, type AutoScanStats, type ScanSummary } from "../../../lib/scannerSession"; import type { ScanViewProps, ScanViewControllerResult } from "../types"; import type { CaptureResult, ClickResult, ReviewSampleRecord } from "../../../types/global"; +import type { StoredArtifactRecord } from "../../../types/storage"; +import { storedArtifactsToGood } from "../../../lib/goodInterop"; import { createRendererRepositories } from "../../../infrastructure/repositories/rendererBridgeRepositories"; export function useScanViewController({ @@ -37,6 +39,7 @@ export function useScanViewController({ const snapshotRepo = repositories?.snapshot; const automationRepo = repositories?.automation; const captureRepo = repositories?.capture; + const exportRepo = repositories?.export; const [detailsOpen, setDetailsOpen] = useState(false); const [diagnosticsOpen, setDiagnosticsOpen] = useState(false); @@ -152,6 +155,25 @@ export function useScanViewController({ setReviewQueueOpen, }); + const canGoodInterop = bridgeReady && Boolean(artifactRepo?.loadAll) && Boolean(artifactRepo?.saveMany); + + const exportGoodFromStore = useCallback(async () => { + if (!artifactRepo?.loadAll || !exportRepo?.exportGood) return { ok: false, count: 0 }; + const loaded = await artifactRepo.loadAll(); + const records = loaded.artifacts ?? []; + const good = storedArtifactsToGood(records); + const result = await exportRepo.exportGood(good); + return { ok: Boolean(result.ok), path: result.path, count: good.artifacts.length }; + }, [artifactRepo, exportRepo]); + + const importGoodArtifacts = useCallback(async (records: StoredArtifactRecord[]) => { + if (!artifactRepo?.saveMany || records.length === 0) return { ok: false, added: 0, updated: 0 }; + const result = await artifactRepo.saveMany(records); + if (typeof result.total === "number") setStoredTotal(result.total); + await onStoredArtifactsChanged?.(); + return { ok: Boolean(result.ok), added: result.added ?? 0, updated: result.updated ?? 0 }; + }, [artifactRepo, onStoredArtifactsChanged]); + useScanViewStateSync({ artifactRepo, latestCapture, @@ -229,5 +251,8 @@ export function useScanViewController({ openReviewQueue: openReviewQueueModal, runAutoReviewScan, runVisibleGridScan, + canGoodInterop, + exportGoodFromStore, + importGoodArtifacts, }; } diff --git a/src/features/scan/types.ts b/src/features/scan/types.ts index 3ca9860..3ce1056 100644 --- a/src/features/scan/types.ts +++ b/src/features/scan/types.ts @@ -1,5 +1,6 @@ import type { AppSnapshot } from "../../types/domain"; import type { BooleanResult, CaptureOptions, CaptureResult, CaptureSourceInfo, RuntimeInfo, ReviewSampleRecord } from "../../types/global"; +import type { StoredArtifactRecord } from "../../types/storage"; import type { AutoScanStats, ScanSummary } from "../../lib/scannerSession"; import type { ParsedArtifactCandidate } from "../../lib/artifactOcrParser"; import type { ScannerLearningRules } from "../../lib/scannerLearning"; @@ -79,4 +80,7 @@ export interface ScanViewControllerResult { openReviewQueue: () => Promise; runAutoReviewScan: () => Promise; runVisibleGridScan: () => Promise; + canGoodInterop: boolean; + exportGoodFromStore: () => Promise<{ ok: boolean; path?: string; count: number }>; + importGoodArtifacts: (records: StoredArtifactRecord[]) => Promise<{ ok: boolean; added: number; updated: number }>; } From 113601f03185f1816112e402969d361174f2c4ef Mon Sep 17 00:00:00 2001 From: AzuTear Date: Mon, 6 Jul 2026 08:11:34 +0200 Subject: [PATCH 10/14] fix(electron): point main + preload + index paths at the real build layout npm run dev failed with ERR_MODULE_NOT_FOUND for dist-electron/services/inputHelper. Root cause: the electron program includes src runtime files (repositories, layoutProfile, ocrPreprocess), so tsc's inferred rootDir is the project root and it emits the entry at dist-electron/electron/main.js (with src at dist-electron/src). package.json "main" still pointed at a stale flat dist-electron/main.js fossil from an older build layout, whose extensionless imports don't resolve under NodeNext ESM. - package.json main -> dist-electron/electron/main.js. - predev/build copy preload.cjs into dist-electron/electron/ (next to main.js, where main.ts resolves it via __dirname). - main.ts loads ../../dist/index.html (one level deeper now) for the packaged window + overlay. Verified: clean electron build emits only the nested layout; electron . loads the main process past module resolution (only ERR_CONNECTION_REFUSED for the dev server, expected standalone); npm run build green. Co-Authored-By: Claude Opus 4.8 --- electron/main.ts | 4 ++-- package.json | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/electron/main.ts b/electron/main.ts index 3196bb6..e2391e6 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -394,7 +394,7 @@ function createMainWindow() { if (isDev) { mainWindow.loadURL(process.env.VITE_DEV_SERVER_URL!); } else { - mainWindow.loadFile(path.join(__dirname, "../dist/index.html")); + mainWindow.loadFile(path.join(__dirname, "../../dist/index.html")); } } @@ -528,7 +528,7 @@ function createOverlayWindow() { if (isDev) { overlayWindow.loadURL(`${process.env.VITE_DEV_SERVER_URL!}?overlay=1`); } else { - overlayWindow.loadFile(path.join(__dirname, "../dist/index.html"), { + overlayWindow.loadFile(path.join(__dirname, "../../dist/index.html"), { query: { overlay: "1" }, }); } diff --git a/package.json b/package.json index ddf1deb..612f525 100644 --- a/package.json +++ b/package.json @@ -3,14 +3,14 @@ "version": "0.1.0", "private": true, "description": "Local Windows assistant for scanning Genshin artifacts and suggesting no-brainer builds.", - "main": "dist-electron/main.js", + "main": "dist-electron/electron/main.js", "type": "module", "scripts": { - "predev": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\kill-stale-instances.ps1 && tsc -p tsconfig.electron.json && copy electron\\preload.cjs dist-electron\\preload.cjs", + "predev": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\kill-stale-instances.ps1 && tsc -p tsconfig.electron.json && copy electron\\preload.cjs dist-electron\\electron\\preload.cjs", "dev": "concurrently -k \"vite --host 127.0.0.1\" \"wait-on tcp:5173 && cross-env VITE_DEV_SERVER_URL=http://127.0.0.1:5173 electron .\"", "dev:web": "vite --host 127.0.0.1", "dev:admin": ".\\dev-admin.cmd", - "build": "tsc && vite build && tsc -p tsconfig.electron.json && copy electron\\\\preload.cjs dist-electron\\\\preload.cjs", + "build": "tsc && vite build && tsc -p tsconfig.electron.json && copy electron\\\\preload.cjs dist-electron\\\\electron\\\\preload.cjs", "preview": "vite preview --host 127.0.0.1", "start": "electron .", "lint": "tsc --noEmit", From c8ae0dd7bffeda999c71d8af0c0c2b2c3d729427 Mon Sep 17 00:00:00 2001 From: AzuTear Date: Mon, 6 Jul 2026 16:07:30 +0200 Subject: [PATCH 11/14] fix(input): force Genshin foreground via AttachThreadInput (auto-scan no longer aborts) Auto-scan aborted immediately with "Genshin konnte nicht in den Vordergrund geholt werden". Root cause: the focus call runs in the background input/capture helper process, and Windows' foreground lock silently refuses SetForegroundWindow from a process that is neither foreground nor the last input source. When the user clicks "Auto-Scan starten" the Electron window is foreground, so the helper's plain SetForegroundWindow is dropped and focus stays false. Fix (both the C# sidecar and the PowerShell fallback): before SetForegroundWindow, attach our thread's input queue to the target (and current-foreground) window thread with AttachThreadInput and clear SPI_..FOREGROUNDLOCKTIMEOUT, then restore. This is the same technique Inventory Kamera and other reliable automators use; it is what our helper was missing after the old ALT-tap workaround was removed on the wrong assumption that equal integrity level is sufficient (that only covers UIPI input injection, not foreground changes). Sidecar recompiled + republished; electron build green. Co-Authored-By: Claude Opus 4.8 --- electron/preload.cjs | 1 + electron/preload.ts | 1 + electron/services/inputHelper.ts | 59 ++++++++--- native/input-helper/Program.cs | 64 +++++++++++- .../scan/hooks/scanViewScanActions.ts | 34 +++++-- .../rendererBridgeRepositoryFactory.ts | 8 ++ .../rendererBridgeRepositoryTypes.ts | 1 + src/lib/autoScanLoop.test.ts | 98 +++++++++++++++++++ src/lib/autoScanLoop.ts | 12 ++- src/services/assistantBridge.ts | 3 + src/types/global.d.ts | 1 + 11 files changed, 255 insertions(+), 27 deletions(-) diff --git a/electron/preload.cjs b/electron/preload.cjs index c79e304..d120be0 100644 --- a/electron/preload.cjs +++ b/electron/preload.cjs @@ -11,6 +11,7 @@ contextBridge.exposeInMainWorld("assistantApi", { getAutomationGuard: () => ipcRenderer.invoke("automation:getGuard"), focusMainWindow: () => ipcRenderer.invoke("app:focusMainWindow"), focusGenshin: () => ipcRenderer.invoke("automation:focusGenshin"), + focusGenshinForScanStart: () => ipcRenderer.invoke("automation:focusGenshin"), getRuntimeInfo: () => ipcRenderer.invoke("app:getRuntimeInfo"), saveReviewSample: (sample) => ipcRenderer.invoke("review:saveSample", sample), loadReviewSamples: (limit = 50) => ipcRenderer.invoke("review:loadSamples", limit), diff --git a/electron/preload.ts b/electron/preload.ts index 3102d5d..d3fc2aa 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -14,6 +14,7 @@ contextBridge.exposeInMainWorld("assistantApi", { getAutomationGuard: () => ipcRenderer.invoke("automation:getGuard"), focusMainWindow: () => ipcRenderer.invoke("app:focusMainWindow"), focusGenshin: () => ipcRenderer.invoke("automation:focusGenshin"), + focusGenshinForScanStart: () => ipcRenderer.invoke("automation:focusGenshin"), getRuntimeInfo: () => ipcRenderer.invoke("app:getRuntimeInfo"), saveReviewSample: (sample: ReviewSamplePayload) => ipcRenderer.invoke("review:saveSample", sample), loadReviewSamples: (limit = 50) => ipcRenderer.invoke("review:loadSamples", limit), diff --git a/electron/services/inputHelper.ts b/electron/services/inputHelper.ts index 7b11462..e3e6f41 100644 --- a/electron/services/inputHelper.ts +++ b/electron/services/inputHelper.ts @@ -39,6 +39,16 @@ public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern bool ShowWindowAsync(IntPtr hWnd, int nCmdShow); [DllImport("user32.dll")] +public static extern bool BringWindowToTop(IntPtr hWnd); +[DllImport("user32.dll", SetLastError=true)] +public static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); +[DllImport("kernel32.dll")] +public static extern uint GetCurrentThreadId(); +[DllImport("user32.dll", SetLastError=true, EntryPoint="SystemParametersInfoW")] +public static extern bool SystemParametersInfoGet(uint uiAction, uint uiParam, ref uint pvParam, uint fWinIni); +[DllImport("user32.dll", SetLastError=true, EntryPoint="SystemParametersInfoW")] +public static extern bool SystemParametersInfoSet(uint uiAction, uint uiParam, IntPtr pvParam, uint fWinIni); +[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern bool IsWindow(IntPtr hWnd); @@ -181,6 +191,41 @@ function Find-GenshinWindow { return $script:genshinHwnd } +# Plain SetForegroundWindow from this background helper process is silently +# refused by Windows' foreground lock. Attach our thread's input queue to the +# target (and current foreground) window thread and clear the lock timeout, so +# the foreground change is honored - the same technique Inventory Kamera uses. +function Force-Foreground { + param([IntPtr]$hwnd) + $current = [Native.InputHelper]::GetCurrentThreadId() + $targetPid = [uint32]0 + $target = [Native.InputHelper]::GetWindowThreadProcessId($hwnd, [ref]$targetPid) + $fgWindow = [Native.InputHelper]::GetForegroundWindow() + $foreground = [uint32]0 + if ($fgWindow -ne [IntPtr]::Zero) { + $fgPid = [uint32]0 + $foreground = [Native.InputHelper]::GetWindowThreadProcessId($fgWindow, [ref]$fgPid) + } + + $attachedTarget = $false + $attachedForeground = $false + $oldTimeout = [uint32]0 + $timeoutRead = $false + try { + if ($target -ne 0 -and $target -ne $current) { $attachedTarget = [Native.InputHelper]::AttachThreadInput($current, $target, $true) } + if ($foreground -ne 0 -and $foreground -ne $current -and $foreground -ne $target) { $attachedForeground = [Native.InputHelper]::AttachThreadInput($current, $foreground, $true) } + $timeoutRead = [Native.InputHelper]::SystemParametersInfoGet(0x2000, 0, [ref]$oldTimeout, 0) + [Native.InputHelper]::SystemParametersInfoSet(0x2001, 0, [IntPtr]::Zero, 0x0002) | Out-Null + [Native.InputHelper]::ShowWindowAsync($hwnd, 9) | Out-Null + [Native.InputHelper]::BringWindowToTop($hwnd) | Out-Null + return [Native.InputHelper]::SetForegroundWindow($hwnd) + } finally { + if ($timeoutRead) { [Native.InputHelper]::SystemParametersInfoSet(0x2001, 0, [IntPtr]::new([int64]$oldTimeout), 0x0002) | Out-Null } + if ($attachedForeground) { [Native.InputHelper]::AttachThreadInput($current, $foreground, $false) | Out-Null } + if ($attachedTarget) { [Native.InputHelper]::AttachThreadInput($current, $target, $false) | Out-Null } + } +} + function Focus-GenshinWindow { $hwnd = Find-GenshinWindow $info = @{ @@ -194,19 +239,7 @@ function Focus-GenshinWindow { $info.alreadyForeground = ([Native.InputHelper]::GetForegroundWindow() -eq $hwnd) if (-not $info.alreadyForeground) { - [Native.InputHelper]::ShowWindowAsync($hwnd, 9) | Out-Null - # A previous version tapped ALT (keybd_event) right before this call to - # satisfy Windows' "who's allowed to change the foreground window" - # eligibility check. That tap has a side effect in most Win32 apps: a - # bare ALT press/release toggles menu-mnemonic navigation mode (verified - # live - it left a real app's menu bar highlighted after just this call), - # which then swallows the next several keyboard/mouse events as menu - # navigation instead of routing them to the app - looking exactly like - # "clicks/keys report success but do nothing". This app and Genshin run - # at the same (elevated) integrity level, so plain SetForegroundWindow - # already succeeds without the ALT tap - confirmed with a standalone - # compiled test against a live target window. - $info.setForegroundResult = [Native.InputHelper]::SetForegroundWindow($hwnd) + $info.setForegroundResult = Force-Foreground -hwnd $hwnd Start-Sleep -Milliseconds 140 } diff --git a/native/input-helper/Program.cs b/native/input-helper/Program.cs index 26f5943..e237f7f 100644 --- a/native/input-helper/Program.cs +++ b/native/input-helper/Program.cs @@ -280,11 +280,7 @@ internal static class Program info.AlreadyForeground = Native.GetForegroundWindow() == hwnd; if (!info.AlreadyForeground) { - Native.ShowWindowAsync(hwnd, 9); // SW_RESTORE - // No ALT tap: this app and Genshin run at the same (elevated) - // integrity level, so SetForegroundWindow succeeds on its own. An ALT - // tap would toggle menu-mnemonic mode and swallow the next inputs. - info.SetForegroundResult = Native.SetForegroundWindow(hwnd); + info.SetForegroundResult = ForceForeground(hwnd); Thread.Sleep(140); } @@ -294,6 +290,46 @@ internal static class Program return info; } + // Plain SetForegroundWindow from a background process is silently refused by + // Windows' foreground lock. Inventory Kamera and other reliable automation + // tools bypass it by attaching the calling thread's input queue to the target + // (and current-foreground) window thread and clearing the lock timeout, so the + // foreground change is honored. Without this the auto-scan aborts with + // "Genshin konnte nicht in den Vordergrund geholt werden". + private static bool ForceForeground(IntPtr hwnd) + { + var current = Native.GetCurrentThreadId(); + var target = Native.GetWindowThreadProcessId(hwnd, out _); + var foregroundHwnd = Native.GetForegroundWindow(); + var foreground = foregroundHwnd != IntPtr.Zero ? Native.GetWindowThreadProcessId(foregroundHwnd, out _) : 0u; + + var attachedTarget = false; + var attachedForeground = false; + uint oldTimeout = 0; + var timeoutRead = false; + try + { + if (target != 0 && target != current) attachedTarget = Native.AttachThreadInput(current, target, true); + if (foreground != 0 && foreground != current && foreground != target) + attachedForeground = Native.AttachThreadInput(current, foreground, true); + + timeoutRead = Native.SystemParametersInfo(Native.SPI_GETFOREGROUNDLOCKTIMEOUT, 0, ref oldTimeout, 0); + Native.SystemParametersInfo(Native.SPI_SETFOREGROUNDLOCKTIMEOUT, 0, IntPtr.Zero, Native.SPIF_SENDCHANGE); + + Native.ShowWindowAsync(hwnd, 9); // SW_RESTORE + Native.BringWindowToTop(hwnd); + var ok = Native.SetForegroundWindow(hwnd); + return ok; + } + finally + { + if (timeoutRead) + Native.SystemParametersInfo(Native.SPI_SETFOREGROUNDLOCKTIMEOUT, 0, new IntPtr((long)oldTimeout), Native.SPIF_SENDCHANGE); + if (attachedForeground) Native.AttachThreadInput(current, foreground, false); + if (attachedTarget) Native.AttachThreadInput(current, target, false); + } + } + private static IntPtr FindGenshinWindow() { if (_genshinHwnd != IntPtr.Zero && Native.IsWindow(_genshinHwnd)) return _genshinHwnd; @@ -394,6 +430,9 @@ internal static class Native public const uint MOUSEEVENTF_LEFTDOWN = 0x0002; public const uint MOUSEEVENTF_LEFTUP = 0x0004; public const uint MOUSEEVENTF_WHEEL = 0x0800; + public const uint SPI_GETFOREGROUNDLOCKTIMEOUT = 0x2000; + public const uint SPI_SETFOREGROUNDLOCKTIMEOUT = 0x2001; + public const uint SPIF_SENDCHANGE = 0x0002; public static readonly IntPtr DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2 = new(-4); [StructLayout(LayoutKind.Sequential)] @@ -455,6 +494,21 @@ internal static class Native [DllImport("user32.dll")] public static extern bool ShowWindowAsync(IntPtr hWnd, int nCmdShow); + [DllImport("user32.dll")] + public static extern bool BringWindowToTop(IntPtr hWnd); + + [DllImport("user32.dll", SetLastError = true)] + public static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); + + [DllImport("kernel32.dll")] + public static extern uint GetCurrentThreadId(); + + [DllImport("user32.dll", SetLastError = true)] + public static extern bool SystemParametersInfo(uint uiAction, uint uiParam, ref uint pvParam, uint fWinIni); + + [DllImport("user32.dll", SetLastError = true)] + public static extern bool SystemParametersInfo(uint uiAction, uint uiParam, IntPtr pvParam, uint fWinIni); + [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); diff --git a/src/features/scan/hooks/scanViewScanActions.ts b/src/features/scan/hooks/scanViewScanActions.ts index 81b72b7..ced3360 100644 --- a/src/features/scan/hooks/scanViewScanActions.ts +++ b/src/features/scan/hooks/scanViewScanActions.ts @@ -4,7 +4,16 @@ import { runAutoScanLoop } from "../../../lib/autoScanLoop"; import { clampScanLimit, emptyAutoScanStats, resolveScanTargetCount, type AutoScanStats, type ScanSummary } from "../../../lib/scannerSession"; import { getAutoReviewReason, wait } from "../../../lib/scanReviewUtils"; import type { AutomationRepositoryPort, RuntimeRepositoryPort } from "../../../infrastructure/repositories/rendererBridgeRepositories"; -import type { AutomationGuard, BooleanResult, CaptureOptions, CaptureResult, ClickResult, RuntimeInfo, ScrollResult } from "../../../types/global"; +import type { + AutomationGuard, + BooleanResult, + CaptureOptions, + CaptureResult, + ClickResult, + FocusGenshinResult, + RuntimeInfo, + ScrollResult, +} from "../../../types/global"; import type { ParsedArtifactCandidate } from "../../../lib/artifactOcrParser"; import type { MutableRefObject } from "react"; import type { Dispatch, SetStateAction } from "react"; @@ -164,7 +173,7 @@ export async function runVisibleGridScan(context: ScanActionContext): Promise null); - if (focusResult) { - appendAutomationLog( - `focus: ${focusResult.focused ? "ok" : "fehlgeschlagen"} found:${focusResult.genshinFound ? "yes" : "no"} setForeground:${focusResult.setForegroundResult ?? "n/a"} target:${focusResult.targetProcess || "?"} fg:${focusResult.foregroundProcess || "?"}`, - ); + let focusResult: FocusGenshinResult | null = null; + for (let attempt = 1; attempt <= 3; attempt += 1) { + const current = await focusGenshinForScanStart().catch(() => null); + if (!current) { + appendAutomationLog(`focus attempt ${attempt}/3: exception`); + } else { + focusResult = current; + appendAutomationLog( + `focus attempt ${attempt}/3: ${current.focused ? "ok" : "failed"} found:${current.genshinFound ? "yes" : "no"} setForeground:${current.setForegroundResult ?? "n/a"} target:${current.targetProcess || "?"} fg:${current.foregroundProcess || "?"}`, + ); + if (current.focused) break; + if (!current.genshinFound) break; + } + if (attempt < 3) await wait(300); } + if (!focusResult?.focused) { setAutoScanRunning(false); const reason = !focusResult?.genshinFound diff --git a/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts b/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts index 4027367..850edd4 100644 --- a/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts +++ b/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts @@ -179,6 +179,14 @@ export function createRendererRepositories(): RendererRepositories | null { () => bridge.getAutomationGuard(), emptyAutomationGuard(), ), + focusGenshinForScanStart: () => + createBridgeSafeCall( + () => + typeof bridge.focusGenshinForScanStart === "function" + ? bridge.focusGenshinForScanStart() + : bridge.focusGenshin(), + emptyFocusGenshinResult(), + ), focusGenshin: () => createBridgeSafeCall( () => bridge.focusGenshin(), diff --git a/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts b/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts index 5e8e536..5a3d992 100644 --- a/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts +++ b/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts @@ -58,6 +58,7 @@ export interface SnapshotRepositoryPort { export interface AutomationRepositoryPort { getAutomationGuard(): Promise; focusGenshin(): Promise; + focusGenshinForScanStart: () => Promise; focusMainWindow(): Promise; clickScreen(x: number, y: number): Promise; scrollScreen(notches: number, anchorX?: number, anchorY?: number): Promise; diff --git a/src/lib/autoScanLoop.test.ts b/src/lib/autoScanLoop.test.ts index 42f6248..a87bbd5 100644 --- a/src/lib/autoScanLoop.test.ts +++ b/src/lib/autoScanLoop.test.ts @@ -1,5 +1,58 @@ import { describe, expect, it } from "vitest"; import { detailFingerprint, fingerprintDataUrl, isRepeatedProcessedPageFingerprint, screenFingerprint } from "./autoScanLoop"; +import { runAutoScanLoop } from "./autoScanLoop"; +import type { AutoScanLoopDependencies } from "./autoScanLoop"; +import type { CaptureResult } from "../types/global"; + +type ScanTestCapture = CaptureResult & { inventoryGrid: NonNullable }; + +const sampleGrid = { + centers: [{ x: 80, y: 90, row: 0, col: 0 }], + rows: 1, + cols: 1, + confidence: 86, + source: "detected" as const, +}; + +const sampleParse = { + name: "A Tiara of Torrents", + slot: "Flower of Life", + level: 16, + mainStat: "HP", + mainValue: "10.7%", + substats: ["ATK+29", "DEF+10"], + setName: "Tenacity of the Millelith", + equipped: "Traveler", + confidence: 84, + notes: [], + fields: { + name: { value: "A Tiara of Torrents", confidence: 84, source: "ocr" as const }, + slot: { value: "Flower of Life", confidence: 84, source: "ocr" as const }, + level: { value: "16", confidence: 84, source: "ocr" as const }, + mainStat: { value: "HP", confidence: 84, source: "ocr" as const }, + mainValue: { value: "10.7%", confidence: 84, source: "ocr" as const }, + setName: { value: "Tenacity of the Millelith", confidence: 84, source: "ocr" as const }, + equipped: { value: "Traveler", confidence: 84, source: "ocr" as const }, + substats: { value: "ATK+29, DEF+10", confidence: 84, source: "ocr" as const }, + }, +}; + +function capture(overrides: Partial = {}): ScanTestCapture { + return { + id: "source", + name: "screen-capture", + width: 1920, + height: 1080, + dataUrl: "data:image/png;base64,AA", + capturedAt: "2026-01-01T00:00:00.000Z", + captureTarget: "desktop-source", + ocr: [], + detailDataUrl: "data:image/png;base64,DETAIL-AAA", + inventoryDataUrl: "data:image/png;base64,GRID-AAA", + inventoryGrid: sampleGrid, + ...overrides, + }; +} describe("autoScanLoop fingerprints", () => { it("distinguishes captures that share the same prefix but differ later", () => { @@ -47,4 +100,49 @@ describe("autoScanLoop fingerprints", () => { expect(isRepeatedProcessedPageFingerprint("abc", seen, 2)).toBe(true); expect(isRepeatedProcessedPageFingerprint("", seen, 3)).toBe(false); }); + + it("does not block before first click when start capture is from the primary screen", async () => { + const startCapture = capture({ + captureTarget: "primary-screen", + detailDataUrl: `data:image/png;base64,${"D".repeat(600)}`, + }); + + let clicked = 0; + const deps: AutoScanLoopDependencies = { + api: { + clickScreen: async () => { + clicked += 1; + return { + ok: true, + x: 0, + y: 0, + cursorX: 0, + cursorY: 0, + clicked: true, + moved: true, + focused: true, + }; + }, + scrollScreen: async () => ({ ok: true, notchesSent: 0 }), + getAutomationGuard: async () => ({ ok: true, escapePressed: false, enterPressed: false, f9Pressed: false }), + }, + captureSelectedSource: async () => capture({ detailDataUrl: `data:image/png;base64,${"E".repeat(600)}` }), + captureFastSelectedSource: async () => startCapture, + parseArtifact: () => sampleParse, + persistParsedArtifact: async () => false, + saveReviewSample: async () => ({ ok: true }), + getAutoReviewReason: () => "", + shouldFlagArtifactForReview: () => false, + appendAutomationLog: () => undefined, + appendClickDiagnostics: () => undefined, + setReviewStatus: () => undefined, + setAutoScanStats: () => undefined, + shouldStop: () => false, + }; + + const result = await runAutoScanLoop(deps, { scanLimit: 1, skipRows: 0, detectedInventoryCount: null }); + expect(result.blockedReason).toBe(""); + expect(result.status).toBe("done"); + expect(clicked).toBe(1); + }); }); diff --git a/src/lib/autoScanLoop.ts b/src/lib/autoScanLoop.ts index 10c29a4..fafdaef 100644 --- a/src/lib/autoScanLoop.ts +++ b/src/lib/autoScanLoop.ts @@ -98,6 +98,8 @@ export async function runAutoScanLoop( let aborted = false; let consecutiveMisses = 0; let rowsQueued = 0; + const primaryScreenStartWarning = + "Start-Capture ist vom Primary-Screen, kein spezifischer Genshin-Client-Marker vorhanden - Auto-Scan wird mit Vorsicht fortgesetzt."; function updateStats() { setAutoScanStats({ ...stats }); @@ -153,7 +155,8 @@ export async function runAutoScanLoop( } let currentCapture = await captureSelectedSource(0, true); - const initialCaptureRejection = captureSourceRejectionReason(currentCapture); + const isPrimaryCapture = currentCapture?.captureTarget === "primary-screen"; + const initialCaptureRejection = isPrimaryCapture ? "" : captureSourceRejectionReason(currentCapture); let gridModel = buildGridModel(currentCapture?.inventoryGrid); if (initialCaptureRejection || !gridModel || gridModel.targets.length === 0) { @@ -162,6 +165,10 @@ export async function runAutoScanLoop( return { status: "blocked", stats, blockedReason: reason, pageCount: 0, gridLabel: reason, targetCount: maxTargets }; } + if (isPrimaryCapture) { + appendAutomationLog(primaryScreenStartWarning); + } + let lastDetailSignature = ""; const initialParsed = parseArtifact(currentCapture); if (initialParsed) lastDetailSignature = sessionSignature(initialParsed); @@ -496,5 +503,6 @@ export function fingerprintDataUrl(dataUrl: string) { } function wait(ms: number): Promise { - return new Promise((resolve) => window.setTimeout(() => resolve(), ms)); + const schedule = typeof window !== "undefined" && window.setTimeout ? window.setTimeout : setTimeout; + return new Promise((resolve) => schedule(() => resolve(), ms)); } diff --git a/src/services/assistantBridge.ts b/src/services/assistantBridge.ts index 5ddedeb..88b8bfc 100644 --- a/src/services/assistantBridge.ts +++ b/src/services/assistantBridge.ts @@ -51,6 +51,7 @@ export interface AssistantBridge { publishScannerStatus: (status: ScannerStatusPayload) => Promise; focusMainWindow: () => Promise; focusGenshin: () => Promise; + focusGenshinForScanStart: () => Promise; clickScreen: (x: number, y: number) => Promise; scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => Promise; onScannerCommand: (callback: (command: "start-auto" | "stop") => void) => () => void; @@ -66,6 +67,7 @@ export function getAssistantBridge(): AssistantBridge | null { const apiRecord = api as unknown as Record; const canAutoScan = hasFunction(apiRecord, "clickScreen") && hasFunction(apiRecord, "scrollScreen"); const canReviewSamples = hasFunction(apiRecord, "loadReviewSamples") && hasFunction(apiRecord, "saveReviewSample"); + const hasFocusGenshinForScanStart = hasFunction(apiRecord, "focusGenshinForScanStart"); return { isAvailable: true, @@ -90,6 +92,7 @@ export function getAssistantBridge(): AssistantBridge | null { publishScannerStatus: (status) => api.publishScannerStatus(status), focusMainWindow: () => api.focusMainWindow(), focusGenshin: () => api.focusGenshin(), + focusGenshinForScanStart: () => (hasFocusGenshinForScanStart ? api.focusGenshinForScanStart() : api.focusGenshin()), clickScreen: (x: number, y: number) => api.clickScreen(x, y), scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => api.scrollScreen(notches, anchorX, anchorY), showOverlay: () => api.showOverlay(), diff --git a/src/types/global.d.ts b/src/types/global.d.ts index 7252905..f8987dc 100644 --- a/src/types/global.d.ts +++ b/src/types/global.d.ts @@ -293,6 +293,7 @@ declare global { getAutomationGuard: () => Promise; focusMainWindow: () => Promise; focusGenshin: () => Promise; + focusGenshinForScanStart: () => Promise; getRuntimeInfo: () => Promise; saveReviewSample: (sample: ReviewSamplePayload) => Promise; loadReviewSamples: (limit?: number) => Promise; From b8309af377017c3b4ce969bfa4c4556ad7e0e2aa Mon Sep 17 00:00:00 2001 From: AzuTear Date: Mon, 6 Jul 2026 16:13:25 +0200 Subject: [PATCH 12/14] fix(input): inject a no-op input event so force-foreground actually works Follow-up to the AttachThreadInput change: verified against an isolated repro (a foreground-stealing window + the helper spawned exactly like the app) that AttachThreadInput + clearing the foreground-lock timeout was NOT sufficient on this Windows build - SetForegroundWindow still returned false and Genshin stayed in the background. The missing condition is "the calling process received the last input event". Injecting a benign no-op input (a 0,0 relative mouse move, no cursor movement, no menu-mnemonic side effect) right before SetForegroundWindow satisfies it. With the nudge the repro now returns focused:true / setForegroundResult:true from a background process while another app holds the foreground - the exact auto-scan start scenario. Applied to both the C# sidecar and the PowerShell fallback. Co-Authored-By: Claude Opus 4.8 --- electron/services/inputHelper.ts | 3 +++ native/input-helper/Program.cs | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/electron/services/inputHelper.ts b/electron/services/inputHelper.ts index e3e6f41..68c87b0 100644 --- a/electron/services/inputHelper.ts +++ b/electron/services/inputHelper.ts @@ -216,6 +216,9 @@ function Force-Foreground { if ($foreground -ne 0 -and $foreground -ne $current -and $foreground -ne $target) { $attachedForeground = [Native.InputHelper]::AttachThreadInput($current, $foreground, $true) } $timeoutRead = [Native.InputHelper]::SystemParametersInfoGet(0x2000, 0, [ref]$oldTimeout, 0) [Native.InputHelper]::SystemParametersInfoSet(0x2001, 0, [IntPtr]::Zero, 0x0002) | Out-Null + # Inject a no-op input (0,0 mouse move) so this process is the last input + # source, which Windows requires before it will honor a foreground change. + Send-MouseInput -flags 0x0001 | Out-Null [Native.InputHelper]::ShowWindowAsync($hwnd, 9) | Out-Null [Native.InputHelper]::BringWindowToTop($hwnd) | Out-Null return [Native.InputHelper]::SetForegroundWindow($hwnd) diff --git a/native/input-helper/Program.cs b/native/input-helper/Program.cs index e237f7f..b469339 100644 --- a/native/input-helper/Program.cs +++ b/native/input-helper/Program.cs @@ -316,6 +316,12 @@ internal static class Program timeoutRead = Native.SystemParametersInfo(Native.SPI_GETFOREGROUNDLOCKTIMEOUT, 0, ref oldTimeout, 0); Native.SystemParametersInfo(Native.SPI_SETFOREGROUNDLOCKTIMEOUT, 0, IntPtr.Zero, Native.SPIF_SENDCHANGE); + // Inject a no-op input (0,0 relative mouse move) so this process counts + // as the last input source - one of the conditions Windows requires to + // allow a foreground change. This is what the removed ALT tap did, but + // without the menu-mnemonic side effect. + NudgeInput(); + Native.ShowWindowAsync(hwnd, 9); // SW_RESTORE Native.BringWindowToTop(hwnd); var ok = Native.SetForegroundWindow(hwnd); @@ -390,6 +396,14 @@ internal static class Program return principal.IsInRole(WindowsBuiltInRole.Administrator); } + private static void NudgeInput() + { + var move = new Native.INPUT[1]; + move[0].type = 0; // INPUT_MOUSE + move[0].mi.dwFlags = Native.MOUSEEVENTF_MOVE; // dx=dy=0 -> no cursor movement + Native.SendInput(1, move, Marshal.SizeOf()); + } + private static uint SendMouseClickBatch() { var inputs = new Native.INPUT[2]; @@ -427,6 +441,7 @@ internal static class Program internal static class Native { + public const uint MOUSEEVENTF_MOVE = 0x0001; public const uint MOUSEEVENTF_LEFTDOWN = 0x0002; public const uint MOUSEEVENTF_LEFTUP = 0x0004; public const uint MOUSEEVENTF_WHEEL = 0x0800; From 7930e369a7db9b691748ce4a84fb22814036f97c Mon Sep 17 00:00:00 2001 From: AzuTear Date: Mon, 6 Jul 2026 16:21:28 +0200 Subject: [PATCH 13/14] feat(ocr): substat-roll validation + rarity inference for GOOD export Adds the accuracy check yas / Genshin Optimizer use: a substat value is only legitimate if it equals round(sum of 1..6 rolls) from that stat's roll table. Values that fit no combination at either rarity are guaranteed OCR misreads. - src/lib/substatRolls.ts: 5-star roll tables (+ 4-star %/crit tables to tell rarities apart), pure isPlausibleSubstat/implausibleSubstats, and inferRarity (level > 16 or roll-table fit; conservative, defaults to 5). Validates against the union of rarities so valid 4-star pieces are not false-flagged. - Wired in: shouldFlagArtifactForReview routes implausible substats to review; the parser adds an explanatory note; goodInterop export replaces the hardcoded rarity:5 with inferRarity (fixes wrong 4-star exports to GO/IK). - 10 new unit tests; 131 total green; eval still 100%. Co-Authored-By: Claude Opus 4.8 --- src/lib/artifactOcrParser.ts | 3 + src/lib/goodInterop.ts | 3 +- src/lib/scannerLearning.ts | 5 ++ src/lib/substatRolls.test.ts | 72 ++++++++++++++++++ src/lib/substatRolls.ts | 139 +++++++++++++++++++++++++++++++++++ 5 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 src/lib/substatRolls.test.ts create mode 100644 src/lib/substatRolls.ts diff --git a/src/lib/artifactOcrParser.ts b/src/lib/artifactOcrParser.ts index 79ff8cf..cd7df7b 100644 --- a/src/lib/artifactOcrParser.ts +++ b/src/lib/artifactOcrParser.ts @@ -19,6 +19,7 @@ import { textReplacements, } from "./genshinData.js"; import { fuzzyFindKnown, simplifyForMatch } from "./fuzzyMatch.js"; +import { implausibleSubstats } from "./substatRolls.js"; type MainStatValueReference = { stat: string; base: number; max: number }; @@ -110,6 +111,8 @@ export function parseArtifactCandidate(capture: CaptureResult | null): ParsedArt if (!mainStatField.value) notes.push("Main stat not confidently parsed."); if (!mainValueField.value) notes.push("Main stat value not confidently parsed."); if (substats.length < 3) notes.push("Substats look incomplete; crop or OCR needs tuning."); + const implausible = implausibleSubstats(substats); + if (implausible.length) notes.push(`Substat value has no valid roll combination (likely OCR misread): ${implausible.join(", ")}.`); if (!setField.value) notes.push("Set name not confidently parsed."); for (const [label, parsedField] of Object.entries({ diff --git a/src/lib/goodInterop.ts b/src/lib/goodInterop.ts index 1a4a492..468e285 100644 --- a/src/lib/goodInterop.ts +++ b/src/lib/goodInterop.ts @@ -2,6 +2,7 @@ import type { GoodExportArtifact } from "../types/global"; import type { StoredArtifactRecord } from "../types/storage"; import { knownSets, mainStatValueReferences, pieceToSet, pieceToSlot } from "./genshinData"; import { simplifyForMatch } from "./fuzzyMatch"; +import { inferRarity } from "./substatRolls"; // GOOD (Genshin Open Object Description) interop for scanned artifacts, so the // local store can round-trip with Genshin Optimizer / Inventory Kamera / Akasha @@ -127,7 +128,7 @@ export function storedArtifactToGood(record: StoredArtifactRecord): GoodExportAr return { setKey: setNameToKey(record.setName), slotKey: SLOT_TO_GOOD[record.slot] ?? "", - rarity: 5, + rarity: inferRarity(record.level ?? 0, record.substats), level: record.level ?? 0, mainStatKey: statDisplayToGoodKey(record.mainStat), substats, diff --git a/src/lib/scannerLearning.ts b/src/lib/scannerLearning.ts index 31e15c1..597ad1a 100644 --- a/src/lib/scannerLearning.ts +++ b/src/lib/scannerLearning.ts @@ -1,5 +1,6 @@ import type { ParsedArtifactCandidate } from "./artifactOcrParser"; import { simplifyForMatch } from "./fuzzyMatch"; +import { implausibleSubstats } from "./substatRolls"; import type { CaptureResult, ReviewSampleRecord } from "../types/global"; import type { ScannerLearningRulePayload } from "../types/global"; @@ -73,6 +74,10 @@ export function shouldFlagArtifactForReview( if (substatCount === 0) return true; if (substatCount < 3 && substatConfidence < 70) return true; + // A substat value that matches no legal roll combination is a guaranteed OCR + // misread - never store it as fact, always review. + if (implausibleSubstats(parsed.substats ?? []).length > 0) return true; + return false; } diff --git a/src/lib/substatRolls.test.ts b/src/lib/substatRolls.test.ts new file mode 100644 index 0000000..7e14059 --- /dev/null +++ b/src/lib/substatRolls.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; +import { + implausibleSubstats, + inferRarity, + isPlausibleSubstat, + isPlausibleSubstatValue, + parseSubstatEntry, +} from "./substatRolls"; + +describe("substatRolls parsing", () => { + it("parses percent and flat entries", () => { + expect(parseSubstatEntry("CRIT DMG+13.2%")).toEqual({ stat: "CRIT DMG", value: 13.2, percent: true }); + expect(parseSubstatEntry("HP+1,509")).toEqual({ stat: "HP", value: 1509, percent: false }); + expect(parseSubstatEntry("garbage")).toBeNull(); + }); +}); + +describe("substatRolls plausibility", () => { + it("accepts real single-roll values (5-star)", () => { + expect(isPlausibleSubstatValue("CRIT DMG", 7.8)).toBe(true); // 7.77 high roll + expect(isPlausibleSubstatValue("CRIT DMG", 5.4)).toBe(true); // 5.44 low roll + expect(isPlausibleSubstatValue("CRIT Rate", 3.9)).toBe(true); // 3.89 + expect(isPlausibleSubstatValue("ATK", 19)).toBe(true); // 19.45 + expect(isPlausibleSubstatValue("HP", 269)).toBe(true); // 268.88 + }); + + it("accepts multi-roll sums", () => { + expect(isPlausibleSubstat("CRIT DMG+13.2%")).toBe(true); // 5.44+7.77 or 6.22+6.99 + expect(isPlausibleSubstat("Elemental Mastery+68")).toBe(true); + expect(isPlausibleSubstat("Energy Recharge+11.7%")).toBe(true); + }); + + it("rejects values that fit no roll combination at either rarity (OCR misreads)", () => { + // 8.1% CRIT DMG sits in the gap: single roll maxes at 7.8 (5-star), and the + // smallest two-roll sum is 8.2 (4-star), so it is unreachable at both. + expect(isPlausibleSubstatValue("CRIT DMG", 8.1)).toBe(false); + // 3.5% CRIT DMG is below the lowest possible roll at either rarity. + expect(isPlausibleSubstatValue("CRIT DMG", 3.5)).toBe(false); + // A dropped digit on flat ATK. + expect(isPlausibleSubstatValue("ATK", 5)).toBe(false); + }); + + it("does not flag unknown stats", () => { + expect(isPlausibleSubstatValue("Mystery Stat", 12.3)).toBe(true); + }); + + it("collects the implausible entries from a substat list", () => { + const bad = implausibleSubstats(["CRIT DMG+13.2%", "CRIT DMG+8.1%", "ATK+19"]); + expect(bad).toEqual(["CRIT DMG+8.1%"]); + }); +}); + +describe("substatRolls rarity inference", () => { + it("is 5-star for any artifact leveled past +16", () => { + expect(inferRarity(20, ["ATK%+3.5%"])).toBe(5); + expect(inferRarity(17, [])).toBe(5); + }); + + it("detects 5-star from a substat that only fits the 5-star table", () => { + // 7.8% CRIT DMG is a 5-star single high roll; not reachable at 4-star. + expect(inferRarity(12, ["CRIT DMG+7.8%"])).toBe(5); + }); + + it("detects 4-star from a substat that only fits the 4-star table", () => { + // 4.1% CRIT DMG is a 4-star single low roll; below any 5-star roll. + expect(inferRarity(12, ["CRIT DMG+4.1%"])).toBe(4); + }); + + it("defaults to 5-star when ambiguous", () => { + expect(inferRarity(8, [])).toBe(5); + }); +}); diff --git a/src/lib/substatRolls.ts b/src/lib/substatRolls.ts new file mode 100644 index 0000000..3ca8eb7 --- /dev/null +++ b/src/lib/substatRolls.ts @@ -0,0 +1,139 @@ +// Substat roll validation (the accuracy trick yas / Genshin Optimizer use). +// Every artifact substat value is the SUM of discrete per-roll increments: a +// substat rolls once when it appears and again at every +4 level, up to 6 rolls +// total. So a displayed value is only legitimate if it equals round(sum of N +// rolls) for some N in 1..6 from that stat's roll table. An OCR value that fits +// no combination is a misread (e.g. a dropped/extra digit) and should go to +// review instead of being stored as fact. +// +// Roll tables are keyed by the app's display stat names. 5-star values are the +// full set; 4-star values are included for the stats used to tell rarities +// apart. Flat 4-star tables are intentionally omitted (kept conservative). + +const MAX_ROLLS = 6; + +const ROLLS_5STAR: Record = { + HP: [209.13, 239.0, 268.88, 298.75], + ATK: [13.62, 15.56, 17.51, 19.45], + DEF: [16.2, 18.52, 20.83, 23.15], + "HP%": [4.08, 4.66, 5.25, 5.83], + "ATK%": [4.08, 4.66, 5.25, 5.83], + "DEF%": [5.1, 5.83, 6.56, 7.29], + "Elemental Mastery": [16.32, 18.65, 20.98, 23.31], + "Energy Recharge": [4.53, 5.18, 5.83, 6.48], + "CRIT Rate": [2.72, 3.11, 3.5, 3.89], + "CRIT DMG": [5.44, 6.22, 6.99, 7.77], +}; + +const ROLLS_4STAR: Record = { + "HP%": [3.06, 3.5, 3.93, 4.37], + "ATK%": [3.06, 3.5, 3.93, 4.37], + "DEF%": [3.83, 4.37, 4.92, 5.47], + "Elemental Mastery": [12.25, 13.99, 15.74, 17.48], + "Energy Recharge": [3.4, 3.89, 4.37, 4.86], + "CRIT Rate": [2.04, 2.33, 2.62, 2.91], + "CRIT DMG": [4.08, 4.66, 5.25, 5.83], +}; + +const PERCENT_STATS = new Set([ + "HP%", + "ATK%", + "DEF%", + "Energy Recharge", + "CRIT Rate", + "CRIT DMG", +]); + +function isPercent(stat: string) { + return PERCENT_STATS.has(stat); +} + +function displayRound(value: number, percent: boolean) { + return percent ? Math.round(value * 10) / 10 : Math.round(value); +} + +// All displayed values reachable by summing 1..MAX_ROLLS rolls from `rolls`. +function buildValidSet(rolls: number[], percent: boolean): Set { + const results = new Set(); + let sums = new Set([0]); + for (let n = 1; n <= MAX_ROLLS; n++) { + const next = new Set(); + for (const sum of sums) { + for (const roll of rolls) next.add(Math.round((sum + roll) * 1000) / 1000); + } + sums = next; + for (const sum of sums) results.add(displayRound(sum, percent)); + } + return results; +} + +const VALID_CACHE = new Map>(); + +function validSet(stat: string, table: Record, tag: string): Set | null { + const rolls = table[stat]; + if (!rolls) return null; + const key = `${tag}:${stat}`; + let cached = VALID_CACHE.get(key); + if (!cached) { + cached = buildValidSet(rolls, isPercent(stat)); + VALID_CACHE.set(key, cached); + } + return cached; +} + +export function parseSubstatEntry(entry: string): { stat: string; value: number; percent: boolean } | null { + const plusIndex = entry.indexOf("+"); + if (plusIndex <= 0) return null; + const stat = entry.slice(0, plusIndex).trim(); + const raw = entry.slice(plusIndex + 1).replace(/,/g, "").replace("%", "").trim(); + const value = Number.parseFloat(raw); + if (!Number.isFinite(value)) return null; + return { stat, value, percent: entry.includes("%") }; +} + +/** A displayed value is plausible if it matches a roll sum for 5-star or 4-star. */ +export function isPlausibleSubstatValue(stat: string, value: number): boolean { + const rounded = displayRound(value, isPercent(stat)); + const five = validSet(stat, ROLLS_5STAR, "5"); + const four = validSet(stat, ROLLS_4STAR, "4"); + // Unknown stat (no table) -> do not claim it is implausible. + if (!five && !four) return true; + return Boolean(five?.has(rounded)) || Boolean(four?.has(rounded)); +} + +export function isPlausibleSubstat(entry: string): boolean { + const parsed = parseSubstatEntry(entry); + if (!parsed) return true; + return isPlausibleSubstatValue(parsed.stat, parsed.value); +} + +/** The substat entries whose value fits no legal roll combination. */ +export function implausibleSubstats(substats: readonly string[]): string[] { + return substats.filter((entry) => !isPlausibleSubstat(entry)); +} + +/** + * Best-effort rarity from level + which roll table the substats fit. Conservative: + * only returns 4 when a substat clearly fits the 4-star table and not 5-star; + * otherwise defaults to 5 (the common case and the previous hardcoded value). + */ +export function inferRarity(level: number, substats: readonly string[]): number { + if (level > 16) return 5; // only 5-star artifacts level past +16 + let fitsFiveOnly = 0; + let fitsFourOnly = 0; + for (const entry of substats) { + const parsed = parseSubstatEntry(entry); + if (!parsed) continue; + const rounded = displayRound(parsed.value, isPercent(parsed.stat)); + const five = validSet(parsed.stat, ROLLS_5STAR, "5"); + const four = validSet(parsed.stat, ROLLS_4STAR, "4"); + if (!five || !four) continue; + const inFive = five.has(rounded); + const inFour = four.has(rounded); + if (inFive && !inFour) fitsFiveOnly++; + else if (inFour && !inFive) fitsFourOnly++; + } + if (fitsFiveOnly > 0) return 5; + if (fitsFourOnly > 0) return 4; + return 5; +} From ef65c3e6a018d6b74d8a4c6e5f79d9c480a7ab8b Mon Sep 17 00:00:00 2001 From: AzuTear Date: Tue, 7 Jul 2026 07:49:22 +0200 Subject: [PATCH 14/14] feat(scanner): validate elevated live automation --- dev-admin.cmd | 23 -- docs/ARCHITECTURE.md | 19 +- docs/AUTOMATION_LIVE_SCAN.md | 154 +++++++++++ docs/DECISIONS.md | 51 ++++ docs/PROJECT.md | 14 +- docs/scanner-rework-status.md | 42 ++- electron/bootstrap/ipcBootstrap.ts | 3 + electron/devControlServer.ts | 256 ++++++++++++++++++ electron/ipc/persistenceHandlers.ts | 7 + electron/main.ts | 121 ++++----- electron/preload.cjs | 1 + electron/preload.ts | 7 +- .../repositories/artifactStoreRepository.ts | 3 + package.json | 2 +- scripts/dev-admin-start.ps1 | 9 + scripts/dev-admin.ps1 | 33 +++ .../scan/components/DiagnosticsView.tsx | 40 +-- .../scan/hooks/scanViewReviewHelpers.ts | 3 +- .../scan/hooks/scanViewScanActions.ts | 9 +- .../scan/hooks/useScanCommandListener.ts | 13 +- src/features/scan/hooks/useScanViewActions.ts | 8 +- .../scan/hooks/useScanViewController.ts | 20 +- src/features/scan/types.ts | 1 + .../rendererBridgeRepositoryFactory.ts | 8 + .../rendererBridgeRepositoryTypes.ts | 5 +- src/lib/artifactOcrParser.test.ts | 21 ++ src/lib/artifactOcrParser.ts | 11 +- src/lib/artifactStore.ts | 3 +- src/lib/layoutProfile.test.ts | 21 +- src/lib/layoutProfile.ts | 78 +++--- src/lib/lockDetection.test.ts | 5 +- src/lib/lockDetection.ts | 12 +- src/lib/storedArtifactAdapter.test.ts | 9 +- src/lib/storedArtifactAdapter.ts | 2 +- src/services/assistantBridge.ts | 6 +- src/types/global.d.ts | 22 +- src/types/storage.ts | 1 + 37 files changed, 826 insertions(+), 217 deletions(-) delete mode 100644 dev-admin.cmd create mode 100644 docs/AUTOMATION_LIVE_SCAN.md create mode 100644 electron/devControlServer.ts create mode 100644 scripts/dev-admin.ps1 diff --git a/dev-admin.cmd b/dev-admin.cmd deleted file mode 100644 index 67dfd40..0000000 --- a/dev-admin.cmd +++ /dev/null @@ -1,23 +0,0 @@ -@echo off -rem Startet den Dev-Modus mit Administratorrechten (ein UAC-Prompt erscheint). -rem Noetig, weil Genshin erhoeht laeuft: Windows (UIPI) verwirft sonst alle -rem simulierten Maus-Eingaben an das Spiel - SendInput meldet trotzdem Erfolg. -rem Der Punkt hinter %~dp0 verhindert, dass der abschliessende Backslash das -rem schliessende Anfuehrungszeichen escaped. -rem -NoExit haelt das erhoehte (innere) Fenster offen, selbst wenn das Skript -rem einen Fehler wirft oder npm run dev sofort wieder beendet. -rem -rem Dieses AEUSSERE Fenster (das du beim Doppelklick oder ueber -rem "npm run dev:admin" siehst) schloss sich frueher sofort, sobald -rem Start-Process zurueckkehrte - auch wenn UAC abgelehnt wurde oder die -rem Elevation ganz fehlschlug, ohne dass davon irgendetwas sichtbar war. -rem try/catch + timeout zeigen jetzt den Fehler und halten das Fenster kurz offen. -"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -ExecutionPolicy Bypass -Command "$script='%~dp0scripts\dev-admin-start.ps1'; $project='%~dp0.'; try { Start-Process powershell -ArgumentList @('-NoProfile','-ExecutionPolicy','Bypass','-NoExit','-File',$script,'-ProjectRoot',$project) -Verb RunAs -ErrorAction Stop; Write-Host ''; Write-Host 'UAC-Abfrage gestartet. Bitte bestaetigen - danach oeffnet sich ein neues Administrator-Fenster mit npm run dev.' -ForegroundColor Green } catch { Write-Host ''; Write-Host 'Admin-Start fehlgeschlagen oder UAC-Abfrage abgelehnt:' -ForegroundColor Red; Write-Host $_.Exception.Message -ForegroundColor Red }" -echo. -echo Dieses Fenster kannst du jetzt schliessen (Taste druecken oder 15s warten). Das eigentliche Programm laeuft im neuen Administrator-Fenster. -rem timeout statt pause/choice: pause und choice warten unter umgeleiteter -rem Standardeingabe (z.B. beim Testen ueber ein Skript) fuer immer, weil sie -rem auf ein echtes Konsolen-Handle angewiesen sind. timeout erkennt eine -rem umgeleitete Eingabe explizit und bricht sofort ab statt zu haengen, waehrend -rem es bei einem echten Doppelklick normal 15s wartet oder bei Tastendruck endet. -timeout /t 15 >nul 2>&1 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 776da75..7910b29 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -49,7 +49,7 @@ flowchart LR | Module | Responsibility | | --- | --- | -| `electron/main.ts` | Window lifecycle, capture source listing, Smart Capture, OCR crop generation, overlay window IPC, persistent PowerShell input/capture helper, JSON artifact store | +| `electron/main.ts` | Window lifecycle, capture source listing, Smart Capture, OCR crop generation, overlay window IPC, input/capture sidecar orchestration, JSON artifact store, dev-only scanner control endpoints | | `electron/preload.cjs` | Safe renderer bridge exposed as `window.assistantApi` | | `src/lib/artifactStore.ts` | Pure signature/id/record helpers for the persistent artifact store | | `src/App.tsx` | Main app shell, scan view, triage view, build view, overlay preview | @@ -111,9 +111,24 @@ sequenceDiagram **Automatic grid scan** is user-triggered input automation limited to clicking detected inventory tiles and wheel-scrolling the inventory. Safety and reliability rules: -- All input goes through one persistent PowerShell helper process (`input-helper.ps1` in userData) that compiles the Win32 interop once and speaks JSON over stdin/stdout (ops: ping, focus, cursor, click, scroll, capture). Mouse movement is sent as iterated relative SendInput deltas (what a real mouse produces): Genshin tracks the cursor via raw input and snaps the OS cursor back to its own position every frame, so SetCursorPos/absolute moves silently stop working once the game owns the cursor. The helper verifies the cursor reached the target and refuses to click otherwise. +- All input goes through the helper service boundary (currently a C# sidecar with + fallback support behind the same JSON protocol). The helper owns focus, cursor + movement, click, scroll, guard-state polling, elevation detection, and GDI + capture. Mouse movement is sent as iterated relative input deltas instead of + relying on a single absolute cursor jump. The helper verifies the cursor + reached the target and refuses to click otherwise. +- `npm run dev:admin` is the validated dev path for automation when elevated + input is required. The elevated PowerShell startup is handled by + `scripts/dev-admin.ps1` and logged to `outputs/admin-start/admin-dev.log`. + The user must approve UAC manually; the app cannot approve the Secure Desktop + prompt itself. - Failsafe: before every click and scroll the renderer polls cursor position and ESC state. Holding ESC or moving the mouse away from the last automated position aborts the scan immediately; the Stop button also aborts. Only the `GetAsyncKeyState` held-down bit (0x8000) is used - the "pressed since last call" bit fires for stale ESC presses from normal Genshin menu navigation and caused false aborts. - SendInput's return value is checked: zero injected events (UIPI, e.g. elevated Genshin vs. non-elevated app) aborts with an explicit hint instead of silently clicking into nothing. +- Dev-only probes under `http://127.0.0.1:17317` are used for live validation: + `/automation/probe-click?index=N` tests one read-only tile selection, and + `/scanner/start?limit=N` starts an auto-scan with a temporary limit payload. + The live known-good result on 2026-07-07 is documented in + [AUTOMATION_LIVE_SCAN.md](AUTOMATION_LIVE_SCAN.md). - Click verification: after each click the parsed detail-panel signature should change. An unchanged signature is a soft miss (it can also mean two OCR-identical neighbor pieces, common among +0 artifacts), so it is retried once with a small offset, logged with the stuck artifact name, and then skipped - never fatal on its own. The scan aborts only when the first ~6 clicks of page 1 produce nothing new (diagnosis hint: elevated Genshin blocks SendInput via UIPI, or grid coordinates are wrong) or a later page yields zero new artifacts. - Scan stats separate clicked (click attempts), parsed (readable captures), stored (persisted), review (review samples), duplicates, and misses, so "scanned" cannot be mistaken for "successfully read". - Scrolling sends one wheel notch per grid row with the cursor anchored over the inventory (assumption: roughly one row per notch; overlap is absorbed by dedupe, and a page without new artifacts stops the scan). diff --git a/docs/AUTOMATION_LIVE_SCAN.md b/docs/AUTOMATION_LIVE_SCAN.md new file mode 100644 index 0000000..c63daf5 --- /dev/null +++ b/docs/AUTOMATION_LIVE_SCAN.md @@ -0,0 +1,154 @@ +# Automation Live Scan Runbook + +This document is the durable reference for automatic artifact scanning, mouse +movement, click input, elevation, and live validation status. + +## Current Known-Good State + +Validated live on 2026-07-07 with Genshin open in the artifact inventory at +1920x1080, English UI: + +- `npm run dev:admin` starts the app elevated after the user confirms UAC. +- Runtime status reported `isElevated: true`, `genshinFound: true`, and + `targetProcess: "GenshinImpact"`. +- The safe probe endpoint `/automation/probe-click?index=1` focused Genshin, + moved the cursor to the second visible inventory tile, clicked it, and changed + the artifact detail panel fingerprint. +- Probe result: `clicked: true`, `inputBlocked: false`, + `foregroundProcess: "GenshinImpact"`, and `changed: true`. +- A bounded live auto-scan via `/scanner/start?limit=2` completed with: + `clicked: 2`, `attempted: 2`, `verified: 2`, `parsed: 2`, `stored: 2`, + `review: 2`, `misses: 0`, `status: "done"`. + +This proves that the current elevated app plus helper path can deliver mouse +movement and click input to the focused Genshin client in this environment. + +## Elevation And UAC + +Use: + +```powershell +npm run dev:admin +``` + +The command runs `scripts/dev-admin.ps1`, which launches a new elevated +PowerShell window running `scripts/dev-admin-start.ps1`. The elevated start is +logged to: + +```text +outputs/admin-start/admin-dev.log +``` + +The user must confirm the Windows UAC prompt. The app cannot and must not click +the Secure Desktop UAC prompt for itself. After confirmation, the app can verify +its own runtime through the dev status endpoint. + +Useful checks: + +```powershell +Invoke-RestMethod http://127.0.0.1:17317/health +Invoke-RestMethod http://127.0.0.1:17317/scanner/status +``` + +Expected runtime facts before automatic scan: + +- `isElevated: true` +- `genshinFound: true` +- `targetProcess: "GenshinImpact"` +- hotkeys registered + +## Mouse And Click Validation + +Use the probe before broad auto-scan work: + +```powershell +Invoke-RestMethod "http://127.0.0.1:17317/automation/probe-click?index=1" | + ConvertTo-Json -Depth 12 +``` + +The probe performs one read-only inventory selection click. It does not delete, +feed, enhance, lock, unlock, spend, or modify game resources. + +Interpretation: + +- `click.ok: true`, `clicked: true`, `inputBlocked: false` means Windows did not + block SendInput/UIPI in the current configuration. +- `focused: true` and `foregroundProcess: "GenshinImpact"` means the click was + sent while Genshin was foreground. +- `changed: true` means the detail panel changed after the click. +- `changed: false` can be benign if the target tile was already selected or two + neighboring artifacts render identically; retry with another `index`, `row`, + or `col`. + +Examples: + +```powershell +# Second visible tile +Invoke-RestMethod "http://127.0.0.1:17317/automation/probe-click?index=1" + +# Specific grid cell +Invoke-RestMethod "http://127.0.0.1:17317/automation/probe-click?row=0&col=3" +``` + +## Bounded Live Auto-Scan + +For live validation, prefer a bounded scan first: + +```powershell +Invoke-RestMethod "http://127.0.0.1:17317/scanner/start?limit=2" +``` + +Then poll: + +```powershell +Invoke-RestMethod "http://127.0.0.1:17317/scanner/status" | + ConvertTo-Json -Depth 12 +``` + +The `/scanner/start?limit=N` endpoint sends a renderer command payload with a +temporary scan limit. It does not change the normal UI setting. The normal +hotkeys and buttons still use the UI's configured scan limit. + +## Anti-Cheat And Safety Boundary + +Do not describe the current implementation as bypassing anti-cheat. The app +does not read memory, hook the process, inject code, modify game files, inspect +packets, or interact with kernel drivers. It uses normal Windows screen capture, +focus, cursor movement, wheel, and click input. + +The practical finding is narrower: + +- A non-elevated app can be blocked by Windows integrity/UIPI when the target + process is elevated or protected. +- Running the app elevated fixed input delivery in the tested environment. +- Genshin's anti-cheat may still affect behavior on other machines, game modes, + overlays, or future versions. Re-run the probe before trusting broad scans. + +Never add automation that deletes, feeds, enhances, locks/unlocks, spends +resources, reads memory, hooks, injects, or modifies Genshin. + +## Live Layout Facts + +The current 16:9 layout profile is calibrated from a 1920x1080 English +artifact-inventory capture: + +- detail rect approximately `x=1308`, `y=120`, `width=492`, `height=838` +- inventory grid: `8 x 5` +- first tile center: `x=179`, `y=254`, `row=0`, `col=0` +- second tile center: `x=325`, `y=254`, `row=0`, `col=1` +- inventory count crop successfully read `2059/2400` in the live session + +The profile is resolution-scaled for 16:9. Off-profile setups should be treated +as higher risk and validated with Smart Capture plus the probe. + +## Validation Checklist + +Before marking an automation change done: + +1. Run `npm run lint`. +2. Run `npx tsc -p tsconfig.electron.json` when Electron/preload/main changed. +3. Run `npm test`. +4. Run `npm run build`. +5. If Genshin is available, run `/automation/probe-click?index=1`. +6. For scan-loop changes, run `/scanner/start?limit=2` before any broader scan. +7. Record new live findings in this file and in `docs/scanner-rework-status.md`. diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index c0c41e1..9980791 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -15,6 +15,7 @@ This document contains Architecture Decision Records. | ADR-007 | Measure OCR accuracy with a labeled eval harness before reworking the scanner | Accepted | 2026-07-05 | | ADR-008 | Replace the PowerShell input/capture helper with a C# sidecar | Accepted | 2026-07-05 | | ADR-009 | Resolution-anchored layout profiles and OCR preprocessing over color detection | Accepted | 2026-07-05 | +| ADR-010 | Elevated dev runner and bounded live automation probes | Accepted | 2026-07-07 | ## ADR-001: Build A Local Electron App First @@ -235,3 +236,53 @@ validated against the ADR-007 eval harness. insufficient. - Non-16:9 or non-borderless setups are explicitly unsupported for the auto scanner; the app should detect and warn rather than silently misread. + +## ADR-010: Elevated Dev Runner And Bounded Live Automation Probes + +### Status + +Accepted + +### Context + +Automatic grid scanning needs read-only mouse movement, click, and wheel input +to reach the focused Genshin window. A lower-integrity app can fail to deliver +input to an elevated or protected target because of Windows UIPI/integrity +boundaries. During live testing, `npm run dev:admin` originally printed that a +new Administrator window was started, but the elevated PowerShell received no +arguments, so the intended dev process did not reliably start. + +The project also needed a smaller live validation path than a full inventory +scan. A full scan is too risky as the first proof of input delivery because it +can click many tiles before a bad coordinate, focus issue, or blocked input is +understood. + +### Decision + +Keep automatic scan input automation read-only and require an elevated runtime +when Windows reports that automation would otherwise be blocked. Replace the +old `dev-admin.cmd` entry with `scripts/dev-admin.ps1`, quote the elevated +PowerShell arguments explicitly, and log elevated startup to +`outputs/admin-start/admin-dev.log`. + +Add dev-only HTTP checks: + +- `/automation/probe-click?index=N` or `?row=R&col=C` performs one safe + inventory selection click and verifies whether the detail panel changed. +- `/scanner/start?limit=N` sends a temporary scan-limit payload to the renderer, + so live auto-scan validation can start with two items instead of the UI + default. + +Document the workflow in [AUTOMATION_LIVE_SCAN.md](AUTOMATION_LIVE_SCAN.md). + +### Consequences + +- The user still has to approve Windows UAC manually; the app must not try to + click the Secure Desktop prompt. +- We can distinguish input delivery from OCR/parser quality with a one-click + probe before running any broader scan. +- Live validation now has a low-risk path: check elevation and Genshin + detection, run a single probe click, then run a bounded `limit=2` scan. +- The implementation remains inside the allowed safety boundary: no memory + reads, hooks, injection, game-file modification, deleting, feeding, enhancing, + locking/unlocking, or spending resources. diff --git a/docs/PROJECT.md b/docs/PROJECT.md index ee2824c..1206ddf 100644 --- a/docs/PROJECT.md +++ b/docs/PROJECT.md @@ -72,7 +72,7 @@ The app is not intended to replace deep min-max tools. It prioritizes time savin | Styling | CSS with dark purple glassmorphism system | Premium fintech-inspired visual direction | | OCR | Tesseract.js prototype plus deterministic normalization/derivation | OCR alone is not trusted as the decision source | | Capture | Electron desktopCapturer plus Windows GDI Smart Capture | GDI path is used for Genshin Smart Capture reliability | -| Input automation | PowerShell sidecar prototype now, native sidecar planned | Current sidecar is good for proving behavior, not the final production path | +| Input automation | C# sidecar with elevated dev runner when needed | Live-validated for read-only inventory selection clicks; see `docs/AUTOMATION_LIVE_SCAN.md` | | Tests | Vitest + TypeScript checks | Current validation baseline; regression samples must expand | | Packaging | electron-builder | Configured in `package.json` | @@ -95,11 +95,15 @@ The app is not intended to replace deep min-max tools. It prioritizes time savin - The parser already uses known sets, pieces, slots, stat aliases, set aliases, character aliases, and derived slot/set mapping. - Review samples, learned replacements, parser notes, and stored artifacts already persist locally. - The auto-scan loop is no longer a naive click spammer: it has preflight, verification, miss handling, page fingerprinting, and stop conditions. +- Elevated live automation is validated in the current dev environment: + `/automation/probe-click?index=1` changed the selected artifact and + `/scanner/start?limit=2` completed with 2/2 verified reads and 0 misses. ### What is still structurally weak - The scan experience is still partly orchestrated from `src/App.tsx`, which makes behavior changes harder than they should be. -- The current PowerShell input sidecar is serviceable for experimentation but not a strong production base for long-running, low-jitter auto-scan. +- Broader scan soak testing still needs to increase the live limit gradually and + validate scroll/page transitions beyond the first visible row. - OCR quality is still inconsistent enough that some fields are recovered by fallback and derivation more often than they should be. - Learned fixes currently focus on text replacements; they do not yet update crop offsets, UI profile variants, or scanner targeting rules in a structured way. - The scan page is cleaner than before, but it still exposes too much operator/debug state in the main flow. @@ -196,7 +200,7 @@ Outcome: - Auto-scan never starts on a session that cannot prove one successful detail-card change. Status: -- Planned +- First live path validated; broader soak testing still needed ### Phase 5 - Learning loop that actually compounds @@ -228,7 +232,7 @@ Status: 1. Finish scan-page cleanup so the main operator view is no longer noisy. 2. Tighten the game data generator and parser contract, then backfill regression tests from real bad samples. 3. Continue moving auto-scan behavior out of `App.tsx` and into isolated scanner modules. -4. Replace or wrap the current PowerShell sidecar with a more stable long-lived automation process. +4. Soak-test the elevated C# helper automation path with gradually larger scan limits and page scroll transitions. 5. Extend the learning system from text-only fixes into crop/UI profile tuning. 6. Resume recommendation work only when scan accuracy is consistently trustworthy. @@ -236,7 +240,7 @@ Status: | Question | Status | | --- | --- | -| Should the production input sidecar be Rust/C++ first, or a transitional Node native addon, for the next iteration? | Open | +| Is the current C# helper sufficient for production packaging, or does a later Rust/C++ sidecar still materially reduce latency or packaging risk? | Open | | When should UI-profile learning be allowed to change crop geometry automatically versus requiring review approval? | Open | | What scan-quality threshold is high enough before recommendations should be considered user-facing again? | Open | | Which Genshin UI languages should be supported after English once the scanner contract is stable? | Open | diff --git a/docs/scanner-rework-status.md b/docs/scanner-rework-status.md index fbe51b4..65fe1df 100644 --- a/docs/scanner-rework-status.md +++ b/docs/scanner-rework-status.md @@ -1,7 +1,9 @@ # Scanner rework status -Progress on the approved scanner/OCR rework. See ADR-007/008/009 in -[DECISIONS.md](DECISIONS.md) for the decisions behind these. +Progress on the approved scanner/OCR rework. See ADR-007/008/009/010 in +[DECISIONS.md](DECISIONS.md) for the decisions behind these. For the current +live automation runbook, see +[AUTOMATION_LIVE_SCAN.md](AUTOMATION_LIVE_SCAN.md). ## Done (implemented, unit-tested, build green) @@ -12,34 +14,44 @@ Progress on the approved scanner/OCR rework. See ADR-007/008/009 in fallback. Verified end-to-end (spawn, runtime, base64 capture). - **Layout profiles + OCR preprocessing** — `src/lib/layoutProfile.ts` (pure geometry, 16:9 detection), `src/lib/ocrPreprocess.ts` (grayscale + Otsu - binarize). main.ts crops via the profile and OCRs an upscaled + binarized copy. + binarize). main.ts now uses calibrated 16:9 detail/count/grid coordinates + first and OCRs an upscaled + binarized copy. - **Card-ready gating** — `src/lib/cardReadyGate.ts` replaces the fixed 280 ms settle with change+stability polling; robust to animation. - **GOOD interop** — `src/lib/goodInterop.ts` (export + best-effort import for - scanned records). + scanned records), Electron file-picker import/export, and store merge. - **Rescan-merge** — `src/lib/artifactMerge.ts` collapses leveled re-scan duplicates by a level-independent identity. - **Data staleness warning** — `src/lib/dataPackageStatus.ts`, surfaced in the Scanner Diagnose data-package line. -- **Lock detection (experimental)** — `src/lib/lockDetection.ts`, pure heuristic, - not yet wired into capture. +- **Lock detection (experimental)** — `src/lib/lockDetection.ts`, wired into + live capture as a read-only `locked` flag and persisted with scanned records. +- **Elevated live automation path** — `npm run dev:admin` now starts through + `scripts/dev-admin.ps1` and logs to `outputs/admin-start/admin-dev.log`. + Live status confirmed `isElevated: true`, `genshinFound: true`, and + `targetProcess: "GenshinImpact"`. +- **Read-only click probe** — `/automation/probe-click?index=1` verified that + the app can focus Genshin, move to a visible inventory tile, click it, and + observe a changed detail panel fingerprint (`clicked: true`, + `inputBlocked: false`, `changed: true`). +- **Bounded auto-scan validation** — `/scanner/start?limit=2` completed live + with 2 clicks, 2 verified detail views, 2 parsed artifacts, 2 stored records, + 2 review samples, and 0 misses. ## Remaining — needs the live environment or a UI pass These cannot be finished/validated without Genshin running at the user's resolution or without UI work best tested live: -1. **Calibrate IK-style fixed crop coordinates** (ADR-009). The layout module is - the structure; the exact per-field fractions still come from a - colour-detected/fallback detail rect. A reference 16:9 screenshot of the - artifact screen lets us pin exact client-relative crop coordinates. -2. **Validate/tune OCR preprocessing** on real captures — confirm invert + +1. **Validate/tune OCR preprocessing** on more real captures — confirm invert + threshold + upscale factor help (not hurt) actual Tesseract reads. The text-level eval harness cannot measure image preprocessing. -3. **Wire GOOD import** — file-picker IPC + merge imported records into the store - (the conversion engine is done and tested). -4. **Wire live lock detection** — calibrate crop position/threshold against a - reference screenshot, then populate a `locked` flag during capture. +2. **Validate locked=true** against a known locked artifact — unlocked/grey lock + was live-checked; a gold locked icon still needs a positive sample. + +3. **Broader scan soak test** — after the bounded two-item live scan passed, + the next automation validation should increase the limit gradually and watch + for repeated pages, scroll behavior, duplicate handling, and OCR review rate. ## Grow the eval corpus diff --git a/electron/bootstrap/ipcBootstrap.ts b/electron/bootstrap/ipcBootstrap.ts index 511dc16..3e290e2 100644 --- a/electron/bootstrap/ipcBootstrap.ts +++ b/electron/bootstrap/ipcBootstrap.ts @@ -17,6 +17,7 @@ import type { SaveResultWithPath, SaveSnapshotResult, GoodDatabase, + GoodImportFileResult, ScannerStatusPayload, } from "../../src/types/global.js"; import type { @@ -51,6 +52,7 @@ interface PersistenceHandlersDependencies { loadScannerLearningRules: () => Promise; writeScannerLearningRules: (rules: ScannerLearningRulePayload) => Promise; exportGood: (payload: GoodDatabase) => Promise; + importGoodFile: () => Promise; } interface CaptureHandlersDependencies { @@ -86,6 +88,7 @@ export function registerIpcHandlers(dependencies: IpcBootstrapDependencies) { loadScannerLearningRules: dependencies.loadScannerLearningRules, writeScannerLearningRules: dependencies.writeScannerLearningRules, exportGood: dependencies.exportGood, + importGoodFile: dependencies.importGoodFile, }); registerCaptureHandlers({ diff --git a/electron/devControlServer.ts b/electron/devControlServer.ts new file mode 100644 index 0000000..bd9cebe --- /dev/null +++ b/electron/devControlServer.ts @@ -0,0 +1,256 @@ +import fs from "node:fs/promises"; +import http, { type Server } from "node:http"; +import path from "node:path"; +import type { + CaptureOptions, + CaptureResult, + CaptureSourceInfo, + ClickResult, + ReviewSampleListResult, + ScannerCommand, + ScannerStatusPayload, +} from "../src/types/global.js"; + +interface DevControlServerDependencies { + registeredHotkeys: Record; + hasMainWindow: () => boolean; + sendScannerCommand: (command: ScannerCommand | "probe-click") => void; + clickScreen: (x: number, y: number) => Promise; + scannerStatus: () => ScannerStatusPayload; + loadReviewSamples: (limit?: number) => Promise; + listCaptureSources: () => Promise; + captureSource: ( + id: string, + delayMs?: number, + focusGenshin?: boolean, + options?: CaptureOptions, + ) => Promise; +} + +function writeDevJson(res: http.ServerResponse, statusCode: number, payload: unknown) { + res.writeHead(statusCode, { + "content-type": "application/json; charset=utf-8", + "cache-control": "no-store", + }); + res.end(JSON.stringify(payload)); +} + +function dataUrlBase64(dataUrl: string) { + return dataUrl.replace(/^data:image\/png;base64,/, ""); +} + +function devCaptureOutputDir() { + return path.join(process.cwd(), "outputs", "live-capture"); +} + +function safeDebugFilePart(value: string) { + return value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture"; +} + +function dataUrlFingerprint(dataUrl: string | undefined) { + if (!dataUrl) return ""; + let hash = 2166136261; + const stride = Math.max(1, Math.floor(dataUrl.length / 4096)); + for (let index = 0; index < dataUrl.length; index += stride) { + hash ^= dataUrl.charCodeAt(index); + hash = Math.imul(hash, 16777619); + } + return `${dataUrl.length.toString(16)}:${(hash >>> 0).toString(16)}`; +} + +async function writeDevCaptureImage(filePath: string, dataUrl: string | undefined) { + if (!dataUrl) return null; + await fs.writeFile(filePath, Buffer.from(dataUrlBase64(dataUrl), "base64")); + return filePath; +} + +async function writeDevCaptureSnapshot(capture: CaptureResult) { + const outputDir = devCaptureOutputDir(); + await fs.mkdir(outputDir, { recursive: true }); + const stamp = new Date().toISOString().replace(/[\\/:]/g, "-").replace(/\..+?$/, ""); + const prefix = safeDebugFilePart(`${stamp}-${capture.name}`); + const files = { + full: await writeDevCaptureImage(path.join(outputDir, `${prefix}-full.png`), capture.dataUrl), + detail: await writeDevCaptureImage(path.join(outputDir, `${prefix}-detail.png`), capture.detailDataUrl), + inventory: await writeDevCaptureImage(path.join(outputDir, `${prefix}-inventory.png`), capture.inventoryDataUrl), + crops: [] as Array<{ id: string; label: string; path: string; rect: { x: number; y: number; width: number; height: number } }>, + }; + + for (const crop of capture.crops ?? []) { + const cropPath = path.join(outputDir, `${prefix}-${safeDebugFilePart(crop.id)}.png`); + const written = await writeDevCaptureImage(cropPath, crop.dataUrl); + if (written) files.crops.push({ id: crop.id, label: crop.label, path: written, rect: crop.rect }); + } + + const summary = { + id: capture.id, + name: capture.name, + width: capture.width, + height: capture.height, + capturedAt: capture.capturedAt, + captureTarget: capture.captureTarget, + ocrSkipped: capture.ocrSkipped, + ocrTimedOut: capture.ocrTimedOut, + layout: capture.layout, + inventoryGrid: capture.inventoryGrid + ? { + rows: capture.inventoryGrid.rows, + cols: capture.inventoryGrid.cols, + confidence: capture.inventoryGrid.confidence, + source: capture.inventoryGrid.source, + firstCenter: capture.inventoryGrid.centers[0] ?? null, + lastCenter: capture.inventoryGrid.centers.at(-1) ?? null, + } + : null, + inventoryCount: capture.inventoryCount ?? null, + locked: capture.locked, + crops: (capture.crops ?? []).map((crop) => ({ id: crop.id, label: crop.label, rect: crop.rect })), + ocr: capture.ocr ?? [], + files, + }; + const summaryPath = path.join(outputDir, `${prefix}-summary.json`); + await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2), "utf8"); + return { ...summary, summaryPath }; +} + +function wait(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function findGenshinSource(sources: CaptureSourceInfo[], sourceId: string | null) { + return sourceId + ? sources.find((entry) => entry.id === sourceId) + : sources.find((entry) => entry.isGenshinCandidate); +} + +function sourceListForError(sources: CaptureSourceInfo[]) { + return sources.map(({ id, name, isGenshinCandidate }) => ({ id, name, isGenshinCandidate })); +} + +export function createDevControlServer(deps: DevControlServerDependencies): Server { + const server = http.createServer((req, res) => { + if (req.socket.remoteAddress && !["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress)) { + writeDevJson(res, 403, { ok: false, error: "local only" }); + return; + } + + const url = new URL(req.url ?? "/", "http://127.0.0.1"); + if (url.pathname === "/health") { + writeDevJson(res, 200, { ok: true, hotkeys: deps.registeredHotkeys, hasWindow: deps.hasMainWindow() }); + return; + } + if (url.pathname === "/scanner/start") { + const limit = Number(url.searchParams.get("limit") ?? Number.NaN); + const command: ScannerCommand = Number.isFinite(limit) && limit > 0 + ? { type: "start-auto", scanLimit: limit } + : "start-auto"; + deps.sendScannerCommand(command); + writeDevJson(res, 200, { ok: true, command }); + return; + } + if (url.pathname === "/scanner/stop") { + deps.sendScannerCommand("stop"); + writeDevJson(res, 200, { ok: true, command: "stop" }); + return; + } + if (url.pathname === "/scanner/probe") { + deps.sendScannerCommand("probe-click"); + writeDevJson(res, 200, { ok: true, command: "probe-click" }); + return; + } + if (url.pathname === "/automation/click") { + const x = Number(url.searchParams.get("x")); + const y = Number(url.searchParams.get("y")); + if (!Number.isFinite(x) || !Number.isFinite(y)) { + writeDevJson(res, 400, { ok: false, error: "x and y query params are required" }); + return; + } + deps.clickScreen(Math.round(x), Math.round(y)) + .then((payload: unknown) => writeDevJson(res, 200, { ok: true, payload })) + .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) })); + return; + } + if (url.pathname === "/scanner/status") { + writeDevJson(res, 200, { ok: true, status: deps.scannerStatus() }); + return; + } + if (url.pathname === "/review/samples") { + deps.loadReviewSamples(Number(url.searchParams.get("limit") ?? 20)) + .then((payload: unknown) => writeDevJson(res, 200, payload)) + .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) })); + return; + } + if (url.pathname === "/capture/smart") { + const sourceId = url.searchParams.get("sourceId"); + const focus = url.searchParams.get("focus") !== "0"; + const skipOcr = url.searchParams.get("skipOcr") === "1"; + deps.listCaptureSources() + .then(async (sources) => { + const source = findGenshinSource(sources, sourceId); + if (!source) { + writeDevJson(res, 404, { ok: false, error: "No Genshin capture source found.", sources: sourceListForError(sources) }); + return; + } + const capture = await deps.captureSource(source.id, 250, focus, { skipOcr }); + const summary = await writeDevCaptureSnapshot(capture); + writeDevJson(res, 200, { ok: true, source: { id: source.id, name: source.name }, summary }); + }) + .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) })); + return; + } + if (url.pathname === "/automation/probe-click") { + const sourceId = url.searchParams.get("sourceId"); + const requestedIndex = Number(url.searchParams.get("index") ?? "1"); + const requestedRow = Number(url.searchParams.get("row") ?? Number.NaN); + const requestedCol = Number(url.searchParams.get("col") ?? Number.NaN); + deps.listCaptureSources() + .then(async (sources) => { + const source = findGenshinSource(sources, sourceId); + if (!source) { + writeDevJson(res, 404, { ok: false, error: "No Genshin capture source found.", sources: sourceListForError(sources) }); + return; + } + + const before = await deps.captureSource(source.id, 150, true, { skipOcr: true }); + const centers = before.inventoryGrid?.centers ?? []; + const target = Number.isFinite(requestedRow) && Number.isFinite(requestedCol) + ? centers.find((center) => center.row === requestedRow && center.col === requestedCol) + : centers[Math.max(0, Math.min(centers.length - 1, Number.isFinite(requestedIndex) ? requestedIndex : 1))]; + if (!target) { + writeDevJson(res, 409, { ok: false, error: "No inventory grid target available.", grid: before.inventoryGrid ?? null }); + return; + } + + const beforeFingerprint = dataUrlFingerprint(before.detailDataUrl); + const click = await deps.clickScreen(target.x, target.y); + await wait(650); + const after = await deps.captureSource(source.id, 0, true, { skipOcr: true }); + const afterFingerprint = dataUrlFingerprint(after.detailDataUrl); + const changed = Boolean(beforeFingerprint && afterFingerprint && beforeFingerprint !== afterFingerprint); + writeDevJson(res, 200, { + ok: Boolean(click.ok && click.clicked && changed), + changed, + target, + click, + before: { + captureTarget: before.captureTarget, + grid: before.inventoryGrid ? { rows: before.inventoryGrid.rows, cols: before.inventoryGrid.cols, source: before.inventoryGrid.source, confidence: before.inventoryGrid.confidence } : null, + detailFingerprint: beforeFingerprint, + }, + after: { + captureTarget: after.captureTarget, + grid: after.inventoryGrid ? { rows: after.inventoryGrid.rows, cols: after.inventoryGrid.cols, source: after.inventoryGrid.source, confidence: after.inventoryGrid.confidence } : null, + detailFingerprint: afterFingerprint, + }, + }); + }) + .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) })); + return; + } + + writeDevJson(res, 404, { ok: false, error: "unknown endpoint" }); + }); + + server.listen(17317, "127.0.0.1"); + return server; +} diff --git a/electron/ipc/persistenceHandlers.ts b/electron/ipc/persistenceHandlers.ts index 1cb48dd..22b1000 100644 --- a/electron/ipc/persistenceHandlers.ts +++ b/electron/ipc/persistenceHandlers.ts @@ -13,6 +13,7 @@ import type { SaveScannerLearningRulesResult, ScannerLearningRulePayload, SaveResultWithPath, + GoodImportFileResult, } from "../../src/types/global.js"; import type { StoredArtifactRecord } from "../../src/types/storage.js"; @@ -28,6 +29,7 @@ interface PersistenceDependencies { loadScannerLearningRules: () => Promise; writeScannerLearningRules: (rules: ScannerLearningRulePayload) => Promise; exportGood: (payload: GoodDatabase) => Promise; + importGoodFile: () => Promise; } export function registerPersistenceHandlers({ @@ -39,6 +41,7 @@ export function registerPersistenceHandlers({ loadScannerLearningRules, writeScannerLearningRules, exportGood, + importGoodFile, }: PersistenceDependencies) { ipcMain.handle("review:saveSample", async (_event, sample: ReviewSamplePayload) => { try { @@ -81,4 +84,8 @@ export function registerPersistenceHandlers({ ipcMain.handle("good:export", async (_event, payload: GoodDatabase) => { return exportGood(payload); }); + + ipcMain.handle("good:importFile", async () => { + return importGoodFile(); + }); } diff --git a/electron/main.ts b/electron/main.ts index e2391e6..e056258 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -1,19 +1,22 @@ -import { app, BrowserWindow, Menu, desktopCapturer, globalShortcut, nativeImage, screen, type NativeImage } from "electron"; +import { app, BrowserWindow, Menu, desktopCapturer, dialog, globalShortcut, nativeImage, screen, type NativeImage } from "electron"; import fs from "node:fs/promises"; import { existsSync } from "node:fs"; -import http, { type Server } from "node:http"; +import type { Server } from "node:http"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { createWorker } from "tesseract.js"; import { createInputHelperService, type InputHelperService } from "./services/inputHelper.js"; import { createRepositoryContext, type RepositoryContext } from "./bootstrap/repositoryContext.js"; import { registerIpcHandlers } from "./bootstrap/ipcBootstrap.js"; +import { createDevControlServer } from "./devControlServer.js"; import type { AppSnapshot } from "../src/types/domain.js"; import type { CaptureOptions, CaptureResult, GoodDatabase, + GoodImportFileResult, SaveResultWithPath, + ScannerCommand, ScannerLearningRulePayload, ScannerStatusPayload, } from "../src/types/global.js"; @@ -33,6 +36,7 @@ import { profileDetailRect, } from "../src/lib/layoutProfile.js"; import { binarizeForOcr } from "../src/lib/ocrPreprocess.js"; +import { detectLockState, lockIconCropRect } from "../src/lib/lockDetection.js"; // Chromium's renderer sandbox can refuse to fully initialize (or silently // crash the GPU/renderer process) when the hosting process runs with a full @@ -416,7 +420,7 @@ function focusMainWindow() { return { ok: true }; } -function sendScannerCommand(command: "start-auto" | "stop" | "probe-click") { +function sendScannerCommand(command: ScannerCommand | "probe-click") { if (!mainWindow || mainWindow.isDestroyed()) return; mainWindow.webContents.send("scanner:command", command); } @@ -431,71 +435,18 @@ function registerScannerHotkeys() { }; } -function writeDevJson(res: http.ServerResponse, statusCode: number, payload: unknown) { - res.writeHead(statusCode, { - "content-type": "application/json; charset=utf-8", - "cache-control": "no-store", - }); - res.end(JSON.stringify(payload)); -} - function startDevControlServer() { if (!isDev || devControlServer) return; - - devControlServer = http.createServer((req, res) => { - if (req.socket.remoteAddress && !["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress)) { - writeDevJson(res, 403, { ok: false, error: "local only" }); - return; - } - - const url = new URL(req.url ?? "/", "http://127.0.0.1"); - if (url.pathname === "/health") { - writeDevJson(res, 200, { ok: true, hotkeys: registeredHotkeys, hasWindow: Boolean(mainWindow && !mainWindow.isDestroyed()) }); - return; - } - if (url.pathname === "/scanner/start") { - sendScannerCommand("start-auto"); - writeDevJson(res, 200, { ok: true, command: "start-auto" }); - return; - } - if (url.pathname === "/scanner/stop") { - sendScannerCommand("stop"); - writeDevJson(res, 200, { ok: true, command: "stop" }); - return; - } - if (url.pathname === "/scanner/probe") { - sendScannerCommand("probe-click"); - writeDevJson(res, 200, { ok: true, command: "probe-click" }); - return; - } - if (url.pathname === "/automation/click") { - const x = Number(url.searchParams.get("x")); - const y = Number(url.searchParams.get("y")); - if (!Number.isFinite(x) || !Number.isFinite(y)) { - writeDevJson(res, 400, { ok: false, error: "x and y query params are required" }); - return; - } - getInputHelperService() - .clickScreen(Math.round(x), Math.round(y)) - .then((payload: unknown) => writeDevJson(res, 200, { ok: true, payload })) - .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) })); - return; - } - if (url.pathname === "/scanner/status") { - writeDevJson(res, 200, { ok: true, status: scannerDevStatus }); - return; - } - if (url.pathname === "/review/samples") { - loadReviewSamples(Number(url.searchParams.get("limit") ?? 20)) - .then((payload: unknown) => writeDevJson(res, 200, payload)) - .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) })); - return; - } - - writeDevJson(res, 404, { ok: false, error: "unknown endpoint" }); + devControlServer = createDevControlServer({ + registeredHotkeys, + hasMainWindow: () => Boolean(mainWindow && !mainWindow.isDestroyed()), + sendScannerCommand, + clickScreen: clickScreenCommand, + scannerStatus: () => scannerDevStatus, + loadReviewSamples, + listCaptureSources, + captureSource, }); - - devControlServer.listen(17317, "127.0.0.1"); } function createOverlayWindow() { @@ -802,6 +753,10 @@ function createCrops( } function inferDetailRect(bitmap: Buffer, imageSize: { width: number; height: number }) { + if (isSixteenNine(imageSize)) { + return profileDetailRect(imageSize); + } + const { width, height } = imageSize; const sampleStrideX = width > 2200 ? 4 : 3; const sampleStrideY = height > 1400 ? 4 : 3; @@ -869,6 +824,12 @@ async function buildCaptureResult( const detailRect = inferDetailRect(bitmap, size); const inventoryRect = inferInventoryRect(size, detailRect); const crops = createCrops(sourceImage, size, detailRect, inventoryRect); + const lockRect = clampCaptureRect(lockIconCropRect(detailRect, size), size); + const lockImage = sourceImage.crop(lockRect); + const lockSize = lockImage.getSize(); + const locked = lockSize.width > 0 && lockSize.height > 0 + ? detectLockState({ data: lockImage.getBitmap(), width: lockSize.width, height: lockSize.height }) + : undefined; const croppedPayload = crops.map((crop) => ({ id: crop.id, label: crop.label, @@ -905,6 +866,7 @@ async function buildCaptureResult( })), inventoryGrid: inferInventoryGrid(size, detailRect), inventoryCount: count, + locked, layout: { aspect: aspectRatioLabel(size), isSixteenNine: isSixteenNine(size), @@ -965,6 +927,34 @@ async function exportGood(payload: GoodDatabase): Promise { } } +async function importGoodFile(): Promise { + const dialogOptions = { + title: "GOOD-Datei importieren", + properties: ["openFile"], + filters: [{ name: "GOOD JSON", extensions: ["json"] }], + } satisfies Electron.OpenDialogOptions; + const dialogResult = mainWindow && !mainWindow.isDestroyed() + ? await dialog.showOpenDialog(mainWindow, dialogOptions) + : await dialog.showOpenDialog(dialogOptions); + + if (dialogResult.canceled || dialogResult.filePaths.length === 0) { + return { ok: false, canceled: true, path: "" }; + } + + const filePath = dialogResult.filePaths[0]; + try { + const text = await fs.readFile(filePath, "utf8"); + return { ok: true, canceled: false, path: filePath, database: JSON.parse(text) }; + } catch (error) { + return { + ok: false, + canceled: false, + path: filePath, + error: error instanceof Error ? error.message : String(error), + }; + } +} + function initializeAppLifecycle() { app.whenReady().then(() => { const userDataPath = app.getPath("userData"); @@ -993,6 +983,7 @@ function initializeAppLifecycle() { loadScannerLearningRules: () => loadScannerLearningRules(), writeScannerLearningRules: (rules: ScannerLearningRulePayload) => writeScannerLearningRules(rules), exportGood: (exportPayload: GoodDatabase) => exportGood(exportPayload), + importGoodFile: () => importGoodFile(), listSources: () => listCaptureSources(), captureSource: ( id: string, diff --git a/electron/preload.cjs b/electron/preload.cjs index d120be0..304f8f9 100644 --- a/electron/preload.cjs +++ b/electron/preload.cjs @@ -20,6 +20,7 @@ contextBridge.exposeInMainWorld("assistantApi", { loadArtifacts: () => ipcRenderer.invoke("artifacts:load"), saveArtifacts: (records) => ipcRenderer.invoke("artifacts:saveMany", records), exportGood: (payload) => ipcRenderer.invoke("good:export", payload), + importGoodFile: () => ipcRenderer.invoke("good:importFile"), publishScannerStatus: (status) => ipcRenderer.invoke("scanner:publishStatus", status), showOverlay: () => ipcRenderer.invoke("overlay:show"), hideOverlay: () => ipcRenderer.invoke("overlay:hide"), diff --git a/electron/preload.ts b/electron/preload.ts index d3fc2aa..12ae90e 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,5 +1,5 @@ import { contextBridge, ipcRenderer } from "electron"; -import type { CaptureOptions, GoodDatabase, ReviewSamplePayload, ScannerStatusPayload, ScannerLearningRulePayload } from "../src/types/global.js"; +import type { CaptureOptions, GoodDatabase, ReviewSamplePayload, ScannerCommand, ScannerStatusPayload, ScannerLearningRulePayload } from "../src/types/global.js"; import type { StoredArtifactRecord } from "../src/types/storage.js"; import type { AppSnapshot } from "../src/types/domain.js"; @@ -23,11 +23,12 @@ contextBridge.exposeInMainWorld("assistantApi", { loadArtifacts: () => ipcRenderer.invoke("artifacts:load"), saveArtifacts: (records: StoredArtifactRecord[]) => ipcRenderer.invoke("artifacts:saveMany", records), exportGood: (payload: GoodDatabase) => ipcRenderer.invoke("good:export", payload), + importGoodFile: () => ipcRenderer.invoke("good:importFile"), publishScannerStatus: (status: ScannerStatusPayload) => ipcRenderer.invoke("scanner:publishStatus", status), showOverlay: () => ipcRenderer.invoke("overlay:show"), hideOverlay: () => ipcRenderer.invoke("overlay:hide"), - onScannerCommand: (callback: (command: "start-auto" | "stop") => void) => { - const listener = (_event: Electron.IpcRendererEvent, command: "start-auto" | "stop") => callback(command); + onScannerCommand: (callback: (command: ScannerCommand) => void) => { + const listener = (_event: Electron.IpcRendererEvent, command: ScannerCommand) => callback(command); ipcRenderer.on("scanner:command", listener); return () => ipcRenderer.removeListener("scanner:command", listener); }, diff --git a/electron/repositories/artifactStoreRepository.ts b/electron/repositories/artifactStoreRepository.ts index bf0797d..3d7a1e5 100644 --- a/electron/repositories/artifactStoreRepository.ts +++ b/electron/repositories/artifactStoreRepository.ts @@ -54,6 +54,7 @@ export class JsonArtifactStoreRepository implements ArtifactStoreRepositoryPort lastSeenAt: now, timesSeen: (existing.timesSeen ?? 1) + 1, confidence: Math.max(existing.confidence ?? 0, record.confidence ?? 0), + locked: typeof record.locked === "boolean" ? record.locked : existing.locked, // A later confident scan clears the review flag; an uncertain rescan // must not downgrade an already confirmed artifact. needsReview: Boolean(existing.needsReview) && Boolean(record.needsReview), @@ -136,6 +137,7 @@ function normalizeStoredArtifactRecordForLoad(record: StoredArtifactRecord) { ...record, timesSeen: reviewOnly ? 1 : normalizedTimesSeen, firstSeenAt: record.firstSeenAt ?? record.lastSeenAt, + locked: typeof record.locked === "boolean" ? record.locked : undefined, }; } @@ -183,6 +185,7 @@ function mergeArtifactRecords(existing: StoredArtifactRecord, incoming: StoredAr substats: [...(preferredSubstats ?? [])], equipped: preferred.equipped && preferred.equipped !== "Not detected" ? preferred.equipped : secondary.equipped, confidence: Math.max(existing.confidence ?? 0, incoming.confidence ?? 0), + locked: typeof incoming.locked === "boolean" ? incoming.locked : existing.locked, needsReview: Boolean(existing.needsReview) && Boolean(incoming.needsReview), source: resolveStoredArtifactSource(existing.source, incoming.source), firstSeenAt: existing.firstSeenAt ?? now, diff --git a/package.json b/package.json index 612f525..b91295f 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "predev": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\kill-stale-instances.ps1 && tsc -p tsconfig.electron.json && copy electron\\preload.cjs dist-electron\\electron\\preload.cjs", "dev": "concurrently -k \"vite --host 127.0.0.1\" \"wait-on tcp:5173 && cross-env VITE_DEV_SERVER_URL=http://127.0.0.1:5173 electron .\"", "dev:web": "vite --host 127.0.0.1", - "dev:admin": ".\\dev-admin.cmd", + "dev:admin": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\dev-admin.ps1 -ProjectRoot .", "build": "tsc && vite build && tsc -p tsconfig.electron.json && copy electron\\\\preload.cjs dist-electron\\\\electron\\\\preload.cjs", "preview": "vite preview --host 127.0.0.1", "start": "electron .", diff --git a/scripts/dev-admin-start.ps1 b/scripts/dev-admin-start.ps1 index fe13513..59f652d 100644 --- a/scripts/dev-admin-start.ps1 +++ b/scripts/dev-admin-start.ps1 @@ -9,8 +9,17 @@ $ErrorActionPreference = "Stop" try { $project = (Resolve-Path -LiteralPath $ProjectRoot).Path + $logDir = Join-Path $project "outputs\admin-start" + New-Item -ItemType Directory -Force -Path $logDir | Out-Null + $logPath = Join-Path $logDir "admin-dev.log" + try { + Start-Transcript -Path $logPath -Append | Out-Null + } catch { + Write-Host "WARNUNG: Admin-Start-Log konnte nicht geschrieben werden: $($_.Exception.Message)" -ForegroundColor Yellow + } Write-Host "Projekt: $project" + Write-Host "Admin-Log: $logPath" # A UAC-elevated process gets its environment rebuilt fresh from the # registry; it does NOT inherit PATH edits that only exist in the calling diff --git a/scripts/dev-admin.ps1 b/scripts/dev-admin.ps1 new file mode 100644 index 0000000..38287f8 --- /dev/null +++ b/scripts/dev-admin.ps1 @@ -0,0 +1,33 @@ +param( + [string]$ProjectRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..")).Path +) + +$ErrorActionPreference = "Stop" + +function Quote-ProcessArgument([string]$Value) { + return '"' + $Value.Replace('"', '\"') + '"' +} + +try { + $project = (Resolve-Path -LiteralPath $ProjectRoot).Path + $script = Join-Path $PSScriptRoot "dev-admin-start.ps1" + $powershellExe = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" + $arguments = @( + "-NoProfile", + "-ExecutionPolicy Bypass", + "-NoExit", + "-File $(Quote-ProcessArgument $script)", + "-ProjectRoot $(Quote-ProcessArgument $project)" + ) -join " " + + Start-Process -FilePath $powershellExe -ArgumentList $arguments -WorkingDirectory $project -Verb RunAs -WindowStyle Normal -ErrorAction Stop + + Write-Host "" + Write-Host "UAC-Abfrage gestartet. Bitte bestaetigen - danach oeffnet sich ein neues Administrator-Fenster mit npm run dev." -ForegroundColor Green + Write-Host "Dieses Fenster kann geschlossen werden; das eigentliche Programm laeuft im neuen Administrator-Fenster." +} catch { + Write-Host "" + Write-Host "Admin-Start fehlgeschlagen oder UAC-Abfrage abgelehnt:" -ForegroundColor Red + Write-Host $_.Exception.Message -ForegroundColor Red + exit 1 +} diff --git a/src/features/scan/components/DiagnosticsView.tsx b/src/features/scan/components/DiagnosticsView.tsx index bd59eea..d13ccbb 100644 --- a/src/features/scan/components/DiagnosticsView.tsx +++ b/src/features/scan/components/DiagnosticsView.tsx @@ -1,8 +1,7 @@ -import { useRef, useState, type ChangeEvent } from "react"; +import { useState } from "react"; import { AlertTriangle, Download, Play, Upload, Wrench } from "lucide-react"; import type { CaptureResult } from "../../../types/global"; import type { ScanViewControllerResult } from "../types"; -import { goodDatabaseToStoredArtifacts, type GoodImportDatabase } from "../../../lib/goodInterop"; import { FieldConfidenceList } from "./ScanResultCards"; import { useScanDiagnosticsModalModel } from "./modals/hooks/useScanDiagnosticsModalModel"; import { useScanDetailsModalModel } from "./modals/hooks/useScanDetailsModalModel"; @@ -59,7 +58,6 @@ export function DiagnosticsView({ controller, latestCapture, captureStatus, onDe }); const [interopStatus, setInteropStatus] = useState(""); - const fileInputRef = useRef(null); const handleExportGood = async () => { setInteropStatus("Exportiere GOOD..."); @@ -71,27 +69,20 @@ export function DiagnosticsView({ controller, latestCapture, captureStatus, onDe ); }; - const handleImportGood = async (event: ChangeEvent) => { - const file = event.target.files?.[0]; - event.target.value = ""; - if (!file) return; - try { - const database = JSON.parse(await file.text()) as GoodImportDatabase; - const records = goodDatabaseToStoredArtifacts(database); - if (records.length === 0) { - setInteropStatus("Keine gueltigen Artifacts in der Datei gefunden."); - return; - } - setInteropStatus(`Importiere ${records.length} Artifacts...`); - const result = await controller.importGoodArtifacts(records); - setInteropStatus( - result.ok - ? `Importiert: ${result.added} neu, ${result.updated} aktualisiert.` - : "Import fehlgeschlagen (App im Electron-Fenster oeffnen).", - ); - } catch { - setInteropStatus("Datei ist kein gueltiges GOOD/JSON."); + const handleImportGood = async () => { + setInteropStatus("Waehle GOOD-Datei..."); + const result = await controller.importGoodFromFile(); + if (result.canceled) { + setInteropStatus("GOOD-Import abgebrochen."); + return; } + setInteropStatus( + result.ok + ? `Importiert: ${result.added} neu, ${result.updated} aktualisiert (${result.count} gelesen).` + : result.error === "No valid GOOD artifacts found." + ? "Keine gueltigen Artifacts in der Datei gefunden." + : "Import fehlgeschlagen (Datei ist kein gueltiges GOOD/JSON oder Bridge fehlt).", + ); }; return ( @@ -182,11 +173,10 @@ export function DiagnosticsView({ controller, latestCapture, captureStatus, onDe GOOD exportieren - -

diff --git a/src/features/scan/hooks/scanViewReviewHelpers.ts b/src/features/scan/hooks/scanViewReviewHelpers.ts index 4c32869..1fa662a 100644 --- a/src/features/scan/hooks/scanViewReviewHelpers.ts +++ b/src/features/scan/hooks/scanViewReviewHelpers.ts @@ -244,7 +244,7 @@ export async function persistParsedArtifact( return false; } try { - const result = await artifactRepo.saveMany([toStoredArtifact(parsed, source, needsReview)]); + const result = await artifactRepo.saveMany([toStoredArtifact(parsed, source, needsReview, capture?.locked)]); if (result?.ok) { setStoredTotal(result.total); void onStoredArtifactsChanged?.(); @@ -297,6 +297,7 @@ export async function saveReviewSample( })), inventoryGrid: capture.inventoryGrid, inventoryCount: capture.inventoryCount, + locked: capture.locked, ocr: capture.ocr, }, parsed, diff --git a/src/features/scan/hooks/scanViewScanActions.ts b/src/features/scan/hooks/scanViewScanActions.ts index ced3360..5d2c1e7 100644 --- a/src/features/scan/hooks/scanViewScanActions.ts +++ b/src/features/scan/hooks/scanViewScanActions.ts @@ -45,6 +45,10 @@ export interface ScanActionContext { focusDashboard: () => Promise; } +export interface VisibleGridScanOptions { + scanLimit?: number; +} + function buildScanSignature(parsed: ParsedArtifactCandidate) { return `${parsed.name}-${parsed.slot}-${parsed.mainStat}-${parsed.mainValue}-${parsed.level ?? 0}`; } @@ -146,7 +150,7 @@ export async function runAutoReviewScan(context: ScanActionContext): Promise { +export async function runVisibleGridScan(context: ScanActionContext, options: VisibleGridScanOptions = {}): Promise { const { autoScanRunning, bridgeReady, @@ -167,11 +171,12 @@ export async function runVisibleGridScan(context: ScanActionContext): Promise void; - runVisibleGridScan: () => Promise; + runVisibleGridScan: (options?: VisibleGridScanOptions) => Promise; } export function useScanCommandListener({ @@ -20,15 +22,16 @@ export function useScanCommandListener({ }: ScanCommandListenerInput) { useEffect(() => { if (!automationRepo?.onCommand) return; - return automationRepo.onCommand((command: "start-auto" | "stop") => { + return automationRepo.onCommand((command: ScannerCommand) => { if (command === "stop") { requestScanStop("Hotkey/Dev-Stop gedrueckt."); return; } - if (command === "start-auto" && !autoScanRunning && !isScanning && selectedSourceId) { - void runVisibleGridScan(); + const commandType = typeof command === "string" ? command : command.type; + if (commandType === "start-auto" && !autoScanRunning && !isScanning && selectedSourceId) { + const options = typeof command === "string" ? undefined : { scanLimit: command.scanLimit }; + void runVisibleGridScan(options); } }); }, [automationRepo, autoScanRunning, isScanning, selectedSourceId, requestScanStop, runVisibleGridScan]); } - diff --git a/src/features/scan/hooks/useScanViewActions.ts b/src/features/scan/hooks/useScanViewActions.ts index 56e665c..efe3ce1 100644 --- a/src/features/scan/hooks/useScanViewActions.ts +++ b/src/features/scan/hooks/useScanViewActions.ts @@ -1,6 +1,6 @@ import { useCallback, useEffect, useMemo } from "react"; import type { Dispatch, MutableRefObject, SetStateAction } from "react"; -import { runAutoReviewScan as runAutoReviewScanAction, runVisibleGridScan as runVisibleGridScanAction } from "./scanViewScanActions"; +import { runAutoReviewScan as runAutoReviewScanAction, runVisibleGridScan as runVisibleGridScanAction, type VisibleGridScanOptions } from "./scanViewScanActions"; import { initializeLearningState, loadReviewQueue as loadReviewQueueFromRepo, @@ -77,7 +77,7 @@ export interface ScanViewActionResult { loadReviewQueue: () => Promise; openReviewQueue: () => Promise; runAutoReviewScan: () => Promise; - runVisibleGridScan: () => Promise; + runVisibleGridScan: (options?: VisibleGridScanOptions) => Promise; } export function useScanViewActions(input: ScanViewActionInput): ScanViewActionResult { @@ -262,11 +262,11 @@ export function useScanViewActions(input: ScanViewActionInput): ScanViewActionRe await runAutoReviewScanAction(scanActionContext); }, [autoScanRunning, canCaptureSource, selectedSourceId, scanActionContext]); - const runVisibleGridScan = useCallback(async () => { + const runVisibleGridScan = useCallback(async (options: VisibleGridScanOptions = {}) => { if (autoScanRunning || !bridgeReady || !selectedSourceId || !automationRepo?.clickScreen || !automationRepo?.scrollScreen) { return; } - await runVisibleGridScanAction(scanActionContext); + await runVisibleGridScanAction(scanActionContext, options); }, [ autoScanRunning, bridgeReady, diff --git a/src/features/scan/hooks/useScanViewController.ts b/src/features/scan/hooks/useScanViewController.ts index 2a51761..468410d 100644 --- a/src/features/scan/hooks/useScanViewController.ts +++ b/src/features/scan/hooks/useScanViewController.ts @@ -17,7 +17,7 @@ import { emptyAutoScanStats, resolveScanTargetCount, type AutoScanStats, type Sc import type { ScanViewProps, ScanViewControllerResult } from "../types"; import type { CaptureResult, ClickResult, ReviewSampleRecord } from "../../../types/global"; import type { StoredArtifactRecord } from "../../../types/storage"; -import { storedArtifactsToGood } from "../../../lib/goodInterop"; +import { goodDatabaseToStoredArtifacts, type GoodImportDatabase, storedArtifactsToGood } from "../../../lib/goodInterop"; import { createRendererRepositories } from "../../../infrastructure/repositories/rendererBridgeRepositories"; export function useScanViewController({ @@ -174,6 +174,23 @@ export function useScanViewController({ return { ok: Boolean(result.ok), added: result.added ?? 0, updated: result.updated ?? 0 }; }, [artifactRepo, onStoredArtifactsChanged]); + const importGoodFromFile = useCallback(async () => { + if (!exportRepo?.importGoodFile || !artifactRepo?.saveMany) { + return { ok: false, added: 0, updated: 0, count: 0, error: "GOOD import is unavailable." }; + } + const fileResult = await exportRepo.importGoodFile(); + if (fileResult.canceled) return { ok: false, added: 0, updated: 0, count: 0, canceled: true }; + if (!fileResult.ok) { + return { ok: false, added: 0, updated: 0, count: 0, path: fileResult.path, error: fileResult.error }; + } + const records = goodDatabaseToStoredArtifacts(fileResult.database as GoodImportDatabase); + if (records.length === 0) { + return { ok: false, added: 0, updated: 0, count: 0, path: fileResult.path, error: "No valid GOOD artifacts found." }; + } + const saved = await importGoodArtifacts(records); + return { ...saved, count: records.length, path: fileResult.path }; + }, [artifactRepo, exportRepo, importGoodArtifacts]); + useScanViewStateSync({ artifactRepo, latestCapture, @@ -253,6 +270,7 @@ export function useScanViewController({ runVisibleGridScan, canGoodInterop, exportGoodFromStore, + importGoodFromFile, importGoodArtifacts, }; } diff --git a/src/features/scan/types.ts b/src/features/scan/types.ts index 3ce1056..76e35bd 100644 --- a/src/features/scan/types.ts +++ b/src/features/scan/types.ts @@ -82,5 +82,6 @@ export interface ScanViewControllerResult { runVisibleGridScan: () => Promise; canGoodInterop: boolean; exportGoodFromStore: () => Promise<{ ok: boolean; path?: string; count: number }>; + importGoodFromFile: () => Promise<{ ok: boolean; added: number; updated: number; count: number; canceled?: boolean; path?: string; error?: string }>; importGoodArtifacts: (records: StoredArtifactRecord[]) => Promise<{ ok: boolean; added: number; updated: number }>; } diff --git a/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts b/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts index 850edd4..2f94dc3 100644 --- a/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts +++ b/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts @@ -26,6 +26,7 @@ import type { ClickResult, ReviewSampleListResult, SaveScannerLearningRulesResult, + GoodImportFileResult, } from "../../types/global"; const EMPTY_SNAPSHOT: AppSnapshot | null = null; @@ -67,6 +68,12 @@ const EMPTY_SAVE_RULES_RESULT: SaveScannerLearningRulesResult = { rules: {}, total: 0, }; +const EMPTY_GOOD_IMPORT_FILE_RESULT: GoodImportFileResult = { + ok: false, + canceled: false, + path: "", + error: "Electron bridge unavailable.", +}; async function createBridgeSafeCall( callback: () => Promise | TResult | null | undefined, @@ -205,6 +212,7 @@ export function createRendererRepositories(): RendererRepositories | null { const exportRepo: ScanExportPort = { exportGood: (payload) => createBridgeSafeCall(() => bridge.exportGood(payload), EMPTY_SAVE_RESULT), + importGoodFile: () => createBridgeSafeCall(() => bridge.importGoodFile(), EMPTY_GOOD_IMPORT_FILE_RESULT), }; return { diff --git a/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts b/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts index 5a3d992..6a19363 100644 --- a/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts +++ b/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts @@ -8,8 +8,10 @@ import type { ScannerStatusPayload, ReviewSamplePayload, GoodDatabase, + GoodImportFileResult, FocusGenshinResult, RuntimeInfo, + ScannerCommand, LoadScannerLearningRulesResult, SaveScannerLearningRulesResult, ArtifactStoreLoadResult, @@ -62,7 +64,7 @@ export interface AutomationRepositoryPort { focusMainWindow(): Promise; clickScreen(x: number, y: number): Promise; scrollScreen(notches: number, anchorX?: number, anchorY?: number): Promise; - onCommand(callback: (command: "start-auto" | "stop") => void): () => void; + onCommand(callback: (command: ScannerCommand) => void): () => void; } export interface OverlayRepositoryPort { @@ -71,6 +73,7 @@ export interface OverlayRepositoryPort { export interface ScanExportPort { exportGood(payload: GoodDatabase): Promise; + importGoodFile(): Promise; } export interface RendererRepositories { diff --git a/src/lib/artifactOcrParser.test.ts b/src/lib/artifactOcrParser.test.ts index 0a8f483..c9372df 100644 --- a/src/lib/artifactOcrParser.test.ts +++ b/src/lib/artifactOcrParser.test.ts @@ -264,6 +264,27 @@ describe("parseArtifactCandidate", () => { expect(parsed?.fields.substats.confidence).toBe(96); }); + it("parses the live calibrated 1080p Conductor circlet capture", () => { + const parsed = parseArtifactCandidate(captureFromOcr({ + "artifact-title": "Conductor's Top Hat", + "artifact-main-stat": "Circlet of Logos\nHP\n7. 0 % i", + "artifact-substats": "a +\n+ Energy Recharge+4.5%\n+ ATK+14\n- Elemental Mastery+19\n- ATK+5.3% (unactivated)", + "artifact-footer": "", + })); + + expect(parsed?.name).toBe("Conductor's Top Hat"); + expect(parsed?.slot).toBe("Circlet of Logos"); + expect(parsed?.setName).toBe("Wanderer's Troupe"); + expect(parsed?.mainStat).toBe("HP%"); + expect(parsed?.mainValue).toBe("7.0%"); + expect(parsed?.substats).toEqual([ + "Energy Recharge+4.5%", + "ATK+14", + "Elemental Mastery+19", + "ATK%+5.3%", + ]); + }); + it("keeps a percent main value even when OCR misses the main stat label", () => { const parsed = parseArtifactCandidate(captureFromOcr({ "artifact-title": "Moonlit Offering's Final\nSands of Eon", diff --git a/src/lib/artifactOcrParser.ts b/src/lib/artifactOcrParser.ts index cd7df7b..66d7dd1 100644 --- a/src/lib/artifactOcrParser.ts +++ b/src/lib/artifactOcrParser.ts @@ -276,14 +276,15 @@ function findMainValue(text: string, mainStat: string, slot: string, level: numb } function extractPercentValue(text: string) { + const percentPattern = /([0-9]{1,3}(?:\s*[.,:\u00B7]\s*[0-9])?)\s*%/; const lineMatches = text .split("\n") - .map((line) => line.match(/([0-9]{1,3}(?:[.,:\u00B7][0-9])?)\s*%/)) + .map((line) => line.match(percentPattern)) .filter((match): match is RegExpMatchArray => Boolean(match)); - const preferred = lineMatches[0] ?? text.match(/([0-9]{1,3}(?:[.,:\u00B7][0-9])?)\s*%/); + const preferred = lineMatches[0] ?? text.match(percentPattern); if (!preferred?.[1]) return ""; - return `${preferred[1].replace(/[:,\u00B7]/g, ".")}%`; + return `${normalizeMainValue(preferred[1])}%`; } function inferMainStat(slot: string, text: string): ParsedField { @@ -302,7 +303,7 @@ function inferMainStat(slot: string, text: string): ParsedField { function findDirectMainStat(text: string) { const compact = simplifyForMatch(text); - const hasPercentValue = /[0-9]{1,3}(?:\.[0-9])?\s*%/.test(text); + const hasPercentValue = /[0-9]{1,3}(?:\s*[.,:\u00B7]\s*[0-9])?\s*%/.test(text); const priority = [ "Physical DMG Bonus", "Elemental Mastery", @@ -481,7 +482,7 @@ function isPercentMainStat(stat: string) { } function promotePercentVariant(stat: string, text: string) { - if (["ATK", "HP", "DEF"].includes(stat) && /[0-9]{1,3}(?:\.[0-9])?\s*%/.test(text)) return `${stat}%`; + if (["ATK", "HP", "DEF"].includes(stat) && /[0-9]{1,3}(?:\s*[.,:\u00B7]\s*[0-9])?\s*%/.test(text)) return `${stat}%`; return stat; } diff --git a/src/lib/artifactStore.ts b/src/lib/artifactStore.ts index 2fbb81a..9082a56 100644 --- a/src/lib/artifactStore.ts +++ b/src/lib/artifactStore.ts @@ -45,7 +45,7 @@ export function hashId(value: string) { return `${hash.toString(16).padStart(8, "0")}-${value.length.toString(16)}`; } -export function toStoredArtifact(parsed: ParsedArtifactCandidate, source: string, needsReview: boolean): StoredArtifactRecord { +export function toStoredArtifact(parsed: ParsedArtifactCandidate, source: string, needsReview: boolean, locked?: boolean): StoredArtifactRecord { return { id: hashId(storeSignature(parsed)), name: parsed.name, @@ -58,6 +58,7 @@ export function toStoredArtifact(parsed: ParsedArtifactCandidate, source: string equipped: parsed.equipped, confidence: parsed.confidence, needsReview, + locked, source, }; } diff --git a/src/lib/layoutProfile.test.ts b/src/lib/layoutProfile.test.ts index b3a0d03..cb95763 100644 --- a/src/lib/layoutProfile.test.ts +++ b/src/lib/layoutProfile.test.ts @@ -42,6 +42,10 @@ describe("layoutProfile", () => { expect(rect.y + rect.height).toBeLessThanOrEqual(QHD.height); }); + it("matches the calibrated 1080p artifact detail panel", () => { + expect(profileDetailRect(HD)).toEqual({ x: 1308, y: 120, width: 492, height: 838 }); + }); + it("produces the four artifact crops in top-to-bottom order, all clamped", () => { const detail = profileDetailRect(QHD); const crops = detailCropRects(detail, QHD); @@ -65,19 +69,28 @@ describe("layoutProfile", () => { const detail = profileDetailRect(QHD); const inv = inventoryRect(QHD, detail); const count = inventoryCountCropRect(inv, QHD); - expect(count.x).toBeGreaterThanOrEqual(inv.x); + expect(count.x).toBeGreaterThan(QHD.width * 0.75); expect(count.x + count.width).toBeLessThanOrEqual(QHD.width); }); - it("builds a 5-column inventory grid on the left", () => { + it("builds the calibrated 8-column inventory grid on the left", () => { const detail = profileDetailRect(QHD); const grid = inventoryGrid(QHD, detail); - expect(grid.cols).toBe(5); + expect(grid.cols).toBe(8); + expect(grid.rows).toBe(5); expect(grid.source).toBe("detected"); - expect(grid.centers.length).toBeGreaterThanOrEqual(10); + expect(grid.centers).toHaveLength(40); expect(grid.centers.every((center) => center.x < detail.x)).toBe(true); }); + it("matches the live 1080p artifact grid centers", () => { + const detail = profileDetailRect(HD); + const grid = inventoryGrid(HD, detail); + expect(grid.centers[0]).toEqual({ x: 179, y: 254, row: 0, col: 0 }); + expect(grid.centers[7]).toEqual({ x: 1201, y: 254, row: 0, col: 7 }); + expect(grid.centers.at(-1)).toEqual({ x: 1201, y: 958, row: 4, col: 7 }); + }); + it("reports a missing grid when the inventory panel is too small", () => { const tiny = { width: 320, height: 180 }; const grid = inventoryGrid(tiny, profileDetailRect(tiny)); diff --git a/src/lib/layoutProfile.ts b/src/lib/layoutProfile.ts index 9244420..fda193c 100644 --- a/src/lib/layoutProfile.ts +++ b/src/lib/layoutProfile.ts @@ -5,9 +5,9 @@ // of that geometry; electron/main.ts consumes it for cropping and keeps a // colour-based detail-rect detector only as a fallback for off-profile setups. // -// NOTE: the per-field detail crop fractions below are the current working values. -// True IK-style fixed coordinates need calibration against a reference 16:9 -// screenshot; the structure here is what those calibrated numbers slot into. +// Calibrated from a 1920x1080 English artifact-inventory screenshot and scaled +// by client size. This follows Inventory Kamera's stable approach: fixed +// 16:9-relative UI regions first, visual detection only as a fallback. export interface LayoutRect { x: number; @@ -81,10 +81,10 @@ export function profileDetailRect(imageSize: { width: number; height: number }): if (width <= 0 || height <= 0) return { x: 0, y: 0, width: Math.max(1, width), height: Math.max(1, height) }; return clampRect( { - x: Math.round(width * 0.5), - y: Math.round(height * 0.08), - width: Math.round(width * 0.46), - height: Math.round(height * 0.74), + x: Math.round(width * 0.681), + y: Math.round(height * 0.111), + width: Math.round(width * 0.256), + height: Math.round(height * 0.776), }, imageSize, ); @@ -97,10 +97,10 @@ export function detailCropRects(detailRect: LayoutRect, imageSize: { width: numb id: "artifact-title", label: "Artifact title", rect: { - x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.05), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.16), + x: Math.round(detailRect.x), + y: Math.round(detailRect.y), + width: Math.round(detailRect.width), + height: Math.round(detailRect.height * 0.07), }, }, { @@ -108,9 +108,9 @@ export function detailCropRects(detailRect: LayoutRect, imageSize: { width: numb label: "Main stat", rect: { x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.2), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.18), + y: Math.round(detailRect.y + detailRect.height * 0.075), + width: Math.round(detailRect.width * 0.58), + height: Math.round(detailRect.height * 0.26), }, }, { @@ -118,9 +118,9 @@ export function detailCropRects(detailRect: LayoutRect, imageSize: { width: numb label: "Substats", rect: { x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.41), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.25), + y: Math.round(detailRect.y + detailRect.height * 0.34), + width: Math.round(detailRect.width * 0.86), + height: Math.round(detailRect.height * 0.27), }, }, { @@ -128,9 +128,9 @@ export function detailCropRects(detailRect: LayoutRect, imageSize: { width: numb label: "Footer", rect: { x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.78), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.16), + y: Math.round(detailRect.y + detailRect.height * 0.82), + width: Math.round(detailRect.width * 0.86), + height: Math.round(detailRect.height * 0.14), }, }, ]; @@ -139,12 +139,13 @@ export function detailCropRects(detailRect: LayoutRect, imageSize: { width: numb } export function inventoryCountCropRect(inventoryRect: LayoutRect, imageSize: { width: number; height: number }): LayoutRect { + const { width, height } = imageSize; return clampRect( { - x: Math.round(inventoryRect.x + inventoryRect.width * 0.62), - y: Math.round(inventoryRect.y + inventoryRect.height * 0.02), - width: Math.round(inventoryRect.width * 0.34), - height: Math.round(inventoryRect.height * 0.09), + x: Math.round(width * 0.795), + y: Math.round(height * 0.02), + width: Math.round(width * 0.145), + height: Math.round(height * 0.055), }, imageSize, ); @@ -152,18 +153,12 @@ export function inventoryCountCropRect(inventoryRect: LayoutRect, imageSize: { w export function inventoryRect(imageSize: { width: number; height: number }, detailRect: LayoutRect): LayoutRect { const { width, height } = imageSize; - const preferredWidth = Math.max(140, Math.round(width * 0.48)); - const x = Math.round(width * 0.03); - const y = Math.round(detailRect.y + detailRect.height * 0.09); - const availableWidth = Math.max(100, detailRect.x - Math.round(width * 0.04)); - const panelWidth = Math.max(100, Math.min(preferredWidth, availableWidth)); - const safeWidth = panelWidth > width * 0.85 ? Math.round(width * 0.55) : panelWidth; return clampRect( { - x, - y, - width: Math.min(safeWidth, Math.max(width - x - Math.round(width * 0.02), 100)), - height: Math.max(140, Math.round(height * 0.7)), + x: Math.round(width * 0.055), + y: Math.round(height * 0.155), + width: Math.max(140, Math.round(Math.min(detailRect.x - width * 0.07, width * 0.63))), + height: Math.max(140, Math.round(height * 0.74)), }, imageSize, ); @@ -171,18 +166,17 @@ export function inventoryRect(imageSize: { width: number; height: number }, deta export function inventoryGrid(imageSize: { width: number; height: number }, detailRect: LayoutRect): InventoryGridLayout { const rect = inventoryRect(imageSize, detailRect); - const cols = 5; - if (rect.width < 160 || rect.height < 140) { + const cols = 8; + if (imageSize.width < 800 || imageSize.height < 450 || rect.width < 160 || rect.height < 140) { return { centers: [], rows: 0, cols: 0, confidence: 0, source: "missing" }; } - const cellWidth = Math.max(56, Math.round(rect.width / cols)); - const stepX = Math.round(cellWidth * 0.96); - const stepY = Math.round(cellWidth * 1.03); - const visibleRows = Math.max(2, Math.min(6, Math.round(rect.height / Math.max(stepY, 1)))); + const stepX = Math.round(imageSize.width * 0.076); + const stepY = Math.round(imageSize.height * 0.163); + const visibleRows = 5; - const startX = rect.x + Math.max(6, Math.round(stepX * 0.45)); - const startY = rect.y + Math.max(6, Math.round(stepY * 0.45)); + const startX = Math.round(imageSize.width * 0.093); + const startY = Math.round(imageSize.height * 0.235); const centers: InventoryGridLayout["centers"] = []; for (let row = 0; row < visibleRows; row++) { for (let col = 0; col < cols; col++) { diff --git a/src/lib/lockDetection.test.ts b/src/lib/lockDetection.test.ts index a727811..1066501 100644 --- a/src/lib/lockDetection.test.ts +++ b/src/lib/lockDetection.test.ts @@ -19,13 +19,14 @@ function bitmap(goldPixels: number, total: number): Bitmap { } describe("lockDetection", () => { - it("places the lock crop in the top-right of the detail card", () => { + it("places the lock crop on the lock button in the substat panel", () => { const size = { width: 2560, height: 1440 }; const detail = profileDetailRect(size); const rect = lockIconCropRect(detail, size); expect(rect.x).toBeGreaterThan(detail.x + detail.width * 0.5); expect(rect.x + rect.width).toBeLessThanOrEqual(size.width); - expect(rect.y).toBeLessThan(detail.y + detail.height * 0.5); + expect(rect.y).toBeGreaterThan(detail.y + detail.height * 0.3); + expect(rect.y).toBeLessThan(detail.y + detail.height * 0.45); }); it("measures the gold-pixel ratio", () => { diff --git a/src/lib/lockDetection.ts b/src/lib/lockDetection.ts index 20e3b7c..279b4ee 100644 --- a/src/lib/lockDetection.ts +++ b/src/lib/lockDetection.ts @@ -1,5 +1,5 @@ -import { clampRect, type LayoutRect } from "./layoutProfile"; -import type { Bitmap } from "./ocrPreprocess"; +import { clampRect, type LayoutRect } from "./layoutProfile.js"; +import type { Bitmap } from "./ocrPreprocess.js"; // EXPERIMENTAL, read-only lock-status detection (nice-to-have). Genshin shows a // padlock at the top-right of the artifact detail card: a bright gold fill when @@ -15,10 +15,10 @@ import type { Bitmap } from "./ocrPreprocess"; export function lockIconCropRect(detailRect: LayoutRect, imageSize: { width: number; height: number }): LayoutRect { return clampRect( { - x: Math.round(detailRect.x + detailRect.width * 0.8), - y: Math.round(detailRect.y + detailRect.height * 0.03), - width: Math.round(detailRect.width * 0.16), - height: Math.round(detailRect.height * 0.09), + x: Math.round(detailRect.x + detailRect.width * 0.735), + y: Math.round(detailRect.y + detailRect.height * 0.355), + width: Math.round(detailRect.width * 0.105), + height: Math.round(detailRect.height * 0.07), }, imageSize, ); diff --git a/src/lib/storedArtifactAdapter.test.ts b/src/lib/storedArtifactAdapter.test.ts index 2023f97..12c21b0 100644 --- a/src/lib/storedArtifactAdapter.test.ts +++ b/src/lib/storedArtifactAdapter.test.ts @@ -23,7 +23,7 @@ function record(overrides: Partial = {}): StoredArtifactRe describe("storedArtifactAdapter", () => { it("converts stored OCR artifacts into recommendation-domain artifacts", () => { - const [artifact] = storedArtifactsToDomain([record()]); + const [artifact] = storedArtifactsToDomain([record({ locked: true })]); expect(artifact.slot).toBe("sands"); expect(artifact.setKey).toBe("viridescent_venerer"); @@ -32,6 +32,13 @@ describe("storedArtifactAdapter", () => { expect(artifact.equipped).toBe("Sucrose"); expect(artifact.confidence).toBe(0.96); expect(artifact.source).toBe("screen"); + expect(artifact.locked).toBe(true); + }); + + it("does not invent lock state from confidence", () => { + const [artifact] = storedArtifactsToDomain([record({ locked: undefined, confidence: 100 })]); + + expect(artifact.locked).toBe(false); }); it("keeps flat and percent ATK substats distinct", () => { diff --git a/src/lib/storedArtifactAdapter.ts b/src/lib/storedArtifactAdapter.ts index f595709..4a19133 100644 --- a/src/lib/storedArtifactAdapter.ts +++ b/src/lib/storedArtifactAdapter.ts @@ -54,7 +54,7 @@ export function storedArtifactsToDomain(records: StoredArtifactRecord[]): Artifa mainStat: normalizeMainStat(record.mainStat, record.mainValue), substats: record.substats.map(parseStoredSubstat).filter(Boolean) as ArtifactSubstat[], equipped: isUsefulEquippedName(record.equipped) ? record.equipped.trim() : undefined, - locked: !record.needsReview && record.confidence >= 90, + locked: Boolean(record.locked), source: toSource(record.source), confidence: Math.max(0, Math.min(1, record.confidence / 100)), lastSeenAt: record.lastSeenAt ?? record.firstSeenAt ?? now, diff --git a/src/services/assistantBridge.ts b/src/services/assistantBridge.ts index 88b8bfc..27d43e0 100644 --- a/src/services/assistantBridge.ts +++ b/src/services/assistantBridge.ts @@ -15,6 +15,8 @@ import type { SaveScannerLearningRulesResult, SaveSnapshotResult, GoodDatabase, + GoodImportFileResult, + ScannerCommand, ScannerStatusPayload, ScannerLearningRulePayload, } from "../types/global"; @@ -39,6 +41,7 @@ export interface AssistantBridge { options?: CaptureOptions, ) => Promise; exportGood: (payload: GoodDatabase) => Promise; + importGoodFile: () => Promise; showOverlay: () => Promise; loadArtifacts: () => Promise; saveArtifacts: (records: StoredArtifactRecord[]) => Promise; @@ -54,7 +57,7 @@ export interface AssistantBridge { focusGenshinForScanStart: () => Promise; clickScreen: (x: number, y: number) => Promise; scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => Promise; - onScannerCommand: (callback: (command: "start-auto" | "stop") => void) => () => void; + onScannerCommand: (callback: (command: ScannerCommand) => void) => () => void; } function hasFunction(api: Record, key: string): boolean { @@ -81,6 +84,7 @@ export function getAssistantBridge(): AssistantBridge | null { listCaptureSources: () => api.listCaptureSources(), captureSource: (sourceId, delayMs, focusGenshin, options) => api.captureSource(sourceId, delayMs, focusGenshin, options), exportGood: (payload) => api.exportGood(payload), + importGoodFile: () => api.importGoodFile(), loadArtifacts: () => api.loadArtifacts(), saveArtifacts: (records) => api.saveArtifacts(records), loadReviewSamples: (limit = 50) => api.loadReviewSamples(limit), diff --git a/src/types/global.d.ts b/src/types/global.d.ts index f8987dc..177ca03 100644 --- a/src/types/global.d.ts +++ b/src/types/global.d.ts @@ -56,6 +56,7 @@ export interface CaptureResult { source: "ocr" | "missing"; text: string; }; + locked?: boolean; layout?: { aspect: string; isSixteenNine: boolean; @@ -125,6 +126,14 @@ export interface SaveResultWithPath { export type SaveSnapshotResult = SaveResultWithPath; +export type ScannerCommand = + | "start-auto" + | "stop" + | { + type: "start-auto"; + scanLimit?: number; + }; + export interface ScannerLearningRulePayload { textReplacements?: Record; } @@ -211,6 +220,7 @@ export interface ReviewSampleRecord { ocr?: OcrResult[]; inventoryGrid?: CaptureResult["inventoryGrid"]; inventoryCount?: CaptureResult["inventoryCount"]; + locked?: boolean; }; }; } @@ -239,6 +249,7 @@ export interface ReviewSamplePayload { ocr?: OcrResult[]; inventoryGrid?: CaptureResult["inventoryGrid"]; inventoryCount?: CaptureResult["inventoryCount"]; + locked?: boolean; }; [key: string]: unknown; } @@ -280,6 +291,14 @@ export interface GoodDatabase { artifacts: GoodExportArtifact[]; } +export interface GoodImportFileResult { + ok: boolean; + canceled: boolean; + path: string; + database?: unknown; + error?: string; +} + declare global { interface Window { assistantApi?: { @@ -302,10 +321,11 @@ declare global { loadArtifacts: () => Promise; saveArtifacts: (records: StoredArtifactRecord[]) => Promise; exportGood: (payload: GoodDatabase) => Promise; + importGoodFile: () => Promise; publishScannerStatus: (status: ScannerStatusPayload) => Promise; showOverlay: () => Promise; hideOverlay: () => Promise; - onScannerCommand: (callback: (command: "start-auto" | "stop") => void) => () => void; + onScannerCommand: (callback: (command: ScannerCommand) => void) => () => void; }; } } diff --git a/src/types/storage.ts b/src/types/storage.ts index 353c220..63574cb 100644 --- a/src/types/storage.ts +++ b/src/types/storage.ts @@ -10,6 +10,7 @@ export interface StoredArtifactRecord { equipped: string; confidence: number; needsReview: boolean; + locked?: boolean; source: string; firstSeenAt?: string; lastSeenAt?: string;