diff --git a/dev-admin.cmd b/dev-admin.cmd deleted file mode 100644 index 67dfd40..0000000 --- a/dev-admin.cmd +++ /dev/null @@ -1,23 +0,0 @@ -@echo off -rem Startet den Dev-Modus mit Administratorrechten (ein UAC-Prompt erscheint). -rem Noetig, weil Genshin erhoeht laeuft: Windows (UIPI) verwirft sonst alle -rem simulierten Maus-Eingaben an das Spiel - SendInput meldet trotzdem Erfolg. -rem Der Punkt hinter %~dp0 verhindert, dass der abschliessende Backslash das -rem schliessende Anfuehrungszeichen escaped. -rem -NoExit haelt das erhoehte (innere) Fenster offen, selbst wenn das Skript -rem einen Fehler wirft oder npm run dev sofort wieder beendet. -rem -rem Dieses AEUSSERE Fenster (das du beim Doppelklick oder ueber -rem "npm run dev:admin" siehst) schloss sich frueher sofort, sobald -rem Start-Process zurueckkehrte - auch wenn UAC abgelehnt wurde oder die -rem Elevation ganz fehlschlug, ohne dass davon irgendetwas sichtbar war. -rem try/catch + timeout zeigen jetzt den Fehler und halten das Fenster kurz offen. -"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" -NoProfile -ExecutionPolicy Bypass -Command "$script='%~dp0scripts\dev-admin-start.ps1'; $project='%~dp0.'; try { Start-Process powershell -ArgumentList @('-NoProfile','-ExecutionPolicy','Bypass','-NoExit','-File',$script,'-ProjectRoot',$project) -Verb RunAs -ErrorAction Stop; Write-Host ''; Write-Host 'UAC-Abfrage gestartet. Bitte bestaetigen - danach oeffnet sich ein neues Administrator-Fenster mit npm run dev.' -ForegroundColor Green } catch { Write-Host ''; Write-Host 'Admin-Start fehlgeschlagen oder UAC-Abfrage abgelehnt:' -ForegroundColor Red; Write-Host $_.Exception.Message -ForegroundColor Red }" -echo. -echo Dieses Fenster kannst du jetzt schliessen (Taste druecken oder 15s warten). Das eigentliche Programm laeuft im neuen Administrator-Fenster. -rem timeout statt pause/choice: pause und choice warten unter umgeleiteter -rem Standardeingabe (z.B. beim Testen ueber ein Skript) fuer immer, weil sie -rem auf ein echtes Konsolen-Handle angewiesen sind. timeout erkennt eine -rem umgeleitete Eingabe explizit und bricht sofort ab statt zu haengen, waehrend -rem es bei einem echten Doppelklick normal 15s wartet oder bei Tastendruck endet. -timeout /t 15 >nul 2>&1 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 776da75..7910b29 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -49,7 +49,7 @@ flowchart LR | Module | Responsibility | | --- | --- | -| `electron/main.ts` | Window lifecycle, capture source listing, Smart Capture, OCR crop generation, overlay window IPC, persistent PowerShell input/capture helper, JSON artifact store | +| `electron/main.ts` | Window lifecycle, capture source listing, Smart Capture, OCR crop generation, overlay window IPC, input/capture sidecar orchestration, JSON artifact store, dev-only scanner control endpoints | | `electron/preload.cjs` | Safe renderer bridge exposed as `window.assistantApi` | | `src/lib/artifactStore.ts` | Pure signature/id/record helpers for the persistent artifact store | | `src/App.tsx` | Main app shell, scan view, triage view, build view, overlay preview | @@ -111,9 +111,24 @@ sequenceDiagram **Automatic grid scan** is user-triggered input automation limited to clicking detected inventory tiles and wheel-scrolling the inventory. Safety and reliability rules: -- All input goes through one persistent PowerShell helper process (`input-helper.ps1` in userData) that compiles the Win32 interop once and speaks JSON over stdin/stdout (ops: ping, focus, cursor, click, scroll, capture). Mouse movement is sent as iterated relative SendInput deltas (what a real mouse produces): Genshin tracks the cursor via raw input and snaps the OS cursor back to its own position every frame, so SetCursorPos/absolute moves silently stop working once the game owns the cursor. The helper verifies the cursor reached the target and refuses to click otherwise. +- All input goes through the helper service boundary (currently a C# sidecar with + fallback support behind the same JSON protocol). The helper owns focus, cursor + movement, click, scroll, guard-state polling, elevation detection, and GDI + capture. Mouse movement is sent as iterated relative input deltas instead of + relying on a single absolute cursor jump. The helper verifies the cursor + reached the target and refuses to click otherwise. +- `npm run dev:admin` is the validated dev path for automation when elevated + input is required. The elevated PowerShell startup is handled by + `scripts/dev-admin.ps1` and logged to `outputs/admin-start/admin-dev.log`. + The user must approve UAC manually; the app cannot approve the Secure Desktop + prompt itself. - Failsafe: before every click and scroll the renderer polls cursor position and ESC state. Holding ESC or moving the mouse away from the last automated position aborts the scan immediately; the Stop button also aborts. Only the `GetAsyncKeyState` held-down bit (0x8000) is used - the "pressed since last call" bit fires for stale ESC presses from normal Genshin menu navigation and caused false aborts. - SendInput's return value is checked: zero injected events (UIPI, e.g. elevated Genshin vs. non-elevated app) aborts with an explicit hint instead of silently clicking into nothing. +- Dev-only probes under `http://127.0.0.1:17317` are used for live validation: + `/automation/probe-click?index=N` tests one read-only tile selection, and + `/scanner/start?limit=N` starts an auto-scan with a temporary limit payload. + The live known-good result on 2026-07-07 is documented in + [AUTOMATION_LIVE_SCAN.md](AUTOMATION_LIVE_SCAN.md). - Click verification: after each click the parsed detail-panel signature should change. An unchanged signature is a soft miss (it can also mean two OCR-identical neighbor pieces, common among +0 artifacts), so it is retried once with a small offset, logged with the stuck artifact name, and then skipped - never fatal on its own. The scan aborts only when the first ~6 clicks of page 1 produce nothing new (diagnosis hint: elevated Genshin blocks SendInput via UIPI, or grid coordinates are wrong) or a later page yields zero new artifacts. - Scan stats separate clicked (click attempts), parsed (readable captures), stored (persisted), review (review samples), duplicates, and misses, so "scanned" cannot be mistaken for "successfully read". - Scrolling sends one wheel notch per grid row with the cursor anchored over the inventory (assumption: roughly one row per notch; overlap is absorbed by dedupe, and a page without new artifacts stops the scan). diff --git a/docs/AUTOMATION_LIVE_SCAN.md b/docs/AUTOMATION_LIVE_SCAN.md new file mode 100644 index 0000000..c63daf5 --- /dev/null +++ b/docs/AUTOMATION_LIVE_SCAN.md @@ -0,0 +1,154 @@ +# Automation Live Scan Runbook + +This document is the durable reference for automatic artifact scanning, mouse +movement, click input, elevation, and live validation status. + +## Current Known-Good State + +Validated live on 2026-07-07 with Genshin open in the artifact inventory at +1920x1080, English UI: + +- `npm run dev:admin` starts the app elevated after the user confirms UAC. +- Runtime status reported `isElevated: true`, `genshinFound: true`, and + `targetProcess: "GenshinImpact"`. +- The safe probe endpoint `/automation/probe-click?index=1` focused Genshin, + moved the cursor to the second visible inventory tile, clicked it, and changed + the artifact detail panel fingerprint. +- Probe result: `clicked: true`, `inputBlocked: false`, + `foregroundProcess: "GenshinImpact"`, and `changed: true`. +- A bounded live auto-scan via `/scanner/start?limit=2` completed with: + `clicked: 2`, `attempted: 2`, `verified: 2`, `parsed: 2`, `stored: 2`, + `review: 2`, `misses: 0`, `status: "done"`. + +This proves that the current elevated app plus helper path can deliver mouse +movement and click input to the focused Genshin client in this environment. + +## Elevation And UAC + +Use: + +```powershell +npm run dev:admin +``` + +The command runs `scripts/dev-admin.ps1`, which launches a new elevated +PowerShell window running `scripts/dev-admin-start.ps1`. The elevated start is +logged to: + +```text +outputs/admin-start/admin-dev.log +``` + +The user must confirm the Windows UAC prompt. The app cannot and must not click +the Secure Desktop UAC prompt for itself. After confirmation, the app can verify +its own runtime through the dev status endpoint. + +Useful checks: + +```powershell +Invoke-RestMethod http://127.0.0.1:17317/health +Invoke-RestMethod http://127.0.0.1:17317/scanner/status +``` + +Expected runtime facts before automatic scan: + +- `isElevated: true` +- `genshinFound: true` +- `targetProcess: "GenshinImpact"` +- hotkeys registered + +## Mouse And Click Validation + +Use the probe before broad auto-scan work: + +```powershell +Invoke-RestMethod "http://127.0.0.1:17317/automation/probe-click?index=1" | + ConvertTo-Json -Depth 12 +``` + +The probe performs one read-only inventory selection click. It does not delete, +feed, enhance, lock, unlock, spend, or modify game resources. + +Interpretation: + +- `click.ok: true`, `clicked: true`, `inputBlocked: false` means Windows did not + block SendInput/UIPI in the current configuration. +- `focused: true` and `foregroundProcess: "GenshinImpact"` means the click was + sent while Genshin was foreground. +- `changed: true` means the detail panel changed after the click. +- `changed: false` can be benign if the target tile was already selected or two + neighboring artifacts render identically; retry with another `index`, `row`, + or `col`. + +Examples: + +```powershell +# Second visible tile +Invoke-RestMethod "http://127.0.0.1:17317/automation/probe-click?index=1" + +# Specific grid cell +Invoke-RestMethod "http://127.0.0.1:17317/automation/probe-click?row=0&col=3" +``` + +## Bounded Live Auto-Scan + +For live validation, prefer a bounded scan first: + +```powershell +Invoke-RestMethod "http://127.0.0.1:17317/scanner/start?limit=2" +``` + +Then poll: + +```powershell +Invoke-RestMethod "http://127.0.0.1:17317/scanner/status" | + ConvertTo-Json -Depth 12 +``` + +The `/scanner/start?limit=N` endpoint sends a renderer command payload with a +temporary scan limit. It does not change the normal UI setting. The normal +hotkeys and buttons still use the UI's configured scan limit. + +## Anti-Cheat And Safety Boundary + +Do not describe the current implementation as bypassing anti-cheat. The app +does not read memory, hook the process, inject code, modify game files, inspect +packets, or interact with kernel drivers. It uses normal Windows screen capture, +focus, cursor movement, wheel, and click input. + +The practical finding is narrower: + +- A non-elevated app can be blocked by Windows integrity/UIPI when the target + process is elevated or protected. +- Running the app elevated fixed input delivery in the tested environment. +- Genshin's anti-cheat may still affect behavior on other machines, game modes, + overlays, or future versions. Re-run the probe before trusting broad scans. + +Never add automation that deletes, feeds, enhances, locks/unlocks, spends +resources, reads memory, hooks, injects, or modifies Genshin. + +## Live Layout Facts + +The current 16:9 layout profile is calibrated from a 1920x1080 English +artifact-inventory capture: + +- detail rect approximately `x=1308`, `y=120`, `width=492`, `height=838` +- inventory grid: `8 x 5` +- first tile center: `x=179`, `y=254`, `row=0`, `col=0` +- second tile center: `x=325`, `y=254`, `row=0`, `col=1` +- inventory count crop successfully read `2059/2400` in the live session + +The profile is resolution-scaled for 16:9. Off-profile setups should be treated +as higher risk and validated with Smart Capture plus the probe. + +## Validation Checklist + +Before marking an automation change done: + +1. Run `npm run lint`. +2. Run `npx tsc -p tsconfig.electron.json` when Electron/preload/main changed. +3. Run `npm test`. +4. Run `npm run build`. +5. If Genshin is available, run `/automation/probe-click?index=1`. +6. For scan-loop changes, run `/scanner/start?limit=2` before any broader scan. +7. Record new live findings in this file and in `docs/scanner-rework-status.md`. diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index c0c41e1..9980791 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -15,6 +15,7 @@ This document contains Architecture Decision Records. | ADR-007 | Measure OCR accuracy with a labeled eval harness before reworking the scanner | Accepted | 2026-07-05 | | ADR-008 | Replace the PowerShell input/capture helper with a C# sidecar | Accepted | 2026-07-05 | | ADR-009 | Resolution-anchored layout profiles and OCR preprocessing over color detection | Accepted | 2026-07-05 | +| ADR-010 | Elevated dev runner and bounded live automation probes | Accepted | 2026-07-07 | ## ADR-001: Build A Local Electron App First @@ -235,3 +236,53 @@ validated against the ADR-007 eval harness. insufficient. - Non-16:9 or non-borderless setups are explicitly unsupported for the auto scanner; the app should detect and warn rather than silently misread. + +## ADR-010: Elevated Dev Runner And Bounded Live Automation Probes + +### Status + +Accepted + +### Context + +Automatic grid scanning needs read-only mouse movement, click, and wheel input +to reach the focused Genshin window. A lower-integrity app can fail to deliver +input to an elevated or protected target because of Windows UIPI/integrity +boundaries. During live testing, `npm run dev:admin` originally printed that a +new Administrator window was started, but the elevated PowerShell received no +arguments, so the intended dev process did not reliably start. + +The project also needed a smaller live validation path than a full inventory +scan. A full scan is too risky as the first proof of input delivery because it +can click many tiles before a bad coordinate, focus issue, or blocked input is +understood. + +### Decision + +Keep automatic scan input automation read-only and require an elevated runtime +when Windows reports that automation would otherwise be blocked. Replace the +old `dev-admin.cmd` entry with `scripts/dev-admin.ps1`, quote the elevated +PowerShell arguments explicitly, and log elevated startup to +`outputs/admin-start/admin-dev.log`. + +Add dev-only HTTP checks: + +- `/automation/probe-click?index=N` or `?row=R&col=C` performs one safe + inventory selection click and verifies whether the detail panel changed. +- `/scanner/start?limit=N` sends a temporary scan-limit payload to the renderer, + so live auto-scan validation can start with two items instead of the UI + default. + +Document the workflow in [AUTOMATION_LIVE_SCAN.md](AUTOMATION_LIVE_SCAN.md). + +### Consequences + +- The user still has to approve Windows UAC manually; the app must not try to + click the Secure Desktop prompt. +- We can distinguish input delivery from OCR/parser quality with a one-click + probe before running any broader scan. +- Live validation now has a low-risk path: check elevation and Genshin + detection, run a single probe click, then run a bounded `limit=2` scan. +- The implementation remains inside the allowed safety boundary: no memory + reads, hooks, injection, game-file modification, deleting, feeding, enhancing, + locking/unlocking, or spending resources. diff --git a/docs/PROJECT.md b/docs/PROJECT.md index ee2824c..1206ddf 100644 --- a/docs/PROJECT.md +++ b/docs/PROJECT.md @@ -72,7 +72,7 @@ The app is not intended to replace deep min-max tools. It prioritizes time savin | Styling | CSS with dark purple glassmorphism system | Premium fintech-inspired visual direction | | OCR | Tesseract.js prototype plus deterministic normalization/derivation | OCR alone is not trusted as the decision source | | Capture | Electron desktopCapturer plus Windows GDI Smart Capture | GDI path is used for Genshin Smart Capture reliability | -| Input automation | PowerShell sidecar prototype now, native sidecar planned | Current sidecar is good for proving behavior, not the final production path | +| Input automation | C# sidecar with elevated dev runner when needed | Live-validated for read-only inventory selection clicks; see `docs/AUTOMATION_LIVE_SCAN.md` | | Tests | Vitest + TypeScript checks | Current validation baseline; regression samples must expand | | Packaging | electron-builder | Configured in `package.json` | @@ -95,11 +95,15 @@ The app is not intended to replace deep min-max tools. It prioritizes time savin - The parser already uses known sets, pieces, slots, stat aliases, set aliases, character aliases, and derived slot/set mapping. - Review samples, learned replacements, parser notes, and stored artifacts already persist locally. - The auto-scan loop is no longer a naive click spammer: it has preflight, verification, miss handling, page fingerprinting, and stop conditions. +- Elevated live automation is validated in the current dev environment: + `/automation/probe-click?index=1` changed the selected artifact and + `/scanner/start?limit=2` completed with 2/2 verified reads and 0 misses. ### What is still structurally weak - The scan experience is still partly orchestrated from `src/App.tsx`, which makes behavior changes harder than they should be. -- The current PowerShell input sidecar is serviceable for experimentation but not a strong production base for long-running, low-jitter auto-scan. +- Broader scan soak testing still needs to increase the live limit gradually and + validate scroll/page transitions beyond the first visible row. - OCR quality is still inconsistent enough that some fields are recovered by fallback and derivation more often than they should be. - Learned fixes currently focus on text replacements; they do not yet update crop offsets, UI profile variants, or scanner targeting rules in a structured way. - The scan page is cleaner than before, but it still exposes too much operator/debug state in the main flow. @@ -196,7 +200,7 @@ Outcome: - Auto-scan never starts on a session that cannot prove one successful detail-card change. Status: -- Planned +- First live path validated; broader soak testing still needed ### Phase 5 - Learning loop that actually compounds @@ -228,7 +232,7 @@ Status: 1. Finish scan-page cleanup so the main operator view is no longer noisy. 2. Tighten the game data generator and parser contract, then backfill regression tests from real bad samples. 3. Continue moving auto-scan behavior out of `App.tsx` and into isolated scanner modules. -4. Replace or wrap the current PowerShell sidecar with a more stable long-lived automation process. +4. Soak-test the elevated C# helper automation path with gradually larger scan limits and page scroll transitions. 5. Extend the learning system from text-only fixes into crop/UI profile tuning. 6. Resume recommendation work only when scan accuracy is consistently trustworthy. @@ -236,7 +240,7 @@ Status: | Question | Status | | --- | --- | -| Should the production input sidecar be Rust/C++ first, or a transitional Node native addon, for the next iteration? | Open | +| Is the current C# helper sufficient for production packaging, or does a later Rust/C++ sidecar still materially reduce latency or packaging risk? | Open | | When should UI-profile learning be allowed to change crop geometry automatically versus requiring review approval? | Open | | What scan-quality threshold is high enough before recommendations should be considered user-facing again? | Open | | Which Genshin UI languages should be supported after English once the scanner contract is stable? | Open | diff --git a/docs/scanner-rework-status.md b/docs/scanner-rework-status.md index fbe51b4..65fe1df 100644 --- a/docs/scanner-rework-status.md +++ b/docs/scanner-rework-status.md @@ -1,7 +1,9 @@ # Scanner rework status -Progress on the approved scanner/OCR rework. See ADR-007/008/009 in -[DECISIONS.md](DECISIONS.md) for the decisions behind these. +Progress on the approved scanner/OCR rework. See ADR-007/008/009/010 in +[DECISIONS.md](DECISIONS.md) for the decisions behind these. For the current +live automation runbook, see +[AUTOMATION_LIVE_SCAN.md](AUTOMATION_LIVE_SCAN.md). ## Done (implemented, unit-tested, build green) @@ -12,34 +14,44 @@ Progress on the approved scanner/OCR rework. See ADR-007/008/009 in fallback. Verified end-to-end (spawn, runtime, base64 capture). - **Layout profiles + OCR preprocessing** — `src/lib/layoutProfile.ts` (pure geometry, 16:9 detection), `src/lib/ocrPreprocess.ts` (grayscale + Otsu - binarize). main.ts crops via the profile and OCRs an upscaled + binarized copy. + binarize). main.ts now uses calibrated 16:9 detail/count/grid coordinates + first and OCRs an upscaled + binarized copy. - **Card-ready gating** — `src/lib/cardReadyGate.ts` replaces the fixed 280 ms settle with change+stability polling; robust to animation. - **GOOD interop** — `src/lib/goodInterop.ts` (export + best-effort import for - scanned records). + scanned records), Electron file-picker import/export, and store merge. - **Rescan-merge** — `src/lib/artifactMerge.ts` collapses leveled re-scan duplicates by a level-independent identity. - **Data staleness warning** — `src/lib/dataPackageStatus.ts`, surfaced in the Scanner Diagnose data-package line. -- **Lock detection (experimental)** — `src/lib/lockDetection.ts`, pure heuristic, - not yet wired into capture. +- **Lock detection (experimental)** — `src/lib/lockDetection.ts`, wired into + live capture as a read-only `locked` flag and persisted with scanned records. +- **Elevated live automation path** — `npm run dev:admin` now starts through + `scripts/dev-admin.ps1` and logs to `outputs/admin-start/admin-dev.log`. + Live status confirmed `isElevated: true`, `genshinFound: true`, and + `targetProcess: "GenshinImpact"`. +- **Read-only click probe** — `/automation/probe-click?index=1` verified that + the app can focus Genshin, move to a visible inventory tile, click it, and + observe a changed detail panel fingerprint (`clicked: true`, + `inputBlocked: false`, `changed: true`). +- **Bounded auto-scan validation** — `/scanner/start?limit=2` completed live + with 2 clicks, 2 verified detail views, 2 parsed artifacts, 2 stored records, + 2 review samples, and 0 misses. ## Remaining — needs the live environment or a UI pass These cannot be finished/validated without Genshin running at the user's resolution or without UI work best tested live: -1. **Calibrate IK-style fixed crop coordinates** (ADR-009). The layout module is - the structure; the exact per-field fractions still come from a - colour-detected/fallback detail rect. A reference 16:9 screenshot of the - artifact screen lets us pin exact client-relative crop coordinates. -2. **Validate/tune OCR preprocessing** on real captures — confirm invert + +1. **Validate/tune OCR preprocessing** on more real captures — confirm invert + threshold + upscale factor help (not hurt) actual Tesseract reads. The text-level eval harness cannot measure image preprocessing. -3. **Wire GOOD import** — file-picker IPC + merge imported records into the store - (the conversion engine is done and tested). -4. **Wire live lock detection** — calibrate crop position/threshold against a - reference screenshot, then populate a `locked` flag during capture. +2. **Validate locked=true** against a known locked artifact — unlocked/grey lock + was live-checked; a gold locked icon still needs a positive sample. + +3. **Broader scan soak test** — after the bounded two-item live scan passed, + the next automation validation should increase the limit gradually and watch + for repeated pages, scroll behavior, duplicate handling, and OCR review rate. ## Grow the eval corpus diff --git a/electron/bootstrap/ipcBootstrap.ts b/electron/bootstrap/ipcBootstrap.ts index 511dc16..3e290e2 100644 --- a/electron/bootstrap/ipcBootstrap.ts +++ b/electron/bootstrap/ipcBootstrap.ts @@ -17,6 +17,7 @@ import type { SaveResultWithPath, SaveSnapshotResult, GoodDatabase, + GoodImportFileResult, ScannerStatusPayload, } from "../../src/types/global.js"; import type { @@ -51,6 +52,7 @@ interface PersistenceHandlersDependencies { loadScannerLearningRules: () => Promise; writeScannerLearningRules: (rules: ScannerLearningRulePayload) => Promise; exportGood: (payload: GoodDatabase) => Promise; + importGoodFile: () => Promise; } interface CaptureHandlersDependencies { @@ -86,6 +88,7 @@ export function registerIpcHandlers(dependencies: IpcBootstrapDependencies) { loadScannerLearningRules: dependencies.loadScannerLearningRules, writeScannerLearningRules: dependencies.writeScannerLearningRules, exportGood: dependencies.exportGood, + importGoodFile: dependencies.importGoodFile, }); registerCaptureHandlers({ diff --git a/electron/devControlServer.ts b/electron/devControlServer.ts new file mode 100644 index 0000000..bd9cebe --- /dev/null +++ b/electron/devControlServer.ts @@ -0,0 +1,256 @@ +import fs from "node:fs/promises"; +import http, { type Server } from "node:http"; +import path from "node:path"; +import type { + CaptureOptions, + CaptureResult, + CaptureSourceInfo, + ClickResult, + ReviewSampleListResult, + ScannerCommand, + ScannerStatusPayload, +} from "../src/types/global.js"; + +interface DevControlServerDependencies { + registeredHotkeys: Record; + hasMainWindow: () => boolean; + sendScannerCommand: (command: ScannerCommand | "probe-click") => void; + clickScreen: (x: number, y: number) => Promise; + scannerStatus: () => ScannerStatusPayload; + loadReviewSamples: (limit?: number) => Promise; + listCaptureSources: () => Promise; + captureSource: ( + id: string, + delayMs?: number, + focusGenshin?: boolean, + options?: CaptureOptions, + ) => Promise; +} + +function writeDevJson(res: http.ServerResponse, statusCode: number, payload: unknown) { + res.writeHead(statusCode, { + "content-type": "application/json; charset=utf-8", + "cache-control": "no-store", + }); + res.end(JSON.stringify(payload)); +} + +function dataUrlBase64(dataUrl: string) { + return dataUrl.replace(/^data:image\/png;base64,/, ""); +} + +function devCaptureOutputDir() { + return path.join(process.cwd(), "outputs", "live-capture"); +} + +function safeDebugFilePart(value: string) { + return value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture"; +} + +function dataUrlFingerprint(dataUrl: string | undefined) { + if (!dataUrl) return ""; + let hash = 2166136261; + const stride = Math.max(1, Math.floor(dataUrl.length / 4096)); + for (let index = 0; index < dataUrl.length; index += stride) { + hash ^= dataUrl.charCodeAt(index); + hash = Math.imul(hash, 16777619); + } + return `${dataUrl.length.toString(16)}:${(hash >>> 0).toString(16)}`; +} + +async function writeDevCaptureImage(filePath: string, dataUrl: string | undefined) { + if (!dataUrl) return null; + await fs.writeFile(filePath, Buffer.from(dataUrlBase64(dataUrl), "base64")); + return filePath; +} + +async function writeDevCaptureSnapshot(capture: CaptureResult) { + const outputDir = devCaptureOutputDir(); + await fs.mkdir(outputDir, { recursive: true }); + const stamp = new Date().toISOString().replace(/[\\/:]/g, "-").replace(/\..+?$/, ""); + const prefix = safeDebugFilePart(`${stamp}-${capture.name}`); + const files = { + full: await writeDevCaptureImage(path.join(outputDir, `${prefix}-full.png`), capture.dataUrl), + detail: await writeDevCaptureImage(path.join(outputDir, `${prefix}-detail.png`), capture.detailDataUrl), + inventory: await writeDevCaptureImage(path.join(outputDir, `${prefix}-inventory.png`), capture.inventoryDataUrl), + crops: [] as Array<{ id: string; label: string; path: string; rect: { x: number; y: number; width: number; height: number } }>, + }; + + for (const crop of capture.crops ?? []) { + const cropPath = path.join(outputDir, `${prefix}-${safeDebugFilePart(crop.id)}.png`); + const written = await writeDevCaptureImage(cropPath, crop.dataUrl); + if (written) files.crops.push({ id: crop.id, label: crop.label, path: written, rect: crop.rect }); + } + + const summary = { + id: capture.id, + name: capture.name, + width: capture.width, + height: capture.height, + capturedAt: capture.capturedAt, + captureTarget: capture.captureTarget, + ocrSkipped: capture.ocrSkipped, + ocrTimedOut: capture.ocrTimedOut, + layout: capture.layout, + inventoryGrid: capture.inventoryGrid + ? { + rows: capture.inventoryGrid.rows, + cols: capture.inventoryGrid.cols, + confidence: capture.inventoryGrid.confidence, + source: capture.inventoryGrid.source, + firstCenter: capture.inventoryGrid.centers[0] ?? null, + lastCenter: capture.inventoryGrid.centers.at(-1) ?? null, + } + : null, + inventoryCount: capture.inventoryCount ?? null, + locked: capture.locked, + crops: (capture.crops ?? []).map((crop) => ({ id: crop.id, label: crop.label, rect: crop.rect })), + ocr: capture.ocr ?? [], + files, + }; + const summaryPath = path.join(outputDir, `${prefix}-summary.json`); + await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2), "utf8"); + return { ...summary, summaryPath }; +} + +function wait(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function findGenshinSource(sources: CaptureSourceInfo[], sourceId: string | null) { + return sourceId + ? sources.find((entry) => entry.id === sourceId) + : sources.find((entry) => entry.isGenshinCandidate); +} + +function sourceListForError(sources: CaptureSourceInfo[]) { + return sources.map(({ id, name, isGenshinCandidate }) => ({ id, name, isGenshinCandidate })); +} + +export function createDevControlServer(deps: DevControlServerDependencies): Server { + const server = http.createServer((req, res) => { + if (req.socket.remoteAddress && !["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress)) { + writeDevJson(res, 403, { ok: false, error: "local only" }); + return; + } + + const url = new URL(req.url ?? "/", "http://127.0.0.1"); + if (url.pathname === "/health") { + writeDevJson(res, 200, { ok: true, hotkeys: deps.registeredHotkeys, hasWindow: deps.hasMainWindow() }); + return; + } + if (url.pathname === "/scanner/start") { + const limit = Number(url.searchParams.get("limit") ?? Number.NaN); + const command: ScannerCommand = Number.isFinite(limit) && limit > 0 + ? { type: "start-auto", scanLimit: limit } + : "start-auto"; + deps.sendScannerCommand(command); + writeDevJson(res, 200, { ok: true, command }); + return; + } + if (url.pathname === "/scanner/stop") { + deps.sendScannerCommand("stop"); + writeDevJson(res, 200, { ok: true, command: "stop" }); + return; + } + if (url.pathname === "/scanner/probe") { + deps.sendScannerCommand("probe-click"); + writeDevJson(res, 200, { ok: true, command: "probe-click" }); + return; + } + if (url.pathname === "/automation/click") { + const x = Number(url.searchParams.get("x")); + const y = Number(url.searchParams.get("y")); + if (!Number.isFinite(x) || !Number.isFinite(y)) { + writeDevJson(res, 400, { ok: false, error: "x and y query params are required" }); + return; + } + deps.clickScreen(Math.round(x), Math.round(y)) + .then((payload: unknown) => writeDevJson(res, 200, { ok: true, payload })) + .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) })); + return; + } + if (url.pathname === "/scanner/status") { + writeDevJson(res, 200, { ok: true, status: deps.scannerStatus() }); + return; + } + if (url.pathname === "/review/samples") { + deps.loadReviewSamples(Number(url.searchParams.get("limit") ?? 20)) + .then((payload: unknown) => writeDevJson(res, 200, payload)) + .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) })); + return; + } + if (url.pathname === "/capture/smart") { + const sourceId = url.searchParams.get("sourceId"); + const focus = url.searchParams.get("focus") !== "0"; + const skipOcr = url.searchParams.get("skipOcr") === "1"; + deps.listCaptureSources() + .then(async (sources) => { + const source = findGenshinSource(sources, sourceId); + if (!source) { + writeDevJson(res, 404, { ok: false, error: "No Genshin capture source found.", sources: sourceListForError(sources) }); + return; + } + const capture = await deps.captureSource(source.id, 250, focus, { skipOcr }); + const summary = await writeDevCaptureSnapshot(capture); + writeDevJson(res, 200, { ok: true, source: { id: source.id, name: source.name }, summary }); + }) + .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) })); + return; + } + if (url.pathname === "/automation/probe-click") { + const sourceId = url.searchParams.get("sourceId"); + const requestedIndex = Number(url.searchParams.get("index") ?? "1"); + const requestedRow = Number(url.searchParams.get("row") ?? Number.NaN); + const requestedCol = Number(url.searchParams.get("col") ?? Number.NaN); + deps.listCaptureSources() + .then(async (sources) => { + const source = findGenshinSource(sources, sourceId); + if (!source) { + writeDevJson(res, 404, { ok: false, error: "No Genshin capture source found.", sources: sourceListForError(sources) }); + return; + } + + const before = await deps.captureSource(source.id, 150, true, { skipOcr: true }); + const centers = before.inventoryGrid?.centers ?? []; + const target = Number.isFinite(requestedRow) && Number.isFinite(requestedCol) + ? centers.find((center) => center.row === requestedRow && center.col === requestedCol) + : centers[Math.max(0, Math.min(centers.length - 1, Number.isFinite(requestedIndex) ? requestedIndex : 1))]; + if (!target) { + writeDevJson(res, 409, { ok: false, error: "No inventory grid target available.", grid: before.inventoryGrid ?? null }); + return; + } + + const beforeFingerprint = dataUrlFingerprint(before.detailDataUrl); + const click = await deps.clickScreen(target.x, target.y); + await wait(650); + const after = await deps.captureSource(source.id, 0, true, { skipOcr: true }); + const afterFingerprint = dataUrlFingerprint(after.detailDataUrl); + const changed = Boolean(beforeFingerprint && afterFingerprint && beforeFingerprint !== afterFingerprint); + writeDevJson(res, 200, { + ok: Boolean(click.ok && click.clicked && changed), + changed, + target, + click, + before: { + captureTarget: before.captureTarget, + grid: before.inventoryGrid ? { rows: before.inventoryGrid.rows, cols: before.inventoryGrid.cols, source: before.inventoryGrid.source, confidence: before.inventoryGrid.confidence } : null, + detailFingerprint: beforeFingerprint, + }, + after: { + captureTarget: after.captureTarget, + grid: after.inventoryGrid ? { rows: after.inventoryGrid.rows, cols: after.inventoryGrid.cols, source: after.inventoryGrid.source, confidence: after.inventoryGrid.confidence } : null, + detailFingerprint: afterFingerprint, + }, + }); + }) + .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) })); + return; + } + + writeDevJson(res, 404, { ok: false, error: "unknown endpoint" }); + }); + + server.listen(17317, "127.0.0.1"); + return server; +} diff --git a/electron/ipc/persistenceHandlers.ts b/electron/ipc/persistenceHandlers.ts index 1cb48dd..22b1000 100644 --- a/electron/ipc/persistenceHandlers.ts +++ b/electron/ipc/persistenceHandlers.ts @@ -13,6 +13,7 @@ import type { SaveScannerLearningRulesResult, ScannerLearningRulePayload, SaveResultWithPath, + GoodImportFileResult, } from "../../src/types/global.js"; import type { StoredArtifactRecord } from "../../src/types/storage.js"; @@ -28,6 +29,7 @@ interface PersistenceDependencies { loadScannerLearningRules: () => Promise; writeScannerLearningRules: (rules: ScannerLearningRulePayload) => Promise; exportGood: (payload: GoodDatabase) => Promise; + importGoodFile: () => Promise; } export function registerPersistenceHandlers({ @@ -39,6 +41,7 @@ export function registerPersistenceHandlers({ loadScannerLearningRules, writeScannerLearningRules, exportGood, + importGoodFile, }: PersistenceDependencies) { ipcMain.handle("review:saveSample", async (_event, sample: ReviewSamplePayload) => { try { @@ -81,4 +84,8 @@ export function registerPersistenceHandlers({ ipcMain.handle("good:export", async (_event, payload: GoodDatabase) => { return exportGood(payload); }); + + ipcMain.handle("good:importFile", async () => { + return importGoodFile(); + }); } diff --git a/electron/main.ts b/electron/main.ts index e2391e6..e056258 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -1,19 +1,22 @@ -import { app, BrowserWindow, Menu, desktopCapturer, globalShortcut, nativeImage, screen, type NativeImage } from "electron"; +import { app, BrowserWindow, Menu, desktopCapturer, dialog, globalShortcut, nativeImage, screen, type NativeImage } from "electron"; import fs from "node:fs/promises"; import { existsSync } from "node:fs"; -import http, { type Server } from "node:http"; +import type { Server } from "node:http"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { createWorker } from "tesseract.js"; import { createInputHelperService, type InputHelperService } from "./services/inputHelper.js"; import { createRepositoryContext, type RepositoryContext } from "./bootstrap/repositoryContext.js"; import { registerIpcHandlers } from "./bootstrap/ipcBootstrap.js"; +import { createDevControlServer } from "./devControlServer.js"; import type { AppSnapshot } from "../src/types/domain.js"; import type { CaptureOptions, CaptureResult, GoodDatabase, + GoodImportFileResult, SaveResultWithPath, + ScannerCommand, ScannerLearningRulePayload, ScannerStatusPayload, } from "../src/types/global.js"; @@ -33,6 +36,7 @@ import { profileDetailRect, } from "../src/lib/layoutProfile.js"; import { binarizeForOcr } from "../src/lib/ocrPreprocess.js"; +import { detectLockState, lockIconCropRect } from "../src/lib/lockDetection.js"; // Chromium's renderer sandbox can refuse to fully initialize (or silently // crash the GPU/renderer process) when the hosting process runs with a full @@ -416,7 +420,7 @@ function focusMainWindow() { return { ok: true }; } -function sendScannerCommand(command: "start-auto" | "stop" | "probe-click") { +function sendScannerCommand(command: ScannerCommand | "probe-click") { if (!mainWindow || mainWindow.isDestroyed()) return; mainWindow.webContents.send("scanner:command", command); } @@ -431,71 +435,18 @@ function registerScannerHotkeys() { }; } -function writeDevJson(res: http.ServerResponse, statusCode: number, payload: unknown) { - res.writeHead(statusCode, { - "content-type": "application/json; charset=utf-8", - "cache-control": "no-store", - }); - res.end(JSON.stringify(payload)); -} - function startDevControlServer() { if (!isDev || devControlServer) return; - - devControlServer = http.createServer((req, res) => { - if (req.socket.remoteAddress && !["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress)) { - writeDevJson(res, 403, { ok: false, error: "local only" }); - return; - } - - const url = new URL(req.url ?? "/", "http://127.0.0.1"); - if (url.pathname === "/health") { - writeDevJson(res, 200, { ok: true, hotkeys: registeredHotkeys, hasWindow: Boolean(mainWindow && !mainWindow.isDestroyed()) }); - return; - } - if (url.pathname === "/scanner/start") { - sendScannerCommand("start-auto"); - writeDevJson(res, 200, { ok: true, command: "start-auto" }); - return; - } - if (url.pathname === "/scanner/stop") { - sendScannerCommand("stop"); - writeDevJson(res, 200, { ok: true, command: "stop" }); - return; - } - if (url.pathname === "/scanner/probe") { - sendScannerCommand("probe-click"); - writeDevJson(res, 200, { ok: true, command: "probe-click" }); - return; - } - if (url.pathname === "/automation/click") { - const x = Number(url.searchParams.get("x")); - const y = Number(url.searchParams.get("y")); - if (!Number.isFinite(x) || !Number.isFinite(y)) { - writeDevJson(res, 400, { ok: false, error: "x and y query params are required" }); - return; - } - getInputHelperService() - .clickScreen(Math.round(x), Math.round(y)) - .then((payload: unknown) => writeDevJson(res, 200, { ok: true, payload })) - .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) })); - return; - } - if (url.pathname === "/scanner/status") { - writeDevJson(res, 200, { ok: true, status: scannerDevStatus }); - return; - } - if (url.pathname === "/review/samples") { - loadReviewSamples(Number(url.searchParams.get("limit") ?? 20)) - .then((payload: unknown) => writeDevJson(res, 200, payload)) - .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) })); - return; - } - - writeDevJson(res, 404, { ok: false, error: "unknown endpoint" }); + devControlServer = createDevControlServer({ + registeredHotkeys, + hasMainWindow: () => Boolean(mainWindow && !mainWindow.isDestroyed()), + sendScannerCommand, + clickScreen: clickScreenCommand, + scannerStatus: () => scannerDevStatus, + loadReviewSamples, + listCaptureSources, + captureSource, }); - - devControlServer.listen(17317, "127.0.0.1"); } function createOverlayWindow() { @@ -802,6 +753,10 @@ function createCrops( } function inferDetailRect(bitmap: Buffer, imageSize: { width: number; height: number }) { + if (isSixteenNine(imageSize)) { + return profileDetailRect(imageSize); + } + const { width, height } = imageSize; const sampleStrideX = width > 2200 ? 4 : 3; const sampleStrideY = height > 1400 ? 4 : 3; @@ -869,6 +824,12 @@ async function buildCaptureResult( const detailRect = inferDetailRect(bitmap, size); const inventoryRect = inferInventoryRect(size, detailRect); const crops = createCrops(sourceImage, size, detailRect, inventoryRect); + const lockRect = clampCaptureRect(lockIconCropRect(detailRect, size), size); + const lockImage = sourceImage.crop(lockRect); + const lockSize = lockImage.getSize(); + const locked = lockSize.width > 0 && lockSize.height > 0 + ? detectLockState({ data: lockImage.getBitmap(), width: lockSize.width, height: lockSize.height }) + : undefined; const croppedPayload = crops.map((crop) => ({ id: crop.id, label: crop.label, @@ -905,6 +866,7 @@ async function buildCaptureResult( })), inventoryGrid: inferInventoryGrid(size, detailRect), inventoryCount: count, + locked, layout: { aspect: aspectRatioLabel(size), isSixteenNine: isSixteenNine(size), @@ -965,6 +927,34 @@ async function exportGood(payload: GoodDatabase): Promise { } } +async function importGoodFile(): Promise { + const dialogOptions = { + title: "GOOD-Datei importieren", + properties: ["openFile"], + filters: [{ name: "GOOD JSON", extensions: ["json"] }], + } satisfies Electron.OpenDialogOptions; + const dialogResult = mainWindow && !mainWindow.isDestroyed() + ? await dialog.showOpenDialog(mainWindow, dialogOptions) + : await dialog.showOpenDialog(dialogOptions); + + if (dialogResult.canceled || dialogResult.filePaths.length === 0) { + return { ok: false, canceled: true, path: "" }; + } + + const filePath = dialogResult.filePaths[0]; + try { + const text = await fs.readFile(filePath, "utf8"); + return { ok: true, canceled: false, path: filePath, database: JSON.parse(text) }; + } catch (error) { + return { + ok: false, + canceled: false, + path: filePath, + error: error instanceof Error ? error.message : String(error), + }; + } +} + function initializeAppLifecycle() { app.whenReady().then(() => { const userDataPath = app.getPath("userData"); @@ -993,6 +983,7 @@ function initializeAppLifecycle() { loadScannerLearningRules: () => loadScannerLearningRules(), writeScannerLearningRules: (rules: ScannerLearningRulePayload) => writeScannerLearningRules(rules), exportGood: (exportPayload: GoodDatabase) => exportGood(exportPayload), + importGoodFile: () => importGoodFile(), listSources: () => listCaptureSources(), captureSource: ( id: string, diff --git a/electron/preload.cjs b/electron/preload.cjs index d120be0..304f8f9 100644 --- a/electron/preload.cjs +++ b/electron/preload.cjs @@ -20,6 +20,7 @@ contextBridge.exposeInMainWorld("assistantApi", { loadArtifacts: () => ipcRenderer.invoke("artifacts:load"), saveArtifacts: (records) => ipcRenderer.invoke("artifacts:saveMany", records), exportGood: (payload) => ipcRenderer.invoke("good:export", payload), + importGoodFile: () => ipcRenderer.invoke("good:importFile"), publishScannerStatus: (status) => ipcRenderer.invoke("scanner:publishStatus", status), showOverlay: () => ipcRenderer.invoke("overlay:show"), hideOverlay: () => ipcRenderer.invoke("overlay:hide"), diff --git a/electron/preload.ts b/electron/preload.ts index d3fc2aa..12ae90e 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,5 +1,5 @@ import { contextBridge, ipcRenderer } from "electron"; -import type { CaptureOptions, GoodDatabase, ReviewSamplePayload, ScannerStatusPayload, ScannerLearningRulePayload } from "../src/types/global.js"; +import type { CaptureOptions, GoodDatabase, ReviewSamplePayload, ScannerCommand, ScannerStatusPayload, ScannerLearningRulePayload } from "../src/types/global.js"; import type { StoredArtifactRecord } from "../src/types/storage.js"; import type { AppSnapshot } from "../src/types/domain.js"; @@ -23,11 +23,12 @@ contextBridge.exposeInMainWorld("assistantApi", { loadArtifacts: () => ipcRenderer.invoke("artifacts:load"), saveArtifacts: (records: StoredArtifactRecord[]) => ipcRenderer.invoke("artifacts:saveMany", records), exportGood: (payload: GoodDatabase) => ipcRenderer.invoke("good:export", payload), + importGoodFile: () => ipcRenderer.invoke("good:importFile"), publishScannerStatus: (status: ScannerStatusPayload) => ipcRenderer.invoke("scanner:publishStatus", status), showOverlay: () => ipcRenderer.invoke("overlay:show"), hideOverlay: () => ipcRenderer.invoke("overlay:hide"), - onScannerCommand: (callback: (command: "start-auto" | "stop") => void) => { - const listener = (_event: Electron.IpcRendererEvent, command: "start-auto" | "stop") => callback(command); + onScannerCommand: (callback: (command: ScannerCommand) => void) => { + const listener = (_event: Electron.IpcRendererEvent, command: ScannerCommand) => callback(command); ipcRenderer.on("scanner:command", listener); return () => ipcRenderer.removeListener("scanner:command", listener); }, diff --git a/electron/repositories/artifactStoreRepository.ts b/electron/repositories/artifactStoreRepository.ts index bf0797d..3d7a1e5 100644 --- a/electron/repositories/artifactStoreRepository.ts +++ b/electron/repositories/artifactStoreRepository.ts @@ -54,6 +54,7 @@ export class JsonArtifactStoreRepository implements ArtifactStoreRepositoryPort lastSeenAt: now, timesSeen: (existing.timesSeen ?? 1) + 1, confidence: Math.max(existing.confidence ?? 0, record.confidence ?? 0), + locked: typeof record.locked === "boolean" ? record.locked : existing.locked, // A later confident scan clears the review flag; an uncertain rescan // must not downgrade an already confirmed artifact. needsReview: Boolean(existing.needsReview) && Boolean(record.needsReview), @@ -136,6 +137,7 @@ function normalizeStoredArtifactRecordForLoad(record: StoredArtifactRecord) { ...record, timesSeen: reviewOnly ? 1 : normalizedTimesSeen, firstSeenAt: record.firstSeenAt ?? record.lastSeenAt, + locked: typeof record.locked === "boolean" ? record.locked : undefined, }; } @@ -183,6 +185,7 @@ function mergeArtifactRecords(existing: StoredArtifactRecord, incoming: StoredAr substats: [...(preferredSubstats ?? [])], equipped: preferred.equipped && preferred.equipped !== "Not detected" ? preferred.equipped : secondary.equipped, confidence: Math.max(existing.confidence ?? 0, incoming.confidence ?? 0), + locked: typeof incoming.locked === "boolean" ? incoming.locked : existing.locked, needsReview: Boolean(existing.needsReview) && Boolean(incoming.needsReview), source: resolveStoredArtifactSource(existing.source, incoming.source), firstSeenAt: existing.firstSeenAt ?? now, diff --git a/package.json b/package.json index 612f525..b91295f 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "predev": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\kill-stale-instances.ps1 && tsc -p tsconfig.electron.json && copy electron\\preload.cjs dist-electron\\electron\\preload.cjs", "dev": "concurrently -k \"vite --host 127.0.0.1\" \"wait-on tcp:5173 && cross-env VITE_DEV_SERVER_URL=http://127.0.0.1:5173 electron .\"", "dev:web": "vite --host 127.0.0.1", - "dev:admin": ".\\dev-admin.cmd", + "dev:admin": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\dev-admin.ps1 -ProjectRoot .", "build": "tsc && vite build && tsc -p tsconfig.electron.json && copy electron\\\\preload.cjs dist-electron\\\\electron\\\\preload.cjs", "preview": "vite preview --host 127.0.0.1", "start": "electron .", diff --git a/scripts/dev-admin-start.ps1 b/scripts/dev-admin-start.ps1 index fe13513..59f652d 100644 --- a/scripts/dev-admin-start.ps1 +++ b/scripts/dev-admin-start.ps1 @@ -9,8 +9,17 @@ $ErrorActionPreference = "Stop" try { $project = (Resolve-Path -LiteralPath $ProjectRoot).Path + $logDir = Join-Path $project "outputs\admin-start" + New-Item -ItemType Directory -Force -Path $logDir | Out-Null + $logPath = Join-Path $logDir "admin-dev.log" + try { + Start-Transcript -Path $logPath -Append | Out-Null + } catch { + Write-Host "WARNUNG: Admin-Start-Log konnte nicht geschrieben werden: $($_.Exception.Message)" -ForegroundColor Yellow + } Write-Host "Projekt: $project" + Write-Host "Admin-Log: $logPath" # A UAC-elevated process gets its environment rebuilt fresh from the # registry; it does NOT inherit PATH edits that only exist in the calling diff --git a/scripts/dev-admin.ps1 b/scripts/dev-admin.ps1 new file mode 100644 index 0000000..38287f8 --- /dev/null +++ b/scripts/dev-admin.ps1 @@ -0,0 +1,33 @@ +param( + [string]$ProjectRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "..")).Path +) + +$ErrorActionPreference = "Stop" + +function Quote-ProcessArgument([string]$Value) { + return '"' + $Value.Replace('"', '\"') + '"' +} + +try { + $project = (Resolve-Path -LiteralPath $ProjectRoot).Path + $script = Join-Path $PSScriptRoot "dev-admin-start.ps1" + $powershellExe = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" + $arguments = @( + "-NoProfile", + "-ExecutionPolicy Bypass", + "-NoExit", + "-File $(Quote-ProcessArgument $script)", + "-ProjectRoot $(Quote-ProcessArgument $project)" + ) -join " " + + Start-Process -FilePath $powershellExe -ArgumentList $arguments -WorkingDirectory $project -Verb RunAs -WindowStyle Normal -ErrorAction Stop + + Write-Host "" + Write-Host "UAC-Abfrage gestartet. Bitte bestaetigen - danach oeffnet sich ein neues Administrator-Fenster mit npm run dev." -ForegroundColor Green + Write-Host "Dieses Fenster kann geschlossen werden; das eigentliche Programm laeuft im neuen Administrator-Fenster." +} catch { + Write-Host "" + Write-Host "Admin-Start fehlgeschlagen oder UAC-Abfrage abgelehnt:" -ForegroundColor Red + Write-Host $_.Exception.Message -ForegroundColor Red + exit 1 +} diff --git a/src/features/scan/components/DiagnosticsView.tsx b/src/features/scan/components/DiagnosticsView.tsx index bd59eea..d13ccbb 100644 --- a/src/features/scan/components/DiagnosticsView.tsx +++ b/src/features/scan/components/DiagnosticsView.tsx @@ -1,8 +1,7 @@ -import { useRef, useState, type ChangeEvent } from "react"; +import { useState } from "react"; import { AlertTriangle, Download, Play, Upload, Wrench } from "lucide-react"; import type { CaptureResult } from "../../../types/global"; import type { ScanViewControllerResult } from "../types"; -import { goodDatabaseToStoredArtifacts, type GoodImportDatabase } from "../../../lib/goodInterop"; import { FieldConfidenceList } from "./ScanResultCards"; import { useScanDiagnosticsModalModel } from "./modals/hooks/useScanDiagnosticsModalModel"; import { useScanDetailsModalModel } from "./modals/hooks/useScanDetailsModalModel"; @@ -59,7 +58,6 @@ export function DiagnosticsView({ controller, latestCapture, captureStatus, onDe }); const [interopStatus, setInteropStatus] = useState(""); - const fileInputRef = useRef(null); const handleExportGood = async () => { setInteropStatus("Exportiere GOOD..."); @@ -71,27 +69,20 @@ export function DiagnosticsView({ controller, latestCapture, captureStatus, onDe ); }; - const handleImportGood = async (event: ChangeEvent) => { - const file = event.target.files?.[0]; - event.target.value = ""; - if (!file) return; - try { - const database = JSON.parse(await file.text()) as GoodImportDatabase; - const records = goodDatabaseToStoredArtifacts(database); - if (records.length === 0) { - setInteropStatus("Keine gueltigen Artifacts in der Datei gefunden."); - return; - } - setInteropStatus(`Importiere ${records.length} Artifacts...`); - const result = await controller.importGoodArtifacts(records); - setInteropStatus( - result.ok - ? `Importiert: ${result.added} neu, ${result.updated} aktualisiert.` - : "Import fehlgeschlagen (App im Electron-Fenster oeffnen).", - ); - } catch { - setInteropStatus("Datei ist kein gueltiges GOOD/JSON."); + const handleImportGood = async () => { + setInteropStatus("Waehle GOOD-Datei..."); + const result = await controller.importGoodFromFile(); + if (result.canceled) { + setInteropStatus("GOOD-Import abgebrochen."); + return; } + setInteropStatus( + result.ok + ? `Importiert: ${result.added} neu, ${result.updated} aktualisiert (${result.count} gelesen).` + : result.error === "No valid GOOD artifacts found." + ? "Keine gueltigen Artifacts in der Datei gefunden." + : "Import fehlgeschlagen (Datei ist kein gueltiges GOOD/JSON oder Bridge fehlt).", + ); }; return ( @@ -182,11 +173,10 @@ export function DiagnosticsView({ controller, latestCapture, captureStatus, onDe GOOD exportieren - -

diff --git a/src/features/scan/hooks/scanViewReviewHelpers.ts b/src/features/scan/hooks/scanViewReviewHelpers.ts index 4c32869..1fa662a 100644 --- a/src/features/scan/hooks/scanViewReviewHelpers.ts +++ b/src/features/scan/hooks/scanViewReviewHelpers.ts @@ -244,7 +244,7 @@ export async function persistParsedArtifact( return false; } try { - const result = await artifactRepo.saveMany([toStoredArtifact(parsed, source, needsReview)]); + const result = await artifactRepo.saveMany([toStoredArtifact(parsed, source, needsReview, capture?.locked)]); if (result?.ok) { setStoredTotal(result.total); void onStoredArtifactsChanged?.(); @@ -297,6 +297,7 @@ export async function saveReviewSample( })), inventoryGrid: capture.inventoryGrid, inventoryCount: capture.inventoryCount, + locked: capture.locked, ocr: capture.ocr, }, parsed, diff --git a/src/features/scan/hooks/scanViewScanActions.ts b/src/features/scan/hooks/scanViewScanActions.ts index ced3360..5d2c1e7 100644 --- a/src/features/scan/hooks/scanViewScanActions.ts +++ b/src/features/scan/hooks/scanViewScanActions.ts @@ -45,6 +45,10 @@ export interface ScanActionContext { focusDashboard: () => Promise; } +export interface VisibleGridScanOptions { + scanLimit?: number; +} + function buildScanSignature(parsed: ParsedArtifactCandidate) { return `${parsed.name}-${parsed.slot}-${parsed.mainStat}-${parsed.mainValue}-${parsed.level ?? 0}`; } @@ -146,7 +150,7 @@ export async function runAutoReviewScan(context: ScanActionContext): Promise { +export async function runVisibleGridScan(context: ScanActionContext, options: VisibleGridScanOptions = {}): Promise { const { autoScanRunning, bridgeReady, @@ -167,11 +171,12 @@ export async function runVisibleGridScan(context: ScanActionContext): Promise void; - runVisibleGridScan: () => Promise; + runVisibleGridScan: (options?: VisibleGridScanOptions) => Promise; } export function useScanCommandListener({ @@ -20,15 +22,16 @@ export function useScanCommandListener({ }: ScanCommandListenerInput) { useEffect(() => { if (!automationRepo?.onCommand) return; - return automationRepo.onCommand((command: "start-auto" | "stop") => { + return automationRepo.onCommand((command: ScannerCommand) => { if (command === "stop") { requestScanStop("Hotkey/Dev-Stop gedrueckt."); return; } - if (command === "start-auto" && !autoScanRunning && !isScanning && selectedSourceId) { - void runVisibleGridScan(); + const commandType = typeof command === "string" ? command : command.type; + if (commandType === "start-auto" && !autoScanRunning && !isScanning && selectedSourceId) { + const options = typeof command === "string" ? undefined : { scanLimit: command.scanLimit }; + void runVisibleGridScan(options); } }); }, [automationRepo, autoScanRunning, isScanning, selectedSourceId, requestScanStop, runVisibleGridScan]); } - diff --git a/src/features/scan/hooks/useScanViewActions.ts b/src/features/scan/hooks/useScanViewActions.ts index 56e665c..efe3ce1 100644 --- a/src/features/scan/hooks/useScanViewActions.ts +++ b/src/features/scan/hooks/useScanViewActions.ts @@ -1,6 +1,6 @@ import { useCallback, useEffect, useMemo } from "react"; import type { Dispatch, MutableRefObject, SetStateAction } from "react"; -import { runAutoReviewScan as runAutoReviewScanAction, runVisibleGridScan as runVisibleGridScanAction } from "./scanViewScanActions"; +import { runAutoReviewScan as runAutoReviewScanAction, runVisibleGridScan as runVisibleGridScanAction, type VisibleGridScanOptions } from "./scanViewScanActions"; import { initializeLearningState, loadReviewQueue as loadReviewQueueFromRepo, @@ -77,7 +77,7 @@ export interface ScanViewActionResult { loadReviewQueue: () => Promise; openReviewQueue: () => Promise; runAutoReviewScan: () => Promise; - runVisibleGridScan: () => Promise; + runVisibleGridScan: (options?: VisibleGridScanOptions) => Promise; } export function useScanViewActions(input: ScanViewActionInput): ScanViewActionResult { @@ -262,11 +262,11 @@ export function useScanViewActions(input: ScanViewActionInput): ScanViewActionRe await runAutoReviewScanAction(scanActionContext); }, [autoScanRunning, canCaptureSource, selectedSourceId, scanActionContext]); - const runVisibleGridScan = useCallback(async () => { + const runVisibleGridScan = useCallback(async (options: VisibleGridScanOptions = {}) => { if (autoScanRunning || !bridgeReady || !selectedSourceId || !automationRepo?.clickScreen || !automationRepo?.scrollScreen) { return; } - await runVisibleGridScanAction(scanActionContext); + await runVisibleGridScanAction(scanActionContext, options); }, [ autoScanRunning, bridgeReady, diff --git a/src/features/scan/hooks/useScanViewController.ts b/src/features/scan/hooks/useScanViewController.ts index 2a51761..468410d 100644 --- a/src/features/scan/hooks/useScanViewController.ts +++ b/src/features/scan/hooks/useScanViewController.ts @@ -17,7 +17,7 @@ import { emptyAutoScanStats, resolveScanTargetCount, type AutoScanStats, type Sc import type { ScanViewProps, ScanViewControllerResult } from "../types"; import type { CaptureResult, ClickResult, ReviewSampleRecord } from "../../../types/global"; import type { StoredArtifactRecord } from "../../../types/storage"; -import { storedArtifactsToGood } from "../../../lib/goodInterop"; +import { goodDatabaseToStoredArtifacts, type GoodImportDatabase, storedArtifactsToGood } from "../../../lib/goodInterop"; import { createRendererRepositories } from "../../../infrastructure/repositories/rendererBridgeRepositories"; export function useScanViewController({ @@ -174,6 +174,23 @@ export function useScanViewController({ return { ok: Boolean(result.ok), added: result.added ?? 0, updated: result.updated ?? 0 }; }, [artifactRepo, onStoredArtifactsChanged]); + const importGoodFromFile = useCallback(async () => { + if (!exportRepo?.importGoodFile || !artifactRepo?.saveMany) { + return { ok: false, added: 0, updated: 0, count: 0, error: "GOOD import is unavailable." }; + } + const fileResult = await exportRepo.importGoodFile(); + if (fileResult.canceled) return { ok: false, added: 0, updated: 0, count: 0, canceled: true }; + if (!fileResult.ok) { + return { ok: false, added: 0, updated: 0, count: 0, path: fileResult.path, error: fileResult.error }; + } + const records = goodDatabaseToStoredArtifacts(fileResult.database as GoodImportDatabase); + if (records.length === 0) { + return { ok: false, added: 0, updated: 0, count: 0, path: fileResult.path, error: "No valid GOOD artifacts found." }; + } + const saved = await importGoodArtifacts(records); + return { ...saved, count: records.length, path: fileResult.path }; + }, [artifactRepo, exportRepo, importGoodArtifacts]); + useScanViewStateSync({ artifactRepo, latestCapture, @@ -253,6 +270,7 @@ export function useScanViewController({ runVisibleGridScan, canGoodInterop, exportGoodFromStore, + importGoodFromFile, importGoodArtifacts, }; } diff --git a/src/features/scan/types.ts b/src/features/scan/types.ts index 3ce1056..76e35bd 100644 --- a/src/features/scan/types.ts +++ b/src/features/scan/types.ts @@ -82,5 +82,6 @@ export interface ScanViewControllerResult { runVisibleGridScan: () => Promise; canGoodInterop: boolean; exportGoodFromStore: () => Promise<{ ok: boolean; path?: string; count: number }>; + importGoodFromFile: () => Promise<{ ok: boolean; added: number; updated: number; count: number; canceled?: boolean; path?: string; error?: string }>; importGoodArtifacts: (records: StoredArtifactRecord[]) => Promise<{ ok: boolean; added: number; updated: number }>; } diff --git a/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts b/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts index 850edd4..2f94dc3 100644 --- a/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts +++ b/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts @@ -26,6 +26,7 @@ import type { ClickResult, ReviewSampleListResult, SaveScannerLearningRulesResult, + GoodImportFileResult, } from "../../types/global"; const EMPTY_SNAPSHOT: AppSnapshot | null = null; @@ -67,6 +68,12 @@ const EMPTY_SAVE_RULES_RESULT: SaveScannerLearningRulesResult = { rules: {}, total: 0, }; +const EMPTY_GOOD_IMPORT_FILE_RESULT: GoodImportFileResult = { + ok: false, + canceled: false, + path: "", + error: "Electron bridge unavailable.", +}; async function createBridgeSafeCall( callback: () => Promise | TResult | null | undefined, @@ -205,6 +212,7 @@ export function createRendererRepositories(): RendererRepositories | null { const exportRepo: ScanExportPort = { exportGood: (payload) => createBridgeSafeCall(() => bridge.exportGood(payload), EMPTY_SAVE_RESULT), + importGoodFile: () => createBridgeSafeCall(() => bridge.importGoodFile(), EMPTY_GOOD_IMPORT_FILE_RESULT), }; return { diff --git a/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts b/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts index 5a3d992..6a19363 100644 --- a/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts +++ b/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts @@ -8,8 +8,10 @@ import type { ScannerStatusPayload, ReviewSamplePayload, GoodDatabase, + GoodImportFileResult, FocusGenshinResult, RuntimeInfo, + ScannerCommand, LoadScannerLearningRulesResult, SaveScannerLearningRulesResult, ArtifactStoreLoadResult, @@ -62,7 +64,7 @@ export interface AutomationRepositoryPort { focusMainWindow(): Promise; clickScreen(x: number, y: number): Promise; scrollScreen(notches: number, anchorX?: number, anchorY?: number): Promise; - onCommand(callback: (command: "start-auto" | "stop") => void): () => void; + onCommand(callback: (command: ScannerCommand) => void): () => void; } export interface OverlayRepositoryPort { @@ -71,6 +73,7 @@ export interface OverlayRepositoryPort { export interface ScanExportPort { exportGood(payload: GoodDatabase): Promise; + importGoodFile(): Promise; } export interface RendererRepositories { diff --git a/src/lib/artifactOcrParser.test.ts b/src/lib/artifactOcrParser.test.ts index 0a8f483..c9372df 100644 --- a/src/lib/artifactOcrParser.test.ts +++ b/src/lib/artifactOcrParser.test.ts @@ -264,6 +264,27 @@ describe("parseArtifactCandidate", () => { expect(parsed?.fields.substats.confidence).toBe(96); }); + it("parses the live calibrated 1080p Conductor circlet capture", () => { + const parsed = parseArtifactCandidate(captureFromOcr({ + "artifact-title": "Conductor's Top Hat", + "artifact-main-stat": "Circlet of Logos\nHP\n7. 0 % i", + "artifact-substats": "a +\n+ Energy Recharge+4.5%\n+ ATK+14\n- Elemental Mastery+19\n- ATK+5.3% (unactivated)", + "artifact-footer": "", + })); + + expect(parsed?.name).toBe("Conductor's Top Hat"); + expect(parsed?.slot).toBe("Circlet of Logos"); + expect(parsed?.setName).toBe("Wanderer's Troupe"); + expect(parsed?.mainStat).toBe("HP%"); + expect(parsed?.mainValue).toBe("7.0%"); + expect(parsed?.substats).toEqual([ + "Energy Recharge+4.5%", + "ATK+14", + "Elemental Mastery+19", + "ATK%+5.3%", + ]); + }); + it("keeps a percent main value even when OCR misses the main stat label", () => { const parsed = parseArtifactCandidate(captureFromOcr({ "artifact-title": "Moonlit Offering's Final\nSands of Eon", diff --git a/src/lib/artifactOcrParser.ts b/src/lib/artifactOcrParser.ts index cd7df7b..66d7dd1 100644 --- a/src/lib/artifactOcrParser.ts +++ b/src/lib/artifactOcrParser.ts @@ -276,14 +276,15 @@ function findMainValue(text: string, mainStat: string, slot: string, level: numb } function extractPercentValue(text: string) { + const percentPattern = /([0-9]{1,3}(?:\s*[.,:\u00B7]\s*[0-9])?)\s*%/; const lineMatches = text .split("\n") - .map((line) => line.match(/([0-9]{1,3}(?:[.,:\u00B7][0-9])?)\s*%/)) + .map((line) => line.match(percentPattern)) .filter((match): match is RegExpMatchArray => Boolean(match)); - const preferred = lineMatches[0] ?? text.match(/([0-9]{1,3}(?:[.,:\u00B7][0-9])?)\s*%/); + const preferred = lineMatches[0] ?? text.match(percentPattern); if (!preferred?.[1]) return ""; - return `${preferred[1].replace(/[:,\u00B7]/g, ".")}%`; + return `${normalizeMainValue(preferred[1])}%`; } function inferMainStat(slot: string, text: string): ParsedField { @@ -302,7 +303,7 @@ function inferMainStat(slot: string, text: string): ParsedField { function findDirectMainStat(text: string) { const compact = simplifyForMatch(text); - const hasPercentValue = /[0-9]{1,3}(?:\.[0-9])?\s*%/.test(text); + const hasPercentValue = /[0-9]{1,3}(?:\s*[.,:\u00B7]\s*[0-9])?\s*%/.test(text); const priority = [ "Physical DMG Bonus", "Elemental Mastery", @@ -481,7 +482,7 @@ function isPercentMainStat(stat: string) { } function promotePercentVariant(stat: string, text: string) { - if (["ATK", "HP", "DEF"].includes(stat) && /[0-9]{1,3}(?:\.[0-9])?\s*%/.test(text)) return `${stat}%`; + if (["ATK", "HP", "DEF"].includes(stat) && /[0-9]{1,3}(?:\s*[.,:\u00B7]\s*[0-9])?\s*%/.test(text)) return `${stat}%`; return stat; } diff --git a/src/lib/artifactStore.ts b/src/lib/artifactStore.ts index 2fbb81a..9082a56 100644 --- a/src/lib/artifactStore.ts +++ b/src/lib/artifactStore.ts @@ -45,7 +45,7 @@ export function hashId(value: string) { return `${hash.toString(16).padStart(8, "0")}-${value.length.toString(16)}`; } -export function toStoredArtifact(parsed: ParsedArtifactCandidate, source: string, needsReview: boolean): StoredArtifactRecord { +export function toStoredArtifact(parsed: ParsedArtifactCandidate, source: string, needsReview: boolean, locked?: boolean): StoredArtifactRecord { return { id: hashId(storeSignature(parsed)), name: parsed.name, @@ -58,6 +58,7 @@ export function toStoredArtifact(parsed: ParsedArtifactCandidate, source: string equipped: parsed.equipped, confidence: parsed.confidence, needsReview, + locked, source, }; } diff --git a/src/lib/layoutProfile.test.ts b/src/lib/layoutProfile.test.ts index b3a0d03..cb95763 100644 --- a/src/lib/layoutProfile.test.ts +++ b/src/lib/layoutProfile.test.ts @@ -42,6 +42,10 @@ describe("layoutProfile", () => { expect(rect.y + rect.height).toBeLessThanOrEqual(QHD.height); }); + it("matches the calibrated 1080p artifact detail panel", () => { + expect(profileDetailRect(HD)).toEqual({ x: 1308, y: 120, width: 492, height: 838 }); + }); + it("produces the four artifact crops in top-to-bottom order, all clamped", () => { const detail = profileDetailRect(QHD); const crops = detailCropRects(detail, QHD); @@ -65,19 +69,28 @@ describe("layoutProfile", () => { const detail = profileDetailRect(QHD); const inv = inventoryRect(QHD, detail); const count = inventoryCountCropRect(inv, QHD); - expect(count.x).toBeGreaterThanOrEqual(inv.x); + expect(count.x).toBeGreaterThan(QHD.width * 0.75); expect(count.x + count.width).toBeLessThanOrEqual(QHD.width); }); - it("builds a 5-column inventory grid on the left", () => { + it("builds the calibrated 8-column inventory grid on the left", () => { const detail = profileDetailRect(QHD); const grid = inventoryGrid(QHD, detail); - expect(grid.cols).toBe(5); + expect(grid.cols).toBe(8); + expect(grid.rows).toBe(5); expect(grid.source).toBe("detected"); - expect(grid.centers.length).toBeGreaterThanOrEqual(10); + expect(grid.centers).toHaveLength(40); expect(grid.centers.every((center) => center.x < detail.x)).toBe(true); }); + it("matches the live 1080p artifact grid centers", () => { + const detail = profileDetailRect(HD); + const grid = inventoryGrid(HD, detail); + expect(grid.centers[0]).toEqual({ x: 179, y: 254, row: 0, col: 0 }); + expect(grid.centers[7]).toEqual({ x: 1201, y: 254, row: 0, col: 7 }); + expect(grid.centers.at(-1)).toEqual({ x: 1201, y: 958, row: 4, col: 7 }); + }); + it("reports a missing grid when the inventory panel is too small", () => { const tiny = { width: 320, height: 180 }; const grid = inventoryGrid(tiny, profileDetailRect(tiny)); diff --git a/src/lib/layoutProfile.ts b/src/lib/layoutProfile.ts index 9244420..fda193c 100644 --- a/src/lib/layoutProfile.ts +++ b/src/lib/layoutProfile.ts @@ -5,9 +5,9 @@ // of that geometry; electron/main.ts consumes it for cropping and keeps a // colour-based detail-rect detector only as a fallback for off-profile setups. // -// NOTE: the per-field detail crop fractions below are the current working values. -// True IK-style fixed coordinates need calibration against a reference 16:9 -// screenshot; the structure here is what those calibrated numbers slot into. +// Calibrated from a 1920x1080 English artifact-inventory screenshot and scaled +// by client size. This follows Inventory Kamera's stable approach: fixed +// 16:9-relative UI regions first, visual detection only as a fallback. export interface LayoutRect { x: number; @@ -81,10 +81,10 @@ export function profileDetailRect(imageSize: { width: number; height: number }): if (width <= 0 || height <= 0) return { x: 0, y: 0, width: Math.max(1, width), height: Math.max(1, height) }; return clampRect( { - x: Math.round(width * 0.5), - y: Math.round(height * 0.08), - width: Math.round(width * 0.46), - height: Math.round(height * 0.74), + x: Math.round(width * 0.681), + y: Math.round(height * 0.111), + width: Math.round(width * 0.256), + height: Math.round(height * 0.776), }, imageSize, ); @@ -97,10 +97,10 @@ export function detailCropRects(detailRect: LayoutRect, imageSize: { width: numb id: "artifact-title", label: "Artifact title", rect: { - x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.05), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.16), + x: Math.round(detailRect.x), + y: Math.round(detailRect.y), + width: Math.round(detailRect.width), + height: Math.round(detailRect.height * 0.07), }, }, { @@ -108,9 +108,9 @@ export function detailCropRects(detailRect: LayoutRect, imageSize: { width: numb label: "Main stat", rect: { x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.2), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.18), + y: Math.round(detailRect.y + detailRect.height * 0.075), + width: Math.round(detailRect.width * 0.58), + height: Math.round(detailRect.height * 0.26), }, }, { @@ -118,9 +118,9 @@ export function detailCropRects(detailRect: LayoutRect, imageSize: { width: numb label: "Substats", rect: { x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.41), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.25), + y: Math.round(detailRect.y + detailRect.height * 0.34), + width: Math.round(detailRect.width * 0.86), + height: Math.round(detailRect.height * 0.27), }, }, { @@ -128,9 +128,9 @@ export function detailCropRects(detailRect: LayoutRect, imageSize: { width: numb label: "Footer", rect: { x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.78), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.16), + y: Math.round(detailRect.y + detailRect.height * 0.82), + width: Math.round(detailRect.width * 0.86), + height: Math.round(detailRect.height * 0.14), }, }, ]; @@ -139,12 +139,13 @@ export function detailCropRects(detailRect: LayoutRect, imageSize: { width: numb } export function inventoryCountCropRect(inventoryRect: LayoutRect, imageSize: { width: number; height: number }): LayoutRect { + const { width, height } = imageSize; return clampRect( { - x: Math.round(inventoryRect.x + inventoryRect.width * 0.62), - y: Math.round(inventoryRect.y + inventoryRect.height * 0.02), - width: Math.round(inventoryRect.width * 0.34), - height: Math.round(inventoryRect.height * 0.09), + x: Math.round(width * 0.795), + y: Math.round(height * 0.02), + width: Math.round(width * 0.145), + height: Math.round(height * 0.055), }, imageSize, ); @@ -152,18 +153,12 @@ export function inventoryCountCropRect(inventoryRect: LayoutRect, imageSize: { w export function inventoryRect(imageSize: { width: number; height: number }, detailRect: LayoutRect): LayoutRect { const { width, height } = imageSize; - const preferredWidth = Math.max(140, Math.round(width * 0.48)); - const x = Math.round(width * 0.03); - const y = Math.round(detailRect.y + detailRect.height * 0.09); - const availableWidth = Math.max(100, detailRect.x - Math.round(width * 0.04)); - const panelWidth = Math.max(100, Math.min(preferredWidth, availableWidth)); - const safeWidth = panelWidth > width * 0.85 ? Math.round(width * 0.55) : panelWidth; return clampRect( { - x, - y, - width: Math.min(safeWidth, Math.max(width - x - Math.round(width * 0.02), 100)), - height: Math.max(140, Math.round(height * 0.7)), + x: Math.round(width * 0.055), + y: Math.round(height * 0.155), + width: Math.max(140, Math.round(Math.min(detailRect.x - width * 0.07, width * 0.63))), + height: Math.max(140, Math.round(height * 0.74)), }, imageSize, ); @@ -171,18 +166,17 @@ export function inventoryRect(imageSize: { width: number; height: number }, deta export function inventoryGrid(imageSize: { width: number; height: number }, detailRect: LayoutRect): InventoryGridLayout { const rect = inventoryRect(imageSize, detailRect); - const cols = 5; - if (rect.width < 160 || rect.height < 140) { + const cols = 8; + if (imageSize.width < 800 || imageSize.height < 450 || rect.width < 160 || rect.height < 140) { return { centers: [], rows: 0, cols: 0, confidence: 0, source: "missing" }; } - const cellWidth = Math.max(56, Math.round(rect.width / cols)); - const stepX = Math.round(cellWidth * 0.96); - const stepY = Math.round(cellWidth * 1.03); - const visibleRows = Math.max(2, Math.min(6, Math.round(rect.height / Math.max(stepY, 1)))); + const stepX = Math.round(imageSize.width * 0.076); + const stepY = Math.round(imageSize.height * 0.163); + const visibleRows = 5; - const startX = rect.x + Math.max(6, Math.round(stepX * 0.45)); - const startY = rect.y + Math.max(6, Math.round(stepY * 0.45)); + const startX = Math.round(imageSize.width * 0.093); + const startY = Math.round(imageSize.height * 0.235); const centers: InventoryGridLayout["centers"] = []; for (let row = 0; row < visibleRows; row++) { for (let col = 0; col < cols; col++) { diff --git a/src/lib/lockDetection.test.ts b/src/lib/lockDetection.test.ts index a727811..1066501 100644 --- a/src/lib/lockDetection.test.ts +++ b/src/lib/lockDetection.test.ts @@ -19,13 +19,14 @@ function bitmap(goldPixels: number, total: number): Bitmap { } describe("lockDetection", () => { - it("places the lock crop in the top-right of the detail card", () => { + it("places the lock crop on the lock button in the substat panel", () => { const size = { width: 2560, height: 1440 }; const detail = profileDetailRect(size); const rect = lockIconCropRect(detail, size); expect(rect.x).toBeGreaterThan(detail.x + detail.width * 0.5); expect(rect.x + rect.width).toBeLessThanOrEqual(size.width); - expect(rect.y).toBeLessThan(detail.y + detail.height * 0.5); + expect(rect.y).toBeGreaterThan(detail.y + detail.height * 0.3); + expect(rect.y).toBeLessThan(detail.y + detail.height * 0.45); }); it("measures the gold-pixel ratio", () => { diff --git a/src/lib/lockDetection.ts b/src/lib/lockDetection.ts index 20e3b7c..279b4ee 100644 --- a/src/lib/lockDetection.ts +++ b/src/lib/lockDetection.ts @@ -1,5 +1,5 @@ -import { clampRect, type LayoutRect } from "./layoutProfile"; -import type { Bitmap } from "./ocrPreprocess"; +import { clampRect, type LayoutRect } from "./layoutProfile.js"; +import type { Bitmap } from "./ocrPreprocess.js"; // EXPERIMENTAL, read-only lock-status detection (nice-to-have). Genshin shows a // padlock at the top-right of the artifact detail card: a bright gold fill when @@ -15,10 +15,10 @@ import type { Bitmap } from "./ocrPreprocess"; export function lockIconCropRect(detailRect: LayoutRect, imageSize: { width: number; height: number }): LayoutRect { return clampRect( { - x: Math.round(detailRect.x + detailRect.width * 0.8), - y: Math.round(detailRect.y + detailRect.height * 0.03), - width: Math.round(detailRect.width * 0.16), - height: Math.round(detailRect.height * 0.09), + x: Math.round(detailRect.x + detailRect.width * 0.735), + y: Math.round(detailRect.y + detailRect.height * 0.355), + width: Math.round(detailRect.width * 0.105), + height: Math.round(detailRect.height * 0.07), }, imageSize, ); diff --git a/src/lib/storedArtifactAdapter.test.ts b/src/lib/storedArtifactAdapter.test.ts index 2023f97..12c21b0 100644 --- a/src/lib/storedArtifactAdapter.test.ts +++ b/src/lib/storedArtifactAdapter.test.ts @@ -23,7 +23,7 @@ function record(overrides: Partial = {}): StoredArtifactRe describe("storedArtifactAdapter", () => { it("converts stored OCR artifacts into recommendation-domain artifacts", () => { - const [artifact] = storedArtifactsToDomain([record()]); + const [artifact] = storedArtifactsToDomain([record({ locked: true })]); expect(artifact.slot).toBe("sands"); expect(artifact.setKey).toBe("viridescent_venerer"); @@ -32,6 +32,13 @@ describe("storedArtifactAdapter", () => { expect(artifact.equipped).toBe("Sucrose"); expect(artifact.confidence).toBe(0.96); expect(artifact.source).toBe("screen"); + expect(artifact.locked).toBe(true); + }); + + it("does not invent lock state from confidence", () => { + const [artifact] = storedArtifactsToDomain([record({ locked: undefined, confidence: 100 })]); + + expect(artifact.locked).toBe(false); }); it("keeps flat and percent ATK substats distinct", () => { diff --git a/src/lib/storedArtifactAdapter.ts b/src/lib/storedArtifactAdapter.ts index f595709..4a19133 100644 --- a/src/lib/storedArtifactAdapter.ts +++ b/src/lib/storedArtifactAdapter.ts @@ -54,7 +54,7 @@ export function storedArtifactsToDomain(records: StoredArtifactRecord[]): Artifa mainStat: normalizeMainStat(record.mainStat, record.mainValue), substats: record.substats.map(parseStoredSubstat).filter(Boolean) as ArtifactSubstat[], equipped: isUsefulEquippedName(record.equipped) ? record.equipped.trim() : undefined, - locked: !record.needsReview && record.confidence >= 90, + locked: Boolean(record.locked), source: toSource(record.source), confidence: Math.max(0, Math.min(1, record.confidence / 100)), lastSeenAt: record.lastSeenAt ?? record.firstSeenAt ?? now, diff --git a/src/services/assistantBridge.ts b/src/services/assistantBridge.ts index 88b8bfc..27d43e0 100644 --- a/src/services/assistantBridge.ts +++ b/src/services/assistantBridge.ts @@ -15,6 +15,8 @@ import type { SaveScannerLearningRulesResult, SaveSnapshotResult, GoodDatabase, + GoodImportFileResult, + ScannerCommand, ScannerStatusPayload, ScannerLearningRulePayload, } from "../types/global"; @@ -39,6 +41,7 @@ export interface AssistantBridge { options?: CaptureOptions, ) => Promise; exportGood: (payload: GoodDatabase) => Promise; + importGoodFile: () => Promise; showOverlay: () => Promise; loadArtifacts: () => Promise; saveArtifacts: (records: StoredArtifactRecord[]) => Promise; @@ -54,7 +57,7 @@ export interface AssistantBridge { focusGenshinForScanStart: () => Promise; clickScreen: (x: number, y: number) => Promise; scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => Promise; - onScannerCommand: (callback: (command: "start-auto" | "stop") => void) => () => void; + onScannerCommand: (callback: (command: ScannerCommand) => void) => () => void; } function hasFunction(api: Record, key: string): boolean { @@ -81,6 +84,7 @@ export function getAssistantBridge(): AssistantBridge | null { listCaptureSources: () => api.listCaptureSources(), captureSource: (sourceId, delayMs, focusGenshin, options) => api.captureSource(sourceId, delayMs, focusGenshin, options), exportGood: (payload) => api.exportGood(payload), + importGoodFile: () => api.importGoodFile(), loadArtifacts: () => api.loadArtifacts(), saveArtifacts: (records) => api.saveArtifacts(records), loadReviewSamples: (limit = 50) => api.loadReviewSamples(limit), diff --git a/src/types/global.d.ts b/src/types/global.d.ts index f8987dc..177ca03 100644 --- a/src/types/global.d.ts +++ b/src/types/global.d.ts @@ -56,6 +56,7 @@ export interface CaptureResult { source: "ocr" | "missing"; text: string; }; + locked?: boolean; layout?: { aspect: string; isSixteenNine: boolean; @@ -125,6 +126,14 @@ export interface SaveResultWithPath { export type SaveSnapshotResult = SaveResultWithPath; +export type ScannerCommand = + | "start-auto" + | "stop" + | { + type: "start-auto"; + scanLimit?: number; + }; + export interface ScannerLearningRulePayload { textReplacements?: Record; } @@ -211,6 +220,7 @@ export interface ReviewSampleRecord { ocr?: OcrResult[]; inventoryGrid?: CaptureResult["inventoryGrid"]; inventoryCount?: CaptureResult["inventoryCount"]; + locked?: boolean; }; }; } @@ -239,6 +249,7 @@ export interface ReviewSamplePayload { ocr?: OcrResult[]; inventoryGrid?: CaptureResult["inventoryGrid"]; inventoryCount?: CaptureResult["inventoryCount"]; + locked?: boolean; }; [key: string]: unknown; } @@ -280,6 +291,14 @@ export interface GoodDatabase { artifacts: GoodExportArtifact[]; } +export interface GoodImportFileResult { + ok: boolean; + canceled: boolean; + path: string; + database?: unknown; + error?: string; +} + declare global { interface Window { assistantApi?: { @@ -302,10 +321,11 @@ declare global { loadArtifacts: () => Promise; saveArtifacts: (records: StoredArtifactRecord[]) => Promise; exportGood: (payload: GoodDatabase) => Promise; + importGoodFile: () => Promise; publishScannerStatus: (status: ScannerStatusPayload) => Promise; showOverlay: () => Promise; hideOverlay: () => Promise; - onScannerCommand: (callback: (command: "start-auto" | "stop") => void) => () => void; + onScannerCommand: (callback: (command: ScannerCommand) => void) => () => void; }; } } diff --git a/src/types/storage.ts b/src/types/storage.ts index 353c220..63574cb 100644 --- a/src/types/storage.ts +++ b/src/types/storage.ts @@ -10,6 +10,7 @@ export interface StoredArtifactRecord { equipped: string; confidence: number; needsReview: boolean; + locked?: boolean; source: string; firstSeenAt?: string; lastSeenAt?: string;