diff --git a/docs/AUTOMATION_LIVE_SCAN.md b/docs/AUTOMATION_LIVE_SCAN.md index 040b5a9..d138568 100644 --- a/docs/AUTOMATION_LIVE_SCAN.md +++ b/docs/AUTOMATION_LIVE_SCAN.md @@ -268,10 +268,20 @@ Latest live timing evidence on 2026-07-08: signature `2026-07-08-direct-gdi-reviewfix` completed `20/20`, `18` stored, `0` review, `2` duplicates, `0` misses, `7973 ms` elapsed, or `399 ms/artifact`. 3 artifacts/second is not proven. +- Follow-up 3 artifacts/second attempts on 2026-07-09: + after the distinctive partial piece parser fix, the best clean repeatability + run reached `336 ms/artifact` with `20/20` parsed, `0` review, `0` misses, + `318 ms` average capture roundtrip, and `138 ms` roundtrip overhead. Later + runs with crop priority and image-payload cleanup stayed clean but ranged + around `346-351 ms/artifact`; the strict `333 ms/artifact` budget remains + unproven. - Rejected speed experiments: detail-region capture, `GAA_OCR_WORKERS=5`, DataURL-to-buffer decode, and substat OCR `PSM.SINGLE_COLUMN` were all live/benchmark tested and were slower - than the direct-GDI baseline. Keep `GAA_OCR_WORKERS=4` for current runs. + than the direct-GDI baseline. Later checks also rejected skipping + `analyzePaimonMenu`, skipping lock-state as a production shortcut, reducing + the artifact-level crop scale, and `GAA_OCR_WORKERS=6` as the default. + Keep `GAA_OCR_WORKERS=4` for current runs. - Quality-gated current-vs-IK comparison: `npm run scan:goal:compare:validated` produced `outputs/live-soak/2026-07-08T18-38-35/scan-performance-assessment.json` @@ -449,15 +459,17 @@ uses the default `current` scan engine; use `scan:goal:ik` for a native IK-traineddata scan pass. Use `scan:goal:compare` to run both scan engines back-to-back with the same limits and one combined CSV. The CSV is the quickest evidence for `averageMsPerParsed`, `activeAverageMsPerParsed`, `averageCaptureMs`, +`averageCaptureRoundTripMs`, `averageCaptureRoundTripOverheadMs`, `captureP50Ms`, `captureP90Ms`, `averageOcrMs`, `ocrP50Ms`, `ocrP90Ms`, `averageCardReadyMs`, `averageScrollReadyMs`, `artifactsPerMinute`, and `projectedMsFor100`. The run also writes `scan-performance-assessment.json`, which groups results by limit, picks the best qualified engine, and labels the dominant bottleneck as -OCR, capture, card-ready, or scroll-ready. A qualified winner must finish the -run, parse the requested count, keep miss rate under 2%, and keep review rate -at or below 15%; review and miss rates are penalized before active average speed -is used as the tie-breaker. For IK-target claims, check `goal100Decision`; it +OCR, capture-roundtrip-overhead, capture, card-ready, or scroll-ready. A +qualified winner must finish the run, parse the requested count, keep miss rate +under 2%, and keep review rate at or below 15%; review and miss rates are +penalized before active average speed is used as the tie-breaker. For IK-target +claims, check `goal100Decision`; it must read `qualified-comparison: winner=`, and `goal100.comparisonComplete` must be `true` so a single-engine 100-artifact run is not mistaken for a current-vs-IK comparison. @@ -478,12 +490,38 @@ instead of the final 100-artifact proof. Add `--summary` when you want a short report-ready PASS/FAIL output that includes the input assessment path and assessment `createdAt` timestamp. +Optional budget flags are useful for the current speed work: + +```powershell +npm run scan:assessment:validate -- --latest --summary --limit=20 --max-active-average-ms=333 --max-capture-roundtrip-overhead-ms=120 +``` + +`--max-active-average-ms=333` is the strict 3 artifacts/second check. Use a +separate `--max-capture-roundtrip-overhead-ms` budget when deciding whether the +next optimization belongs in native capture transport instead of OCR. +Single-engine repeatability runs may be validated with `--allow-single-engine`, +but that mode is only for repeatability evidence and must not be used for IK +parity claims. + `npm run scan:goal:compare:validated` is the preferred final command: it runs the live preflight first, then the full comparison, and then validates the newest assessment with `--summary`. Use `npm run scan:goal:compare:validated:wait` for the same final flow when starting immediately after UAC. +For later-session repeatability without changing OCR engines, use: + +```powershell +npm run scan:repeatability:wait +``` + +That command runs the current visible-inventory engine at 20, 45, and 100 +artifacts, then validates the 100-artifact result as single-engine evidence. +It uses the `live-soak.ps1 -RepeatabilityRun` switch instead of passing a +comma-separated `-Limits` value through npm/cmd, because Windows argument +parsing can collapse `20,45,100` into one unsafe number. The script also refuses +limits above 1800 as a final guard. + The assessment ranking can be verified without Genshin or the Electron app: ```powershell diff --git a/docs/ocr-eval.md b/docs/ocr-eval.md index 8719f3a..8c30ba7 100644 --- a/docs/ocr-eval.md +++ b/docs/ocr-eval.md @@ -15,6 +15,7 @@ npm run eval:review-candidates # export unconfirmed review samples for human la npm test # runs the eval gate alongside the rest of the suite npm run scan:assessment:test # verifies quality-first scan ranking logic npm run scan:iterate:compare:validated:wait # 20-artifact live comparison +npm run scan:repeatability:wait # 20/45/100 current-engine repeatability npm run scan:goal:compare:validated:wait # final 100-artifact live comparison ``` @@ -22,6 +23,10 @@ The report prints exact-match rate, overall field accuracy, a per-field breakdown (critical fields marked with `*`), and every failing case with an `expected "..." got "..."` diff. +Repeatability runs are single-engine evidence. Use them to prove that the +visible-inventory scanner stays stable across later sessions, but keep +current-vs-IK claims on `scan:goal:compare:validated:*`. + ## How it works - `src/eval/ocrEvalHarness.ts` - pure metric functions. `runOcrEval(cases)` diff --git a/docs/scanner-rework-status.md b/docs/scanner-rework-status.md index 3ce1fd3..3989cd1 100644 --- a/docs/scanner-rework-status.md +++ b/docs/scanner-rework-status.md @@ -144,6 +144,37 @@ Current status after the 2026-07-09 merge to `main`: end-to-end capture roundtrip time. Use `averageCaptureRoundTripMs` and `averageCaptureRoundTripOverheadMs` in the next `limit=20` live iteration to decide whether the next cut belongs in native capture transport or OCR. +- **Repeatability and capture-overhead guardrails (2026-07-09)** - + `scripts/live-soak.ps1` now writes capture roundtrip and roundtrip-overhead + timing into `scan-performance-assessment.json`. The assessment validator can + enforce optional speed budgets with `--max-active-average-ms` and + `--max-capture-roundtrip-overhead-ms`, and `npm run scan:repeatability:wait` + runs 20/45/100 current-engine passes as repeatability evidence without + presenting them as an IK comparison. `-RepeatabilityRun` now sets those limits + inside PowerShell, and the script refuses unsafe limits above 1800 so npm/cmd + argument parsing cannot accidentally turn `20,45,100` into one oversized run. +- **Distinctive partial piece recovery (2026-07-09)** - a live repeatability run + exposed four identical OCR misses where the piece name was read as + `Wontiroms Creation pan`. The parser now derives a piece only when a long OCR + fragment uniquely matches exactly one known artifact piece. This recovered the + local case as `Sharpness That Ceased Upon Wondrous Creation` / + `Disenchantment in Deep Shadow` without adding a broad fuzzy exception. +- **Repeatability live pass after parser fix (2026-07-09)** - + `outputs/live-soak/2026-07-09T09-29-11/scan-performance-assessment.json` + captured a clean current-engine 20-artifact run: `20/20` parsed, `0` review, + `0` misses, `336 ms/artifact` active average, `318 ms` average capture + roundtrip, and `138 ms` average roundtrip overhead. The strict 3 artifacts per + second budget still failed by 3 ms (`336 ms` vs `333 ms`). +- **3/s follow-up experiments (2026-07-09)** - tested and rejected several + shortcut-style optimizations because live runs got slower or added risk: + skipping Paimon-menu analysis, skipping lock-state as a production shortcut, + reducing the artifact-level crop scale, and raising the OCR worker pool to 6. + The kept low-risk changes are fast-profile OCR crop priority and avoiding a + duplicate DataURL string when the native helper already returns Base64. A + follow-up clean 20-artifact run after payload cleanup reached `351 ms/artifact`, + `331 ms` capture roundtrip, and `146 ms` roundtrip overhead, so the next + credible 3/s work is native capture transport/roundtrip reduction, not UI + recommendation work. - **3/s live attempt (2026-07-08)** - the missing-detail-preview review trigger was fixed and tested. The best clean 20-artifact run reached `7285 ms` (`364 ms/artifact`, about `2.75 artifacts/second`) with 0 review and 0 misses. @@ -218,14 +249,17 @@ resolution or without UI work best tested live: substat OCR time, or a larger capture pipeline change that avoids full-frame PNG/Base64 transport without hurting safety checks. The target remains `<= 6667 ms` elapsed for 20 parsed artifacts with 0 misses and no silent OCR - review regression. + review regression. The latest clean 20-artifact repeatability run reached + `336 ms/artifact`, so 3/s remains close but unproven. 6. **Broader scan soak test** — direct-GDI current-engine runs now passed at - `20/20`, `45/45`, and `100/100` with 0 misses. Continue with repeat runs if - duplicate rate needs tuning. + `20/20`, `45/45`, and `100/100` with 0 misses. Continue with + `npm run scan:repeatability:wait` in later sessions to check duplicate rate, + scroll behavior, and capture roundtrip timing without changing defaults. 7. **Repeatability pass** — repeat the qualified current-vs-IK-traineddata run in a later live session before making major OCR-engine defaults or speed - claims beyond this environment. + claims beyond this environment. Current-engine-only repeatability is useful + evidence, but it is not an IK parity claim. Visible-page limits up to 20, scroll/page-transition limit 45, the final 100-artifact current-vs-IK-traineddata comparison, equipped footer live smokes, diff --git a/electron/main.ts b/electron/main.ts index 0554730..841bb2f 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -407,6 +407,12 @@ async function captureSourceFromGdi(sourceId: string, sourceName: string, option } function nativeImageFromGdiCapture(gdi: Awaited>) { + if (gdi.imageBase64) { + return nativeImage.createFromBuffer(Buffer.from(gdi.imageBase64, "base64")); + } + if (!gdi.dataUrl) { + throw new Error("GDI capture returned no image payload."); + } return nativeImage.createFromDataURL(gdi.dataUrl); } @@ -1296,6 +1302,14 @@ async function buildCaptureResult( const omitInventoryPreview = Boolean(options.omitInventoryPreview || options.ocrMode === "artifact"); const detailFingerprint = imageCropFingerprint(sourceImage, detailRect, size); const inventoryFingerprint = imageCropFingerprint(sourceImage, inventoryRect, size); + const fastOcrPriority: Record = { + "artifact-substats": 0, + "artifact-name": 1, + "artifact-footer": 2, + "artifact-slot": 3, + "artifact-main-stat-label": 4, + "artifact-level": 5, + }; const croppedPayload = crops .filter((crop) => crop.ocrEnabled !== false) .map((crop) => { @@ -1307,7 +1321,11 @@ async function buildCaptureResult( } : null; }) - .filter((crop): crop is OcrCropPayload => Boolean(crop)); + .filter((crop): crop is OcrCropPayload => Boolean(crop)) + .sort((left, right) => { + if (options.ocrMode !== "artifact" || options.ocrProfile !== "fast") return 0; + return (fastOcrPriority[left.id] ?? 100) - (fastOcrPriority[right.id] ?? 100); + }); const prepareMs = Date.now() - buildStartedAt; const ocrEngine = ocrEngineFromOptions(options); const ocrStartedAt = Date.now(); diff --git a/electron/services/inputHelper.ts b/electron/services/inputHelper.ts index 11a289c..42c8058 100644 --- a/electron/services/inputHelper.ts +++ b/electron/services/inputHelper.ts @@ -292,7 +292,9 @@ export function createInputHelperService(options: { userDataPath: string; exePat : "primary-screen"; return { - dataUrl: `data:image/png;base64,${base64}`, + ...(typeof result.imageBase64 === "string" && result.imageBase64 + ? { imageBase64: base64 } + : { dataUrl: `data:image/png;base64,${base64}` }), width: Number(result.width), height: Number(result.height), originX: Number(result.originX), diff --git a/package.json b/package.json index 7a121a9..71546d8 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,8 @@ "scan:iterate:compare": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\live-soak.ps1 -Limits 20 -ScanEngine compare -BenchmarkOcr", "scan:iterate:compare:validated": "npm run scan:live:preflight && npm run scan:iterate:compare && npm run scan:assessment:validate -- --latest --summary --limit=20", "scan:iterate:compare:validated:wait": "npm run scan:live:preflight:wait && npm run scan:iterate:compare && npm run scan:assessment:validate -- --latest --summary --limit=20", + "scan:repeatability": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\live-soak.ps1 -RepeatabilityRun -ScanEngine current", + "scan:repeatability:wait": "npm run scan:live:preflight:wait && npm run scan:repeatability && npm run scan:assessment:validate -- --latest --summary --limit=100 --expect-winner=current --allow-single-engine", "scan:live:preflight": "node scripts/live-preflight.cjs", "scan:live:preflight:wait": "node scripts/live-preflight.cjs --wait=120", "scan:assessment:test": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\live-soak.ps1 -SelfTestAssessment", diff --git a/scripts/live-soak.ps1 b/scripts/live-soak.ps1 index dd94fb9..de5fa4a 100644 --- a/scripts/live-soak.ps1 +++ b/scripts/live-soak.ps1 @@ -6,6 +6,7 @@ param( [int]$TimeoutSeconds = 600, [string]$OutputRoot = (Join-Path (Resolve-Path -LiteralPath ".").Path "outputs\live-soak"), [switch]$GoalRun, + [switch]$RepeatabilityRun, [ValidateSet("current", "ik-traineddata", "compare")] [string]$ScanEngine = "current", [switch]$BenchmarkOcr, @@ -138,6 +139,8 @@ function Convert-ScanTimingSummary([object]$StatusPayload, [int]$Limit, [string] projectedMsFor100 = [int]$stats.projectedMsFor100 activeProjectedMsFor100 = [int]$stats.activeProjectedMsFor100 averageCaptureMs = [int]$stats.averageCaptureMs + averageCaptureRoundTripMs = [int]$stats.averageCaptureRoundTripMs + averageCaptureRoundTripOverheadMs = [int]$stats.averageCaptureRoundTripOverheadMs averageOcrMs = [int]$stats.averageOcrMs captureP50Ms = [int]$stats.captureP50Ms captureP90Ms = [int]$stats.captureP90Ms @@ -150,13 +153,15 @@ function Convert-ScanTimingSummary([object]$StatusPayload, [int]$Limit, [string] scrollReadyCount = [int]$stats.scrollReadyCount averageScrollReadyMs = [int]$stats.averageScrollReadyMs captureMs = [int]$stats.captureMs + captureRoundTripMs = [int]$stats.captureRoundTripMs + captureRoundTripOverheadMs = [int]$stats.captureRoundTripOverheadMs ocrMs = [int]$stats.ocrMs reviewStatus = [string]$StatusPayload.status.reviewStatus } } function Write-ScanTimingLine([object]$Timing) { - Write-Host ("engine={0} limit={1} status={2} parsed={3} review={4} miss={5} elapsed={6}ms active={7}ms flush={8}ms avg={9}ms activeAvg={10}ms captureAvg={11}ms captureP50={12}ms captureP90={13}ms ocrAvg={14}ms ocrP50={15}ms ocrP90={16}ms cardReadyAvg={17}ms scrollReadyAvg={18}ms ppm={19} activePpm={20} projected100={21}ms activeProjected100={22}ms" -f ` + Write-Host ("engine={0} limit={1} status={2} parsed={3} review={4} miss={5} elapsed={6}ms active={7}ms flush={8}ms avg={9}ms activeAvg={10}ms captureAvg={11}ms roundtripAvg={12}ms roundtripOverheadAvg={13}ms captureP50={14}ms captureP90={15}ms ocrAvg={16}ms ocrP50={17}ms ocrP90={18}ms cardReadyAvg={19}ms scrollReadyAvg={20}ms ppm={21} activePpm={22} projected100={23}ms activeProjected100={24}ms" -f ` $Timing.engine, $Timing.limit, $Timing.status, @@ -169,6 +174,8 @@ function Write-ScanTimingLine([object]$Timing) { $Timing.averageMsPerParsed, $Timing.activeAverageMsPerParsed, $Timing.averageCaptureMs, + $Timing.averageCaptureRoundTripMs, + $Timing.averageCaptureRoundTripOverheadMs, $Timing.captureP50Ms, $Timing.captureP90Ms, $Timing.averageOcrMs, @@ -185,6 +192,7 @@ function Write-ScanTimingLine([object]$Timing) { function Get-TimingBottleneck([object]$Timing) { $parts = @( [pscustomobject]@{ name = "ocr"; value = [int]$Timing.averageOcrMs }, + [pscustomobject]@{ name = "capture-roundtrip-overhead"; value = [int]$Timing.averageCaptureRoundTripOverheadMs }, [pscustomobject]@{ name = "capture"; value = [int]$Timing.averageCaptureMs }, [pscustomobject]@{ name = "card-ready"; value = [int]$Timing.averageCardReadyMs }, [pscustomobject]@{ name = "scroll-ready"; value = [int]$Timing.averageScrollReadyMs } @@ -198,6 +206,7 @@ function Get-TimingRecommendation([object]$Timing) { $bottleneck = Get-TimingBottleneck $Timing switch ($bottleneck) { "ocr" { return "OCR dominates; compare engine, crop count, worker pool, and parser-derived fields first." } + "capture-roundtrip-overhead" { return "Capture transport overhead dominates; inspect native encode, IPC payload size, and Base64/DataURL conversion before OCR changes." } "capture" { return "Capture dominates; reduce payloads/crops and avoid full-frame or Base64 work in the hot loop." } "card-ready" { return "Card-ready dominates; tune detail fingerprint polling against IK's 200ms item wait." } "scroll-ready" { return "Scroll-ready dominates; tune page fingerprint polling against IK's 100ms fast-scroll wait." } @@ -274,6 +283,8 @@ function New-PerformanceAssessment([object[]]$Summaries) { activeProjectedMsFor100 = $entry.activeProjectedMsFor100 averageOcrMs = $entry.averageOcrMs averageCaptureMs = $entry.averageCaptureMs + averageCaptureRoundTripMs = $entry.averageCaptureRoundTripMs + averageCaptureRoundTripOverheadMs = $entry.averageCaptureRoundTripOverheadMs averageCardReadyMs = $entry.averageCardReadyMs averageScrollReadyMs = $entry.averageScrollReadyMs bottleneck = Get-TimingBottleneck $entry @@ -292,6 +303,8 @@ function New-PerformanceAssessment([object[]]$Summaries) { winnerReviewRate = $winner.reviewRate winnerActiveAverageMsPerParsed = $winner.activeAverageMsPerParsed winnerActiveProjectedMsFor100 = $winner.activeProjectedMsFor100 + winnerAverageCaptureRoundTripMs = $winner.averageCaptureRoundTripMs + winnerAverageCaptureRoundTripOverheadMs = $winner.averageCaptureRoundTripOverheadMs engines = $engineReports } } @@ -322,7 +335,7 @@ function Write-PerformanceAssessment([object]$Assessment) { Write-Host "assessment goal100: $($Assessment.goal100Decision)" } foreach ($limit in @($Assessment.limits)) { - Write-Host ("assessment limit={0}: winner={1} qualified={2} completeCompare={3} engines={4} missRate={5:P1} reviewRate={6:P1} activeAvg={7}ms projected100={8}ms" -f ` + Write-Host ("assessment limit={0}: winner={1} qualified={2} completeCompare={3} engines={4} missRate={5:P1} reviewRate={6:P1} activeAvg={7}ms projected100={8}ms roundtripOverheadAvg={9}ms" -f ` $limit.limit, $limit.winnerEngine, $limit.winnerQualified, @@ -331,9 +344,10 @@ function Write-PerformanceAssessment([object]$Assessment) { $limit.winnerMissRate, $limit.winnerReviewRate, $limit.winnerActiveAverageMsPerParsed, - $limit.winnerActiveProjectedMsFor100) + $limit.winnerActiveProjectedMsFor100, + $limit.winnerAverageCaptureRoundTripOverheadMs) foreach ($engine in @($limit.engines)) { - Write-Host (" {0}: qualified={1}, missRate={2:P1}, reviewRate={3:P1}, bottleneck={4}, activeAvg={5}ms, ocr={6}ms, capture={7}ms, cardReady={8}ms, scrollReady={9}ms, decision={10}" -f ` + Write-Host (" {0}: qualified={1}, missRate={2:P1}, reviewRate={3:P1}, bottleneck={4}, activeAvg={5}ms, ocr={6}ms, capture={7}ms, roundtrip={8}ms, roundtripOverhead={9}ms, cardReady={10}ms, scrollReady={11}ms, decision={12}" -f ` $engine.engine, $engine.qualified, $engine.missRate, @@ -342,6 +356,8 @@ function Write-PerformanceAssessment([object]$Assessment) { $engine.activeAverageMsPerParsed, $engine.averageOcrMs, $engine.averageCaptureMs, + $engine.averageCaptureRoundTripMs, + $engine.averageCaptureRoundTripOverheadMs, $engine.averageCardReadyMs, $engine.averageScrollReadyMs, $engine.qualityDecision) @@ -363,6 +379,8 @@ function Invoke-AssessmentSelfTest { activeProjectedMsFor100 = 70000 averageOcrMs = 260 averageCaptureMs = 160 + averageCaptureRoundTripMs = 340 + averageCaptureRoundTripOverheadMs = 180 averageCardReadyMs = 190 averageScrollReadyMs = 60 }, @@ -378,6 +396,8 @@ function Invoke-AssessmentSelfTest { activeProjectedMsFor100 = 82000 averageOcrMs = 210 averageCaptureMs = 170 + averageCaptureRoundTripMs = 300 + averageCaptureRoundTripOverheadMs = 130 averageCardReadyMs = 205 averageScrollReadyMs = 80 }, @@ -393,6 +413,8 @@ function Invoke-AssessmentSelfTest { activeProjectedMsFor100 = 39000 averageOcrMs = 150 averageCaptureMs = 130 + averageCaptureRoundTripMs = 280 + averageCaptureRoundTripOverheadMs = 150 averageCardReadyMs = 80 averageScrollReadyMs = 0 }, @@ -408,6 +430,8 @@ function Invoke-AssessmentSelfTest { activeProjectedMsFor100 = 46000 averageOcrMs = 190 averageCaptureMs = 130 + averageCaptureRoundTripMs = 310 + averageCaptureRoundTripOverheadMs = 180 averageCardReadyMs = 80 averageScrollReadyMs = 0 }, @@ -423,6 +447,8 @@ function Invoke-AssessmentSelfTest { activeProjectedMsFor100 = 30000 averageOcrMs = 100 averageCaptureMs = 90 + averageCaptureRoundTripMs = 190 + averageCaptureRoundTripOverheadMs = 100 averageCardReadyMs = 40 averageScrollReadyMs = 20 }, @@ -438,6 +464,8 @@ function Invoke-AssessmentSelfTest { activeProjectedMsFor100 = 60000 averageOcrMs = 230 averageCaptureMs = 130 + averageCaptureRoundTripMs = 300 + averageCaptureRoundTripOverheadMs = 170 averageCardReadyMs = 160 averageScrollReadyMs = 50 } @@ -492,6 +520,8 @@ function Invoke-AssessmentSelfTest { activeProjectedMsFor100 = 50000 averageOcrMs = 180 averageCaptureMs = 120 + averageCaptureRoundTripMs = 260 + averageCaptureRoundTripOverheadMs = 140 averageCardReadyMs = 100 averageScrollReadyMs = 50 } @@ -665,6 +695,17 @@ if ($GoalRun) { $Limits = @(2, 5, 20, 45, 100) $BenchmarkOcr = $true } +if ($RepeatabilityRun) { + $Limits = @(20, 45, 100) +} + +$Limits = @($Limits | ForEach-Object { + $limit = [int]$_ + if ($limit -lt 1 -or $limit -gt 1800) { + throw "Refusing unsafe scan limit '$limit'. Expected a bounded value from 1 to 1800." + } + $limit +}) $ScanEngines = if ($ScanEngine -eq "compare") { @("current", "ik-traineddata") } else { @($ScanEngine) } $RunSummaries = @() diff --git a/scripts/validate-scan-assessment.cjs b/scripts/validate-scan-assessment.cjs index 9b1207d..f4ca537 100644 --- a/scripts/validate-scan-assessment.cjs +++ b/scripts/validate-scan-assessment.cjs @@ -51,6 +51,13 @@ function validateAssessment(assessment, options = {}) { const errors = []; const expectedWinner = options.expectedWinner || "any"; const expectedLimit = options.limit === undefined ? 100 : Number(options.limit); + const allowSingleEngine = options.allowSingleEngine === true; + const maxActiveAverageMsPerParsed = options.maxActiveAverageMsPerParsed === undefined + ? null + : Number(options.maxActiveAverageMsPerParsed); + const maxCaptureRoundTripOverheadMs = options.maxCaptureRoundTripOverheadMs === undefined + ? null + : Number(options.maxCaptureRoundTripOverheadMs); if (!assessment || typeof assessment !== "object") { return { ok: false, errors: ["Assessment must be a JSON object."] }; } @@ -58,20 +65,26 @@ function validateAssessment(assessment, options = {}) { if (!Number.isInteger(expectedLimit) || expectedLimit < 1) { errors.push(`--limit must be a positive integer, got ${options.limit}.`); } + if (maxActiveAverageMsPerParsed !== null && (!Number.isFinite(maxActiveAverageMsPerParsed) || maxActiveAverageMsPerParsed <= 0)) { + errors.push(`--max-active-average-ms must be a positive finite number, got ${options.maxActiveAverageMsPerParsed}.`); + } + if (maxCaptureRoundTripOverheadMs !== null && (!Number.isFinite(maxCaptureRoundTripOverheadMs) || maxCaptureRoundTripOverheadMs < 0)) { + errors.push(`--max-capture-roundtrip-overhead-ms must be a non-negative finite number, got ${options.maxCaptureRoundTripOverheadMs}.`); + } const limitAssessment = findLimitAssessment(assessment, expectedLimit); if (!limitAssessment || typeof limitAssessment !== "object") { errors.push(`Missing limit=${expectedLimit} assessment.`); } - if (expectedLimit === 100 && assessment.goal100Decision !== `qualified-comparison: winner=${limitAssessment?.winnerEngine}`) { + if (!allowSingleEngine && expectedLimit === 100 && assessment.goal100Decision !== `qualified-comparison: winner=${limitAssessment?.winnerEngine}`) { errors.push(`goal100Decision is not a qualified comparison: ${assessment.goal100Decision || ""}`); } if (limitAssessment?.limit !== expectedLimit) { errors.push(`limit assessment must be ${expectedLimit}, got ${limitAssessment?.limit ?? ""}.`); } - if (limitAssessment?.comparisonComplete !== true) errors.push(`limit=${expectedLimit}.comparisonComplete must be true.`); + if (!allowSingleEngine && limitAssessment?.comparisonComplete !== true) errors.push(`limit=${expectedLimit}.comparisonComplete must be true.`); if (limitAssessment?.winnerQualified !== true) errors.push(`limit=${expectedLimit}.winnerQualified must be true.`); if (expectedWinner !== "any" && limitAssessment?.winnerEngine !== expectedWinner) { errors.push(`Expected winner '${expectedWinner}', got '${limitAssessment?.winnerEngine ?? ""}'.`); @@ -79,8 +92,12 @@ function validateAssessment(assessment, options = {}) { const engines = Array.isArray(limitAssessment?.engines) ? limitAssessment.engines : []; const engineNames = new Set(engines.map((entry) => entry?.engine)); - for (const required of ["current", "ik-traineddata"]) { - if (!engineNames.has(required)) errors.push(`limit=${expectedLimit} is missing engine result: ${required}.`); + if (!allowSingleEngine) { + for (const required of ["current", "ik-traineddata"]) { + if (!engineNames.has(required)) errors.push(`limit=${expectedLimit} is missing engine result: ${required}.`); + } + } else if (engines.length < 1) { + errors.push(`limit=${expectedLimit} must include at least one engine result.`); } const winner = engines.find((entry) => entry?.engine === limitAssessment?.winnerEngine); @@ -93,6 +110,7 @@ function validateAssessment(assessment, options = {}) { const summaryWinnerReviewRate = Number(limitAssessment?.winnerReviewRate); const winnerActiveAverageMsPerParsed = Number(limitAssessment?.winnerActiveAverageMsPerParsed); const winnerActiveProjectedMsFor100 = Number(limitAssessment?.winnerActiveProjectedMsFor100); + const winnerAverageCaptureRoundTripOverheadMs = Number(limitAssessment?.winnerAverageCaptureRoundTripOverheadMs); if (winner.qualified !== true) errors.push("Winner engine result must be qualified."); if (!Number.isFinite(winnerMissRate)) { errors.push(`Winner missRate must be a finite number, got ${winner.missRate ?? ""}.`); @@ -116,10 +134,19 @@ function validateAssessment(assessment, options = {}) { } if (!Number.isFinite(winnerActiveAverageMsPerParsed) || winnerActiveAverageMsPerParsed <= 0) { errors.push(`Winner active average timing must be a positive finite number, got ${limitAssessment?.winnerActiveAverageMsPerParsed ?? ""}.`); + } else if (maxActiveAverageMsPerParsed !== null && Number.isFinite(maxActiveAverageMsPerParsed) && winnerActiveAverageMsPerParsed > maxActiveAverageMsPerParsed) { + errors.push(`Winner active average timing exceeds ${maxActiveAverageMsPerParsed}ms: ${winnerActiveAverageMsPerParsed}.`); } if (!Number.isFinite(winnerActiveProjectedMsFor100) || winnerActiveProjectedMsFor100 <= 0) { errors.push(`Winner projected100 timing must be a positive finite number, got ${limitAssessment?.winnerActiveProjectedMsFor100 ?? ""}.`); } + if (maxCaptureRoundTripOverheadMs !== null && Number.isFinite(maxCaptureRoundTripOverheadMs)) { + if (!Number.isFinite(winnerAverageCaptureRoundTripOverheadMs)) { + errors.push(`Winner capture roundtrip overhead must be a finite number, got ${limitAssessment?.winnerAverageCaptureRoundTripOverheadMs ?? ""}.`); + } else if (winnerAverageCaptureRoundTripOverheadMs > maxCaptureRoundTripOverheadMs) { + errors.push(`Winner capture roundtrip overhead exceeds ${maxCaptureRoundTripOverheadMs}ms: ${winnerAverageCaptureRoundTripOverheadMs}.`); + } + } } return { @@ -130,6 +157,8 @@ function validateAssessment(assessment, options = {}) { winnerEngine: limitAssessment?.winnerEngine, winnerActiveAverageMsPerParsed: limitAssessment?.winnerActiveAverageMsPerParsed, winnerActiveProjectedMsFor100: limitAssessment?.winnerActiveProjectedMsFor100, + winnerAverageCaptureRoundTripMs: limitAssessment?.winnerAverageCaptureRoundTripMs, + winnerAverageCaptureRoundTripOverheadMs: limitAssessment?.winnerAverageCaptureRoundTripOverheadMs, winnerMissRate: limitAssessment?.winnerMissRate, winnerReviewRate: limitAssessment?.winnerReviewRate, }; @@ -145,6 +174,8 @@ function formatSummary(result) { `winner: ${result.winnerEngine || "unknown"}`, `activeAvg: ${result.winnerActiveAverageMsPerParsed ?? "unknown"}ms/artifact`, `projected100: ${result.winnerActiveProjectedMsFor100 ?? "unknown"}ms`, + `captureRoundTrip: ${result.winnerAverageCaptureRoundTripMs ?? "unknown"}ms`, + `captureRoundTripOverhead: ${result.winnerAverageCaptureRoundTripOverheadMs ?? "unknown"}ms`, `missRate: ${result.winnerMissRate ?? "unknown"}`, `reviewRate: ${result.winnerReviewRate ?? "unknown"}`, ]; @@ -165,7 +196,15 @@ function main() { } const limit = Number(argValue("limit", "100")); const assessment = loadAssessment(inputPath); - const result = validateAssessment(assessment, { expectedWinner, limit }); + const maxActiveAverageMsPerParsed = argValue("max-active-average-ms", ""); + const maxCaptureRoundTripOverheadMs = argValue("max-capture-roundtrip-overhead-ms", ""); + const result = validateAssessment(assessment, { + expectedWinner, + limit, + allowSingleEngine: hasFlag("allow-single-engine"), + maxActiveAverageMsPerParsed: maxActiveAverageMsPerParsed === "" ? undefined : maxActiveAverageMsPerParsed, + maxCaptureRoundTripOverheadMs: maxCaptureRoundTripOverheadMs === "" ? undefined : maxCaptureRoundTripOverheadMs, + }); const payload = { inputPath: path.resolve(inputPath), ...result }; console.log(hasFlag("summary") ? formatSummary(payload) : JSON.stringify(payload, null, 2)); if (!result.ok) process.exit(1); diff --git a/src/eval/scanAssessmentValidatorScript.test.ts b/src/eval/scanAssessmentValidatorScript.test.ts index 421955e..c2d676a 100644 --- a/src/eval/scanAssessmentValidatorScript.test.ts +++ b/src/eval/scanAssessmentValidatorScript.test.ts @@ -15,11 +15,13 @@ function validAssessment() { winnerQualified: true, winnerActiveAverageMsPerParsed: 820, winnerActiveProjectedMsFor100: 82000, + winnerAverageCaptureRoundTripMs: 420, + winnerAverageCaptureRoundTripOverheadMs: 160, winnerMissRate: 0, winnerReviewRate: 0.04, engines: [ - { engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.04 }, - { engine: "current", qualified: true, missRate: 0, reviewRate: 0.06 }, + { engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.04, averageCaptureRoundTripMs: 420, averageCaptureRoundTripOverheadMs: 160 }, + { engine: "current", qualified: true, missRate: 0, reviewRate: 0.06, averageCaptureRoundTripMs: 460, averageCaptureRoundTripOverheadMs: 190 }, ], }, limits: [ @@ -30,11 +32,13 @@ function validAssessment() { winnerQualified: true, winnerActiveAverageMsPerParsed: 390, winnerActiveProjectedMsFor100: 39000, + winnerAverageCaptureRoundTripMs: 364, + winnerAverageCaptureRoundTripOverheadMs: 152, winnerMissRate: 0, winnerReviewRate: 0.05, engines: [ - { engine: "current", qualified: true, missRate: 0, reviewRate: 0.05 }, - { engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.1 }, + { engine: "current", qualified: true, missRate: 0, reviewRate: 0.05, averageCaptureRoundTripMs: 364, averageCaptureRoundTripOverheadMs: 152 }, + { engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.1, averageCaptureRoundTripMs: 410, averageCaptureRoundTripOverheadMs: 180 }, ], }, ], @@ -94,6 +98,7 @@ describe("scan assessment validator", () => { expect(output).toContain("limit: 100"); expect(output).toContain("winner: ik-traineddata"); expect(output).toContain("activeAvg: 820ms/artifact"); + expect(output).toContain("captureRoundTripOverhead: 160ms"); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -120,6 +125,65 @@ describe("scan assessment validator", () => { } }); + it("accepts explicit speed and capture-overhead budgets when the winner stays inside them", () => { + const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-")); + try { + const inputPath = writeAssessment(dir, validAssessment()); + const output = execFileSync( + "node", + [ + "scripts/validate-scan-assessment.cjs", + `--input=${inputPath}`, + "--summary", + "--limit=20", + "--max-active-average-ms=400", + "--max-capture-roundtrip-overhead-ms=160", + ], + { + cwd: process.cwd(), + encoding: "utf8", + }, + ); + expect(output).toContain("scan assessment: PASS"); + expect(output).toContain("captureRoundTripOverhead: 152ms"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects speed and capture-overhead budgets when the winner exceeds them", () => { + const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-")); + try { + const inputPath = writeAssessment(dir, validAssessment()); + let stdout = ""; + try { + execFileSync( + "node", + [ + "scripts/validate-scan-assessment.cjs", + `--input=${inputPath}`, + "--summary", + "--limit=20", + "--max-active-average-ms=333", + "--max-capture-roundtrip-overhead-ms=120", + ], + { + cwd: process.cwd(), + encoding: "utf8", + stdio: "pipe", + }, + ); + } catch (error) { + stdout = String((error as { stdout?: string }).stdout || ""); + } + expect(stdout).toContain("scan assessment: FAIL"); + expect(stdout).toContain("Winner active average timing exceeds 333ms"); + expect(stdout).toContain("Winner capture roundtrip overhead exceeds 120ms"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("rejects invalid requested limits", () => { const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-")); try { @@ -179,6 +243,12 @@ describe("scan assessment validator", () => { expect(packageJson.scripts["scan:iterate:compare:validated:wait"]).toBe( "npm run scan:live:preflight:wait && npm run scan:iterate:compare && npm run scan:assessment:validate -- --latest --summary --limit=20", ); + expect(packageJson.scripts["scan:repeatability"]).toBe( + "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\live-soak.ps1 -RepeatabilityRun -ScanEngine current", + ); + expect(packageJson.scripts["scan:repeatability:wait"]).toBe( + "npm run scan:live:preflight:wait && npm run scan:repeatability && npm run scan:assessment:validate -- --latest --summary --limit=100 --expect-winner=current --allow-single-engine", + ); }); it("rejects a mismatched expected winner", () => { @@ -223,7 +293,9 @@ describe("scan assessment validator", () => { const payload = validAssessment(); payload.goal100Decision = "not-comparable: current and ik-traineddata were not both run"; payload.goal100.comparisonComplete = false; - payload.goal100.engines = [{ engine: "current", qualified: true, missRate: 0, reviewRate: 0 }]; + payload.goal100.engines = [ + { engine: "current", qualified: true, missRate: 0, reviewRate: 0, averageCaptureRoundTripMs: 360, averageCaptureRoundTripOverheadMs: 150 }, + ]; const inputPath = writeAssessment(dir, payload); expect(() => execFileSync("node", ["scripts/validate-scan-assessment.cjs", `--input=${inputPath}`], { @@ -236,6 +308,38 @@ describe("scan assessment validator", () => { } }); + it("accepts a single-engine 100-artifact repeatability run only with the explicit flag", () => { + const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-")); + try { + const payload = validAssessment(); + payload.goal100Decision = "not-comparable: current and ik-traineddata were not both run"; + payload.goal100.comparisonComplete = false; + payload.goal100.winnerEngine = "current"; + payload.goal100.engines = [ + { engine: "current", qualified: true, missRate: 0, reviewRate: 0, averageCaptureRoundTripMs: 360, averageCaptureRoundTripOverheadMs: 150 }, + ]; + const inputPath = writeAssessment(dir, payload); + const output = execFileSync( + "node", + [ + "scripts/validate-scan-assessment.cjs", + `--input=${inputPath}`, + "--summary", + "--expect-winner=current", + "--allow-single-engine", + ], + { + cwd: process.cwd(), + encoding: "utf8", + }, + ); + expect(output).toContain("scan assessment: PASS"); + expect(output).toContain("winner: current"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("rejects an unqualified winner", () => { const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-")); try { diff --git a/src/features/scan/components/DiagnosticsView.tsx b/src/features/scan/components/DiagnosticsView.tsx index 57e6feb..03f4ed0 100644 --- a/src/features/scan/components/DiagnosticsView.tsx +++ b/src/features/scan/components/DiagnosticsView.tsx @@ -31,10 +31,10 @@ const appDiagnosisSections = [ tone: "warn", icon: ClipboardList, items: [ - "Paimon-Menue-Einstieg ist gebaut, aber live noch nicht mit 2/20/45 Limits validiert.", - "Native/IK-Tesseract ist nur als Benchmark-Pfad vorbereitet, noch nicht Standard.", - "Positive locked=true Probe und erneuter Equipped-Footer-Livebeweis an bekannten Artifacts fehlen.", - "Empfehlungen bleiben Nebenfunktion, bis Scanner-Vertrauen und Review-Rate stabil genug sind.", + "Empfehlungen und Build-UX sind bewusst noch nicht der naechste Hauptfokus.", + "Auto-Entry-Modi wie paimon-menu und direct-inventory bleiben Dev-Control Experimente, nicht Produktionspfad.", + "3 Artifacts/Sekunde ist noch nicht bewiesen; der sichtbare Pfad liegt knapp darunter.", + "Mehr bestaetigte OCR-/Review-Corpus-Faelle fehlen, bevor wir breite Qualitaet behaupten.", ], }, { @@ -42,10 +42,10 @@ const appDiagnosisSections = [ tone: "risk", icon: AlertTriangle, items: [ - "OCR ist weiterhin der Haupt-Risikofaktor; einige Felder landen noch in Fallback, Ableitung oder Review.", - "Bild-Preprocessing kann nur an echten Captures bewertet werden, nicht allein mit Text-Eval.", - "Auto-Scan braucht bei erhoehtem Genshin auch eine erhoehte App-Laufzeit.", - "Groessere Runs brauchen weiter Beobachtung auf Scroll-Uebergaenge, Wiederholseiten und Review-Quote.", + "Scanner ist stark im aktuellen sichtbaren Inventory-Pfad; Repeatability ueber spaetere Sessions ist der naechste Pruefpunkt.", + "OCR ist schnell genug fuer gute Runs, bleibt aber der groesste Qualitaets- und Speed-Hebel.", + "Capture-Roundtrip ist sichtbar: Roundtrip und Overhead muessen bei 20/45/100 Runs mit bewertet werden.", + "Groessere Runs brauchen weiter Beobachtung auf Scroll-Uebergaenge, Wiederholseiten, Duplicate-Rate und Review-Quote.", ], }, { @@ -53,11 +53,11 @@ const appDiagnosisSections = [ tone: "next", icon: Target, items: [ - "Review-Corpus aus echten Samples vergroessern und mit `npm run eval` messbar halten.", - "Review-Export fuer Equipped-Footer und locked=true Kandidaten nutzen.", - "OCR-Benchmark gegen identische Crops fahren und erst danach Engine-Standard wechseln.", - "Paimon-Menue-Pfad live pruefen und bei Blockade sichtbar auf visible-inventory zurueckfallen.", - "Diagnose weiter als Operator-Cockpit halten: Live-Status, Evidenz und naechster sicherer Schritt.", + "Mehr Live-Wiederholungen: 20/45/100 Runs in spaeteren Sessions, sauber als Repeatability statt IK-Claim markieren.", + "Review-Samples sauber labeln und ueber `npm run eval:prepare-confirmed` ins Eval-Corpus uebernehmen.", + "Scanner-UI weiter beruhigen, aber den funktionierenden visible-inventory Pfad nicht umbauen.", + "Capture-Roundtrip-Overhead als eigenen Optimierungspunkt pruefen, bevor weitere OCR-Engine-Wechsel priorisiert werden.", + "Erst danach Empfehlungen wieder staerker nach vorne ziehen.", ], }, ]; diff --git a/src/lib/artifactOcrParser.test.ts b/src/lib/artifactOcrParser.test.ts index 3006ede..78430e7 100644 --- a/src/lib/artifactOcrParser.test.ts +++ b/src/lib/artifactOcrParser.test.ts @@ -81,6 +81,22 @@ describe("parseArtifactCandidate", () => { expect(parsed?.equipped).toBe("Aino"); }); + it("derives a unique piece and set from a distinctive partial OCR fragment", () => { + const parsed = parseArtifactCandidate(captureFromOcr({ + "artifact-name": "Wontiroms Creation pan", + "artifact-slot": "Plume of Death", + "artifact-main-stat-label": "ATK", + "artifact-level": "+0", + "artifact-substats": "+ HP+239\n+ Energy Recharge+4.5%\n+ Elemental Mastery+23\n- ATK+4.1% unactivated", + })); + + expect(parsed?.name).toBe("Sharpness That Ceased Upon Wondrous Creation"); + expect(parsed?.slot).toBe("Plume of Death"); + expect(parsed?.setName).toBe("Disenchantment in Deep Shadow"); + expect(parsed?.fields.name.source).toBe("fallback"); + expect(parsed?.fields.name.confidence).toBe(72); + }); + it("keeps goblet elemental damage main stats separate from crit substats", () => { const parsed = parseArtifactCandidate(captureFromOcr({ "artifact-title": "Deep Gallery's Bestowed Banquet\nGoblet of Eonothem", diff --git a/src/lib/artifactOcrParser.ts b/src/lib/artifactOcrParser.ts index da7c024..4b8e34d 100644 --- a/src/lib/artifactOcrParser.ts +++ b/src/lib/artifactOcrParser.ts @@ -178,6 +178,9 @@ function parseArtifactName(titleText: string): ParsedField { if (alias) return field(alias, 96, "database"); } + const partialPiece = derivePieceFromDistinctivePartialName(titleText); + if (partialPiece) return field(partialPiece, 72, "fallback"); + const knownPiece = fuzzyFindKnown(titleText, knownPieceNames, 0.72); if (knownPiece) return field(knownPiece.value, Math.round(knownPiece.score * 100), knownPiece.score >= 0.98 ? "database" : "fallback"); @@ -294,6 +297,20 @@ function deriveSetFromPartialPieceName(text: string) { return sets.length === 1 ? sets[0] : ""; } +function derivePieceFromDistinctivePartialName(text: string) { + const words = cleanupOcrLabel(text) + .split(/\s+/) + .map((word) => simplifyForMatch(word)) + .filter((word) => word.length >= 8); + if (words.length === 0) return ""; + + const candidates = knownPieceNames.filter((piece) => { + const normalizedPiece = simplifyForMatch(piece); + return words.some((word) => normalizedPiece.includes(word)); + }); + return candidates.length === 1 ? candidates[0] : ""; +} + function findMainValue(text: string, mainStat: string, slot: string, level: number | null): ParsedField { const cleaned = text.replace(/\b20\b/g, " ").replace(/[Oo]/g, "0"); const percentValue = extractPercentValue(cleaned); diff --git a/src/types/global.d.ts b/src/types/global.d.ts index 9f5fb68..c1d9760 100644 --- a/src/types/global.d.ts +++ b/src/types/global.d.ts @@ -248,7 +248,8 @@ export interface AutomationGuard { } export interface GdiCaptureResult { - dataUrl: string; + dataUrl?: string; + imageBase64?: string; width: number; height: number; originX: number;