Improve scanner repeatability guardrails

This commit is contained in:
AzuTear
2026-07-09 10:21:13 +02:00
parent 13fd46c104
commit c10f17b4ef
13 changed files with 356 additions and 39 deletions
+43 -5
View File
@@ -268,10 +268,20 @@ Latest live timing evidence on 2026-07-08:
signature `2026-07-08-direct-gdi-reviewfix` completed `20/20`, `18` stored, signature `2026-07-08-direct-gdi-reviewfix` completed `20/20`, `18` stored,
`0` review, `2` duplicates, `0` misses, `7973 ms` elapsed, or `0` review, `2` duplicates, `0` misses, `7973 ms` elapsed, or
`399 ms/artifact`. 3 artifacts/second is not proven. `399 ms/artifact`. 3 artifacts/second is not proven.
- Follow-up 3 artifacts/second attempts on 2026-07-09:
after the distinctive partial piece parser fix, the best clean repeatability
run reached `336 ms/artifact` with `20/20` parsed, `0` review, `0` misses,
`318 ms` average capture roundtrip, and `138 ms` roundtrip overhead. Later
runs with crop priority and image-payload cleanup stayed clean but ranged
around `346-351 ms/artifact`; the strict `333 ms/artifact` budget remains
unproven.
- Rejected speed experiments: - Rejected speed experiments:
detail-region capture, `GAA_OCR_WORKERS=5`, DataURL-to-buffer decode, and detail-region capture, `GAA_OCR_WORKERS=5`, DataURL-to-buffer decode, and
substat OCR `PSM.SINGLE_COLUMN` were all live/benchmark tested and were slower substat OCR `PSM.SINGLE_COLUMN` were all live/benchmark tested and were slower
than the direct-GDI baseline. Keep `GAA_OCR_WORKERS=4` for current runs. than the direct-GDI baseline. Later checks also rejected skipping
`analyzePaimonMenu`, skipping lock-state as a production shortcut, reducing
the artifact-level crop scale, and `GAA_OCR_WORKERS=6` as the default.
Keep `GAA_OCR_WORKERS=4` for current runs.
- Quality-gated current-vs-IK comparison: - Quality-gated current-vs-IK comparison:
`npm run scan:goal:compare:validated` produced `npm run scan:goal:compare:validated` produced
`outputs/live-soak/2026-07-08T18-38-35/scan-performance-assessment.json` `outputs/live-soak/2026-07-08T18-38-35/scan-performance-assessment.json`
@@ -449,15 +459,17 @@ uses the default `current` scan engine; use `scan:goal:ik` for a native
IK-traineddata scan pass. Use `scan:goal:compare` to run both scan engines IK-traineddata scan pass. Use `scan:goal:compare` to run both scan engines
back-to-back with the same limits and one combined CSV. The CSV is the quickest evidence for back-to-back with the same limits and one combined CSV. The CSV is the quickest evidence for
`averageMsPerParsed`, `activeAverageMsPerParsed`, `averageCaptureMs`, `averageMsPerParsed`, `activeAverageMsPerParsed`, `averageCaptureMs`,
`averageCaptureRoundTripMs`, `averageCaptureRoundTripOverheadMs`,
`captureP50Ms`, `captureP90Ms`, `averageOcrMs`, `ocrP50Ms`, `ocrP90Ms`, `captureP50Ms`, `captureP90Ms`, `averageOcrMs`, `ocrP50Ms`, `ocrP90Ms`,
`averageCardReadyMs`, `averageScrollReadyMs`, `artifactsPerMinute`, and `averageCardReadyMs`, `averageScrollReadyMs`, `artifactsPerMinute`, and
`projectedMsFor100`. `projectedMsFor100`.
The run also writes `scan-performance-assessment.json`, which groups results by The run also writes `scan-performance-assessment.json`, which groups results by
limit, picks the best qualified engine, and labels the dominant bottleneck as limit, picks the best qualified engine, and labels the dominant bottleneck as
OCR, capture, card-ready, or scroll-ready. A qualified winner must finish the OCR, capture-roundtrip-overhead, capture, card-ready, or scroll-ready. A
run, parse the requested count, keep miss rate under 2%, and keep review rate qualified winner must finish the run, parse the requested count, keep miss rate
at or below 15%; review and miss rates are penalized before active average speed under 2%, and keep review rate at or below 15%; review and miss rates are
is used as the tie-breaker. For IK-target claims, check `goal100Decision`; it penalized before active average speed is used as the tie-breaker. For IK-target
claims, check `goal100Decision`; it
must read `qualified-comparison: winner=<engine>`, and must read `qualified-comparison: winner=<engine>`, and
`goal100.comparisonComplete` must be `true` so a single-engine 100-artifact run `goal100.comparisonComplete` must be `true` so a single-engine 100-artifact run
is not mistaken for a current-vs-IK comparison. is not mistaken for a current-vs-IK comparison.
@@ -478,12 +490,38 @@ instead of the final 100-artifact proof. Add `--summary` when you want a short
report-ready PASS/FAIL output that includes the input assessment path and report-ready PASS/FAIL output that includes the input assessment path and
assessment `createdAt` timestamp. assessment `createdAt` timestamp.
Optional budget flags are useful for the current speed work:
```powershell
npm run scan:assessment:validate -- --latest --summary --limit=20 --max-active-average-ms=333 --max-capture-roundtrip-overhead-ms=120
```
`--max-active-average-ms=333` is the strict 3 artifacts/second check. Use a
separate `--max-capture-roundtrip-overhead-ms` budget when deciding whether the
next optimization belongs in native capture transport instead of OCR.
Single-engine repeatability runs may be validated with `--allow-single-engine`,
but that mode is only for repeatability evidence and must not be used for IK
parity claims.
`npm run scan:goal:compare:validated` is the preferred final command: it runs `npm run scan:goal:compare:validated` is the preferred final command: it runs
the live preflight first, then the full comparison, and then validates the the live preflight first, then the full comparison, and then validates the
newest assessment with `--summary`. Use newest assessment with `--summary`. Use
`npm run scan:goal:compare:validated:wait` for the same final flow when starting `npm run scan:goal:compare:validated:wait` for the same final flow when starting
immediately after UAC. immediately after UAC.
For later-session repeatability without changing OCR engines, use:
```powershell
npm run scan:repeatability:wait
```
That command runs the current visible-inventory engine at 20, 45, and 100
artifacts, then validates the 100-artifact result as single-engine evidence.
It uses the `live-soak.ps1 -RepeatabilityRun` switch instead of passing a
comma-separated `-Limits` value through npm/cmd, because Windows argument
parsing can collapse `20,45,100` into one unsafe number. The script also refuses
limits above 1800 as a final guard.
The assessment ranking can be verified without Genshin or the Electron app: The assessment ranking can be verified without Genshin or the Electron app:
```powershell ```powershell
+5
View File
@@ -15,6 +15,7 @@ npm run eval:review-candidates # export unconfirmed review samples for human la
npm test # runs the eval gate alongside the rest of the suite npm test # runs the eval gate alongside the rest of the suite
npm run scan:assessment:test # verifies quality-first scan ranking logic npm run scan:assessment:test # verifies quality-first scan ranking logic
npm run scan:iterate:compare:validated:wait # 20-artifact live comparison npm run scan:iterate:compare:validated:wait # 20-artifact live comparison
npm run scan:repeatability:wait # 20/45/100 current-engine repeatability
npm run scan:goal:compare:validated:wait # final 100-artifact live comparison npm run scan:goal:compare:validated:wait # final 100-artifact live comparison
``` ```
@@ -22,6 +23,10 @@ The report prints exact-match rate, overall field accuracy, a per-field
breakdown (critical fields marked with `*`), and every failing case with an breakdown (critical fields marked with `*`), and every failing case with an
`expected "..." got "..."` diff. `expected "..." got "..."` diff.
Repeatability runs are single-engine evidence. Use them to prove that the
visible-inventory scanner stays stable across later sessions, but keep
current-vs-IK claims on `scan:goal:compare:validated:*`.
## How it works ## How it works
- `src/eval/ocrEvalHarness.ts` - pure metric functions. `runOcrEval(cases)` - `src/eval/ocrEvalHarness.ts` - pure metric functions. `runOcrEval(cases)`
+38 -4
View File
@@ -144,6 +144,37 @@ Current status after the 2026-07-09 merge to `main`:
end-to-end capture roundtrip time. Use `averageCaptureRoundTripMs` and end-to-end capture roundtrip time. Use `averageCaptureRoundTripMs` and
`averageCaptureRoundTripOverheadMs` in the next `limit=20` live iteration to `averageCaptureRoundTripOverheadMs` in the next `limit=20` live iteration to
decide whether the next cut belongs in native capture transport or OCR. decide whether the next cut belongs in native capture transport or OCR.
- **Repeatability and capture-overhead guardrails (2026-07-09)** -
`scripts/live-soak.ps1` now writes capture roundtrip and roundtrip-overhead
timing into `scan-performance-assessment.json`. The assessment validator can
enforce optional speed budgets with `--max-active-average-ms` and
`--max-capture-roundtrip-overhead-ms`, and `npm run scan:repeatability:wait`
runs 20/45/100 current-engine passes as repeatability evidence without
presenting them as an IK comparison. `-RepeatabilityRun` now sets those limits
inside PowerShell, and the script refuses unsafe limits above 1800 so npm/cmd
argument parsing cannot accidentally turn `20,45,100` into one oversized run.
- **Distinctive partial piece recovery (2026-07-09)** - a live repeatability run
exposed four identical OCR misses where the piece name was read as
`Wontiroms Creation pan`. The parser now derives a piece only when a long OCR
fragment uniquely matches exactly one known artifact piece. This recovered the
local case as `Sharpness That Ceased Upon Wondrous Creation` /
`Disenchantment in Deep Shadow` without adding a broad fuzzy exception.
- **Repeatability live pass after parser fix (2026-07-09)** -
`outputs/live-soak/2026-07-09T09-29-11/scan-performance-assessment.json`
captured a clean current-engine 20-artifact run: `20/20` parsed, `0` review,
`0` misses, `336 ms/artifact` active average, `318 ms` average capture
roundtrip, and `138 ms` average roundtrip overhead. The strict 3 artifacts per
second budget still failed by 3 ms (`336 ms` vs `333 ms`).
- **3/s follow-up experiments (2026-07-09)** - tested and rejected several
shortcut-style optimizations because live runs got slower or added risk:
skipping Paimon-menu analysis, skipping lock-state as a production shortcut,
reducing the artifact-level crop scale, and raising the OCR worker pool to 6.
The kept low-risk changes are fast-profile OCR crop priority and avoiding a
duplicate DataURL string when the native helper already returns Base64. A
follow-up clean 20-artifact run after payload cleanup reached `351 ms/artifact`,
`331 ms` capture roundtrip, and `146 ms` roundtrip overhead, so the next
credible 3/s work is native capture transport/roundtrip reduction, not UI
recommendation work.
- **3/s live attempt (2026-07-08)** - the missing-detail-preview review trigger - **3/s live attempt (2026-07-08)** - the missing-detail-preview review trigger
was fixed and tested. The best clean 20-artifact run reached `7285 ms` was fixed and tested. The best clean 20-artifact run reached `7285 ms`
(`364 ms/artifact`, about `2.75 artifacts/second`) with 0 review and 0 misses. (`364 ms/artifact`, about `2.75 artifacts/second`) with 0 review and 0 misses.
@@ -218,14 +249,17 @@ resolution or without UI work best tested live:
substat OCR time, or a larger capture pipeline change that avoids full-frame substat OCR time, or a larger capture pipeline change that avoids full-frame
PNG/Base64 transport without hurting safety checks. The target remains PNG/Base64 transport without hurting safety checks. The target remains
`<= 6667 ms` elapsed for 20 parsed artifacts with 0 misses and no silent OCR `<= 6667 ms` elapsed for 20 parsed artifacts with 0 misses and no silent OCR
review regression. review regression. The latest clean 20-artifact repeatability run reached
`336 ms/artifact`, so 3/s remains close but unproven.
6. **Broader scan soak test** — direct-GDI current-engine runs now passed at 6. **Broader scan soak test** — direct-GDI current-engine runs now passed at
`20/20`, `45/45`, and `100/100` with 0 misses. Continue with repeat runs if `20/20`, `45/45`, and `100/100` with 0 misses. Continue with
duplicate rate needs tuning. `npm run scan:repeatability:wait` in later sessions to check duplicate rate,
scroll behavior, and capture roundtrip timing without changing defaults.
7. **Repeatability pass** — repeat the qualified current-vs-IK-traineddata run 7. **Repeatability pass** — repeat the qualified current-vs-IK-traineddata run
in a later live session before making major OCR-engine defaults or speed in a later live session before making major OCR-engine defaults or speed
claims beyond this environment. claims beyond this environment. Current-engine-only repeatability is useful
evidence, but it is not an IK parity claim.
Visible-page limits up to 20, scroll/page-transition limit 45, the final Visible-page limits up to 20, scroll/page-transition limit 45, the final
100-artifact current-vs-IK-traineddata comparison, equipped footer live smokes, 100-artifact current-vs-IK-traineddata comparison, equipped footer live smokes,
+19 -1
View File
@@ -407,6 +407,12 @@ async function captureSourceFromGdi(sourceId: string, sourceName: string, option
} }
function nativeImageFromGdiCapture(gdi: Awaited<ReturnType<InputHelperService["capturePrimaryScreenViaGdi"]>>) { function nativeImageFromGdiCapture(gdi: Awaited<ReturnType<InputHelperService["capturePrimaryScreenViaGdi"]>>) {
if (gdi.imageBase64) {
return nativeImage.createFromBuffer(Buffer.from(gdi.imageBase64, "base64"));
}
if (!gdi.dataUrl) {
throw new Error("GDI capture returned no image payload.");
}
return nativeImage.createFromDataURL(gdi.dataUrl); return nativeImage.createFromDataURL(gdi.dataUrl);
} }
@@ -1296,6 +1302,14 @@ async function buildCaptureResult(
const omitInventoryPreview = Boolean(options.omitInventoryPreview || options.ocrMode === "artifact"); const omitInventoryPreview = Boolean(options.omitInventoryPreview || options.ocrMode === "artifact");
const detailFingerprint = imageCropFingerprint(sourceImage, detailRect, size); const detailFingerprint = imageCropFingerprint(sourceImage, detailRect, size);
const inventoryFingerprint = imageCropFingerprint(sourceImage, inventoryRect, size); const inventoryFingerprint = imageCropFingerprint(sourceImage, inventoryRect, size);
const fastOcrPriority: Record<string, number> = {
"artifact-substats": 0,
"artifact-name": 1,
"artifact-footer": 2,
"artifact-slot": 3,
"artifact-main-stat-label": 4,
"artifact-level": 5,
};
const croppedPayload = crops const croppedPayload = crops
.filter((crop) => crop.ocrEnabled !== false) .filter((crop) => crop.ocrEnabled !== false)
.map((crop) => { .map((crop) => {
@@ -1307,7 +1321,11 @@ async function buildCaptureResult(
} }
: null; : null;
}) })
.filter((crop): crop is OcrCropPayload => Boolean(crop)); .filter((crop): crop is OcrCropPayload => Boolean(crop))
.sort((left, right) => {
if (options.ocrMode !== "artifact" || options.ocrProfile !== "fast") return 0;
return (fastOcrPriority[left.id] ?? 100) - (fastOcrPriority[right.id] ?? 100);
});
const prepareMs = Date.now() - buildStartedAt; const prepareMs = Date.now() - buildStartedAt;
const ocrEngine = ocrEngineFromOptions(options); const ocrEngine = ocrEngineFromOptions(options);
const ocrStartedAt = Date.now(); const ocrStartedAt = Date.now();
+3 -1
View File
@@ -292,7 +292,9 @@ export function createInputHelperService(options: { userDataPath: string; exePat
: "primary-screen"; : "primary-screen";
return { return {
dataUrl: `data:image/png;base64,${base64}`, ...(typeof result.imageBase64 === "string" && result.imageBase64
? { imageBase64: base64 }
: { dataUrl: `data:image/png;base64,${base64}` }),
width: Number(result.width), width: Number(result.width),
height: Number(result.height), height: Number(result.height),
originX: Number(result.originX), originX: Number(result.originX),
+2
View File
@@ -28,6 +28,8 @@
"scan:iterate:compare": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\live-soak.ps1 -Limits 20 -ScanEngine compare -BenchmarkOcr", "scan:iterate:compare": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\live-soak.ps1 -Limits 20 -ScanEngine compare -BenchmarkOcr",
"scan:iterate:compare:validated": "npm run scan:live:preflight && npm run scan:iterate:compare && npm run scan:assessment:validate -- --latest --summary --limit=20", "scan:iterate:compare:validated": "npm run scan:live:preflight && npm run scan:iterate:compare && npm run scan:assessment:validate -- --latest --summary --limit=20",
"scan:iterate:compare:validated:wait": "npm run scan:live:preflight:wait && npm run scan:iterate:compare && npm run scan:assessment:validate -- --latest --summary --limit=20", "scan:iterate:compare:validated:wait": "npm run scan:live:preflight:wait && npm run scan:iterate:compare && npm run scan:assessment:validate -- --latest --summary --limit=20",
"scan:repeatability": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\live-soak.ps1 -RepeatabilityRun -ScanEngine current",
"scan:repeatability:wait": "npm run scan:live:preflight:wait && npm run scan:repeatability && npm run scan:assessment:validate -- --latest --summary --limit=100 --expect-winner=current --allow-single-engine",
"scan:live:preflight": "node scripts/live-preflight.cjs", "scan:live:preflight": "node scripts/live-preflight.cjs",
"scan:live:preflight:wait": "node scripts/live-preflight.cjs --wait=120", "scan:live:preflight:wait": "node scripts/live-preflight.cjs --wait=120",
"scan:assessment:test": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\live-soak.ps1 -SelfTestAssessment", "scan:assessment:test": "powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\live-soak.ps1 -SelfTestAssessment",
+45 -4
View File
@@ -6,6 +6,7 @@ param(
[int]$TimeoutSeconds = 600, [int]$TimeoutSeconds = 600,
[string]$OutputRoot = (Join-Path (Resolve-Path -LiteralPath ".").Path "outputs\live-soak"), [string]$OutputRoot = (Join-Path (Resolve-Path -LiteralPath ".").Path "outputs\live-soak"),
[switch]$GoalRun, [switch]$GoalRun,
[switch]$RepeatabilityRun,
[ValidateSet("current", "ik-traineddata", "compare")] [ValidateSet("current", "ik-traineddata", "compare")]
[string]$ScanEngine = "current", [string]$ScanEngine = "current",
[switch]$BenchmarkOcr, [switch]$BenchmarkOcr,
@@ -138,6 +139,8 @@ function Convert-ScanTimingSummary([object]$StatusPayload, [int]$Limit, [string]
projectedMsFor100 = [int]$stats.projectedMsFor100 projectedMsFor100 = [int]$stats.projectedMsFor100
activeProjectedMsFor100 = [int]$stats.activeProjectedMsFor100 activeProjectedMsFor100 = [int]$stats.activeProjectedMsFor100
averageCaptureMs = [int]$stats.averageCaptureMs averageCaptureMs = [int]$stats.averageCaptureMs
averageCaptureRoundTripMs = [int]$stats.averageCaptureRoundTripMs
averageCaptureRoundTripOverheadMs = [int]$stats.averageCaptureRoundTripOverheadMs
averageOcrMs = [int]$stats.averageOcrMs averageOcrMs = [int]$stats.averageOcrMs
captureP50Ms = [int]$stats.captureP50Ms captureP50Ms = [int]$stats.captureP50Ms
captureP90Ms = [int]$stats.captureP90Ms captureP90Ms = [int]$stats.captureP90Ms
@@ -150,13 +153,15 @@ function Convert-ScanTimingSummary([object]$StatusPayload, [int]$Limit, [string]
scrollReadyCount = [int]$stats.scrollReadyCount scrollReadyCount = [int]$stats.scrollReadyCount
averageScrollReadyMs = [int]$stats.averageScrollReadyMs averageScrollReadyMs = [int]$stats.averageScrollReadyMs
captureMs = [int]$stats.captureMs captureMs = [int]$stats.captureMs
captureRoundTripMs = [int]$stats.captureRoundTripMs
captureRoundTripOverheadMs = [int]$stats.captureRoundTripOverheadMs
ocrMs = [int]$stats.ocrMs ocrMs = [int]$stats.ocrMs
reviewStatus = [string]$StatusPayload.status.reviewStatus reviewStatus = [string]$StatusPayload.status.reviewStatus
} }
} }
function Write-ScanTimingLine([object]$Timing) { function Write-ScanTimingLine([object]$Timing) {
Write-Host ("engine={0} limit={1} status={2} parsed={3} review={4} miss={5} elapsed={6}ms active={7}ms flush={8}ms avg={9}ms activeAvg={10}ms captureAvg={11}ms captureP50={12}ms captureP90={13}ms ocrAvg={14}ms ocrP50={15}ms ocrP90={16}ms cardReadyAvg={17}ms scrollReadyAvg={18}ms ppm={19} activePpm={20} projected100={21}ms activeProjected100={22}ms" -f ` Write-Host ("engine={0} limit={1} status={2} parsed={3} review={4} miss={5} elapsed={6}ms active={7}ms flush={8}ms avg={9}ms activeAvg={10}ms captureAvg={11}ms roundtripAvg={12}ms roundtripOverheadAvg={13}ms captureP50={14}ms captureP90={15}ms ocrAvg={16}ms ocrP50={17}ms ocrP90={18}ms cardReadyAvg={19}ms scrollReadyAvg={20}ms ppm={21} activePpm={22} projected100={23}ms activeProjected100={24}ms" -f `
$Timing.engine, $Timing.engine,
$Timing.limit, $Timing.limit,
$Timing.status, $Timing.status,
@@ -169,6 +174,8 @@ function Write-ScanTimingLine([object]$Timing) {
$Timing.averageMsPerParsed, $Timing.averageMsPerParsed,
$Timing.activeAverageMsPerParsed, $Timing.activeAverageMsPerParsed,
$Timing.averageCaptureMs, $Timing.averageCaptureMs,
$Timing.averageCaptureRoundTripMs,
$Timing.averageCaptureRoundTripOverheadMs,
$Timing.captureP50Ms, $Timing.captureP50Ms,
$Timing.captureP90Ms, $Timing.captureP90Ms,
$Timing.averageOcrMs, $Timing.averageOcrMs,
@@ -185,6 +192,7 @@ function Write-ScanTimingLine([object]$Timing) {
function Get-TimingBottleneck([object]$Timing) { function Get-TimingBottleneck([object]$Timing) {
$parts = @( $parts = @(
[pscustomobject]@{ name = "ocr"; value = [int]$Timing.averageOcrMs }, [pscustomobject]@{ name = "ocr"; value = [int]$Timing.averageOcrMs },
[pscustomobject]@{ name = "capture-roundtrip-overhead"; value = [int]$Timing.averageCaptureRoundTripOverheadMs },
[pscustomobject]@{ name = "capture"; value = [int]$Timing.averageCaptureMs }, [pscustomobject]@{ name = "capture"; value = [int]$Timing.averageCaptureMs },
[pscustomobject]@{ name = "card-ready"; value = [int]$Timing.averageCardReadyMs }, [pscustomobject]@{ name = "card-ready"; value = [int]$Timing.averageCardReadyMs },
[pscustomobject]@{ name = "scroll-ready"; value = [int]$Timing.averageScrollReadyMs } [pscustomobject]@{ name = "scroll-ready"; value = [int]$Timing.averageScrollReadyMs }
@@ -198,6 +206,7 @@ function Get-TimingRecommendation([object]$Timing) {
$bottleneck = Get-TimingBottleneck $Timing $bottleneck = Get-TimingBottleneck $Timing
switch ($bottleneck) { switch ($bottleneck) {
"ocr" { return "OCR dominates; compare engine, crop count, worker pool, and parser-derived fields first." } "ocr" { return "OCR dominates; compare engine, crop count, worker pool, and parser-derived fields first." }
"capture-roundtrip-overhead" { return "Capture transport overhead dominates; inspect native encode, IPC payload size, and Base64/DataURL conversion before OCR changes." }
"capture" { return "Capture dominates; reduce payloads/crops and avoid full-frame or Base64 work in the hot loop." } "capture" { return "Capture dominates; reduce payloads/crops and avoid full-frame or Base64 work in the hot loop." }
"card-ready" { return "Card-ready dominates; tune detail fingerprint polling against IK's 200ms item wait." } "card-ready" { return "Card-ready dominates; tune detail fingerprint polling against IK's 200ms item wait." }
"scroll-ready" { return "Scroll-ready dominates; tune page fingerprint polling against IK's 100ms fast-scroll wait." } "scroll-ready" { return "Scroll-ready dominates; tune page fingerprint polling against IK's 100ms fast-scroll wait." }
@@ -274,6 +283,8 @@ function New-PerformanceAssessment([object[]]$Summaries) {
activeProjectedMsFor100 = $entry.activeProjectedMsFor100 activeProjectedMsFor100 = $entry.activeProjectedMsFor100
averageOcrMs = $entry.averageOcrMs averageOcrMs = $entry.averageOcrMs
averageCaptureMs = $entry.averageCaptureMs averageCaptureMs = $entry.averageCaptureMs
averageCaptureRoundTripMs = $entry.averageCaptureRoundTripMs
averageCaptureRoundTripOverheadMs = $entry.averageCaptureRoundTripOverheadMs
averageCardReadyMs = $entry.averageCardReadyMs averageCardReadyMs = $entry.averageCardReadyMs
averageScrollReadyMs = $entry.averageScrollReadyMs averageScrollReadyMs = $entry.averageScrollReadyMs
bottleneck = Get-TimingBottleneck $entry bottleneck = Get-TimingBottleneck $entry
@@ -292,6 +303,8 @@ function New-PerformanceAssessment([object[]]$Summaries) {
winnerReviewRate = $winner.reviewRate winnerReviewRate = $winner.reviewRate
winnerActiveAverageMsPerParsed = $winner.activeAverageMsPerParsed winnerActiveAverageMsPerParsed = $winner.activeAverageMsPerParsed
winnerActiveProjectedMsFor100 = $winner.activeProjectedMsFor100 winnerActiveProjectedMsFor100 = $winner.activeProjectedMsFor100
winnerAverageCaptureRoundTripMs = $winner.averageCaptureRoundTripMs
winnerAverageCaptureRoundTripOverheadMs = $winner.averageCaptureRoundTripOverheadMs
engines = $engineReports engines = $engineReports
} }
} }
@@ -322,7 +335,7 @@ function Write-PerformanceAssessment([object]$Assessment) {
Write-Host "assessment goal100: $($Assessment.goal100Decision)" Write-Host "assessment goal100: $($Assessment.goal100Decision)"
} }
foreach ($limit in @($Assessment.limits)) { foreach ($limit in @($Assessment.limits)) {
Write-Host ("assessment limit={0}: winner={1} qualified={2} completeCompare={3} engines={4} missRate={5:P1} reviewRate={6:P1} activeAvg={7}ms projected100={8}ms" -f ` Write-Host ("assessment limit={0}: winner={1} qualified={2} completeCompare={3} engines={4} missRate={5:P1} reviewRate={6:P1} activeAvg={7}ms projected100={8}ms roundtripOverheadAvg={9}ms" -f `
$limit.limit, $limit.limit,
$limit.winnerEngine, $limit.winnerEngine,
$limit.winnerQualified, $limit.winnerQualified,
@@ -331,9 +344,10 @@ function Write-PerformanceAssessment([object]$Assessment) {
$limit.winnerMissRate, $limit.winnerMissRate,
$limit.winnerReviewRate, $limit.winnerReviewRate,
$limit.winnerActiveAverageMsPerParsed, $limit.winnerActiveAverageMsPerParsed,
$limit.winnerActiveProjectedMsFor100) $limit.winnerActiveProjectedMsFor100,
$limit.winnerAverageCaptureRoundTripOverheadMs)
foreach ($engine in @($limit.engines)) { foreach ($engine in @($limit.engines)) {
Write-Host (" {0}: qualified={1}, missRate={2:P1}, reviewRate={3:P1}, bottleneck={4}, activeAvg={5}ms, ocr={6}ms, capture={7}ms, cardReady={8}ms, scrollReady={9}ms, decision={10}" -f ` Write-Host (" {0}: qualified={1}, missRate={2:P1}, reviewRate={3:P1}, bottleneck={4}, activeAvg={5}ms, ocr={6}ms, capture={7}ms, roundtrip={8}ms, roundtripOverhead={9}ms, cardReady={10}ms, scrollReady={11}ms, decision={12}" -f `
$engine.engine, $engine.engine,
$engine.qualified, $engine.qualified,
$engine.missRate, $engine.missRate,
@@ -342,6 +356,8 @@ function Write-PerformanceAssessment([object]$Assessment) {
$engine.activeAverageMsPerParsed, $engine.activeAverageMsPerParsed,
$engine.averageOcrMs, $engine.averageOcrMs,
$engine.averageCaptureMs, $engine.averageCaptureMs,
$engine.averageCaptureRoundTripMs,
$engine.averageCaptureRoundTripOverheadMs,
$engine.averageCardReadyMs, $engine.averageCardReadyMs,
$engine.averageScrollReadyMs, $engine.averageScrollReadyMs,
$engine.qualityDecision) $engine.qualityDecision)
@@ -363,6 +379,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 70000 activeProjectedMsFor100 = 70000
averageOcrMs = 260 averageOcrMs = 260
averageCaptureMs = 160 averageCaptureMs = 160
averageCaptureRoundTripMs = 340
averageCaptureRoundTripOverheadMs = 180
averageCardReadyMs = 190 averageCardReadyMs = 190
averageScrollReadyMs = 60 averageScrollReadyMs = 60
}, },
@@ -378,6 +396,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 82000 activeProjectedMsFor100 = 82000
averageOcrMs = 210 averageOcrMs = 210
averageCaptureMs = 170 averageCaptureMs = 170
averageCaptureRoundTripMs = 300
averageCaptureRoundTripOverheadMs = 130
averageCardReadyMs = 205 averageCardReadyMs = 205
averageScrollReadyMs = 80 averageScrollReadyMs = 80
}, },
@@ -393,6 +413,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 39000 activeProjectedMsFor100 = 39000
averageOcrMs = 150 averageOcrMs = 150
averageCaptureMs = 130 averageCaptureMs = 130
averageCaptureRoundTripMs = 280
averageCaptureRoundTripOverheadMs = 150
averageCardReadyMs = 80 averageCardReadyMs = 80
averageScrollReadyMs = 0 averageScrollReadyMs = 0
}, },
@@ -408,6 +430,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 46000 activeProjectedMsFor100 = 46000
averageOcrMs = 190 averageOcrMs = 190
averageCaptureMs = 130 averageCaptureMs = 130
averageCaptureRoundTripMs = 310
averageCaptureRoundTripOverheadMs = 180
averageCardReadyMs = 80 averageCardReadyMs = 80
averageScrollReadyMs = 0 averageScrollReadyMs = 0
}, },
@@ -423,6 +447,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 30000 activeProjectedMsFor100 = 30000
averageOcrMs = 100 averageOcrMs = 100
averageCaptureMs = 90 averageCaptureMs = 90
averageCaptureRoundTripMs = 190
averageCaptureRoundTripOverheadMs = 100
averageCardReadyMs = 40 averageCardReadyMs = 40
averageScrollReadyMs = 20 averageScrollReadyMs = 20
}, },
@@ -438,6 +464,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 60000 activeProjectedMsFor100 = 60000
averageOcrMs = 230 averageOcrMs = 230
averageCaptureMs = 130 averageCaptureMs = 130
averageCaptureRoundTripMs = 300
averageCaptureRoundTripOverheadMs = 170
averageCardReadyMs = 160 averageCardReadyMs = 160
averageScrollReadyMs = 50 averageScrollReadyMs = 50
} }
@@ -492,6 +520,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 50000 activeProjectedMsFor100 = 50000
averageOcrMs = 180 averageOcrMs = 180
averageCaptureMs = 120 averageCaptureMs = 120
averageCaptureRoundTripMs = 260
averageCaptureRoundTripOverheadMs = 140
averageCardReadyMs = 100 averageCardReadyMs = 100
averageScrollReadyMs = 50 averageScrollReadyMs = 50
} }
@@ -665,6 +695,17 @@ if ($GoalRun) {
$Limits = @(2, 5, 20, 45, 100) $Limits = @(2, 5, 20, 45, 100)
$BenchmarkOcr = $true $BenchmarkOcr = $true
} }
if ($RepeatabilityRun) {
$Limits = @(20, 45, 100)
}
$Limits = @($Limits | ForEach-Object {
$limit = [int]$_
if ($limit -lt 1 -or $limit -gt 1800) {
throw "Refusing unsafe scan limit '$limit'. Expected a bounded value from 1 to 1800."
}
$limit
})
$ScanEngines = if ($ScanEngine -eq "compare") { @("current", "ik-traineddata") } else { @($ScanEngine) } $ScanEngines = if ($ScanEngine -eq "compare") { @("current", "ik-traineddata") } else { @($ScanEngine) }
$RunSummaries = @() $RunSummaries = @()
+42 -3
View File
@@ -51,6 +51,13 @@ function validateAssessment(assessment, options = {}) {
const errors = []; const errors = [];
const expectedWinner = options.expectedWinner || "any"; const expectedWinner = options.expectedWinner || "any";
const expectedLimit = options.limit === undefined ? 100 : Number(options.limit); const expectedLimit = options.limit === undefined ? 100 : Number(options.limit);
const allowSingleEngine = options.allowSingleEngine === true;
const maxActiveAverageMsPerParsed = options.maxActiveAverageMsPerParsed === undefined
? null
: Number(options.maxActiveAverageMsPerParsed);
const maxCaptureRoundTripOverheadMs = options.maxCaptureRoundTripOverheadMs === undefined
? null
: Number(options.maxCaptureRoundTripOverheadMs);
if (!assessment || typeof assessment !== "object") { if (!assessment || typeof assessment !== "object") {
return { ok: false, errors: ["Assessment must be a JSON object."] }; return { ok: false, errors: ["Assessment must be a JSON object."] };
} }
@@ -58,20 +65,26 @@ function validateAssessment(assessment, options = {}) {
if (!Number.isInteger(expectedLimit) || expectedLimit < 1) { if (!Number.isInteger(expectedLimit) || expectedLimit < 1) {
errors.push(`--limit must be a positive integer, got ${options.limit}.`); errors.push(`--limit must be a positive integer, got ${options.limit}.`);
} }
if (maxActiveAverageMsPerParsed !== null && (!Number.isFinite(maxActiveAverageMsPerParsed) || maxActiveAverageMsPerParsed <= 0)) {
errors.push(`--max-active-average-ms must be a positive finite number, got ${options.maxActiveAverageMsPerParsed}.`);
}
if (maxCaptureRoundTripOverheadMs !== null && (!Number.isFinite(maxCaptureRoundTripOverheadMs) || maxCaptureRoundTripOverheadMs < 0)) {
errors.push(`--max-capture-roundtrip-overhead-ms must be a non-negative finite number, got ${options.maxCaptureRoundTripOverheadMs}.`);
}
const limitAssessment = findLimitAssessment(assessment, expectedLimit); const limitAssessment = findLimitAssessment(assessment, expectedLimit);
if (!limitAssessment || typeof limitAssessment !== "object") { if (!limitAssessment || typeof limitAssessment !== "object") {
errors.push(`Missing limit=${expectedLimit} assessment.`); errors.push(`Missing limit=${expectedLimit} assessment.`);
} }
if (expectedLimit === 100 && assessment.goal100Decision !== `qualified-comparison: winner=${limitAssessment?.winnerEngine}`) { if (!allowSingleEngine && expectedLimit === 100 && assessment.goal100Decision !== `qualified-comparison: winner=${limitAssessment?.winnerEngine}`) {
errors.push(`goal100Decision is not a qualified comparison: ${assessment.goal100Decision || "<missing>"}`); errors.push(`goal100Decision is not a qualified comparison: ${assessment.goal100Decision || "<missing>"}`);
} }
if (limitAssessment?.limit !== expectedLimit) { if (limitAssessment?.limit !== expectedLimit) {
errors.push(`limit assessment must be ${expectedLimit}, got ${limitAssessment?.limit ?? "<missing>"}.`); errors.push(`limit assessment must be ${expectedLimit}, got ${limitAssessment?.limit ?? "<missing>"}.`);
} }
if (limitAssessment?.comparisonComplete !== true) errors.push(`limit=${expectedLimit}.comparisonComplete must be true.`); if (!allowSingleEngine && limitAssessment?.comparisonComplete !== true) errors.push(`limit=${expectedLimit}.comparisonComplete must be true.`);
if (limitAssessment?.winnerQualified !== true) errors.push(`limit=${expectedLimit}.winnerQualified must be true.`); if (limitAssessment?.winnerQualified !== true) errors.push(`limit=${expectedLimit}.winnerQualified must be true.`);
if (expectedWinner !== "any" && limitAssessment?.winnerEngine !== expectedWinner) { if (expectedWinner !== "any" && limitAssessment?.winnerEngine !== expectedWinner) {
errors.push(`Expected winner '${expectedWinner}', got '${limitAssessment?.winnerEngine ?? "<missing>"}'.`); errors.push(`Expected winner '${expectedWinner}', got '${limitAssessment?.winnerEngine ?? "<missing>"}'.`);
@@ -79,9 +92,13 @@ function validateAssessment(assessment, options = {}) {
const engines = Array.isArray(limitAssessment?.engines) ? limitAssessment.engines : []; const engines = Array.isArray(limitAssessment?.engines) ? limitAssessment.engines : [];
const engineNames = new Set(engines.map((entry) => entry?.engine)); const engineNames = new Set(engines.map((entry) => entry?.engine));
if (!allowSingleEngine) {
for (const required of ["current", "ik-traineddata"]) { for (const required of ["current", "ik-traineddata"]) {
if (!engineNames.has(required)) errors.push(`limit=${expectedLimit} is missing engine result: ${required}.`); if (!engineNames.has(required)) errors.push(`limit=${expectedLimit} is missing engine result: ${required}.`);
} }
} else if (engines.length < 1) {
errors.push(`limit=${expectedLimit} must include at least one engine result.`);
}
const winner = engines.find((entry) => entry?.engine === limitAssessment?.winnerEngine); const winner = engines.find((entry) => entry?.engine === limitAssessment?.winnerEngine);
if (!winner) { if (!winner) {
@@ -93,6 +110,7 @@ function validateAssessment(assessment, options = {}) {
const summaryWinnerReviewRate = Number(limitAssessment?.winnerReviewRate); const summaryWinnerReviewRate = Number(limitAssessment?.winnerReviewRate);
const winnerActiveAverageMsPerParsed = Number(limitAssessment?.winnerActiveAverageMsPerParsed); const winnerActiveAverageMsPerParsed = Number(limitAssessment?.winnerActiveAverageMsPerParsed);
const winnerActiveProjectedMsFor100 = Number(limitAssessment?.winnerActiveProjectedMsFor100); const winnerActiveProjectedMsFor100 = Number(limitAssessment?.winnerActiveProjectedMsFor100);
const winnerAverageCaptureRoundTripOverheadMs = Number(limitAssessment?.winnerAverageCaptureRoundTripOverheadMs);
if (winner.qualified !== true) errors.push("Winner engine result must be qualified."); if (winner.qualified !== true) errors.push("Winner engine result must be qualified.");
if (!Number.isFinite(winnerMissRate)) { if (!Number.isFinite(winnerMissRate)) {
errors.push(`Winner missRate must be a finite number, got ${winner.missRate ?? "<missing>"}.`); errors.push(`Winner missRate must be a finite number, got ${winner.missRate ?? "<missing>"}.`);
@@ -116,10 +134,19 @@ function validateAssessment(assessment, options = {}) {
} }
if (!Number.isFinite(winnerActiveAverageMsPerParsed) || winnerActiveAverageMsPerParsed <= 0) { if (!Number.isFinite(winnerActiveAverageMsPerParsed) || winnerActiveAverageMsPerParsed <= 0) {
errors.push(`Winner active average timing must be a positive finite number, got ${limitAssessment?.winnerActiveAverageMsPerParsed ?? "<missing>"}.`); errors.push(`Winner active average timing must be a positive finite number, got ${limitAssessment?.winnerActiveAverageMsPerParsed ?? "<missing>"}.`);
} else if (maxActiveAverageMsPerParsed !== null && Number.isFinite(maxActiveAverageMsPerParsed) && winnerActiveAverageMsPerParsed > maxActiveAverageMsPerParsed) {
errors.push(`Winner active average timing exceeds ${maxActiveAverageMsPerParsed}ms: ${winnerActiveAverageMsPerParsed}.`);
} }
if (!Number.isFinite(winnerActiveProjectedMsFor100) || winnerActiveProjectedMsFor100 <= 0) { if (!Number.isFinite(winnerActiveProjectedMsFor100) || winnerActiveProjectedMsFor100 <= 0) {
errors.push(`Winner projected100 timing must be a positive finite number, got ${limitAssessment?.winnerActiveProjectedMsFor100 ?? "<missing>"}.`); errors.push(`Winner projected100 timing must be a positive finite number, got ${limitAssessment?.winnerActiveProjectedMsFor100 ?? "<missing>"}.`);
} }
if (maxCaptureRoundTripOverheadMs !== null && Number.isFinite(maxCaptureRoundTripOverheadMs)) {
if (!Number.isFinite(winnerAverageCaptureRoundTripOverheadMs)) {
errors.push(`Winner capture roundtrip overhead must be a finite number, got ${limitAssessment?.winnerAverageCaptureRoundTripOverheadMs ?? "<missing>"}.`);
} else if (winnerAverageCaptureRoundTripOverheadMs > maxCaptureRoundTripOverheadMs) {
errors.push(`Winner capture roundtrip overhead exceeds ${maxCaptureRoundTripOverheadMs}ms: ${winnerAverageCaptureRoundTripOverheadMs}.`);
}
}
} }
return { return {
@@ -130,6 +157,8 @@ function validateAssessment(assessment, options = {}) {
winnerEngine: limitAssessment?.winnerEngine, winnerEngine: limitAssessment?.winnerEngine,
winnerActiveAverageMsPerParsed: limitAssessment?.winnerActiveAverageMsPerParsed, winnerActiveAverageMsPerParsed: limitAssessment?.winnerActiveAverageMsPerParsed,
winnerActiveProjectedMsFor100: limitAssessment?.winnerActiveProjectedMsFor100, winnerActiveProjectedMsFor100: limitAssessment?.winnerActiveProjectedMsFor100,
winnerAverageCaptureRoundTripMs: limitAssessment?.winnerAverageCaptureRoundTripMs,
winnerAverageCaptureRoundTripOverheadMs: limitAssessment?.winnerAverageCaptureRoundTripOverheadMs,
winnerMissRate: limitAssessment?.winnerMissRate, winnerMissRate: limitAssessment?.winnerMissRate,
winnerReviewRate: limitAssessment?.winnerReviewRate, winnerReviewRate: limitAssessment?.winnerReviewRate,
}; };
@@ -145,6 +174,8 @@ function formatSummary(result) {
`winner: ${result.winnerEngine || "unknown"}`, `winner: ${result.winnerEngine || "unknown"}`,
`activeAvg: ${result.winnerActiveAverageMsPerParsed ?? "unknown"}ms/artifact`, `activeAvg: ${result.winnerActiveAverageMsPerParsed ?? "unknown"}ms/artifact`,
`projected100: ${result.winnerActiveProjectedMsFor100 ?? "unknown"}ms`, `projected100: ${result.winnerActiveProjectedMsFor100 ?? "unknown"}ms`,
`captureRoundTrip: ${result.winnerAverageCaptureRoundTripMs ?? "unknown"}ms`,
`captureRoundTripOverhead: ${result.winnerAverageCaptureRoundTripOverheadMs ?? "unknown"}ms`,
`missRate: ${result.winnerMissRate ?? "unknown"}`, `missRate: ${result.winnerMissRate ?? "unknown"}`,
`reviewRate: ${result.winnerReviewRate ?? "unknown"}`, `reviewRate: ${result.winnerReviewRate ?? "unknown"}`,
]; ];
@@ -165,7 +196,15 @@ function main() {
} }
const limit = Number(argValue("limit", "100")); const limit = Number(argValue("limit", "100"));
const assessment = loadAssessment(inputPath); const assessment = loadAssessment(inputPath);
const result = validateAssessment(assessment, { expectedWinner, limit }); const maxActiveAverageMsPerParsed = argValue("max-active-average-ms", "");
const maxCaptureRoundTripOverheadMs = argValue("max-capture-roundtrip-overhead-ms", "");
const result = validateAssessment(assessment, {
expectedWinner,
limit,
allowSingleEngine: hasFlag("allow-single-engine"),
maxActiveAverageMsPerParsed: maxActiveAverageMsPerParsed === "" ? undefined : maxActiveAverageMsPerParsed,
maxCaptureRoundTripOverheadMs: maxCaptureRoundTripOverheadMs === "" ? undefined : maxCaptureRoundTripOverheadMs,
});
const payload = { inputPath: path.resolve(inputPath), ...result }; const payload = { inputPath: path.resolve(inputPath), ...result };
console.log(hasFlag("summary") ? formatSummary(payload) : JSON.stringify(payload, null, 2)); console.log(hasFlag("summary") ? formatSummary(payload) : JSON.stringify(payload, null, 2));
if (!result.ok) process.exit(1); if (!result.ok) process.exit(1);
+109 -5
View File
@@ -15,11 +15,13 @@ function validAssessment() {
winnerQualified: true, winnerQualified: true,
winnerActiveAverageMsPerParsed: 820, winnerActiveAverageMsPerParsed: 820,
winnerActiveProjectedMsFor100: 82000, winnerActiveProjectedMsFor100: 82000,
winnerAverageCaptureRoundTripMs: 420,
winnerAverageCaptureRoundTripOverheadMs: 160,
winnerMissRate: 0, winnerMissRate: 0,
winnerReviewRate: 0.04, winnerReviewRate: 0.04,
engines: [ engines: [
{ engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.04 }, { engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.04, averageCaptureRoundTripMs: 420, averageCaptureRoundTripOverheadMs: 160 },
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0.06 }, { engine: "current", qualified: true, missRate: 0, reviewRate: 0.06, averageCaptureRoundTripMs: 460, averageCaptureRoundTripOverheadMs: 190 },
], ],
}, },
limits: [ limits: [
@@ -30,11 +32,13 @@ function validAssessment() {
winnerQualified: true, winnerQualified: true,
winnerActiveAverageMsPerParsed: 390, winnerActiveAverageMsPerParsed: 390,
winnerActiveProjectedMsFor100: 39000, winnerActiveProjectedMsFor100: 39000,
winnerAverageCaptureRoundTripMs: 364,
winnerAverageCaptureRoundTripOverheadMs: 152,
winnerMissRate: 0, winnerMissRate: 0,
winnerReviewRate: 0.05, winnerReviewRate: 0.05,
engines: [ engines: [
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0.05 }, { engine: "current", qualified: true, missRate: 0, reviewRate: 0.05, averageCaptureRoundTripMs: 364, averageCaptureRoundTripOverheadMs: 152 },
{ engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.1 }, { engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.1, averageCaptureRoundTripMs: 410, averageCaptureRoundTripOverheadMs: 180 },
], ],
}, },
], ],
@@ -94,6 +98,7 @@ describe("scan assessment validator", () => {
expect(output).toContain("limit: 100"); expect(output).toContain("limit: 100");
expect(output).toContain("winner: ik-traineddata"); expect(output).toContain("winner: ik-traineddata");
expect(output).toContain("activeAvg: 820ms/artifact"); expect(output).toContain("activeAvg: 820ms/artifact");
expect(output).toContain("captureRoundTripOverhead: 160ms");
} finally { } finally {
rmSync(dir, { recursive: true, force: true }); rmSync(dir, { recursive: true, force: true });
} }
@@ -120,6 +125,65 @@ describe("scan assessment validator", () => {
} }
}); });
it("accepts explicit speed and capture-overhead budgets when the winner stays inside them", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {
const inputPath = writeAssessment(dir, validAssessment());
const output = execFileSync(
"node",
[
"scripts/validate-scan-assessment.cjs",
`--input=${inputPath}`,
"--summary",
"--limit=20",
"--max-active-average-ms=400",
"--max-capture-roundtrip-overhead-ms=160",
],
{
cwd: process.cwd(),
encoding: "utf8",
},
);
expect(output).toContain("scan assessment: PASS");
expect(output).toContain("captureRoundTripOverhead: 152ms");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("rejects speed and capture-overhead budgets when the winner exceeds them", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {
const inputPath = writeAssessment(dir, validAssessment());
let stdout = "";
try {
execFileSync(
"node",
[
"scripts/validate-scan-assessment.cjs",
`--input=${inputPath}`,
"--summary",
"--limit=20",
"--max-active-average-ms=333",
"--max-capture-roundtrip-overhead-ms=120",
],
{
cwd: process.cwd(),
encoding: "utf8",
stdio: "pipe",
},
);
} catch (error) {
stdout = String((error as { stdout?: string }).stdout || "");
}
expect(stdout).toContain("scan assessment: FAIL");
expect(stdout).toContain("Winner active average timing exceeds 333ms");
expect(stdout).toContain("Winner capture roundtrip overhead exceeds 120ms");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("rejects invalid requested limits", () => { it("rejects invalid requested limits", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-")); const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try { try {
@@ -179,6 +243,12 @@ describe("scan assessment validator", () => {
expect(packageJson.scripts["scan:iterate:compare:validated:wait"]).toBe( expect(packageJson.scripts["scan:iterate:compare:validated:wait"]).toBe(
"npm run scan:live:preflight:wait && npm run scan:iterate:compare && npm run scan:assessment:validate -- --latest --summary --limit=20", "npm run scan:live:preflight:wait && npm run scan:iterate:compare && npm run scan:assessment:validate -- --latest --summary --limit=20",
); );
expect(packageJson.scripts["scan:repeatability"]).toBe(
"powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\live-soak.ps1 -RepeatabilityRun -ScanEngine current",
);
expect(packageJson.scripts["scan:repeatability:wait"]).toBe(
"npm run scan:live:preflight:wait && npm run scan:repeatability && npm run scan:assessment:validate -- --latest --summary --limit=100 --expect-winner=current --allow-single-engine",
);
}); });
it("rejects a mismatched expected winner", () => { it("rejects a mismatched expected winner", () => {
@@ -223,7 +293,9 @@ describe("scan assessment validator", () => {
const payload = validAssessment(); const payload = validAssessment();
payload.goal100Decision = "not-comparable: current and ik-traineddata were not both run"; payload.goal100Decision = "not-comparable: current and ik-traineddata were not both run";
payload.goal100.comparisonComplete = false; payload.goal100.comparisonComplete = false;
payload.goal100.engines = [{ engine: "current", qualified: true, missRate: 0, reviewRate: 0 }]; payload.goal100.engines = [
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0, averageCaptureRoundTripMs: 360, averageCaptureRoundTripOverheadMs: 150 },
];
const inputPath = writeAssessment(dir, payload); const inputPath = writeAssessment(dir, payload);
expect(() => expect(() =>
execFileSync("node", ["scripts/validate-scan-assessment.cjs", `--input=${inputPath}`], { execFileSync("node", ["scripts/validate-scan-assessment.cjs", `--input=${inputPath}`], {
@@ -236,6 +308,38 @@ describe("scan assessment validator", () => {
} }
}); });
it("accepts a single-engine 100-artifact repeatability run only with the explicit flag", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {
const payload = validAssessment();
payload.goal100Decision = "not-comparable: current and ik-traineddata were not both run";
payload.goal100.comparisonComplete = false;
payload.goal100.winnerEngine = "current";
payload.goal100.engines = [
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0, averageCaptureRoundTripMs: 360, averageCaptureRoundTripOverheadMs: 150 },
];
const inputPath = writeAssessment(dir, payload);
const output = execFileSync(
"node",
[
"scripts/validate-scan-assessment.cjs",
`--input=${inputPath}`,
"--summary",
"--expect-winner=current",
"--allow-single-engine",
],
{
cwd: process.cwd(),
encoding: "utf8",
},
);
expect(output).toContain("scan assessment: PASS");
expect(output).toContain("winner: current");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("rejects an unqualified winner", () => { it("rejects an unqualified winner", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-")); const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try { try {
@@ -31,10 +31,10 @@ const appDiagnosisSections = [
tone: "warn", tone: "warn",
icon: ClipboardList, icon: ClipboardList,
items: [ items: [
"Paimon-Menue-Einstieg ist gebaut, aber live noch nicht mit 2/20/45 Limits validiert.", "Empfehlungen und Build-UX sind bewusst noch nicht der naechste Hauptfokus.",
"Native/IK-Tesseract ist nur als Benchmark-Pfad vorbereitet, noch nicht Standard.", "Auto-Entry-Modi wie paimon-menu und direct-inventory bleiben Dev-Control Experimente, nicht Produktionspfad.",
"Positive locked=true Probe und erneuter Equipped-Footer-Livebeweis an bekannten Artifacts fehlen.", "3 Artifacts/Sekunde ist noch nicht bewiesen; der sichtbare Pfad liegt knapp darunter.",
"Empfehlungen bleiben Nebenfunktion, bis Scanner-Vertrauen und Review-Rate stabil genug sind.", "Mehr bestaetigte OCR-/Review-Corpus-Faelle fehlen, bevor wir breite Qualitaet behaupten.",
], ],
}, },
{ {
@@ -42,10 +42,10 @@ const appDiagnosisSections = [
tone: "risk", tone: "risk",
icon: AlertTriangle, icon: AlertTriangle,
items: [ items: [
"OCR ist weiterhin der Haupt-Risikofaktor; einige Felder landen noch in Fallback, Ableitung oder Review.", "Scanner ist stark im aktuellen sichtbaren Inventory-Pfad; Repeatability ueber spaetere Sessions ist der naechste Pruefpunkt.",
"Bild-Preprocessing kann nur an echten Captures bewertet werden, nicht allein mit Text-Eval.", "OCR ist schnell genug fuer gute Runs, bleibt aber der groesste Qualitaets- und Speed-Hebel.",
"Auto-Scan braucht bei erhoehtem Genshin auch eine erhoehte App-Laufzeit.", "Capture-Roundtrip ist sichtbar: Roundtrip und Overhead muessen bei 20/45/100 Runs mit bewertet werden.",
"Groessere Runs brauchen weiter Beobachtung auf Scroll-Uebergaenge, Wiederholseiten und Review-Quote.", "Groessere Runs brauchen weiter Beobachtung auf Scroll-Uebergaenge, Wiederholseiten, Duplicate-Rate und Review-Quote.",
], ],
}, },
{ {
@@ -53,11 +53,11 @@ const appDiagnosisSections = [
tone: "next", tone: "next",
icon: Target, icon: Target,
items: [ items: [
"Review-Corpus aus echten Samples vergroessern und mit `npm run eval` messbar halten.", "Mehr Live-Wiederholungen: 20/45/100 Runs in spaeteren Sessions, sauber als Repeatability statt IK-Claim markieren.",
"Review-Export fuer Equipped-Footer und locked=true Kandidaten nutzen.", "Review-Samples sauber labeln und ueber `npm run eval:prepare-confirmed` ins Eval-Corpus uebernehmen.",
"OCR-Benchmark gegen identische Crops fahren und erst danach Engine-Standard wechseln.", "Scanner-UI weiter beruhigen, aber den funktionierenden visible-inventory Pfad nicht umbauen.",
"Paimon-Menue-Pfad live pruefen und bei Blockade sichtbar auf visible-inventory zurueckfallen.", "Capture-Roundtrip-Overhead als eigenen Optimierungspunkt pruefen, bevor weitere OCR-Engine-Wechsel priorisiert werden.",
"Diagnose weiter als Operator-Cockpit halten: Live-Status, Evidenz und naechster sicherer Schritt.", "Erst danach Empfehlungen wieder staerker nach vorne ziehen.",
], ],
}, },
]; ];
+16
View File
@@ -81,6 +81,22 @@ describe("parseArtifactCandidate", () => {
expect(parsed?.equipped).toBe("Aino"); expect(parsed?.equipped).toBe("Aino");
}); });
it("derives a unique piece and set from a distinctive partial OCR fragment", () => {
const parsed = parseArtifactCandidate(captureFromOcr({
"artifact-name": "Wontiroms Creation pan",
"artifact-slot": "Plume of Death",
"artifact-main-stat-label": "ATK",
"artifact-level": "+0",
"artifact-substats": "+ HP+239\n+ Energy Recharge+4.5%\n+ Elemental Mastery+23\n- ATK+4.1% unactivated",
}));
expect(parsed?.name).toBe("Sharpness That Ceased Upon Wondrous Creation");
expect(parsed?.slot).toBe("Plume of Death");
expect(parsed?.setName).toBe("Disenchantment in Deep Shadow");
expect(parsed?.fields.name.source).toBe("fallback");
expect(parsed?.fields.name.confidence).toBe(72);
});
it("keeps goblet elemental damage main stats separate from crit substats", () => { it("keeps goblet elemental damage main stats separate from crit substats", () => {
const parsed = parseArtifactCandidate(captureFromOcr({ const parsed = parseArtifactCandidate(captureFromOcr({
"artifact-title": "Deep Gallery's Bestowed Banquet\nGoblet of Eonothem", "artifact-title": "Deep Gallery's Bestowed Banquet\nGoblet of Eonothem",
+17
View File
@@ -178,6 +178,9 @@ function parseArtifactName(titleText: string): ParsedField {
if (alias) return field(alias, 96, "database"); if (alias) return field(alias, 96, "database");
} }
const partialPiece = derivePieceFromDistinctivePartialName(titleText);
if (partialPiece) return field(partialPiece, 72, "fallback");
const knownPiece = fuzzyFindKnown(titleText, knownPieceNames, 0.72); const knownPiece = fuzzyFindKnown(titleText, knownPieceNames, 0.72);
if (knownPiece) return field(knownPiece.value, Math.round(knownPiece.score * 100), knownPiece.score >= 0.98 ? "database" : "fallback"); if (knownPiece) return field(knownPiece.value, Math.round(knownPiece.score * 100), knownPiece.score >= 0.98 ? "database" : "fallback");
@@ -294,6 +297,20 @@ function deriveSetFromPartialPieceName(text: string) {
return sets.length === 1 ? sets[0] : ""; return sets.length === 1 ? sets[0] : "";
} }
function derivePieceFromDistinctivePartialName(text: string) {
const words = cleanupOcrLabel(text)
.split(/\s+/)
.map((word) => simplifyForMatch(word))
.filter((word) => word.length >= 8);
if (words.length === 0) return "";
const candidates = knownPieceNames.filter((piece) => {
const normalizedPiece = simplifyForMatch(piece);
return words.some((word) => normalizedPiece.includes(word));
});
return candidates.length === 1 ? candidates[0] : "";
}
function findMainValue(text: string, mainStat: string, slot: string, level: number | null): ParsedField { function findMainValue(text: string, mainStat: string, slot: string, level: number | null): ParsedField {
const cleaned = text.replace(/\b20\b/g, " ").replace(/[Oo]/g, "0"); const cleaned = text.replace(/\b20\b/g, " ").replace(/[Oo]/g, "0");
const percentValue = extractPercentValue(cleaned); const percentValue = extractPercentValue(cleaned);
+2 -1
View File
@@ -248,7 +248,8 @@ export interface AutomationGuard {
} }
export interface GdiCaptureResult { export interface GdiCaptureResult {
dataUrl: string; dataUrl?: string;
imageBase64?: string;
width: number; width: number;
height: number; height: number;
originX: number; originX: number;