bao/genshin-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve scanner repeatability guardrails

Browse Source
This commit is contained in:
AzuTear
2026-07-09 10:21:13 +02:00
parent 13fd46c104
commit c10f17b4ef
13 changed files with 356 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/live-soak.ps1
+45 -4
View File
@@ -6,6 +6,7 @@ param(
[int]$TimeoutSeconds = 600,
[string]$OutputRoot = (Join-Path (Resolve-Path -LiteralPath ".").Path "outputs\live-soak"),
[switch]$GoalRun,
[switch]$RepeatabilityRun,
[ValidateSet("current", "ik-traineddata", "compare")]
[string]$ScanEngine = "current",
[switch]$BenchmarkOcr,
@@ -138,6 +139,8 @@ function Convert-ScanTimingSummary([object]$StatusPayload, [int]$Limit, [string]
projectedMsFor100 = [int]$stats.projectedMsFor100
activeProjectedMsFor100 = [int]$stats.activeProjectedMsFor100
averageCaptureMs = [int]$stats.averageCaptureMs
averageCaptureRoundTripMs = [int]$stats.averageCaptureRoundTripMs
averageCaptureRoundTripOverheadMs = [int]$stats.averageCaptureRoundTripOverheadMs
averageOcrMs = [int]$stats.averageOcrMs
captureP50Ms = [int]$stats.captureP50Ms
captureP90Ms = [int]$stats.captureP90Ms
@@ -150,13 +153,15 @@ function Convert-ScanTimingSummary([object]$StatusPayload, [int]$Limit, [string]
scrollReadyCount = [int]$stats.scrollReadyCount
averageScrollReadyMs = [int]$stats.averageScrollReadyMs
captureMs = [int]$stats.captureMs
captureRoundTripMs = [int]$stats.captureRoundTripMs
captureRoundTripOverheadMs = [int]$stats.captureRoundTripOverheadMs
ocrMs = [int]$stats.ocrMs
reviewStatus = [string]$StatusPayload.status.reviewStatus
}
}
function Write-ScanTimingLine([object]$Timing) {
Write-Host ("engine={0} limit={1} status={2} parsed={3} review={4} miss={5} elapsed={6}ms active={7}ms flush={8}ms avg={9}ms activeAvg={10}ms captureAvg={11}ms captureP50={12}ms captureP90={13}ms ocrAvg={14}ms ocrP50={15}ms ocrP90={16}ms cardReadyAvg={17}ms scrollReadyAvg={18}ms ppm={19} activePpm={20} projected100={21}ms activeProjected100={22}ms" -f `
Write-Host ("engine={0} limit={1} status={2} parsed={3} review={4} miss={5} elapsed={6}ms active={7}ms flush={8}ms avg={9}ms activeAvg={10}ms captureAvg={11}ms roundtripAvg={12}ms roundtripOverheadAvg={13}ms captureP50={14}ms captureP90={15}ms ocrAvg={16}ms ocrP50={17}ms ocrP90={18}ms cardReadyAvg={19}ms scrollReadyAvg={20}ms ppm={21} activePpm={22} projected100={23}ms activeProjected100={24}ms" -f `
$Timing.engine,
$Timing.limit,
$Timing.status,
@@ -169,6 +174,8 @@ function Write-ScanTimingLine([object]$Timing) {
$Timing.averageMsPerParsed,
$Timing.activeAverageMsPerParsed,
$Timing.averageCaptureMs,
$Timing.averageCaptureRoundTripMs,
$Timing.averageCaptureRoundTripOverheadMs,
$Timing.captureP50Ms,
$Timing.captureP90Ms,
$Timing.averageOcrMs,
@@ -185,6 +192,7 @@ function Write-ScanTimingLine([object]$Timing) {
function Get-TimingBottleneck([object]$Timing) {
$parts = @(
[pscustomobject]@{ name = "ocr"; value = [int]$Timing.averageOcrMs },
[pscustomobject]@{ name = "capture-roundtrip-overhead"; value = [int]$Timing.averageCaptureRoundTripOverheadMs },
[pscustomobject]@{ name = "capture"; value = [int]$Timing.averageCaptureMs },
[pscustomobject]@{ name = "card-ready"; value = [int]$Timing.averageCardReadyMs },
[pscustomobject]@{ name = "scroll-ready"; value = [int]$Timing.averageScrollReadyMs }
@@ -198,6 +206,7 @@ function Get-TimingRecommendation([object]$Timing) {
$bottleneck = Get-TimingBottleneck $Timing
switch ($bottleneck) {
"ocr" { return "OCR dominates; compare engine, crop count, worker pool, and parser-derived fields first." }
"capture-roundtrip-overhead" { return "Capture transport overhead dominates; inspect native encode, IPC payload size, and Base64/DataURL conversion before OCR changes." }
"capture" { return "Capture dominates; reduce payloads/crops and avoid full-frame or Base64 work in the hot loop." }
"card-ready" { return "Card-ready dominates; tune detail fingerprint polling against IK's 200ms item wait." }
"scroll-ready" { return "Scroll-ready dominates; tune page fingerprint polling against IK's 100ms fast-scroll wait." }
@@ -274,6 +283,8 @@ function New-PerformanceAssessment([object[]]$Summaries) {
activeProjectedMsFor100 = $entry.activeProjectedMsFor100
averageOcrMs = $entry.averageOcrMs
averageCaptureMs = $entry.averageCaptureMs
averageCaptureRoundTripMs = $entry.averageCaptureRoundTripMs
averageCaptureRoundTripOverheadMs = $entry.averageCaptureRoundTripOverheadMs
averageCardReadyMs = $entry.averageCardReadyMs
averageScrollReadyMs = $entry.averageScrollReadyMs
bottleneck = Get-TimingBottleneck $entry
@@ -292,6 +303,8 @@ function New-PerformanceAssessment([object[]]$Summaries) {
winnerReviewRate = $winner.reviewRate
winnerActiveAverageMsPerParsed = $winner.activeAverageMsPerParsed
winnerActiveProjectedMsFor100 = $winner.activeProjectedMsFor100
winnerAverageCaptureRoundTripMs = $winner.averageCaptureRoundTripMs
winnerAverageCaptureRoundTripOverheadMs = $winner.averageCaptureRoundTripOverheadMs
engines = $engineReports
}
}
@@ -322,7 +335,7 @@ function Write-PerformanceAssessment([object]$Assessment) {
Write-Host "assessment goal100: $($Assessment.goal100Decision)"
}
foreach ($limit in @($Assessment.limits)) {
Write-Host ("assessment limit={0}: winner={1} qualified={2} completeCompare={3} engines={4} missRate={5:P1} reviewRate={6:P1} activeAvg={7}ms projected100={8}ms" -f `
Write-Host ("assessment limit={0}: winner={1} qualified={2} completeCompare={3} engines={4} missRate={5:P1} reviewRate={6:P1} activeAvg={7}ms projected100={8}ms roundtripOverheadAvg={9}ms" -f `
$limit.limit,
$limit.winnerEngine,
$limit.winnerQualified,
@@ -331,9 +344,10 @@ function Write-PerformanceAssessment([object]$Assessment) {
$limit.winnerMissRate,
$limit.winnerReviewRate,
$limit.winnerActiveAverageMsPerParsed,
$limit.winnerActiveProjectedMsFor100)
$limit.winnerActiveProjectedMsFor100,
$limit.winnerAverageCaptureRoundTripOverheadMs)
foreach ($engine in @($limit.engines)) {
Write-Host (" {0}: qualified={1}, missRate={2:P1}, reviewRate={3:P1}, bottleneck={4}, activeAvg={5}ms, ocr={6}ms, capture={7}ms, cardReady={8}ms, scrollReady={9}ms, decision={10}" -f `
Write-Host (" {0}: qualified={1}, missRate={2:P1}, reviewRate={3:P1}, bottleneck={4}, activeAvg={5}ms, ocr={6}ms, capture={7}ms, roundtrip={8}ms, roundtripOverhead={9}ms, cardReady={10}ms, scrollReady={11}ms, decision={12}" -f `
$engine.engine,
$engine.qualified,
$engine.missRate,
@@ -342,6 +356,8 @@ function Write-PerformanceAssessment([object]$Assessment) {
$engine.activeAverageMsPerParsed,
$engine.averageOcrMs,
$engine.averageCaptureMs,
$engine.averageCaptureRoundTripMs,
$engine.averageCaptureRoundTripOverheadMs,
$engine.averageCardReadyMs,
$engine.averageScrollReadyMs,
$engine.qualityDecision)
@@ -363,6 +379,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 70000
averageOcrMs = 260
averageCaptureMs = 160
averageCaptureRoundTripMs = 340
averageCaptureRoundTripOverheadMs = 180
averageCardReadyMs = 190
averageScrollReadyMs = 60
},
@@ -378,6 +396,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 82000
averageOcrMs = 210
averageCaptureMs = 170
averageCaptureRoundTripMs = 300
averageCaptureRoundTripOverheadMs = 130
averageCardReadyMs = 205
averageScrollReadyMs = 80
},
@@ -393,6 +413,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 39000
averageOcrMs = 150
averageCaptureMs = 130
averageCaptureRoundTripMs = 280
averageCaptureRoundTripOverheadMs = 150
averageCardReadyMs = 80
averageScrollReadyMs = 0
},
@@ -408,6 +430,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 46000
averageOcrMs = 190
averageCaptureMs = 130
averageCaptureRoundTripMs = 310
averageCaptureRoundTripOverheadMs = 180
averageCardReadyMs = 80
averageScrollReadyMs = 0
},
@@ -423,6 +447,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 30000
averageOcrMs = 100
averageCaptureMs = 90
averageCaptureRoundTripMs = 190
averageCaptureRoundTripOverheadMs = 100
averageCardReadyMs = 40
averageScrollReadyMs = 20
},
@@ -438,6 +464,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 60000
averageOcrMs = 230
averageCaptureMs = 130
averageCaptureRoundTripMs = 300
averageCaptureRoundTripOverheadMs = 170
averageCardReadyMs = 160
averageScrollReadyMs = 50
}
@@ -492,6 +520,8 @@ function Invoke-AssessmentSelfTest {
activeProjectedMsFor100 = 50000
averageOcrMs = 180
averageCaptureMs = 120
averageCaptureRoundTripMs = 260
averageCaptureRoundTripOverheadMs = 140
averageCardReadyMs = 100
averageScrollReadyMs = 50
}
@@ -665,6 +695,17 @@ if ($GoalRun) {
$Limits = @(2, 5, 20, 45, 100)
$BenchmarkOcr = $true
}
if ($RepeatabilityRun) {
$Limits = @(20, 45, 100)
}
$Limits = @($Limits | ForEach-Object {
$limit = [int]$_
if ($limit -lt 1 -or $limit -gt 1800) {
throw "Refusing unsafe scan limit '$limit'. Expected a bounded value from 1 to 1800."
}
$limit
})
$ScanEngines = if ($ScanEngine -eq "compare") { @("current", "ik-traineddata") } else { @($ScanEngine) }
$RunSummaries = @()