AzuTear
832 lines
32 KiB
PowerShell
832 lines
32 KiB
PowerShell
param(
|
|
[string]$BaseUrl = "http://127.0.0.1:17317",
|
|
[int[]]$ProbeIndices = @(1, 3),
|
|
[int[]]$Limits = @(2, 5, 10, 20),
|
|
[int]$PollIntervalSeconds = 2,
|
|
[int]$TimeoutSeconds = 600,
|
|
[string]$OutputRoot = (Join-Path (Resolve-Path -LiteralPath ".").Path "outputs\live-soak"),
|
|
[switch]$GoalRun,
|
|
[switch]$RepeatabilityRun,
|
|
[ValidateSet("current", "ik-traineddata", "compare")]
|
|
[string]$ScanEngine = "current",
|
|
[switch]$BenchmarkOcr,
|
|
[int]$BenchmarkLimit = 5,
|
|
[ValidateSet("current", "ik-traineddata", "compare")]
|
|
[string]$BenchmarkEngine = "compare",
|
|
[ValidateSet("fast", "full")]
|
|
[string]$BenchmarkProfile = "fast",
|
|
[switch]$SkipSmartCapture,
|
|
[switch]$SaveFullReviewSamples,
|
|
[switch]$AllowStaleBuild,
|
|
[string]$ExpectedAppSignature,
|
|
[switch]$ContinueAfterBlocked,
|
|
[switch]$SelfTestAssessment
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
if ([string]::IsNullOrWhiteSpace($ExpectedAppSignature)) {
|
|
$mainPath = Join-Path (Resolve-Path -LiteralPath ".").Path "electron\main.ts"
|
|
if (Test-Path -LiteralPath $mainPath) {
|
|
$mainSource = Get-Content -LiteralPath $mainPath -Raw
|
|
$match = [regex]::Match($mainSource, 'APP_RUNTIME_SIGNATURE\s*=\s*"([^"]+)"')
|
|
if ($match.Success) {
|
|
$ExpectedAppSignature = $match.Groups[1].Value
|
|
}
|
|
}
|
|
}
|
|
|
|
function ConvertTo-SafeFilePart([string]$Value) {
|
|
$safe = $Value -replace "[^A-Za-z0-9._-]+", "-"
|
|
$safe = $safe.Trim("-")
|
|
if ($safe.Length -eq 0) { return "item" }
|
|
if ($safe.Length -gt 80) { return $safe.Substring(0, 80) }
|
|
return $safe
|
|
}
|
|
|
|
function Invoke-DevJson([string]$Path) {
|
|
$uri = if ($Path.StartsWith("http")) { $Path } else { "$BaseUrl$Path" }
|
|
Invoke-RestMethod -Method Get -Uri $uri -TimeoutSec 60
|
|
}
|
|
|
|
function Save-Json([string]$Name, [object]$Payload) {
|
|
$path = Join-Path $RunDir "$(ConvertTo-SafeFilePart $Name).json"
|
|
$Payload | ConvertTo-Json -Depth 30 | Set-Content -LiteralPath $path -Encoding UTF8
|
|
return $path
|
|
}
|
|
|
|
function Assert-CurrentAppBuild([object]$HealthPayload) {
|
|
if ($AllowStaleBuild) { return }
|
|
|
|
if ($null -eq $HealthPayload.appBuild) {
|
|
throw "Dev endpoint is stale: /health has no appBuild signature. Close the old elevated Genshin Artifact Assistant/Electron instance, restart with npm run dev:admin, then rerun this script. Use -AllowStaleBuild only for debugging old instances."
|
|
}
|
|
|
|
if ([string]::IsNullOrWhiteSpace([string]$HealthPayload.appBuild.signature)) {
|
|
throw "Dev endpoint is stale: appBuild.signature is empty. Restart the elevated app before live scanner timing."
|
|
}
|
|
|
|
if (-not [string]::IsNullOrWhiteSpace($ExpectedAppSignature) -and [string]$HealthPayload.appBuild.signature -ne $ExpectedAppSignature) {
|
|
throw "Dev endpoint is stale: appBuild.signature='$($HealthPayload.appBuild.signature)' but current source expects '$ExpectedAppSignature'. Close the old Electron instance, restart with npm run dev:admin, then rerun this script. Use -AllowStaleBuild only for debugging old instances."
|
|
}
|
|
}
|
|
|
|
function Get-ScannerStatus {
|
|
Invoke-DevJson "/scanner/status"
|
|
}
|
|
|
|
function Test-ProbeSucceeded([object]$ProbePayload) {
|
|
if ($ProbePayload.ok) { return $true }
|
|
if ($ProbePayload.changed) { return $true }
|
|
if ($ProbePayload.click -and $ProbePayload.click.clicked -and $ProbePayload.click.moved -and -not $ProbePayload.click.inputBlocked) { return $true }
|
|
return $false
|
|
}
|
|
|
|
function Get-CompletedScanSummary([object]$StatusPayload, [int]$Limit) {
|
|
if ($null -ne $StatusPayload.status.summary) {
|
|
return $StatusPayload.status.summary
|
|
}
|
|
|
|
$reviewStatus = [string]$StatusPayload.status.reviewStatus
|
|
$status = "done"
|
|
if ($reviewStatus -match "blockiert") {
|
|
$status = "blocked"
|
|
} elseif ($reviewStatus -match "gestoppt") {
|
|
$status = "stopped"
|
|
}
|
|
|
|
$stats = $StatusPayload.status.stats
|
|
return [pscustomobject]@{
|
|
mode = "Automatischer Scan"
|
|
status = $status
|
|
clicked = [int]$stats.clicked
|
|
attempted = [int]$stats.attempted
|
|
verified = [int]$stats.verified
|
|
parsed = [int]$stats.parsed
|
|
stored = [int]$stats.stored
|
|
review = [int]$stats.review
|
|
duplicates = [int]$stats.duplicates
|
|
misses = [int]$stats.misses
|
|
pages = [int]$stats.pages
|
|
targetCount = $Limit
|
|
gridLabel = $reviewStatus
|
|
}
|
|
}
|
|
|
|
function Convert-ScanTimingSummary([object]$StatusPayload, [int]$Limit, [string]$Engine) {
|
|
$summary = Get-CompletedScanSummary $StatusPayload $Limit
|
|
$stats = $StatusPayload.status.stats
|
|
return [pscustomobject]@{
|
|
engine = $Engine
|
|
limit = $Limit
|
|
status = $summary.status
|
|
clicked = $summary.clicked
|
|
attempted = $summary.attempted
|
|
verified = $summary.verified
|
|
parsed = $summary.parsed
|
|
stored = $summary.stored
|
|
review = $summary.review
|
|
duplicates = $summary.duplicates
|
|
misses = $summary.misses
|
|
pages = $summary.pages
|
|
elapsedMs = [int]$stats.elapsedMs
|
|
activeScanMs = [int]$stats.activeScanMs
|
|
writeFlushMs = [int]$stats.writeFlushMs
|
|
averageMsPerParsed = [int]$stats.averageMsPerParsed
|
|
activeAverageMsPerParsed = [int]$stats.activeAverageMsPerParsed
|
|
artifactsPerMinute = [double]$stats.artifactsPerMinute
|
|
activeArtifactsPerMinute = [double]$stats.activeArtifactsPerMinute
|
|
projectedMsFor100 = [int]$stats.projectedMsFor100
|
|
activeProjectedMsFor100 = [int]$stats.activeProjectedMsFor100
|
|
averageCaptureMs = [int]$stats.averageCaptureMs
|
|
averageCaptureRoundTripMs = [int]$stats.averageCaptureRoundTripMs
|
|
averageCaptureRoundTripOverheadMs = [int]$stats.averageCaptureRoundTripOverheadMs
|
|
averageOcrMs = [int]$stats.averageOcrMs
|
|
captureP50Ms = [int]$stats.captureP50Ms
|
|
captureP90Ms = [int]$stats.captureP90Ms
|
|
ocrP50Ms = [int]$stats.ocrP50Ms
|
|
ocrP90Ms = [int]$stats.ocrP90Ms
|
|
cardReadyMs = [int]$stats.cardReadyMs
|
|
cardReadyCount = [int]$stats.cardReadyCount
|
|
averageCardReadyMs = [int]$stats.averageCardReadyMs
|
|
scrollReadyMs = [int]$stats.scrollReadyMs
|
|
scrollReadyCount = [int]$stats.scrollReadyCount
|
|
averageScrollReadyMs = [int]$stats.averageScrollReadyMs
|
|
captureMs = [int]$stats.captureMs
|
|
captureRoundTripMs = [int]$stats.captureRoundTripMs
|
|
captureRoundTripOverheadMs = [int]$stats.captureRoundTripOverheadMs
|
|
ocrMs = [int]$stats.ocrMs
|
|
reviewStatus = [string]$StatusPayload.status.reviewStatus
|
|
}
|
|
}
|
|
|
|
function Write-ScanTimingLine([object]$Timing) {
|
|
Write-Host ("engine={0} limit={1} status={2} parsed={3} review={4} miss={5} elapsed={6}ms active={7}ms flush={8}ms avg={9}ms activeAvg={10}ms captureAvg={11}ms roundtripAvg={12}ms roundtripOverheadAvg={13}ms captureP50={14}ms captureP90={15}ms ocrAvg={16}ms ocrP50={17}ms ocrP90={18}ms cardReadyAvg={19}ms scrollReadyAvg={20}ms ppm={21} activePpm={22} projected100={23}ms activeProjected100={24}ms" -f `
|
|
$Timing.engine,
|
|
$Timing.limit,
|
|
$Timing.status,
|
|
$Timing.parsed,
|
|
$Timing.review,
|
|
$Timing.misses,
|
|
$Timing.elapsedMs,
|
|
$Timing.activeScanMs,
|
|
$Timing.writeFlushMs,
|
|
$Timing.averageMsPerParsed,
|
|
$Timing.activeAverageMsPerParsed,
|
|
$Timing.averageCaptureMs,
|
|
$Timing.averageCaptureRoundTripMs,
|
|
$Timing.averageCaptureRoundTripOverheadMs,
|
|
$Timing.captureP50Ms,
|
|
$Timing.captureP90Ms,
|
|
$Timing.averageOcrMs,
|
|
$Timing.ocrP50Ms,
|
|
$Timing.ocrP90Ms,
|
|
$Timing.averageCardReadyMs,
|
|
$Timing.averageScrollReadyMs,
|
|
$Timing.artifactsPerMinute,
|
|
$Timing.activeArtifactsPerMinute,
|
|
$Timing.projectedMsFor100,
|
|
$Timing.activeProjectedMsFor100)
|
|
}
|
|
|
|
function Get-TimingBottleneck([object]$Timing) {
|
|
$parts = @(
|
|
[pscustomobject]@{ name = "ocr"; value = [int]$Timing.averageOcrMs },
|
|
[pscustomobject]@{ name = "capture-roundtrip-overhead"; value = [int]$Timing.averageCaptureRoundTripOverheadMs },
|
|
[pscustomobject]@{ name = "capture"; value = [int]$Timing.averageCaptureMs },
|
|
[pscustomobject]@{ name = "card-ready"; value = [int]$Timing.averageCardReadyMs },
|
|
[pscustomobject]@{ name = "scroll-ready"; value = [int]$Timing.averageScrollReadyMs }
|
|
) | Sort-Object -Property value -Descending
|
|
|
|
if ($parts.Count -eq 0 -or $parts[0].value -le 0) { return "unknown" }
|
|
return $parts[0].name
|
|
}
|
|
|
|
function Get-TimingRecommendation([object]$Timing) {
|
|
$bottleneck = Get-TimingBottleneck $Timing
|
|
switch ($bottleneck) {
|
|
"ocr" { return "OCR dominates; compare engine, crop count, worker pool, and parser-derived fields first." }
|
|
"capture-roundtrip-overhead" { return "Capture transport overhead dominates; inspect native encode, IPC payload size, and Base64/DataURL conversion before OCR changes." }
|
|
"capture" { return "Capture dominates; reduce payloads/crops and avoid full-frame or Base64 work in the hot loop." }
|
|
"card-ready" { return "Card-ready dominates; tune detail fingerprint polling against IK's 200ms item wait." }
|
|
"scroll-ready" { return "Scroll-ready dominates; tune page fingerprint polling against IK's 100ms fast-scroll wait." }
|
|
default { return "No dominant timing component detected; inspect misses/review/duplicates and raw diagnostic events." }
|
|
}
|
|
}
|
|
|
|
function Get-ScanQuality([object]$Timing) {
|
|
$parsed = [int]$Timing.parsed
|
|
$misses = [int]$Timing.misses
|
|
$review = [int]$Timing.review
|
|
$target = [int]$Timing.limit
|
|
$processed = [Math]::Max(1, $parsed + $misses)
|
|
$parseCoverage = if ($target -gt 0) { [Math]::Round($parsed / $target, 4) } else { 0 }
|
|
$missRate = [Math]::Round($misses / $processed, 4)
|
|
$reviewRate = if ($parsed -gt 0) { [Math]::Round($review / $parsed, 4) } else { 1 }
|
|
$statusOk = [string]$Timing.status -eq "done"
|
|
$coverageOk = $parsed -ge $target
|
|
$missOk = $missRate -le 0.02
|
|
$reviewOk = $reviewRate -le 0.15
|
|
$qualityPenalty = [Math]::Round(($missRate * 1000000) + ($reviewRate * 250000) + ((1 - $parseCoverage) * 1000000), 0)
|
|
$qualityDecision = if (-not $statusOk) {
|
|
"not-qualified: scan did not finish cleanly"
|
|
} elseif (-not $coverageOk) {
|
|
"not-qualified: parsed fewer artifacts than requested"
|
|
} elseif (-not $missOk) {
|
|
"not-qualified: miss rate above 2%"
|
|
} elseif (-not $reviewOk) {
|
|
"not-qualified: review rate above 15%"
|
|
} else {
|
|
"qualified"
|
|
}
|
|
|
|
return [pscustomobject]@{
|
|
parseCoverage = $parseCoverage
|
|
missRate = $missRate
|
|
reviewRate = $reviewRate
|
|
qualityDecision = $qualityDecision
|
|
qualityPenalty = $qualityPenalty
|
|
qualified = ($statusOk -and $coverageOk -and $missOk -and $reviewOk)
|
|
}
|
|
}
|
|
|
|
function New-PerformanceAssessment([object[]]$Summaries) {
|
|
$limitReports = @()
|
|
foreach ($group in ($Summaries | Group-Object -Property limit | Sort-Object { [int]$_.Name })) {
|
|
$entries = @($group.Group | ForEach-Object {
|
|
$quality = Get-ScanQuality $_
|
|
$_ | Add-Member -NotePropertyName parseCoverage -NotePropertyValue $quality.parseCoverage -Force
|
|
$_ | Add-Member -NotePropertyName missRate -NotePropertyValue $quality.missRate -Force
|
|
$_ | Add-Member -NotePropertyName reviewRate -NotePropertyValue $quality.reviewRate -Force
|
|
$_ | Add-Member -NotePropertyName qualityDecision -NotePropertyValue $quality.qualityDecision -Force
|
|
$_ | Add-Member -NotePropertyName qualityPenalty -NotePropertyValue $quality.qualityPenalty -Force
|
|
$_ | Add-Member -NotePropertyName qualified -NotePropertyValue $quality.qualified -Force
|
|
$_
|
|
} | Sort-Object -Property @{ Expression = "qualified"; Descending = $true }, qualityPenalty, activeAverageMsPerParsed, averageMsPerParsed)
|
|
if ($entries.Count -eq 0) { continue }
|
|
$winner = $entries[0]
|
|
$engineReports = @()
|
|
foreach ($entry in $entries) {
|
|
$engineReports += [pscustomobject]@{
|
|
engine = $entry.engine
|
|
status = $entry.status
|
|
parsed = $entry.parsed
|
|
review = $entry.review
|
|
misses = $entry.misses
|
|
parseCoverage = $entry.parseCoverage
|
|
missRate = $entry.missRate
|
|
reviewRate = $entry.reviewRate
|
|
qualified = $entry.qualified
|
|
qualityDecision = $entry.qualityDecision
|
|
qualityPenalty = $entry.qualityPenalty
|
|
activeAverageMsPerParsed = $entry.activeAverageMsPerParsed
|
|
activeProjectedMsFor100 = $entry.activeProjectedMsFor100
|
|
averageOcrMs = $entry.averageOcrMs
|
|
averageCaptureMs = $entry.averageCaptureMs
|
|
averageCaptureRoundTripMs = $entry.averageCaptureRoundTripMs
|
|
averageCaptureRoundTripOverheadMs = $entry.averageCaptureRoundTripOverheadMs
|
|
averageCardReadyMs = $entry.averageCardReadyMs
|
|
averageScrollReadyMs = $entry.averageScrollReadyMs
|
|
bottleneck = Get-TimingBottleneck $entry
|
|
recommendation = Get-TimingRecommendation $entry
|
|
}
|
|
}
|
|
|
|
$limitReports += [pscustomobject]@{
|
|
limit = [int]$group.Name
|
|
engineCount = $entries.Count
|
|
enginesCompared = @($entries | ForEach-Object { $_.engine })
|
|
comparisonComplete = (@($entries | Where-Object { $_.engine -eq "current" }).Count -gt 0 -and @($entries | Where-Object { $_.engine -eq "ik-traineddata" }).Count -gt 0)
|
|
winnerEngine = $winner.engine
|
|
winnerQualified = $winner.qualified
|
|
winnerMissRate = $winner.missRate
|
|
winnerReviewRate = $winner.reviewRate
|
|
winnerActiveAverageMsPerParsed = $winner.activeAverageMsPerParsed
|
|
winnerActiveProjectedMsFor100 = $winner.activeProjectedMsFor100
|
|
winnerAverageCaptureRoundTripMs = $winner.averageCaptureRoundTripMs
|
|
winnerAverageCaptureRoundTripOverheadMs = $winner.averageCaptureRoundTripOverheadMs
|
|
engines = $engineReports
|
|
}
|
|
}
|
|
|
|
$goal100 = @($limitReports | Where-Object { $_.limit -eq 100 } | Select-Object -First 1)
|
|
$goal100Decision = "not-run: missing 100-artifact assessment"
|
|
if ($goal100.Count -gt 0) {
|
|
if (-not $goal100[0].comparisonComplete) {
|
|
$goal100Decision = "not-comparable: current and ik-traineddata were not both run"
|
|
} elseif (-not $goal100[0].winnerQualified) {
|
|
$goal100Decision = "not-qualified: 100-artifact winner failed quality gates"
|
|
} else {
|
|
$goal100Decision = "qualified-comparison: winner=$($goal100[0].winnerEngine)"
|
|
}
|
|
}
|
|
return [pscustomobject]@{
|
|
createdAt = (Get-Date).ToString("o")
|
|
goalLimit = 100
|
|
goalEngines = @("current", "ik-traineddata")
|
|
goal100Decision = $goal100Decision
|
|
goal100 = if ($goal100.Count -gt 0) { $goal100[0] } else { $null }
|
|
limits = $limitReports
|
|
}
|
|
}
|
|
|
|
function Write-PerformanceAssessment([object]$Assessment) {
|
|
if ($Assessment.goal100Decision) {
|
|
Write-Host "assessment goal100: $($Assessment.goal100Decision)"
|
|
}
|
|
foreach ($limit in @($Assessment.limits)) {
|
|
Write-Host ("assessment limit={0}: winner={1} qualified={2} completeCompare={3} engines={4} missRate={5:P1} reviewRate={6:P1} activeAvg={7}ms projected100={8}ms roundtripOverheadAvg={9}ms" -f `
|
|
$limit.limit,
|
|
$limit.winnerEngine,
|
|
$limit.winnerQualified,
|
|
$limit.comparisonComplete,
|
|
($limit.enginesCompared -join ","),
|
|
$limit.winnerMissRate,
|
|
$limit.winnerReviewRate,
|
|
$limit.winnerActiveAverageMsPerParsed,
|
|
$limit.winnerActiveProjectedMsFor100,
|
|
$limit.winnerAverageCaptureRoundTripOverheadMs)
|
|
foreach ($engine in @($limit.engines)) {
|
|
Write-Host (" {0}: qualified={1}, missRate={2:P1}, reviewRate={3:P1}, bottleneck={4}, activeAvg={5}ms, ocr={6}ms, capture={7}ms, roundtrip={8}ms, roundtripOverhead={9}ms, cardReady={10}ms, scrollReady={11}ms, decision={12}" -f `
|
|
$engine.engine,
|
|
$engine.qualified,
|
|
$engine.missRate,
|
|
$engine.reviewRate,
|
|
$engine.bottleneck,
|
|
$engine.activeAverageMsPerParsed,
|
|
$engine.averageOcrMs,
|
|
$engine.averageCaptureMs,
|
|
$engine.averageCaptureRoundTripMs,
|
|
$engine.averageCaptureRoundTripOverheadMs,
|
|
$engine.averageCardReadyMs,
|
|
$engine.averageScrollReadyMs,
|
|
$engine.qualityDecision)
|
|
}
|
|
}
|
|
}
|
|
|
|
function Invoke-AssessmentSelfTest {
|
|
$synthetic = @(
|
|
[pscustomobject]@{
|
|
engine = "current"
|
|
limit = 100
|
|
status = "done"
|
|
parsed = 100
|
|
review = 22
|
|
misses = 0
|
|
activeAverageMsPerParsed = 700
|
|
averageMsPerParsed = 760
|
|
activeProjectedMsFor100 = 70000
|
|
averageOcrMs = 260
|
|
averageCaptureMs = 160
|
|
averageCaptureRoundTripMs = 340
|
|
averageCaptureRoundTripOverheadMs = 180
|
|
averageCardReadyMs = 190
|
|
averageScrollReadyMs = 60
|
|
},
|
|
[pscustomobject]@{
|
|
engine = "ik-traineddata"
|
|
limit = 100
|
|
status = "done"
|
|
parsed = 100
|
|
review = 4
|
|
misses = 0
|
|
activeAverageMsPerParsed = 820
|
|
averageMsPerParsed = 870
|
|
activeProjectedMsFor100 = 82000
|
|
averageOcrMs = 210
|
|
averageCaptureMs = 170
|
|
averageCaptureRoundTripMs = 300
|
|
averageCaptureRoundTripOverheadMs = 130
|
|
averageCardReadyMs = 205
|
|
averageScrollReadyMs = 80
|
|
},
|
|
[pscustomobject]@{
|
|
engine = "current"
|
|
limit = 20
|
|
status = "done"
|
|
parsed = 20
|
|
review = 1
|
|
misses = 0
|
|
activeAverageMsPerParsed = 390
|
|
averageMsPerParsed = 405
|
|
activeProjectedMsFor100 = 39000
|
|
averageOcrMs = 150
|
|
averageCaptureMs = 130
|
|
averageCaptureRoundTripMs = 280
|
|
averageCaptureRoundTripOverheadMs = 150
|
|
averageCardReadyMs = 80
|
|
averageScrollReadyMs = 0
|
|
},
|
|
[pscustomobject]@{
|
|
engine = "ik-traineddata"
|
|
limit = 20
|
|
status = "done"
|
|
parsed = 20
|
|
review = 2
|
|
misses = 0
|
|
activeAverageMsPerParsed = 460
|
|
averageMsPerParsed = 475
|
|
activeProjectedMsFor100 = 46000
|
|
averageOcrMs = 190
|
|
averageCaptureMs = 130
|
|
averageCaptureRoundTripMs = 310
|
|
averageCaptureRoundTripOverheadMs = 180
|
|
averageCardReadyMs = 80
|
|
averageScrollReadyMs = 0
|
|
},
|
|
[pscustomobject]@{
|
|
engine = "broken-fast"
|
|
limit = 45
|
|
status = "done"
|
|
parsed = 45
|
|
review = 0
|
|
misses = 3
|
|
activeAverageMsPerParsed = 300
|
|
averageMsPerParsed = 330
|
|
activeProjectedMsFor100 = 30000
|
|
averageOcrMs = 100
|
|
averageCaptureMs = 90
|
|
averageCaptureRoundTripMs = 190
|
|
averageCaptureRoundTripOverheadMs = 100
|
|
averageCardReadyMs = 40
|
|
averageScrollReadyMs = 20
|
|
},
|
|
[pscustomobject]@{
|
|
engine = "current"
|
|
limit = 45
|
|
status = "done"
|
|
parsed = 45
|
|
review = 1
|
|
misses = 0
|
|
activeAverageMsPerParsed = 600
|
|
averageMsPerParsed = 650
|
|
activeProjectedMsFor100 = 60000
|
|
averageOcrMs = 230
|
|
averageCaptureMs = 130
|
|
averageCaptureRoundTripMs = 300
|
|
averageCaptureRoundTripOverheadMs = 170
|
|
averageCardReadyMs = 160
|
|
averageScrollReadyMs = 50
|
|
}
|
|
)
|
|
|
|
$assessment = New-PerformanceAssessment -Summaries $synthetic
|
|
$goal100 = $assessment.goal100
|
|
$limit20 = @($assessment.limits | Where-Object { $_.limit -eq 20 } | Select-Object -First 1)[0]
|
|
$limit45 = @($assessment.limits | Where-Object { $_.limit -eq 45 } | Select-Object -First 1)[0]
|
|
|
|
if ($goal100.winnerEngine -ne "ik-traineddata") {
|
|
throw "Assessment self-test failed: expected ik-traineddata to win limit=100, got '$($goal100.winnerEngine)'."
|
|
}
|
|
if (-not $goal100.winnerQualified) {
|
|
throw "Assessment self-test failed: expected limit=100 winner to be qualified."
|
|
}
|
|
if (-not $goal100.comparisonComplete) {
|
|
throw "Assessment self-test failed: expected limit=100 to be a complete current vs ik-traineddata comparison."
|
|
}
|
|
if ($assessment.goal100Decision -ne "qualified-comparison: winner=ik-traineddata") {
|
|
throw "Assessment self-test failed: unexpected goal100Decision '$($assessment.goal100Decision)'."
|
|
}
|
|
if ($limit20.winnerEngine -ne "current") {
|
|
throw "Assessment self-test failed: expected current to win limit=20, got '$($limit20.winnerEngine)'."
|
|
}
|
|
if (-not $limit20.comparisonComplete) {
|
|
throw "Assessment self-test failed: expected limit=20 to be a complete current vs ik-traineddata comparison."
|
|
}
|
|
if (-not $limit20.winnerQualified) {
|
|
throw "Assessment self-test failed: expected limit=20 winner to be qualified."
|
|
}
|
|
if ($limit45.winnerEngine -ne "current") {
|
|
throw "Assessment self-test failed: expected current to win limit=45, got '$($limit45.winnerEngine)'."
|
|
}
|
|
if (@($goal100.engines | Where-Object { $_.engine -eq "current" })[0].qualityDecision -ne "not-qualified: review rate above 15%") {
|
|
throw "Assessment self-test failed: expected high-review current run to be rejected."
|
|
}
|
|
if (@($limit45.engines | Where-Object { $_.engine -eq "broken-fast" })[0].qualityDecision -ne "not-qualified: miss rate above 2%") {
|
|
throw "Assessment self-test failed: expected broken-fast run to be rejected for miss rate."
|
|
}
|
|
|
|
$singleEngineAssessment = New-PerformanceAssessment -Summaries @(
|
|
[pscustomobject]@{
|
|
engine = "current"
|
|
limit = 100
|
|
status = "done"
|
|
parsed = 100
|
|
review = 0
|
|
misses = 0
|
|
activeAverageMsPerParsed = 500
|
|
averageMsPerParsed = 520
|
|
activeProjectedMsFor100 = 50000
|
|
averageOcrMs = 180
|
|
averageCaptureMs = 120
|
|
averageCaptureRoundTripMs = 260
|
|
averageCaptureRoundTripOverheadMs = 140
|
|
averageCardReadyMs = 100
|
|
averageScrollReadyMs = 50
|
|
}
|
|
)
|
|
if ($singleEngineAssessment.goal100Decision -ne "not-comparable: current and ik-traineddata were not both run") {
|
|
throw "Assessment self-test failed: expected single-engine 100 run to be not-comparable, got '$($singleEngineAssessment.goal100Decision)'."
|
|
}
|
|
|
|
Write-PerformanceAssessment $assessment
|
|
Write-Host "Assessment self-test passed." -ForegroundColor Green
|
|
return $assessment
|
|
}
|
|
|
|
function Invoke-OcrBenchmark {
|
|
param(
|
|
[int]$Limit,
|
|
[string]$Engine,
|
|
[string]$Profile
|
|
)
|
|
|
|
Write-Host "Warming current OCR workers..."
|
|
$warmCurrent = Invoke-DevJson "/scanner/ocr/warmup?engine=current"
|
|
Save-Json "benchmark-warmup-current" $warmCurrent | Out-Null
|
|
|
|
if ($Engine -eq "compare" -or $Engine -eq "ik-traineddata") {
|
|
Write-Host "Warming IK-traineddata OCR workers..."
|
|
$warmIk = Invoke-DevJson "/scanner/ocr/warmup?engine=ik-traineddata"
|
|
Save-Json "benchmark-warmup-ik-traineddata" $warmIk | Out-Null
|
|
}
|
|
|
|
Write-Host "Running OCR benchmark engine=$Engine profile=$Profile limit=$Limit"
|
|
$benchmark = Invoke-DevJson "/scanner/benchmark-ocr?limit=$Limit&engine=$Engine&profile=$Profile"
|
|
Save-Json "benchmark-ocr-$Engine-$Profile-limit-$Limit" $benchmark | Out-Null
|
|
|
|
if ($benchmark.summary.mode -eq "compare") {
|
|
foreach ($engineSummary in @($benchmark.summary.engines)) {
|
|
Write-Host ("benchmark {0}: avg={1}ms ocrAvg={2}ms p50={3}ms p90={4}ms projected100={5}ms skipped={6} pool={7}" -f `
|
|
$engineSummary.engine,
|
|
$engineSummary.averageMs,
|
|
$engineSummary.averageOcrMs,
|
|
$engineSummary.p50Ms,
|
|
$engineSummary.p90Ms,
|
|
$engineSummary.projectedMs.artifacts100,
|
|
$engineSummary.skippedOcrCaptures,
|
|
$engineSummary.workerPoolSize)
|
|
}
|
|
} else {
|
|
$summary = $benchmark.summary
|
|
Write-Host ("benchmark {0}: avg={1}ms ocrAvg={2}ms p50={3}ms p90={4}ms projected100={5}ms skipped={6} pool={7}" -f `
|
|
$summary.engine,
|
|
$summary.averageMs,
|
|
$summary.averageOcrMs,
|
|
$summary.p50Ms,
|
|
$summary.p90Ms,
|
|
$summary.projectedMs.artifacts100,
|
|
$summary.skippedOcrCaptures,
|
|
$summary.workerPoolSize)
|
|
}
|
|
|
|
return $benchmark
|
|
}
|
|
|
|
function Convert-ReviewSamplesSummary([object]$ReviewPayload) {
|
|
$samples = @()
|
|
foreach ($record in @($ReviewPayload.samples)) {
|
|
$parsed = $record.sample.parsed
|
|
$capture = $record.sample.capture
|
|
$ocr = @()
|
|
foreach ($entry in @($capture.ocr)) {
|
|
$ocr += [pscustomobject]@{
|
|
id = $entry.id
|
|
label = $entry.label
|
|
text = $entry.text
|
|
confidence = $entry.confidence
|
|
}
|
|
}
|
|
$samples += [pscustomobject]@{
|
|
savedAt = $record.savedAt
|
|
reason = $record.sample.reason
|
|
parsed = [pscustomobject]@{
|
|
name = $parsed.name
|
|
slot = $parsed.slot
|
|
level = $parsed.level
|
|
mainStat = $parsed.mainStat
|
|
mainValue = $parsed.mainValue
|
|
setName = $parsed.setName
|
|
equipped = $parsed.equipped
|
|
confidence = $parsed.confidence
|
|
notes = $parsed.notes
|
|
fields = $parsed.fields
|
|
}
|
|
capture = [pscustomobject]@{
|
|
name = $capture.name
|
|
width = $capture.width
|
|
height = $capture.height
|
|
captureTarget = $capture.captureTarget
|
|
capturedAt = $capture.capturedAt
|
|
inventoryGrid = if ($capture.inventoryGrid) {
|
|
[pscustomobject]@{
|
|
rows = $capture.inventoryGrid.rows
|
|
cols = $capture.inventoryGrid.cols
|
|
confidence = $capture.inventoryGrid.confidence
|
|
source = $capture.inventoryGrid.source
|
|
}
|
|
} else { $null }
|
|
inventoryCount = $capture.inventoryCount
|
|
locked = $capture.locked
|
|
ocr = $ocr
|
|
}
|
|
}
|
|
}
|
|
|
|
return [pscustomobject]@{
|
|
ok = $ReviewPayload.ok
|
|
total = $ReviewPayload.total
|
|
path = $ReviewPayload.path
|
|
samples = $samples
|
|
}
|
|
}
|
|
|
|
function Wait-ForScannerIdle([int]$Limit, [string]$Engine) {
|
|
$startedAt = Get-Date
|
|
$pollIndex = 0
|
|
$lastStatus = $null
|
|
$observedMatchingRun = $false
|
|
|
|
while ($true) {
|
|
Start-Sleep -Seconds $PollIntervalSeconds
|
|
$pollIndex += 1
|
|
$statusPayload = Get-ScannerStatus
|
|
$lastStatus = $statusPayload
|
|
Save-Json "scan-$Engine-limit-$Limit-poll-$pollIndex" $statusPayload | Out-Null
|
|
|
|
$running = [bool]$statusPayload.status.running
|
|
$summaryTarget = if ($statusPayload.status.summary) { [int]$statusPayload.status.summary.targetCount } else { 0 }
|
|
$scanStart = @($statusPayload.status.diagnosticEvents | Where-Object { $_.phase -eq "scan-start" } | Select-Object -Last 1)
|
|
$scanStartDetails = if ($scanStart.Count -gt 0) { $scanStart[0].details } else { $null }
|
|
$scanStartLimit = if ($scanStartDetails -and $scanStartDetails.scanLimit) { [int]$scanStartDetails.scanLimit } else { 0 }
|
|
$scanStartEngine = if ($scanStartDetails -and $scanStartDetails.ocrEngine) { [string]$scanStartDetails.ocrEngine } else { "" }
|
|
$matchesScanStart = $scanStartLimit -eq $Limit -and ($scanStartEngine -eq "" -or $scanStartEngine -eq $Engine)
|
|
$matchesFinalSummary = $summaryTarget -eq $Limit
|
|
if ($running -and ($matchesScanStart -or $matchesFinalSummary)) {
|
|
$observedMatchingRun = $true
|
|
}
|
|
if (-not $running) {
|
|
if ($observedMatchingRun -or $matchesScanStart -or $matchesFinalSummary) {
|
|
return $statusPayload
|
|
}
|
|
Write-Host "Waiting for scanner run limit=$Limit engine=$Engine to appear; ignoring unrelated idle status." -ForegroundColor DarkGray
|
|
}
|
|
|
|
$elapsed = ((Get-Date) - $startedAt).TotalSeconds
|
|
if ($elapsed -gt $TimeoutSeconds) {
|
|
$stop = Invoke-DevJson "/scanner/stop"
|
|
Save-Json "scan-$Engine-limit-$Limit-timeout-stop" $stop | Out-Null
|
|
throw "Scanner timed out after $TimeoutSeconds seconds for limit=$Limit engine=$Engine. Stop command was sent."
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($SelfTestAssessment) {
|
|
Invoke-AssessmentSelfTest | Out-Null
|
|
exit 0
|
|
}
|
|
|
|
$stamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss"
|
|
$RunDir = Join-Path $OutputRoot $stamp
|
|
New-Item -ItemType Directory -Force -Path $RunDir | Out-Null
|
|
|
|
if ($GoalRun) {
|
|
$Limits = @(2, 5, 20, 45, 100)
|
|
$BenchmarkOcr = $true
|
|
}
|
|
if ($RepeatabilityRun) {
|
|
$Limits = @(20, 45, 100)
|
|
}
|
|
|
|
$Limits = @($Limits | ForEach-Object {
|
|
$limit = [int]$_
|
|
if ($limit -lt 1 -or $limit -gt 1800) {
|
|
throw "Refusing unsafe scan limit '$limit'. Expected a bounded value from 1 to 1800."
|
|
}
|
|
$limit
|
|
})
|
|
|
|
$ScanEngines = if ($ScanEngine -eq "compare") { @("current", "ik-traineddata") } else { @($ScanEngine) }
|
|
$RunSummaries = @()
|
|
|
|
$transcriptPath = Join-Path $RunDir "transcript.log"
|
|
try {
|
|
Start-Transcript -Path $transcriptPath -Append | Out-Null
|
|
} catch {
|
|
Write-Warning "Could not start transcript: $($_.Exception.Message)"
|
|
}
|
|
|
|
try {
|
|
Write-Host "Live soak output: $RunDir"
|
|
Write-Host "Checking dev control server at $BaseUrl"
|
|
|
|
$health = Invoke-DevJson "/health"
|
|
Save-Json "00-health" $health | Out-Null
|
|
Assert-CurrentAppBuild $health
|
|
|
|
if ($health.appBuild) {
|
|
Write-Host "App build: signature=$($health.appBuild.signature), pid=$($health.appBuild.pid), startedAt=$($health.appBuild.startedAt), ocrWorkers=$($health.appBuild.expectedOcrWorkerPoolSize)"
|
|
if ($health.appBuild.expectedOcrWorkerPoolSize -lt 4) {
|
|
Write-Host "WARNUNG: OCR worker pool is below 4. This is valid for constrained debugging, but not ideal for IK-speed comparison." -ForegroundColor Yellow
|
|
}
|
|
}
|
|
|
|
$initialStatus = Get-ScannerStatus
|
|
Save-Json "01-status-before" $initialStatus | Out-Null
|
|
|
|
if ($initialStatus.status.runtimeInfo) {
|
|
$runtime = $initialStatus.status.runtimeInfo
|
|
Write-Host "Runtime: elevated=$($runtime.isElevated), genshinFound=$($runtime.genshinFound), target=$($runtime.targetProcess)"
|
|
} else {
|
|
Write-Host "Runtime info is not published yet. Open the app scanner view before running broad scans." -ForegroundColor Yellow
|
|
}
|
|
|
|
if ($BenchmarkOcr) {
|
|
Invoke-OcrBenchmark -Limit $BenchmarkLimit -Engine $BenchmarkEngine -Profile $BenchmarkProfile | Out-Null
|
|
}
|
|
|
|
if (-not $SkipSmartCapture) {
|
|
Write-Host "Capturing smart preflight snapshot without OCR..."
|
|
$capture = Invoke-DevJson "/capture/smart?skipOcr=1"
|
|
Save-Json "02-smart-capture-skip-ocr" $capture | Out-Null
|
|
}
|
|
|
|
foreach ($index in $ProbeIndices) {
|
|
Write-Host "Running probe click index=$index"
|
|
$probe = Invoke-DevJson "/automation/probe-click?index=$index"
|
|
Save-Json "probe-index-$index" $probe | Out-Null
|
|
if (-not (Test-ProbeSucceeded $probe)) {
|
|
Write-Host "Probe index=$index did not fully pass. Review the saved JSON before broader scans." -ForegroundColor Yellow
|
|
if (-not $ContinueAfterBlocked) {
|
|
throw "Stopping after failed probe index=$index. Re-run with -ContinueAfterBlocked only if you deliberately want to continue."
|
|
}
|
|
} elseif (-not $probe.ok -and $probe.changed) {
|
|
Write-Host "Probe index=$index changed the detail panel even though helper cursor/click readback was not clean; continuing." -ForegroundColor Yellow
|
|
} elseif (-not $probe.ok -and $probe.click -and $probe.click.clicked -and $probe.click.moved -and -not $probe.click.inputBlocked) {
|
|
Write-Host "Probe index=$index delivered input but detail did not change; continuing because the target may already be selected." -ForegroundColor Yellow
|
|
}
|
|
}
|
|
|
|
foreach ($engine in $ScanEngines) {
|
|
foreach ($limit in $Limits) {
|
|
if ($limit -lt 1) { continue }
|
|
Write-Host "Starting bounded scanner run limit=$limit engine=$engine"
|
|
$start = Invoke-DevJson "/scanner/start?entry=visible-inventory&limit=$limit&engine=$engine"
|
|
Save-Json "scan-$engine-limit-$limit-start" $start | Out-Null
|
|
|
|
$finalStatus = Wait-ForScannerIdle -Limit $limit -Engine $engine
|
|
Save-Json "scan-$engine-limit-$limit-final" $finalStatus | Out-Null
|
|
|
|
$summary = Get-CompletedScanSummary $finalStatus $limit
|
|
Write-Host "engine=$engine limit=$limit summary: status=$($summary.status), attempted=$($summary.attempted), verified=$($summary.verified), parsed=$($summary.parsed), stored=$($summary.stored), review=$($summary.review), misses=$($summary.misses), pages=$($summary.pages)"
|
|
$timing = Convert-ScanTimingSummary $finalStatus $limit $engine
|
|
$RunSummaries += $timing
|
|
Write-ScanTimingLine $timing
|
|
|
|
if (($summary.status -eq "blocked" -or $summary.status -eq "stopped") -and -not $ContinueAfterBlocked) {
|
|
throw "Stopping after scan status '$($summary.status)' for limit=$limit engine=$engine."
|
|
}
|
|
}
|
|
}
|
|
|
|
$review = Invoke-DevJson "/review/samples?limit=30"
|
|
Save-Json "review-samples-tail-summary" (Convert-ReviewSamplesSummary $review) | Out-Null
|
|
if ($SaveFullReviewSamples) {
|
|
Save-Json "review-samples-tail-full" $review | Out-Null
|
|
}
|
|
|
|
$afterStatus = Get-ScannerStatus
|
|
Save-Json "99-status-after" $afterStatus | Out-Null
|
|
Save-Json "scan-run-summary" ([pscustomobject]@{
|
|
createdAt = (Get-Date).ToString("o")
|
|
goalRun = [bool]$GoalRun
|
|
scanEngine = $ScanEngine
|
|
scanEngines = $ScanEngines
|
|
benchmarkOcr = [bool]$BenchmarkOcr
|
|
limits = $Limits
|
|
summaries = $RunSummaries
|
|
}) | Out-Null
|
|
|
|
if ($RunSummaries.Count -gt 0) {
|
|
$assessment = New-PerformanceAssessment -Summaries $RunSummaries
|
|
Save-Json "scan-performance-assessment" $assessment | Out-Null
|
|
Write-PerformanceAssessment $assessment
|
|
|
|
$csvPath = Join-Path $RunDir "scan-run-summary.csv"
|
|
$RunSummaries | Export-Csv -LiteralPath $csvPath -NoTypeInformation -Encoding UTF8
|
|
Write-Host "Timing summary CSV: $csvPath"
|
|
}
|
|
|
|
Write-Host "Live soak complete. Output: $RunDir" -ForegroundColor Green
|
|
} catch {
|
|
Write-Host "Live soak failed: $($_.Exception.Message)" -ForegroundColor Red
|
|
Write-Host "Output so far: $RunDir" -ForegroundColor Yellow
|
|
exit 1
|
|
} finally {
|
|
try {
|
|
Stop-Transcript | Out-Null
|
|
} catch {
|
|
}
|
|
}
|