Prepare scanner branch for merge
This commit is contained in:
AzuTear
+108
-3
@@ -77,6 +77,7 @@ function Get-ScannerStatus {
|
||||
function Test-ProbeSucceeded([object]$ProbePayload) {
|
||||
if ($ProbePayload.ok) { return $true }
|
||||
if ($ProbePayload.changed) { return $true }
|
||||
if ($ProbePayload.click -and $ProbePayload.click.clicked -and $ProbePayload.click.moved -and -not $ProbePayload.click.inputBlocked) { return $true }
|
||||
return $false
|
||||
}
|
||||
|
||||
@@ -282,6 +283,9 @@ function New-PerformanceAssessment([object[]]$Summaries) {
|
||||
|
||||
$limitReports += [pscustomobject]@{
|
||||
limit = [int]$group.Name
|
||||
engineCount = $entries.Count
|
||||
enginesCompared = @($entries | ForEach-Object { $_.engine })
|
||||
comparisonComplete = (@($entries | Where-Object { $_.engine -eq "current" }).Count -gt 0 -and @($entries | Where-Object { $_.engine -eq "ik-traineddata" }).Count -gt 0)
|
||||
winnerEngine = $winner.engine
|
||||
winnerQualified = $winner.qualified
|
||||
winnerMissRate = $winner.missRate
|
||||
@@ -293,19 +297,37 @@ function New-PerformanceAssessment([object[]]$Summaries) {
|
||||
}
|
||||
|
||||
$goal100 = @($limitReports | Where-Object { $_.limit -eq 100 } | Select-Object -First 1)
|
||||
$goal100Decision = "not-run: missing 100-artifact assessment"
|
||||
if ($goal100.Count -gt 0) {
|
||||
if (-not $goal100[0].comparisonComplete) {
|
||||
$goal100Decision = "not-comparable: current and ik-traineddata were not both run"
|
||||
} elseif (-not $goal100[0].winnerQualified) {
|
||||
$goal100Decision = "not-qualified: 100-artifact winner failed quality gates"
|
||||
} else {
|
||||
$goal100Decision = "qualified-comparison: winner=$($goal100[0].winnerEngine)"
|
||||
}
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
createdAt = (Get-Date).ToString("o")
|
||||
goalLimit = 100
|
||||
goalEngines = @("current", "ik-traineddata")
|
||||
goal100Decision = $goal100Decision
|
||||
goal100 = if ($goal100.Count -gt 0) { $goal100[0] } else { $null }
|
||||
limits = $limitReports
|
||||
}
|
||||
}
|
||||
|
||||
function Write-PerformanceAssessment([object]$Assessment) {
|
||||
if ($Assessment.goal100Decision) {
|
||||
Write-Host "assessment goal100: $($Assessment.goal100Decision)"
|
||||
}
|
||||
foreach ($limit in @($Assessment.limits)) {
|
||||
Write-Host ("assessment limit={0}: winner={1} qualified={2} missRate={3:P1} reviewRate={4:P1} activeAvg={5}ms projected100={6}ms" -f `
|
||||
Write-Host ("assessment limit={0}: winner={1} qualified={2} completeCompare={3} engines={4} missRate={5:P1} reviewRate={6:P1} activeAvg={7}ms projected100={8}ms" -f `
|
||||
$limit.limit,
|
||||
$limit.winnerEngine,
|
||||
$limit.winnerQualified,
|
||||
$limit.comparisonComplete,
|
||||
($limit.enginesCompared -join ","),
|
||||
$limit.winnerMissRate,
|
||||
$limit.winnerReviewRate,
|
||||
$limit.winnerActiveAverageMsPerParsed,
|
||||
@@ -359,6 +381,36 @@ function Invoke-AssessmentSelfTest {
|
||||
averageCardReadyMs = 205
|
||||
averageScrollReadyMs = 80
|
||||
},
|
||||
[pscustomobject]@{
|
||||
engine = "current"
|
||||
limit = 20
|
||||
status = "done"
|
||||
parsed = 20
|
||||
review = 1
|
||||
misses = 0
|
||||
activeAverageMsPerParsed = 390
|
||||
averageMsPerParsed = 405
|
||||
activeProjectedMsFor100 = 39000
|
||||
averageOcrMs = 150
|
||||
averageCaptureMs = 130
|
||||
averageCardReadyMs = 80
|
||||
averageScrollReadyMs = 0
|
||||
},
|
||||
[pscustomobject]@{
|
||||
engine = "ik-traineddata"
|
||||
limit = 20
|
||||
status = "done"
|
||||
parsed = 20
|
||||
review = 2
|
||||
misses = 0
|
||||
activeAverageMsPerParsed = 460
|
||||
averageMsPerParsed = 475
|
||||
activeProjectedMsFor100 = 46000
|
||||
averageOcrMs = 190
|
||||
averageCaptureMs = 130
|
||||
averageCardReadyMs = 80
|
||||
averageScrollReadyMs = 0
|
||||
},
|
||||
[pscustomobject]@{
|
||||
engine = "broken-fast"
|
||||
limit = 45
|
||||
@@ -393,6 +445,7 @@ function Invoke-AssessmentSelfTest {
|
||||
|
||||
$assessment = New-PerformanceAssessment -Summaries $synthetic
|
||||
$goal100 = $assessment.goal100
|
||||
$limit20 = @($assessment.limits | Where-Object { $_.limit -eq 20 } | Select-Object -First 1)[0]
|
||||
$limit45 = @($assessment.limits | Where-Object { $_.limit -eq 45 } | Select-Object -First 1)[0]
|
||||
|
||||
if ($goal100.winnerEngine -ne "ik-traineddata") {
|
||||
@@ -401,6 +454,21 @@ function Invoke-AssessmentSelfTest {
|
||||
if (-not $goal100.winnerQualified) {
|
||||
throw "Assessment self-test failed: expected limit=100 winner to be qualified."
|
||||
}
|
||||
if (-not $goal100.comparisonComplete) {
|
||||
throw "Assessment self-test failed: expected limit=100 to be a complete current vs ik-traineddata comparison."
|
||||
}
|
||||
if ($assessment.goal100Decision -ne "qualified-comparison: winner=ik-traineddata") {
|
||||
throw "Assessment self-test failed: unexpected goal100Decision '$($assessment.goal100Decision)'."
|
||||
}
|
||||
if ($limit20.winnerEngine -ne "current") {
|
||||
throw "Assessment self-test failed: expected current to win limit=20, got '$($limit20.winnerEngine)'."
|
||||
}
|
||||
if (-not $limit20.comparisonComplete) {
|
||||
throw "Assessment self-test failed: expected limit=20 to be a complete current vs ik-traineddata comparison."
|
||||
}
|
||||
if (-not $limit20.winnerQualified) {
|
||||
throw "Assessment self-test failed: expected limit=20 winner to be qualified."
|
||||
}
|
||||
if ($limit45.winnerEngine -ne "current") {
|
||||
throw "Assessment self-test failed: expected current to win limit=45, got '$($limit45.winnerEngine)'."
|
||||
}
|
||||
@@ -411,6 +479,27 @@ function Invoke-AssessmentSelfTest {
|
||||
throw "Assessment self-test failed: expected broken-fast run to be rejected for miss rate."
|
||||
}
|
||||
|
||||
$singleEngineAssessment = New-PerformanceAssessment -Summaries @(
|
||||
[pscustomobject]@{
|
||||
engine = "current"
|
||||
limit = 100
|
||||
status = "done"
|
||||
parsed = 100
|
||||
review = 0
|
||||
misses = 0
|
||||
activeAverageMsPerParsed = 500
|
||||
averageMsPerParsed = 520
|
||||
activeProjectedMsFor100 = 50000
|
||||
averageOcrMs = 180
|
||||
averageCaptureMs = 120
|
||||
averageCardReadyMs = 100
|
||||
averageScrollReadyMs = 50
|
||||
}
|
||||
)
|
||||
if ($singleEngineAssessment.goal100Decision -ne "not-comparable: current and ik-traineddata were not both run") {
|
||||
throw "Assessment self-test failed: expected single-engine 100 run to be not-comparable, got '$($singleEngineAssessment.goal100Decision)'."
|
||||
}
|
||||
|
||||
Write-PerformanceAssessment $assessment
|
||||
Write-Host "Assessment self-test passed." -ForegroundColor Green
|
||||
return $assessment
|
||||
@@ -527,6 +616,7 @@ function Wait-ForScannerIdle([int]$Limit, [string]$Engine) {
|
||||
$startedAt = Get-Date
|
||||
$pollIndex = 0
|
||||
$lastStatus = $null
|
||||
$observedMatchingRun = $false
|
||||
|
||||
while ($true) {
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
@@ -536,8 +626,21 @@ function Wait-ForScannerIdle([int]$Limit, [string]$Engine) {
|
||||
Save-Json "scan-$Engine-limit-$Limit-poll-$pollIndex" $statusPayload | Out-Null
|
||||
|
||||
$running = [bool]$statusPayload.status.running
|
||||
$summaryTarget = if ($statusPayload.status.summary) { [int]$statusPayload.status.summary.targetCount } else { 0 }
|
||||
$scanStart = @($statusPayload.status.diagnosticEvents | Where-Object { $_.phase -eq "scan-start" } | Select-Object -Last 1)
|
||||
$scanStartDetails = if ($scanStart.Count -gt 0) { $scanStart[0].details } else { $null }
|
||||
$scanStartLimit = if ($scanStartDetails -and $scanStartDetails.scanLimit) { [int]$scanStartDetails.scanLimit } else { 0 }
|
||||
$scanStartEngine = if ($scanStartDetails -and $scanStartDetails.ocrEngine) { [string]$scanStartDetails.ocrEngine } else { "" }
|
||||
$matchesScanStart = $scanStartLimit -eq $Limit -and ($scanStartEngine -eq "" -or $scanStartEngine -eq $Engine)
|
||||
$matchesFinalSummary = $summaryTarget -eq $Limit
|
||||
if ($running -and ($matchesScanStart -or $matchesFinalSummary)) {
|
||||
$observedMatchingRun = $true
|
||||
}
|
||||
if (-not $running) {
|
||||
return $statusPayload
|
||||
if ($observedMatchingRun -or $matchesScanStart -or $matchesFinalSummary) {
|
||||
return $statusPayload
|
||||
}
|
||||
Write-Host "Waiting for scanner run limit=$Limit engine=$Engine to appear; ignoring unrelated idle status." -ForegroundColor DarkGray
|
||||
}
|
||||
|
||||
$elapsed = ((Get-Date) - $startedAt).TotalSeconds
|
||||
@@ -619,6 +722,8 @@ try {
|
||||
}
|
||||
} elseif (-not $probe.ok -and $probe.changed) {
|
||||
Write-Host "Probe index=$index changed the detail panel even though helper cursor/click readback was not clean; continuing." -ForegroundColor Yellow
|
||||
} elseif (-not $probe.ok -and $probe.click -and $probe.click.clicked -and $probe.click.moved -and -not $probe.click.inputBlocked) {
|
||||
Write-Host "Probe index=$index delivered input but detail did not change; continuing because the target may already be selected." -ForegroundColor Yellow
|
||||
}
|
||||
}
|
||||
|
||||
@@ -626,7 +731,7 @@ try {
|
||||
foreach ($limit in $Limits) {
|
||||
if ($limit -lt 1) { continue }
|
||||
Write-Host "Starting bounded scanner run limit=$limit engine=$engine"
|
||||
$start = Invoke-DevJson "/scanner/start?limit=$limit&engine=$engine"
|
||||
$start = Invoke-DevJson "/scanner/start?entry=visible-inventory&limit=$limit&engine=$engine"
|
||||
Save-Json "scan-$engine-limit-$limit-start" $start | Out-Null
|
||||
|
||||
$finalStatus = Wait-ForScannerIdle -Limit $limit -Engine $engine
|
||||
|
||||
Reference in New Issue
Block a user