feat(scanner): add native artifact pipeline
Add native IK-style capture processing, Artifact Inventory, explicit promotion and single-result review. Confirm the three live OCR corrections in the eval corpus and preserve extraction/value separation.
This commit is contained in:
AzuTear
+57
-103
@@ -7,12 +7,12 @@ param(
|
||||
[string]$OutputRoot = (Join-Path (Resolve-Path -LiteralPath ".").Path "outputs\live-soak"),
|
||||
[switch]$GoalRun,
|
||||
[switch]$RepeatabilityRun,
|
||||
[ValidateSet("current", "ik-traineddata", "compare")]
|
||||
[ValidateSet("current")]
|
||||
[string]$ScanEngine = "current",
|
||||
[switch]$BenchmarkOcr,
|
||||
[int]$BenchmarkLimit = 5,
|
||||
[ValidateSet("current", "ik-traineddata", "compare")]
|
||||
[string]$BenchmarkEngine = "compare",
|
||||
[ValidateSet("current")]
|
||||
[string]$BenchmarkEngine = "current",
|
||||
[ValidateSet("fast", "full")]
|
||||
[string]$BenchmarkProfile = "fast",
|
||||
[switch]$SkipSmartCapture,
|
||||
@@ -205,11 +205,11 @@ function Get-TimingBottleneck([object]$Timing) {
|
||||
function Get-TimingRecommendation([object]$Timing) {
|
||||
$bottleneck = Get-TimingBottleneck $Timing
|
||||
switch ($bottleneck) {
|
||||
"ocr" { return "OCR dominates; compare engine, crop count, worker pool, and parser-derived fields first." }
|
||||
"ocr" { return "OCR dominates; inspect crop count, worker pool, and parser-derived fields first." }
|
||||
"capture-roundtrip-overhead" { return "Capture transport overhead dominates; inspect native encode, IPC payload size, and Base64/DataURL conversion before OCR changes." }
|
||||
"capture" { return "Capture dominates; reduce payloads/crops and avoid full-frame or Base64 work in the hot loop." }
|
||||
"card-ready" { return "Card-ready dominates; tune detail fingerprint polling against IK's 200ms item wait." }
|
||||
"scroll-ready" { return "Scroll-ready dominates; tune page fingerprint polling against IK's 100ms fast-scroll wait." }
|
||||
"card-ready" { return "Card-ready dominates; tune detail fingerprint polling before OCR changes." }
|
||||
"scroll-ready" { return "Scroll-ready dominates; tune page fingerprint polling before OCR changes." }
|
||||
default { return "No dominant timing component detected; inspect misses/review/duplicates and raw diagnostic events." }
|
||||
}
|
||||
}
|
||||
@@ -296,7 +296,7 @@ function New-PerformanceAssessment([object[]]$Summaries) {
|
||||
limit = [int]$group.Name
|
||||
engineCount = $entries.Count
|
||||
enginesCompared = @($entries | ForEach-Object { $_.engine })
|
||||
comparisonComplete = (@($entries | Where-Object { $_.engine -eq "current" }).Count -gt 0 -and @($entries | Where-Object { $_.engine -eq "ik-traineddata" }).Count -gt 0)
|
||||
singleEngine = $true
|
||||
winnerEngine = $winner.engine
|
||||
winnerQualified = $winner.qualified
|
||||
winnerMissRate = $winner.missRate
|
||||
@@ -312,18 +312,16 @@ function New-PerformanceAssessment([object[]]$Summaries) {
|
||||
$goal100 = @($limitReports | Where-Object { $_.limit -eq 100 } | Select-Object -First 1)
|
||||
$goal100Decision = "not-run: missing 100-artifact assessment"
|
||||
if ($goal100.Count -gt 0) {
|
||||
if (-not $goal100[0].comparisonComplete) {
|
||||
$goal100Decision = "not-comparable: current and ik-traineddata were not both run"
|
||||
} elseif (-not $goal100[0].winnerQualified) {
|
||||
if (-not $goal100[0].winnerQualified) {
|
||||
$goal100Decision = "not-qualified: 100-artifact winner failed quality gates"
|
||||
} else {
|
||||
$goal100Decision = "qualified-comparison: winner=$($goal100[0].winnerEngine)"
|
||||
$goal100Decision = "qualified: winner=$($goal100[0].winnerEngine)"
|
||||
}
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
createdAt = (Get-Date).ToString("o")
|
||||
goalLimit = 100
|
||||
goalEngines = @("current", "ik-traineddata")
|
||||
goalEngines = @("current")
|
||||
goal100Decision = $goal100Decision
|
||||
goal100 = if ($goal100.Count -gt 0) { $goal100[0] } else { $null }
|
||||
limits = $limitReports
|
||||
@@ -335,11 +333,10 @@ function Write-PerformanceAssessment([object]$Assessment) {
|
||||
Write-Host "assessment goal100: $($Assessment.goal100Decision)"
|
||||
}
|
||||
foreach ($limit in @($Assessment.limits)) {
|
||||
Write-Host ("assessment limit={0}: winner={1} qualified={2} completeCompare={3} engines={4} missRate={5:P1} reviewRate={6:P1} activeAvg={7}ms projected100={8}ms roundtripOverheadAvg={9}ms" -f `
|
||||
Write-Host ("assessment limit={0}: winner={1} qualified={2} engines={3} missRate={4:P1} reviewRate={5:P1} activeAvg={6}ms projected100={7}ms roundtripOverheadAvg={8}ms" -f `
|
||||
$limit.limit,
|
||||
$limit.winnerEngine,
|
||||
$limit.winnerQualified,
|
||||
$limit.comparisonComplete,
|
||||
($limit.enginesCompared -join ","),
|
||||
$limit.winnerMissRate,
|
||||
$limit.winnerReviewRate,
|
||||
@@ -372,34 +369,34 @@ function Invoke-AssessmentSelfTest {
|
||||
limit = 100
|
||||
status = "done"
|
||||
parsed = 100
|
||||
review = 22
|
||||
review = 0
|
||||
misses = 0
|
||||
activeAverageMsPerParsed = 700
|
||||
averageMsPerParsed = 760
|
||||
activeProjectedMsFor100 = 70000
|
||||
averageOcrMs = 260
|
||||
averageCaptureMs = 160
|
||||
averageCaptureRoundTripMs = 340
|
||||
averageCaptureRoundTripOverheadMs = 180
|
||||
averageCardReadyMs = 190
|
||||
averageScrollReadyMs = 60
|
||||
activeAverageMsPerParsed = 500
|
||||
averageMsPerParsed = 520
|
||||
activeProjectedMsFor100 = 50000
|
||||
averageOcrMs = 180
|
||||
averageCaptureMs = 120
|
||||
averageCaptureRoundTripMs = 260
|
||||
averageCaptureRoundTripOverheadMs = 140
|
||||
averageCardReadyMs = 100
|
||||
averageScrollReadyMs = 50
|
||||
},
|
||||
[pscustomobject]@{
|
||||
engine = "ik-traineddata"
|
||||
engine = "high-review"
|
||||
limit = 100
|
||||
status = "done"
|
||||
parsed = 100
|
||||
review = 4
|
||||
review = 22
|
||||
misses = 0
|
||||
activeAverageMsPerParsed = 820
|
||||
averageMsPerParsed = 870
|
||||
activeProjectedMsFor100 = 82000
|
||||
averageOcrMs = 210
|
||||
averageCaptureMs = 170
|
||||
averageCaptureRoundTripMs = 300
|
||||
averageCaptureRoundTripOverheadMs = 130
|
||||
averageCardReadyMs = 205
|
||||
averageScrollReadyMs = 80
|
||||
activeAverageMsPerParsed = 300
|
||||
averageMsPerParsed = 330
|
||||
activeProjectedMsFor100 = 30000
|
||||
averageOcrMs = 100
|
||||
averageCaptureMs = 90
|
||||
averageCaptureRoundTripMs = 190
|
||||
averageCaptureRoundTripOverheadMs = 100
|
||||
averageCardReadyMs = 40
|
||||
averageScrollReadyMs = 20
|
||||
},
|
||||
[pscustomobject]@{
|
||||
engine = "current"
|
||||
@@ -418,23 +415,6 @@ function Invoke-AssessmentSelfTest {
|
||||
averageCardReadyMs = 80
|
||||
averageScrollReadyMs = 0
|
||||
},
|
||||
[pscustomobject]@{
|
||||
engine = "ik-traineddata"
|
||||
limit = 20
|
||||
status = "done"
|
||||
parsed = 20
|
||||
review = 2
|
||||
misses = 0
|
||||
activeAverageMsPerParsed = 460
|
||||
averageMsPerParsed = 475
|
||||
activeProjectedMsFor100 = 46000
|
||||
averageOcrMs = 190
|
||||
averageCaptureMs = 130
|
||||
averageCaptureRoundTripMs = 310
|
||||
averageCaptureRoundTripOverheadMs = 180
|
||||
averageCardReadyMs = 80
|
||||
averageScrollReadyMs = 0
|
||||
},
|
||||
[pscustomobject]@{
|
||||
engine = "broken-fast"
|
||||
limit = 45
|
||||
@@ -476,43 +456,37 @@ function Invoke-AssessmentSelfTest {
|
||||
$limit20 = @($assessment.limits | Where-Object { $_.limit -eq 20 } | Select-Object -First 1)[0]
|
||||
$limit45 = @($assessment.limits | Where-Object { $_.limit -eq 45 } | Select-Object -First 1)[0]
|
||||
|
||||
if ($goal100.winnerEngine -ne "ik-traineddata") {
|
||||
throw "Assessment self-test failed: expected ik-traineddata to win limit=100, got '$($goal100.winnerEngine)'."
|
||||
if ($goal100.winnerEngine -ne "current") {
|
||||
throw "Assessment self-test failed: expected current to win limit=100, got '$($goal100.winnerEngine)'."
|
||||
}
|
||||
if (-not $goal100.winnerQualified) {
|
||||
throw "Assessment self-test failed: expected limit=100 winner to be qualified."
|
||||
}
|
||||
if (-not $goal100.comparisonComplete) {
|
||||
throw "Assessment self-test failed: expected limit=100 to be a complete current vs ik-traineddata comparison."
|
||||
}
|
||||
if ($assessment.goal100Decision -ne "qualified-comparison: winner=ik-traineddata") {
|
||||
if ($assessment.goal100Decision -ne "qualified: winner=current") {
|
||||
throw "Assessment self-test failed: unexpected goal100Decision '$($assessment.goal100Decision)'."
|
||||
}
|
||||
if ($limit20.winnerEngine -ne "current") {
|
||||
throw "Assessment self-test failed: expected current to win limit=20, got '$($limit20.winnerEngine)'."
|
||||
}
|
||||
if (-not $limit20.comparisonComplete) {
|
||||
throw "Assessment self-test failed: expected limit=20 to be a complete current vs ik-traineddata comparison."
|
||||
}
|
||||
if (-not $limit20.winnerQualified) {
|
||||
throw "Assessment self-test failed: expected limit=20 winner to be qualified."
|
||||
}
|
||||
if ($limit45.winnerEngine -ne "current") {
|
||||
throw "Assessment self-test failed: expected current to win limit=45, got '$($limit45.winnerEngine)'."
|
||||
}
|
||||
if (@($goal100.engines | Where-Object { $_.engine -eq "current" })[0].qualityDecision -ne "not-qualified: review rate above 15%") {
|
||||
throw "Assessment self-test failed: expected high-review current run to be rejected."
|
||||
if (@($goal100.engines | Where-Object { $_.engine -eq "high-review" })[0].qualityDecision -ne "not-qualified: review rate above 15%") {
|
||||
throw "Assessment self-test failed: expected high-review run to be rejected."
|
||||
}
|
||||
if (@($limit45.engines | Where-Object { $_.engine -eq "broken-fast" })[0].qualityDecision -ne "not-qualified: miss rate above 2%") {
|
||||
throw "Assessment self-test failed: expected broken-fast run to be rejected for miss rate."
|
||||
}
|
||||
|
||||
$singleEngineAssessment = New-PerformanceAssessment -Summaries @(
|
||||
$missingGoalAssessment = New-PerformanceAssessment -Summaries @(
|
||||
[pscustomobject]@{
|
||||
engine = "current"
|
||||
limit = 100
|
||||
limit = 20
|
||||
status = "done"
|
||||
parsed = 100
|
||||
parsed = 20
|
||||
review = 0
|
||||
misses = 0
|
||||
activeAverageMsPerParsed = 500
|
||||
@@ -526,8 +500,8 @@ function Invoke-AssessmentSelfTest {
|
||||
averageScrollReadyMs = 50
|
||||
}
|
||||
)
|
||||
if ($singleEngineAssessment.goal100Decision -ne "not-comparable: current and ik-traineddata were not both run") {
|
||||
throw "Assessment self-test failed: expected single-engine 100 run to be not-comparable, got '$($singleEngineAssessment.goal100Decision)'."
|
||||
if ($missingGoalAssessment.goal100Decision -ne "not-run: missing 100-artifact assessment") {
|
||||
throw "Assessment self-test failed: expected missing 100 run to be not-run, got '$($missingGoalAssessment.goal100Decision)'."
|
||||
}
|
||||
|
||||
Write-PerformanceAssessment $assessment
|
||||
@@ -543,43 +517,23 @@ function Invoke-OcrBenchmark {
|
||||
)
|
||||
|
||||
Write-Host "Warming current OCR workers..."
|
||||
$warmCurrent = Invoke-DevJson "/scanner/ocr/warmup?engine=current"
|
||||
$warmCurrent = Invoke-DevJson "/scanner/ocr/warmup"
|
||||
Save-Json "benchmark-warmup-current" $warmCurrent | Out-Null
|
||||
|
||||
if ($Engine -eq "compare" -or $Engine -eq "ik-traineddata") {
|
||||
Write-Host "Warming IK-traineddata OCR workers..."
|
||||
$warmIk = Invoke-DevJson "/scanner/ocr/warmup?engine=ik-traineddata"
|
||||
Save-Json "benchmark-warmup-ik-traineddata" $warmIk | Out-Null
|
||||
}
|
||||
|
||||
Write-Host "Running OCR benchmark engine=$Engine profile=$Profile limit=$Limit"
|
||||
$benchmark = Invoke-DevJson "/scanner/benchmark-ocr?limit=$Limit&engine=$Engine&profile=$Profile"
|
||||
$benchmark = Invoke-DevJson "/scanner/benchmark-ocr?limit=$Limit&profile=$Profile"
|
||||
Save-Json "benchmark-ocr-$Engine-$Profile-limit-$Limit" $benchmark | Out-Null
|
||||
|
||||
if ($benchmark.summary.mode -eq "compare") {
|
||||
foreach ($engineSummary in @($benchmark.summary.engines)) {
|
||||
Write-Host ("benchmark {0}: avg={1}ms ocrAvg={2}ms p50={3}ms p90={4}ms projected100={5}ms skipped={6} pool={7}" -f `
|
||||
$engineSummary.engine,
|
||||
$engineSummary.averageMs,
|
||||
$engineSummary.averageOcrMs,
|
||||
$engineSummary.p50Ms,
|
||||
$engineSummary.p90Ms,
|
||||
$engineSummary.projectedMs.artifacts100,
|
||||
$engineSummary.skippedOcrCaptures,
|
||||
$engineSummary.workerPoolSize)
|
||||
}
|
||||
} else {
|
||||
$summary = $benchmark.summary
|
||||
Write-Host ("benchmark {0}: avg={1}ms ocrAvg={2}ms p50={3}ms p90={4}ms projected100={5}ms skipped={6} pool={7}" -f `
|
||||
$summary.engine,
|
||||
$summary.averageMs,
|
||||
$summary.averageOcrMs,
|
||||
$summary.p50Ms,
|
||||
$summary.p90Ms,
|
||||
$summary.projectedMs.artifacts100,
|
||||
$summary.skippedOcrCaptures,
|
||||
$summary.workerPoolSize)
|
||||
}
|
||||
$summary = $benchmark.summary
|
||||
Write-Host ("benchmark {0}: avg={1}ms ocrAvg={2}ms p50={3}ms p90={4}ms projected100={5}ms skipped={6} pool={7}" -f `
|
||||
$summary.engine,
|
||||
$summary.averageMs,
|
||||
$summary.averageOcrMs,
|
||||
$summary.p50Ms,
|
||||
$summary.p90Ms,
|
||||
$summary.projectedMs.artifacts100,
|
||||
$summary.skippedOcrCaptures,
|
||||
$summary.workerPoolSize)
|
||||
|
||||
return $benchmark
|
||||
}
|
||||
@@ -707,7 +661,7 @@ $Limits = @($Limits | ForEach-Object {
|
||||
$limit
|
||||
})
|
||||
|
||||
$ScanEngines = if ($ScanEngine -eq "compare") { @("current", "ik-traineddata") } else { @($ScanEngine) }
|
||||
$ScanEngines = @($ScanEngine)
|
||||
$RunSummaries = @()
|
||||
|
||||
$transcriptPath = Join-Path $RunDir "transcript.log"
|
||||
@@ -728,7 +682,7 @@ try {
|
||||
if ($health.appBuild) {
|
||||
Write-Host "App build: signature=$($health.appBuild.signature), pid=$($health.appBuild.pid), startedAt=$($health.appBuild.startedAt), ocrWorkers=$($health.appBuild.expectedOcrWorkerPoolSize)"
|
||||
if ($health.appBuild.expectedOcrWorkerPoolSize -lt 4) {
|
||||
Write-Host "WARNUNG: OCR worker pool is below 4. This is valid for constrained debugging, but not ideal for IK-speed comparison." -ForegroundColor Yellow
|
||||
Write-Host "WARNUNG: OCR worker pool is below 4. This is valid for constrained debugging, but not ideal for scanner timing." -ForegroundColor Yellow
|
||||
}
|
||||
}
|
||||
|
||||
@@ -772,7 +726,7 @@ try {
|
||||
foreach ($limit in $Limits) {
|
||||
if ($limit -lt 1) { continue }
|
||||
Write-Host "Starting bounded scanner run limit=$limit engine=$engine"
|
||||
$start = Invoke-DevJson "/scanner/start?entry=visible-inventory&limit=$limit&engine=$engine"
|
||||
$start = Invoke-DevJson "/scanner/start?entry=visible-inventory&limit=$limit"
|
||||
Save-Json "scan-$engine-limit-$limit-start" $start | Out-Null
|
||||
|
||||
$finalStatus = Wait-ForScannerIdle -Limit $limit -Engine $engine
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
param(
|
||||
[string]$BaseUrl = "http://127.0.0.1:17317",
|
||||
[int]$Limit = 2,
|
||||
[ValidateSet("artifacts", "weapons", "characters", "materials")]
|
||||
[string]$Category = "artifacts",
|
||||
[int]$PollIntervalSeconds = 1,
|
||||
[int]$TimeoutSeconds = 180,
|
||||
[string]$OutputRoot = (Join-Path (Resolve-Path -LiteralPath ".").Path "outputs\native-live-smoke"),
|
||||
[switch]$SkipProbe,
|
||||
[switch]$Persist
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function ConvertTo-SafeFilePart([string]$Value) {
|
||||
$safe = $Value -replace "[^A-Za-z0-9._-]+", "-"
|
||||
$safe = $safe.Trim("-")
|
||||
if ($safe.Length -eq 0) { return "item" }
|
||||
if ($safe.Length -gt 80) { return $safe.Substring(0, 80) }
|
||||
return $safe
|
||||
}
|
||||
|
||||
function Invoke-DevJson([string]$Path) {
|
||||
$uri = if ($Path.StartsWith("http")) { $Path } else { "$BaseUrl$Path" }
|
||||
try {
|
||||
Invoke-RestMethod -Method Get -Uri $uri -TimeoutSec 90
|
||||
} catch {
|
||||
$response = $_.Exception.Response
|
||||
if ($response) {
|
||||
$stream = $response.GetResponseStream()
|
||||
if ($stream) {
|
||||
$reader = New-Object System.IO.StreamReader($stream)
|
||||
$body = $reader.ReadToEnd()
|
||||
if (-not [string]::IsNullOrWhiteSpace($body)) {
|
||||
try {
|
||||
return $body | ConvertFrom-Json
|
||||
} catch {
|
||||
throw "Dev endpoint $uri returned HTTP error with non-JSON body: $body"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
function Save-Json([string]$Name, [object]$Payload) {
|
||||
$path = Join-Path $RunDir "$(ConvertTo-SafeFilePart $Name).json"
|
||||
$Payload | ConvertTo-Json -Depth 40 | Set-Content -LiteralPath $path -Encoding UTF8
|
||||
return $path
|
||||
}
|
||||
|
||||
function Test-ProbeSucceeded([object]$ProbePayload) {
|
||||
if ($ProbePayload.ok) { return $true }
|
||||
if ($ProbePayload.changed) { return $true }
|
||||
if ($ProbePayload.click -and $ProbePayload.click.clicked -and $ProbePayload.click.moved -and -not $ProbePayload.click.inputBlocked) { return $true }
|
||||
return $false
|
||||
}
|
||||
|
||||
function Get-NativeScanner([object]$StatusPayload) {
|
||||
if ($StatusPayload.nativeScanner) { return $StatusPayload.nativeScanner }
|
||||
if ($StatusPayload.scanner) { return $StatusPayload.scanner }
|
||||
return $null
|
||||
}
|
||||
|
||||
function Get-ExpectedAppSignature() {
|
||||
$mainPath = Join-Path (Resolve-Path -LiteralPath ".").Path "electron\main.ts"
|
||||
$mainSource = Get-Content -LiteralPath $mainPath -Raw
|
||||
$match = [regex]::Match($mainSource, 'APP_RUNTIME_SIGNATURE\s*=\s*"([^"]+)"')
|
||||
if (-not $match.Success) {
|
||||
throw "APP_RUNTIME_SIGNATURE not found in electron/main.ts."
|
||||
}
|
||||
return $match.Groups[1].Value
|
||||
}
|
||||
|
||||
function UriEscape([string]$Value) {
|
||||
return [System.Uri]::EscapeDataString($Value)
|
||||
}
|
||||
|
||||
$safeLimit = [Math]::Max(1, [Math]::Min(100, $Limit))
|
||||
$timestamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$RunDir = Join-Path $OutputRoot $timestamp
|
||||
New-Item -ItemType Directory -Force -Path $RunDir | Out-Null
|
||||
|
||||
$summary = [ordered]@{
|
||||
ok = $false
|
||||
createdAt = (Get-Date).ToString("o")
|
||||
limit = $safeLimit
|
||||
category = $Category
|
||||
expectedSignature = ""
|
||||
outputDir = $RunDir
|
||||
persistRequested = [bool]$Persist
|
||||
health = $null
|
||||
data = $null
|
||||
preflight = $null
|
||||
probe = $null
|
||||
finalStatus = $null
|
||||
process = $null
|
||||
results = $null
|
||||
errors = @()
|
||||
}
|
||||
|
||||
try {
|
||||
$expectedSignature = Get-ExpectedAppSignature
|
||||
$summary.expectedSignature = $expectedSignature
|
||||
$health = Invoke-DevJson "/health"
|
||||
$summary.health = @{
|
||||
ok = [bool]$health.ok
|
||||
signature = if ($health.appBuild) { [string]$health.appBuild.signature } else { "" }
|
||||
}
|
||||
Save-Json "01-health" $health | Out-Null
|
||||
if (-not $health.appBuild -or [string]$health.appBuild.signature -ne $expectedSignature) {
|
||||
throw "Dev endpoint is stale: /health signature '$($summary.health.signature)' does not match source '$expectedSignature'. Restart the elevated app."
|
||||
}
|
||||
|
||||
$data = Invoke-DevJson "/scanner/native/data"
|
||||
$summary.data = @{
|
||||
ok = [bool]$data.ok
|
||||
version = [string]$data.status.version
|
||||
totalEntries = [int]$data.status.totalEntries
|
||||
valid = [bool]$data.status.valid
|
||||
}
|
||||
Save-Json "02-native-data" $data | Out-Null
|
||||
if (-not $data.ok) {
|
||||
throw "Native IK data check failed."
|
||||
}
|
||||
|
||||
$preflight = Invoke-DevJson "/scanner/native/preflight?category=$Category"
|
||||
$summary.preflight = @{
|
||||
ok = [bool]$preflight.ok
|
||||
ready = [bool]$preflight.status.ready
|
||||
category = [string]$preflight.status.category
|
||||
categoryReady = [bool]$preflight.status.categoryReady
|
||||
genshinFound = [bool]$preflight.status.genshinFound
|
||||
isSixteenNine = [bool]$preflight.status.isSixteenNine
|
||||
gridCount = [int]$preflight.status.grid.count
|
||||
visualReady = if ($preflight.status.visual) { [bool]$preflight.status.visual.ready } else { $null }
|
||||
visualWhitePct = if ($preflight.status.visual) { [double]$preflight.status.visual.whitePct } else { $null }
|
||||
visualDarkPct = if ($preflight.status.visual) { [double]$preflight.status.visual.darkPct } else { $null }
|
||||
visualColorPct = if ($preflight.status.visual) { [double]$preflight.status.visual.colorPct } else { $null }
|
||||
visualLumaStdDev = if ($preflight.status.visual) { [double]$preflight.status.visual.lumaStdDev } else { $null }
|
||||
blockReason = [string]$preflight.status.blockReason
|
||||
}
|
||||
Save-Json "03-native-preflight" $preflight | Out-Null
|
||||
if (-not $preflight.ok) {
|
||||
$preflightReason = if ($preflight.status.blockReason) { [string]$preflight.status.blockReason } else { "unknown preflight block reason" }
|
||||
throw "Native scanner preflight is not ready for category '$Category': $preflightReason"
|
||||
}
|
||||
|
||||
if (-not $SkipProbe) {
|
||||
$probe = Invoke-DevJson "/automation/probe-click?index=1"
|
||||
$summary.probe = @{
|
||||
ok = [bool](Test-ProbeSucceeded $probe)
|
||||
endpointOk = [bool]$probe.ok
|
||||
changed = [bool]$probe.changed
|
||||
}
|
||||
Save-Json "04-probe-click" $probe | Out-Null
|
||||
if (-not (Test-ProbeSucceeded $probe)) {
|
||||
throw "Probe click did not prove safe input delivery."
|
||||
}
|
||||
}
|
||||
|
||||
$start = Invoke-DevJson "/scanner/start?limit=$safeLimit&category=$Category"
|
||||
Save-Json "05-native-start" $start | Out-Null
|
||||
$startedScanner = Get-NativeScanner $start
|
||||
if ($null -eq $startedScanner) {
|
||||
throw "Native scanner start returned no scanner payload."
|
||||
}
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
$polls = @()
|
||||
do {
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
$status = Invoke-DevJson "/scanner/status"
|
||||
$scanner = Get-NativeScanner $status
|
||||
$polls += [pscustomobject]@{
|
||||
at = (Get-Date).ToString("o")
|
||||
status = if ($scanner) { [string]$scanner.status } else { "missing" }
|
||||
running = if ($scanner) { [bool]$scanner.running } else { $false }
|
||||
captured = if ($scanner) { [int]$scanner.captured } else { 0 }
|
||||
activeMs = if ($scanner) { [int]$scanner.activeMs } else { 0 }
|
||||
message = if ($scanner) { [string]$scanner.message } else { "missing native scanner payload" }
|
||||
}
|
||||
if ($null -ne $scanner -and -not $scanner.running) {
|
||||
break
|
||||
}
|
||||
} while ((Get-Date) -lt $deadline)
|
||||
|
||||
Save-Json "06-native-polls" $polls | Out-Null
|
||||
$final = Invoke-DevJson "/scanner/status"
|
||||
Save-Json "07-native-final-status" $final | Out-Null
|
||||
$finalScanner = Get-NativeScanner $final
|
||||
if ($null -eq $finalScanner) {
|
||||
throw "Native scanner final status returned no scanner payload."
|
||||
}
|
||||
|
||||
$summary.finalStatus = @{
|
||||
status = [string]$finalScanner.status
|
||||
running = [bool]$finalScanner.running
|
||||
runDir = [string]$finalScanner.runDir
|
||||
captured = [int]$finalScanner.captured
|
||||
clicked = [int]$finalScanner.clicked
|
||||
pages = [int]$finalScanner.pages
|
||||
activeMs = [int]$finalScanner.activeMs
|
||||
message = [string]$finalScanner.message
|
||||
}
|
||||
|
||||
if ($finalScanner.running) {
|
||||
throw "Native scanner did not finish before timeout."
|
||||
}
|
||||
if ([string]$finalScanner.status -ne "done") {
|
||||
throw "Native scanner finished with status '$($finalScanner.status)': $($finalScanner.message)"
|
||||
}
|
||||
if ([int]$finalScanner.captured -lt $safeLimit) {
|
||||
throw "Native scanner captured $($finalScanner.captured), expected at least $safeLimit."
|
||||
}
|
||||
|
||||
$runDirParam = UriEscape([string]$finalScanner.runDir)
|
||||
$persistParam = if ($Persist) { "1" } else { "0" }
|
||||
$process = Invoke-DevJson "/scanner/native/process?runDir=$runDirParam&limit=$safeLimit&persist=$persistParam"
|
||||
Save-Json "08-native-process" $process | Out-Null
|
||||
$summary.process = @{
|
||||
ok = [bool]$process.ok
|
||||
processed = [int]$process.status.processed
|
||||
parsed = [int]$process.status.parsed
|
||||
review = [int]$process.status.review
|
||||
errors = [int]$process.status.errors
|
||||
stored = [int]$process.status.stored
|
||||
persisted = [bool]$process.status.persisted
|
||||
queueConcurrency = [int]$process.status.queueConcurrency
|
||||
elapsedMs = [int]$process.status.elapsedMs
|
||||
}
|
||||
if (-not $process.ok) {
|
||||
throw "Native post-capture processing failed."
|
||||
}
|
||||
|
||||
$results = Invoke-DevJson "/scanner/native/results?runDir=$runDirParam&limit=$safeLimit"
|
||||
Save-Json "09-native-results" $results | Out-Null
|
||||
$summary.results = @{
|
||||
ok = [bool]$results.ok
|
||||
total = [int]$results.status.total
|
||||
loaded = [int]$results.status.results.Count
|
||||
}
|
||||
if (-not $results.ok) {
|
||||
throw "Native scan results could not be loaded."
|
||||
}
|
||||
|
||||
$summary.ok = $true
|
||||
} catch {
|
||||
$summary.errors += [string]$_.Exception.Message
|
||||
throw
|
||||
} finally {
|
||||
$summaryPath = Save-Json "native-live-smoke-summary" ([pscustomobject]$summary)
|
||||
Write-Host "Native live smoke summary: $summaryPath"
|
||||
if ($summary.ok) {
|
||||
Write-Host ("ok limit={0} captured={1} parsed={2} review={3} errors={4} stored={5} persisted={6}" -f `
|
||||
$summary.limit,
|
||||
$summary.finalStatus.captured,
|
||||
$summary.process.parsed,
|
||||
$summary.process.review,
|
||||
$summary.process.errors,
|
||||
$summary.process.stored,
|
||||
$summary.process.persisted)
|
||||
}
|
||||
}
|
||||
@@ -51,7 +51,6 @@ function validateAssessment(assessment, options = {}) {
|
||||
const errors = [];
|
||||
const expectedWinner = options.expectedWinner || "any";
|
||||
const expectedLimit = options.limit === undefined ? 100 : Number(options.limit);
|
||||
const allowSingleEngine = options.allowSingleEngine === true;
|
||||
const maxActiveAverageMsPerParsed = options.maxActiveAverageMsPerParsed === undefined
|
||||
? null
|
||||
: Number(options.maxActiveAverageMsPerParsed);
|
||||
@@ -77,26 +76,20 @@ function validateAssessment(assessment, options = {}) {
|
||||
errors.push(`Missing limit=${expectedLimit} assessment.`);
|
||||
}
|
||||
|
||||
if (!allowSingleEngine && expectedLimit === 100 && assessment.goal100Decision !== `qualified-comparison: winner=${limitAssessment?.winnerEngine}`) {
|
||||
errors.push(`goal100Decision is not a qualified comparison: ${assessment.goal100Decision || "<missing>"}`);
|
||||
if (expectedLimit === 100 && assessment.goal100Decision !== `qualified: winner=${limitAssessment?.winnerEngine}`) {
|
||||
errors.push(`goal100Decision is not a qualified 100-artifact run: ${assessment.goal100Decision || "<missing>"}`);
|
||||
}
|
||||
|
||||
if (limitAssessment?.limit !== expectedLimit) {
|
||||
errors.push(`limit assessment must be ${expectedLimit}, got ${limitAssessment?.limit ?? "<missing>"}.`);
|
||||
}
|
||||
if (!allowSingleEngine && limitAssessment?.comparisonComplete !== true) errors.push(`limit=${expectedLimit}.comparisonComplete must be true.`);
|
||||
if (limitAssessment?.winnerQualified !== true) errors.push(`limit=${expectedLimit}.winnerQualified must be true.`);
|
||||
if (expectedWinner !== "any" && limitAssessment?.winnerEngine !== expectedWinner) {
|
||||
errors.push(`Expected winner '${expectedWinner}', got '${limitAssessment?.winnerEngine ?? "<missing>"}'.`);
|
||||
}
|
||||
|
||||
const engines = Array.isArray(limitAssessment?.engines) ? limitAssessment.engines : [];
|
||||
const engineNames = new Set(engines.map((entry) => entry?.engine));
|
||||
if (!allowSingleEngine) {
|
||||
for (const required of ["current", "ik-traineddata"]) {
|
||||
if (!engineNames.has(required)) errors.push(`limit=${expectedLimit} is missing engine result: ${required}.`);
|
||||
}
|
||||
} else if (engines.length < 1) {
|
||||
if (engines.length < 1) {
|
||||
errors.push(`limit=${expectedLimit} must include at least one engine result.`);
|
||||
}
|
||||
|
||||
@@ -191,8 +184,8 @@ function main() {
|
||||
? findLatestAssessment(argValue("root", defaultAssessmentRoot()))
|
||||
: argValue("input", process.argv[2] || "");
|
||||
const expectedWinner = argValue("expect-winner", "any");
|
||||
if (!["any", "current", "ik-traineddata"].includes(expectedWinner)) {
|
||||
throw new Error("--expect-winner must be one of: any, current, ik-traineddata.");
|
||||
if (!["any", "current"].includes(expectedWinner)) {
|
||||
throw new Error("--expect-winner must be one of: any, current.");
|
||||
}
|
||||
const limit = Number(argValue("limit", "100"));
|
||||
const assessment = loadAssessment(inputPath);
|
||||
@@ -201,7 +194,6 @@ function main() {
|
||||
const result = validateAssessment(assessment, {
|
||||
expectedWinner,
|
||||
limit,
|
||||
allowSingleEngine: hasFlag("allow-single-engine"),
|
||||
maxActiveAverageMsPerParsed: maxActiveAverageMsPerParsed === "" ? undefined : maxActiveAverageMsPerParsed,
|
||||
maxCaptureRoundTripOverheadMs: maxCaptureRoundTripOverheadMs === "" ? undefined : maxCaptureRoundTripOverheadMs,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user