feat(scanner): add native artifact pipeline
Add native IK-style capture processing, Artifact Inventory, explicit promotion and single-result review. Confirm the three live OCR corrections in the eval corpus and preserve extraction/value separation.
This commit is contained in:
AzuTear
+57
-103
@@ -7,12 +7,12 @@ param(
|
||||
[string]$OutputRoot = (Join-Path (Resolve-Path -LiteralPath ".").Path "outputs\live-soak"),
|
||||
[switch]$GoalRun,
|
||||
[switch]$RepeatabilityRun,
|
||||
[ValidateSet("current", "ik-traineddata", "compare")]
|
||||
[ValidateSet("current")]
|
||||
[string]$ScanEngine = "current",
|
||||
[switch]$BenchmarkOcr,
|
||||
[int]$BenchmarkLimit = 5,
|
||||
[ValidateSet("current", "ik-traineddata", "compare")]
|
||||
[string]$BenchmarkEngine = "compare",
|
||||
[ValidateSet("current")]
|
||||
[string]$BenchmarkEngine = "current",
|
||||
[ValidateSet("fast", "full")]
|
||||
[string]$BenchmarkProfile = "fast",
|
||||
[switch]$SkipSmartCapture,
|
||||
@@ -205,11 +205,11 @@ function Get-TimingBottleneck([object]$Timing) {
|
||||
function Get-TimingRecommendation([object]$Timing) {
|
||||
$bottleneck = Get-TimingBottleneck $Timing
|
||||
switch ($bottleneck) {
|
||||
"ocr" { return "OCR dominates; compare engine, crop count, worker pool, and parser-derived fields first." }
|
||||
"ocr" { return "OCR dominates; inspect crop count, worker pool, and parser-derived fields first." }
|
||||
"capture-roundtrip-overhead" { return "Capture transport overhead dominates; inspect native encode, IPC payload size, and Base64/DataURL conversion before OCR changes." }
|
||||
"capture" { return "Capture dominates; reduce payloads/crops and avoid full-frame or Base64 work in the hot loop." }
|
||||
"card-ready" { return "Card-ready dominates; tune detail fingerprint polling against IK's 200ms item wait." }
|
||||
"scroll-ready" { return "Scroll-ready dominates; tune page fingerprint polling against IK's 100ms fast-scroll wait." }
|
||||
"card-ready" { return "Card-ready dominates; tune detail fingerprint polling before OCR changes." }
|
||||
"scroll-ready" { return "Scroll-ready dominates; tune page fingerprint polling before OCR changes." }
|
||||
default { return "No dominant timing component detected; inspect misses/review/duplicates and raw diagnostic events." }
|
||||
}
|
||||
}
|
||||
@@ -296,7 +296,7 @@ function New-PerformanceAssessment([object[]]$Summaries) {
|
||||
limit = [int]$group.Name
|
||||
engineCount = $entries.Count
|
||||
enginesCompared = @($entries | ForEach-Object { $_.engine })
|
||||
comparisonComplete = (@($entries | Where-Object { $_.engine -eq "current" }).Count -gt 0 -and @($entries | Where-Object { $_.engine -eq "ik-traineddata" }).Count -gt 0)
|
||||
singleEngine = $true
|
||||
winnerEngine = $winner.engine
|
||||
winnerQualified = $winner.qualified
|
||||
winnerMissRate = $winner.missRate
|
||||
@@ -312,18 +312,16 @@ function New-PerformanceAssessment([object[]]$Summaries) {
|
||||
$goal100 = @($limitReports | Where-Object { $_.limit -eq 100 } | Select-Object -First 1)
|
||||
$goal100Decision = "not-run: missing 100-artifact assessment"
|
||||
if ($goal100.Count -gt 0) {
|
||||
if (-not $goal100[0].comparisonComplete) {
|
||||
$goal100Decision = "not-comparable: current and ik-traineddata were not both run"
|
||||
} elseif (-not $goal100[0].winnerQualified) {
|
||||
if (-not $goal100[0].winnerQualified) {
|
||||
$goal100Decision = "not-qualified: 100-artifact winner failed quality gates"
|
||||
} else {
|
||||
$goal100Decision = "qualified-comparison: winner=$($goal100[0].winnerEngine)"
|
||||
$goal100Decision = "qualified: winner=$($goal100[0].winnerEngine)"
|
||||
}
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
createdAt = (Get-Date).ToString("o")
|
||||
goalLimit = 100
|
||||
goalEngines = @("current", "ik-traineddata")
|
||||
goalEngines = @("current")
|
||||
goal100Decision = $goal100Decision
|
||||
goal100 = if ($goal100.Count -gt 0) { $goal100[0] } else { $null }
|
||||
limits = $limitReports
|
||||
@@ -335,11 +333,10 @@ function Write-PerformanceAssessment([object]$Assessment) {
|
||||
Write-Host "assessment goal100: $($Assessment.goal100Decision)"
|
||||
}
|
||||
foreach ($limit in @($Assessment.limits)) {
|
||||
Write-Host ("assessment limit={0}: winner={1} qualified={2} completeCompare={3} engines={4} missRate={5:P1} reviewRate={6:P1} activeAvg={7}ms projected100={8}ms roundtripOverheadAvg={9}ms" -f `
|
||||
Write-Host ("assessment limit={0}: winner={1} qualified={2} engines={3} missRate={4:P1} reviewRate={5:P1} activeAvg={6}ms projected100={7}ms roundtripOverheadAvg={8}ms" -f `
|
||||
$limit.limit,
|
||||
$limit.winnerEngine,
|
||||
$limit.winnerQualified,
|
||||
$limit.comparisonComplete,
|
||||
($limit.enginesCompared -join ","),
|
||||
$limit.winnerMissRate,
|
||||
$limit.winnerReviewRate,
|
||||
@@ -372,34 +369,34 @@ function Invoke-AssessmentSelfTest {
|
||||
limit = 100
|
||||
status = "done"
|
||||
parsed = 100
|
||||
review = 22
|
||||
review = 0
|
||||
misses = 0
|
||||
activeAverageMsPerParsed = 700
|
||||
averageMsPerParsed = 760
|
||||
activeProjectedMsFor100 = 70000
|
||||
averageOcrMs = 260
|
||||
averageCaptureMs = 160
|
||||
averageCaptureRoundTripMs = 340
|
||||
averageCaptureRoundTripOverheadMs = 180
|
||||
averageCardReadyMs = 190
|
||||
averageScrollReadyMs = 60
|
||||
activeAverageMsPerParsed = 500
|
||||
averageMsPerParsed = 520
|
||||
activeProjectedMsFor100 = 50000
|
||||
averageOcrMs = 180
|
||||
averageCaptureMs = 120
|
||||
averageCaptureRoundTripMs = 260
|
||||
averageCaptureRoundTripOverheadMs = 140
|
||||
averageCardReadyMs = 100
|
||||
averageScrollReadyMs = 50
|
||||
},
|
||||
[pscustomobject]@{
|
||||
engine = "ik-traineddata"
|
||||
engine = "high-review"
|
||||
limit = 100
|
||||
status = "done"
|
||||
parsed = 100
|
||||
review = 4
|
||||
review = 22
|
||||
misses = 0
|
||||
activeAverageMsPerParsed = 820
|
||||
averageMsPerParsed = 870
|
||||
activeProjectedMsFor100 = 82000
|
||||
averageOcrMs = 210
|
||||
averageCaptureMs = 170
|
||||
averageCaptureRoundTripMs = 300
|
||||
averageCaptureRoundTripOverheadMs = 130
|
||||
averageCardReadyMs = 205
|
||||
averageScrollReadyMs = 80
|
||||
activeAverageMsPerParsed = 300
|
||||
averageMsPerParsed = 330
|
||||
activeProjectedMsFor100 = 30000
|
||||
averageOcrMs = 100
|
||||
averageCaptureMs = 90
|
||||
averageCaptureRoundTripMs = 190
|
||||
averageCaptureRoundTripOverheadMs = 100
|
||||
averageCardReadyMs = 40
|
||||
averageScrollReadyMs = 20
|
||||
},
|
||||
[pscustomobject]@{
|
||||
engine = "current"
|
||||
@@ -418,23 +415,6 @@ function Invoke-AssessmentSelfTest {
|
||||
averageCardReadyMs = 80
|
||||
averageScrollReadyMs = 0
|
||||
},
|
||||
[pscustomobject]@{
|
||||
engine = "ik-traineddata"
|
||||
limit = 20
|
||||
status = "done"
|
||||
parsed = 20
|
||||
review = 2
|
||||
misses = 0
|
||||
activeAverageMsPerParsed = 460
|
||||
averageMsPerParsed = 475
|
||||
activeProjectedMsFor100 = 46000
|
||||
averageOcrMs = 190
|
||||
averageCaptureMs = 130
|
||||
averageCaptureRoundTripMs = 310
|
||||
averageCaptureRoundTripOverheadMs = 180
|
||||
averageCardReadyMs = 80
|
||||
averageScrollReadyMs = 0
|
||||
},
|
||||
[pscustomobject]@{
|
||||
engine = "broken-fast"
|
||||
limit = 45
|
||||
@@ -476,43 +456,37 @@ function Invoke-AssessmentSelfTest {
|
||||
$limit20 = @($assessment.limits | Where-Object { $_.limit -eq 20 } | Select-Object -First 1)[0]
|
||||
$limit45 = @($assessment.limits | Where-Object { $_.limit -eq 45 } | Select-Object -First 1)[0]
|
||||
|
||||
if ($goal100.winnerEngine -ne "ik-traineddata") {
|
||||
throw "Assessment self-test failed: expected ik-traineddata to win limit=100, got '$($goal100.winnerEngine)'."
|
||||
if ($goal100.winnerEngine -ne "current") {
|
||||
throw "Assessment self-test failed: expected current to win limit=100, got '$($goal100.winnerEngine)'."
|
||||
}
|
||||
if (-not $goal100.winnerQualified) {
|
||||
throw "Assessment self-test failed: expected limit=100 winner to be qualified."
|
||||
}
|
||||
if (-not $goal100.comparisonComplete) {
|
||||
throw "Assessment self-test failed: expected limit=100 to be a complete current vs ik-traineddata comparison."
|
||||
}
|
||||
if ($assessment.goal100Decision -ne "qualified-comparison: winner=ik-traineddata") {
|
||||
if ($assessment.goal100Decision -ne "qualified: winner=current") {
|
||||
throw "Assessment self-test failed: unexpected goal100Decision '$($assessment.goal100Decision)'."
|
||||
}
|
||||
if ($limit20.winnerEngine -ne "current") {
|
||||
throw "Assessment self-test failed: expected current to win limit=20, got '$($limit20.winnerEngine)'."
|
||||
}
|
||||
if (-not $limit20.comparisonComplete) {
|
||||
throw "Assessment self-test failed: expected limit=20 to be a complete current vs ik-traineddata comparison."
|
||||
}
|
||||
if (-not $limit20.winnerQualified) {
|
||||
throw "Assessment self-test failed: expected limit=20 winner to be qualified."
|
||||
}
|
||||
if ($limit45.winnerEngine -ne "current") {
|
||||
throw "Assessment self-test failed: expected current to win limit=45, got '$($limit45.winnerEngine)'."
|
||||
}
|
||||
if (@($goal100.engines | Where-Object { $_.engine -eq "current" })[0].qualityDecision -ne "not-qualified: review rate above 15%") {
|
||||
throw "Assessment self-test failed: expected high-review current run to be rejected."
|
||||
if (@($goal100.engines | Where-Object { $_.engine -eq "high-review" })[0].qualityDecision -ne "not-qualified: review rate above 15%") {
|
||||
throw "Assessment self-test failed: expected high-review run to be rejected."
|
||||
}
|
||||
if (@($limit45.engines | Where-Object { $_.engine -eq "broken-fast" })[0].qualityDecision -ne "not-qualified: miss rate above 2%") {
|
||||
throw "Assessment self-test failed: expected broken-fast run to be rejected for miss rate."
|
||||
}
|
||||
|
||||
$singleEngineAssessment = New-PerformanceAssessment -Summaries @(
|
||||
$missingGoalAssessment = New-PerformanceAssessment -Summaries @(
|
||||
[pscustomobject]@{
|
||||
engine = "current"
|
||||
limit = 100
|
||||
limit = 20
|
||||
status = "done"
|
||||
parsed = 100
|
||||
parsed = 20
|
||||
review = 0
|
||||
misses = 0
|
||||
activeAverageMsPerParsed = 500
|
||||
@@ -526,8 +500,8 @@ function Invoke-AssessmentSelfTest {
|
||||
averageScrollReadyMs = 50
|
||||
}
|
||||
)
|
||||
if ($singleEngineAssessment.goal100Decision -ne "not-comparable: current and ik-traineddata were not both run") {
|
||||
throw "Assessment self-test failed: expected single-engine 100 run to be not-comparable, got '$($singleEngineAssessment.goal100Decision)'."
|
||||
if ($missingGoalAssessment.goal100Decision -ne "not-run: missing 100-artifact assessment") {
|
||||
throw "Assessment self-test failed: expected missing 100 run to be not-run, got '$($missingGoalAssessment.goal100Decision)'."
|
||||
}
|
||||
|
||||
Write-PerformanceAssessment $assessment
|
||||
@@ -543,43 +517,23 @@ function Invoke-OcrBenchmark {
|
||||
)
|
||||
|
||||
Write-Host "Warming current OCR workers..."
|
||||
$warmCurrent = Invoke-DevJson "/scanner/ocr/warmup?engine=current"
|
||||
$warmCurrent = Invoke-DevJson "/scanner/ocr/warmup"
|
||||
Save-Json "benchmark-warmup-current" $warmCurrent | Out-Null
|
||||
|
||||
if ($Engine -eq "compare" -or $Engine -eq "ik-traineddata") {
|
||||
Write-Host "Warming IK-traineddata OCR workers..."
|
||||
$warmIk = Invoke-DevJson "/scanner/ocr/warmup?engine=ik-traineddata"
|
||||
Save-Json "benchmark-warmup-ik-traineddata" $warmIk | Out-Null
|
||||
}
|
||||
|
||||
Write-Host "Running OCR benchmark engine=$Engine profile=$Profile limit=$Limit"
|
||||
$benchmark = Invoke-DevJson "/scanner/benchmark-ocr?limit=$Limit&engine=$Engine&profile=$Profile"
|
||||
$benchmark = Invoke-DevJson "/scanner/benchmark-ocr?limit=$Limit&profile=$Profile"
|
||||
Save-Json "benchmark-ocr-$Engine-$Profile-limit-$Limit" $benchmark | Out-Null
|
||||
|
||||
if ($benchmark.summary.mode -eq "compare") {
|
||||
foreach ($engineSummary in @($benchmark.summary.engines)) {
|
||||
Write-Host ("benchmark {0}: avg={1}ms ocrAvg={2}ms p50={3}ms p90={4}ms projected100={5}ms skipped={6} pool={7}" -f `
|
||||
$engineSummary.engine,
|
||||
$engineSummary.averageMs,
|
||||
$engineSummary.averageOcrMs,
|
||||
$engineSummary.p50Ms,
|
||||
$engineSummary.p90Ms,
|
||||
$engineSummary.projectedMs.artifacts100,
|
||||
$engineSummary.skippedOcrCaptures,
|
||||
$engineSummary.workerPoolSize)
|
||||
}
|
||||
} else {
|
||||
$summary = $benchmark.summary
|
||||
Write-Host ("benchmark {0}: avg={1}ms ocrAvg={2}ms p50={3}ms p90={4}ms projected100={5}ms skipped={6} pool={7}" -f `
|
||||
$summary.engine,
|
||||
$summary.averageMs,
|
||||
$summary.averageOcrMs,
|
||||
$summary.p50Ms,
|
||||
$summary.p90Ms,
|
||||
$summary.projectedMs.artifacts100,
|
||||
$summary.skippedOcrCaptures,
|
||||
$summary.workerPoolSize)
|
||||
}
|
||||
$summary = $benchmark.summary
|
||||
Write-Host ("benchmark {0}: avg={1}ms ocrAvg={2}ms p50={3}ms p90={4}ms projected100={5}ms skipped={6} pool={7}" -f `
|
||||
$summary.engine,
|
||||
$summary.averageMs,
|
||||
$summary.averageOcrMs,
|
||||
$summary.p50Ms,
|
||||
$summary.p90Ms,
|
||||
$summary.projectedMs.artifacts100,
|
||||
$summary.skippedOcrCaptures,
|
||||
$summary.workerPoolSize)
|
||||
|
||||
return $benchmark
|
||||
}
|
||||
@@ -707,7 +661,7 @@ $Limits = @($Limits | ForEach-Object {
|
||||
$limit
|
||||
})
|
||||
|
||||
$ScanEngines = if ($ScanEngine -eq "compare") { @("current", "ik-traineddata") } else { @($ScanEngine) }
|
||||
$ScanEngines = @($ScanEngine)
|
||||
$RunSummaries = @()
|
||||
|
||||
$transcriptPath = Join-Path $RunDir "transcript.log"
|
||||
@@ -728,7 +682,7 @@ try {
|
||||
if ($health.appBuild) {
|
||||
Write-Host "App build: signature=$($health.appBuild.signature), pid=$($health.appBuild.pid), startedAt=$($health.appBuild.startedAt), ocrWorkers=$($health.appBuild.expectedOcrWorkerPoolSize)"
|
||||
if ($health.appBuild.expectedOcrWorkerPoolSize -lt 4) {
|
||||
Write-Host "WARNUNG: OCR worker pool is below 4. This is valid for constrained debugging, but not ideal for IK-speed comparison." -ForegroundColor Yellow
|
||||
Write-Host "WARNUNG: OCR worker pool is below 4. This is valid for constrained debugging, but not ideal for scanner timing." -ForegroundColor Yellow
|
||||
}
|
||||
}
|
||||
|
||||
@@ -772,7 +726,7 @@ try {
|
||||
foreach ($limit in $Limits) {
|
||||
if ($limit -lt 1) { continue }
|
||||
Write-Host "Starting bounded scanner run limit=$limit engine=$engine"
|
||||
$start = Invoke-DevJson "/scanner/start?entry=visible-inventory&limit=$limit&engine=$engine"
|
||||
$start = Invoke-DevJson "/scanner/start?entry=visible-inventory&limit=$limit"
|
||||
Save-Json "scan-$engine-limit-$limit-start" $start | Out-Null
|
||||
|
||||
$finalStatus = Wait-ForScannerIdle -Limit $limit -Engine $engine
|
||||
|
||||
Reference in New Issue
Block a user