Improve scanner repeatability guardrails
This commit is contained in:
AzuTear
+45
-4
@@ -6,6 +6,7 @@ param(
|
||||
[int]$TimeoutSeconds = 600,
|
||||
[string]$OutputRoot = (Join-Path (Resolve-Path -LiteralPath ".").Path "outputs\live-soak"),
|
||||
[switch]$GoalRun,
|
||||
[switch]$RepeatabilityRun,
|
||||
[ValidateSet("current", "ik-traineddata", "compare")]
|
||||
[string]$ScanEngine = "current",
|
||||
[switch]$BenchmarkOcr,
|
||||
@@ -138,6 +139,8 @@ function Convert-ScanTimingSummary([object]$StatusPayload, [int]$Limit, [string]
|
||||
projectedMsFor100 = [int]$stats.projectedMsFor100
|
||||
activeProjectedMsFor100 = [int]$stats.activeProjectedMsFor100
|
||||
averageCaptureMs = [int]$stats.averageCaptureMs
|
||||
averageCaptureRoundTripMs = [int]$stats.averageCaptureRoundTripMs
|
||||
averageCaptureRoundTripOverheadMs = [int]$stats.averageCaptureRoundTripOverheadMs
|
||||
averageOcrMs = [int]$stats.averageOcrMs
|
||||
captureP50Ms = [int]$stats.captureP50Ms
|
||||
captureP90Ms = [int]$stats.captureP90Ms
|
||||
@@ -150,13 +153,15 @@ function Convert-ScanTimingSummary([object]$StatusPayload, [int]$Limit, [string]
|
||||
scrollReadyCount = [int]$stats.scrollReadyCount
|
||||
averageScrollReadyMs = [int]$stats.averageScrollReadyMs
|
||||
captureMs = [int]$stats.captureMs
|
||||
captureRoundTripMs = [int]$stats.captureRoundTripMs
|
||||
captureRoundTripOverheadMs = [int]$stats.captureRoundTripOverheadMs
|
||||
ocrMs = [int]$stats.ocrMs
|
||||
reviewStatus = [string]$StatusPayload.status.reviewStatus
|
||||
}
|
||||
}
|
||||
|
||||
function Write-ScanTimingLine([object]$Timing) {
|
||||
Write-Host ("engine={0} limit={1} status={2} parsed={3} review={4} miss={5} elapsed={6}ms active={7}ms flush={8}ms avg={9}ms activeAvg={10}ms captureAvg={11}ms captureP50={12}ms captureP90={13}ms ocrAvg={14}ms ocrP50={15}ms ocrP90={16}ms cardReadyAvg={17}ms scrollReadyAvg={18}ms ppm={19} activePpm={20} projected100={21}ms activeProjected100={22}ms" -f `
|
||||
Write-Host ("engine={0} limit={1} status={2} parsed={3} review={4} miss={5} elapsed={6}ms active={7}ms flush={8}ms avg={9}ms activeAvg={10}ms captureAvg={11}ms roundtripAvg={12}ms roundtripOverheadAvg={13}ms captureP50={14}ms captureP90={15}ms ocrAvg={16}ms ocrP50={17}ms ocrP90={18}ms cardReadyAvg={19}ms scrollReadyAvg={20}ms ppm={21} activePpm={22} projected100={23}ms activeProjected100={24}ms" -f `
|
||||
$Timing.engine,
|
||||
$Timing.limit,
|
||||
$Timing.status,
|
||||
@@ -169,6 +174,8 @@ function Write-ScanTimingLine([object]$Timing) {
|
||||
$Timing.averageMsPerParsed,
|
||||
$Timing.activeAverageMsPerParsed,
|
||||
$Timing.averageCaptureMs,
|
||||
$Timing.averageCaptureRoundTripMs,
|
||||
$Timing.averageCaptureRoundTripOverheadMs,
|
||||
$Timing.captureP50Ms,
|
||||
$Timing.captureP90Ms,
|
||||
$Timing.averageOcrMs,
|
||||
@@ -185,6 +192,7 @@ function Write-ScanTimingLine([object]$Timing) {
|
||||
function Get-TimingBottleneck([object]$Timing) {
|
||||
$parts = @(
|
||||
[pscustomobject]@{ name = "ocr"; value = [int]$Timing.averageOcrMs },
|
||||
[pscustomobject]@{ name = "capture-roundtrip-overhead"; value = [int]$Timing.averageCaptureRoundTripOverheadMs },
|
||||
[pscustomobject]@{ name = "capture"; value = [int]$Timing.averageCaptureMs },
|
||||
[pscustomobject]@{ name = "card-ready"; value = [int]$Timing.averageCardReadyMs },
|
||||
[pscustomobject]@{ name = "scroll-ready"; value = [int]$Timing.averageScrollReadyMs }
|
||||
@@ -198,6 +206,7 @@ function Get-TimingRecommendation([object]$Timing) {
|
||||
$bottleneck = Get-TimingBottleneck $Timing
|
||||
switch ($bottleneck) {
|
||||
"ocr" { return "OCR dominates; compare engine, crop count, worker pool, and parser-derived fields first." }
|
||||
"capture-roundtrip-overhead" { return "Capture transport overhead dominates; inspect native encode, IPC payload size, and Base64/DataURL conversion before OCR changes." }
|
||||
"capture" { return "Capture dominates; reduce payloads/crops and avoid full-frame or Base64 work in the hot loop." }
|
||||
"card-ready" { return "Card-ready dominates; tune detail fingerprint polling against IK's 200ms item wait." }
|
||||
"scroll-ready" { return "Scroll-ready dominates; tune page fingerprint polling against IK's 100ms fast-scroll wait." }
|
||||
@@ -274,6 +283,8 @@ function New-PerformanceAssessment([object[]]$Summaries) {
|
||||
activeProjectedMsFor100 = $entry.activeProjectedMsFor100
|
||||
averageOcrMs = $entry.averageOcrMs
|
||||
averageCaptureMs = $entry.averageCaptureMs
|
||||
averageCaptureRoundTripMs = $entry.averageCaptureRoundTripMs
|
||||
averageCaptureRoundTripOverheadMs = $entry.averageCaptureRoundTripOverheadMs
|
||||
averageCardReadyMs = $entry.averageCardReadyMs
|
||||
averageScrollReadyMs = $entry.averageScrollReadyMs
|
||||
bottleneck = Get-TimingBottleneck $entry
|
||||
@@ -292,6 +303,8 @@ function New-PerformanceAssessment([object[]]$Summaries) {
|
||||
winnerReviewRate = $winner.reviewRate
|
||||
winnerActiveAverageMsPerParsed = $winner.activeAverageMsPerParsed
|
||||
winnerActiveProjectedMsFor100 = $winner.activeProjectedMsFor100
|
||||
winnerAverageCaptureRoundTripMs = $winner.averageCaptureRoundTripMs
|
||||
winnerAverageCaptureRoundTripOverheadMs = $winner.averageCaptureRoundTripOverheadMs
|
||||
engines = $engineReports
|
||||
}
|
||||
}
|
||||
@@ -322,7 +335,7 @@ function Write-PerformanceAssessment([object]$Assessment) {
|
||||
Write-Host "assessment goal100: $($Assessment.goal100Decision)"
|
||||
}
|
||||
foreach ($limit in @($Assessment.limits)) {
|
||||
Write-Host ("assessment limit={0}: winner={1} qualified={2} completeCompare={3} engines={4} missRate={5:P1} reviewRate={6:P1} activeAvg={7}ms projected100={8}ms" -f `
|
||||
Write-Host ("assessment limit={0}: winner={1} qualified={2} completeCompare={3} engines={4} missRate={5:P1} reviewRate={6:P1} activeAvg={7}ms projected100={8}ms roundtripOverheadAvg={9}ms" -f `
|
||||
$limit.limit,
|
||||
$limit.winnerEngine,
|
||||
$limit.winnerQualified,
|
||||
@@ -331,9 +344,10 @@ function Write-PerformanceAssessment([object]$Assessment) {
|
||||
$limit.winnerMissRate,
|
||||
$limit.winnerReviewRate,
|
||||
$limit.winnerActiveAverageMsPerParsed,
|
||||
$limit.winnerActiveProjectedMsFor100)
|
||||
$limit.winnerActiveProjectedMsFor100,
|
||||
$limit.winnerAverageCaptureRoundTripOverheadMs)
|
||||
foreach ($engine in @($limit.engines)) {
|
||||
Write-Host (" {0}: qualified={1}, missRate={2:P1}, reviewRate={3:P1}, bottleneck={4}, activeAvg={5}ms, ocr={6}ms, capture={7}ms, cardReady={8}ms, scrollReady={9}ms, decision={10}" -f `
|
||||
Write-Host (" {0}: qualified={1}, missRate={2:P1}, reviewRate={3:P1}, bottleneck={4}, activeAvg={5}ms, ocr={6}ms, capture={7}ms, roundtrip={8}ms, roundtripOverhead={9}ms, cardReady={10}ms, scrollReady={11}ms, decision={12}" -f `
|
||||
$engine.engine,
|
||||
$engine.qualified,
|
||||
$engine.missRate,
|
||||
@@ -342,6 +356,8 @@ function Write-PerformanceAssessment([object]$Assessment) {
|
||||
$engine.activeAverageMsPerParsed,
|
||||
$engine.averageOcrMs,
|
||||
$engine.averageCaptureMs,
|
||||
$engine.averageCaptureRoundTripMs,
|
||||
$engine.averageCaptureRoundTripOverheadMs,
|
||||
$engine.averageCardReadyMs,
|
||||
$engine.averageScrollReadyMs,
|
||||
$engine.qualityDecision)
|
||||
@@ -363,6 +379,8 @@ function Invoke-AssessmentSelfTest {
|
||||
activeProjectedMsFor100 = 70000
|
||||
averageOcrMs = 260
|
||||
averageCaptureMs = 160
|
||||
averageCaptureRoundTripMs = 340
|
||||
averageCaptureRoundTripOverheadMs = 180
|
||||
averageCardReadyMs = 190
|
||||
averageScrollReadyMs = 60
|
||||
},
|
||||
@@ -378,6 +396,8 @@ function Invoke-AssessmentSelfTest {
|
||||
activeProjectedMsFor100 = 82000
|
||||
averageOcrMs = 210
|
||||
averageCaptureMs = 170
|
||||
averageCaptureRoundTripMs = 300
|
||||
averageCaptureRoundTripOverheadMs = 130
|
||||
averageCardReadyMs = 205
|
||||
averageScrollReadyMs = 80
|
||||
},
|
||||
@@ -393,6 +413,8 @@ function Invoke-AssessmentSelfTest {
|
||||
activeProjectedMsFor100 = 39000
|
||||
averageOcrMs = 150
|
||||
averageCaptureMs = 130
|
||||
averageCaptureRoundTripMs = 280
|
||||
averageCaptureRoundTripOverheadMs = 150
|
||||
averageCardReadyMs = 80
|
||||
averageScrollReadyMs = 0
|
||||
},
|
||||
@@ -408,6 +430,8 @@ function Invoke-AssessmentSelfTest {
|
||||
activeProjectedMsFor100 = 46000
|
||||
averageOcrMs = 190
|
||||
averageCaptureMs = 130
|
||||
averageCaptureRoundTripMs = 310
|
||||
averageCaptureRoundTripOverheadMs = 180
|
||||
averageCardReadyMs = 80
|
||||
averageScrollReadyMs = 0
|
||||
},
|
||||
@@ -423,6 +447,8 @@ function Invoke-AssessmentSelfTest {
|
||||
activeProjectedMsFor100 = 30000
|
||||
averageOcrMs = 100
|
||||
averageCaptureMs = 90
|
||||
averageCaptureRoundTripMs = 190
|
||||
averageCaptureRoundTripOverheadMs = 100
|
||||
averageCardReadyMs = 40
|
||||
averageScrollReadyMs = 20
|
||||
},
|
||||
@@ -438,6 +464,8 @@ function Invoke-AssessmentSelfTest {
|
||||
activeProjectedMsFor100 = 60000
|
||||
averageOcrMs = 230
|
||||
averageCaptureMs = 130
|
||||
averageCaptureRoundTripMs = 300
|
||||
averageCaptureRoundTripOverheadMs = 170
|
||||
averageCardReadyMs = 160
|
||||
averageScrollReadyMs = 50
|
||||
}
|
||||
@@ -492,6 +520,8 @@ function Invoke-AssessmentSelfTest {
|
||||
activeProjectedMsFor100 = 50000
|
||||
averageOcrMs = 180
|
||||
averageCaptureMs = 120
|
||||
averageCaptureRoundTripMs = 260
|
||||
averageCaptureRoundTripOverheadMs = 140
|
||||
averageCardReadyMs = 100
|
||||
averageScrollReadyMs = 50
|
||||
}
|
||||
@@ -665,6 +695,17 @@ if ($GoalRun) {
|
||||
$Limits = @(2, 5, 20, 45, 100)
|
||||
$BenchmarkOcr = $true
|
||||
}
|
||||
if ($RepeatabilityRun) {
|
||||
$Limits = @(20, 45, 100)
|
||||
}
|
||||
|
||||
$Limits = @($Limits | ForEach-Object {
|
||||
$limit = [int]$_
|
||||
if ($limit -lt 1 -or $limit -gt 1800) {
|
||||
throw "Refusing unsafe scan limit '$limit'. Expected a bounded value from 1 to 1800."
|
||||
}
|
||||
$limit
|
||||
})
|
||||
|
||||
$ScanEngines = if ($ScanEngine -eq "compare") { @("current", "ik-traineddata") } else { @($ScanEngine) }
|
||||
$RunSummaries = @()
|
||||
|
||||
@@ -51,6 +51,13 @@ function validateAssessment(assessment, options = {}) {
|
||||
const errors = [];
|
||||
const expectedWinner = options.expectedWinner || "any";
|
||||
const expectedLimit = options.limit === undefined ? 100 : Number(options.limit);
|
||||
const allowSingleEngine = options.allowSingleEngine === true;
|
||||
const maxActiveAverageMsPerParsed = options.maxActiveAverageMsPerParsed === undefined
|
||||
? null
|
||||
: Number(options.maxActiveAverageMsPerParsed);
|
||||
const maxCaptureRoundTripOverheadMs = options.maxCaptureRoundTripOverheadMs === undefined
|
||||
? null
|
||||
: Number(options.maxCaptureRoundTripOverheadMs);
|
||||
if (!assessment || typeof assessment !== "object") {
|
||||
return { ok: false, errors: ["Assessment must be a JSON object."] };
|
||||
}
|
||||
@@ -58,20 +65,26 @@ function validateAssessment(assessment, options = {}) {
|
||||
if (!Number.isInteger(expectedLimit) || expectedLimit < 1) {
|
||||
errors.push(`--limit must be a positive integer, got ${options.limit}.`);
|
||||
}
|
||||
if (maxActiveAverageMsPerParsed !== null && (!Number.isFinite(maxActiveAverageMsPerParsed) || maxActiveAverageMsPerParsed <= 0)) {
|
||||
errors.push(`--max-active-average-ms must be a positive finite number, got ${options.maxActiveAverageMsPerParsed}.`);
|
||||
}
|
||||
if (maxCaptureRoundTripOverheadMs !== null && (!Number.isFinite(maxCaptureRoundTripOverheadMs) || maxCaptureRoundTripOverheadMs < 0)) {
|
||||
errors.push(`--max-capture-roundtrip-overhead-ms must be a non-negative finite number, got ${options.maxCaptureRoundTripOverheadMs}.`);
|
||||
}
|
||||
|
||||
const limitAssessment = findLimitAssessment(assessment, expectedLimit);
|
||||
if (!limitAssessment || typeof limitAssessment !== "object") {
|
||||
errors.push(`Missing limit=${expectedLimit} assessment.`);
|
||||
}
|
||||
|
||||
if (expectedLimit === 100 && assessment.goal100Decision !== `qualified-comparison: winner=${limitAssessment?.winnerEngine}`) {
|
||||
if (!allowSingleEngine && expectedLimit === 100 && assessment.goal100Decision !== `qualified-comparison: winner=${limitAssessment?.winnerEngine}`) {
|
||||
errors.push(`goal100Decision is not a qualified comparison: ${assessment.goal100Decision || "<missing>"}`);
|
||||
}
|
||||
|
||||
if (limitAssessment?.limit !== expectedLimit) {
|
||||
errors.push(`limit assessment must be ${expectedLimit}, got ${limitAssessment?.limit ?? "<missing>"}.`);
|
||||
}
|
||||
if (limitAssessment?.comparisonComplete !== true) errors.push(`limit=${expectedLimit}.comparisonComplete must be true.`);
|
||||
if (!allowSingleEngine && limitAssessment?.comparisonComplete !== true) errors.push(`limit=${expectedLimit}.comparisonComplete must be true.`);
|
||||
if (limitAssessment?.winnerQualified !== true) errors.push(`limit=${expectedLimit}.winnerQualified must be true.`);
|
||||
if (expectedWinner !== "any" && limitAssessment?.winnerEngine !== expectedWinner) {
|
||||
errors.push(`Expected winner '${expectedWinner}', got '${limitAssessment?.winnerEngine ?? "<missing>"}'.`);
|
||||
@@ -79,8 +92,12 @@ function validateAssessment(assessment, options = {}) {
|
||||
|
||||
const engines = Array.isArray(limitAssessment?.engines) ? limitAssessment.engines : [];
|
||||
const engineNames = new Set(engines.map((entry) => entry?.engine));
|
||||
for (const required of ["current", "ik-traineddata"]) {
|
||||
if (!engineNames.has(required)) errors.push(`limit=${expectedLimit} is missing engine result: ${required}.`);
|
||||
if (!allowSingleEngine) {
|
||||
for (const required of ["current", "ik-traineddata"]) {
|
||||
if (!engineNames.has(required)) errors.push(`limit=${expectedLimit} is missing engine result: ${required}.`);
|
||||
}
|
||||
} else if (engines.length < 1) {
|
||||
errors.push(`limit=${expectedLimit} must include at least one engine result.`);
|
||||
}
|
||||
|
||||
const winner = engines.find((entry) => entry?.engine === limitAssessment?.winnerEngine);
|
||||
@@ -93,6 +110,7 @@ function validateAssessment(assessment, options = {}) {
|
||||
const summaryWinnerReviewRate = Number(limitAssessment?.winnerReviewRate);
|
||||
const winnerActiveAverageMsPerParsed = Number(limitAssessment?.winnerActiveAverageMsPerParsed);
|
||||
const winnerActiveProjectedMsFor100 = Number(limitAssessment?.winnerActiveProjectedMsFor100);
|
||||
const winnerAverageCaptureRoundTripOverheadMs = Number(limitAssessment?.winnerAverageCaptureRoundTripOverheadMs);
|
||||
if (winner.qualified !== true) errors.push("Winner engine result must be qualified.");
|
||||
if (!Number.isFinite(winnerMissRate)) {
|
||||
errors.push(`Winner missRate must be a finite number, got ${winner.missRate ?? "<missing>"}.`);
|
||||
@@ -116,10 +134,19 @@ function validateAssessment(assessment, options = {}) {
|
||||
}
|
||||
if (!Number.isFinite(winnerActiveAverageMsPerParsed) || winnerActiveAverageMsPerParsed <= 0) {
|
||||
errors.push(`Winner active average timing must be a positive finite number, got ${limitAssessment?.winnerActiveAverageMsPerParsed ?? "<missing>"}.`);
|
||||
} else if (maxActiveAverageMsPerParsed !== null && Number.isFinite(maxActiveAverageMsPerParsed) && winnerActiveAverageMsPerParsed > maxActiveAverageMsPerParsed) {
|
||||
errors.push(`Winner active average timing exceeds ${maxActiveAverageMsPerParsed}ms: ${winnerActiveAverageMsPerParsed}.`);
|
||||
}
|
||||
if (!Number.isFinite(winnerActiveProjectedMsFor100) || winnerActiveProjectedMsFor100 <= 0) {
|
||||
errors.push(`Winner projected100 timing must be a positive finite number, got ${limitAssessment?.winnerActiveProjectedMsFor100 ?? "<missing>"}.`);
|
||||
}
|
||||
if (maxCaptureRoundTripOverheadMs !== null && Number.isFinite(maxCaptureRoundTripOverheadMs)) {
|
||||
if (!Number.isFinite(winnerAverageCaptureRoundTripOverheadMs)) {
|
||||
errors.push(`Winner capture roundtrip overhead must be a finite number, got ${limitAssessment?.winnerAverageCaptureRoundTripOverheadMs ?? "<missing>"}.`);
|
||||
} else if (winnerAverageCaptureRoundTripOverheadMs > maxCaptureRoundTripOverheadMs) {
|
||||
errors.push(`Winner capture roundtrip overhead exceeds ${maxCaptureRoundTripOverheadMs}ms: ${winnerAverageCaptureRoundTripOverheadMs}.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -130,6 +157,8 @@ function validateAssessment(assessment, options = {}) {
|
||||
winnerEngine: limitAssessment?.winnerEngine,
|
||||
winnerActiveAverageMsPerParsed: limitAssessment?.winnerActiveAverageMsPerParsed,
|
||||
winnerActiveProjectedMsFor100: limitAssessment?.winnerActiveProjectedMsFor100,
|
||||
winnerAverageCaptureRoundTripMs: limitAssessment?.winnerAverageCaptureRoundTripMs,
|
||||
winnerAverageCaptureRoundTripOverheadMs: limitAssessment?.winnerAverageCaptureRoundTripOverheadMs,
|
||||
winnerMissRate: limitAssessment?.winnerMissRate,
|
||||
winnerReviewRate: limitAssessment?.winnerReviewRate,
|
||||
};
|
||||
@@ -145,6 +174,8 @@ function formatSummary(result) {
|
||||
`winner: ${result.winnerEngine || "unknown"}`,
|
||||
`activeAvg: ${result.winnerActiveAverageMsPerParsed ?? "unknown"}ms/artifact`,
|
||||
`projected100: ${result.winnerActiveProjectedMsFor100 ?? "unknown"}ms`,
|
||||
`captureRoundTrip: ${result.winnerAverageCaptureRoundTripMs ?? "unknown"}ms`,
|
||||
`captureRoundTripOverhead: ${result.winnerAverageCaptureRoundTripOverheadMs ?? "unknown"}ms`,
|
||||
`missRate: ${result.winnerMissRate ?? "unknown"}`,
|
||||
`reviewRate: ${result.winnerReviewRate ?? "unknown"}`,
|
||||
];
|
||||
@@ -165,7 +196,15 @@ function main() {
|
||||
}
|
||||
const limit = Number(argValue("limit", "100"));
|
||||
const assessment = loadAssessment(inputPath);
|
||||
const result = validateAssessment(assessment, { expectedWinner, limit });
|
||||
const maxActiveAverageMsPerParsed = argValue("max-active-average-ms", "");
|
||||
const maxCaptureRoundTripOverheadMs = argValue("max-capture-roundtrip-overhead-ms", "");
|
||||
const result = validateAssessment(assessment, {
|
||||
expectedWinner,
|
||||
limit,
|
||||
allowSingleEngine: hasFlag("allow-single-engine"),
|
||||
maxActiveAverageMsPerParsed: maxActiveAverageMsPerParsed === "" ? undefined : maxActiveAverageMsPerParsed,
|
||||
maxCaptureRoundTripOverheadMs: maxCaptureRoundTripOverheadMs === "" ? undefined : maxCaptureRoundTripOverheadMs,
|
||||
});
|
||||
const payload = { inputPath: path.resolve(inputPath), ...result };
|
||||
console.log(hasFlag("summary") ? formatSummary(payload) : JSON.stringify(payload, null, 2));
|
||||
if (!result.ok) process.exit(1);
|
||||
|
||||
Reference in New Issue
Block a user