Improve scanner repeatability guardrails

scripts/validate-scan-assessment.cjs

@@ -51,6 +51,13 @@ function validateAssessment(assessment, options = {}) {
   const errors = [];
   const expectedWinner = options.expectedWinner || "any";
   const expectedLimit = options.limit === undefined ? 100 : Number(options.limit);
+  const allowSingleEngine = options.allowSingleEngine === true;
+  const maxActiveAverageMsPerParsed = options.maxActiveAverageMsPerParsed === undefined
+    ? null
+    : Number(options.maxActiveAverageMsPerParsed);
+  const maxCaptureRoundTripOverheadMs = options.maxCaptureRoundTripOverheadMs === undefined
+    ? null
+    : Number(options.maxCaptureRoundTripOverheadMs);
   if (!assessment || typeof assessment !== "object") {
     return { ok: false, errors: ["Assessment must be a JSON object."] };
   }
@@ -58,20 +65,26 @@ function validateAssessment(assessment, options = {}) {
   if (!Number.isInteger(expectedLimit) || expectedLimit < 1) {
     errors.push(`--limit must be a positive integer, got ${options.limit}.`);
   }
+  if (maxActiveAverageMsPerParsed !== null && (!Number.isFinite(maxActiveAverageMsPerParsed) || maxActiveAverageMsPerParsed <= 0)) {
+    errors.push(`--max-active-average-ms must be a positive finite number, got ${options.maxActiveAverageMsPerParsed}.`);
+  }
+  if (maxCaptureRoundTripOverheadMs !== null && (!Number.isFinite(maxCaptureRoundTripOverheadMs) || maxCaptureRoundTripOverheadMs < 0)) {
+    errors.push(`--max-capture-roundtrip-overhead-ms must be a non-negative finite number, got ${options.maxCaptureRoundTripOverheadMs}.`);
+  }
 
   const limitAssessment = findLimitAssessment(assessment, expectedLimit);
   if (!limitAssessment || typeof limitAssessment !== "object") {
     errors.push(`Missing limit=${expectedLimit} assessment.`);
   }
 
-  if (expectedLimit === 100 && assessment.goal100Decision !== `qualified-comparison: winner=${limitAssessment?.winnerEngine}`) {
+  if (!allowSingleEngine && expectedLimit === 100 && assessment.goal100Decision !== `qualified-comparison: winner=${limitAssessment?.winnerEngine}`) {
     errors.push(`goal100Decision is not a qualified comparison: ${assessment.goal100Decision || "<missing>"}`);
   }
 
   if (limitAssessment?.limit !== expectedLimit) {
     errors.push(`limit assessment must be ${expectedLimit}, got ${limitAssessment?.limit ?? "<missing>"}.`);
   }
-  if (limitAssessment?.comparisonComplete !== true) errors.push(`limit=${expectedLimit}.comparisonComplete must be true.`);
+  if (!allowSingleEngine && limitAssessment?.comparisonComplete !== true) errors.push(`limit=${expectedLimit}.comparisonComplete must be true.`);
   if (limitAssessment?.winnerQualified !== true) errors.push(`limit=${expectedLimit}.winnerQualified must be true.`);
   if (expectedWinner !== "any" && limitAssessment?.winnerEngine !== expectedWinner) {
     errors.push(`Expected winner '${expectedWinner}', got '${limitAssessment?.winnerEngine ?? "<missing>"}'.`);
@@ -79,8 +92,12 @@ function validateAssessment(assessment, options = {}) {
 
   const engines = Array.isArray(limitAssessment?.engines) ? limitAssessment.engines : [];
   const engineNames = new Set(engines.map((entry) => entry?.engine));
-  for (const required of ["current", "ik-traineddata"]) {
-    if (!engineNames.has(required)) errors.push(`limit=${expectedLimit} is missing engine result: ${required}.`);
+  if (!allowSingleEngine) {
+    for (const required of ["current", "ik-traineddata"]) {
+      if (!engineNames.has(required)) errors.push(`limit=${expectedLimit} is missing engine result: ${required}.`);
+    }
+  } else if (engines.length < 1) {
+    errors.push(`limit=${expectedLimit} must include at least one engine result.`);
   }
 
   const winner = engines.find((entry) => entry?.engine === limitAssessment?.winnerEngine);
@@ -93,6 +110,7 @@ function validateAssessment(assessment, options = {}) {
     const summaryWinnerReviewRate = Number(limitAssessment?.winnerReviewRate);
     const winnerActiveAverageMsPerParsed = Number(limitAssessment?.winnerActiveAverageMsPerParsed);
     const winnerActiveProjectedMsFor100 = Number(limitAssessment?.winnerActiveProjectedMsFor100);
+    const winnerAverageCaptureRoundTripOverheadMs = Number(limitAssessment?.winnerAverageCaptureRoundTripOverheadMs);
     if (winner.qualified !== true) errors.push("Winner engine result must be qualified.");
     if (!Number.isFinite(winnerMissRate)) {
       errors.push(`Winner missRate must be a finite number, got ${winner.missRate ?? "<missing>"}.`);
@@ -116,10 +134,19 @@ function validateAssessment(assessment, options = {}) {
     }
     if (!Number.isFinite(winnerActiveAverageMsPerParsed) || winnerActiveAverageMsPerParsed <= 0) {
       errors.push(`Winner active average timing must be a positive finite number, got ${limitAssessment?.winnerActiveAverageMsPerParsed ?? "<missing>"}.`);
+    } else if (maxActiveAverageMsPerParsed !== null && Number.isFinite(maxActiveAverageMsPerParsed) && winnerActiveAverageMsPerParsed > maxActiveAverageMsPerParsed) {
+      errors.push(`Winner active average timing exceeds ${maxActiveAverageMsPerParsed}ms: ${winnerActiveAverageMsPerParsed}.`);
     }
     if (!Number.isFinite(winnerActiveProjectedMsFor100) || winnerActiveProjectedMsFor100 <= 0) {
       errors.push(`Winner projected100 timing must be a positive finite number, got ${limitAssessment?.winnerActiveProjectedMsFor100 ?? "<missing>"}.`);
     }
+    if (maxCaptureRoundTripOverheadMs !== null && Number.isFinite(maxCaptureRoundTripOverheadMs)) {
+      if (!Number.isFinite(winnerAverageCaptureRoundTripOverheadMs)) {
+        errors.push(`Winner capture roundtrip overhead must be a finite number, got ${limitAssessment?.winnerAverageCaptureRoundTripOverheadMs ?? "<missing>"}.`);
+      } else if (winnerAverageCaptureRoundTripOverheadMs > maxCaptureRoundTripOverheadMs) {
+        errors.push(`Winner capture roundtrip overhead exceeds ${maxCaptureRoundTripOverheadMs}ms: ${winnerAverageCaptureRoundTripOverheadMs}.`);
+      }
+    }
   }
 
   return {
@@ -130,6 +157,8 @@ function validateAssessment(assessment, options = {}) {
     winnerEngine: limitAssessment?.winnerEngine,
     winnerActiveAverageMsPerParsed: limitAssessment?.winnerActiveAverageMsPerParsed,
     winnerActiveProjectedMsFor100: limitAssessment?.winnerActiveProjectedMsFor100,
+    winnerAverageCaptureRoundTripMs: limitAssessment?.winnerAverageCaptureRoundTripMs,
+    winnerAverageCaptureRoundTripOverheadMs: limitAssessment?.winnerAverageCaptureRoundTripOverheadMs,
     winnerMissRate: limitAssessment?.winnerMissRate,
     winnerReviewRate: limitAssessment?.winnerReviewRate,
   };
@@ -145,6 +174,8 @@ function formatSummary(result) {
     `winner: ${result.winnerEngine || "unknown"}`,
     `activeAvg: ${result.winnerActiveAverageMsPerParsed ?? "unknown"}ms/artifact`,
     `projected100: ${result.winnerActiveProjectedMsFor100 ?? "unknown"}ms`,
+    `captureRoundTrip: ${result.winnerAverageCaptureRoundTripMs ?? "unknown"}ms`,
+    `captureRoundTripOverhead: ${result.winnerAverageCaptureRoundTripOverheadMs ?? "unknown"}ms`,
     `missRate: ${result.winnerMissRate ?? "unknown"}`,
     `reviewRate: ${result.winnerReviewRate ?? "unknown"}`,
   ];
@@ -165,7 +196,15 @@ function main() {
   }
   const limit = Number(argValue("limit", "100"));
   const assessment = loadAssessment(inputPath);
-  const result = validateAssessment(assessment, { expectedWinner, limit });
+  const maxActiveAverageMsPerParsed = argValue("max-active-average-ms", "");
+  const maxCaptureRoundTripOverheadMs = argValue("max-capture-roundtrip-overhead-ms", "");
+  const result = validateAssessment(assessment, {
+    expectedWinner,
+    limit,
+    allowSingleEngine: hasFlag("allow-single-engine"),
+    maxActiveAverageMsPerParsed: maxActiveAverageMsPerParsed === "" ? undefined : maxActiveAverageMsPerParsed,
+    maxCaptureRoundTripOverheadMs: maxCaptureRoundTripOverheadMs === "" ? undefined : maxCaptureRoundTripOverheadMs,
+  });
   const payload = { inputPath: path.resolve(inputPath), ...result };
   console.log(hasFlag("summary") ? formatSummary(payload) : JSON.stringify(payload, null, 2));
   if (!result.ok) process.exit(1);