AzuTear
639b0b7f59
Add native IK-style capture processing, Artifact Inventory, explicit promotion and single-result review. Confirm the three live OCR corrections in the eval corpus and preserve extraction/value separation.
221 lines
11 KiB
JavaScript
221 lines
11 KiB
JavaScript
const fs = require("node:fs");
|
|
const path = require("node:path");
|
|
|
|
function argValue(name, fallback = "") {
|
|
const prefix = `--${name}=`;
|
|
const match = process.argv.find((entry) => entry.startsWith(prefix));
|
|
return match ? match.slice(prefix.length) : fallback;
|
|
}
|
|
|
|
function hasFlag(name) {
|
|
return process.argv.includes(`--${name}`);
|
|
}
|
|
|
|
function defaultAssessmentRoot() {
|
|
return path.join(process.cwd(), "outputs", "live-soak");
|
|
}
|
|
|
|
function findLatestAssessment(rootDir = defaultAssessmentRoot()) {
|
|
const resolvedRoot = path.resolve(rootDir);
|
|
if (!fs.existsSync(resolvedRoot)) throw new Error(`Assessment root not found: ${resolvedRoot}`);
|
|
const candidates = fs.readdirSync(resolvedRoot, { withFileTypes: true })
|
|
.filter((entry) => entry.isDirectory())
|
|
.map((entry) => {
|
|
const assessmentPath = path.join(resolvedRoot, entry.name, "scan-performance-assessment.json");
|
|
if (!fs.existsSync(assessmentPath)) return null;
|
|
const stat = fs.statSync(assessmentPath);
|
|
return { assessmentPath, runName: entry.name, mtimeMs: stat.mtimeMs };
|
|
})
|
|
.filter(Boolean)
|
|
.sort((left, right) => {
|
|
const mtimeDiff = right.mtimeMs - left.mtimeMs;
|
|
if (Math.abs(mtimeDiff) > 1) return mtimeDiff;
|
|
return right.runName.localeCompare(left.runName);
|
|
});
|
|
if (candidates.length === 0) throw new Error(`No scan-performance-assessment.json found under: ${resolvedRoot}`);
|
|
return candidates[0].assessmentPath;
|
|
}
|
|
|
|
function loadAssessment(inputPath) {
|
|
if (!inputPath) throw new Error("Missing assessment file. Pass --input=<scan-performance-assessment.json>.");
|
|
return JSON.parse(fs.readFileSync(path.resolve(inputPath), "utf8").replace(/^\uFEFF/, ""));
|
|
}
|
|
|
|
function findLimitAssessment(assessment, limit) {
|
|
if (limit === 100 && assessment?.goal100) return assessment.goal100;
|
|
const limits = Array.isArray(assessment?.limits) ? assessment.limits : [];
|
|
return limits.find((entry) => Number(entry?.limit) === limit) || null;
|
|
}
|
|
|
|
function validateAssessment(assessment, options = {}) {
|
|
const errors = [];
|
|
const expectedWinner = options.expectedWinner || "any";
|
|
const expectedLimit = options.limit === undefined ? 100 : Number(options.limit);
|
|
const maxActiveAverageMsPerParsed = options.maxActiveAverageMsPerParsed === undefined
|
|
? null
|
|
: Number(options.maxActiveAverageMsPerParsed);
|
|
const maxCaptureRoundTripOverheadMs = options.maxCaptureRoundTripOverheadMs === undefined
|
|
? null
|
|
: Number(options.maxCaptureRoundTripOverheadMs);
|
|
if (!assessment || typeof assessment !== "object") {
|
|
return { ok: false, errors: ["Assessment must be a JSON object."] };
|
|
}
|
|
|
|
if (!Number.isInteger(expectedLimit) || expectedLimit < 1) {
|
|
errors.push(`--limit must be a positive integer, got ${options.limit}.`);
|
|
}
|
|
if (maxActiveAverageMsPerParsed !== null && (!Number.isFinite(maxActiveAverageMsPerParsed) || maxActiveAverageMsPerParsed <= 0)) {
|
|
errors.push(`--max-active-average-ms must be a positive finite number, got ${options.maxActiveAverageMsPerParsed}.`);
|
|
}
|
|
if (maxCaptureRoundTripOverheadMs !== null && (!Number.isFinite(maxCaptureRoundTripOverheadMs) || maxCaptureRoundTripOverheadMs < 0)) {
|
|
errors.push(`--max-capture-roundtrip-overhead-ms must be a non-negative finite number, got ${options.maxCaptureRoundTripOverheadMs}.`);
|
|
}
|
|
|
|
const limitAssessment = findLimitAssessment(assessment, expectedLimit);
|
|
if (!limitAssessment || typeof limitAssessment !== "object") {
|
|
errors.push(`Missing limit=${expectedLimit} assessment.`);
|
|
}
|
|
|
|
if (expectedLimit === 100 && assessment.goal100Decision !== `qualified: winner=${limitAssessment?.winnerEngine}`) {
|
|
errors.push(`goal100Decision is not a qualified 100-artifact run: ${assessment.goal100Decision || "<missing>"}`);
|
|
}
|
|
|
|
if (limitAssessment?.limit !== expectedLimit) {
|
|
errors.push(`limit assessment must be ${expectedLimit}, got ${limitAssessment?.limit ?? "<missing>"}.`);
|
|
}
|
|
if (limitAssessment?.winnerQualified !== true) errors.push(`limit=${expectedLimit}.winnerQualified must be true.`);
|
|
if (expectedWinner !== "any" && limitAssessment?.winnerEngine !== expectedWinner) {
|
|
errors.push(`Expected winner '${expectedWinner}', got '${limitAssessment?.winnerEngine ?? "<missing>"}'.`);
|
|
}
|
|
|
|
const engines = Array.isArray(limitAssessment?.engines) ? limitAssessment.engines : [];
|
|
if (engines.length < 1) {
|
|
errors.push(`limit=${expectedLimit} must include at least one engine result.`);
|
|
}
|
|
|
|
const winner = engines.find((entry) => entry?.engine === limitAssessment?.winnerEngine);
|
|
if (!winner) {
|
|
errors.push(`Winner engine is missing from limit=${expectedLimit}.engines: ${limitAssessment?.winnerEngine ?? "<missing>"}.`);
|
|
} else {
|
|
const winnerMissRate = Number(winner.missRate);
|
|
const winnerReviewRate = Number(winner.reviewRate);
|
|
const summaryWinnerMissRate = Number(limitAssessment?.winnerMissRate);
|
|
const summaryWinnerReviewRate = Number(limitAssessment?.winnerReviewRate);
|
|
const winnerActiveAverageMsPerParsed = Number(limitAssessment?.winnerActiveAverageMsPerParsed);
|
|
const winnerActiveProjectedMsFor100 = Number(limitAssessment?.winnerActiveProjectedMsFor100);
|
|
const winnerAverageCaptureRoundTripOverheadMs = Number(limitAssessment?.winnerAverageCaptureRoundTripOverheadMs);
|
|
if (winner.qualified !== true) errors.push("Winner engine result must be qualified.");
|
|
if (!Number.isFinite(winnerMissRate)) {
|
|
errors.push(`Winner missRate must be a finite number, got ${winner.missRate ?? "<missing>"}.`);
|
|
} else if (winnerMissRate > 0.02) {
|
|
errors.push(`Winner missRate exceeds 2%: ${winner.missRate}.`);
|
|
}
|
|
if (!Number.isFinite(winnerReviewRate)) {
|
|
errors.push(`Winner reviewRate must be a finite number, got ${winner.reviewRate ?? "<missing>"}.`);
|
|
} else if (winnerReviewRate > 0.15) {
|
|
errors.push(`Winner reviewRate exceeds 15%: ${winner.reviewRate}.`);
|
|
}
|
|
if (!Number.isFinite(summaryWinnerMissRate)) {
|
|
errors.push(`Winner summary missRate must be a finite number, got ${limitAssessment?.winnerMissRate ?? "<missing>"}.`);
|
|
} else if (summaryWinnerMissRate > 0.02) {
|
|
errors.push(`Winner summary missRate exceeds 2%: ${limitAssessment.winnerMissRate}.`);
|
|
}
|
|
if (!Number.isFinite(summaryWinnerReviewRate)) {
|
|
errors.push(`Winner summary reviewRate must be a finite number, got ${limitAssessment?.winnerReviewRate ?? "<missing>"}.`);
|
|
} else if (summaryWinnerReviewRate > 0.15) {
|
|
errors.push(`Winner summary reviewRate exceeds 15%: ${limitAssessment.winnerReviewRate}.`);
|
|
}
|
|
if (!Number.isFinite(winnerActiveAverageMsPerParsed) || winnerActiveAverageMsPerParsed <= 0) {
|
|
errors.push(`Winner active average timing must be a positive finite number, got ${limitAssessment?.winnerActiveAverageMsPerParsed ?? "<missing>"}.`);
|
|
} else if (maxActiveAverageMsPerParsed !== null && Number.isFinite(maxActiveAverageMsPerParsed) && winnerActiveAverageMsPerParsed > maxActiveAverageMsPerParsed) {
|
|
errors.push(`Winner active average timing exceeds ${maxActiveAverageMsPerParsed}ms: ${winnerActiveAverageMsPerParsed}.`);
|
|
}
|
|
if (!Number.isFinite(winnerActiveProjectedMsFor100) || winnerActiveProjectedMsFor100 <= 0) {
|
|
errors.push(`Winner projected100 timing must be a positive finite number, got ${limitAssessment?.winnerActiveProjectedMsFor100 ?? "<missing>"}.`);
|
|
}
|
|
if (maxCaptureRoundTripOverheadMs !== null && Number.isFinite(maxCaptureRoundTripOverheadMs)) {
|
|
if (!Number.isFinite(winnerAverageCaptureRoundTripOverheadMs)) {
|
|
errors.push(`Winner capture roundtrip overhead must be a finite number, got ${limitAssessment?.winnerAverageCaptureRoundTripOverheadMs ?? "<missing>"}.`);
|
|
} else if (winnerAverageCaptureRoundTripOverheadMs > maxCaptureRoundTripOverheadMs) {
|
|
errors.push(`Winner capture roundtrip overhead exceeds ${maxCaptureRoundTripOverheadMs}ms: ${winnerAverageCaptureRoundTripOverheadMs}.`);
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
ok: errors.length === 0,
|
|
errors,
|
|
createdAt: assessment.createdAt || "",
|
|
limit: expectedLimit,
|
|
winnerEngine: limitAssessment?.winnerEngine,
|
|
winnerActiveAverageMsPerParsed: limitAssessment?.winnerActiveAverageMsPerParsed,
|
|
winnerActiveProjectedMsFor100: limitAssessment?.winnerActiveProjectedMsFor100,
|
|
winnerAverageCaptureRoundTripMs: limitAssessment?.winnerAverageCaptureRoundTripMs,
|
|
winnerAverageCaptureRoundTripOverheadMs: limitAssessment?.winnerAverageCaptureRoundTripOverheadMs,
|
|
winnerMissRate: limitAssessment?.winnerMissRate,
|
|
winnerReviewRate: limitAssessment?.winnerReviewRate,
|
|
};
|
|
}
|
|
|
|
function formatSummary(result) {
|
|
const status = result.ok ? "PASS" : "FAIL";
|
|
const lines = [
|
|
`scan assessment: ${status}`,
|
|
`input: ${result.inputPath || "unknown"}`,
|
|
`createdAt: ${result.createdAt || "unknown"}`,
|
|
`limit: ${result.limit ?? "unknown"}`,
|
|
`winner: ${result.winnerEngine || "unknown"}`,
|
|
`activeAvg: ${result.winnerActiveAverageMsPerParsed ?? "unknown"}ms/artifact`,
|
|
`projected100: ${result.winnerActiveProjectedMsFor100 ?? "unknown"}ms`,
|
|
`captureRoundTrip: ${result.winnerAverageCaptureRoundTripMs ?? "unknown"}ms`,
|
|
`captureRoundTripOverhead: ${result.winnerAverageCaptureRoundTripOverheadMs ?? "unknown"}ms`,
|
|
`missRate: ${result.winnerMissRate ?? "unknown"}`,
|
|
`reviewRate: ${result.winnerReviewRate ?? "unknown"}`,
|
|
];
|
|
if (!result.ok) {
|
|
lines.push("errors:");
|
|
for (const error of result.errors) lines.push(`- ${error}`);
|
|
}
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function main() {
|
|
const inputPath = hasFlag("latest")
|
|
? findLatestAssessment(argValue("root", defaultAssessmentRoot()))
|
|
: argValue("input", process.argv[2] || "");
|
|
const expectedWinner = argValue("expect-winner", "any");
|
|
if (!["any", "current"].includes(expectedWinner)) {
|
|
throw new Error("--expect-winner must be one of: any, current.");
|
|
}
|
|
const limit = Number(argValue("limit", "100"));
|
|
const assessment = loadAssessment(inputPath);
|
|
const maxActiveAverageMsPerParsed = argValue("max-active-average-ms", "");
|
|
const maxCaptureRoundTripOverheadMs = argValue("max-capture-roundtrip-overhead-ms", "");
|
|
const result = validateAssessment(assessment, {
|
|
expectedWinner,
|
|
limit,
|
|
maxActiveAverageMsPerParsed: maxActiveAverageMsPerParsed === "" ? undefined : maxActiveAverageMsPerParsed,
|
|
maxCaptureRoundTripOverheadMs: maxCaptureRoundTripOverheadMs === "" ? undefined : maxCaptureRoundTripOverheadMs,
|
|
});
|
|
const payload = { inputPath: path.resolve(inputPath), ...result };
|
|
console.log(hasFlag("summary") ? formatSummary(payload) : JSON.stringify(payload, null, 2));
|
|
if (!result.ok) process.exit(1);
|
|
}
|
|
|
|
if (require.main === module) {
|
|
try {
|
|
main();
|
|
} catch (error) {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
findLatestAssessment,
|
|
findLimitAssessment,
|
|
formatSummary,
|
|
loadAssessment,
|
|
validateAssessment,
|
|
};
|