bao/genshin-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve scanner repeatability guardrails

Browse Source
This commit is contained in:
AzuTear
2026-07-09 10:21:13 +02:00
parent 13fd46c104
commit c10f17b4ef
13 changed files with 356 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/eval/scanAssessmentValidatorScript.test.ts
+109 -5
View File
@@ -15,11 +15,13 @@ function validAssessment() {
winnerQualified: true,
winnerActiveAverageMsPerParsed: 820,
winnerActiveProjectedMsFor100: 82000,
winnerAverageCaptureRoundTripMs: 420,
winnerAverageCaptureRoundTripOverheadMs: 160,
winnerMissRate: 0,
winnerReviewRate: 0.04,
engines: [
{ engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.04 },
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0.06 },
{ engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.04, averageCaptureRoundTripMs: 420, averageCaptureRoundTripOverheadMs: 160 },
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0.06, averageCaptureRoundTripMs: 460, averageCaptureRoundTripOverheadMs: 190 },
],
},
limits: [
@@ -30,11 +32,13 @@ function validAssessment() {
winnerQualified: true,
winnerActiveAverageMsPerParsed: 390,
winnerActiveProjectedMsFor100: 39000,
winnerAverageCaptureRoundTripMs: 364,
winnerAverageCaptureRoundTripOverheadMs: 152,
winnerMissRate: 0,
winnerReviewRate: 0.05,
engines: [
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0.05 },
{ engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.1 },
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0.05, averageCaptureRoundTripMs: 364, averageCaptureRoundTripOverheadMs: 152 },
{ engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.1, averageCaptureRoundTripMs: 410, averageCaptureRoundTripOverheadMs: 180 },
],
},
],
@@ -94,6 +98,7 @@ describe("scan assessment validator", () => {
expect(output).toContain("limit: 100");
expect(output).toContain("winner: ik-traineddata");
expect(output).toContain("activeAvg: 820ms/artifact");
expect(output).toContain("captureRoundTripOverhead: 160ms");
} finally {
rmSync(dir, { recursive: true, force: true });
}
@@ -120,6 +125,65 @@ describe("scan assessment validator", () => {
}
});
it("accepts explicit speed and capture-overhead budgets when the winner stays inside them", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {
const inputPath = writeAssessment(dir, validAssessment());
const output = execFileSync(
"node",
[
"scripts/validate-scan-assessment.cjs",
`--input=${inputPath}`,
"--summary",
"--limit=20",
"--max-active-average-ms=400",
"--max-capture-roundtrip-overhead-ms=160",
],
{
cwd: process.cwd(),
encoding: "utf8",
},
);
expect(output).toContain("scan assessment: PASS");
expect(output).toContain("captureRoundTripOverhead: 152ms");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("rejects speed and capture-overhead budgets when the winner exceeds them", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {
const inputPath = writeAssessment(dir, validAssessment());
let stdout = "";
try {
execFileSync(
"node",
[
"scripts/validate-scan-assessment.cjs",
`--input=${inputPath}`,
"--summary",
"--limit=20",
"--max-active-average-ms=333",
"--max-capture-roundtrip-overhead-ms=120",
],
{
cwd: process.cwd(),
encoding: "utf8",
stdio: "pipe",
},
);
} catch (error) {
stdout = String((error as { stdout?: string }).stdout || "");
}
expect(stdout).toContain("scan assessment: FAIL");
expect(stdout).toContain("Winner active average timing exceeds 333ms");
expect(stdout).toContain("Winner capture roundtrip overhead exceeds 120ms");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("rejects invalid requested limits", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {
@@ -179,6 +243,12 @@ describe("scan assessment validator", () => {
expect(packageJson.scripts["scan:iterate:compare:validated:wait"]).toBe(
"npm run scan:live:preflight:wait && npm run scan:iterate:compare && npm run scan:assessment:validate -- --latest --summary --limit=20",
);
expect(packageJson.scripts["scan:repeatability"]).toBe(
"powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\live-soak.ps1 -RepeatabilityRun -ScanEngine current",
);
expect(packageJson.scripts["scan:repeatability:wait"]).toBe(
"npm run scan:live:preflight:wait && npm run scan:repeatability && npm run scan:assessment:validate -- --latest --summary --limit=100 --expect-winner=current --allow-single-engine",
);
});
it("rejects a mismatched expected winner", () => {
@@ -223,7 +293,9 @@ describe("scan assessment validator", () => {
const payload = validAssessment();
payload.goal100Decision = "not-comparable: current and ik-traineddata were not both run";
payload.goal100.comparisonComplete = false;
payload.goal100.engines = [{ engine: "current", qualified: true, missRate: 0, reviewRate: 0 }];
payload.goal100.engines = [
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0, averageCaptureRoundTripMs: 360, averageCaptureRoundTripOverheadMs: 150 },
];
const inputPath = writeAssessment(dir, payload);
expect(() =>
execFileSync("node", ["scripts/validate-scan-assessment.cjs", `--input=${inputPath}`], {
@@ -236,6 +308,38 @@ describe("scan assessment validator", () => {
}
});
it("accepts a single-engine 100-artifact repeatability run only with the explicit flag", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {
const payload = validAssessment();
payload.goal100Decision = "not-comparable: current and ik-traineddata were not both run";
payload.goal100.comparisonComplete = false;
payload.goal100.winnerEngine = "current";
payload.goal100.engines = [
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0, averageCaptureRoundTripMs: 360, averageCaptureRoundTripOverheadMs: 150 },
];
const inputPath = writeAssessment(dir, payload);
const output = execFileSync(
"node",
[
"scripts/validate-scan-assessment.cjs",
`--input=${inputPath}`,
"--summary",
"--expect-winner=current",
"--allow-single-engine",
],
{
cwd: process.cwd(),
encoding: "utf8",
},
);
expect(output).toContain("scan assessment: PASS");
expect(output).toContain("winner: current");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("rejects an unqualified winner", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {