bao/genshin-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve scanner repeatability guardrails

Browse Source
This commit is contained in:
AzuTear
2026-07-09 10:21:13 +02:00
parent 13fd46c104
commit c10f17b4ef
13 changed files with 356 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/eval/scanAssessmentValidatorScript.test.ts
+109 -5
View File
@@ -15,11 +15,13 @@ function validAssessment() {
winnerQualified: true,
winnerActiveAverageMsPerParsed: 820,
winnerActiveProjectedMsFor100: 82000,
winnerAverageCaptureRoundTripMs: 420,
winnerAverageCaptureRoundTripOverheadMs: 160,
winnerMissRate: 0,
winnerReviewRate: 0.04,
engines: [
{ engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.04 },
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0.06 },
{ engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.04, averageCaptureRoundTripMs: 420, averageCaptureRoundTripOverheadMs: 160 },
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0.06, averageCaptureRoundTripMs: 460, averageCaptureRoundTripOverheadMs: 190 },
],
},
limits: [
@@ -30,11 +32,13 @@ function validAssessment() {
winnerQualified: true,
winnerActiveAverageMsPerParsed: 390,
winnerActiveProjectedMsFor100: 39000,
winnerAverageCaptureRoundTripMs: 364,
winnerAverageCaptureRoundTripOverheadMs: 152,
winnerMissRate: 0,
winnerReviewRate: 0.05,
engines: [
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0.05 },
{ engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.1 },
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0.05, averageCaptureRoundTripMs: 364, averageCaptureRoundTripOverheadMs: 152 },
{ engine: "ik-traineddata", qualified: true, missRate: 0, reviewRate: 0.1, averageCaptureRoundTripMs: 410, averageCaptureRoundTripOverheadMs: 180 },
],
},
],
@@ -94,6 +98,7 @@ describe("scan assessment validator", () => {
expect(output).toContain("limit: 100");
expect(output).toContain("winner: ik-traineddata");
expect(output).toContain("activeAvg: 820ms/artifact");
expect(output).toContain("captureRoundTripOverhead: 160ms");
} finally {
rmSync(dir, { recursive: true, force: true });
}
@@ -120,6 +125,65 @@ describe("scan assessment validator", () => {
}
});
it("accepts explicit speed and capture-overhead budgets when the winner stays inside them", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {
const inputPath = writeAssessment(dir, validAssessment());
const output = execFileSync(
"node",
[
"scripts/validate-scan-assessment.cjs",
`--input=${inputPath}`,
"--summary",
"--limit=20",
"--max-active-average-ms=400",
"--max-capture-roundtrip-overhead-ms=160",
],
{
cwd: process.cwd(),
encoding: "utf8",
},
);
expect(output).toContain("scan assessment: PASS");
expect(output).toContain("captureRoundTripOverhead: 152ms");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("rejects speed and capture-overhead budgets when the winner exceeds them", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {
const inputPath = writeAssessment(dir, validAssessment());
let stdout = "";
try {
execFileSync(
"node",
[
"scripts/validate-scan-assessment.cjs",
`--input=${inputPath}`,
"--summary",
"--limit=20",
"--max-active-average-ms=333",
"--max-capture-roundtrip-overhead-ms=120",
],
{
cwd: process.cwd(),
encoding: "utf8",
stdio: "pipe",
},
);
} catch (error) {
stdout = String((error as { stdout?: string }).stdout || "");
}
expect(stdout).toContain("scan assessment: FAIL");
expect(stdout).toContain("Winner active average timing exceeds 333ms");
expect(stdout).toContain("Winner capture roundtrip overhead exceeds 120ms");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("rejects invalid requested limits", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {
@@ -179,6 +243,12 @@ describe("scan assessment validator", () => {
expect(packageJson.scripts["scan:iterate:compare:validated:wait"]).toBe(
"npm run scan:live:preflight:wait && npm run scan:iterate:compare && npm run scan:assessment:validate -- --latest --summary --limit=20",
);
expect(packageJson.scripts["scan:repeatability"]).toBe(
"powershell -NoProfile -ExecutionPolicy Bypass -File scripts\\live-soak.ps1 -RepeatabilityRun -ScanEngine current",
);
expect(packageJson.scripts["scan:repeatability:wait"]).toBe(
"npm run scan:live:preflight:wait && npm run scan:repeatability && npm run scan:assessment:validate -- --latest --summary --limit=100 --expect-winner=current --allow-single-engine",
);
});
it("rejects a mismatched expected winner", () => {
@@ -223,7 +293,9 @@ describe("scan assessment validator", () => {
const payload = validAssessment();
payload.goal100Decision = "not-comparable: current and ik-traineddata were not both run";
payload.goal100.comparisonComplete = false;
payload.goal100.engines = [{ engine: "current", qualified: true, missRate: 0, reviewRate: 0 }];
payload.goal100.engines = [
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0, averageCaptureRoundTripMs: 360, averageCaptureRoundTripOverheadMs: 150 },
];
const inputPath = writeAssessment(dir, payload);
expect(() =>
execFileSync("node", ["scripts/validate-scan-assessment.cjs", `--input=${inputPath}`], {
@@ -236,6 +308,38 @@ describe("scan assessment validator", () => {
}
});
it("accepts a single-engine 100-artifact repeatability run only with the explicit flag", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {
const payload = validAssessment();
payload.goal100Decision = "not-comparable: current and ik-traineddata were not both run";
payload.goal100.comparisonComplete = false;
payload.goal100.winnerEngine = "current";
payload.goal100.engines = [
{ engine: "current", qualified: true, missRate: 0, reviewRate: 0, averageCaptureRoundTripMs: 360, averageCaptureRoundTripOverheadMs: 150 },
];
const inputPath = writeAssessment(dir, payload);
const output = execFileSync(
"node",
[
"scripts/validate-scan-assessment.cjs",
`--input=${inputPath}`,
"--summary",
"--expect-winner=current",
"--allow-single-engine",
],
{
cwd: process.cwd(),
encoding: "utf8",
},
);
expect(output).toContain("scan assessment: PASS");
expect(output).toContain("winner: current");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("rejects an unqualified winner", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-assessment-"));
try {
src/features/scan/components/DiagnosticsView.tsx
+13 -13
View File
@@ -31,10 +31,10 @@ const appDiagnosisSections = [
tone: "warn",
icon: ClipboardList,
items: [
"Paimon-Menue-Einstieg ist gebaut, aber live noch nicht mit 2/20/45 Limits validiert.",
"Native/IK-Tesseract ist nur als Benchmark-Pfad vorbereitet, noch nicht Standard.",
"Positive locked=true Probe und erneuter Equipped-Footer-Livebeweis an bekannten Artifacts fehlen.",
"Empfehlungen bleiben Nebenfunktion, bis Scanner-Vertrauen und Review-Rate stabil genug sind.",
"Empfehlungen und Build-UX sind bewusst noch nicht der naechste Hauptfokus.",
"Auto-Entry-Modi wie paimon-menu und direct-inventory bleiben Dev-Control Experimente, nicht Produktionspfad.",
"3 Artifacts/Sekunde ist noch nicht bewiesen; der sichtbare Pfad liegt knapp darunter.",
"Mehr bestaetigte OCR-/Review-Corpus-Faelle fehlen, bevor wir breite Qualitaet behaupten.",
],
},
{
@@ -42,10 +42,10 @@ const appDiagnosisSections = [
tone: "risk",
icon: AlertTriangle,
items: [
"OCR ist weiterhin der Haupt-Risikofaktor; einige Felder landen noch in Fallback, Ableitung oder Review.",
"Bild-Preprocessing kann nur an echten Captures bewertet werden, nicht allein mit Text-Eval.",
"Auto-Scan braucht bei erhoehtem Genshin auch eine erhoehte App-Laufzeit.",
"Groessere Runs brauchen weiter Beobachtung auf Scroll-Uebergaenge, Wiederholseiten und Review-Quote.",
"Scanner ist stark im aktuellen sichtbaren Inventory-Pfad; Repeatability ueber spaetere Sessions ist der naechste Pruefpunkt.",
"OCR ist schnell genug fuer gute Runs, bleibt aber der groesste Qualitaets- und Speed-Hebel.",
"Capture-Roundtrip ist sichtbar: Roundtrip und Overhead muessen bei 20/45/100 Runs mit bewertet werden.",
"Groessere Runs brauchen weiter Beobachtung auf Scroll-Uebergaenge, Wiederholseiten, Duplicate-Rate und Review-Quote.",
],
},
{
@@ -53,11 +53,11 @@ const appDiagnosisSections = [
tone: "next",
icon: Target,
items: [
"Review-Corpus aus echten Samples vergroessern und mit `npm run eval` messbar halten.",
"Review-Export fuer Equipped-Footer und locked=true Kandidaten nutzen.",
"OCR-Benchmark gegen identische Crops fahren und erst danach Engine-Standard wechseln.",
"Paimon-Menue-Pfad live pruefen und bei Blockade sichtbar auf visible-inventory zurueckfallen.",
"Diagnose weiter als Operator-Cockpit halten: Live-Status, Evidenz und naechster sicherer Schritt.",
"Mehr Live-Wiederholungen: 20/45/100 Runs in spaeteren Sessions, sauber als Repeatability statt IK-Claim markieren.",
"Review-Samples sauber labeln und ueber `npm run eval:prepare-confirmed` ins Eval-Corpus uebernehmen.",
"Scanner-UI weiter beruhigen, aber den funktionierenden visible-inventory Pfad nicht umbauen.",
"Capture-Roundtrip-Overhead als eigenen Optimierungspunkt pruefen, bevor weitere OCR-Engine-Wechsel priorisiert werden.",
"Erst danach Empfehlungen wieder staerker nach vorne ziehen.",
],
},
];
src/lib/artifactOcrParser.test.ts
+16
View File
@@ -81,6 +81,22 @@ describe("parseArtifactCandidate", () => {
expect(parsed?.equipped).toBe("Aino");
});
it("derives a unique piece and set from a distinctive partial OCR fragment", () => {
const parsed = parseArtifactCandidate(captureFromOcr({
"artifact-name": "Wontiroms Creation pan",
"artifact-slot": "Plume of Death",
"artifact-main-stat-label": "ATK",
"artifact-level": "+0",
"artifact-substats": "+ HP+239\n+ Energy Recharge+4.5%\n+ Elemental Mastery+23\n- ATK+4.1% unactivated",
}));
expect(parsed?.name).toBe("Sharpness That Ceased Upon Wondrous Creation");
expect(parsed?.slot).toBe("Plume of Death");
expect(parsed?.setName).toBe("Disenchantment in Deep Shadow");
expect(parsed?.fields.name.source).toBe("fallback");
expect(parsed?.fields.name.confidence).toBe(72);
});
it("keeps goblet elemental damage main stats separate from crit substats", () => {
const parsed = parseArtifactCandidate(captureFromOcr({
"artifact-title": "Deep Gallery's Bestowed Banquet\nGoblet of Eonothem",
src/lib/artifactOcrParser.ts
+17
View File
@@ -178,6 +178,9 @@ function parseArtifactName(titleText: string): ParsedField {
if (alias) return field(alias, 96, "database");
}
const partialPiece = derivePieceFromDistinctivePartialName(titleText);
if (partialPiece) return field(partialPiece, 72, "fallback");
const knownPiece = fuzzyFindKnown(titleText, knownPieceNames, 0.72);
if (knownPiece) return field(knownPiece.value, Math.round(knownPiece.score * 100), knownPiece.score >= 0.98 ? "database" : "fallback");
@@ -294,6 +297,20 @@ function deriveSetFromPartialPieceName(text: string) {
return sets.length === 1 ? sets[0] : "";
}
function derivePieceFromDistinctivePartialName(text: string) {
const words = cleanupOcrLabel(text)
.split(/\s+/)
.map((word) => simplifyForMatch(word))
.filter((word) => word.length >= 8);
if (words.length === 0) return "";
const candidates = knownPieceNames.filter((piece) => {
const normalizedPiece = simplifyForMatch(piece);
return words.some((word) => normalizedPiece.includes(word));
});
return candidates.length === 1 ? candidates[0] : "";
}
function findMainValue(text: string, mainStat: string, slot: string, level: number | null): ParsedField {
const cleaned = text.replace(/\b20\b/g, " ").replace(/[Oo]/g, "0");
const percentValue = extractPercentValue(cleaned);
src/types/global.d.ts
Vendored
+2 -1
View File
@@ -248,7 +248,8 @@ export interface AutomationGuard {
}
export interface GdiCaptureResult {
dataUrl: string;
dataUrl?: string;
imageBase64?: string;
width: number;
height: number;
originX: number;