bao/genshin-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(corpus): require retrievable visual review evidence

Browse Source
This commit is contained in:
AzuTear
2026-07-11 16:13:02 +02:00
parent 8b9f948c6b
commit 3e2ab61c97
14 changed files with 325 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/eval/prepareConfirmedReviewCaseScript.test.ts
+43 -2
View File
@@ -4,8 +4,9 @@ import { tmpdir } from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
function writeCandidatePayload(dir: string) {
function writeCandidatePayload(dir: string, visualEvidenceStatus: "available" | "unavailable" = "available") {
const inputPath = path.join(dir, "review-eval-candidates.json");
const localCropPath = path.join(dir, "private-artifact-crop.png");
writeFileSync(
inputPath,
JSON.stringify({
@@ -15,7 +16,16 @@ function writeCandidatePayload(dir: string) {
id: "review-2026-07-08T15-02-39-765Z-152",
savedAt: "2026-07-08T15:02:39.765Z",
reason: "automatic:parser-notes:p1:r2c3",
resolution: "1920x1080",
resolution: localCropPath,
captureId: "20260711-154827:10",
capturedAt: "2026-07-11T15:48:28.000Z",
capturePath: localCropPath,
visualEvidence: {
status: visualEvidenceStatus,
source: visualEvidenceStatus === "available" ? "local-crop" : "none",
reference: "20260711-154827:10",
localPath: localCropPath,
},
ocr: {
"artifact-name": "Pristine Plume of the Blessed",
"artifact-slot": "Plume of Death",
@@ -67,6 +77,10 @@ describe("prepare confirmed review case script", () => {
expect(snippet).toContain("Pristine Plume of the Blessed");
expect(snippet).toContain('"artifact-name": "Pristine Plume of the Blessed"');
expect(snippet).toContain("sourceCandidate=review-2026-07-08T15-02-39-765Z-152");
expect(snippet).toContain("visualEvidence=available");
expect(snippet).toContain("captureId=20260711-154827:10");
expect(snippet).toContain("capturedAt=2026-07-11T15:48:28.000Z");
expect(snippet).not.toContain(path.join(dir, "private-artifact-crop.png"));
expect(snippet).not.toContain("confirmed: false");
} finally {
rmSync(dir, { recursive: true, force: true });
@@ -88,4 +102,31 @@ describe("prepare confirmed review case script", () => {
rmSync(dir, { recursive: true, force: true });
}
});
it("rejects candidates without retrievable visual evidence", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-confirmed-review-"));
try {
const inputPath = writeCandidatePayload(dir, "unavailable");
let thrown: unknown;
try {
execFileSync(
"node",
[
"scripts/prepare-confirmed-review-case.cjs",
`--input=${inputPath}`,
"--candidate=review-2026-07-08T15-02-39-765Z-152",
'--expect-json={"name":"Pristine Plume of the Blessed"}',
],
{ cwd: process.cwd(), stdio: "pipe" },
);
} catch (error) {
thrown = error;
}
expect(thrown).toBeDefined();
const stderr = (thrown as { stderr?: Buffer | string }).stderr;
expect(String(stderr)).toContain("retrievable visual evidence");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
});
src/eval/reviewEvalCandidatesScript.test.ts
+68 -1
View File
@@ -4,12 +4,19 @@ import { tmpdir } from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
function reviewRecord(savedAt: string, reason: string, ocr: Array<{ id: string; text: string }>, locked?: boolean) {
function reviewRecord(
savedAt: string,
reason: string,
ocr: Array<{ id: string; text: string }>,
locked?: boolean,
captureDetails: Record<string, unknown> = {},
) {
return {
savedAt,
sample: {
reason,
capture: {
...captureDetails,
width: 1920,
height: 1080,
ocr,
@@ -74,7 +81,10 @@ describe("review eval candidate exporter", () => {
expect(payload.summary.exportStats.likelyStaleCaptures).toBe(1);
expect(payload.summary.exportStats.equippedFooterCandidates).toBe(2);
expect(payload.summary.exportStats.lockedTrueCandidates).toBe(1);
expect(payload.summary.exportStats.visualEvidenceAvailable).toBe(0);
expect(payload.summary.exportStats.visualEvidenceUnavailable).toBe(2);
expect(payload.candidates[0].missingFastFields).toEqual([]);
expect(payload.candidates[0].visualEvidence.status).toBe("unavailable");
expect(payload.candidates[0].parsed.equipped).toBe("Aino");
expect(payload.candidates[0].reviewPrompt.expectedFields.equipped).toBe("Aino");
expect(payload.candidates[1].likelyStaleCapture).toBe(true);
@@ -83,6 +93,63 @@ describe("review eval candidate exporter", () => {
expect(markdown).toContain("- artifact-slot: 1");
expect(markdown).toContain("equipped=Aino");
expect(markdown).toContain("Locked=true candidates: 1");
expect(markdown).toContain("Visual evidence unavailable: 2");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
it("prioritizes retrievable local crops and does not export their local paths", () => {
const dir = mkdtempSync(path.join(tmpdir(), "gaa-review-evidence-"));
try {
const inputPath = path.join(dir, "review-samples.jsonl");
const outDir = path.join(dir, "out");
const cropPath = path.join(dir, "artifact-0010.png");
writeFileSync(cropPath, "png", "utf8");
const staleOcr = completeOcr.filter((entry) => entry.id !== "artifact-slot");
writeFileSync(
inputPath,
[
JSON.stringify(reviewRecord("2026-07-11T10:00:00.000Z", "automatic:low-field:p1:r0c0", completeOcr)),
JSON.stringify(reviewRecord("2026-07-11T10:01:00.000Z", "automatic:low-field:p1:r0c0", completeOcr, false, {
id: "20260711-154827:10",
name: cropPath,
capturedAt: "2026-07-11T10:01:01.000Z",
})),
JSON.stringify(reviewRecord("2026-07-11T10:02:00.000Z", "automatic:capture-rejected:p1:r0c1", staleOcr, false, {
id: "20260711-154827:11",
name: cropPath,
capturedAt: "2026-07-11T10:02:01.000Z",
})),
JSON.stringify(reviewRecord("2026-07-11T10:03:00.000Z", "automatic:low-field:p1:r0c2", completeOcr)),
].join("\n"),
"utf8",
);
execFileSync("node", ["scripts/export-review-eval-candidates.cjs", `--input=${inputPath}`, `--out=${outDir}`, "--limit=10"], {
cwd: process.cwd(),
stdio: "pipe",
});
const payload = JSON.parse(readFileSync(path.join(outDir, "review-eval-candidates.json"), "utf8"));
const markdown = readFileSync(path.join(outDir, "review-eval-candidates.md"), "utf8");
expect(payload.summary.uniqueCandidates).toBe(3);
expect(payload.summary.exportStats.visualEvidenceAvailable).toBe(2);
expect(payload.summary.exportStats.visualEvidenceUnavailable).toBe(1);
expect(payload.candidates.map((candidate: { captureId: string | null }) => candidate.captureId)).toEqual([
"20260711-154827:10",
"20260711-154827:11",
null,
]);
expect(payload.candidates.map((candidate: { visualEvidence: { status: string } }) => candidate.visualEvidence.status)).toEqual([
"available",
"available",
"unavailable",
]);
expect(JSON.stringify(payload)).not.toContain(cropPath);
expect(markdown).not.toContain(cropPath);
expect(markdown).toContain("Only manually label candidates with `visualEvidence: available`");
} finally {
rmSync(dir, { recursive: true, force: true });
}