bao/genshin-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(corpus): require retrievable visual review evidence

Browse Source
This commit is contained in:
AzuTear
2026-07-11 16:13:02 +02:00
parent 8b9f948c6b
commit 3e2ab61c97
14 changed files with 325 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/export-review-eval-candidates.cjs
+77 -11
View File
@@ -23,6 +23,43 @@ function simplifyId(value) {
.slice(0, 80) || "unknown";
}
function safeCaptureReference(value) {
const reference = typeof value === "string" ? value.trim() : "";
if (!reference || reference.length > 160 || /[\r\n]/.test(reference)) return null;
if (path.isAbsolute(reference) || /^[A-Za-z]:[\\/]/.test(reference) || reference.startsWith("\\\\")) return null;
return reference;
}
function safeCapturedAt(value) {
const capturedAt = typeof value === "string" ? value.trim() : "";
if (!capturedAt || capturedAt.length > 80 || /[\r\n]/.test(capturedAt)) return null;
return capturedAt;
}
function hasRetrievableLocalPng(value) {
if (typeof value !== "string" || path.extname(value).toLowerCase() !== ".png") return false;
try {
return fs.statSync(value).isFile();
} catch {
return false;
}
}
function visualEvidenceForCapture(capture) {
const captureId = safeCaptureReference(capture?.id);
const capturedAt = safeCapturedAt(capture?.capturedAt);
const available = hasRetrievableLocalPng(capture?.name);
return {
captureId,
capturedAt,
visualEvidence: {
status: available ? "available" : "unavailable",
source: available ? "local-crop" : "none",
reference: captureId,
},
};
}
function parsedSummary(parsed) {
if (!parsed || typeof parsed !== "object") return {};
return {
@@ -66,6 +103,7 @@ function candidateFromRecord(record, index) {
const reason = record?.sample?.reason || "missing-reason";
const savedAt = record?.savedAt || "unknown";
const capture = record?.sample?.capture || {};
const visualEvidence = visualEvidenceForCapture(capture);
const ocrFieldIds = Object.keys(ocr).sort();
const missingFastFields = REQUIRED_FAST_OCR_FIELDS.filter((field) => !ocr[field]);
const hasEquippedFooterOcr = Boolean(ocr["artifact-footer"]);
@@ -76,6 +114,9 @@ function candidateFromRecord(record, index) {
reason,
confirmed: false,
resolution: capture.width && capture.height ? `${capture.width}x${capture.height}` : "",
captureId: visualEvidence.captureId,
capturedAt: visualEvidence.capturedAt,
visualEvidence: visualEvidence.visualEvidence,
ocrFieldIds,
missingFastFields,
hasEquippedFooterOcr,
@@ -98,6 +139,9 @@ function candidateFromRecord(record, index) {
validationChecks: {
equipped: hasEquippedFooterOcr ? "Confirm character name or mark Not detected." : "No footer OCR in this sample.",
locked: locked === null ? "No lock-state payload in this sample." : `Confirm locked=${locked}.`,
visualEvidence: visualEvidence.visualEvidence.status === "available"
? "A local PNG was retrievable at export. Inspect it before confirming labels."
: "No retrievable local PNG. Do not promote this case to the confirmed corpus.",
},
},
};
@@ -128,9 +172,19 @@ function candidatePriority(candidate) {
return 5;
}
function candidateComparator(left, right) {
const visualEvidencePriority = Number(left.visualEvidence?.status !== "available")
- Number(right.visualEvidence?.status !== "available");
if (visualEvidencePriority) return visualEvidencePriority;
const priority = candidatePriority(left) - candidatePriority(right);
if (priority) return priority;
const missingDiff = (left.missingFastFields?.length || 0) - (right.missingFastFields?.length || 0);
if (missingDiff) return missingDiff;
return String(right.savedAt).localeCompare(String(left.savedAt));
}
async function readCandidates(inputPath, limit) {
const candidates = [];
const seen = new Set();
const candidatesByKey = new Map();
let total = 0;
let invalid = 0;
const rl = readline.createInterface({ input: fs.createReadStream(inputPath, { encoding: "utf8" }) });
@@ -147,16 +201,14 @@ async function readCandidates(inputPath, limit) {
const candidate = candidateFromRecord(record, total);
if (!candidate) continue;
const key = candidateKey(candidate);
if (seen.has(key)) continue;
seen.add(key);
candidates.push(candidate);
const existing = candidatesByKey.get(key);
if (!existing || candidateComparator(candidate, existing) < 0) {
candidatesByKey.set(key, candidate);
}
}
const candidates = [...candidatesByKey.values()];
candidates.sort((left, right) => {
const priority = candidatePriority(left) - candidatePriority(right);
if (priority) return priority;
const missingDiff = (left.missingFastFields?.length || 0) - (right.missingFastFields?.length || 0);
if (missingDiff) return missingDiff;
return String(right.savedAt).localeCompare(String(left.savedAt));
return candidateComparator(left, right);
});
return { total, invalid, candidates: candidates.slice(0, limit), uniqueCandidates: candidates.length };
}
@@ -174,12 +226,16 @@ function buildExportStats(candidates) {
let equippedFooterCandidates = 0;
let lockedTrueCandidates = 0;
let lockedFalseCandidates = 0;
let visualEvidenceAvailable = 0;
let visualEvidenceUnavailable = 0;
for (const candidate of candidates) {
increment(reasonCounts, candidate.reason);
if (candidate.likelyStaleCapture) likelyStaleCaptures++;
if (candidate.hasEquippedFooterOcr) equippedFooterCandidates++;
if (candidate.locked === true) lockedTrueCandidates++;
if (candidate.locked === false) lockedFalseCandidates++;
if (candidate.visualEvidence?.status === "available") visualEvidenceAvailable++;
else visualEvidenceUnavailable++;
const missingFields = candidate.missingFastFields || [];
if (missingFields.length === 0) completeFastFields++;
for (const field of missingFields) increment(missingFastFieldCounts, field);
@@ -190,6 +246,8 @@ function buildExportStats(candidates) {
equippedFooterCandidates,
lockedTrueCandidates,
lockedFalseCandidates,
visualEvidenceAvailable,
visualEvidenceUnavailable,
reasonCounts,
missingFastFieldCounts,
};
@@ -209,8 +267,10 @@ function markdownFor(summary, candidates) {
`Equipped footer candidates: ${summary.exportStats.equippedFooterCandidates}`,
`Locked=true candidates: ${summary.exportStats.lockedTrueCandidates}`,
`Locked=false candidates: ${summary.exportStats.lockedFalseCandidates}`,
`Visual evidence available: ${summary.exportStats.visualEvidenceAvailable}`,
`Visual evidence unavailable: ${summary.exportStats.visualEvidenceUnavailable}`,
"",
"These cases are not ground truth yet. Confirm or correct the expected fields before committing any case into `src/eval/corpus/`.",
"These cases are not ground truth yet. Only manually label candidates with `visualEvidence: available`; then confirm or correct the expected fields before committing any case into `src/eval/corpus/`.",
"",
"## Export stats",
"",
@@ -228,6 +288,9 @@ function markdownFor(summary, candidates) {
lines.push("");
lines.push(`- savedAt: ${candidate.savedAt}`);
lines.push(`- reason: ${candidate.reason}`);
lines.push(`- captureId: ${candidate.captureId || "unavailable"}`);
lines.push(`- capturedAt: ${candidate.capturedAt || "unavailable"}`);
lines.push(`- visualEvidence: ${candidate.visualEvidence?.status || "unavailable"} (${candidate.visualEvidence?.source || "none"})`);
lines.push(`- missingFastFields: ${candidate.missingFastFields.join(", ") || "none"}`);
lines.push(`- likelyStaleCapture: ${candidate.likelyStaleCapture ? "yes" : "no"}`);
lines.push(`- hasEquippedFooterOcr: ${candidate.hasEquippedFooterOcr ? "yes" : "no"}`);
@@ -281,8 +344,11 @@ if (require.main === module) {
module.exports = {
buildExportStats,
candidateFromRecord,
candidateComparator,
candidatePriority,
defaultReviewSamplesPath,
hasRetrievableLocalPng,
markdownFor,
readCandidates,
visualEvidenceForCapture,
};