feat(scanner): complete localized artifact quality checkpoint
This commit is contained in:
AzuTear
@@ -0,0 +1,411 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import { loadNativeScannerResultTombstones } from "../electron/services/nativeScannerResultTombstones.js";
|
||||
import { evaluateStoredScanResult, projectArtifactUpgrade } from "../src/lib/artifactEvaluation.js";
|
||||
import { replayNativeScanResults } from "../src/eval/nativeScanReplay.js";
|
||||
import type { ArtifactUpgradeProjection, StoredScanResultEntry } from "../src/types/storage.js";
|
||||
|
||||
type NativeRunManifest = {
|
||||
runId?: string;
|
||||
category?: string;
|
||||
target?: number;
|
||||
};
|
||||
|
||||
type NativeRunStatus = {
|
||||
scanner?: {
|
||||
status?: string;
|
||||
runId?: string;
|
||||
target?: number;
|
||||
captured?: number;
|
||||
queued?: number;
|
||||
clicked?: number;
|
||||
pages?: number;
|
||||
activeMs?: number;
|
||||
};
|
||||
};
|
||||
|
||||
type NativeProcessingReport = {
|
||||
ok?: boolean;
|
||||
processed?: number;
|
||||
review?: number;
|
||||
stored?: number;
|
||||
errors?: number;
|
||||
elapsedMs?: number;
|
||||
queueConcurrency?: number;
|
||||
persisted?: boolean;
|
||||
};
|
||||
|
||||
type NativeCaptureJob = {
|
||||
sequence?: number;
|
||||
page?: number;
|
||||
category?: string;
|
||||
absolutePath?: string;
|
||||
relativePath?: string;
|
||||
downstream?: string;
|
||||
};
|
||||
|
||||
type NativeReviewLogEntry = {
|
||||
ok?: boolean;
|
||||
action?: string;
|
||||
resultId?: string;
|
||||
};
|
||||
|
||||
type RunValidationSummary = {
|
||||
runId: string;
|
||||
runDir: string;
|
||||
target: number;
|
||||
issues: string[];
|
||||
evidence: {
|
||||
jobs: number;
|
||||
pngs: number;
|
||||
results: number;
|
||||
captured: number;
|
||||
pages: number;
|
||||
processingErrors: number;
|
||||
persisted: boolean;
|
||||
originalReview: number;
|
||||
currentReview: number;
|
||||
approvedReviewLogs: number;
|
||||
duplicatePngHashGroups: number;
|
||||
repeatedPagePairs: string[];
|
||||
identicalBoundaryPairs: string[];
|
||||
};
|
||||
performance: {
|
||||
captureMs: number;
|
||||
capturePerSecond: number | null;
|
||||
processingMs: number;
|
||||
processingPerSecond: number | null;
|
||||
};
|
||||
replay: ReturnType<typeof replayNativeScanResults>;
|
||||
};
|
||||
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const scanRoot = options.scanRoot || path.join(
|
||||
process.env.APPDATA || path.join(os.homedir(), "AppData", "Roaming"),
|
||||
"genshin-artifact-assistant",
|
||||
"native-scans",
|
||||
);
|
||||
const selectedRuns = await selectLatestRuns(scanRoot, options.targets);
|
||||
const runs: RunValidationSummary[] = [];
|
||||
|
||||
for (const target of options.targets) {
|
||||
const runDir = selectedRuns.get(target);
|
||||
if (!runDir) throw new Error(`No saved native run with target ${target} found below ${scanRoot}`);
|
||||
runs.push(await validateRun(runDir, target, options.repeats));
|
||||
}
|
||||
|
||||
const result = {
|
||||
version: "saved-native-run-validation-v1",
|
||||
createdAt: new Date().toISOString(),
|
||||
scanRoot,
|
||||
targets: options.targets,
|
||||
repeats: options.repeats,
|
||||
ok: runs.every((run) => run.issues.length === 0),
|
||||
runs,
|
||||
};
|
||||
const outputDir = path.resolve("outputs", "native-replay");
|
||||
await fs.mkdir(outputDir, { recursive: true });
|
||||
const reportPath = path.join(outputDir, "saved-native-validation-report.json");
|
||||
await fs.writeFile(reportPath, JSON.stringify(result, null, 2), "utf8");
|
||||
|
||||
console.log(`Saved native validation: ${result.ok ? "PASS" : "FAIL"}`);
|
||||
for (const run of runs) {
|
||||
const replay = run.replay;
|
||||
console.log(
|
||||
`${run.target}: ${run.runId}; capture=${formatRate(run.performance.capturePerSecond)}/s; `
|
||||
+ `processing=${formatRate(run.performance.processingPerSecond)}/s; `
|
||||
+ `evaluated=${replay.values.evaluated}; excluded=${replay.values.excluded}; `
|
||||
+ `review=${replay.values.review}; unknown=${replay.values.unknown}; `
|
||||
+ `projection=${replay.projections.available}/${replay.projections.complete}; issues=${run.issues.length}`,
|
||||
);
|
||||
for (const issue of run.issues) console.error(` - ${issue}`);
|
||||
}
|
||||
console.log(`Report: ${reportPath}`);
|
||||
if (!result.ok) process.exitCode = 1;
|
||||
|
||||
async function validateRun(runDir: string, target: number, repeats: number): Promise<RunValidationSummary> {
|
||||
const issues: string[] = [];
|
||||
const manifestPath = path.join(runDir, "manifest.json");
|
||||
const statusPath = path.join(runDir, "status.json");
|
||||
const jobsPath = path.join(runDir, "capture-jobs.jsonl");
|
||||
const resultsPath = path.join(runDir, "scan-results.json");
|
||||
const processingPath = path.join(runDir, "processing-report.json");
|
||||
const [manifest, statusPayload, jobs, rawResults, processing, reviewLogs] = await Promise.all([
|
||||
readJson<NativeRunManifest>(manifestPath),
|
||||
readJson<NativeRunStatus>(statusPath),
|
||||
readJsonLines<NativeCaptureJob>(jobsPath),
|
||||
readJson<unknown[]>(resultsPath),
|
||||
readJson<NativeProcessingReport>(processingPath),
|
||||
readOptionalJsonLines<NativeReviewLogEntry>(path.join(runDir, "review-log.jsonl")),
|
||||
]);
|
||||
const status = statusPayload.scanner ?? {};
|
||||
const results = Array.isArray(rawResults) ? rawResults.filter(isStoredScanResultEntry) : [];
|
||||
const runId = manifest.runId || path.basename(runDir);
|
||||
|
||||
expectEqual(issues, "manifest target", manifest.target, target);
|
||||
expectEqual(issues, "manifest category", manifest.category, "artifacts");
|
||||
expectEqual(issues, "status run id", status.runId, runId);
|
||||
if (!new Set(["done", "completed"]).has(status.status ?? "")) issues.push(`scanner status is ${status.status || "missing"}`);
|
||||
expectEqual(issues, "status target", status.target, target);
|
||||
expectEqual(issues, "captured count", status.captured, target);
|
||||
expectEqual(issues, "queued count", status.queued, target);
|
||||
expectEqual(issues, "job count", jobs.length, target);
|
||||
expectEqual(issues, "raw result count", rawResults.length, target);
|
||||
expectEqual(issues, "valid result count", results.length, target);
|
||||
expectEqual(issues, "processed count", processing.processed, target);
|
||||
expectEqual(issues, "processing errors", processing.errors, 0);
|
||||
expectEqual(issues, "processing persisted", processing.persisted, false);
|
||||
if (processing.ok !== true) issues.push("processing report is not ok");
|
||||
if (!Number.isFinite(processing.queueConcurrency) || (processing.queueConcurrency ?? 0) < 1) {
|
||||
issues.push("processing queue concurrency is missing or invalid");
|
||||
}
|
||||
|
||||
validateSequences(issues, "capture job", jobs.map((job) => job.sequence), target);
|
||||
validateSequences(issues, "scan result", results.map((entry) => entry.sequence), target);
|
||||
const resolvedRunDir = path.resolve(runDir);
|
||||
let pngs = 0;
|
||||
const pngHashes: Array<{ sequence: number; page: number; hash: string }> = [];
|
||||
for (const job of jobs) {
|
||||
if (job.category !== "artifacts") issues.push(`job #${job.sequence ?? "?"} has category ${job.category ?? "missing"}`);
|
||||
if (job.downstream !== "ocr-parse-store") issues.push(`job #${job.sequence ?? "?"} has unexpected downstream contract`);
|
||||
const absolutePath = typeof job.absolutePath === "string" ? path.resolve(job.absolutePath) : "";
|
||||
if (!absolutePath || !isPathInside(resolvedRunDir, absolutePath)) {
|
||||
issues.push(`job #${job.sequence ?? "?"} image escapes or misses the run directory`);
|
||||
continue;
|
||||
}
|
||||
if (path.extname(absolutePath).toLowerCase() !== ".png") {
|
||||
issues.push(`job #${job.sequence ?? "?"} image is not PNG`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const stat = await fs.stat(absolutePath);
|
||||
if (!stat.isFile() || stat.size === 0) issues.push(`job #${job.sequence ?? "?"} PNG is empty`);
|
||||
else {
|
||||
pngs += 1;
|
||||
const bytes = await fs.readFile(absolutePath);
|
||||
pngHashes.push({
|
||||
sequence: finiteNumber(job.sequence),
|
||||
page: finiteNumber(job.page),
|
||||
hash: createHash("sha256").update(bytes).digest("hex"),
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
issues.push(`job #${job.sequence ?? "?"} PNG is missing`);
|
||||
}
|
||||
}
|
||||
expectEqual(issues, "PNG count", pngs, target);
|
||||
const duplicatePngHashGroups = duplicateHashGroupCount(pngHashes);
|
||||
const repeatedPagePairs = identicalPagePairs(pngHashes);
|
||||
const identicalBoundaryPairs = identicalScrollBoundaryPairs(pngHashes);
|
||||
if (repeatedPagePairs.length > 0) {
|
||||
issues.push(`entire captured pages repeat exactly: ${repeatedPagePairs.join(", ")}`);
|
||||
}
|
||||
|
||||
for (const entry of results) {
|
||||
expectEqual(issues, `result #${entry.sequence} run id`, entry.runId, runId);
|
||||
expectEqual(issues, `result #${entry.sequence} category`, entry.category, "artifact");
|
||||
const evaluated = evaluateStoredScanResult(entry);
|
||||
if (evaluated.valueScore !== null && (evaluated.valueScore < 0 || evaluated.valueScore > 100)) {
|
||||
issues.push(`result #${entry.sequence} has score outside 0-100`);
|
||||
}
|
||||
validateProjection(issues, entry.sequence, evaluated);
|
||||
}
|
||||
|
||||
const replay = replayNativeScanResults({ entries: results, repeats, runId, sourcePath: resultsPath });
|
||||
if (!replay.deterministic) issues.push("replay payload is not deterministic");
|
||||
if (replay.cleanUnevaluated.length > 0) issues.push(`${replay.cleanUnevaluated.length} clean results remain unevaluated`);
|
||||
const currentReview = results.filter((entry) => entry.extractionStatus === "review" || entry.needsReview).length;
|
||||
const originalReview = Number(processing.review ?? 0);
|
||||
const approvedReviewLogs = reviewLogs.filter((entry) => entry.ok && entry.action === "approve" && entry.resultId).length;
|
||||
const resolvedReview = Math.max(0, originalReview - currentReview);
|
||||
if (resolvedReview > approvedReviewLogs) {
|
||||
issues.push(`${resolvedReview} processing reviews disappeared but only ${approvedReviewLogs} approvals are logged`);
|
||||
}
|
||||
|
||||
const captureMs = finiteNumber(status.activeMs);
|
||||
const processingMs = finiteNumber(processing.elapsedMs);
|
||||
return {
|
||||
runId,
|
||||
runDir,
|
||||
target,
|
||||
issues: [...new Set(issues)],
|
||||
evidence: {
|
||||
jobs: jobs.length,
|
||||
pngs,
|
||||
results: results.length,
|
||||
captured: finiteNumber(status.captured),
|
||||
pages: finiteNumber(status.pages),
|
||||
processingErrors: finiteNumber(processing.errors),
|
||||
persisted: processing.persisted === true,
|
||||
originalReview,
|
||||
currentReview,
|
||||
approvedReviewLogs,
|
||||
duplicatePngHashGroups,
|
||||
repeatedPagePairs,
|
||||
identicalBoundaryPairs,
|
||||
},
|
||||
performance: {
|
||||
captureMs,
|
||||
capturePerSecond: rate(target, captureMs),
|
||||
processingMs,
|
||||
processingPerSecond: rate(target, processingMs),
|
||||
},
|
||||
replay,
|
||||
};
|
||||
}
|
||||
|
||||
function validateProjection(issues: string[], sequence: number, entry: StoredScanResultEntry) {
|
||||
const evaluation = entry.valueEvaluation;
|
||||
if (!evaluation) {
|
||||
issues.push(`result #${sequence} has no derived evaluation`);
|
||||
return;
|
||||
}
|
||||
const projection = projectArtifactUpgrade(entry.artifact, evaluation);
|
||||
if (projection.status === "available") {
|
||||
const scores = [projection.worstScore, projection.middleScore, projection.bestScore];
|
||||
if (scores.some((score) => typeof score !== "number")) issues.push(`result #${sequence} projection misses a score`);
|
||||
else if (!(scores[0]! <= scores[1]! && scores[1]! <= scores[2]!)) issues.push(`result #${sequence} projection order is invalid`);
|
||||
if (evaluation.rarity !== 5 || (entry.artifact?.level ?? 20) >= 20 || (entry.artifact?.substats.length ?? 0) < 4) {
|
||||
issues.push(`result #${sequence} projection violates its 5-star under-level boundary`);
|
||||
}
|
||||
}
|
||||
validateProjectionScores(issues, sequence, projection);
|
||||
}
|
||||
|
||||
function validateProjectionScores(issues: string[], sequence: number, projection: ArtifactUpgradeProjection) {
|
||||
for (const score of [projection.worstScore, projection.middleScore, projection.bestScore]) {
|
||||
if (score !== null && (score < 0 || score > 100)) issues.push(`result #${sequence} projection score is outside 0-100`);
|
||||
}
|
||||
}
|
||||
|
||||
async function selectLatestRuns(scanRoot: string, targets: number[]) {
|
||||
const selected = new Map<number, string>();
|
||||
const directories = (await fs.readdir(scanRoot, { withFileTypes: true }))
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.sort()
|
||||
.reverse();
|
||||
for (const directory of directories) {
|
||||
const runDir = path.join(scanRoot, directory);
|
||||
try {
|
||||
const manifest = await readJson<NativeRunManifest>(path.join(runDir, "manifest.json"));
|
||||
const target = Number(manifest.target);
|
||||
// A user may intentionally remove a bad local row and its crop. Such a
|
||||
// run is still valid for the app UI but no longer immutable acceptance
|
||||
// evidence, so select an untouched run for validation instead.
|
||||
if ((await loadNativeScannerResultTombstones(runDir)).length > 0) continue;
|
||||
if (targets.includes(target) && !selected.has(target)) selected.set(target, runDir);
|
||||
} catch {
|
||||
// Ignore unrelated or incomplete directories; selected evidence is validated strictly below.
|
||||
}
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
|
||||
function validateSequences(issues: string[], label: string, values: Array<number | undefined>, target: number) {
|
||||
const sequences = values.filter((value): value is number => Number.isInteger(value)).sort((left, right) => left - right);
|
||||
const expected = Array.from({ length: target }, (_, index) => index + 1);
|
||||
if (sequences.length !== expected.length || sequences.some((value, index) => value !== expected[index])) {
|
||||
issues.push(`${label} sequence is not exactly 1-${target}`);
|
||||
}
|
||||
}
|
||||
|
||||
function duplicateHashGroupCount(entries: Array<{ hash: string }>) {
|
||||
const counts = new Map<string, number>();
|
||||
for (const entry of entries) counts.set(entry.hash, (counts.get(entry.hash) ?? 0) + 1);
|
||||
return [...counts.values()].filter((count) => count > 1).length;
|
||||
}
|
||||
|
||||
function identicalPagePairs(entries: Array<{ page: number; sequence: number; hash: string }>) {
|
||||
const pages = new Map<number, Array<{ sequence: number; hash: string }>>();
|
||||
for (const entry of entries) {
|
||||
const page = pages.get(entry.page) ?? [];
|
||||
page.push({ sequence: entry.sequence, hash: entry.hash });
|
||||
pages.set(entry.page, page);
|
||||
}
|
||||
const ordered = [...pages.entries()]
|
||||
.map(([page, values]) => ({ page, hashes: values.sort((left, right) => left.sequence - right.sequence).map((value) => value.hash) }))
|
||||
.sort((left, right) => left.page - right.page);
|
||||
const repeated: string[] = [];
|
||||
for (let leftIndex = 0; leftIndex < ordered.length; leftIndex += 1) {
|
||||
for (let rightIndex = leftIndex + 1; rightIndex < ordered.length; rightIndex += 1) {
|
||||
const left = ordered[leftIndex];
|
||||
const right = ordered[rightIndex];
|
||||
if (left.hashes.length > 0 && left.hashes.length === right.hashes.length && left.hashes.every((hash, index) => hash === right.hashes[index])) {
|
||||
repeated.push(`${left.page}/${right.page}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return repeated;
|
||||
}
|
||||
|
||||
function identicalScrollBoundaryPairs(entries: Array<{ sequence: number; hash: string }>) {
|
||||
const hashes = new Map(entries.map((entry) => [entry.sequence, entry.hash]));
|
||||
return [[32, 33], [64, 65], [96, 97]]
|
||||
.filter(([left, right]) => hashes.has(left) && hashes.get(left) === hashes.get(right))
|
||||
.map(([left, right]) => `${left}/${right}`);
|
||||
}
|
||||
|
||||
function isStoredScanResultEntry(value: unknown): value is StoredScanResultEntry {
|
||||
if (!value || typeof value !== "object") return false;
|
||||
const entry = value as Partial<StoredScanResultEntry>;
|
||||
return typeof entry.id === "string"
|
||||
&& typeof entry.runId === "string"
|
||||
&& Number.isFinite(entry.sequence)
|
||||
&& typeof entry.extractionStatus === "string"
|
||||
&& typeof entry.valueStatus === "string"
|
||||
&& Array.isArray(entry.notes);
|
||||
}
|
||||
|
||||
function isPathInside(root: string, candidate: string) {
|
||||
const relative = path.relative(root, candidate);
|
||||
return relative !== "" && !relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative);
|
||||
}
|
||||
|
||||
function expectEqual(issues: string[], label: string, actual: unknown, expected: unknown) {
|
||||
if (actual !== expected) issues.push(`${label}: expected ${String(expected)}, got ${String(actual)}`);
|
||||
}
|
||||
|
||||
function rate(count: number, milliseconds: number) {
|
||||
return milliseconds > 0 ? Math.round((count * 100_000 / milliseconds)) / 100 : null;
|
||||
}
|
||||
|
||||
function formatRate(value: number | null) {
|
||||
return value === null ? "n/a" : value.toFixed(2);
|
||||
}
|
||||
|
||||
function finiteNumber(value: unknown) {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
||||
}
|
||||
|
||||
async function readJson<T>(filePath: string): Promise<T> {
|
||||
return JSON.parse(await fs.readFile(filePath, "utf8")) as T;
|
||||
}
|
||||
|
||||
async function readJsonLines<T>(filePath: string): Promise<T[]> {
|
||||
return (await fs.readFile(filePath, "utf8"))
|
||||
.split(/\r?\n/)
|
||||
.filter((line) => line.trim())
|
||||
.map((line) => JSON.parse(line.replace(/^\uFEFF/, "")) as T);
|
||||
}
|
||||
|
||||
async function readOptionalJsonLines<T>(filePath: string): Promise<T[]> {
|
||||
try {
|
||||
return await readJsonLines<T>(filePath);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]) {
|
||||
const targetArg = args.find((argument) => argument.startsWith("--targets="))?.slice("--targets=".length) ?? "20,50,100";
|
||||
const targets = [...new Set(targetArg.split(",").map(Number).filter((value) => Number.isInteger(value) && value > 0))];
|
||||
if (targets.length === 0) throw new Error("--targets must contain at least one positive integer.");
|
||||
const repeatArg = Number(args.find((argument) => argument.startsWith("--repeats="))?.slice("--repeats=".length) ?? 5);
|
||||
const repeats = Number.isFinite(repeatArg) ? Math.max(2, Math.min(10, Math.round(repeatArg))) : 5;
|
||||
const scanRoot = args.find((argument) => argument.startsWith("--scan-root="))?.slice("--scan-root=".length) ?? "";
|
||||
return { targets, repeats, scanRoot: scanRoot ? path.resolve(scanRoot) : "" };
|
||||
}
|
||||
Reference in New Issue
Block a user