import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { createHash } from "node:crypto"; import { loadNativeScannerResultTombstones } from "../electron/services/nativeScannerResultTombstones.js"; import { evaluateStoredScanResult, projectArtifactUpgrade } from "../src/lib/artifactEvaluation.js"; import { replayNativeScanResults } from "../src/eval/nativeScanReplay.js"; import type { ArtifactUpgradeProjection, StoredScanResultEntry } from "../src/types/storage.js"; type NativeRunManifest = { runId?: string; category?: string; target?: number; }; type NativeRunStatus = { scanner?: { status?: string; runId?: string; target?: number; captured?: number; queued?: number; clicked?: number; pages?: number; activeMs?: number; }; }; type NativeProcessingReport = { ok?: boolean; processed?: number; review?: number; stored?: number; errors?: number; elapsedMs?: number; queueConcurrency?: number; persisted?: boolean; }; type NativeCaptureJob = { sequence?: number; page?: number; category?: string; absolutePath?: string; relativePath?: string; downstream?: string; }; type NativeReviewLogEntry = { ok?: boolean; action?: string; resultId?: string; }; type RunValidationSummary = { runId: string; runDir: string; target: number; issues: string[]; evidence: { jobs: number; pngs: number; results: number; captured: number; pages: number; processingErrors: number; persisted: boolean; originalReview: number; currentReview: number; approvedReviewLogs: number; duplicatePngHashGroups: number; repeatedPagePairs: string[]; identicalBoundaryPairs: string[]; }; performance: { captureMs: number; capturePerSecond: number | null; processingMs: number; processingPerSecond: number | null; }; replay: ReturnType; }; const options = parseArgs(process.argv.slice(2)); const scanRoot = options.scanRoot || path.join( process.env.APPDATA || path.join(os.homedir(), "AppData", "Roaming"), "genshin-artifact-assistant", "native-scans", ); const selectedRuns = await selectLatestRuns(scanRoot, options.targets); const runs: RunValidationSummary[] = []; for (const target of options.targets) { const runDir = selectedRuns.get(target); if (!runDir) throw new Error(`No saved native run with target ${target} found below ${scanRoot}`); runs.push(await validateRun(runDir, target, options.repeats)); } const result = { version: "saved-native-run-validation-v1", createdAt: new Date().toISOString(), scanRoot, targets: options.targets, repeats: options.repeats, ok: runs.every((run) => run.issues.length === 0), runs, }; const outputDir = path.resolve("outputs", "native-replay"); await fs.mkdir(outputDir, { recursive: true }); const reportPath = path.join(outputDir, "saved-native-validation-report.json"); await fs.writeFile(reportPath, JSON.stringify(result, null, 2), "utf8"); console.log(`Saved native validation: ${result.ok ? "PASS" : "FAIL"}`); for (const run of runs) { const replay = run.replay; console.log( `${run.target}: ${run.runId}; capture=${formatRate(run.performance.capturePerSecond)}/s; ` + `processing=${formatRate(run.performance.processingPerSecond)}/s; ` + `evaluated=${replay.values.evaluated}; excluded=${replay.values.excluded}; ` + `review=${replay.values.review}; unknown=${replay.values.unknown}; ` + `projection=${replay.projections.available}/${replay.projections.complete}; issues=${run.issues.length}`, ); for (const issue of run.issues) console.error(` - ${issue}`); } console.log(`Report: ${reportPath}`); if (!result.ok) process.exitCode = 1; async function validateRun(runDir: string, target: number, repeats: number): Promise { const issues: string[] = []; const manifestPath = path.join(runDir, "manifest.json"); const statusPath = path.join(runDir, "status.json"); const jobsPath = path.join(runDir, "capture-jobs.jsonl"); const resultsPath = path.join(runDir, "scan-results.json"); const processingPath = path.join(runDir, "processing-report.json"); const [manifest, statusPayload, jobs, rawResults, processing, reviewLogs] = await Promise.all([ readJson(manifestPath), readJson(statusPath), readJsonLines(jobsPath), readJson(resultsPath), readJson(processingPath), readOptionalJsonLines(path.join(runDir, "review-log.jsonl")), ]); const status = statusPayload.scanner ?? {}; const results = Array.isArray(rawResults) ? rawResults.filter(isStoredScanResultEntry) : []; const runId = manifest.runId || path.basename(runDir); expectEqual(issues, "manifest target", manifest.target, target); expectEqual(issues, "manifest category", manifest.category, "artifacts"); expectEqual(issues, "status run id", status.runId, runId); if (!new Set(["done", "completed"]).has(status.status ?? "")) issues.push(`scanner status is ${status.status || "missing"}`); expectEqual(issues, "status target", status.target, target); expectEqual(issues, "captured count", status.captured, target); expectEqual(issues, "queued count", status.queued, target); expectEqual(issues, "job count", jobs.length, target); expectEqual(issues, "raw result count", rawResults.length, target); expectEqual(issues, "valid result count", results.length, target); expectEqual(issues, "processed count", processing.processed, target); expectEqual(issues, "processing errors", processing.errors, 0); expectEqual(issues, "processing persisted", processing.persisted, false); if (processing.ok !== true) issues.push("processing report is not ok"); if (!Number.isFinite(processing.queueConcurrency) || (processing.queueConcurrency ?? 0) < 1) { issues.push("processing queue concurrency is missing or invalid"); } validateSequences(issues, "capture job", jobs.map((job) => job.sequence), target); validateSequences(issues, "scan result", results.map((entry) => entry.sequence), target); const resolvedRunDir = path.resolve(runDir); let pngs = 0; const pngHashes: Array<{ sequence: number; page: number; hash: string }> = []; for (const job of jobs) { if (job.category !== "artifacts") issues.push(`job #${job.sequence ?? "?"} has category ${job.category ?? "missing"}`); if (job.downstream !== "ocr-parse-store") issues.push(`job #${job.sequence ?? "?"} has unexpected downstream contract`); const absolutePath = typeof job.absolutePath === "string" ? path.resolve(job.absolutePath) : ""; if (!absolutePath || !isPathInside(resolvedRunDir, absolutePath)) { issues.push(`job #${job.sequence ?? "?"} image escapes or misses the run directory`); continue; } if (path.extname(absolutePath).toLowerCase() !== ".png") { issues.push(`job #${job.sequence ?? "?"} image is not PNG`); continue; } try { const stat = await fs.stat(absolutePath); if (!stat.isFile() || stat.size === 0) issues.push(`job #${job.sequence ?? "?"} PNG is empty`); else { pngs += 1; const bytes = await fs.readFile(absolutePath); pngHashes.push({ sequence: finiteNumber(job.sequence), page: finiteNumber(job.page), hash: createHash("sha256").update(bytes).digest("hex"), }); } } catch { issues.push(`job #${job.sequence ?? "?"} PNG is missing`); } } expectEqual(issues, "PNG count", pngs, target); const duplicatePngHashGroups = duplicateHashGroupCount(pngHashes); const repeatedPagePairs = identicalPagePairs(pngHashes); const identicalBoundaryPairs = identicalScrollBoundaryPairs(pngHashes); if (repeatedPagePairs.length > 0) { issues.push(`entire captured pages repeat exactly: ${repeatedPagePairs.join(", ")}`); } for (const entry of results) { expectEqual(issues, `result #${entry.sequence} run id`, entry.runId, runId); expectEqual(issues, `result #${entry.sequence} category`, entry.category, "artifact"); const evaluated = evaluateStoredScanResult(entry); if (evaluated.valueScore !== null && (evaluated.valueScore < 0 || evaluated.valueScore > 100)) { issues.push(`result #${entry.sequence} has score outside 0-100`); } validateProjection(issues, entry.sequence, evaluated); } const replay = replayNativeScanResults({ entries: results, repeats, runId, sourcePath: resultsPath }); if (!replay.deterministic) issues.push("replay payload is not deterministic"); if (replay.cleanUnevaluated.length > 0) issues.push(`${replay.cleanUnevaluated.length} clean results remain unevaluated`); const currentReview = results.filter((entry) => entry.extractionStatus === "review" || entry.needsReview).length; const originalReview = Number(processing.review ?? 0); const approvedReviewLogs = reviewLogs.filter((entry) => entry.ok && entry.action === "approve" && entry.resultId).length; const resolvedReview = Math.max(0, originalReview - currentReview); if (resolvedReview > approvedReviewLogs) { issues.push(`${resolvedReview} processing reviews disappeared but only ${approvedReviewLogs} approvals are logged`); } const captureMs = finiteNumber(status.activeMs); const processingMs = finiteNumber(processing.elapsedMs); return { runId, runDir, target, issues: [...new Set(issues)], evidence: { jobs: jobs.length, pngs, results: results.length, captured: finiteNumber(status.captured), pages: finiteNumber(status.pages), processingErrors: finiteNumber(processing.errors), persisted: processing.persisted === true, originalReview, currentReview, approvedReviewLogs, duplicatePngHashGroups, repeatedPagePairs, identicalBoundaryPairs, }, performance: { captureMs, capturePerSecond: rate(target, captureMs), processingMs, processingPerSecond: rate(target, processingMs), }, replay, }; } function validateProjection(issues: string[], sequence: number, entry: StoredScanResultEntry) { const evaluation = entry.valueEvaluation; if (!evaluation) { issues.push(`result #${sequence} has no derived evaluation`); return; } const projection = projectArtifactUpgrade(entry.artifact, evaluation); if (projection.status === "available") { const scores = [projection.worstScore, projection.middleScore, projection.bestScore]; if (scores.some((score) => typeof score !== "number")) issues.push(`result #${sequence} projection misses a score`); else if (!(scores[0]! <= scores[1]! && scores[1]! <= scores[2]!)) issues.push(`result #${sequence} projection order is invalid`); if (evaluation.rarity !== 5 || (entry.artifact?.level ?? 20) >= 20 || (entry.artifact?.substats.length ?? 0) < 4) { issues.push(`result #${sequence} projection violates its 5-star under-level boundary`); } } validateProjectionScores(issues, sequence, projection); } function validateProjectionScores(issues: string[], sequence: number, projection: ArtifactUpgradeProjection) { for (const score of [projection.worstScore, projection.middleScore, projection.bestScore]) { if (score !== null && (score < 0 || score > 100)) issues.push(`result #${sequence} projection score is outside 0-100`); } } async function selectLatestRuns(scanRoot: string, targets: number[]) { const selected = new Map(); const directories = (await fs.readdir(scanRoot, { withFileTypes: true })) .filter((entry) => entry.isDirectory()) .map((entry) => entry.name) .sort() .reverse(); for (const directory of directories) { const runDir = path.join(scanRoot, directory); try { const manifest = await readJson(path.join(runDir, "manifest.json")); const target = Number(manifest.target); // A user may intentionally remove a bad local row and its crop. Such a // run is still valid for the app UI but no longer immutable acceptance // evidence, so select an untouched run for validation instead. if ((await loadNativeScannerResultTombstones(runDir)).length > 0) continue; if (targets.includes(target) && !selected.has(target)) selected.set(target, runDir); } catch { // Ignore unrelated or incomplete directories; selected evidence is validated strictly below. } } return selected; } function validateSequences(issues: string[], label: string, values: Array, target: number) { const sequences = values.filter((value): value is number => Number.isInteger(value)).sort((left, right) => left - right); const expected = Array.from({ length: target }, (_, index) => index + 1); if (sequences.length !== expected.length || sequences.some((value, index) => value !== expected[index])) { issues.push(`${label} sequence is not exactly 1-${target}`); } } function duplicateHashGroupCount(entries: Array<{ hash: string }>) { const counts = new Map(); for (const entry of entries) counts.set(entry.hash, (counts.get(entry.hash) ?? 0) + 1); return [...counts.values()].filter((count) => count > 1).length; } function identicalPagePairs(entries: Array<{ page: number; sequence: number; hash: string }>) { const pages = new Map>(); for (const entry of entries) { const page = pages.get(entry.page) ?? []; page.push({ sequence: entry.sequence, hash: entry.hash }); pages.set(entry.page, page); } const ordered = [...pages.entries()] .map(([page, values]) => ({ page, hashes: values.sort((left, right) => left.sequence - right.sequence).map((value) => value.hash) })) .sort((left, right) => left.page - right.page); const repeated: string[] = []; for (let leftIndex = 0; leftIndex < ordered.length; leftIndex += 1) { for (let rightIndex = leftIndex + 1; rightIndex < ordered.length; rightIndex += 1) { const left = ordered[leftIndex]; const right = ordered[rightIndex]; if (left.hashes.length > 0 && left.hashes.length === right.hashes.length && left.hashes.every((hash, index) => hash === right.hashes[index])) { repeated.push(`${left.page}/${right.page}`); } } } return repeated; } function identicalScrollBoundaryPairs(entries: Array<{ sequence: number; hash: string }>) { const hashes = new Map(entries.map((entry) => [entry.sequence, entry.hash])); return [[32, 33], [64, 65], [96, 97]] .filter(([left, right]) => hashes.has(left) && hashes.get(left) === hashes.get(right)) .map(([left, right]) => `${left}/${right}`); } function isStoredScanResultEntry(value: unknown): value is StoredScanResultEntry { if (!value || typeof value !== "object") return false; const entry = value as Partial; return typeof entry.id === "string" && typeof entry.runId === "string" && Number.isFinite(entry.sequence) && typeof entry.extractionStatus === "string" && typeof entry.valueStatus === "string" && Array.isArray(entry.notes); } function isPathInside(root: string, candidate: string) { const relative = path.relative(root, candidate); return relative !== "" && !relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative); } function expectEqual(issues: string[], label: string, actual: unknown, expected: unknown) { if (actual !== expected) issues.push(`${label}: expected ${String(expected)}, got ${String(actual)}`); } function rate(count: number, milliseconds: number) { return milliseconds > 0 ? Math.round((count * 100_000 / milliseconds)) / 100 : null; } function formatRate(value: number | null) { return value === null ? "n/a" : value.toFixed(2); } function finiteNumber(value: unknown) { return typeof value === "number" && Number.isFinite(value) ? value : 0; } async function readJson(filePath: string): Promise { return JSON.parse(await fs.readFile(filePath, "utf8")) as T; } async function readJsonLines(filePath: string): Promise { return (await fs.readFile(filePath, "utf8")) .split(/\r?\n/) .filter((line) => line.trim()) .map((line) => JSON.parse(line.replace(/^\uFEFF/, "")) as T); } async function readOptionalJsonLines(filePath: string): Promise { try { return await readJsonLines(filePath); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; throw error; } } function parseArgs(args: string[]) { const targetArg = args.find((argument) => argument.startsWith("--targets="))?.slice("--targets=".length) ?? "20,50,100"; const targets = [...new Set(targetArg.split(",").map(Number).filter((value) => Number.isInteger(value) && value > 0))]; if (targets.length === 0) throw new Error("--targets must contain at least one positive integer."); const repeatArg = Number(args.find((argument) => argument.startsWith("--repeats="))?.slice("--repeats=".length) ?? 5); const repeats = Number.isFinite(repeatArg) ? Math.max(2, Math.min(10, Math.round(repeatArg))) : 5; const scanRoot = args.find((argument) => argument.startsWith("--scan-root="))?.slice("--scan-root=".length) ?? ""; return { targets, repeats, scanRoot: scanRoot ? path.resolve(scanRoot) : "" }; }