AzuTear
412 lines
17 KiB
TypeScript
412 lines
17 KiB
TypeScript
import fs from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { createHash } from "node:crypto";
|
|
import { loadNativeScannerResultTombstones } from "../electron/services/nativeScannerResultTombstones.js";
|
|
import { evaluateStoredScanResult, projectArtifactUpgrade } from "../src/lib/artifactEvaluation.js";
|
|
import { replayNativeScanResults } from "../src/eval/nativeScanReplay.js";
|
|
import type { ArtifactUpgradeProjection, StoredScanResultEntry } from "../src/types/storage.js";
|
|
|
|
type NativeRunManifest = {
|
|
runId?: string;
|
|
category?: string;
|
|
target?: number;
|
|
};
|
|
|
|
type NativeRunStatus = {
|
|
scanner?: {
|
|
status?: string;
|
|
runId?: string;
|
|
target?: number;
|
|
captured?: number;
|
|
queued?: number;
|
|
clicked?: number;
|
|
pages?: number;
|
|
activeMs?: number;
|
|
};
|
|
};
|
|
|
|
type NativeProcessingReport = {
|
|
ok?: boolean;
|
|
processed?: number;
|
|
review?: number;
|
|
stored?: number;
|
|
errors?: number;
|
|
elapsedMs?: number;
|
|
queueConcurrency?: number;
|
|
persisted?: boolean;
|
|
};
|
|
|
|
type NativeCaptureJob = {
|
|
sequence?: number;
|
|
page?: number;
|
|
category?: string;
|
|
absolutePath?: string;
|
|
relativePath?: string;
|
|
downstream?: string;
|
|
};
|
|
|
|
type NativeReviewLogEntry = {
|
|
ok?: boolean;
|
|
action?: string;
|
|
resultId?: string;
|
|
};
|
|
|
|
type RunValidationSummary = {
|
|
runId: string;
|
|
runDir: string;
|
|
target: number;
|
|
issues: string[];
|
|
evidence: {
|
|
jobs: number;
|
|
pngs: number;
|
|
results: number;
|
|
captured: number;
|
|
pages: number;
|
|
processingErrors: number;
|
|
persisted: boolean;
|
|
originalReview: number;
|
|
currentReview: number;
|
|
approvedReviewLogs: number;
|
|
duplicatePngHashGroups: number;
|
|
repeatedPagePairs: string[];
|
|
identicalBoundaryPairs: string[];
|
|
};
|
|
performance: {
|
|
captureMs: number;
|
|
capturePerSecond: number | null;
|
|
processingMs: number;
|
|
processingPerSecond: number | null;
|
|
};
|
|
replay: ReturnType<typeof replayNativeScanResults>;
|
|
};
|
|
|
|
const options = parseArgs(process.argv.slice(2));
|
|
const scanRoot = options.scanRoot || path.join(
|
|
process.env.APPDATA || path.join(os.homedir(), "AppData", "Roaming"),
|
|
"genshin-artifact-assistant",
|
|
"native-scans",
|
|
);
|
|
const selectedRuns = await selectLatestRuns(scanRoot, options.targets);
|
|
const runs: RunValidationSummary[] = [];
|
|
|
|
for (const target of options.targets) {
|
|
const runDir = selectedRuns.get(target);
|
|
if (!runDir) throw new Error(`No saved native run with target ${target} found below ${scanRoot}`);
|
|
runs.push(await validateRun(runDir, target, options.repeats));
|
|
}
|
|
|
|
const result = {
|
|
version: "saved-native-run-validation-v1",
|
|
createdAt: new Date().toISOString(),
|
|
scanRoot,
|
|
targets: options.targets,
|
|
repeats: options.repeats,
|
|
ok: runs.every((run) => run.issues.length === 0),
|
|
runs,
|
|
};
|
|
const outputDir = path.resolve("outputs", "native-replay");
|
|
await fs.mkdir(outputDir, { recursive: true });
|
|
const reportPath = path.join(outputDir, "saved-native-validation-report.json");
|
|
await fs.writeFile(reportPath, JSON.stringify(result, null, 2), "utf8");
|
|
|
|
console.log(`Saved native validation: ${result.ok ? "PASS" : "FAIL"}`);
|
|
for (const run of runs) {
|
|
const replay = run.replay;
|
|
console.log(
|
|
`${run.target}: ${run.runId}; capture=${formatRate(run.performance.capturePerSecond)}/s; `
|
|
+ `processing=${formatRate(run.performance.processingPerSecond)}/s; `
|
|
+ `evaluated=${replay.values.evaluated}; excluded=${replay.values.excluded}; `
|
|
+ `review=${replay.values.review}; unknown=${replay.values.unknown}; `
|
|
+ `projection=${replay.projections.available}/${replay.projections.complete}; issues=${run.issues.length}`,
|
|
);
|
|
for (const issue of run.issues) console.error(` - ${issue}`);
|
|
}
|
|
console.log(`Report: ${reportPath}`);
|
|
if (!result.ok) process.exitCode = 1;
|
|
|
|
async function validateRun(runDir: string, target: number, repeats: number): Promise<RunValidationSummary> {
|
|
const issues: string[] = [];
|
|
const manifestPath = path.join(runDir, "manifest.json");
|
|
const statusPath = path.join(runDir, "status.json");
|
|
const jobsPath = path.join(runDir, "capture-jobs.jsonl");
|
|
const resultsPath = path.join(runDir, "scan-results.json");
|
|
const processingPath = path.join(runDir, "processing-report.json");
|
|
const [manifest, statusPayload, jobs, rawResults, processing, reviewLogs] = await Promise.all([
|
|
readJson<NativeRunManifest>(manifestPath),
|
|
readJson<NativeRunStatus>(statusPath),
|
|
readJsonLines<NativeCaptureJob>(jobsPath),
|
|
readJson<unknown[]>(resultsPath),
|
|
readJson<NativeProcessingReport>(processingPath),
|
|
readOptionalJsonLines<NativeReviewLogEntry>(path.join(runDir, "review-log.jsonl")),
|
|
]);
|
|
const status = statusPayload.scanner ?? {};
|
|
const results = Array.isArray(rawResults) ? rawResults.filter(isStoredScanResultEntry) : [];
|
|
const runId = manifest.runId || path.basename(runDir);
|
|
|
|
expectEqual(issues, "manifest target", manifest.target, target);
|
|
expectEqual(issues, "manifest category", manifest.category, "artifacts");
|
|
expectEqual(issues, "status run id", status.runId, runId);
|
|
if (!new Set(["done", "completed"]).has(status.status ?? "")) issues.push(`scanner status is ${status.status || "missing"}`);
|
|
expectEqual(issues, "status target", status.target, target);
|
|
expectEqual(issues, "captured count", status.captured, target);
|
|
expectEqual(issues, "queued count", status.queued, target);
|
|
expectEqual(issues, "job count", jobs.length, target);
|
|
expectEqual(issues, "raw result count", rawResults.length, target);
|
|
expectEqual(issues, "valid result count", results.length, target);
|
|
expectEqual(issues, "processed count", processing.processed, target);
|
|
expectEqual(issues, "processing errors", processing.errors, 0);
|
|
expectEqual(issues, "processing persisted", processing.persisted, false);
|
|
if (processing.ok !== true) issues.push("processing report is not ok");
|
|
if (!Number.isFinite(processing.queueConcurrency) || (processing.queueConcurrency ?? 0) < 1) {
|
|
issues.push("processing queue concurrency is missing or invalid");
|
|
}
|
|
|
|
validateSequences(issues, "capture job", jobs.map((job) => job.sequence), target);
|
|
validateSequences(issues, "scan result", results.map((entry) => entry.sequence), target);
|
|
const resolvedRunDir = path.resolve(runDir);
|
|
let pngs = 0;
|
|
const pngHashes: Array<{ sequence: number; page: number; hash: string }> = [];
|
|
for (const job of jobs) {
|
|
if (job.category !== "artifacts") issues.push(`job #${job.sequence ?? "?"} has category ${job.category ?? "missing"}`);
|
|
if (job.downstream !== "ocr-parse-store") issues.push(`job #${job.sequence ?? "?"} has unexpected downstream contract`);
|
|
const absolutePath = typeof job.absolutePath === "string" ? path.resolve(job.absolutePath) : "";
|
|
if (!absolutePath || !isPathInside(resolvedRunDir, absolutePath)) {
|
|
issues.push(`job #${job.sequence ?? "?"} image escapes or misses the run directory`);
|
|
continue;
|
|
}
|
|
if (path.extname(absolutePath).toLowerCase() !== ".png") {
|
|
issues.push(`job #${job.sequence ?? "?"} image is not PNG`);
|
|
continue;
|
|
}
|
|
try {
|
|
const stat = await fs.stat(absolutePath);
|
|
if (!stat.isFile() || stat.size === 0) issues.push(`job #${job.sequence ?? "?"} PNG is empty`);
|
|
else {
|
|
pngs += 1;
|
|
const bytes = await fs.readFile(absolutePath);
|
|
pngHashes.push({
|
|
sequence: finiteNumber(job.sequence),
|
|
page: finiteNumber(job.page),
|
|
hash: createHash("sha256").update(bytes).digest("hex"),
|
|
});
|
|
}
|
|
} catch {
|
|
issues.push(`job #${job.sequence ?? "?"} PNG is missing`);
|
|
}
|
|
}
|
|
expectEqual(issues, "PNG count", pngs, target);
|
|
const duplicatePngHashGroups = duplicateHashGroupCount(pngHashes);
|
|
const repeatedPagePairs = identicalPagePairs(pngHashes);
|
|
const identicalBoundaryPairs = identicalScrollBoundaryPairs(pngHashes);
|
|
if (repeatedPagePairs.length > 0) {
|
|
issues.push(`entire captured pages repeat exactly: ${repeatedPagePairs.join(", ")}`);
|
|
}
|
|
|
|
for (const entry of results) {
|
|
expectEqual(issues, `result #${entry.sequence} run id`, entry.runId, runId);
|
|
expectEqual(issues, `result #${entry.sequence} category`, entry.category, "artifact");
|
|
const evaluated = evaluateStoredScanResult(entry);
|
|
if (evaluated.valueScore !== null && (evaluated.valueScore < 0 || evaluated.valueScore > 100)) {
|
|
issues.push(`result #${entry.sequence} has score outside 0-100`);
|
|
}
|
|
validateProjection(issues, entry.sequence, evaluated);
|
|
}
|
|
|
|
const replay = replayNativeScanResults({ entries: results, repeats, runId, sourcePath: resultsPath });
|
|
if (!replay.deterministic) issues.push("replay payload is not deterministic");
|
|
if (replay.cleanUnevaluated.length > 0) issues.push(`${replay.cleanUnevaluated.length} clean results remain unevaluated`);
|
|
const currentReview = results.filter((entry) => entry.extractionStatus === "review" || entry.needsReview).length;
|
|
const originalReview = Number(processing.review ?? 0);
|
|
const approvedReviewLogs = reviewLogs.filter((entry) => entry.ok && entry.action === "approve" && entry.resultId).length;
|
|
const resolvedReview = Math.max(0, originalReview - currentReview);
|
|
if (resolvedReview > approvedReviewLogs) {
|
|
issues.push(`${resolvedReview} processing reviews disappeared but only ${approvedReviewLogs} approvals are logged`);
|
|
}
|
|
|
|
const captureMs = finiteNumber(status.activeMs);
|
|
const processingMs = finiteNumber(processing.elapsedMs);
|
|
return {
|
|
runId,
|
|
runDir,
|
|
target,
|
|
issues: [...new Set(issues)],
|
|
evidence: {
|
|
jobs: jobs.length,
|
|
pngs,
|
|
results: results.length,
|
|
captured: finiteNumber(status.captured),
|
|
pages: finiteNumber(status.pages),
|
|
processingErrors: finiteNumber(processing.errors),
|
|
persisted: processing.persisted === true,
|
|
originalReview,
|
|
currentReview,
|
|
approvedReviewLogs,
|
|
duplicatePngHashGroups,
|
|
repeatedPagePairs,
|
|
identicalBoundaryPairs,
|
|
},
|
|
performance: {
|
|
captureMs,
|
|
capturePerSecond: rate(target, captureMs),
|
|
processingMs,
|
|
processingPerSecond: rate(target, processingMs),
|
|
},
|
|
replay,
|
|
};
|
|
}
|
|
|
|
function validateProjection(issues: string[], sequence: number, entry: StoredScanResultEntry) {
|
|
const evaluation = entry.valueEvaluation;
|
|
if (!evaluation) {
|
|
issues.push(`result #${sequence} has no derived evaluation`);
|
|
return;
|
|
}
|
|
const projection = projectArtifactUpgrade(entry.artifact, evaluation);
|
|
if (projection.status === "available") {
|
|
const scores = [projection.worstScore, projection.middleScore, projection.bestScore];
|
|
if (scores.some((score) => typeof score !== "number")) issues.push(`result #${sequence} projection misses a score`);
|
|
else if (!(scores[0]! <= scores[1]! && scores[1]! <= scores[2]!)) issues.push(`result #${sequence} projection order is invalid`);
|
|
if (evaluation.rarity !== 5 || (entry.artifact?.level ?? 20) >= 20 || (entry.artifact?.substats.length ?? 0) < 4) {
|
|
issues.push(`result #${sequence} projection violates its 5-star under-level boundary`);
|
|
}
|
|
}
|
|
validateProjectionScores(issues, sequence, projection);
|
|
}
|
|
|
|
function validateProjectionScores(issues: string[], sequence: number, projection: ArtifactUpgradeProjection) {
|
|
for (const score of [projection.worstScore, projection.middleScore, projection.bestScore]) {
|
|
if (score !== null && (score < 0 || score > 100)) issues.push(`result #${sequence} projection score is outside 0-100`);
|
|
}
|
|
}
|
|
|
|
async function selectLatestRuns(scanRoot: string, targets: number[]) {
|
|
const selected = new Map<number, string>();
|
|
const directories = (await fs.readdir(scanRoot, { withFileTypes: true }))
|
|
.filter((entry) => entry.isDirectory())
|
|
.map((entry) => entry.name)
|
|
.sort()
|
|
.reverse();
|
|
for (const directory of directories) {
|
|
const runDir = path.join(scanRoot, directory);
|
|
try {
|
|
const manifest = await readJson<NativeRunManifest>(path.join(runDir, "manifest.json"));
|
|
const target = Number(manifest.target);
|
|
// A user may intentionally remove a bad local row and its crop. Such a
|
|
// run is still valid for the app UI but no longer immutable acceptance
|
|
// evidence, so select an untouched run for validation instead.
|
|
if ((await loadNativeScannerResultTombstones(runDir)).length > 0) continue;
|
|
if (targets.includes(target) && !selected.has(target)) selected.set(target, runDir);
|
|
} catch {
|
|
// Ignore unrelated or incomplete directories; selected evidence is validated strictly below.
|
|
}
|
|
}
|
|
return selected;
|
|
}
|
|
|
|
function validateSequences(issues: string[], label: string, values: Array<number | undefined>, target: number) {
|
|
const sequences = values.filter((value): value is number => Number.isInteger(value)).sort((left, right) => left - right);
|
|
const expected = Array.from({ length: target }, (_, index) => index + 1);
|
|
if (sequences.length !== expected.length || sequences.some((value, index) => value !== expected[index])) {
|
|
issues.push(`${label} sequence is not exactly 1-${target}`);
|
|
}
|
|
}
|
|
|
|
function duplicateHashGroupCount(entries: Array<{ hash: string }>) {
|
|
const counts = new Map<string, number>();
|
|
for (const entry of entries) counts.set(entry.hash, (counts.get(entry.hash) ?? 0) + 1);
|
|
return [...counts.values()].filter((count) => count > 1).length;
|
|
}
|
|
|
|
function identicalPagePairs(entries: Array<{ page: number; sequence: number; hash: string }>) {
|
|
const pages = new Map<number, Array<{ sequence: number; hash: string }>>();
|
|
for (const entry of entries) {
|
|
const page = pages.get(entry.page) ?? [];
|
|
page.push({ sequence: entry.sequence, hash: entry.hash });
|
|
pages.set(entry.page, page);
|
|
}
|
|
const ordered = [...pages.entries()]
|
|
.map(([page, values]) => ({ page, hashes: values.sort((left, right) => left.sequence - right.sequence).map((value) => value.hash) }))
|
|
.sort((left, right) => left.page - right.page);
|
|
const repeated: string[] = [];
|
|
for (let leftIndex = 0; leftIndex < ordered.length; leftIndex += 1) {
|
|
for (let rightIndex = leftIndex + 1; rightIndex < ordered.length; rightIndex += 1) {
|
|
const left = ordered[leftIndex];
|
|
const right = ordered[rightIndex];
|
|
if (left.hashes.length > 0 && left.hashes.length === right.hashes.length && left.hashes.every((hash, index) => hash === right.hashes[index])) {
|
|
repeated.push(`${left.page}/${right.page}`);
|
|
}
|
|
}
|
|
}
|
|
return repeated;
|
|
}
|
|
|
|
function identicalScrollBoundaryPairs(entries: Array<{ sequence: number; hash: string }>) {
|
|
const hashes = new Map(entries.map((entry) => [entry.sequence, entry.hash]));
|
|
return [[32, 33], [64, 65], [96, 97]]
|
|
.filter(([left, right]) => hashes.has(left) && hashes.get(left) === hashes.get(right))
|
|
.map(([left, right]) => `${left}/${right}`);
|
|
}
|
|
|
|
function isStoredScanResultEntry(value: unknown): value is StoredScanResultEntry {
|
|
if (!value || typeof value !== "object") return false;
|
|
const entry = value as Partial<StoredScanResultEntry>;
|
|
return typeof entry.id === "string"
|
|
&& typeof entry.runId === "string"
|
|
&& Number.isFinite(entry.sequence)
|
|
&& typeof entry.extractionStatus === "string"
|
|
&& typeof entry.valueStatus === "string"
|
|
&& Array.isArray(entry.notes);
|
|
}
|
|
|
|
function isPathInside(root: string, candidate: string) {
|
|
const relative = path.relative(root, candidate);
|
|
return relative !== "" && !relative.startsWith(`..${path.sep}`) && relative !== ".." && !path.isAbsolute(relative);
|
|
}
|
|
|
|
function expectEqual(issues: string[], label: string, actual: unknown, expected: unknown) {
|
|
if (actual !== expected) issues.push(`${label}: expected ${String(expected)}, got ${String(actual)}`);
|
|
}
|
|
|
|
function rate(count: number, milliseconds: number) {
|
|
return milliseconds > 0 ? Math.round((count * 100_000 / milliseconds)) / 100 : null;
|
|
}
|
|
|
|
function formatRate(value: number | null) {
|
|
return value === null ? "n/a" : value.toFixed(2);
|
|
}
|
|
|
|
function finiteNumber(value: unknown) {
|
|
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
}
|
|
|
|
async function readJson<T>(filePath: string): Promise<T> {
|
|
return JSON.parse(await fs.readFile(filePath, "utf8")) as T;
|
|
}
|
|
|
|
async function readJsonLines<T>(filePath: string): Promise<T[]> {
|
|
return (await fs.readFile(filePath, "utf8"))
|
|
.split(/\r?\n/)
|
|
.filter((line) => line.trim())
|
|
.map((line) => JSON.parse(line.replace(/^\uFEFF/, "")) as T);
|
|
}
|
|
|
|
async function readOptionalJsonLines<T>(filePath: string): Promise<T[]> {
|
|
try {
|
|
return await readJsonLines<T>(filePath);
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function parseArgs(args: string[]) {
|
|
const targetArg = args.find((argument) => argument.startsWith("--targets="))?.slice("--targets=".length) ?? "20,50,100";
|
|
const targets = [...new Set(targetArg.split(",").map(Number).filter((value) => Number.isInteger(value) && value > 0))];
|
|
if (targets.length === 0) throw new Error("--targets must contain at least one positive integer.");
|
|
const repeatArg = Number(args.find((argument) => argument.startsWith("--repeats="))?.slice("--repeats=".length) ?? 5);
|
|
const repeats = Number.isFinite(repeatArg) ? Math.max(2, Math.min(10, Math.round(repeatArg))) : 5;
|
|
const scanRoot = args.find((argument) => argument.startsWith("--scan-root="))?.slice("--scan-root=".length) ?? "";
|
|
return { targets, repeats, scanRoot: scanRoot ? path.resolve(scanRoot) : "" };
|
|
}
|