feat(scanner): complete localized artifact quality checkpoint

This commit is contained in:
AzuTear
2026-07-11 15:59:19 +02:00
parent 639b0b7f59
commit 8b9f948c6b
215 changed files with 35440 additions and 7273 deletions
@@ -0,0 +1,11 @@
export function isLikelyGenshinSourceName(sourceName: string) {
const normalized = sourceName.trim().toLowerCase();
if (!normalized) return false;
return (
normalized === "genshin"
|| /\bgenshin\s*impact\b/.test(normalized)
|| normalized.includes("genshinimpact")
|| normalized.includes("yuanshen")
|| sourceName.includes("\u539f\u795e")
);
}
+13 -8
View File
@@ -166,6 +166,18 @@ export interface InputHelperService {
dispose(): void;
}
export function automationGuardFromHelperResponse(result: HelperOperationResponse): AutomationGuard {
return {
ok: true,
cursorX: typeof result.cursorX === "number" ? result.cursorX : undefined,
cursorY: typeof result.cursorY === "number" ? result.cursorY : undefined,
escapePressed: Boolean(result.escapePressed),
enterPressed: Boolean(result.enterPressed),
f9Pressed: Boolean(result.f9Pressed),
isElevated: typeof result.isElevated === "boolean" ? result.isElevated : undefined,
};
}
export function createInputHelperService(options: { userDataPath: string; exePath?: string | null }): InputHelperService {
const inputHelper = new InputHelperClient({ scriptUserDataPath: options.userDataPath, exePath: options.exePath ?? null });
@@ -273,14 +285,7 @@ export function createInputHelperService(options: { userDataPath: string; exePat
async function getAutomationGuard() {
const result = await request("cursor", {}, 4000);
return {
ok: true,
cursorX: typeof result.cursorX === "number" ? result.cursorX : undefined,
cursorY: typeof result.cursorY === "number" ? result.cursorY : undefined,
escapePressed: Boolean(result.escapePressed),
enterPressed: Boolean(result.enterPressed),
f9Pressed: Boolean(result.f9Pressed),
};
return automationGuardFromHelperResponse(result);
}
async function capturePrimaryScreenViaGdi() {
@@ -197,8 +197,12 @@ function Get-GenshinClientBounds {
}
function Find-GenshinWindow {
if ($script:genshinHwnd -ne [IntPtr]::Zero -and [Native.InputHelper]::IsWindow($script:genshinHwnd)) { return $script:genshinHwnd }
$proc = Get-Process | Where-Object { $_.ProcessName -match 'GenshinImpact|YuanShen|Genshin' -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
if ($script:genshinHwnd -ne [IntPtr]::Zero -and [Native.InputHelper]::IsWindow($script:genshinHwnd)) {
$cachedName = Get-ProcessNameFromHwnd -hwnd $script:genshinHwnd
if ($cachedName -in @('GenshinImpact', 'YuanShen')) { return $script:genshinHwnd }
$script:genshinHwnd = [IntPtr]::Zero
}
$proc = Get-Process | Where-Object { $_.ProcessName -in @('GenshinImpact', 'YuanShen') -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
if ($proc) { $script:genshinHwnd = $proc.MainWindowHandle } else { $script:genshinHwnd = [IntPtr]::Zero }
return $script:genshinHwnd
}
@@ -283,6 +287,7 @@ while ($true) {
$response.escapePressed = $state.escapePressed
$response.enterPressed = $state.enterPressed
$response.f9Pressed = $state.f9Pressed
$response.isElevated = Get-CurrentProcessElevation
}
"runtime" {
$response.isElevated = Get-CurrentProcessElevation
@@ -2,23 +2,30 @@ import fs from "node:fs/promises";
import path from "node:path";
import { pngBufferToBitmap } from "./pngBitmap.js";
import { createNativeScannerResultWorkflowService } from "./nativeScannerResultWorkflowService.js";
import { loadNativeScannerDeletedResultIds } from "./nativeScannerResultTombstones.js";
import { parseArtifactCandidate } from "../../src/lib/artifactOcrParser.js";
import { evaluateStoredScanResult } from "../../src/lib/artifactEvaluation.js";
import { toStoredArtifact } from "../../src/lib/artifactStore.js";
import { matchParsedArtifactToIk, type IkArtifactCatalog } from "../../src/lib/ikArtifactMatcher.js";
import { createStoredScanResultEntry } from "../../src/lib/scanResultEntry.js";
import { shouldFlagArtifactForReview } from "../../src/lib/scannerLearning.js";
import type {
ArtifactStoreLoadResult,
ArtifactStoreSaveResult,
CaptureResult,
NativeScannerImageLoadStatus,
NativeScannerDeleteResultStatus,
NativeScannerProcessStatus,
NativeScannerProcessingProgressStatus,
NativeScannerPromotionStatus,
NativeScannerReviewArtifactInput,
NativeScannerReviewStatus,
NativeScannerResultsLoadStatus,
NativeScannerRunTiming,
NativeScannerRunTimingPatch,
ReviewSamplePayload,
} from "../../src/types/global.js";
import type { ScanResultCategory, StoredArtifactRecord, StoredScanResultEntry } from "../../src/types/storage.js";
import type { ArtifactRarity, ScanResultCategory, StoredArtifactRecord, StoredScanResultEntry } from "../../src/types/storage.js";
export type NativeCaptureJobPayload = {
sequence: number;
@@ -29,11 +36,21 @@ export type NativeCaptureJobPayload = {
capturedAt?: string;
relativePath?: string;
absolutePath?: string;
/** Direct visual evidence from the native card crop; absent on legacy jobs. */
starCount?: number | null;
starConfidence?: number;
starSource?: string | null;
};
export interface NativeScannerProcessingService {
processRun(options?: { runDir?: string; persist?: boolean; limit?: number }): Promise<NativeScannerProcessStatus>;
loadResults(options?: { runDir?: string; limit?: number }): Promise<NativeScannerResultsLoadStatus>;
processRun(options?: {
runDir?: string;
persist?: boolean;
limit?: number;
stream?: boolean;
expectedTotal?: number;
}): Promise<NativeScannerProcessStatus>;
loadResults(options?: { runDir?: string; limit?: number; afterSequence?: number }): Promise<NativeScannerResultsLoadStatus>;
promoteResults(options: { runDir?: string; resultIds: string[] }): Promise<NativeScannerPromotionStatus>;
reviewResult(options: {
runDir?: string;
@@ -42,6 +59,11 @@ export interface NativeScannerProcessingService {
artifact?: NativeScannerReviewArtifactInput;
note?: string;
}): Promise<NativeScannerReviewStatus>;
deleteResult(options: {
runDir?: string;
resultId: string;
removeLinkedStoreRecord?: boolean;
}): Promise<NativeScannerDeleteResultStatus>;
loadImage(options: { runDir?: string; imagePath: string }): Promise<NativeScannerImageLoadStatus>;
}
@@ -50,8 +72,14 @@ interface NativeScannerProcessingServiceDependencies {
buildCaptureResult(imagePath: string, job: NativeCaptureJobPayload): Promise<CaptureResult>;
loadArtifacts?: () => Promise<ArtifactStoreLoadResult>;
saveArtifacts(records: StoredArtifactRecord[]): Promise<Pick<ArtifactStoreSaveResult, "added" | "updated"> & Partial<ArtifactStoreSaveResult>>;
removeArtifacts?: (ids: string[]) => Promise<{ ok: boolean; removed: number }>;
saveReviewSample?: (sample: ReviewSamplePayload) => Promise<{ ok: boolean }>;
loadIkArtifactCatalog?: () => Promise<IkArtifactCatalog | null>;
/** Durable timing is observational only and must never block OCR/parse work. */
recordRunTiming?: (runDir: string, patch: NativeScannerRunTimingPatch) => Promise<NativeScannerRunTiming>;
loadRunTiming?: (runDir: string) => Promise<NativeScannerRunTiming | null>;
onProgress?: (progress: NativeScannerProcessingProgressStatus) => void;
onResult?: (result: StoredScanResultEntry) => void;
}
type ProcessedNativeJob = {
@@ -61,70 +89,55 @@ type ProcessedNativeJob = {
};
const POST_CAPTURE_QUEUE_CONCURRENCY = 4;
const STREAM_QUEUE_HIGH_WATERMARK = 8;
const STREAM_READ_CHUNK_BYTES = 64 * 1024;
const STREAM_POLL_INTERVAL_MS = 40;
const STREAM_TERMINAL_DRAIN_GRACE_MS = 2_000;
const MAX_CACHED_LIVE_RUNS = 8;
type ProcessRunOptions = {
runDir?: string;
persist?: boolean;
limit?: number;
stream?: boolean;
expectedTotal?: number;
};
type NativeCaptureProducerStatus = {
running: boolean;
status: string;
target: number;
captured: number;
};
const activeProcessingRuns = new Map<string, Promise<NativeScannerProcessStatus>>();
const liveResultsByRunDir = new Map<string, StoredScanResultEntry[]>();
export function createNativeScannerProcessingService(
deps: NativeScannerProcessingServiceDependencies,
): NativeScannerProcessingService {
const resultWorkflows = createNativeScannerResultWorkflowService(deps);
const resultWorkflows = createNativeScannerResultWorkflowService({
...deps,
isProcessingRunActive: (runDir) => activeProcessingRuns.has(runDir),
});
return {
async processRun(options = {}) {
const started = Date.now();
processRun(options: ProcessRunOptions = {}) {
const runDir = deps.resolveRunDir(options.runDir);
if (!runDir) {
return emptyProcessStatus("No native scanner runDir available.");
return Promise.resolve(emptyProcessStatus("No native scanner runDir available."));
}
const active = activeProcessingRuns.get(runDir);
if (active) return active;
const { jobsPath, jobs } = await readNativeCaptureJobs(runDir);
const limit = Math.max(1, Math.min(jobs.length, Math.round(options.limit ?? jobs.length)));
const selectedJobs = jobs.slice(0, limit);
const runId = path.basename(runDir);
const ikCatalog = deps.loadIkArtifactCatalog
? await deps.loadIkArtifactCatalog().catch(() => null)
: null;
const processedJobs = await mapWithConcurrency(
selectedJobs,
POST_CAPTURE_QUEUE_CONCURRENCY,
(job) => processNativeCaptureJob({
deps,
ikCatalog,
job,
persist: Boolean(options.persist),
runDir,
runId,
}),
);
const results = processedJobs.map((entry) => entry.result);
const scanResults = processedJobs.map((entry) => entry.scanResult);
const recordsToPersist = processedJobs
.map((entry) => entry.storedRecord)
.filter((record): record is StoredArtifactRecord => Boolean(record));
let stored = 0;
if (options.persist && recordsToPersist.length > 0) {
const saved = await deps.saveArtifacts(recordsToPersist);
stored = saved.added + saved.updated;
}
const status: NativeScannerProcessStatus = {
ok: true,
runDir,
jobsPath,
reportPath: path.join(runDir, "processing-report.json"),
scanResultsPath: path.join(runDir, "scan-results.json"),
processed: results.length,
parsed: results.filter((result) => result.parsed).length,
review: results.filter((result) => result.needsReview).length,
stored,
errors: results.filter((result) => result.error).length,
elapsedMs: Date.now() - started,
queueConcurrency: Math.min(POST_CAPTURE_QUEUE_CONCURRENCY, selectedJobs.length),
persisted: Boolean(options.persist),
results,
};
await fs.writeFile(status.scanResultsPath, JSON.stringify(scanResults, null, 2), "utf8");
await fs.writeFile(status.reportPath, JSON.stringify(status, null, 2), "utf8");
return status;
replaceCachedLiveResults(runDir, []);
const runPromise = processNativeScannerRun({ deps, options, runDir })
.finally(() => {
if (activeProcessingRuns.get(runDir) === runPromise) {
activeProcessingRuns.delete(runDir);
}
});
activeProcessingRuns.set(runDir, runPromise);
return runPromise;
},
async loadResults(options = {}) {
@@ -135,20 +148,32 @@ export function createNativeScannerProcessingService(
const resultsPath = path.join(runDir, "scan-results.json");
try {
const raw = await fs.readFile(resultsPath, "utf8");
const parsed = JSON.parse(raw);
const results = Array.isArray(parsed)
? parsed.filter(isStoredScanResultEntry)
: [];
const cached = liveResultsByRunDir.get(runDir);
const rawResults = cached
? [...cached]
: await readStoredScanResults(resultsPath);
const deletedIds = await loadNativeScannerDeletedResultIds(runDir);
const results = rawResults.filter((entry) => !deletedIds.has(entry.id));
const afterSequence = Number.isFinite(options.afterSequence)
? Math.max(0, Math.round(options.afterSequence ?? 0))
: null;
const candidates = afterSequence === null
? results
: results.filter((entry) => entry.sequence > afterSequence);
const limit = Number.isFinite(options.limit)
? Math.max(1, Math.min(results.length, Math.round(options.limit ?? results.length)))
: results.length;
? Math.max(1, Math.min(candidates.length, Math.round(options.limit ?? candidates.length)))
: candidates.length;
const selectedResults = afterSequence === null
? candidates.slice(-limit)
: candidates.slice(0, limit);
const timing = await recordResultReconciliationIfComplete(deps, runDir, results);
return {
ok: true,
runDir,
path: resultsPath,
total: results.length,
results: results.slice(-limit),
results: selectedResults,
...(timing ? { timing } : {}),
};
} catch (error) {
return {
@@ -159,8 +184,21 @@ export function createNativeScannerProcessingService(
}
},
promoteResults: resultWorkflows.promoteResults,
reviewResult: resultWorkflows.reviewResult,
async promoteResults(options) {
const status = await resultWorkflows.promoteResults(options);
await refreshCachedResultsAfterMutation(deps, options.runDir);
return status;
},
async reviewResult(options) {
const status = await resultWorkflows.reviewResult(options);
await refreshCachedResultsAfterMutation(deps, options.runDir);
return status;
},
async deleteResult(options) {
const status = await resultWorkflows.deleteResult(options);
await refreshCachedResultsAfterMutation(deps, options.runDir);
return status;
},
async loadImage(options) {
const runDir = deps.resolveRunDir(options.runDir);
@@ -205,6 +243,617 @@ export function createNativeScannerProcessingService(
};
}
async function processNativeScannerRun({
deps,
options,
runDir,
}: {
deps: NativeScannerProcessingServiceDependencies;
options: ProcessRunOptions;
runDir: string;
}) {
await updateRunTiming(deps, runDir, { processingStartedAt: new Date().toISOString() });
return options.stream
? processStreamingNativeScannerRun({ deps, options, runDir })
: processBatchNativeScannerRun({ deps, options, runDir });
}
async function processBatchNativeScannerRun({
deps,
options,
runDir,
}: {
deps: NativeScannerProcessingServiceDependencies;
options: ProcessRunOptions;
runDir: string;
}): Promise<NativeScannerProcessStatus> {
const started = Date.now();
const { jobsPath, jobs } = await readNativeCaptureJobs(runDir);
const requestedLimit = Number.isFinite(options.limit)
? Math.max(1, Math.round(options.limit ?? jobs.length))
: jobs.length;
const selectedJobs = jobs.slice(0, Math.min(jobs.length, requestedLimit));
const runId = path.basename(runDir);
const ikCatalog = deps.loadIkArtifactCatalog
? await deps.loadIkArtifactCatalog().catch(() => null)
: null;
let processedCount = 0;
let parsedCount = 0;
let reviewCount = 0;
let errorCount = 0;
const publishProgress = (running: boolean, stored = 0) => safePublishProgress(deps, {
running,
runDir,
total: selectedJobs.length,
processed: processedCount,
parsed: parsedCount,
review: reviewCount,
stored,
errors: errorCount,
elapsedMs: Date.now() - started,
});
publishProgress(true);
const processedJobs = await mapWithConcurrency(
selectedJobs,
POST_CAPTURE_QUEUE_CONCURRENCY,
async (job) => {
const processedJob = await processNativeCaptureJob({
deps,
ikCatalog,
job,
persist: Boolean(options.persist),
runDir,
runId,
});
processedCount += 1;
if (processedJob.result.parsed) parsedCount += 1;
if (processedJob.result.needsReview) reviewCount += 1;
if (processedJob.result.error) errorCount += 1;
publishProgress(true);
return processedJob;
},
);
const scanResults = processedJobs.map((entry) => entry.scanResult);
replaceCachedLiveResults(runDir, scanResults);
for (const result of scanResults) safePublishResult(deps, result);
return finalizeProcessedRun({
deps,
jobsPath,
ok: true,
options,
processedJobs,
runDir,
started,
});
}
async function processStreamingNativeScannerRun({
deps,
options,
runDir,
}: {
deps: NativeScannerProcessingServiceDependencies;
options: ProcessRunOptions;
runDir: string;
}): Promise<NativeScannerProcessStatus> {
const started = Date.now();
const runId = path.basename(runDir);
const jobsPath = path.join(runDir, "capture-jobs.jsonl");
const reportPath = path.join(runDir, "processing-report.json");
const scanResultsPath = path.join(runDir, "scan-results.json");
const streamLimit = Number.isFinite(options.limit)
? Math.max(1, Math.round(options.limit ?? 1))
: Number.POSITIVE_INFINITY;
const configuredTotal = Number.isFinite(options.expectedTotal)
? Math.max(0, Math.round(options.expectedTotal ?? 0))
: Number.isFinite(options.limit)
? streamLimit
: 0;
const ikCatalog = deps.loadIkArtifactCatalog
? await deps.loadIkArtifactCatalog().catch(() => null)
: null;
const tailState: NativeCaptureJobTailState = { offset: 0 };
const pendingJobs: NativeCaptureJobPayload[] = [];
const orderedSequences: number[] = [];
const seenJobsBySequence = new Map<number, string>();
const activeJobs = new Map<number, Promise<void>>();
const completedBySequence = new Map<number, ProcessedNativeJob>();
const publishedJobs: ProcessedNativeJob[] = [];
let nextPublishIndex = 0;
let processedCount = 0;
let parsedCount = 0;
let reviewCount = 0;
let errorCount = 0;
let producerStatus: NativeCaptureProducerStatus | null = null;
let terminalObservedAt = 0;
let integrityError = "";
let snapshotWriteQueue = Promise.resolve();
const progressTotal = () => {
if (producerStatus && !producerStatus.running) {
return Math.min(producerStatus.captured, streamLimit);
}
return configuredTotal
|| producerStatus?.target
|| Math.min(orderedSequences.length, streamLimit);
};
const publishProgress = (running: boolean, stored = 0) => safePublishProgress(deps, {
running,
runDir,
total: progressTotal(),
processed: processedCount,
parsed: parsedCount,
review: reviewCount,
stored,
errors: errorCount + (integrityError ? 1 : 0),
elapsedMs: Date.now() - started,
...(integrityError ? { error: integrityError } : {}),
});
const queueDurableSnapshot = () => {
const snapshot = [...publishedJobs];
const status = buildProcessStatus({
jobsPath,
ok: true,
options,
processedJobs: snapshot,
runDir,
started,
stored: 0,
});
snapshotWriteQueue = snapshotWriteQueue
.catch(() => undefined)
.then(() => writeProcessingOutputs({ reportPath, scanResultsPath, status, processedJobs: snapshot }));
};
const flushOrderedResults = () => {
let published = false;
while (nextPublishIndex < orderedSequences.length) {
const sequence = orderedSequences[nextPublishIndex];
const completed = completedBySequence.get(sequence);
if (!completed) break;
completedBySequence.delete(sequence);
nextPublishIndex += 1;
publishedJobs.push(completed);
appendCachedLiveResult(runDir, completed.scanResult);
safePublishResult(deps, completed.scanResult);
published = true;
}
if (published && (publishedJobs.length === 1 || publishedJobs.length % 8 === 0)) {
queueDurableSnapshot();
}
};
const scheduleAvailableJobs = () => {
while (activeJobs.size < POST_CAPTURE_QUEUE_CONCURRENCY && pendingJobs.length > 0) {
const job = pendingJobs.shift();
if (!job) break;
const task = processNativeCaptureJob({
deps,
ikCatalog,
job,
persist: Boolean(options.persist),
runDir,
runId,
}).then((processedJob) => {
processedCount += 1;
if (processedJob.result.parsed) parsedCount += 1;
if (processedJob.result.needsReview) reviewCount += 1;
if (processedJob.result.error) errorCount += 1;
completedBySequence.set(job.sequence, processedJob);
flushOrderedResults();
publishProgress(true);
}).finally(() => {
activeJobs.delete(job.sequence);
});
activeJobs.set(job.sequence, task);
}
};
publishProgress(true);
while (true) {
const queueCapacity = Math.max(
0,
STREAM_QUEUE_HIGH_WATERMARK - pendingJobs.length - activeJobs.size,
);
const appendedJobs = queueCapacity > 0
? await readAppendedNativeCaptureJobs(jobsPath, tailState, queueCapacity)
: [];
for (const job of appendedJobs) {
const identity = nativeCaptureJobIdentity(job);
const existingIdentity = seenJobsBySequence.get(job.sequence);
if (existingIdentity) {
if (existingIdentity !== identity && !integrityError) {
integrityError = `Conflicting capture jobs use sequence ${job.sequence}.`;
}
continue;
}
if (orderedSequences.length >= streamLimit) continue;
seenJobsBySequence.set(job.sequence, identity);
orderedSequences.push(job.sequence);
pendingJobs.push(job);
}
scheduleAvailableJobs();
const nextProducerStatus = await readNativeCaptureProducerStatus(runDir);
if (nextProducerStatus) producerStatus = nextProducerStatus;
if (producerStatus && !producerStatus.running && terminalObservedAt === 0) {
terminalObservedAt = Date.now();
}
const expectedCapturedJobs = producerStatus && !producerStatus.running
? Math.min(producerStatus.captured, streamLimit)
: null;
const producerDrained = expectedCapturedJobs !== null
&& orderedSequences.length >= expectedCapturedJobs;
if (producerDrained && pendingJobs.length === 0 && activeJobs.size === 0) break;
if (
expectedCapturedJobs !== null
&& terminalObservedAt > 0
&& Date.now() - terminalObservedAt >= STREAM_TERMINAL_DRAIN_GRACE_MS
&& pendingJobs.length === 0
&& activeJobs.size === 0
) {
integrityError = `Capture job stream ended with ${orderedSequences.length}/${expectedCapturedJobs} jobs.`;
break;
}
if (activeJobs.size > 0) {
await Promise.race([
Promise.race(activeJobs.values()),
delay(STREAM_POLL_INTERVAL_MS),
]);
} else {
await delay(STREAM_POLL_INTERVAL_MS);
}
}
if (activeJobs.size > 0) await Promise.all(activeJobs.values());
flushOrderedResults();
await snapshotWriteQueue.catch(() => undefined);
const status = await finalizeProcessedRun({
deps,
jobsPath,
ok: !integrityError,
options,
processedJobs: publishedJobs,
runDir,
started,
error: integrityError || undefined,
});
return status;
}
async function finalizeProcessedRun({
deps,
error,
jobsPath,
ok,
options,
processedJobs,
runDir,
started,
}: {
deps: NativeScannerProcessingServiceDependencies;
error?: string;
jobsPath: string;
ok: boolean;
options: ProcessRunOptions;
processedJobs: ProcessedNativeJob[];
runDir: string;
started: number;
}) {
const recordsToPersist = processedJobs
.map((entry) => entry.storedRecord)
.filter((record): record is StoredArtifactRecord => Boolean(record));
let stored = 0;
if (options.persist && recordsToPersist.length > 0) {
const saved = await deps.saveArtifacts(recordsToPersist);
stored = saved.added + saved.updated;
}
const processingTiming = await updateRunTiming(deps, runDir, {
processingCompletedAt: new Date().toISOString(),
});
let status = buildProcessStatus({
error,
jobsPath,
ok,
options,
processedJobs,
runDir,
started,
stored,
timing: processingTiming,
});
// The complete results file is the durable hand-off boundary. Keep its
// timestamp distinct from a later renderer/service reconciliation read.
await writeJsonAtomic(status.scanResultsPath, processedJobs.map((entry) => entry.scanResult));
const durableTiming = await updateRunTiming(deps, runDir, {
resultsDurableAt: new Date().toISOString(),
});
status = buildProcessStatus({
error,
jobsPath,
ok,
options,
processedJobs,
runDir,
started,
stored,
timing: durableTiming ?? processingTiming,
});
await writeJsonAtomic(status.reportPath, status);
replaceCachedLiveResults(runDir, processedJobs.map((entry) => entry.scanResult));
safePublishProgress(deps, {
running: false,
runDir,
total: processedJobs.length,
processed: status.processed,
parsed: status.parsed,
review: status.review,
stored: status.stored,
errors: status.errors,
elapsedMs: status.elapsedMs,
...(status.error ? { error: status.error } : {}),
});
return status;
}
function buildProcessStatus({
error,
jobsPath,
ok,
options,
processedJobs,
runDir,
started,
stored,
timing,
}: {
error?: string;
jobsPath: string;
ok: boolean;
options: ProcessRunOptions;
processedJobs: ProcessedNativeJob[];
runDir: string;
started: number;
stored: number;
timing?: NativeScannerRunTiming;
}): NativeScannerProcessStatus {
const results = processedJobs.map((entry) => entry.result);
return {
ok,
runDir,
jobsPath,
reportPath: path.join(runDir, "processing-report.json"),
scanResultsPath: path.join(runDir, "scan-results.json"),
processed: results.length,
parsed: results.filter((result) => result.parsed).length,
review: results.filter((result) => result.needsReview).length,
stored,
errors: results.filter((result) => result.error).length + (error ? 1 : 0),
elapsedMs: Date.now() - started,
queueConcurrency: Math.min(POST_CAPTURE_QUEUE_CONCURRENCY, processedJobs.length),
persisted: Boolean(options.persist),
results,
...(timing ? { timing } : {}),
...(error ? { error } : {}),
};
}
async function writeProcessingOutputs({
processedJobs,
reportPath,
scanResultsPath,
status,
}: {
processedJobs: ProcessedNativeJob[];
reportPath: string;
scanResultsPath: string;
status: NativeScannerProcessStatus;
}) {
await writeJsonAtomic(scanResultsPath, processedJobs.map((entry) => entry.scanResult));
await writeJsonAtomic(reportPath, status);
}
async function writeJsonAtomic(filePath: string, payload: unknown) {
const temporaryPath = `${filePath}.tmp-${process.pid}`;
await fs.writeFile(temporaryPath, JSON.stringify(payload, null, 2), "utf8");
await fs.rename(temporaryPath, filePath);
}
type NativeCaptureJobTailState = {
offset: number;
};
async function readAppendedNativeCaptureJobs(
jobsPath: string,
state: NativeCaptureJobTailState,
maxJobs: number,
): Promise<NativeCaptureJobPayload[]> {
let handle: Awaited<ReturnType<typeof fs.open>> | null = null;
try {
handle = await fs.open(jobsPath, "r");
const stat = await handle.stat();
if (stat.size < state.offset) {
state.offset = 0;
}
const length = Math.min(stat.size - state.offset, STREAM_READ_CHUNK_BYTES);
if (length <= 0) return [];
const chunk = Buffer.alloc(length);
const { bytesRead } = await handle.read(chunk, 0, length, state.offset);
const jobs: NativeCaptureJobPayload[] = [];
let lineStart = 0;
let consumedBytes = 0;
for (let index = 0; index < bytesRead; index += 1) {
if (chunk[index] !== 0x0a) continue;
const line = chunk.subarray(lineStart, index).toString("utf8").trim();
lineStart = index + 1;
consumedBytes = lineStart;
if (!line) continue;
const job = JSON.parse(line) as NativeCaptureJobPayload;
if (Number.isFinite(job.sequence)) jobs.push(job);
if (jobs.length >= maxJobs) break;
}
state.offset += consumedBytes;
return jobs;
} catch (error) {
if (isMissingFileError(error)) return [];
throw error;
} finally {
await handle?.close().catch(() => undefined);
}
}
async function readNativeCaptureProducerStatus(runDir: string): Promise<NativeCaptureProducerStatus | null> {
try {
const raw = JSON.parse(await fs.readFile(path.join(runDir, "status.json"), "utf8"));
const scanner = raw?.scanner ?? raw;
if (!scanner || typeof scanner !== "object") return null;
return {
running: Boolean(scanner.running),
status: typeof scanner.status === "string" ? scanner.status : "unknown",
target: Math.max(0, Math.round(Number(scanner.target) || 0)),
captured: Math.max(0, Math.round(Number(scanner.captured) || 0)),
};
} catch (error) {
if (isMissingFileError(error) || error instanceof SyntaxError) return null;
throw error;
}
}
async function readStoredScanResults(resultsPath: string) {
const raw = await fs.readFile(resultsPath, "utf8");
const parsed = JSON.parse(raw);
return Array.isArray(parsed)
? parsed.filter(isStoredScanResultEntry).map(evaluateStoredScanResult)
: [];
}
async function refreshCachedResultsAfterMutation(
deps: NativeScannerProcessingServiceDependencies,
requestedRunDir?: string,
) {
const runDir = deps.resolveRunDir(requestedRunDir);
if (!runDir) return;
try {
const [results, deletedIds] = await Promise.all([
readStoredScanResults(path.join(runDir, "scan-results.json")),
loadNativeScannerDeletedResultIds(runDir),
]);
replaceCachedLiveResults(runDir, results.filter((entry) => !deletedIds.has(entry.id)));
} catch {
// Mutation status remains authoritative; a later disk load can retry.
liveResultsByRunDir.delete(runDir);
}
}
async function recordResultReconciliationIfComplete(
deps: NativeScannerProcessingServiceDependencies,
runDir: string,
results: StoredScanResultEntry[],
) {
const currentTiming = await loadRunTiming(deps, runDir);
// Legacy runs intentionally stay untouched. A reconciliation timestamp is
// meaningful only for a run that already owns the new capture/processing
// timeline and whose complete result file was durably written by this path.
if (!currentTiming?.resultsDurableAt || currentTiming.resultsReconciledAt) return currentTiming ?? undefined;
const producer = await readNativeCaptureProducerStatus(runDir).catch(() => null);
const terminal = producer
&& !producer.running
&& (producer.status === "done" || producer.status === "completed")
&& producer.captured > 0
&& results.length === producer.captured;
if (!terminal) return currentTiming;
return updateRunTiming(deps, runDir, { resultsReconciledAt: new Date().toISOString() }) ?? currentTiming;
}
async function updateRunTiming(
deps: NativeScannerProcessingServiceDependencies,
runDir: string,
patch: NativeScannerRunTimingPatch,
) {
if (!deps.recordRunTiming || !runDir) return undefined;
try {
return await deps.recordRunTiming(runDir, patch);
} catch {
// Timing is evidence, never a dependency of scanner correctness. A later
// caller can still load the durable results and report the missing marker.
return undefined;
}
}
async function loadRunTiming(
deps: NativeScannerProcessingServiceDependencies,
runDir: string,
) {
if (!deps.loadRunTiming || !runDir) return null;
try {
return await deps.loadRunTiming(runDir);
} catch {
return null;
}
}
function replaceCachedLiveResults(runDir: string, results: StoredScanResultEntry[]) {
liveResultsByRunDir.delete(runDir);
liveResultsByRunDir.set(runDir, [...results]);
while (liveResultsByRunDir.size > MAX_CACHED_LIVE_RUNS) {
const oldest = liveResultsByRunDir.keys().next().value;
if (typeof oldest !== "string") break;
liveResultsByRunDir.delete(oldest);
}
}
function appendCachedLiveResult(runDir: string, result: StoredScanResultEntry) {
const current = liveResultsByRunDir.get(runDir) ?? [];
if (current.some((entry) => entry.id === result.id)) return;
current.push(result);
liveResultsByRunDir.set(runDir, current);
}
function safePublishProgress(
deps: NativeScannerProcessingServiceDependencies,
progress: NativeScannerProcessingProgressStatus,
) {
try {
deps.onProgress?.(progress);
} catch {
// UI progress must never abort durable OCR/parse processing.
}
}
function safePublishResult(deps: NativeScannerProcessingServiceDependencies, result: StoredScanResultEntry) {
try {
deps.onResult?.(result);
} catch {
// UI delivery is best effort; loadResults can reconcile from memory or disk.
}
}
function nativeCaptureJobIdentity(job: NativeCaptureJobPayload) {
return JSON.stringify({
sequence: job.sequence,
category: job.category ?? "",
page: job.page ?? null,
row: job.row ?? null,
col: job.col ?? null,
relativePath: job.relativePath ?? "",
absolutePath: job.absolutePath ?? "",
starCount: job.starCount ?? null,
starConfidence: job.starConfidence ?? null,
starSource: job.starSource ?? "",
});
}
function isMissingFileError(error: unknown) {
return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
}
function delay(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export function nativeScannerProcessStats(status: NativeScannerProcessStatus): Record<string, number> {
return {
processed: status.processed,
@@ -252,13 +901,53 @@ async function processNativeCaptureJob({
const capture = await deps.buildCaptureResult(imagePath, job);
const parsed = parseArtifactCandidate(capture);
const ikMatch = parsed && ikCatalog ? matchParsedArtifactToIk(parsed, ikCatalog) : null;
const notes = [...new Set([...(parsed?.notes ?? []), ...(ikMatch?.notes ?? [])])];
const needsReview = parsedArtifactNeedsReview(parsed) || Boolean(ikMatch && !ikMatch.matched);
const rarityEvidence = nativeJobRarityEvidence(job);
const notes = [...new Set([
...(parsed?.notes ?? []),
...(ikMatch?.notes ?? []),
...(rarityEvidence.uncertain ? ["Artifact star row could not be confirmed; keep this result in Review."] : []),
])];
const needsReview = shouldFlagArtifactForReview(parsed)
|| Boolean(ikMatch && !ikMatch.matched)
|| rarityEvidence.uncertain;
const canPersist = parsedArtifactCanPersist(parsed, needsReview);
const shouldPersist = Boolean(persist && parsed && canPersist && !needsReview);
const baseScanResult = createStoredScanResultEntry({
runId,
sequence: job.sequence,
page: job.page,
row: job.row,
col: job.col,
category: category.scanResultCategory,
source: "native-ik-scan",
imagePath,
parsed,
needsReview,
confidence: parsed?.confidence ?? 0,
capturedAt: job.capturedAt,
persistedArtifact: false,
notes,
locked: capture.locked,
ikMatch: ikMatch ?? undefined,
rarity: rarityEvidence.rarity,
rarityConfidence: rarityEvidence.confidence,
raritySource: rarityEvidence.rarity ? "native" : undefined,
});
const shouldPersist = Boolean(
persist
&& parsed
&& canPersist
&& !needsReview
&& baseScanResult.valueStatus === "evaluated"
&& baseScanResult.valueEvaluation?.status === "evaluated"
&& baseScanResult.valueEvaluation.score !== null
&& Number.isFinite(baseScanResult.valueEvaluation.score),
);
const storedRecord = parsed && shouldPersist
? toStoredArtifact(parsed, "native-ik-scan", needsReview, capture.locked)
: null;
const scanResult = storedRecord
? { ...baseScanResult, artifactRecordId: storedRecord.id, persistedArtifact: true }
: baseScanResult;
return {
result: {
sequence: job.sequence,
@@ -273,30 +962,14 @@ async function processNativeCaptureJob({
slot: parsed?.slot,
confidence: parsed?.confidence ?? 0,
needsReview,
rarity: rarityEvidence.rarity,
rarityConfidence: rarityEvidence.confidence,
persisted: shouldPersist,
ikMatch: ikMatch ?? undefined,
notes,
ocr: capture.ocr ?? [],
},
scanResult: createStoredScanResultEntry({
runId,
sequence: job.sequence,
page: job.page,
row: job.row,
col: job.col,
category: category.scanResultCategory,
source: "native-ik-scan",
imagePath,
parsed,
needsReview,
confidence: parsed?.confidence ?? 0,
capturedAt: job.capturedAt,
artifactRecordId: storedRecord?.id,
persistedArtifact: shouldPersist,
notes,
locked: capture.locked,
ikMatch: ikMatch ?? undefined,
}),
scanResult,
storedRecord,
};
} catch (error) {
@@ -305,6 +978,26 @@ async function processNativeCaptureJob({
}
}
function nativeJobRarityEvidence(job: NativeCaptureJobPayload): {
rarity?: ArtifactRarity;
confidence?: number;
/** The newer helper attempted direct visual detection but could not prove a count. */
uncertain: boolean;
} {
const hasDetectionPayload = Object.prototype.hasOwnProperty.call(job, "starCount")
|| Object.prototype.hasOwnProperty.call(job, "starConfidence")
|| Object.prototype.hasOwnProperty.call(job, "starSource");
if (!hasDetectionPayload) return { uncertain: false };
const starCount = Number(job.starCount);
const confidence = Number(job.starConfidence);
const validRarity = Number.isInteger(starCount) && starCount >= 1 && starCount <= 5;
const confident = Number.isFinite(confidence) && confidence >= 0.8 && confidence <= 1;
const hasSource = typeof job.starSource === "string" && job.starSource.trim().length > 0;
if (!validRarity || !confident || !hasSource) return { uncertain: true };
return { rarity: starCount as ArtifactRarity, confidence, uncertain: false };
}
function reviewJobResult({
category,
error,
@@ -473,15 +1166,6 @@ function isStoredScanResultEntry(value: unknown): value is StoredScanResultEntry
&& Array.isArray(entry.notes);
}
function parsedArtifactNeedsReview(parsed: ReturnType<typeof parseArtifactCandidate>) {
if (!parsed) return true;
if (parsed.confidence < 78) return true;
const criticalFields: Array<"name" | "slot" | "mainStat" | "mainValue" | "setName"> = ["name", "slot", "mainStat", "mainValue", "setName"];
if (criticalFields.some((field) => (parsed.fields[field]?.confidence ?? 0) < 70)) return true;
if (parsed.substats.length === 0) return true;
return parsed.notes.some((note) => /not confidently parsed|substats look incomplete|likely OCR misread/i.test(note));
}
function parsedArtifactCanPersist(parsed: ReturnType<typeof parseArtifactCandidate>, needsReview: boolean) {
if (!parsed) return false;
if (parsed.name === "Unknown artifact") return false;
@@ -0,0 +1,72 @@
import fs from "node:fs/promises";
import path from "node:path";
/**
* Native runs remain replayable evidence. Instead of rewriting their raw result
* array, locally removed rows are recorded here and filtered at load time.
*/
export const NATIVE_SCANNER_RESULT_TOMBSTONES_FILE = "deleted-results.jsonl";
export interface NativeScannerResultTombstone {
version: 1;
resultId: string;
deletedAt: string;
imagePath?: string;
removeLinkedStoreRecord: boolean;
storeRecordId?: string;
}
export function nativeScannerResultTombstonePath(runDir: string) {
return path.join(runDir, NATIVE_SCANNER_RESULT_TOMBSTONES_FILE);
}
export async function loadNativeScannerResultTombstones(runDir: string): Promise<NativeScannerResultTombstone[]> {
try {
const raw = await fs.readFile(nativeScannerResultTombstonePath(runDir), "utf8");
return raw
.split(/\r?\n/)
.filter(Boolean)
.map(parseTombstone)
.filter((entry): entry is NativeScannerResultTombstone => Boolean(entry));
} catch (error) {
if (isMissingFileError(error)) return [];
throw error;
}
}
export async function loadNativeScannerDeletedResultIds(runDir: string) {
const tombstones = await loadNativeScannerResultTombstones(runDir);
return new Set(tombstones.map((entry) => entry.resultId));
}
export async function appendNativeScannerResultTombstone(
runDir: string,
tombstone: NativeScannerResultTombstone,
) {
const filePath = nativeScannerResultTombstonePath(runDir);
await fs.appendFile(filePath, `${JSON.stringify(tombstone)}\n`, "utf8");
return filePath;
}
function parseTombstone(value: string): NativeScannerResultTombstone | null {
try {
const parsed = JSON.parse(value) as Partial<NativeScannerResultTombstone>;
const resultId = typeof parsed.resultId === "string" ? parsed.resultId.trim() : "";
const deletedAt = typeof parsed.deletedAt === "string" ? parsed.deletedAt : "";
if (!resultId || !deletedAt) return null;
return {
version: 1,
resultId,
deletedAt,
...(typeof parsed.imagePath === "string" ? { imagePath: parsed.imagePath } : {}),
removeLinkedStoreRecord: Boolean(parsed.removeLinkedStoreRecord),
...(typeof parsed.storeRecordId === "string" ? { storeRecordId: parsed.storeRecordId } : {}),
};
} catch {
return null;
}
}
function isMissingFileError(error: unknown) {
return typeof error === "object" && error !== null && "code" in error && (error as { code?: string }).code === "ENOENT";
}
@@ -1,23 +1,33 @@
import fs from "node:fs/promises";
import path from "node:path";
import { reviewedMainValueError, type ParsedArtifactCandidate } from "../../src/lib/artifactOcrParser.js";
import { evaluateArtifactValue, evaluateStoredScanResult } from "../../src/lib/artifactEvaluation.js";
import { matchParsedArtifactToIk, type IkArtifactCatalog } from "../../src/lib/ikArtifactMatcher.js";
import { buildScanResultPromotionSummary, scanResultToStoredArtifact } from "../../src/lib/scanResultPromotion.js";
import { implausibleSubstats } from "../../src/lib/substatRolls.js";
import type {
ArtifactStoreLoadResult,
ArtifactStoreSaveResult,
NativeScannerDeleteResultStatus,
NativeScannerPromotionStatus,
NativeScannerReviewArtifactInput,
NativeScannerReviewStatus,
ReviewSamplePayload,
} from "../../src/types/global.js";
import type { StoredArtifactRecord, StoredScanResultEntry } from "../../src/types/storage.js";
import type { ArtifactRarity, ScanResultArtifactIdentity, StoredArtifactRecord, StoredScanResultEntry } from "../../src/types/storage.js";
import {
appendNativeScannerResultTombstone,
loadNativeScannerDeletedResultIds,
loadNativeScannerResultTombstones,
nativeScannerResultTombstonePath,
} from "./nativeScannerResultTombstones.js";
export interface NativeScannerResultWorkflowDependencies {
resolveRunDir(runDir?: string): string;
loadArtifacts?: () => Promise<ArtifactStoreLoadResult>;
saveArtifacts(records: StoredArtifactRecord[]): Promise<Pick<ArtifactStoreSaveResult, "added" | "updated"> & Partial<ArtifactStoreSaveResult>>;
removeArtifacts?: (ids: string[]) => Promise<{ ok: boolean; removed: number }>;
isProcessingRunActive?: (runDir: string) => boolean;
saveReviewSample?: (sample: ReviewSamplePayload) => Promise<{ ok: boolean }>;
loadIkArtifactCatalog?: () => Promise<IkArtifactCatalog | null>;
}
@@ -31,6 +41,11 @@ export interface NativeScannerResultWorkflowService {
artifact?: NativeScannerReviewArtifactInput;
note?: string;
}): Promise<NativeScannerReviewStatus>;
deleteResult(options: {
runDir?: string;
resultId: string;
removeLinkedStoreRecord?: boolean;
}): Promise<NativeScannerDeleteResultStatus>;
}
export function createNativeScannerResultWorkflowService(
@@ -46,6 +61,10 @@ export function createNativeScannerResultWorkflowService(
}
try {
const deletedIds = await loadNativeScannerDeletedResultIds(runDir);
if (requestedIds.some((id) => deletedIds.has(id))) {
return emptyPromotionStatus(runDir, logPath, requestedIds.length, "Deleted scan results cannot be promoted.");
}
const { path: resultsPath, results: scanResults } = await loadScanResults(runDir);
const selectedIds = new Set(requestedIds);
const selectedResults = scanResults.filter((entry) => selectedIds.has(entry.id));
@@ -102,6 +121,10 @@ export function createNativeScannerResultWorkflowService(
}
try {
const deletedIds = await loadNativeScannerDeletedResultIds(runDir);
if (deletedIds.has(resultId)) {
return emptyReviewStatus(runDir, logPath, resultId, options.action, "Deleted scan results cannot be reviewed.");
}
const { path: resultsPath, results: scanResults } = await loadScanResults(runDir);
const index = scanResults.findIndex((entry) => entry.id === resultId);
if (index < 0) return emptyReviewStatus(runDir, logPath, resultId, options.action, "Selected scan result was not found.");
@@ -118,7 +141,7 @@ export function createNativeScannerResultWorkflowService(
updated = rejectResult(current, reviewedAt, note);
} else {
const artifact = normalizeReviewedArtifact(options.artifact);
const errors = reviewedArtifactErrors(artifact);
const errors = reviewedArtifactErrors(artifact, current.artifact?.rarity);
if (errors.length > 0) return emptyReviewStatus(runDir, logPath, resultId, options.action, errors.join(" "));
const parsed = reviewedArtifactToParsed(artifact);
const catalog = deps.loadIkArtifactCatalog ? await deps.loadIkArtifactCatalog() : null;
@@ -148,13 +171,102 @@ export function createNativeScannerResultWorkflowService(
return emptyReviewStatus(runDir, logPath, resultId, options.action, errorMessage(error));
}
},
async deleteResult(options) {
const runDir = deps.resolveRunDir(options.runDir);
const resultId = String(options.resultId ?? "").trim();
const tombstonePath = runDir ? nativeScannerResultTombstonePath(runDir) : "";
if (!runDir || !resultId) {
return emptyDeleteStatus(runDir, tombstonePath, resultId, "Deleting a local scan result requires a run directory and result ID.");
}
try {
if (await isNativeRunActive(runDir) || deps.isProcessingRunActive?.(runDir)) {
return emptyDeleteStatus(runDir, tombstonePath, resultId, "An active native scan cannot be changed. Stop and finish the scan first.");
}
const existingTombstones = await loadNativeScannerResultTombstones(runDir);
if (existingTombstones.some((entry) => entry.resultId === resultId)) {
return {
ok: true,
runDir,
tombstonePath,
resultId,
deletedResult: false,
alreadyDeleted: true,
cropPath: "",
deletedCrop: false,
deletedStoreRecord: false,
warnings: [],
};
}
const { results } = await loadScanResults(runDir);
const current = results.find((entry) => entry.id === resultId);
if (!current) {
return emptyDeleteStatus(runDir, tombstonePath, resultId, "Selected scan result was not found.");
}
const removeLinkedStoreRecord = Boolean(options.removeLinkedStoreRecord);
const storeRecordId = removeLinkedStoreRecord ? current.artifactRecordId?.trim() : undefined;
await appendNativeScannerResultTombstone(runDir, {
version: 1,
resultId,
deletedAt: new Date().toISOString(),
...(current.imagePath ? { imagePath: current.imagePath } : {}),
removeLinkedStoreRecord,
...(storeRecordId ? { storeRecordId } : {}),
});
const warnings: string[] = [];
const crop = await removeLocalCrop(runDir, current.imagePath);
if (crop.warning) warnings.push(crop.warning);
let deletedStoreRecord = false;
if (removeLinkedStoreRecord) {
if (!storeRecordId) {
warnings.push("This scan result has no explicitly linked local store record to remove.");
} else if (!deps.removeArtifacts) {
warnings.push("The linked local store record was retained because local-store deletion is unavailable.");
} else {
try {
const removal = await deps.removeArtifacts([storeRecordId]);
if (!removal.ok) {
warnings.push("The scan result was removed, but the linked local store record could not be removed.");
} else {
deletedStoreRecord = removal.removed > 0;
if (!deletedStoreRecord) warnings.push("The linked local store record was already absent.");
}
} catch {
warnings.push("The scan result was removed, but the linked local store record could not be removed.");
}
}
}
return {
ok: true,
runDir,
tombstonePath,
resultId,
deletedResult: true,
alreadyDeleted: false,
cropPath: crop.path,
deletedCrop: crop.deleted,
...(storeRecordId ? { storeRecordId } : {}),
deletedStoreRecord,
warnings,
};
} catch (error) {
return emptyDeleteStatus(runDir, tombstonePath, resultId, errorMessage(error));
}
},
};
}
async function loadScanResults(runDir: string) {
const resultsPath = path.join(runDir, "scan-results.json");
const raw = JSON.parse(await fs.readFile(resultsPath, "utf8"));
const results = Array.isArray(raw) ? raw.filter(isStoredScanResultEntry) : [];
const results = Array.isArray(raw) ? raw.filter(isStoredScanResultEntry).map(evaluateStoredScanResult) : [];
return { path: resultsPath, results };
}
@@ -167,14 +279,14 @@ async function appendWorkflowLog(logPath: string, payload: object) {
}
function rejectResult(current: StoredScanResultEntry, reviewedAt: string, note: string): StoredScanResultEntry {
return {
return evaluateStoredScanResult({
...current,
extractionStatus: "review",
needsReview: true,
valueStatus: "review",
notes: [...new Set([...current.notes, note || "Manual review rejected this result."])],
review: { status: "rejected", reviewedAt, note: note || undefined, correctedFields: [] },
};
});
}
function approveResult(
@@ -185,22 +297,34 @@ function approveResult(
note: string,
correctedFields: string[],
): StoredScanResultEntry {
return {
const preservedRarity = current.artifact?.rarity;
const preservedRarityMetadata = preservedRarity === undefined
? {}
: {
rarity: preservedRarity,
...(current.artifact?.rarityConfidence === undefined ? {} : { rarityConfidence: current.artifact.rarityConfidence }),
...(current.artifact?.raritySource === undefined ? {} : { raritySource: current.artifact.raritySource }),
};
const approvedArtifact: ScanResultArtifactIdentity = {
...artifact,
...preservedRarityMetadata,
};
return evaluateStoredScanResult({
...current,
extractionStatus: "parsed",
extractionConfidence: 100,
needsReview: false,
valueStatus: "deferred",
valueScore: null,
artifact,
artifact: approvedArtifact,
ikMatch,
fieldConfidences: reviewedFieldConfidences(artifact),
fieldConfidences: reviewedFieldConfidences(artifact, current.artifact),
artifactRecordId: undefined,
persistedArtifact: false,
notes: note ? [`Manual review approved: ${note}`] : ["Manual review approved."],
error: undefined,
review: { status: "approved", reviewedAt, note: note || undefined, correctedFields },
};
});
}
async function saveApprovedEvalSample(
@@ -258,6 +382,73 @@ function emptyReviewStatus(
return { ok: false, runDir, logPath, resultId, action, evalSampleSaved: false, correctedFields: [], error };
}
function emptyDeleteStatus(
runDir: string,
tombstonePath: string,
resultId: string,
error: string,
): NativeScannerDeleteResultStatus {
return {
ok: false,
runDir,
tombstonePath,
resultId,
deletedResult: false,
alreadyDeleted: false,
cropPath: "",
deletedCrop: false,
deletedStoreRecord: false,
warnings: [],
error,
};
}
async function isNativeRunActive(runDir: string) {
try {
const raw = JSON.parse(await fs.readFile(path.join(runDir, "status.json"), "utf8"));
const scanner = raw?.scanner ?? raw;
return Boolean(scanner?.running);
} catch {
// Older offline runs may not have a status file. A missing status is not
// evidence of a running producer, while a present running state blocks.
return false;
}
}
async function removeLocalCrop(runDir: string, imagePath: string) {
const resolvedRunDir = path.resolve(runDir);
const candidate = imagePath
? path.isAbsolute(imagePath) ? path.resolve(imagePath) : path.resolve(resolvedRunDir, imagePath)
: "";
if (!candidate) {
return { path: "", deleted: false, warning: "No crop image was recorded for this local scan result." };
}
if (!isPathInside(resolvedRunDir, candidate)) {
return { path: candidate, deleted: false, warning: "The recorded crop path is outside the native scan run and was retained." };
}
if (!/\.png$/i.test(candidate)) {
return { path: candidate, deleted: false, warning: "The recorded crop is not a PNG file and was retained." };
}
try {
await fs.unlink(candidate);
return { path: candidate, deleted: true };
} catch (error) {
if (isMissingFileError(error)) {
return { path: candidate, deleted: false, warning: "The crop image was already absent." };
}
return { path: candidate, deleted: false, warning: "The scan result was removed, but its crop image could not be removed." };
}
}
function isPathInside(root: string, candidate: string) {
const relative = path.relative(root, candidate);
return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
}
function isMissingFileError(error: unknown) {
return typeof error === "object" && error !== null && "code" in error && (error as { code?: string }).code === "ENOENT";
}
function normalizeReviewedArtifact(input?: NativeScannerReviewArtifactInput): NativeScannerReviewArtifactInput {
return {
name: String(input?.name ?? "").trim(),
@@ -272,7 +463,7 @@ function normalizeReviewedArtifact(input?: NativeScannerReviewArtifactInput): Na
};
}
function reviewedArtifactErrors(artifact: NativeScannerReviewArtifactInput) {
function reviewedArtifactErrors(artifact: NativeScannerReviewArtifactInput, confirmedRarity?: ArtifactRarity) {
const errors: string[] = [];
if (!artifact.name || artifact.name === "Unknown artifact") errors.push("Artifact name is required.");
if (!artifact.slot || artifact.slot === "Unknown slot") errors.push("Artifact slot is required.");
@@ -283,8 +474,15 @@ function reviewedArtifactErrors(artifact: NativeScannerReviewArtifactInput) {
if (artifact.substats.length === 0) errors.push("At least one substat is required.");
const mainValueError = reviewedMainValueError(artifact.slot, artifact.mainStat, artifact.level, artifact.mainValue);
if (mainValueError) errors.push(mainValueError);
const implausible = implausibleSubstats(artifact.substats, artifact.level > 16 ? 5 : undefined);
const implausible = implausibleSubstats(artifact.substats, confirmedRarity === 5 || artifact.level > 16 ? 5 : undefined);
if (implausible.length > 0) errors.push(`Implausible substats: ${implausible.join(", ")}.`);
if (errors.length === 0) {
const evaluation = evaluateArtifactValue({
...artifact,
...(confirmedRarity === undefined ? {} : { rarity: confirmedRarity }),
});
if (evaluation.status !== "evaluated" && evaluation.status !== "excluded") errors.push(evaluation.summary);
}
return errors;
}
@@ -316,8 +514,20 @@ function artifactChangedFields(
.filter((key) => JSON.stringify(before[key]) !== JSON.stringify(after[key]));
}
function reviewedFieldConfidences(artifact: NativeScannerReviewArtifactInput) {
return [
function reviewedFieldConfidences(
artifact: NativeScannerReviewArtifactInput,
previous?: ScanResultArtifactIdentity,
) {
const rarityField = previous?.rarity === undefined
? []
: [{
key: "rarity",
label: "Stars",
value: `${previous.rarity}`,
confidence: Math.round((previous.rarityConfidence ?? 0) * 100),
source: "visual" as const,
}];
const manualFields = [
{ key: "name", label: "Name", value: artifact.name },
{ key: "slot", label: "Slot", value: artifact.slot },
{ key: "level", label: "Level", value: String(artifact.level) },
@@ -327,6 +537,7 @@ function reviewedFieldConfidences(artifact: NativeScannerReviewArtifactInput) {
{ key: "equipped", label: "Equipped", value: artifact.equipped },
{ key: "substats", label: "Substats", value: artifact.substats.join(", ") },
].map((field) => ({ ...field, confidence: 100, source: "database" as const }));
return [...manualFields, ...rarityField];
}
async function loadReviewOcr(runDir: string, sequence: number) {
@@ -0,0 +1,109 @@
import {
existsSync,
readFileSync,
readdirSync,
realpathSync,
statSync,
} from "node:fs";
import path from "node:path";
const COMPLETE_RUN_FILES = [
"manifest.json",
"status.json",
"capture-jobs.jsonl",
"scan-results.json",
"processing-report.json",
] as const;
export interface NativeScannerRunDirectoryOptions {
outputRoot: string;
requestedRunDir?: string;
activeRunDir?: string;
}
export function resolveNativeScannerRunDirectory({
outputRoot,
requestedRunDir,
activeRunDir,
}: NativeScannerRunDirectoryOptions) {
const root = path.resolve(outputRoot);
const requested = requestedRunDir?.trim();
if (requested) return resolveContainedDirectory(root, requested);
const active = activeRunDir?.trim();
if (active) return resolveContainedDirectory(root, active);
return newestCompleteNativeScannerRun(root);
}
export function newestCompleteNativeScannerRun(outputRoot: string) {
const root = path.resolve(outputRoot);
if (!isDirectory(root)) return "";
try {
const candidates = readdirSync(root, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => completeRunCandidate(root, path.join(root, entry.name)))
.filter((entry): entry is CompleteRunCandidate => Boolean(entry))
.sort((left, right) => right.createdAt - left.createdAt || right.name.localeCompare(left.name));
return candidates[0]?.runDir ?? "";
} catch {
return "";
}
}
interface CompleteRunCandidate {
runDir: string;
name: string;
createdAt: number;
}
function completeRunCandidate(root: string, candidate: string): CompleteRunCandidate | null {
const runDir = resolveContainedDirectory(root, candidate);
if (!runDir || COMPLETE_RUN_FILES.some((file) => !existsSync(path.join(runDir, file)))) return null;
try {
const statusPayload = JSON.parse(readFileSync(path.join(runDir, "status.json"), "utf8"));
const status = statusPayload?.scanner ?? statusPayload;
const target = Number(status?.target ?? 0);
const captured = Number(status?.captured ?? 0);
if (status?.running !== false || status?.status !== "done" || target < 1 || captured < target) return null;
const results = JSON.parse(readFileSync(path.join(runDir, "scan-results.json"), "utf8"));
if (!Array.isArray(results) || results.length < captured) return null;
const manifest = JSON.parse(readFileSync(path.join(runDir, "manifest.json"), "utf8"));
const manifestCreatedAt = Date.parse(String(manifest?.createdAt ?? ""));
return {
runDir,
name: path.basename(runDir),
createdAt: Number.isFinite(manifestCreatedAt) ? manifestCreatedAt : statSync(runDir).mtimeMs,
};
} catch {
return null;
}
}
function resolveContainedDirectory(root: string, candidate: string) {
try {
const resolvedRoot = realpathSync(root);
const resolvedCandidate = realpathSync(path.resolve(candidate));
if (!isPathInside(resolvedRoot, resolvedCandidate) || !isDirectory(resolvedCandidate)) return "";
return resolvedCandidate;
} catch {
return "";
}
}
function isDirectory(candidate: string) {
try {
return statSync(candidate).isDirectory();
} catch {
return false;
}
}
function isPathInside(root: string, candidate: string) {
const relative = path.relative(root, candidate);
return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
}
+203
View File
@@ -0,0 +1,203 @@
import fs from "node:fs/promises";
import path from "node:path";
import type { NativeScannerRunTiming, NativeScannerRunTimingPatch } from "../../src/types/global.js";
export const NATIVE_SCANNER_RUN_TIMING_FILE = "run-timing.json";
export const NATIVE_SCANNER_RUN_TIMING_VERSION = "native-scanner-run-timing-v1" as const;
const timingWriteQueues = new Map<string, Promise<void>>();
const timestampFields = [
"requestStartedAt",
"scannerStartedAt",
"captureStartedAt",
"captureCompletedAt",
"processingStartedAt",
"processingCompletedAt",
"resultsDurableAt",
"resultsReconciledAt",
] as const satisfies ReadonlyArray<keyof NativeScannerRunTimingPatch>;
type TimingTimestampField = typeof timestampFields[number];
export function nativeScannerRunTimingPath(runDir: string) {
return path.join(path.resolve(runDir), NATIVE_SCANNER_RUN_TIMING_FILE);
}
export async function readNativeScannerRunTiming(runDir: string): Promise<NativeScannerRunTiming | null> {
const timingPath = nativeScannerRunTimingPath(runDir);
try {
const raw = JSON.parse(await fs.readFile(timingPath, "utf8")) as unknown;
return normalizeTiming(raw);
} catch (error) {
if (isMissingFileError(error)) return null;
throw error;
}
}
/**
* Adds first-observed lifecycle markers to one run's durable timing record.
* A marker is deliberately immutable after the first valid observation so
* polling or a later renderer retry cannot rewrite the measured interval.
*/
export async function recordNativeScannerRunTiming(
runDir: string,
patch: NativeScannerRunTimingPatch,
): Promise<NativeScannerRunTiming> {
const resolvedRunDir = path.resolve(runDir);
const previous = timingWriteQueues.get(resolvedRunDir) ?? Promise.resolve();
const task = previous
.catch(() => undefined)
.then(async () => {
const current = await readNativeScannerRunTiming(resolvedRunDir);
const next = mergeTiming(current, patch);
if (!sameTiming(current, next)) {
await writeJsonAtomic(nativeScannerRunTimingPath(resolvedRunDir), next);
}
return next;
});
const settled = task.then(() => undefined, () => undefined);
timingWriteQueues.set(resolvedRunDir, settled);
void settled.finally(() => {
if (timingWriteQueues.get(resolvedRunDir) === settled) timingWriteQueues.delete(resolvedRunDir);
});
return task;
}
export function validateNativeScannerRunTiming(timing: NativeScannerRunTiming): string[] {
const issues: string[] = [];
if (timing.version !== NATIVE_SCANNER_RUN_TIMING_VERSION) {
issues.push(`unsupported timing version ${String(timing.version)}`);
}
for (const field of timestampFields) {
const value = timing[field];
if (value !== undefined && !isValidTimestamp(value)) {
issues.push(`${field} is not a valid ISO timestamp`);
}
}
validateOrder(issues, timing, "requestStartedAt", "scannerStartedAt");
validateOrder(issues, timing, "requestStartedAt", "captureStartedAt");
validateOrder(issues, timing, "requestStartedAt", "processingStartedAt");
validateOrder(issues, timing, "captureStartedAt", "captureCompletedAt");
validateOrder(issues, timing, "captureCompletedAt", "processingCompletedAt");
validateOrder(issues, timing, "processingStartedAt", "processingCompletedAt");
validateOrder(issues, timing, "processingCompletedAt", "resultsDurableAt");
validateOrder(issues, timing, "resultsDurableAt", "resultsReconciledAt");
validateDuration(issues, timing, "requestToResultsDurableMs", "requestStartedAt", "resultsDurableAt");
validateDuration(issues, timing, "requestToResultsReconciledMs", "requestStartedAt", "resultsReconciledAt");
return issues;
}
export function hasCompleteNativeScannerRunTiming(timing: NativeScannerRunTiming | null | undefined) {
return Boolean(
timing?.requestStartedAt
&& timing.scannerStartedAt
&& timing.captureCompletedAt
&& timing.processingStartedAt
&& timing.processingCompletedAt
&& timing.resultsDurableAt
&& timing.resultsReconciledAt,
);
}
function mergeTiming(current: NativeScannerRunTiming | null, patch: NativeScannerRunTimingPatch): NativeScannerRunTiming {
const next: NativeScannerRunTiming = { version: NATIVE_SCANNER_RUN_TIMING_VERSION };
for (const field of timestampFields) {
const value = firstValidTimestamp(current?.[field], patch[field]);
if (value) next[field] = value;
}
const durableMs = durationMs(next.requestStartedAt, next.resultsDurableAt);
if (durableMs !== null) next.requestToResultsDurableMs = durableMs;
const reconciledMs = durationMs(next.requestStartedAt, next.resultsReconciledAt);
if (reconciledMs !== null) next.requestToResultsReconciledMs = reconciledMs;
return next;
}
function normalizeTiming(value: unknown): NativeScannerRunTiming {
if (!value || typeof value !== "object") throw new Error("Native scanner timing file is not an object.");
const candidate = value as Partial<NativeScannerRunTiming>;
const raw: NativeScannerRunTiming = {
version: candidate.version as NativeScannerRunTiming["version"],
};
for (const field of timestampFields) {
if (candidate[field] !== undefined) raw[field] = candidate[field] as string;
}
if (candidate.requestToResultsDurableMs !== undefined) {
raw.requestToResultsDurableMs = candidate.requestToResultsDurableMs as number;
}
if (candidate.requestToResultsReconciledMs !== undefined) {
raw.requestToResultsReconciledMs = candidate.requestToResultsReconciledMs as number;
}
const issues = validateNativeScannerRunTiming(raw);
if (issues.length > 0) throw new Error(`Invalid native scanner timing file: ${issues.join("; ")}`);
return mergeTiming(null, raw);
}
function validateOrder(
issues: string[],
timing: NativeScannerRunTiming,
earlier: TimingTimestampField,
later: TimingTimestampField,
) {
const earlierMs = timestampMs(timing[earlier]);
const laterMs = timestampMs(timing[later]);
if (earlierMs !== null && laterMs !== null && laterMs < earlierMs) {
issues.push(`${later} is before ${earlier}`);
}
}
function validateDuration(
issues: string[],
timing: NativeScannerRunTiming,
field: "requestToResultsDurableMs" | "requestToResultsReconciledMs",
startedField: TimingTimestampField,
completedField: TimingTimestampField,
) {
const declared = timing[field];
const expected = durationMs(timing[startedField], timing[completedField]);
if (declared !== undefined && (!Number.isInteger(declared) || declared < 0)) {
issues.push(`${field} is not a non-negative integer`);
}
if (expected !== null && declared !== expected) {
issues.push(`${field} does not match ${startedField} to ${completedField}`);
}
if (expected === null && declared !== undefined) {
issues.push(`${field} exists without both endpoint timestamps`);
}
}
function firstValidTimestamp(...values: Array<string | null | undefined>) {
return values.find((value): value is string => isValidTimestamp(value));
}
function isValidTimestamp(value: unknown): value is string {
return typeof value === "string" && value.length > 0 && Number.isFinite(Date.parse(value));
}
function timestampMs(value: unknown) {
return isValidTimestamp(value) ? Date.parse(value) : null;
}
function durationMs(startedAt: string | undefined, completedAt: string | undefined) {
const started = timestampMs(startedAt);
const completed = timestampMs(completedAt);
if (started === null || completed === null || completed < started) return null;
return Math.round(completed - started);
}
function sameTiming(left: NativeScannerRunTiming | null, right: NativeScannerRunTiming) {
return JSON.stringify(left) === JSON.stringify(right);
}
async function writeJsonAtomic(filePath: string, payload: unknown) {
const temporaryPath = `${filePath}.tmp-${process.pid}`;
await fs.writeFile(temporaryPath, JSON.stringify(payload, null, 2), "utf8");
await fs.rename(temporaryPath, filePath);
}
function isMissingFileError(error: unknown) {
return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
}