feat(scanner): add native artifact pipeline

Add native IK-style capture processing, Artifact Inventory, explicit promotion and single-result review. Confirm the three live OCR corrections in the eval corpus and preserve extraction/value separation.
2026-07-09 23:30:42 +02:00
parent 28d60eb915
commit 639b0b7f59
92 changed files with 13606 additions and 1703 deletions
@@ -6,17 +6,33 @@ import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { createWorker, PSM } from "tesseract.js";
 import { createInputHelperService, type InputHelperService } from "./services/inputHelper.js";
+import {
+  createNativeScannerProcessingService,
+  nativeScannerProcessStats,
+  type NativeCaptureJobPayload,
+} from "./services/nativeScannerProcessingService.js";
 import { pngBufferToBitmap } from "./services/pngBitmap.js";
 import { createRepositoryContext, type RepositoryContext } from "./bootstrap/repositoryContext.js";
 import { registerIpcHandlers } from "./bootstrap/ipcBootstrap.js";
 import { createDevControlServer } from "./devControlServer.js";
 import { createAppWindowManager, type AppWindowManager } from "./appWindowManager.js";
 import { createGoodFileService, type GoodFileService } from "./services/goodFileService.js";
+import type { IkArtifactCatalog } from "../src/lib/ikArtifactMatcher.js";
 import type { AppSnapshot } from "../src/types/domain.js";
 import type {
  CaptureOptions,
  CaptureResult,
  GoodDatabase,
+  NativeScannerCatalogStatus,
+  NativeScannerDataStatus,
+  NativeScannerImageLoadStatus,
+  NativeScannerPreflightStatus,
+  NativeScannerProcessStatus,
+  NativeScannerPromotionStatus,
+  NativeScannerReviewArtifactInput,
+  NativeScannerReviewStatus,
+  NativeScannerResultsLoadStatus,
+  NativeScannerRunStatus,
  OcrResult,
  AppRuntimeInfo,
  ScannerCommand,
@@ -54,7 +70,7 @@ app.commandLine.appendSwitch("disable-gpu-sandbox");
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const isDev = Boolean(process.env.VITE_DEV_SERVER_URL);
 const APP_RUNTIME_STARTED_AT = new Date().toISOString();
-const APP_RUNTIME_SIGNATURE = "2026-07-08-direct-gdi-reviewfix";
+const APP_RUNTIME_SIGNATURE = "2026-07-09-native-artifact-pipeline";

 let registeredHotkeys: Record<string, boolean> = {};
 let devControlServer: Server | null = null;
@@ -214,6 +230,305 @@ async function publishScannerStatus(status: ScannerStatusPayload) {
  return { ok: true };
 }

+function nativeScannerDataDir() {
+  const candidates = [
+    process.env.IK_INVENTORYLISTS_DIR,
+    path.join(process.cwd(), "data", "ik-inventorylists"),
+    path.join(app.getAppPath(), "data", "ik-inventorylists"),
+    path.join(process.resourcesPath, "ik-inventorylists"),
+  ].filter((candidate): candidate is string => Boolean(candidate));
+  return candidates.find((candidate) => existsSync(path.join(candidate, "version.txt"))) ?? candidates[0];
+}
+
+function nativeScannerOutputRoot() {
+  return path.join(app.getPath("userData"), "native-scans");
+}
+
+function nativeScannerStats(status: NativeScannerRunStatus): Record<string, number> {
+  return {
+    clicked: Number(status.clicked ?? 0),
+    attempted: Number(status.captured ?? 0),
+    verified: Number(status.captured ?? 0),
+    parsed: 0,
+    stored: 0,
+    review: 0,
+    duplicates: 0,
+    misses: 0,
+    pages: Number(status.pages ?? 0),
+    elapsedMs: Number(status.activeMs ?? 0),
+    activeScanMs: Number(status.activeMs ?? 0),
+    queued: Number(status.queued ?? 0),
+    captured: Number(status.captured ?? 0),
+  };
+}
+
+function publishNativeScannerStatus(status: NativeScannerRunStatus, extra: Partial<ScannerStatusPayload> = {}) {
+  const summary = {
+    mode: "Native IK Scanner",
+    status: status.status,
+    ...nativeScannerStats(status),
+    targetCount: status.target,
+    gridLabel: status.message,
+    runId: status.runId,
+    outputRoot: status.outputRoot,
+    runDir: status.runDir,
+    manifestPath: status.manifestPath,
+    jobsPath: status.jobsPath,
+    lastArtifactPath: status.lastArtifactPath,
+  };
+  scannerDevStatus = {
+    ...scannerDevStatus,
+    running: Boolean(status.running),
+    reviewStatus: status.message || "Native scanner ready.",
+    captureStatus: status.status,
+    stats: nativeScannerStats(status),
+    summary,
+    automationLog: [
+      ...(scannerDevStatus.automationLog ?? []).slice(-10),
+      status.message || `native scanner ${status.status}`,
+    ],
+    nativeScanner: status,
+    ...extra,
+    updatedAt: new Date().toISOString(),
+  };
+  return scannerDevStatus;
+}
+
+async function nativeScannerDataStatus(): Promise<NativeScannerDataStatus> {
+  return getInputHelperService().nativeScannerDataStatus(nativeScannerDataDir());
+}
+
+async function nativeScannerCatalog(): Promise<NativeScannerCatalogStatus> {
+  return getInputHelperService().nativeScannerCatalog(nativeScannerDataDir());
+}
+
+async function loadNativeIkArtifactCatalog(): Promise<IkArtifactCatalog | null> {
+  const catalog = await nativeScannerCatalog();
+  return catalog.data.valid ? { artifacts: catalog.artifacts } : null;
+}
+
+async function nativeScannerPreflight(options: { category?: string } = {}): Promise<NativeScannerPreflightStatus> {
+  const preflight = await getInputHelperService().nativeScannerPreflight(nativeScannerDataDir(), options.category ?? "artifacts");
+  scannerDevStatus = {
+    ...scannerDevStatus,
+    nativeScannerPreflight: preflight,
+    lookupStatus: preflight.data,
+    updatedAt: new Date().toISOString(),
+  };
+  return preflight;
+}
+
+async function nativeScannerStart(options: { limit?: number; category?: string } = {}): Promise<NativeScannerRunStatus> {
+  const status = await getInputHelperService().nativeScannerStart({
+    dataDir: nativeScannerDataDir(),
+    outputRoot: nativeScannerOutputRoot(),
+    limit: options.limit ?? 100,
+    category: options.category ?? "artifacts",
+  });
+  publishNativeScannerStatus(status, { lookupStatus: await nativeScannerDataStatus().catch(() => undefined) });
+  return status;
+}
+
+async function nativeScannerStop(): Promise<NativeScannerRunStatus> {
+  const status = await getInputHelperService().nativeScannerStop();
+  publishNativeScannerStatus(status);
+  return status;
+}
+
+async function nativeScannerStatus(): Promise<NativeScannerRunStatus> {
+  const status = await getInputHelperService().nativeScannerStatus();
+  publishNativeScannerStatus(status);
+  return status;
+}
+
+function resolveNativeProcessRunDir(runDir?: string) {
+  const native = scannerDevStatus.nativeScanner as NativeScannerRunStatus | undefined;
+  const candidate = runDir?.trim() || native?.runDir || "";
+  if (!candidate) return "";
+  const root = path.resolve(nativeScannerOutputRoot());
+  const resolved = path.resolve(candidate);
+  const relative = path.relative(root, resolved);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) return "";
+  return resolved;
+}
+
+async function buildNativeCardCaptureResult(
+  imagePath: string,
+  job: NativeCaptureJobPayload,
+): Promise<CaptureResult> {
+  const sourceImage = nativeImage.createFromPath(imagePath);
+  const size = sourceImage.getSize();
+  if (!size.width || !size.height) throw new Error(`Native card crop is empty: ${imagePath}`);
+  const detailRect = { x: 0, y: 0, width: size.width, height: size.height };
+  const inventoryRect = { x: 0, y: 0, width: 1, height: 1 };
+  const crops = createCrops(
+    sourceImage,
+    size,
+    detailRect,
+    inventoryRect,
+    {
+      ocrMode: "artifact",
+      ocrProfile: "full",
+      omitFullFrame: true,
+      omitInventoryPreview: true,
+      omitCropImages: true,
+      omitLockState: true,
+    },
+    undefined,
+    { skipOcr: false },
+  );
+  const croppedPayload = crops
+    .filter((crop) => crop.ocrEnabled !== false)
+    .map((crop) => crop.ocrImage
+      ? {
+          id: crop.id,
+          label: crop.label,
+          image: crop.ocrImage,
+        }
+      : null)
+    .filter((crop): crop is OcrCropPayload => Boolean(crop));
+  const recognized = await runOcrOnCropsWithTimeout(croppedPayload, "current", 6500);
+  return {
+    id: `native-card-${job.sequence}`,
+    name: path.basename(imagePath),
+    width: size.width,
+    height: size.height,
+    dataUrl: "",
+    capturedAt: new Date().toISOString(),
+    captureTarget: "genshin-client",
+    detailFingerprint: `${path.basename(imagePath)}:${size.width}x${size.height}`,
+    ocr: recognized.ocr,
+    ocrTimedOut: recognized.timedOut,
+    ocrSkipped: false,
+    crops: crops.map((crop) => ({
+      id: crop.id,
+      label: crop.label,
+      rect: {
+        x: crop.rect.x,
+        y: crop.rect.y,
+        width: crop.rect.width,
+        height: crop.rect.height,
+      },
+      dataUrl: crop.dataUrl,
+    })),
+    artifactDetail: {
+      present: true,
+      confidence: 70,
+      orangeHits: 0,
+      greenHits: 0,
+      textHits: 0,
+    },
+    layout: {
+      aspect: aspectRatioLabel(size),
+      isSixteenNine: false,
+      warning: "Native card-crop processing uses the detail card as its own coordinate space.",
+    },
+    timings: {
+      prepareMs: 0,
+      ocrMs: 0,
+      totalMs: 0,
+      ocrProfile: "full",
+      ocrEngine: "current",
+      ocrWorkerPoolSize: OCR_WORKER_POOL_SIZE,
+      cropCount: croppedPayload.length,
+      ocrSkipped: false,
+    },
+  };
+}
+
+function createNativeScannerService() {
+  return createNativeScannerProcessingService({
+    resolveRunDir: resolveNativeProcessRunDir,
+    buildCaptureResult: buildNativeCardCaptureResult,
+    loadArtifacts: () => getArtifactStoreRepository().loadAll(),
+    saveArtifacts: (records) => getArtifactStoreRepository().saveMany(records),
+    loadIkArtifactCatalog: loadNativeIkArtifactCatalog,
+    saveReviewSample: (sample) => getReviewSamplesRepository().append(sample),
+  });
+}
+
+async function nativeScannerProcessRun(options: { runDir?: string; persist?: boolean; limit?: number } = {}): Promise<NativeScannerProcessStatus> {
+  const service = createNativeScannerService();
+  const status = await service.processRun(options);
+  scannerDevStatus = {
+    ...scannerDevStatus,
+    reviewStatus: `Native post-processing: ${status.parsed}/${status.processed} parsed, ${status.review} review, ${status.stored} stored.`,
+    stats: {
+      ...(scannerDevStatus.stats ?? {}),
+      ...nativeScannerProcessStats(status),
+    },
+    nativeScannerProcessing: status,
+    automationLog: [
+      ...(scannerDevStatus.automationLog ?? []).slice(-10),
+      `native process: ${status.parsed}/${status.processed} parsed, report ${status.reportPath}`,
+    ],
+    updatedAt: new Date().toISOString(),
+  };
+  return status;
+}
+
+async function nativeScannerLoadResults(options: { runDir?: string; limit?: number } = {}): Promise<NativeScannerResultsLoadStatus> {
+  const service = createNativeScannerService();
+  const loaded = await service.loadResults(options);
+  scannerDevStatus = {
+    ...scannerDevStatus,
+    nativeScannerResults: loaded,
+    automationLog: [
+      ...(scannerDevStatus.automationLog ?? []).slice(-10),
+      `native results: ${loaded.results.length}/${loaded.total} loaded from ${loaded.path || "scan-results.json"}`,
+    ],
+    updatedAt: new Date().toISOString(),
+  };
+  return loaded;
+}
+
+async function nativeScannerPromoteResults(options: { runDir?: string; resultIds: string[] }): Promise<NativeScannerPromotionStatus> {
+  const service = createNativeScannerService();
+  const status = await service.promoteResults(options);
+  scannerDevStatus = {
+    ...scannerDevStatus,
+    reviewStatus: status.ok
+      ? `Native promotion: ${status.promoted} promoted, ${status.alreadyStored} already stored.`
+      : `Native promotion blocked: ${status.error ?? "unknown error"}`,
+    nativeScannerPromotion: status,
+    automationLog: [
+      ...(scannerDevStatus.automationLog ?? []).slice(-10),
+      `native promotion: ${status.promoted}/${status.requested}, log ${status.logPath || "unavailable"}`,
+    ],
+    updatedAt: new Date().toISOString(),
+  };
+  return status;
+}
+
+async function nativeScannerReviewResult(options: {
+  runDir?: string;
+  resultId: string;
+  action: "approve" | "reject";
+  artifact?: NativeScannerReviewArtifactInput;
+  note?: string;
+}): Promise<NativeScannerReviewStatus> {
+  const service = createNativeScannerService();
+  const status = await service.reviewResult(options);
+  scannerDevStatus = {
+    ...scannerDevStatus,
+    reviewStatus: status.ok
+      ? `Native review ${status.action}: ${status.resultId}`
+      : `Native review blocked: ${status.error ?? "unknown error"}`,
+    nativeScannerReview: status,
+    automationLog: [
+      ...(scannerDevStatus.automationLog ?? []).slice(-10),
+      `native review ${status.action}: ${status.ok ? "ok" : status.error}`,
+    ],
+    updatedAt: new Date().toISOString(),
+  };
+  return status;
+}
+
+async function nativeScannerLoadImage(options: { runDir?: string; imagePath: string }): Promise<NativeScannerImageLoadStatus> {
+  const service = createNativeScannerService();
+  return service.loadImage(options);
+}
+
 async function readRuntimeInfo() {
  try {
    const result = await getInputHelperService().getRuntimeInfo();
@@ -446,6 +761,9 @@ function focusMainWindow() {
 }

 function sendScannerCommand(command: ScannerCommand | "probe-click") {
+  if (command === "stop") {
+    void nativeScannerStop().catch(() => undefined);
+  }
  getAppWindowManager().sendScannerCommand(command);
 }

@@ -468,6 +786,15 @@ function startDevControlServer() {
    sendScannerCommand,
    clickScreen: clickScreenCommand,
    scannerStatus: () => ({ ...scannerDevStatus, appBuild: appRuntimeInfo(), ocrWarmup: getOcrWarmupStatus() }),
+    nativeScannerDataStatus,
+    nativeScannerCatalog,
+    nativeScannerPreflight,
+    nativeScannerStart,
+    nativeScannerStop,
+    nativeScannerStatus,
+    nativeScannerProcessRun,
+    nativeScannerLoadResults,
+    nativeScannerLoadImage,
    warmOcr: (engine) => warmOcrWorkerPool(engine),
    loadReviewSamples,
    listCaptureSources,
@@ -483,15 +810,13 @@ function createOverlayWindow() {
  getAppWindowManager().createOverlayWindow();
 }

-// Inventory Kamera keeps a pool of native Tesseract engines and scans artifact
-// fields concurrently. Our fast artifact profile has four useful OCR parameter
-// groups, so the default pool is four workers unless the machine is smaller or
-// GAA_OCR_WORKERS explicitly overrides it.
+// The fast artifact profile has four useful OCR parameter groups, so the
+// default pool is four workers unless the machine is smaller or GAA_OCR_WORKERS
+// explicitly overrides it.
 const OCR_WORKER_POOL_SIZE = resolveOcrWorkerPoolSize();
-const IK_TRAINEDDATA_LANG = "genshin_fast_09_04_21";

 type OcrWorker = Awaited<ReturnType<typeof createWorker>>;
-type OcrWorkerEngine = "current" | "ik-traineddata";
+type OcrWorkerEngine = "current";
 type OcrCropPayload = { id: string; label: string; image: Buffer };
 type OcrWarmupStatus = {
  engine: OcrWorkerEngine;
@@ -513,11 +838,9 @@ type OcrWorkerPoolState = {

 const ocrWorkerPools: Record<OcrWorkerEngine, OcrWorkerPoolState> = {
  current: { poolPromise: null, runQueue: Promise.resolve() },
-  "ik-traineddata": { poolPromise: null, runQueue: Promise.resolve() },
 };
 const ocrWarmupStatuses: Record<OcrWorkerEngine, OcrWarmupStatus> = {
  current: { engine: "current", status: "cold", workerPoolSize: OCR_WORKER_POOL_SIZE },
-  "ik-traineddata": { engine: "ik-traineddata", status: "cold", workerPoolSize: OCR_WORKER_POOL_SIZE },
 };

 function resolveOcrWorkerPoolSize() {
@@ -541,42 +864,13 @@ function appRuntimeInfo(): AppRuntimeInfo {
 }

 function ocrEngineFromOptions(options: CaptureOptions = {}): OcrWorkerEngine {
-  return options.ocrEngine === "ik-traineddata" ? "ik-traineddata" : "current";
-}
-
-function ikTessdataCandidates() {
-  const envDir = process.env.IK_TESSDATA_DIR;
-  return [
-    envDir,
-    path.resolve(process.cwd(), "data", "tessdata"),
-    path.resolve(process.cwd(), "work", "Inventory_Kamera", "InventoryKamera", "tessdata"),
-    path.resolve(process.cwd(), "work", "refs", "Inventory_Kamera", "InventoryKamera", "tessdata"),
-    path.resolve(process.cwd(), "..", "_ik_ref_fork", "InventoryKamera", "tessdata"),
-    path.resolve(process.cwd(), "..", "_ik_ref", "InventoryKamera", "tessdata"),
-    path.resolve(process.env.USERPROFILE ?? "", "Desktop", "_ik_ref_fork", "InventoryKamera", "tessdata"),
-    path.resolve(process.env.USERPROFILE ?? "", "Desktop", "_ik_ref", "InventoryKamera", "tessdata"),
-    process.resourcesPath ? path.resolve(process.resourcesPath, "tessdata") : "",
-  ].filter(Boolean) as string[];
-}
-
-function findIkTessdataDir() {
-  return ikTessdataCandidates().find((candidate) => existsSync(path.join(candidate, `${IK_TRAINEDDATA_LANG}.traineddata`))) ?? "";
+  void options;
+  return "current";
 }

 function getOcrWorkerOptions(engine: OcrWorkerEngine) {
-  if (engine !== "ik-traineddata") return { lang: "eng", options: undefined };
-  const langPath = findIkTessdataDir();
-  if (!langPath) {
-    throw new Error(`IK traineddata not found. Set IK_TESSDATA_DIR or place ${IK_TRAINEDDATA_LANG}.traineddata in data/tessdata.`);
-  }
-  return {
-    lang: IK_TRAINEDDATA_LANG,
-    options: {
-      langPath,
-      gzip: false,
-      cachePath: app.isReady() ? path.join(app.getPath("userData"), "tessdata-cache") : path.resolve(process.cwd(), "outputs", "tessdata-cache"),
-    },
-  };
+  void engine;
+  return { lang: "eng", options: undefined };
 }

 function getOcrWorkerPool(engine: OcrWorkerEngine) {
@@ -594,7 +888,7 @@ function getOcrWorkerPool(engine: OcrWorkerEngine) {
 }

 async function resetOcrWorker(engine?: OcrWorkerEngine) {
-  const engines: OcrWorkerEngine[] = engine ? [engine] : ["current", "ik-traineddata"];
+  const engines: OcrWorkerEngine[] = engine ? [engine] : ["current"];
  await Promise.all(engines.map(async (engineId) => {
    const state = ocrWorkerPools[engineId];
    const broken = state.poolPromise;
@@ -710,7 +1004,6 @@ function warmOcrWorkerPool(engine: OcrWorkerEngine = "current") {
 function getOcrWarmupStatus() {
  return {
    current: ocrWarmupStatuses.current,
-    "ik-traineddata": ocrWarmupStatuses["ik-traineddata"],
  };
 }

@@ -1334,7 +1627,7 @@ async function buildCaptureResult(
    : await runOcrOnCropsWithTimeout(croppedPayload, ocrEngine);
  const ocrMs = Date.now() - ocrStartedAt;
  const totalMs = Date.now() - buildStartedAt;
-  const captureOcrEngine: CaptureOptions["ocrEngine"] = ocrEngine === "ik-traineddata" ? "ik-traineddata" : "current";
+  const captureOcrEngine: CaptureOptions["ocrEngine"] = "current";

  const count = parseInventoryCount(recognized.ocr);
  return {
@@ -1458,6 +1751,23 @@ function initializeAppLifecycle() {
      moveMainWindowOffGenshin: async () => moveMainWindowOffGenshin(),
      focusGenshinForScanStart: () => focusGenshinForScanStart(),
      publishScannerStatus: (status: ScannerStatusPayload) => publishScannerStatus(status),
+      nativeScannerDataStatus: () => nativeScannerDataStatus(),
+      nativeScannerCatalog: () => nativeScannerCatalog(),
+      nativeScannerPreflight: (options?: { category?: string }) => nativeScannerPreflight(options),
+      nativeScannerStart: (options?: { limit?: number; category?: string }) => nativeScannerStart(options),
+      nativeScannerStop: () => nativeScannerStop(),
+      nativeScannerStatus: () => nativeScannerStatus(),
+      nativeScannerProcessRun: (options?: { runDir?: string; persist?: boolean; limit?: number }) => nativeScannerProcessRun(options),
+      nativeScannerLoadResults: (options?: { runDir?: string; limit?: number }) => nativeScannerLoadResults(options),
+      nativeScannerPromoteResults: (options: { runDir?: string; resultIds: string[] }) => nativeScannerPromoteResults(options),
+      nativeScannerReviewResult: (options: {
+        runDir?: string;
+        resultId: string;
+        action: "approve" | "reject";
+        artifact?: NativeScannerReviewArtifactInput;
+        note?: string;
+      }) => nativeScannerReviewResult(options),
+      nativeScannerLoadImage: (options: { runDir?: string; imagePath: string }) => nativeScannerLoadImage(options),
      readRuntimeInfo: () => readRuntimeInfo(),
      loadSnapshotFromDisk: () => loadSnapshotFromDisk(),
      saveSnapshotToDisk: (snapshot: AppSnapshot) => saveSnapshotToDisk(snapshot),