feat(scanner): add native artifact pipeline

Add native IK-style capture processing, Artifact Inventory, explicit promotion and single-result review. Confirm the three live OCR corrections in the eval corpus and preserve extraction/value separation.
2026-07-09 23:30:42 +02:00
parent 28d60eb915
commit 639b0b7f59
92 changed files with 13606 additions and 1703 deletions
@@ -9,6 +9,13 @@ import type {
  CaptureSourceInfo,
  ClickResult,
  ReviewSampleListResult,
+  NativeScannerCatalogStatus,
+  NativeScannerDataStatus,
+  NativeScannerImageLoadStatus,
+  NativeScannerPreflightStatus,
+  NativeScannerProcessStatus,
+  NativeScannerResultsLoadStatus,
+  NativeScannerRunStatus,
  ScannerCommand,
  ScannerStatusPayload,
  AppRuntimeInfo,
@@ -24,7 +31,16 @@ interface DevControlServerDependencies {
  sendScannerCommand: (command: ScannerCommand | "probe-click") => void;
  clickScreen: (x: number, y: number) => Promise<ClickResult>;
  scannerStatus: () => ScannerStatusPayload;
-  warmOcr: (engine: "current" | "ik-traineddata") => Promise<unknown>;
+  nativeScannerDataStatus: () => Promise<NativeScannerDataStatus>;
+  nativeScannerCatalog: () => Promise<NativeScannerCatalogStatus>;
+  nativeScannerPreflight: (options?: { category?: string }) => Promise<NativeScannerPreflightStatus>;
+  nativeScannerStart: (options?: { limit?: number; category?: string }) => Promise<NativeScannerRunStatus>;
+  nativeScannerStop: () => Promise<NativeScannerRunStatus>;
+  nativeScannerStatus: () => Promise<NativeScannerRunStatus>;
+  nativeScannerProcessRun: (options?: { runDir?: string; persist?: boolean; limit?: number }) => Promise<NativeScannerProcessStatus>;
+  nativeScannerLoadResults: (options?: { runDir?: string; limit?: number }) => Promise<NativeScannerResultsLoadStatus>;
+  nativeScannerLoadImage: (options: { runDir?: string; imagePath: string }) => Promise<NativeScannerImageLoadStatus>;
+  warmOcr: (engine: "current") => Promise<unknown>;
  loadReviewSamples: (limit?: number) => Promise<ReviewSampleListResult>;
  listCaptureSources: () => Promise<CaptureSourceInfo[]>;
  captureSource: (
@@ -161,23 +177,17 @@ export function createDevControlServer(deps: DevControlServerDependencies): Serv
    }
    if (url.pathname === "/scanner/start") {
      const limit = Number(url.searchParams.get("limit") ?? Number.NaN);
-      const entry = url.searchParams.get("entry");
-      const engine = url.searchParams.get("engine");
-      const scanEntryMode = entry === "paimon-menu" || entry === "visible-inventory" || entry === "direct-inventory" || entry === "auto-entry"
-        ? entry
-        : undefined;
-      const ocrEngine = engine === "ik-traineddata" ? "ik-traineddata" : engine === "current" ? "current" : undefined;
      const hasLimit = Number.isFinite(limit) && limit > 0;
-      const command: ScannerCommand = hasLimit || scanEntryMode || ocrEngine
-        ? { type: "start-auto", scanLimit: hasLimit ? limit : undefined, scanEntryMode, ocrEngine }
-        : "start-auto";
-      deps.sendScannerCommand(command);
-      writeDevJson(res, 200, { ok: true, command });
+      const category = url.searchParams.get("category") ?? undefined;
+      deps.nativeScannerStart({ ...(hasLimit ? { limit } : {}), ...(category ? { category } : {}) })
+        .then((scanner) => writeDevJson(res, 200, { ok: true, scanner }))
+        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
      return;
    }
    if (url.pathname === "/scanner/stop") {
-      deps.sendScannerCommand("stop");
-      writeDevJson(res, 200, { ok: true, command: "stop" });
+      deps.nativeScannerStop()
+        .then((scanner) => writeDevJson(res, 200, { ok: true, scanner }))
+        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
      return;
    }
    if (url.pathname === "/scanner/probe") {
@@ -198,13 +208,68 @@ export function createDevControlServer(deps: DevControlServerDependencies): Serv
      return;
    }
    if (url.pathname === "/scanner/status") {
-      writeDevJson(res, 200, { ok: true, status: deps.scannerStatus() });
+      deps.nativeScannerStatus()
+        .then((nativeScanner) => writeDevJson(res, 200, { ok: true, status: deps.scannerStatus(), nativeScanner }))
+        .catch(() => writeDevJson(res, 200, { ok: true, status: deps.scannerStatus() }));
+      return;
+    }
+    if (url.pathname === "/scanner/native/data") {
+      deps.nativeScannerDataStatus()
+        .then((status) => writeDevJson(res, status.valid ? 200 : 409, { ok: status.valid, status }))
+        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
+      return;
+    }
+    if (url.pathname === "/scanner/native/catalog") {
+      deps.nativeScannerCatalog()
+        .then((catalog) => writeDevJson(res, catalog.data.valid ? 200 : 409, { ok: catalog.data.valid, catalog }))
+        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
+      return;
+    }
+    if (url.pathname === "/scanner/native/preflight") {
+      const category = url.searchParams.get("category") ?? undefined;
+      deps.nativeScannerPreflight(category ? { category } : undefined)
+        .then((status) => writeDevJson(res, status.ready ? 200 : 409, { ok: status.ready, status }))
+        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
+      return;
+    }
+    if (url.pathname === "/scanner/native/process") {
+      const runDir = url.searchParams.get("runDir") ?? undefined;
+      const persist = url.searchParams.get("persist") === "1";
+      const limit = Number(url.searchParams.get("limit") ?? Number.NaN);
+      deps.nativeScannerProcessRun({
+        runDir,
+        persist,
+        limit: Number.isFinite(limit) && limit > 0 ? limit : undefined,
+      })
+        .then((status) => writeDevJson(res, status.ok ? 200 : 409, { ok: status.ok, status }))
+        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
+      return;
+    }
+    if (url.pathname === "/scanner/native/results") {
+      const runDir = url.searchParams.get("runDir") ?? undefined;
+      const limit = Number(url.searchParams.get("limit") ?? Number.NaN);
+      deps.nativeScannerLoadResults({
+        runDir,
+        limit: Number.isFinite(limit) && limit > 0 ? limit : undefined,
+      })
+        .then((status) => writeDevJson(res, status.ok ? 200 : 409, { ok: status.ok, status }))
+        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
+      return;
+    }
+    if (url.pathname === "/scanner/native/image") {
+      const runDir = url.searchParams.get("runDir") ?? undefined;
+      const imagePath = url.searchParams.get("imagePath");
+      if (!imagePath) {
+        writeDevJson(res, 400, { ok: false, error: "imagePath query param is required" });
+        return;
+      }
+      deps.nativeScannerLoadImage({ runDir, imagePath })
+        .then((status) => writeDevJson(res, status.ok ? 200 : 409, { ok: status.ok, status }))
+        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
      return;
    }
    if (url.pathname === "/scanner/ocr/warmup") {
-      const engineParam = url.searchParams.get("engine");
-      const engine = engineParam === "ik-traineddata" ? "ik-traineddata" : "current";
-      deps.warmOcr(engine)
+      deps.warmOcr("current")
        .then((status) => writeDevJson(res, 200, { ok: true, status }))
        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
      return;
@@ -226,14 +291,8 @@ export function createDevControlServer(deps: DevControlServerDependencies): Serv
    if (url.pathname === "/scanner/benchmark-ocr") {
      const limit = Math.max(1, Math.min(100, Number(url.searchParams.get("limit") ?? 1) || 1));
      const sourceId = url.searchParams.get("sourceId");
-      const engineParam = url.searchParams.get("engine");
      const profileParam = url.searchParams.get("profile");
      const ocrProfile: "full" | "fast" = profileParam === "full" ? "full" : "fast";
-      const engines: Array<"current" | "ik-traineddata"> = engineParam === "compare"
-        ? ["current", "ik-traineddata"]
-        : engineParam === "ik-traineddata"
-          ? ["ik-traineddata"]
-          : ["current"];
      deps.listCaptureSources()
        .then(async (sources) => {
          const source = findGenshinSource(sources, sourceId);
@@ -243,7 +302,7 @@ export function createDevControlServer(deps: DevControlServerDependencies): Serv
          }
          const benchmarkSource = source;

-          async function runEngineBenchmark(engine: "current" | "ik-traineddata") {
+          async function runCurrentBenchmark() {
            const startedAt = Date.now();
            const captures: Array<{
              index: number;
@@ -265,7 +324,7 @@ export function createDevControlServer(deps: DevControlServerDependencies): Serv
              const capture = await deps.captureSource(benchmarkSource.id, index === 0 ? 150 : 0, true, {
                ocrMode: "artifact",
                ocrProfile,
-                ocrEngine: engine,
+                ocrEngine: "current",
                omitFullFrame: true,
                omitInventoryPreview: true,
                skipOcrUnlessArtifactDetail: true,
@@ -316,7 +375,7 @@ export function createDevControlServer(deps: DevControlServerDependencies): Serv
              ]),
            );
            return {
-              engine,
+              engine: "current",
              nativeTesseract: "not-enabled",
              workerPoolSize: captures.find((capture) => capture.ocrWorkerPoolSize)?.ocrWorkerPoolSize ?? null,
              ocrProfile,
@@ -340,13 +399,10 @@ export function createDevControlServer(deps: DevControlServerDependencies): Serv
              captures,
            };
          }
-          const summaries = [];
-          for (const engine of engines) {
-            summaries.push(await runEngineBenchmark(engine));
-          }
+          const summary = await runCurrentBenchmark();
          writeDevJson(res, 200, {
            ok: true,
-            summary: summaries.length === 1 ? summaries[0] : { mode: "compare", limit, ocrProfile, engines: summaries },
+            summary,
          });
        })
        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
@@ -390,6 +446,22 @@ export function createDevControlServer(deps: DevControlServerDependencies): Serv
          }

          const before = await deps.captureSource(source.id, 150, true, { skipOcr: true });
+          const grid = before.inventoryGrid;
+          const detail = before.artifactDetail;
+          const gridReady = before.captureTarget === "genshin-client"
+            && grid?.source === "detected"
+            && (grid.confidence ?? 0) >= 60;
+          const detailReady = Boolean(detail?.present && detail.confidence >= 45);
+          if (!gridReady || !detailReady) {
+            writeDevJson(res, 409, {
+              ok: false,
+              error: "Artifact inventory detail view is not ready for probe-click.",
+              captureTarget: before.captureTarget,
+              grid: grid ? { rows: grid.rows, cols: grid.cols, source: grid.source, confidence: grid.confidence } : null,
+              artifactDetail: detail ?? null,
+            });
+            return;
+          }
          const centers = before.inventoryGrid?.centers ?? [];
          const target = Number.isFinite(requestedRow) && Number.isFinite(requestedCol)
            ? centers.find((center) => center.row === requestedRow && center.col === requestedCol)