feat(scanner): validate elevated live automation

2026-07-07 07:49:22 +02:00
parent 7930e369a7
commit ef65c3e6a0
37 changed files with 826 additions and 217 deletions
@@ -0,0 +1,256 @@
+import fs from "node:fs/promises";
+import http, { type Server } from "node:http";
+import path from "node:path";
+import type {
+  CaptureOptions,
+  CaptureResult,
+  CaptureSourceInfo,
+  ClickResult,
+  ReviewSampleListResult,
+  ScannerCommand,
+  ScannerStatusPayload,
+} from "../src/types/global.js";
+
+interface DevControlServerDependencies {
+  registeredHotkeys: Record<string, boolean>;
+  hasMainWindow: () => boolean;
+  sendScannerCommand: (command: ScannerCommand | "probe-click") => void;
+  clickScreen: (x: number, y: number) => Promise<ClickResult>;
+  scannerStatus: () => ScannerStatusPayload;
+  loadReviewSamples: (limit?: number) => Promise<ReviewSampleListResult>;
+  listCaptureSources: () => Promise<CaptureSourceInfo[]>;
+  captureSource: (
+    id: string,
+    delayMs?: number,
+    focusGenshin?: boolean,
+    options?: CaptureOptions,
+  ) => Promise<CaptureResult>;
+}
+
+function writeDevJson(res: http.ServerResponse, statusCode: number, payload: unknown) {
+  res.writeHead(statusCode, {
+    "content-type": "application/json; charset=utf-8",
+    "cache-control": "no-store",
+  });
+  res.end(JSON.stringify(payload));
+}
+
+function dataUrlBase64(dataUrl: string) {
+  return dataUrl.replace(/^data:image\/png;base64,/, "");
+}
+
+function devCaptureOutputDir() {
+  return path.join(process.cwd(), "outputs", "live-capture");
+}
+
+function safeDebugFilePart(value: string) {
+  return value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "capture";
+}
+
+function dataUrlFingerprint(dataUrl: string | undefined) {
+  if (!dataUrl) return "";
+  let hash = 2166136261;
+  const stride = Math.max(1, Math.floor(dataUrl.length / 4096));
+  for (let index = 0; index < dataUrl.length; index += stride) {
+    hash ^= dataUrl.charCodeAt(index);
+    hash = Math.imul(hash, 16777619);
+  }
+  return `${dataUrl.length.toString(16)}:${(hash >>> 0).toString(16)}`;
+}
+
+async function writeDevCaptureImage(filePath: string, dataUrl: string | undefined) {
+  if (!dataUrl) return null;
+  await fs.writeFile(filePath, Buffer.from(dataUrlBase64(dataUrl), "base64"));
+  return filePath;
+}
+
+async function writeDevCaptureSnapshot(capture: CaptureResult) {
+  const outputDir = devCaptureOutputDir();
+  await fs.mkdir(outputDir, { recursive: true });
+  const stamp = new Date().toISOString().replace(/[\\/:]/g, "-").replace(/\..+?$/, "");
+  const prefix = safeDebugFilePart(`${stamp}-${capture.name}`);
+  const files = {
+    full: await writeDevCaptureImage(path.join(outputDir, `${prefix}-full.png`), capture.dataUrl),
+    detail: await writeDevCaptureImage(path.join(outputDir, `${prefix}-detail.png`), capture.detailDataUrl),
+    inventory: await writeDevCaptureImage(path.join(outputDir, `${prefix}-inventory.png`), capture.inventoryDataUrl),
+    crops: [] as Array<{ id: string; label: string; path: string; rect: { x: number; y: number; width: number; height: number } }>,
+  };
+
+  for (const crop of capture.crops ?? []) {
+    const cropPath = path.join(outputDir, `${prefix}-${safeDebugFilePart(crop.id)}.png`);
+    const written = await writeDevCaptureImage(cropPath, crop.dataUrl);
+    if (written) files.crops.push({ id: crop.id, label: crop.label, path: written, rect: crop.rect });
+  }
+
+  const summary = {
+    id: capture.id,
+    name: capture.name,
+    width: capture.width,
+    height: capture.height,
+    capturedAt: capture.capturedAt,
+    captureTarget: capture.captureTarget,
+    ocrSkipped: capture.ocrSkipped,
+    ocrTimedOut: capture.ocrTimedOut,
+    layout: capture.layout,
+    inventoryGrid: capture.inventoryGrid
+      ? {
+          rows: capture.inventoryGrid.rows,
+          cols: capture.inventoryGrid.cols,
+          confidence: capture.inventoryGrid.confidence,
+          source: capture.inventoryGrid.source,
+          firstCenter: capture.inventoryGrid.centers[0] ?? null,
+          lastCenter: capture.inventoryGrid.centers.at(-1) ?? null,
+        }
+      : null,
+    inventoryCount: capture.inventoryCount ?? null,
+    locked: capture.locked,
+    crops: (capture.crops ?? []).map((crop) => ({ id: crop.id, label: crop.label, rect: crop.rect })),
+    ocr: capture.ocr ?? [],
+    files,
+  };
+  const summaryPath = path.join(outputDir, `${prefix}-summary.json`);
+  await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2), "utf8");
+  return { ...summary, summaryPath };
+}
+
+function wait(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function findGenshinSource(sources: CaptureSourceInfo[], sourceId: string | null) {
+  return sourceId
+    ? sources.find((entry) => entry.id === sourceId)
+    : sources.find((entry) => entry.isGenshinCandidate);
+}
+
+function sourceListForError(sources: CaptureSourceInfo[]) {
+  return sources.map(({ id, name, isGenshinCandidate }) => ({ id, name, isGenshinCandidate }));
+}
+
+export function createDevControlServer(deps: DevControlServerDependencies): Server {
+  const server = http.createServer((req, res) => {
+    if (req.socket.remoteAddress && !["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress)) {
+      writeDevJson(res, 403, { ok: false, error: "local only" });
+      return;
+    }
+
+    const url = new URL(req.url ?? "/", "http://127.0.0.1");
+    if (url.pathname === "/health") {
+      writeDevJson(res, 200, { ok: true, hotkeys: deps.registeredHotkeys, hasWindow: deps.hasMainWindow() });
+      return;
+    }
+    if (url.pathname === "/scanner/start") {
+      const limit = Number(url.searchParams.get("limit") ?? Number.NaN);
+      const command: ScannerCommand = Number.isFinite(limit) && limit > 0
+        ? { type: "start-auto", scanLimit: limit }
+        : "start-auto";
+      deps.sendScannerCommand(command);
+      writeDevJson(res, 200, { ok: true, command });
+      return;
+    }
+    if (url.pathname === "/scanner/stop") {
+      deps.sendScannerCommand("stop");
+      writeDevJson(res, 200, { ok: true, command: "stop" });
+      return;
+    }
+    if (url.pathname === "/scanner/probe") {
+      deps.sendScannerCommand("probe-click");
+      writeDevJson(res, 200, { ok: true, command: "probe-click" });
+      return;
+    }
+    if (url.pathname === "/automation/click") {
+      const x = Number(url.searchParams.get("x"));
+      const y = Number(url.searchParams.get("y"));
+      if (!Number.isFinite(x) || !Number.isFinite(y)) {
+        writeDevJson(res, 400, { ok: false, error: "x and y query params are required" });
+        return;
+      }
+      deps.clickScreen(Math.round(x), Math.round(y))
+        .then((payload: unknown) => writeDevJson(res, 200, { ok: true, payload }))
+        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
+      return;
+    }
+    if (url.pathname === "/scanner/status") {
+      writeDevJson(res, 200, { ok: true, status: deps.scannerStatus() });
+      return;
+    }
+    if (url.pathname === "/review/samples") {
+      deps.loadReviewSamples(Number(url.searchParams.get("limit") ?? 20))
+        .then((payload: unknown) => writeDevJson(res, 200, payload))
+        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
+      return;
+    }
+    if (url.pathname === "/capture/smart") {
+      const sourceId = url.searchParams.get("sourceId");
+      const focus = url.searchParams.get("focus") !== "0";
+      const skipOcr = url.searchParams.get("skipOcr") === "1";
+      deps.listCaptureSources()
+        .then(async (sources) => {
+          const source = findGenshinSource(sources, sourceId);
+          if (!source) {
+            writeDevJson(res, 404, { ok: false, error: "No Genshin capture source found.", sources: sourceListForError(sources) });
+            return;
+          }
+          const capture = await deps.captureSource(source.id, 250, focus, { skipOcr });
+          const summary = await writeDevCaptureSnapshot(capture);
+          writeDevJson(res, 200, { ok: true, source: { id: source.id, name: source.name }, summary });
+        })
+        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
+      return;
+    }
+    if (url.pathname === "/automation/probe-click") {
+      const sourceId = url.searchParams.get("sourceId");
+      const requestedIndex = Number(url.searchParams.get("index") ?? "1");
+      const requestedRow = Number(url.searchParams.get("row") ?? Number.NaN);
+      const requestedCol = Number(url.searchParams.get("col") ?? Number.NaN);
+      deps.listCaptureSources()
+        .then(async (sources) => {
+          const source = findGenshinSource(sources, sourceId);
+          if (!source) {
+            writeDevJson(res, 404, { ok: false, error: "No Genshin capture source found.", sources: sourceListForError(sources) });
+            return;
+          }
+
+          const before = await deps.captureSource(source.id, 150, true, { skipOcr: true });
+          const centers = before.inventoryGrid?.centers ?? [];
+          const target = Number.isFinite(requestedRow) && Number.isFinite(requestedCol)
+            ? centers.find((center) => center.row === requestedRow && center.col === requestedCol)
+            : centers[Math.max(0, Math.min(centers.length - 1, Number.isFinite(requestedIndex) ? requestedIndex : 1))];
+          if (!target) {
+            writeDevJson(res, 409, { ok: false, error: "No inventory grid target available.", grid: before.inventoryGrid ?? null });
+            return;
+          }
+
+          const beforeFingerprint = dataUrlFingerprint(before.detailDataUrl);
+          const click = await deps.clickScreen(target.x, target.y);
+          await wait(650);
+          const after = await deps.captureSource(source.id, 0, true, { skipOcr: true });
+          const afterFingerprint = dataUrlFingerprint(after.detailDataUrl);
+          const changed = Boolean(beforeFingerprint && afterFingerprint && beforeFingerprint !== afterFingerprint);
+          writeDevJson(res, 200, {
+            ok: Boolean(click.ok && click.clicked && changed),
+            changed,
+            target,
+            click,
+            before: {
+              captureTarget: before.captureTarget,
+              grid: before.inventoryGrid ? { rows: before.inventoryGrid.rows, cols: before.inventoryGrid.cols, source: before.inventoryGrid.source, confidence: before.inventoryGrid.confidence } : null,
+              detailFingerprint: beforeFingerprint,
+            },
+            after: {
+              captureTarget: after.captureTarget,
+              grid: after.inventoryGrid ? { rows: after.inventoryGrid.rows, cols: after.inventoryGrid.cols, source: after.inventoryGrid.source, confidence: after.inventoryGrid.confidence } : null,
+              detailFingerprint: afterFingerprint,
+            },
+          });
+        })
+        .catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
+      return;
+    }
+
+    writeDevJson(res, 404, { ok: false, error: "unknown endpoint" });
+  });
+
+  server.listen(17317, "127.0.0.1");
+  return server;
+}