feat(scanner): validate elevated live automation

This commit is contained in:
AzuTear
2026-07-07 07:49:22 +02:00
parent 7930e369a7
commit ef65c3e6a0
37 changed files with 826 additions and 217 deletions
+56 -65
View File
@@ -1,19 +1,22 @@
import { app, BrowserWindow, Menu, desktopCapturer, globalShortcut, nativeImage, screen, type NativeImage } from "electron";
import { app, BrowserWindow, Menu, desktopCapturer, dialog, globalShortcut, nativeImage, screen, type NativeImage } from "electron";
import fs from "node:fs/promises";
import { existsSync } from "node:fs";
import http, { type Server } from "node:http";
import type { Server } from "node:http";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { createWorker } from "tesseract.js";
import { createInputHelperService, type InputHelperService } from "./services/inputHelper.js";
import { createRepositoryContext, type RepositoryContext } from "./bootstrap/repositoryContext.js";
import { registerIpcHandlers } from "./bootstrap/ipcBootstrap.js";
import { createDevControlServer } from "./devControlServer.js";
import type { AppSnapshot } from "../src/types/domain.js";
import type {
CaptureOptions,
CaptureResult,
GoodDatabase,
GoodImportFileResult,
SaveResultWithPath,
ScannerCommand,
ScannerLearningRulePayload,
ScannerStatusPayload,
} from "../src/types/global.js";
@@ -33,6 +36,7 @@ import {
profileDetailRect,
} from "../src/lib/layoutProfile.js";
import { binarizeForOcr } from "../src/lib/ocrPreprocess.js";
import { detectLockState, lockIconCropRect } from "../src/lib/lockDetection.js";
// Chromium's renderer sandbox can refuse to fully initialize (or silently
// crash the GPU/renderer process) when the hosting process runs with a full
@@ -416,7 +420,7 @@ function focusMainWindow() {
return { ok: true };
}
function sendScannerCommand(command: "start-auto" | "stop" | "probe-click") {
function sendScannerCommand(command: ScannerCommand | "probe-click") {
if (!mainWindow || mainWindow.isDestroyed()) return;
mainWindow.webContents.send("scanner:command", command);
}
@@ -431,71 +435,18 @@ function registerScannerHotkeys() {
};
}
function writeDevJson(res: http.ServerResponse, statusCode: number, payload: unknown) {
res.writeHead(statusCode, {
"content-type": "application/json; charset=utf-8",
"cache-control": "no-store",
});
res.end(JSON.stringify(payload));
}
function startDevControlServer() {
if (!isDev || devControlServer) return;
devControlServer = http.createServer((req, res) => {
if (req.socket.remoteAddress && !["127.0.0.1", "::1", "::ffff:127.0.0.1"].includes(req.socket.remoteAddress)) {
writeDevJson(res, 403, { ok: false, error: "local only" });
return;
}
const url = new URL(req.url ?? "/", "http://127.0.0.1");
if (url.pathname === "/health") {
writeDevJson(res, 200, { ok: true, hotkeys: registeredHotkeys, hasWindow: Boolean(mainWindow && !mainWindow.isDestroyed()) });
return;
}
if (url.pathname === "/scanner/start") {
sendScannerCommand("start-auto");
writeDevJson(res, 200, { ok: true, command: "start-auto" });
return;
}
if (url.pathname === "/scanner/stop") {
sendScannerCommand("stop");
writeDevJson(res, 200, { ok: true, command: "stop" });
return;
}
if (url.pathname === "/scanner/probe") {
sendScannerCommand("probe-click");
writeDevJson(res, 200, { ok: true, command: "probe-click" });
return;
}
if (url.pathname === "/automation/click") {
const x = Number(url.searchParams.get("x"));
const y = Number(url.searchParams.get("y"));
if (!Number.isFinite(x) || !Number.isFinite(y)) {
writeDevJson(res, 400, { ok: false, error: "x and y query params are required" });
return;
}
getInputHelperService()
.clickScreen(Math.round(x), Math.round(y))
.then((payload: unknown) => writeDevJson(res, 200, { ok: true, payload }))
.catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
return;
}
if (url.pathname === "/scanner/status") {
writeDevJson(res, 200, { ok: true, status: scannerDevStatus });
return;
}
if (url.pathname === "/review/samples") {
loadReviewSamples(Number(url.searchParams.get("limit") ?? 20))
.then((payload: unknown) => writeDevJson(res, 200, payload))
.catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
return;
}
writeDevJson(res, 404, { ok: false, error: "unknown endpoint" });
devControlServer = createDevControlServer({
registeredHotkeys,
hasMainWindow: () => Boolean(mainWindow && !mainWindow.isDestroyed()),
sendScannerCommand,
clickScreen: clickScreenCommand,
scannerStatus: () => scannerDevStatus,
loadReviewSamples,
listCaptureSources,
captureSource,
});
devControlServer.listen(17317, "127.0.0.1");
}
function createOverlayWindow() {
@@ -802,6 +753,10 @@ function createCrops(
}
function inferDetailRect(bitmap: Buffer, imageSize: { width: number; height: number }) {
if (isSixteenNine(imageSize)) {
return profileDetailRect(imageSize);
}
const { width, height } = imageSize;
const sampleStrideX = width > 2200 ? 4 : 3;
const sampleStrideY = height > 1400 ? 4 : 3;
@@ -869,6 +824,12 @@ async function buildCaptureResult(
const detailRect = inferDetailRect(bitmap, size);
const inventoryRect = inferInventoryRect(size, detailRect);
const crops = createCrops(sourceImage, size, detailRect, inventoryRect);
const lockRect = clampCaptureRect(lockIconCropRect(detailRect, size), size);
const lockImage = sourceImage.crop(lockRect);
const lockSize = lockImage.getSize();
const locked = lockSize.width > 0 && lockSize.height > 0
? detectLockState({ data: lockImage.getBitmap(), width: lockSize.width, height: lockSize.height })
: undefined;
const croppedPayload = crops.map((crop) => ({
id: crop.id,
label: crop.label,
@@ -905,6 +866,7 @@ async function buildCaptureResult(
})),
inventoryGrid: inferInventoryGrid(size, detailRect),
inventoryCount: count,
locked,
layout: {
aspect: aspectRatioLabel(size),
isSixteenNine: isSixteenNine(size),
@@ -965,6 +927,34 @@ async function exportGood(payload: GoodDatabase): Promise<SaveResultWithPath> {
}
}
async function importGoodFile(): Promise<GoodImportFileResult> {
const dialogOptions = {
title: "GOOD-Datei importieren",
properties: ["openFile"],
filters: [{ name: "GOOD JSON", extensions: ["json"] }],
} satisfies Electron.OpenDialogOptions;
const dialogResult = mainWindow && !mainWindow.isDestroyed()
? await dialog.showOpenDialog(mainWindow, dialogOptions)
: await dialog.showOpenDialog(dialogOptions);
if (dialogResult.canceled || dialogResult.filePaths.length === 0) {
return { ok: false, canceled: true, path: "" };
}
const filePath = dialogResult.filePaths[0];
try {
const text = await fs.readFile(filePath, "utf8");
return { ok: true, canceled: false, path: filePath, database: JSON.parse(text) };
} catch (error) {
return {
ok: false,
canceled: false,
path: filePath,
error: error instanceof Error ? error.message : String(error),
};
}
}
function initializeAppLifecycle() {
app.whenReady().then(() => {
const userDataPath = app.getPath("userData");
@@ -993,6 +983,7 @@ function initializeAppLifecycle() {
loadScannerLearningRules: () => loadScannerLearningRules(),
writeScannerLearningRules: (rules: ScannerLearningRulePayload) => writeScannerLearningRules(rules),
exportGood: (exportPayload: GoodDatabase) => exportGood(exportPayload),
importGoodFile: () => importGoodFile(),
listSources: () => listCaptureSources(),
captureSource: (
id: string,