AzuTear
1519 lines
53 KiB
TypeScript
1519 lines
53 KiB
TypeScript
import { app, desktopCapturer, globalShortcut, nativeImage, screen, type NativeImage } from "electron";
|
|
import { existsSync } from "node:fs";
|
|
import type { Server } from "node:http";
|
|
import { cpus } from "node:os";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { createWorker, PSM } from "tesseract.js";
|
|
import { createInputHelperService, type InputHelperService } from "./services/inputHelper.js";
|
|
import { pngBufferToBitmap } from "./services/pngBitmap.js";
|
|
import { createRepositoryContext, type RepositoryContext } from "./bootstrap/repositoryContext.js";
|
|
import { registerIpcHandlers } from "./bootstrap/ipcBootstrap.js";
|
|
import { createDevControlServer } from "./devControlServer.js";
|
|
import { createAppWindowManager, type AppWindowManager } from "./appWindowManager.js";
|
|
import { createGoodFileService, type GoodFileService } from "./services/goodFileService.js";
|
|
import type { AppSnapshot } from "../src/types/domain.js";
|
|
import type {
|
|
CaptureOptions,
|
|
CaptureResult,
|
|
GoodDatabase,
|
|
OcrResult,
|
|
AppRuntimeInfo,
|
|
ScannerCommand,
|
|
ScannerLearningRulePayload,
|
|
ScannerStatusPayload,
|
|
} from "../src/types/global.js";
|
|
import type {
|
|
ArtifactStoreRepositoryPort,
|
|
ReviewSamplesRepositoryPort,
|
|
ScannerLearningRepositoryPort,
|
|
} from "./repositories/index.js";
|
|
import {
|
|
aspectRatioLabel,
|
|
detailCropRects,
|
|
inventoryCountCropRect,
|
|
inventoryGrid as layoutInventoryGrid,
|
|
inventoryRect as layoutInventoryRect,
|
|
isSixteenNine,
|
|
layoutSupportWarning,
|
|
profileDetailRect,
|
|
} from "../src/lib/layoutProfile.js";
|
|
import { binarizeForOcr } from "../src/lib/ocrPreprocess.js";
|
|
import { DEFAULT_LOCK_THRESHOLD, isLocked, lockIconCropRect, lockSignalRatio } from "../src/lib/lockDetection.js";
|
|
|
|
// Chromium's renderer sandbox can refuse to fully initialize (or silently
|
|
// crash the GPU/renderer process) when the hosting process runs with a full
|
|
// Administrator token - a well-known Electron-on-Windows-elevation quirk.
|
|
// This app already requires elevation for input automation and never loads
|
|
// untrusted remote content, so the renderer sandbox has little security
|
|
// value here; disabling it avoids "works normally, fails only when run as
|
|
// admin" failures with no visible error. Must run before app is ready.
|
|
app.commandLine.appendSwitch("no-sandbox");
|
|
app.commandLine.appendSwitch("disable-gpu-sandbox");
|
|
|
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
const isDev = Boolean(process.env.VITE_DEV_SERVER_URL);
|
|
const APP_RUNTIME_STARTED_AT = new Date().toISOString();
|
|
const APP_RUNTIME_SIGNATURE = "2026-07-08-direct-gdi-reviewfix";
|
|
|
|
let registeredHotkeys: Record<string, boolean> = {};
|
|
let devControlServer: Server | null = null;
|
|
const captureSourceNameCache = new Map<string, string>();
|
|
let scannerDevStatus: ScannerStatusPayload = {
|
|
running: false,
|
|
reviewStatus: "",
|
|
captureStatus: "",
|
|
selectedSource: null,
|
|
stats: {},
|
|
summary: null,
|
|
snapshotArtifacts: 0,
|
|
snapshotCharacters: 0,
|
|
snapshotRecommendations: 0,
|
|
snapshotBuilds: 0,
|
|
grid: null,
|
|
automationLog: [],
|
|
runtimeInfo: null,
|
|
storedTotal: null,
|
|
learningRuleCount: 0,
|
|
updatedAt: null,
|
|
};
|
|
let repositoryContext: RepositoryContext | null = null;
|
|
let artifactStoreRepository: ArtifactStoreRepositoryPort | null = null;
|
|
let reviewSamplesRepository: ReviewSamplesRepositoryPort | null = null;
|
|
let scannerLearningRepository: ScannerLearningRepositoryPort | null = null;
|
|
let inputHelperService: InputHelperService | null = null;
|
|
let appWindowManager: AppWindowManager | null = null;
|
|
let goodFileService: GoodFileService | null = null;
|
|
|
|
function getInputHelperService() {
|
|
if (!inputHelperService) {
|
|
throw new Error("Input-helper service has not been initialized.");
|
|
}
|
|
return inputHelperService;
|
|
}
|
|
|
|
// Locate the compiled C# input/capture sidecar (ADR-008). Falls back to null so
|
|
// the service uses the embedded PowerShell helper when the exe was never built.
|
|
function resolveInputHelperExePath(): string | null {
|
|
const candidates = [
|
|
process.env.INPUT_HELPER_EXE,
|
|
path.join(process.resourcesPath, "input-helper", "InputHelper.exe"),
|
|
path.join(process.cwd(), "native", "input-helper", "bin", "publish", "InputHelper.exe"),
|
|
path.join(app.getAppPath(), "native", "input-helper", "bin", "publish", "InputHelper.exe"),
|
|
].filter((candidate): candidate is string => Boolean(candidate));
|
|
|
|
for (const candidate of candidates) {
|
|
try {
|
|
if (existsSync(candidate)) return candidate;
|
|
} catch {
|
|
// Unreadable path; try the next candidate.
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function getRepositoryContext() {
|
|
if (!repositoryContext) {
|
|
throw new Error("Repository context has not been initialized.");
|
|
}
|
|
return repositoryContext;
|
|
}
|
|
|
|
function artifactStorePath() {
|
|
return getRepositoryContext().artifactStorePath;
|
|
}
|
|
|
|
function reviewSamplesPath() {
|
|
return getRepositoryContext().reviewSamplesPath;
|
|
}
|
|
|
|
function scannerLearningPath() {
|
|
return getRepositoryContext().scannerLearningPath;
|
|
}
|
|
|
|
function getReviewSamplesRepository() {
|
|
if (!reviewSamplesRepository) {
|
|
throw new Error("Review-sample repository is not initialized.");
|
|
}
|
|
return reviewSamplesRepository;
|
|
}
|
|
|
|
async function loadReviewSamples(limit = 50) {
|
|
try {
|
|
const repository = getReviewSamplesRepository();
|
|
return repository.list(Math.max(1, Math.min(200, Number(limit) || 50)));
|
|
} catch {
|
|
return { ok: true, samples: [], total: 0, path: reviewSamplesPath() };
|
|
}
|
|
}
|
|
|
|
async function loadScannerLearningRules() {
|
|
try {
|
|
return await getScannerLearningRepository().load();
|
|
} catch {
|
|
return { ok: true, path: scannerLearningPath(), rules: { textReplacements: {} } };
|
|
}
|
|
}
|
|
|
|
function getScannerLearningRepository() {
|
|
if (!scannerLearningRepository) {
|
|
throw new Error("Scanner-learning repository is not initialized.");
|
|
}
|
|
return scannerLearningRepository;
|
|
}
|
|
|
|
async function writeScannerLearningRules(rules: ScannerLearningRulePayload) {
|
|
const safeRules = rules && typeof rules === "object" ? rules : {};
|
|
try {
|
|
return await getScannerLearningRepository().save(safeRules);
|
|
} catch {
|
|
return { ok: true, path: scannerLearningPath(), rules: { textReplacements: {} }, total: 0 };
|
|
}
|
|
}
|
|
|
|
function getArtifactStoreRepository() {
|
|
if (!artifactStoreRepository) {
|
|
throw new Error("Artifact store repository is not initialized.");
|
|
}
|
|
return artifactStoreRepository;
|
|
}
|
|
|
|
function getSnapshotRepository() {
|
|
const context = getRepositoryContext();
|
|
if (!context.snapshotRepository) {
|
|
throw new Error("Snapshot repository is not initialized.");
|
|
}
|
|
return context.snapshotRepository;
|
|
}
|
|
|
|
function getGoodFileService() {
|
|
if (!goodFileService) {
|
|
throw new Error("GOOD file service is not initialized.");
|
|
}
|
|
return goodFileService;
|
|
}
|
|
|
|
async function loadSnapshotFromDisk() {
|
|
try {
|
|
return await getSnapshotRepository().load();
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function saveSnapshotToDisk(snapshot: AppSnapshot) {
|
|
try {
|
|
return await getSnapshotRepository().save(snapshot);
|
|
} catch {
|
|
return { ok: false, path: "" };
|
|
}
|
|
}
|
|
|
|
async function publishScannerStatus(status: ScannerStatusPayload) {
|
|
scannerDevStatus = { ...status, updatedAt: new Date().toISOString() };
|
|
return { ok: true };
|
|
}
|
|
|
|
async function readRuntimeInfo() {
|
|
try {
|
|
const result = await getInputHelperService().getRuntimeInfo();
|
|
return {
|
|
ok: true,
|
|
isElevated: result.isElevated,
|
|
platform: result.platform,
|
|
appBuild: appRuntimeInfo(),
|
|
hotkeys: registeredHotkeys,
|
|
genshinFound: result.genshinFound,
|
|
genshinHwnd: result.genshinHwnd ?? undefined,
|
|
targetProcess: result.targetProcess,
|
|
foregroundProcess: result.foregroundProcess,
|
|
foregroundHwnd: result.foregroundHwnd ?? undefined,
|
|
helperPid: result.helperPid,
|
|
};
|
|
} catch {
|
|
return { ok: false, isElevated: false, platform: process.platform, appBuild: appRuntimeInfo(), hotkeys: registeredHotkeys };
|
|
}
|
|
}
|
|
|
|
async function focusGenshinWindow() {
|
|
try {
|
|
const result = await getInputHelperService().focusGenshinWindow();
|
|
return result;
|
|
} catch {
|
|
return { focused: false, alreadyForeground: false, genshinFound: false };
|
|
}
|
|
}
|
|
|
|
async function focusGenshinForScanStart() {
|
|
try {
|
|
return await getInputHelperService().focusGenshinForScanStart();
|
|
} catch {
|
|
return { focused: false, alreadyForeground: false, genshinFound: false };
|
|
}
|
|
}
|
|
|
|
async function getGenshinWindowBounds() {
|
|
try {
|
|
return await getInputHelperService().getGenshinWindowBounds();
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// Our own dashboard window defaulted to Electron's normal placement, which
|
|
// (on this exact reported bug) ended up sitting directly on top of Genshin's
|
|
// fullscreen window on the same monitor - so every simulated click that
|
|
// looked correct (focused, on-target, injected) was actually landing on our
|
|
// own window, not the game, since mouse hit-testing goes by which window is
|
|
// topmost at that screen pixel, not by which window has keyboard focus. If a
|
|
// second display exists and isn't the one Genshin occupies, move the
|
|
// dashboard there so it can never cover the grid we're about to click.
|
|
async function moveMainWindowOffGenshin() {
|
|
const mainWindow = getAppWindowManager().getMainWindow();
|
|
if (!mainWindow || mainWindow.isDestroyed()) return;
|
|
const genshinBounds = await getGenshinWindowBounds();
|
|
const displays = screen.getAllDisplays();
|
|
if (displays.length < 2) return;
|
|
|
|
const genshinCenter = genshinBounds
|
|
? { x: genshinBounds.x + genshinBounds.width / 2, y: genshinBounds.y + genshinBounds.height / 2 }
|
|
: null;
|
|
const genshinDisplay = genshinCenter
|
|
? screen.getDisplayNearestPoint(genshinCenter)
|
|
: screen.getPrimaryDisplay();
|
|
|
|
const otherDisplay = displays.find((d) => d.id !== genshinDisplay.id);
|
|
if (!otherDisplay) return;
|
|
|
|
const currentBounds = mainWindow.getBounds();
|
|
const alreadyOnOtherDisplay = screen.getDisplayMatching(currentBounds).id === otherDisplay.id;
|
|
if (alreadyOnOtherDisplay) return;
|
|
|
|
const area = otherDisplay.workArea;
|
|
const width = Math.min(currentBounds.width, area.width - 40);
|
|
const height = Math.min(currentBounds.height, area.height - 40);
|
|
mainWindow.setBounds({
|
|
x: Math.round(area.x + (area.width - width) / 2),
|
|
y: Math.round(area.y + (area.height - height) / 2),
|
|
width: Math.round(width),
|
|
height: Math.round(height),
|
|
});
|
|
}
|
|
|
|
async function runMockScan() {
|
|
return null;
|
|
}
|
|
|
|
async function showOverlayWindow() {
|
|
createOverlayWindow();
|
|
return { ok: true };
|
|
}
|
|
|
|
async function hideOverlayWindow() {
|
|
return getAppWindowManager().hideOverlayWindow();
|
|
}
|
|
|
|
async function listCaptureSources() {
|
|
const displays = screen.getAllDisplays();
|
|
const maxSize = displays.reduce(
|
|
(size, display) => ({
|
|
width: Math.max(size.width, display.size.width),
|
|
height: Math.max(size.height, display.size.height),
|
|
}),
|
|
{ width: 1920, height: 1080 },
|
|
);
|
|
|
|
const sources = await desktopCapturer.getSources({
|
|
types: ["window", "screen"],
|
|
thumbnailSize: maxSize,
|
|
fetchWindowIcons: true,
|
|
});
|
|
|
|
return sources.map((source) => {
|
|
captureSourceNameCache.set(source.id, source.name);
|
|
return {
|
|
id: source.id,
|
|
name: source.name,
|
|
isGenshinCandidate: isLikelyGenshinSourceName(source.name),
|
|
thumbnailDataUrl: source.thumbnail.resize({ width: 420 }).toDataURL(),
|
|
};
|
|
});
|
|
}
|
|
|
|
async function toScreenPoint(x: number, y: number) {
|
|
const point = { x: Math.round(x), y: Math.round(y) };
|
|
const bounds = await getGenshinWindowBounds();
|
|
if (!bounds) return point;
|
|
|
|
const looksClientRelative =
|
|
point.x >= 0 &&
|
|
point.y >= 0 &&
|
|
point.x <= bounds.width &&
|
|
point.y <= bounds.height &&
|
|
(bounds.x !== 0 || bounds.y !== 0);
|
|
if (!looksClientRelative) return point;
|
|
|
|
return {
|
|
x: bounds.x + point.x,
|
|
y: bounds.y + point.y,
|
|
};
|
|
}
|
|
|
|
async function clickScreenCommand(x: number, y: number) {
|
|
const point = await toScreenPoint(x, y);
|
|
return getInputHelperService().clickScreen(point.x, point.y);
|
|
}
|
|
|
|
async function scrollScreenCommand(notches: number, anchorX?: number, anchorY?: number) {
|
|
if (typeof anchorX === "number" && typeof anchorY === "number") {
|
|
const point = await toScreenPoint(anchorX, anchorY);
|
|
return getInputHelperService().scrollScreen(notches, point.x, point.y);
|
|
}
|
|
return getInputHelperService().scrollScreen(notches);
|
|
}
|
|
|
|
async function keyPressCommand(key: string) {
|
|
return getInputHelperService().keyPress(key);
|
|
}
|
|
|
|
async function getAutomationGuardCommand() {
|
|
const result = await getInputHelperService().getAutomationGuard();
|
|
return {
|
|
...result,
|
|
ok: true,
|
|
// Preserve historical automation metadata shape expected by callers.
|
|
isElevated: typeof result.isElevated === "boolean" ? result.isElevated : false,
|
|
};
|
|
}
|
|
|
|
function isLikelyGenshinSourceName(sourceName: string) {
|
|
const lowered = sourceName.toLowerCase();
|
|
return (
|
|
lowered.includes("genshin")
|
|
|| lowered.includes("genshinimpact")
|
|
|| lowered.includes("yuanshen")
|
|
|| sourceName.includes("\u539f\u795e")
|
|
);
|
|
}
|
|
|
|
async function capturePrimaryScreenViaGdi() {
|
|
return getInputHelperService().capturePrimaryScreenViaGdi();
|
|
}
|
|
|
|
async function captureSourceFromGdi(sourceId: string, sourceName: string, options: CaptureOptions = {}) {
|
|
const gdi = await capturePrimaryScreenViaGdi();
|
|
const sourceImage = nativeImageFromGdiCapture(gdi);
|
|
return await buildCaptureResult(sourceImage, sourceId, sourceName, gdi.captureTarget, options);
|
|
}
|
|
|
|
function nativeImageFromGdiCapture(gdi: Awaited<ReturnType<InputHelperService["capturePrimaryScreenViaGdi"]>>) {
|
|
if (gdi.imageBase64) {
|
|
return nativeImage.createFromBuffer(Buffer.from(gdi.imageBase64, "base64"));
|
|
}
|
|
if (!gdi.dataUrl) {
|
|
throw new Error("GDI capture returned no image payload.");
|
|
}
|
|
return nativeImage.createFromDataURL(gdi.dataUrl);
|
|
}
|
|
|
|
function shouldUseDirectGdiHotPath(options: CaptureOptions = {}) {
|
|
return Boolean(
|
|
options.ocrMode === "artifact" ||
|
|
options.skipOcrUnlessArtifactDetail ||
|
|
options.skipOcr ||
|
|
options.omitCrops,
|
|
);
|
|
}
|
|
|
|
function getAppWindowManager() {
|
|
if (!appWindowManager) {
|
|
appWindowManager = createAppWindowManager({
|
|
preloadPath: path.join(__dirname, "preload.cjs"),
|
|
rendererUrl: process.env.VITE_DEV_SERVER_URL,
|
|
rendererFilePath: path.join(__dirname, "../../dist/index.html"),
|
|
onMainReadyToShow: moveMainWindowOffGenshin,
|
|
});
|
|
}
|
|
return appWindowManager;
|
|
}
|
|
|
|
function createMainWindow() {
|
|
getAppWindowManager().createMainWindow();
|
|
}
|
|
|
|
function focusMainWindow() {
|
|
return getAppWindowManager().focusMainWindow();
|
|
}
|
|
|
|
function sendScannerCommand(command: ScannerCommand | "probe-click") {
|
|
getAppWindowManager().sendScannerCommand(command);
|
|
}
|
|
|
|
function registerScannerHotkeys() {
|
|
globalShortcut.unregisterAll();
|
|
registeredHotkeys = {
|
|
"Ctrl+Shift+S": globalShortcut.register("CommandOrControl+Shift+S", () => sendScannerCommand("start-auto")),
|
|
"Ctrl+Shift+X": globalShortcut.register("CommandOrControl+Shift+X", () => sendScannerCommand("stop")),
|
|
F8: globalShortcut.register("F8", () => sendScannerCommand("start-auto")),
|
|
F9: globalShortcut.register("F9", () => sendScannerCommand("stop")),
|
|
};
|
|
}
|
|
|
|
function startDevControlServer() {
|
|
if (!isDev || devControlServer) return;
|
|
devControlServer = createDevControlServer({
|
|
registeredHotkeys,
|
|
appBuild: appRuntimeInfo(),
|
|
hasMainWindow: () => getAppWindowManager().hasMainWindow(),
|
|
sendScannerCommand,
|
|
clickScreen: clickScreenCommand,
|
|
scannerStatus: () => ({ ...scannerDevStatus, appBuild: appRuntimeInfo(), ocrWarmup: getOcrWarmupStatus() }),
|
|
warmOcr: (engine) => warmOcrWorkerPool(engine),
|
|
loadReviewSamples,
|
|
listCaptureSources,
|
|
captureSource,
|
|
requestShutdown: (reason) => {
|
|
console.log(`[dev-control] shutdown requested: ${reason}`);
|
|
app.quit();
|
|
},
|
|
});
|
|
}
|
|
|
|
function createOverlayWindow() {
|
|
getAppWindowManager().createOverlayWindow();
|
|
}
|
|
|
|
// Inventory Kamera keeps a pool of native Tesseract engines and scans artifact
|
|
// fields concurrently. Our fast artifact profile has four useful OCR parameter
|
|
// groups, so the default pool is four workers unless the machine is smaller or
|
|
// GAA_OCR_WORKERS explicitly overrides it.
|
|
const OCR_WORKER_POOL_SIZE = resolveOcrWorkerPoolSize();
|
|
const IK_TRAINEDDATA_LANG = "genshin_fast_09_04_21";
|
|
|
|
type OcrWorker = Awaited<ReturnType<typeof createWorker>>;
|
|
type OcrWorkerEngine = "current" | "ik-traineddata";
|
|
type OcrCropPayload = { id: string; label: string; image: Buffer };
|
|
type OcrWarmupStatus = {
|
|
engine: OcrWorkerEngine;
|
|
status: "cold" | "warming" | "ready" | "error";
|
|
workerPoolSize: number;
|
|
startedAt?: string;
|
|
readyAt?: string;
|
|
elapsedMs?: number;
|
|
error?: string;
|
|
};
|
|
type OcrWorkerSlot = {
|
|
worker: OcrWorker;
|
|
parametersKey: string;
|
|
};
|
|
type OcrWorkerPoolState = {
|
|
poolPromise: Promise<OcrWorkerSlot[]> | null;
|
|
runQueue: Promise<unknown>;
|
|
};
|
|
|
|
const ocrWorkerPools: Record<OcrWorkerEngine, OcrWorkerPoolState> = {
|
|
current: { poolPromise: null, runQueue: Promise.resolve() },
|
|
"ik-traineddata": { poolPromise: null, runQueue: Promise.resolve() },
|
|
};
|
|
const ocrWarmupStatuses: Record<OcrWorkerEngine, OcrWarmupStatus> = {
|
|
current: { engine: "current", status: "cold", workerPoolSize: OCR_WORKER_POOL_SIZE },
|
|
"ik-traineddata": { engine: "ik-traineddata", status: "cold", workerPoolSize: OCR_WORKER_POOL_SIZE },
|
|
};
|
|
|
|
function resolveOcrWorkerPoolSize() {
|
|
const requested = Number(process.env.GAA_OCR_WORKERS ?? Number.NaN);
|
|
if (Number.isFinite(requested) && requested > 0) {
|
|
return Math.max(1, Math.min(8, Math.floor(requested)));
|
|
}
|
|
const logicalCores = cpus().length || 4;
|
|
return Math.max(2, Math.min(4, logicalCores - 1));
|
|
}
|
|
|
|
function appRuntimeInfo(): AppRuntimeInfo {
|
|
return {
|
|
signature: APP_RUNTIME_SIGNATURE,
|
|
pid: process.pid,
|
|
startedAt: APP_RUNTIME_STARTED_AT,
|
|
cwd: process.cwd(),
|
|
isDev,
|
|
expectedOcrWorkerPoolSize: OCR_WORKER_POOL_SIZE,
|
|
};
|
|
}
|
|
|
|
function ocrEngineFromOptions(options: CaptureOptions = {}): OcrWorkerEngine {
|
|
return options.ocrEngine === "ik-traineddata" ? "ik-traineddata" : "current";
|
|
}
|
|
|
|
function ikTessdataCandidates() {
|
|
const envDir = process.env.IK_TESSDATA_DIR;
|
|
return [
|
|
envDir,
|
|
path.resolve(process.cwd(), "data", "tessdata"),
|
|
path.resolve(process.cwd(), "work", "Inventory_Kamera", "InventoryKamera", "tessdata"),
|
|
path.resolve(process.cwd(), "work", "refs", "Inventory_Kamera", "InventoryKamera", "tessdata"),
|
|
path.resolve(process.cwd(), "..", "_ik_ref_fork", "InventoryKamera", "tessdata"),
|
|
path.resolve(process.cwd(), "..", "_ik_ref", "InventoryKamera", "tessdata"),
|
|
path.resolve(process.env.USERPROFILE ?? "", "Desktop", "_ik_ref_fork", "InventoryKamera", "tessdata"),
|
|
path.resolve(process.env.USERPROFILE ?? "", "Desktop", "_ik_ref", "InventoryKamera", "tessdata"),
|
|
process.resourcesPath ? path.resolve(process.resourcesPath, "tessdata") : "",
|
|
].filter(Boolean) as string[];
|
|
}
|
|
|
|
function findIkTessdataDir() {
|
|
return ikTessdataCandidates().find((candidate) => existsSync(path.join(candidate, `${IK_TRAINEDDATA_LANG}.traineddata`))) ?? "";
|
|
}
|
|
|
|
function getOcrWorkerOptions(engine: OcrWorkerEngine) {
|
|
if (engine !== "ik-traineddata") return { lang: "eng", options: undefined };
|
|
const langPath = findIkTessdataDir();
|
|
if (!langPath) {
|
|
throw new Error(`IK traineddata not found. Set IK_TESSDATA_DIR or place ${IK_TRAINEDDATA_LANG}.traineddata in data/tessdata.`);
|
|
}
|
|
return {
|
|
lang: IK_TRAINEDDATA_LANG,
|
|
options: {
|
|
langPath,
|
|
gzip: false,
|
|
cachePath: app.isReady() ? path.join(app.getPath("userData"), "tessdata-cache") : path.resolve(process.cwd(), "outputs", "tessdata-cache"),
|
|
},
|
|
};
|
|
}
|
|
|
|
function getOcrWorkerPool(engine: OcrWorkerEngine) {
|
|
const state = ocrWorkerPools[engine];
|
|
if (!state.poolPromise) {
|
|
const { lang, options } = getOcrWorkerOptions(engine);
|
|
state.poolPromise = Promise.all(
|
|
Array.from({ length: OCR_WORKER_POOL_SIZE }, async () => ({
|
|
worker: await createWorker(lang, 1, options),
|
|
parametersKey: "",
|
|
})),
|
|
);
|
|
}
|
|
return state.poolPromise;
|
|
}
|
|
|
|
async function resetOcrWorker(engine?: OcrWorkerEngine) {
|
|
const engines: OcrWorkerEngine[] = engine ? [engine] : ["current", "ik-traineddata"];
|
|
await Promise.all(engines.map(async (engineId) => {
|
|
const state = ocrWorkerPools[engineId];
|
|
const broken = state.poolPromise;
|
|
state.poolPromise = null;
|
|
state.runQueue = Promise.resolve();
|
|
ocrWarmupStatuses[engineId] = { engine: engineId, status: "cold", workerPoolSize: OCR_WORKER_POOL_SIZE };
|
|
if (broken) {
|
|
try {
|
|
const pool = await broken;
|
|
await Promise.all(pool.map((slot) => slot.worker.terminate().catch(() => undefined)));
|
|
} catch {
|
|
// Workers never initialized; nothing to clean up.
|
|
}
|
|
}
|
|
}));
|
|
}
|
|
|
|
function ocrParametersForCrop(cropId: string) {
|
|
switch (cropId) {
|
|
case "artifact-level":
|
|
return {
|
|
tessedit_pageseg_mode: PSM.SINGLE_WORD,
|
|
tessedit_char_whitelist: "0123456789+",
|
|
};
|
|
case "artifact-main-stat-value":
|
|
return {
|
|
tessedit_pageseg_mode: PSM.SINGLE_LINE,
|
|
tessedit_char_whitelist: "0123456789.,%+",
|
|
};
|
|
case "inventory-count":
|
|
return {
|
|
tessedit_pageseg_mode: PSM.SINGLE_LINE,
|
|
tessedit_char_whitelist: "0123456789/",
|
|
};
|
|
case "artifact-name":
|
|
case "artifact-slot":
|
|
case "artifact-main-stat-label":
|
|
case "artifact-footer":
|
|
return {
|
|
tessedit_pageseg_mode: PSM.SINGLE_LINE,
|
|
tessedit_char_whitelist: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .:'%-",
|
|
};
|
|
case "artifact-main-stat":
|
|
return {
|
|
tessedit_pageseg_mode: PSM.SINGLE_BLOCK,
|
|
tessedit_char_whitelist: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .,%+",
|
|
};
|
|
case "artifact-substats":
|
|
return {
|
|
tessedit_pageseg_mode: PSM.SINGLE_BLOCK,
|
|
tessedit_char_whitelist: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .,%+-",
|
|
};
|
|
default:
|
|
return {
|
|
tessedit_pageseg_mode: PSM.SINGLE_BLOCK,
|
|
tessedit_char_whitelist: "",
|
|
};
|
|
}
|
|
}
|
|
|
|
function ocrParametersKey(cropId: string) {
|
|
const params = ocrParametersForCrop(cropId);
|
|
return `${params.tessedit_pageseg_mode}:${params.tessedit_char_whitelist}`;
|
|
}
|
|
|
|
async function applyOcrParameters(slot: OcrWorkerSlot, cropId: string) {
|
|
const params = ocrParametersForCrop(cropId);
|
|
const key = ocrParametersKey(cropId);
|
|
if (key === slot.parametersKey) return;
|
|
await slot.worker.setParameters(params);
|
|
slot.parametersKey = key;
|
|
}
|
|
|
|
function warmOcrWorkerPool(engine: OcrWorkerEngine = "current") {
|
|
const current = ocrWarmupStatuses[engine];
|
|
if (current.status === "warming" || current.status === "ready") return Promise.resolve(current);
|
|
|
|
const started = Date.now();
|
|
ocrWarmupStatuses[engine] = {
|
|
engine,
|
|
status: "warming",
|
|
workerPoolSize: OCR_WORKER_POOL_SIZE,
|
|
startedAt: new Date(started).toISOString(),
|
|
};
|
|
|
|
return getOcrWorkerPool(engine)
|
|
.then(async (pool) => {
|
|
await Promise.all(pool.map((slot) => applyOcrParameters(slot, "artifact-name")));
|
|
const readyAt = Date.now();
|
|
ocrWarmupStatuses[engine] = {
|
|
engine,
|
|
status: "ready",
|
|
workerPoolSize: OCR_WORKER_POOL_SIZE,
|
|
startedAt: ocrWarmupStatuses[engine].startedAt,
|
|
readyAt: new Date(readyAt).toISOString(),
|
|
elapsedMs: readyAt - started,
|
|
};
|
|
return ocrWarmupStatuses[engine];
|
|
})
|
|
.catch((error: unknown) => {
|
|
ocrWarmupStatuses[engine] = {
|
|
engine,
|
|
status: "error",
|
|
workerPoolSize: OCR_WORKER_POOL_SIZE,
|
|
startedAt: ocrWarmupStatuses[engine].startedAt,
|
|
elapsedMs: Date.now() - started,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
};
|
|
return ocrWarmupStatuses[engine];
|
|
});
|
|
}
|
|
|
|
function getOcrWarmupStatus() {
|
|
return {
|
|
current: ocrWarmupStatuses.current,
|
|
"ik-traineddata": ocrWarmupStatuses["ik-traineddata"],
|
|
};
|
|
}
|
|
|
|
async function runOcrOnCrops(crops: OcrCropPayload[], engine: OcrWorkerEngine) {
|
|
try {
|
|
const pool = await getOcrWorkerPool(engine);
|
|
const results: OcrResult[] = new Array(crops.length);
|
|
let nextCropIndex = 0;
|
|
const workers = pool.slice(0, Math.min(pool.length, crops.length || 1));
|
|
|
|
await Promise.all(workers.map(async (slot) => {
|
|
while (nextCropIndex < crops.length) {
|
|
const index = nextCropIndex++;
|
|
const crop = crops[index];
|
|
if (!crop) continue;
|
|
results[index] = await recognizeCropWithSlot(slot, crop);
|
|
}
|
|
}));
|
|
return results;
|
|
} catch (error) {
|
|
await resetOcrWorker(engine);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function recognizeCropWithSlot(slot: OcrWorkerSlot, crop: OcrCropPayload): Promise<OcrResult> {
|
|
const startedAt = Date.now();
|
|
await applyOcrParameters(slot, crop.id);
|
|
const recognized = await slot.worker.recognize(crop.image);
|
|
return {
|
|
id: crop.id,
|
|
label: crop.label,
|
|
text: cleanOcrText(crop.id, recognized.data.text),
|
|
confidence: Math.round(recognized.data.confidence),
|
|
elapsedMs: Date.now() - startedAt,
|
|
};
|
|
}
|
|
|
|
function runQueuedOcrOnCrops(crops: OcrCropPayload[], engine: OcrWorkerEngine) {
|
|
const state = ocrWorkerPools[engine];
|
|
const run = state.runQueue.catch(() => undefined).then(() => runOcrOnCrops(crops, engine));
|
|
state.runQueue = run.catch(() => undefined);
|
|
return run;
|
|
}
|
|
|
|
async function runOcrOnCropsWithTimeout(crops: OcrCropPayload[], engine: OcrWorkerEngine, timeoutMs = 6500) {
|
|
let timeout: NodeJS.Timeout | undefined;
|
|
try {
|
|
return await Promise.race([
|
|
runQueuedOcrOnCrops(crops, engine).then((ocr) => ({ ocr, timedOut: false })),
|
|
new Promise<{ ocr: Awaited<ReturnType<typeof runOcrOnCrops>>; timedOut: boolean }>((resolve) => {
|
|
timeout = setTimeout(() => resolve({ ocr: [], timedOut: true }), timeoutMs);
|
|
}),
|
|
]);
|
|
} finally {
|
|
if (timeout) clearTimeout(timeout);
|
|
}
|
|
}
|
|
|
|
function cleanOcrText(cropId: string, text: string) {
|
|
const normalized = text
|
|
.replace(/[\u201c\u201d]/g, '"')
|
|
.replace(/[\u2019]/g, "'")
|
|
.replace(/\r/g, "")
|
|
.split("\n")
|
|
.map((line) => line.replace(/\s+/g, " ").trim())
|
|
.filter(Boolean);
|
|
|
|
if (cropId === "artifact-footer") {
|
|
const equipped = normalized.find((line) => /equipped/i.test(line));
|
|
if (!equipped) return "";
|
|
|
|
const match = /equipped\s*:?\s*([A-Za-z][A-Za-z'\-\s]{1,32})/i.exec(equipped);
|
|
return match ? `Equipped: ${match[1].replace(/[^A-Za-z'\-\s]/g, "").trim()}` : equipped;
|
|
}
|
|
|
|
if (cropId === "artifact-title" || cropId === "artifact-name" || cropId === "artifact-slot" || cropId === "artifact-main-stat-label") {
|
|
return normalized
|
|
.filter((line) => /[A-Za-z]/.test(line))
|
|
.slice(0, 2)
|
|
.join("\n");
|
|
}
|
|
|
|
if (cropId === "artifact-main-stat") {
|
|
return normalized
|
|
.filter((line) => /[A-Za-z0-9]/.test(line))
|
|
.slice(0, 3)
|
|
.join("\n");
|
|
}
|
|
|
|
if (cropId === "artifact-main-stat-value") {
|
|
return normalized
|
|
.map((line) => line.replace(/[^0-9.,%]/g, ""))
|
|
.find((line) => /[0-9]/.test(line)) ?? "";
|
|
}
|
|
|
|
if (cropId === "artifact-level") {
|
|
const value = normalized
|
|
.map((line) => line.replace(/[^0-9+]/g, ""))
|
|
.find((line) => /[0-9]/.test(line)) ?? "";
|
|
return value.startsWith("+") || value === "" ? value : `+${value}`;
|
|
}
|
|
|
|
if (cropId === "artifact-substats") {
|
|
return normalized
|
|
.filter((line) => /(\+|CRIT|ATK|DEF|HP|Energy|Elemental)/i.test(line))
|
|
.slice(0, 6)
|
|
.join("\n");
|
|
}
|
|
|
|
if (cropId === "inventory-count") {
|
|
return normalized
|
|
.map((line) => line.replace(/[^0-9/]/g, ""))
|
|
.find((line) => /[0-9]/.test(line)) ?? "";
|
|
}
|
|
|
|
return normalized.join("\n");
|
|
}
|
|
|
|
function parseInventoryCount(ocr: Array<{ id: string; text: string; confidence: number }>) {
|
|
const entry = ocr.find((item) => item.id === "inventory-count");
|
|
if (!entry?.text) return { current: 0, total: 0, confidence: 0, source: "missing" as const, text: "" };
|
|
const cleaned = entry.text.replace(/[^0-9/]/g, "");
|
|
const match = /^(\d{1,4})\/(\d{3,4})$/.exec(cleaned);
|
|
if (match) {
|
|
return {
|
|
current: Number(match[1]),
|
|
total: Number(match[2]),
|
|
confidence: Math.max(0, Math.min(100, entry.confidence)),
|
|
source: "ocr" as const,
|
|
text: cleaned,
|
|
};
|
|
}
|
|
|
|
const fallback = cleaned.match(/(\d{1,4})(\d{4})$/);
|
|
if (fallback) {
|
|
return {
|
|
current: Number(fallback[1]),
|
|
total: Number(fallback[2]),
|
|
confidence: Math.max(0, Math.min(84, entry.confidence)),
|
|
source: "ocr" as const,
|
|
text: cleaned,
|
|
};
|
|
}
|
|
|
|
return { current: 0, total: 0, confidence: 0, source: "missing" as const, text: cleaned };
|
|
}
|
|
|
|
function isArtifactTitleOrange(bitmap: Buffer, index: number) {
|
|
const blue = bitmap[index];
|
|
const green = bitmap[index + 1];
|
|
const red = bitmap[index + 2];
|
|
|
|
return red >= 135 && green >= 70 && green <= 155 && blue <= 95 && red > green + 35 && green > blue + 20;
|
|
}
|
|
|
|
function isEquippedFooterYellow(bitmap: Buffer, index: number) {
|
|
const blue = bitmap[index];
|
|
const green = bitmap[index + 1];
|
|
const red = bitmap[index + 2];
|
|
|
|
return red >= 220 && green >= 185 && blue >= 125 && red > blue + 35 && green > blue + 20;
|
|
}
|
|
|
|
function isSetTitleGreen(bitmap: Buffer, index: number) {
|
|
const blue = bitmap[index];
|
|
const green = bitmap[index + 1];
|
|
const red = bitmap[index + 2];
|
|
|
|
return green >= 110 && red <= 170 && blue <= 160 && green > red + 12 && green > blue + 10;
|
|
}
|
|
|
|
function isArtifactTextColor(bitmap: Buffer, index: number) {
|
|
const blue = bitmap[index];
|
|
const green = bitmap[index + 1];
|
|
const red = bitmap[index + 2];
|
|
|
|
return green >= 180 && red >= 140 && blue <= 95 && green > blue + 35 && red > blue + 10;
|
|
}
|
|
|
|
function hasEquippedFooterMarker(bitmap: Buffer, imageSize: { width: number; height: number }, rect: Electron.Rectangle) {
|
|
const safeRect = clampCaptureRect(rect, imageSize);
|
|
const strideX = Math.max(1, Math.floor(safeRect.width / 48));
|
|
const strideY = Math.max(1, Math.floor(safeRect.height / 18));
|
|
let hits = 0;
|
|
|
|
for (let y = safeRect.y; y < safeRect.y + safeRect.height; y += strideY) {
|
|
const rowOffset = y * imageSize.width * 4;
|
|
for (let x = safeRect.x; x < safeRect.x + safeRect.width; x += strideX) {
|
|
if (isEquippedFooterYellow(bitmap, rowOffset + x * 4)) {
|
|
hits++;
|
|
if (hits >= 10) return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
function isSanctifiedArtifactPurple(bitmap: Buffer, index: number) {
|
|
const blue = bitmap[index];
|
|
const green = bitmap[index + 1];
|
|
const red = bitmap[index + 2];
|
|
return blue >= 220 && red >= 180 && red <= 245 && green >= 155 && green <= 220 && blue > red + 10;
|
|
}
|
|
|
|
function detectSanctifiedArtifactDetail(bitmap: Buffer, imageSize: { width: number; height: number }, detailRect: Electron.Rectangle) {
|
|
const safeRect = clampCaptureRect(detailRect, imageSize);
|
|
const x0 = Math.max(safeRect.x, Math.round(safeRect.x + safeRect.width * 0.0));
|
|
const x1 = Math.min(imageSize.width - 1, Math.round(safeRect.x + safeRect.width * 0.0606));
|
|
const y0 = Math.max(safeRect.y, Math.round(safeRect.y + safeRect.height * 0.3333));
|
|
const y1 = Math.min(imageSize.height - 1, Math.round(y0 + safeRect.height * 0.0526));
|
|
const strideX = Math.max(1, Math.floor(Math.max(1, x1 - x0) / 12));
|
|
const strideY = Math.max(1, Math.floor(Math.max(1, y1 - y0) / 10));
|
|
let hits = 0;
|
|
|
|
for (let y = y0; y <= y1; y += strideY) {
|
|
const rowOffset = y * imageSize.width * 4;
|
|
for (let x = x0; x <= x1; x += strideX) {
|
|
if (isSanctifiedArtifactPurple(bitmap, rowOffset + x * 4)) {
|
|
hits++;
|
|
if (hits >= 3) return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
function analyzeArtifactDetailPanel(bitmap: Buffer, imageSize: { width: number; height: number }, rect: Electron.Rectangle) {
|
|
const safeRect = clampCaptureRect(rect, imageSize);
|
|
const strideX = Math.max(1, Math.floor(safeRect.width / 96));
|
|
const strideY = Math.max(1, Math.floor(safeRect.height / 160));
|
|
let orangeHits = 0;
|
|
let greenHits = 0;
|
|
let textHits = 0;
|
|
let titleOrangeHits = 0;
|
|
let upperTextHits = 0;
|
|
let lowerGreenHits = 0;
|
|
|
|
for (let y = safeRect.y; y < safeRect.y + safeRect.height; y += strideY) {
|
|
const rowOffset = y * imageSize.width * 4;
|
|
for (let x = safeRect.x; x < safeRect.x + safeRect.width; x += strideX) {
|
|
const index = rowOffset + x * 4;
|
|
const relativeY = (y - safeRect.y) / safeRect.height;
|
|
const isOrange = isArtifactTitleOrange(bitmap, index);
|
|
const isGreen = isSetTitleGreen(bitmap, index);
|
|
const isText = isArtifactTextColor(bitmap, index);
|
|
if (isOrange) {
|
|
orangeHits++;
|
|
if (relativeY <= 0.085) titleOrangeHits++;
|
|
}
|
|
if (isGreen) {
|
|
greenHits++;
|
|
if (relativeY >= 0.37 && relativeY <= 0.84) lowerGreenHits++;
|
|
}
|
|
if (isText) {
|
|
textHits++;
|
|
if (relativeY >= 0.08 && relativeY <= 0.37) upperTextHits++;
|
|
}
|
|
}
|
|
}
|
|
|
|
const confidence = Math.max(
|
|
0,
|
|
Math.min(100, Math.round(titleOrangeHits * 1.6 + upperTextHits * 1.2 + lowerGreenHits * 0.7)),
|
|
);
|
|
return {
|
|
present: titleOrangeHits >= 40 && (upperTextHits >= 20 || lowerGreenHits >= 40) && confidence >= 58,
|
|
confidence,
|
|
orangeHits,
|
|
greenHits,
|
|
textHits,
|
|
titleOrangeHits,
|
|
upperTextHits,
|
|
lowerGreenHits,
|
|
};
|
|
}
|
|
|
|
function isPaimonProfileLight(bitmap: Buffer, index: number) {
|
|
const blue = bitmap[index];
|
|
const green = bitmap[index + 1];
|
|
const red = bitmap[index + 2];
|
|
return red > 150 && green > 150 && blue > 150;
|
|
}
|
|
|
|
function isPaimonProfileCream(bitmap: Buffer, index: number) {
|
|
const blue = bitmap[index];
|
|
const green = bitmap[index + 1];
|
|
const red = bitmap[index + 2];
|
|
return red >= 195 && green >= 185 && blue >= 155;
|
|
}
|
|
|
|
function isPaimonMenuTileDark(bitmap: Buffer, index: number) {
|
|
const blue = bitmap[index];
|
|
const green = bitmap[index + 1];
|
|
const red = bitmap[index + 2];
|
|
return red >= 45 && red <= 115 && green >= 55 && green <= 125 && blue >= 70 && blue <= 150;
|
|
}
|
|
|
|
function sampleScreenZone(
|
|
bitmap: Buffer,
|
|
imageSize: { width: number; height: number },
|
|
zone: { x0: number; x1: number; y0: number; y1: number },
|
|
predicate: (bitmap: Buffer, index: number) => boolean,
|
|
) {
|
|
const x0 = Math.max(0, Math.floor(imageSize.width * zone.x0));
|
|
const x1 = Math.min(imageSize.width, Math.ceil(imageSize.width * zone.x1));
|
|
const y0 = Math.max(0, Math.floor(imageSize.height * zone.y0));
|
|
const y1 = Math.min(imageSize.height, Math.ceil(imageSize.height * zone.y1));
|
|
const strideX = Math.max(1, Math.floor((x1 - x0) / 135));
|
|
const strideY = Math.max(1, Math.floor((y1 - y0) / 90));
|
|
let hits = 0;
|
|
let samples = 0;
|
|
|
|
for (let y = y0; y < y1; y += strideY) {
|
|
const rowOffset = y * imageSize.width * 4;
|
|
for (let x = x0; x < x1; x += strideX) {
|
|
samples++;
|
|
if (predicate(bitmap, rowOffset + x * 4)) hits++;
|
|
}
|
|
}
|
|
|
|
return samples > 0 ? (hits / samples) * 100 : 0;
|
|
}
|
|
|
|
function analyzePaimonMenu(bitmap: Buffer, imageSize: { width: number; height: number }) {
|
|
const profileZone = { x0: 0.05, x1: 0.40, y0: 0, y1: 0.30 };
|
|
const menuTileZone = { x0: 0.06, x1: 0.39, y0: 0.32, y1: 0.98 };
|
|
const profileLightPct = sampleScreenZone(bitmap, imageSize, profileZone, isPaimonProfileLight);
|
|
const profileCreamPct = sampleScreenZone(bitmap, imageSize, profileZone, isPaimonProfileCream);
|
|
const menuTileDarkPct = sampleScreenZone(bitmap, imageSize, menuTileZone, isPaimonMenuTileDark);
|
|
const confidence = Math.max(0, Math.min(100, Math.round((profileLightPct - 20) * 1.2 + (profileCreamPct - 12) * 1.4 + (menuTileDarkPct - 25) * 1.1)));
|
|
|
|
return {
|
|
present: profileLightPct >= 35 && profileCreamPct >= 20 && menuTileDarkPct >= 40,
|
|
confidence,
|
|
profileLightPct: Math.round(profileLightPct * 10) / 10,
|
|
profileCreamPct: Math.round(profileCreamPct * 10) / 10,
|
|
menuTileDarkPct: Math.round(menuTileDarkPct * 10) / 10,
|
|
};
|
|
}
|
|
|
|
function waitDelay(ms: number) {
|
|
return new Promise<void>((resolve) => setTimeout(resolve, Math.max(0, Math.floor(ms))));
|
|
}
|
|
|
|
type CropTemplate = { id: string; label: string; rect: Electron.Rectangle };
|
|
|
|
function getCaptureSourceListOptions() {
|
|
const displays = screen.getAllDisplays();
|
|
const maxSize = displays.reduce(
|
|
(size, display) => ({
|
|
width: Math.max(size.width, display.size.width),
|
|
height: Math.max(size.height, display.size.height),
|
|
}),
|
|
{ width: 1920, height: 1080 },
|
|
);
|
|
|
|
return { maxSize, fetchWindowIcons: true };
|
|
}
|
|
|
|
async function getAllSources() {
|
|
const { maxSize, fetchWindowIcons } = getCaptureSourceListOptions();
|
|
return desktopCapturer.getSources({ types: ["window", "screen"], thumbnailSize: maxSize, fetchWindowIcons });
|
|
}
|
|
|
|
async function findCaptureSourceById(sourceId: string) {
|
|
const sources = await getAllSources();
|
|
for (const source of sources) captureSourceNameCache.set(source.id, source.name);
|
|
return sources.find((source) => source.id === sourceId) ?? null;
|
|
}
|
|
|
|
function clampCaptureRect(rect: Electron.Rectangle, imageSize: { width: number; height: number }) {
|
|
const clamped = {
|
|
x: Math.max(0, Math.min(imageSize.width - 1, rect.x)),
|
|
y: Math.max(0, Math.min(imageSize.height - 1, rect.y)),
|
|
};
|
|
const maxWidth = Math.max(1, imageSize.width - clamped.x);
|
|
const maxHeight = Math.max(1, imageSize.height - clamped.y);
|
|
return {
|
|
...clamped,
|
|
width: Math.max(1, Math.min(maxWidth, rect.width)),
|
|
height: Math.max(1, Math.min(maxHeight, rect.height)),
|
|
};
|
|
}
|
|
|
|
function imageCropDataUrl(sourceImage: NativeImage, rect: Electron.Rectangle, imageSize: { width: number; height: number }) {
|
|
const safeRect = clampCaptureRect(rect, imageSize);
|
|
return sourceImage.crop(safeRect).toDataURL();
|
|
}
|
|
|
|
function imageCropFingerprint(sourceImage: NativeImage, rect: Electron.Rectangle, imageSize: { width: number; height: number }) {
|
|
const safeRect = clampCaptureRect(rect, imageSize);
|
|
const bitmap = sourceImage.crop(safeRect).getBitmap();
|
|
let hash = 2166136261;
|
|
const stride = Math.max(4, Math.floor(bitmap.length / 4096) * 4);
|
|
for (let index = 0; index < bitmap.length; index += stride) {
|
|
hash ^= bitmap[index] ?? 0;
|
|
hash = Math.imul(hash, 16777619);
|
|
hash ^= bitmap[index + 1] ?? 0;
|
|
hash = Math.imul(hash, 16777619);
|
|
hash ^= bitmap[index + 2] ?? 0;
|
|
hash = Math.imul(hash, 16777619);
|
|
}
|
|
return `${safeRect.width}x${safeRect.height}:${(hash >>> 0).toString(16)}`;
|
|
}
|
|
|
|
// Preprocessed copy of a crop for OCR (ADR-009): upscale for more pixels, then
|
|
// grayscale + Otsu-binarize with inversion (artifact text is the bright
|
|
// foreground). The original crop is kept separately for the diagnostics UI.
|
|
// OCR receives a PNG buffer directly to avoid DataURL encode/decode churn in
|
|
// the auto-scan hot path.
|
|
function preprocessedCropPngBuffer(sourceImage: NativeImage, rect: Electron.Rectangle, imageSize: { width: number; height: number }, cropId = "") {
|
|
const safeRect = clampCaptureRect(rect, imageSize);
|
|
const scale = cropId === "artifact-level" ? 3 : 2;
|
|
const upscaled = sourceImage.crop(safeRect).resize({ width: Math.max(1, Math.round(safeRect.width * scale)), quality: "best" });
|
|
const size = upscaled.getSize();
|
|
if (!size.width || !size.height) return upscaled.toPNG();
|
|
const binarized = binarizeForOcr(
|
|
{ data: upscaled.getBitmap(), width: size.width, height: size.height },
|
|
cropId === "artifact-level" ? { contrast: 80 } : {},
|
|
);
|
|
return nativeImage
|
|
.createFromBitmap(Buffer.from(binarized.data), { width: binarized.width, height: binarized.height })
|
|
.toPNG();
|
|
}
|
|
|
|
function createCrops(
|
|
sourceImage: NativeImage,
|
|
imageSize: { width: number; height: number },
|
|
detailRect: Electron.Rectangle,
|
|
inventoryRect: Electron.Rectangle,
|
|
options: CaptureOptions = {},
|
|
bitmap?: Buffer,
|
|
cropOptions: { sanctified?: boolean; skipOcr?: boolean } = {},
|
|
) {
|
|
const isArtifactScanMode = options.ocrMode === "artifact";
|
|
const fastArtifactProfile = isArtifactScanMode && options.ocrProfile === "fast";
|
|
const omitCropImages = Boolean(options.omitCropImages);
|
|
const skipCropOcr = Boolean(cropOptions.skipOcr);
|
|
const templates: CropTemplate[] = detailCropRects(detailRect, imageSize, { ...cropOptions, fastProfile: fastArtifactProfile })
|
|
.filter((template) => {
|
|
if (fastArtifactProfile && (
|
|
template.id === "artifact-set-effects" ||
|
|
template.id === "artifact-main-stat-value"
|
|
)) return false;
|
|
if (template.id === "artifact-footer" && options.omitEquippedOcr) return false;
|
|
if (!isArtifactScanMode || template.id !== "artifact-footer" || !bitmap) return true;
|
|
return hasEquippedFooterMarker(bitmap, imageSize, template.rect);
|
|
});
|
|
|
|
if (!isArtifactScanMode && inventoryRect.width > 120 && inventoryRect.height > 80) {
|
|
templates.push({
|
|
id: "inventory-count",
|
|
label: "Inventory count",
|
|
rect: inventoryCountCropRect(inventoryRect, imageSize),
|
|
});
|
|
}
|
|
|
|
return templates
|
|
.map((template) => {
|
|
const rect = clampCaptureRect(template.rect, imageSize);
|
|
const ocrEnabled = !skipCropOcr;
|
|
return {
|
|
id: template.id,
|
|
label: template.label,
|
|
rect,
|
|
dataUrl: omitCropImages ? undefined : imageCropDataUrl(sourceImage, rect, imageSize),
|
|
ocrImage: ocrEnabled ? preprocessedCropPngBuffer(sourceImage, rect, imageSize, template.id) : undefined,
|
|
ocrEnabled,
|
|
};
|
|
})
|
|
.filter((crop) => crop.rect.width > 0 && crop.rect.height > 0);
|
|
}
|
|
|
|
function inferDetailRect(bitmap: Buffer, imageSize: { width: number; height: number }) {
|
|
if (isSixteenNine(imageSize)) {
|
|
return profileDetailRect(imageSize);
|
|
}
|
|
|
|
const { width, height } = imageSize;
|
|
const sampleStrideX = width > 2200 ? 4 : 3;
|
|
const sampleStrideY = height > 1400 ? 4 : 3;
|
|
const candidates: Array<{ x: number; y: number }> = [];
|
|
|
|
const search = {
|
|
x0: Math.floor(width * 0.40),
|
|
x1: Math.floor(width * 0.98),
|
|
y0: Math.floor(height * 0.04),
|
|
y1: Math.floor(height * 0.83),
|
|
};
|
|
|
|
for (let y = search.y0; y < search.y1; y += sampleStrideY) {
|
|
const rowOffset = y * width * 4;
|
|
for (let x = search.x0; x < search.x1; x += sampleStrideX) {
|
|
const index = rowOffset + x * 4;
|
|
if (isArtifactTitleOrange(bitmap, index) || isSetTitleGreen(bitmap, index) || isArtifactTextColor(bitmap, index)) {
|
|
candidates.push({ x, y });
|
|
}
|
|
}
|
|
}
|
|
|
|
if (candidates.length >= 180) {
|
|
const xValues = candidates.map((item) => item.x);
|
|
const yValues = candidates.map((item) => item.y);
|
|
const xMin = Math.min(...xValues);
|
|
const xMax = Math.max(...xValues);
|
|
const yMin = Math.min(...yValues);
|
|
const yMax = Math.max(...yValues);
|
|
const spanX = Math.max(1, xMax - xMin);
|
|
const spanY = Math.max(1, yMax - yMin);
|
|
const widthGuess = Math.max(Math.round(width * 0.30), Math.min(Math.round(width * 0.52), Math.round(spanX * 3.6)));
|
|
const left = Math.max(Math.round(width * 0.42), Math.round((xMin + xMax) / 2 - widthGuess * 0.52));
|
|
const top = Math.max(0, Math.min(height - 1, Math.round(yMin - spanY * 0.4)));
|
|
const heightGuess = Math.max(Math.round(height * 0.58), Math.min(Math.round(height * 0.74), Math.round(spanY * 4.1)));
|
|
return clampCaptureRect({ x: left, y: top, width: widthGuess, height: heightGuess }, imageSize);
|
|
}
|
|
|
|
// Colour detection found nothing usable; fall back to the resolution-anchored
|
|
// profile rect (single source of truth in layoutProfile).
|
|
return profileDetailRect(imageSize);
|
|
}
|
|
|
|
function inferInventoryRect(imageSize: { width: number; height: number }, detailRect: Electron.Rectangle) {
|
|
return layoutInventoryRect(imageSize, detailRect);
|
|
}
|
|
|
|
function inferInventoryGrid(imageSize: { width: number; height: number }, detailRect: Electron.Rectangle) {
|
|
return layoutInventoryGrid(imageSize, detailRect);
|
|
}
|
|
|
|
async function buildCaptureResult(
|
|
sourceImage: NativeImage,
|
|
sourceId: string,
|
|
sourceName: string,
|
|
captureTarget: CaptureResult["captureTarget"],
|
|
options: CaptureOptions = {},
|
|
) {
|
|
const buildStartedAt = Date.now();
|
|
const size = sourceImage.getSize();
|
|
if (!size.width || !size.height) {
|
|
throw new Error("Capture produced an empty image.");
|
|
}
|
|
|
|
const bitmap = sourceImage.getBitmap();
|
|
const detailRect = inferDetailRect(bitmap, size);
|
|
const artifactDetail = analyzeArtifactDetailPanel(bitmap, size, detailRect);
|
|
const paimonMenu = analyzePaimonMenu(bitmap, size);
|
|
const sanctified = detectSanctifiedArtifactDetail(bitmap, size, detailRect);
|
|
const skipOcrForMissingDetail = Boolean(options.skipOcrUnlessArtifactDetail && !artifactDetail.present);
|
|
const shouldSkipOcr = Boolean(options.skipOcr || skipOcrForMissingDetail);
|
|
const inventoryRect = inferInventoryRect(size, detailRect);
|
|
const omitCrops = Boolean(options.omitCrops);
|
|
const crops = omitCrops
|
|
? []
|
|
: createCrops(sourceImage, size, detailRect, inventoryRect, options, bitmap, { sanctified, skipOcr: shouldSkipOcr });
|
|
const lockSignal = options.omitLockState
|
|
? undefined
|
|
: (() => {
|
|
const lockRect = clampCaptureRect(lockIconCropRect(detailRect, size), size);
|
|
const lockImage = sourceImage.crop(lockRect);
|
|
const lockSize = lockImage.getSize();
|
|
const ratio = lockSize.width > 0 && lockSize.height > 0
|
|
? (() => {
|
|
try {
|
|
return lockSignalRatio(pngBufferToBitmap(lockImage.toPNG()));
|
|
} catch {
|
|
return lockSignalRatio({ data: lockImage.getBitmap(), width: lockSize.width, height: lockSize.height });
|
|
}
|
|
})()
|
|
: undefined;
|
|
return typeof ratio === "number"
|
|
? {
|
|
ratio,
|
|
threshold: DEFAULT_LOCK_THRESHOLD,
|
|
rect: {
|
|
x: lockRect.x,
|
|
y: lockRect.y,
|
|
width: lockRect.width,
|
|
height: lockRect.height,
|
|
},
|
|
}
|
|
: undefined;
|
|
})();
|
|
const locked = lockSignal ? isLocked(lockSignal.ratio, lockSignal.threshold) : undefined;
|
|
const omitFullFrame = Boolean(options.omitFullFrame || options.ocrMode === "artifact");
|
|
const omitDetailPreview = Boolean(options.omitDetailPreview);
|
|
const omitInventoryPreview = Boolean(options.omitInventoryPreview || options.ocrMode === "artifact");
|
|
const detailFingerprint = imageCropFingerprint(sourceImage, detailRect, size);
|
|
const inventoryFingerprint = imageCropFingerprint(sourceImage, inventoryRect, size);
|
|
const fastOcrPriority: Record<string, number> = {
|
|
"artifact-substats": 0,
|
|
"artifact-name": 1,
|
|
"artifact-footer": 2,
|
|
"artifact-slot": 3,
|
|
"artifact-main-stat-label": 4,
|
|
"artifact-level": 5,
|
|
};
|
|
const croppedPayload = crops
|
|
.filter((crop) => crop.ocrEnabled !== false)
|
|
.map((crop) => {
|
|
return crop.ocrImage
|
|
? {
|
|
id: crop.id,
|
|
label: crop.label,
|
|
image: crop.ocrImage,
|
|
}
|
|
: null;
|
|
})
|
|
.filter((crop): crop is OcrCropPayload => Boolean(crop))
|
|
.sort((left, right) => {
|
|
if (options.ocrMode !== "artifact" || options.ocrProfile !== "fast") return 0;
|
|
return (fastOcrPriority[left.id] ?? 100) - (fastOcrPriority[right.id] ?? 100);
|
|
});
|
|
const prepareMs = Date.now() - buildStartedAt;
|
|
const ocrEngine = ocrEngineFromOptions(options);
|
|
const ocrStartedAt = Date.now();
|
|
const recognized = shouldSkipOcr
|
|
? { ocr: [], timedOut: false }
|
|
: await runOcrOnCropsWithTimeout(croppedPayload, ocrEngine);
|
|
const ocrMs = Date.now() - ocrStartedAt;
|
|
const totalMs = Date.now() - buildStartedAt;
|
|
const captureOcrEngine: CaptureOptions["ocrEngine"] = ocrEngine === "ik-traineddata" ? "ik-traineddata" : "current";
|
|
|
|
const count = parseInventoryCount(recognized.ocr);
|
|
return {
|
|
id: sourceId,
|
|
name: sourceName,
|
|
width: size.width,
|
|
height: size.height,
|
|
dataUrl: omitFullFrame ? "" : sourceImage.toDataURL(),
|
|
capturedAt: new Date().toISOString(),
|
|
captureTarget,
|
|
detailDataUrl: omitDetailPreview ? undefined : imageCropDataUrl(sourceImage, detailRect, size),
|
|
inventoryDataUrl: omitInventoryPreview ? undefined : imageCropDataUrl(sourceImage, inventoryRect, size),
|
|
detailFingerprint,
|
|
inventoryFingerprint,
|
|
ocr: recognized.ocr,
|
|
ocrTimedOut: recognized.timedOut,
|
|
ocrSkipped: shouldSkipOcr,
|
|
crops: crops.map((crop) => ({
|
|
id: crop.id,
|
|
label: crop.label,
|
|
rect: {
|
|
x: crop.rect.x,
|
|
y: crop.rect.y,
|
|
width: crop.rect.width,
|
|
height: crop.rect.height,
|
|
},
|
|
dataUrl: crop.dataUrl,
|
|
})),
|
|
inventoryGrid: inferInventoryGrid(size, detailRect),
|
|
artifactDetail,
|
|
paimonMenu,
|
|
inventoryCount: count,
|
|
locked,
|
|
lockSignal,
|
|
sanctified,
|
|
layout: {
|
|
aspect: aspectRatioLabel(size),
|
|
isSixteenNine: isSixteenNine(size),
|
|
warning: layoutSupportWarning(size),
|
|
},
|
|
timings: {
|
|
totalMs,
|
|
prepareMs,
|
|
ocrMs: shouldSkipOcr ? 0 : ocrMs,
|
|
cropCount: croppedPayload.length,
|
|
ocrEngine: captureOcrEngine,
|
|
ocrWorkerPoolSize: OCR_WORKER_POOL_SIZE,
|
|
ocrProfile: options.ocrProfile ?? "full",
|
|
ocrFieldMs: recognized.ocr.reduce<Record<string, number>>((fields, result) => {
|
|
if (typeof result.elapsedMs === "number") fields[result.id] = result.elapsedMs;
|
|
return fields;
|
|
}, {}),
|
|
ocrSkipped: shouldSkipOcr,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function captureSource(sourceId: string, delayMs = 0, focusGenshin = false, options?: CaptureOptions): Promise<CaptureResult> {
|
|
const captureStartedAt = Date.now();
|
|
if (!Number.isFinite(delayMs) || delayMs < 0) {
|
|
delayMs = 0;
|
|
}
|
|
|
|
const withElapsed = (capture: CaptureResult): CaptureResult => ({
|
|
...capture,
|
|
elapsedMs: Math.max(0, Date.now() - captureStartedAt),
|
|
});
|
|
|
|
await waitDelay(Math.floor(delayMs));
|
|
|
|
if (focusGenshin) {
|
|
await focusGenshinForScanStart();
|
|
}
|
|
|
|
if (shouldUseDirectGdiHotPath(options ?? {})) {
|
|
const cachedName = captureSourceNameCache.get(sourceId) ?? "Genshin GDI Capture";
|
|
return withElapsed(await captureSourceFromGdi(sourceId, cachedName, options ?? {}));
|
|
}
|
|
|
|
const source = await findCaptureSourceById(sourceId);
|
|
if (!source) {
|
|
throw new Error("Capture source not found.");
|
|
}
|
|
|
|
const isGenshinCandidate = isLikelyGenshinSourceName(source.name);
|
|
if (isGenshinCandidate) {
|
|
try {
|
|
return withElapsed(await captureSourceFromGdi(sourceId, source.name, options ?? {}));
|
|
} catch {
|
|
// Fall back to desktop thumbnail capture for robustness in low-permission
|
|
// or transient capture failures. OCR will still produce a best-effort result.
|
|
}
|
|
}
|
|
|
|
const sourceImage = source.thumbnail;
|
|
if (sourceImage.isEmpty()) {
|
|
return withElapsed(await captureSourceFromGdi(sourceId, source.name, options ?? {}));
|
|
}
|
|
|
|
return withElapsed(await buildCaptureResult(
|
|
sourceImage,
|
|
sourceId,
|
|
source.name,
|
|
sourceId.startsWith("screen:") ? "desktop-source" : "genshin-client",
|
|
options ?? {},
|
|
));
|
|
}
|
|
|
|
function initializeAppLifecycle() {
|
|
app.whenReady().then(() => {
|
|
const userDataPath = app.getPath("userData");
|
|
repositoryContext = createRepositoryContext(userDataPath);
|
|
artifactStoreRepository = repositoryContext.artifactStoreRepository;
|
|
reviewSamplesRepository = repositoryContext.reviewSamplesRepository;
|
|
scannerLearningRepository = repositoryContext.scannerLearningRepository;
|
|
inputHelperService = createInputHelperService({ userDataPath, exePath: resolveInputHelperExePath() });
|
|
goodFileService = createGoodFileService(path.join(userDataPath, "exports"));
|
|
|
|
registerIpcHandlers({
|
|
focusMainWindow: () => focusMainWindow(),
|
|
moveMainWindowOffGenshin: async () => moveMainWindowOffGenshin(),
|
|
focusGenshinForScanStart: () => focusGenshinForScanStart(),
|
|
publishScannerStatus: (status: ScannerStatusPayload) => publishScannerStatus(status),
|
|
readRuntimeInfo: () => readRuntimeInfo(),
|
|
loadSnapshotFromDisk: () => loadSnapshotFromDisk(),
|
|
saveSnapshotToDisk: (snapshot: AppSnapshot) => saveSnapshotToDisk(snapshot),
|
|
runMockScan: () => runMockScan(),
|
|
showOverlayWindow: () => showOverlayWindow(),
|
|
hideOverlayWindow: () => hideOverlayWindow(),
|
|
getArtifactStoreRepository: () => getArtifactStoreRepository(),
|
|
getReviewSamplesRepository: () => getReviewSamplesRepository(),
|
|
artifactStorePath: () => artifactStorePath(),
|
|
reviewSamplesPath: () => reviewSamplesPath(),
|
|
loadReviewSamples: (limit?: number) => loadReviewSamples(limit),
|
|
loadScannerLearningRules: () => loadScannerLearningRules(),
|
|
writeScannerLearningRules: (rules: ScannerLearningRulePayload) => writeScannerLearningRules(rules),
|
|
exportGood: (exportPayload: GoodDatabase) => getGoodFileService().exportGood(exportPayload),
|
|
importGoodFile: () => getGoodFileService().importGoodFile(getAppWindowManager().getMainWindow()),
|
|
listSources: () => listCaptureSources(),
|
|
captureSource: (
|
|
id: string,
|
|
delayMs?: number,
|
|
focus?: boolean,
|
|
captureOptions?: CaptureOptions,
|
|
) => captureSource(id, delayMs, focus, captureOptions),
|
|
clickScreen: (x: number, y: number) => clickScreenCommand(x, y),
|
|
scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => scrollScreenCommand(notches, anchorX, anchorY),
|
|
keyPress: (key: string) => keyPressCommand(key),
|
|
getAutomationGuard: () => getAutomationGuardCommand(),
|
|
});
|
|
|
|
createMainWindow();
|
|
registerScannerHotkeys();
|
|
startDevControlServer();
|
|
void warmOcrWorkerPool("current");
|
|
});
|
|
|
|
app.on("activate", () => {
|
|
if (!getAppWindowManager().hasMainWindow()) {
|
|
createMainWindow();
|
|
}
|
|
});
|
|
|
|
app.on("window-all-closed", () => {
|
|
if (process.platform !== "darwin") {
|
|
app.quit();
|
|
}
|
|
});
|
|
|
|
app.on("will-quit", async () => {
|
|
globalShortcut.unregisterAll();
|
|
if (devControlServer) {
|
|
devControlServer.close();
|
|
devControlServer = null;
|
|
}
|
|
await resetOcrWorker();
|
|
inputHelperService?.dispose();
|
|
});
|
|
}
|
|
|
|
initializeAppLifecycle();
|