Improve IK-style artifact scanner pipeline
This commit is contained in:
AzuTear
+603
-71
@@ -2,9 +2,10 @@ import { app, BrowserWindow, Menu, desktopCapturer, dialog, globalShortcut, nati
|
||||
import fs from "node:fs/promises";
|
||||
import { existsSync } from "node:fs";
|
||||
import type { Server } from "node:http";
|
||||
import { cpus } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createWorker } from "tesseract.js";
|
||||
import { createWorker, PSM } from "tesseract.js";
|
||||
import { createInputHelperService, type InputHelperService } from "./services/inputHelper.js";
|
||||
import { createRepositoryContext, type RepositoryContext } from "./bootstrap/repositoryContext.js";
|
||||
import { registerIpcHandlers } from "./bootstrap/ipcBootstrap.js";
|
||||
@@ -15,6 +16,8 @@ import type {
|
||||
CaptureResult,
|
||||
GoodDatabase,
|
||||
GoodImportFileResult,
|
||||
OcrResult,
|
||||
AppRuntimeInfo,
|
||||
SaveResultWithPath,
|
||||
ScannerCommand,
|
||||
ScannerLearningRulePayload,
|
||||
@@ -50,6 +53,8 @@ app.commandLine.appendSwitch("disable-gpu-sandbox");
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const isDev = Boolean(process.env.VITE_DEV_SERVER_URL);
|
||||
const APP_RUNTIME_STARTED_AT = new Date().toISOString();
|
||||
const APP_RUNTIME_SIGNATURE = "2026-07-07-ik32-fastsubstats-active-timing";
|
||||
|
||||
let mainWindow: BrowserWindow | null = null;
|
||||
let overlayWindow: BrowserWindow | null = null;
|
||||
@@ -211,6 +216,7 @@ async function readRuntimeInfo() {
|
||||
ok: true,
|
||||
isElevated: result.isElevated,
|
||||
platform: result.platform,
|
||||
appBuild: appRuntimeInfo(),
|
||||
hotkeys: registeredHotkeys,
|
||||
genshinFound: result.genshinFound,
|
||||
genshinHwnd: result.genshinHwnd ?? undefined,
|
||||
@@ -220,7 +226,7 @@ async function readRuntimeInfo() {
|
||||
helperPid: result.helperPid,
|
||||
};
|
||||
} catch {
|
||||
return { ok: false, isElevated: false, platform: process.platform, hotkeys: registeredHotkeys };
|
||||
return { ok: false, isElevated: false, platform: process.platform, appBuild: appRuntimeInfo(), hotkeys: registeredHotkeys };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -326,12 +332,40 @@ async function listCaptureSources() {
|
||||
}));
|
||||
}
|
||||
|
||||
async function toScreenPoint(x: number, y: number) {
|
||||
const point = { x: Math.round(x), y: Math.round(y) };
|
||||
const bounds = await getGenshinWindowBounds();
|
||||
if (!bounds) return point;
|
||||
|
||||
const looksClientRelative =
|
||||
point.x >= 0 &&
|
||||
point.y >= 0 &&
|
||||
point.x <= bounds.width &&
|
||||
point.y <= bounds.height &&
|
||||
(bounds.x !== 0 || bounds.y !== 0);
|
||||
if (!looksClientRelative) return point;
|
||||
|
||||
return {
|
||||
x: bounds.x + point.x,
|
||||
y: bounds.y + point.y,
|
||||
};
|
||||
}
|
||||
|
||||
async function clickScreenCommand(x: number, y: number) {
|
||||
return getInputHelperService().clickScreen(Math.round(x), Math.round(y));
|
||||
const point = await toScreenPoint(x, y);
|
||||
return getInputHelperService().clickScreen(point.x, point.y);
|
||||
}
|
||||
|
||||
async function scrollScreenCommand(notches: number, anchorX?: number, anchorY?: number) {
|
||||
return getInputHelperService().scrollScreen(notches, anchorX, anchorY);
|
||||
if (typeof anchorX === "number" && typeof anchorY === "number") {
|
||||
const point = await toScreenPoint(anchorX, anchorY);
|
||||
return getInputHelperService().scrollScreen(notches, point.x, point.y);
|
||||
}
|
||||
return getInputHelperService().scrollScreen(notches);
|
||||
}
|
||||
|
||||
async function keyPressCommand(key: string) {
|
||||
return getInputHelperService().keyPress(key);
|
||||
}
|
||||
|
||||
async function getAutomationGuardCommand() {
|
||||
@@ -439,13 +473,19 @@ function startDevControlServer() {
|
||||
if (!isDev || devControlServer) return;
|
||||
devControlServer = createDevControlServer({
|
||||
registeredHotkeys,
|
||||
appBuild: appRuntimeInfo(),
|
||||
hasMainWindow: () => Boolean(mainWindow && !mainWindow.isDestroyed()),
|
||||
sendScannerCommand,
|
||||
clickScreen: clickScreenCommand,
|
||||
scannerStatus: () => scannerDevStatus,
|
||||
scannerStatus: () => ({ ...scannerDevStatus, appBuild: appRuntimeInfo(), ocrWarmup: getOcrWarmupStatus() }),
|
||||
warmOcr: (engine) => warmOcrWorkerPool(engine),
|
||||
loadReviewSamples,
|
||||
listCaptureSources,
|
||||
captureSource,
|
||||
requestShutdown: (reason) => {
|
||||
console.log(`[dev-control] shutdown requested: ${reason}`);
|
||||
app.quit();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -489,60 +529,284 @@ function createOverlayWindow() {
|
||||
});
|
||||
}
|
||||
|
||||
function dataUrlToBuffer(dataUrl: string) {
|
||||
const base64 = dataUrl.replace(/^data:image\/png;base64,/, "");
|
||||
return Buffer.from(base64, "base64");
|
||||
}
|
||||
// Inventory Kamera keeps a pool of native Tesseract engines and scans artifact
|
||||
// fields concurrently. Our fast artifact profile has four useful OCR parameter
|
||||
// groups, so the default pool is four workers unless the machine is smaller or
|
||||
// GAA_OCR_WORKERS explicitly overrides it.
|
||||
const OCR_WORKER_POOL_SIZE = resolveOcrWorkerPoolSize();
|
||||
const IK_TRAINEDDATA_LANG = "genshin_fast_09_04_21";
|
||||
|
||||
// One shared OCR worker. Creating a Tesseract worker per capture added ~1s
|
||||
// to every artifact during batch scans.
|
||||
let ocrWorkerPromise: ReturnType<typeof createWorker> | null = null;
|
||||
type OcrWorker = Awaited<ReturnType<typeof createWorker>>;
|
||||
type OcrWorkerEngine = "current" | "ik-traineddata";
|
||||
type OcrCropPayload = { id: string; label: string; image: Buffer };
|
||||
type OcrWarmupStatus = {
|
||||
engine: OcrWorkerEngine;
|
||||
status: "cold" | "warming" | "ready" | "error";
|
||||
workerPoolSize: number;
|
||||
startedAt?: string;
|
||||
readyAt?: string;
|
||||
elapsedMs?: number;
|
||||
error?: string;
|
||||
};
|
||||
type OcrWorkerSlot = {
|
||||
worker: OcrWorker;
|
||||
parametersKey: string;
|
||||
};
|
||||
type OcrWorkerPoolState = {
|
||||
poolPromise: Promise<OcrWorkerSlot[]> | null;
|
||||
runQueue: Promise<unknown>;
|
||||
};
|
||||
|
||||
function getOcrWorker() {
|
||||
if (!ocrWorkerPromise) {
|
||||
ocrWorkerPromise = createWorker("eng");
|
||||
const ocrWorkerPools: Record<OcrWorkerEngine, OcrWorkerPoolState> = {
|
||||
current: { poolPromise: null, runQueue: Promise.resolve() },
|
||||
"ik-traineddata": { poolPromise: null, runQueue: Promise.resolve() },
|
||||
};
|
||||
const ocrWarmupStatuses: Record<OcrWorkerEngine, OcrWarmupStatus> = {
|
||||
current: { engine: "current", status: "cold", workerPoolSize: OCR_WORKER_POOL_SIZE },
|
||||
"ik-traineddata": { engine: "ik-traineddata", status: "cold", workerPoolSize: OCR_WORKER_POOL_SIZE },
|
||||
};
|
||||
|
||||
function resolveOcrWorkerPoolSize() {
|
||||
const requested = Number(process.env.GAA_OCR_WORKERS ?? Number.NaN);
|
||||
if (Number.isFinite(requested) && requested > 0) {
|
||||
return Math.max(1, Math.min(8, Math.floor(requested)));
|
||||
}
|
||||
return ocrWorkerPromise;
|
||||
const logicalCores = cpus().length || 4;
|
||||
return Math.max(2, Math.min(4, logicalCores - 1));
|
||||
}
|
||||
|
||||
async function resetOcrWorker() {
|
||||
const broken = ocrWorkerPromise;
|
||||
ocrWorkerPromise = null;
|
||||
if (broken) {
|
||||
try {
|
||||
const worker = await broken;
|
||||
await worker.terminate();
|
||||
} catch {
|
||||
// Worker never initialized; nothing to clean up.
|
||||
function appRuntimeInfo(): AppRuntimeInfo {
|
||||
return {
|
||||
signature: APP_RUNTIME_SIGNATURE,
|
||||
pid: process.pid,
|
||||
startedAt: APP_RUNTIME_STARTED_AT,
|
||||
cwd: process.cwd(),
|
||||
isDev,
|
||||
expectedOcrWorkerPoolSize: OCR_WORKER_POOL_SIZE,
|
||||
};
|
||||
}
|
||||
|
||||
function ocrEngineFromOptions(options: CaptureOptions = {}): OcrWorkerEngine {
|
||||
return options.ocrEngine === "ik-traineddata" ? "ik-traineddata" : "current";
|
||||
}
|
||||
|
||||
function ikTessdataCandidates() {
|
||||
const envDir = process.env.IK_TESSDATA_DIR;
|
||||
return [
|
||||
envDir,
|
||||
path.resolve(process.cwd(), "data", "tessdata"),
|
||||
path.resolve(process.cwd(), "work", "Inventory_Kamera", "InventoryKamera", "tessdata"),
|
||||
path.resolve(process.cwd(), "work", "refs", "Inventory_Kamera", "InventoryKamera", "tessdata"),
|
||||
path.resolve(process.cwd(), "..", "_ik_ref_fork", "InventoryKamera", "tessdata"),
|
||||
path.resolve(process.cwd(), "..", "_ik_ref", "InventoryKamera", "tessdata"),
|
||||
path.resolve(process.env.USERPROFILE ?? "", "Desktop", "_ik_ref_fork", "InventoryKamera", "tessdata"),
|
||||
path.resolve(process.env.USERPROFILE ?? "", "Desktop", "_ik_ref", "InventoryKamera", "tessdata"),
|
||||
process.resourcesPath ? path.resolve(process.resourcesPath, "tessdata") : "",
|
||||
].filter(Boolean) as string[];
|
||||
}
|
||||
|
||||
function findIkTessdataDir() {
|
||||
return ikTessdataCandidates().find((candidate) => existsSync(path.join(candidate, `${IK_TRAINEDDATA_LANG}.traineddata`))) ?? "";
|
||||
}
|
||||
|
||||
function getOcrWorkerOptions(engine: OcrWorkerEngine) {
|
||||
if (engine !== "ik-traineddata") return { lang: "eng", options: undefined };
|
||||
const langPath = findIkTessdataDir();
|
||||
if (!langPath) {
|
||||
throw new Error(`IK traineddata not found. Set IK_TESSDATA_DIR or place ${IK_TRAINEDDATA_LANG}.traineddata in data/tessdata.`);
|
||||
}
|
||||
return {
|
||||
lang: IK_TRAINEDDATA_LANG,
|
||||
options: {
|
||||
langPath,
|
||||
gzip: false,
|
||||
cachePath: app.isReady() ? path.join(app.getPath("userData"), "tessdata-cache") : path.resolve(process.cwd(), "outputs", "tessdata-cache"),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function getOcrWorkerPool(engine: OcrWorkerEngine) {
|
||||
const state = ocrWorkerPools[engine];
|
||||
if (!state.poolPromise) {
|
||||
const { lang, options } = getOcrWorkerOptions(engine);
|
||||
state.poolPromise = Promise.all(
|
||||
Array.from({ length: OCR_WORKER_POOL_SIZE }, async () => ({
|
||||
worker: await createWorker(lang, 1, options),
|
||||
parametersKey: "",
|
||||
})),
|
||||
);
|
||||
}
|
||||
return state.poolPromise;
|
||||
}
|
||||
|
||||
async function resetOcrWorker(engine?: OcrWorkerEngine) {
|
||||
const engines: OcrWorkerEngine[] = engine ? [engine] : ["current", "ik-traineddata"];
|
||||
await Promise.all(engines.map(async (engineId) => {
|
||||
const state = ocrWorkerPools[engineId];
|
||||
const broken = state.poolPromise;
|
||||
state.poolPromise = null;
|
||||
state.runQueue = Promise.resolve();
|
||||
ocrWarmupStatuses[engineId] = { engine: engineId, status: "cold", workerPoolSize: OCR_WORKER_POOL_SIZE };
|
||||
if (broken) {
|
||||
try {
|
||||
const pool = await broken;
|
||||
await Promise.all(pool.map((slot) => slot.worker.terminate().catch(() => undefined)));
|
||||
} catch {
|
||||
// Workers never initialized; nothing to clean up.
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
function ocrParametersForCrop(cropId: string) {
|
||||
switch (cropId) {
|
||||
case "artifact-level":
|
||||
return {
|
||||
tessedit_pageseg_mode: PSM.SINGLE_WORD,
|
||||
tessedit_char_whitelist: "0123456789+",
|
||||
};
|
||||
case "artifact-main-stat-value":
|
||||
return {
|
||||
tessedit_pageseg_mode: PSM.SINGLE_LINE,
|
||||
tessedit_char_whitelist: "0123456789.,%+",
|
||||
};
|
||||
case "inventory-count":
|
||||
return {
|
||||
tessedit_pageseg_mode: PSM.SINGLE_LINE,
|
||||
tessedit_char_whitelist: "0123456789/",
|
||||
};
|
||||
case "artifact-name":
|
||||
case "artifact-slot":
|
||||
case "artifact-main-stat-label":
|
||||
case "artifact-footer":
|
||||
return {
|
||||
tessedit_pageseg_mode: PSM.SINGLE_LINE,
|
||||
tessedit_char_whitelist: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .:'%-",
|
||||
};
|
||||
case "artifact-main-stat":
|
||||
return {
|
||||
tessedit_pageseg_mode: PSM.SINGLE_BLOCK,
|
||||
tessedit_char_whitelist: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .,%+",
|
||||
};
|
||||
case "artifact-substats":
|
||||
return {
|
||||
tessedit_pageseg_mode: PSM.SINGLE_BLOCK,
|
||||
tessedit_char_whitelist: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .,%+-",
|
||||
};
|
||||
default:
|
||||
return {
|
||||
tessedit_pageseg_mode: PSM.SINGLE_BLOCK,
|
||||
tessedit_char_whitelist: "",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function runOcrOnCrops(crops: Array<{ id: string; label: string; dataUrl: string }>) {
|
||||
function ocrParametersKey(cropId: string) {
|
||||
const params = ocrParametersForCrop(cropId);
|
||||
return `${params.tessedit_pageseg_mode}:${params.tessedit_char_whitelist}`;
|
||||
}
|
||||
|
||||
async function applyOcrParameters(slot: OcrWorkerSlot, cropId: string) {
|
||||
const params = ocrParametersForCrop(cropId);
|
||||
const key = ocrParametersKey(cropId);
|
||||
if (key === slot.parametersKey) return;
|
||||
await slot.worker.setParameters(params);
|
||||
slot.parametersKey = key;
|
||||
}
|
||||
|
||||
function warmOcrWorkerPool(engine: OcrWorkerEngine = "current") {
|
||||
const current = ocrWarmupStatuses[engine];
|
||||
if (current.status === "warming" || current.status === "ready") return Promise.resolve(current);
|
||||
|
||||
const started = Date.now();
|
||||
ocrWarmupStatuses[engine] = {
|
||||
engine,
|
||||
status: "warming",
|
||||
workerPoolSize: OCR_WORKER_POOL_SIZE,
|
||||
startedAt: new Date(started).toISOString(),
|
||||
};
|
||||
|
||||
return getOcrWorkerPool(engine)
|
||||
.then(async (pool) => {
|
||||
await Promise.all(pool.map((slot) => applyOcrParameters(slot, "artifact-name")));
|
||||
const readyAt = Date.now();
|
||||
ocrWarmupStatuses[engine] = {
|
||||
engine,
|
||||
status: "ready",
|
||||
workerPoolSize: OCR_WORKER_POOL_SIZE,
|
||||
startedAt: ocrWarmupStatuses[engine].startedAt,
|
||||
readyAt: new Date(readyAt).toISOString(),
|
||||
elapsedMs: readyAt - started,
|
||||
};
|
||||
return ocrWarmupStatuses[engine];
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
ocrWarmupStatuses[engine] = {
|
||||
engine,
|
||||
status: "error",
|
||||
workerPoolSize: OCR_WORKER_POOL_SIZE,
|
||||
startedAt: ocrWarmupStatuses[engine].startedAt,
|
||||
elapsedMs: Date.now() - started,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
};
|
||||
return ocrWarmupStatuses[engine];
|
||||
});
|
||||
}
|
||||
|
||||
function getOcrWarmupStatus() {
|
||||
return {
|
||||
current: ocrWarmupStatuses.current,
|
||||
"ik-traineddata": ocrWarmupStatuses["ik-traineddata"],
|
||||
};
|
||||
}
|
||||
|
||||
async function runOcrOnCrops(crops: OcrCropPayload[], engine: OcrWorkerEngine) {
|
||||
try {
|
||||
const worker = await getOcrWorker();
|
||||
const results = [];
|
||||
for (const crop of crops) {
|
||||
const recognized = await worker.recognize(dataUrlToBuffer(crop.dataUrl));
|
||||
results.push({
|
||||
id: crop.id,
|
||||
label: crop.label,
|
||||
text: cleanOcrText(crop.id, recognized.data.text),
|
||||
confidence: Math.round(recognized.data.confidence),
|
||||
});
|
||||
}
|
||||
const pool = await getOcrWorkerPool(engine);
|
||||
const results: OcrResult[] = new Array(crops.length);
|
||||
let nextCropIndex = 0;
|
||||
const workers = pool.slice(0, Math.min(pool.length, crops.length || 1));
|
||||
|
||||
await Promise.all(workers.map(async (slot) => {
|
||||
while (nextCropIndex < crops.length) {
|
||||
const index = nextCropIndex++;
|
||||
const crop = crops[index];
|
||||
if (!crop) continue;
|
||||
results[index] = await recognizeCropWithSlot(slot, crop);
|
||||
}
|
||||
}));
|
||||
return results;
|
||||
} catch (error) {
|
||||
await resetOcrWorker();
|
||||
await resetOcrWorker(engine);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function runOcrOnCropsWithTimeout(crops: Array<{ id: string; label: string; dataUrl: string }>, timeoutMs = 6500) {
|
||||
async function recognizeCropWithSlot(slot: OcrWorkerSlot, crop: OcrCropPayload): Promise<OcrResult> {
|
||||
const startedAt = Date.now();
|
||||
await applyOcrParameters(slot, crop.id);
|
||||
const recognized = await slot.worker.recognize(crop.image);
|
||||
return {
|
||||
id: crop.id,
|
||||
label: crop.label,
|
||||
text: cleanOcrText(crop.id, recognized.data.text),
|
||||
confidence: Math.round(recognized.data.confidence),
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function runQueuedOcrOnCrops(crops: OcrCropPayload[], engine: OcrWorkerEngine) {
|
||||
const state = ocrWorkerPools[engine];
|
||||
const run = state.runQueue.catch(() => undefined).then(() => runOcrOnCrops(crops, engine));
|
||||
state.runQueue = run.catch(() => undefined);
|
||||
return run;
|
||||
}
|
||||
|
||||
async function runOcrOnCropsWithTimeout(crops: OcrCropPayload[], engine: OcrWorkerEngine, timeoutMs = 6500) {
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
runOcrOnCrops(crops).then((ocr) => ({ ocr, timedOut: false })),
|
||||
runQueuedOcrOnCrops(crops, engine).then((ocr) => ({ ocr, timedOut: false })),
|
||||
new Promise<{ ocr: Awaited<ReturnType<typeof runOcrOnCrops>>; timedOut: boolean }>((resolve) => {
|
||||
timeout = setTimeout(() => resolve({ ocr: [], timedOut: true }), timeoutMs);
|
||||
}),
|
||||
@@ -569,7 +833,7 @@ function cleanOcrText(cropId: string, text: string) {
|
||||
return match ? `Equipped: ${match[1].replace(/[^A-Za-z'\-\s]/g, "").trim()}` : equipped;
|
||||
}
|
||||
|
||||
if (cropId === "artifact-title") {
|
||||
if (cropId === "artifact-title" || cropId === "artifact-name" || cropId === "artifact-slot" || cropId === "artifact-main-stat-label") {
|
||||
return normalized
|
||||
.filter((line) => /[A-Za-z]/.test(line))
|
||||
.slice(0, 2)
|
||||
@@ -583,10 +847,23 @@ function cleanOcrText(cropId: string, text: string) {
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
if (cropId === "artifact-main-stat-value") {
|
||||
return normalized
|
||||
.map((line) => line.replace(/[^0-9.,%]/g, ""))
|
||||
.find((line) => /[0-9]/.test(line)) ?? "";
|
||||
}
|
||||
|
||||
if (cropId === "artifact-level") {
|
||||
const value = normalized
|
||||
.map((line) => line.replace(/[^0-9+]/g, ""))
|
||||
.find((line) => /[0-9]/.test(line)) ?? "";
|
||||
return value.startsWith("+") || value === "" ? value : `+${value}`;
|
||||
}
|
||||
|
||||
if (cropId === "artifact-substats") {
|
||||
return normalized
|
||||
.filter((line) => /(\+|CRIT|ATK|DEF|HP|Energy|Elemental)/i.test(line))
|
||||
.slice(0, 5)
|
||||
.slice(0, 6)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
@@ -660,6 +937,169 @@ function isArtifactTextColor(bitmap: Buffer, index: number) {
|
||||
return green >= 180 && red >= 140 && blue <= 95 && green > blue + 35 && red > blue + 10;
|
||||
}
|
||||
|
||||
function hasEquippedFooterMarker(bitmap: Buffer, imageSize: { width: number; height: number }, rect: Electron.Rectangle) {
|
||||
const safeRect = clampCaptureRect(rect, imageSize);
|
||||
const strideX = Math.max(1, Math.floor(safeRect.width / 48));
|
||||
const strideY = Math.max(1, Math.floor(safeRect.height / 18));
|
||||
let hits = 0;
|
||||
|
||||
for (let y = safeRect.y; y < safeRect.y + safeRect.height; y += strideY) {
|
||||
const rowOffset = y * imageSize.width * 4;
|
||||
for (let x = safeRect.x; x < safeRect.x + safeRect.width; x += strideX) {
|
||||
if (isEquippedFooterYellow(bitmap, rowOffset + x * 4)) {
|
||||
hits++;
|
||||
if (hits >= 10) return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function isSanctifiedArtifactPurple(bitmap: Buffer, index: number) {
|
||||
const blue = bitmap[index];
|
||||
const green = bitmap[index + 1];
|
||||
const red = bitmap[index + 2];
|
||||
return blue >= 220 && red >= 180 && red <= 245 && green >= 155 && green <= 220 && blue > red + 10;
|
||||
}
|
||||
|
||||
function detectSanctifiedArtifactDetail(bitmap: Buffer, imageSize: { width: number; height: number }, detailRect: Electron.Rectangle) {
|
||||
const safeRect = clampCaptureRect(detailRect, imageSize);
|
||||
const x0 = Math.max(safeRect.x, Math.round(safeRect.x + safeRect.width * 0.0));
|
||||
const x1 = Math.min(imageSize.width - 1, Math.round(safeRect.x + safeRect.width * 0.0606));
|
||||
const y0 = Math.max(safeRect.y, Math.round(safeRect.y + safeRect.height * 0.3333));
|
||||
const y1 = Math.min(imageSize.height - 1, Math.round(y0 + safeRect.height * 0.0526));
|
||||
const strideX = Math.max(1, Math.floor(Math.max(1, x1 - x0) / 12));
|
||||
const strideY = Math.max(1, Math.floor(Math.max(1, y1 - y0) / 10));
|
||||
let hits = 0;
|
||||
|
||||
for (let y = y0; y <= y1; y += strideY) {
|
||||
const rowOffset = y * imageSize.width * 4;
|
||||
for (let x = x0; x <= x1; x += strideX) {
|
||||
if (isSanctifiedArtifactPurple(bitmap, rowOffset + x * 4)) {
|
||||
hits++;
|
||||
if (hits >= 3) return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function analyzeArtifactDetailPanel(bitmap: Buffer, imageSize: { width: number; height: number }, rect: Electron.Rectangle) {
|
||||
const safeRect = clampCaptureRect(rect, imageSize);
|
||||
const strideX = Math.max(1, Math.floor(safeRect.width / 96));
|
||||
const strideY = Math.max(1, Math.floor(safeRect.height / 160));
|
||||
let orangeHits = 0;
|
||||
let greenHits = 0;
|
||||
let textHits = 0;
|
||||
let titleOrangeHits = 0;
|
||||
let upperTextHits = 0;
|
||||
let lowerGreenHits = 0;
|
||||
|
||||
for (let y = safeRect.y; y < safeRect.y + safeRect.height; y += strideY) {
|
||||
const rowOffset = y * imageSize.width * 4;
|
||||
for (let x = safeRect.x; x < safeRect.x + safeRect.width; x += strideX) {
|
||||
const index = rowOffset + x * 4;
|
||||
const relativeY = (y - safeRect.y) / safeRect.height;
|
||||
const isOrange = isArtifactTitleOrange(bitmap, index);
|
||||
const isGreen = isSetTitleGreen(bitmap, index);
|
||||
const isText = isArtifactTextColor(bitmap, index);
|
||||
if (isOrange) {
|
||||
orangeHits++;
|
||||
if (relativeY <= 0.085) titleOrangeHits++;
|
||||
}
|
||||
if (isGreen) {
|
||||
greenHits++;
|
||||
if (relativeY >= 0.37 && relativeY <= 0.84) lowerGreenHits++;
|
||||
}
|
||||
if (isText) {
|
||||
textHits++;
|
||||
if (relativeY >= 0.08 && relativeY <= 0.37) upperTextHits++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const confidence = Math.max(
|
||||
0,
|
||||
Math.min(100, Math.round(titleOrangeHits * 1.6 + upperTextHits * 1.2 + lowerGreenHits * 0.7)),
|
||||
);
|
||||
return {
|
||||
present: titleOrangeHits >= 40 && (upperTextHits >= 20 || lowerGreenHits >= 40) && confidence >= 58,
|
||||
confidence,
|
||||
orangeHits,
|
||||
greenHits,
|
||||
textHits,
|
||||
titleOrangeHits,
|
||||
upperTextHits,
|
||||
lowerGreenHits,
|
||||
};
|
||||
}
|
||||
|
||||
function isPaimonProfileLight(bitmap: Buffer, index: number) {
|
||||
const blue = bitmap[index];
|
||||
const green = bitmap[index + 1];
|
||||
const red = bitmap[index + 2];
|
||||
return red > 150 && green > 150 && blue > 150;
|
||||
}
|
||||
|
||||
function isPaimonProfileCream(bitmap: Buffer, index: number) {
|
||||
const blue = bitmap[index];
|
||||
const green = bitmap[index + 1];
|
||||
const red = bitmap[index + 2];
|
||||
return red >= 195 && green >= 185 && blue >= 155;
|
||||
}
|
||||
|
||||
function isPaimonMenuTileDark(bitmap: Buffer, index: number) {
|
||||
const blue = bitmap[index];
|
||||
const green = bitmap[index + 1];
|
||||
const red = bitmap[index + 2];
|
||||
return red >= 45 && red <= 115 && green >= 55 && green <= 125 && blue >= 70 && blue <= 150;
|
||||
}
|
||||
|
||||
function sampleScreenZone(
|
||||
bitmap: Buffer,
|
||||
imageSize: { width: number; height: number },
|
||||
zone: { x0: number; x1: number; y0: number; y1: number },
|
||||
predicate: (bitmap: Buffer, index: number) => boolean,
|
||||
) {
|
||||
const x0 = Math.max(0, Math.floor(imageSize.width * zone.x0));
|
||||
const x1 = Math.min(imageSize.width, Math.ceil(imageSize.width * zone.x1));
|
||||
const y0 = Math.max(0, Math.floor(imageSize.height * zone.y0));
|
||||
const y1 = Math.min(imageSize.height, Math.ceil(imageSize.height * zone.y1));
|
||||
const strideX = Math.max(1, Math.floor((x1 - x0) / 135));
|
||||
const strideY = Math.max(1, Math.floor((y1 - y0) / 90));
|
||||
let hits = 0;
|
||||
let samples = 0;
|
||||
|
||||
for (let y = y0; y < y1; y += strideY) {
|
||||
const rowOffset = y * imageSize.width * 4;
|
||||
for (let x = x0; x < x1; x += strideX) {
|
||||
samples++;
|
||||
if (predicate(bitmap, rowOffset + x * 4)) hits++;
|
||||
}
|
||||
}
|
||||
|
||||
return samples > 0 ? (hits / samples) * 100 : 0;
|
||||
}
|
||||
|
||||
function analyzePaimonMenu(bitmap: Buffer, imageSize: { width: number; height: number }) {
|
||||
const profileZone = { x0: 0.05, x1: 0.40, y0: 0, y1: 0.30 };
|
||||
const menuTileZone = { x0: 0.06, x1: 0.39, y0: 0.32, y1: 0.98 };
|
||||
const profileLightPct = sampleScreenZone(bitmap, imageSize, profileZone, isPaimonProfileLight);
|
||||
const profileCreamPct = sampleScreenZone(bitmap, imageSize, profileZone, isPaimonProfileCream);
|
||||
const menuTileDarkPct = sampleScreenZone(bitmap, imageSize, menuTileZone, isPaimonMenuTileDark);
|
||||
const confidence = Math.max(0, Math.min(100, Math.round((profileLightPct - 20) * 1.2 + (profileCreamPct - 12) * 1.4 + (menuTileDarkPct - 25) * 1.1)));
|
||||
|
||||
return {
|
||||
present: profileLightPct >= 35 && profileCreamPct >= 20 && menuTileDarkPct >= 40,
|
||||
confidence,
|
||||
profileLightPct: Math.round(profileLightPct * 10) / 10,
|
||||
profileCreamPct: Math.round(profileCreamPct * 10) / 10,
|
||||
menuTileDarkPct: Math.round(menuTileDarkPct * 10) / 10,
|
||||
};
|
||||
}
|
||||
|
||||
function waitDelay(ms: number) {
|
||||
return new Promise<void>((resolve) => setTimeout(resolve, Math.max(0, Math.floor(ms))));
|
||||
}
|
||||
@@ -708,18 +1148,40 @@ function imageCropDataUrl(sourceImage: NativeImage, rect: Electron.Rectangle, im
|
||||
return sourceImage.crop(safeRect).toDataURL();
|
||||
}
|
||||
|
||||
function imageCropFingerprint(sourceImage: NativeImage, rect: Electron.Rectangle, imageSize: { width: number; height: number }) {
|
||||
const safeRect = clampCaptureRect(rect, imageSize);
|
||||
const bitmap = sourceImage.crop(safeRect).getBitmap();
|
||||
let hash = 2166136261;
|
||||
const stride = Math.max(4, Math.floor(bitmap.length / 4096) * 4);
|
||||
for (let index = 0; index < bitmap.length; index += stride) {
|
||||
hash ^= bitmap[index] ?? 0;
|
||||
hash = Math.imul(hash, 16777619);
|
||||
hash ^= bitmap[index + 1] ?? 0;
|
||||
hash = Math.imul(hash, 16777619);
|
||||
hash ^= bitmap[index + 2] ?? 0;
|
||||
hash = Math.imul(hash, 16777619);
|
||||
}
|
||||
return `${safeRect.width}x${safeRect.height}:${(hash >>> 0).toString(16)}`;
|
||||
}
|
||||
|
||||
// Preprocessed copy of a crop for OCR (ADR-009): upscale for more pixels, then
|
||||
// grayscale + Otsu-binarize with inversion (artifact text is the bright
|
||||
// foreground). The original crop is kept separately for the diagnostics UI.
|
||||
function preprocessedCropDataUrl(sourceImage: NativeImage, rect: Electron.Rectangle, imageSize: { width: number; height: number }) {
|
||||
// OCR receives a PNG buffer directly to avoid DataURL encode/decode churn in
|
||||
// the auto-scan hot path.
|
||||
function preprocessedCropPngBuffer(sourceImage: NativeImage, rect: Electron.Rectangle, imageSize: { width: number; height: number }, cropId = "") {
|
||||
const safeRect = clampCaptureRect(rect, imageSize);
|
||||
const upscaled = sourceImage.crop(safeRect).resize({ width: Math.max(1, safeRect.width * 2), quality: "best" });
|
||||
const scale = cropId === "artifact-level" ? 3 : 2;
|
||||
const upscaled = sourceImage.crop(safeRect).resize({ width: Math.max(1, safeRect.width * scale), quality: "best" });
|
||||
const size = upscaled.getSize();
|
||||
if (!size.width || !size.height) return upscaled.toDataURL();
|
||||
const binarized = binarizeForOcr({ data: upscaled.getBitmap(), width: size.width, height: size.height });
|
||||
if (!size.width || !size.height) return upscaled.toPNG();
|
||||
const binarized = binarizeForOcr(
|
||||
{ data: upscaled.getBitmap(), width: size.width, height: size.height },
|
||||
cropId === "artifact-level" ? { contrast: 80 } : {},
|
||||
);
|
||||
return nativeImage
|
||||
.createFromBitmap(Buffer.from(binarized.data), { width: binarized.width, height: binarized.height })
|
||||
.toDataURL();
|
||||
.toPNG();
|
||||
}
|
||||
|
||||
function createCrops(
|
||||
@@ -727,10 +1189,27 @@ function createCrops(
|
||||
imageSize: { width: number; height: number },
|
||||
detailRect: Electron.Rectangle,
|
||||
inventoryRect: Electron.Rectangle,
|
||||
options: CaptureOptions = {},
|
||||
bitmap?: Buffer,
|
||||
cropOptions: { sanctified?: boolean; skipOcr?: boolean } = {},
|
||||
) {
|
||||
const templates: CropTemplate[] = detailCropRects(detailRect, imageSize);
|
||||
const isArtifactScanMode = options.ocrMode === "artifact";
|
||||
const fastArtifactProfile = isArtifactScanMode && options.ocrProfile === "fast";
|
||||
const omitCropImages = Boolean(options.omitCropImages);
|
||||
const skipCropOcr = Boolean(cropOptions.skipOcr);
|
||||
const templates: CropTemplate[] = detailCropRects(detailRect, imageSize, { ...cropOptions, fastProfile: fastArtifactProfile })
|
||||
.filter((template) => {
|
||||
if (fastArtifactProfile && (
|
||||
template.id === "artifact-set-effects" ||
|
||||
template.id === "artifact-slot" ||
|
||||
template.id === "artifact-main-stat-value"
|
||||
)) return false;
|
||||
if (template.id === "artifact-footer" && (options.omitEquippedOcr || fastArtifactProfile)) return false;
|
||||
if (!isArtifactScanMode || template.id !== "artifact-footer" || !bitmap) return true;
|
||||
return hasEquippedFooterMarker(bitmap, imageSize, template.rect);
|
||||
});
|
||||
|
||||
if (inventoryRect.width > 120 && inventoryRect.height > 80) {
|
||||
if (!isArtifactScanMode && inventoryRect.width > 120 && inventoryRect.height > 80) {
|
||||
templates.push({
|
||||
id: "inventory-count",
|
||||
label: "Inventory count",
|
||||
@@ -741,12 +1220,14 @@ function createCrops(
|
||||
return templates
|
||||
.map((template) => {
|
||||
const rect = clampCaptureRect(template.rect, imageSize);
|
||||
const ocrEnabled = !skipCropOcr;
|
||||
return {
|
||||
id: template.id,
|
||||
label: template.label,
|
||||
rect,
|
||||
dataUrl: imageCropDataUrl(sourceImage, rect, imageSize),
|
||||
ocrDataUrl: preprocessedCropDataUrl(sourceImage, rect, imageSize),
|
||||
dataUrl: omitCropImages ? undefined : imageCropDataUrl(sourceImage, rect, imageSize),
|
||||
ocrImage: ocrEnabled ? preprocessedCropPngBuffer(sourceImage, rect, imageSize, template.id) : undefined,
|
||||
ocrEnabled,
|
||||
};
|
||||
})
|
||||
.filter((crop) => crop.rect.width > 0 && crop.rect.height > 0);
|
||||
@@ -815,6 +1296,7 @@ async function buildCaptureResult(
|
||||
captureTarget: CaptureResult["captureTarget"],
|
||||
options: CaptureOptions = {},
|
||||
) {
|
||||
const buildStartedAt = Date.now();
|
||||
const size = sourceImage.getSize();
|
||||
if (!size.width || !size.height) {
|
||||
throw new Error("Capture produced an empty image.");
|
||||
@@ -822,22 +1304,52 @@ async function buildCaptureResult(
|
||||
|
||||
const bitmap = sourceImage.getBitmap();
|
||||
const detailRect = inferDetailRect(bitmap, size);
|
||||
const artifactDetail = analyzeArtifactDetailPanel(bitmap, size, detailRect);
|
||||
const paimonMenu = analyzePaimonMenu(bitmap, size);
|
||||
const sanctified = detectSanctifiedArtifactDetail(bitmap, size, detailRect);
|
||||
const skipOcrForMissingDetail = Boolean(options.skipOcrUnlessArtifactDetail && !artifactDetail.present);
|
||||
const shouldSkipOcr = Boolean(options.skipOcr || skipOcrForMissingDetail);
|
||||
const inventoryRect = inferInventoryRect(size, detailRect);
|
||||
const crops = createCrops(sourceImage, size, detailRect, inventoryRect);
|
||||
const lockRect = clampCaptureRect(lockIconCropRect(detailRect, size), size);
|
||||
const lockImage = sourceImage.crop(lockRect);
|
||||
const lockSize = lockImage.getSize();
|
||||
const locked = lockSize.width > 0 && lockSize.height > 0
|
||||
? detectLockState({ data: lockImage.getBitmap(), width: lockSize.width, height: lockSize.height })
|
||||
: undefined;
|
||||
const croppedPayload = crops.map((crop) => ({
|
||||
id: crop.id,
|
||||
label: crop.label,
|
||||
// OCR reads the preprocessed (upscaled + binarized) crop; the original is
|
||||
// kept below for the diagnostics UI.
|
||||
dataUrl: crop.ocrDataUrl ?? crop.dataUrl,
|
||||
}));
|
||||
const recognized = options.skipOcr ? { ocr: [], timedOut: false } : await runOcrOnCropsWithTimeout(croppedPayload);
|
||||
const omitCrops = Boolean(options.omitCrops);
|
||||
const crops = omitCrops
|
||||
? []
|
||||
: createCrops(sourceImage, size, detailRect, inventoryRect, options, bitmap, { sanctified, skipOcr: shouldSkipOcr });
|
||||
const locked = options.omitLockState
|
||||
? undefined
|
||||
: (() => {
|
||||
const lockRect = clampCaptureRect(lockIconCropRect(detailRect, size), size);
|
||||
const lockImage = sourceImage.crop(lockRect);
|
||||
const lockSize = lockImage.getSize();
|
||||
return lockSize.width > 0 && lockSize.height > 0
|
||||
? detectLockState({ data: lockImage.getBitmap(), width: lockSize.width, height: lockSize.height })
|
||||
: undefined;
|
||||
})();
|
||||
const omitFullFrame = Boolean(options.omitFullFrame || options.ocrMode === "artifact");
|
||||
const omitDetailPreview = Boolean(options.omitDetailPreview);
|
||||
const omitInventoryPreview = Boolean(options.omitInventoryPreview || options.ocrMode === "artifact");
|
||||
const detailFingerprint = imageCropFingerprint(sourceImage, detailRect, size);
|
||||
const inventoryFingerprint = imageCropFingerprint(sourceImage, inventoryRect, size);
|
||||
const croppedPayload = crops
|
||||
.filter((crop) => crop.ocrEnabled !== false)
|
||||
.map((crop) => {
|
||||
return crop.ocrImage
|
||||
? {
|
||||
id: crop.id,
|
||||
label: crop.label,
|
||||
image: crop.ocrImage,
|
||||
}
|
||||
: null;
|
||||
})
|
||||
.filter((crop): crop is OcrCropPayload => Boolean(crop));
|
||||
const prepareMs = Date.now() - buildStartedAt;
|
||||
const ocrEngine = ocrEngineFromOptions(options);
|
||||
const ocrStartedAt = Date.now();
|
||||
const recognized = shouldSkipOcr
|
||||
? { ocr: [], timedOut: false }
|
||||
: await runOcrOnCropsWithTimeout(croppedPayload, ocrEngine);
|
||||
const ocrMs = Date.now() - ocrStartedAt;
|
||||
const totalMs = Date.now() - buildStartedAt;
|
||||
const captureOcrEngine: CaptureOptions["ocrEngine"] = ocrEngine === "ik-traineddata" ? "ik-traineddata" : "current";
|
||||
|
||||
const count = parseInventoryCount(recognized.ocr);
|
||||
return {
|
||||
@@ -845,14 +1357,16 @@ async function buildCaptureResult(
|
||||
name: sourceName,
|
||||
width: size.width,
|
||||
height: size.height,
|
||||
dataUrl: sourceImage.toDataURL(),
|
||||
dataUrl: omitFullFrame ? "" : sourceImage.toDataURL(),
|
||||
capturedAt: new Date().toISOString(),
|
||||
captureTarget,
|
||||
detailDataUrl: imageCropDataUrl(sourceImage, detailRect, size),
|
||||
inventoryDataUrl: imageCropDataUrl(sourceImage, inventoryRect, size),
|
||||
detailDataUrl: omitDetailPreview ? undefined : imageCropDataUrl(sourceImage, detailRect, size),
|
||||
inventoryDataUrl: omitInventoryPreview ? undefined : imageCropDataUrl(sourceImage, inventoryRect, size),
|
||||
detailFingerprint,
|
||||
inventoryFingerprint,
|
||||
ocr: recognized.ocr,
|
||||
ocrTimedOut: recognized.timedOut,
|
||||
ocrSkipped: Boolean(options.skipOcr),
|
||||
ocrSkipped: shouldSkipOcr,
|
||||
crops: crops.map((crop) => ({
|
||||
id: crop.id,
|
||||
label: crop.label,
|
||||
@@ -865,13 +1379,30 @@ async function buildCaptureResult(
|
||||
dataUrl: crop.dataUrl,
|
||||
})),
|
||||
inventoryGrid: inferInventoryGrid(size, detailRect),
|
||||
artifactDetail,
|
||||
paimonMenu,
|
||||
inventoryCount: count,
|
||||
locked,
|
||||
sanctified,
|
||||
layout: {
|
||||
aspect: aspectRatioLabel(size),
|
||||
isSixteenNine: isSixteenNine(size),
|
||||
warning: layoutSupportWarning(size),
|
||||
},
|
||||
timings: {
|
||||
totalMs,
|
||||
prepareMs,
|
||||
ocrMs: shouldSkipOcr ? 0 : ocrMs,
|
||||
cropCount: croppedPayload.length,
|
||||
ocrEngine: captureOcrEngine,
|
||||
ocrWorkerPoolSize: OCR_WORKER_POOL_SIZE,
|
||||
ocrProfile: options.ocrProfile ?? "full",
|
||||
ocrFieldMs: recognized.ocr.reduce<Record<string, number>>((fields, result) => {
|
||||
if (typeof result.elapsedMs === "number") fields[result.id] = result.elapsedMs;
|
||||
return fields;
|
||||
}, {}),
|
||||
ocrSkipped: shouldSkipOcr,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -993,12 +1524,14 @@ function initializeAppLifecycle() {
|
||||
) => captureSource(id, delayMs, focus, captureOptions),
|
||||
clickScreen: (x: number, y: number) => clickScreenCommand(x, y),
|
||||
scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => scrollScreenCommand(notches, anchorX, anchorY),
|
||||
keyPress: (key: string) => keyPressCommand(key),
|
||||
getAutomationGuard: () => getAutomationGuardCommand(),
|
||||
});
|
||||
|
||||
createMainWindow();
|
||||
registerScannerHotkeys();
|
||||
startDevControlServer();
|
||||
void warmOcrWorkerPool("current");
|
||||
});
|
||||
|
||||
app.on("activate", () => {
|
||||
@@ -1025,4 +1558,3 @@ function initializeAppLifecycle() {
|
||||
}
|
||||
|
||||
initializeAppLifecycle();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user