Improve IK-style artifact scanner pipeline

This commit is contained in:
AzuTear
2026-07-07 22:02:24 +02:00
parent 8ebbe91c39
commit f791d1464c
70 changed files with 7408 additions and 445 deletions
+3
View File
@@ -14,6 +14,7 @@ import type {
ClickResult,
AutomationGuard,
ScrollResult,
KeyPressResult,
SaveResultWithPath,
SaveSnapshotResult,
GoodDatabase,
@@ -60,6 +61,7 @@ interface CaptureHandlersDependencies {
captureSource: (sourceId: string, delayMs?: number, focusGenshin?: boolean, options?: CaptureOptions) => Promise<CaptureResult>;
clickScreen: (x: number, y: number) => Promise<ClickResult>;
scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => Promise<ScrollResult>;
keyPress: (key: string) => Promise<KeyPressResult>;
getAutomationGuard: () => Promise<AutomationGuard>;
}
@@ -96,6 +98,7 @@ export function registerIpcHandlers(dependencies: IpcBootstrapDependencies) {
captureSource: dependencies.captureSource,
clickScreen: dependencies.clickScreen,
scrollScreen: dependencies.scrollScreen,
keyPress: dependencies.keyPress,
getAutomationGuard: dependencies.getAutomationGuard,
});
}
+177 -3
View File
@@ -1,6 +1,8 @@
import fs from "node:fs/promises";
import http, { type Server } from "node:http";
import path from "node:path";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import type {
CaptureOptions,
CaptureResult,
@@ -9,14 +11,20 @@ import type {
ReviewSampleListResult,
ScannerCommand,
ScannerStatusPayload,
AppRuntimeInfo,
} from "../src/types/global.js";
import { validateLookupPackage } from "../src/lib/genshinLookup.js";
const execFileAsync = promisify(execFile);
interface DevControlServerDependencies {
registeredHotkeys: Record<string, boolean>;
appBuild: AppRuntimeInfo;
hasMainWindow: () => boolean;
sendScannerCommand: (command: ScannerCommand | "probe-click") => void;
clickScreen: (x: number, y: number) => Promise<ClickResult>;
scannerStatus: () => ScannerStatusPayload;
warmOcr: (engine: "current" | "ik-traineddata") => Promise<unknown>;
loadReviewSamples: (limit?: number) => Promise<ReviewSampleListResult>;
listCaptureSources: () => Promise<CaptureSourceInfo[]>;
captureSource: (
@@ -25,6 +33,7 @@ interface DevControlServerDependencies {
focusGenshin?: boolean,
options?: CaptureOptions,
) => Promise<CaptureResult>;
requestShutdown?: (reason: string) => void;
}
function writeDevJson(res: http.ServerResponse, statusCode: number, payload: unknown) {
@@ -136,13 +145,30 @@ export function createDevControlServer(deps: DevControlServerDependencies): Serv
const url = new URL(req.url ?? "/", "http://127.0.0.1");
if (url.pathname === "/health") {
writeDevJson(res, 200, { ok: true, hotkeys: deps.registeredHotkeys, hasWindow: deps.hasMainWindow() });
writeDevJson(res, 200, { ok: true, hotkeys: deps.registeredHotkeys, hasWindow: deps.hasMainWindow(), appBuild: deps.appBuild });
return;
}
if (url.pathname === "/dev/shutdown") {
if (!deps.requestShutdown) {
writeDevJson(res, 501, { ok: false, error: "shutdown not supported" });
return;
}
const reason = url.searchParams.get("reason") || "dev-control shutdown requested";
writeDevJson(res, 200, { ok: true, appBuild: deps.appBuild, reason });
setTimeout(() => deps.requestShutdown?.(reason), 50);
return;
}
if (url.pathname === "/scanner/start") {
const limit = Number(url.searchParams.get("limit") ?? Number.NaN);
const command: ScannerCommand = Number.isFinite(limit) && limit > 0
? { type: "start-auto", scanLimit: limit }
const entry = url.searchParams.get("entry");
const engine = url.searchParams.get("engine");
const scanEntryMode = entry === "paimon-menu" || entry === "visible-inventory" || entry === "direct-inventory" || entry === "auto-entry"
? entry
: undefined;
const ocrEngine = engine === "ik-traineddata" ? "ik-traineddata" : engine === "current" ? "current" : undefined;
const hasLimit = Number.isFinite(limit) && limit > 0;
const command: ScannerCommand = hasLimit || scanEntryMode || ocrEngine
? { type: "start-auto", scanLimit: hasLimit ? limit : undefined, scanEntryMode, ocrEngine }
: "start-auto";
deps.sendScannerCommand(command);
writeDevJson(res, 200, { ok: true, command });
@@ -174,6 +200,154 @@ export function createDevControlServer(deps: DevControlServerDependencies): Serv
writeDevJson(res, 200, { ok: true, status: deps.scannerStatus() });
return;
}
if (url.pathname === "/scanner/ocr/warmup") {
const engineParam = url.searchParams.get("engine");
const engine = engineParam === "ik-traineddata" ? "ik-traineddata" : "current";
deps.warmOcr(engine)
.then((status) => writeDevJson(res, 200, { ok: true, status }))
.catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
return;
}
if (url.pathname === "/scanner/lookup/status") {
const status = validateLookupPackage();
writeDevJson(res, status.valid ? 200 : 409, { ok: status.valid, status });
return;
}
if (url.pathname === "/scanner/lookup/regenerate") {
execFileAsync("node", ["scripts/generate-genshin-data.cjs"], { cwd: process.cwd(), windowsHide: true, timeout: 120000 })
.then(({ stdout, stderr }) => {
const status = validateLookupPackage();
writeDevJson(res, status.valid ? 200 : 409, { ok: status.valid, status, stdout, stderr });
})
.catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
return;
}
if (url.pathname === "/scanner/benchmark-ocr") {
const limit = Math.max(1, Math.min(100, Number(url.searchParams.get("limit") ?? 1) || 1));
const sourceId = url.searchParams.get("sourceId");
const engineParam = url.searchParams.get("engine");
const profileParam = url.searchParams.get("profile");
const ocrProfile: "full" | "fast" = profileParam === "full" ? "full" : "fast";
const engines: Array<"current" | "ik-traineddata"> = engineParam === "compare"
? ["current", "ik-traineddata"]
: engineParam === "ik-traineddata"
? ["ik-traineddata"]
: ["current"];
deps.listCaptureSources()
.then(async (sources) => {
const source = findGenshinSource(sources, sourceId);
if (!source) {
writeDevJson(res, 404, { ok: false, error: "No Genshin capture source found.", sources: sourceListForError(sources) });
return;
}
const benchmarkSource = source;
async function runEngineBenchmark(engine: "current" | "ik-traineddata") {
const startedAt = Date.now();
const captures: Array<{
index: number;
elapsedMs: number;
ocrFields: number;
timedOut: boolean;
ocrSkipped: boolean;
artifactDetailConfidence: number;
sanctified: boolean;
prepareMs: number;
ocrMs: number;
totalMs: number;
ocrProfile?: "full" | "fast";
ocrWorkerPoolSize?: number;
ocrFieldMs?: Record<string, number>;
}> = [];
for (let index = 0; index < limit; index += 1) {
const captureStartedAt = Date.now();
const capture = await deps.captureSource(benchmarkSource.id, index === 0 ? 150 : 0, true, {
ocrMode: "artifact",
ocrProfile,
ocrEngine: engine,
omitFullFrame: true,
omitInventoryPreview: true,
skipOcrUnlessArtifactDetail: true,
});
captures.push({
index,
elapsedMs: Date.now() - captureStartedAt,
ocrFields: capture.ocr?.length ?? 0,
timedOut: Boolean(capture.ocrTimedOut),
ocrSkipped: Boolean(capture.ocrSkipped),
artifactDetailConfidence: capture.artifactDetail?.confidence ?? 0,
sanctified: Boolean(capture.sanctified),
prepareMs: capture.timings?.prepareMs ?? 0,
ocrMs: capture.timings?.ocrMs ?? 0,
totalMs: capture.timings?.totalMs ?? 0,
ocrProfile: capture.timings?.ocrProfile,
ocrWorkerPoolSize: capture.timings?.ocrWorkerPoolSize,
ocrFieldMs: capture.timings?.ocrFieldMs,
});
}
const elapsedMs = Date.now() - startedAt;
const timings = captures.map((capture) => capture.elapsedMs).sort((left, right) => left - right);
const ocrTimings = captures.map((capture) => capture.ocrMs).filter((value) => value > 0).sort((left, right) => left - right);
const averageMs = Math.round(elapsedMs / limit);
const averageOcrMs = ocrTimings.length > 0
? Math.round(ocrTimings.reduce((total, value) => total + value, 0) / ocrTimings.length)
: 0;
const percentile = (ratio: number) => timings[Math.min(timings.length - 1, Math.max(0, Math.ceil(timings.length * ratio) - 1))] ?? 0;
const ocrPercentile = (ratio: number) => ocrTimings[Math.min(ocrTimings.length - 1, Math.max(0, Math.ceil(ocrTimings.length * ratio) - 1))] ?? 0;
const ocrFieldTotals = captures.reduce<Record<string, { totalMs: number; count: number; maxMs: number }>>((fields, capture) => {
for (const [field, elapsed] of Object.entries(capture.ocrFieldMs ?? {})) {
const current = fields[field] ?? { totalMs: 0, count: 0, maxMs: 0 };
current.totalMs += elapsed;
current.count += 1;
current.maxMs = Math.max(current.maxMs, elapsed);
fields[field] = current;
}
return fields;
}, {});
const ocrFieldAverages = Object.fromEntries(
Object.entries(ocrFieldTotals).map(([field, timing]) => [
field,
{
averageMs: Math.round(timing.totalMs / Math.max(1, timing.count)),
maxMs: timing.maxMs,
count: timing.count,
},
]),
);
return {
engine,
nativeTesseract: "not-enabled",
workerPoolSize: captures.find((capture) => capture.ocrWorkerPoolSize)?.ocrWorkerPoolSize ?? null,
ocrProfile,
limit,
elapsedMs,
averageMs,
averageOcrMs,
minMs: timings[0] ?? 0,
p50Ms: percentile(0.5),
p90Ms: percentile(0.9),
maxMs: timings[timings.length - 1] ?? 0,
ocrP50Ms: ocrPercentile(0.5),
ocrP90Ms: ocrPercentile(0.9),
ocrFieldAverages,
projectedMs: {
artifacts20: averageMs * 20,
artifacts45: averageMs * 45,
artifacts100: averageMs * 100,
},
skippedOcrCaptures: captures.filter((capture) => capture.ocrSkipped).length,
captures,
};
}
const summaries = await Promise.all(engines.map((engine) => runEngineBenchmark(engine)));
writeDevJson(res, 200, {
ok: true,
summary: summaries.length === 1 ? summaries[0] : { mode: "compare", limit, ocrProfile, engines: summaries },
});
})
.catch((error: unknown) => writeDevJson(res, 500, { ok: false, error: error instanceof Error ? error.message : String(error) }));
return;
}
if (url.pathname === "/review/samples") {
deps.loadReviewSamples(Number(url.searchParams.get("limit") ?? 20))
.then((payload: unknown) => writeDevJson(res, 200, payload))
+4 -1
View File
@@ -1,11 +1,12 @@
import { ipcMain } from "electron";
import type { CaptureOptions, CaptureResult, CaptureSourceInfo, ClickResult, ScrollResult, AutomationGuard } from "../../src/types/global.js";
import type { CaptureOptions, CaptureResult, CaptureSourceInfo, ClickResult, ScrollResult, AutomationGuard, KeyPressResult } from "../../src/types/global.js";
interface CaptureCommandDependencies {
listSources: () => Promise<CaptureSourceInfo[]>;
captureSource: (sourceId: string, delayMs?: number, focusGenshin?: boolean, options?: CaptureOptions) => Promise<CaptureResult>;
clickScreen: (x: number, y: number) => Promise<ClickResult>;
scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => Promise<ScrollResult>;
keyPress: (key: string) => Promise<KeyPressResult>;
getAutomationGuard: () => Promise<AutomationGuard>;
}
@@ -14,6 +15,7 @@ export function registerCaptureHandlers({
captureSource,
clickScreen,
scrollScreen,
keyPress,
getAutomationGuard,
}: CaptureCommandDependencies) {
ipcMain.handle("capture:listSources", async () => listSources());
@@ -22,5 +24,6 @@ export function registerCaptureHandlers({
});
ipcMain.handle("automation:clickScreen", async (_event, x: number, y: number) => clickScreen(x, y));
ipcMain.handle("automation:scrollScreen", async (_event, notches: number, anchorX?: number, anchorY?: number) => scrollScreen(notches, anchorX, anchorY));
ipcMain.handle("automation:keyPress", async (_event, key: string) => keyPress(key));
ipcMain.handle("automation:getGuard", async () => getAutomationGuard());
}
+603 -71
View File
@@ -2,9 +2,10 @@ import { app, BrowserWindow, Menu, desktopCapturer, dialog, globalShortcut, nati
import fs from "node:fs/promises";
import { existsSync } from "node:fs";
import type { Server } from "node:http";
import { cpus } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { createWorker } from "tesseract.js";
import { createWorker, PSM } from "tesseract.js";
import { createInputHelperService, type InputHelperService } from "./services/inputHelper.js";
import { createRepositoryContext, type RepositoryContext } from "./bootstrap/repositoryContext.js";
import { registerIpcHandlers } from "./bootstrap/ipcBootstrap.js";
@@ -15,6 +16,8 @@ import type {
CaptureResult,
GoodDatabase,
GoodImportFileResult,
OcrResult,
AppRuntimeInfo,
SaveResultWithPath,
ScannerCommand,
ScannerLearningRulePayload,
@@ -50,6 +53,8 @@ app.commandLine.appendSwitch("disable-gpu-sandbox");
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const isDev = Boolean(process.env.VITE_DEV_SERVER_URL);
const APP_RUNTIME_STARTED_AT = new Date().toISOString();
const APP_RUNTIME_SIGNATURE = "2026-07-07-ik32-fastsubstats-active-timing";
let mainWindow: BrowserWindow | null = null;
let overlayWindow: BrowserWindow | null = null;
@@ -211,6 +216,7 @@ async function readRuntimeInfo() {
ok: true,
isElevated: result.isElevated,
platform: result.platform,
appBuild: appRuntimeInfo(),
hotkeys: registeredHotkeys,
genshinFound: result.genshinFound,
genshinHwnd: result.genshinHwnd ?? undefined,
@@ -220,7 +226,7 @@ async function readRuntimeInfo() {
helperPid: result.helperPid,
};
} catch {
return { ok: false, isElevated: false, platform: process.platform, hotkeys: registeredHotkeys };
return { ok: false, isElevated: false, platform: process.platform, appBuild: appRuntimeInfo(), hotkeys: registeredHotkeys };
}
}
@@ -326,12 +332,40 @@ async function listCaptureSources() {
}));
}
async function toScreenPoint(x: number, y: number) {
const point = { x: Math.round(x), y: Math.round(y) };
const bounds = await getGenshinWindowBounds();
if (!bounds) return point;
const looksClientRelative =
point.x >= 0 &&
point.y >= 0 &&
point.x <= bounds.width &&
point.y <= bounds.height &&
(bounds.x !== 0 || bounds.y !== 0);
if (!looksClientRelative) return point;
return {
x: bounds.x + point.x,
y: bounds.y + point.y,
};
}
async function clickScreenCommand(x: number, y: number) {
return getInputHelperService().clickScreen(Math.round(x), Math.round(y));
const point = await toScreenPoint(x, y);
return getInputHelperService().clickScreen(point.x, point.y);
}
async function scrollScreenCommand(notches: number, anchorX?: number, anchorY?: number) {
return getInputHelperService().scrollScreen(notches, anchorX, anchorY);
if (typeof anchorX === "number" && typeof anchorY === "number") {
const point = await toScreenPoint(anchorX, anchorY);
return getInputHelperService().scrollScreen(notches, point.x, point.y);
}
return getInputHelperService().scrollScreen(notches);
}
async function keyPressCommand(key: string) {
return getInputHelperService().keyPress(key);
}
async function getAutomationGuardCommand() {
@@ -439,13 +473,19 @@ function startDevControlServer() {
if (!isDev || devControlServer) return;
devControlServer = createDevControlServer({
registeredHotkeys,
appBuild: appRuntimeInfo(),
hasMainWindow: () => Boolean(mainWindow && !mainWindow.isDestroyed()),
sendScannerCommand,
clickScreen: clickScreenCommand,
scannerStatus: () => scannerDevStatus,
scannerStatus: () => ({ ...scannerDevStatus, appBuild: appRuntimeInfo(), ocrWarmup: getOcrWarmupStatus() }),
warmOcr: (engine) => warmOcrWorkerPool(engine),
loadReviewSamples,
listCaptureSources,
captureSource,
requestShutdown: (reason) => {
console.log(`[dev-control] shutdown requested: ${reason}`);
app.quit();
},
});
}
@@ -489,60 +529,284 @@ function createOverlayWindow() {
});
}
function dataUrlToBuffer(dataUrl: string) {
const base64 = dataUrl.replace(/^data:image\/png;base64,/, "");
return Buffer.from(base64, "base64");
}
// Inventory Kamera keeps a pool of native Tesseract engines and scans artifact
// fields concurrently. Our fast artifact profile has four useful OCR parameter
// groups, so the default pool is four workers unless the machine is smaller or
// GAA_OCR_WORKERS explicitly overrides it.
const OCR_WORKER_POOL_SIZE = resolveOcrWorkerPoolSize();
const IK_TRAINEDDATA_LANG = "genshin_fast_09_04_21";
// One shared OCR worker. Creating a Tesseract worker per capture added ~1s
// to every artifact during batch scans.
let ocrWorkerPromise: ReturnType<typeof createWorker> | null = null;
type OcrWorker = Awaited<ReturnType<typeof createWorker>>;
type OcrWorkerEngine = "current" | "ik-traineddata";
type OcrCropPayload = { id: string; label: string; image: Buffer };
type OcrWarmupStatus = {
engine: OcrWorkerEngine;
status: "cold" | "warming" | "ready" | "error";
workerPoolSize: number;
startedAt?: string;
readyAt?: string;
elapsedMs?: number;
error?: string;
};
type OcrWorkerSlot = {
worker: OcrWorker;
parametersKey: string;
};
type OcrWorkerPoolState = {
poolPromise: Promise<OcrWorkerSlot[]> | null;
runQueue: Promise<unknown>;
};
function getOcrWorker() {
if (!ocrWorkerPromise) {
ocrWorkerPromise = createWorker("eng");
const ocrWorkerPools: Record<OcrWorkerEngine, OcrWorkerPoolState> = {
current: { poolPromise: null, runQueue: Promise.resolve() },
"ik-traineddata": { poolPromise: null, runQueue: Promise.resolve() },
};
const ocrWarmupStatuses: Record<OcrWorkerEngine, OcrWarmupStatus> = {
current: { engine: "current", status: "cold", workerPoolSize: OCR_WORKER_POOL_SIZE },
"ik-traineddata": { engine: "ik-traineddata", status: "cold", workerPoolSize: OCR_WORKER_POOL_SIZE },
};
function resolveOcrWorkerPoolSize() {
const requested = Number(process.env.GAA_OCR_WORKERS ?? Number.NaN);
if (Number.isFinite(requested) && requested > 0) {
return Math.max(1, Math.min(8, Math.floor(requested)));
}
return ocrWorkerPromise;
const logicalCores = cpus().length || 4;
return Math.max(2, Math.min(4, logicalCores - 1));
}
async function resetOcrWorker() {
const broken = ocrWorkerPromise;
ocrWorkerPromise = null;
if (broken) {
try {
const worker = await broken;
await worker.terminate();
} catch {
// Worker never initialized; nothing to clean up.
function appRuntimeInfo(): AppRuntimeInfo {
return {
signature: APP_RUNTIME_SIGNATURE,
pid: process.pid,
startedAt: APP_RUNTIME_STARTED_AT,
cwd: process.cwd(),
isDev,
expectedOcrWorkerPoolSize: OCR_WORKER_POOL_SIZE,
};
}
function ocrEngineFromOptions(options: CaptureOptions = {}): OcrWorkerEngine {
return options.ocrEngine === "ik-traineddata" ? "ik-traineddata" : "current";
}
function ikTessdataCandidates() {
const envDir = process.env.IK_TESSDATA_DIR;
return [
envDir,
path.resolve(process.cwd(), "data", "tessdata"),
path.resolve(process.cwd(), "work", "Inventory_Kamera", "InventoryKamera", "tessdata"),
path.resolve(process.cwd(), "work", "refs", "Inventory_Kamera", "InventoryKamera", "tessdata"),
path.resolve(process.cwd(), "..", "_ik_ref_fork", "InventoryKamera", "tessdata"),
path.resolve(process.cwd(), "..", "_ik_ref", "InventoryKamera", "tessdata"),
path.resolve(process.env.USERPROFILE ?? "", "Desktop", "_ik_ref_fork", "InventoryKamera", "tessdata"),
path.resolve(process.env.USERPROFILE ?? "", "Desktop", "_ik_ref", "InventoryKamera", "tessdata"),
process.resourcesPath ? path.resolve(process.resourcesPath, "tessdata") : "",
].filter(Boolean) as string[];
}
function findIkTessdataDir() {
return ikTessdataCandidates().find((candidate) => existsSync(path.join(candidate, `${IK_TRAINEDDATA_LANG}.traineddata`))) ?? "";
}
function getOcrWorkerOptions(engine: OcrWorkerEngine) {
if (engine !== "ik-traineddata") return { lang: "eng", options: undefined };
const langPath = findIkTessdataDir();
if (!langPath) {
throw new Error(`IK traineddata not found. Set IK_TESSDATA_DIR or place ${IK_TRAINEDDATA_LANG}.traineddata in data/tessdata.`);
}
return {
lang: IK_TRAINEDDATA_LANG,
options: {
langPath,
gzip: false,
cachePath: app.isReady() ? path.join(app.getPath("userData"), "tessdata-cache") : path.resolve(process.cwd(), "outputs", "tessdata-cache"),
},
};
}
function getOcrWorkerPool(engine: OcrWorkerEngine) {
const state = ocrWorkerPools[engine];
if (!state.poolPromise) {
const { lang, options } = getOcrWorkerOptions(engine);
state.poolPromise = Promise.all(
Array.from({ length: OCR_WORKER_POOL_SIZE }, async () => ({
worker: await createWorker(lang, 1, options),
parametersKey: "",
})),
);
}
return state.poolPromise;
}
async function resetOcrWorker(engine?: OcrWorkerEngine) {
const engines: OcrWorkerEngine[] = engine ? [engine] : ["current", "ik-traineddata"];
await Promise.all(engines.map(async (engineId) => {
const state = ocrWorkerPools[engineId];
const broken = state.poolPromise;
state.poolPromise = null;
state.runQueue = Promise.resolve();
ocrWarmupStatuses[engineId] = { engine: engineId, status: "cold", workerPoolSize: OCR_WORKER_POOL_SIZE };
if (broken) {
try {
const pool = await broken;
await Promise.all(pool.map((slot) => slot.worker.terminate().catch(() => undefined)));
} catch {
// Workers never initialized; nothing to clean up.
}
}
}));
}
function ocrParametersForCrop(cropId: string) {
switch (cropId) {
case "artifact-level":
return {
tessedit_pageseg_mode: PSM.SINGLE_WORD,
tessedit_char_whitelist: "0123456789+",
};
case "artifact-main-stat-value":
return {
tessedit_pageseg_mode: PSM.SINGLE_LINE,
tessedit_char_whitelist: "0123456789.,%+",
};
case "inventory-count":
return {
tessedit_pageseg_mode: PSM.SINGLE_LINE,
tessedit_char_whitelist: "0123456789/",
};
case "artifact-name":
case "artifact-slot":
case "artifact-main-stat-label":
case "artifact-footer":
return {
tessedit_pageseg_mode: PSM.SINGLE_LINE,
tessedit_char_whitelist: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .:'%-",
};
case "artifact-main-stat":
return {
tessedit_pageseg_mode: PSM.SINGLE_BLOCK,
tessedit_char_whitelist: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .,%+",
};
case "artifact-substats":
return {
tessedit_pageseg_mode: PSM.SINGLE_BLOCK,
tessedit_char_whitelist: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .,%+-",
};
default:
return {
tessedit_pageseg_mode: PSM.SINGLE_BLOCK,
tessedit_char_whitelist: "",
};
}
}
async function runOcrOnCrops(crops: Array<{ id: string; label: string; dataUrl: string }>) {
function ocrParametersKey(cropId: string) {
const params = ocrParametersForCrop(cropId);
return `${params.tessedit_pageseg_mode}:${params.tessedit_char_whitelist}`;
}
async function applyOcrParameters(slot: OcrWorkerSlot, cropId: string) {
const params = ocrParametersForCrop(cropId);
const key = ocrParametersKey(cropId);
if (key === slot.parametersKey) return;
await slot.worker.setParameters(params);
slot.parametersKey = key;
}
function warmOcrWorkerPool(engine: OcrWorkerEngine = "current") {
const current = ocrWarmupStatuses[engine];
if (current.status === "warming" || current.status === "ready") return Promise.resolve(current);
const started = Date.now();
ocrWarmupStatuses[engine] = {
engine,
status: "warming",
workerPoolSize: OCR_WORKER_POOL_SIZE,
startedAt: new Date(started).toISOString(),
};
return getOcrWorkerPool(engine)
.then(async (pool) => {
await Promise.all(pool.map((slot) => applyOcrParameters(slot, "artifact-name")));
const readyAt = Date.now();
ocrWarmupStatuses[engine] = {
engine,
status: "ready",
workerPoolSize: OCR_WORKER_POOL_SIZE,
startedAt: ocrWarmupStatuses[engine].startedAt,
readyAt: new Date(readyAt).toISOString(),
elapsedMs: readyAt - started,
};
return ocrWarmupStatuses[engine];
})
.catch((error: unknown) => {
ocrWarmupStatuses[engine] = {
engine,
status: "error",
workerPoolSize: OCR_WORKER_POOL_SIZE,
startedAt: ocrWarmupStatuses[engine].startedAt,
elapsedMs: Date.now() - started,
error: error instanceof Error ? error.message : String(error),
};
return ocrWarmupStatuses[engine];
});
}
function getOcrWarmupStatus() {
return {
current: ocrWarmupStatuses.current,
"ik-traineddata": ocrWarmupStatuses["ik-traineddata"],
};
}
async function runOcrOnCrops(crops: OcrCropPayload[], engine: OcrWorkerEngine) {
try {
const worker = await getOcrWorker();
const results = [];
for (const crop of crops) {
const recognized = await worker.recognize(dataUrlToBuffer(crop.dataUrl));
results.push({
id: crop.id,
label: crop.label,
text: cleanOcrText(crop.id, recognized.data.text),
confidence: Math.round(recognized.data.confidence),
});
}
const pool = await getOcrWorkerPool(engine);
const results: OcrResult[] = new Array(crops.length);
let nextCropIndex = 0;
const workers = pool.slice(0, Math.min(pool.length, crops.length || 1));
await Promise.all(workers.map(async (slot) => {
while (nextCropIndex < crops.length) {
const index = nextCropIndex++;
const crop = crops[index];
if (!crop) continue;
results[index] = await recognizeCropWithSlot(slot, crop);
}
}));
return results;
} catch (error) {
await resetOcrWorker();
await resetOcrWorker(engine);
throw error;
}
}
async function runOcrOnCropsWithTimeout(crops: Array<{ id: string; label: string; dataUrl: string }>, timeoutMs = 6500) {
async function recognizeCropWithSlot(slot: OcrWorkerSlot, crop: OcrCropPayload): Promise<OcrResult> {
const startedAt = Date.now();
await applyOcrParameters(slot, crop.id);
const recognized = await slot.worker.recognize(crop.image);
return {
id: crop.id,
label: crop.label,
text: cleanOcrText(crop.id, recognized.data.text),
confidence: Math.round(recognized.data.confidence),
elapsedMs: Date.now() - startedAt,
};
}
function runQueuedOcrOnCrops(crops: OcrCropPayload[], engine: OcrWorkerEngine) {
const state = ocrWorkerPools[engine];
const run = state.runQueue.catch(() => undefined).then(() => runOcrOnCrops(crops, engine));
state.runQueue = run.catch(() => undefined);
return run;
}
async function runOcrOnCropsWithTimeout(crops: OcrCropPayload[], engine: OcrWorkerEngine, timeoutMs = 6500) {
let timeout: NodeJS.Timeout | undefined;
try {
return await Promise.race([
runOcrOnCrops(crops).then((ocr) => ({ ocr, timedOut: false })),
runQueuedOcrOnCrops(crops, engine).then((ocr) => ({ ocr, timedOut: false })),
new Promise<{ ocr: Awaited<ReturnType<typeof runOcrOnCrops>>; timedOut: boolean }>((resolve) => {
timeout = setTimeout(() => resolve({ ocr: [], timedOut: true }), timeoutMs);
}),
@@ -569,7 +833,7 @@ function cleanOcrText(cropId: string, text: string) {
return match ? `Equipped: ${match[1].replace(/[^A-Za-z'\-\s]/g, "").trim()}` : equipped;
}
if (cropId === "artifact-title") {
if (cropId === "artifact-title" || cropId === "artifact-name" || cropId === "artifact-slot" || cropId === "artifact-main-stat-label") {
return normalized
.filter((line) => /[A-Za-z]/.test(line))
.slice(0, 2)
@@ -583,10 +847,23 @@ function cleanOcrText(cropId: string, text: string) {
.join("\n");
}
if (cropId === "artifact-main-stat-value") {
return normalized
.map((line) => line.replace(/[^0-9.,%]/g, ""))
.find((line) => /[0-9]/.test(line)) ?? "";
}
if (cropId === "artifact-level") {
const value = normalized
.map((line) => line.replace(/[^0-9+]/g, ""))
.find((line) => /[0-9]/.test(line)) ?? "";
return value.startsWith("+") || value === "" ? value : `+${value}`;
}
if (cropId === "artifact-substats") {
return normalized
.filter((line) => /(\+|CRIT|ATK|DEF|HP|Energy|Elemental)/i.test(line))
.slice(0, 5)
.slice(0, 6)
.join("\n");
}
@@ -660,6 +937,169 @@ function isArtifactTextColor(bitmap: Buffer, index: number) {
return green >= 180 && red >= 140 && blue <= 95 && green > blue + 35 && red > blue + 10;
}
function hasEquippedFooterMarker(bitmap: Buffer, imageSize: { width: number; height: number }, rect: Electron.Rectangle) {
const safeRect = clampCaptureRect(rect, imageSize);
const strideX = Math.max(1, Math.floor(safeRect.width / 48));
const strideY = Math.max(1, Math.floor(safeRect.height / 18));
let hits = 0;
for (let y = safeRect.y; y < safeRect.y + safeRect.height; y += strideY) {
const rowOffset = y * imageSize.width * 4;
for (let x = safeRect.x; x < safeRect.x + safeRect.width; x += strideX) {
if (isEquippedFooterYellow(bitmap, rowOffset + x * 4)) {
hits++;
if (hits >= 10) return true;
}
}
}
return false;
}
function isSanctifiedArtifactPurple(bitmap: Buffer, index: number) {
const blue = bitmap[index];
const green = bitmap[index + 1];
const red = bitmap[index + 2];
return blue >= 220 && red >= 180 && red <= 245 && green >= 155 && green <= 220 && blue > red + 10;
}
function detectSanctifiedArtifactDetail(bitmap: Buffer, imageSize: { width: number; height: number }, detailRect: Electron.Rectangle) {
const safeRect = clampCaptureRect(detailRect, imageSize);
const x0 = Math.max(safeRect.x, Math.round(safeRect.x + safeRect.width * 0.0));
const x1 = Math.min(imageSize.width - 1, Math.round(safeRect.x + safeRect.width * 0.0606));
const y0 = Math.max(safeRect.y, Math.round(safeRect.y + safeRect.height * 0.3333));
const y1 = Math.min(imageSize.height - 1, Math.round(y0 + safeRect.height * 0.0526));
const strideX = Math.max(1, Math.floor(Math.max(1, x1 - x0) / 12));
const strideY = Math.max(1, Math.floor(Math.max(1, y1 - y0) / 10));
let hits = 0;
for (let y = y0; y <= y1; y += strideY) {
const rowOffset = y * imageSize.width * 4;
for (let x = x0; x <= x1; x += strideX) {
if (isSanctifiedArtifactPurple(bitmap, rowOffset + x * 4)) {
hits++;
if (hits >= 3) return true;
}
}
}
return false;
}
function analyzeArtifactDetailPanel(bitmap: Buffer, imageSize: { width: number; height: number }, rect: Electron.Rectangle) {
const safeRect = clampCaptureRect(rect, imageSize);
const strideX = Math.max(1, Math.floor(safeRect.width / 96));
const strideY = Math.max(1, Math.floor(safeRect.height / 160));
let orangeHits = 0;
let greenHits = 0;
let textHits = 0;
let titleOrangeHits = 0;
let upperTextHits = 0;
let lowerGreenHits = 0;
for (let y = safeRect.y; y < safeRect.y + safeRect.height; y += strideY) {
const rowOffset = y * imageSize.width * 4;
for (let x = safeRect.x; x < safeRect.x + safeRect.width; x += strideX) {
const index = rowOffset + x * 4;
const relativeY = (y - safeRect.y) / safeRect.height;
const isOrange = isArtifactTitleOrange(bitmap, index);
const isGreen = isSetTitleGreen(bitmap, index);
const isText = isArtifactTextColor(bitmap, index);
if (isOrange) {
orangeHits++;
if (relativeY <= 0.085) titleOrangeHits++;
}
if (isGreen) {
greenHits++;
if (relativeY >= 0.37 && relativeY <= 0.84) lowerGreenHits++;
}
if (isText) {
textHits++;
if (relativeY >= 0.08 && relativeY <= 0.37) upperTextHits++;
}
}
}
const confidence = Math.max(
0,
Math.min(100, Math.round(titleOrangeHits * 1.6 + upperTextHits * 1.2 + lowerGreenHits * 0.7)),
);
return {
present: titleOrangeHits >= 40 && (upperTextHits >= 20 || lowerGreenHits >= 40) && confidence >= 58,
confidence,
orangeHits,
greenHits,
textHits,
titleOrangeHits,
upperTextHits,
lowerGreenHits,
};
}
function isPaimonProfileLight(bitmap: Buffer, index: number) {
const blue = bitmap[index];
const green = bitmap[index + 1];
const red = bitmap[index + 2];
return red > 150 && green > 150 && blue > 150;
}
function isPaimonProfileCream(bitmap: Buffer, index: number) {
const blue = bitmap[index];
const green = bitmap[index + 1];
const red = bitmap[index + 2];
return red >= 195 && green >= 185 && blue >= 155;
}
function isPaimonMenuTileDark(bitmap: Buffer, index: number) {
const blue = bitmap[index];
const green = bitmap[index + 1];
const red = bitmap[index + 2];
return red >= 45 && red <= 115 && green >= 55 && green <= 125 && blue >= 70 && blue <= 150;
}
function sampleScreenZone(
bitmap: Buffer,
imageSize: { width: number; height: number },
zone: { x0: number; x1: number; y0: number; y1: number },
predicate: (bitmap: Buffer, index: number) => boolean,
) {
const x0 = Math.max(0, Math.floor(imageSize.width * zone.x0));
const x1 = Math.min(imageSize.width, Math.ceil(imageSize.width * zone.x1));
const y0 = Math.max(0, Math.floor(imageSize.height * zone.y0));
const y1 = Math.min(imageSize.height, Math.ceil(imageSize.height * zone.y1));
const strideX = Math.max(1, Math.floor((x1 - x0) / 135));
const strideY = Math.max(1, Math.floor((y1 - y0) / 90));
let hits = 0;
let samples = 0;
for (let y = y0; y < y1; y += strideY) {
const rowOffset = y * imageSize.width * 4;
for (let x = x0; x < x1; x += strideX) {
samples++;
if (predicate(bitmap, rowOffset + x * 4)) hits++;
}
}
return samples > 0 ? (hits / samples) * 100 : 0;
}
function analyzePaimonMenu(bitmap: Buffer, imageSize: { width: number; height: number }) {
const profileZone = { x0: 0.05, x1: 0.40, y0: 0, y1: 0.30 };
const menuTileZone = { x0: 0.06, x1: 0.39, y0: 0.32, y1: 0.98 };
const profileLightPct = sampleScreenZone(bitmap, imageSize, profileZone, isPaimonProfileLight);
const profileCreamPct = sampleScreenZone(bitmap, imageSize, profileZone, isPaimonProfileCream);
const menuTileDarkPct = sampleScreenZone(bitmap, imageSize, menuTileZone, isPaimonMenuTileDark);
const confidence = Math.max(0, Math.min(100, Math.round((profileLightPct - 20) * 1.2 + (profileCreamPct - 12) * 1.4 + (menuTileDarkPct - 25) * 1.1)));
return {
present: profileLightPct >= 35 && profileCreamPct >= 20 && menuTileDarkPct >= 40,
confidence,
profileLightPct: Math.round(profileLightPct * 10) / 10,
profileCreamPct: Math.round(profileCreamPct * 10) / 10,
menuTileDarkPct: Math.round(menuTileDarkPct * 10) / 10,
};
}
function waitDelay(ms: number) {
return new Promise<void>((resolve) => setTimeout(resolve, Math.max(0, Math.floor(ms))));
}
@@ -708,18 +1148,40 @@ function imageCropDataUrl(sourceImage: NativeImage, rect: Electron.Rectangle, im
return sourceImage.crop(safeRect).toDataURL();
}
function imageCropFingerprint(sourceImage: NativeImage, rect: Electron.Rectangle, imageSize: { width: number; height: number }) {
const safeRect = clampCaptureRect(rect, imageSize);
const bitmap = sourceImage.crop(safeRect).getBitmap();
let hash = 2166136261;
const stride = Math.max(4, Math.floor(bitmap.length / 4096) * 4);
for (let index = 0; index < bitmap.length; index += stride) {
hash ^= bitmap[index] ?? 0;
hash = Math.imul(hash, 16777619);
hash ^= bitmap[index + 1] ?? 0;
hash = Math.imul(hash, 16777619);
hash ^= bitmap[index + 2] ?? 0;
hash = Math.imul(hash, 16777619);
}
return `${safeRect.width}x${safeRect.height}:${(hash >>> 0).toString(16)}`;
}
// Preprocessed copy of a crop for OCR (ADR-009): upscale for more pixels, then
// grayscale + Otsu-binarize with inversion (artifact text is the bright
// foreground). The original crop is kept separately for the diagnostics UI.
function preprocessedCropDataUrl(sourceImage: NativeImage, rect: Electron.Rectangle, imageSize: { width: number; height: number }) {
// OCR receives a PNG buffer directly to avoid DataURL encode/decode churn in
// the auto-scan hot path.
function preprocessedCropPngBuffer(sourceImage: NativeImage, rect: Electron.Rectangle, imageSize: { width: number; height: number }, cropId = "") {
const safeRect = clampCaptureRect(rect, imageSize);
const upscaled = sourceImage.crop(safeRect).resize({ width: Math.max(1, safeRect.width * 2), quality: "best" });
const scale = cropId === "artifact-level" ? 3 : 2;
const upscaled = sourceImage.crop(safeRect).resize({ width: Math.max(1, safeRect.width * scale), quality: "best" });
const size = upscaled.getSize();
if (!size.width || !size.height) return upscaled.toDataURL();
const binarized = binarizeForOcr({ data: upscaled.getBitmap(), width: size.width, height: size.height });
if (!size.width || !size.height) return upscaled.toPNG();
const binarized = binarizeForOcr(
{ data: upscaled.getBitmap(), width: size.width, height: size.height },
cropId === "artifact-level" ? { contrast: 80 } : {},
);
return nativeImage
.createFromBitmap(Buffer.from(binarized.data), { width: binarized.width, height: binarized.height })
.toDataURL();
.toPNG();
}
function createCrops(
@@ -727,10 +1189,27 @@ function createCrops(
imageSize: { width: number; height: number },
detailRect: Electron.Rectangle,
inventoryRect: Electron.Rectangle,
options: CaptureOptions = {},
bitmap?: Buffer,
cropOptions: { sanctified?: boolean; skipOcr?: boolean } = {},
) {
const templates: CropTemplate[] = detailCropRects(detailRect, imageSize);
const isArtifactScanMode = options.ocrMode === "artifact";
const fastArtifactProfile = isArtifactScanMode && options.ocrProfile === "fast";
const omitCropImages = Boolean(options.omitCropImages);
const skipCropOcr = Boolean(cropOptions.skipOcr);
const templates: CropTemplate[] = detailCropRects(detailRect, imageSize, { ...cropOptions, fastProfile: fastArtifactProfile })
.filter((template) => {
if (fastArtifactProfile && (
template.id === "artifact-set-effects" ||
template.id === "artifact-slot" ||
template.id === "artifact-main-stat-value"
)) return false;
if (template.id === "artifact-footer" && (options.omitEquippedOcr || fastArtifactProfile)) return false;
if (!isArtifactScanMode || template.id !== "artifact-footer" || !bitmap) return true;
return hasEquippedFooterMarker(bitmap, imageSize, template.rect);
});
if (inventoryRect.width > 120 && inventoryRect.height > 80) {
if (!isArtifactScanMode && inventoryRect.width > 120 && inventoryRect.height > 80) {
templates.push({
id: "inventory-count",
label: "Inventory count",
@@ -741,12 +1220,14 @@ function createCrops(
return templates
.map((template) => {
const rect = clampCaptureRect(template.rect, imageSize);
const ocrEnabled = !skipCropOcr;
return {
id: template.id,
label: template.label,
rect,
dataUrl: imageCropDataUrl(sourceImage, rect, imageSize),
ocrDataUrl: preprocessedCropDataUrl(sourceImage, rect, imageSize),
dataUrl: omitCropImages ? undefined : imageCropDataUrl(sourceImage, rect, imageSize),
ocrImage: ocrEnabled ? preprocessedCropPngBuffer(sourceImage, rect, imageSize, template.id) : undefined,
ocrEnabled,
};
})
.filter((crop) => crop.rect.width > 0 && crop.rect.height > 0);
@@ -815,6 +1296,7 @@ async function buildCaptureResult(
captureTarget: CaptureResult["captureTarget"],
options: CaptureOptions = {},
) {
const buildStartedAt = Date.now();
const size = sourceImage.getSize();
if (!size.width || !size.height) {
throw new Error("Capture produced an empty image.");
@@ -822,22 +1304,52 @@ async function buildCaptureResult(
const bitmap = sourceImage.getBitmap();
const detailRect = inferDetailRect(bitmap, size);
const artifactDetail = analyzeArtifactDetailPanel(bitmap, size, detailRect);
const paimonMenu = analyzePaimonMenu(bitmap, size);
const sanctified = detectSanctifiedArtifactDetail(bitmap, size, detailRect);
const skipOcrForMissingDetail = Boolean(options.skipOcrUnlessArtifactDetail && !artifactDetail.present);
const shouldSkipOcr = Boolean(options.skipOcr || skipOcrForMissingDetail);
const inventoryRect = inferInventoryRect(size, detailRect);
const crops = createCrops(sourceImage, size, detailRect, inventoryRect);
const lockRect = clampCaptureRect(lockIconCropRect(detailRect, size), size);
const lockImage = sourceImage.crop(lockRect);
const lockSize = lockImage.getSize();
const locked = lockSize.width > 0 && lockSize.height > 0
? detectLockState({ data: lockImage.getBitmap(), width: lockSize.width, height: lockSize.height })
: undefined;
const croppedPayload = crops.map((crop) => ({
id: crop.id,
label: crop.label,
// OCR reads the preprocessed (upscaled + binarized) crop; the original is
// kept below for the diagnostics UI.
dataUrl: crop.ocrDataUrl ?? crop.dataUrl,
}));
const recognized = options.skipOcr ? { ocr: [], timedOut: false } : await runOcrOnCropsWithTimeout(croppedPayload);
const omitCrops = Boolean(options.omitCrops);
const crops = omitCrops
? []
: createCrops(sourceImage, size, detailRect, inventoryRect, options, bitmap, { sanctified, skipOcr: shouldSkipOcr });
const locked = options.omitLockState
? undefined
: (() => {
const lockRect = clampCaptureRect(lockIconCropRect(detailRect, size), size);
const lockImage = sourceImage.crop(lockRect);
const lockSize = lockImage.getSize();
return lockSize.width > 0 && lockSize.height > 0
? detectLockState({ data: lockImage.getBitmap(), width: lockSize.width, height: lockSize.height })
: undefined;
})();
const omitFullFrame = Boolean(options.omitFullFrame || options.ocrMode === "artifact");
const omitDetailPreview = Boolean(options.omitDetailPreview);
const omitInventoryPreview = Boolean(options.omitInventoryPreview || options.ocrMode === "artifact");
const detailFingerprint = imageCropFingerprint(sourceImage, detailRect, size);
const inventoryFingerprint = imageCropFingerprint(sourceImage, inventoryRect, size);
const croppedPayload = crops
.filter((crop) => crop.ocrEnabled !== false)
.map((crop) => {
return crop.ocrImage
? {
id: crop.id,
label: crop.label,
image: crop.ocrImage,
}
: null;
})
.filter((crop): crop is OcrCropPayload => Boolean(crop));
const prepareMs = Date.now() - buildStartedAt;
const ocrEngine = ocrEngineFromOptions(options);
const ocrStartedAt = Date.now();
const recognized = shouldSkipOcr
? { ocr: [], timedOut: false }
: await runOcrOnCropsWithTimeout(croppedPayload, ocrEngine);
const ocrMs = Date.now() - ocrStartedAt;
const totalMs = Date.now() - buildStartedAt;
const captureOcrEngine: CaptureOptions["ocrEngine"] = ocrEngine === "ik-traineddata" ? "ik-traineddata" : "current";
const count = parseInventoryCount(recognized.ocr);
return {
@@ -845,14 +1357,16 @@ async function buildCaptureResult(
name: sourceName,
width: size.width,
height: size.height,
dataUrl: sourceImage.toDataURL(),
dataUrl: omitFullFrame ? "" : sourceImage.toDataURL(),
capturedAt: new Date().toISOString(),
captureTarget,
detailDataUrl: imageCropDataUrl(sourceImage, detailRect, size),
inventoryDataUrl: imageCropDataUrl(sourceImage, inventoryRect, size),
detailDataUrl: omitDetailPreview ? undefined : imageCropDataUrl(sourceImage, detailRect, size),
inventoryDataUrl: omitInventoryPreview ? undefined : imageCropDataUrl(sourceImage, inventoryRect, size),
detailFingerprint,
inventoryFingerprint,
ocr: recognized.ocr,
ocrTimedOut: recognized.timedOut,
ocrSkipped: Boolean(options.skipOcr),
ocrSkipped: shouldSkipOcr,
crops: crops.map((crop) => ({
id: crop.id,
label: crop.label,
@@ -865,13 +1379,30 @@ async function buildCaptureResult(
dataUrl: crop.dataUrl,
})),
inventoryGrid: inferInventoryGrid(size, detailRect),
artifactDetail,
paimonMenu,
inventoryCount: count,
locked,
sanctified,
layout: {
aspect: aspectRatioLabel(size),
isSixteenNine: isSixteenNine(size),
warning: layoutSupportWarning(size),
},
timings: {
totalMs,
prepareMs,
ocrMs: shouldSkipOcr ? 0 : ocrMs,
cropCount: croppedPayload.length,
ocrEngine: captureOcrEngine,
ocrWorkerPoolSize: OCR_WORKER_POOL_SIZE,
ocrProfile: options.ocrProfile ?? "full",
ocrFieldMs: recognized.ocr.reduce<Record<string, number>>((fields, result) => {
if (typeof result.elapsedMs === "number") fields[result.id] = result.elapsedMs;
return fields;
}, {}),
ocrSkipped: shouldSkipOcr,
},
};
}
@@ -993,12 +1524,14 @@ function initializeAppLifecycle() {
) => captureSource(id, delayMs, focus, captureOptions),
clickScreen: (x: number, y: number) => clickScreenCommand(x, y),
scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => scrollScreenCommand(notches, anchorX, anchorY),
keyPress: (key: string) => keyPressCommand(key),
getAutomationGuard: () => getAutomationGuardCommand(),
});
createMainWindow();
registerScannerHotkeys();
startDevControlServer();
void warmOcrWorkerPool("current");
});
app.on("activate", () => {
@@ -1025,4 +1558,3 @@ function initializeAppLifecycle() {
}
initializeAppLifecycle();
+1
View File
@@ -8,6 +8,7 @@ contextBridge.exposeInMainWorld("assistantApi", {
captureSource: (sourceId, delayMs = 0, focusGenshin = false, options) => ipcRenderer.invoke("capture:captureSource", sourceId, delayMs, focusGenshin, options),
clickScreen: (x, y) => ipcRenderer.invoke("automation:clickScreen", x, y),
scrollScreen: (notches, anchorX, anchorY) => ipcRenderer.invoke("automation:scrollScreen", notches, anchorX, anchorY),
keyPress: (key) => ipcRenderer.invoke("automation:keyPress", key),
getAutomationGuard: () => ipcRenderer.invoke("automation:getGuard"),
focusMainWindow: () => ipcRenderer.invoke("app:focusMainWindow"),
focusGenshin: () => ipcRenderer.invoke("automation:focusGenshin"),
+1
View File
@@ -11,6 +11,7 @@ contextBridge.exposeInMainWorld("assistantApi", {
captureSource: (sourceId: string, delayMs = 0, focusGenshin = false, options?: CaptureOptions) => ipcRenderer.invoke("capture:captureSource", sourceId, delayMs, focusGenshin, options),
clickScreen: (x: number, y: number) => ipcRenderer.invoke("automation:clickScreen", x, y),
scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => ipcRenderer.invoke("automation:scrollScreen", notches, anchorX, anchorY),
keyPress: (key: string) => ipcRenderer.invoke("automation:keyPress", key),
getAutomationGuard: () => ipcRenderer.invoke("automation:getGuard"),
focusMainWindow: () => ipcRenderer.invoke("app:focusMainWindow"),
focusGenshin: () => ipcRenderer.invoke("automation:focusGenshin"),
@@ -3,6 +3,9 @@ import path from "node:path";
import type { ReviewSampleListResult, ReviewSamplesRepositoryPort, ReviewSamplePayload } from "./contracts.js";
import type { ReviewSampleRecord, SaveResultWithPath } from "../../src/types/global.js";
const SMALL_FILE_LIMIT_BYTES = 8 * 1024 * 1024;
const TAIL_READ_LIMIT_BYTES = 32 * 1024 * 1024;
export class ReviewSamplesRepository implements ReviewSamplesRepositoryPort {
private readonly filePath: string;
@@ -12,9 +15,12 @@ export class ReviewSamplesRepository implements ReviewSamplesRepositoryPort {
async list(limit = 50): Promise<ReviewSampleListResult> {
try {
const raw = await fs.readFile(this.filePath, "utf8");
const lines = raw.split(/\r?\n/).filter(Boolean);
const safeLimit = Math.max(1, Math.min(200, Number(limit) || 50));
const stats = await fs.stat(this.filePath);
const raw = stats.size <= SMALL_FILE_LIMIT_BYTES
? await fs.readFile(this.filePath, "utf8")
: await readTailText(this.filePath, stats.size, TAIL_READ_LIMIT_BYTES);
const lines = raw.split(/\r?\n/).filter(Boolean);
const samples = lines
.slice(-safeLimit)
.map((line) => {
@@ -25,7 +31,8 @@ export class ReviewSamplesRepository implements ReviewSamplesRepositoryPort {
}
})
.filter(Boolean) as ReviewSampleRecord[];
return { ok: true, samples: samples.reverse(), total: lines.length, path: this.filePath };
const total = stats.size <= SMALL_FILE_LIMIT_BYTES ? lines.length : Math.max(samples.length, lines.length);
return { ok: true, samples: samples.reverse(), total, path: this.filePath };
} catch {
return { ok: true, samples: [], total: 0, path: this.filePath };
}
@@ -37,3 +44,17 @@ export class ReviewSamplesRepository implements ReviewSamplesRepositoryPort {
return { ok: true, path: this.filePath };
}
}
async function readTailText(filePath: string, fileSize: number, maxBytes: number) {
const bytesToRead = Math.min(fileSize, maxBytes);
const handle = await fs.open(filePath, "r");
try {
const buffer = Buffer.alloc(bytesToRead);
await handle.read(buffer, 0, bytesToRead, fileSize - bytesToRead);
const text = buffer.toString("utf8");
const firstNewline = text.indexOf("\n");
return fileSize > bytesToRead && firstNewline >= 0 ? text.slice(firstNewline + 1) : text;
} finally {
await handle.close();
}
}
+58
View File
@@ -7,6 +7,7 @@ import type {
FocusGenshinResult,
GdiCaptureResult,
HelperOperationResponse,
KeyPressResult,
WindowBounds,
RuntimeInfo,
ScrollResult,
@@ -115,6 +116,32 @@ function Send-MouseClickBatch {
return [Native.InputHelper]::SendInput(2, [Native.InputHelper+INPUT[]]@($down, $up), $inputSize)
}
function Send-KeyPressBatch {
param([int]$virtualKey)
$down = New-Object Native.InputHelper+INPUT
$down.type = 1
$down.mi.dx = $virtualKey
$up = New-Object Native.InputHelper+INPUT
$up.type = 1
$up.mi.dx = $virtualKey
# Same union bytes as KEYBDINPUT: dx low word = wVk, dy = dwFlags.
$up.mi.dy = 0x0002
return [Native.InputHelper]::SendInput(2, [Native.InputHelper+INPUT[]]@($down, $up), $inputSize)
}
function Resolve-VirtualKey {
param([string]$key)
switch ($key.ToUpperInvariant()) {
"ESC" { return 27 }
"ESCAPE" { return 27 }
"ENTER" { return 13 }
"B" { return 66 }
"C" { return 67 }
"1" { return 49 }
default { throw "unsupported key: $key" }
}
}
function Get-CursorPoint {
$pt = New-Object Native.InputHelper+POINT
[Native.InputHelper]::GetCursorPos([ref]$pt) | Out-Null
@@ -358,6 +385,21 @@ while ($true) {
$response.notchesSent = $sentTotal
$response.inputBlocked = (($count -gt 0) -and ($sentTotal -eq 0))
}
"key" {
$focusInfo = Focus-GenshinWindow
if ($focusInfo.focused -and -not $focusInfo.alreadyForeground) {
Start-Sleep -Milliseconds 120
}
$vk = Resolve-VirtualKey -key "$($cmd.key)"
$sent = Send-KeyPressBatch -virtualKey $vk
$response.key = "$($cmd.key)"
$response.focused = $focusInfo.focused
$response.foregroundProcess = $focusInfo.foregroundProcess
$response.targetProcess = $focusInfo.targetProcess
$response.isElevated = Get-CurrentProcessElevation
$response.eventsSent = $sent
$response.inputBlocked = ($sent -lt 2)
}
"bounds" {
$clientBounds = Get-GenshinClientBounds
if ($null -eq $clientBounds) {
@@ -545,6 +587,7 @@ export interface InputHelperService {
getGenshinWindowBounds(): Promise<WindowBounds | null>;
clickScreen(x: number, y: number): Promise<ClickResult>;
scrollScreen(notches: number, anchorX?: number, anchorY?: number): Promise<ScrollResult>;
keyPress(key: string): Promise<KeyPressResult>;
getAutomationGuard(): Promise<AutomationGuard>;
capturePrimaryScreenViaGdi(): Promise<GdiCaptureResult>;
dispose(): void;
@@ -641,6 +684,20 @@ export function createInputHelperService(options: { userDataPath: string; exePat
};
}
async function keyPress(key: string) {
const result = (await request("key", { key }, 8000)) as HelperOperationResponse;
return {
ok: Boolean(result.ok) && Number(result.eventsSent ?? 0) >= 2,
key,
focused: Boolean(result.focused),
foregroundProcess: typeof result.foregroundProcess === "string" ? result.foregroundProcess : undefined,
targetProcess: typeof result.targetProcess === "string" ? result.targetProcess : undefined,
isElevated: typeof result.isElevated === "boolean" ? result.isElevated : undefined,
inputBlocked: Boolean(result.inputBlocked),
eventsSent: Number(result.eventsSent ?? 0),
};
}
async function getAutomationGuard() {
const result = await request("cursor", {}, 4000);
return {
@@ -688,6 +745,7 @@ export function createInputHelperService(options: { userDataPath: string; exePat
getGenshinWindowBounds,
clickScreen,
scrollScreen,
keyPress,
getAutomationGuard,
capturePrimaryScreenViaGdi,
dispose: () => inputHelper.dispose(),