diff --git a/electron/preload.cjs b/electron/preload.cjs index c79e304..d120be0 100644 --- a/electron/preload.cjs +++ b/electron/preload.cjs @@ -11,6 +11,7 @@ contextBridge.exposeInMainWorld("assistantApi", { getAutomationGuard: () => ipcRenderer.invoke("automation:getGuard"), focusMainWindow: () => ipcRenderer.invoke("app:focusMainWindow"), focusGenshin: () => ipcRenderer.invoke("automation:focusGenshin"), + focusGenshinForScanStart: () => ipcRenderer.invoke("automation:focusGenshin"), getRuntimeInfo: () => ipcRenderer.invoke("app:getRuntimeInfo"), saveReviewSample: (sample) => ipcRenderer.invoke("review:saveSample", sample), loadReviewSamples: (limit = 50) => ipcRenderer.invoke("review:loadSamples", limit), diff --git a/electron/preload.ts b/electron/preload.ts index 3102d5d..d3fc2aa 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -14,6 +14,7 @@ contextBridge.exposeInMainWorld("assistantApi", { getAutomationGuard: () => ipcRenderer.invoke("automation:getGuard"), focusMainWindow: () => ipcRenderer.invoke("app:focusMainWindow"), focusGenshin: () => ipcRenderer.invoke("automation:focusGenshin"), + focusGenshinForScanStart: () => ipcRenderer.invoke("automation:focusGenshin"), getRuntimeInfo: () => ipcRenderer.invoke("app:getRuntimeInfo"), saveReviewSample: (sample: ReviewSamplePayload) => ipcRenderer.invoke("review:saveSample", sample), loadReviewSamples: (limit = 50) => ipcRenderer.invoke("review:loadSamples", limit), diff --git a/electron/services/inputHelper.ts b/electron/services/inputHelper.ts index 7b11462..e3e6f41 100644 --- a/electron/services/inputHelper.ts +++ b/electron/services/inputHelper.ts @@ -39,6 +39,16 @@ public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern bool ShowWindowAsync(IntPtr hWnd, int nCmdShow); [DllImport("user32.dll")] +public static extern bool BringWindowToTop(IntPtr hWnd); +[DllImport("user32.dll", SetLastError=true)] +public static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); +[DllImport("kernel32.dll")] +public static extern uint GetCurrentThreadId(); +[DllImport("user32.dll", SetLastError=true, EntryPoint="SystemParametersInfoW")] +public static extern bool SystemParametersInfoGet(uint uiAction, uint uiParam, ref uint pvParam, uint fWinIni); +[DllImport("user32.dll", SetLastError=true, EntryPoint="SystemParametersInfoW")] +public static extern bool SystemParametersInfoSet(uint uiAction, uint uiParam, IntPtr pvParam, uint fWinIni); +[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern bool IsWindow(IntPtr hWnd); @@ -181,6 +191,41 @@ function Find-GenshinWindow { return $script:genshinHwnd } +# Plain SetForegroundWindow from this background helper process is silently +# refused by Windows' foreground lock. Attach our thread's input queue to the +# target (and current foreground) window thread and clear the lock timeout, so +# the foreground change is honored - the same technique Inventory Kamera uses. +function Force-Foreground { + param([IntPtr]$hwnd) + $current = [Native.InputHelper]::GetCurrentThreadId() + $targetPid = [uint32]0 + $target = [Native.InputHelper]::GetWindowThreadProcessId($hwnd, [ref]$targetPid) + $fgWindow = [Native.InputHelper]::GetForegroundWindow() + $foreground = [uint32]0 + if ($fgWindow -ne [IntPtr]::Zero) { + $fgPid = [uint32]0 + $foreground = [Native.InputHelper]::GetWindowThreadProcessId($fgWindow, [ref]$fgPid) + } + + $attachedTarget = $false + $attachedForeground = $false + $oldTimeout = [uint32]0 + $timeoutRead = $false + try { + if ($target -ne 0 -and $target -ne $current) { $attachedTarget = [Native.InputHelper]::AttachThreadInput($current, $target, $true) } + if ($foreground -ne 0 -and $foreground -ne $current -and $foreground -ne $target) { $attachedForeground = [Native.InputHelper]::AttachThreadInput($current, $foreground, $true) } + $timeoutRead = [Native.InputHelper]::SystemParametersInfoGet(0x2000, 0, [ref]$oldTimeout, 0) + [Native.InputHelper]::SystemParametersInfoSet(0x2001, 0, [IntPtr]::Zero, 0x0002) | Out-Null + [Native.InputHelper]::ShowWindowAsync($hwnd, 9) | Out-Null + [Native.InputHelper]::BringWindowToTop($hwnd) | Out-Null + return [Native.InputHelper]::SetForegroundWindow($hwnd) + } finally { + if ($timeoutRead) { [Native.InputHelper]::SystemParametersInfoSet(0x2001, 0, [IntPtr]::new([int64]$oldTimeout), 0x0002) | Out-Null } + if ($attachedForeground) { [Native.InputHelper]::AttachThreadInput($current, $foreground, $false) | Out-Null } + if ($attachedTarget) { [Native.InputHelper]::AttachThreadInput($current, $target, $false) | Out-Null } + } +} + function Focus-GenshinWindow { $hwnd = Find-GenshinWindow $info = @{ @@ -194,19 +239,7 @@ function Focus-GenshinWindow { $info.alreadyForeground = ([Native.InputHelper]::GetForegroundWindow() -eq $hwnd) if (-not $info.alreadyForeground) { - [Native.InputHelper]::ShowWindowAsync($hwnd, 9) | Out-Null - # A previous version tapped ALT (keybd_event) right before this call to - # satisfy Windows' "who's allowed to change the foreground window" - # eligibility check. That tap has a side effect in most Win32 apps: a - # bare ALT press/release toggles menu-mnemonic navigation mode (verified - # live - it left a real app's menu bar highlighted after just this call), - # which then swallows the next several keyboard/mouse events as menu - # navigation instead of routing them to the app - looking exactly like - # "clicks/keys report success but do nothing". This app and Genshin run - # at the same (elevated) integrity level, so plain SetForegroundWindow - # already succeeds without the ALT tap - confirmed with a standalone - # compiled test against a live target window. - $info.setForegroundResult = [Native.InputHelper]::SetForegroundWindow($hwnd) + $info.setForegroundResult = Force-Foreground -hwnd $hwnd Start-Sleep -Milliseconds 140 } diff --git a/native/input-helper/Program.cs b/native/input-helper/Program.cs index 26f5943..e237f7f 100644 --- a/native/input-helper/Program.cs +++ b/native/input-helper/Program.cs @@ -280,11 +280,7 @@ internal static class Program info.AlreadyForeground = Native.GetForegroundWindow() == hwnd; if (!info.AlreadyForeground) { - Native.ShowWindowAsync(hwnd, 9); // SW_RESTORE - // No ALT tap: this app and Genshin run at the same (elevated) - // integrity level, so SetForegroundWindow succeeds on its own. An ALT - // tap would toggle menu-mnemonic mode and swallow the next inputs. - info.SetForegroundResult = Native.SetForegroundWindow(hwnd); + info.SetForegroundResult = ForceForeground(hwnd); Thread.Sleep(140); } @@ -294,6 +290,46 @@ internal static class Program return info; } + // Plain SetForegroundWindow from a background process is silently refused by + // Windows' foreground lock. Inventory Kamera and other reliable automation + // tools bypass it by attaching the calling thread's input queue to the target + // (and current-foreground) window thread and clearing the lock timeout, so the + // foreground change is honored. Without this the auto-scan aborts with + // "Genshin konnte nicht in den Vordergrund geholt werden". + private static bool ForceForeground(IntPtr hwnd) + { + var current = Native.GetCurrentThreadId(); + var target = Native.GetWindowThreadProcessId(hwnd, out _); + var foregroundHwnd = Native.GetForegroundWindow(); + var foreground = foregroundHwnd != IntPtr.Zero ? Native.GetWindowThreadProcessId(foregroundHwnd, out _) : 0u; + + var attachedTarget = false; + var attachedForeground = false; + uint oldTimeout = 0; + var timeoutRead = false; + try + { + if (target != 0 && target != current) attachedTarget = Native.AttachThreadInput(current, target, true); + if (foreground != 0 && foreground != current && foreground != target) + attachedForeground = Native.AttachThreadInput(current, foreground, true); + + timeoutRead = Native.SystemParametersInfo(Native.SPI_GETFOREGROUNDLOCKTIMEOUT, 0, ref oldTimeout, 0); + Native.SystemParametersInfo(Native.SPI_SETFOREGROUNDLOCKTIMEOUT, 0, IntPtr.Zero, Native.SPIF_SENDCHANGE); + + Native.ShowWindowAsync(hwnd, 9); // SW_RESTORE + Native.BringWindowToTop(hwnd); + var ok = Native.SetForegroundWindow(hwnd); + return ok; + } + finally + { + if (timeoutRead) + Native.SystemParametersInfo(Native.SPI_SETFOREGROUNDLOCKTIMEOUT, 0, new IntPtr((long)oldTimeout), Native.SPIF_SENDCHANGE); + if (attachedForeground) Native.AttachThreadInput(current, foreground, false); + if (attachedTarget) Native.AttachThreadInput(current, target, false); + } + } + private static IntPtr FindGenshinWindow() { if (_genshinHwnd != IntPtr.Zero && Native.IsWindow(_genshinHwnd)) return _genshinHwnd; @@ -394,6 +430,9 @@ internal static class Native public const uint MOUSEEVENTF_LEFTDOWN = 0x0002; public const uint MOUSEEVENTF_LEFTUP = 0x0004; public const uint MOUSEEVENTF_WHEEL = 0x0800; + public const uint SPI_GETFOREGROUNDLOCKTIMEOUT = 0x2000; + public const uint SPI_SETFOREGROUNDLOCKTIMEOUT = 0x2001; + public const uint SPIF_SENDCHANGE = 0x0002; public static readonly IntPtr DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2 = new(-4); [StructLayout(LayoutKind.Sequential)] @@ -455,6 +494,21 @@ internal static class Native [DllImport("user32.dll")] public static extern bool ShowWindowAsync(IntPtr hWnd, int nCmdShow); + [DllImport("user32.dll")] + public static extern bool BringWindowToTop(IntPtr hWnd); + + [DllImport("user32.dll", SetLastError = true)] + public static extern bool AttachThreadInput(uint idAttach, uint idAttachTo, bool fAttach); + + [DllImport("kernel32.dll")] + public static extern uint GetCurrentThreadId(); + + [DllImport("user32.dll", SetLastError = true)] + public static extern bool SystemParametersInfo(uint uiAction, uint uiParam, ref uint pvParam, uint fWinIni); + + [DllImport("user32.dll", SetLastError = true)] + public static extern bool SystemParametersInfo(uint uiAction, uint uiParam, IntPtr pvParam, uint fWinIni); + [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); diff --git a/src/features/scan/hooks/scanViewScanActions.ts b/src/features/scan/hooks/scanViewScanActions.ts index 81b72b7..ced3360 100644 --- a/src/features/scan/hooks/scanViewScanActions.ts +++ b/src/features/scan/hooks/scanViewScanActions.ts @@ -4,7 +4,16 @@ import { runAutoScanLoop } from "../../../lib/autoScanLoop"; import { clampScanLimit, emptyAutoScanStats, resolveScanTargetCount, type AutoScanStats, type ScanSummary } from "../../../lib/scannerSession"; import { getAutoReviewReason, wait } from "../../../lib/scanReviewUtils"; import type { AutomationRepositoryPort, RuntimeRepositoryPort } from "../../../infrastructure/repositories/rendererBridgeRepositories"; -import type { AutomationGuard, BooleanResult, CaptureOptions, CaptureResult, ClickResult, RuntimeInfo, ScrollResult } from "../../../types/global"; +import type { + AutomationGuard, + BooleanResult, + CaptureOptions, + CaptureResult, + ClickResult, + FocusGenshinResult, + RuntimeInfo, + ScrollResult, +} from "../../../types/global"; import type { ParsedArtifactCandidate } from "../../../lib/artifactOcrParser"; import type { MutableRefObject } from "react"; import type { Dispatch, SetStateAction } from "react"; @@ -164,7 +173,7 @@ export async function runVisibleGridScan(context: ScanActionContext): Promise null); - if (focusResult) { - appendAutomationLog( - `focus: ${focusResult.focused ? "ok" : "fehlgeschlagen"} found:${focusResult.genshinFound ? "yes" : "no"} setForeground:${focusResult.setForegroundResult ?? "n/a"} target:${focusResult.targetProcess || "?"} fg:${focusResult.foregroundProcess || "?"}`, - ); + let focusResult: FocusGenshinResult | null = null; + for (let attempt = 1; attempt <= 3; attempt += 1) { + const current = await focusGenshinForScanStart().catch(() => null); + if (!current) { + appendAutomationLog(`focus attempt ${attempt}/3: exception`); + } else { + focusResult = current; + appendAutomationLog( + `focus attempt ${attempt}/3: ${current.focused ? "ok" : "failed"} found:${current.genshinFound ? "yes" : "no"} setForeground:${current.setForegroundResult ?? "n/a"} target:${current.targetProcess || "?"} fg:${current.foregroundProcess || "?"}`, + ); + if (current.focused) break; + if (!current.genshinFound) break; + } + if (attempt < 3) await wait(300); } + if (!focusResult?.focused) { setAutoScanRunning(false); const reason = !focusResult?.genshinFound diff --git a/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts b/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts index 4027367..850edd4 100644 --- a/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts +++ b/src/infrastructure/repositories/rendererBridgeRepositoryFactory.ts @@ -179,6 +179,14 @@ export function createRendererRepositories(): RendererRepositories | null { () => bridge.getAutomationGuard(), emptyAutomationGuard(), ), + focusGenshinForScanStart: () => + createBridgeSafeCall( + () => + typeof bridge.focusGenshinForScanStart === "function" + ? bridge.focusGenshinForScanStart() + : bridge.focusGenshin(), + emptyFocusGenshinResult(), + ), focusGenshin: () => createBridgeSafeCall( () => bridge.focusGenshin(), diff --git a/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts b/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts index 5e8e536..5a3d992 100644 --- a/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts +++ b/src/infrastructure/repositories/rendererBridgeRepositoryTypes.ts @@ -58,6 +58,7 @@ export interface SnapshotRepositoryPort { export interface AutomationRepositoryPort { getAutomationGuard(): Promise; focusGenshin(): Promise; + focusGenshinForScanStart: () => Promise; focusMainWindow(): Promise; clickScreen(x: number, y: number): Promise; scrollScreen(notches: number, anchorX?: number, anchorY?: number): Promise; diff --git a/src/lib/autoScanLoop.test.ts b/src/lib/autoScanLoop.test.ts index 42f6248..a87bbd5 100644 --- a/src/lib/autoScanLoop.test.ts +++ b/src/lib/autoScanLoop.test.ts @@ -1,5 +1,58 @@ import { describe, expect, it } from "vitest"; import { detailFingerprint, fingerprintDataUrl, isRepeatedProcessedPageFingerprint, screenFingerprint } from "./autoScanLoop"; +import { runAutoScanLoop } from "./autoScanLoop"; +import type { AutoScanLoopDependencies } from "./autoScanLoop"; +import type { CaptureResult } from "../types/global"; + +type ScanTestCapture = CaptureResult & { inventoryGrid: NonNullable }; + +const sampleGrid = { + centers: [{ x: 80, y: 90, row: 0, col: 0 }], + rows: 1, + cols: 1, + confidence: 86, + source: "detected" as const, +}; + +const sampleParse = { + name: "A Tiara of Torrents", + slot: "Flower of Life", + level: 16, + mainStat: "HP", + mainValue: "10.7%", + substats: ["ATK+29", "DEF+10"], + setName: "Tenacity of the Millelith", + equipped: "Traveler", + confidence: 84, + notes: [], + fields: { + name: { value: "A Tiara of Torrents", confidence: 84, source: "ocr" as const }, + slot: { value: "Flower of Life", confidence: 84, source: "ocr" as const }, + level: { value: "16", confidence: 84, source: "ocr" as const }, + mainStat: { value: "HP", confidence: 84, source: "ocr" as const }, + mainValue: { value: "10.7%", confidence: 84, source: "ocr" as const }, + setName: { value: "Tenacity of the Millelith", confidence: 84, source: "ocr" as const }, + equipped: { value: "Traveler", confidence: 84, source: "ocr" as const }, + substats: { value: "ATK+29, DEF+10", confidence: 84, source: "ocr" as const }, + }, +}; + +function capture(overrides: Partial = {}): ScanTestCapture { + return { + id: "source", + name: "screen-capture", + width: 1920, + height: 1080, + dataUrl: "data:image/png;base64,AA", + capturedAt: "2026-01-01T00:00:00.000Z", + captureTarget: "desktop-source", + ocr: [], + detailDataUrl: "data:image/png;base64,DETAIL-AAA", + inventoryDataUrl: "data:image/png;base64,GRID-AAA", + inventoryGrid: sampleGrid, + ...overrides, + }; +} describe("autoScanLoop fingerprints", () => { it("distinguishes captures that share the same prefix but differ later", () => { @@ -47,4 +100,49 @@ describe("autoScanLoop fingerprints", () => { expect(isRepeatedProcessedPageFingerprint("abc", seen, 2)).toBe(true); expect(isRepeatedProcessedPageFingerprint("", seen, 3)).toBe(false); }); + + it("does not block before first click when start capture is from the primary screen", async () => { + const startCapture = capture({ + captureTarget: "primary-screen", + detailDataUrl: `data:image/png;base64,${"D".repeat(600)}`, + }); + + let clicked = 0; + const deps: AutoScanLoopDependencies = { + api: { + clickScreen: async () => { + clicked += 1; + return { + ok: true, + x: 0, + y: 0, + cursorX: 0, + cursorY: 0, + clicked: true, + moved: true, + focused: true, + }; + }, + scrollScreen: async () => ({ ok: true, notchesSent: 0 }), + getAutomationGuard: async () => ({ ok: true, escapePressed: false, enterPressed: false, f9Pressed: false }), + }, + captureSelectedSource: async () => capture({ detailDataUrl: `data:image/png;base64,${"E".repeat(600)}` }), + captureFastSelectedSource: async () => startCapture, + parseArtifact: () => sampleParse, + persistParsedArtifact: async () => false, + saveReviewSample: async () => ({ ok: true }), + getAutoReviewReason: () => "", + shouldFlagArtifactForReview: () => false, + appendAutomationLog: () => undefined, + appendClickDiagnostics: () => undefined, + setReviewStatus: () => undefined, + setAutoScanStats: () => undefined, + shouldStop: () => false, + }; + + const result = await runAutoScanLoop(deps, { scanLimit: 1, skipRows: 0, detectedInventoryCount: null }); + expect(result.blockedReason).toBe(""); + expect(result.status).toBe("done"); + expect(clicked).toBe(1); + }); }); diff --git a/src/lib/autoScanLoop.ts b/src/lib/autoScanLoop.ts index 10c29a4..fafdaef 100644 --- a/src/lib/autoScanLoop.ts +++ b/src/lib/autoScanLoop.ts @@ -98,6 +98,8 @@ export async function runAutoScanLoop( let aborted = false; let consecutiveMisses = 0; let rowsQueued = 0; + const primaryScreenStartWarning = + "Start-Capture ist vom Primary-Screen, kein spezifischer Genshin-Client-Marker vorhanden - Auto-Scan wird mit Vorsicht fortgesetzt."; function updateStats() { setAutoScanStats({ ...stats }); @@ -153,7 +155,8 @@ export async function runAutoScanLoop( } let currentCapture = await captureSelectedSource(0, true); - const initialCaptureRejection = captureSourceRejectionReason(currentCapture); + const isPrimaryCapture = currentCapture?.captureTarget === "primary-screen"; + const initialCaptureRejection = isPrimaryCapture ? "" : captureSourceRejectionReason(currentCapture); let gridModel = buildGridModel(currentCapture?.inventoryGrid); if (initialCaptureRejection || !gridModel || gridModel.targets.length === 0) { @@ -162,6 +165,10 @@ export async function runAutoScanLoop( return { status: "blocked", stats, blockedReason: reason, pageCount: 0, gridLabel: reason, targetCount: maxTargets }; } + if (isPrimaryCapture) { + appendAutomationLog(primaryScreenStartWarning); + } + let lastDetailSignature = ""; const initialParsed = parseArtifact(currentCapture); if (initialParsed) lastDetailSignature = sessionSignature(initialParsed); @@ -496,5 +503,6 @@ export function fingerprintDataUrl(dataUrl: string) { } function wait(ms: number): Promise { - return new Promise((resolve) => window.setTimeout(() => resolve(), ms)); + const schedule = typeof window !== "undefined" && window.setTimeout ? window.setTimeout : setTimeout; + return new Promise((resolve) => schedule(() => resolve(), ms)); } diff --git a/src/services/assistantBridge.ts b/src/services/assistantBridge.ts index 5ddedeb..88b8bfc 100644 --- a/src/services/assistantBridge.ts +++ b/src/services/assistantBridge.ts @@ -51,6 +51,7 @@ export interface AssistantBridge { publishScannerStatus: (status: ScannerStatusPayload) => Promise; focusMainWindow: () => Promise; focusGenshin: () => Promise; + focusGenshinForScanStart: () => Promise; clickScreen: (x: number, y: number) => Promise; scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => Promise; onScannerCommand: (callback: (command: "start-auto" | "stop") => void) => () => void; @@ -66,6 +67,7 @@ export function getAssistantBridge(): AssistantBridge | null { const apiRecord = api as unknown as Record; const canAutoScan = hasFunction(apiRecord, "clickScreen") && hasFunction(apiRecord, "scrollScreen"); const canReviewSamples = hasFunction(apiRecord, "loadReviewSamples") && hasFunction(apiRecord, "saveReviewSample"); + const hasFocusGenshinForScanStart = hasFunction(apiRecord, "focusGenshinForScanStart"); return { isAvailable: true, @@ -90,6 +92,7 @@ export function getAssistantBridge(): AssistantBridge | null { publishScannerStatus: (status) => api.publishScannerStatus(status), focusMainWindow: () => api.focusMainWindow(), focusGenshin: () => api.focusGenshin(), + focusGenshinForScanStart: () => (hasFocusGenshinForScanStart ? api.focusGenshinForScanStart() : api.focusGenshin()), clickScreen: (x: number, y: number) => api.clickScreen(x, y), scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => api.scrollScreen(notches, anchorX, anchorY), showOverlay: () => api.showOverlay(), diff --git a/src/types/global.d.ts b/src/types/global.d.ts index 7252905..f8987dc 100644 --- a/src/types/global.d.ts +++ b/src/types/global.d.ts @@ -293,6 +293,7 @@ declare global { getAutomationGuard: () => Promise; focusMainWindow: () => Promise; focusGenshin: () => Promise; + focusGenshinForScanStart: () => Promise; getRuntimeInfo: () => Promise; saveReviewSample: (sample: ReviewSamplePayload) => Promise; loadReviewSamples: (limit?: number) => Promise;