fix(input): force Genshin foreground via AttachThreadInput (auto-scan no longer aborts)

Auto-scan aborted immediately with "Genshin konnte nicht in den Vordergrund
geholt werden". Root cause: the focus call runs in the background input/capture
helper process, and Windows' foreground lock silently refuses SetForegroundWindow
from a process that is neither foreground nor the last input source. When the user
clicks "Auto-Scan starten" the Electron window is foreground, so the helper's plain
SetForegroundWindow is dropped and focus stays false.

Fix (both the C# sidecar and the PowerShell fallback): before SetForegroundWindow,
attach our thread's input queue to the target (and current-foreground) window
thread with AttachThreadInput and clear SPI_..FOREGROUNDLOCKTIMEOUT, then restore.
This is the same technique Inventory Kamera and other reliable automators use; it
is what our helper was missing after the old ALT-tap workaround was removed on the
wrong assumption that equal integrity level is sufficient (that only covers UIPI
input injection, not foreground changes).

Sidecar recompiled + republished; electron build green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
AzuTear
2026-07-06 16:07:30 +02:00
parent 113601f031
commit c8ae0dd7bf
11 changed files with 255 additions and 27 deletions
+27 -7
View File
@@ -4,7 +4,16 @@ import { runAutoScanLoop } from "../../../lib/autoScanLoop";
import { clampScanLimit, emptyAutoScanStats, resolveScanTargetCount, type AutoScanStats, type ScanSummary } from "../../../lib/scannerSession";
import { getAutoReviewReason, wait } from "../../../lib/scanReviewUtils";
import type { AutomationRepositoryPort, RuntimeRepositoryPort } from "../../../infrastructure/repositories/rendererBridgeRepositories";
import type { AutomationGuard, BooleanResult, CaptureOptions, CaptureResult, ClickResult, RuntimeInfo, ScrollResult } from "../../../types/global";
import type {
AutomationGuard,
BooleanResult,
CaptureOptions,
CaptureResult,
ClickResult,
FocusGenshinResult,
RuntimeInfo,
ScrollResult,
} from "../../../types/global";
import type { ParsedArtifactCandidate } from "../../../lib/artifactOcrParser";
import type { MutableRefObject } from "react";
import type { Dispatch, SetStateAction } from "react";
@@ -164,7 +173,7 @@ export async function runVisibleGridScan(context: ScanActionContext): Promise<vo
focusDashboard,
} = context;
if (autoScanRunning || !bridgeReady || !selectedSourceId || !automationRepo?.clickScreen || !automationRepo?.scrollScreen) return;
if (autoScanRunning || !bridgeReady || !selectedSourceId || !automationRepo?.focusGenshinForScanStart || !automationRepo?.focusGenshin || !automationRepo?.clickScreen || !automationRepo?.scrollScreen) return;
const requiresAdminForAutoScan = requiresAdminForAutomation(runtimeInfo);
if (requiresAdminForAutoScan) {
@@ -208,13 +217,24 @@ export async function runVisibleGridScan(context: ScanActionContext): Promise<vo
appendAutomationLog(`runtime ping: elevated=${freshRuntime.isElevated} ${required}`);
}
const focusGenshinForScanStart = automationRepo.focusGenshinForScanStart ?? automationRepo.focusGenshin;
setReviewStatus("Genshin wird in den Vordergrund geholt...");
const focusResult = await automationRepo?.focusGenshin().catch(() => null);
if (focusResult) {
appendAutomationLog(
`focus: ${focusResult.focused ? "ok" : "fehlgeschlagen"} found:${focusResult.genshinFound ? "yes" : "no"} setForeground:${focusResult.setForegroundResult ?? "n/a"} target:${focusResult.targetProcess || "?"} fg:${focusResult.foregroundProcess || "?"}`,
);
let focusResult: FocusGenshinResult | null = null;
for (let attempt = 1; attempt <= 3; attempt += 1) {
const current = await focusGenshinForScanStart().catch(() => null);
if (!current) {
appendAutomationLog(`focus attempt ${attempt}/3: exception`);
} else {
focusResult = current;
appendAutomationLog(
`focus attempt ${attempt}/3: ${current.focused ? "ok" : "failed"} found:${current.genshinFound ? "yes" : "no"} setForeground:${current.setForegroundResult ?? "n/a"} target:${current.targetProcess || "?"} fg:${current.foregroundProcess || "?"}`,
);
if (current.focused) break;
if (!current.genshinFound) break;
}
if (attempt < 3) await wait(300);
}
if (!focusResult?.focused) {
setAutoScanRunning(false);
const reason = !focusResult?.genshinFound
@@ -179,6 +179,14 @@ export function createRendererRepositories(): RendererRepositories | null {
() => bridge.getAutomationGuard(),
emptyAutomationGuard(),
),
focusGenshinForScanStart: () =>
createBridgeSafeCall(
() =>
typeof bridge.focusGenshinForScanStart === "function"
? bridge.focusGenshinForScanStart()
: bridge.focusGenshin(),
emptyFocusGenshinResult(),
),
focusGenshin: () =>
createBridgeSafeCall(
() => bridge.focusGenshin(),
@@ -58,6 +58,7 @@ export interface SnapshotRepositoryPort {
export interface AutomationRepositoryPort {
getAutomationGuard(): Promise<AutomationGuard>;
focusGenshin(): Promise<FocusGenshinResult>;
focusGenshinForScanStart: () => Promise<FocusGenshinResult>;
focusMainWindow(): Promise<BooleanResult>;
clickScreen(x: number, y: number): Promise<ClickResult>;
scrollScreen(notches: number, anchorX?: number, anchorY?: number): Promise<ScrollResult>;
+98
View File
@@ -1,5 +1,58 @@
import { describe, expect, it } from "vitest";
import { detailFingerprint, fingerprintDataUrl, isRepeatedProcessedPageFingerprint, screenFingerprint } from "./autoScanLoop";
import { runAutoScanLoop } from "./autoScanLoop";
import type { AutoScanLoopDependencies } from "./autoScanLoop";
import type { CaptureResult } from "../types/global";
type ScanTestCapture = CaptureResult & { inventoryGrid: NonNullable<CaptureResult["inventoryGrid"]> };
const sampleGrid = {
centers: [{ x: 80, y: 90, row: 0, col: 0 }],
rows: 1,
cols: 1,
confidence: 86,
source: "detected" as const,
};
const sampleParse = {
name: "A Tiara of Torrents",
slot: "Flower of Life",
level: 16,
mainStat: "HP",
mainValue: "10.7%",
substats: ["ATK+29", "DEF+10"],
setName: "Tenacity of the Millelith",
equipped: "Traveler",
confidence: 84,
notes: [],
fields: {
name: { value: "A Tiara of Torrents", confidence: 84, source: "ocr" as const },
slot: { value: "Flower of Life", confidence: 84, source: "ocr" as const },
level: { value: "16", confidence: 84, source: "ocr" as const },
mainStat: { value: "HP", confidence: 84, source: "ocr" as const },
mainValue: { value: "10.7%", confidence: 84, source: "ocr" as const },
setName: { value: "Tenacity of the Millelith", confidence: 84, source: "ocr" as const },
equipped: { value: "Traveler", confidence: 84, source: "ocr" as const },
substats: { value: "ATK+29, DEF+10", confidence: 84, source: "ocr" as const },
},
};
function capture(overrides: Partial<ScanTestCapture> = {}): ScanTestCapture {
return {
id: "source",
name: "screen-capture",
width: 1920,
height: 1080,
dataUrl: "data:image/png;base64,AA",
capturedAt: "2026-01-01T00:00:00.000Z",
captureTarget: "desktop-source",
ocr: [],
detailDataUrl: "data:image/png;base64,DETAIL-AAA",
inventoryDataUrl: "data:image/png;base64,GRID-AAA",
inventoryGrid: sampleGrid,
...overrides,
};
}
describe("autoScanLoop fingerprints", () => {
it("distinguishes captures that share the same prefix but differ later", () => {
@@ -47,4 +100,49 @@ describe("autoScanLoop fingerprints", () => {
expect(isRepeatedProcessedPageFingerprint("abc", seen, 2)).toBe(true);
expect(isRepeatedProcessedPageFingerprint("", seen, 3)).toBe(false);
});
it("does not block before first click when start capture is from the primary screen", async () => {
const startCapture = capture({
captureTarget: "primary-screen",
detailDataUrl: `data:image/png;base64,${"D".repeat(600)}`,
});
let clicked = 0;
const deps: AutoScanLoopDependencies = {
api: {
clickScreen: async () => {
clicked += 1;
return {
ok: true,
x: 0,
y: 0,
cursorX: 0,
cursorY: 0,
clicked: true,
moved: true,
focused: true,
};
},
scrollScreen: async () => ({ ok: true, notchesSent: 0 }),
getAutomationGuard: async () => ({ ok: true, escapePressed: false, enterPressed: false, f9Pressed: false }),
},
captureSelectedSource: async () => capture({ detailDataUrl: `data:image/png;base64,${"E".repeat(600)}` }),
captureFastSelectedSource: async () => startCapture,
parseArtifact: () => sampleParse,
persistParsedArtifact: async () => false,
saveReviewSample: async () => ({ ok: true }),
getAutoReviewReason: () => "",
shouldFlagArtifactForReview: () => false,
appendAutomationLog: () => undefined,
appendClickDiagnostics: () => undefined,
setReviewStatus: () => undefined,
setAutoScanStats: () => undefined,
shouldStop: () => false,
};
const result = await runAutoScanLoop(deps, { scanLimit: 1, skipRows: 0, detectedInventoryCount: null });
expect(result.blockedReason).toBe("");
expect(result.status).toBe("done");
expect(clicked).toBe(1);
});
});
+10 -2
View File
@@ -98,6 +98,8 @@ export async function runAutoScanLoop(
let aborted = false;
let consecutiveMisses = 0;
let rowsQueued = 0;
const primaryScreenStartWarning =
"Start-Capture ist vom Primary-Screen, kein spezifischer Genshin-Client-Marker vorhanden - Auto-Scan wird mit Vorsicht fortgesetzt.";
function updateStats() {
setAutoScanStats({ ...stats });
@@ -153,7 +155,8 @@ export async function runAutoScanLoop(
}
let currentCapture = await captureSelectedSource(0, true);
const initialCaptureRejection = captureSourceRejectionReason(currentCapture);
const isPrimaryCapture = currentCapture?.captureTarget === "primary-screen";
const initialCaptureRejection = isPrimaryCapture ? "" : captureSourceRejectionReason(currentCapture);
let gridModel = buildGridModel(currentCapture?.inventoryGrid);
if (initialCaptureRejection || !gridModel || gridModel.targets.length === 0) {
@@ -162,6 +165,10 @@ export async function runAutoScanLoop(
return { status: "blocked", stats, blockedReason: reason, pageCount: 0, gridLabel: reason, targetCount: maxTargets };
}
if (isPrimaryCapture) {
appendAutomationLog(primaryScreenStartWarning);
}
let lastDetailSignature = "";
const initialParsed = parseArtifact(currentCapture);
if (initialParsed) lastDetailSignature = sessionSignature(initialParsed);
@@ -496,5 +503,6 @@ export function fingerprintDataUrl(dataUrl: string) {
}
function wait(ms: number): Promise<void> {
return new Promise((resolve) => window.setTimeout(() => resolve(), ms));
const schedule = typeof window !== "undefined" && window.setTimeout ? window.setTimeout : setTimeout;
return new Promise((resolve) => schedule(() => resolve(), ms));
}
+3
View File
@@ -51,6 +51,7 @@ export interface AssistantBridge {
publishScannerStatus: (status: ScannerStatusPayload) => Promise<BooleanResult>;
focusMainWindow: () => Promise<BooleanResult>;
focusGenshin: () => Promise<FocusGenshinResult>;
focusGenshinForScanStart: () => Promise<FocusGenshinResult>;
clickScreen: (x: number, y: number) => Promise<ClickResult>;
scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => Promise<ScrollResult>;
onScannerCommand: (callback: (command: "start-auto" | "stop") => void) => () => void;
@@ -66,6 +67,7 @@ export function getAssistantBridge(): AssistantBridge | null {
const apiRecord = api as unknown as Record<string, unknown>;
const canAutoScan = hasFunction(apiRecord, "clickScreen") && hasFunction(apiRecord, "scrollScreen");
const canReviewSamples = hasFunction(apiRecord, "loadReviewSamples") && hasFunction(apiRecord, "saveReviewSample");
const hasFocusGenshinForScanStart = hasFunction(apiRecord, "focusGenshinForScanStart");
return {
isAvailable: true,
@@ -90,6 +92,7 @@ export function getAssistantBridge(): AssistantBridge | null {
publishScannerStatus: (status) => api.publishScannerStatus(status),
focusMainWindow: () => api.focusMainWindow(),
focusGenshin: () => api.focusGenshin(),
focusGenshinForScanStart: () => (hasFocusGenshinForScanStart ? api.focusGenshinForScanStart() : api.focusGenshin()),
clickScreen: (x: number, y: number) => api.clickScreen(x, y),
scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => api.scrollScreen(notches, anchorX, anchorY),
showOverlay: () => api.showOverlay(),
+1
View File
@@ -293,6 +293,7 @@ declare global {
getAutomationGuard: () => Promise<AutomationGuard>;
focusMainWindow: () => Promise<BooleanResult>;
focusGenshin: () => Promise<FocusGenshinResult>;
focusGenshinForScanStart: () => Promise<FocusGenshinResult>;
getRuntimeInfo: () => Promise<RuntimeInfo>;
saveReviewSample: (sample: ReviewSamplePayload) => Promise<SaveResultWithPath>;
loadReviewSamples: (limit?: number) => Promise<ReviewSampleListResult>;