bao/genshin-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(native): C# input/capture sidecar replacing the PowerShell helper

Browse Source 
Implements ADR-008. native/input-helper is a self-contained .NET 9 console exe
speaking the identical JSON-over-stdin/stdout protocol as the old PowerShell
helper (ping/cursor/runtime/focus/click/scroll/bounds/capture), so the
InputHelperService interface is unchanged.

- Win32 interop compiled once (native exe), not per call.
- PerMonitorV2 DPI via manifest so click/capture coordinates stay correct on
  mixed-DPI multi-monitor setups.
- capture returns base64 PNG bytes inline (imageBase64) instead of writing a
  temp file per frame; the client handles both base64 and the PowerShell path.
- InputHelperClient prefers the exe and falls back to the embedded PowerShell
  helper when the exe is absent, so the app still runs without the .NET build.
- main.ts resolves the exe (INPUT_HELPER_EXE env -> packaged resources/input-helper
  -> native/input-helper/bin/publish). electron-builder ships it via extraResources.
- npm run helper:build; README documents the build + fallback.

Verified end-to-end through the compiled client: sidecar spawns, runtime info
and a base64 primary-screen capture return correctly. Build stays green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
AzuTear
2026-07-05 21:33:26 +02:00
parent 92345ef51a
commit c7138b541d
8 changed files with 597 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
native/input-helper/Program.cs
+466
View File
@@ -0,0 +1,466 @@
using System.Diagnostics;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using System.Security.Principal;
using System.Text;
using System.Text.Json;
using System.Windows.Forms;
// Long-lived input/capture sidecar for the Genshin Artifact Assistant.
// Drop-in replacement for the old PowerShell helper (see ADR-008): identical
// JSON-over-stdin/stdout protocol - one JSON request per line, one JSON response
// per line - so the Electron-side InputHelperService is unchanged. Win32 interop
// is compiled once (this is a native exe), and capture returns base64 PNG bytes
// directly instead of writing a temp file per frame.
namespace GenshinAssistant.InputHelper;
internal static class Program
{
private static IntPtr _genshinHwnd = IntPtr.Zero;
private static int Main()
{
// Manifest already declares PerMonitorV2; this is a belt-and-suspenders
// call for hosts that ignore the manifest.
try { Native.SetProcessDpiAwarenessContext(Native.DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2); }
catch { try { Native.SetProcessDpiAwareness(2); } catch { /* oldest fallback */ Native.SetProcessDPIAware(); } }
Console.OutputEncoding = Encoding.UTF8;
var stdout = Console.Out;
string? line;
while ((line = Console.In.ReadLine()) != null)
{
if (line.Trim().Length == 0) continue;
var response = new Dictionary<string, object?> { ["id"] = "", ["ok"] = true };
try
{
using var doc = JsonDocument.Parse(line);
var root = doc.RootElement;
response["id"] = GetString(root, "id");
var op = GetString(root, "op");
Handle(op, root, response);
}
catch (Exception ex)
{
response["ok"] = false;
response["error"] = ex.Message;
}
stdout.WriteLine(JsonSerializer.Serialize(response));
stdout.Flush();
}
return 0;
}
private static void Handle(string op, JsonElement root, Dictionary<string, object?> response)
{
switch (op)
{
case "ping":
response["pong"] = true;
break;
case "cursor":
{
var state = GetCursorState();
response["cursorX"] = state.X;
response["cursorY"] = state.Y;
response["escapePressed"] = state.Escape;
response["enterPressed"] = state.Enter;
response["f9Pressed"] = state.F9;
break;
}
case "runtime":
{
var hwnd = FindGenshinWindow();
var fgHwnd = Native.GetForegroundWindow();
response["isElevated"] = IsElevated();
response["genshinFound"] = hwnd != IntPtr.Zero;
response["genshinHwnd"] = hwnd.ToInt64();
response["targetProcess"] = ProcessNameFromHwnd(hwnd);
response["foregroundProcess"] = ProcessNameFromHwnd(fgHwnd);
response["foregroundHwnd"] = fgHwnd.ToInt64();
response["helperPid"] = Environment.ProcessId;
break;
}
case "focus":
{
var info = FocusGenshinWindow();
response["focused"] = info.Focused;
response["alreadyForeground"] = info.AlreadyForeground;
response["foregroundProcess"] = info.ForegroundProcess;
response["targetProcess"] = info.TargetProcess;
response["genshinFound"] = info.Hwnd != IntPtr.Zero;
response["setForegroundResult"] = info.SetForegroundResult;
break;
}
case "click":
{
var info = FocusGenshinWindow();
if (info.Focused && !info.AlreadyForeground) Thread.Sleep(120);
var targetX = GetInt(root, "x");
var targetY = GetInt(root, "y");
// Bare SetCursorPos then a batched down+up click, matching the
// verified Inventory Kamera sequence: no extra move event, no
// gap between move and click.
Native.SetCursorPos(targetX, targetY);
Native.GetCursorPos(out var pt);
var onTarget = Math.Abs(targetX - pt.X) <= 2 && Math.Abs(targetY - pt.Y) <= 2;
var clickEventsSent = onTarget ? SendMouseClickBatch() : 0u;
var state = GetCursorState();
response["cursorX"] = state.X;
response["cursorY"] = state.Y;
response["escapePressed"] = state.Escape;
response["enterPressed"] = state.Enter;
response["f9Pressed"] = state.F9;
response["moved"] = onTarget;
response["focused"] = info.Focused;
response["alreadyForeground"] = info.AlreadyForeground;
response["foregroundProcess"] = info.ForegroundProcess;
response["targetProcess"] = info.TargetProcess;
response["isElevated"] = IsElevated();
// Only report a click when the cursor is verifiably on target and
// SendInput injected both events; real acceptance is proven later
// by the detail-panel fingerprint.
response["clicked"] = onTarget && clickEventsSent >= 2;
response["inputBlocked"] = onTarget && clickEventsSent < 2;
break;
}
case "scroll":
{
var info = FocusGenshinWindow();
if (info.Focused && !info.AlreadyForeground) Thread.Sleep(120);
if (TryGetInt(root, "x", out var ax) && TryGetInt(root, "y", out var ay))
{
Native.SetCursorPos(ax, ay);
Thread.Sleep(30);
}
Native.GetCursorPos(out var pt);
response["cursorX"] = pt.X;
response["cursorY"] = pt.Y;
response["focused"] = info.Focused;
response["foregroundProcess"] = info.ForegroundProcess;
response["isElevated"] = IsElevated();
var notches = GetInt(root, "notches");
var stepDelta = notches < 0 ? -120 : 120;
var count = Math.Min(60, Math.Abs(notches));
uint sentTotal = 0;
for (var i = 0; i < count; i++)
{
sentTotal += SendMouseWheel(stepDelta);
Thread.Sleep(45);
}
response["notchesSent"] = sentTotal;
response["inputBlocked"] = count > 0 && sentTotal == 0;
break;
}
case "bounds":
{
var bounds = GetGenshinClientBounds();
if (bounds == null)
{
response["found"] = false;
}
else
{
response["found"] = true;
response["left"] = bounds.Value.Left;
response["top"] = bounds.Value.Top;
response["width"] = bounds.Value.Width;
response["height"] = bounds.Value.Height;
}
break;
}
case "capture":
{
var bounds = GetGenshinClientBounds();
string captureTarget;
Rect area;
if (bounds == null)
{
var screen = Screen.PrimaryScreen!.Bounds;
area = new Rect { Left = screen.Left, Top = screen.Top, Width = screen.Width, Height = screen.Height };
captureTarget = "primary-screen";
}
else
{
area = bounds.Value;
captureTarget = "genshin-client";
}
using var bitmap = new Bitmap(area.Width, area.Height, PixelFormat.Format32bppArgb);
using (var graphics = Graphics.FromImage(bitmap))
{
graphics.CopyFromScreen(area.Left, area.Top, 0, 0, bitmap.Size, CopyPixelOperation.SourceCopy);
}
using var stream = new MemoryStream();
bitmap.Save(stream, ImageFormat.Png);
response["imageBase64"] = Convert.ToBase64String(stream.ToArray());
response["width"] = area.Width;
response["height"] = area.Height;
response["originX"] = area.Left;
response["originY"] = area.Top;
response["captureTarget"] = captureTarget;
break;
}
default:
response["ok"] = false;
response["error"] = "unknown op";
break;
}
}
private readonly struct Rect
{
public int Left { get; init; }
public int Top { get; init; }
public int Width { get; init; }
public int Height { get; init; }
}
private struct CursorState
{
public int X;
public int Y;
public bool Escape;
public bool Enter;
public bool F9;
}
private struct FocusInfo
{
public IntPtr Hwnd;
public bool Focused;
public bool AlreadyForeground;
public string ForegroundProcess;
public string TargetProcess;
public bool SetForegroundResult;
}
private static CursorState GetCursorState()
{
Native.GetCursorPos(out var pt);
// Only 0x8000 (held right now). The 0x0001 "pressed since last call" bit
// is unreliable and fires for ESC presses used to navigate Genshin menus.
var esc = (Native.GetAsyncKeyState(0x1B) & 0x8000) != 0;
var enter = (Native.GetAsyncKeyState(0x0D) & 0x8000) != 0;
var f9 = (Native.GetAsyncKeyState(0x78) & 0x8000) != 0;
return new CursorState { X = pt.X, Y = pt.Y, Escape = esc, Enter = enter, F9 = f9 };
}
private static FocusInfo FocusGenshinWindow()
{
var hwnd = FindGenshinWindow();
var info = new FocusInfo
{
Hwnd = hwnd,
ForegroundProcess = "",
TargetProcess = ProcessNameFromHwnd(hwnd),
};
if (hwnd == IntPtr.Zero) return info;
info.AlreadyForeground = Native.GetForegroundWindow() == hwnd;
if (!info.AlreadyForeground)
{
Native.ShowWindowAsync(hwnd, 9); // SW_RESTORE
// No ALT tap: this app and Genshin run at the same (elevated)
// integrity level, so SetForegroundWindow succeeds on its own. An ALT
// tap would toggle menu-mnemonic mode and swallow the next inputs.
info.SetForegroundResult = Native.SetForegroundWindow(hwnd);
Thread.Sleep(140);
}
var foreground = Native.GetForegroundWindow();
info.Focused = foreground == hwnd;
info.ForegroundProcess = ProcessNameFromHwnd(foreground);
return info;
}
private static IntPtr FindGenshinWindow()
{
if (_genshinHwnd != IntPtr.Zero && Native.IsWindow(_genshinHwnd)) return _genshinHwnd;
foreach (var proc in Process.GetProcesses())
{
try
{
var name = proc.ProcessName;
if ((name.Contains("GenshinImpact", StringComparison.OrdinalIgnoreCase)
|| name.Contains("YuanShen", StringComparison.OrdinalIgnoreCase)
|| name.Contains("Genshin", StringComparison.OrdinalIgnoreCase))
&& proc.MainWindowHandle != IntPtr.Zero)
{
_genshinHwnd = proc.MainWindowHandle;
return _genshinHwnd;
}
}
catch
{
// Process exited between enumeration and inspection; ignore.
}
}
_genshinHwnd = IntPtr.Zero;
return _genshinHwnd;
}
private static Rect? GetGenshinClientBounds()
{
var hwnd = FindGenshinWindow();
if (hwnd == IntPtr.Zero) return null;
if (!Native.GetClientRect(hwnd, out var rect)) return null;
var topLeft = new Native.POINT { X = 0, Y = 0 };
if (!Native.ClientToScreen(hwnd, ref topLeft)) return null;
var width = rect.Right - rect.Left;
var height = rect.Bottom - rect.Top;
if (width <= 0 || height <= 0) return null;
return new Rect { Left = topLeft.X, Top = topLeft.Y, Width = width, Height = height };
}
private static string ProcessNameFromHwnd(IntPtr hwnd)
{
if (hwnd == IntPtr.Zero) return "";
Native.GetWindowThreadProcessId(hwnd, out var pid);
if (pid == 0) return "";
try { return Process.GetProcessById((int)pid).ProcessName; }
catch { return ""; }
}
private static bool IsElevated()
{
using var identity = WindowsIdentity.GetCurrent();
var principal = new WindowsPrincipal(identity);
return principal.IsInRole(WindowsBuiltInRole.Administrator);
}
private static uint SendMouseClickBatch()
{
var inputs = new Native.INPUT[2];
inputs[0].type = 0; // INPUT_MOUSE
inputs[0].mi.dwFlags = Native.MOUSEEVENTF_LEFTDOWN;
inputs[1].type = 0;
inputs[1].mi.dwFlags = Native.MOUSEEVENTF_LEFTUP;
return Native.SendInput(2, inputs, Marshal.SizeOf<Native.INPUT>());
}
private static uint SendMouseWheel(int wheelData)
{
var inputs = new Native.INPUT[1];
inputs[0].type = 0;
inputs[0].mi.mouseData = unchecked((uint)wheelData);
inputs[0].mi.dwFlags = Native.MOUSEEVENTF_WHEEL;
return Native.SendInput(1, inputs, Marshal.SizeOf<Native.INPUT>());
}
private static string GetString(JsonElement root, string name)
=> root.TryGetProperty(name, out var value) ? value.ToString() : "";
private static int GetInt(JsonElement root, string name)
=> TryGetInt(root, name, out var value) ? value : 0;
private static bool TryGetInt(JsonElement root, string name, out int value)
{
value = 0;
if (!root.TryGetProperty(name, out var element)) return false;
if (element.ValueKind == JsonValueKind.Number && element.TryGetInt32(out value)) return true;
if (element.ValueKind == JsonValueKind.String && int.TryParse(element.GetString(), out value)) return true;
return false;
}
}
internal static class Native
{
public const uint MOUSEEVENTF_LEFTDOWN = 0x0002;
public const uint MOUSEEVENTF_LEFTUP = 0x0004;
public const uint MOUSEEVENTF_WHEEL = 0x0800;
public static readonly IntPtr DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2 = new(-4);
[StructLayout(LayoutKind.Sequential)]
public struct POINT { public int X; public int Y; }
[StructLayout(LayoutKind.Sequential)]
public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
// Binary-compatible with the Win32 INPUT for mouse-only use on x64:
// type(4) + 4 pad + MOUSEINPUT(32) = 40 bytes = sizeof(INPUT).
[StructLayout(LayoutKind.Sequential)]
public struct MOUSEINPUT
{
public int dx;
public int dy;
public uint mouseData;
public uint dwFlags;
public uint time;
public UIntPtr dwExtraInfo;
}
[StructLayout(LayoutKind.Sequential)]
public struct INPUT
{
public int type;
public MOUSEINPUT mi;
}
[DllImport("user32.dll")]
public static extern bool SetProcessDPIAware();
[DllImport("shcore.dll")]
public static extern int SetProcessDpiAwareness(int value);
[DllImport("user32.dll")]
public static extern bool SetProcessDpiAwarenessContext(IntPtr value);
[DllImport("user32.dll")]
public static extern bool SetCursorPos(int x, int y);
[DllImport("user32.dll")]
public static extern bool GetCursorPos(out POINT lpPoint);
[DllImport("user32.dll", SetLastError = true)]
public static extern bool GetClientRect(IntPtr hWnd, out RECT lpRect);
[DllImport("user32.dll", SetLastError = true)]
public static extern bool ClientToScreen(IntPtr hWnd, ref POINT lpPoint);
[DllImport("user32.dll")]
public static extern short GetAsyncKeyState(int vKey);
[DllImport("user32.dll", SetLastError = true)]
public static extern uint SendInput(uint nInputs, INPUT[] pInputs, int cbSize);
[DllImport("user32.dll")]
public static extern bool SetForegroundWindow(IntPtr hWnd);
[DllImport("user32.dll")]
public static extern bool ShowWindowAsync(IntPtr hWnd, int nCmdShow);
[DllImport("user32.dll")]
public static extern IntPtr GetForegroundWindow();
[DllImport("user32.dll")]
public static extern bool IsWindow(IntPtr hWnd);
[DllImport("user32.dll")]
public static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint lpdwProcessId);
}