Files
genshin-assistant/src/types/global.d.ts
T
AzuTear 639b0b7f59 feat(scanner): add native artifact pipeline
Add native IK-style capture processing, Artifact Inventory, explicit promotion and single-result review. Confirm the three live OCR corrections in the eval corpus and preserve extraction/value separation.
2026-07-09 23:30:42 +02:00

685 lines
18 KiB
TypeScript

import type { AppSnapshot } from "./domain";
import type { ScanResultCategory, ScanResultIkMatch, StoredArtifactRecord, StoredScanResultEntry } from "./storage";
export interface CaptureOptions {
skipOcr?: boolean;
skipOcrUnlessArtifactDetail?: boolean;
ocrMode?: "full" | "artifact";
ocrProfile?: "full" | "fast";
ocrEngine?: "current";
scanEntryMode?: "visible-inventory" | "direct-inventory" | "paimon-menu" | "auto-entry";
omitFullFrame?: boolean;
omitDetailPreview?: boolean;
omitInventoryPreview?: boolean;
omitCrops?: boolean;
omitCropImages?: boolean;
omitEquippedOcr?: boolean;
omitLockState?: boolean;
}
export interface CaptureSourceInfo {
id: string;
name: string;
isGenshinCandidate: boolean;
thumbnailDataUrl: string;
}
export interface CaptureCrop {
id: string;
label: string;
rect: { x: number; y: number; width: number; height: number };
dataUrl?: string;
}
export interface OcrResult {
id: string;
label: string;
text: string;
confidence: number;
elapsedMs?: number;
}
export type CaptureTarget = "genshin-client" | "primary-screen" | "desktop-source";
export interface CaptureResult {
id: string;
name: string;
width: number;
height: number;
dataUrl: string;
capturedAt: string;
captureTarget?: CaptureTarget;
detailDataUrl?: string;
inventoryDataUrl?: string;
detailFingerprint?: string;
inventoryFingerprint?: string;
ocrSkipped?: boolean;
ocrTimedOut?: boolean;
crops?: CaptureCrop[];
ocr?: OcrResult[];
inventoryGrid?: {
centers: Array<{ x: number; y: number; row: number; col: number }>;
rows: number;
cols: number;
confidence: number;
source: "detected" | "fallback" | "missing";
};
artifactDetail?: {
present: boolean;
confidence: number;
orangeHits: number;
greenHits: number;
textHits: number;
titleOrangeHits?: number;
upperTextHits?: number;
lowerGreenHits?: number;
};
paimonMenu?: {
present: boolean;
confidence: number;
profileLightPct: number;
profileCreamPct: number;
menuTileDarkPct: number;
};
inventoryCount?: {
current: number;
total: number;
confidence: number;
source: "ocr" | "missing";
text: string;
};
locked?: boolean;
lockSignal?: {
ratio: number;
threshold: number;
rect: {
x: number;
y: number;
width: number;
height: number;
};
};
sanctified?: boolean;
elapsedMs?: number;
layout?: {
aspect: string;
isSixteenNine: boolean;
warning: string;
};
timings?: {
totalMs: number;
prepareMs: number;
ocrMs: number;
cropCount: number;
ocrEngine: CaptureOptions["ocrEngine"];
ocrWorkerPoolSize?: number;
ocrProfile?: CaptureOptions["ocrProfile"];
ocrFieldMs?: Record<string, number>;
ocrSkipped: boolean;
};
}
export interface WindowBounds {
x: number;
y: number;
width: number;
height: number;
}
export interface ClickResult {
ok: boolean;
x: number;
y: number;
cursorX?: number;
cursorY?: number;
escapePressed?: boolean;
enterPressed?: boolean;
f9Pressed?: boolean;
/** Cursor verifiably reached the target position before clicking. */
moved?: boolean;
/** Mouse down+up were injected (only attempted when moved is true). */
clicked?: boolean;
/** SendInput injected zero events, e.g. blocked by UIPI (elevated game). */
inputBlocked?: boolean;
focused?: boolean;
alreadyForeground?: boolean;
foregroundProcess?: string;
targetProcess?: string;
isElevated?: boolean;
}
export interface RuntimeInfo {
ok: boolean;
isElevated: boolean;
platform: string;
appBuild?: AppRuntimeInfo;
hotkeys?: Record<string, boolean>;
genshinFound?: boolean;
genshinHwnd?: number;
targetProcess?: string;
foregroundProcess?: string;
foregroundHwnd?: number;
helperPid?: number;
}
export interface AppRuntimeInfo {
signature: string;
pid: number;
startedAt: string;
cwd: string;
isDev: boolean;
expectedOcrWorkerPoolSize?: number;
}
export interface FocusGenshinResult {
focused: boolean;
alreadyForeground: boolean;
foregroundProcess?: string;
targetProcess?: string;
genshinFound?: boolean;
setForegroundResult?: boolean;
}
export interface BooleanResult {
ok: boolean;
}
export interface SaveResultWithPath {
ok: boolean;
path: string;
}
export type SaveSnapshotResult = SaveResultWithPath;
export type ScannerCommand =
| "start-auto"
| "stop"
| {
type: "start-auto";
scanLimit?: number;
scanEntryMode?: CaptureOptions["scanEntryMode"];
};
export interface ScannerLearningRulePayload {
textReplacements?: Record<string, string>;
fieldAliases?: Record<string, Record<string, string>>;
constrainedFixes?: Record<string, string>;
cropAdjustments?: Record<string, {
dx?: number;
dy?: number;
dw?: number;
dh?: number;
reason?: string;
approved?: boolean;
}>;
uiProfileAdjustments?: Record<string, {
fieldId: string;
dx?: number;
dy?: number;
dw?: number;
dh?: number;
reason?: string;
approved?: boolean;
}>;
}
export interface NativeScannerDataStatus {
directory: string;
version: string;
artifactSets: number;
artifactPieces: number;
weapons: number;
characters: number;
materials: number;
totalEntries: number;
categories: {
artifacts: NativeScannerArtifactCategoryStatus;
weapons: NativeScannerSimpleCategoryStatus;
characters: NativeScannerSimpleCategoryStatus;
materials: NativeScannerSimpleCategoryStatus;
};
missing: string[];
valid: boolean;
source: string;
}
export type NativeScannerCategoryScanStatus = "native_capture" | "catalog_only" | "missing_data";
export type NativeScannerStartCategory = "artifacts" | "weapons" | "characters" | "materials";
export interface NativeScannerCategoryStatusBase {
file: string;
supported: boolean;
catalogAvailable?: boolean;
nativeCaptureSupported?: boolean;
scanStatus?: NativeScannerCategoryScanStatus;
}
export interface NativeScannerArtifactCategoryStatus extends NativeScannerCategoryStatusBase {
setCount: number;
pieceCount: number;
}
export interface NativeScannerSimpleCategoryStatus extends NativeScannerCategoryStatusBase {
count: number;
}
export interface NativeScannerGridStatus {
rows: number;
cols: number;
count: number;
anchorX: number;
anchorY: number;
first?: { x: number; y: number; row: number; col: number };
last?: { x: number; y: number; row: number; col: number };
}
export interface NativeScannerCatalogStatus {
data: NativeScannerDataStatus;
artifacts: Array<{
normalizedName: string;
setName: string;
good: string;
pieces: Array<{ slot: string; artifactName: string; good: string; normalizedName: string }>;
}>;
weapons: Array<{ normalizedName: string; good: string }>;
characters: Array<{
normalizedName: string;
good: string;
element: string;
weaponType: number;
constellationName: string;
}>;
materials: Array<{ normalizedName: string; good: string }>;
}
export interface NativeScannerRunStatus {
running: boolean;
status: "idle" | "running" | "done" | "blocked" | "stopped" | string;
runId: string;
category?: NativeScannerStartCategory | string;
target: number;
captured: number;
queued: number;
clicked: number;
pages: number;
activeMs: number;
message: string;
outputRoot: string;
runDir: string;
manifestPath: string;
jobsPath: string;
statusPath: string;
dataVersion: string;
lastArtifactPath: string;
supportedCategories?: NativeScannerDataStatus["categories"];
lastJob?: {
sequence: number;
category?: NativeScannerStartCategory | string;
page: number;
row: number;
col: number;
clientX: number;
clientY: number;
screenX: number;
screenY: number;
clickEventsSent: number;
relativePath: string;
absolutePath: string;
capturedAt: string;
detail: { width: number; height: number };
kind: string;
downstream: string;
};
}
export interface NativeScannerPreflightStatus {
data: NativeScannerDataStatus;
category?: NativeScannerStartCategory | string;
supportedCategories?: NativeScannerDataStatus["categories"];
categoryReady?: boolean;
genshinFound: boolean;
bounds: { left: number; top: number; width: number; height: number } | null;
isSixteenNine: boolean;
grid: NativeScannerGridStatus;
visual?: {
ready: boolean;
samples: number;
whitePct: number;
darkPct: number;
colorPct: number;
lumaStdDev: number;
blockReason?: string;
};
ready: boolean;
blockReason?: string;
}
export interface NativeScannerProcessStatus {
ok: boolean;
runDir: string;
jobsPath: string;
reportPath: string;
scanResultsPath: string;
processed: number;
parsed: number;
review: number;
stored: number;
errors: number;
elapsedMs: number;
queueConcurrency: number;
persisted: boolean;
error?: string;
results: Array<{
sequence: number;
category?: ScanResultCategory | string;
page?: number;
row?: number;
col?: number;
imagePath: string;
parsed: boolean;
artifactName?: string;
setName?: string;
slot?: string;
confidence: number;
needsReview: boolean;
persisted?: boolean;
ikMatch?: ScanResultIkMatch;
notes?: string[];
ocr: OcrResult[];
error?: string;
}>;
}
export interface NativeScannerResultsLoadStatus {
ok: boolean;
runDir: string;
path: string;
total: number;
results: StoredScanResultEntry[];
error?: string;
}
export interface NativeScannerPromotionStatus {
ok: boolean;
runDir: string;
logPath: string;
requested: number;
selected: number;
promoted: number;
alreadyStored: number;
review: number;
blocked: number;
added: number;
updated: number;
total: number;
promotedResultIds: string[];
error?: string;
}
export interface NativeScannerReviewArtifactInput {
name: string;
slot: string;
level: number;
setName: string;
mainStat: string;
mainValue: string;
substats: string[];
equipped: string;
locked?: boolean;
}
export interface NativeScannerReviewStatus {
ok: boolean;
runDir: string;
logPath: string;
resultId: string;
action: "approve" | "reject";
evalSampleSaved: boolean;
correctedFields: string[];
error?: string;
}
export interface NativeScannerImageLoadStatus {
ok: boolean;
runDir: string;
path: string;
dataUrl: string;
width: number;
height: number;
error?: string;
}
export interface NativeScannerResult {
ok: boolean;
scanner: NativeScannerRunStatus | NativeScannerPreflightStatus | NativeScannerDataStatus | NativeScannerCatalogStatus | NativeScannerProcessStatus | NativeScannerResultsLoadStatus | NativeScannerImageLoadStatus;
}
export interface LoadScannerLearningRulesResult {
ok: boolean;
path: string;
rules: ScannerLearningRulePayload;
}
export interface SaveScannerLearningRulesResult {
ok: boolean;
path: string;
rules: ScannerLearningRulePayload;
total: number;
}
export interface AutomationGuard {
ok: boolean;
cursorX?: number;
cursorY?: number;
escapePressed: boolean;
enterPressed?: boolean;
f9Pressed?: boolean;
isElevated?: boolean;
}
export interface GdiCaptureResult {
dataUrl?: string;
imageBase64?: string;
width: number;
height: number;
originX: number;
originY: number;
captureTarget: Extract<CaptureTarget, "primary-screen" | "genshin-client">;
}
export interface HelperOperationResponse {
id?: string;
ok: boolean;
error?: string;
[key: string]: unknown;
}
export interface ScrollResult {
ok: boolean;
notchesSent: number;
inputBlocked?: boolean;
isElevated?: boolean;
}
export interface KeyPressResult {
ok: boolean;
key: string;
focused?: boolean;
foregroundProcess?: string;
targetProcess?: string;
isElevated?: boolean;
inputBlocked?: boolean;
eventsSent?: number;
}
export type ArtifactSaveResult = ArtifactStoreSaveResult;
export interface ArtifactStoreLoadResult {
ok: boolean;
artifacts: StoredArtifactRecord[];
total: number;
path: string;
}
export interface ArtifactStoreSaveResult {
ok: boolean;
added: number;
updated: number;
total: number;
path: string;
}
export interface ReviewSampleRecord {
savedAt: string;
sample?: {
reason?: string;
parsed?: unknown;
capture?: {
id?: string;
name?: string;
width?: number;
height?: number;
dataUrl?: string;
detailDataUrl?: string;
inventoryDataUrl?: string;
captureTarget?: CaptureResult["captureTarget"];
capturedAt?: string;
crops?: Array<{ id: string; label: string; rect: { x: number; y: number; width: number; height: number }; dataUrl?: string }>;
ocr?: OcrResult[];
inventoryGrid?: CaptureResult["inventoryGrid"];
inventoryCount?: CaptureResult["inventoryCount"];
locked?: boolean;
};
};
}
export interface ReviewSampleListResult {
ok: boolean;
samples: ReviewSampleRecord[];
total: number;
path: string;
}
export interface ReviewSamplePayload {
reason?: string;
parsed?: unknown;
capture?: {
id?: string;
name?: string;
width?: number;
height?: number;
dataUrl?: string;
detailDataUrl?: string;
inventoryDataUrl?: string;
captureTarget?: CaptureResult["captureTarget"];
capturedAt?: string;
crops?: Array<{ id: string; label: string; rect: { x: number; y: number; width: number; height: number }; dataUrl?: string }>;
ocr?: OcrResult[];
inventoryGrid?: CaptureResult["inventoryGrid"];
inventoryCount?: CaptureResult["inventoryCount"];
locked?: boolean;
};
[key: string]: unknown;
}
export interface ScannerStatusPayload {
running: boolean;
reviewStatus: string;
captureStatus: string;
selectedSource: string | null;
stats: Record<string, number>;
summary: unknown;
snapshotArtifacts: number;
snapshotCharacters: number;
snapshotRecommendations: number;
snapshotBuilds: number;
grid: CaptureResult["inventoryGrid"] | null;
automationLog: string[];
diagnosticEvents?: unknown[];
runtimeInfo: RuntimeInfo | null;
appBuild?: AppRuntimeInfo;
storedTotal: number | null;
learningRuleCount: number;
lookupStatus?: unknown;
ocrEngine?: CaptureOptions["ocrEngine"];
ocrWarmup?: unknown;
entryMode?: CaptureOptions["scanEntryMode"];
benchmarkSummary?: unknown;
updatedAt: string | null;
[key: string]: unknown;
}
export interface GoodExportArtifact {
setKey: string;
slotKey: string;
rarity: number;
level: number;
mainStatKey: string;
substats: Array<{ key: string; value: number }>;
lock: boolean;
}
export interface GoodDatabase {
format: "GOOD";
version: number;
source: string;
artifacts: GoodExportArtifact[];
}
export interface GoodImportFileResult {
ok: boolean;
canceled: boolean;
path: string;
database?: unknown;
error?: string;
}
declare global {
interface Window {
assistantApi?: {
loadSnapshot: () => Promise<AppSnapshot | null>;
saveSnapshot: (snapshot: AppSnapshot) => Promise<SaveSnapshotResult>;
runMockScan: () => Promise<AppSnapshot | null>;
listCaptureSources: () => Promise<CaptureSourceInfo[]>;
captureSource: (sourceId: string, delayMs?: number, focusGenshin?: boolean, options?: CaptureOptions) => Promise<CaptureResult>;
clickScreen: (x: number, y: number) => Promise<ClickResult>;
scrollScreen: (notches: number, anchorX?: number, anchorY?: number) => Promise<ScrollResult>;
keyPress: (key: string) => Promise<KeyPressResult>;
getAutomationGuard: () => Promise<AutomationGuard>;
focusMainWindow: () => Promise<BooleanResult>;
focusGenshin: () => Promise<FocusGenshinResult>;
focusGenshinForScanStart: () => Promise<FocusGenshinResult>;
getRuntimeInfo: () => Promise<RuntimeInfo>;
saveReviewSample: (sample: ReviewSamplePayload) => Promise<SaveResultWithPath>;
loadReviewSamples: (limit?: number) => Promise<ReviewSampleListResult>;
loadScannerLearningRules: () => Promise<LoadScannerLearningRulesResult>;
saveScannerLearningRules: (rules: ScannerLearningRulePayload) => Promise<SaveScannerLearningRulesResult>;
loadArtifacts: () => Promise<ArtifactStoreLoadResult>;
saveArtifacts: (records: StoredArtifactRecord[]) => Promise<ArtifactStoreSaveResult>;
exportGood: (payload: GoodDatabase) => Promise<SaveResultWithPath>;
importGoodFile: () => Promise<GoodImportFileResult>;
publishScannerStatus: (status: ScannerStatusPayload) => Promise<BooleanResult>;
nativeScannerDataStatus: () => Promise<NativeScannerDataStatus>;
nativeScannerCatalog: () => Promise<NativeScannerCatalogStatus>;
nativeScannerPreflight: (options?: { category?: NativeScannerStartCategory }) => Promise<NativeScannerPreflightStatus>;
nativeScannerStart: (options?: { limit?: number; category?: NativeScannerStartCategory }) => Promise<NativeScannerRunStatus>;
nativeScannerStop: () => Promise<NativeScannerRunStatus>;
nativeScannerStatus: () => Promise<NativeScannerRunStatus>;
nativeScannerProcessRun: (options?: { runDir?: string; persist?: boolean; limit?: number }) => Promise<NativeScannerProcessStatus>;
nativeScannerLoadResults: (options?: { runDir?: string; limit?: number }) => Promise<NativeScannerResultsLoadStatus>;
nativeScannerPromoteResults: (options: { runDir?: string; resultIds: string[] }) => Promise<NativeScannerPromotionStatus>;
nativeScannerReviewResult: (options: {
runDir?: string;
resultId: string;
action: "approve" | "reject";
artifact?: NativeScannerReviewArtifactInput;
note?: string;
}) => Promise<NativeScannerReviewStatus>;
nativeScannerLoadImage: (options: { runDir?: string; imagePath: string }) => Promise<NativeScannerImageLoadStatus>;
showOverlay: () => Promise<BooleanResult>;
hideOverlay: () => Promise<BooleanResult>;
onScannerCommand: (callback: (command: ScannerCommand) => void) => () => void;
};
}
}