feat(ocr): resolution-anchored layout module + crop preprocessing

Implements ADR-009 (structure + preprocessing; exact IK fixed coordinates still
need calibration against a reference 16:9 screenshot).

- src/lib/layoutProfile.ts: pure, unit-tested geometry for the artifact screen -
  detail rect, the four detail crops, inventory rect/count crop, 5-col grid,
  16:9 detection, aspect label, and an off-16:9 support warning. Single source of
  truth; electron/main.ts now delegates all crop/grid geometry to it and keeps
  colour detection only as the detail-rect fallback.
- src/lib/ocrPreprocess.ts: pure, unit-tested Otsu binarization with inversion
  (artifact text is the bright foreground) over a BGRA bitmap.
- main.ts: OCR now reads an upscaled + binarized copy of each crop; the original
  crop is retained for the diagnostics UI. CaptureResult carries layout info
  { aspect, isSixteenNine, warning }.

NOTE: image preprocessing changes the OCR input and cannot be validated by the
text-level eval harness; it needs a live Genshin 16:9 capture to confirm/tune
(threshold, invert, upscale factor). 88 tests + build green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/lib/layoutProfile.test.ts

@@ -0,0 +1,87 @@
+import { describe, expect, it } from "vitest";
+import {
+  aspectRatioLabel,
+  detailCropRects,
+  inventoryCountCropRect,
+  inventoryGrid,
+  inventoryRect,
+  isSixteenNine,
+  layoutSupportWarning,
+  profileDetailRect,
+} from "./layoutProfile";
+
+const HD = { width: 1920, height: 1080 };
+const QHD = { width: 2560, height: 1440 };
+const ULTRAWIDE = { width: 3440, height: 1440 };
+
+describe("layoutProfile", () => {
+  it("detects 16:9 across common resolutions and rejects ultrawide", () => {
+    expect(isSixteenNine(HD)).toBe(true);
+    expect(isSixteenNine(QHD)).toBe(true);
+    expect(isSixteenNine({ width: 3840, height: 2160 })).toBe(true);
+    expect(isSixteenNine(ULTRAWIDE)).toBe(false);
+    expect(isSixteenNine({ width: 1920, height: 1200 })).toBe(false); // 16:10
+  });
+
+  it("labels the aspect ratio", () => {
+    expect(aspectRatioLabel(HD)).toBe("1.78:1");
+    expect(aspectRatioLabel({ width: 0, height: 0 })).toBe("unknown");
+  });
+
+  it("warns only for non-16:9 resolutions", () => {
+    expect(layoutSupportWarning(HD)).toBe("");
+    expect(layoutSupportWarning(QHD)).toBe("");
+    expect(layoutSupportWarning(ULTRAWIDE)).toContain("nicht 16:9");
+    expect(layoutSupportWarning({ width: 0, height: 0 })).toBe("");
+  });
+
+  it("keeps the detail rect inside the image and on the right half", () => {
+    const rect = profileDetailRect(QHD);
+    expect(rect.x).toBeGreaterThanOrEqual(QHD.width * 0.45);
+    expect(rect.x + rect.width).toBeLessThanOrEqual(QHD.width);
+    expect(rect.y + rect.height).toBeLessThanOrEqual(QHD.height);
+  });
+
+  it("produces the four artifact crops in top-to-bottom order, all clamped", () => {
+    const detail = profileDetailRect(QHD);
+    const crops = detailCropRects(detail, QHD);
+    expect(crops.map((crop) => crop.id)).toEqual([
+      "artifact-title",
+      "artifact-main-stat",
+      "artifact-substats",
+      "artifact-footer",
+    ]);
+    let previousY = -1;
+    for (const crop of crops) {
+      expect(crop.rect.x).toBeGreaterThanOrEqual(0);
+      expect(crop.rect.y).toBeGreaterThan(previousY);
+      expect(crop.rect.x + crop.rect.width).toBeLessThanOrEqual(QHD.width);
+      expect(crop.rect.y + crop.rect.height).toBeLessThanOrEqual(QHD.height);
+      previousY = crop.rect.y;
+    }
+  });
+
+  it("places the inventory count crop inside the inventory panel", () => {
+    const detail = profileDetailRect(QHD);
+    const inv = inventoryRect(QHD, detail);
+    const count = inventoryCountCropRect(inv, QHD);
+    expect(count.x).toBeGreaterThanOrEqual(inv.x);
+    expect(count.x + count.width).toBeLessThanOrEqual(QHD.width);
+  });
+
+  it("builds a 5-column inventory grid on the left", () => {
+    const detail = profileDetailRect(QHD);
+    const grid = inventoryGrid(QHD, detail);
+    expect(grid.cols).toBe(5);
+    expect(grid.source).toBe("detected");
+    expect(grid.centers.length).toBeGreaterThanOrEqual(10);
+    expect(grid.centers.every((center) => center.x < detail.x)).toBe(true);
+  });
+
+  it("reports a missing grid when the inventory panel is too small", () => {
+    const tiny = { width: 320, height: 180 };
+    const grid = inventoryGrid(tiny, profileDetailRect(tiny));
+    expect(grid.source).toBe("missing");
+    expect(grid.centers).toHaveLength(0);
+  });
+});

src/lib/layoutProfile.ts

@@ -0,0 +1,205 @@
+// Resolution-anchored layout geometry for the artifact inventory screen
+// (ADR-009). Inventory Kamera's proven approach is to require borderless 16:9 and
+// derive crop/grid coordinates from the client rectangle instead of detecting the
+// panel by colour each frame. This module is the single, pure, unit-tested source
+// of that geometry; electron/main.ts consumes it for cropping and keeps a
+// colour-based detail-rect detector only as a fallback for off-profile setups.
+//
+// NOTE: the per-field detail crop fractions below are the current working values.
+// True IK-style fixed coordinates need calibration against a reference 16:9
+// screenshot; the structure here is what those calibrated numbers slot into.
+
+export interface LayoutRect {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+export interface CropTemplateRect {
+  id: string;
+  label: string;
+  rect: LayoutRect;
+}
+
+export interface InventoryGridLayout {
+  centers: Array<{ x: number; y: number; row: number; col: number }>;
+  rows: number;
+  cols: number;
+  confidence: number;
+  source: "detected" | "missing";
+}
+
+const SIXTEEN_NINE = 16 / 9;
+
+export function aspectRatio(size: { width: number; height: number }): number {
+  if (!size.height) return 0;
+  return size.width / size.height;
+}
+
+export function aspectRatioLabel(size: { width: number; height: number }): string {
+  const ratio = aspectRatio(size);
+  if (ratio === 0) return "unknown";
+  return `${ratio.toFixed(2)}:1`;
+}
+
+// Genshin's UI is authored for 16:9; other aspect ratios letterbox or reflow and
+// the anchored crops no longer line up. Allow a small tolerance for rounding.
+export function isSixteenNine(size: { width: number; height: number }, tolerance = 0.02): boolean {
+  const ratio = aspectRatio(size);
+  if (ratio === 0) return false;
+  return Math.abs(ratio - SIXTEEN_NINE) <= SIXTEEN_NINE * tolerance;
+}
+
+// Empty when the client is a supported 16:9; otherwise a warning explaining that
+// the anchored crops are unreliable off-profile (ADR-009: non-16:9 is explicitly
+// unsupported for the auto scanner).
+export function layoutSupportWarning(size: { width: number; height: number }): string {
+  if (size.width <= 0 || size.height <= 0) return "";
+  if (isSixteenNine(size)) return "";
+  return `Aufloesung ${size.width}x${size.height} ist nicht 16:9 (${aspectRatioLabel(size)}). Der Auto-Scan ist auf 16:9 im randlosen Fenstermodus ausgelegt; die Erkennung kann daneben liegen.`;
+}
+
+export function clampRect(rect: LayoutRect, imageSize: { width: number; height: number }): LayoutRect {
+  const x = Math.max(0, Math.min(imageSize.width - 1, rect.x));
+  const y = Math.max(0, Math.min(imageSize.height - 1, rect.y));
+  const maxWidth = Math.max(1, imageSize.width - x);
+  const maxHeight = Math.max(1, imageSize.height - y);
+  return {
+    x,
+    y,
+    width: Math.max(1, Math.min(maxWidth, rect.width)),
+    height: Math.max(1, Math.min(maxHeight, rect.height)),
+  };
+}
+
+// Anchored guess for the artifact detail panel on the right of the screen. Used
+// as the primary rect for a clean 16:9 client and as the fallback when colour
+// detection cannot find the panel.
+export function profileDetailRect(imageSize: { width: number; height: number }): LayoutRect {
+  const { width, height } = imageSize;
+  if (width <= 0 || height <= 0) return { x: 0, y: 0, width: Math.max(1, width), height: Math.max(1, height) };
+  return clampRect(
+    {
+      x: Math.round(width * 0.5),
+      y: Math.round(height * 0.08),
+      width: Math.round(width * 0.46),
+      height: Math.round(height * 0.74),
+    },
+    imageSize,
+  );
+}
+
+// The four OCR crops inside the detail panel, as fractions of the detail rect.
+export function detailCropRects(detailRect: LayoutRect, imageSize: { width: number; height: number }): CropTemplateRect[] {
+  const templates: CropTemplateRect[] = [
+    {
+      id: "artifact-title",
+      label: "Artifact title",
+      rect: {
+        x: Math.round(detailRect.x + detailRect.width * 0.055),
+        y: Math.round(detailRect.y + detailRect.height * 0.05),
+        width: Math.round(detailRect.width * 0.82),
+        height: Math.round(detailRect.height * 0.16),
+      },
+    },
+    {
+      id: "artifact-main-stat",
+      label: "Main stat",
+      rect: {
+        x: Math.round(detailRect.x + detailRect.width * 0.055),
+        y: Math.round(detailRect.y + detailRect.height * 0.2),
+        width: Math.round(detailRect.width * 0.82),
+        height: Math.round(detailRect.height * 0.18),
+      },
+    },
+    {
+      id: "artifact-substats",
+      label: "Substats",
+      rect: {
+        x: Math.round(detailRect.x + detailRect.width * 0.055),
+        y: Math.round(detailRect.y + detailRect.height * 0.41),
+        width: Math.round(detailRect.width * 0.82),
+        height: Math.round(detailRect.height * 0.25),
+      },
+    },
+    {
+      id: "artifact-footer",
+      label: "Footer",
+      rect: {
+        x: Math.round(detailRect.x + detailRect.width * 0.055),
+        y: Math.round(detailRect.y + detailRect.height * 0.78),
+        width: Math.round(detailRect.width * 0.82),
+        height: Math.round(detailRect.height * 0.16),
+      },
+    },
+  ];
+
+  return templates.map((template) => ({ ...template, rect: clampRect(template.rect, imageSize) }));
+}
+
+export function inventoryCountCropRect(inventoryRect: LayoutRect, imageSize: { width: number; height: number }): LayoutRect {
+  return clampRect(
+    {
+      x: Math.round(inventoryRect.x + inventoryRect.width * 0.62),
+      y: Math.round(inventoryRect.y + inventoryRect.height * 0.02),
+      width: Math.round(inventoryRect.width * 0.34),
+      height: Math.round(inventoryRect.height * 0.09),
+    },
+    imageSize,
+  );
+}
+
+export function inventoryRect(imageSize: { width: number; height: number }, detailRect: LayoutRect): LayoutRect {
+  const { width, height } = imageSize;
+  const preferredWidth = Math.max(140, Math.round(width * 0.48));
+  const x = Math.round(width * 0.03);
+  const y = Math.round(detailRect.y + detailRect.height * 0.09);
+  const availableWidth = Math.max(100, detailRect.x - Math.round(width * 0.04));
+  const panelWidth = Math.max(100, Math.min(preferredWidth, availableWidth));
+  const safeWidth = panelWidth > width * 0.85 ? Math.round(width * 0.55) : panelWidth;
+  return clampRect(
+    {
+      x,
+      y,
+      width: Math.min(safeWidth, Math.max(width - x - Math.round(width * 0.02), 100)),
+      height: Math.max(140, Math.round(height * 0.7)),
+    },
+    imageSize,
+  );
+}
+
+export function inventoryGrid(imageSize: { width: number; height: number }, detailRect: LayoutRect): InventoryGridLayout {
+  const rect = inventoryRect(imageSize, detailRect);
+  const cols = 5;
+  if (rect.width < 160 || rect.height < 140) {
+    return { centers: [], rows: 0, cols: 0, confidence: 0, source: "missing" };
+  }
+
+  const cellWidth = Math.max(56, Math.round(rect.width / cols));
+  const stepX = Math.round(cellWidth * 0.96);
+  const stepY = Math.round(cellWidth * 1.03);
+  const visibleRows = Math.max(2, Math.min(6, Math.round(rect.height / Math.max(stepY, 1))));
+
+  const startX = rect.x + Math.max(6, Math.round(stepX * 0.45));
+  const startY = rect.y + Math.max(6, Math.round(stepY * 0.45));
+  const centers: InventoryGridLayout["centers"] = [];
+  for (let row = 0; row < visibleRows; row++) {
+    for (let col = 0; col < cols; col++) {
+      const x = startX + col * stepX;
+      const y = startY + row * stepY;
+      if (x < imageSize.width && y < imageSize.height) {
+        centers.push({ x, y, row, col });
+      }
+    }
+  }
+
+  const trimmed = centers.filter((center) => center.x > 0 && center.y > 0);
+  return {
+    centers: trimmed,
+    rows: visibleRows,
+    cols,
+    confidence: trimmed.length >= cols * 2 ? 76 : trimmed.length >= cols ? 58 : 36,
+    source: "detected",
+  };
+}

src/lib/ocrPreprocess.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, it } from "vitest";
+import { binarizeForOcr, computeLuminanceHistogram, otsuThreshold, type Bitmap } from "./ocrPreprocess";
+
+// Build a BGRA bitmap from a grid of [b,g,r] pixels.
+function bitmapFrom(pixels: Array<[number, number, number]>, width: number, height: number): Bitmap {
+  const data = Buffer.alloc(width * height * 4);
+  pixels.forEach(([b, g, r], index) => {
+    data[index * 4] = b;
+    data[index * 4 + 1] = g;
+    data[index * 4 + 2] = r;
+    data[index * 4 + 3] = 255;
+  });
+  return { data, width, height };
+}
+
+describe("ocrPreprocess", () => {
+  it("computes a luminance histogram over all pixels", () => {
+    const bitmap = bitmapFrom([
+      [0, 0, 0],
+      [255, 255, 255],
+      [0, 0, 0],
+      [255, 255, 255],
+    ], 2, 2);
+    const histogram = computeLuminanceHistogram(bitmap);
+    expect(histogram[0]).toBe(2);
+    expect(histogram[255]).toBe(2);
+    expect(histogram.reduce((sum, count) => sum + count, 0)).toBe(4);
+  });
+
+  it("otsu splits a clean bimodal image between the two peaks", () => {
+    const histogram = new Array<number>(256).fill(0);
+    histogram[20] = 50;
+    histogram[220] = 50;
+    const threshold = otsuThreshold(histogram);
+    expect(threshold).toBeGreaterThanOrEqual(20);
+    expect(threshold).toBeLessThan(220);
+  });
+
+  it("otsu is safe on an empty histogram", () => {
+    expect(otsuThreshold(new Array<number>(256).fill(0))).toBe(127);
+  });
+
+  it("inverts bright foreground to black-on-white by default", () => {
+    // Bright text pixel + dark background pixel.
+    const bitmap = bitmapFrom([
+      [255, 255, 255], // bright -> should become black
+      [0, 0, 0], // dark -> should become white
+    ], 2, 1);
+    const out = binarizeForOcr(bitmap, { threshold: 128 });
+    expect([out.data[0], out.data[1], out.data[2]]).toEqual([0, 0, 0]);
+    expect([out.data[4], out.data[5], out.data[6]]).toEqual([255, 255, 255]);
+    expect(out.data[3]).toBe(255);
+  });
+
+  it("keeps bright foreground white when inversion is disabled", () => {
+    const bitmap = bitmapFrom([
+      [255, 255, 255],
+      [0, 0, 0],
+    ], 2, 1);
+    const out = binarizeForOcr(bitmap, { threshold: 128, invertBrightForeground: false });
+    expect(out.data[0]).toBe(255);
+    expect(out.data[4]).toBe(0);
+  });
+
+  it("preserves dimensions and always emits opaque pixels", () => {
+    const bitmap = bitmapFrom(Array.from({ length: 9 }, () => [100, 100, 100] as [number, number, number]), 3, 3);
+    const out = binarizeForOcr(bitmap);
+    expect(out.width).toBe(3);
+    expect(out.height).toBe(3);
+    for (let pixel = 0; pixel < 9; pixel++) {
+      expect(out.data[pixel * 4 + 3]).toBe(255);
+    }
+  });
+});

src/lib/ocrPreprocess.ts

@@ -0,0 +1,101 @@
+// OCR preprocessing for artifact crops (ADR-009). Tesseract reads a clean, high
+// contrast, dark-text-on-light image far more reliably than Genshin's native
+// bright-text-on-dark UI. This binarizes a crop with Otsu thresholding and (by
+// default) inverts, because artifact text is the bright foreground.
+//
+// Works on a raw BGRA bitmap (Electron NativeImage.getBitmap() layout on
+// Windows). Kept pure and channel-order-agnostic for luminance so it is unit
+// testable without Electron. Upscaling is done separately via NativeImage.resize
+// before this runs - interpolated upscaling of small crops is a big Tesseract win
+// and NativeImage does it better than hand-rolled JS.
+
+export interface Bitmap {
+  data: Uint8Array | Buffer;
+  width: number;
+  height: number;
+}
+
+export interface BinarizeOptions {
+  /** Artifact text is the bright foreground, so invert to dark-on-light. */
+  invertBrightForeground?: boolean;
+  /** Override Otsu with a fixed 0-255 luminance threshold. */
+  threshold?: number;
+}
+
+const BYTES_PER_PIXEL = 4;
+
+// Rec. 601 luma. Channel order does not matter for a weighted sum as long as we
+// read the same three bytes; BGRA and RGBA give the same luminance here because
+// we weight by position-independent coefficients applied to the actual R/G/B.
+function luminanceAt(data: Uint8Array | Buffer, index: number): number {
+  // NativeImage on Windows is BGRA: byte0=B, byte1=G, byte2=R.
+  const b = data[index];
+  const g = data[index + 1];
+  const r = data[index + 2];
+  return 0.299 * r + 0.587 * g + 0.114 * b;
+}
+
+export function computeLuminanceHistogram(bitmap: Bitmap): number[] {
+  const histogram = new Array<number>(256).fill(0);
+  const { data, width, height } = bitmap;
+  const pixels = width * height;
+  for (let pixel = 0; pixel < pixels; pixel++) {
+    const value = Math.round(luminanceAt(data, pixel * BYTES_PER_PIXEL));
+    histogram[Math.max(0, Math.min(255, value))]++;
+  }
+  return histogram;
+}
+
+// Otsu's method: pick the threshold that maximizes between-class variance.
+export function otsuThreshold(histogram: readonly number[]): number {
+  const total = histogram.reduce((sum, count) => sum + count, 0);
+  if (total === 0) return 127;
+
+  let sumAll = 0;
+  for (let level = 0; level < 256; level++) sumAll += level * histogram[level];
+
+  let sumBackground = 0;
+  let weightBackground = 0;
+  let maxVariance = -1;
+  let threshold = 127;
+
+  for (let level = 0; level < 256; level++) {
+    weightBackground += histogram[level];
+    if (weightBackground === 0) continue;
+    const weightForeground = total - weightBackground;
+    if (weightForeground === 0) break;
+
+    sumBackground += level * histogram[level];
+    const meanBackground = sumBackground / weightBackground;
+    const meanForeground = (sumAll - sumBackground) / weightForeground;
+    const betweenVariance = weightBackground * weightForeground * (meanBackground - meanForeground) ** 2;
+
+    if (betweenVariance > maxVariance) {
+      maxVariance = betweenVariance;
+      threshold = level;
+    }
+  }
+
+  return threshold;
+}
+
+export function binarizeForOcr(bitmap: Bitmap, options: BinarizeOptions = {}): Bitmap {
+  const { data, width, height } = bitmap;
+  const invert = options.invertBrightForeground ?? true;
+  const threshold = options.threshold ?? otsuThreshold(computeLuminanceHistogram(bitmap));
+
+  const output = Buffer.alloc(width * height * BYTES_PER_PIXEL);
+  const pixels = width * height;
+  for (let pixel = 0; pixel < pixels; pixel++) {
+    const index = pixel * BYTES_PER_PIXEL;
+    const isBright = luminanceAt(data, index) > threshold;
+    // Bright foreground text -> black; dark background -> white (inverted).
+    const value = invert ? (isBright ? 0 : 255) : (isBright ? 255 : 0);
+    output[index] = value;
+    output[index + 1] = value;
+    output[index + 2] = value;
+    output[index + 3] = 255;
+  }
+
+  return { data: output, width, height };
+}

src/types/global.d.ts

@@ -56,6 +56,11 @@ export interface CaptureResult {
     source: "ocr" | "missing";
     text: string;
   };
+  layout?: {
+    aspect: string;
+    isSixteenNine: boolean;
+    warning: string;
+  };
 }
 
 export interface WindowBounds {