diff --git a/electron/main.ts b/electron/main.ts index f8cbc11..3196bb6 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -22,6 +22,17 @@ import type { ReviewSamplesRepositoryPort, ScannerLearningRepositoryPort, } from "./repositories/index.js"; +import { + aspectRatioLabel, + detailCropRects, + inventoryCountCropRect, + inventoryGrid as layoutInventoryGrid, + inventoryRect as layoutInventoryRect, + isSixteenNine, + layoutSupportWarning, + profileDetailRect, +} from "../src/lib/layoutProfile.js"; +import { binarizeForOcr } from "../src/lib/ocrPreprocess.js"; // Chromium's renderer sandbox can refuse to fully initialize (or silently // crash the GPU/renderer process) when the hosting process runs with a full @@ -746,84 +757,48 @@ function imageCropDataUrl(sourceImage: NativeImage, rect: Electron.Rectangle, im return sourceImage.crop(safeRect).toDataURL(); } +// Preprocessed copy of a crop for OCR (ADR-009): upscale for more pixels, then +// grayscale + Otsu-binarize with inversion (artifact text is the bright +// foreground). The original crop is kept separately for the diagnostics UI. +function preprocessedCropDataUrl(sourceImage: NativeImage, rect: Electron.Rectangle, imageSize: { width: number; height: number }) { + const safeRect = clampCaptureRect(rect, imageSize); + const upscaled = sourceImage.crop(safeRect).resize({ width: Math.max(1, safeRect.width * 2), quality: "best" }); + const size = upscaled.getSize(); + if (!size.width || !size.height) return upscaled.toDataURL(); + const binarized = binarizeForOcr({ data: upscaled.getBitmap(), width: size.width, height: size.height }); + return nativeImage + .createFromBitmap(Buffer.from(binarized.data), { width: binarized.width, height: binarized.height }) + .toDataURL(); +} + function createCrops( sourceImage: NativeImage, imageSize: { width: number; height: number }, detailRect: Electron.Rectangle, inventoryRect: Electron.Rectangle, ) { - const templates: CropTemplate[] = [ - { - id: "artifact-title", - label: "Artifact title", - rect: { - x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.05), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.16), - }, - }, - { - id: "artifact-main-stat", - label: "Main stat", - rect: { - x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.20), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.18), - }, - }, - { - id: "artifact-substats", - label: "Substats", - rect: { - x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.41), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.25), - }, - }, - { - id: "artifact-footer", - label: "Footer", - rect: { - x: Math.round(detailRect.x + detailRect.width * 0.055), - y: Math.round(detailRect.y + detailRect.height * 0.78), - width: Math.round(detailRect.width * 0.82), - height: Math.round(detailRect.height * 0.16), - }, - }, - ]; + const templates: CropTemplate[] = detailCropRects(detailRect, imageSize); if (inventoryRect.width > 120 && inventoryRect.height > 80) { templates.push({ id: "inventory-count", label: "Inventory count", - rect: { - x: Math.round(inventoryRect.x + inventoryRect.width * 0.62), - y: Math.round(inventoryRect.y + inventoryRect.height * 0.02), - width: Math.round(inventoryRect.width * 0.34), - height: Math.round(inventoryRect.height * 0.09), - }, + rect: inventoryCountCropRect(inventoryRect, imageSize), }); } return templates - .map((template) => ({ - ...template, - rect: clampCaptureRect(template.rect, imageSize), - dataUrl: imageCropDataUrl(sourceImage, template.rect, imageSize), - })) - .filter((crop) => crop.rect.width > 0 && crop.rect.height > 0) - .map((crop) => ({ - ...crop, - rect: { - x: crop.rect.x, - y: crop.rect.y, - width: crop.rect.width, - height: crop.rect.height, - }, - })); + .map((template) => { + const rect = clampCaptureRect(template.rect, imageSize); + return { + id: template.id, + label: template.label, + rect, + dataUrl: imageCropDataUrl(sourceImage, rect, imageSize), + ocrDataUrl: preprocessedCropDataUrl(sourceImage, rect, imageSize), + }; + }) + .filter((crop) => crop.rect.width > 0 && crop.rect.height > 0); } function inferDetailRect(bitmap: Buffer, imageSize: { width: number; height: number }) { @@ -865,79 +840,17 @@ function inferDetailRect(bitmap: Buffer, imageSize: { width: number; height: num return clampCaptureRect({ x: left, y: top, width: widthGuess, height: heightGuess }, imageSize); } - if (width > 0 && height > 0) { - return clampCaptureRect( - { - x: Math.round(width * 0.50), - y: Math.round(height * 0.08), - width: Math.round(width * 0.46), - height: Math.round(height * 0.74), - }, - imageSize, - ); - } - - return { x: 0, y: 0, width, height }; + // Colour detection found nothing usable; fall back to the resolution-anchored + // profile rect (single source of truth in layoutProfile). + return profileDetailRect(imageSize); } function inferInventoryRect(imageSize: { width: number; height: number }, detailRect: Electron.Rectangle) { - const { width, height } = imageSize; - const preferredWidth = Math.max(140, Math.round(width * 0.48)); - const x = Math.round(width * 0.03); - const y = Math.round(detailRect.y + detailRect.height * 0.09); - const availableWidth = Math.max(100, detailRect.x - Math.round(width * 0.04)); - const panelWidth = Math.max(100, Math.min(preferredWidth, availableWidth)); - const safeWidth = panelWidth > width * 0.85 ? Math.round(width * 0.55) : panelWidth; - return clampCaptureRect( - { - x, - y, - width: Math.min(safeWidth, Math.max(width - x - Math.round(width * 0.02), 100)), - height: Math.max(140, Math.round(height * 0.70)), - }, - imageSize, - ); + return layoutInventoryRect(imageSize, detailRect); } function inferInventoryGrid(imageSize: { width: number; height: number }, detailRect: Electron.Rectangle) { - const inventoryRect = inferInventoryRect(imageSize, detailRect); - const cols = 5; - if (inventoryRect.width < 160 || inventoryRect.height < 140) { - return { - centers: [], - rows: 0, - cols: 0, - confidence: 0, - source: "missing" as const, - }; - } - - const cellWidth = Math.max(56, Math.round(inventoryRect.width / cols)); - const stepX = Math.round(cellWidth * 0.96); - const stepY = Math.round(cellWidth * 1.03); - const visibleRows = Math.max(2, Math.min(6, Math.round(inventoryRect.height / Math.max(stepY, 1)))); - - const startX = inventoryRect.x + Math.max(6, Math.round(stepX * 0.45)); - const startY = inventoryRect.y + Math.max(6, Math.round(stepY * 0.45)); - const centers = []; - for (let row = 0; row < visibleRows; row++) { - for (let col = 0; col < cols; col++) { - const x = startX + col * stepX; - const y = startY + row * stepY; - if (x < imageSize.width && y < imageSize.height) { - centers.push({ x, y, row, col }); - } - } - } - - const trimmed = centers.filter((center) => center.x > 0 && center.y > 0); - return { - centers: trimmed, - rows: visibleRows, - cols, - confidence: trimmed.length >= cols * 2 ? 76 : trimmed.length >= cols ? 58 : 36, - source: "detected" as const, - }; + return layoutInventoryGrid(imageSize, detailRect); } async function buildCaptureResult( @@ -959,7 +872,9 @@ async function buildCaptureResult( const croppedPayload = crops.map((crop) => ({ id: crop.id, label: crop.label, - dataUrl: crop.dataUrl, + // OCR reads the preprocessed (upscaled + binarized) crop; the original is + // kept below for the diagnostics UI. + dataUrl: crop.ocrDataUrl ?? crop.dataUrl, })); const recognized = options.skipOcr ? { ocr: [], timedOut: false } : await runOcrOnCropsWithTimeout(croppedPayload); @@ -990,6 +905,11 @@ async function buildCaptureResult( })), inventoryGrid: inferInventoryGrid(size, detailRect), inventoryCount: count, + layout: { + aspect: aspectRatioLabel(size), + isSixteenNine: isSixteenNine(size), + warning: layoutSupportWarning(size), + }, }; } diff --git a/src/lib/layoutProfile.test.ts b/src/lib/layoutProfile.test.ts new file mode 100644 index 0000000..b3a0d03 --- /dev/null +++ b/src/lib/layoutProfile.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { + aspectRatioLabel, + detailCropRects, + inventoryCountCropRect, + inventoryGrid, + inventoryRect, + isSixteenNine, + layoutSupportWarning, + profileDetailRect, +} from "./layoutProfile"; + +const HD = { width: 1920, height: 1080 }; +const QHD = { width: 2560, height: 1440 }; +const ULTRAWIDE = { width: 3440, height: 1440 }; + +describe("layoutProfile", () => { + it("detects 16:9 across common resolutions and rejects ultrawide", () => { + expect(isSixteenNine(HD)).toBe(true); + expect(isSixteenNine(QHD)).toBe(true); + expect(isSixteenNine({ width: 3840, height: 2160 })).toBe(true); + expect(isSixteenNine(ULTRAWIDE)).toBe(false); + expect(isSixteenNine({ width: 1920, height: 1200 })).toBe(false); // 16:10 + }); + + it("labels the aspect ratio", () => { + expect(aspectRatioLabel(HD)).toBe("1.78:1"); + expect(aspectRatioLabel({ width: 0, height: 0 })).toBe("unknown"); + }); + + it("warns only for non-16:9 resolutions", () => { + expect(layoutSupportWarning(HD)).toBe(""); + expect(layoutSupportWarning(QHD)).toBe(""); + expect(layoutSupportWarning(ULTRAWIDE)).toContain("nicht 16:9"); + expect(layoutSupportWarning({ width: 0, height: 0 })).toBe(""); + }); + + it("keeps the detail rect inside the image and on the right half", () => { + const rect = profileDetailRect(QHD); + expect(rect.x).toBeGreaterThanOrEqual(QHD.width * 0.45); + expect(rect.x + rect.width).toBeLessThanOrEqual(QHD.width); + expect(rect.y + rect.height).toBeLessThanOrEqual(QHD.height); + }); + + it("produces the four artifact crops in top-to-bottom order, all clamped", () => { + const detail = profileDetailRect(QHD); + const crops = detailCropRects(detail, QHD); + expect(crops.map((crop) => crop.id)).toEqual([ + "artifact-title", + "artifact-main-stat", + "artifact-substats", + "artifact-footer", + ]); + let previousY = -1; + for (const crop of crops) { + expect(crop.rect.x).toBeGreaterThanOrEqual(0); + expect(crop.rect.y).toBeGreaterThan(previousY); + expect(crop.rect.x + crop.rect.width).toBeLessThanOrEqual(QHD.width); + expect(crop.rect.y + crop.rect.height).toBeLessThanOrEqual(QHD.height); + previousY = crop.rect.y; + } + }); + + it("places the inventory count crop inside the inventory panel", () => { + const detail = profileDetailRect(QHD); + const inv = inventoryRect(QHD, detail); + const count = inventoryCountCropRect(inv, QHD); + expect(count.x).toBeGreaterThanOrEqual(inv.x); + expect(count.x + count.width).toBeLessThanOrEqual(QHD.width); + }); + + it("builds a 5-column inventory grid on the left", () => { + const detail = profileDetailRect(QHD); + const grid = inventoryGrid(QHD, detail); + expect(grid.cols).toBe(5); + expect(grid.source).toBe("detected"); + expect(grid.centers.length).toBeGreaterThanOrEqual(10); + expect(grid.centers.every((center) => center.x < detail.x)).toBe(true); + }); + + it("reports a missing grid when the inventory panel is too small", () => { + const tiny = { width: 320, height: 180 }; + const grid = inventoryGrid(tiny, profileDetailRect(tiny)); + expect(grid.source).toBe("missing"); + expect(grid.centers).toHaveLength(0); + }); +}); diff --git a/src/lib/layoutProfile.ts b/src/lib/layoutProfile.ts new file mode 100644 index 0000000..9244420 --- /dev/null +++ b/src/lib/layoutProfile.ts @@ -0,0 +1,205 @@ +// Resolution-anchored layout geometry for the artifact inventory screen +// (ADR-009). Inventory Kamera's proven approach is to require borderless 16:9 and +// derive crop/grid coordinates from the client rectangle instead of detecting the +// panel by colour each frame. This module is the single, pure, unit-tested source +// of that geometry; electron/main.ts consumes it for cropping and keeps a +// colour-based detail-rect detector only as a fallback for off-profile setups. +// +// NOTE: the per-field detail crop fractions below are the current working values. +// True IK-style fixed coordinates need calibration against a reference 16:9 +// screenshot; the structure here is what those calibrated numbers slot into. + +export interface LayoutRect { + x: number; + y: number; + width: number; + height: number; +} + +export interface CropTemplateRect { + id: string; + label: string; + rect: LayoutRect; +} + +export interface InventoryGridLayout { + centers: Array<{ x: number; y: number; row: number; col: number }>; + rows: number; + cols: number; + confidence: number; + source: "detected" | "missing"; +} + +const SIXTEEN_NINE = 16 / 9; + +export function aspectRatio(size: { width: number; height: number }): number { + if (!size.height) return 0; + return size.width / size.height; +} + +export function aspectRatioLabel(size: { width: number; height: number }): string { + const ratio = aspectRatio(size); + if (ratio === 0) return "unknown"; + return `${ratio.toFixed(2)}:1`; +} + +// Genshin's UI is authored for 16:9; other aspect ratios letterbox or reflow and +// the anchored crops no longer line up. Allow a small tolerance for rounding. +export function isSixteenNine(size: { width: number; height: number }, tolerance = 0.02): boolean { + const ratio = aspectRatio(size); + if (ratio === 0) return false; + return Math.abs(ratio - SIXTEEN_NINE) <= SIXTEEN_NINE * tolerance; +} + +// Empty when the client is a supported 16:9; otherwise a warning explaining that +// the anchored crops are unreliable off-profile (ADR-009: non-16:9 is explicitly +// unsupported for the auto scanner). +export function layoutSupportWarning(size: { width: number; height: number }): string { + if (size.width <= 0 || size.height <= 0) return ""; + if (isSixteenNine(size)) return ""; + return `Aufloesung ${size.width}x${size.height} ist nicht 16:9 (${aspectRatioLabel(size)}). Der Auto-Scan ist auf 16:9 im randlosen Fenstermodus ausgelegt; die Erkennung kann daneben liegen.`; +} + +export function clampRect(rect: LayoutRect, imageSize: { width: number; height: number }): LayoutRect { + const x = Math.max(0, Math.min(imageSize.width - 1, rect.x)); + const y = Math.max(0, Math.min(imageSize.height - 1, rect.y)); + const maxWidth = Math.max(1, imageSize.width - x); + const maxHeight = Math.max(1, imageSize.height - y); + return { + x, + y, + width: Math.max(1, Math.min(maxWidth, rect.width)), + height: Math.max(1, Math.min(maxHeight, rect.height)), + }; +} + +// Anchored guess for the artifact detail panel on the right of the screen. Used +// as the primary rect for a clean 16:9 client and as the fallback when colour +// detection cannot find the panel. +export function profileDetailRect(imageSize: { width: number; height: number }): LayoutRect { + const { width, height } = imageSize; + if (width <= 0 || height <= 0) return { x: 0, y: 0, width: Math.max(1, width), height: Math.max(1, height) }; + return clampRect( + { + x: Math.round(width * 0.5), + y: Math.round(height * 0.08), + width: Math.round(width * 0.46), + height: Math.round(height * 0.74), + }, + imageSize, + ); +} + +// The four OCR crops inside the detail panel, as fractions of the detail rect. +export function detailCropRects(detailRect: LayoutRect, imageSize: { width: number; height: number }): CropTemplateRect[] { + const templates: CropTemplateRect[] = [ + { + id: "artifact-title", + label: "Artifact title", + rect: { + x: Math.round(detailRect.x + detailRect.width * 0.055), + y: Math.round(detailRect.y + detailRect.height * 0.05), + width: Math.round(detailRect.width * 0.82), + height: Math.round(detailRect.height * 0.16), + }, + }, + { + id: "artifact-main-stat", + label: "Main stat", + rect: { + x: Math.round(detailRect.x + detailRect.width * 0.055), + y: Math.round(detailRect.y + detailRect.height * 0.2), + width: Math.round(detailRect.width * 0.82), + height: Math.round(detailRect.height * 0.18), + }, + }, + { + id: "artifact-substats", + label: "Substats", + rect: { + x: Math.round(detailRect.x + detailRect.width * 0.055), + y: Math.round(detailRect.y + detailRect.height * 0.41), + width: Math.round(detailRect.width * 0.82), + height: Math.round(detailRect.height * 0.25), + }, + }, + { + id: "artifact-footer", + label: "Footer", + rect: { + x: Math.round(detailRect.x + detailRect.width * 0.055), + y: Math.round(detailRect.y + detailRect.height * 0.78), + width: Math.round(detailRect.width * 0.82), + height: Math.round(detailRect.height * 0.16), + }, + }, + ]; + + return templates.map((template) => ({ ...template, rect: clampRect(template.rect, imageSize) })); +} + +export function inventoryCountCropRect(inventoryRect: LayoutRect, imageSize: { width: number; height: number }): LayoutRect { + return clampRect( + { + x: Math.round(inventoryRect.x + inventoryRect.width * 0.62), + y: Math.round(inventoryRect.y + inventoryRect.height * 0.02), + width: Math.round(inventoryRect.width * 0.34), + height: Math.round(inventoryRect.height * 0.09), + }, + imageSize, + ); +} + +export function inventoryRect(imageSize: { width: number; height: number }, detailRect: LayoutRect): LayoutRect { + const { width, height } = imageSize; + const preferredWidth = Math.max(140, Math.round(width * 0.48)); + const x = Math.round(width * 0.03); + const y = Math.round(detailRect.y + detailRect.height * 0.09); + const availableWidth = Math.max(100, detailRect.x - Math.round(width * 0.04)); + const panelWidth = Math.max(100, Math.min(preferredWidth, availableWidth)); + const safeWidth = panelWidth > width * 0.85 ? Math.round(width * 0.55) : panelWidth; + return clampRect( + { + x, + y, + width: Math.min(safeWidth, Math.max(width - x - Math.round(width * 0.02), 100)), + height: Math.max(140, Math.round(height * 0.7)), + }, + imageSize, + ); +} + +export function inventoryGrid(imageSize: { width: number; height: number }, detailRect: LayoutRect): InventoryGridLayout { + const rect = inventoryRect(imageSize, detailRect); + const cols = 5; + if (rect.width < 160 || rect.height < 140) { + return { centers: [], rows: 0, cols: 0, confidence: 0, source: "missing" }; + } + + const cellWidth = Math.max(56, Math.round(rect.width / cols)); + const stepX = Math.round(cellWidth * 0.96); + const stepY = Math.round(cellWidth * 1.03); + const visibleRows = Math.max(2, Math.min(6, Math.round(rect.height / Math.max(stepY, 1)))); + + const startX = rect.x + Math.max(6, Math.round(stepX * 0.45)); + const startY = rect.y + Math.max(6, Math.round(stepY * 0.45)); + const centers: InventoryGridLayout["centers"] = []; + for (let row = 0; row < visibleRows; row++) { + for (let col = 0; col < cols; col++) { + const x = startX + col * stepX; + const y = startY + row * stepY; + if (x < imageSize.width && y < imageSize.height) { + centers.push({ x, y, row, col }); + } + } + } + + const trimmed = centers.filter((center) => center.x > 0 && center.y > 0); + return { + centers: trimmed, + rows: visibleRows, + cols, + confidence: trimmed.length >= cols * 2 ? 76 : trimmed.length >= cols ? 58 : 36, + source: "detected", + }; +} diff --git a/src/lib/ocrPreprocess.test.ts b/src/lib/ocrPreprocess.test.ts new file mode 100644 index 0000000..eaa97f2 --- /dev/null +++ b/src/lib/ocrPreprocess.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { binarizeForOcr, computeLuminanceHistogram, otsuThreshold, type Bitmap } from "./ocrPreprocess"; + +// Build a BGRA bitmap from a grid of [b,g,r] pixels. +function bitmapFrom(pixels: Array<[number, number, number]>, width: number, height: number): Bitmap { + const data = Buffer.alloc(width * height * 4); + pixels.forEach(([b, g, r], index) => { + data[index * 4] = b; + data[index * 4 + 1] = g; + data[index * 4 + 2] = r; + data[index * 4 + 3] = 255; + }); + return { data, width, height }; +} + +describe("ocrPreprocess", () => { + it("computes a luminance histogram over all pixels", () => { + const bitmap = bitmapFrom([ + [0, 0, 0], + [255, 255, 255], + [0, 0, 0], + [255, 255, 255], + ], 2, 2); + const histogram = computeLuminanceHistogram(bitmap); + expect(histogram[0]).toBe(2); + expect(histogram[255]).toBe(2); + expect(histogram.reduce((sum, count) => sum + count, 0)).toBe(4); + }); + + it("otsu splits a clean bimodal image between the two peaks", () => { + const histogram = new Array(256).fill(0); + histogram[20] = 50; + histogram[220] = 50; + const threshold = otsuThreshold(histogram); + expect(threshold).toBeGreaterThanOrEqual(20); + expect(threshold).toBeLessThan(220); + }); + + it("otsu is safe on an empty histogram", () => { + expect(otsuThreshold(new Array(256).fill(0))).toBe(127); + }); + + it("inverts bright foreground to black-on-white by default", () => { + // Bright text pixel + dark background pixel. + const bitmap = bitmapFrom([ + [255, 255, 255], // bright -> should become black + [0, 0, 0], // dark -> should become white + ], 2, 1); + const out = binarizeForOcr(bitmap, { threshold: 128 }); + expect([out.data[0], out.data[1], out.data[2]]).toEqual([0, 0, 0]); + expect([out.data[4], out.data[5], out.data[6]]).toEqual([255, 255, 255]); + expect(out.data[3]).toBe(255); + }); + + it("keeps bright foreground white when inversion is disabled", () => { + const bitmap = bitmapFrom([ + [255, 255, 255], + [0, 0, 0], + ], 2, 1); + const out = binarizeForOcr(bitmap, { threshold: 128, invertBrightForeground: false }); + expect(out.data[0]).toBe(255); + expect(out.data[4]).toBe(0); + }); + + it("preserves dimensions and always emits opaque pixels", () => { + const bitmap = bitmapFrom(Array.from({ length: 9 }, () => [100, 100, 100] as [number, number, number]), 3, 3); + const out = binarizeForOcr(bitmap); + expect(out.width).toBe(3); + expect(out.height).toBe(3); + for (let pixel = 0; pixel < 9; pixel++) { + expect(out.data[pixel * 4 + 3]).toBe(255); + } + }); +}); diff --git a/src/lib/ocrPreprocess.ts b/src/lib/ocrPreprocess.ts new file mode 100644 index 0000000..18c08c3 --- /dev/null +++ b/src/lib/ocrPreprocess.ts @@ -0,0 +1,101 @@ +// OCR preprocessing for artifact crops (ADR-009). Tesseract reads a clean, high +// contrast, dark-text-on-light image far more reliably than Genshin's native +// bright-text-on-dark UI. This binarizes a crop with Otsu thresholding and (by +// default) inverts, because artifact text is the bright foreground. +// +// Works on a raw BGRA bitmap (Electron NativeImage.getBitmap() layout on +// Windows). Kept pure and channel-order-agnostic for luminance so it is unit +// testable without Electron. Upscaling is done separately via NativeImage.resize +// before this runs - interpolated upscaling of small crops is a big Tesseract win +// and NativeImage does it better than hand-rolled JS. + +export interface Bitmap { + data: Uint8Array | Buffer; + width: number; + height: number; +} + +export interface BinarizeOptions { + /** Artifact text is the bright foreground, so invert to dark-on-light. */ + invertBrightForeground?: boolean; + /** Override Otsu with a fixed 0-255 luminance threshold. */ + threshold?: number; +} + +const BYTES_PER_PIXEL = 4; + +// Rec. 601 luma. Channel order does not matter for a weighted sum as long as we +// read the same three bytes; BGRA and RGBA give the same luminance here because +// we weight by position-independent coefficients applied to the actual R/G/B. +function luminanceAt(data: Uint8Array | Buffer, index: number): number { + // NativeImage on Windows is BGRA: byte0=B, byte1=G, byte2=R. + const b = data[index]; + const g = data[index + 1]; + const r = data[index + 2]; + return 0.299 * r + 0.587 * g + 0.114 * b; +} + +export function computeLuminanceHistogram(bitmap: Bitmap): number[] { + const histogram = new Array(256).fill(0); + const { data, width, height } = bitmap; + const pixels = width * height; + for (let pixel = 0; pixel < pixels; pixel++) { + const value = Math.round(luminanceAt(data, pixel * BYTES_PER_PIXEL)); + histogram[Math.max(0, Math.min(255, value))]++; + } + return histogram; +} + +// Otsu's method: pick the threshold that maximizes between-class variance. +export function otsuThreshold(histogram: readonly number[]): number { + const total = histogram.reduce((sum, count) => sum + count, 0); + if (total === 0) return 127; + + let sumAll = 0; + for (let level = 0; level < 256; level++) sumAll += level * histogram[level]; + + let sumBackground = 0; + let weightBackground = 0; + let maxVariance = -1; + let threshold = 127; + + for (let level = 0; level < 256; level++) { + weightBackground += histogram[level]; + if (weightBackground === 0) continue; + const weightForeground = total - weightBackground; + if (weightForeground === 0) break; + + sumBackground += level * histogram[level]; + const meanBackground = sumBackground / weightBackground; + const meanForeground = (sumAll - sumBackground) / weightForeground; + const betweenVariance = weightBackground * weightForeground * (meanBackground - meanForeground) ** 2; + + if (betweenVariance > maxVariance) { + maxVariance = betweenVariance; + threshold = level; + } + } + + return threshold; +} + +export function binarizeForOcr(bitmap: Bitmap, options: BinarizeOptions = {}): Bitmap { + const { data, width, height } = bitmap; + const invert = options.invertBrightForeground ?? true; + const threshold = options.threshold ?? otsuThreshold(computeLuminanceHistogram(bitmap)); + + const output = Buffer.alloc(width * height * BYTES_PER_PIXEL); + const pixels = width * height; + for (let pixel = 0; pixel < pixels; pixel++) { + const index = pixel * BYTES_PER_PIXEL; + const isBright = luminanceAt(data, index) > threshold; + // Bright foreground text -> black; dark background -> white (inverted). + const value = invert ? (isBright ? 0 : 255) : (isBright ? 255 : 0); + output[index] = value; + output[index + 1] = value; + output[index + 2] = value; + output[index + 3] = 255; + } + + return { data: output, width, height }; +} diff --git a/src/types/global.d.ts b/src/types/global.d.ts index 0c8437e..7252905 100644 --- a/src/types/global.d.ts +++ b/src/types/global.d.ts @@ -56,6 +56,11 @@ export interface CaptureResult { source: "ocr" | "missing"; text: string; }; + layout?: { + aspect: string; + isSixteenNine: boolean; + warning: string; + }; } export interface WindowBounds {