bao/genshin-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ocr): resolution-anchored layout module + crop preprocessing

Browse Source 
Implements ADR-009 (structure + preprocessing; exact IK fixed coordinates still
need calibration against a reference 16:9 screenshot).

- src/lib/layoutProfile.ts: pure, unit-tested geometry for the artifact screen -
  detail rect, the four detail crops, inventory rect/count crop, 5-col grid,
  16:9 detection, aspect label, and an off-16:9 support warning. Single source of
  truth; electron/main.ts now delegates all crop/grid geometry to it and keeps
  colour detection only as the detail-rect fallback.
- src/lib/ocrPreprocess.ts: pure, unit-tested Otsu binarization with inversion
  (artifact text is the bright foreground) over a BGRA bitmap.
- main.ts: OCR now reads an upscaled + binarized copy of each crop; the original
  crop is retained for the diagnostics UI. CaptureResult carries layout info
  { aspect, isSixteenNine, warning }.

NOTE: image preprocessing changes the OCR input and cannot be validated by the
text-level eval harness; it needs a live Genshin 16:9 capture to confirm/tune
(threshold, invert, upscale factor). 88 tests + build green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
AzuTear
2026-07-05 21:42:16 +02:00
parent c7138b541d
commit 2c6c1a8b31
6 changed files with 523 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
electron/main.ts
+51 -131
View File
@@ -22,6 +22,17 @@ import type {
ReviewSamplesRepositoryPort,
ScannerLearningRepositoryPort,
} from "./repositories/index.js";
import {
aspectRatioLabel,
detailCropRects,
inventoryCountCropRect,
inventoryGrid as layoutInventoryGrid,
inventoryRect as layoutInventoryRect,
isSixteenNine,
layoutSupportWarning,
profileDetailRect,
} from "../src/lib/layoutProfile.js";
import { binarizeForOcr } from "../src/lib/ocrPreprocess.js";
// Chromium's renderer sandbox can refuse to fully initialize (or silently
// crash the GPU/renderer process) when the hosting process runs with a full
@@ -746,84 +757,48 @@ function imageCropDataUrl(sourceImage: NativeImage, rect: Electron.Rectangle, im
return sourceImage.crop(safeRect).toDataURL();
}
// Preprocessed copy of a crop for OCR (ADR-009): upscale for more pixels, then
// grayscale + Otsu-binarize with inversion (artifact text is the bright
// foreground). The original crop is kept separately for the diagnostics UI.
function preprocessedCropDataUrl(sourceImage: NativeImage, rect: Electron.Rectangle, imageSize: { width: number; height: number }) {
const safeRect = clampCaptureRect(rect, imageSize);
const upscaled = sourceImage.crop(safeRect).resize({ width: Math.max(1, safeRect.width * 2), quality: "best" });
const size = upscaled.getSize();
if (!size.width || !size.height) return upscaled.toDataURL();
const binarized = binarizeForOcr({ data: upscaled.getBitmap(), width: size.width, height: size.height });
return nativeImage
.createFromBitmap(Buffer.from(binarized.data), { width: binarized.width, height: binarized.height })
.toDataURL();
}
function createCrops(
sourceImage: NativeImage,
imageSize: { width: number; height: number },
detailRect: Electron.Rectangle,
inventoryRect: Electron.Rectangle,
) {
const templates: CropTemplate[] = [
{
id: "artifact-title",
label: "Artifact title",
rect: {
x: Math.round(detailRect.x + detailRect.width * 0.055),
y: Math.round(detailRect.y + detailRect.height * 0.05),
width: Math.round(detailRect.width * 0.82),
height: Math.round(detailRect.height * 0.16),
},
},
{
id: "artifact-main-stat",
label: "Main stat",
rect: {
x: Math.round(detailRect.x + detailRect.width * 0.055),
y: Math.round(detailRect.y + detailRect.height * 0.20),
width: Math.round(detailRect.width * 0.82),
height: Math.round(detailRect.height * 0.18),
},
},
{
id: "artifact-substats",
label: "Substats",
rect: {
x: Math.round(detailRect.x + detailRect.width * 0.055),
y: Math.round(detailRect.y + detailRect.height * 0.41),
width: Math.round(detailRect.width * 0.82),
height: Math.round(detailRect.height * 0.25),
},
},
{
id: "artifact-footer",
label: "Footer",
rect: {
x: Math.round(detailRect.x + detailRect.width * 0.055),
y: Math.round(detailRect.y + detailRect.height * 0.78),
width: Math.round(detailRect.width * 0.82),
height: Math.round(detailRect.height * 0.16),
},
},
];
const templates: CropTemplate[] = detailCropRects(detailRect, imageSize);
if (inventoryRect.width > 120 && inventoryRect.height > 80) {
templates.push({
id: "inventory-count",
label: "Inventory count",
rect: {
x: Math.round(inventoryRect.x + inventoryRect.width * 0.62),
y: Math.round(inventoryRect.y + inventoryRect.height * 0.02),
width: Math.round(inventoryRect.width * 0.34),
height: Math.round(inventoryRect.height * 0.09),
},
rect: inventoryCountCropRect(inventoryRect, imageSize),
});
}
return templates
.map((template) => ({
...template,
rect: clampCaptureRect(template.rect, imageSize),
dataUrl: imageCropDataUrl(sourceImage, template.rect, imageSize),
}))
.filter((crop) => crop.rect.width > 0 && crop.rect.height > 0)
.map((crop) => ({
...crop,
rect: {
x: crop.rect.x,
y: crop.rect.y,
width: crop.rect.width,
height: crop.rect.height,
},
}));
.map((template) => {
const rect = clampCaptureRect(template.rect, imageSize);
return {
id: template.id,
label: template.label,
rect,
dataUrl: imageCropDataUrl(sourceImage, rect, imageSize),
ocrDataUrl: preprocessedCropDataUrl(sourceImage, rect, imageSize),
};
})
.filter((crop) => crop.rect.width > 0 && crop.rect.height > 0);
}
function inferDetailRect(bitmap: Buffer, imageSize: { width: number; height: number }) {
@@ -865,79 +840,17 @@ function inferDetailRect(bitmap: Buffer, imageSize: { width: number; height: num
return clampCaptureRect({ x: left, y: top, width: widthGuess, height: heightGuess }, imageSize);
}
if (width > 0 && height > 0) {
return clampCaptureRect(
{
x: Math.round(width * 0.50),
y: Math.round(height * 0.08),
width: Math.round(width * 0.46),
height: Math.round(height * 0.74),
},
imageSize,
);
}
return { x: 0, y: 0, width, height };
// Colour detection found nothing usable; fall back to the resolution-anchored
// profile rect (single source of truth in layoutProfile).
return profileDetailRect(imageSize);
}
function inferInventoryRect(imageSize: { width: number; height: number }, detailRect: Electron.Rectangle) {
const { width, height } = imageSize;
const preferredWidth = Math.max(140, Math.round(width * 0.48));
const x = Math.round(width * 0.03);
const y = Math.round(detailRect.y + detailRect.height * 0.09);
const availableWidth = Math.max(100, detailRect.x - Math.round(width * 0.04));
const panelWidth = Math.max(100, Math.min(preferredWidth, availableWidth));
const safeWidth = panelWidth > width * 0.85 ? Math.round(width * 0.55) : panelWidth;
return clampCaptureRect(
{
x,
y,
width: Math.min(safeWidth, Math.max(width - x - Math.round(width * 0.02), 100)),
height: Math.max(140, Math.round(height * 0.70)),
},
imageSize,
);
return layoutInventoryRect(imageSize, detailRect);
}
function inferInventoryGrid(imageSize: { width: number; height: number }, detailRect: Electron.Rectangle) {
const inventoryRect = inferInventoryRect(imageSize, detailRect);
const cols = 5;
if (inventoryRect.width < 160 || inventoryRect.height < 140) {
return {
centers: [],
rows: 0,
cols: 0,
confidence: 0,
source: "missing" as const,
};
}
const cellWidth = Math.max(56, Math.round(inventoryRect.width / cols));
const stepX = Math.round(cellWidth * 0.96);
const stepY = Math.round(cellWidth * 1.03);
const visibleRows = Math.max(2, Math.min(6, Math.round(inventoryRect.height / Math.max(stepY, 1))));
const startX = inventoryRect.x + Math.max(6, Math.round(stepX * 0.45));
const startY = inventoryRect.y + Math.max(6, Math.round(stepY * 0.45));
const centers = [];
for (let row = 0; row < visibleRows; row++) {
for (let col = 0; col < cols; col++) {
const x = startX + col * stepX;
const y = startY + row * stepY;
if (x < imageSize.width && y < imageSize.height) {
centers.push({ x, y, row, col });
}
}
}
const trimmed = centers.filter((center) => center.x > 0 && center.y > 0);
return {
centers: trimmed,
rows: visibleRows,
cols,
confidence: trimmed.length >= cols * 2 ? 76 : trimmed.length >= cols ? 58 : 36,
source: "detected" as const,
};
return layoutInventoryGrid(imageSize, detailRect);
}
async function buildCaptureResult(
@@ -959,7 +872,9 @@ async function buildCaptureResult(
const croppedPayload = crops.map((crop) => ({
id: crop.id,
label: crop.label,
dataUrl: crop.dataUrl,
// OCR reads the preprocessed (upscaled + binarized) crop; the original is
// kept below for the diagnostics UI.
dataUrl: crop.ocrDataUrl ?? crop.dataUrl,
}));
const recognized = options.skipOcr ? { ocr: [], timedOut: false } : await runOcrOnCropsWithTimeout(croppedPayload);
@@ -990,6 +905,11 @@ async function buildCaptureResult(
})),
inventoryGrid: inferInventoryGrid(size, detailRect),
inventoryCount: count,
layout: {
aspect: aspectRatioLabel(size),
isSixteenNine: isSixteenNine(size),
warning: layoutSupportWarning(size),
},
};
}