diff --git a/server/app/image_pipeline.py b/server/app/image_pipeline.py index f758635..b991511 100644 --- a/server/app/image_pipeline.py +++ b/server/app/image_pipeline.py @@ -5,14 +5,13 @@ from __future__ import annotations import io import math -import numpy as np from PIL import Image, ImageDraw, ImageEnhance, ImageFont, ImageOps EPD_WIDTH = 800 EPD_HEIGHT = 480 # PIL's TrueType rendering antialiases by default (graduated gray edge -# pixels). Those survive straight into _quantize's error-diffusion +# pixels). Those survive straight into _quantize's Floyd-Steinberg # dithering, which -- confirmed visually -- turns them into scattered # colored speckles along every glyph edge once forced onto the panel's 6 # colors, since a mid-gray input has no close palette match and the @@ -149,24 +148,20 @@ def logical_to_native(x: float, y: float, orientation: str) -> tuple[int, int]: return int(logical_h - 1 - y), int(x) return int(x), int(y) -# Measured sRGB appearance of each of the panel's 6 ink colors on an -# actual Spectra 6 panel -- sourced from epdoptimize's "spectra6" palette -# (github.com/paperlesspaper/epdoptimize, src/dither/data/default-palettes -# .json), not our own calibration, but a much better starting point than a -# guess: e-ink ink never reaches full sRGB saturation/contrast, so this is -# uniformly darker and more muted than the naive (0,0,0)/(255,255,255)/pure -# hues this used to be. This is the fallback for any frame that hasn't -# tuned its own (Frame.palette_rgb, set from a frame's Configuration tab -# -- "Advanced configuration" -- once you can compare a rendered test -# image against the real panel; different panel units can vary enough to -# be worth calibrating per frame). +# Approximate sRGB for each of the panel's 6 ink colors -- reasonable +# placeholders, not measured values (Waveshare doesn't publish exact +# color primaries for this panel). This is the fallback for any frame +# that hasn't tuned its own (Frame.palette_rgb, set from a frame's +# Configuration tab -- "Advanced configuration" -- once you can compare +# a rendered test image against the real panel; different panel units +# can vary enough to be worth calibrating per frame). DEFAULT_PALETTE_RGB = [ - (31, 34, 38), # BLACK - (185, 199, 201), # WHITE - (193, 187, 30), # YELLOW - (98, 32, 30), # RED - (35, 63, 142), # BLUE - (53, 86, 58), # GREEN + (0, 0, 0), # BLACK + (255, 255, 255), # WHITE + (255, 219, 0), # YELLOW + (207, 0, 15), # RED + (0, 39, 133), # BLUE + (0, 133, 55), # GREEN ] PALETTE_LABELS = ["Black", "White", "Yellow", "Red", "Blue", "Green"] @@ -227,137 +222,10 @@ def hex_to_rgb(hex_str: str) -> tuple[int, int, int] | None: return None -def _rgb_to_oklab(rgb: "np.ndarray") -> "np.ndarray": - """(...,3) uint8/float sRGB -> (...,3) float32 OKLab (Bjorn Ottosson's - formulation, https://bottosson.github.io/posts/oklab/). Used instead - of raw RGB distance for palette matching/error diffusion below -- - Euclidean distance in OKLab tracks perceived color difference far - better than in RGB, which matters a lot once the "colors" being - matched against are a 6-entry palette this coarse.""" - linear = (rgb.astype(np.float32) / 255.0) - linear = np.where(linear <= 0.04045, linear / 12.92, ((linear + 0.055) / 1.055) ** 2.4) - r, g, b = linear[..., 0], linear[..., 1], linear[..., 2] - - l = 0.4122214708 * r + 0.5363325363 * g + 0.0514459929 * b - m = 0.2119034982 * r + 0.6806995451 * g + 0.1073969566 * b - s = 0.0883024619 * r + 0.2817188376 * g + 0.6299787005 * b - l_, m_, s_ = np.cbrt(l), np.cbrt(m), np.cbrt(s) - - L = 0.2104542553 * l_ + 0.7936177850 * m_ - 0.0040720468 * s_ - a = 1.9779984951 * l_ - 2.4285922050 * m_ + 0.4505937099 * s_ - b2 = 0.0259040371 * l_ + 0.7827717662 * m_ - 0.8086757660 * s_ - return np.stack([L, a, b2], axis=-1) - - -# Lightness is weighted down relative to a/b when *choosing* the nearest -# palette entry (established color-difference formulas -- CIE94, CMC -- -# do the same, on the general principle that a lightness mismatch reads -# as less objectionable than a hue mismatch). Not optional polish: this -# palette's ink colors are far darker/lighter than their sRGB namesakes -# (e.g. "red" ink is a dark #62201E, "yellow" ink is a bright #C1BB1E), -# so unweighted OKLab distance lets that lightness gap dominate and pure -# saturated red (high L) ends up nearer "yellow" (L=0.77) than "red" -# (L=0.35) even though red is unambiguously closer in hue/chroma (a/b) -- -# confirmed both analytically and by DEFAULT_PALETTE_RGB's own test -# coverage (test_render_size_invariants.py's pure-red/pure-blue check). -_LIGHTNESS_MATCH_WEIGHT = 0.5 - - -def _nearest_palette_indices(oklab_pixels: "np.ndarray", palette_oklab: "np.ndarray") -> "np.ndarray": - """(H,W,3) OKLab pixels, (K,3) OKLab palette -> (H,W) index array, no - error diffusion -- the "flat"/undithered quantization, vectorized - (K is always 6, so brute-force all-pairs distance is cheap and this - stays a single numpy call rather than a per-pixel Python loop).""" - diffs2 = (oklab_pixels[:, :, None, :] - palette_oklab[None, None, :, :]) ** 2 - weights = np.array([_LIGHTNESS_MATCH_WEIGHT, 1.0, 1.0], dtype=np.float32) - dist2 = (diffs2 * weights).sum(axis=-1) - return np.argmin(dist2, axis=2) - - -def _bayer_matrix(n: int) -> "np.ndarray": - """Recursive construction of the standard n x n (n a power of 2) - Bayer ordered-dithering threshold matrix, values 0..n*n-1, each used - exactly once -- the classic recursive doubling - (https://en.wikipedia.org/wiki/Ordered_dithering).""" - if n == 1: - return np.zeros((1, 1)) - smaller = _bayer_matrix(n // 2) - return np.block([ - [4 * smaller, 4 * smaller + 2], - [4 * smaller + 3, 4 * smaller + 1], - ]) - - -# Normalized to [0, 1): a deterministic per-pixel threshold tiled across -# the image, used (like classic ordered/Bayer dithering) to decide, for -# each pixel, whether it plots as its nearest or second-nearest palette -# color -- see _ordered_dither_oklab. -_BAYER_8 = (_bayer_matrix(8) + 0.5) / 64.0 - - -def _ordered_dither_oklab(oklab_pixels: "np.ndarray", palette_oklab: "np.ndarray") -> "np.ndarray": - """(H,W,3) OKLab pixels, (K,3) OKLab palette -> (H,W) uint8 index - array, ordered (Bayer matrix) dithering -- picked over error - diffusion (Floyd-Steinberg/Atkinson/etc.) specifically because it's - fully vectorizable: no pixel-to-pixel dependency to chain through a - Python loop, just a fixed number of numpy calls over the whole - image. A straight per-pixel error-diffusion loop in Python was - measured at ~1s for a full 800x480 panel -- see - test_widgets_render_concurrently's latency budget (the whole reason - widgets render concurrently in the first place, see git history) -- - which this avoids entirely. - - Finds each pixel's true nearest and second-nearest palette color and - mixes between exactly those two, using the Bayer threshold as the - per-pixel coin flip -- the standard generalization of ordered - dithering to a palette whose entries aren't evenly spaced (unlike, - say, dithering 0-255 gray down to a handful of even steps). The - mixing fraction is the pixel's projection onto the segment from its - nearest color to its second-nearest, NOT distance-to-nearest over - total distance (d0/(d0+d1)) -- an earlier version used that ratio - and it's wrong whenever the second-nearest color is simply far away - in an unrelated direction rather than genuinely "on the other side" - of the pixel: d1 being large made the ratio look small-mixing-needed - only when d0 was *also* comparably large, so a pixel sitting almost - exactly on its nearest color still got a large fraction of an - unrelated second color -- confirmed visually as entire regions - (e.g. a pale sky, clearly nearest White) rendering as flat blocks of - a wrong, unrelated color (Yellow) instead of White. Projection onto - the actual nearest-neighbor segment doesn't have that failure mode: - a pixel essentially at c0 projects to ~0 regardless of where c1 is.""" - weights = np.array([_LIGHTNESS_MATCH_WEIGHT, 1.0, 1.0], dtype=np.float32) - scale = np.sqrt(weights) - pixels_w = oklab_pixels * scale - palette_w = palette_oklab * scale - - dist2 = ((pixels_w[:, :, None, :] - palette_w[None, None, :, :]) ** 2).sum(axis=-1) # (H, W, K) - order = np.argsort(dist2, axis=-1) - idx0, idx1 = order[..., 0], order[..., 1] - - c0 = palette_w[idx0] # (H, W, 3) - c1 = palette_w[idx1] # (H, W, 3) - segment = c1 - c0 - to_pixel = pixels_w - c0 - segment_len2 = (segment * segment).sum(axis=-1) - t = np.divide((to_pixel * segment).sum(axis=-1), segment_len2, - out=np.zeros_like(segment_len2), where=segment_len2 > 1e-12) - t = np.clip(t, 0.0, 1.0) - - h, w, _ = oklab_pixels.shape - threshold = np.tile(_BAYER_8, (h // 8 + 1, w // 8 + 1))[:h, :w] - use_second = threshold < t - return np.where(use_second, idx1, idx0).astype(np.uint8) - - -def _index_array_to_p_image(idx_array: "np.ndarray", palette_rgb: list) -> Image.Image: - """(H,W) palette-index array -> a PIL "P"-mode image carrying - `palette_rgb` as its palette, so downstream code (as_png's - .convert("RGB"), _transpose_and_pack's pixels[x, y] index lookups) - behaves exactly as it did with PIL's own quantize().""" - img = Image.fromarray(idx_array, mode="P") - padded = list(palette_rgb) + [(0, 0, 0)] * (256 - len(palette_rgb)) - img.putpalette([channel for rgb in padded for channel in rgb]) - return img +def _build_palette_image(palette_rgb: list) -> Image.Image: + pal_img = Image.new("P", (1, 1)) + pal_img.putpalette([channel for rgb in palette_rgb for channel in rgb]) + return pal_img def _plain_center_crop_box( @@ -535,39 +403,21 @@ def _enhance(img: Image.Image, color_boost: float, contrast_boost: float) -> Ima def _quantize(img: Image.Image, palette_rgb: list | None, dither_strength: float) -> Image.Image: """RGB -> palette-quantized P-mode image, same size/orientation as - `img` (no rotation here). Matches against the palette in OKLab space - (perceptual distance, not raw RGB -- see _rgb_to_oklab/ - _nearest_palette_indices) and, when dithering, jitters that match - with a Bayer ordered-dither pattern rather than Floyd-Steinberg error - diffusion -- see _ordered_dither_oklab for why (short version: error - diffusion is inherently a serial per-pixel loop, and doing that in - Python for a full 800x480 panel blew well past this project's - render-latency budget). dither_strength blends `img` toward its own - flat (undithered) quantization before dithering the blend: at 0 the - blend IS the flat quantization (nothing left for the jitter to push - across a color boundary, so no dithering texture at all); at 1 it's - `img` unchanged (full-strength dithering, this project's original - always-on behavior); values between give a smooth continuum of - dithering intensity rather than an on/off toggle.""" - palette_rgb = palette_rgb or DEFAULT_PALETTE_RGB - palette_oklab = _rgb_to_oklab(np.asarray(palette_rgb, dtype=np.float32)) - - rgb_array = np.asarray(img.convert("RGB")) - oklab_pixels = _rgb_to_oklab(rgb_array) - + `img` (no rotation here). dither_strength blends `img` toward its own + flat (undithered) quantization before running Floyd-Steinberg on the + blend: at 0 there's zero quantization error left to diffuse (so the + result IS the flat quantization, no dithering texture at all); at 1 + it's `img` unchanged (full-strength dithering, this project's + original always-on behavior); values between give a smooth continuum + of dithering intensity rather than an on/off toggle.""" + palette_image = _build_palette_image(palette_rgb or DEFAULT_PALETTE_RGB) + if dither_strength >= 1.0: + return img.quantize(palette=palette_image, dither=Image.Dither.FLOYDSTEINBERG) if dither_strength <= 0.0: - flat_idx = _nearest_palette_indices(oklab_pixels, palette_oklab) - return _index_array_to_p_image(flat_idx.astype(np.uint8), palette_rgb) - - if dither_strength < 1.0: - flat_idx = _nearest_palette_indices(oklab_pixels, palette_oklab) - palette_arr = np.asarray(palette_rgb, dtype=np.uint8) - flat_rgb = Image.fromarray(palette_arr[flat_idx], mode="RGB") - blended = Image.blend(flat_rgb, img.convert("RGB"), dither_strength) - oklab_pixels = _rgb_to_oklab(np.asarray(blended)) - - dithered_idx = _ordered_dither_oklab(oklab_pixels, palette_oklab) - return _index_array_to_p_image(dithered_idx, palette_rgb) + return img.quantize(palette=palette_image, dither=Image.Dither.NONE) + flat = img.quantize(palette=palette_image, dither=Image.Dither.NONE).convert("RGB") + blended = Image.blend(flat, img, dither_strength) + return blended.quantize(palette=palette_image, dither=Image.Dither.FLOYDSTEINBERG) def _transpose_and_pack(quantized: Image.Image, orientation: str) -> bytes: diff --git a/server/requirements.txt b/server/requirements.txt index 5d67f88..0b675fb 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -3,7 +3,6 @@ starlette==0.41.3 uvicorn[standard]==0.34.0 httpx==0.28.1 pillow==12.3.0 -numpy==2.5.1 python-multipart==0.0.20 jinja2==3.1.5 sqlalchemy==2.0.51 diff --git a/server/tests/test_widget_border.py b/server/tests/test_widget_border.py index bef608f..9d158de 100644 --- a/server/tests/test_widget_border.py +++ b/server/tests/test_widget_border.py @@ -135,7 +135,7 @@ def test_solid_border_draws_the_configured_palette_color(client, db_session): assert resp.status_code == 200, resp.text img = _preview_pixels(client) - assert img.getpixel((0, 0)) == (98, 32, 30) # DEFAULT_PALETTE_RGB[3], top-left corner of the stroke + assert img.getpixel((0, 0)) == (207, 0, 15) # DEFAULT_PALETTE_RGB[3], top-left corner of the stroke def test_no_border_leaves_the_edge_unmarked(client, db_session): @@ -146,4 +146,4 @@ def test_no_border_leaves_the_edge_unmarked(client, db_session): render path already painting it that color.""" client.post("/setup", data={"username": "alice", "password": "hunter22"}) img = _preview_pixels(client) - assert img.getpixel((0, 0)) != (98, 32, 30) + assert img.getpixel((0, 0)) != (207, 0, 15)