Display mode: crop to fill, crop to faces, stretch to fill, shrink to fit

Replaces the smart_crop_faces boolean with a 4-way display_mode select
on each frame's Configuration tab (image_pipeline.DISPLAY_MODES):

- Crop to fill / Crop to faces: the previous False/True behavior,
  unchanged (center-crop trimming excess, optionally shifted to keep
  faces on screen).
- Stretch to fill (new): fills the panel exactly, aspect ratio not
  preserved -- a plain resize, no crop.
- Shrink to fit (new): the whole photo visible, letterboxed with white
  where it doesn't fill the panel.

Named-face overlay label positioning (face_labels.py, the manage menu's
"who's in this photo") now goes through a shared _placement_transform()
in image_pipeline.py instead of duplicating crop-box math, so label
placement stays correct (and in-bounds) under all four modes, not just
the two crop ones -- letterbox/stretch never crop a face out, so labels
just use straight scale+offset math there.

Schema migration v5 adds display_mode, backfills it from the old
boolean (True/False -> crop_faces/crop_fill), and drops the boolean.

Verified against the live-shaped test database: the migration
(existing frames correctly preserved as crop_faces), the config page's
new 4-option select, actual renders under letterbox (confirmed real
white letterbox padding in the packed panel-code bytes) and
stretch_fill, invalid-input fallback, and the standing legacy-device
curl suite.

server/app/face_labels.py

@@ -15,12 +15,7 @@ import io
 
 from PIL import Image, ImageOps
 
-from .image_pipeline import (
-    _face_aware_crop_box,
-    _plain_center_crop_box,
-    logical_render_size,
-    logical_to_native,
-)
+from .image_pipeline import _placement_transform, logical_render_size, logical_to_native
 
 # Small caps, not arbitrary: each label is its own malloc'd overlay
 # buffer on the device (see firmware/main/manage_qr_overlay.c), and the
@@ -31,20 +26,21 @@ MAX_LABELED_FACES = 4
 NAME_MAX_LEN = 10
 
 
-def compute_face_labels(preview_bytes: bytes, faces: list[dict], smart_crop_faces: bool,
+def compute_face_labels(preview_bytes: bytes, faces: list[dict], display_mode: str,
                         orientation: str = "landscape") -> list[dict]:
     """Returns up to MAX_LABELED_FACES [{"name", "x", "y"}], x/y in native
     800x480 panel pixel space at each named face's bottom-center point.
     Faces without an Immich-identified person name are skipped entirely.
     preview_bytes must be the same preview image render_frame() used for
-    the currently-displayed frame, and smart_crop_faces/orientation must
-    match the settings that were active then -- otherwise the crop box and
-    rotation computed here won't match what's actually on screen.
+    the currently-displayed frame, and display_mode/orientation must
+    match the settings that were active then -- otherwise the placement
+    and rotation computed here won't match what's actually on screen.
 
-    The crop math runs in logical (pre-rotation) space, matching
-    render_frame()'s composition step; each anchor is then rotated into
-    native panel coordinates via logical_to_native(), since the firmware
-    draws labels in native space.
+    The placement math runs in logical (pre-rotation) space, matching
+    render_frame()'s composition step (see image_pipeline._placement_transform,
+    shared so the two can't drift apart); each anchor is then rotated
+    into native panel coordinates via logical_to_native(), since the
+    firmware draws labels in native space.
     """
     named = [face for face in faces if (face.get("person") or {}).get("name")]
     if not named:
@@ -53,24 +49,22 @@ def compute_face_labels(preview_bytes: bytes, faces: list[dict], smart_crop_face
     logical_w, logical_h = logical_render_size(orientation)
     fitted = ImageOps.exif_transpose(Image.open(io.BytesIO(preview_bytes)).convert("RGB"))
 
-    if smart_crop_faces and faces:
-        left, top, right, bottom = _face_aware_crop_box(fitted.width, fitted.height, logical_w, logical_h, faces)
-        crop_w, crop_h = right - left, bottom - top
-    else:
-        left, top, crop_w, crop_h = _plain_center_crop_box(fitted.width, fitted.height, logical_w, logical_h)
+    scale_x, scale_y, offset_x, offset_y = _placement_transform(
+        fitted.width, fitted.height, logical_w, logical_h, display_mode, faces
+    )
 
     labels = []
     for face in named[:MAX_LABELED_FACES]:
         face_w = face.get("imageWidth") or fitted.width
         face_h = face.get("imageHeight") or fitted.height
-        scale_x = fitted.width / face_w
-        scale_y = fitted.height / face_h
+        img_scale_x = fitted.width / face_w
+        img_scale_y = fitted.height / face_h
 
-        center_x = (face["boundingBoxX1"] + face["boundingBoxX2"]) / 2 * scale_x
-        bottom_y = face["boundingBoxY2"] * scale_y
+        center_x = (face["boundingBoxX1"] + face["boundingBoxX2"]) / 2 * img_scale_x
+        bottom_y = face["boundingBoxY2"] * img_scale_y
 
-        frame_x = (center_x - left) * (logical_w / crop_w)
-        frame_y = (bottom_y - top) * (logical_h / crop_h)
+        frame_x = center_x * scale_x + offset_x
+        frame_y = bottom_y * scale_y + offset_y
 
         if not (0 <= frame_x <= logical_w and 0 <= frame_y <= logical_h):
             continue  # this face got cropped out of the final frame entirely

server/app/image_pipeline.py

@@ -169,14 +169,58 @@ def _face_aware_crop_box(
     return (int(left), int(top), int(left) + crop_w, int(top) + crop_h)
 
 
+# Display modes: how a photo's aspect ratio gets reconciled with the
+# panel's. "crop_faces" falls back to "crop_fill" behavior when no faces
+# were detected/passed. DEFAULT_DISPLAY_MODE matches this project's old
+# always-on smart_crop_faces=True default.
+DISPLAY_MODES = ["crop_fill", "crop_faces", "stretch_fill", "letterbox"]
+DISPLAY_MODE_LABELS = {
+    "crop_fill": "Crop to fill",
+    "crop_faces": "Crop to faces",
+    "stretch_fill": "Stretch to fill",
+    "letterbox": "Shrink to fit",
+}
+DEFAULT_DISPLAY_MODE = "crop_faces"
+LETTERBOX_BG = (255, 255, 255)
+
+
+def _placement_transform(
+    img_width: int, img_height: int, target_w: int, target_h: int,
+    display_mode: str, faces: list[dict] | None = None,
+) -> tuple[float, float, float, float]:
+    """Returns (scale_x, scale_y, offset_x, offset_y) mapping a point in
+    source-image pixel space to a point in target logical space, for the
+    given display_mode. Shared by render_frame (which also does the
+    actual pixel crop/resize/pad) and face_labels.py (label position
+    math) -- they must stay in exact agreement or overlay labels drift
+    off the people they're meant to point at."""
+    if display_mode == "stretch_fill":
+        return target_w / img_width, target_h / img_height, 0.0, 0.0
+    if display_mode == "letterbox":
+        scale = min(target_w / img_width, target_h / img_height)
+        return scale, scale, (target_w - img_width * scale) / 2, (target_h - img_height * scale) / 2
+    if display_mode == "crop_faces" and faces:
+        left, top, right, bottom = _face_aware_crop_box(img_width, img_height, target_w, target_h, faces)
+        crop_w, crop_h = right - left, bottom - top
+    else:
+        left, top, crop_w, crop_h = _plain_center_crop_box(img_width, img_height, target_w, target_h)
+    scale_x, scale_y = target_w / crop_w, target_h / crop_h
+    return scale_x, scale_y, -left * scale_x, -top * scale_y
+
+
 def render_frame(source: Image.Image, faces: list[dict] | None = None,
-                 orientation: str = "landscape", palette_rgb: list | None = None) -> bytes:
+                 orientation: str = "landscape", palette_rgb: list | None = None,
+                 display_mode: str = DEFAULT_DISPLAY_MODE) -> bytes:
     """Fits `source` to the panel's resolution, quantizes it to the 6-color
     palette with Floyd-Steinberg dithering, and packs 2 pixels/byte the way
     epd7in3e.c expects. Always returns exactly EPD_WIDTH*EPD_HEIGHT/2 bytes.
 
-    If `faces` (from ImmichClient.get_asset_faces) is non-empty, crops
-    toward keeping them on screen instead of a plain center-crop.
+    `display_mode` (see DISPLAY_MODES) picks how the photo's aspect ratio
+    is reconciled with the panel's: crop_fill (center-crop to fill,
+    excess trimmed), crop_faces (as crop_fill, but shifts the crop to
+    keep `faces` on screen -- falls back to crop_fill if none), stretch_fill
+    (fills exactly, aspect ratio not preserved), letterbox (whole photo
+    visible, letterboxed with LETTERBOX_BG where it doesn't fill).
 
     `orientation` (see ORIENTATION_TRANSPOSE) composes the photo for how
     the frame physically hangs, then rotates into native panel space --
@@ -188,10 +232,19 @@ def render_frame(source: Image.Image, faces: list[dict] | None = None,
     logical_w, logical_h = logical_render_size(orientation)
     fitted = ImageOps.exif_transpose(source.convert("RGB"))
 
-    if faces:
+    if display_mode == "stretch_fill":
+        fitted = fitted.resize((logical_w, logical_h), Image.LANCZOS)
+    elif display_mode == "letterbox":
+        scale = min(logical_w / fitted.width, logical_h / fitted.height)
+        new_w, new_h = max(1, round(fitted.width * scale)), max(1, round(fitted.height * scale))
+        resized = fitted.resize((new_w, new_h), Image.LANCZOS)
+        canvas = Image.new("RGB", (logical_w, logical_h), LETTERBOX_BG)
+        canvas.paste(resized, ((logical_w - new_w) // 2, (logical_h - new_h) // 2))
+        fitted = canvas
+    elif display_mode == "crop_faces" and faces:
         box = _face_aware_crop_box(fitted.width, fitted.height, logical_w, logical_h, faces)
         fitted = fitted.crop(box).resize((logical_w, logical_h), Image.LANCZOS)
-    else:
+    else:  # crop_fill, or crop_faces with no faces detected
         fitted = ImageOps.fit(fitted, (logical_w, logical_h), method=Image.LANCZOS)
 
     return _quantize_and_pack(fitted, orientation, palette_rgb)

server/app/migration.py

@@ -59,11 +59,23 @@ def _migration_4(conn) -> None:
     conn.execute(text("ALTER TABLE frames ADD COLUMN palette_rgb TEXT"))
 
 
+def _migration_5(conn) -> None:
+    """Replaces the smart_crop_faces boolean with display_mode (see
+    image_pipeline.DISPLAY_MODES) -- crop_faces/crop_fill are exactly
+    the old True/False behavior, stretch_fill/letterbox are new."""
+    conn.execute(text("ALTER TABLE frames ADD COLUMN display_mode TEXT NOT NULL DEFAULT 'crop_faces'"))
+    conn.execute(text(
+        "UPDATE frames SET display_mode = CASE WHEN smart_crop_faces THEN 'crop_faces' ELSE 'crop_fill' END"
+    ))
+    conn.execute(text("ALTER TABLE frames DROP COLUMN smart_crop_faces"))
+
+
 MIGRATIONS = [
     (1, _migration_1),
     (2, _migration_2),
     (3, _migration_3),
     (4, _migration_4),
+    (5, _migration_5),
 ]
 
 
@@ -126,7 +138,7 @@ def _ensure_frame_one() -> None:
             quiet_hours_start=cfg.quiet_hours_start,
             quiet_hours_end=cfg.quiet_hours_end,
             timezone=cfg.timezone,
-            smart_crop_faces=cfg.smart_crop_faces,
+            display_mode="crop_faces" if cfg.smart_crop_faces else "crop_fill",
             orientation=cfg.orientation,
             queue_target_len=cfg.queue_target_len,
             current_asset_id=cfg.current_asset_id,

server/app/models.py

@@ -121,7 +121,9 @@ class Frame(Base):
     quiet_hours_start: Mapped[str] = mapped_column(String, default="22:00")
     quiet_hours_end: Mapped[str] = mapped_column(String, default="07:00")
     timezone: Mapped[str] = mapped_column(String, default="UTC")
-    smart_crop_faces: Mapped[bool] = mapped_column(Boolean, default=True)
+    # How a photo's aspect ratio is reconciled with the panel's -- see
+    # image_pipeline.DISPLAY_MODES.
+    display_mode: Mapped[str] = mapped_column(String, default="crop_faces")
     orientation: Mapped[str] = mapped_column(String, default="landscape")
     queue_target_len: Mapped[int] = mapped_column(Integer, default=20)
     # Advanced configuration: [[r,g,b], ...] x6 (black/white/yellow/red/

server/app/routers/api_frames.py

@@ -28,7 +28,7 @@ from sqlalchemy.orm import Session
 from .. import gitea_releases, photo_queue, quiet_hours
 from ..auth import require_frame_control, require_frame_view, require_user_api
 from ..db import frame_locked, get_db
-from ..image_pipeline import PALETTE_LABELS, hex_to_rgb
+from ..image_pipeline import DEFAULT_DISPLAY_MODE, DISPLAY_MODES, PALETTE_LABELS, hex_to_rgb
 from ..firmware import firmware_path, parse_app_version
 from ..models import BatteryLog, Frame
 from .common import (
@@ -70,7 +70,7 @@ def api_config_save(
     album_id: str | None = Form(None),
     order: str | None = Form(None),
     refresh_interval_s: int | None = Form(None),
-    smart_crop_faces: bool | None = Form(None),
+    display_mode: str | None = Form(None),
     queue_target_len: int | None = Form(None),
     orientation: str | None = Form(None),
     quiet_hours_enabled: bool | None = Form(None),
@@ -104,8 +104,8 @@ def api_config_save(
             cfg.refresh_interval_s = max(
                 MIN_REFRESH_INTERVAL_S, min(MAX_REFRESH_INTERVAL_S, refresh_interval_s)
             )
-        if smart_crop_faces is not None:
-            cfg.smart_crop_faces = smart_crop_faces
+        if display_mode is not None:
+            cfg.display_mode = display_mode if display_mode in DISPLAY_MODES else DEFAULT_DISPLAY_MODE
         if queue_target_len is not None:
             cfg.queue_target_len = max(MIN_QUEUE_TARGET_LEN, min(MAX_QUEUE_TARGET_LEN, queue_target_len))
         if orientation is not None:

server/app/routers/common.py

@@ -76,7 +76,7 @@ def render_asset(client: ImmichClient, frame: Frame, asset_id: str) -> bytes:
         raise HTTPException(502, f"Could not download asset from Immich: {e}") from e
 
     faces = None
-    if frame.smart_crop_faces:
+    if frame.display_mode == "crop_faces":
         try:
             faces = client.get_asset_faces(asset_id)
         except httpx.HTTPError as e:
@@ -85,7 +85,8 @@ def render_asset(client: ImmichClient, frame: Frame, asset_id: str) -> bytes:
             logger.warning("Could not fetch faces for asset %s: %s", asset_id, e)
 
     source = Image.open(io.BytesIO(jpeg_bytes))
-    return render_frame(source, faces=faces, orientation=frame.orientation, palette_rgb=frame.palette_rgb)
+    return render_frame(source, faces=faces, orientation=frame.orientation,
+                        palette_rgb=frame.palette_rgb, display_mode=frame.display_mode)
 
 
 def battery_estimate_s(frame: Frame) -> int | None:

server/app/routers/device.py

@@ -410,7 +410,7 @@ def frame_face_labels(frame: Frame = Depends(require_device), db: Session = Depe
     with frame_locked(db, frame.id) as locked:
         photo_queue.get_current(locked, assets, in_quiet_hours=quiet_hours.in_quiet_hours(locked))
         asset_id = locked.current_asset_id
-        smart_crop = locked.smart_crop_faces
+        display_mode = locked.display_mode
         orientation = locked.orientation
 
     if not asset_id:
@@ -431,7 +431,7 @@ def frame_face_labels(frame: Frame = Depends(require_device), db: Session = Depe
         logger.warning("Could not download asset %s for face-label mapping: %s", asset_id, e)
         return {"count": 0}
 
-    labels = compute_face_labels(preview_bytes, faces, smart_crop, orientation)
+    labels = compute_face_labels(preview_bytes, faces, display_mode, orientation)
 
     result: dict[str, object] = {"count": len(labels)}
     for i, label in enumerate(labels):

server/app/routers/frame_pages.py

@@ -12,7 +12,12 @@ from sqlalchemy.orm import Session
 
 from ..auth import can_view_frame, current_user
 from ..db import get_db
-from ..image_pipeline import DEFAULT_PALETTE_RGB, PALETTE_LABELS, palette_to_hex
+from ..image_pipeline import (
+    DEFAULT_PALETTE_RGB,
+    DISPLAY_MODE_LABELS,
+    PALETTE_LABELS,
+    palette_to_hex,
+)
 from ..models import Frame
 from ..quiet_hours import ALL_TIMEZONES
 from .common import shell_context
@@ -46,6 +51,7 @@ def frame_config_page(frame_id: int, request: Request, db: Session = Depends(get
         palette_labels=PALETTE_LABELS,
         default_palette_rgb=DEFAULT_PALETTE_RGB,
         palette_to_hex=palette_to_hex,
+        display_mode_labels=DISPLAY_MODE_LABELS,
     )
 
 

server/app/static/frame_config.js

@@ -10,7 +10,7 @@ async function saveConfig() {
     order: document.getElementById('order').value,
     orientation: document.getElementById('orientation').value,
     refresh_interval_s: String(minutes * 60),
-    smart_crop_faces: String(document.getElementById('smart_crop_faces').checked),
+    display_mode: document.getElementById('display_mode').value,
     quiet_hours_enabled: String(document.getElementById('quiet_hours_enabled').checked),
     quiet_hours_start: document.getElementById('quiet_hours_start').value || '22:00',
     quiet_hours_end: document.getElementById('quiet_hours_end').value || '07:00',

server/app/templates/frame_config.html

@@ -37,10 +37,21 @@
             <input type="number" id="refresh_interval_minutes" min="1" max="1440"
                    value="{{ (frame.refresh_interval_s // 60) or 60 }}" required>
           </label>
-          <div class="checkbox-row">
-            <input type="checkbox" id="smart_crop_faces" {% if frame.smart_crop_faces %}checked{% endif %}>
-            <label for="smart_crop_faces">Center faces in crop</label>
-          </div>
+          <label>Display mode
+            <select id="display_mode">
+              {% for mode, label in display_mode_labels.items() %}
+                <option value="{{ mode }}" {% if frame.display_mode == mode %}selected{% endif %}>{{ label }}</option>
+              {% endfor %}
+            </select>
+          </label>
+          <p class="sub" style="margin-top: 8px;">How a photo's aspect ratio
+            is reconciled with the panel's: <strong>Crop to fill</strong>
+            trims the excess; <strong>Crop to faces</strong> does the same
+            but shifts the crop to keep people on screen; <strong>Stretch to
+            fill</strong> fills the panel exactly without cropping (photos
+            not matching the panel's aspect ratio look stretched);
+            <strong>Shrink to fit</strong> shows the whole photo, letterboxed
+            if needed.</p>
           <div class="checkbox-row">
             <input type="checkbox" id="quiet_hours_enabled" {% if frame.quiet_hours_enabled %}checked{% endif %}>
             <label for="quiet_hours_enabled">Quiet hours (don't wake overnight)</label>