Render widgets concurrently instead of one at a time

A layout with several network-backed widgets (photos, weather,
calendar) paid their fetch latency serially in one /frame/* request,
which could exceed the firmware's fixed HTTP timeout and show a false
"server failed" status screen even though the server was still
working -- most visibly on the hold-triggered "cycle layouts" action,
which swaps in a whole new, cold-started widget set. Each widget now
renders on its own DB session in a thread pool (a plain Session isn't
thread-safe to share, but the per-frame threading.Lock in
frame_locked/widget_locked already made this kind of concurrency safe
by design -- see app/db.py); regions are still collected in
sort_order so overlapping widgets paint in the same z-order as before.

server/app/routers/device.py

@@ -14,6 +14,7 @@ from __future__ import annotations
 
 import logging
 import time
+from concurrent.futures import ThreadPoolExecutor
 
 from fastapi import APIRouter, Depends, HTTPException, Request
 from fastapi.responses import FileResponse, Response
@@ -23,7 +24,7 @@ from sqlalchemy.orm import Session
 
 from .. import grid, mail, quiet_hours
 from ..auth import get_server_settings, require_device
-from ..db import frame_locked, get_db
+from ..db import SessionLocal, frame_locked, get_db
 from ..firmware import firmware_path
 from ..global_actions import GLOBAL_ACTIONS
 from ..image_pipeline import draw_widget_border, logical_render_size, render_panel, render_placeholder, resolve_border_color
@@ -82,6 +83,40 @@ def _setup_placeholder(frame: Frame, request: Request, manage: dict | None = Non
     )
 
 
+def _render_one_widget(frame_id: int, widget_id: int, orientation: str, panel_w: int, panel_h: int,
+                       cell: tuple[int, int, int, int], is_normal_wake: bool,
+                       ) -> tuple[tuple[int, int, int, int], object] | None:
+    """Renders exactly one widget on its own DB session, so several of
+    these can run concurrently in a thread pool -- see app/db.py's
+    module docstring: handlers already run multi-threaded (sync
+    handlers in FastAPI's threadpool, one process), and frame_locked/
+    widget_locked's per-frame threading.Lock is what makes that safe,
+    not anything about which Session object is in play. A SQLAlchemy
+    Session itself is never safe to share across threads, so each
+    concurrent render gets a fresh one rather than reusing the
+    request's. Most of a widget's render time is spent waiting on an
+    external call (Immich, a weather provider, CalDAV) with the DB
+    untouched, which is exactly the time this buys back."""
+    db = SessionLocal()
+    try:
+        frame = db.get(Frame, frame_id)
+        widget = db.get(Widget, widget_id)
+        if widget is None:
+            return None  # deleted between the listing query and this fetch -- skip it, not a 500
+        module = WIDGET_TYPES.get(widget.widget_type)
+        if module is None:
+            return None  # unrecognized widget_type -- shouldn't happen, skip defensively rather than 500
+        px, py, pw, ph = grid.cell_to_pixels(orientation, panel_w, panel_h, cell)
+        img = module.render(db, frame, widget, pw, ph, is_normal_wake=is_normal_wake)
+        draw_widget_border(
+            img, widget.border_style, widget.border_thickness,
+            resolve_border_color(widget.border_color_index, frame.palette_rgb),
+        )
+        return (px, py, pw, ph), img
+    finally:
+        db.close()
+
+
 def _render_widgets(db: Session, frame: Frame, manage: dict | None, is_normal_wake: bool,
                     as_png: bool = False, capture_snapshot: bool = False) -> bytes | tuple[bytes, bytes]:
     """The widget-system compositor: renders every widget on this frame
@@ -92,25 +127,35 @@ def _render_widgets(db: Session, frame: Frame, manage: dict | None, is_normal_wa
     image_pipeline.render_panel for the single shared paste/enhance/
     overlay/quantize/pack pass. Replaces the old per-mode RENDERERS
     dict -- a frame can now show several widgets at once instead of
-    exactly one mode owning the whole panel."""
+    exactly one mode owning the whole panel.
+
+    Widgets render concurrently (_render_one_widget, each on its own DB
+    session) rather than one at a time -- a layout with several
+    network-backed widgets (photos, weather, calendar) previously paid
+    their fetch latency serially, which could push a single /frame/*
+    response past the firmware's fixed HTTP timeout and show a
+    misleading "server failed" status screen even though the server
+    was simply still working. Futures are submitted in sort_order and
+    collected in that same order (not completion order) -- overlapping
+    widgets must still paint in the original z-order."""
     all_widgets = db.scalars(
         select(Widget).where(Widget.frame_id == frame.id).order_by(Widget.sort_order)
     ).all()
     panel_w, panel_h = logical_render_size(frame.orientation)
     regions = []
-    for widget in all_widgets:
-        module = WIDGET_TYPES.get(widget.widget_type)
-        if module is None:
-            continue  # unrecognized widget_type -- shouldn't happen, skip defensively rather than 500
-        px, py, pw, ph = grid.cell_to_pixels(
-            frame.orientation, panel_w, panel_h, (widget.x, widget.y, widget.w, widget.h)
-        )
-        img = module.render(db, frame, widget, pw, ph, is_normal_wake=is_normal_wake)
-        draw_widget_border(
-            img, widget.border_style, widget.border_thickness,
-            resolve_border_color(widget.border_color_index, frame.palette_rgb),
-        )
-        regions.append(((px, py, pw, ph), img))
+    if all_widgets:
+        with ThreadPoolExecutor(max_workers=min(len(all_widgets), 8)) as pool:
+            futures = [
+                pool.submit(
+                    _render_one_widget, frame.id, widget.id, frame.orientation, panel_w, panel_h,
+                    (widget.x, widget.y, widget.w, widget.h), is_normal_wake,
+                )
+                for widget in all_widgets
+            ]
+            for future in futures:
+                result = future.result()
+                if result is not None:
+                    regions.append(result)
     return render_panel(
         regions, orientation=frame.orientation, palette_rgb=frame.palette_rgb,
         color_boost=frame.color_boost, contrast_boost=frame.contrast_boost,