diff --git a/firmware/README.md b/firmware/README.md index b19c51a..7d2bb0c 100644 --- a/firmware/README.md +++ b/firmware/README.md @@ -112,14 +112,20 @@ the photo visible and unchanged: link for that exact photo. The link is only created once someone actually scans it, and expires 30 minutes after that. -After 30 seconds it automatically reverts to the plain photo. See -`FRAME_MANAGE_BUTTON_GPIO` above to change the pin or disable the -feature. +After 30 seconds with no further presses it automatically reverts to the +plain photo. Pressing the button *again* while this is up escalates to a +second menu level -- everything above, plus the name of anyone Immich +has identified labeled right next to their face in the photo (skipped +for faces Immich hasn't been told a name for; no face detection happens +on the device or the server, this is entirely Immich's own People +feature). A third press exits immediately rather than waiting out the +30-second timer. See `FRAME_MANAGE_BUTTON_GPIO` above to change the pin +or disable the feature. -The device stays awake for the full 30 seconds (two physical refreshes, -one for the overlay and one to revert), so this costs meaningfully more -power than a normal wake -- expected for a deliberate, occasional action, -same tradeoff as the other two buttons. +The device stays awake the whole time (up to three physical refreshes: +the base overlay, the escalated one, and reverting), so this costs +meaningfully more power than a normal wake -- expected for a deliberate, +occasional action, same tradeoff as the other two buttons. ## Resetting to provisioning mode diff --git a/firmware/main/frame_client.c b/firmware/main/frame_client.c index 5f84c3e..a7211b8 100644 --- a/firmware/main/frame_client.c +++ b/firmware/main/frame_client.c @@ -14,6 +14,7 @@ #include "epd7in3e.h" #include "status_screen.h" #include "manage_qr_overlay.h" +#include "manage_button.h" #include "frame_client.h" @@ -246,13 +247,15 @@ static frame_server_config_t fetch_frame_config(const char *toolsserver) * e.g. no GPS EXIF to geocode, or no capture date) and a share_url built * from the returned asset_id, same construction pattern as * run_fetch_cycle()'s management_url. Any failure (unreachable, no - * current photo, etc.) just leaves all three outputs empty -- the caller + * current photo, etc.) just leaves all outputs empty -- the caller * treats that as "skip these optional overlay regions", not a hard * error, since the base "scan to manage" QR should still show. */ -static void fetch_photo_info(const char *toolsserver, char *location, size_t location_size, char *taken_at, - size_t taken_at_size, char *share_url, size_t share_url_size) +static void fetch_photo_info(const char *toolsserver, char *location_line1, size_t location_line1_size, + char *location_line2, size_t location_line2_size, char *taken_at, size_t taken_at_size, + char *share_url, size_t share_url_size) { - location[0] = '\0'; + location_line1[0] = '\0'; + location_line2[0] = '\0'; taken_at[0] = '\0'; share_url[0] = '\0'; @@ -293,7 +296,8 @@ static void fetch_photo_info(const char *toolsserver, char *location, size_t loc esp_http_client_close(client); esp_http_client_cleanup(client); - json_extract_string(body, "location", location, location_size); + json_extract_string(body, "location_line1", location_line1, location_line1_size); + json_extract_string(body, "location_line2", location_line2, location_line2_size); json_extract_string(body, "taken_at", taken_at, taken_at_size); char asset_id[48]; @@ -302,6 +306,83 @@ static void fetch_photo_info(const char *toolsserver, char *location, size_t loc } } +/* GETs the server's /frame/face-labels for the manage-button's escalated + * "level 2" menu -- named-face positions, if Immich has any for the + * current photo. Response is a flattened, fixed-slot shape ("count", + * then name_0/x_0/y_0, name_1/x_1/y_1, ...) rather than a real JSON + * array, read with the same flat-scalar helpers as everywhere else in + * this file instead of needing an actual array parser. Any failure + * (unreachable, malformed response, etc.) just returns 0 -- named faces + * are a "nice to have" addition to the menu, not worth failing it over. */ +static int fetch_face_labels(const char *toolsserver, manage_face_label_t *out, int max_labels) +{ + char url[160]; + snprintf(url, sizeof(url), "http://%s/frame/face-labels", toolsserver); + + esp_http_client_config_t config = { + .url = url, + .method = HTTP_METHOD_GET, + .timeout_ms = CONFIG_FRAME_SERVER_CHECK_TIMEOUT_MS, + }; + esp_http_client_handle_t client = esp_http_client_init(&config); + + esp_err_t err = esp_http_client_open(client, 0); + if (err != ESP_OK) { + ESP_LOGW(TAG, "'%s' not reachable: %s", url, esp_err_to_name(err)); + esp_http_client_cleanup(client); + return 0; + } + + int status = esp_http_client_fetch_headers(client) >= 0 ? esp_http_client_get_status_code(client) : -1; + if (status != 200) { + ESP_LOGW(TAG, "'%s' returned HTTP %d", url, status); + esp_http_client_close(client); + esp_http_client_cleanup(client); + return 0; + } + + char body[768]; + int total = 0; + int n; + while (total < (int)sizeof(body) - 1 && + (n = esp_http_client_read(client, body + total, sizeof(body) - 1 - total)) > 0) { + total += n; + } + body[total] = '\0'; + + esp_http_client_close(client); + esp_http_client_cleanup(client); + + uint32_t count = 0; + json_extract_uint(body, "count", &count); + if ((int)count > max_labels) { + count = (uint32_t)max_labels; + } + + int found = 0; + for (uint32_t i = 0; i < count; i++) { + char key[16]; + snprintf(key, sizeof(key), "name_%u", (unsigned)i); + if (!json_extract_string(body, key, out[found].name, sizeof(out[found].name))) { + continue; + } + snprintf(key, sizeof(key), "x_%u", (unsigned)i); + uint32_t x; + if (!json_extract_uint(body, key, &x)) { + continue; + } + snprintf(key, sizeof(key), "y_%u", (unsigned)i); + uint32_t y; + if (!json_extract_uint(body, key, &y)) { + continue; + } + out[found].x = (int)x; + out[found].y = (int)y; + found++; + } + return found; +} + typedef struct { esp_http_client_handle_t client; size_t stream_pos; /* running absolute offset into the frame, for overlay splicing */ @@ -429,64 +510,133 @@ static esp_err_t fetch_and_display(const frame_config_t *cfg, bool force_advance return err; } +#define MANAGE_MENU_MAX_LEVEL 2 +#define MANAGE_MENU_LEVEL_TIMEOUT_MS 30000 +#define MANAGE_MENU_POLL_MS 150 +#define MANAGE_MENU_DEBOUNCE_MS 30 + +/* Polls the manage button for up to timeout_ms for a new press. On + * detecting one, waits for release before returning true, so a single + * physical press-and-release is always exactly one event to the caller + * -- without that, a press held across multiple poll intervals would + * register as multiple escalations. Returns false if timeout_ms elapses + * with no press. */ +static bool wait_for_button_press(uint32_t timeout_ms) +{ + uint32_t elapsed_ms = 0; + while (elapsed_ms < timeout_ms) { + vTaskDelay(pdMS_TO_TICKS(MANAGE_MENU_POLL_MS)); + elapsed_ms += MANAGE_MENU_POLL_MS; + + if (!manage_button_is_pressed()) { + continue; + } + vTaskDelay(pdMS_TO_TICKS(MANAGE_MENU_DEBOUNCE_MS)); + if (!manage_button_is_pressed()) { + continue; /* noise, not a real press */ + } + while (manage_button_is_pressed()) { + vTaskDelay(pdMS_TO_TICKS(MANAGE_MENU_POLL_MS)); + } + return true; + } + return false; +} + +/* Builds and shows one level of the manage menu: level 1 is the base + * overlay (management QR + location/date/share-QR wherever the server + * had that data); level 2 adds named-face labels on top. force_advance + * only applies at level 1 -- escalating to level 2 redisplays the same + * photo, so it never re-advances. */ +static esp_err_t show_menu_level(const frame_config_t *cfg, bool force_advance, int level) +{ + char management_url[160]; + snprintf(management_url, sizeof(management_url), "http://%s/", cfg->toolsserver); + + char location_line1[32]; + char location_line2[32]; + char taken_at[32]; + char share_url[160]; + fetch_photo_info(cfg->toolsserver, location_line1, sizeof(location_line1), location_line2, + sizeof(location_line2), taken_at, sizeof(taken_at), share_url, sizeof(share_url)); + + manage_face_label_t face_labels[MANAGE_FACE_LABELS_MAX]; + int face_label_count = 0; + if (level >= 2) { + face_label_count = fetch_face_labels(cfg->toolsserver, face_labels, MANAGE_FACE_LABELS_MAX); + } + + manage_overlay_content_t content = { + .management_url = management_url, + .location_line1 = location_line1[0] != '\0' ? location_line1 : NULL, + .location_line2 = location_line2[0] != '\0' ? location_line2 : NULL, + .taken_at = taken_at[0] != '\0' ? taken_at : NULL, + .share_url = share_url[0] != '\0' ? share_url : NULL, + .face_labels = face_labels, + .face_label_count = face_label_count, + }; + + manage_overlay_set_t overlay; + esp_err_t err = manage_overlay_render(&content, &overlay); + if (err != ESP_OK) { + manage_overlay_free(&overlay); + return err; + } + + err = fetch_and_display(cfg, force_advance, &overlay); + manage_overlay_free(&overlay); + return err; +} + +/* Runs the manage-button menu: level 1 (the base overlay) shows first; + * from there, each further press within 30s escalates one level (up to + * MANAGE_MENU_MAX_LEVEL, which adds named-face labels), and a press once + * already at the max level exits immediately instead of escalating + * further. A 30s timeout at any level also exits. Device stays awake + * throughout (doesn't sleep the panel or the chip). Returns non-ESP_OK + * only if the very first (level 1) render/fetch failed; failures after + * that (escalating, or the final revert) are logged but don't count as + * an overall failure -- something was already shown successfully, which + * was the point of the button. */ +static esp_err_t run_management_menu(const frame_config_t *cfg, bool force_advance) +{ + int level = 1; + esp_err_t err = show_menu_level(cfg, force_advance, level); + if (err != ESP_OK) { + ESP_LOGW(TAG, "Could not render management overlay (%s), showing photo normally", esp_err_to_name(err)); + return fetch_and_display(cfg, force_advance, NULL); + } + + for (;;) { + ESP_LOGI(TAG, "Showing management menu level %d, waiting up to 30s", level); + bool pressed = wait_for_button_press(MANAGE_MENU_LEVEL_TIMEOUT_MS); + if (!pressed || level >= MANAGE_MENU_MAX_LEVEL) { + break; /* timeout at any level, or a press while already maxed out -- exit */ + } + level++; + esp_err_t level_err = show_menu_level(cfg, false, level); + if (level_err != ESP_OK) { + ESP_LOGW(TAG, "Could not render menu level %d (%s), reverting", level, esp_err_to_name(level_err)); + break; + } + } + + esp_err_t revert_err = fetch_and_display(cfg, false, NULL); + if (revert_err != ESP_OK) { + ESP_LOGW(TAG, "Failed to revert management overlay (%s)", esp_err_to_name(revert_err)); + } + return ESP_OK; +} + /* Runs the appropriate fetch for this cycle: a plain fetch, or -- if - * show_management_qr -- a fetch with the manage overlay spliced in (the - * top-right "scan to manage" QR always, plus location/date-taken/ - * share-QR corners wherever the server had that data -- see - * fetch_photo_info()), held on screen for 30s (device stays awake, - * doesn't sleep the panel or the chip), then reverted with a second - * plain fetch. Returns non-ESP_OK only if the FIRST fetch failed; a - * revert failure afterward is logged but doesn't count as an overall - * failure -- the overlay itself displayed fine, which was the point of - * the button. */ + * show_management_qr -- the escalating manage menu (see + * run_management_menu()). */ static esp_err_t run_fetch_cycle(const frame_config_t *cfg, bool force_advance, bool show_management_qr) { if (!show_management_qr) { return fetch_and_display(cfg, force_advance, NULL); } - - char management_url[160]; - snprintf(management_url, sizeof(management_url), "http://%s/", cfg->toolsserver); - - char location[64]; - char taken_at[32]; - char share_url[160]; - fetch_photo_info(cfg->toolsserver, location, sizeof(location), taken_at, sizeof(taken_at), share_url, - sizeof(share_url)); - - manage_overlay_content_t content = { - .management_url = management_url, - .location = location[0] != '\0' ? location : NULL, - .taken_at = taken_at[0] != '\0' ? taken_at : NULL, - .share_url = share_url[0] != '\0' ? share_url : NULL, - }; - - manage_overlay_set_t overlay; - esp_err_t overlay_err = manage_overlay_render(&content, &overlay); - if (overlay_err != ESP_OK) { - manage_overlay_free(&overlay); - ESP_LOGW(TAG, "Could not render management overlay (%s), showing photo normally", - esp_err_to_name(overlay_err)); - return fetch_and_display(cfg, force_advance, NULL); - } - - esp_err_t err = fetch_and_display(cfg, force_advance, &overlay); - manage_overlay_free(&overlay); - if (err != ESP_OK) { - return err; - } - - ESP_LOGI(TAG, "Showing management overlay for 30s"); - vTaskDelay(pdMS_TO_TICKS(30000)); - - /* force_advance is always false here -- reverting shouldn't skip - * ahead a second time. */ - esp_err_t revert_err = fetch_and_display(cfg, false, NULL); - if (revert_err != ESP_OK) { - ESP_LOGW(TAG, "Failed to revert management overlay (%s)", esp_err_to_name(revert_err)); - } - - return ESP_OK; + return run_management_menu(cfg, force_advance); } void frame_client_run(const frame_config_t *cfg, bool force_advance, bool show_management_qr) diff --git a/firmware/main/manage_button.c b/firmware/main/manage_button.c index e79fd95..e02e578 100644 --- a/firmware/main/manage_button.c +++ b/firmware/main/manage_button.c @@ -56,9 +56,15 @@ bool manage_button_check(void) return true; } +bool manage_button_is_pressed(void) +{ + return gpio_get_level(MANAGE_BUTTON_GPIO) == 0; +} + #else void manage_button_init(void) {} bool manage_button_check(void) { return false; } +bool manage_button_is_pressed(void) { return false; } #endif diff --git a/firmware/main/manage_button.h b/firmware/main/manage_button.h index 4f2bbeb..27eea00 100644 --- a/firmware/main/manage_button.h +++ b/firmware/main/manage_button.h @@ -19,3 +19,14 @@ void manage_button_init(void); * low-stakes and should feel immediate. */ bool manage_button_check(void); + +/** + * Bare, undebounced "is the pin low right now" read -- unlike + * manage_button_check(), this doesn't consult the latched deep-sleep + * wakeup status (only meaningful once, immediately after waking from + * sleep) and isn't meant to detect what woke the device. Used for + * polling for a subsequent press while already awake and the manage + * overlay is up (see frame_client.c's wait_for_button_press()), which + * does its own debounce/release-wait around repeated calls to this. + */ +bool manage_button_is_pressed(void); diff --git a/firmware/main/manage_qr_overlay.c b/firmware/main/manage_qr_overlay.c index 8689eef..141f2be 100644 --- a/firmware/main/manage_qr_overlay.c +++ b/firmware/main/manage_qr_overlay.c @@ -134,13 +134,21 @@ static esp_err_t render_qr_region(const char *payload, const char *line1, const return ESP_OK; } -/* White-padded box with a single centered line of text -- used for the - * top-left location and bottom-right date-taken labels. */ -static esp_err_t render_text_region(const char *text, overlay_corner_t corner, manage_overlay_region_t *out) +/* White-padded box with one or two centered lines of text (line2 may be + * NULL) -- used for the top-left location (city + state/country, two + * lines rather than cramming both onto one to keep the box from + * threatening to overlap the top-right QR box) and the bottom-right + * date-taken label (one line). */ +static esp_err_t render_text_region(const char *line1, const char *line2, overlay_corner_t corner, + manage_overlay_region_t *out) { - int text_w = (int)strlen(text) * Font24.Width; + int w1 = (int)strlen(line1) * Font24.Width; + int w2 = line2 != NULL ? (int)strlen(line2) * Font24.Width : 0; + int text_w = w1 > w2 ? w1 : w2; + int text_h = Font24.Height + (line2 != NULL ? LINE_GAP + Font24.Height : 0); + int w = text_w + PADDING * 2; - int h = Font24.Height + PADDING * 2; + int h = text_h + PADDING * 2; w += w % 2; int stride = w / 2; @@ -148,7 +156,13 @@ static esp_err_t render_text_region(const char *text, overlay_corner_t corner, m ESP_RETURN_ON_FALSE(buf != NULL, ESP_ERR_NO_MEM, TAG, "Failed to allocate overlay region"); memset(buf, (EPD_COLOR_WHITE << 4) | EPD_COLOR_WHITE, (size_t)stride * h); - epd_draw_text_centered_ex(buf, stride, w, h, &Font24, text, w / 2, PADDING); + int center_x = w / 2; + int y = PADDING; + epd_draw_text_centered_ex(buf, stride, w, h, &Font24, line1, center_x, y); + if (line2 != NULL) { + y += Font24.Height + LINE_GAP; + epd_draw_text_centered_ex(buf, stride, w, h, &Font24, line2, center_x, y); + } out->buf = buf; out->w = w; @@ -157,6 +171,58 @@ static esp_err_t render_text_region(const char *text, overlay_corner_t corner, m return ESP_OK; } +/* Deliberately tighter than PADDING (used for the fixed QR/text corner + * boxes) -- these labels sit right next to a face rather than needing + * generous QR-scanning margin, and there can be several of them + * simultaneously (see MANAGE_FACE_LABELS_MAX's memory-budget note in + * the header). */ +#define FACE_LABEL_PADDING 8 +#define FACE_LABEL_GAP 4 /* distance from the face's anchor point to the label box */ + +/* White-padded single-line name label positioned near an arbitrary + * (anchor_x, anchor_y) face position, rather than a fixed corner -- + * unlike the four corner regions (always in-bounds by construction), + * this needs real clamping since a face can be anywhere, including near + * an edge. Centered horizontally on the face, placed just below it by + * default, flipped above if there's no room below. */ +static esp_err_t render_face_label_region(const char *name, int anchor_x, int anchor_y, manage_overlay_region_t *out) +{ + int text_w = (int)strlen(name) * Font24.Width; + int w = text_w + FACE_LABEL_PADDING * 2; + int h = Font24.Height + FACE_LABEL_PADDING * 2; + w += w % 2; + + int stride = w / 2; + uint8_t *buf = malloc((size_t)stride * h); + ESP_RETURN_ON_FALSE(buf != NULL, ESP_ERR_NO_MEM, TAG, "Failed to allocate overlay region"); + memset(buf, (EPD_COLOR_WHITE << 4) | EPD_COLOR_WHITE, (size_t)stride * h); + epd_draw_text_centered_ex(buf, stride, w, h, &Font24, name, w / 2, FACE_LABEL_PADDING); + + int x0 = anchor_x - w / 2; + int y0 = anchor_y + FACE_LABEL_GAP; + if (y0 + h > EPD_HEIGHT) { + y0 = anchor_y - FACE_LABEL_GAP - h; /* no room below -- place above the face instead */ + } + if (x0 < 0) { + x0 = 0; + } else if (x0 + w > EPD_WIDTH) { + x0 = EPD_WIDTH - w; + } + if (y0 < 0) { + y0 = 0; + } else if (y0 + h > EPD_HEIGHT) { + y0 = EPD_HEIGHT - h; + } + x0 -= x0 % 2; /* keep byte-aligned (2px/byte) */ + + out->buf = buf; + out->w = w; + out->h = h; + out->x0 = x0; + out->y0 = y0; + return ESP_OK; +} + esp_err_t manage_overlay_render(const manage_overlay_content_t *content, manage_overlay_set_t *out) { out->count = 0; @@ -168,21 +234,38 @@ esp_err_t manage_overlay_render(const manage_overlay_content_t *content, manage_ } out->count++; - if (content->location != NULL && content->location[0] != '\0') { - if (render_text_region(content->location, CORNER_TOP_LEFT, &out->regions[out->count]) == ESP_OK) { + if (content->location_line1 != NULL && content->location_line1[0] != '\0') { + const char *line2 = + (content->location_line2 != NULL && content->location_line2[0] != '\0') ? content->location_line2 : NULL; + if (render_text_region(content->location_line1, line2, CORNER_TOP_LEFT, &out->regions[out->count]) == + ESP_OK) { out->count++; } } if (content->taken_at != NULL && content->taken_at[0] != '\0') { - if (render_text_region(content->taken_at, CORNER_BOTTOM_RIGHT, &out->regions[out->count]) == ESP_OK) { + if (render_text_region(content->taken_at, NULL, CORNER_BOTTOM_RIGHT, &out->regions[out->count]) == ESP_OK) { out->count++; } } if (content->share_url != NULL && content->share_url[0] != '\0') { - if (render_qr_region(content->share_url, NULL, NULL, CORNER_BOTTOM_LEFT, &out->regions[out->count]) == - ESP_OK) { + if (render_qr_region(content->share_url, "SCAN TO", "DOWNLOAD", CORNER_BOTTOM_LEFT, + &out->regions[out->count]) == ESP_OK) { + out->count++; + } + } + + int face_count = content->face_label_count; + if (face_count > MANAGE_FACE_LABELS_MAX) { + face_count = MANAGE_FACE_LABELS_MAX; + } + for (int i = 0; content->face_labels != NULL && i < face_count; i++) { + const manage_face_label_t *label = &content->face_labels[i]; + if (label->name[0] == '\0') { + continue; + } + if (render_face_label_region(label->name, label->x, label->y, &out->regions[out->count]) == ESP_OK) { out->count++; } } diff --git a/firmware/main/manage_qr_overlay.h b/firmware/main/manage_qr_overlay.h index fbaff80..14132eb 100644 --- a/firmware/main/manage_qr_overlay.h +++ b/firmware/main/manage_qr_overlay.h @@ -4,7 +4,15 @@ #include "esp_err.h" -#define MANAGE_OVERLAY_MAX_REGIONS 4 +/* 4 fixed corner regions (manage QR, location, date, share QR) plus up + * to MANAGE_FACE_LABELS_MAX arbitrary-position named-face labels (see + * manage_face_label_t below). MANAGE_FACE_LABELS_MAX is capped small + * deliberately, not arbitrarily -- each label is its own malloc'd + * buffer, and the 4 fixed regions alone already use a meaningful chunk + * of the ESP32-C6's limited RAM; this keeps worst-case overlay memory + * well clear of what the WiFi/HTTP stack needs alongside it. */ +#define MANAGE_FACE_LABELS_MAX 4 +#define MANAGE_OVERLAY_MAX_REGIONS (4 + MANAGE_FACE_LABELS_MAX) typedef struct { uint8_t *buf; /* malloc'd (w/2)*h bytes, packed 2px/byte; owned by the region */ @@ -18,22 +26,33 @@ typedef struct { } manage_overlay_set_t; typedef struct { - const char *management_url; /* top-right QR + "SCAN TO"/"MANAGE" caption -- always shown */ - const char *location; /* top-left text; NULL/empty skips this region */ - const char *taken_at; /* bottom-right text; NULL/empty skips this region */ - const char *share_url; /* bottom-left QR (no caption); NULL/empty skips this region */ + char name[16]; + int x, y; /* anchor point (bottom-center of the face), panel pixel coordinates */ +} manage_face_label_t; + +typedef struct { + const char *management_url; /* top-right QR + "SCAN TO"/"MANAGE" caption -- always shown */ + const char *location_line1; /* top-left text, line 1 (city); NULL/empty skips this region */ + const char *location_line2; /* top-left text, line 2 (state/country); NULL/empty is fine if line1 is set */ + const char *taken_at; /* bottom-right text; NULL/empty skips this region */ + const char *share_url; /* bottom-left QR + "SCAN TO"/"DOWNLOAD" caption; NULL/empty skips this region */ + const manage_face_label_t *face_labels; /* named-face labels ("level 2" menu); NULL/empty count skips these */ + int face_label_count; /* clamped to MANAGE_FACE_LABELS_MAX internally */ } manage_overlay_content_t; /** * Renders the manage-button overlay: always a "scan to manage" QR in the - * top-right corner, plus whichever of location/taken_at/share_url are - * non-NULL/non-empty in their own corners (top-left, bottom-right, - * bottom-left respectively). Each region is its own separately malloc'd - * small buffer (not a full EPD_FRAME_BYTES frame). A failure rendering - * the top-right region fails the whole call; a failure rendering one of - * the optional regions just skips that region and keeps going. Caller - * must call manage_overlay_free() on out regardless of the return value - * (out->count reflects however many regions were actually populated). + * top-right corner, plus whichever of location_line1/taken_at/share_url + * are non-NULL/non-empty in their own corners (top-left, bottom-right, + * bottom-left respectively), plus one region per entry in face_labels + * (positioned near that face rather than a fixed corner -- see + * render_face_label_region() in the .c file for the clamping logic). + * Each region is its own separately malloc'd small buffer (not a full + * EPD_FRAME_BYTES frame). A failure rendering the top-right region fails + * the whole call; a failure rendering any other region just skips that + * region and keeps going. Caller must call manage_overlay_free() on out + * regardless of the return value (out->count reflects however many + * regions were actually populated). */ esp_err_t manage_overlay_render(const manage_overlay_content_t *content, manage_overlay_set_t *out); diff --git a/server/README.md b/server/README.md index bb6bcf7..46e2090 100644 --- a/server/README.md +++ b/server/README.md @@ -48,13 +48,16 @@ algorithm itself -- it just streams the response straight to the panel. button (see `firmware/README.md`). - `GET /frame/config` -- `{"refresh_interval_s": ...}`, polled by the frame each wake alongside its reachability check -- `GET /frame/photo-info` -- `{"asset_id": ..., "location": ... | null, - "taken_at": ... | null}` for the current photo (same idempotent - current-photo semantics as `/frame/image`). `location` is `city, state` - (or `city, country`, or just `city`) if Immich reverse-geocoded the - photo's GPS EXIF, else `null`; `taken_at` is `MM/DD/YY` from the - photo's EXIF capture date, else `null`. Used by the device's manage - button to build its overlay text +- `GET /frame/photo-info` -- `{"asset_id": ..., "location_line1": ... | + null, "location_line2": ... | null, "taken_at": ... | null}` for the + current photo (same idempotent current-photo semantics as + `/frame/image`). `location_line1`/`location_line2` are `city` / + `state-or-country` if Immich reverse-geocoded the photo's GPS EXIF + (both `null` if not) -- for US/Canada, the region line is the + abbreviated state/province (`"CA"`, `"ON"`); elsewhere it's the full + country name. `taken_at` is `MM/DD/YY` from the photo's EXIF capture + date, else `null`. Used by the device's manage button to build its + overlay text - `GET /frame/share/{asset_id}` -- creates a 30-minute public, view-only Immich share link for `asset_id` and redirects (302) to it. Only works for the photo currently showing or in the upcoming queue on this frame @@ -62,6 +65,13 @@ algorithm itself -- it just streams the response straight to the panel. (i.e. when someone actually scans the manage overlay's share QR), not when the button's pressed, so the 30-minute window starts when it's actually used +- `GET /frame/face-labels` -- `{"count": N, "name_0": ..., "x_0": ..., + "y_0": ..., ...}` (up to 4 slots) -- named people from Immich's face + recognition, positioned in final 800x480 frame pixel space. Only faces + Immich already has an identified name for are included (no face + detection/recognition happens in this project, see + `app/face_labels.py`); `count: 0` if none are named. Used by the + device manage button's escalated second menu level - `GET /api/queue` -- `{"current": {...} | null, "upcoming": [...]}`, each entry an asset id + thumbnail URL; used by the config UI - `POST /api/queue/reorder` -- reorders the upcoming queue; body is diff --git a/server/app/face_labels.py b/server/app/face_labels.py new file mode 100644 index 0000000..e0f4172 --- /dev/null +++ b/server/app/face_labels.py @@ -0,0 +1,72 @@ +"""Maps named faces (from Immich's own face recognition/People feature) +onto their position in the final rendered 800x480 frame, for the +manage-button overlay's escalated "who's in this photo" menu level. + +No face detection or recognition happens here or anywhere else in this +project -- Immich's GET /api/faces?id={assetId} already returns each +detected face's bounding box plus a nullable `person` object (with a +`name`, if the user has identified them in Immich); this module only +does the coordinate math to place a label next to a *named* one. +""" + +from __future__ import annotations + +import io + +from PIL import Image, ImageOps + +from .image_pipeline import EPD_HEIGHT, EPD_WIDTH, _face_aware_crop_box, _plain_center_crop_box + +# Small caps, not arbitrary: each label is its own malloc'd overlay +# buffer on the device (see firmware/main/manage_qr_overlay.c), and the +# four existing fixed corner regions already use a meaningful chunk of +# the ESP32-C6's limited RAM. Capping at 4 short names keeps the total +# overlay memory budget well clear of the WiFi/HTTP stack's own needs. +MAX_LABELED_FACES = 4 +NAME_MAX_LEN = 10 + + +def compute_face_labels(preview_bytes: bytes, faces: list[dict], smart_crop_faces: bool) -> list[dict]: + """Returns up to MAX_LABELED_FACES [{"name", "x", "y"}], x/y in final + 800x480 frame pixel space at each named face's bottom-center point. + Faces without an Immich-identified person name are skipped entirely. + preview_bytes must be the same preview image render_frame() used for + the currently-displayed frame, and smart_crop_faces must match the + setting that was active then -- otherwise the crop box computed here + won't match what's actually on screen. + """ + named = [face for face in faces if (face.get("person") or {}).get("name")] + if not named: + return [] + + fitted = ImageOps.exif_transpose(Image.open(io.BytesIO(preview_bytes)).convert("RGB")) + + if smart_crop_faces and faces: + left, top, right, bottom = _face_aware_crop_box(fitted.width, fitted.height, EPD_WIDTH, EPD_HEIGHT, faces) + crop_w, crop_h = right - left, bottom - top + else: + left, top, crop_w, crop_h = _plain_center_crop_box(fitted.width, fitted.height, EPD_WIDTH, EPD_HEIGHT) + + labels = [] + for face in named[:MAX_LABELED_FACES]: + face_w = face.get("imageWidth") or fitted.width + face_h = face.get("imageHeight") or fitted.height + scale_x = fitted.width / face_w + scale_y = fitted.height / face_h + + center_x = (face["boundingBoxX1"] + face["boundingBoxX2"]) / 2 * scale_x + bottom_y = face["boundingBoxY2"] * scale_y + + frame_x = (center_x - left) * (EPD_WIDTH / crop_w) + frame_y = (bottom_y - top) * (EPD_HEIGHT / crop_h) + + if not (0 <= frame_x <= EPD_WIDTH and 0 <= frame_y <= EPD_HEIGHT): + continue # this face got cropped out of the final frame entirely + + name = face["person"]["name"] + if len(name) > NAME_MAX_LEN: + name = name[: NAME_MAX_LEN - 3] + "..." + + labels.append({"name": name, "x": int(frame_x), "y": int(frame_y)}) + + return labels diff --git a/server/app/image_pipeline.py b/server/app/image_pipeline.py index 36dbe62..c98a373 100644 --- a/server/app/image_pipeline.py +++ b/server/app/image_pipeline.py @@ -34,6 +34,27 @@ def _build_palette_image() -> Image.Image: _PALETTE_IMAGE = _build_palette_image() +def _plain_center_crop_box( + img_width: int, img_height: int, target_width: int, target_height: int +) -> tuple[float, float, int, int]: + """The largest target_width:target_height window centered in the + source image -- the same box ImageOps.fit() computes internally when + there's no face-aware shift to apply. Returns (left, top, crop_w, + crop_h); left/top are floats (not yet rounded) since callers that go + on to face-shift this box need the unrounded center point.""" + target_ratio = target_width / target_height + if img_width / img_height > target_ratio: + crop_h = img_height + crop_w = int(crop_h * target_ratio) + else: + crop_w = img_width + crop_h = int(crop_w / target_ratio) + + left = (img_width - crop_w) / 2 + top = (img_height - crop_h) / 2 + return left, top, crop_w, crop_h + + def _face_aware_crop_box( img_width: int, img_height: int, target_width: int, target_height: int, faces: list[dict] ) -> tuple[int, int, int, int]: @@ -63,16 +84,7 @@ def _face_aware_crop_box( min_y = min(min_y, face["boundingBoxY1"] * scale_y) max_y = max(max_y, face["boundingBoxY2"] * scale_y) - target_ratio = target_width / target_height - if img_width / img_height > target_ratio: - crop_h = img_height - crop_w = int(crop_h * target_ratio) - else: - crop_w = img_width - crop_h = int(crop_w / target_ratio) - - left = (img_width - crop_w) / 2 - top = (img_height - crop_h) / 2 + left, top, crop_w, crop_h = _plain_center_crop_box(img_width, img_height, target_width, target_height) if max_x - min_x <= crop_w: if min_x < left: diff --git a/server/app/main.py b/server/app/main.py index 05c0bf0..f6098cd 100644 --- a/server/app/main.py +++ b/server/app/main.py @@ -15,6 +15,7 @@ from PIL import Image from pydantic import BaseModel from . import config, photo_queue +from .face_labels import compute_face_labels from .image_pipeline import render_frame from .immich_client import ImmichClient @@ -163,24 +164,64 @@ def frame_advance(): return Response(content=_render_asset(client, cfg, cfg.current_asset_id), media_type="application/octet-stream") -LOCATION_MAX_LEN = 14 +LOCATION_LINE_MAX_LEN = 14 + +US_STATE_ABBR = { + "alabama": "AL", "alaska": "AK", "arizona": "AZ", "arkansas": "AR", "california": "CA", + "colorado": "CO", "connecticut": "CT", "delaware": "DE", "florida": "FL", "georgia": "GA", + "hawaii": "HI", "idaho": "ID", "illinois": "IL", "indiana": "IN", "iowa": "IA", + "kansas": "KS", "kentucky": "KY", "louisiana": "LA", "maine": "ME", "maryland": "MD", + "massachusetts": "MA", "michigan": "MI", "minnesota": "MN", "mississippi": "MS", "missouri": "MO", + "montana": "MT", "nebraska": "NE", "nevada": "NV", "new hampshire": "NH", "new jersey": "NJ", + "new mexico": "NM", "new york": "NY", "north carolina": "NC", "north dakota": "ND", "ohio": "OH", + "oklahoma": "OK", "oregon": "OR", "pennsylvania": "PA", "rhode island": "RI", "south carolina": "SC", + "south dakota": "SD", "tennessee": "TN", "texas": "TX", "utah": "UT", "vermont": "VT", + "virginia": "VA", "washington": "WA", "west virginia": "WV", "wisconsin": "WI", "wyoming": "WY", + "district of columbia": "DC", +} + +CA_PROVINCE_ABBR = { + "alberta": "AB", "british columbia": "BC", "manitoba": "MB", "new brunswick": "NB", + "newfoundland and labrador": "NL", "northwest territories": "NT", "nova scotia": "NS", + "nunavut": "NU", "ontario": "ON", "prince edward island": "PE", "quebec": "QC", + "saskatchewan": "SK", "yukon": "YT", +} + +US_COUNTRY_NAMES = {"united states", "united states of america", "usa", "us"} +CA_COUNTRY_NAMES = {"canada"} -def _format_location(exif: dict) -> str | None: +def _truncate(text: str, max_len: int) -> str: + if len(text) <= max_len: + return text + return text[: max_len - 3] + "..." + + +def _format_location(exif: dict) -> tuple[str, str] | None: + """Returns (city_line, region_line), each independently truncated to + fit its own corner-overlay line, or None if Immich hasn't geocoded + this photo. region_line is the abbreviated state/province for US/CAN + locations (e.g. "CA", "ON"), else the full country name.""" city = exif.get("city") if not city: return None + state = exif.get("state") country = exif.get("country") - if state: - location = f"{city}, {state}" + country_key = (country or "").strip().lower() + + if state and country_key in US_COUNTRY_NAMES: + region = US_STATE_ABBR.get(state.strip().lower(), state) + elif state and country_key in CA_COUNTRY_NAMES: + region = CA_PROVINCE_ABBR.get(state.strip().lower(), state) elif country: - location = f"{city}, {country}" + region = country + elif state: + region = state else: - location = city - if len(location) > LOCATION_MAX_LEN: - location = location[: LOCATION_MAX_LEN - 3] + "..." - return location + region = "" + + return _truncate(city, LOCATION_LINE_MAX_LEN), _truncate(region, LOCATION_LINE_MAX_LEN) def _format_taken_at(exif: dict) -> str | None: @@ -217,9 +258,11 @@ def frame_photo_info(): raise HTTPException(502, f"Could not reach Immich: {e}") from e exif = asset.get("exifInfo") or {} + location = _format_location(exif) return { "asset_id": cfg.current_asset_id, - "location": _format_location(exif), + "location_line1": location[0] if location else None, + "location_line2": location[1] if location and location[1] else None, "taken_at": _format_taken_at(exif), } @@ -249,6 +292,55 @@ def frame_share(asset_id: str): return RedirectResponse(share_url) +@app.get("/frame/face-labels") +def frame_face_labels(): + """Named-face positions for the manage button's escalated "level 2" + menu -- who's in the current photo, per Immich's own face + recognition (no detection/recognition happens here, see + app/face_labels.py). Response is a flattened, fixed-slot shape + (name_0/x_0/y_0, name_1/x_1/y_1, ...) rather than a JSON array, so + the device's hand-rolled parser can read it with the same flat- + scalar helpers it already has, instead of needing a real array + parser. Empty (count: 0) if no faces are named, or if anything about + fetching them fails -- this is a "nice to have" addition to the + overlay, not worth failing the whole menu over.""" + cfg = config.load() + _require_configured(cfg) + + client = ImmichClient(cfg.immich_url, cfg.immich_api_key) + assets = _list_assets(client, cfg) + + if photo_queue.get_current(cfg, assets): + config.save(cfg) + + if not cfg.current_asset_id: + return {"count": 0} + + try: + faces = client.get_asset_faces(cfg.current_asset_id) + except httpx.HTTPError as e: + logger.warning("Could not fetch faces for asset %s: %s", cfg.current_asset_id, e) + return {"count": 0} + + if not any((face.get("person") or {}).get("name") for face in faces): + return {"count": 0} # skip the extra preview download in the common no-named-faces case + + try: + preview_bytes = client.download_asset_preview(cfg.current_asset_id) + except httpx.HTTPError as e: + logger.warning("Could not download asset %s for face-label mapping: %s", cfg.current_asset_id, e) + return {"count": 0} + + labels = compute_face_labels(preview_bytes, faces, cfg.smart_crop_faces) + + result: dict[str, object] = {"count": len(labels)} + for i, label in enumerate(labels): + result[f"name_{i}"] = label["name"] + result[f"x_{i}"] = label["x"] + result[f"y_{i}"] = label["y"] + return result + + @app.get("/api/queue") def api_queue(): cfg = config.load()