From 15e37c77cd06e34dc4af8a32189be47e5800c87e Mon Sep 17 00:00:00 2001 From: Thomas Faour Date: Wed, 22 Jul 2026 19:06:49 -0400 Subject: [PATCH] Simplify firmware: manage overlay now composited server-side Deletes manage_qr_overlay.c/.h; frame_client.c's manage-button flow is now one fetch with &manage=1 instead of on-device QR/text generation plus separate photo-info/face-labels requests. --- firmware/main/CMakeLists.txt | 2 +- firmware/main/frame_client.c | 397 ++++------------------------------- firmware/main/frame_client.h | 13 +- 3 files changed, 54 insertions(+), 358 deletions(-) diff --git a/firmware/main/CMakeLists.txt b/firmware/main/CMakeLists.txt index 1a44032..a1c3140 100644 --- a/firmware/main/CMakeLists.txt +++ b/firmware/main/CMakeLists.txt @@ -1,3 +1,3 @@ -idf_component_register(SRCS main.c wifi_provisioning.c frame_client.c qr_onboarding.c status_screen.c epd_draw.c next_button.c back_button.c combo_button.c manage_qr_overlay.c battery.c ota_update.c board_antenna.c +idf_component_register(SRCS main.c wifi_provisioning.c frame_client.c qr_onboarding.c status_screen.c epd_draw.c next_button.c back_button.c combo_button.c battery.c ota_update.c board_antenna.c PRIV_REQUIRES esp_event nvs_flash esp_wifi esp_netif esp_http_server esp_http_client mbedtls dns_server epd7in3e qrcode epaper_fonts esp_driver_gpio esp_adc esp_https_ota app_update esp_app_format EMBED_FILES root.html) diff --git a/firmware/main/frame_client.c b/firmware/main/frame_client.c index 1ee16d4..ea1cfb8 100644 --- a/firmware/main/frame_client.c +++ b/firmware/main/frame_client.c @@ -16,7 +16,6 @@ #include "epd7in3e.h" #include "status_screen.h" -#include "manage_qr_overlay.h" #include "combo_button.h" #include "ota_update.h" #include "board_antenna.h" @@ -314,35 +313,6 @@ static bool json_extract_uint(const char *json, const char *key, uint32_t *out) return true; } -/* Same as json_extract_uint(), but signed -- e.g. battery_percent's -1 - * ("no reading") sentinel. strtoul() would silently wrap a leading '-' - * into a huge unsigned value instead of failing, so this needs its own - * strtol()-based parse rather than reusing json_extract_uint(). */ -static bool json_extract_int(const char *json, const char *key, int *out) -{ - char needle[48]; - snprintf(needle, sizeof(needle), "\"%s\"", key); - const char *pos = strstr(json, needle); - if (pos == NULL) { - return false; - } - pos = strchr(pos, ':'); - if (pos == NULL) { - return false; - } - pos++; - while (*pos == ' ') { - pos++; - } - char *end; - long value = strtol(pos, &end, 10); - if (end == pos) { - return false; - } - *out = (int)value; - return true; -} - /* Finds the string value associated with "key" in a small, flat JSON * blob, e.g. "San Francisco, CA" in {"location": "San Francisco, CA"}. * Same rationale as json_extract_uint() -- not a general parser. Returns @@ -455,248 +425,35 @@ static frame_server_config_t fetch_frame_config(const frame_config_t *cfg) return result; } -/* GETs the server's /frame/photo-info for the manage-button overlay: - * location/taken_at text (left empty if the server didn't have them -- - * e.g. no GPS EXIF to geocode, or no capture date), a share_url built - * from the returned asset_id (same construction pattern as - * run_fetch_cycle()'s management_url), and the last battery percent this - * frame reported (-1 if none yet). The overlay uses that last-known - * value rather than a fresh local reading -- it's needed before this - * photo is composited and pushed to the panel, i.e. before this cycle's - * own reading (taken later, right before it's reported -- see - * frame_client_run()) even exists yet. Any failure (unreachable, no - * current photo, etc.) just leaves all outputs empty/-1 -- the caller - * treats that as "skip these optional overlay regions", not a hard - * error, since the base "scan to manage" QR should still show. */ -static void fetch_photo_info(const frame_config_t *cfg, char *location_line1, size_t location_line1_size, - char *location_line2, size_t location_line2_size, char *taken_at, size_t taken_at_size, - char *share_url, size_t share_url_size, int *battery_percent) -{ - location_line1[0] = '\0'; - location_line2[0] = '\0'; - taken_at[0] = '\0'; - share_url[0] = '\0'; - *battery_percent = -1; - - char url[256]; - build_url(url, sizeof(url), cfg, "frame/photo-info"); - - /* CONFIG_FRAME_FETCH_TIMEOUT_MS, not the shorter SERVER_CHECK one: - * unlike fetch_frame_config() (always called after the image fetch - * has already warmed the connection, see frame_client_run()), this - * is the *first* network call of the wake cycle whenever the manage - * menu is opened -- same cold-connection latency spike that made - * the short timeout unreliable for /frame/config before, now worse - * with a real TLS handshake on top. Confirmed on hardware: this - * timed out under CONFIG_FRAME_SERVER_CHECK_TIMEOUT_MS while the - * rest of the cycle (a fresh connection, but not the *first* one) - * succeeded fine. */ - esp_http_client_config_t config = { - .url = url, - .method = HTTP_METHOD_GET, - .timeout_ms = CONFIG_FRAME_FETCH_TIMEOUT_MS, - .crt_bundle_attach = esp_crt_bundle_attach, - }; - esp_http_client_handle_t client = esp_http_client_init(&config); - - esp_err_t err = esp_http_client_open(client, 0); - if (err != ESP_OK) { - ESP_LOGW(TAG, "'%s' not reachable: %s", url, esp_err_to_name(err)); - esp_http_client_cleanup(client); - return; - } - - int status = esp_http_client_fetch_headers(client) >= 0 ? esp_http_client_get_status_code(client) : -1; - if (status != 200) { - ESP_LOGW(TAG, "'%s' returned HTTP %d", url, status); - esp_http_client_close(client); - esp_http_client_cleanup(client); - return; - } - - char body[384]; - int total = 0; - int n; - while (total < (int)sizeof(body) - 1 && - (n = esp_http_client_read(client, body + total, sizeof(body) - 1 - total)) > 0) { - total += n; - } - body[total] = '\0'; - - esp_http_client_close(client); - esp_http_client_cleanup(client); - - json_extract_string(body, "location_line1", location_line1, location_line1_size); - json_extract_string(body, "location_line2", location_line2, location_line2_size); - json_extract_string(body, "taken_at", taken_at, taken_at_size); - json_extract_int(body, "battery_percent", battery_percent); - - char asset_id[48]; - if (json_extract_string(body, "asset_id", asset_id, sizeof(asset_id))) { - char path[80]; - snprintf(path, sizeof(path), "frame/share/%s", asset_id); - build_url(share_url, share_url_size, cfg, path); - } -} - -/* GETs the server's /frame/face-labels for the manage-button's escalated - * "level 2" menu -- named-face positions, if Immich has any for the - * current photo. Response is a flattened, fixed-slot shape ("count", - * then name_0/x_0/y_0, name_1/x_1/y_1, ...) rather than a real JSON - * array, read with the same flat-scalar helpers as everywhere else in - * this file instead of needing an actual array parser. Any failure - * (unreachable, malformed response, etc.) just returns 0 -- named faces - * are a "nice to have" addition to the menu, not worth failing it over. */ -static int fetch_face_labels(const frame_config_t *cfg, manage_face_label_t *out, int max_labels) -{ - char url[256]; - build_url(url, sizeof(url), cfg, "frame/face-labels"); - - /* Same reasoning as fetch_photo_info() -- this is a manage-menu - * request too, not a warmed-connection reachability check. */ - esp_http_client_config_t config = { - .url = url, - .method = HTTP_METHOD_GET, - .timeout_ms = CONFIG_FRAME_FETCH_TIMEOUT_MS, - .crt_bundle_attach = esp_crt_bundle_attach, - }; - esp_http_client_handle_t client = esp_http_client_init(&config); - - esp_err_t err = esp_http_client_open(client, 0); - if (err != ESP_OK) { - ESP_LOGW(TAG, "'%s' not reachable: %s", url, esp_err_to_name(err)); - esp_http_client_cleanup(client); - return 0; - } - - int status = esp_http_client_fetch_headers(client) >= 0 ? esp_http_client_get_status_code(client) : -1; - if (status != 200) { - ESP_LOGW(TAG, "'%s' returned HTTP %d", url, status); - esp_http_client_close(client); - esp_http_client_cleanup(client); - return 0; - } - - char body[768]; - int total = 0; - int n; - while (total < (int)sizeof(body) - 1 && - (n = esp_http_client_read(client, body + total, sizeof(body) - 1 - total)) > 0) { - total += n; - } - body[total] = '\0'; - - esp_http_client_close(client); - esp_http_client_cleanup(client); - - uint32_t count = 0; - json_extract_uint(body, "count", &count); - /* Unsigned compare: casting count to int first let a server-supplied - * value >= 2^31 (still a perfectly ordinary decimal in the JSON) go - * negative, skipping this clamp entirely and driving the loop below - * with the full attacker/server-controlled count -- out[found] is a - * fixed MANAGE_FACE_LABELS_MAX-element caller stack array. */ - if (count > (uint32_t)max_labels) { - count = (uint32_t)max_labels; - } - - int found = 0; - for (uint32_t i = 0; i < count; i++) { - char key[16]; - snprintf(key, sizeof(key), "name_%u", (unsigned)i); - if (!json_extract_string(body, key, out[found].name, sizeof(out[found].name))) { - continue; - } - snprintf(key, sizeof(key), "x_%u", (unsigned)i); - uint32_t x; - if (!json_extract_uint(body, key, &x)) { - continue; - } - snprintf(key, sizeof(key), "y_%u", (unsigned)i); - uint32_t y; - if (!json_extract_uint(body, key, &y)) { - continue; - } - out[found].x = (int)x; - out[found].y = (int)y; - found++; - } - return found; -} - typedef struct { esp_http_client_handle_t client; - size_t stream_pos; /* running absolute offset into the frame, for overlay splicing */ - const manage_overlay_set_t *overlay; /* NULL = no overlay this fetch */ } http_read_ctx_t; -/* Splices one overlay region's pixels over the real photo bytes in chunk - * wherever chunk's absolute byte range [chunk_start, chunk_start+chunk_len) - * within the full frame intersects that region's rectangle. Rows/chunks - * outside the region's footprint are left completely untouched. - * region->x0 is always even (see manage_qr_overlay.h), so byte_x0 below - * is exact. */ -static void splice_overlay_region(uint8_t *chunk, size_t chunk_len, size_t chunk_start, - const manage_overlay_region_t *region) -{ - int byte_x0 = region->x0 / 2; - int byte_w = region->w / 2; - size_t chunk_end = chunk_start + chunk_len; - - for (int row = region->y0; row < region->y0 + region->h; row++) { - size_t row_start = (size_t)row * EPD_BYTES_PER_ROW + (size_t)byte_x0; - size_t row_end = row_start + (size_t)byte_w; - - size_t lo = row_start > chunk_start ? row_start : chunk_start; - size_t hi = row_end < chunk_end ? row_end : chunk_end; - if (lo >= hi) { - continue; - } - - size_t region_row_offset = (size_t)(row - region->y0) * (size_t)byte_w + (lo - row_start); - memcpy(chunk + (lo - chunk_start), region->buf + region_row_offset, hi - lo); - } -} - -static void splice_overlay(uint8_t *chunk, size_t chunk_len, size_t chunk_start, const manage_overlay_set_t *overlay) -{ - for (int i = 0; i < overlay->count; i++) { - splice_overlay_region(chunk, chunk_len, chunk_start, &overlay->regions[i]); - } -} - /* Pulls the next chunk straight out of the in-progress HTTP response -- * epd_write_frame() calls this to feed the panel without ever holding - * the full ~192KB frame in RAM. Splices in ctx->overlay's regions (if - * set) as chunks pass through, so the panel driver never needs to know - * an overlay exists at all. */ + * the full ~192KB frame in RAM. Just a plain relay: the manage overlay + * (scan-to-manage QR, battery, location/date, share-QR, named face + * labels) is composited server-side now (see server/app/manage_overlay.py), + * baked into the same image bytes as any other render -- this function, + * like the rest of this file, has no idea an overlay exists. */ static size_t http_read_fn(uint8_t *chunk, size_t chunk_size, void *ctx_) { http_read_ctx_t *ctx = (http_read_ctx_t *)ctx_; int n = esp_http_client_read(ctx->client, (char *)chunk, (int)chunk_size); - if (n <= 0) { - return 0; - } - - if (ctx->overlay != NULL) { - splice_overlay(chunk, (size_t)n, ctx->stream_pos, ctx->overlay); - } - ctx->stream_pos += (size_t)n; - - return (size_t)n; + return n > 0 ? (size_t)n : 0; } /* GETs /frame/image (FETCH_NORMAL), or POSTs /frame/advance or * /frame/back to force a move in either direction (FETCH_ADVANCE / - * FETCH_BACK -- the next-photo / back-photo buttons), and streams the - * response directly into the panel, splicing in overlay's pixels (if - * non-NULL) as it streams. Returning non-ESP_OK means the panel was - * never actually refreshed -- epd_display_stream() (see epd7in3e.c) - * refuses to trigger a physical refresh on a short/wrong-size stream, - * so a failure here always leaves the visible screen exactly as it - * was. */ -static esp_err_t fetch_and_display(const frame_config_t *cfg, fetch_action_t action, - const manage_overlay_set_t *overlay) + * FETCH_BACK -- the next-photo / back-photo buttons). manage=true (the + * manage button) appends &manage=1, telling the server to bake its + * overlay into this same response instead of returning the bare + * content -- see server/app/routers/device.py. Returning non-ESP_OK + * means the panel was never actually refreshed -- epd_display_stream() + * (see epd7in3e.c) refuses to trigger a physical refresh on a short/ + * wrong-size stream, so a failure here always leaves the visible screen + * exactly as it was. */ +static esp_err_t fetch_and_display(const frame_config_t *cfg, fetch_action_t action, bool manage) { const char *path = "frame/image"; if (action == FETCH_ADVANCE) { @@ -707,6 +464,12 @@ static esp_err_t fetch_and_display(const frame_config_t *cfg, fetch_action_t act char url[256]; build_url(url, sizeof(url), cfg, path); + if (manage) { + size_t len = strlen(url); + if (len + strlen("&manage=1") < sizeof(url)) { + strcpy(url + len, "&manage=1"); + } + } esp_http_client_config_t config = { .url = url, @@ -733,7 +496,7 @@ static esp_err_t fetch_and_display(const frame_config_t *cfg, fetch_action_t act } ESP_LOGI(TAG, "Fetching frame (%d bytes) from '%s'", content_length, url); - http_read_ctx_t ctx = { .client = client, .overlay = overlay }; + http_read_ctx_t ctx = { .client = client }; uint32_t crc = 0; err = epd_write_frame(http_read_fn, &ctx, &crc); @@ -761,8 +524,7 @@ static esp_err_t fetch_and_display(const frame_config_t *cfg, fetch_action_t act return err; } -#define MANAGE_MENU_MAX_LEVEL 2 -#define MANAGE_MENU_LEVEL_TIMEOUT_MS 30000 +#define MANAGE_MENU_TIMEOUT_MS 30000 #define MANAGE_MENU_POLL_MS 150 #define MANAGE_MENU_DEBOUNCE_MS 30 @@ -794,108 +556,40 @@ static bool wait_for_button_press(uint32_t timeout_ms) return false; } -/* Builds and shows one level of the manage menu: level 1 is the base - * overlay (management QR + location/date/share-QR wherever the server - * had that data); level 2 adds named-face labels on top. action only - * applies at level 1 -- escalating to level 2 redisplays the same - * photo, so it never re-advances/-backs. */ -static esp_err_t show_menu_level(const frame_config_t *cfg, fetch_action_t action, int level) -{ - char management_url[256]; - build_url(management_url, sizeof(management_url), cfg, ""); - - char location_line1[32]; - char location_line2[32]; - char taken_at[32]; - /* Wider than the other URL buffers in this file: unlike a fixed path, - * this one stacks toolsserver (up to 128) + "/frame/share/" + an - * asset_id (up to 47) + "?token=" + an access_token (up to 64) -- - * worst case ~266 bytes, which a 256-byte buffer could silently - * truncate the token off of (build_url()'s bounds check avoids an - * overflow, but a truncated/dropped token still means the resulting - * request just 401s with no obvious cause). */ - char share_url[320]; - int battery_percent; - fetch_photo_info(cfg, location_line1, sizeof(location_line1), location_line2, - sizeof(location_line2), taken_at, sizeof(taken_at), share_url, sizeof(share_url), - &battery_percent); - - manage_face_label_t face_labels[MANAGE_FACE_LABELS_MAX]; - int face_label_count = 0; - if (level >= 2) { - face_label_count = fetch_face_labels(cfg, face_labels, MANAGE_FACE_LABELS_MAX); - } - - manage_overlay_content_t content = { - .management_url = management_url, - .location_line1 = location_line1[0] != '\0' ? location_line1 : NULL, - .location_line2 = location_line2[0] != '\0' ? location_line2 : NULL, - .taken_at = taken_at[0] != '\0' ? taken_at : NULL, - .share_url = share_url[0] != '\0' ? share_url : NULL, - .face_labels = face_labels, - .face_label_count = face_label_count, - .battery_percent = battery_percent, - }; - - manage_overlay_set_t overlay; - esp_err_t err = manage_overlay_render(&content, &overlay); - if (err != ESP_OK) { - manage_overlay_free(&overlay); - return err; - } - - err = fetch_and_display(cfg, action, &overlay); - manage_overlay_free(&overlay); - return err; -} - -/* Runs the manage-button menu: level 1 (the base overlay) shows first; - * from there, each further press within 30s escalates one level (up to - * MANAGE_MENU_MAX_LEVEL, which adds named-face labels), and a press once - * already at the max level exits immediately instead of escalating - * further. A 30s timeout at any level also exits. Device stays awake - * throughout (doesn't sleep the panel or the chip). Returns non-ESP_OK - * only if the very first (level 1) render/fetch failed; failures after - * that (escalating, or the final revert) are logged but don't count as - * an overall failure -- something was already shown successfully, which - * was the point of the button. */ +/* Runs the manage-button view: fetches once with manage=1 (the server + * bakes its whole overlay -- scan-to-manage QR, battery, location/date/ + * share-QR, every named face label, no more RAM-driven cap on how many -- + * into the response), shows it, then waits up to 30s for either another + * press or the timeout before reverting to a plain fetch. Device stays + * awake throughout (doesn't sleep the panel or the chip). Returns + * non-ESP_OK only if the manage fetch itself failed; a revert failure + * after that is logged but doesn't count as an overall failure -- + * something was already shown successfully, which was the point of the + * button. */ static esp_err_t run_management_menu(const frame_config_t *cfg, fetch_action_t action) { - int level = 1; - esp_err_t err = show_menu_level(cfg, action, level); + esp_err_t err = fetch_and_display(cfg, action, true); if (err != ESP_OK) { - ESP_LOGW(TAG, "Could not render management overlay (%s), showing photo normally", esp_err_to_name(err)); - return fetch_and_display(cfg, action, NULL); + ESP_LOGW(TAG, "Could not fetch manage view (%s), showing photo normally", esp_err_to_name(err)); + return fetch_and_display(cfg, action, false); } - for (;;) { - ESP_LOGI(TAG, "Showing management menu level %d, waiting up to 30s", level); - bool pressed = wait_for_button_press(MANAGE_MENU_LEVEL_TIMEOUT_MS); - if (!pressed || level >= MANAGE_MENU_MAX_LEVEL) { - break; /* timeout at any level, or a press while already maxed out -- exit */ - } - level++; - esp_err_t level_err = show_menu_level(cfg, FETCH_NORMAL, level); - if (level_err != ESP_OK) { - ESP_LOGW(TAG, "Could not render menu level %d (%s), reverting", level, esp_err_to_name(level_err)); - break; - } - } + ESP_LOGI(TAG, "Showing manage view, waiting up to 30s"); + wait_for_button_press(MANAGE_MENU_TIMEOUT_MS); - esp_err_t revert_err = fetch_and_display(cfg, FETCH_NORMAL, NULL); + esp_err_t revert_err = fetch_and_display(cfg, FETCH_NORMAL, false); if (revert_err != ESP_OK) { - ESP_LOGW(TAG, "Failed to revert management overlay (%s)", esp_err_to_name(revert_err)); + ESP_LOGW(TAG, "Failed to revert manage view (%s)", esp_err_to_name(revert_err)); } return ESP_OK; } /* Runs the appropriate fetch for this cycle: a plain fetch, or -- if - * show_management_qr -- the escalating manage menu (see - * run_management_menu()). */ + * show_management_qr -- the manage view (see run_management_menu()). */ static esp_err_t run_fetch_cycle(const frame_config_t *cfg, fetch_action_t action, bool show_management_qr) { if (!show_management_qr) { - return fetch_and_display(cfg, action, NULL); + return fetch_and_display(cfg, action, false); } return run_management_menu(cfg, action); } @@ -1011,9 +705,10 @@ void frame_client_run(const frame_config_t *cfg, fetch_action_t action, bool sho esp_ota_mark_app_valid_cancel_rollback(); /* Read now, not at boot: the photo (and, if shown, the manage - * overlay -- which gets its own battery number from the server's - * last-known value, not a local reading, see fetch_photo_info()) - * is already on the panel, so there's no display deadline to beat. + * overlay -- entirely server-composited now, using the server's + * own last-known battery value, not a local reading, see + * server/app/manage_overlay.py) is already on the panel, so + * there's no display deadline to beat. * Reading here instead of right after waking sidesteps taking the * ADC sample while the rail's still settling from whatever the * boot/reset just did, with no need to guess a settle delay -- diff --git a/firmware/main/frame_client.h b/firmware/main/frame_client.h index 2d2e4c8..f1bc442 100644 --- a/firmware/main/frame_client.h +++ b/firmware/main/frame_client.h @@ -33,12 +33,13 @@ esp_err_t frame_wifi_connect_sta(const frame_config_t *cfg); * deep-sleep until the next refresh. * * If show_management_qr is true (the manage button was held), the - * displayed photo gets a small "scan to manage" QR overlay in the - * top-right corner linking to the server's config page, held for 30 - * seconds (the device stays awake), then reverted back to the plain - * photo before proceeding to the normal sleep-interval logic. Its - * battery indicator shows the server's last-known reading, not a fresh - * one -- see fetch_photo_info() in frame_client.c. + * request for that cycle carries &manage=1, and the server bakes its + * whole manage overlay (scan-to-manage QR, battery, location/date, + * share-QR, named face labels) directly into the image it returns -- + * see server/app/manage_overlay.py; this device is otherwise unaware + * any of that exists, it just displays whatever comes back. Held for 30 + * seconds (the device stays awake), then reverted back to a plain fetch + * before proceeding to the normal sleep-interval logic. * * Reads the battery (see battery_read_percent()) itself, once, after the * photo is already on the panel, and reports it to the server on a