From 9f92ad1b90c45c51f6eeb12d970547d5200af88f Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 8 Sep 2026 11:36:50 +0200 Subject: [PATCH 1/6] fix(web): show startup and background history import progress --- .changeset/clear-history-import-progress.md | 5 ++ go/cmd/ftw/boothealth.go | 76 +++++++++++++---- go/cmd/ftw/boothealth_test.go | 82 ++++++++++++++++++ go/cmd/ftw/main.go | 5 +- web/app.js | 19 ++++- web/boot.html | 49 +++++++++++ web/history-migration.js | 86 +++++++++++++++++++ web/history-migration.test.mjs | 31 +++++++ web/update-badge.js | 94 +++++++++++++++++---- web/update-channel-wiring.test.mjs | 2 +- web/update-dialog.test.mjs | 44 +++++++++- 11 files changed, 456 insertions(+), 37 deletions(-) create mode 100644 .changeset/clear-history-import-progress.md create mode 100644 go/cmd/ftw/boothealth_test.go create mode 100644 web/boot.html create mode 100644 web/history-migration.js create mode 100644 web/history-migration.test.mjs diff --git a/.changeset/clear-history-import-progress.md b/.changeset/clear-history-import-progress.md new file mode 100644 index 000000000..c6664d813 --- /dev/null +++ b/.changeset/clear-history-import-progress.md @@ -0,0 +1,5 @@ +--- +"ftw": patch +--- + +Show startup and background history import progress, keep incomplete history visible, and resume the update view when another client starts the work. diff --git a/go/cmd/ftw/boothealth.go b/go/cmd/ftw/boothealth.go index 946c60d39..cf98e2779 100644 --- a/go/cmd/ftw/boothealth.go +++ b/go/cmd/ftw/boothealth.go @@ -1,8 +1,14 @@ package main import ( + "encoding/json" "net/http" + "os" + "path/filepath" + "strings" "sync/atomic" + + "github.com/srcfl/ftw/go/internal/state" ) // swappableHandler lets the API port be bound before slow boot work (state @@ -28,21 +34,61 @@ func (s *swappableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { (*s.h.Load()).ServeHTTP(w, r) } -// bootPhaseHandler answers health probes 200 while the process initializes, -// so a legitimately slow boot is distinguishable from a dead one. Everything -// else gets 503 + Retry-After so clients and the UI know to come back. +type bootHandler struct { + webDir string + migration atomic.Pointer[json.RawMessage] +} + func bootPhaseHandler() http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/api/health", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"status":"starting","phase":"initializing state"}`)) - }) - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Retry-After", "10") + return newBootPhaseHandler("") +} + +func newBootPhaseHandler(webDir string) *bootHandler { + return &bootHandler{webDir: webDir} +} + +func (b *bootHandler) setMigration(progress state.HistoryMigrationStatus) { + data, err := json.Marshal(progress) + if err != nil { + return + } + snapshot := json.RawMessage(data) + b.migration.Store(&snapshot) +} + +// Health proves process liveness. Every other API stays unavailable until +// the fully wired handler replaces this one. Browser reloads get a progress +// page instead of a JSON error, without changing the updater's readiness test. +func (b *bootHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + if b.webDir != "" && (r.Method == http.MethodGet || r.Method == http.MethodHead) { + if r.URL.Path == "/history-migration.js" { + http.ServeFile(w, r, filepath.Join(b.webDir, "history-migration.js")) + return + } + if !strings.HasPrefix(r.URL.Path, "/api/") { + if page, err := os.ReadFile(filepath.Join(b.webDir, "boot.html")); err == nil { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Retry-After", "2") + w.WriteHeader(http.StatusServiceUnavailable) + if r.Method != http.MethodHead { + _, _ = w.Write(page) + } + return + } + } + } + w.Header().Set("Content-Type", "application/json") + payload := map[string]any{"phase": "initializing state"} + if migration := b.migration.Load(); migration != nil { + payload["migration"] = *migration + } + if r.URL.Path == "/api/health" { + payload["status"] = "starting" + } else { + payload["error"] = "starting" + w.Header().Set("Retry-After", "2") w.WriteHeader(http.StatusServiceUnavailable) - _, _ = w.Write([]byte(`{"error":"starting","phase":"initializing state"}`)) - }) - return mux + } + _ = json.NewEncoder(w).Encode(payload) } diff --git a/go/cmd/ftw/boothealth_test.go b/go/cmd/ftw/boothealth_test.go new file mode 100644 index 000000000..a015963da --- /dev/null +++ b/go/cmd/ftw/boothealth_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/srcfl/ftw/go/internal/state" +) + +func TestBootProgressNeverClaimsAPIReadiness(t *testing.T) { + dir := t.TempDir() + for name, body := range map[string]string{"boot.html": "

Preparing FTW

", "history-migration.js": "export const fixture=true;"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil { + t.Fatal(err) + } + } + boot := newBootPhaseHandler(dir) + boot.setMigration(state.HistoryMigrationStatus{State: "starting", Phase: "seed", RowsDone: 2048}) + for _, path := range []string{"/api/health", "/api/status", "/api/version/update/status", "/", "/history-migration.js"} { + t.Run(path, func(t *testing.T) { + w := httptest.NewRecorder() + boot.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + want := http.StatusServiceUnavailable + if path == "/api/health" || path == "/history-migration.js" { + want = http.StatusOK + } + if w.Code != want { + t.Fatalf("status=%d want=%d", w.Code, want) + } + if w.Header().Get("Cache-Control") != "no-store" { + t.Fatal("startup responses must not be cached") + } + if strings.HasPrefix(path, "/api/") { + var body struct { + Status string + Error string + Migration state.HistoryMigrationStatus + } + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.Migration.RowsDone != 2048 || body.Migration.HistoryComplete { + t.Fatalf("wrong progress: %+v", body) + } + if path == "/api/health" && body.Status != "starting" { + t.Fatal("health must say starting") + } + if path != "/api/health" && body.Error != "starting" { + t.Fatal("full API must stay unavailable") + } + } else if path == "/" && !strings.Contains(w.Body.String(), "Preparing FTW") { + t.Fatal("browser must get startup page") + } + }) + } +} + +func TestBootProgressConcurrentReadersAndUpdates(t *testing.T) { + boot := newBootPhaseHandler("") + var wg sync.WaitGroup + for worker := 0; worker < 4; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := int64(0); i < 50; i++ { + boot.setMigration(state.HistoryMigrationStatus{State: "starting", Phase: "seed", RowsDone: i}) + w := httptest.NewRecorder() + boot.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/health", nil)) + if !json.Valid(w.Body.Bytes()) { + t.Error("partial progress response") + } + } + }() + } + wg.Wait() +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 109526844..5791ea22e 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -432,7 +432,8 @@ func main() { bootPolicy := apiMutationPolicy() bootPolicy.LANAuthEnabled = lanAuth.Enabled bootPolicy.VerifyLANSecret = lanAuth.Verify - apiHandler := newSwappableHandler(bootPhaseHandler()) + boot := newBootPhaseHandler(*webDir) + apiHandler := newSwappableHandler(boot) httpSrv := &http.Server{ Addr: fmt.Sprintf(":%d", cfg.API.Port), Handler: api.WithSecurityHeaders(api.Authenticate(apiHandler, bootPolicy)), @@ -445,7 +446,7 @@ func main() { } }() - st, err := state.OpenWithLegacyHistory(statePath, coldDir) + st, err := state.OpenWithBackgroundHistory(statePath, coldDir, boot.setMigration) if err != nil { slog.Error("open state", "err", err) os.Exit(1) diff --git a/web/app.js b/web/app.js index 2a3ed3609..d94ab9698 100644 --- a/web/app.js +++ b/web/app.js @@ -4,6 +4,11 @@ "use strict"; const POLL_INTERVAL = 2000; // status poll cadence — snappier cards + const historyMigrationUI = import("/history-migration.js").catch(function () { return null; }); + function updateHistoryMigration(health) { + historyMigrationUI.then(function (ui) { if (ui) ui.updateMigrationBanner(health); }).catch(function () {}); + if (updateBadge && typeof updateBadge.setBootHealth === "function") updateBadge.setBootHealth(health); + } // Prices arrive as minor units per kWh; what to call them depends on the // configured currency. window.FTWUnits is set when @@ -2200,13 +2205,21 @@ function fetchStatus() { return Promise.all([ boundedApiRead("/api/status", function (r) { - if (!r.ok) throw new Error("HTTP " + r.status); + if (!r.ok) return r.json().catch(function () { return {}; }).then(function (body) { + var error = new Error("HTTP " + r.status); + error.starting = body.error === "starting"; + throw error; + }); return r.json(); }), boundedApiRead("/api/loadpoints", function (r) { return r.ok ? r.json() : null; }) .catch(function () { return null; }), boundedApiRead("/api/health", function (r) { return r.ok ? r.json() : null; }) - .catch(function () { return null; }), + .catch(function () { return null; }) + .then(function (health) { + updateHistoryMigration(health); + return health; + }), ]) .then(function (results) { var data = results[0]; @@ -2243,7 +2256,7 @@ console.warn("status fetch failed:", e); updateChargingNotice(null); setConnected(false); - if (firstLoad) { showSetupBanner(); } + if (firstLoad && !e.starting) { showSetupBanner(); } }); } diff --git a/web/boot.html b/web/boot.html new file mode 100644 index 000000000..4d4cd94bc --- /dev/null +++ b/web/boot.html @@ -0,0 +1,49 @@ + + + + +FTW is starting + +
+
FTW
+

FTW is starting

Preparing your site. Control has not started yet.

Keep the box powered. This page checks progress automatically.

+

Connecting to the box…

+

Reload status · Reloading this page does not restart the box.

+
+ + diff --git a/web/history-migration.js b/web/history-migration.js new file mode 100644 index 000000000..77ff51bdd --- /dev/null +++ b/web/history-migration.js @@ -0,0 +1,86 @@ +// Shared by the startup page, update dialog and live dashboard. +const count = value => Math.max(0, Number.isFinite(Number(value)) ? Number(value) : 0); +const number = value => count(value).toLocaleString("en-US"); +const escape = value => String(value ?? "").replace(/[&<>"']/g, c => ({ + "&": "&", "<": "<", ">": ">", '"': """, "'": "'", +})[c]); + +export function migrationView(migration, { boot = false, connected = true, now = Date.now() } = {}) { + if (!migration || migration.history_complete === true) return null; + const failed = migration.state === "failed"; + const archive = migration.phase === "parquet"; + const total = count(archive ? migration.current_source_rows_total : migration.rows_total); + const done = count(archive ? migration.current_source_rows_done : migration.rows_done); + const progress = total > 0 ? { value: Math.min(done, total), max: total, label: archive ? "Current archive file" : "History import progress" } : null; + const details = []; + if (total > 0) details.push(`${archive ? "Current archive: " : ""}${number(done)} of ${number(total)} readings imported`); + if ((archive || !total) && count(migration.rows_done) > 0) details.push(`${number(migration.rows_done)} readings imported in total`); + if (count(migration.files_total) > 0) details.push(`${number(migration.files_done)} of ${number(migration.files_total)} archive files complete`); + const from = Number(migration.incomplete_from_ms), until = Number(migration.incomplete_until_ms); + const coverage = Number.isFinite(from) && Number.isFinite(until) && from > 0 && until >= from + ? `History may be incomplete from ${new Date(from).toLocaleDateString("en-GB")} to ${new Date(until).toLocaleDateString("en-GB")}.` + : "Older history is not complete yet."; + const updated = Number(migration.updated_at_ms); + const age = Number.isFinite(updated) && updated > 0 ? Math.max(0, Math.floor((now - updated) / 1000)) : null; + const elapsed = age < 60 ? age : Math.floor(age / 60); + const unit = age < 60 ? "second" : "minute"; + const activity = age === null ? "Waiting for the first import report." + : `Last import report ${elapsed} ${unit}${elapsed === 1 ? "" : "s"} ago.`; + return { + title: !connected ? "History import status unavailable" : failed ? "History import paused" : boot ? "Preparing FTW" : "Importing older history", + description: !connected ? "The box is not responding. Showing its last import report." + : failed ? "The import needs attention. Your original history files are still kept." + : boot ? "FTW is preparing the data it needs to start. Control has not started yet." + : "Core is running while FTW imports older readings in the background.", + details: details.join(" · "), coverage, activity, progress, + error: failed ? migration.last_error || "The box could not finish importing history." : "", + guidance: failed || !connected ? "Keep the original data and backup. Reload this page to check the current status." + : "Keep the box powered. You can close this page and return later. Import progress is saved so it can resume after a restart.", + }; +} + +export function migrationHTML(migration, options) { + const view = migrationView(migration, options); + if (!view) return ""; + const progress = view.progress + ? `` + : ``; + const heading = options?.boot ? "h1" : "h3"; + return `
+ <${heading} role="status">${escape(view.title)}

${escape(view.description)}

+ ${progress}

${escape(view.details)}

${escape(view.coverage)}

+

${escape(view.activity)}

+ ${view.error ? `

${escape(view.error)}

` : ""} +

${escape(view.guidance)}

`; +} + +let lastMigration = null; +let lastHealth = null; +export function updateMigrationBanner(health) { + const current = health?.history_storage?.migration || health?.migration; + if (current && (current.history_complete === true || !lastMigration || count(current.updated_at_ms) >= count(lastMigration.updated_at_ms))) lastMigration = current; + if (health) lastHealth = health; + const view = migrationView(lastMigration, { boot: lastHealth?.status === "starting", connected: !!health }); + let banner = document.getElementById("history-import-banner"); + if (!view) { banner?.remove(); return; } + if (!banner) { + const main = document.querySelector("main"); + if (!main) return; + banner = document.createElement("section"); + banner.id = "history-import-banner"; + banner.className = "storage-banner"; + banner.setAttribute("aria-label", "History import"); + banner.style.cssText = "display:block;text-align:left;padding:16px 24px"; + banner.innerHTML = '

'; + main.parentNode.insertBefore(banner, main); + } + for (const key of ["title", "description", "details", "coverage", "activity", "error", "guidance"]) { + const element = banner.querySelector(`[data-field="${key}"]`); + if (element.textContent !== view[key]) element.textContent = view[key]; + element.hidden = !view[key]; + } + const progress = banner.querySelector("progress"); + progress.setAttribute("aria-label", view.progress?.label || "History import progress"); + if (view.progress) { progress.max = view.progress.max; progress.value = view.progress.value; } + else progress.removeAttribute("value"); +} diff --git a/web/history-migration.test.mjs b/web/history-migration.test.mjs new file mode 100644 index 000000000..e1dedadc3 --- /dev/null +++ b/web/history-migration.test.mjs @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { migrationHTML, migrationView } from "./history-migration.js"; + +const running = { state:"running", phase:"parquet", history_complete:false, files_done:3, files_total:12, rows_done:6000, updated_at_ms:100000 }; +test("unknown totals stay indeterminate and incomplete history stays explicit", () => { + const view = migrationView(running, {now:130000}); + assert.equal(view.progress, null); + assert.match(view.details, /6,000 readings.*3 of 12/); + assert.match(view.coverage, /not complete yet/); + assert.equal(view.activity, "Last import report 30 seconds ago."); + assert.match(migrationHTML(running), /<\/progress>/); +}); +test("completed history, not a row percentage, removes the notice", () => { + assert.ok(migrationView({...running, rows_total:6000})); + assert.equal(migrationView({...running, history_complete:true}), null); +}); +test("startup, live background import and lost contact make distinct claims", () => { + assert.match(migrationView(running, {boot:true}).description, /Control has not started/); + assert.match(migrationView(running).description, /Core is running/); + assert.match(migrationView(running, {connected:false}).description, /last import report/); + assert.doesNotMatch(migrationView(running, {connected:false}).description, /Core is running/); +}); +test("failure keeps incomplete coverage and escapes diagnostic text", () => { + const failed = {...running, state:"failed", last_error:'bad '}; + assert.equal(migrationView(failed).title, "History import paused"); + const html = migrationHTML(failed); + assert.doesNotMatch(html, /