From c8846da8f8dfe69d1ac0ad1611bb56c83201faf4 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sun, 15 Nov 2026 14:49:13 +0100 Subject: [PATCH 1/2] feat(support): make the support file answer for the last week A support file sent at 12:54 could not say why the battery discharged overnight: its log covered 75 minutes and its telemetry one hour. The box held the night in storage; the file did not take it. The help report gains "What happened over the last week", per day (06-22) and night (22-06): battery charge range, battery, car and grid energy, replans and why, plan slots missed and the energy gap, fuse time, devices out of contact, fallback plans, warnings and Core starts, with what stands out stated first. The archive carries week/: every series in five-minute buckets, the control journal, plan snapshots with solver fields, notifications and start/stop events. Warnings and errors are counted per message and hour in a new state.db table, log_digest, so a week of them outlasts the log ring and a restart. A seeded week of 113 series gave a 2.8 MB archive. The redacted config and logs no longer show the owner's e-mail address or account username. `ftw support` waits up to two minutes. Co-Authored-By: Claude Opus 5.5 --- .changeset/support-file-week.md | 16 + go/cmd/ftw/log_digest.go | 62 ++ go/cmd/ftw/main.go | 5 + go/internal/api/api_drivers_debug.go | 59 +- go/internal/api/api_drivers_debug_test.go | 4 +- go/internal/api/api_support_dump_zip_test.go | 75 ++ go/internal/api/api_support_report.go | 23 +- go/internal/api/api_support_week.go | 988 +++++++++++++++++++ go/internal/api/api_support_week_test.go | 306 ++++++ go/internal/ftwcli/backup.go | 6 +- go/internal/state/diagnostics.go | 51 + go/internal/state/log_digest.go | 101 ++ go/internal/state/log_digest_test.go | 112 +++ go/internal/state/store.go | 15 + go/internal/telemetry/logring.go | 177 +++- go/internal/telemetry/logring_test.go | 95 ++ web/diagnostics-modal.js | 2 +- web/plan.js | 5 +- 18 files changed, 2074 insertions(+), 28 deletions(-) create mode 100644 .changeset/support-file-week.md create mode 100644 go/cmd/ftw/log_digest.go create mode 100644 go/internal/api/api_support_week.go create mode 100644 go/internal/api/api_support_week_test.go create mode 100644 go/internal/state/log_digest.go create mode 100644 go/internal/state/log_digest_test.go create mode 100644 go/internal/telemetry/logring_test.go diff --git a/.changeset/support-file-week.md b/.changeset/support-file-week.md new file mode 100644 index 000000000..4c43c868a --- /dev/null +++ b/.changeset/support-file-week.md @@ -0,0 +1,16 @@ +--- +"ftw": minor +--- + +The support file now answers for the last seven days, not the last hour. +Its help report gains "What happened over the last week": per day and +night, the battery's lowest and highest charge, energy in and out of the +battery, to the car and over the grid, how often the plan changed and why, +how far the battery strayed from the plan, fuse limits, devices out of +contact, plans that fell back to Core and restarts, with the notable ones +stated first. The archive carries the week behind it in `week/`: every +series in five-minute buckets, the control journal, plan snapshots, +notifications, start and stop events, and a week of warnings and errors +counted per hour, which the box now stores so they outlast the log buffer +and a restart. The redacted config and logs no longer show the owner's +e-mail address or account username. diff --git a/go/cmd/ftw/log_digest.go b/go/cmd/ftw/log_digest.go new file mode 100644 index 000000000..e208d42c7 --- /dev/null +++ b/go/cmd/ftw/log_digest.go @@ -0,0 +1,62 @@ +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +const ( + // logDigestEvery is how often warning counts reach the store. A crash + // loses at most this much of the digest. + logDigestEvery = time.Minute + // logDigestKeep covers the support file's week with a day to spare. + logDigestKeep = 8 * 24 * time.Hour +) + +// runLogDigest writes the log ring's warning and error counts to the store +// until ctx ends, then writes once more. +func runLogDigest(ctx context.Context, ring *telemetry.LogRing, st *state.Store) { + t := time.NewTicker(logDigestEvery) + defer t.Stop() + var prunedAt time.Time + for { + select { + case <-ctx.Done(): + flushLogDigest(context.Background(), ring, st, time.Now()) + return + case now := <-t.C: + flushLogDigest(ctx, ring, st, now) + if now.Sub(prunedAt) < 24*time.Hour { + continue + } + prunedAt = now + if _, err := st.PruneLogDigest(ctx, now.Add(-logDigestKeep).UnixMilli()); err != nil { + slog.Warn("log digest: prune failed", "err", err) + } + } + } +} + +func flushLogDigest(ctx context.Context, ring *telemetry.LogRing, st *state.Store, now time.Time) { + entries := ring.TakeDigest(now) + if len(entries) == 0 { + return + } + rows := make([]state.LogDigest, len(entries)) + for i, e := range entries { + rows[i] = state.LogDigest{ + HourMs: e.Hour.UnixMilli(), Level: e.Level, Driver: e.Driver, Pattern: e.Pattern, + Msg: e.Msg, Attrs: e.Attrs, Count: int64(e.Count), + FirstMs: e.First.UnixMilli(), LastMs: e.Last.UnixMilli(), + } + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := st.AddLogDigest(ctx, rows); err != nil { + slog.Warn("log digest: write failed", "rows", len(rows), "err", err) + } +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index aef6324fb..fb6b04e3c 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -2730,6 +2730,9 @@ func main() { // in control?", the one GET /api/status serves. notifSvc.SetControlReader(srv.ControlStates) ctrlJournal.start(ctx, srv.JournalVerdicts, st) + // Warning and error counts per hour, so the support file covers a week + // of them after the log ring has turned over or the box has restarted. + go runLogDigest(ctx, logRing, st) // FuseReader: on each HealthTick the fuse_over_limit rule reads // the site meter's live per-phase currents from telemetry and // compares against cfg.Fuse.MaxAmps. Closes over cfg + cfgMu so @@ -2938,6 +2941,7 @@ func main() { cloudLink.Bye(cloudwire.ByeShutdown, cloudByeWait) markNativeCleanExit(nativeRoot) ctrlJournal.shutdown() + flushLogDigest(context.Background(), logRing, st, time.Now()) flushHistoryOnStop(st) if err := st.RecordEvent("shutdown"); err != nil { slog.Warn("failed to persist shutdown event", "err", err) @@ -2954,6 +2958,7 @@ func main() { cloudLink.Bye(cloudwire.ByeUpdate, cloudByeWait) markNativeCleanExit(nativeRoot) ctrlJournal.shutdown() + flushLogDigest(context.Background(), logRing, st, time.Now()) flushHistoryOnStop(st) if err := st.RecordEvent("restart"); err != nil { slog.Warn("failed to persist restart event", "err", err) diff --git a/go/internal/api/api_drivers_debug.go b/go/internal/api/api_drivers_debug.go index 5f5608bc0..424707773 100644 --- a/go/internal/api/api_drivers_debug.go +++ b/go/internal/api/api_drivers_debug.go @@ -738,12 +738,14 @@ func (s *Server) handleGlobalLogs(w http.ResponseWriter, r *http.Request) { } // GET /api/support/dump — zip archive with everything a developer needs -// to triage a support incident: redacted config, full driver health JSON, -// recent global + per-driver logs (secrets and IPv4 addresses stripped), -// last 1 h of TS samples per (driver, metric), and a -// manifest. SQLite is NOT included; the dump is intended to be small -// enough to attach to a chat message — measured at ~6 kB on a two-driver -// install. +// to triage a support incident: the help report, redacted config, driver +// health, the log ring (secrets and IPv4 addresses stripped), the last hour +// of samples at stored resolution, and seven days from the box's own +// storage in week/: every series in five-minute buckets, the control +// journal, plan snapshots, notifications, start and stop events and the +// hourly warning counts. SQLite is NOT included. A seeded week with five +// drivers, 113 series, 6,000 journal spans and 1,050 plans gave a 2.8 MB +// archive, small enough to mail; week/series/ is three quarters of it. // // Zip rather than tar.gz because this file's whole purpose is to be // handed to somebody else. Windows and every chat client open a zip @@ -782,7 +784,8 @@ func (s *Server) handleSupportDump(w http.ResponseWriter, r *http.Request) { // below it is what you reach for when the report does not settle the // question. Shipping them together means the person asking for help // sends one file and never has to pick the right one. - addFile("00-help-report.md", []byte(redactDumpLog(s.buildSupportReport(r.Context(), now)))) + week := s.loadSupportWeek(r.Context(), now, true) + addFile("00-help-report.md", []byte(redactDumpLog(s.buildSupportReportWith(r.Context(), now, week)))) manifest := map[string]any{ "generated_at": now.Format(time.RFC3339), @@ -800,7 +803,20 @@ func (s *Server) handleSupportDump(w http.ResponseWriter, r *http.Request) { "loadpoints.json", "logs/global.log", "logs/.log", + "logs/warnings-week.log", "telemetry/__.csv", + "week/summary.json", + "week/series/__.csv", + "week/control-journal.json", + "week/plans.json", + "week/notifications.json", + "week/events.json", + }, + "covers": map[string]string{ + "logs/*.log except warnings-week.log": "the in-memory log ring: the latest lines, often under an hour", + "logs/warnings-week.log": "seven days of warnings and errors, one line per message pattern and hour", + "telemetry/": "the last hour of samples at stored resolution", + "week/": "seven days from the box's storage; series in five-minute buckets with min, max and sample count", }, } manifestBody, _ := json.MarshalIndent(manifest, "", " ") @@ -831,9 +847,11 @@ func (s *Server) handleSupportDump(w http.ResponseWriter, r *http.Request) { addFile("logs/"+sanitizeName(d)+".log", []byte(formatLogs(entries))) } - // Last 1 h of TS samples per (driver, metric) — small enough to - // keep the bundle email-able. Only metrics the operator actually - // emitted in this window appear. + addWeekFiles(addFile, week) + + // Last 1 h of TS samples per (driver, metric) at stored resolution, + // next to week/'s five-minute buckets. Only metrics the operator + // actually emitted in this window appear. if s.deps.State != nil { untilMs := now.UnixMilli() sinceMs := untilMs - 3600*1000 @@ -927,8 +945,8 @@ func (s *Server) redactedConfig() []byte { } // redactSensitive walks an unmarshalled JSON tree and replaces any value -// at a key whose lowercased name matches a sensitive substring. Recursive, -// in-place. +// at a key whose lowercased name matches a sensitive substring, and any +// e-mail address wherever it sits. Recursive, in-place. func redactSensitive(v any) { switch x := v.(type) { case map[string]any: @@ -938,10 +956,18 @@ func redactSensitive(v any) { x[k] = "***" continue } + if str, ok := val.(string); ok { + x[k] = dumpEmailRe.ReplaceAllString(str, "[email omitted]") + continue + } redactSensitive(val) } case []any: - for _, e := range x { + for i, e := range x { + if str, ok := e.(string); ok { + x[i] = dumpEmailRe.ReplaceAllString(str, "[email omitted]") + continue + } redactSensitive(e) } } @@ -955,6 +981,9 @@ func isSensitiveKey(k string) bool { for _, s := range []string{ "password", "passwd", "secret", "token", "api_key", "apikey", "private_key", "client_secret", "authorization", "credential", + // Who the owner is, not how they sign in: a cloud account's + // username is often their e-mail address. + "email", "e-mail", "username", "user_name", } { if strings.Contains(k, s) { return true @@ -985,9 +1014,10 @@ var ( dumpJSONSecretRe = regexp.MustCompile(`(?i)("([^"\\]+)"\s*:\s*)("(?:\\.|[^"\\])*")`) dumpFormSecretRe = regexp.MustCompile(`(?i)\b([A-Za-z][A-Za-z0-9_-]*)=([^\s&"]+)`) dumpBasicAuthRe = regexp.MustCompile(`(?i)basic\s+[A-Za-z0-9+/=_-]+`) + dumpEmailRe = regexp.MustCompile(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}`) ) -// redactDumpLog removes IPv4 addresses and credentials, including JSON and +// redactDumpLog removes IPv4 addresses, e-mail addresses and credentials, including JSON and // form fields whose keys isSensitiveKey would catch. Stricter on OAuth // bodies that Lua logs as `HTTP %d: %s`. func redactDumpLog(s string) string { @@ -995,6 +1025,7 @@ func redactDumpLog(s string) string { return s } s = dumpIPv4Re.ReplaceAllString(s, "[ip omitted]") + s = dumpEmailRe.ReplaceAllString(s, "[email omitted]") s = dumpBearerRe.ReplaceAllString(s, "Bearer [omitted]") s = dumpBasicAuthRe.ReplaceAllString(s, "Basic [omitted]") s = dumpJSONSecretRe.ReplaceAllStringFunc(s, func(m string) string { diff --git a/go/internal/api/api_drivers_debug_test.go b/go/internal/api/api_drivers_debug_test.go index 87d8eae0e..a0e850d1f 100644 --- a/go/internal/api/api_drivers_debug_test.go +++ b/go/internal/api/api_drivers_debug_test.go @@ -587,6 +587,8 @@ func TestIsSensitiveKey(t *testing.T) { "refresh_token", "access_token", "api_key", "apikey", "private_key", "authorization", "Authorization", "credential", "credentials", "auth", "AUTH", "auth_header", "x-auth", + // Who the owner is: a cloud account's username is often an e-mail. + "username", "ev_charger_username", "email", "Email", "user_name", } for _, k := range sensitive { if !isSensitiveKey(k) { @@ -594,7 +596,7 @@ func TestIsSensitiveKey(t *testing.T) { } } keep := []string{ - "host", "username", "name", "oauth", "oauth_client_id", "client_id", + "host", "user", "name", "oauth", "oauth_client_id", "client_id", "port", "lua", } for _, k := range keep { diff --git a/go/internal/api/api_support_dump_zip_test.go b/go/internal/api/api_support_dump_zip_test.go index cb554464f..1cbe2bdd8 100644 --- a/go/internal/api/api_support_dump_zip_test.go +++ b/go/internal/api/api_support_dump_zip_test.go @@ -205,3 +205,78 @@ func TestSupportDumpRedactsSecretsInLogsAndConfig(t *testing.T) { t.Error("dump must not include state.db") } } + +// A support file from a real site carried the owner's e-mail twice: the +// cloud charger's username and a driver's `email` setting. Neither is a +// password, so key-based secret redaction let both through. The address +// identifies the household wherever it sits, so it is removed by key and +// by shape, in the config and in the logs. +func TestSupportDumpRemovesTheOwnersEmail(t *testing.T) { + const ( + email = "bjorn.example@example.se" + username = "bjorn-cloud-login" + inLog = "owner.name@example.org" + ) + tel := telemetry.NewStore() + ring := telemetry.NewLogRing() + ring.Append(telemetry.LogEntry{Level: "WARN", Driver: "easee", Msg: "login failed for " + inLog}) + cfg := &config.Config{ + Drivers: []config.Driver{{ + Name: "easee", Lua: "easee_cloud.lua", + Config: map[string]any{"email": email, "site": "home", "note": "contact " + email}, + }}, + EVCharger: &config.EVCharger{Provider: "easee", Username: username, Serial: "EH123456"}, + } + srv := New(&Deps{ + Ctrl: control.NewState(0, 50, "meter"), CtrlMu: &sync.Mutex{}, + Tel: tel, LogRing: ring, Version: "test", + Cfg: cfg, CfgMu: &sync.RWMutex{}, + }) + files := readSupportZip(t, srv) + for name, body := range files { + for _, leak := range []string{email, username, inLog} { + if strings.Contains(body, leak) { + t.Errorf("%s leaked %q", name, leak) + } + } + } + cfgJSON := files["config.redacted.yaml"] + if !strings.Contains(cfgJSON, `"site": "home"`) || !strings.Contains(cfgJSON, "EH123456") { + t.Errorf("redaction removed settings that are not personal: %s", cfgJSON) + } +} + +// readSupportZip fetches /api/support/dump and returns each file by its path +// inside the archive's top folder. +func readSupportZip(t *testing.T, srv *Server) map[string]string { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/api/support/dump", nil) + rec := httptest.NewRecorder() + srv.Handler().ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("GET /api/support/dump = %d, want 200", rec.Code) + } + body := rec.Body.Bytes() + zr, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + t.Fatalf("archive does not open as a zip: %v", err) + } + out := map[string]string{} + for _, f := range zr.File { + rc, err := f.Open() + if err != nil { + t.Fatalf("open %s: %v", f.Name, err) + } + raw, err := io.ReadAll(rc) + rc.Close() + if err != nil { + t.Fatalf("read %s: %v", f.Name, err) + } + name := f.Name + if i := strings.IndexByte(name, '/'); i >= 0 { + name = name[i+1:] + } + out[name] = string(raw) + } + return out +} diff --git a/go/internal/api/api_support_report.go b/go/internal/api/api_support_report.go index 9741f1f08..55c5b5b65 100644 --- a/go/internal/api/api_support_report.go +++ b/go/internal/api/api_support_report.go @@ -203,6 +203,12 @@ func (s *Server) liveNow(ctrl control.State, now time.Time) liveSnapshot { } func (s *Server) buildSupportReport(ctx context.Context, now time.Time) string { + return s.buildSupportReportWith(ctx, now, s.loadSupportWeek(ctx, now, false)) +} + +// buildSupportReportWith writes the report around a week already read, so +// the support archive reads storage once for the report and its files. +func (s *Server) buildSupportReportWith(ctx context.Context, now time.Time, week *supportWeek) string { s.deps.CtrlMu.Lock() ctrl := *s.deps.Ctrl targets := append([]control.DispatchTarget{}, s.deps.Ctrl.LastTargets...) @@ -227,6 +233,7 @@ func (s *Server) buildSupportReport(ctx context.Context, now time.Time) string { var b strings.Builder writeReportHeader(&b, s.deps.Version, now) writeFindings(&b, findings) + writeWeekSection(&b, week) if feedback, err := json.MarshalIndent(s.controlFeedback(now), "", " "); err == nil { b.WriteString("## Command feedback\n\nDriver-call results, readbacks and measured power are separate evidence. Missing values are unknown.\n\n```json\n") b.Write(feedback) @@ -272,10 +279,11 @@ func writeReportHeader(b *strings.Builder, version string, now time.Time) { "(importing, charging the battery); **negative W = power out** " + "(exporting, discharging). This holds in every table below.\n\n") b.WriteString("Read `Findings` first — it lists what looks wrong without " + - "needing the tables. `Right now` is what the system is doing at this " + - "instant; `Plan` is what it intends to do next. Those two answer " + - "different questions and often disagree, which is usually the point " + - "of the question.\n\n") + "needing the tables. `What happened over the last week` answers " + + "questions about the past seven days, night by night. `Right now` is " + + "what the system is doing at this instant; `Plan` is what it intends " + + "to do next. Those two answer different questions and often disagree, " + + "which is usually the point of the question.\n\n") } func writeFindings(b *strings.Builder, findings []finding) { @@ -688,9 +696,10 @@ func (s *Server) writeLogSection(b *strings.Builder) { func writeReportFooter(b *strings.Builder) { b.WriteString("---\n\n") - b.WriteString("Deeper data lives in `/api/support/dump` (logs, telemetry, " + - "redacted config) and `/api/mpc/diagnose` (every slot the optimizer " + - "saw). This report deliberately omits both to stay readable.\n") + b.WriteString("Deeper data lives in `/api/support/dump` (logs, a week of " + + "series, control journal, plans and warnings, redacted config) and " + + "`/api/mpc/diagnose` (every slot the optimizer saw). This report " + + "deliberately omits both to stay readable.\n") } // ---- findings ---- diff --git a/go/internal/api/api_support_week.go b/go/internal/api/api_support_week.go new file mode 100644 index 000000000..0f372eb9b --- /dev/null +++ b/go/internal/api/api_support_week.go @@ -0,0 +1,988 @@ +package api + +// The support file's week. +// +// A tester asked why the battery emptied overnight, and the support file +// could not say: its log covered 75 minutes and its telemetry one hour. The +// box keeps far more than that. This reads seven days back from its own +// storage (series, control journal, plan snapshots, notifications, start +// and stop events, warning counts), sums them per day and night for the +// help report, and ships them in lower resolution in the archive's week/. + +import ( + "context" + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +const ( + supportWeekSpan = 7 * 24 * time.Hour + // supportWeekStep is the series resolution. Five-minute buckets keep + // their min/max envelope, so a short spike survives, and a site with a + // hundred series still makes a file that can be mailed. + supportWeekStep = 5 * time.Minute + // supportWeekBudget bounds the reads. The archive must finish well inside + // the HTTP write timeout on a Raspberry Pi; what is left out is named. + supportWeekBudget = 60 * time.Second + // Day and night split at these local hours, so "last night" is one row. + supportDayStartHour = 6 + supportNightStartHour = 22 + + supportWeekSpanLimit = 20000 + supportWeekReplanLimit = 10000 + supportWeekNoticeLimit = 2000 + supportWeekDigestLimit = 5000 + supportWeekEventLimit = 1000 + + // supportPlanGapWh is the battery energy over a period, beyond what the + // plan's own slots asked for, worth a warning. One kilowatt-hour is a + // tenth of a typical home battery, past rounding and slot-edge effects. + supportPlanGapWh = 1000 + // supportNightNoteWh is the night discharge worth listing even when it + // followed the plan, because it is what people ask about. + supportNightNoteWh = 500 + // supportSourceDownMin is the offline time a device needs before the + // week calls it out. Shorter stretches are polls that came back. + supportSourceDownMin = 5 * time.Minute + // supportRepeatWarnings is how often one warning has to recur in the + // week before it is a finding of its own. + supportRepeatWarnings = 100 + // supportBusyReplans: a period with this many replans and three times + // the median is unusual enough to name. + supportBusyReplans = 30 +) + +// supportPeriod is one day (06–22) or night (22–06) in household time. +type supportPeriod struct { + Start, End time.Time + Night bool + // Partial marks the period that is still running. + Partial bool +} + +// Label names the period the way a person asks about it. +func (p supportPeriod) Label() string { + if !p.Night { + return p.Start.Format("Mon 2 Jan") + ", day" + } + next := p.Start.AddDate(0, 0, 1) + from := p.Start.Format("Mon 2") + if next.Month() != p.Start.Month() { + from = p.Start.Format("Mon 2 Jan") + } + return from + " → " + next.Format("Mon 2 Jan") + ", night" +} + +// supportWeekPeriods splits the last seven days at 06:00 and 22:00 in now's +// zone, from the first boundary inside the window to now. Clock changes +// make a night eight or ten hours long rather than moving the boundary. +func supportWeekPeriods(now time.Time) []supportPeriod { + loc := now.Location() + since := now.Add(-supportWeekSpan) + var bounds []time.Time + y, m, d := since.Date() + for day := time.Date(y, m, d, 0, 0, 0, 0, loc); !day.After(now); day = day.AddDate(0, 0, 1) { + for _, h := range []int{supportDayStartHour, supportNightStartHour} { + b := time.Date(day.Year(), day.Month(), day.Day(), h, 0, 0, 0, loc) + if !b.Before(since) && b.Before(now) { + bounds = append(bounds, b) + } + } + } + out := make([]supportPeriod, 0, len(bounds)) + for i, b := range bounds { + p := supportPeriod{Start: b, End: now, Night: b.Hour() == supportNightStartHour, Partial: true} + if i+1 < len(bounds) { + p.End, p.Partial = bounds[i+1], false + } + out = append(out, p) + } + return out +} + +type supportSeries struct { + Driver, Metric, Unit string + Points []state.SeriesPoint +} + +type supportCount struct { + Name string `json:"name"` + Count int `json:"count"` +} + +// supportPeriodSummary is one row of the week. Nil means not measured. +type supportPeriodSummary struct { + period supportPeriod + Label string `json:"label"` + StartMs int64 `json:"start_ms"` + EndMs int64 `json:"end_ms"` + Night bool `json:"night"` + Partial bool `json:"partial,omitempty"` + SoCMinPct *float64 `json:"soc_min_pct"` + SoCMaxPct *float64 `json:"soc_max_pct"` + BatChargedWh *float64 `json:"battery_charged_wh"` + BatDischargedWh *float64 `json:"battery_discharged_wh"` + ImportWh *float64 `json:"import_wh"` + ExportWh *float64 `json:"export_wh"` + // EVWh integrates the chargers' five-minute series, so it is close, + // not metered. + EVWh *float64 `json:"ev_wh_approx"` + // The plan's battery energy over the slots Core scored in the period, + // against what the batteries did in those slots. Out is discharge. + SlotsScored int `json:"slots_scored"` + SlotsOffPlan int `json:"slots_off_plan"` + PlannedInWh float64 `json:"planned_in_wh"` + PlannedOutWh float64 `json:"planned_out_wh"` + ActualInWh float64 `json:"actual_in_wh"` + ActualOutWh float64 `json:"actual_out_wh"` + SlotGapWh float64 `json:"slot_gap_wh"` + Replans int `json:"replans"` + ReplanReasons []supportCount `json:"replan_reasons,omitempty"` + Fallbacks int `json:"fallbacks"` + FuseLimitedS float64 `json:"fuse_limited_s"` + SourceDown []supportCount `json:"sources_down_s,omitempty"` // seconds per driver + Warnings int64 `json:"warnings"` + Errors int64 `json:"errors"` + Starts int `json:"core_starts"` + UncleanStarts int `json:"core_starts_without_clean_stop"` + EnergyIncomplete bool `json:"energy_incomplete,omitempty"` +} + +// supportWeek is everything the week reads, loaded once per request. +type supportWeek struct { + Since, Until time.Time + Series []supportSeries + Spans []state.ControlSpan + SpansMore bool + Replans []state.PlannerReplan + Notices []state.NotificationEntry + NoticesMore bool + Events []state.Event + // EventBefore is the last start or stop event before the window, so + // the first start inside it can be judged. Nil on a new box. + EventBefore *state.Event + // FirstData is the earliest stored series point in the window. Zero + // when there is none. + FirstData time.Time + Digest []state.LogDigest + DigestMore bool + Periods []supportPeriodSummary + // Gaps names what could not be read, so a missing row is not mistaken + // for a quiet week. + Gaps []string +} + +func supportWeekPoints() int { return int(supportWeekSpan / supportWeekStep) } + +// supportSummaryMetric is a series the period table reads. The help report +// alone loads only these; the archive loads every series. +func supportSummaryMetric(metric string) bool { + return metric == telemetry.DerBattery.String()+"_soc" || metric == telemetry.DerEV.String()+"_w" +} + +// loadSupportWeek reads the last seven days from storage. allSeries loads +// every (driver, metric) series for the archive. It returns nil without a +// store. +func (s *Server) loadSupportWeek(ctx context.Context, now time.Time, allSeries bool) *supportWeek { + if s.deps.State == nil { + return nil + } + now = now.In(time.Local) + ctx, cancel := context.WithTimeout(ctx, supportWeekBudget) + defer cancel() + st := s.deps.State + w := &supportWeek{Since: now.Add(-supportWeekSpan), Until: now} + sinceMs, untilMs := w.Since.UnixMilli(), w.Until.UnixMilli() + gap := func(what string, err error) { + if err != nil { + w.Gaps = append(w.Gaps, what+": "+err.Error()) + } + } + + var err error + w.Spans, w.SpansMore, err = st.ControlSpans(ctx, state.ControlSpanQuery{SinceMs: sinceMs, UntilMs: untilMs, Limit: supportWeekSpanLimit}) + gap("control journal", err) + w.Replans, err = st.PlannerReplans(ctx, sinceMs, untilMs, supportWeekReplanLimit) + gap("plan snapshots", err) + w.Notices, w.NoticesMore, err = st.NotificationsBetween(ctx, sinceMs, untilMs, "", supportWeekNoticeLimit) + gap("notifications", err) + w.Digest, w.DigestMore, err = st.LogDigestBetween(ctx, sinceMs, untilMs, supportWeekDigestLimit) + gap("warning counts", err) + events, err := st.RecentEvents(supportWeekEventLimit) + gap("start and stop events", err) + for i := len(events) - 1; i >= 0; i-- { // newest first from the store + e := events[i] + switch { + case e.TsMs >= sinceMs && e.TsMs <= untilMs: + w.Events = append(w.Events, e) + case e.TsMs < sinceMs && isStartStop(e.Event): + w.EventBefore = &e + } + } + + units := s.metricUnits() + drivers, err := st.DriverNames() + gap("series drivers", err) + metrics, err := st.MetricNames() + gap("series metrics", err) + sort.Strings(drivers) + sort.Strings(metrics) + skipped := 0 + for _, d := range drivers { + for _, m := range metrics { + if !allSeries && !supportSummaryMetric(m) { + continue + } + if ctx.Err() != nil { + skipped++ + continue + } + pts, err := st.LoadSeriesBucketsOrRawContext(ctx, d, m, sinceMs, untilMs, supportWeekPoints()) + if err != nil { + if ctx.Err() == nil { + gap("series "+d+"/"+m, err) + } else { + skipped++ + } + continue + } + if len(pts) > 0 { + w.Series = append(w.Series, supportSeries{Driver: d, Metric: m, Unit: units[m], Points: pts}) + firstMs := pts[0].TsMs // a bucket's newest sample + if pts[0].FirstMS > 0 { + firstMs = pts[0].FirstMS + } + if first := time.UnixMilli(firstMs); w.FirstData.IsZero() || first.Before(w.FirstData) { + w.FirstData = first + } + } + } + } + if skipped > 0 { + w.Gaps = append(w.Gaps, fmt.Sprintf("series: %d left out when the %s read budget ran out", skipped, supportWeekBudget)) + } + + for _, p := range supportWeekPeriods(now) { + w.Periods = append(w.Periods, s.summarisePeriod(ctx, w, p)) + } + if ctx.Err() != nil { + w.Gaps = append(w.Gaps, "energy per period: the read budget ran out") + } + return w +} + +func (s *Server) summarisePeriod(ctx context.Context, w *supportWeek, p supportPeriod) supportPeriodSummary { + startMs, endMs := p.Start.UnixMilli(), p.End.UnixMilli() + out := supportPeriodSummary{period: p, Label: p.Label(), StartMs: startMs, EndMs: endMs, Night: p.Night, Partial: p.Partial} + in := func(ts int64) bool { return ts >= startMs && ts < endMs } + + if ctx.Err() == nil { + if de, err := s.deps.State.DailyEnergy(startMs, endMs); err == nil { + if de.Intervals == 0 { + de.Missing |= state.MissingPower + } + out.BatChargedWh = measured(de.Missing, state.MissingBattery, de.BatChargedWh) + out.BatDischargedWh = measured(de.Missing, state.MissingBattery, de.BatDischargedWh) + out.ImportWh = measured(de.Missing, state.MissingGrid, de.ImportWh) + out.ExportWh = measured(de.Missing, state.MissingGrid, de.ExportWh) + out.EnergyIncomplete = de.Missing&(state.MissingBattery|state.MissingGrid) != 0 + } + } + + widthH := float64(state.BucketWidthMs(w.Since.UnixMilli(), w.Until.UnixMilli(), supportWeekPoints())) / 3_600_000 + for _, ser := range w.Series { + switch { + case ser.Metric == telemetry.DerBattery.String()+"_soc": + for _, pt := range ser.Points { + if !in(pt.TsMs) { + continue + } + out.SoCMinPct = minPtr(out.SoCMinPct, pt.Min*100) + out.SoCMaxPct = maxPtr(out.SoCMaxPct, pt.Max*100) + } + case ser.Metric == telemetry.DerEV.String()+"_w": + for _, pt := range ser.Points { + if in(pt.TsMs) { + v := math.Max(pt.V, 0) * widthH + if out.EVWh == nil { + out.EVWh = new(float64) + } + *out.EVWh += v + } + } + } + } + + down := map[string]float64{} + for _, sp := range w.Spans { + end := sp.EndMs + if end == 0 { + end = w.Until.UnixMilli() + } + if sp.Kind == "slot" { + if end <= startMs || end > endMs { + continue + } + var e struct { + PlannedWh float64 `json:"planned_wh"` + ActualWh float64 `json:"actual_wh"` + } + if json.Unmarshal(sp.First, &e) != nil { + continue + } + out.SlotsScored++ + if sp.Status != "on_target" { + out.SlotsOffPlan++ + } + out.PlannedInWh += math.Max(e.PlannedWh, 0) + out.PlannedOutWh += math.Max(-e.PlannedWh, 0) + out.ActualInWh += math.Max(e.ActualWh, 0) + out.ActualOutWh += math.Max(-e.ActualWh, 0) + out.SlotGapWh += math.Abs(e.PlannedWh - e.ActualWh) + continue + } + overlap := float64(min(end, endMs)-max(sp.StartMs, startMs)) / 1000 + if overlap <= 0 { + continue + } + if sp.Reason == "fuse_limit" || sp.Reason == "fuse_cooldown" { + out.FuseLimitedS += overlap + } + if sp.Kind == "source" && (sp.Status == "offline" || sp.Status == "fault") { + down[sp.Driver+" "+sp.Status] += overlap + } + } + out.SourceDown = sortedCounts(down) + + reasons := map[string]float64{} + for _, r := range w.Replans { + if !in(r.TsMs) { + continue + } + out.Replans++ + reasons[r.Reason]++ + if r.Fallback { + out.Fallbacks++ + } + } + out.ReplanReasons = sortedCounts(reasons) + + for _, g := range w.Digest { + if !in(g.FirstMs) { + continue + } + if g.Level == "ERROR" { + out.Errors += g.Count + } else { + out.Warnings += g.Count + } + } + + prev := w.EventBefore + for i := range w.Events { + e := w.Events[i] + if !isStartStop(e.Event) { + continue + } + if e.Event == "startup" && in(e.TsMs) { + out.Starts++ + // A start right after another start had no stop between them. + // The very first start of a new box has nothing before it. + if prev != nil && prev.Event == "startup" { + out.UncleanStarts++ + } + } + prev = &w.Events[i] + } + return out +} + +func isStartStop(event string) bool { + return event == "startup" || event == "shutdown" || event == "restart" +} + +func measured(m state.HistoryMissing, flag state.HistoryMissing, v float64) *float64 { + if m&flag != 0 { + return nil + } + return &v +} + +func minPtr(p *float64, v float64) *float64 { + if p == nil || v < *p { + return &v + } + return p +} + +func maxPtr(p *float64, v float64) *float64 { + if p == nil || v > *p { + return &v + } + return p +} + +func sortedCounts(m map[string]float64) []supportCount { + out := make([]supportCount, 0, len(m)) + for k, v := range m { + out = append(out, supportCount{Name: k, Count: int(math.Round(v))}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + return out[i].Name < out[j].Name + }) + return out +} + +// ---- the report section ---- + +func writeWeekSection(b *strings.Builder, w *supportWeek) { + b.WriteString("## What happened over the last week\n\n") + if w == nil { + b.WriteString("The box has no store to read the week from.\n\n") + return + } + fmt.Fprintf(b, "From %s to %s, household time, split into days (06–22) and nights (22–06). "+ + "Built from the box's stored history, control journal, plan snapshots, notifications and "+ + "warning counts, not from the log buffer, which holds about the last hour.\n\n", + w.Since.Format("Mon 2 Jan 15:04"), w.Until.Format("Mon 2 Jan 15:04")) + + findings := weekFindings(w) + if len(findings) == 0 { + b.WriteString("Nothing in the stored week stands out.\n\n") + } else { + writeFindingList(b, findings) + } + + b.WriteString("| Period | Charge | Battery in / out | EV | Grid in / out | Plan vs battery | Replans | Limits and faults |\n") + b.WriteString("|---|---|---|---|---|---|---|---|\n") + periods := w.Periods + if empty := periodsBeforeData(w); empty > 0 { + fmt.Fprintf(b, "| %s to %s | no stored data | | | | | | |\n", + periods[0].period.Start.Format("Mon 2 Jan 15:04"), periods[empty-1].period.End.Format("Mon 2 Jan 15:04")) + periods = periods[empty:] + } + for _, p := range periods { + label := p.Label + if p.Partial { + label += " (so far)" + } + fmt.Fprintf(b, "| %s | %s | %s / %s | %s | %s / %s | %s | %s | %s |\n", + label, fmtSoCRange(p.SoCMinPct, p.SoCMaxPct), + fmtWhPtr(p.BatChargedWh), fmtWhPtr(p.BatDischargedWh), fmtEVWh(p.EVWh), + fmtWhPtr(p.ImportWh), fmtWhPtr(p.ExportWh), + fmtPlanVsBattery(p), fmtReplans(p), fmtLimits(p)) + } + b.WriteString("\n`Plan vs battery` compares the battery energy each scored plan slot asked for " + + "with what the batteries did in it; `off` counts slots outside half to one and a half " + + "times the plan, or in the wrong direction. EV energy is integrated from five-minute " + + "averages, so it is close rather than metered. `—` means not measured.\n\n") + + if top := topDigest(w.Digest, 5); len(top) > 0 { + b.WriteString("Most repeated warnings and errors this week (numbers folded to `#`):\n\n```\n") + for _, g := range top { + fmt.Fprintf(b, "%s ×%d", g.Level, g.Count) + if g.Driver != "" { + fmt.Fprintf(b, " [%s]", g.Driver) + } + fmt.Fprintf(b, " %s (last %s)\n", redactDumpLog(truncateReport(g.Msg, 160)), time.UnixMilli(g.LastMs).In(time.Local).Format("Mon 15:04")) + } + b.WriteString("```\n\n") + } + var notes []string + if w.SpansMore { + notes = append(notes, fmt.Sprintf("the control journal had more than %d spans; the oldest are left out", supportWeekSpanLimit)) + } + if w.DigestMore { + notes = append(notes, fmt.Sprintf("more than %d warning rows; the oldest are left out", supportWeekDigestLimit)) + } + notes = append(notes, w.Gaps...) + if len(notes) > 0 { + b.WriteString("Not complete: " + strings.Join(notes, "; ") + ".\n\n") + } + b.WriteString("The data behind this table is in `week/` in `/api/support/dump`.\n\n") +} + +func writeFindingList(b *strings.Builder, findings []finding) { + order := map[string]int{sevProblem: 0, sevWarning: 1, sevNote: 2} + sort.SliceStable(findings, func(i, j int) bool { + return order[findings[i].Severity] < order[findings[j].Severity] + }) + for _, f := range findings { + fmt.Fprintf(b, "- **%s — %s.** %s\n", f.Severity, f.Title, f.Detail) + } + b.WriteString("\n") +} + +// weekFindings states what stands out in the week, so the section reads +// without the table. +func weekFindings(w *supportWeek) []finding { + var out []finding + var followed []string + for _, p := range w.Periods { + label := p.Label + if p.Partial { + label += " (so far)" + } + charge := "" + if p.SoCMinPct != nil { + charge = fmt.Sprintf(" (charge %s)", fmtSoCRange(p.SoCMinPct, p.SoCMaxPct)) + } + discharged := p.ActualOutWh + if p.BatDischargedWh != nil { + discharged = *p.BatDischargedWh + } + charged := p.ActualInWh + if p.BatChargedWh != nil { + charged = *p.BatChargedWh + } + if p.SlotsScored > 0 { + var parts []string + if d := p.ActualOutWh - p.PlannedOutWh; math.Abs(d) >= supportPlanGapWh { + parts = append(parts, fmt.Sprintf("discharged %s%s, %s than the plan asked for: its %d scored slots planned %s out and measured %s out", + fmtReportWh(discharged), charge, moreOrLess(d), p.SlotsScored, fmtReportWh(p.PlannedOutWh), fmtReportWh(p.ActualOutWh))) + } + if d := p.ActualInWh - p.PlannedInWh; math.Abs(d) >= supportPlanGapWh { + parts = append(parts, fmt.Sprintf("charged %s, %s than the plan asked for: planned %s in, measured %s in", + fmtReportWh(charged), moreOrLess(d), fmtReportWh(p.PlannedInWh), fmtReportWh(p.ActualInWh))) + } + if len(parts) > 0 { + detail := "The battery " + strings.Join(parts, "; and ") + "." + if p.Replans > 0 { + detail += " " + fmtReplans(p) + " that period." + } + detail += " Safety limits, a device that does not follow its command, or a plan replaced mid-slot " + + "look like this; the control journal in `week/` shows which." + out = append(out, finding{sevWarning, "The battery did not follow the plan on " + label, detail}) + continue + } + } + if p.Night && discharged >= supportNightNoteWh { + s := fmt.Sprintf("%s %s%s", label, fmtReportWh(discharged), charge) + if p.SlotsScored > 0 { + s += fmt.Sprintf(", plan %s", fmtReportWh(p.PlannedOutWh)) + } + followed = append(followed, s) + } + } + if len(followed) > 0 { + out = append(out, finding{sevNote, "Night discharge", + "The battery discharged at night: " + strings.Join(followed, "; ") + + ". Where a plan figure is given, the battery stayed within " + fmtReportWh(supportPlanGapWh) + " of it."}) + } + + // Devices that lost contact or reported a fault. + type down struct { + total, longest float64 + from int64 + } + downs := map[string]*down{} + for _, sp := range w.Spans { + if sp.Kind != "source" || (sp.Status != "offline" && sp.Status != "fault") { + continue + } + end := sp.EndMs + if end == 0 { + end = w.Until.UnixMilli() + } + start := max(sp.StartMs, w.Since.UnixMilli()) + secs := float64(end-start) / 1000 + if secs <= 0 { + continue + } + key := sp.Driver + "\x00" + sp.Status + d := downs[key] + if d == nil { + d = &down{} + downs[key] = d + } + d.total += secs + if secs > d.longest { + d.longest, d.from = secs, start + } + } + keys := make([]string, 0, len(downs)) + for k := range downs { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + d := downs[k] + if time.Duration(d.total*float64(time.Second)) < supportSourceDownMin { + continue + } + driver, status, _ := strings.Cut(k, "\x00") + verb, sev, title := "was offline", sevWarning, driver+" lost contact this week" + effect := "A device out of contact gets its safe default mode and is left out of the plan, so its share becomes grid flow." + if status == "fault" { + verb, sev, title = "reported a fault", sevProblem, driver+" reported a fault this week" + effect = "A device in fault can be reached but cannot act, so the plan leaves it out and its share becomes grid flow." + } + out = append(out, finding{sev, title, + fmt.Sprintf("%s %s for %s in total; the longest stretch, %s, began %s. %s", + driver, verb, fmtHours(d.total), fmtHours(d.longest), time.UnixMilli(d.from).In(time.Local).Format("Mon 2 Jan 15:04"), effect)}) + } + var fuseS, worstFuse float64 + worstLabel := "" + for _, p := range w.Periods { + fuseS += p.FuseLimitedS + if p.FuseLimitedS > worstFuse { + worstFuse, worstLabel = p.FuseLimitedS, p.Label + } + } + if fuseS >= 60 { + out = append(out, finding{sevNote, "The fuse limit held power back", + fmt.Sprintf("Charging or discharging was held under the main fuse for %s this week, most on %s. "+ + "That is protection working; it explains slower charging, not a fault.", fmtHours(fuseS), worstLabel)}) + } + + fallbacks, last := 0, state.PlannerReplan{} + for _, r := range w.Replans { + if r.Fallback { + fallbacks++ + last = r + } + } + if fallbacks > 0 { + out = append(out, finding{sevWarning, "Plans fell back to Core", + fmt.Sprintf("%d of %d plans this week came from Core's own planner instead of the optimizer. "+ + "The last, %s: %s.", fallbacks, len(w.Replans), + time.UnixMilli(last.TsMs).In(time.Local).Format("Mon 2 Jan 15:04"), + strings.TrimSuffix(redactDumpLog(truncateReport(last.FallbackReason, 200)), "."))}) + } + + if busy, ok := busyPeriod(w.Periods); ok { + out = append(out, finding{sevNote, "Many replans on " + busy.Label, + fmt.Sprintf("%s. Frequent replans usually follow a forecast that keeps missing, "+ + "a battery that drifts from its planned charge, or a device that comes and goes.", fmtReplans(busy))}) + } + + var starts, unclean int + var startTimes []string + for _, p := range w.Periods { + starts += p.Starts + unclean += p.UncleanStarts + } + for _, e := range w.Events { + if e.Event == "startup" { + startTimes = append(startTimes, time.UnixMilli(e.TsMs).In(time.Local).Format("Mon 15:04")) + } + } + stops := 0 + for _, e := range w.Events { + if e.Event == "restart" || e.Event == "shutdown" { + stops++ + } + } + switch { + case unclean > 0: + out = append(out, finding{sevWarning, "Core stopped without a clean shutdown", + fmt.Sprintf("Core started %s this week (%s); %d of those came after no recorded stop, "+ + "which is a crash, a power cut or a killed process.", plural(starts, "time"), strings.Join(startTimes, ", "), unclean)}) + case starts > 0 || stops > 0: + out = append(out, finding{sevNote, "Core restarted", + fmt.Sprintf("Core recorded %s (restart or shutdown) and %s this week%s. Each restart replans "+ + "and briefly hands devices their default mode.", plural(stops, "stop"), plural(starts, "start"), listSuffix(startTimes))}) + } + + if top := topDigest(w.Digest, 1); len(top) > 0 && top[0].Count >= supportRepeatWarnings { + g := top[0] + sev := sevNote + if g.Level == "ERROR" { + sev = sevWarning + } + who := "" + if g.Driver != "" { + who = " from " + g.Driver + } + out = append(out, finding{sev, "One message dominates the week's log", + fmt.Sprintf("%q%s was logged %d times. The table of repeated messages below has the rest.", + redactDumpLog(truncateReport(g.Msg, 160)), who, g.Count)}) + } + + empty := periodsBeforeData(w) + if empty > 0 { + out = append(out, finding{sevNote, "The stored week starts late", + "The first stored reading is from " + w.FirstData.In(time.Local).Format("Mon 2 Jan 15:04") + + ", so the box is newer than the week or was off before then. The periods before it have no data."}) + } + // The period holding the first reading is incomplete by definition. + holding := -1 + if empty < len(w.Periods) && !w.FirstData.IsZero() && w.FirstData.After(w.Periods[empty].period.Start) { + holding = empty + } + var incomplete []string + for i, p := range w.Periods { + if i < empty || i == holding || p.Partial || !p.EnergyIncomplete { + continue + } + incomplete = append(incomplete, p.Label) + } + if len(incomplete) > 0 { + which := strings.Join(incomplete, "; ") + if len(incomplete) > 3 { + which = fmt.Sprintf("%d periods, from %s to %s", len(incomplete), incomplete[0], incomplete[len(incomplete)-1]) + } + out = append(out, finding{sevNote, "Stored history is incomplete", + "Battery or grid energy is missing for " + which + + ". The box was off, a meter was not read, or history was not written."}) + } + return out +} + +// periodsBeforeData counts the leading periods that end before the first +// stored series point. They have nothing to show. +func periodsBeforeData(w *supportWeek) int { + if w.FirstData.IsZero() { + return 0 + } + n := 0 + for _, p := range w.Periods { + if !p.period.End.After(w.FirstData) { + n++ + continue + } + break + } + return n +} + +func plural(n int, word string) string { + if n == 1 { + return "1 " + word + } + return fmt.Sprintf("%d %ss", n, word) +} + +func listSuffix(items []string) string { + if len(items) == 0 { + return "" + } + return " (starts " + strings.Join(items, ", ") + ")" +} + +func moreOrLess(delta float64) string { + if delta > 0 { + return "more" + } + return "less" +} + +// busyPeriod is the period with the most replans when it has at least +// supportBusyReplans and three times the median. +func busyPeriod(ps []supportPeriodSummary) (supportPeriodSummary, bool) { + if len(ps) < 3 { + return supportPeriodSummary{}, false + } + counts := make([]int, len(ps)) + best := 0 + for i, p := range ps { + counts[i] = p.Replans + if p.Replans > ps[best].Replans { + best = i + } + } + sort.Ints(counts) + median := counts[len(counts)/2] + if ps[best].Replans >= supportBusyReplans && ps[best].Replans >= 3*max(median, 1) { + return ps[best], true + } + return supportPeriodSummary{}, false +} + +// topDigest sums the week's rows per pattern and returns the n most frequent. +func topDigest(rows []state.LogDigest, n int) []state.LogDigest { + type key struct{ level, driver, pattern string } + sum := map[key]*state.LogDigest{} + for _, r := range rows { + k := key{r.Level, r.Driver, r.Pattern} + g := sum[k] + if g == nil { + c := r + sum[k] = &c + continue + } + g.Count += r.Count + if r.LastMs >= g.LastMs { + g.LastMs, g.Msg, g.Attrs = r.LastMs, r.Msg, r.Attrs + } + } + out := make([]state.LogDigest, 0, len(sum)) + for _, g := range sum { + out = append(out, *g) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + return out[i].LastMs > out[j].LastMs + }) + if len(out) > n { + out = out[:n] + } + return out +} + +// csvNumber keeps three decimals: a bucket average otherwise prints sixteen +// digits, while an energy counter in Wh still needs all its whole ones. +func csvNumber(v float64) string { + return strconv.FormatFloat(math.Round(v*1000)/1000, 'f', -1, 64) +} + +func fmtSoCRange(lo, hi *float64) string { + if lo == nil || hi == nil { + return "—" + } + return fmt.Sprintf("%.0f–%.0f%%", *lo, *hi) +} + +func fmtWhPtr(v *float64) string { + if v == nil { + return "—" + } + return fmtReportWh(*v) +} + +func fmtEVWh(v *float64) string { + if v == nil { + return "—" + } + return "≈ " + fmtReportWh(*v) +} + +func fmtHours(secs float64) string { + if secs < 3600 { + return fmt.Sprintf("%.0f min", secs/60) + } + return fmt.Sprintf("%.1f h", secs/3600) +} + +func fmtPlanVsBattery(p supportPeriodSummary) string { + if p.SlotsScored == 0 { + return "—" + } + return fmt.Sprintf("%d of %d off · gap %s", p.SlotsOffPlan, p.SlotsScored, fmtReportWh(p.SlotGapWh)) +} + +func fmtReplans(p supportPeriodSummary) string { + if p.Replans == 0 { + return "0 replans" + } + s := fmt.Sprintf("%d replans", p.Replans) + if p.Replans == 1 { + s = "1 replan" + } + var rs []string + for i, r := range p.ReplanReasons { + if i == 3 { + break + } + rs = append(rs, fmt.Sprintf("%s %d", r.Name, r.Count)) + } + if len(rs) > 0 { + s += " (" + strings.Join(rs, ", ") + ")" + } + if p.Fallbacks > 0 { + s += fmt.Sprintf(" · %d fallback", p.Fallbacks) + } + return s +} + +func fmtLimits(p supportPeriodSummary) string { + var parts []string + if p.FuseLimitedS >= 60 { + parts = append(parts, "fuse "+fmtHours(p.FuseLimitedS)) + } + for _, d := range p.SourceDown { + parts = append(parts, d.Name+" "+fmtHours(float64(d.Count))) + } + if p.Warnings > 0 || p.Errors > 0 { + parts = append(parts, fmt.Sprintf("%d warnings, %d errors", p.Warnings, p.Errors)) + } + if p.Starts > 0 { + parts = append(parts, plural(p.Starts, "start")) + } + if len(parts) == 0 { + return "—" + } + return strings.Join(parts, " · ") +} + +// ---- the archive's week/ ---- + +// addWeekFiles writes the week's data into the archive. JSON passes the +// same redaction as the logs: device errors in the journal carry addresses. +func addWeekFiles(add func(string, []byte), w *supportWeek) { + if w == nil { + return + } + redacted := func(v any) []byte { + raw, err := json.MarshalIndent(v, "", " ") + if err != nil { + return []byte(`{"error":` + fmt.Sprintf("%q", err.Error()) + `}`) + } + return []byte(redactDumpLog(string(raw))) + } + window := func(extra map[string]any) map[string]any { + extra["since_ms"], extra["until_ms"] = w.Since.UnixMilli(), w.Until.UnixMilli() + return extra + } + index := make([]map[string]any, 0, len(w.Series)) + for _, ser := range w.Series { + index = append(index, map[string]any{"driver": ser.Driver, "metric": ser.Metric, "unit": ser.Unit, "buckets": len(ser.Points)}) + } + add("week/summary.json", redacted(window(map[string]any{ + "tz": w.Until.Location().String(), "bucket_ms": state.BucketWidthMs(w.Since.UnixMilli(), w.Until.UnixMilli(), supportWeekPoints()), + "periods": w.Periods, "series": index, "gaps": append([]string{}, w.Gaps...), + }))) + spans := make([]journalSpan, 0, len(w.Spans)) + for _, sp := range w.Spans { + spans = append(spans, toJournalSpan(sp, w.Until.UnixMilli())) + } + add("week/control-journal.json", redacted(window(map[string]any{"spans": spans, "more": w.SpansMore}))) + add("week/plans.json", redacted(window(map[string]any{"replans": w.Replans}))) + add("week/notifications.json", redacted(window(map[string]any{"notifications": w.Notices, "more": w.NoticesMore}))) + events := make([]map[string]any, 0, len(w.Events)) + for _, e := range w.Events { + events = append(events, map[string]any{"ts_ms": e.TsMs, "event": e.Event}) + } + add("week/events.json", redacted(window(map[string]any{"events": events}))) + + for _, ser := range w.Series { + var b strings.Builder + b.WriteString("ts_ms,v,min,max,n\n") + for _, p := range ser.Points { + fmt.Fprintf(&b, "%d,%s,%s,%s,%d\n", p.TsMs, csvNumber(p.V), csvNumber(p.Min), csvNumber(p.Max), p.N) + } + add("week/series/"+sanitizeName(ser.Driver)+"__"+sanitizeName(ser.Metric)+".csv", []byte(b.String())) + } + + var b strings.Builder + if len(w.Digest) == 0 { + b.WriteString("No warnings or errors stored for the week.\n") + } + for _, g := range w.Digest { + fmt.Fprintf(&b, "%s %s ", time.UnixMilli(g.HourMs).UTC().Format("2006-01-02T15Z"), g.Level) + if g.Driver != "" { + fmt.Fprintf(&b, "[%s] ", g.Driver) + } + b.WriteString(redactDumpLog(g.Msg)) + fmt.Fprintf(&b, " ×%d first %s last %s", g.Count, + time.UnixMilli(g.FirstMs).UTC().Format("15:04:05"), time.UnixMilli(g.LastMs).UTC().Format("15:04:05")) + if g.Attrs != "" { + b.WriteString(" " + redactDumpLog(g.Attrs)) + } + b.WriteByte('\n') + } + add("logs/warnings-week.log", []byte(b.String())) +} diff --git a/go/internal/api/api_support_week_test.go b/go/internal/api/api_support_week_test.go new file mode 100644 index 000000000..ee451c419 --- /dev/null +++ b/go/internal/api/api_support_week_test.go @@ -0,0 +1,306 @@ +package api + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/control" + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// A tester asked why the battery emptied during the night. The support file +// covered the last hour and could not say. This seeds a week where, one +// night, the battery discharged 8 kWh while every plan slot asked it to +// hold, a charger lost contact and the fuse limited charging, and checks +// that the file answers from the box's own storage. +func TestSupportDumpAnswersForTheLastWeek(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + ctx := context.Background() + + now := time.Now() + y, m, d := now.AddDate(0, 0, -1).Date() + midnight := time.Date(y, m, d, 0, 0, 0, 0, time.Local) // inside last night but one + from, to := midnight.Add(-2*time.Hour), midnight.Add(6*time.Hour) + + var samples []state.Sample + var history []state.HistoryPoint + for ts := from; !ts.After(to); ts = ts.Add(time.Minute) { + batW, soc := 0.0, 0.8 + switch { + case !ts.Before(midnight) && ts.Before(midnight.Add(4*time.Hour)): + batW = -2000 + soc = 0.8 - 0.5*ts.Sub(midnight).Hours()/4 + case !ts.Before(midnight.Add(4 * time.Hour)): + soc = 0.3 + } + ms := ts.UnixMilli() + samples = append(samples, + state.Sample{Driver: "pixii", Metric: "battery_w", TsMs: ms, Value: batW, Unit: "W"}, + state.Sample{Driver: "pixii", Metric: "battery_soc", TsMs: ms, Value: soc}, + state.Sample{Driver: "easee", Metric: "ev_w", TsMs: ms, Value: 0, Unit: "W"}, + ) + history = append(history, state.HistoryPoint{TsMs: ms, GridW: 400, BatW: batW, LoadW: 2400, BatSoC: soc, JSON: `{}`}) + } + if err := st.RecordSamples(samples); err != nil { + t.Fatal(err) + } + if err := st.BulkRecordHistory(history); err != nil { + t.Fatal(err) + } + + // Every 15-minute slot that night asked the battery to hold. + var spans []*state.ControlSpan + for s := midnight; s.Before(midnight.Add(4 * time.Hour)); s = s.Add(15 * time.Minute) { + first, _ := json.Marshal(map[string]any{"planned_wh": 0.0, "actual_wh": -500.0, "ratio": 0, "replans": 1}) + spans = append(spans, &state.ControlSpan{Subject: "slot:batteries", Kind: "slot", Status: "over_delivered", + Severity: "info", StartMs: s.UnixMilli(), EndMs: s.Add(15 * time.Minute).UnixMilli(), + UpdatedMs: s.Add(15 * time.Minute).UnixMilli(), Samples: 1, First: first, Last: first}) + } + spans = append(spans, + &state.ControlSpan{Subject: "source:easee", Driver: "easee", Kind: "source", Status: "offline", + Reason: "telemetry_stale", Severity: "warning", StartMs: midnight.Add(time.Hour).UnixMilli(), + EndMs: midnight.Add(3 * time.Hour).UnixMilli(), UpdatedMs: midnight.Add(3 * time.Hour).UnixMilli(), Samples: 1440}, + &state.ControlSpan{Subject: "ev:easee", Driver: "easee", Kind: "ev", Status: "limited", + Reason: "fuse_limit", Severity: "info", StartMs: midnight.Add(-90 * time.Minute).UnixMilli(), + EndMs: midnight.Add(-60 * time.Minute).UnixMilli(), UpdatedMs: midnight.Add(-60 * time.Minute).UnixMilli(), Samples: 360}, + ) + if err := st.SaveControlSpans(ctx, spans); err != nil { + t.Fatal(err) + } + for i, reason := range []string{"scheduled", "soc_drift", "soc_drift"} { + js := `{"solver":{"engine":"energyplan","status":"optimal"}}` + if i == 2 { + js = `{"solver":{"engine":"core","backend":"dp","status":"fallback","fallback":true,"fallback_reason":"optimizer timed out"}}` + } + if err := st.SaveDiagnostic(midnight.Add(time.Duration(i)*time.Hour).UnixMilli(), reason, "SE3", 10, 96, js); err != nil { + t.Fatal(err) + } + } + if err := st.RecordNotification(state.NotificationEntry{TsMs: midnight.Add(time.Hour).UnixMilli(), + EventType: "driver_offline", Driver: "easee", Title: "Charger offline", Body: "easee stopped answering", Status: "sent"}); err != nil { + t.Fatal(err) + } + hour := midnight.Add(time.Hour).Truncate(time.Hour) + if err := st.AddLogDigest(ctx, []state.LogDigest{{HourMs: hour.UnixMilli(), Level: "WARN", Driver: "easee", + Pattern: "poll failed: HTTP #", Msg: "poll failed: HTTP 503", Count: 240, + FirstMs: hour.UnixMilli(), LastMs: hour.Add(59 * time.Minute).UnixMilli()}}); err != nil { + t.Fatal(err) + } + if err := st.RecordEvent("restart"); err != nil { + t.Fatal(err) + } + time.Sleep(2 * time.Millisecond) // events are keyed by the millisecond + if err := st.RecordEvent("startup"); err != nil { + t.Fatal(err) + } + + srv := New(&Deps{ + Ctrl: control.NewState(0, 50, "meter"), CtrlMu: &sync.Mutex{}, + Tel: telemetry.NewStore(), LogRing: telemetry.NewLogRing(), Version: "test", State: st, + }) + files := readSupportZip(t, srv) + + for _, want := range []string{ + "week/summary.json", "week/series/pixii__battery_w.csv", "week/series/pixii__battery_soc.csv", + "week/control-journal.json", "week/plans.json", "week/notifications.json", "week/events.json", + "logs/warnings-week.log", + } { + if _, ok := files[want]; !ok { + t.Errorf("archive is missing %s", want) + } + } + var manifest struct { + Contents []string `json:"contents"` + } + if err := json.Unmarshal([]byte(files["manifest.json"]), &manifest); err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(manifest.Contents, " "), "week/series/__.csv") { + t.Errorf("manifest does not list the week's series: %v", manifest.Contents) + } + + // The series come from storage at five-minute resolution, far past the + // last hour the old file stopped at. + csv := strings.Split(strings.TrimSpace(files["week/series/pixii__battery_w.csv"]), "\n") + if csv[0] != "ts_ms,v,min,max,n" { + t.Errorf("series header = %q", csv[0]) + } + if rows := len(csv) - 1; rows < 90 || rows > 100 { + t.Errorf("eight hours of minute samples gave %d rows, want about 96 five-minute buckets", rows) + } + if !strings.Contains(files["week/series/pixii__battery_w.csv"], ",-2000,-2000,") { + t.Error("series lost the discharge") + } + if !strings.Contains(files["week/control-journal.json"], "source:easee") { + t.Error("control journal file lacks the charger's offline span") + } + if !strings.Contains(files["week/plans.json"], "optimizer timed out") { + t.Error("plans file lacks the fallback reason") + } + if !strings.Contains(files["logs/warnings-week.log"], "poll failed: HTTP 503") || !strings.Contains(files["logs/warnings-week.log"], "×240") { + t.Errorf("warnings-week.log = %q", files["logs/warnings-week.log"]) + } + + report := files["00-help-report.md"] + week := sectionOf(report, "## What happened over the last week") + if week == "" { + t.Fatalf("report has no week section:\n%s", report) + } + for _, want := range []string{ + "discharged 8.00 kWh", // the night's measured discharge + "30–80%", // its charge range + "more than the plan", // and that the plan did not ask for it + "easee", // the charger that went offline + "offline for 2.0 h", // and for how long + "fuse", // the fuse limit on charging + "Core's own planner", // the fallback plan + "optimizer timed out", // with its reason + "restart", // the box restarted + "poll failed: HTTP 503", // the week's most repeated warning + "| Period |", // and the table behind the bullets + } { + if !strings.Contains(week, want) { + t.Errorf("week section lacks %q:\n%s", want, week) + } + } + if strings.Index(report, "## What happened over the last week") > strings.Index(report, "## Right now") { + t.Error("the week section belongs before Right now, next to Findings") + } +} + +// Periods split the week at 06:00 and 22:00 household time, so a question +// about "last night" lands in one row. The window opens at the first +// boundary inside the last seven days and ends now. +func TestSupportWeekPeriods(t *testing.T) { + loc, err := time.LoadLocation("Europe/Stockholm") + if err != nil { + t.Skip("no zone database") + } + now := time.Date(2026, 11, 15, 12, 54, 0, 0, loc) + ps := supportWeekPeriods(now) + if len(ps) != 14 { + t.Fatalf("got %d periods, want 7 nights and 7 days", len(ps)) + } + if !ps[0].Start.Equal(time.Date(2026, 11, 8, 22, 0, 0, 0, loc)) || !ps[0].Night { + t.Errorf("first period = %+v, want the night from 8 Nov 22:00", ps[0]) + } + last := ps[len(ps)-1] + if last.Night || !last.Start.Equal(time.Date(2026, 11, 15, 6, 0, 0, 0, loc)) || !last.End.Equal(now) || !last.Partial { + t.Errorf("last period = %+v, want today's day so far", last) + } + for i := 1; i < len(ps); i++ { + if !ps[i].Start.Equal(ps[i-1].End) { + t.Fatalf("gap between %v and %v", ps[i-1].End, ps[i].Start) + } + } + if got := ps[0].Label(); got != "Sun 8 → Mon 9 Nov, night" { + t.Errorf("night label = %q", got) + } + if got := ps[1].Label(); got != "Mon 9 Nov, day" { + t.Errorf("day label = %q", got) + } + + // The autumn clock change makes one night nine hours long. + dst := supportWeekPeriods(time.Date(2026, 10, 27, 12, 0, 0, 0, loc)) + for _, p := range dst { + if p.Night && p.Start.Day() == 24 { + if h := p.End.Sub(p.Start).Hours(); h != 9 { + t.Errorf("night over the clock change is %.1f h, want 9", h) + } + } + } +} + +func sectionOf(report, heading string) string { + i := strings.Index(report, heading) + if i < 0 { + return "" + } + rest := report[i+len(heading):] + if j := strings.Index(rest, "\n## "); j >= 0 { + rest = rest[:j] + } + return rest +} + +// A night that discharged as planned is still listed, because it is what +// people ask about; a start with no stop before it is a crash. +func TestWeekFindingsNightDischargeAndCrash(t *testing.T) { + loc := time.FixedZone("test", 3600) + start := time.Date(2026, 11, 13, 22, 0, 0, 0, loc) + night := supportPeriod{Start: start, End: start.Add(8 * time.Hour), Night: true} + out, soc := 3100.0, 20.0 + full := 90.0 + w := &supportWeek{ + Since: start.Add(-24 * time.Hour), Until: start.Add(24 * time.Hour), + Periods: []supportPeriodSummary{{ + period: night, Label: night.Label(), Night: true, BatDischargedWh: &out, + SoCMinPct: &soc, SoCMaxPct: &full, SlotsScored: 32, PlannedOutWh: 3000, ActualOutWh: 3100, + Starts: 2, UncleanStarts: 1, + }}, + Events: []state.Event{ + {TsMs: start.Add(time.Hour).UnixMilli(), Event: "startup"}, + {TsMs: start.Add(2 * time.Hour).UnixMilli(), Event: "startup"}, + }, + } + var text strings.Builder + writeFindingList(&text, weekFindings(w)) + got := text.String() + for _, want := range []string{ + "NOTE — Night discharge.", "3.10 kWh (charge 20–90%), plan 3.00 kWh", + "WARNING — Core stopped without a clean shutdown.", "Core started 2 times", + } { + if !strings.Contains(got, want) { + t.Errorf("findings lack %q:\n%s", want, got) + } + } + if strings.Contains(got, "did not follow the plan") { + t.Errorf("a night within 1 kWh of its plan is not a miss:\n%s", got) + } +} + +// A box installed today has one start and an hour of data. Its first start +// had no stop before it because there was no run before it, and the empty +// days before the install fold into one row. +func TestSupportWeekOnANewBox(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + if err := st.RecordEvent("startup"); err != nil { + t.Fatal(err) + } + now := time.Now() + var samples []state.Sample + for ts := now.Add(-time.Hour); ts.Before(now); ts = ts.Add(time.Minute) { + samples = append(samples, state.Sample{Driver: "sungrow", Metric: "battery_soc", TsMs: ts.UnixMilli(), Value: 0.5}) + } + if err := st.RecordSamples(samples); err != nil { + t.Fatal(err) + } + srv := New(&Deps{ + Ctrl: control.NewState(0, 50, "meter"), CtrlMu: &sync.Mutex{}, + Tel: telemetry.NewStore(), LogRing: telemetry.NewLogRing(), Version: "test", State: st, + }) + week := sectionOf(srv.buildSupportReport(context.Background(), now), "## What happened over the last week") + if strings.Contains(week, "clean shutdown") { + t.Errorf("a new box's first start is not a crash:\n%s", week) + } + if !strings.Contains(week, "no stored data") || !strings.Contains(week, "The stored week starts late") { + t.Errorf("the empty days before the install should fold into one row:\n%s", week) + } + if rows := strings.Count(week, "\n| "); rows > 4 { + t.Errorf("week table has %d rows for an hour of data:\n%s", rows, week) + } +} diff --git a/go/internal/ftwcli/backup.go b/go/internal/ftwcli/backup.go index 2b41fc15e..7ae66097d 100644 --- a/go/internal/ftwcli/backup.go +++ b/go/internal/ftwcli/backup.go @@ -221,6 +221,10 @@ func (c *client) copyBackup(ctx context.Context, out io.Writer, dir string, b ba return nil } +// supportFileTimeout matches Core's HTTP write timeout. The file reads a +// week from storage, which takes a Raspberry Pi longer than a status call. +const supportFileTimeout = 2 * time.Minute + func runSupport(args []string, out io.Writer, e env) error { var output string base, err := parse(args, out, "ftw support [--output FILE] [--url URL]", func(fs *flag.FlagSet) { @@ -233,7 +237,7 @@ func runSupport(args []string, out io.Writer, e env) error { output = "ftw-support-" + e.now().Format("20060102-150405") + ".zip" } c := newClient(base, e) - ctx, cancel := context.WithTimeout(context.Background(), 2*e.requestTimeout) + ctx, cancel := context.WithTimeout(context.Background(), max(2*e.requestTimeout, supportFileTimeout)) defer cancel() resp, err := c.open(ctx, "/api/support/dump") if err != nil { diff --git a/go/internal/state/diagnostics.go b/go/internal/state/diagnostics.go index 5c38ef15a..19d5bde5e 100644 --- a/go/internal/state/diagnostics.go +++ b/go/internal/state/diagnostics.go @@ -86,6 +86,57 @@ func (s *Store) LoadDiagnosticsInRange(sinceMs, untilMs int64, return out, rows.Err() } +// PlannerReplan is a stored replan with the solver fields the support week +// counts: which engine made the plan, and whether it was Core's fallback. +type PlannerReplan struct { + DiagnosticSummary + Engine string `json:"engine,omitempty"` + Backend string `json:"backend,omitempty"` + Status string `json:"status,omitempty"` + Fallback bool `json:"fallback,omitempty"` + FallbackReason string `json:"fallback_reason,omitempty"` +} + +// PlannerReplans returns the replans in [sinceMs, untilMs], oldest first. +// A positive limit keeps the newest. SQLite reads the solver fields from +// each blob, so Go never holds the full snapshots. +func (s *Store) PlannerReplans(ctx context.Context, sinceMs, untilMs int64, limit int) ([]PlannerReplan, error) { + q := `SELECT ts_ms, reason, zone, total_cost_ore, horizon_slots, + COALESCE(json_extract(json, '$.solver.engine'), ''), + COALESCE(json_extract(json, '$.solver.backend'), ''), + COALESCE(json_extract(json, '$.solver.status'), ''), + COALESCE(json_extract(json, '$.solver.fallback') = 1, 0), + COALESCE(json_extract(json, '$.solver.fallback_reason'), '') + FROM planner_diagnostics WHERE ts_ms >= ? AND ts_ms <= ? + ORDER BY ts_ms DESC` + args := []any{sinceMs, untilMs} + if limit > 0 { + q += ` LIMIT ?` + args = append(args, limit) + } + rows, err := s.db.QueryContext(ctx, q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := []PlannerReplan{} + for rows.Next() { + var r PlannerReplan + if err := rows.Scan(&r.TsMs, &r.Reason, &r.Zone, &r.TotalCostOre, &r.HorizonSlots, + &r.Engine, &r.Backend, &r.Status, &r.Fallback, &r.FallbackReason); err != nil { + return nil, err + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + return nil, err + } + for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] + } + return out, nil +} + // PlannerDay sums the plans kept since sinceMs: how many, the longest any // took to solve in milliseconds, and how many were a fallback plan. The // anonymous fleet report coarsens these into buckets; nothing else of a plan diff --git a/go/internal/state/log_digest.go b/go/internal/state/log_digest.go new file mode 100644 index 000000000..10b099047 --- /dev/null +++ b/go/internal/state/log_digest.go @@ -0,0 +1,101 @@ +package state + +import ( + "context" + "fmt" +) + +// LogDigest is one warning or error pattern as logged within one hour: how +// often, when first and last, and the latest full text. The support file +// reads a week of these. The in-memory log ring cannot hold that long: a +// driver logging every control tick filled it in 75 minutes on a real site, +// and a restart empties it. +type LogDigest struct { + HourMs int64 `json:"hour_ms"` + Level string `json:"level"` + Driver string `json:"driver,omitempty"` + // Pattern is the message with its numbers folded, the key that groups + // repeats whose figures differ. + Pattern string `json:"pattern"` + Msg string `json:"msg"` + Attrs string `json:"attrs,omitempty"` + Count int64 `json:"count"` + FirstMs int64 `json:"first_ms"` + LastMs int64 `json:"last_ms"` +} + +// AddLogDigest adds counts to the stored rows. A row seen again in the same +// hour keeps its first sighting and takes the latest text. +func (s *Store) AddLogDigest(ctx context.Context, rows []LogDigest) error { + if len(rows) == 0 { + return nil + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("log digest: begin: %w", err) + } + defer tx.Rollback() + for _, r := range rows { + if _, err := tx.ExecContext(ctx, `INSERT INTO log_digest + (hour_ms, level, driver, pattern, msg, attrs, count, first_ms, last_ms) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (hour_ms, level, driver, pattern) DO UPDATE SET + count = count + excluded.count, + first_ms = MIN(first_ms, excluded.first_ms), + msg = CASE WHEN excluded.last_ms >= last_ms THEN excluded.msg ELSE msg END, + attrs = CASE WHEN excluded.last_ms >= last_ms THEN excluded.attrs ELSE attrs END, + last_ms = MAX(last_ms, excluded.last_ms)`, + r.HourMs, r.Level, r.Driver, r.Pattern, r.Msg, r.Attrs, r.Count, r.FirstMs, r.LastMs); err != nil { + return fmt.Errorf("log digest: add: %w", err) + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("log digest: commit: %w", err) + } + return nil +} + +// LogDigestBetween returns the rows whose hour starts in [sinceMs, untilMs], +// oldest first, and whether more matched than the limit allowed. The limit +// keeps the newest. +func (s *Store) LogDigestBetween(ctx context.Context, sinceMs, untilMs int64, limit int) ([]LogDigest, bool, error) { + if limit <= 0 { + limit = 1000 + } + // A row's hour may begin before sinceMs and still hold sightings inside. + rows, err := s.db.QueryContext(ctx, `SELECT hour_ms, level, driver, pattern, msg, attrs, count, first_ms, last_ms + FROM log_digest WHERE hour_ms <= ? AND last_ms >= ? + ORDER BY hour_ms DESC, first_ms DESC LIMIT ?`, untilMs, sinceMs, limit+1) + if err != nil { + return nil, false, fmt.Errorf("log digest: query: %w", err) + } + defer rows.Close() + out := []LogDigest{} + for rows.Next() { + var r LogDigest + if err := rows.Scan(&r.HourMs, &r.Level, &r.Driver, &r.Pattern, &r.Msg, &r.Attrs, &r.Count, &r.FirstMs, &r.LastMs); err != nil { + return nil, false, fmt.Errorf("log digest: scan: %w", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + return nil, false, fmt.Errorf("log digest: rows: %w", err) + } + more := len(out) > limit + if more { + out = out[:limit] + } + for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] + } + return out, more, nil +} + +// PruneLogDigest deletes the hours that began before beforeMs. +func (s *Store) PruneLogDigest(ctx context.Context, beforeMs int64) (int64, error) { + res, err := s.db.ExecContext(ctx, `DELETE FROM log_digest WHERE hour_ms < ?`, beforeMs) + if err != nil { + return 0, fmt.Errorf("log digest: prune: %w", err) + } + return res.RowsAffected() +} diff --git a/go/internal/state/log_digest_test.go b/go/internal/state/log_digest_test.go new file mode 100644 index 000000000..4ff63a5d3 --- /dev/null +++ b/go/internal/state/log_digest_test.go @@ -0,0 +1,112 @@ +package state + +import ( + "context" + "testing" +) + +// The digest keeps a week of warnings in a few rows. Two flushes of the +// same message in the same hour must add up, keep the earliest first +// sighting and the latest text, and stay one row. +func TestLogDigestAddsUpRepeatsWithinAnHour(t *testing.T) { + st := openTestStore(t) + ctx := context.Background() + const hour = int64(3_600_000) * 500_000 + first := LogDigest{HourMs: hour, Level: "WARN", Driver: "pixii", Pattern: "write failed reg=#", + Msg: "write failed reg=40", Attrs: "err=timeout", Count: 3, FirstMs: hour + 1_000, LastMs: hour + 9_000} + if err := st.AddLogDigest(ctx, []LogDigest{first}); err != nil { + t.Fatal(err) + } + later := first + later.Msg, later.Attrs, later.Count = "write failed reg=41", "err=reset", 2 + later.FirstMs, later.LastMs = hour+60_000, hour+70_000 + other := LogDigest{HourMs: hour + 3_600_000, Level: "ERROR", Pattern: "optimizer: deadline", + Msg: "optimizer: deadline", Count: 1, FirstMs: hour + 3_700_000, LastMs: hour + 3_700_000} + if err := st.AddLogDigest(ctx, []LogDigest{later, other}); err != nil { + t.Fatal(err) + } + + got, more, err := st.LogDigestBetween(ctx, hour, hour+2*3_600_000, 100) + if err != nil || more { + t.Fatalf("LogDigestBetween: more=%v err=%v", more, err) + } + if len(got) != 2 { + t.Fatalf("got %d rows, want 2: %+v", len(got), got) + } + w := got[0] + if w.Count != 5 || w.FirstMs != hour+1_000 || w.LastMs != hour+70_000 { + t.Errorf("repeat row = count %d first %d last %d, want 5, %d, %d", w.Count, w.FirstMs, w.LastMs, hour+1_000, hour+70_000) + } + if w.Msg != "write failed reg=41" || w.Attrs != "err=reset" { + t.Errorf("repeat row keeps %q %q, want the latest text", w.Msg, w.Attrs) + } + if got[1].Level != "ERROR" || got[1].Driver != "" { + t.Errorf("second row = %+v, want the core error", got[1]) + } + + // A window that ends before the second hour leaves it out. + got, _, err = st.LogDigestBetween(ctx, hour, hour+3_599_999, 100) + if err != nil || len(got) != 1 { + t.Fatalf("first hour only: %+v err=%v", got, err) + } + // The limit is honoured and says more matched. + got, more, err = st.LogDigestBetween(ctx, hour, hour+2*3_600_000, 1) + if err != nil || len(got) != 1 || !more { + t.Fatalf("limit 1: %d rows more=%v err=%v", len(got), more, err) + } + + n, err := st.PruneLogDigest(ctx, hour+3_600_000) + if err != nil || n != 1 { + t.Fatalf("prune: n=%d err=%v, want the first hour gone", n, err) + } + got, _, _ = st.LogDigestBetween(ctx, 0, hour+10*3_600_000, 100) + if len(got) != 1 || got[0].Level != "ERROR" { + t.Fatalf("after prune: %+v", got) + } +} + +// The week view counts fallback plans from the stored diagnostics without +// handing an 85 kB blob per replan to Go. +func TestPlannerReplansReadSolverFields(t *testing.T) { + st := openTestStore(t) + ctx := context.Background() + if err := st.SaveDiagnostic(1_000, "scheduled", "SE3", 12.5, 96, + `{"solver":{"engine":"energyplan","backend":"milp","status":"optimal"}}`); err != nil { + t.Fatal(err) + } + if err := st.SaveDiagnostic(2_000, "soc_drift", "SE3", 14, 96, + `{"solver":{"engine":"core","backend":"dp","status":"fallback","fallback":true,"fallback_reason":"optimizer timed out"}}`); err != nil { + t.Fatal(err) + } + if err := st.SaveDiagnostic(3_000, "scheduled", "SE3", 9, 96, `{}`); err != nil { + t.Fatal(err) + } + got, err := st.PlannerReplans(ctx, 0, 2_500, 0) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("got %d replans, want 2 inside the window: %+v", len(got), got) + } + if got[0].TsMs != 1_000 || got[1].TsMs != 2_000 { + t.Fatalf("replans not oldest first: %+v", got) + } + if got[0].Engine != "energyplan" || got[0].Fallback { + t.Errorf("optimizer plan = %+v", got[0]) + } + fb := got[1] + if !fb.Fallback || fb.Engine != "core" || fb.Backend != "dp" || fb.FallbackReason != "optimizer timed out" || fb.Reason != "soc_drift" { + t.Errorf("fallback plan = %+v", fb) + } + if fb.TotalCostOre != 14 || fb.HorizonSlots != 96 || fb.Zone != "SE3" { + t.Errorf("fallback summary = %+v", fb.DiagnosticSummary) + } + // A plan without solver data reads as empty, not as an error. + all, err := st.PlannerReplans(ctx, 0, 10_000, 0) + if err != nil || len(all) != 3 || all[2].Engine != "" || all[2].Fallback { + t.Fatalf("plan without solver: %+v err=%v", all, err) + } + if lim, err := st.PlannerReplans(ctx, 0, 10_000, 1); err != nil || len(lim) != 1 || lim[0].TsMs != 3_000 { + t.Fatalf("limit keeps the newest: %+v err=%v", lim, err) + } +} diff --git a/go/internal/state/store.go b/go/internal/state/store.go index b2cdd95cd..d800879fc 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -1019,6 +1019,21 @@ func (s *Store) migrate() error { ) STRICT`, `CREATE INDEX IF NOT EXISTS idx_control_journal_start ON control_journal(start_ms DESC)`, `CREATE INDEX IF NOT EXISTS idx_control_journal_open ON control_journal(end_ms) WHERE end_ms IS NULL`, + // log_digest: warning and error patterns per hour (log_digest.go), + // so the support file can show a week of them. At most a few hundred + // rows an hour, pruned after eight days; kept in recovery snapshots. + `CREATE TABLE IF NOT EXISTS log_digest ( + hour_ms INTEGER NOT NULL, + level TEXT NOT NULL, + driver TEXT NOT NULL DEFAULT '', + pattern TEXT NOT NULL, + msg TEXT NOT NULL, + attrs TEXT NOT NULL DEFAULT '', + count INTEGER NOT NULL, + first_ms INTEGER NOT NULL, + last_ms INTEGER NOT NULL, + PRIMARY KEY (hour_ms, level, driver, pattern) + ) STRICT`, // Independently installed Lua drivers. Content lives on disk; SQLite // records activation history and the exact previous artifact used for diff --git a/go/internal/telemetry/logring.go b/go/internal/telemetry/logring.go index ef6e883ea..3b54d237b 100644 --- a/go/internal/telemetry/logring.go +++ b/go/internal/telemetry/logring.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "log/slog" + "sort" "strings" "sync" "time" @@ -31,14 +32,59 @@ const ( ringGlobal = 2000 ) +const ( + // digestPatternsPerHour caps the distinct warning patterns kept per + // hour. A driver that puts a changing word in every warning would + // otherwise add a row per line; past the cap they are only counted. + digestPatternsPerHour = 200 + // digestOverflowPattern is the row that counts them. + digestOverflowPattern = "(more distinct messages this hour)" + // digestTextMax bounds the stored message and attribute text. + digestTextMax = 300 + // digestHeldMax is how long rows wait in memory for TakeDigest. + digestHeldMax = 24 * time.Hour +) + // LogRing is an in-memory ring of recent log entries, split into a // global stream and one stream per driver name. Thread-safe. +// +// Warnings and errors also go to a digest, one row per pattern and hour, +// which Core writes to the store (TakeDigest). Lines logged every control +// tick can push them out of the rings within an hour; the digest keeps a +// week of them. type LogRing struct { mu sync.RWMutex global []LogEntry gNext int gFilled bool drivers map[string]*driverRing + + digest map[digestKey]*digestRow + digestHours map[time.Time]int // distinct patterns per hour +} + +// DigestEntry is the count of one warning or error pattern in one hour +// since the previous TakeDigest. +type DigestEntry struct { + Hour time.Time + Level string + Driver string + Pattern string + Msg string + Attrs string + Count int + First time.Time + Last time.Time +} + +type digestKey struct { + hour time.Time + level, driver, pattern string +} + +type digestRow struct { + DigestEntry + sent int // Count already handed out } type driverRing struct { @@ -50,9 +96,135 @@ type driverRing struct { // NewLogRing constructs an empty ring. func NewLogRing() *LogRing { return &LogRing{ - global: make([]LogEntry, ringGlobal), - drivers: make(map[string]*driverRing), + global: make([]LogEntry, ringGlobal), + drivers: make(map[string]*driverRing), + digest: make(map[digestKey]*digestRow), + digestHours: make(map[time.Time]int), + } +} + +// addDigest counts a warning or error. Caller holds r.mu. +func (r *LogRing) addDigest(e LogEntry) { + level := strings.ToUpper(e.Level) + if level == "WARNING" { + level = "WARN" + } + if level != "WARN" && level != "ERROR" { + return + } + ts := e.TS + if ts.IsZero() { + ts = time.Now() + } + ts = ts.UTC() + hour := ts.Truncate(time.Hour) + k := digestKey{hour: hour, level: level, driver: e.Driver, pattern: clip(logPattern(e.Msg))} + row, ok := r.digest[k] + if !ok && r.digestHours[hour] >= digestPatternsPerHour { + k.driver, k.pattern = "", digestOverflowPattern + row, ok = r.digest[k] + } + if _, seen := r.digestHours[hour]; !seen { + // Nothing has taken the digest for a day: keep memory bounded. + for old := range r.digest { + if old.hour.Before(hour.Add(-digestHeldMax)) { + delete(r.digest, old) + delete(r.digestHours, old.hour) + } + } + r.digestHours[hour] = 0 + } + if !ok { + row = &digestRow{DigestEntry: DigestEntry{Hour: hour, Level: level, Driver: k.driver, Pattern: k.pattern, First: ts}} + r.digest[k] = row + if k.pattern != digestOverflowPattern { + r.digestHours[hour]++ + } + } + row.Count++ + if ts.Before(row.First) { + row.First = ts + } + if !ts.Before(row.Last) { + row.Last = ts + if k.pattern == digestOverflowPattern { + row.Msg = "warnings and errors past the hourly limit of distinct messages" + } else { + row.Msg, row.Attrs = clip(e.Msg), clip(e.Attrs) + } + } +} + +// TakeDigest returns the counts added since the last call, oldest hour +// first, and forgets every hour before now's once its counts are out. The +// store adds what it is given, so a row appears again only with new counts. +func (r *LogRing) TakeDigest(now time.Time) []DigestEntry { + r.mu.Lock() + defer r.mu.Unlock() + current := now.UTC().Truncate(time.Hour) + var out []DigestEntry + for k, row := range r.digest { + if delta := row.Count - row.sent; delta > 0 { + e := row.DigestEntry + e.Count = delta + out = append(out, e) + row.sent = row.Count + } + if k.hour.Before(current) { + delete(r.digest, k) + } + } + for h := range r.digestHours { + if h.Before(current) { + delete(r.digestHours, h) + } + } + sort.Slice(out, func(i, j int) bool { + if !out[i].Hour.Equal(out[j].Hour) { + return out[i].Hour.Before(out[j].Hour) + } + return out[i].First.Before(out[j].First) + }) + return out +} + +// logPattern folds every number in a message to "#", so the same warning +// with different figures counts as one. +func logPattern(msg string) string { + var b strings.Builder + b.Grow(len(msg)) + for i := 0; i < len(msg); { + if !isDigit(msg[i]) { + b.WriteByte(msg[i]) + i++ + continue + } + // One number: digits, joined by single . : or , between digits. + j := i + 1 + for j < len(msg) { + if isDigit(msg[j]) { + j++ + continue + } + if (msg[j] == '.' || msg[j] == ':' || msg[j] == ',') && j+1 < len(msg) && isDigit(msg[j+1]) { + j += 2 + continue + } + break + } + b.WriteByte('#') + i = j + } + return b.String() +} + +func isDigit(c byte) bool { return c >= '0' && c <= '9' } + +func clip(s string) string { + if len(s) <= digestTextMax { + return s } + return s[:digestTextMax] } // Append records one entry. If e.Driver is non-empty it lands in BOTH @@ -61,6 +233,7 @@ func NewLogRing() *LogRing { func (r *LogRing) Append(e LogEntry) { r.mu.Lock() defer r.mu.Unlock() + r.addDigest(e) r.global[r.gNext] = e r.gNext++ if r.gNext == len(r.global) { diff --git a/go/internal/telemetry/logring_test.go b/go/internal/telemetry/logring_test.go new file mode 100644 index 000000000..fb0df6676 --- /dev/null +++ b/go/internal/telemetry/logring_test.go @@ -0,0 +1,95 @@ +package telemetry + +import ( + "fmt" + "testing" + "time" +) + +// A driver that logs every control tick at INFO filled the 2000-line ring +// in 75 minutes on a real site. Warnings and errors must not depend on that +// ring: they go to a digest that keeps one row per message and hour. +func TestLogRingDigestKeepsWarningsPastTickChatter(t *testing.T) { + r := NewLogRing() + hour := time.Date(2026, 11, 15, 2, 0, 0, 0, time.UTC) + r.Append(LogEntry{TS: hour.Add(time.Minute), Level: "WARN", Driver: "pixii", Msg: "write failed reg=40", Attrs: "err=timeout"}) + for i := 0; i < 5000; i++ { + r.Append(LogEntry{TS: hour.Add(2 * time.Minute), Level: "INFO", Driver: "pixii", Msg: "Pixii: setpoint ems_w=" + fmt.Sprint(i)}) + } + r.Append(LogEntry{TS: hour.Add(3 * time.Minute), Level: "WARN", Driver: "pixii", Msg: "write failed reg=41", Attrs: "err=reset"}) + r.Append(LogEntry{TS: hour.Add(4 * time.Minute), Level: "ERROR", Msg: "optimizer: deadline exceeded"}) + r.Append(LogEntry{TS: hour.Add(5 * time.Minute), Level: "DEBUG", Msg: "noise"}) + + rows := r.TakeDigest(hour.Add(10 * time.Minute)) + if len(rows) != 2 { + t.Fatalf("digest has %d rows, want the warning pattern and the error: %+v", len(rows), rows) + } + w := rows[0] + if w.Level != "WARN" || w.Driver != "pixii" || w.Pattern != "write failed reg=#" || w.Count != 2 { + t.Errorf("warning row = %+v, want one pattern seen twice", w) + } + if w.Msg != "write failed reg=41" || w.Attrs != "err=reset" { + t.Errorf("warning row keeps %q %q, want the latest text", w.Msg, w.Attrs) + } + if !w.Hour.Equal(hour) || !w.First.Equal(hour.Add(time.Minute)) || !w.Last.Equal(hour.Add(3*time.Minute)) { + t.Errorf("warning row times = %v %v %v", w.Hour, w.First, w.Last) + } + if rows[1].Level != "ERROR" || rows[1].Count != 1 { + t.Errorf("error row = %+v", rows[1]) + } + + // A second take sends only what arrived since, so the store can add. + if again := r.TakeDigest(hour.Add(11 * time.Minute)); len(again) != 0 { + t.Fatalf("nothing new, got %+v", again) + } + r.Append(LogEntry{TS: hour.Add(12 * time.Minute), Level: "WARN", Driver: "pixii", Msg: "write failed reg=7"}) + delta := r.TakeDigest(hour.Add(13 * time.Minute)) + if len(delta) != 1 || delta[0].Count != 1 || delta[0].Pattern != "write failed reg=#" { + t.Fatalf("delta = %+v, want one more of the same pattern", delta) + } + + // Once the hour has been sent and has passed, it is dropped from memory. + r.TakeDigest(hour.Add(2 * time.Hour)) + r.mu.RLock() + left := len(r.digest) + r.mu.RUnlock() + if left != 0 { + t.Errorf("%d digest rows still held after their hour was sent", left) + } +} + +// Messages that differ in more than numbers would each be a new row. Past +// the hourly cap they are counted in one overflow row instead, so a driver +// that puts a timestamp in every warning cannot grow the store. +func TestLogRingDigestCapsDistinctMessagesPerHour(t *testing.T) { + r := NewLogRing() + hour := time.Date(2026, 11, 15, 3, 0, 0, 0, time.UTC) + for i := 0; i < digestPatternsPerHour+50; i++ { + r.Append(LogEntry{TS: hour.Add(time.Second), Level: "WARN", Msg: "odd " + string(rune('a'+i%26)) + string(rune('a'+i/26))}) + } + rows := r.TakeDigest(hour.Add(time.Minute)) + if len(rows) != digestPatternsPerHour+1 { + t.Fatalf("got %d rows, want %d patterns plus one overflow row", len(rows), digestPatternsPerHour+1) + } + var overflow *DigestEntry + for i := range rows { + if rows[i].Pattern == digestOverflowPattern { + overflow = &rows[i] + } + } + if overflow == nil || overflow.Count != 50 { + t.Fatalf("overflow row = %+v, want 50 messages", overflow) + } +} + +func TestLogPatternFoldsNumbers(t *testing.T) { + for in, want := range map[string]string{ + "write failed reg=40": "write failed reg=#", + "read 12 of 3.5 kW at 02:13:59": "read # of # kW at #", + "no numbers": "no numbers", + } { + if got := logPattern(in); got != want { + t.Errorf("logPattern(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/web/diagnostics-modal.js b/web/diagnostics-modal.js index f5162984d..e49089df6 100644 --- a/web/diagnostics-modal.js +++ b/web/diagnostics-modal.js @@ -497,7 +497,7 @@ downloadWithFeedback( "/api/support/dump", "Preparing support bundle", - "Collecting logs, redacted config, driver health, and recent telemetry.", + "Collecting logs, redacted config, driver health, and the last seven days of telemetry, plans and control.", "ftw-support.zip" ); }); diff --git a/web/plan.js b/web/plan.js index 6cdfee3e1..234e1f3c9 100644 --- a/web/plan.js +++ b/web/plan.js @@ -1541,8 +1541,9 @@ import { } // Pulls GET /api/support/dump and saves it. That archive leads with - // 00-help-report.md — the readable answer to "why is it doing that?" — - // and carries the logs, config and telemetry behind it. One button and + // 00-help-report.md — the readable answer to "why is it doing that?" and + // "what happened last night?" — and carries the logs, config and a week + // of telemetry, plans and control journal behind it. One button and // one file, because a user asking for help should not have to work out // which of two downloads we wanted. The bare report is still at // /api/support/report for anyone who wants only the text. From 6b3495a184cde51b08d86c5ea77bd94ec1c43b7f Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Sun, 15 Nov 2026 16:07:45 +0100 Subject: [PATCH 2/2] fix(support): hide the site's location and the ntfy topic Round coordinates to whole degrees in the redacted config, planner.json and log text (URL query fields and JSON numbers in logged bodies), and hide notifications.ntfy.topic. A failed publish quotes the topic in its URL, so every file in the support archive and GET /api/support/report also replace the configured topic wherever it appears. Co-Authored-By: Claude Opus 5.5 --- .changeset/support-file-week.md | 3 +- go/internal/api/api_drivers_debug.go | 118 ++++++++++++++++++- go/internal/api/api_drivers_debug_test.go | 18 +++ go/internal/api/api_support_dump_zip_test.go | 56 +++++++++ go/internal/api/api_support_planner.go | 7 +- go/internal/api/api_support_report.go | 4 +- 6 files changed, 196 insertions(+), 10 deletions(-) diff --git a/.changeset/support-file-week.md b/.changeset/support-file-week.md index 4c43c868a..d0f288b7f 100644 --- a/.changeset/support-file-week.md +++ b/.changeset/support-file-week.md @@ -13,4 +13,5 @@ series in five-minute buckets, the control journal, plan snapshots, notifications, start and stop events, and a week of warnings and errors counted per hour, which the box now stores so they outlast the log buffer and a restart. The redacted config and logs no longer show the owner's -e-mail address or account username. +e-mail address, account username or ntfy topic, and they round the site's +coordinates to whole degrees. diff --git a/go/internal/api/api_drivers_debug.go b/go/internal/api/api_drivers_debug.go index 424707773..4a336f001 100644 --- a/go/internal/api/api_drivers_debug.go +++ b/go/internal/api/api_drivers_debug.go @@ -8,10 +8,12 @@ package api import ( "archive/zip" + "bytes" "context" "encoding/json" "fmt" "log/slog" + "math" "net" "net/http" "net/url" @@ -19,6 +21,7 @@ import ( "path/filepath" "regexp" "runtime" + "slices" "sort" "strconv" "strings" @@ -765,7 +768,9 @@ func (s *Server) handleSupportDump(w http.ResponseWriter, r *http.Request) { zw := zip.NewWriter(w) defer zw.Close() + topic := s.ntfyTopic() addFile := func(name string, body []byte) { + body = hideNtfyTopic(body, topic) hdr := &zip.FileHeader{ Name: "ftw-support-" + stamp + "/" + name, Method: zip.Deflate, @@ -937,6 +942,11 @@ func (s *Server) redactedConfig() []byte { return cfgBytes } redactSensitive(asMap) + top, _ := asMap.(map[string]any) + notifications, _ := top["notifications"].(map[string]any) + if ntfy, ok := notifications["ntfy"].(map[string]any); ok && ntfy["topic"] != nil { + ntfy["topic"] = "***" + } pretty, err := json.MarshalIndent(asMap, "", " ") if err != nil { return cfgBytes @@ -956,6 +966,10 @@ func redactSensitive(v any) { x[k] = "***" continue } + if isCoordinateKey(lk) { + x[k] = roundCoordinate(val) + continue + } if str, ok := val.(string); ok { x[k] = dumpEmailRe.ReplaceAllString(str, "[email omitted]") continue @@ -992,6 +1006,74 @@ func isSensitiveKey(k string) bool { return hasAuthSegment(k) } +// coordinateWords name a value that says where something is. The support +// file keeps whole degrees, about 100 km: enough to show a missing location +// or a wrong sign, not enough to find the house. +var coordinateWords = []string{"lat", "lon", "lng", "latitude", "longitude"} + +func isCoordinateKey(k string) bool { + words := strings.FieldsFunc(strings.ToLower(k), func(r rune) bool { return r == '_' || r == '-' || r == '.' }) + for _, w := range words { + if slices.Contains(coordinateWords, w) { + return true + } + } + return false +} + +// roundCoordinate rounds a coordinate to whole degrees and keeps its type. +// Anything that is not a number goes. +func roundCoordinate(v any) any { + switch x := v.(type) { + case nil, bool: + return v + case float64: + return math.Round(x) + case json.Number: + if f, err := x.Float64(); err == nil { + return json.Number(strconv.FormatFloat(math.Round(f), 'f', -1, 64)) + } + case string: + if strings.TrimSpace(x) == "" { + return x + } + return roundCoordinateText(x) + } + return "***" +} + +func roundCoordinateText(s string) string { + f, err := strconv.ParseFloat(strings.TrimSpace(s), 64) + if err != nil || math.IsNaN(f) || math.IsInf(f, 0) { + return "***" + } + return strconv.FormatFloat(math.Round(f), 'f', -1, 64) +} + +// ntfyTopic is the configured ntfy topic, or "". +func (s *Server) ntfyTopic() string { + if s.deps.Cfg == nil || s.deps.CfgMu == nil { + return "" + } + s.deps.CfgMu.RLock() + defer s.deps.CfgMu.RUnlock() + if n := s.deps.Cfg.Notifications; n != nil && n.Ntfy != nil { + return strings.TrimSpace(n.Ntfy.Topic) + } + return "" +} + +// hideNtfyTopic removes the ntfy topic from text. On the public server the +// topic is the key that reads the owner's alerts, and a failed publish +// quotes it in the URL. A topic under three characters stays: replacing it +// would scrub ordinary words, and the config hides it anyway. +func hideNtfyTopic(body []byte, topic string) []byte { + if len(topic) < 3 || !bytes.Contains(body, []byte(topic)) { + return body + } + return bytes.ReplaceAll(body, []byte(topic), []byte("[ntfy topic omitted]")) +} + // hasAuthSegment is true for a bare "auth" path segment so auth, // auth_header and x-auth redact, while oauth and oauth_client_id do not. func hasAuthSegment(k string) bool { @@ -1015,11 +1097,15 @@ var ( dumpFormSecretRe = regexp.MustCompile(`(?i)\b([A-Za-z][A-Za-z0-9_-]*)=([^\s&"]+)`) dumpBasicAuthRe = regexp.MustCompile(`(?i)basic\s+[A-Za-z0-9+/=_-]+`) dumpEmailRe = regexp.MustCompile(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}`) + // dumpJSONCoordRe finds a JSON number under a coordinate key, such as a + // car's position in an HTTP body a driver logs. + dumpJSONCoordRe = regexp.MustCompile(`(?i)("(?:[a-z0-9]+[_.-])*(?:lat|lon|lng|latitude|longitude)(?:[_.-][a-z0-9]+)*"\s*:\s*)(-?\d+(?:\.\d+)?(?:e[-+]?\d+)?)`) ) -// redactDumpLog removes IPv4 addresses, e-mail addresses and credentials, including JSON and -// form fields whose keys isSensitiveKey would catch. Stricter on OAuth -// bodies that Lua logs as `HTTP %d: %s`. +// redactDumpLog removes IPv4 addresses, e-mail addresses and credentials, +// including JSON and form fields whose keys isSensitiveKey would catch, and +// rounds coordinates to whole degrees. Stricter on OAuth bodies that Lua +// logs as `HTTP %d: %s`. func redactDumpLog(s string) string { if s == "" { return s @@ -1030,14 +1116,36 @@ func redactDumpLog(s string) string { s = dumpBasicAuthRe.ReplaceAllString(s, "Basic [omitted]") s = dumpJSONSecretRe.ReplaceAllStringFunc(s, func(m string) string { parts := dumpJSONSecretRe.FindStringSubmatch(m) - if len(parts) != 4 || !isSensitiveKey(parts[2]) { + if len(parts) != 4 { + return m + } + if isCoordinateKey(parts[2]) { + if v, err := strconv.Unquote(parts[3]); err == nil { + return parts[1] + strconv.Quote(roundCoordinateText(v)) + } + return parts[1] + `"***"` + } + if !isSensitiveKey(parts[2]) { return m } return parts[1] + `"***"` }) + s = dumpJSONCoordRe.ReplaceAllStringFunc(s, func(m string) string { + parts := dumpJSONCoordRe.FindStringSubmatch(m) + if len(parts) != 3 { + return m + } + return parts[1] + roundCoordinateText(parts[2]) + }) s = dumpFormSecretRe.ReplaceAllStringFunc(s, func(m string) string { parts := dumpFormSecretRe.FindStringSubmatch(m) - if len(parts) != 3 || !isSensitiveKey(parts[1]) { + if len(parts) != 3 { + return m + } + if isCoordinateKey(parts[1]) { + return parts[1] + "=" + roundCoordinateText(parts[2]) + } + if !isSensitiveKey(parts[1]) { return m } return parts[1] + "=***" diff --git a/go/internal/api/api_drivers_debug_test.go b/go/internal/api/api_drivers_debug_test.go index a0e850d1f..b24f79e18 100644 --- a/go/internal/api/api_drivers_debug_test.go +++ b/go/internal/api/api_drivers_debug_test.go @@ -581,6 +581,24 @@ func TestRejectUnsafeProbeTargetsAllowsOnlyConfiguredLoopbackException(t *testin } } +func TestIsCoordinateKey(t *testing.T) { + for _, k := range []string{"lat", "lon", "lng", "latitude", "Longitude", "home_lat", "latitude_deg", "site.lon"} { + if !isCoordinateKey(k) { + t.Errorf("isCoordinateKey(%q) = false, want true", k) + } + } + for _, k := range []string{"latest_ms", "latency_ms", "longest", "relative", "plateau", "salon", "flat"} { + if isCoordinateKey(k) { + t.Errorf("isCoordinateKey(%q) = true, want false", k) + } + } + got := redactDumpLog(`{"lat":"56.7","position":{"latitude":-12.6},"latest_ms":56123} lat=56.7 lon=n/a`) + want := `{"lat":"57","position":{"latitude":-13},"latest_ms":56123} lat=57 lon=***` + if got != want { + t.Errorf("redactDumpLog\n got %s\nwant %s", got, want) + } +} + func TestIsSensitiveKey(t *testing.T) { sensitive := []string{ "password", "Password", "mqtt_password", "passwd", "client_secret", diff --git a/go/internal/api/api_support_dump_zip_test.go b/go/internal/api/api_support_dump_zip_test.go index 1cbe2bdd8..5c75e37b1 100644 --- a/go/internal/api/api_support_dump_zip_test.go +++ b/go/internal/api/api_support_dump_zip_test.go @@ -246,6 +246,62 @@ func TestSupportDumpRemovesTheOwnersEmail(t *testing.T) { } } +// The site's coordinates say where the house is, and an ntfy topic on the +// public server is the key that reads its alerts. Neither may leave in the +// support file, from the config or from a log line that quotes a URL or an +// HTTP body. Whole degrees stay, so a wrong sign or a missing location shows. +func TestSupportDumpHidesWhereTheHouseIsAndTheNtfyTopic(t *testing.T) { + const topic = "ftw-home-7f3k2q9x" + tel := telemetry.NewStore() + ring := telemetry.NewLogRing() + ring.Append(telemetry.LogEntry{Level: "WARN", Msg: "forecast: fetch failed", + Attrs: `err=Get "https://api.met.no/weatherapi/locationforecast/2.0/compact?lat=56.1235&lon=13.6543": timeout`}) + ring.Append(telemetry.LogEntry{Level: "WARN", Msg: "notifications: publish failed", + Attrs: `err=ntfy: Post "https://ntfy.sh/` + topic + `": dial tcp: lookup ntfy.sh: no such host`}) + ring.Append(telemetry.LogEntry{Level: "INFO", Driver: "vw", Msg: `car: {"position":{"latitude":59.334591,"longitude":18.063240},"latest_ms":1763200000000}`}) + cfg := &config.Config{ + Weather: &config.Weather{Provider: "met_no", Latitude: 56.123456, Longitude: 13.654321}, + Notifications: &config.Notifications{Enabled: true, Provider: "ntfy", + Ntfy: &config.NtfyConfig{Server: "https://ntfy.sh", Topic: topic}}, + Drivers: []config.Driver{{ + Name: "vw", Lua: "vw.lua", + Config: map[string]any{"home_lat": "56.123456", "home_lon": 13.654321, "topic_prefix": "ftw/site"}, + }}, + } + srv := New(&Deps{ + Ctrl: control.NewState(0, 50, "meter"), CtrlMu: &sync.Mutex{}, + Tel: tel, LogRing: ring, Version: "test", + Cfg: cfg, CfgMu: &sync.RWMutex{}, + }) + files := readSupportZip(t, srv) + for name, body := range files { + for _, leak := range []string{topic, "56.12", "13.65", "59.33", "18.06"} { + if strings.Contains(body, leak) { + t.Errorf("%s leaked %q", name, leak) + } + } + } + cfgJSON := files["config.redacted.yaml"] + for _, want := range []string{`"latitude": 56`, `"longitude": 14`, `"home_lat": "56"`, `"home_lon": 14`, `"topic_prefix": "ftw/site"`, `"server": "https://ntfy.sh"`} { + if !strings.Contains(cfgJSON, want) { + t.Errorf("config.redacted.yaml lacks %s:\n%s", want, cfgJSON) + } + } + if log := files["logs/global.log"]; !strings.Contains(log, "lat=56&lon=14") || !strings.Contains(log, `"latitude":59`) || !strings.Contains(log, `"latest_ms":1763200000000`) { + t.Errorf("global.log lost what is not personal:\n%s", log) + } + + req := httptest.NewRequest(http.MethodGet, "/api/support/report", nil) + rec := httptest.NewRecorder() + srv.Handler().ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("GET /api/support/report = %d, want 200", rec.Code) + } + if strings.Contains(rec.Body.String(), topic) { + t.Errorf("help report leaked the ntfy topic:\n%s", rec.Body.String()) + } +} + // readSupportZip fetches /api/support/dump and returns each file by its path // inside the archive's top folder. func readSupportZip(t *testing.T, srv *Server) map[string]string { diff --git a/go/internal/api/api_support_planner.go b/go/internal/api/api_support_planner.go index fde3b5a8e..0aa938556 100644 --- a/go/internal/api/api_support_planner.go +++ b/go/internal/api/api_support_planner.go @@ -48,9 +48,12 @@ func supportJSON(v any) []byte { switch x := v.(type) { case map[string]any: for k, val := range x { - if isSensitiveKey(k) { + switch { + case isSensitiveKey(k): x[k] = "***" - } else { + case isCoordinateKey(k): + x[k] = roundCoordinate(val) + default: x[k] = redact(val) } } diff --git a/go/internal/api/api_support_report.go b/go/internal/api/api_support_report.go index 55c5b5b65..32facb735 100644 --- a/go/internal/api/api_support_report.go +++ b/go/internal/api/api_support_report.go @@ -135,12 +135,12 @@ func comparePlanControlDecision(plan *mpc.Plan, slot control.SlotEnergySnapshot) func (s *Server) handleSupportReport(w http.ResponseWriter, r *http.Request) { now := time.Now() - body := s.buildSupportReport(r.Context(), now) + body := hideNtfyTopic([]byte(s.buildSupportReport(r.Context(), now)), s.ntfyTopic()) stamp := now.UTC().Format("20060102-150405") w.Header().Set("Content-Type", "text/markdown; charset=utf-8") w.Header().Set("Content-Disposition", `attachment; filename="ftw-help-`+stamp+`.md"`) w.Header().Set("Cache-Control", "no-store") - _, _ = w.Write([]byte(body)) + _, _ = w.Write(body) } // liveSnapshot adds forecast context to the same qualified live power that