diff --git a/.changeset/support-file-week.md b/.changeset/support-file-week.md new file mode 100644 index 000000000..d0f288b7f --- /dev/null +++ b/.changeset/support-file-week.md @@ -0,0 +1,17 @@ +--- +"ftw": minor +--- + +The support file now answers for the last seven days, not the last hour. +Its help report gains "What happened over the last week": per day and +night, the battery's lowest and highest charge, energy in and out of the +battery, to the car and over the grid, how often the plan changed and why, +how far the battery strayed from the plan, fuse limits, devices out of +contact, plans that fell back to Core and restarts, with the notable ones +stated first. The archive carries the week behind it in `week/`: every +series in five-minute buckets, the control journal, plan snapshots, +notifications, start and stop events, and a week of warnings and errors +counted per hour, which the box now stores so they outlast the log buffer +and a restart. The redacted config and logs no longer show the owner's +e-mail address, account username or ntfy topic, and they round the site's +coordinates to whole degrees. diff --git a/go/cmd/ftw/log_digest.go b/go/cmd/ftw/log_digest.go new file mode 100644 index 000000000..e208d42c7 --- /dev/null +++ b/go/cmd/ftw/log_digest.go @@ -0,0 +1,62 @@ +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +const ( + // logDigestEvery is how often warning counts reach the store. A crash + // loses at most this much of the digest. + logDigestEvery = time.Minute + // logDigestKeep covers the support file's week with a day to spare. + logDigestKeep = 8 * 24 * time.Hour +) + +// runLogDigest writes the log ring's warning and error counts to the store +// until ctx ends, then writes once more. +func runLogDigest(ctx context.Context, ring *telemetry.LogRing, st *state.Store) { + t := time.NewTicker(logDigestEvery) + defer t.Stop() + var prunedAt time.Time + for { + select { + case <-ctx.Done(): + flushLogDigest(context.Background(), ring, st, time.Now()) + return + case now := <-t.C: + flushLogDigest(ctx, ring, st, now) + if now.Sub(prunedAt) < 24*time.Hour { + continue + } + prunedAt = now + if _, err := st.PruneLogDigest(ctx, now.Add(-logDigestKeep).UnixMilli()); err != nil { + slog.Warn("log digest: prune failed", "err", err) + } + } + } +} + +func flushLogDigest(ctx context.Context, ring *telemetry.LogRing, st *state.Store, now time.Time) { + entries := ring.TakeDigest(now) + if len(entries) == 0 { + return + } + rows := make([]state.LogDigest, len(entries)) + for i, e := range entries { + rows[i] = state.LogDigest{ + HourMs: e.Hour.UnixMilli(), Level: e.Level, Driver: e.Driver, Pattern: e.Pattern, + Msg: e.Msg, Attrs: e.Attrs, Count: int64(e.Count), + FirstMs: e.First.UnixMilli(), LastMs: e.Last.UnixMilli(), + } + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := st.AddLogDigest(ctx, rows); err != nil { + slog.Warn("log digest: write failed", "rows", len(rows), "err", err) + } +} diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index aef6324fb..fb6b04e3c 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -2730,6 +2730,9 @@ func main() { // in control?", the one GET /api/status serves. notifSvc.SetControlReader(srv.ControlStates) ctrlJournal.start(ctx, srv.JournalVerdicts, st) + // Warning and error counts per hour, so the support file covers a week + // of them after the log ring has turned over or the box has restarted. + go runLogDigest(ctx, logRing, st) // FuseReader: on each HealthTick the fuse_over_limit rule reads // the site meter's live per-phase currents from telemetry and // compares against cfg.Fuse.MaxAmps. Closes over cfg + cfgMu so @@ -2938,6 +2941,7 @@ func main() { cloudLink.Bye(cloudwire.ByeShutdown, cloudByeWait) markNativeCleanExit(nativeRoot) ctrlJournal.shutdown() + flushLogDigest(context.Background(), logRing, st, time.Now()) flushHistoryOnStop(st) if err := st.RecordEvent("shutdown"); err != nil { slog.Warn("failed to persist shutdown event", "err", err) @@ -2954,6 +2958,7 @@ func main() { cloudLink.Bye(cloudwire.ByeUpdate, cloudByeWait) markNativeCleanExit(nativeRoot) ctrlJournal.shutdown() + flushLogDigest(context.Background(), logRing, st, time.Now()) flushHistoryOnStop(st) if err := st.RecordEvent("restart"); err != nil { slog.Warn("failed to persist restart event", "err", err) diff --git a/go/internal/api/api_drivers_debug.go b/go/internal/api/api_drivers_debug.go index 5f5608bc0..4a336f001 100644 --- a/go/internal/api/api_drivers_debug.go +++ b/go/internal/api/api_drivers_debug.go @@ -8,10 +8,12 @@ package api import ( "archive/zip" + "bytes" "context" "encoding/json" "fmt" "log/slog" + "math" "net" "net/http" "net/url" @@ -19,6 +21,7 @@ import ( "path/filepath" "regexp" "runtime" + "slices" "sort" "strconv" "strings" @@ -738,12 +741,14 @@ func (s *Server) handleGlobalLogs(w http.ResponseWriter, r *http.Request) { } // GET /api/support/dump — zip archive with everything a developer needs -// to triage a support incident: redacted config, full driver health JSON, -// recent global + per-driver logs (secrets and IPv4 addresses stripped), -// last 1 h of TS samples per (driver, metric), and a -// manifest. SQLite is NOT included; the dump is intended to be small -// enough to attach to a chat message — measured at ~6 kB on a two-driver -// install. +// to triage a support incident: the help report, redacted config, driver +// health, the log ring (secrets and IPv4 addresses stripped), the last hour +// of samples at stored resolution, and seven days from the box's own +// storage in week/: every series in five-minute buckets, the control +// journal, plan snapshots, notifications, start and stop events and the +// hourly warning counts. SQLite is NOT included. A seeded week with five +// drivers, 113 series, 6,000 journal spans and 1,050 plans gave a 2.8 MB +// archive, small enough to mail; week/series/ is three quarters of it. // // Zip rather than tar.gz because this file's whole purpose is to be // handed to somebody else. Windows and every chat client open a zip @@ -763,7 +768,9 @@ func (s *Server) handleSupportDump(w http.ResponseWriter, r *http.Request) { zw := zip.NewWriter(w) defer zw.Close() + topic := s.ntfyTopic() addFile := func(name string, body []byte) { + body = hideNtfyTopic(body, topic) hdr := &zip.FileHeader{ Name: "ftw-support-" + stamp + "/" + name, Method: zip.Deflate, @@ -782,7 +789,8 @@ func (s *Server) handleSupportDump(w http.ResponseWriter, r *http.Request) { // below it is what you reach for when the report does not settle the // question. Shipping them together means the person asking for help // sends one file and never has to pick the right one. - addFile("00-help-report.md", []byte(redactDumpLog(s.buildSupportReport(r.Context(), now)))) + week := s.loadSupportWeek(r.Context(), now, true) + addFile("00-help-report.md", []byte(redactDumpLog(s.buildSupportReportWith(r.Context(), now, week)))) manifest := map[string]any{ "generated_at": now.Format(time.RFC3339), @@ -800,7 +808,20 @@ func (s *Server) handleSupportDump(w http.ResponseWriter, r *http.Request) { "loadpoints.json", "logs/global.log", "logs/.log", + "logs/warnings-week.log", "telemetry/__.csv", + "week/summary.json", + "week/series/__.csv", + "week/control-journal.json", + "week/plans.json", + "week/notifications.json", + "week/events.json", + }, + "covers": map[string]string{ + "logs/*.log except warnings-week.log": "the in-memory log ring: the latest lines, often under an hour", + "logs/warnings-week.log": "seven days of warnings and errors, one line per message pattern and hour", + "telemetry/": "the last hour of samples at stored resolution", + "week/": "seven days from the box's storage; series in five-minute buckets with min, max and sample count", }, } manifestBody, _ := json.MarshalIndent(manifest, "", " ") @@ -831,9 +852,11 @@ func (s *Server) handleSupportDump(w http.ResponseWriter, r *http.Request) { addFile("logs/"+sanitizeName(d)+".log", []byte(formatLogs(entries))) } - // Last 1 h of TS samples per (driver, metric) — small enough to - // keep the bundle email-able. Only metrics the operator actually - // emitted in this window appear. + addWeekFiles(addFile, week) + + // Last 1 h of TS samples per (driver, metric) at stored resolution, + // next to week/'s five-minute buckets. Only metrics the operator + // actually emitted in this window appear. if s.deps.State != nil { untilMs := now.UnixMilli() sinceMs := untilMs - 3600*1000 @@ -919,6 +942,11 @@ func (s *Server) redactedConfig() []byte { return cfgBytes } redactSensitive(asMap) + top, _ := asMap.(map[string]any) + notifications, _ := top["notifications"].(map[string]any) + if ntfy, ok := notifications["ntfy"].(map[string]any); ok && ntfy["topic"] != nil { + ntfy["topic"] = "***" + } pretty, err := json.MarshalIndent(asMap, "", " ") if err != nil { return cfgBytes @@ -927,8 +955,8 @@ func (s *Server) redactedConfig() []byte { } // redactSensitive walks an unmarshalled JSON tree and replaces any value -// at a key whose lowercased name matches a sensitive substring. Recursive, -// in-place. +// at a key whose lowercased name matches a sensitive substring, and any +// e-mail address wherever it sits. Recursive, in-place. func redactSensitive(v any) { switch x := v.(type) { case map[string]any: @@ -938,10 +966,22 @@ func redactSensitive(v any) { x[k] = "***" continue } + if isCoordinateKey(lk) { + x[k] = roundCoordinate(val) + continue + } + if str, ok := val.(string); ok { + x[k] = dumpEmailRe.ReplaceAllString(str, "[email omitted]") + continue + } redactSensitive(val) } case []any: - for _, e := range x { + for i, e := range x { + if str, ok := e.(string); ok { + x[i] = dumpEmailRe.ReplaceAllString(str, "[email omitted]") + continue + } redactSensitive(e) } } @@ -955,6 +995,9 @@ func isSensitiveKey(k string) bool { for _, s := range []string{ "password", "passwd", "secret", "token", "api_key", "apikey", "private_key", "client_secret", "authorization", "credential", + // Who the owner is, not how they sign in: a cloud account's + // username is often their e-mail address. + "email", "e-mail", "username", "user_name", } { if strings.Contains(k, s) { return true @@ -963,6 +1006,74 @@ func isSensitiveKey(k string) bool { return hasAuthSegment(k) } +// coordinateWords name a value that says where something is. The support +// file keeps whole degrees, about 100 km: enough to show a missing location +// or a wrong sign, not enough to find the house. +var coordinateWords = []string{"lat", "lon", "lng", "latitude", "longitude"} + +func isCoordinateKey(k string) bool { + words := strings.FieldsFunc(strings.ToLower(k), func(r rune) bool { return r == '_' || r == '-' || r == '.' }) + for _, w := range words { + if slices.Contains(coordinateWords, w) { + return true + } + } + return false +} + +// roundCoordinate rounds a coordinate to whole degrees and keeps its type. +// Anything that is not a number goes. +func roundCoordinate(v any) any { + switch x := v.(type) { + case nil, bool: + return v + case float64: + return math.Round(x) + case json.Number: + if f, err := x.Float64(); err == nil { + return json.Number(strconv.FormatFloat(math.Round(f), 'f', -1, 64)) + } + case string: + if strings.TrimSpace(x) == "" { + return x + } + return roundCoordinateText(x) + } + return "***" +} + +func roundCoordinateText(s string) string { + f, err := strconv.ParseFloat(strings.TrimSpace(s), 64) + if err != nil || math.IsNaN(f) || math.IsInf(f, 0) { + return "***" + } + return strconv.FormatFloat(math.Round(f), 'f', -1, 64) +} + +// ntfyTopic is the configured ntfy topic, or "". +func (s *Server) ntfyTopic() string { + if s.deps.Cfg == nil || s.deps.CfgMu == nil { + return "" + } + s.deps.CfgMu.RLock() + defer s.deps.CfgMu.RUnlock() + if n := s.deps.Cfg.Notifications; n != nil && n.Ntfy != nil { + return strings.TrimSpace(n.Ntfy.Topic) + } + return "" +} + +// hideNtfyTopic removes the ntfy topic from text. On the public server the +// topic is the key that reads the owner's alerts, and a failed publish +// quotes it in the URL. A topic under three characters stays: replacing it +// would scrub ordinary words, and the config hides it anyway. +func hideNtfyTopic(body []byte, topic string) []byte { + if len(topic) < 3 || !bytes.Contains(body, []byte(topic)) { + return body + } + return bytes.ReplaceAll(body, []byte(topic), []byte("[ntfy topic omitted]")) +} + // hasAuthSegment is true for a bare "auth" path segment so auth, // auth_header and x-auth redact, while oauth and oauth_client_id do not. func hasAuthSegment(k string) bool { @@ -985,28 +1096,56 @@ var ( dumpJSONSecretRe = regexp.MustCompile(`(?i)("([^"\\]+)"\s*:\s*)("(?:\\.|[^"\\])*")`) dumpFormSecretRe = regexp.MustCompile(`(?i)\b([A-Za-z][A-Za-z0-9_-]*)=([^\s&"]+)`) dumpBasicAuthRe = regexp.MustCompile(`(?i)basic\s+[A-Za-z0-9+/=_-]+`) + dumpEmailRe = regexp.MustCompile(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}`) + // dumpJSONCoordRe finds a JSON number under a coordinate key, such as a + // car's position in an HTTP body a driver logs. + dumpJSONCoordRe = regexp.MustCompile(`(?i)("(?:[a-z0-9]+[_.-])*(?:lat|lon|lng|latitude|longitude)(?:[_.-][a-z0-9]+)*"\s*:\s*)(-?\d+(?:\.\d+)?(?:e[-+]?\d+)?)`) ) -// redactDumpLog removes IPv4 addresses and credentials, including JSON and -// form fields whose keys isSensitiveKey would catch. Stricter on OAuth -// bodies that Lua logs as `HTTP %d: %s`. +// redactDumpLog removes IPv4 addresses, e-mail addresses and credentials, +// including JSON and form fields whose keys isSensitiveKey would catch, and +// rounds coordinates to whole degrees. Stricter on OAuth bodies that Lua +// logs as `HTTP %d: %s`. func redactDumpLog(s string) string { if s == "" { return s } s = dumpIPv4Re.ReplaceAllString(s, "[ip omitted]") + s = dumpEmailRe.ReplaceAllString(s, "[email omitted]") s = dumpBearerRe.ReplaceAllString(s, "Bearer [omitted]") s = dumpBasicAuthRe.ReplaceAllString(s, "Basic [omitted]") s = dumpJSONSecretRe.ReplaceAllStringFunc(s, func(m string) string { parts := dumpJSONSecretRe.FindStringSubmatch(m) - if len(parts) != 4 || !isSensitiveKey(parts[2]) { + if len(parts) != 4 { + return m + } + if isCoordinateKey(parts[2]) { + if v, err := strconv.Unquote(parts[3]); err == nil { + return parts[1] + strconv.Quote(roundCoordinateText(v)) + } + return parts[1] + `"***"` + } + if !isSensitiveKey(parts[2]) { return m } return parts[1] + `"***"` }) + s = dumpJSONCoordRe.ReplaceAllStringFunc(s, func(m string) string { + parts := dumpJSONCoordRe.FindStringSubmatch(m) + if len(parts) != 3 { + return m + } + return parts[1] + roundCoordinateText(parts[2]) + }) s = dumpFormSecretRe.ReplaceAllStringFunc(s, func(m string) string { parts := dumpFormSecretRe.FindStringSubmatch(m) - if len(parts) != 3 || !isSensitiveKey(parts[1]) { + if len(parts) != 3 { + return m + } + if isCoordinateKey(parts[1]) { + return parts[1] + "=" + roundCoordinateText(parts[2]) + } + if !isSensitiveKey(parts[1]) { return m } return parts[1] + "=***" diff --git a/go/internal/api/api_drivers_debug_test.go b/go/internal/api/api_drivers_debug_test.go index 87d8eae0e..b24f79e18 100644 --- a/go/internal/api/api_drivers_debug_test.go +++ b/go/internal/api/api_drivers_debug_test.go @@ -581,12 +581,32 @@ func TestRejectUnsafeProbeTargetsAllowsOnlyConfiguredLoopbackException(t *testin } } +func TestIsCoordinateKey(t *testing.T) { + for _, k := range []string{"lat", "lon", "lng", "latitude", "Longitude", "home_lat", "latitude_deg", "site.lon"} { + if !isCoordinateKey(k) { + t.Errorf("isCoordinateKey(%q) = false, want true", k) + } + } + for _, k := range []string{"latest_ms", "latency_ms", "longest", "relative", "plateau", "salon", "flat"} { + if isCoordinateKey(k) { + t.Errorf("isCoordinateKey(%q) = true, want false", k) + } + } + got := redactDumpLog(`{"lat":"56.7","position":{"latitude":-12.6},"latest_ms":56123} lat=56.7 lon=n/a`) + want := `{"lat":"57","position":{"latitude":-13},"latest_ms":56123} lat=57 lon=***` + if got != want { + t.Errorf("redactDumpLog\n got %s\nwant %s", got, want) + } +} + func TestIsSensitiveKey(t *testing.T) { sensitive := []string{ "password", "Password", "mqtt_password", "passwd", "client_secret", "refresh_token", "access_token", "api_key", "apikey", "private_key", "authorization", "Authorization", "credential", "credentials", "auth", "AUTH", "auth_header", "x-auth", + // Who the owner is: a cloud account's username is often an e-mail. + "username", "ev_charger_username", "email", "Email", "user_name", } for _, k := range sensitive { if !isSensitiveKey(k) { @@ -594,7 +614,7 @@ func TestIsSensitiveKey(t *testing.T) { } } keep := []string{ - "host", "username", "name", "oauth", "oauth_client_id", "client_id", + "host", "user", "name", "oauth", "oauth_client_id", "client_id", "port", "lua", } for _, k := range keep { diff --git a/go/internal/api/api_support_dump_zip_test.go b/go/internal/api/api_support_dump_zip_test.go index cb554464f..5c75e37b1 100644 --- a/go/internal/api/api_support_dump_zip_test.go +++ b/go/internal/api/api_support_dump_zip_test.go @@ -205,3 +205,134 @@ func TestSupportDumpRedactsSecretsInLogsAndConfig(t *testing.T) { t.Error("dump must not include state.db") } } + +// A support file from a real site carried the owner's e-mail twice: the +// cloud charger's username and a driver's `email` setting. Neither is a +// password, so key-based secret redaction let both through. The address +// identifies the household wherever it sits, so it is removed by key and +// by shape, in the config and in the logs. +func TestSupportDumpRemovesTheOwnersEmail(t *testing.T) { + const ( + email = "bjorn.example@example.se" + username = "bjorn-cloud-login" + inLog = "owner.name@example.org" + ) + tel := telemetry.NewStore() + ring := telemetry.NewLogRing() + ring.Append(telemetry.LogEntry{Level: "WARN", Driver: "easee", Msg: "login failed for " + inLog}) + cfg := &config.Config{ + Drivers: []config.Driver{{ + Name: "easee", Lua: "easee_cloud.lua", + Config: map[string]any{"email": email, "site": "home", "note": "contact " + email}, + }}, + EVCharger: &config.EVCharger{Provider: "easee", Username: username, Serial: "EH123456"}, + } + srv := New(&Deps{ + Ctrl: control.NewState(0, 50, "meter"), CtrlMu: &sync.Mutex{}, + Tel: tel, LogRing: ring, Version: "test", + Cfg: cfg, CfgMu: &sync.RWMutex{}, + }) + files := readSupportZip(t, srv) + for name, body := range files { + for _, leak := range []string{email, username, inLog} { + if strings.Contains(body, leak) { + t.Errorf("%s leaked %q", name, leak) + } + } + } + cfgJSON := files["config.redacted.yaml"] + if !strings.Contains(cfgJSON, `"site": "home"`) || !strings.Contains(cfgJSON, "EH123456") { + t.Errorf("redaction removed settings that are not personal: %s", cfgJSON) + } +} + +// The site's coordinates say where the house is, and an ntfy topic on the +// public server is the key that reads its alerts. Neither may leave in the +// support file, from the config or from a log line that quotes a URL or an +// HTTP body. Whole degrees stay, so a wrong sign or a missing location shows. +func TestSupportDumpHidesWhereTheHouseIsAndTheNtfyTopic(t *testing.T) { + const topic = "ftw-home-7f3k2q9x" + tel := telemetry.NewStore() + ring := telemetry.NewLogRing() + ring.Append(telemetry.LogEntry{Level: "WARN", Msg: "forecast: fetch failed", + Attrs: `err=Get "https://api.met.no/weatherapi/locationforecast/2.0/compact?lat=56.1235&lon=13.6543": timeout`}) + ring.Append(telemetry.LogEntry{Level: "WARN", Msg: "notifications: publish failed", + Attrs: `err=ntfy: Post "https://ntfy.sh/` + topic + `": dial tcp: lookup ntfy.sh: no such host`}) + ring.Append(telemetry.LogEntry{Level: "INFO", Driver: "vw", Msg: `car: {"position":{"latitude":59.334591,"longitude":18.063240},"latest_ms":1763200000000}`}) + cfg := &config.Config{ + Weather: &config.Weather{Provider: "met_no", Latitude: 56.123456, Longitude: 13.654321}, + Notifications: &config.Notifications{Enabled: true, Provider: "ntfy", + Ntfy: &config.NtfyConfig{Server: "https://ntfy.sh", Topic: topic}}, + Drivers: []config.Driver{{ + Name: "vw", Lua: "vw.lua", + Config: map[string]any{"home_lat": "56.123456", "home_lon": 13.654321, "topic_prefix": "ftw/site"}, + }}, + } + srv := New(&Deps{ + Ctrl: control.NewState(0, 50, "meter"), CtrlMu: &sync.Mutex{}, + Tel: tel, LogRing: ring, Version: "test", + Cfg: cfg, CfgMu: &sync.RWMutex{}, + }) + files := readSupportZip(t, srv) + for name, body := range files { + for _, leak := range []string{topic, "56.12", "13.65", "59.33", "18.06"} { + if strings.Contains(body, leak) { + t.Errorf("%s leaked %q", name, leak) + } + } + } + cfgJSON := files["config.redacted.yaml"] + for _, want := range []string{`"latitude": 56`, `"longitude": 14`, `"home_lat": "56"`, `"home_lon": 14`, `"topic_prefix": "ftw/site"`, `"server": "https://ntfy.sh"`} { + if !strings.Contains(cfgJSON, want) { + t.Errorf("config.redacted.yaml lacks %s:\n%s", want, cfgJSON) + } + } + if log := files["logs/global.log"]; !strings.Contains(log, "lat=56&lon=14") || !strings.Contains(log, `"latitude":59`) || !strings.Contains(log, `"latest_ms":1763200000000`) { + t.Errorf("global.log lost what is not personal:\n%s", log) + } + + req := httptest.NewRequest(http.MethodGet, "/api/support/report", nil) + rec := httptest.NewRecorder() + srv.Handler().ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("GET /api/support/report = %d, want 200", rec.Code) + } + if strings.Contains(rec.Body.String(), topic) { + t.Errorf("help report leaked the ntfy topic:\n%s", rec.Body.String()) + } +} + +// readSupportZip fetches /api/support/dump and returns each file by its path +// inside the archive's top folder. +func readSupportZip(t *testing.T, srv *Server) map[string]string { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/api/support/dump", nil) + rec := httptest.NewRecorder() + srv.Handler().ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("GET /api/support/dump = %d, want 200", rec.Code) + } + body := rec.Body.Bytes() + zr, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + t.Fatalf("archive does not open as a zip: %v", err) + } + out := map[string]string{} + for _, f := range zr.File { + rc, err := f.Open() + if err != nil { + t.Fatalf("open %s: %v", f.Name, err) + } + raw, err := io.ReadAll(rc) + rc.Close() + if err != nil { + t.Fatalf("read %s: %v", f.Name, err) + } + name := f.Name + if i := strings.IndexByte(name, '/'); i >= 0 { + name = name[i+1:] + } + out[name] = string(raw) + } + return out +} diff --git a/go/internal/api/api_support_planner.go b/go/internal/api/api_support_planner.go index fde3b5a8e..0aa938556 100644 --- a/go/internal/api/api_support_planner.go +++ b/go/internal/api/api_support_planner.go @@ -48,9 +48,12 @@ func supportJSON(v any) []byte { switch x := v.(type) { case map[string]any: for k, val := range x { - if isSensitiveKey(k) { + switch { + case isSensitiveKey(k): x[k] = "***" - } else { + case isCoordinateKey(k): + x[k] = roundCoordinate(val) + default: x[k] = redact(val) } } diff --git a/go/internal/api/api_support_report.go b/go/internal/api/api_support_report.go index 9741f1f08..32facb735 100644 --- a/go/internal/api/api_support_report.go +++ b/go/internal/api/api_support_report.go @@ -135,12 +135,12 @@ func comparePlanControlDecision(plan *mpc.Plan, slot control.SlotEnergySnapshot) func (s *Server) handleSupportReport(w http.ResponseWriter, r *http.Request) { now := time.Now() - body := s.buildSupportReport(r.Context(), now) + body := hideNtfyTopic([]byte(s.buildSupportReport(r.Context(), now)), s.ntfyTopic()) stamp := now.UTC().Format("20060102-150405") w.Header().Set("Content-Type", "text/markdown; charset=utf-8") w.Header().Set("Content-Disposition", `attachment; filename="ftw-help-`+stamp+`.md"`) w.Header().Set("Cache-Control", "no-store") - _, _ = w.Write([]byte(body)) + _, _ = w.Write(body) } // liveSnapshot adds forecast context to the same qualified live power that @@ -203,6 +203,12 @@ func (s *Server) liveNow(ctrl control.State, now time.Time) liveSnapshot { } func (s *Server) buildSupportReport(ctx context.Context, now time.Time) string { + return s.buildSupportReportWith(ctx, now, s.loadSupportWeek(ctx, now, false)) +} + +// buildSupportReportWith writes the report around a week already read, so +// the support archive reads storage once for the report and its files. +func (s *Server) buildSupportReportWith(ctx context.Context, now time.Time, week *supportWeek) string { s.deps.CtrlMu.Lock() ctrl := *s.deps.Ctrl targets := append([]control.DispatchTarget{}, s.deps.Ctrl.LastTargets...) @@ -227,6 +233,7 @@ func (s *Server) buildSupportReport(ctx context.Context, now time.Time) string { var b strings.Builder writeReportHeader(&b, s.deps.Version, now) writeFindings(&b, findings) + writeWeekSection(&b, week) if feedback, err := json.MarshalIndent(s.controlFeedback(now), "", " "); err == nil { b.WriteString("## Command feedback\n\nDriver-call results, readbacks and measured power are separate evidence. Missing values are unknown.\n\n```json\n") b.Write(feedback) @@ -272,10 +279,11 @@ func writeReportHeader(b *strings.Builder, version string, now time.Time) { "(importing, charging the battery); **negative W = power out** " + "(exporting, discharging). This holds in every table below.\n\n") b.WriteString("Read `Findings` first — it lists what looks wrong without " + - "needing the tables. `Right now` is what the system is doing at this " + - "instant; `Plan` is what it intends to do next. Those two answer " + - "different questions and often disagree, which is usually the point " + - "of the question.\n\n") + "needing the tables. `What happened over the last week` answers " + + "questions about the past seven days, night by night. `Right now` is " + + "what the system is doing at this instant; `Plan` is what it intends " + + "to do next. Those two answer different questions and often disagree, " + + "which is usually the point of the question.\n\n") } func writeFindings(b *strings.Builder, findings []finding) { @@ -688,9 +696,10 @@ func (s *Server) writeLogSection(b *strings.Builder) { func writeReportFooter(b *strings.Builder) { b.WriteString("---\n\n") - b.WriteString("Deeper data lives in `/api/support/dump` (logs, telemetry, " + - "redacted config) and `/api/mpc/diagnose` (every slot the optimizer " + - "saw). This report deliberately omits both to stay readable.\n") + b.WriteString("Deeper data lives in `/api/support/dump` (logs, a week of " + + "series, control journal, plans and warnings, redacted config) and " + + "`/api/mpc/diagnose` (every slot the optimizer saw). This report " + + "deliberately omits both to stay readable.\n") } // ---- findings ---- diff --git a/go/internal/api/api_support_week.go b/go/internal/api/api_support_week.go new file mode 100644 index 000000000..0f372eb9b --- /dev/null +++ b/go/internal/api/api_support_week.go @@ -0,0 +1,988 @@ +package api + +// The support file's week. +// +// A tester asked why the battery emptied overnight, and the support file +// could not say: its log covered 75 minutes and its telemetry one hour. The +// box keeps far more than that. This reads seven days back from its own +// storage (series, control journal, plan snapshots, notifications, start +// and stop events, warning counts), sums them per day and night for the +// help report, and ships them in lower resolution in the archive's week/. + +import ( + "context" + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +const ( + supportWeekSpan = 7 * 24 * time.Hour + // supportWeekStep is the series resolution. Five-minute buckets keep + // their min/max envelope, so a short spike survives, and a site with a + // hundred series still makes a file that can be mailed. + supportWeekStep = 5 * time.Minute + // supportWeekBudget bounds the reads. The archive must finish well inside + // the HTTP write timeout on a Raspberry Pi; what is left out is named. + supportWeekBudget = 60 * time.Second + // Day and night split at these local hours, so "last night" is one row. + supportDayStartHour = 6 + supportNightStartHour = 22 + + supportWeekSpanLimit = 20000 + supportWeekReplanLimit = 10000 + supportWeekNoticeLimit = 2000 + supportWeekDigestLimit = 5000 + supportWeekEventLimit = 1000 + + // supportPlanGapWh is the battery energy over a period, beyond what the + // plan's own slots asked for, worth a warning. One kilowatt-hour is a + // tenth of a typical home battery, past rounding and slot-edge effects. + supportPlanGapWh = 1000 + // supportNightNoteWh is the night discharge worth listing even when it + // followed the plan, because it is what people ask about. + supportNightNoteWh = 500 + // supportSourceDownMin is the offline time a device needs before the + // week calls it out. Shorter stretches are polls that came back. + supportSourceDownMin = 5 * time.Minute + // supportRepeatWarnings is how often one warning has to recur in the + // week before it is a finding of its own. + supportRepeatWarnings = 100 + // supportBusyReplans: a period with this many replans and three times + // the median is unusual enough to name. + supportBusyReplans = 30 +) + +// supportPeriod is one day (06–22) or night (22–06) in household time. +type supportPeriod struct { + Start, End time.Time + Night bool + // Partial marks the period that is still running. + Partial bool +} + +// Label names the period the way a person asks about it. +func (p supportPeriod) Label() string { + if !p.Night { + return p.Start.Format("Mon 2 Jan") + ", day" + } + next := p.Start.AddDate(0, 0, 1) + from := p.Start.Format("Mon 2") + if next.Month() != p.Start.Month() { + from = p.Start.Format("Mon 2 Jan") + } + return from + " → " + next.Format("Mon 2 Jan") + ", night" +} + +// supportWeekPeriods splits the last seven days at 06:00 and 22:00 in now's +// zone, from the first boundary inside the window to now. Clock changes +// make a night eight or ten hours long rather than moving the boundary. +func supportWeekPeriods(now time.Time) []supportPeriod { + loc := now.Location() + since := now.Add(-supportWeekSpan) + var bounds []time.Time + y, m, d := since.Date() + for day := time.Date(y, m, d, 0, 0, 0, 0, loc); !day.After(now); day = day.AddDate(0, 0, 1) { + for _, h := range []int{supportDayStartHour, supportNightStartHour} { + b := time.Date(day.Year(), day.Month(), day.Day(), h, 0, 0, 0, loc) + if !b.Before(since) && b.Before(now) { + bounds = append(bounds, b) + } + } + } + out := make([]supportPeriod, 0, len(bounds)) + for i, b := range bounds { + p := supportPeriod{Start: b, End: now, Night: b.Hour() == supportNightStartHour, Partial: true} + if i+1 < len(bounds) { + p.End, p.Partial = bounds[i+1], false + } + out = append(out, p) + } + return out +} + +type supportSeries struct { + Driver, Metric, Unit string + Points []state.SeriesPoint +} + +type supportCount struct { + Name string `json:"name"` + Count int `json:"count"` +} + +// supportPeriodSummary is one row of the week. Nil means not measured. +type supportPeriodSummary struct { + period supportPeriod + Label string `json:"label"` + StartMs int64 `json:"start_ms"` + EndMs int64 `json:"end_ms"` + Night bool `json:"night"` + Partial bool `json:"partial,omitempty"` + SoCMinPct *float64 `json:"soc_min_pct"` + SoCMaxPct *float64 `json:"soc_max_pct"` + BatChargedWh *float64 `json:"battery_charged_wh"` + BatDischargedWh *float64 `json:"battery_discharged_wh"` + ImportWh *float64 `json:"import_wh"` + ExportWh *float64 `json:"export_wh"` + // EVWh integrates the chargers' five-minute series, so it is close, + // not metered. + EVWh *float64 `json:"ev_wh_approx"` + // The plan's battery energy over the slots Core scored in the period, + // against what the batteries did in those slots. Out is discharge. + SlotsScored int `json:"slots_scored"` + SlotsOffPlan int `json:"slots_off_plan"` + PlannedInWh float64 `json:"planned_in_wh"` + PlannedOutWh float64 `json:"planned_out_wh"` + ActualInWh float64 `json:"actual_in_wh"` + ActualOutWh float64 `json:"actual_out_wh"` + SlotGapWh float64 `json:"slot_gap_wh"` + Replans int `json:"replans"` + ReplanReasons []supportCount `json:"replan_reasons,omitempty"` + Fallbacks int `json:"fallbacks"` + FuseLimitedS float64 `json:"fuse_limited_s"` + SourceDown []supportCount `json:"sources_down_s,omitempty"` // seconds per driver + Warnings int64 `json:"warnings"` + Errors int64 `json:"errors"` + Starts int `json:"core_starts"` + UncleanStarts int `json:"core_starts_without_clean_stop"` + EnergyIncomplete bool `json:"energy_incomplete,omitempty"` +} + +// supportWeek is everything the week reads, loaded once per request. +type supportWeek struct { + Since, Until time.Time + Series []supportSeries + Spans []state.ControlSpan + SpansMore bool + Replans []state.PlannerReplan + Notices []state.NotificationEntry + NoticesMore bool + Events []state.Event + // EventBefore is the last start or stop event before the window, so + // the first start inside it can be judged. Nil on a new box. + EventBefore *state.Event + // FirstData is the earliest stored series point in the window. Zero + // when there is none. + FirstData time.Time + Digest []state.LogDigest + DigestMore bool + Periods []supportPeriodSummary + // Gaps names what could not be read, so a missing row is not mistaken + // for a quiet week. + Gaps []string +} + +func supportWeekPoints() int { return int(supportWeekSpan / supportWeekStep) } + +// supportSummaryMetric is a series the period table reads. The help report +// alone loads only these; the archive loads every series. +func supportSummaryMetric(metric string) bool { + return metric == telemetry.DerBattery.String()+"_soc" || metric == telemetry.DerEV.String()+"_w" +} + +// loadSupportWeek reads the last seven days from storage. allSeries loads +// every (driver, metric) series for the archive. It returns nil without a +// store. +func (s *Server) loadSupportWeek(ctx context.Context, now time.Time, allSeries bool) *supportWeek { + if s.deps.State == nil { + return nil + } + now = now.In(time.Local) + ctx, cancel := context.WithTimeout(ctx, supportWeekBudget) + defer cancel() + st := s.deps.State + w := &supportWeek{Since: now.Add(-supportWeekSpan), Until: now} + sinceMs, untilMs := w.Since.UnixMilli(), w.Until.UnixMilli() + gap := func(what string, err error) { + if err != nil { + w.Gaps = append(w.Gaps, what+": "+err.Error()) + } + } + + var err error + w.Spans, w.SpansMore, err = st.ControlSpans(ctx, state.ControlSpanQuery{SinceMs: sinceMs, UntilMs: untilMs, Limit: supportWeekSpanLimit}) + gap("control journal", err) + w.Replans, err = st.PlannerReplans(ctx, sinceMs, untilMs, supportWeekReplanLimit) + gap("plan snapshots", err) + w.Notices, w.NoticesMore, err = st.NotificationsBetween(ctx, sinceMs, untilMs, "", supportWeekNoticeLimit) + gap("notifications", err) + w.Digest, w.DigestMore, err = st.LogDigestBetween(ctx, sinceMs, untilMs, supportWeekDigestLimit) + gap("warning counts", err) + events, err := st.RecentEvents(supportWeekEventLimit) + gap("start and stop events", err) + for i := len(events) - 1; i >= 0; i-- { // newest first from the store + e := events[i] + switch { + case e.TsMs >= sinceMs && e.TsMs <= untilMs: + w.Events = append(w.Events, e) + case e.TsMs < sinceMs && isStartStop(e.Event): + w.EventBefore = &e + } + } + + units := s.metricUnits() + drivers, err := st.DriverNames() + gap("series drivers", err) + metrics, err := st.MetricNames() + gap("series metrics", err) + sort.Strings(drivers) + sort.Strings(metrics) + skipped := 0 + for _, d := range drivers { + for _, m := range metrics { + if !allSeries && !supportSummaryMetric(m) { + continue + } + if ctx.Err() != nil { + skipped++ + continue + } + pts, err := st.LoadSeriesBucketsOrRawContext(ctx, d, m, sinceMs, untilMs, supportWeekPoints()) + if err != nil { + if ctx.Err() == nil { + gap("series "+d+"/"+m, err) + } else { + skipped++ + } + continue + } + if len(pts) > 0 { + w.Series = append(w.Series, supportSeries{Driver: d, Metric: m, Unit: units[m], Points: pts}) + firstMs := pts[0].TsMs // a bucket's newest sample + if pts[0].FirstMS > 0 { + firstMs = pts[0].FirstMS + } + if first := time.UnixMilli(firstMs); w.FirstData.IsZero() || first.Before(w.FirstData) { + w.FirstData = first + } + } + } + } + if skipped > 0 { + w.Gaps = append(w.Gaps, fmt.Sprintf("series: %d left out when the %s read budget ran out", skipped, supportWeekBudget)) + } + + for _, p := range supportWeekPeriods(now) { + w.Periods = append(w.Periods, s.summarisePeriod(ctx, w, p)) + } + if ctx.Err() != nil { + w.Gaps = append(w.Gaps, "energy per period: the read budget ran out") + } + return w +} + +func (s *Server) summarisePeriod(ctx context.Context, w *supportWeek, p supportPeriod) supportPeriodSummary { + startMs, endMs := p.Start.UnixMilli(), p.End.UnixMilli() + out := supportPeriodSummary{period: p, Label: p.Label(), StartMs: startMs, EndMs: endMs, Night: p.Night, Partial: p.Partial} + in := func(ts int64) bool { return ts >= startMs && ts < endMs } + + if ctx.Err() == nil { + if de, err := s.deps.State.DailyEnergy(startMs, endMs); err == nil { + if de.Intervals == 0 { + de.Missing |= state.MissingPower + } + out.BatChargedWh = measured(de.Missing, state.MissingBattery, de.BatChargedWh) + out.BatDischargedWh = measured(de.Missing, state.MissingBattery, de.BatDischargedWh) + out.ImportWh = measured(de.Missing, state.MissingGrid, de.ImportWh) + out.ExportWh = measured(de.Missing, state.MissingGrid, de.ExportWh) + out.EnergyIncomplete = de.Missing&(state.MissingBattery|state.MissingGrid) != 0 + } + } + + widthH := float64(state.BucketWidthMs(w.Since.UnixMilli(), w.Until.UnixMilli(), supportWeekPoints())) / 3_600_000 + for _, ser := range w.Series { + switch { + case ser.Metric == telemetry.DerBattery.String()+"_soc": + for _, pt := range ser.Points { + if !in(pt.TsMs) { + continue + } + out.SoCMinPct = minPtr(out.SoCMinPct, pt.Min*100) + out.SoCMaxPct = maxPtr(out.SoCMaxPct, pt.Max*100) + } + case ser.Metric == telemetry.DerEV.String()+"_w": + for _, pt := range ser.Points { + if in(pt.TsMs) { + v := math.Max(pt.V, 0) * widthH + if out.EVWh == nil { + out.EVWh = new(float64) + } + *out.EVWh += v + } + } + } + } + + down := map[string]float64{} + for _, sp := range w.Spans { + end := sp.EndMs + if end == 0 { + end = w.Until.UnixMilli() + } + if sp.Kind == "slot" { + if end <= startMs || end > endMs { + continue + } + var e struct { + PlannedWh float64 `json:"planned_wh"` + ActualWh float64 `json:"actual_wh"` + } + if json.Unmarshal(sp.First, &e) != nil { + continue + } + out.SlotsScored++ + if sp.Status != "on_target" { + out.SlotsOffPlan++ + } + out.PlannedInWh += math.Max(e.PlannedWh, 0) + out.PlannedOutWh += math.Max(-e.PlannedWh, 0) + out.ActualInWh += math.Max(e.ActualWh, 0) + out.ActualOutWh += math.Max(-e.ActualWh, 0) + out.SlotGapWh += math.Abs(e.PlannedWh - e.ActualWh) + continue + } + overlap := float64(min(end, endMs)-max(sp.StartMs, startMs)) / 1000 + if overlap <= 0 { + continue + } + if sp.Reason == "fuse_limit" || sp.Reason == "fuse_cooldown" { + out.FuseLimitedS += overlap + } + if sp.Kind == "source" && (sp.Status == "offline" || sp.Status == "fault") { + down[sp.Driver+" "+sp.Status] += overlap + } + } + out.SourceDown = sortedCounts(down) + + reasons := map[string]float64{} + for _, r := range w.Replans { + if !in(r.TsMs) { + continue + } + out.Replans++ + reasons[r.Reason]++ + if r.Fallback { + out.Fallbacks++ + } + } + out.ReplanReasons = sortedCounts(reasons) + + for _, g := range w.Digest { + if !in(g.FirstMs) { + continue + } + if g.Level == "ERROR" { + out.Errors += g.Count + } else { + out.Warnings += g.Count + } + } + + prev := w.EventBefore + for i := range w.Events { + e := w.Events[i] + if !isStartStop(e.Event) { + continue + } + if e.Event == "startup" && in(e.TsMs) { + out.Starts++ + // A start right after another start had no stop between them. + // The very first start of a new box has nothing before it. + if prev != nil && prev.Event == "startup" { + out.UncleanStarts++ + } + } + prev = &w.Events[i] + } + return out +} + +func isStartStop(event string) bool { + return event == "startup" || event == "shutdown" || event == "restart" +} + +func measured(m state.HistoryMissing, flag state.HistoryMissing, v float64) *float64 { + if m&flag != 0 { + return nil + } + return &v +} + +func minPtr(p *float64, v float64) *float64 { + if p == nil || v < *p { + return &v + } + return p +} + +func maxPtr(p *float64, v float64) *float64 { + if p == nil || v > *p { + return &v + } + return p +} + +func sortedCounts(m map[string]float64) []supportCount { + out := make([]supportCount, 0, len(m)) + for k, v := range m { + out = append(out, supportCount{Name: k, Count: int(math.Round(v))}) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + return out[i].Name < out[j].Name + }) + return out +} + +// ---- the report section ---- + +func writeWeekSection(b *strings.Builder, w *supportWeek) { + b.WriteString("## What happened over the last week\n\n") + if w == nil { + b.WriteString("The box has no store to read the week from.\n\n") + return + } + fmt.Fprintf(b, "From %s to %s, household time, split into days (06–22) and nights (22–06). "+ + "Built from the box's stored history, control journal, plan snapshots, notifications and "+ + "warning counts, not from the log buffer, which holds about the last hour.\n\n", + w.Since.Format("Mon 2 Jan 15:04"), w.Until.Format("Mon 2 Jan 15:04")) + + findings := weekFindings(w) + if len(findings) == 0 { + b.WriteString("Nothing in the stored week stands out.\n\n") + } else { + writeFindingList(b, findings) + } + + b.WriteString("| Period | Charge | Battery in / out | EV | Grid in / out | Plan vs battery | Replans | Limits and faults |\n") + b.WriteString("|---|---|---|---|---|---|---|---|\n") + periods := w.Periods + if empty := periodsBeforeData(w); empty > 0 { + fmt.Fprintf(b, "| %s to %s | no stored data | | | | | | |\n", + periods[0].period.Start.Format("Mon 2 Jan 15:04"), periods[empty-1].period.End.Format("Mon 2 Jan 15:04")) + periods = periods[empty:] + } + for _, p := range periods { + label := p.Label + if p.Partial { + label += " (so far)" + } + fmt.Fprintf(b, "| %s | %s | %s / %s | %s | %s / %s | %s | %s | %s |\n", + label, fmtSoCRange(p.SoCMinPct, p.SoCMaxPct), + fmtWhPtr(p.BatChargedWh), fmtWhPtr(p.BatDischargedWh), fmtEVWh(p.EVWh), + fmtWhPtr(p.ImportWh), fmtWhPtr(p.ExportWh), + fmtPlanVsBattery(p), fmtReplans(p), fmtLimits(p)) + } + b.WriteString("\n`Plan vs battery` compares the battery energy each scored plan slot asked for " + + "with what the batteries did in it; `off` counts slots outside half to one and a half " + + "times the plan, or in the wrong direction. EV energy is integrated from five-minute " + + "averages, so it is close rather than metered. `—` means not measured.\n\n") + + if top := topDigest(w.Digest, 5); len(top) > 0 { + b.WriteString("Most repeated warnings and errors this week (numbers folded to `#`):\n\n```\n") + for _, g := range top { + fmt.Fprintf(b, "%s ×%d", g.Level, g.Count) + if g.Driver != "" { + fmt.Fprintf(b, " [%s]", g.Driver) + } + fmt.Fprintf(b, " %s (last %s)\n", redactDumpLog(truncateReport(g.Msg, 160)), time.UnixMilli(g.LastMs).In(time.Local).Format("Mon 15:04")) + } + b.WriteString("```\n\n") + } + var notes []string + if w.SpansMore { + notes = append(notes, fmt.Sprintf("the control journal had more than %d spans; the oldest are left out", supportWeekSpanLimit)) + } + if w.DigestMore { + notes = append(notes, fmt.Sprintf("more than %d warning rows; the oldest are left out", supportWeekDigestLimit)) + } + notes = append(notes, w.Gaps...) + if len(notes) > 0 { + b.WriteString("Not complete: " + strings.Join(notes, "; ") + ".\n\n") + } + b.WriteString("The data behind this table is in `week/` in `/api/support/dump`.\n\n") +} + +func writeFindingList(b *strings.Builder, findings []finding) { + order := map[string]int{sevProblem: 0, sevWarning: 1, sevNote: 2} + sort.SliceStable(findings, func(i, j int) bool { + return order[findings[i].Severity] < order[findings[j].Severity] + }) + for _, f := range findings { + fmt.Fprintf(b, "- **%s — %s.** %s\n", f.Severity, f.Title, f.Detail) + } + b.WriteString("\n") +} + +// weekFindings states what stands out in the week, so the section reads +// without the table. +func weekFindings(w *supportWeek) []finding { + var out []finding + var followed []string + for _, p := range w.Periods { + label := p.Label + if p.Partial { + label += " (so far)" + } + charge := "" + if p.SoCMinPct != nil { + charge = fmt.Sprintf(" (charge %s)", fmtSoCRange(p.SoCMinPct, p.SoCMaxPct)) + } + discharged := p.ActualOutWh + if p.BatDischargedWh != nil { + discharged = *p.BatDischargedWh + } + charged := p.ActualInWh + if p.BatChargedWh != nil { + charged = *p.BatChargedWh + } + if p.SlotsScored > 0 { + var parts []string + if d := p.ActualOutWh - p.PlannedOutWh; math.Abs(d) >= supportPlanGapWh { + parts = append(parts, fmt.Sprintf("discharged %s%s, %s than the plan asked for: its %d scored slots planned %s out and measured %s out", + fmtReportWh(discharged), charge, moreOrLess(d), p.SlotsScored, fmtReportWh(p.PlannedOutWh), fmtReportWh(p.ActualOutWh))) + } + if d := p.ActualInWh - p.PlannedInWh; math.Abs(d) >= supportPlanGapWh { + parts = append(parts, fmt.Sprintf("charged %s, %s than the plan asked for: planned %s in, measured %s in", + fmtReportWh(charged), moreOrLess(d), fmtReportWh(p.PlannedInWh), fmtReportWh(p.ActualInWh))) + } + if len(parts) > 0 { + detail := "The battery " + strings.Join(parts, "; and ") + "." + if p.Replans > 0 { + detail += " " + fmtReplans(p) + " that period." + } + detail += " Safety limits, a device that does not follow its command, or a plan replaced mid-slot " + + "look like this; the control journal in `week/` shows which." + out = append(out, finding{sevWarning, "The battery did not follow the plan on " + label, detail}) + continue + } + } + if p.Night && discharged >= supportNightNoteWh { + s := fmt.Sprintf("%s %s%s", label, fmtReportWh(discharged), charge) + if p.SlotsScored > 0 { + s += fmt.Sprintf(", plan %s", fmtReportWh(p.PlannedOutWh)) + } + followed = append(followed, s) + } + } + if len(followed) > 0 { + out = append(out, finding{sevNote, "Night discharge", + "The battery discharged at night: " + strings.Join(followed, "; ") + + ". Where a plan figure is given, the battery stayed within " + fmtReportWh(supportPlanGapWh) + " of it."}) + } + + // Devices that lost contact or reported a fault. + type down struct { + total, longest float64 + from int64 + } + downs := map[string]*down{} + for _, sp := range w.Spans { + if sp.Kind != "source" || (sp.Status != "offline" && sp.Status != "fault") { + continue + } + end := sp.EndMs + if end == 0 { + end = w.Until.UnixMilli() + } + start := max(sp.StartMs, w.Since.UnixMilli()) + secs := float64(end-start) / 1000 + if secs <= 0 { + continue + } + key := sp.Driver + "\x00" + sp.Status + d := downs[key] + if d == nil { + d = &down{} + downs[key] = d + } + d.total += secs + if secs > d.longest { + d.longest, d.from = secs, start + } + } + keys := make([]string, 0, len(downs)) + for k := range downs { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + d := downs[k] + if time.Duration(d.total*float64(time.Second)) < supportSourceDownMin { + continue + } + driver, status, _ := strings.Cut(k, "\x00") + verb, sev, title := "was offline", sevWarning, driver+" lost contact this week" + effect := "A device out of contact gets its safe default mode and is left out of the plan, so its share becomes grid flow." + if status == "fault" { + verb, sev, title = "reported a fault", sevProblem, driver+" reported a fault this week" + effect = "A device in fault can be reached but cannot act, so the plan leaves it out and its share becomes grid flow." + } + out = append(out, finding{sev, title, + fmt.Sprintf("%s %s for %s in total; the longest stretch, %s, began %s. %s", + driver, verb, fmtHours(d.total), fmtHours(d.longest), time.UnixMilli(d.from).In(time.Local).Format("Mon 2 Jan 15:04"), effect)}) + } + var fuseS, worstFuse float64 + worstLabel := "" + for _, p := range w.Periods { + fuseS += p.FuseLimitedS + if p.FuseLimitedS > worstFuse { + worstFuse, worstLabel = p.FuseLimitedS, p.Label + } + } + if fuseS >= 60 { + out = append(out, finding{sevNote, "The fuse limit held power back", + fmt.Sprintf("Charging or discharging was held under the main fuse for %s this week, most on %s. "+ + "That is protection working; it explains slower charging, not a fault.", fmtHours(fuseS), worstLabel)}) + } + + fallbacks, last := 0, state.PlannerReplan{} + for _, r := range w.Replans { + if r.Fallback { + fallbacks++ + last = r + } + } + if fallbacks > 0 { + out = append(out, finding{sevWarning, "Plans fell back to Core", + fmt.Sprintf("%d of %d plans this week came from Core's own planner instead of the optimizer. "+ + "The last, %s: %s.", fallbacks, len(w.Replans), + time.UnixMilli(last.TsMs).In(time.Local).Format("Mon 2 Jan 15:04"), + strings.TrimSuffix(redactDumpLog(truncateReport(last.FallbackReason, 200)), "."))}) + } + + if busy, ok := busyPeriod(w.Periods); ok { + out = append(out, finding{sevNote, "Many replans on " + busy.Label, + fmt.Sprintf("%s. Frequent replans usually follow a forecast that keeps missing, "+ + "a battery that drifts from its planned charge, or a device that comes and goes.", fmtReplans(busy))}) + } + + var starts, unclean int + var startTimes []string + for _, p := range w.Periods { + starts += p.Starts + unclean += p.UncleanStarts + } + for _, e := range w.Events { + if e.Event == "startup" { + startTimes = append(startTimes, time.UnixMilli(e.TsMs).In(time.Local).Format("Mon 15:04")) + } + } + stops := 0 + for _, e := range w.Events { + if e.Event == "restart" || e.Event == "shutdown" { + stops++ + } + } + switch { + case unclean > 0: + out = append(out, finding{sevWarning, "Core stopped without a clean shutdown", + fmt.Sprintf("Core started %s this week (%s); %d of those came after no recorded stop, "+ + "which is a crash, a power cut or a killed process.", plural(starts, "time"), strings.Join(startTimes, ", "), unclean)}) + case starts > 0 || stops > 0: + out = append(out, finding{sevNote, "Core restarted", + fmt.Sprintf("Core recorded %s (restart or shutdown) and %s this week%s. Each restart replans "+ + "and briefly hands devices their default mode.", plural(stops, "stop"), plural(starts, "start"), listSuffix(startTimes))}) + } + + if top := topDigest(w.Digest, 1); len(top) > 0 && top[0].Count >= supportRepeatWarnings { + g := top[0] + sev := sevNote + if g.Level == "ERROR" { + sev = sevWarning + } + who := "" + if g.Driver != "" { + who = " from " + g.Driver + } + out = append(out, finding{sev, "One message dominates the week's log", + fmt.Sprintf("%q%s was logged %d times. The table of repeated messages below has the rest.", + redactDumpLog(truncateReport(g.Msg, 160)), who, g.Count)}) + } + + empty := periodsBeforeData(w) + if empty > 0 { + out = append(out, finding{sevNote, "The stored week starts late", + "The first stored reading is from " + w.FirstData.In(time.Local).Format("Mon 2 Jan 15:04") + + ", so the box is newer than the week or was off before then. The periods before it have no data."}) + } + // The period holding the first reading is incomplete by definition. + holding := -1 + if empty < len(w.Periods) && !w.FirstData.IsZero() && w.FirstData.After(w.Periods[empty].period.Start) { + holding = empty + } + var incomplete []string + for i, p := range w.Periods { + if i < empty || i == holding || p.Partial || !p.EnergyIncomplete { + continue + } + incomplete = append(incomplete, p.Label) + } + if len(incomplete) > 0 { + which := strings.Join(incomplete, "; ") + if len(incomplete) > 3 { + which = fmt.Sprintf("%d periods, from %s to %s", len(incomplete), incomplete[0], incomplete[len(incomplete)-1]) + } + out = append(out, finding{sevNote, "Stored history is incomplete", + "Battery or grid energy is missing for " + which + + ". The box was off, a meter was not read, or history was not written."}) + } + return out +} + +// periodsBeforeData counts the leading periods that end before the first +// stored series point. They have nothing to show. +func periodsBeforeData(w *supportWeek) int { + if w.FirstData.IsZero() { + return 0 + } + n := 0 + for _, p := range w.Periods { + if !p.period.End.After(w.FirstData) { + n++ + continue + } + break + } + return n +} + +func plural(n int, word string) string { + if n == 1 { + return "1 " + word + } + return fmt.Sprintf("%d %ss", n, word) +} + +func listSuffix(items []string) string { + if len(items) == 0 { + return "" + } + return " (starts " + strings.Join(items, ", ") + ")" +} + +func moreOrLess(delta float64) string { + if delta > 0 { + return "more" + } + return "less" +} + +// busyPeriod is the period with the most replans when it has at least +// supportBusyReplans and three times the median. +func busyPeriod(ps []supportPeriodSummary) (supportPeriodSummary, bool) { + if len(ps) < 3 { + return supportPeriodSummary{}, false + } + counts := make([]int, len(ps)) + best := 0 + for i, p := range ps { + counts[i] = p.Replans + if p.Replans > ps[best].Replans { + best = i + } + } + sort.Ints(counts) + median := counts[len(counts)/2] + if ps[best].Replans >= supportBusyReplans && ps[best].Replans >= 3*max(median, 1) { + return ps[best], true + } + return supportPeriodSummary{}, false +} + +// topDigest sums the week's rows per pattern and returns the n most frequent. +func topDigest(rows []state.LogDigest, n int) []state.LogDigest { + type key struct{ level, driver, pattern string } + sum := map[key]*state.LogDigest{} + for _, r := range rows { + k := key{r.Level, r.Driver, r.Pattern} + g := sum[k] + if g == nil { + c := r + sum[k] = &c + continue + } + g.Count += r.Count + if r.LastMs >= g.LastMs { + g.LastMs, g.Msg, g.Attrs = r.LastMs, r.Msg, r.Attrs + } + } + out := make([]state.LogDigest, 0, len(sum)) + for _, g := range sum { + out = append(out, *g) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + return out[i].LastMs > out[j].LastMs + }) + if len(out) > n { + out = out[:n] + } + return out +} + +// csvNumber keeps three decimals: a bucket average otherwise prints sixteen +// digits, while an energy counter in Wh still needs all its whole ones. +func csvNumber(v float64) string { + return strconv.FormatFloat(math.Round(v*1000)/1000, 'f', -1, 64) +} + +func fmtSoCRange(lo, hi *float64) string { + if lo == nil || hi == nil { + return "—" + } + return fmt.Sprintf("%.0f–%.0f%%", *lo, *hi) +} + +func fmtWhPtr(v *float64) string { + if v == nil { + return "—" + } + return fmtReportWh(*v) +} + +func fmtEVWh(v *float64) string { + if v == nil { + return "—" + } + return "≈ " + fmtReportWh(*v) +} + +func fmtHours(secs float64) string { + if secs < 3600 { + return fmt.Sprintf("%.0f min", secs/60) + } + return fmt.Sprintf("%.1f h", secs/3600) +} + +func fmtPlanVsBattery(p supportPeriodSummary) string { + if p.SlotsScored == 0 { + return "—" + } + return fmt.Sprintf("%d of %d off · gap %s", p.SlotsOffPlan, p.SlotsScored, fmtReportWh(p.SlotGapWh)) +} + +func fmtReplans(p supportPeriodSummary) string { + if p.Replans == 0 { + return "0 replans" + } + s := fmt.Sprintf("%d replans", p.Replans) + if p.Replans == 1 { + s = "1 replan" + } + var rs []string + for i, r := range p.ReplanReasons { + if i == 3 { + break + } + rs = append(rs, fmt.Sprintf("%s %d", r.Name, r.Count)) + } + if len(rs) > 0 { + s += " (" + strings.Join(rs, ", ") + ")" + } + if p.Fallbacks > 0 { + s += fmt.Sprintf(" · %d fallback", p.Fallbacks) + } + return s +} + +func fmtLimits(p supportPeriodSummary) string { + var parts []string + if p.FuseLimitedS >= 60 { + parts = append(parts, "fuse "+fmtHours(p.FuseLimitedS)) + } + for _, d := range p.SourceDown { + parts = append(parts, d.Name+" "+fmtHours(float64(d.Count))) + } + if p.Warnings > 0 || p.Errors > 0 { + parts = append(parts, fmt.Sprintf("%d warnings, %d errors", p.Warnings, p.Errors)) + } + if p.Starts > 0 { + parts = append(parts, plural(p.Starts, "start")) + } + if len(parts) == 0 { + return "—" + } + return strings.Join(parts, " · ") +} + +// ---- the archive's week/ ---- + +// addWeekFiles writes the week's data into the archive. JSON passes the +// same redaction as the logs: device errors in the journal carry addresses. +func addWeekFiles(add func(string, []byte), w *supportWeek) { + if w == nil { + return + } + redacted := func(v any) []byte { + raw, err := json.MarshalIndent(v, "", " ") + if err != nil { + return []byte(`{"error":` + fmt.Sprintf("%q", err.Error()) + `}`) + } + return []byte(redactDumpLog(string(raw))) + } + window := func(extra map[string]any) map[string]any { + extra["since_ms"], extra["until_ms"] = w.Since.UnixMilli(), w.Until.UnixMilli() + return extra + } + index := make([]map[string]any, 0, len(w.Series)) + for _, ser := range w.Series { + index = append(index, map[string]any{"driver": ser.Driver, "metric": ser.Metric, "unit": ser.Unit, "buckets": len(ser.Points)}) + } + add("week/summary.json", redacted(window(map[string]any{ + "tz": w.Until.Location().String(), "bucket_ms": state.BucketWidthMs(w.Since.UnixMilli(), w.Until.UnixMilli(), supportWeekPoints()), + "periods": w.Periods, "series": index, "gaps": append([]string{}, w.Gaps...), + }))) + spans := make([]journalSpan, 0, len(w.Spans)) + for _, sp := range w.Spans { + spans = append(spans, toJournalSpan(sp, w.Until.UnixMilli())) + } + add("week/control-journal.json", redacted(window(map[string]any{"spans": spans, "more": w.SpansMore}))) + add("week/plans.json", redacted(window(map[string]any{"replans": w.Replans}))) + add("week/notifications.json", redacted(window(map[string]any{"notifications": w.Notices, "more": w.NoticesMore}))) + events := make([]map[string]any, 0, len(w.Events)) + for _, e := range w.Events { + events = append(events, map[string]any{"ts_ms": e.TsMs, "event": e.Event}) + } + add("week/events.json", redacted(window(map[string]any{"events": events}))) + + for _, ser := range w.Series { + var b strings.Builder + b.WriteString("ts_ms,v,min,max,n\n") + for _, p := range ser.Points { + fmt.Fprintf(&b, "%d,%s,%s,%s,%d\n", p.TsMs, csvNumber(p.V), csvNumber(p.Min), csvNumber(p.Max), p.N) + } + add("week/series/"+sanitizeName(ser.Driver)+"__"+sanitizeName(ser.Metric)+".csv", []byte(b.String())) + } + + var b strings.Builder + if len(w.Digest) == 0 { + b.WriteString("No warnings or errors stored for the week.\n") + } + for _, g := range w.Digest { + fmt.Fprintf(&b, "%s %s ", time.UnixMilli(g.HourMs).UTC().Format("2006-01-02T15Z"), g.Level) + if g.Driver != "" { + fmt.Fprintf(&b, "[%s] ", g.Driver) + } + b.WriteString(redactDumpLog(g.Msg)) + fmt.Fprintf(&b, " ×%d first %s last %s", g.Count, + time.UnixMilli(g.FirstMs).UTC().Format("15:04:05"), time.UnixMilli(g.LastMs).UTC().Format("15:04:05")) + if g.Attrs != "" { + b.WriteString(" " + redactDumpLog(g.Attrs)) + } + b.WriteByte('\n') + } + add("logs/warnings-week.log", []byte(b.String())) +} diff --git a/go/internal/api/api_support_week_test.go b/go/internal/api/api_support_week_test.go new file mode 100644 index 000000000..ee451c419 --- /dev/null +++ b/go/internal/api/api_support_week_test.go @@ -0,0 +1,306 @@ +package api + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/control" + "github.com/srcfl/ftw/go/internal/state" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// A tester asked why the battery emptied during the night. The support file +// covered the last hour and could not say. This seeds a week where, one +// night, the battery discharged 8 kWh while every plan slot asked it to +// hold, a charger lost contact and the fuse limited charging, and checks +// that the file answers from the box's own storage. +func TestSupportDumpAnswersForTheLastWeek(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + ctx := context.Background() + + now := time.Now() + y, m, d := now.AddDate(0, 0, -1).Date() + midnight := time.Date(y, m, d, 0, 0, 0, 0, time.Local) // inside last night but one + from, to := midnight.Add(-2*time.Hour), midnight.Add(6*time.Hour) + + var samples []state.Sample + var history []state.HistoryPoint + for ts := from; !ts.After(to); ts = ts.Add(time.Minute) { + batW, soc := 0.0, 0.8 + switch { + case !ts.Before(midnight) && ts.Before(midnight.Add(4*time.Hour)): + batW = -2000 + soc = 0.8 - 0.5*ts.Sub(midnight).Hours()/4 + case !ts.Before(midnight.Add(4 * time.Hour)): + soc = 0.3 + } + ms := ts.UnixMilli() + samples = append(samples, + state.Sample{Driver: "pixii", Metric: "battery_w", TsMs: ms, Value: batW, Unit: "W"}, + state.Sample{Driver: "pixii", Metric: "battery_soc", TsMs: ms, Value: soc}, + state.Sample{Driver: "easee", Metric: "ev_w", TsMs: ms, Value: 0, Unit: "W"}, + ) + history = append(history, state.HistoryPoint{TsMs: ms, GridW: 400, BatW: batW, LoadW: 2400, BatSoC: soc, JSON: `{}`}) + } + if err := st.RecordSamples(samples); err != nil { + t.Fatal(err) + } + if err := st.BulkRecordHistory(history); err != nil { + t.Fatal(err) + } + + // Every 15-minute slot that night asked the battery to hold. + var spans []*state.ControlSpan + for s := midnight; s.Before(midnight.Add(4 * time.Hour)); s = s.Add(15 * time.Minute) { + first, _ := json.Marshal(map[string]any{"planned_wh": 0.0, "actual_wh": -500.0, "ratio": 0, "replans": 1}) + spans = append(spans, &state.ControlSpan{Subject: "slot:batteries", Kind: "slot", Status: "over_delivered", + Severity: "info", StartMs: s.UnixMilli(), EndMs: s.Add(15 * time.Minute).UnixMilli(), + UpdatedMs: s.Add(15 * time.Minute).UnixMilli(), Samples: 1, First: first, Last: first}) + } + spans = append(spans, + &state.ControlSpan{Subject: "source:easee", Driver: "easee", Kind: "source", Status: "offline", + Reason: "telemetry_stale", Severity: "warning", StartMs: midnight.Add(time.Hour).UnixMilli(), + EndMs: midnight.Add(3 * time.Hour).UnixMilli(), UpdatedMs: midnight.Add(3 * time.Hour).UnixMilli(), Samples: 1440}, + &state.ControlSpan{Subject: "ev:easee", Driver: "easee", Kind: "ev", Status: "limited", + Reason: "fuse_limit", Severity: "info", StartMs: midnight.Add(-90 * time.Minute).UnixMilli(), + EndMs: midnight.Add(-60 * time.Minute).UnixMilli(), UpdatedMs: midnight.Add(-60 * time.Minute).UnixMilli(), Samples: 360}, + ) + if err := st.SaveControlSpans(ctx, spans); err != nil { + t.Fatal(err) + } + for i, reason := range []string{"scheduled", "soc_drift", "soc_drift"} { + js := `{"solver":{"engine":"energyplan","status":"optimal"}}` + if i == 2 { + js = `{"solver":{"engine":"core","backend":"dp","status":"fallback","fallback":true,"fallback_reason":"optimizer timed out"}}` + } + if err := st.SaveDiagnostic(midnight.Add(time.Duration(i)*time.Hour).UnixMilli(), reason, "SE3", 10, 96, js); err != nil { + t.Fatal(err) + } + } + if err := st.RecordNotification(state.NotificationEntry{TsMs: midnight.Add(time.Hour).UnixMilli(), + EventType: "driver_offline", Driver: "easee", Title: "Charger offline", Body: "easee stopped answering", Status: "sent"}); err != nil { + t.Fatal(err) + } + hour := midnight.Add(time.Hour).Truncate(time.Hour) + if err := st.AddLogDigest(ctx, []state.LogDigest{{HourMs: hour.UnixMilli(), Level: "WARN", Driver: "easee", + Pattern: "poll failed: HTTP #", Msg: "poll failed: HTTP 503", Count: 240, + FirstMs: hour.UnixMilli(), LastMs: hour.Add(59 * time.Minute).UnixMilli()}}); err != nil { + t.Fatal(err) + } + if err := st.RecordEvent("restart"); err != nil { + t.Fatal(err) + } + time.Sleep(2 * time.Millisecond) // events are keyed by the millisecond + if err := st.RecordEvent("startup"); err != nil { + t.Fatal(err) + } + + srv := New(&Deps{ + Ctrl: control.NewState(0, 50, "meter"), CtrlMu: &sync.Mutex{}, + Tel: telemetry.NewStore(), LogRing: telemetry.NewLogRing(), Version: "test", State: st, + }) + files := readSupportZip(t, srv) + + for _, want := range []string{ + "week/summary.json", "week/series/pixii__battery_w.csv", "week/series/pixii__battery_soc.csv", + "week/control-journal.json", "week/plans.json", "week/notifications.json", "week/events.json", + "logs/warnings-week.log", + } { + if _, ok := files[want]; !ok { + t.Errorf("archive is missing %s", want) + } + } + var manifest struct { + Contents []string `json:"contents"` + } + if err := json.Unmarshal([]byte(files["manifest.json"]), &manifest); err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(manifest.Contents, " "), "week/series/__.csv") { + t.Errorf("manifest does not list the week's series: %v", manifest.Contents) + } + + // The series come from storage at five-minute resolution, far past the + // last hour the old file stopped at. + csv := strings.Split(strings.TrimSpace(files["week/series/pixii__battery_w.csv"]), "\n") + if csv[0] != "ts_ms,v,min,max,n" { + t.Errorf("series header = %q", csv[0]) + } + if rows := len(csv) - 1; rows < 90 || rows > 100 { + t.Errorf("eight hours of minute samples gave %d rows, want about 96 five-minute buckets", rows) + } + if !strings.Contains(files["week/series/pixii__battery_w.csv"], ",-2000,-2000,") { + t.Error("series lost the discharge") + } + if !strings.Contains(files["week/control-journal.json"], "source:easee") { + t.Error("control journal file lacks the charger's offline span") + } + if !strings.Contains(files["week/plans.json"], "optimizer timed out") { + t.Error("plans file lacks the fallback reason") + } + if !strings.Contains(files["logs/warnings-week.log"], "poll failed: HTTP 503") || !strings.Contains(files["logs/warnings-week.log"], "×240") { + t.Errorf("warnings-week.log = %q", files["logs/warnings-week.log"]) + } + + report := files["00-help-report.md"] + week := sectionOf(report, "## What happened over the last week") + if week == "" { + t.Fatalf("report has no week section:\n%s", report) + } + for _, want := range []string{ + "discharged 8.00 kWh", // the night's measured discharge + "30–80%", // its charge range + "more than the plan", // and that the plan did not ask for it + "easee", // the charger that went offline + "offline for 2.0 h", // and for how long + "fuse", // the fuse limit on charging + "Core's own planner", // the fallback plan + "optimizer timed out", // with its reason + "restart", // the box restarted + "poll failed: HTTP 503", // the week's most repeated warning + "| Period |", // and the table behind the bullets + } { + if !strings.Contains(week, want) { + t.Errorf("week section lacks %q:\n%s", want, week) + } + } + if strings.Index(report, "## What happened over the last week") > strings.Index(report, "## Right now") { + t.Error("the week section belongs before Right now, next to Findings") + } +} + +// Periods split the week at 06:00 and 22:00 household time, so a question +// about "last night" lands in one row. The window opens at the first +// boundary inside the last seven days and ends now. +func TestSupportWeekPeriods(t *testing.T) { + loc, err := time.LoadLocation("Europe/Stockholm") + if err != nil { + t.Skip("no zone database") + } + now := time.Date(2026, 11, 15, 12, 54, 0, 0, loc) + ps := supportWeekPeriods(now) + if len(ps) != 14 { + t.Fatalf("got %d periods, want 7 nights and 7 days", len(ps)) + } + if !ps[0].Start.Equal(time.Date(2026, 11, 8, 22, 0, 0, 0, loc)) || !ps[0].Night { + t.Errorf("first period = %+v, want the night from 8 Nov 22:00", ps[0]) + } + last := ps[len(ps)-1] + if last.Night || !last.Start.Equal(time.Date(2026, 11, 15, 6, 0, 0, 0, loc)) || !last.End.Equal(now) || !last.Partial { + t.Errorf("last period = %+v, want today's day so far", last) + } + for i := 1; i < len(ps); i++ { + if !ps[i].Start.Equal(ps[i-1].End) { + t.Fatalf("gap between %v and %v", ps[i-1].End, ps[i].Start) + } + } + if got := ps[0].Label(); got != "Sun 8 → Mon 9 Nov, night" { + t.Errorf("night label = %q", got) + } + if got := ps[1].Label(); got != "Mon 9 Nov, day" { + t.Errorf("day label = %q", got) + } + + // The autumn clock change makes one night nine hours long. + dst := supportWeekPeriods(time.Date(2026, 10, 27, 12, 0, 0, 0, loc)) + for _, p := range dst { + if p.Night && p.Start.Day() == 24 { + if h := p.End.Sub(p.Start).Hours(); h != 9 { + t.Errorf("night over the clock change is %.1f h, want 9", h) + } + } + } +} + +func sectionOf(report, heading string) string { + i := strings.Index(report, heading) + if i < 0 { + return "" + } + rest := report[i+len(heading):] + if j := strings.Index(rest, "\n## "); j >= 0 { + rest = rest[:j] + } + return rest +} + +// A night that discharged as planned is still listed, because it is what +// people ask about; a start with no stop before it is a crash. +func TestWeekFindingsNightDischargeAndCrash(t *testing.T) { + loc := time.FixedZone("test", 3600) + start := time.Date(2026, 11, 13, 22, 0, 0, 0, loc) + night := supportPeriod{Start: start, End: start.Add(8 * time.Hour), Night: true} + out, soc := 3100.0, 20.0 + full := 90.0 + w := &supportWeek{ + Since: start.Add(-24 * time.Hour), Until: start.Add(24 * time.Hour), + Periods: []supportPeriodSummary{{ + period: night, Label: night.Label(), Night: true, BatDischargedWh: &out, + SoCMinPct: &soc, SoCMaxPct: &full, SlotsScored: 32, PlannedOutWh: 3000, ActualOutWh: 3100, + Starts: 2, UncleanStarts: 1, + }}, + Events: []state.Event{ + {TsMs: start.Add(time.Hour).UnixMilli(), Event: "startup"}, + {TsMs: start.Add(2 * time.Hour).UnixMilli(), Event: "startup"}, + }, + } + var text strings.Builder + writeFindingList(&text, weekFindings(w)) + got := text.String() + for _, want := range []string{ + "NOTE — Night discharge.", "3.10 kWh (charge 20–90%), plan 3.00 kWh", + "WARNING — Core stopped without a clean shutdown.", "Core started 2 times", + } { + if !strings.Contains(got, want) { + t.Errorf("findings lack %q:\n%s", want, got) + } + } + if strings.Contains(got, "did not follow the plan") { + t.Errorf("a night within 1 kWh of its plan is not a miss:\n%s", got) + } +} + +// A box installed today has one start and an hour of data. Its first start +// had no stop before it because there was no run before it, and the empty +// days before the install fold into one row. +func TestSupportWeekOnANewBox(t *testing.T) { + st, err := state.Open(filepath.Join(t.TempDir(), "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + if err := st.RecordEvent("startup"); err != nil { + t.Fatal(err) + } + now := time.Now() + var samples []state.Sample + for ts := now.Add(-time.Hour); ts.Before(now); ts = ts.Add(time.Minute) { + samples = append(samples, state.Sample{Driver: "sungrow", Metric: "battery_soc", TsMs: ts.UnixMilli(), Value: 0.5}) + } + if err := st.RecordSamples(samples); err != nil { + t.Fatal(err) + } + srv := New(&Deps{ + Ctrl: control.NewState(0, 50, "meter"), CtrlMu: &sync.Mutex{}, + Tel: telemetry.NewStore(), LogRing: telemetry.NewLogRing(), Version: "test", State: st, + }) + week := sectionOf(srv.buildSupportReport(context.Background(), now), "## What happened over the last week") + if strings.Contains(week, "clean shutdown") { + t.Errorf("a new box's first start is not a crash:\n%s", week) + } + if !strings.Contains(week, "no stored data") || !strings.Contains(week, "The stored week starts late") { + t.Errorf("the empty days before the install should fold into one row:\n%s", week) + } + if rows := strings.Count(week, "\n| "); rows > 4 { + t.Errorf("week table has %d rows for an hour of data:\n%s", rows, week) + } +} diff --git a/go/internal/ftwcli/backup.go b/go/internal/ftwcli/backup.go index 2b41fc15e..7ae66097d 100644 --- a/go/internal/ftwcli/backup.go +++ b/go/internal/ftwcli/backup.go @@ -221,6 +221,10 @@ func (c *client) copyBackup(ctx context.Context, out io.Writer, dir string, b ba return nil } +// supportFileTimeout matches Core's HTTP write timeout. The file reads a +// week from storage, which takes a Raspberry Pi longer than a status call. +const supportFileTimeout = 2 * time.Minute + func runSupport(args []string, out io.Writer, e env) error { var output string base, err := parse(args, out, "ftw support [--output FILE] [--url URL]", func(fs *flag.FlagSet) { @@ -233,7 +237,7 @@ func runSupport(args []string, out io.Writer, e env) error { output = "ftw-support-" + e.now().Format("20060102-150405") + ".zip" } c := newClient(base, e) - ctx, cancel := context.WithTimeout(context.Background(), 2*e.requestTimeout) + ctx, cancel := context.WithTimeout(context.Background(), max(2*e.requestTimeout, supportFileTimeout)) defer cancel() resp, err := c.open(ctx, "/api/support/dump") if err != nil { diff --git a/go/internal/state/diagnostics.go b/go/internal/state/diagnostics.go index 5c38ef15a..19d5bde5e 100644 --- a/go/internal/state/diagnostics.go +++ b/go/internal/state/diagnostics.go @@ -86,6 +86,57 @@ func (s *Store) LoadDiagnosticsInRange(sinceMs, untilMs int64, return out, rows.Err() } +// PlannerReplan is a stored replan with the solver fields the support week +// counts: which engine made the plan, and whether it was Core's fallback. +type PlannerReplan struct { + DiagnosticSummary + Engine string `json:"engine,omitempty"` + Backend string `json:"backend,omitempty"` + Status string `json:"status,omitempty"` + Fallback bool `json:"fallback,omitempty"` + FallbackReason string `json:"fallback_reason,omitempty"` +} + +// PlannerReplans returns the replans in [sinceMs, untilMs], oldest first. +// A positive limit keeps the newest. SQLite reads the solver fields from +// each blob, so Go never holds the full snapshots. +func (s *Store) PlannerReplans(ctx context.Context, sinceMs, untilMs int64, limit int) ([]PlannerReplan, error) { + q := `SELECT ts_ms, reason, zone, total_cost_ore, horizon_slots, + COALESCE(json_extract(json, '$.solver.engine'), ''), + COALESCE(json_extract(json, '$.solver.backend'), ''), + COALESCE(json_extract(json, '$.solver.status'), ''), + COALESCE(json_extract(json, '$.solver.fallback') = 1, 0), + COALESCE(json_extract(json, '$.solver.fallback_reason'), '') + FROM planner_diagnostics WHERE ts_ms >= ? AND ts_ms <= ? + ORDER BY ts_ms DESC` + args := []any{sinceMs, untilMs} + if limit > 0 { + q += ` LIMIT ?` + args = append(args, limit) + } + rows, err := s.db.QueryContext(ctx, q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := []PlannerReplan{} + for rows.Next() { + var r PlannerReplan + if err := rows.Scan(&r.TsMs, &r.Reason, &r.Zone, &r.TotalCostOre, &r.HorizonSlots, + &r.Engine, &r.Backend, &r.Status, &r.Fallback, &r.FallbackReason); err != nil { + return nil, err + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + return nil, err + } + for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] + } + return out, nil +} + // PlannerDay sums the plans kept since sinceMs: how many, the longest any // took to solve in milliseconds, and how many were a fallback plan. The // anonymous fleet report coarsens these into buckets; nothing else of a plan diff --git a/go/internal/state/log_digest.go b/go/internal/state/log_digest.go new file mode 100644 index 000000000..10b099047 --- /dev/null +++ b/go/internal/state/log_digest.go @@ -0,0 +1,101 @@ +package state + +import ( + "context" + "fmt" +) + +// LogDigest is one warning or error pattern as logged within one hour: how +// often, when first and last, and the latest full text. The support file +// reads a week of these. The in-memory log ring cannot hold that long: a +// driver logging every control tick filled it in 75 minutes on a real site, +// and a restart empties it. +type LogDigest struct { + HourMs int64 `json:"hour_ms"` + Level string `json:"level"` + Driver string `json:"driver,omitempty"` + // Pattern is the message with its numbers folded, the key that groups + // repeats whose figures differ. + Pattern string `json:"pattern"` + Msg string `json:"msg"` + Attrs string `json:"attrs,omitempty"` + Count int64 `json:"count"` + FirstMs int64 `json:"first_ms"` + LastMs int64 `json:"last_ms"` +} + +// AddLogDigest adds counts to the stored rows. A row seen again in the same +// hour keeps its first sighting and takes the latest text. +func (s *Store) AddLogDigest(ctx context.Context, rows []LogDigest) error { + if len(rows) == 0 { + return nil + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("log digest: begin: %w", err) + } + defer tx.Rollback() + for _, r := range rows { + if _, err := tx.ExecContext(ctx, `INSERT INTO log_digest + (hour_ms, level, driver, pattern, msg, attrs, count, first_ms, last_ms) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (hour_ms, level, driver, pattern) DO UPDATE SET + count = count + excluded.count, + first_ms = MIN(first_ms, excluded.first_ms), + msg = CASE WHEN excluded.last_ms >= last_ms THEN excluded.msg ELSE msg END, + attrs = CASE WHEN excluded.last_ms >= last_ms THEN excluded.attrs ELSE attrs END, + last_ms = MAX(last_ms, excluded.last_ms)`, + r.HourMs, r.Level, r.Driver, r.Pattern, r.Msg, r.Attrs, r.Count, r.FirstMs, r.LastMs); err != nil { + return fmt.Errorf("log digest: add: %w", err) + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("log digest: commit: %w", err) + } + return nil +} + +// LogDigestBetween returns the rows whose hour starts in [sinceMs, untilMs], +// oldest first, and whether more matched than the limit allowed. The limit +// keeps the newest. +func (s *Store) LogDigestBetween(ctx context.Context, sinceMs, untilMs int64, limit int) ([]LogDigest, bool, error) { + if limit <= 0 { + limit = 1000 + } + // A row's hour may begin before sinceMs and still hold sightings inside. + rows, err := s.db.QueryContext(ctx, `SELECT hour_ms, level, driver, pattern, msg, attrs, count, first_ms, last_ms + FROM log_digest WHERE hour_ms <= ? AND last_ms >= ? + ORDER BY hour_ms DESC, first_ms DESC LIMIT ?`, untilMs, sinceMs, limit+1) + if err != nil { + return nil, false, fmt.Errorf("log digest: query: %w", err) + } + defer rows.Close() + out := []LogDigest{} + for rows.Next() { + var r LogDigest + if err := rows.Scan(&r.HourMs, &r.Level, &r.Driver, &r.Pattern, &r.Msg, &r.Attrs, &r.Count, &r.FirstMs, &r.LastMs); err != nil { + return nil, false, fmt.Errorf("log digest: scan: %w", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + return nil, false, fmt.Errorf("log digest: rows: %w", err) + } + more := len(out) > limit + if more { + out = out[:limit] + } + for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] + } + return out, more, nil +} + +// PruneLogDigest deletes the hours that began before beforeMs. +func (s *Store) PruneLogDigest(ctx context.Context, beforeMs int64) (int64, error) { + res, err := s.db.ExecContext(ctx, `DELETE FROM log_digest WHERE hour_ms < ?`, beforeMs) + if err != nil { + return 0, fmt.Errorf("log digest: prune: %w", err) + } + return res.RowsAffected() +} diff --git a/go/internal/state/log_digest_test.go b/go/internal/state/log_digest_test.go new file mode 100644 index 000000000..4ff63a5d3 --- /dev/null +++ b/go/internal/state/log_digest_test.go @@ -0,0 +1,112 @@ +package state + +import ( + "context" + "testing" +) + +// The digest keeps a week of warnings in a few rows. Two flushes of the +// same message in the same hour must add up, keep the earliest first +// sighting and the latest text, and stay one row. +func TestLogDigestAddsUpRepeatsWithinAnHour(t *testing.T) { + st := openTestStore(t) + ctx := context.Background() + const hour = int64(3_600_000) * 500_000 + first := LogDigest{HourMs: hour, Level: "WARN", Driver: "pixii", Pattern: "write failed reg=#", + Msg: "write failed reg=40", Attrs: "err=timeout", Count: 3, FirstMs: hour + 1_000, LastMs: hour + 9_000} + if err := st.AddLogDigest(ctx, []LogDigest{first}); err != nil { + t.Fatal(err) + } + later := first + later.Msg, later.Attrs, later.Count = "write failed reg=41", "err=reset", 2 + later.FirstMs, later.LastMs = hour+60_000, hour+70_000 + other := LogDigest{HourMs: hour + 3_600_000, Level: "ERROR", Pattern: "optimizer: deadline", + Msg: "optimizer: deadline", Count: 1, FirstMs: hour + 3_700_000, LastMs: hour + 3_700_000} + if err := st.AddLogDigest(ctx, []LogDigest{later, other}); err != nil { + t.Fatal(err) + } + + got, more, err := st.LogDigestBetween(ctx, hour, hour+2*3_600_000, 100) + if err != nil || more { + t.Fatalf("LogDigestBetween: more=%v err=%v", more, err) + } + if len(got) != 2 { + t.Fatalf("got %d rows, want 2: %+v", len(got), got) + } + w := got[0] + if w.Count != 5 || w.FirstMs != hour+1_000 || w.LastMs != hour+70_000 { + t.Errorf("repeat row = count %d first %d last %d, want 5, %d, %d", w.Count, w.FirstMs, w.LastMs, hour+1_000, hour+70_000) + } + if w.Msg != "write failed reg=41" || w.Attrs != "err=reset" { + t.Errorf("repeat row keeps %q %q, want the latest text", w.Msg, w.Attrs) + } + if got[1].Level != "ERROR" || got[1].Driver != "" { + t.Errorf("second row = %+v, want the core error", got[1]) + } + + // A window that ends before the second hour leaves it out. + got, _, err = st.LogDigestBetween(ctx, hour, hour+3_599_999, 100) + if err != nil || len(got) != 1 { + t.Fatalf("first hour only: %+v err=%v", got, err) + } + // The limit is honoured and says more matched. + got, more, err = st.LogDigestBetween(ctx, hour, hour+2*3_600_000, 1) + if err != nil || len(got) != 1 || !more { + t.Fatalf("limit 1: %d rows more=%v err=%v", len(got), more, err) + } + + n, err := st.PruneLogDigest(ctx, hour+3_600_000) + if err != nil || n != 1 { + t.Fatalf("prune: n=%d err=%v, want the first hour gone", n, err) + } + got, _, _ = st.LogDigestBetween(ctx, 0, hour+10*3_600_000, 100) + if len(got) != 1 || got[0].Level != "ERROR" { + t.Fatalf("after prune: %+v", got) + } +} + +// The week view counts fallback plans from the stored diagnostics without +// handing an 85 kB blob per replan to Go. +func TestPlannerReplansReadSolverFields(t *testing.T) { + st := openTestStore(t) + ctx := context.Background() + if err := st.SaveDiagnostic(1_000, "scheduled", "SE3", 12.5, 96, + `{"solver":{"engine":"energyplan","backend":"milp","status":"optimal"}}`); err != nil { + t.Fatal(err) + } + if err := st.SaveDiagnostic(2_000, "soc_drift", "SE3", 14, 96, + `{"solver":{"engine":"core","backend":"dp","status":"fallback","fallback":true,"fallback_reason":"optimizer timed out"}}`); err != nil { + t.Fatal(err) + } + if err := st.SaveDiagnostic(3_000, "scheduled", "SE3", 9, 96, `{}`); err != nil { + t.Fatal(err) + } + got, err := st.PlannerReplans(ctx, 0, 2_500, 0) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("got %d replans, want 2 inside the window: %+v", len(got), got) + } + if got[0].TsMs != 1_000 || got[1].TsMs != 2_000 { + t.Fatalf("replans not oldest first: %+v", got) + } + if got[0].Engine != "energyplan" || got[0].Fallback { + t.Errorf("optimizer plan = %+v", got[0]) + } + fb := got[1] + if !fb.Fallback || fb.Engine != "core" || fb.Backend != "dp" || fb.FallbackReason != "optimizer timed out" || fb.Reason != "soc_drift" { + t.Errorf("fallback plan = %+v", fb) + } + if fb.TotalCostOre != 14 || fb.HorizonSlots != 96 || fb.Zone != "SE3" { + t.Errorf("fallback summary = %+v", fb.DiagnosticSummary) + } + // A plan without solver data reads as empty, not as an error. + all, err := st.PlannerReplans(ctx, 0, 10_000, 0) + if err != nil || len(all) != 3 || all[2].Engine != "" || all[2].Fallback { + t.Fatalf("plan without solver: %+v err=%v", all, err) + } + if lim, err := st.PlannerReplans(ctx, 0, 10_000, 1); err != nil || len(lim) != 1 || lim[0].TsMs != 3_000 { + t.Fatalf("limit keeps the newest: %+v err=%v", lim, err) + } +} diff --git a/go/internal/state/store.go b/go/internal/state/store.go index b2cdd95cd..d800879fc 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -1019,6 +1019,21 @@ func (s *Store) migrate() error { ) STRICT`, `CREATE INDEX IF NOT EXISTS idx_control_journal_start ON control_journal(start_ms DESC)`, `CREATE INDEX IF NOT EXISTS idx_control_journal_open ON control_journal(end_ms) WHERE end_ms IS NULL`, + // log_digest: warning and error patterns per hour (log_digest.go), + // so the support file can show a week of them. At most a few hundred + // rows an hour, pruned after eight days; kept in recovery snapshots. + `CREATE TABLE IF NOT EXISTS log_digest ( + hour_ms INTEGER NOT NULL, + level TEXT NOT NULL, + driver TEXT NOT NULL DEFAULT '', + pattern TEXT NOT NULL, + msg TEXT NOT NULL, + attrs TEXT NOT NULL DEFAULT '', + count INTEGER NOT NULL, + first_ms INTEGER NOT NULL, + last_ms INTEGER NOT NULL, + PRIMARY KEY (hour_ms, level, driver, pattern) + ) STRICT`, // Independently installed Lua drivers. Content lives on disk; SQLite // records activation history and the exact previous artifact used for diff --git a/go/internal/telemetry/logring.go b/go/internal/telemetry/logring.go index ef6e883ea..3b54d237b 100644 --- a/go/internal/telemetry/logring.go +++ b/go/internal/telemetry/logring.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "log/slog" + "sort" "strings" "sync" "time" @@ -31,14 +32,59 @@ const ( ringGlobal = 2000 ) +const ( + // digestPatternsPerHour caps the distinct warning patterns kept per + // hour. A driver that puts a changing word in every warning would + // otherwise add a row per line; past the cap they are only counted. + digestPatternsPerHour = 200 + // digestOverflowPattern is the row that counts them. + digestOverflowPattern = "(more distinct messages this hour)" + // digestTextMax bounds the stored message and attribute text. + digestTextMax = 300 + // digestHeldMax is how long rows wait in memory for TakeDigest. + digestHeldMax = 24 * time.Hour +) + // LogRing is an in-memory ring of recent log entries, split into a // global stream and one stream per driver name. Thread-safe. +// +// Warnings and errors also go to a digest, one row per pattern and hour, +// which Core writes to the store (TakeDigest). Lines logged every control +// tick can push them out of the rings within an hour; the digest keeps a +// week of them. type LogRing struct { mu sync.RWMutex global []LogEntry gNext int gFilled bool drivers map[string]*driverRing + + digest map[digestKey]*digestRow + digestHours map[time.Time]int // distinct patterns per hour +} + +// DigestEntry is the count of one warning or error pattern in one hour +// since the previous TakeDigest. +type DigestEntry struct { + Hour time.Time + Level string + Driver string + Pattern string + Msg string + Attrs string + Count int + First time.Time + Last time.Time +} + +type digestKey struct { + hour time.Time + level, driver, pattern string +} + +type digestRow struct { + DigestEntry + sent int // Count already handed out } type driverRing struct { @@ -50,9 +96,135 @@ type driverRing struct { // NewLogRing constructs an empty ring. func NewLogRing() *LogRing { return &LogRing{ - global: make([]LogEntry, ringGlobal), - drivers: make(map[string]*driverRing), + global: make([]LogEntry, ringGlobal), + drivers: make(map[string]*driverRing), + digest: make(map[digestKey]*digestRow), + digestHours: make(map[time.Time]int), + } +} + +// addDigest counts a warning or error. Caller holds r.mu. +func (r *LogRing) addDigest(e LogEntry) { + level := strings.ToUpper(e.Level) + if level == "WARNING" { + level = "WARN" + } + if level != "WARN" && level != "ERROR" { + return + } + ts := e.TS + if ts.IsZero() { + ts = time.Now() + } + ts = ts.UTC() + hour := ts.Truncate(time.Hour) + k := digestKey{hour: hour, level: level, driver: e.Driver, pattern: clip(logPattern(e.Msg))} + row, ok := r.digest[k] + if !ok && r.digestHours[hour] >= digestPatternsPerHour { + k.driver, k.pattern = "", digestOverflowPattern + row, ok = r.digest[k] + } + if _, seen := r.digestHours[hour]; !seen { + // Nothing has taken the digest for a day: keep memory bounded. + for old := range r.digest { + if old.hour.Before(hour.Add(-digestHeldMax)) { + delete(r.digest, old) + delete(r.digestHours, old.hour) + } + } + r.digestHours[hour] = 0 + } + if !ok { + row = &digestRow{DigestEntry: DigestEntry{Hour: hour, Level: level, Driver: k.driver, Pattern: k.pattern, First: ts}} + r.digest[k] = row + if k.pattern != digestOverflowPattern { + r.digestHours[hour]++ + } + } + row.Count++ + if ts.Before(row.First) { + row.First = ts + } + if !ts.Before(row.Last) { + row.Last = ts + if k.pattern == digestOverflowPattern { + row.Msg = "warnings and errors past the hourly limit of distinct messages" + } else { + row.Msg, row.Attrs = clip(e.Msg), clip(e.Attrs) + } + } +} + +// TakeDigest returns the counts added since the last call, oldest hour +// first, and forgets every hour before now's once its counts are out. The +// store adds what it is given, so a row appears again only with new counts. +func (r *LogRing) TakeDigest(now time.Time) []DigestEntry { + r.mu.Lock() + defer r.mu.Unlock() + current := now.UTC().Truncate(time.Hour) + var out []DigestEntry + for k, row := range r.digest { + if delta := row.Count - row.sent; delta > 0 { + e := row.DigestEntry + e.Count = delta + out = append(out, e) + row.sent = row.Count + } + if k.hour.Before(current) { + delete(r.digest, k) + } + } + for h := range r.digestHours { + if h.Before(current) { + delete(r.digestHours, h) + } + } + sort.Slice(out, func(i, j int) bool { + if !out[i].Hour.Equal(out[j].Hour) { + return out[i].Hour.Before(out[j].Hour) + } + return out[i].First.Before(out[j].First) + }) + return out +} + +// logPattern folds every number in a message to "#", so the same warning +// with different figures counts as one. +func logPattern(msg string) string { + var b strings.Builder + b.Grow(len(msg)) + for i := 0; i < len(msg); { + if !isDigit(msg[i]) { + b.WriteByte(msg[i]) + i++ + continue + } + // One number: digits, joined by single . : or , between digits. + j := i + 1 + for j < len(msg) { + if isDigit(msg[j]) { + j++ + continue + } + if (msg[j] == '.' || msg[j] == ':' || msg[j] == ',') && j+1 < len(msg) && isDigit(msg[j+1]) { + j += 2 + continue + } + break + } + b.WriteByte('#') + i = j + } + return b.String() +} + +func isDigit(c byte) bool { return c >= '0' && c <= '9' } + +func clip(s string) string { + if len(s) <= digestTextMax { + return s } + return s[:digestTextMax] } // Append records one entry. If e.Driver is non-empty it lands in BOTH @@ -61,6 +233,7 @@ func NewLogRing() *LogRing { func (r *LogRing) Append(e LogEntry) { r.mu.Lock() defer r.mu.Unlock() + r.addDigest(e) r.global[r.gNext] = e r.gNext++ if r.gNext == len(r.global) { diff --git a/go/internal/telemetry/logring_test.go b/go/internal/telemetry/logring_test.go new file mode 100644 index 000000000..fb0df6676 --- /dev/null +++ b/go/internal/telemetry/logring_test.go @@ -0,0 +1,95 @@ +package telemetry + +import ( + "fmt" + "testing" + "time" +) + +// A driver that logs every control tick at INFO filled the 2000-line ring +// in 75 minutes on a real site. Warnings and errors must not depend on that +// ring: they go to a digest that keeps one row per message and hour. +func TestLogRingDigestKeepsWarningsPastTickChatter(t *testing.T) { + r := NewLogRing() + hour := time.Date(2026, 11, 15, 2, 0, 0, 0, time.UTC) + r.Append(LogEntry{TS: hour.Add(time.Minute), Level: "WARN", Driver: "pixii", Msg: "write failed reg=40", Attrs: "err=timeout"}) + for i := 0; i < 5000; i++ { + r.Append(LogEntry{TS: hour.Add(2 * time.Minute), Level: "INFO", Driver: "pixii", Msg: "Pixii: setpoint ems_w=" + fmt.Sprint(i)}) + } + r.Append(LogEntry{TS: hour.Add(3 * time.Minute), Level: "WARN", Driver: "pixii", Msg: "write failed reg=41", Attrs: "err=reset"}) + r.Append(LogEntry{TS: hour.Add(4 * time.Minute), Level: "ERROR", Msg: "optimizer: deadline exceeded"}) + r.Append(LogEntry{TS: hour.Add(5 * time.Minute), Level: "DEBUG", Msg: "noise"}) + + rows := r.TakeDigest(hour.Add(10 * time.Minute)) + if len(rows) != 2 { + t.Fatalf("digest has %d rows, want the warning pattern and the error: %+v", len(rows), rows) + } + w := rows[0] + if w.Level != "WARN" || w.Driver != "pixii" || w.Pattern != "write failed reg=#" || w.Count != 2 { + t.Errorf("warning row = %+v, want one pattern seen twice", w) + } + if w.Msg != "write failed reg=41" || w.Attrs != "err=reset" { + t.Errorf("warning row keeps %q %q, want the latest text", w.Msg, w.Attrs) + } + if !w.Hour.Equal(hour) || !w.First.Equal(hour.Add(time.Minute)) || !w.Last.Equal(hour.Add(3*time.Minute)) { + t.Errorf("warning row times = %v %v %v", w.Hour, w.First, w.Last) + } + if rows[1].Level != "ERROR" || rows[1].Count != 1 { + t.Errorf("error row = %+v", rows[1]) + } + + // A second take sends only what arrived since, so the store can add. + if again := r.TakeDigest(hour.Add(11 * time.Minute)); len(again) != 0 { + t.Fatalf("nothing new, got %+v", again) + } + r.Append(LogEntry{TS: hour.Add(12 * time.Minute), Level: "WARN", Driver: "pixii", Msg: "write failed reg=7"}) + delta := r.TakeDigest(hour.Add(13 * time.Minute)) + if len(delta) != 1 || delta[0].Count != 1 || delta[0].Pattern != "write failed reg=#" { + t.Fatalf("delta = %+v, want one more of the same pattern", delta) + } + + // Once the hour has been sent and has passed, it is dropped from memory. + r.TakeDigest(hour.Add(2 * time.Hour)) + r.mu.RLock() + left := len(r.digest) + r.mu.RUnlock() + if left != 0 { + t.Errorf("%d digest rows still held after their hour was sent", left) + } +} + +// Messages that differ in more than numbers would each be a new row. Past +// the hourly cap they are counted in one overflow row instead, so a driver +// that puts a timestamp in every warning cannot grow the store. +func TestLogRingDigestCapsDistinctMessagesPerHour(t *testing.T) { + r := NewLogRing() + hour := time.Date(2026, 11, 15, 3, 0, 0, 0, time.UTC) + for i := 0; i < digestPatternsPerHour+50; i++ { + r.Append(LogEntry{TS: hour.Add(time.Second), Level: "WARN", Msg: "odd " + string(rune('a'+i%26)) + string(rune('a'+i/26))}) + } + rows := r.TakeDigest(hour.Add(time.Minute)) + if len(rows) != digestPatternsPerHour+1 { + t.Fatalf("got %d rows, want %d patterns plus one overflow row", len(rows), digestPatternsPerHour+1) + } + var overflow *DigestEntry + for i := range rows { + if rows[i].Pattern == digestOverflowPattern { + overflow = &rows[i] + } + } + if overflow == nil || overflow.Count != 50 { + t.Fatalf("overflow row = %+v, want 50 messages", overflow) + } +} + +func TestLogPatternFoldsNumbers(t *testing.T) { + for in, want := range map[string]string{ + "write failed reg=40": "write failed reg=#", + "read 12 of 3.5 kW at 02:13:59": "read # of # kW at #", + "no numbers": "no numbers", + } { + if got := logPattern(in); got != want { + t.Errorf("logPattern(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/web/diagnostics-modal.js b/web/diagnostics-modal.js index f5162984d..e49089df6 100644 --- a/web/diagnostics-modal.js +++ b/web/diagnostics-modal.js @@ -497,7 +497,7 @@ downloadWithFeedback( "/api/support/dump", "Preparing support bundle", - "Collecting logs, redacted config, driver health, and recent telemetry.", + "Collecting logs, redacted config, driver health, and the last seven days of telemetry, plans and control.", "ftw-support.zip" ); }); diff --git a/web/plan.js b/web/plan.js index 6cdfee3e1..234e1f3c9 100644 --- a/web/plan.js +++ b/web/plan.js @@ -1541,8 +1541,9 @@ import { } // Pulls GET /api/support/dump and saves it. That archive leads with - // 00-help-report.md — the readable answer to "why is it doing that?" — - // and carries the logs, config and telemetry behind it. One button and + // 00-help-report.md — the readable answer to "why is it doing that?" and + // "what happened last night?" — and carries the logs, config and a week + // of telemetry, plans and control journal behind it. One button and // one file, because a user asking for help should not have to work out // which of two downloads we wanted. The bare report is still at // /api/support/report for anyone who wants only the text.