From 1e32d9d1a8d4e5dc315591a268efe1c89b796941 Mon Sep 17 00:00:00 2001 From: hannesb90 Date: Tue, 10 Nov 2026 10:08:12 +0100 Subject: [PATCH 1/2] feat(thermaltwin): learn each declared room's thermal twin in shadow Step 3 of ADR 0012 (#1994). For every room sensor a driver declares, FTW fits dT/dt = a(Tout - Tin) + b*P + c by recursive least squares over 30-minute windows: inertia, capacity, heat loss and other gains. The twin is judged by simulating each three-hour stretch from the parameters it had at the start with the inputs that really happened, and reports learning, trusted, drifting or no_heat_input. It reads LatestMetric and the load model's outdoor temperature, persists through modelstate, and is shown at GET /api/thermal/twins with shadow: true. It commands nothing and nothing plans from it. Refs #520 Co-Authored-By: Claude Opus 5.5 Signed-off-by: hannesb90 --- .changeset/thermal-twin-shadow.md | 5 + go/cmd/ftw/main.go | 25 ++ go/cmd/ftw/thermal_twin.go | 51 ++++ go/cmd/ftw/thermal_twin_test.go | 53 ++++ go/internal/api/api.go | 6 +- go/internal/api/api_agent_catalog.go | 1 + go/internal/api/api_thermal_twin.go | 23 ++ go/internal/api/api_thermal_twin_test.go | 55 ++++ go/internal/thermaltwin/model.go | 337 ++++++++++++++++++++++ go/internal/thermaltwin/model_test.go | 217 ++++++++++++++ go/internal/thermaltwin/service.go | 344 +++++++++++++++++++++++ go/internal/thermaltwin/service_test.go | 186 ++++++++++++ 12 files changed, 1302 insertions(+), 1 deletion(-) create mode 100644 .changeset/thermal-twin-shadow.md create mode 100644 go/cmd/ftw/thermal_twin.go create mode 100644 go/cmd/ftw/thermal_twin_test.go create mode 100644 go/internal/api/api_thermal_twin.go create mode 100644 go/internal/api/api_thermal_twin_test.go create mode 100644 go/internal/thermaltwin/model.go create mode 100644 go/internal/thermaltwin/model_test.go create mode 100644 go/internal/thermaltwin/service.go create mode 100644 go/internal/thermaltwin/service_test.go diff --git a/.changeset/thermal-twin-shadow.md b/.changeset/thermal-twin-shadow.md new file mode 100644 index 000000000..10ef87023 --- /dev/null +++ b/.changeset/thermal-twin-shadow.md @@ -0,0 +1,5 @@ +--- +"ftw": patch +--- + +FTW now learns a thermal twin, in shadow, for each room sensor a driver declares (`sensors` with `asset = "space_heat"`). Every 30 minutes it fits the room's inertia, heat capacity, heat loss and other gains from the room temperature, the outdoor temperature and the heat pump's `hp_power_w`, and checks itself by predicting three hours ahead against what the sensor then measured. `GET /api/thermal/twins` shows each twin as learning, trusted or drifting, or says it has no heat input to learn from. The twin commands nothing, and nothing plans from it yet. diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index aef6324fb..32c108e6d 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -75,6 +75,7 @@ import ( "github.com/srcfl/ftw/go/internal/selfupdate" "github.com/srcfl/ftw/go/internal/state" "github.com/srcfl/ftw/go/internal/telemetry" + "github.com/srcfl/ftw/go/internal/thermaltwin" "github.com/srcfl/ftw/go/protocol/cloudwire" ) @@ -1468,6 +1469,29 @@ func main() { defer loadSvc.Stop() slog.Info("loadmodel started", "peak_w", loadPeakW, "quality", loadSvc.Model().Quality()) + // ---- Thermal twins, shadow only (docs/adr/0012) ---- + // Learns each declared room sensor's inertia and heat response from the + // same outdoor temperature as the load model. It reads and stores, and + // nothing plans or dispatches from it. + twinDrivers := func() []config.Driver { + cfgMu.RLock() + defer cfgMu.RUnlock() + return append([]config.Driver(nil), cfg.Drivers...) + } + thermalTwins := thermaltwin.NewService(st, tel) + thermalTwins.Temp = loadSvc.Temp + thermalTwins.Zones = func() []thermaltwin.Zone { + catalog, err := drivers.LoadCatalogSources(drivers.CatalogSource{Dir: *userDriversDirFlag, Source: "local"}, drivers.CatalogSource{Dir: config.ManagedDriversDirOverride, Source: "managed"}, drivers.CatalogSource{Dir: resolveDriverDir(), Source: "bundled"}) + if err != nil { + slog.Warn("thermal twin: driver catalog unavailable", "err", err) + return nil + } + return thermalTwinZones(twinDrivers(), catalog) + } + thermalTwins.PowerDrivers = func() []string { return thermalTwinPowerDrivers(twinDrivers()) } + thermalTwins.Start(ctx) + defer thermalTwins.Stop() + // ---- Start OCPP 1.6J Central System (optional) ---- // Chargers dial us, so there is nothing to add to cfg.Drivers and no Lua // driver involved. A charge point whose identity (the last segment of the @@ -2598,6 +2622,7 @@ func main() { PlannerPrefs: plannerPrefs, PVModel: pvSvc, LoadModel: loadSvc, + ThermalTwin: thermalTwins, ForecastLearning: forecastTrackerSvc, Loadpoints: lpMgr, Cars: carsRT, diff --git a/go/cmd/ftw/thermal_twin.go b/go/cmd/ftw/thermal_twin.go new file mode 100644 index 000000000..cb0254f0c --- /dev/null +++ b/go/cmd/ftw/thermal_twin.go @@ -0,0 +1,51 @@ +package main + +import ( + "sort" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/drivers" + "github.com/srcfl/ftw/go/internal/thermal" + "github.com/srcfl/ftw/go/internal/thermaltwin" +) + +// thermalTwinZones lists one shadow twin zone per room sensor an enabled +// driver declares. Only space heat for now: a tank or buffer has the same +// form but loses heat to the room it stands in, not to outdoors, and the +// service has no such temperature yet. +func thermalTwinZones(cfgDrivers []config.Driver, catalog []drivers.CatalogEntry) []thermaltwin.Zone { + var out []thermaltwin.Zone + for _, d := range cfgDrivers { + if d.Disabled || d.Lua == "" { + continue + } + entry := drivers.EntryForDriver(catalog, d.Lua) + if entry == nil { + continue + } + for _, s := range entry.Sensors { + if s.Asset != thermal.AssetSpaceHeat { + continue + } + out = append(out, thermaltwin.Zone{ + Key: d.Name + "/" + s.Metric, + SensorDriver: d.Name, + SensorMetric: s.Metric, + }) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) + return out +} + +// thermalTwinPowerDrivers lists the enabled drivers that may report heat +// pump power. +func thermalTwinPowerDrivers(cfgDrivers []config.Driver) []string { + var out []string + for _, d := range cfgDrivers { + if !d.Disabled { + out = append(out, d.Name) + } + } + return out +} diff --git a/go/cmd/ftw/thermal_twin_test.go b/go/cmd/ftw/thermal_twin_test.go new file mode 100644 index 000000000..a0c351d0c --- /dev/null +++ b/go/cmd/ftw/thermal_twin_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/drivers" + "github.com/srcfl/ftw/go/internal/thermaltwin" +) + +func TestThermalTwinZonesFromDeclaredSensors(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + write("thermo.lua", `DRIVER = { + id = "thermo", + sensors = { + { metric = "room_living_temp_c", asset = "space_heat", zone = "living" }, + { metric = "room_bed_temp_c", asset = "space_heat" }, + { metric = "tank_temp_c", asset = "dhw" }, + }, +} +`) + write("meter.lua", `DRIVER = { id = "meter" } +`) + catalog, err := drivers.LoadCatalog(dir) + if err != nil { + t.Fatal(err) + } + cfg := []config.Driver{ + {Name: "wiser", Lua: filepath.Join(dir, "thermo.lua")}, + {Name: "spare", Lua: filepath.Join(dir, "thermo.lua"), Disabled: true}, + {Name: "grid", Lua: filepath.Join(dir, "meter.lua")}, + } + + got := thermalTwinZones(cfg, catalog) + want := []thermaltwin.Zone{ + {Key: "wiser/room_bed_temp_c", SensorDriver: "wiser", SensorMetric: "room_bed_temp_c"}, + {Key: "wiser/room_living_temp_c", SensorDriver: "wiser", SensorMetric: "room_living_temp_c"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("zones = %+v, want %+v (no tank, no disabled driver)", got, want) + } + if p := thermalTwinPowerDrivers(cfg); !reflect.DeepEqual(p, []string{"wiser", "grid"}) { + t.Errorf("power drivers = %v", p) + } +} diff --git a/go/internal/api/api.go b/go/internal/api/api.go index d6de34afb..eac762af2 100644 --- a/go/internal/api/api.go +++ b/go/internal/api/api.go @@ -61,6 +61,7 @@ import ( "github.com/srcfl/ftw/go/internal/selfupdate" "github.com/srcfl/ftw/go/internal/state" "github.com/srcfl/ftw/go/internal/telemetry" + "github.com/srcfl/ftw/go/internal/thermaltwin" "github.com/srcfl/ftw/go/internal/units" v2xpolicy "github.com/srcfl/ftw/go/internal/v2x" "github.com/srcfl/ftw/go/protocol" @@ -174,7 +175,9 @@ type Deps struct { PVModel *pvmodel.Service // Optional: load digital-twin self-learner. - LoadModel *loadmodel.Service + LoadModel *loadmodel.Service + // Optional: shadow thermal twins of declared room sensors. + ThermalTwin *thermaltwin.Service ForecastLearning ForecastLearning // Optional: EV loadpoint state consumed by the API and MPC. @@ -544,6 +547,7 @@ func (s *Server) routes() { s.handle("GET /api/pvmodel", Read, s.handlePVModel) s.handle("POST /api/pvmodel/reset", Configure, s.handlePVModelReset) s.handle("GET /api/loadmodel", Read, s.handleLoadModel) + s.handle("GET /api/thermal/twins", Read, s.handleThermalTwins) s.handle("POST /api/loadmodel/profile", Configure, s.handleLoadModelProfile) s.handle("POST /api/loadmodel/reset", Configure, s.handleLoadModelReset) s.handle("GET /api/research/load/dump", Read, s.handleLoadResearchDump) diff --git a/go/internal/api/api_agent_catalog.go b/go/internal/api/api_agent_catalog.go index 4c2463074..42a75d3ed 100644 --- a/go/internal/api/api_agent_catalog.go +++ b/go/internal/api/api_agent_catalog.go @@ -25,6 +25,7 @@ var agentAPIHints = []protocol.AgentAPIRoute{ {Path: "/api/data-sources", Description: "Available data sources and freshness"}, {Path: "/api/battery_models", Description: "Learned battery model and efficiency"}, {Path: "/api/loadmodel", Description: "Load forecast model and learning"}, + {Path: "/api/thermal/twins", Description: "Learned thermal twins of room sensors; shadow only, nothing controls from them"}, {Path: "/api/pvmodel", Description: "Solar forecast model and learning"}, {Path: "/api/forecast", Description: "Load and solar forecasts", Query: map[string]string{"range": "Time ahead, such as 24h"}}, {Path: "/api/prices", Description: "Stored import and export prices", Query: map[string]string{"range": "Time ahead, such as 24h", "since_ms": "Unix milliseconds", "until_ms": "Unix milliseconds"}}, diff --git a/go/internal/api/api_thermal_twin.go b/go/internal/api/api_thermal_twin.go new file mode 100644 index 000000000..b524bc2fd --- /dev/null +++ b/go/internal/api/api_thermal_twin.go @@ -0,0 +1,23 @@ +package api + +import ( + "net/http" + + "github.com/srcfl/ftw/go/internal/thermaltwin" +) + +// GET /api/thermal/twins — each declared room sensor's learned twin, in +// shadow. `shadow: true` is part of the answer, not decoration: nothing in +// Core plans or commands from these, and a reader must not infer that a +// trusted twin is steering anything. +func (s *Server) handleThermalTwins(w http.ResponseWriter, r *http.Request) { + twins := []thermaltwin.ZoneStatus{} + if s.deps.ThermalTwin != nil { + twins = s.deps.ThermalTwin.Snapshot() + } + writeJSON(w, http.StatusOK, map[string]any{ + "shadow": true, + "enabled": s.deps.ThermalTwin != nil, + "twins": twins, + }) +} diff --git a/go/internal/api/api_thermal_twin_test.go b/go/internal/api/api_thermal_twin_test.go new file mode 100644 index 000000000..11fdafd69 --- /dev/null +++ b/go/internal/api/api_thermal_twin_test.go @@ -0,0 +1,55 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/thermaltwin" +) + +type twinTele struct{} + +func (twinTele) LatestMetric(string, string) (float64, time.Time, bool) { return 0, time.Time{}, false } + +func getThermalTwins(t *testing.T, srv *Server) map[string]json.RawMessage { + t.Helper() + rec := httptest.NewRecorder() + srv.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/thermal/twins", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("GET /api/thermal/twins = %d, body %s", rec.Code, rec.Body.String()) + } + var got map[string]json.RawMessage + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + return got +} + +// Every answer says shadow, whether or not twins exist, so no reader can +// take a trusted twin for one that steers anything. +func TestThermalTwinsAreAlwaysMarkedShadow(t *testing.T) { + got := getThermalTwins(t, New(&Deps{})) + if string(got["shadow"]) != "true" || string(got["enabled"]) != "false" || string(got["twins"]) != "[]" { + t.Errorf("without a service = %v", got) + } + + svc := thermaltwin.NewService(nil, twinTele{}) + svc.Zones = func() []thermaltwin.Zone { + return []thermaltwin.Zone{{Key: "wiser/room_temp_c", SensorDriver: "wiser", SensorMetric: "room_temp_c"}} + } + svc.Tick(time.Now()) + got = getThermalTwins(t, New(&Deps{ThermalTwin: svc})) + if string(got["shadow"]) != "true" || string(got["enabled"]) != "true" { + t.Errorf("with a service = %v", got) + } + var twins []thermaltwin.ZoneStatus + if err := json.Unmarshal(got["twins"], &twins); err != nil { + t.Fatal(err) + } + if len(twins) != 1 || twins[0].Key != "wiser/room_temp_c" || twins[0].Status != thermaltwin.StatusNoHeatInput { + t.Errorf("twins = %+v", twins) + } +} diff --git a/go/internal/thermaltwin/model.go b/go/internal/thermaltwin/model.go new file mode 100644 index 000000000..b701aa277 --- /dev/null +++ b/go/internal/thermaltwin/model.go @@ -0,0 +1,337 @@ +// Package thermaltwin learns how a heated zone behaves, in shadow. +// +// One first-order model per zone, proposed in docs/adr/0012: +// +// dT_in/dt = a·(T_out − T_in) + b·P + c +// +// a is 1/τ, the zone's inertia; b is how many kelvin one kWh of heat input +// moves it; c is what neither explains (sun, people, other heat). The heat +// loss UA follows as a/b. Learned online by recursive least squares over +// fixed windows. +// +// The twin only predicts. Nothing in Core plans or dispatches from it. It is +// judged the only way a model may be: against measured temperature, by +// simulating each three-hour stretch from the parameters it had at the +// start, fed the inputs that really happened. A prediction never verifies +// itself. +package thermaltwin + +import ( + "math" + "time" + + "github.com/srcfl/ftw/go/internal/modelstate" +) + +const ( + // Window is the length of one learning sample. Room sensors report in + // 0.1 K steps and a zone moves a few tenths of a kelvin an hour, so a + // shorter window would learn mostly quantisation. + Window = 30 * time.Minute + // EvalHorizonWindows is how far ahead a twin is judged: three hours, + // the scale of a price peak the planner would heat ahead of. + EvalHorizonWindows = 6 + + windowH = 0.5 + + // Forgetting factor. A half-life of about 350 windows, a week, follows + // the season without losing a cold snap within days. + lambda = 0.998 +) + +// Proposed thresholds, ADR 0012. Not used for control by anything yet. +const ( + minSamplesForTrust = 7 * 48 // a week of windows + minEvaluationsForTrust = 48 // a day of three-hour judgements + trustMAEK = 0.3 + driftMAEK = 0.5 + maeMemory = 24 // evaluations, one per window +) + +// Status is what a twin can honestly say about itself. +type Status string + +const ( + StatusNoHeatInput Status = "no_heat_input" + StatusLearning Status = "learning" + StatusTrusted Status = "trusted" + StatusDrifting Status = "drifting" +) + +// Sample is one closed window, as the service measured it. +type Sample struct { + TInStartC float64 // zone temperature at the window's start + TInEndC float64 // zone temperature at the window's end + TOutC float64 // outdoor temperature over the window + PowerKW float64 // mean heat input over the window; 0 without a source + // DurationH is the time between the two temperatures. Zero means a + // whole window. + DurationH float64 + // Contiguous is false when the previous window was not observed. A gap + // breaks the three-hour simulations that span it. + Contiguous bool +} + +func (s Sample) durationH() float64 { + if s.DurationH > 0 { + return s.DurationH + } + return windowH +} + +type pending struct { + TInStartC float64 `json:"t_in_start_c"` + TInEndC float64 `json:"t_in_end_c"` + TOutC float64 `json:"t_out_c"` + PowerKW float64 `json:"power_kw"` + DurationH float64 `json:"duration_h"` + Theta [3]float64 `json:"theta"` +} + +// Twin is one zone's learned model. Its JSON is the persisted state. +type Twin struct { + Theta [3]float64 `json:"theta"` + P [3][3]float64 `json:"p"` + + Samples int64 `json:"samples"` + PowerSamples int64 `json:"power_samples"` + Rejected int64 `json:"rejected"` + + MAE3hK float64 `json:"mae_3h_k"` + Evaluations int64 `json:"evaluations"` + WasTrusted bool `json:"was_trusted"` + + // PowerSource names where heat input came from. A twin fitted to one + // source means nothing against another, so a change starts it over. + PowerSource string `json:"power_source,omitempty"` + + Pending []pending `json:"pending,omitempty"` +} + +// New returns a twin at its prior: τ = 50 h, 20 kWh/K, no unexplained gain. +// The covariance is wide, so a week of data outweighs the prior. +func New() *Twin { + t := &Twin{Theta: [3]float64{1.0 / 50, 1.0 / 20, 0}} + t.P[0][0], t.P[1][1], t.P[2][2] = 0.01, 0.01, 0.25 + return t +} + +// Observe learns from one window. It returns false, and learns nothing, for +// a window no physical zone produces: a sensor glitch, an open window, a +// missing input. +func (t *Twin) Observe(s Sample) bool { + if !plausible(s) { + t.Rejected++ + t.Pending = nil + return false + } + if !s.Contiguous { + t.Pending = nil + } + t.Pending = append(t.Pending, pending{ + TInStartC: s.TInStartC, TInEndC: s.TInEndC, TOutC: s.TOutC, + PowerKW: s.PowerKW, DurationH: s.durationH(), Theta: t.Theta, + }) + if len(t.Pending) >= EvalHorizonWindows { + t.evaluate() + t.Pending = t.Pending[1:] + } + + x := regressor(s.TInStartC, s.TOutC, s.PowerKW) + y := (s.TInEndC - s.TInStartC) / s.durationH() + t.update(x, y) + t.Samples++ + if s.PowerKW > 0 { + t.PowerSamples++ + } + // Records a first trust here, not only when someone reads the status: + // a twin nobody looked at must still be able to drift. + _ = t.Status() + return true +} + +func plausible(s Sample) bool { + for _, v := range []float64{s.TInStartC, s.TInEndC, s.TOutC, s.PowerKW} { + if math.IsNaN(v) || math.IsInf(v, 0) { + return false + } + } + switch { + case s.TInStartC < -10 || s.TInStartC > 45 || s.TInEndC < -10 || s.TInEndC > 45: + return false + case s.TOutC < -50 || s.TOutC > 50: + return false + case s.PowerKW < 0 || s.PowerKW > 50: + return false + case s.DurationH < 0 || s.DurationH > 2*windowH: + return false + case math.Abs(s.TInEndC-s.TInStartC)/s.durationH() > 3: + // Faster than any heated building: a window, a sensor in the sun, a + // sensor moved. + return false + } + return true +} + +func regressor(tInC, tOutC, powerKW float64) [3]float64 { + return [3]float64{tOutC - tInC, powerKW, 1} +} + +// update is one recursive-least-squares step with forgetting. +func (t *Twin) update(x [3]float64, y float64) { + var px [3]float64 + for i := 0; i < 3; i++ { + for j := 0; j < 3; j++ { + px[i] += t.P[i][j] * x[j] + } + } + denom := lambda + for i := 0; i < 3; i++ { + denom += x[i] * px[i] + } + if denom <= 0 || math.IsNaN(denom) || math.IsInf(denom, 0) { + return + } + var k [3]float64 + for i := 0; i < 3; i++ { + k[i] = px[i] / denom + } + errY := y - dot(t.Theta, x) + next := t.Theta + for i := 0; i < 3; i++ { + next[i] += k[i] * errY + } + var nextP [3][3]float64 + for i := 0; i < 3; i++ { + for j := 0; j < 3; j++ { + nextP[i][j] = (t.P[i][j] - k[i]*px[j]) / lambda + } + } + // Keep P symmetric and bounded; an unbounded covariance is how RLS with + // forgetting blows up during a long stretch of identical inputs. + for i := 0; i < 3; i++ { + for j := i + 1; j < 3; j++ { + m := (nextP[i][j] + nextP[j][i]) / 2 + nextP[i][j], nextP[j][i] = m, m + } + if nextP[i][i] > 1e3 { + return + } + } + if !finiteVec(next) { + return + } + t.Theta, t.P = next, nextP +} + +// evaluate simulates the oldest pending window forward to the newest one's +// end, from the parameters the twin had when the stretch began. +func (t *Twin) evaluate() { + first := t.Pending[0] + temp := first.TInStartC + for i, w := range t.Pending { + if i > 0 { + // The gap between one window's end and the next one's start + // is the zone drifting with the inputs of the window before. + prev := t.Pending[i-1] + if gapH := windowH - prev.DurationH; gapH > 0 { + temp += gapH * dot(first.Theta, regressor(temp, prev.TOutC, prev.PowerKW)) + } + } + temp += w.DurationH * dot(first.Theta, regressor(temp, w.TOutC, w.PowerKW)) + } + errK := math.Abs(temp - t.Pending[len(t.Pending)-1].TInEndC) + if math.IsNaN(errK) || math.IsInf(errK, 0) { + return + } + t.Evaluations++ + // A running mean at first, then about half a day of memory: long + // enough that one odd evening does not cost trust, short enough that a + // house that changed shows as drifting the same day. + n := float64(t.Evaluations) + if n > maeMemory { + n = maeMemory + } + t.MAE3hK += (errK - t.MAE3hK) / n +} + +// Params is the twin read as physics. +type Params struct { + TauH *float64 `json:"tau_h,omitempty"` + CapacityKWhPerK *float64 `json:"capacity_kwh_per_k,omitempty"` + UAWPerK *float64 `json:"ua_w_per_k,omitempty"` + OtherGainKPerH float64 `json:"other_gain_k_per_h"` +} + +// Params derives physical quantities. A non-positive coefficient has no +// physical reading and is left out rather than shown as nonsense. +func (t *Twin) Params() Params { + a, b, c := t.Theta[0], t.Theta[1], t.Theta[2] + p := Params{OtherGainKPerH: c} + if a > 0 { + tau := 1 / a + p.TauH = &tau + } + if b > 0 && t.PowerSamples > 0 { + cap := 1 / b + p.CapacityKWhPerK = &cap + if a > 0 { + ua := 1000 * a / b + p.UAWPerK = &ua + } + } + return p +} + +func (t *Twin) physical() bool { + p := t.Params() + return p.TauH != nil && *p.TauH >= 1 && *p.TauH <= 2000 && + p.CapacityKWhPerK != nil && *p.CapacityKWhPerK >= 1 && *p.CapacityKWhPerK <= 500 +} + +// Status reports the twin's standing and records a first trust, so later +// error above the drift limit reads as drifting rather than as learning. +func (t *Twin) Status() Status { + if t.PowerSamples == 0 { + return StatusNoHeatInput + } + if t.Samples < minSamplesForTrust || t.Evaluations < minEvaluationsForTrust || !t.physical() { + if t.WasTrusted && t.Evaluations >= minEvaluationsForTrust { + return StatusDrifting + } + return StatusLearning + } + switch { + case t.MAE3hK <= trustMAEK: + t.WasTrusted = true + return StatusTrusted + case t.WasTrusted && t.MAE3hK <= driftMAEK: + return StatusTrusted + case t.WasTrusted: + return StatusDrifting + } + return StatusLearning +} + +// FeatureHash fingerprints the model's form, so stored state from a +// different form cold starts instead of restoring meaningless coefficients. +func FeatureHash() string { + var probe []float64 + for _, v := range [][3]float64{{20, -5, 2}, {21.5, 10, 0}, {18, 0, 4.5}} { + x := regressor(v[0], v[1], v[2]) + probe = append(probe, x[:]...) + } + probe = append(probe, windowH, lambda, EvalHorizonWindows) + return modelstate.Fingerprint("thermaltwin: dT/dt K/h = a(Tout-Tin) + b*P_kW + c; heat input hp_power_w", probe) +} + +func dot(a, b [3]float64) float64 { return a[0]*b[0] + a[1]*b[1] + a[2]*b[2] } + +func finiteVec(v [3]float64) bool { + for _, x := range v { + if math.IsNaN(x) || math.IsInf(x, 0) { + return false + } + } + return true +} diff --git a/go/internal/thermaltwin/model_test.go b/go/internal/thermaltwin/model_test.go new file mode 100644 index 000000000..a3ce97434 --- /dev/null +++ b/go/internal/thermaltwin/model_test.go @@ -0,0 +1,217 @@ +package thermaltwin + +import ( + "math" + "testing" +) + +// house is a zone with known physics, integrated finely so the twin is +// judged against a continuous system rather than its own discretisation. +type house struct { + a, b, c float64 // 1/h, K/kWh, K/h + tIn float64 +} + +func (h *house) step(tOut, powerKW, hours float64) { + const sub = 30 + dt := hours / sub + for i := 0; i < sub; i++ { + h.tIn += dt * (h.a*(tOut-h.tIn) + h.b*powerKW + h.c) + } +} + +// drive runs the house for n windows with a daily outdoor swing and a heat +// pump that cycles, and feeds every window to the twin. +func drive(tw *Twin, h *house, n int, quantize bool) { + q := func(v float64) float64 { + if !quantize { + return v + } + return math.Round(v*10) / 10 + } + for i := 0; i < n; i++ { + hour := float64(i) * windowH + tOut := -2 + 6*math.Sin(2*math.Pi*hour/24) + power := 0.0 + // On for 2 h, off for 1 h, with a stronger burst every evening: + // the variation a twin needs to separate inertia from input. + if int(hour)%3 != 2 { + power = 1.6 + } + if int(hour)%24 >= 17 && int(hour)%24 < 19 { + power = 3.0 + } + start := q(h.tIn) + h.step(tOut, power, windowH) + tw.Observe(Sample{TInStartC: start, TInEndC: q(h.tIn), TOutC: tOut, PowerKW: power, Contiguous: true}) + } +} + +func TestTwinLearnsKnownPhysics(t *testing.T) { + h := &house{a: 1.0 / 40, b: 1.0 / 15, c: 0.05, tIn: 20} + tw := New() + drive(tw, h, 14*48, false) + + p := tw.Params() + if p.TauH == nil || math.Abs(*p.TauH-40)/40 > 0.1 { + t.Errorf("tau = %v h, want 40 ±10%%", deref(p.TauH)) + } + if p.CapacityKWhPerK == nil || math.Abs(*p.CapacityKWhPerK-15)/15 > 0.1 { + t.Errorf("capacity = %v kWh/K, want 15 ±10%%", deref(p.CapacityKWhPerK)) + } + wantUA := 1000 * (1.0 / 40) / (1.0 / 15) + if p.UAWPerK == nil || math.Abs(*p.UAWPerK-wantUA)/wantUA > 0.15 { + t.Errorf("UA = %v W/K, want %.0f ±15%%", deref(p.UAWPerK), wantUA) + } + if tw.MAE3hK > 0.05 { + t.Errorf("3 h error = %.3f K on noise-free data", tw.MAE3hK) + } + if got := tw.Status(); got != StatusTrusted { + t.Errorf("status = %s, want trusted", got) + } +} + +// Real sensors step in 0.1 K. The twin must still predict three hours ahead +// well enough to earn trust. +func TestTwinEarnsTrustThroughSensorQuantisation(t *testing.T) { + h := &house{a: 1.0 / 40, b: 1.0 / 15, c: 0.05, tIn: 20} + tw := New() + drive(tw, h, 14*48, true) + if tw.MAE3hK > trustMAEK { + t.Errorf("3 h error = %.3f K, want ≤ %.1f", tw.MAE3hK, trustMAEK) + } + if got := tw.Status(); got != StatusTrusted { + t.Errorf("status = %s, want trusted (params %+v)", got, tw.Params()) + } +} + +// A week is the minimum. Good data for a day earns nothing yet. +func TestTwinDoesNotTrustEarly(t *testing.T) { + h := &house{a: 1.0 / 40, b: 1.0 / 15, c: 0.05, tIn: 20} + tw := New() + drive(tw, h, 48, false) + if got := tw.Status(); got != StatusLearning { + t.Errorf("status after a day = %s, want learning", got) + } +} + +func TestTwinWithoutHeatInputSaysSo(t *testing.T) { + tw := New() + for i := 0; i < 400; i++ { + tw.Observe(Sample{TInStartC: 20, TInEndC: 19.95, TOutC: 0, Contiguous: true}) + } + if got := tw.Status(); got != StatusNoHeatInput { + t.Errorf("status = %s, want no_heat_input", got) + } + if p := tw.Params(); p.CapacityKWhPerK != nil || p.UAWPerK != nil { + t.Errorf("params = %+v; capacity and UA need heat input", p) + } +} + +// A house that changes in a way the model can follow (new insulation, a +// colder season) is relearned and stays trusted: trust is about prediction, +// not about the parameters staying put. +func TestTwinFollowsAChangedHouse(t *testing.T) { + h := &house{a: 1.0 / 40, b: 1.0 / 15, c: 0.05, tIn: 20} + tw := New() + drive(tw, h, 14*48, false) + h.a = 2.0 / 40 + // A week: the forgetting half-life, chosen to follow seasons, not + // to chase a single cold night. + drive(tw, h, 7*48, false) + if got := tw.Status(); got != StatusTrusted || tw.MAE3hK > trustMAEK { + t.Errorf("status = %s with 3 h error %.2f K, want trusted within %.1f", got, tw.MAE3hK, trustMAEK) + } + // The inertia itself follows more slowly than the prediction recovers: + // for a while the unexplained gain c absorbs the difference. That is + // enough for three hours ahead and not for how long a house can coast, + // which is why a planner must not use τ from a twin whose house just + // changed. Measured here so the lag is a known number, not a surprise. + p := tw.Params() + if p.TauH == nil || *p.TauH >= 40 { + t.Fatalf("tau = %v h after a week, want moving from 40 toward 20", deref(p.TauH)) + } + drive(tw, h, 14*48, false) + if p := tw.Params(); p.TauH == nil || math.Abs(*p.TauH-20)/20 > 0.2 { + t.Errorf("tau = %v h after three weeks, want 20 ±20%%", deref(p.TauH)) + } +} + +// A trusted twin that stops predicting, here a sensor moved next to a heat +// source that comes and goes with no pattern, must say drifting the same +// day, not quietly keep its trust. +func TestTwinReportsDrift(t *testing.T) { + h := &house{a: 1.0 / 40, b: 1.0 / 15, c: 0.05, tIn: 20} + tw := New() + drive(tw, h, 14*48, false) + if tw.Status() != StatusTrusted { + t.Fatalf("precondition: twin not trusted") + } + seed := uint32(1) + gain := func() float64 { + seed = seed*1664525 + 1013904223 + return 2.5 * (float64(seed>>8)/float64(1<<24) - 0.5) // ±1.25 K/h + } + sawDrift := false + for i := 0; i < 48 && !sawDrift; i++ { + h.c = 0.05 + gain() + drive(tw, h, 1, false) + sawDrift = tw.Status() == StatusDrifting + } + if !sawDrift { + t.Errorf("status stayed %s with 3 h error %.2f K", tw.Status(), tw.MAE3hK) + } +} + +func TestTwinRejectsImplausibleWindows(t *testing.T) { + cases := map[string]Sample{ + "NaN temperature": {TInStartC: math.NaN(), TInEndC: 20, TOutC: 0}, + "jump of 5 K": {TInStartC: 20, TInEndC: 25, TOutC: 0}, + "freezing sensor": {TInStartC: -30, TInEndC: -30, TOutC: 0}, + "negative power": {TInStartC: 20, TInEndC: 20, TOutC: 0, PowerKW: -1}, + "absurd outdoor": {TInStartC: 20, TInEndC: 20, TOutC: 80}, + "window too long": {TInStartC: 20, TInEndC: 20, TOutC: 0, DurationH: 3}, + "infinite power": {TInStartC: 20, TInEndC: 20, TOutC: 0, PowerKW: math.Inf(1)}, + } + for name, s := range cases { + t.Run(name, func(t *testing.T) { + tw := New() + before := tw.Theta + if tw.Observe(s) { + t.Fatal("accepted") + } + if tw.Theta != before || tw.Samples != 0 || tw.Rejected != 1 { + t.Errorf("rejected window changed the twin: %+v", tw) + } + }) + } +} + +// A gap breaks the three-hour stretches that span it. Judging across one +// would score the twin on a simulation fed inputs that never arrived. +func TestTwinGapRestartsEvaluation(t *testing.T) { + tw := New() + for i := 0; i < EvalHorizonWindows-1; i++ { + tw.Observe(Sample{TInStartC: 20, TInEndC: 20, TOutC: 0, PowerKW: 1, Contiguous: true}) + } + tw.Observe(Sample{TInStartC: 20, TInEndC: 20, TOutC: 0, PowerKW: 1, Contiguous: false}) + if tw.Evaluations != 0 { + t.Errorf("evaluations = %d across a gap, want 0", tw.Evaluations) + } + if len(tw.Pending) != 1 { + t.Errorf("pending = %d after a gap, want 1", len(tw.Pending)) + } +} + +func TestFeatureHashIsStable(t *testing.T) { + if a, b := FeatureHash(), FeatureHash(); a != b || len(a) != 16 { + t.Errorf("hash = %q, %q", a, b) + } +} + +func deref(v *float64) any { + if v == nil { + return nil + } + return *v +} diff --git a/go/internal/thermaltwin/service.go b/go/internal/thermaltwin/service.go new file mode 100644 index 000000000..7fc04475c --- /dev/null +++ b/go/internal/thermaltwin/service.go @@ -0,0 +1,344 @@ +package thermaltwin + +import ( + "context" + "log/slog" + "sort" + "sync" + "time" + + "github.com/srcfl/ftw/go/internal/modelstate" +) + +// HeatPowerMetric is the canonical electrical input of a heat pump, the same +// metric the Heating view discovers pumps by (docs/nibe-local.md). +const HeatPowerMetric = "hp_power_w" + +const ( + tick = time.Minute + // A reading older than this is not the zone now. + maxReadingAge = 5 * time.Minute + // A window needs most of its minutes; a sensor that reported for ten of + // thirty is not a measurement of the window. + minWindowSamples = 25 + // Temperatures at each end of a window are means of this many readings, + // which is what keeps 0.1 K sensor steps from dominating. + edgeSamples = 3 + stateKeyRoot = "thermaltwin/" +) + +// Zone is one declared temperature sensor the twin learns. +type Zone struct { + // Key identifies the zone across restarts: driver name and metric. + Key string `json:"key"` + SensorDriver string `json:"sensor_driver"` + SensorMetric string `json:"sensor_metric"` +} + +// MetricReader is the part of the telemetry store the service reads. +type MetricReader interface { + LatestMetric(driver, name string) (float64, time.Time, bool) +} + +// ConfigStore is the part of the state store the service persists to. +type ConfigStore interface { + LoadConfig(key string) (string, bool) + SaveConfig(key, value string) error +} + +// Service samples every declared zone once a minute and feeds closed windows +// to its twin. It reads and stores; it commands nothing. +type Service struct { + Store ConfigStore + Tele MetricReader + // Temp is outdoor temperature at a time, the same source the load model + // learns its heating coefficient from. + Temp func(time.Time) (float64, bool) + // Zones lists the declared sensors to learn. Called at start and at each + // window boundary, so a config change is picked up within a window. + Zones func() []Zone + // PowerDrivers lists configured drivers that may report HeatPowerMetric. + PowerDrivers func() []string + + mu sync.Mutex + zones map[string]*zoneState + refreshed time.Time + + stop chan struct{} + done chan struct{} +} + +type reading struct { + at time.Time + v float64 +} + +type zoneState struct { + zone Zone + twin *Twin + start time.Time + tIn []reading + power []float64 + src string + gap bool + // lastEnd is the end of the last observed window, for contiguity. + lastEnd time.Time +} + +// NewService returns a stopped service. +func NewService(store ConfigStore, tele MetricReader) *Service { + return &Service{Store: store, Tele: tele, zones: map[string]*zoneState{}} +} + +// Start runs the sampling loop until Stop or ctx ends. +func (s *Service) Start(ctx context.Context) { + s.stop = make(chan struct{}) + s.done = make(chan struct{}) + go func() { + defer close(s.done) + t := time.NewTicker(tick) + defer t.Stop() + s.Tick(time.Now()) + for { + select { + case <-ctx.Done(): + return + case <-s.stop: + return + case now := <-t.C: + s.Tick(now) + } + } + }() +} + +// Stop ends the loop and waits for it. +func (s *Service) Stop() { + if s.stop == nil { + return + } + close(s.stop) + <-s.done + s.stop = nil +} + +// Tick takes one minute's readings. Exported for tests and for a caller +// that drives time itself. +func (s *Service) Tick(now time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + windowStart := now.Truncate(Window) + if !s.refreshed.Equal(windowStart) { + s.refreshZonesLocked() + s.refreshed = windowStart + } + for _, zs := range s.zones { + if !zs.start.Equal(windowStart) { + if !zs.start.IsZero() { + s.closeWindowLocked(zs) + } + zs.start, zs.tIn, zs.power, zs.src, zs.gap = windowStart, nil, nil, "", false + } + s.sampleLocked(zs, now) + } +} + +func (s *Service) refreshZonesLocked() { + if s.Zones == nil { + return + } + want := map[string]Zone{} + for _, z := range s.Zones() { + if z.Key != "" { + want[z.Key] = z + } + } + for key := range s.zones { + if _, ok := want[key]; !ok { + delete(s.zones, key) + } + } + for key, z := range want { + if zs, ok := s.zones[key]; ok { + zs.zone = z + continue + } + s.zones[key] = &zoneState{zone: z, twin: s.restore(key)} + } +} + +func (s *Service) sampleLocked(zs *zoneState, now time.Time) { + v, at, ok := s.read(zs.zone.SensorDriver, zs.zone.SensorMetric, now) + if !ok { + zs.gap = true + return + } + src := s.powerSourceLocked(zs.zone, now) + if len(zs.tIn) > 0 && src != zs.src { + // Heat input appeared, vanished or moved mid-window; the window's + // mean would mix two meanings. + zs.gap = true + } + zs.src = src + if src != "" { + p, _, ok := s.read(src, HeatPowerMetric, now) + if !ok { + zs.gap = true + return + } + zs.power = append(zs.power, p/1000) + } + zs.tIn = append(zs.tIn, reading{at: at, v: v}) +} + +// powerSourceLocked prefers the sensor's own driver, then the single other +// driver reporting heat power. Two candidates is no answer: the twin cannot +// tell which one heats this zone. +func (s *Service) powerSourceLocked(z Zone, now time.Time) string { + if _, _, ok := s.read(z.SensorDriver, HeatPowerMetric, now); ok { + return z.SensorDriver + } + if s.PowerDrivers == nil { + return "" + } + var found []string + for _, d := range s.PowerDrivers() { + if _, _, ok := s.read(d, HeatPowerMetric, now); ok { + found = append(found, d) + } + } + if len(found) == 1 { + return found[0] + } + return "" +} + +func (s *Service) read(driver, metric string, now time.Time) (float64, time.Time, bool) { + if s.Tele == nil || driver == "" { + return 0, time.Time{}, false + } + v, at, ok := s.Tele.LatestMetric(driver, metric) + if !ok || now.Sub(at) > maxReadingAge || at.After(now.Add(time.Minute)) { + return 0, time.Time{}, false + } + return v, at, true +} + +func (s *Service) closeWindowLocked(zs *zoneState) { + end := zs.start.Add(Window) + if zs.gap || len(zs.tIn) < minWindowSamples || (zs.src != "" && len(zs.power) != len(zs.tIn)) { + return + } + if s.Temp == nil { + return + } + tOut, ok := s.Temp(zs.start.Add(Window / 2)) + if !ok { + return + } + startC, startAt := edgeMean(zs.tIn[:edgeSamples]) + endC, endAt := edgeMean(zs.tIn[len(zs.tIn)-edgeSamples:]) + var powerKW float64 + for _, p := range zs.power { + powerKW += p + } + if len(zs.power) > 0 { + powerKW /= float64(len(zs.power)) + } + + if zs.src != "" && zs.twin.PowerSource != zs.src { + if zs.twin.Samples > 0 { + slog.Info("thermal twin restarts: heat input source changed", + "zone", zs.zone.Key, "from", zs.twin.PowerSource, "to", zs.src) + zs.twin = New() + } + zs.twin.PowerSource = zs.src + } + learned := zs.twin.Observe(Sample{ + TInStartC: startC, + TInEndC: endC, + TOutC: tOut, + PowerKW: powerKW, + DurationH: endAt.Sub(startAt).Hours(), + Contiguous: zs.lastEnd.Equal(zs.start), + }) + if learned { + zs.lastEnd = end + } + s.persist(zs) +} + +func edgeMean(rs []reading) (float64, time.Time) { + var sum float64 + var at int64 + for _, r := range rs { + sum += r.v + at += r.at.UnixNano() + } + n := int64(len(rs)) + return sum / float64(n), time.Unix(0, at/n) +} + +func (s *Service) restore(key string) *Twin { + t := New() + if s.Store == nil { + return t + } + js, ok := s.Store.LoadConfig(stateKeyRoot + key) + if !ok { + return t + } + var stored Twin + res := modelstate.Unwrap(js, FeatureHash(), "", &stored) + if !res.OK() { + slog.Info("thermal twin cold start", "zone", key, "reason", res.Reason) + return t + } + return &stored +} + +func (s *Service) persist(zs *zoneState) { + if s.Store == nil { + return + } + js, err := modelstate.Wrap(FeatureHash(), zs.twin) + if err == nil { + err = s.Store.SaveConfig(stateKeyRoot+zs.zone.Key, js) + } + if err != nil { + slog.Warn("thermal twin persist", "zone", zs.zone.Key, "err", err) + } +} + +// ZoneStatus is one zone's twin, as the API shows it. +type ZoneStatus struct { + Zone + PowerSource string `json:"power_source,omitempty"` + Status Status `json:"status"` + Params Params `json:"params"` + Samples int64 `json:"samples"` + Rejected int64 `json:"rejected"` + Evaluations int64 `json:"evaluations"` + MAE3hK float64 `json:"mae_3h_k"` +} + +// Snapshot returns every zone's twin, sorted by key. +func (s *Service) Snapshot() []ZoneStatus { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]ZoneStatus, 0, len(s.zones)) + for _, zs := range s.zones { + out = append(out, ZoneStatus{ + Zone: zs.zone, + PowerSource: zs.twin.PowerSource, + Status: zs.twin.Status(), + Params: zs.twin.Params(), + Samples: zs.twin.Samples, + Rejected: zs.twin.Rejected, + Evaluations: zs.twin.Evaluations, + MAE3hK: zs.twin.MAE3hK, + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) + return out +} diff --git a/go/internal/thermaltwin/service_test.go b/go/internal/thermaltwin/service_test.go new file mode 100644 index 000000000..d2e8c81dc --- /dev/null +++ b/go/internal/thermaltwin/service_test.go @@ -0,0 +1,186 @@ +package thermaltwin + +import ( + "strings" + "testing" + "time" +) + +type metric struct { + v float64 + at time.Time +} + +type fakeTele map[string]metric + +func (f fakeTele) LatestMetric(driver, name string) (float64, time.Time, bool) { + m, ok := f[driver+"/"+name] + return m.v, m.at, ok +} + +type fakeStore map[string]string + +func (f fakeStore) LoadConfig(key string) (string, bool) { v, ok := f[key]; return v, ok } +func (f fakeStore) SaveConfig(key, value string) error { f[key] = value; return nil } + +var t0 = time.Date(2026, 11, 10, 6, 0, 0, 0, time.UTC) + +var livingRoom = Zone{Key: "wiser/room_temp_c", SensorDriver: "wiser", SensorMetric: "room_temp_c"} + +func newTestService(tele fakeTele, store fakeStore, zones ...Zone) *Service { + s := NewService(store, tele) + s.Temp = func(time.Time) (float64, bool) { return 0, true } + s.Zones = func() []Zone { return zones } + s.PowerDrivers = func() []string { return []string{"wiser", "heishamon", "nibe"} } + return s +} + +// run ticks once a minute for the given minutes, letting update set this +// minute's readings first. +func run(s *Service, from time.Time, minutes int, update func(now time.Time)) time.Time { + now := from + for i := 0; i < minutes; i++ { + update(now) + s.Tick(now) + now = now.Add(time.Minute) + } + return now +} + +func zoneStatus(t *testing.T, s *Service, key string) ZoneStatus { + t.Helper() + for _, z := range s.Snapshot() { + if z.Key == key { + return z + } + } + t.Fatalf("zone %s not in snapshot %+v", key, s.Snapshot()) + return ZoneStatus{} +} + +func TestServiceLearnsAndPersistsAZone(t *testing.T) { + tele, store := fakeTele{}, fakeStore{} + s := newTestService(tele, store, livingRoom) + h := &house{a: 1.0 / 40, b: 1.0 / 15, c: 0.05, tIn: 20} + + // Three whole windows plus the minute that closes the third. + run(s, t0, 3*30+1, func(now time.Time) { + h.step(0, 2, 1.0/60) + tele["wiser/room_temp_c"] = metric{h.tIn, now} + tele["heishamon/hp_power_w"] = metric{2000, now} + }) + + z := zoneStatus(t, s, livingRoom.Key) + if z.Samples != 3 || z.Rejected != 0 { + t.Fatalf("samples = %d, rejected = %d, want 3 and 0", z.Samples, z.Rejected) + } + if z.PowerSource != "heishamon" { + t.Errorf("power source = %q, want the single driver reporting %s", z.PowerSource, HeatPowerMetric) + } + if z.Status != StatusLearning { + t.Errorf("status = %s, want learning", z.Status) + } + + js, ok := store[stateKeyRoot+livingRoom.Key] + if !ok || !strings.Contains(js, FeatureHash()) { + t.Fatalf("persisted state = %q", js) + } + restored := newTestService(fakeTele{}, store, livingRoom) + restored.Tick(t0.Add(10 * time.Hour)) + if got := zoneStatus(t, restored, livingRoom.Key); got.Samples != 3 || got.PowerSource != "heishamon" { + t.Errorf("restored = %+v, want the persisted twin", got) + } +} + +// State fitted to another form of the model cold starts instead of +// restoring coefficients that mean something else. +func TestServiceColdStartsOnForeignState(t *testing.T) { + store := fakeStore{stateKeyRoot + livingRoom.Key: `{"schema_version":1,"feature_hash":"0000000000000000","model":{"samples":999}}`} + s := newTestService(fakeTele{}, store, livingRoom) + s.Tick(t0) + if got := zoneStatus(t, s, livingRoom.Key); got.Samples != 0 { + t.Errorf("samples = %d, want a cold start", got.Samples) + } +} + +// A sensor that stops reporting for part of a window yields no sample: an +// old reading is not the zone now. +func TestServiceSkipsAWindowWithAStaleSensor(t *testing.T) { + tele := fakeTele{} + s := newTestService(tele, fakeStore{}, livingRoom) + run(s, t0, 31, func(now time.Time) { + at := now + if now.Sub(t0) >= 10*time.Minute { + at = t0.Add(10 * time.Minute) // frozen from minute ten + } + tele["wiser/room_temp_c"] = metric{20, at} + tele["heishamon/hp_power_w"] = metric{1500, now} + }) + if got := zoneStatus(t, s, livingRoom.Key); got.Samples != 0 { + t.Errorf("samples = %d, want 0 for a window with a stale sensor", got.Samples) + } +} + +// Two drivers reporting heat power and neither the sensor's own: the twin +// cannot tell which heats this zone, so it learns without heat input and +// says so, rather than guess. +func TestServiceRefusesAnAmbiguousPowerSource(t *testing.T) { + tele := fakeTele{} + s := newTestService(tele, fakeStore{}, livingRoom) + run(s, t0, 31, func(now time.Time) { + tele["wiser/room_temp_c"] = metric{20, now} + tele["heishamon/hp_power_w"] = metric{1500, now} + tele["nibe/hp_power_w"] = metric{900, now} + }) + z := zoneStatus(t, s, livingRoom.Key) + if z.PowerSource != "" || z.Status != StatusNoHeatInput { + t.Errorf("zone = %+v, want no power source and no_heat_input", z) + } +} + +// The sensor's own driver reporting heat power wins over any other. +func TestServicePrefersTheSensorsOwnPower(t *testing.T) { + tele := fakeTele{} + pump := Zone{Key: "nibe/hp_indoor_temp_c", SensorDriver: "nibe", SensorMetric: "hp_indoor_temp_c"} + s := newTestService(tele, fakeStore{}, pump) + run(s, t0, 31, func(now time.Time) { + tele["nibe/hp_indoor_temp_c"] = metric{20, now} + tele["nibe/hp_power_w"] = metric{900, now} + tele["heishamon/hp_power_w"] = metric{1500, now} + }) + if z := zoneStatus(t, s, pump.Key); z.PowerSource != "nibe" { + t.Errorf("power source = %q, want nibe", z.PowerSource) + } +} + +func TestServiceDropsARemovedZone(t *testing.T) { + tele := fakeTele{} + zones := []Zone{livingRoom} + s := NewService(fakeStore{}, tele) + s.Temp = func(time.Time) (float64, bool) { return 0, true } + s.Zones = func() []Zone { return zones } + s.Tick(t0) + if len(s.Snapshot()) != 1 { + t.Fatalf("snapshot = %+v", s.Snapshot()) + } + zones = nil + s.Tick(t0.Add(Window)) + if got := s.Snapshot(); len(got) != 0 { + t.Errorf("snapshot = %+v after the zone was removed", got) + } +} + +// Without an outdoor temperature a window teaches nothing true, so it is +// skipped rather than learned against a guessed zero. +func TestServiceNeedsOutdoorTemperature(t *testing.T) { + tele := fakeTele{} + s := newTestService(tele, fakeStore{}, livingRoom) + s.Temp = func(time.Time) (float64, bool) { return 0, false } + run(s, t0, 31, func(now time.Time) { + tele["wiser/room_temp_c"] = metric{20, now} + tele["heishamon/hp_power_w"] = metric{1500, now} + }) + if got := zoneStatus(t, s, livingRoom.Key); got.Samples != 0 { + t.Errorf("samples = %d without outdoor temperature, want 0", got.Samples) + } +} From 5ea4f360e492c52fd08e61978849e1a09c02ec45 Mon Sep 17 00:00:00 2001 From: hannesb90 <279736760+hannesb90@users.noreply.github.com> Date: Sun, 15 Nov 2026 08:20:29 +0100 Subject: [PATCH 2/2] fix(thermaltwin): trust a twin only when it shows skill Review of #1996: a twin reached "trusted" in a house a thermostat holds still, at 0.08 K error, although outdoor temperature and heat input move together there and the room never moves, so inertia and capacity cannot be learned. The error was never compared with "the temperature stays as it is". A twin is now trusted only when, over a week of three-hour judgements, its error is at most 0.8 of the error of assuming no change, and heat input varied within at least a fifth of the judged stretches. A twin whose error is within the limit without that reads "unproven". The API shows both errors and the share. Stored twins of the earlier form cold start. Also from the review: - The service no longer reads the driver catalog. main hands it the zones from the catalog it already loads at start and on each applied configuration, so nothing is parsed every 30 minutes or under the service's lock. - A zone that is no longer declared has its stored twin deleted, also when it went away while FTW was not running. An incomplete zone list (a driver with no catalog entry) forgets nothing. - hp_power_w includes hot-water charging and defrost, which biases b; said where the metric is named and in the changeset. Co-Authored-By: Claude Fable 5.2 Signed-off-by: hannesb90 <279736760+hannesb90@users.noreply.github.com> --- .changeset/thermal-twin-shadow.md | 2 +- go/cmd/ftw/main.go | 21 +++--- go/cmd/ftw/thermal_twin.go | 9 ++- go/cmd/ftw/thermal_twin_test.go | 14 +++- go/internal/api/api_thermal_twin_test.go | 4 +- go/internal/state/store.go | 8 +++ go/internal/state/store_test.go | 22 ++++++ go/internal/thermaltwin/model.go | 84 ++++++++++++++++++---- go/internal/thermaltwin/model_test.go | 71 ++++++++++++++++++ go/internal/thermaltwin/service.go | 91 ++++++++++++++++++------ go/internal/thermaltwin/service_test.go | 59 ++++++++++++--- 11 files changed, 326 insertions(+), 59 deletions(-) diff --git a/.changeset/thermal-twin-shadow.md b/.changeset/thermal-twin-shadow.md index 10ef87023..4370badec 100644 --- a/.changeset/thermal-twin-shadow.md +++ b/.changeset/thermal-twin-shadow.md @@ -2,4 +2,4 @@ "ftw": patch --- -FTW now learns a thermal twin, in shadow, for each room sensor a driver declares (`sensors` with `asset = "space_heat"`). Every 30 minutes it fits the room's inertia, heat capacity, heat loss and other gains from the room temperature, the outdoor temperature and the heat pump's `hp_power_w`, and checks itself by predicting three hours ahead against what the sensor then measured. `GET /api/thermal/twins` shows each twin as learning, trusted or drifting, or says it has no heat input to learn from. The twin commands nothing, and nothing plans from it yet. +FTW now learns a thermal twin, in shadow, for each room sensor a driver declares (`sensors` with `asset = "space_heat"`). Every 30 minutes it fits the room's inertia, heat capacity, heat loss and other gains from the room temperature, the outdoor temperature and the heat pump's `hp_power_w`, and checks itself by predicting three hours ahead against what the sensor then measured. `GET /api/thermal/twins` shows each twin as learning, unproven, trusted or drifting, or says it has no heat input to learn from. A twin is trusted only when its error is small, it beats assuming the temperature stays as it is, and the heat input varied enough to learn from; a room a thermostat holds still stays unproven. `hp_power_w` is the pump's whole electrical power, so hot-water charging and defrost count as room heat until a driver can report space heat alone. A twin is deleted when its sensor is no longer declared. The twin commands nothing, and nothing plans from it yet. diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 32c108e6d..4c071f219 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -1065,6 +1065,9 @@ func main() { // the telemetry store + state DB are wired. Closure capture // works because Go closes over the variable, not its value. var loadSvc *loadmodel.Service + // Likewise constructed further down; the applier hands it the zones a + // changed configuration or driver catalog declares. + var thermalTwins *thermaltwin.Service // Caller holds forecastConfigMu for a live transition. Keep persisted state // intact during a short unconfirmed startup; only confirmed live bindings @@ -1194,6 +1197,9 @@ func main() { } observeOnly = config.ObserveOnlyDriverSet(newCfg) capMu.Unlock() + if thermalTwins != nil { + thermalTwins.SetZones(thermalTwinZones(newCfg.Drivers, reloadCatalog)) + } warnIfEVHasBatteryCapacity(newCfg.Drivers, newCfg.Loadpoints, reloadCatalog) // Swap inverter-group tags (#143) and per-driver power @@ -1473,21 +1479,18 @@ func main() { // Learns each declared room sensor's inertia and heat response from the // same outdoor temperature as the load model. It reads and stores, and // nothing plans or dispatches from it. + // + // The zones come from the catalog already loaded for the drivers, here + // and again whenever a saved configuration reloads it, so the service + // reads no driver file itself. twinDrivers := func() []config.Driver { cfgMu.RLock() defer cfgMu.RUnlock() return append([]config.Driver(nil), cfg.Drivers...) } - thermalTwins := thermaltwin.NewService(st, tel) + thermalTwins = thermaltwin.NewService(st, tel) thermalTwins.Temp = loadSvc.Temp - thermalTwins.Zones = func() []thermaltwin.Zone { - catalog, err := drivers.LoadCatalogSources(drivers.CatalogSource{Dir: *userDriversDirFlag, Source: "local"}, drivers.CatalogSource{Dir: config.ManagedDriversDirOverride, Source: "managed"}, drivers.CatalogSource{Dir: resolveDriverDir(), Source: "bundled"}) - if err != nil { - slog.Warn("thermal twin: driver catalog unavailable", "err", err) - return nil - } - return thermalTwinZones(twinDrivers(), catalog) - } + thermalTwins.SetZones(thermalTwinZones(twinDrivers(), driverCatalog)) thermalTwins.PowerDrivers = func() []string { return thermalTwinPowerDrivers(twinDrivers()) } thermalTwins.Start(ctx) defer thermalTwins.Stop() diff --git a/go/cmd/ftw/thermal_twin.go b/go/cmd/ftw/thermal_twin.go index cb0254f0c..d0c964dda 100644 --- a/go/cmd/ftw/thermal_twin.go +++ b/go/cmd/ftw/thermal_twin.go @@ -13,14 +13,19 @@ import ( // driver declares. Only space heat for now: a tank or buffer has the same // form but loses heat to the room it stands in, not to outdoors, and the // service has no such temperature yet. -func thermalTwinZones(cfgDrivers []config.Driver, catalog []drivers.CatalogEntry) []thermaltwin.Zone { +// +// complete is false when an enabled driver has no catalog entry, so its +// sensors are unknown rather than absent. The service then forgets no twin. +func thermalTwinZones(cfgDrivers []config.Driver, catalog []drivers.CatalogEntry) (zones []thermaltwin.Zone, complete bool) { var out []thermaltwin.Zone + complete = true for _, d := range cfgDrivers { if d.Disabled || d.Lua == "" { continue } entry := drivers.EntryForDriver(catalog, d.Lua) if entry == nil { + complete = false continue } for _, s := range entry.Sensors { @@ -35,7 +40,7 @@ func thermalTwinZones(cfgDrivers []config.Driver, catalog []drivers.CatalogEntry } } sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) - return out + return out, complete } // thermalTwinPowerDrivers lists the enabled drivers that may report heat diff --git a/go/cmd/ftw/thermal_twin_test.go b/go/cmd/ftw/thermal_twin_test.go index a0c351d0c..79a4e440b 100644 --- a/go/cmd/ftw/thermal_twin_test.go +++ b/go/cmd/ftw/thermal_twin_test.go @@ -39,7 +39,10 @@ func TestThermalTwinZonesFromDeclaredSensors(t *testing.T) { {Name: "grid", Lua: filepath.Join(dir, "meter.lua")}, } - got := thermalTwinZones(cfg, catalog) + got, complete := thermalTwinZones(cfg, catalog) + if !complete { + t.Error("complete = false with every enabled driver in the catalog") + } want := []thermaltwin.Zone{ {Key: "wiser/room_bed_temp_c", SensorDriver: "wiser", SensorMetric: "room_bed_temp_c"}, {Key: "wiser/room_living_temp_c", SensorDriver: "wiser", SensorMetric: "room_living_temp_c"}, @@ -47,6 +50,15 @@ func TestThermalTwinZonesFromDeclaredSensors(t *testing.T) { if !reflect.DeepEqual(got, want) { t.Errorf("zones = %+v, want %+v (no tank, no disabled driver)", got, want) } + // A driver the catalog cannot describe has unknown sensors, not none: + // the list is incomplete and no twin may be forgotten on its strength. + missing := append(cfg, config.Driver{Name: "gone", Lua: filepath.Join(dir, "gone.lua")}) + if got, complete := thermalTwinZones(missing, catalog); complete || !reflect.DeepEqual(got, want) { + t.Errorf("with an unreadable driver: zones = %+v, complete = %v; want the same zones, incomplete", got, complete) + } + if got, complete := thermalTwinZones(cfg, nil); complete || len(got) != 0 { + t.Errorf("without a catalog: zones = %+v, complete = %v; want none, incomplete", got, complete) + } if p := thermalTwinPowerDrivers(cfg); !reflect.DeepEqual(p, []string{"wiser", "grid"}) { t.Errorf("power drivers = %v", p) } diff --git a/go/internal/api/api_thermal_twin_test.go b/go/internal/api/api_thermal_twin_test.go index 11fdafd69..90359c86e 100644 --- a/go/internal/api/api_thermal_twin_test.go +++ b/go/internal/api/api_thermal_twin_test.go @@ -37,9 +37,7 @@ func TestThermalTwinsAreAlwaysMarkedShadow(t *testing.T) { } svc := thermaltwin.NewService(nil, twinTele{}) - svc.Zones = func() []thermaltwin.Zone { - return []thermaltwin.Zone{{Key: "wiser/room_temp_c", SensorDriver: "wiser", SensorMetric: "room_temp_c"}} - } + svc.SetZones([]thermaltwin.Zone{{Key: "wiser/room_temp_c", SensorDriver: "wiser", SensorMetric: "room_temp_c"}}, true) svc.Tick(time.Now()) got = getThermalTwins(t, New(&Deps{ThermalTwin: svc})) if string(got["shadow"]) != "true" || string(got["enabled"]) != "true" { diff --git a/go/internal/state/store.go b/go/internal/state/store.go index b2cdd95cd..f26d7c0a3 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -1293,6 +1293,14 @@ func (s *Store) SaveConfig(key, value string) error { }) } +// DeleteConfig removes a config k/v. A missing key is not an error. +func (s *Store) DeleteConfig(key string) error { + return s.durableConfigWrite(func(tx *sql.Tx) error { + _, err := tx.Exec(`DELETE FROM config WHERE key = ?`, key) + return err + }) +} + // LoadConfig returns the value for key, or ok=false if missing. func (s *Store) LoadConfig(key string) (string, bool) { var v string diff --git a/go/internal/state/store_test.go b/go/internal/state/store_test.go index 2fdf1f1d8..b202d7f22 100644 --- a/go/internal/state/store_test.go +++ b/go/internal/state/store_test.go @@ -150,6 +150,28 @@ func TestTimeSeriesInternCacheIsPerStore(t *testing.T) { } } +func TestDeleteConfig(t *testing.T) { + s := freshStore(t) + if err := s.SaveConfig("thermaltwin/a", "1"); err != nil { + t.Fatal(err) + } + if err := s.SaveConfig("thermaltwin/b", "2"); err != nil { + t.Fatal(err) + } + if err := s.DeleteConfig("thermaltwin/a"); err != nil { + t.Fatal(err) + } + if _, ok := s.LoadConfig("thermaltwin/a"); ok { + t.Error("deleted key still loads") + } + if v, ok := s.LoadConfig("thermaltwin/b"); !ok || v != "2" { + t.Errorf("other key = %q ok=%v, want kept", v, ok) + } + if err := s.DeleteConfig("missing"); err != nil { + t.Errorf("deleting a missing key: %v", err) + } +} + func TestConfigRoundtrip(t *testing.T) { s := freshStore(t) if err := s.SaveConfig("mode", "self_consumption"); err != nil { diff --git a/go/internal/thermaltwin/model.go b/go/internal/thermaltwin/model.go index b701aa277..142c1e290 100644 --- a/go/internal/thermaltwin/model.go +++ b/go/internal/thermaltwin/model.go @@ -14,6 +14,13 @@ // simulating each three-hour stretch from the parameters it had at the // start, fed the inputs that really happened. A prediction never verifies // itself. +// +// A small error is not yet skill. In a house a thermostat holds still, "the +// temperature stays as it is" is as accurate as any model, and τ and the +// capacity cannot be learned there: outdoor temperature and heat input move +// together and the room does not move at all. So a twin is trusted only +// when it also beats that guess over the same stretches, and when the heat +// input varied within enough of them to tell the two terms apart. package thermaltwin import ( @@ -46,6 +53,23 @@ const ( trustMAEK = 0.3 driftMAEK = 0.5 maeMemory = 24 // evaluations, one per window + + // Skill is judged over a week of evaluations, the span the twin needs + // to learn at all: one still day must not cost a twin its standing, and + // one lively day must not earn it. + skillMemory = 7 * 48 + // The twin's three-hour error must be at most this share of the error + // of assuming no change. At 1.0 a twin that predicts "no change" would + // pass by a rounding step. + skillVsPersistence = 0.8 + // Heat input varied within a stretch when its lowest window is at most + // this share of its highest, and the highest is real heat rather than + // standby. A pump that follows the cold moves a few per cent in three + // hours; one that cycles, or is set back, moves far more. + heatVariedRatio = 0.7 + heatVariedMinKW = 0.1 + // The share of judged stretches in which heat input must have varied. + minHeatVariedShare = 0.2 ) // Status is what a twin can honestly say about itself. @@ -54,8 +78,12 @@ type Status string const ( StatusNoHeatInput Status = "no_heat_input" StatusLearning Status = "learning" - StatusTrusted Status = "trusted" - StatusDrifting Status = "drifting" + // StatusUnproven is a twin whose error is within the limit but which + // has not shown skill: it does not beat "no change", or heat input + // varied too little to learn from. Nothing may plan from it. + StatusUnproven Status = "unproven" + StatusTrusted Status = "trusted" + StatusDrifting Status = "drifting" ) // Sample is one closed window, as the service measured it. @@ -101,6 +129,13 @@ type Twin struct { Evaluations int64 `json:"evaluations"` WasTrusted bool `json:"was_trusted"` + // The same three-hour error over a week, beside the error of assuming + // the temperature stays where the stretch began, and the share of those + // stretches in which heat input varied. + SkillMAE3hK float64 `json:"skill_mae_3h_k"` + PersistenceMAE3hK float64 `json:"persistence_mae_3h_k"` + HeatVariedShare float64 `json:"heat_varied_share"` + // PowerSource names where heat input came from. A twin fitted to one // source means nothing against another, so a change starts it over. PowerSource string `json:"power_source,omitempty"` @@ -240,11 +275,25 @@ func (t *Twin) evaluate() { } temp += w.DurationH * dot(first.Theta, regressor(temp, w.TOutC, w.PowerKW)) } - errK := math.Abs(temp - t.Pending[len(t.Pending)-1].TInEndC) + measured := t.Pending[len(t.Pending)-1].TInEndC + errK := math.Abs(temp - measured) if math.IsNaN(errK) || math.IsInf(errK, 0) { return } + persistenceK := math.Abs(first.TInStartC - measured) + minKW, maxKW := first.PowerKW, first.PowerKW + for _, w := range t.Pending { + minKW, maxKW = math.Min(minKW, w.PowerKW), math.Max(maxKW, w.PowerKW) + } + varied := 0.0 + if maxKW >= heatVariedMinKW && minKW <= heatVariedRatio*maxKW { + varied = 1 + } t.Evaluations++ + week := math.Min(float64(t.Evaluations), skillMemory) + t.SkillMAE3hK += (errK - t.SkillMAE3hK) / week + t.PersistenceMAE3hK += (persistenceK - t.PersistenceMAE3hK) / week + t.HeatVariedShare += (varied - t.HeatVariedShare) / week // A running mean at first, then about half a day of memory: long // enough that one odd evening does not cost trust, short enough that a // house that changed shows as drifting the same day. @@ -289,6 +338,13 @@ func (t *Twin) physical() bool { p.CapacityKWhPerK != nil && *p.CapacityKWhPerK >= 1 && *p.CapacityKWhPerK <= 500 } +// skilled is whether the twin has shown it knows something: over the last +// week it beat "no change", in stretches where heat input varied. +func (t *Twin) skilled() bool { + return t.SkillMAE3hK <= skillVsPersistence*t.PersistenceMAE3hK && + t.HeatVariedShare >= minHeatVariedShare +} + // Status reports the twin's standing and records a first trust, so later // error above the drift limit reads as drifting rather than as learning. func (t *Twin) Status() Status { @@ -301,16 +357,20 @@ func (t *Twin) Status() Status { } return StatusLearning } + limit := trustMAEK + if t.WasTrusted { + limit = driftMAEK + } switch { - case t.MAE3hK <= trustMAEK: - t.WasTrusted = true - return StatusTrusted - case t.WasTrusted && t.MAE3hK <= driftMAEK: - return StatusTrusted - case t.WasTrusted: + case t.MAE3hK > limit && t.WasTrusted: return StatusDrifting + case t.MAE3hK > limit: + return StatusLearning + case !t.skilled(): + return StatusUnproven } - return StatusLearning + t.WasTrusted = true + return StatusTrusted } // FeatureHash fingerprints the model's form, so stored state from a @@ -321,8 +381,8 @@ func FeatureHash() string { x := regressor(v[0], v[1], v[2]) probe = append(probe, x[:]...) } - probe = append(probe, windowH, lambda, EvalHorizonWindows) - return modelstate.Fingerprint("thermaltwin: dT/dt K/h = a(Tout-Tin) + b*P_kW + c; heat input hp_power_w", probe) + probe = append(probe, windowH, lambda, EvalHorizonWindows, skillMemory) + return modelstate.Fingerprint("thermaltwin: dT/dt K/h = a(Tout-Tin) + b*P_kW + c; heat input hp_power_w; judged against persistence", probe) } func dot(a, b [3]float64) float64 { return a[0]*b[0] + a[1]*b[1] + a[2]*b[2] } diff --git a/go/internal/thermaltwin/model_test.go b/go/internal/thermaltwin/model_test.go index a3ce97434..c0a6f620a 100644 --- a/go/internal/thermaltwin/model_test.go +++ b/go/internal/thermaltwin/model_test.go @@ -215,3 +215,74 @@ func deref(v *float64) any { } return *v } + +// held runs a house whose thermostat holds the room still: heat input +// follows the cold exactly, and the sensor reads the setpoint with noise. +// This is the reviewer's probe on #1996. Nothing in it can teach inertia or +// capacity, however small the prediction error gets. +func held(tw *Twin, n int) { + const setpoint = 21.0 + a, b := 1.0/40, 1.0/15 + seed := uint32(7) + read := func() float64 { + seed = seed*1664525 + 1013904223 + noise := 0.3 * (float64(seed>>8)/float64(1<<24) - 0.5) // ±0.15 K + return math.Round((setpoint+noise)*10) / 10 + } + tIn := read() + for i := 0; i < n; i++ { + hour := float64(i) * windowH + tOut := -2 + 6*math.Sin(2*math.Pi*hour/24) + power := a * (setpoint - tOut) / b + next := read() + tw.Observe(Sample{TInStartC: tIn, TInEndC: next, TOutC: tOut, PowerKW: power, Contiguous: true}) + tIn = next + } +} + +// A small error in a room that never moves is not skill: "no change" does +// as well, and heat input never varied apart from the weather. +func TestTwinDoesNotTrustAThermostatHeldHouse(t *testing.T) { + tw := New() + held(tw, 21*48) + if tw.MAE3hK > trustMAEK { + t.Fatalf("precondition: 3 h error = %.2f K, want within the old limit %.1f", tw.MAE3hK, trustMAEK) + } + if got := tw.Status(); got != StatusUnproven || tw.WasTrusted { + t.Errorf("status = %s (was trusted %v), want unproven, with 3 h error %.2f K against %.2f K for no change, heat varied in %.0f%% of stretches", + got, tw.WasTrusted, tw.MAE3hK, tw.PersistenceMAE3hK, 100*tw.HeatVariedShare) + } +} + +// The two conditions stand apart: beating "no change" is not enough where +// heat input never varied, and varied heat input is not enough for a twin +// that does no better than "no change". +func TestTwinTrustNeedsSkillAndVariedHeat(t *testing.T) { + accurate := func() *Twin { + tw := New() + tw.Theta = [3]float64{1.0 / 40, 1.0 / 15, 0} + tw.Samples, tw.PowerSamples, tw.Evaluations = minSamplesForTrust, minSamplesForTrust, skillMemory + tw.MAE3hK, tw.SkillMAE3hK, tw.PersistenceMAE3hK, tw.HeatVariedShare = 0.1, 0.1, 0.4, 0.5 + return tw + } + if got := accurate().Status(); got != StatusTrusted { + t.Fatalf("precondition: status = %s, want trusted", got) + } + still := accurate() + still.PersistenceMAE3hK = 0.1 + if got := still.Status(); got != StatusUnproven { + t.Errorf("no better than no change: status = %s, want unproven", got) + } + steady := accurate() + steady.HeatVariedShare = 0.05 + if got := steady.Status(); got != StatusUnproven { + t.Errorf("heat input never varied: status = %s, want unproven", got) + } + // A twin that was trusted and then loses its footing says unproven, not + // drifting: its error is still small, it has only stopped being tested. + was := accurate() + was.WasTrusted, was.HeatVariedShare = true, 0.05 + if got := was.Status(); got != StatusUnproven { + t.Errorf("trusted twin in a house that went still: status = %s, want unproven", got) + } +} diff --git a/go/internal/thermaltwin/service.go b/go/internal/thermaltwin/service.go index 7fc04475c..7dda8fdea 100644 --- a/go/internal/thermaltwin/service.go +++ b/go/internal/thermaltwin/service.go @@ -4,6 +4,7 @@ import ( "context" "log/slog" "sort" + "strings" "sync" "time" @@ -12,6 +13,12 @@ import ( // HeatPowerMetric is the canonical electrical input of a heat pump, the same // metric the Heating view discovers pumps by (docs/nibe-local.md). +// +// It is the pump's whole electrical power, so it includes hot-water charging +// and defrost, which put no heat in the room. That biases b low (the +// capacity reads high) by the share of power spent on them. A space-heat +// power metric declared by the driver would remove the bias; none exists +// yet. const HeatPowerMetric = "hp_power_w" const ( @@ -44,6 +51,8 @@ type MetricReader interface { type ConfigStore interface { LoadConfig(key string) (string, bool) SaveConfig(key, value string) error + LoadConfigByPrefix(prefix string) (map[string]string, error) + DeleteConfig(key string) error } // Service samples every declared zone once a minute and feeds closed windows @@ -54,15 +63,14 @@ type Service struct { // Temp is outdoor temperature at a time, the same source the load model // learns its heating coefficient from. Temp func(time.Time) (float64, bool) - // Zones lists the declared sensors to learn. Called at start and at each - // window boundary, so a config change is picked up within a window. - Zones func() []Zone // PowerDrivers lists configured drivers that may report HeatPowerMetric. PowerDrivers func() []string - mu sync.Mutex - zones map[string]*zoneState - refreshed time.Time + mu sync.Mutex + zones map[string]*zoneState + // swept is set once stored twins of zones that went away while the + // process was down have been deleted. + swept bool stop chan struct{} done chan struct{} @@ -128,10 +136,6 @@ func (s *Service) Tick(now time.Time) { s.mu.Lock() defer s.mu.Unlock() windowStart := now.Truncate(Window) - if !s.refreshed.Equal(windowStart) { - s.refreshZonesLocked() - s.refreshed = windowStart - } for _, zs := range s.zones { if !zs.start.Equal(windowStart) { if !zs.start.IsZero() { @@ -143,21 +147,24 @@ func (s *Service) Tick(now time.Time) { } } -func (s *Service) refreshZonesLocked() { - if s.Zones == nil { - return - } +// SetZones declares the sensors to learn, replacing the previous list. The +// caller computes it when the configuration or the driver catalog changes, +// so the service never reads either. +// +// A zone that left the list is forgotten, its stored twin with it: learned +// state must not outlive the declaration it was learned for. complete is +// false when the list may be missing zones, as when a driver's catalog +// entry could not be read. Then zones are added and nothing is forgotten, +// since weeks of learning should not be lost to an unreadable file. +func (s *Service) SetZones(zones []Zone, complete bool) { + s.mu.Lock() + defer s.mu.Unlock() want := map[string]Zone{} - for _, z := range s.Zones() { + for _, z := range zones { if z.Key != "" { want[z.Key] = z } } - for key := range s.zones { - if _, ok := want[key]; !ok { - delete(s.zones, key) - } - } for key, z := range want { if zs, ok := s.zones[key]; ok { zs.zone = z @@ -165,6 +172,40 @@ func (s *Service) refreshZonesLocked() { } s.zones[key] = &zoneState{zone: z, twin: s.restore(key)} } + if !complete { + return + } + for key := range s.zones { + if _, ok := want[key]; !ok { + delete(s.zones, key) + s.forget(key) + } + } + if s.swept || s.Store == nil { + return + } + stored, err := s.Store.LoadConfigByPrefix(stateKeyRoot) + if err != nil { + slog.Warn("thermal twin: stored twins unreadable", "err", err) + return + } + s.swept = true + for storeKey := range stored { + if _, ok := want[strings.TrimPrefix(storeKey, stateKeyRoot)]; !ok { + s.forget(strings.TrimPrefix(storeKey, stateKeyRoot)) + } + } +} + +func (s *Service) forget(key string) { + if s.Store == nil { + return + } + if err := s.Store.DeleteConfig(stateKeyRoot + key); err != nil { + slog.Warn("thermal twin: stored twin not deleted", "zone", key, "err", err) + return + } + slog.Info("thermal twin forgotten: zone no longer declared", "zone", key) } func (s *Service) sampleLocked(zs *zoneState, now time.Time) { @@ -320,6 +361,12 @@ type ZoneStatus struct { Rejected int64 `json:"rejected"` Evaluations int64 `json:"evaluations"` MAE3hK float64 `json:"mae_3h_k"` + // What trust is judged on beside MAE3hK: the twin's error and the + // error of assuming no change, both over a week, and the share of + // judged stretches in which heat input varied. + SkillMAE3hK float64 `json:"skill_mae_3h_k"` + PersistenceMAE3hK float64 `json:"persistence_mae_3h_k"` + HeatVariedShare float64 `json:"heat_varied_share"` } // Snapshot returns every zone's twin, sorted by key. @@ -337,6 +384,10 @@ func (s *Service) Snapshot() []ZoneStatus { Rejected: zs.twin.Rejected, Evaluations: zs.twin.Evaluations, MAE3hK: zs.twin.MAE3hK, + + SkillMAE3hK: zs.twin.SkillMAE3hK, + PersistenceMAE3hK: zs.twin.PersistenceMAE3hK, + HeatVariedShare: zs.twin.HeatVariedShare, }) } sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) diff --git a/go/internal/thermaltwin/service_test.go b/go/internal/thermaltwin/service_test.go index d2e8c81dc..1fdb4f234 100644 --- a/go/internal/thermaltwin/service_test.go +++ b/go/internal/thermaltwin/service_test.go @@ -22,6 +22,16 @@ type fakeStore map[string]string func (f fakeStore) LoadConfig(key string) (string, bool) { v, ok := f[key]; return v, ok } func (f fakeStore) SaveConfig(key, value string) error { f[key] = value; return nil } +func (f fakeStore) DeleteConfig(key string) error { delete(f, key); return nil } +func (f fakeStore) LoadConfigByPrefix(prefix string) (map[string]string, error) { + out := map[string]string{} + for k, v := range f { + if strings.HasPrefix(k, prefix) { + out[k] = v + } + } + return out, nil +} var t0 = time.Date(2026, 11, 10, 6, 0, 0, 0, time.UTC) @@ -30,7 +40,7 @@ var livingRoom = Zone{Key: "wiser/room_temp_c", SensorDriver: "wiser", SensorMet func newTestService(tele fakeTele, store fakeStore, zones ...Zone) *Service { s := NewService(store, tele) s.Temp = func(time.Time) (float64, bool) { return 0, true } - s.Zones = func() []Zone { return zones } + s.SetZones(zones, true) s.PowerDrivers = func() []string { return []string{"wiser", "heishamon", "nibe"} } return s } @@ -153,21 +163,48 @@ func TestServicePrefersTheSensorsOwnPower(t *testing.T) { } } -func TestServiceDropsARemovedZone(t *testing.T) { +// A zone that is no longer declared is forgotten, its stored twin with it; +// so is a twin stored for a zone that went away while FTW was not running. +func TestServiceForgetsARemovedZone(t *testing.T) { + store := fakeStore{stateKeyRoot + "old/room_temp_c": "{}", "unrelated": "kept"} tele := fakeTele{} - zones := []Zone{livingRoom} - s := NewService(fakeStore{}, tele) - s.Temp = func(time.Time) (float64, bool) { return 0, true } - s.Zones = func() []Zone { return zones } - s.Tick(t0) - if len(s.Snapshot()) != 1 { - t.Fatalf("snapshot = %+v", s.Snapshot()) + s := newTestService(tele, store, livingRoom) + if _, ok := store[stateKeyRoot+"old/room_temp_c"]; ok { + t.Error("the stored twin of a zone gone before start was kept") } - zones = nil - s.Tick(t0.Add(Window)) + run(s, t0, 31, func(now time.Time) { + tele["wiser/room_temp_c"] = metric{20, now} + tele["heishamon/hp_power_w"] = metric{1500, now} + }) + if _, ok := store[stateKeyRoot+livingRoom.Key]; !ok { + t.Fatal("precondition: the zone's twin was not stored") + } + s.SetZones(nil, true) if got := s.Snapshot(); len(got) != 0 { t.Errorf("snapshot = %+v after the zone was removed", got) } + if _, ok := store[stateKeyRoot+livingRoom.Key]; ok { + t.Error("the removed zone's stored twin was kept") + } + if store["unrelated"] != "kept" { + t.Error("a row outside the twins' prefix was deleted") + } +} + +// An incomplete list says some zones could not be read, not that they are +// gone. Weeks of learning are not dropped on that. +func TestServiceKeepsZonesWhenTheListIsIncomplete(t *testing.T) { + store := fakeStore{stateKeyRoot + "old/room_temp_c": "{}"} + bedroom := Zone{Key: "wiser/bed_temp_c", SensorDriver: "wiser", SensorMetric: "bed_temp_c"} + s := NewService(store, fakeTele{}) + s.SetZones([]Zone{livingRoom}, false) + s.SetZones([]Zone{bedroom}, false) + if got := s.Snapshot(); len(got) != 2 { + t.Errorf("snapshot = %+v, want both zones kept", got) + } + if _, ok := store[stateKeyRoot+"old/room_temp_c"]; !ok { + t.Error("a stored twin was deleted on an incomplete list") + } } // Without an outdoor temperature a window teaches nothing true, so it is