From 4c48376fcb9696e56d932903ea4335c5b70fb13e Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 27 Oct 2026 20:39:49 +0100 Subject: [PATCH 1/3] test(site): R8 counts a forced battery from when Core has the inverter back The random judge's R8 check counted 10 s from the inverter's return. The Modbus reconnect backoff (up to 60 s) holds Core's first good read back, so the check could not pass even with C1 fixed (207539: 55.7 s). The count now starts at Core's first good read of the inverter after an outage or a restart, at the latest 60 s after the return. After an outage it allows the registry's 30 s default retry (defaultRetryMax) until #1915 (C11) lands. While Core is stopped with SIGTERM, a running inverter must leave its forced mode within 10 s of the stop: the driver cleanup writes the default (triage item 10). Co-Authored-By: Claude Opus 5.5 --- go/test/site/evaluate_test.go | 5 +- go/test/site/random_test.go | 102 ++++++++++++++++++---- go/test/site/random_unit_test.go | 143 ++++++++++++++++++++++++------- 3 files changed, 204 insertions(+), 46 deletions(-) diff --git a/go/test/site/evaluate_test.go b/go/test/site/evaluate_test.go index 70a65fb06..7a2f93145 100644 --- a/go/test/site/evaluate_test.go +++ b/go/test/site/evaluate_test.go @@ -70,8 +70,9 @@ type ftwDispatch struct { } type ftwDriver struct { - Status string `json:"status"` - EVSessionWh *float64 `json:"ev_session_wh"` // a charger's session energy + Status string `json:"status"` + EVSessionWh *float64 `json:"ev_session_wh"` // a charger's session energy + ConsecutiveErrors *int `json:"consecutive_errors"` // failed polls in a row; 0 once a poll reads } type ftwLoadpoint struct { diff --git a/go/test/site/random_test.go b/go/test/site/random_test.go index beab667e1..1f07cfa98 100644 --- a/go/test/site/random_test.go +++ b/go/test/site/random_test.go @@ -225,6 +225,60 @@ func within(ws []eventWindow, t, afterS float64) bool { return false } +// inverterBackFrom gives, for a time t, when the R8 count may start after +// the latest return before t: the inverter's from an outage (inverterOut) +// or Core's from a restart (down). Core has the inverter back at its first +// good read of it (no consecutive errors), which the Modbus reconnect +// backoff may hold back up to reconnectBackoffMaxS. After an outage, a +// default that failed while the inverter was away waits on the registry's +// retry timer, up to defaultRetryMaxS more, until #1915 (C11) sends it on +// the first good poll. A Core that starts fresh has no such wait. +func (d *runData) inverterBackFrom(inverterOut, down []eventWindow) func(t float64) float64 { + type back struct { + at, from float64 + } + var backs []back + add := func(at float64, outage bool) { + from := at + reconnectBackoffMaxS + for _, r := range d.samples { + if r.T >= at && r.T < from && readsInverter(r) { + from = r.T + break + } + } + if outage { + from += defaultRetryMaxS + } + backs = append(backs, back{at, from}) + } + for _, w := range inverterOut { + add(w.to, true) + } + for _, w := range down { + add(w.to, false) + } + return func(t float64) float64 { + at, from := math.Inf(-1), math.Inf(-1) + for _, b := range backs { + if b.at <= t && b.at >= at { + at, from = b.at, b.from + } + } + return from + } +} + +// readsInverter is whether Core's last poll of the inverter succeeded. A +// status without the driver's error count says nothing, and counts as a +// read. +func readsInverter(r *sample) bool { + if r.Status == nil { + return false + } + dr, ok := r.Status.Drivers[inverterDriver] + return !ok || dr.ConsecutiveErrors == nil || *dr.ConsecutiveErrors == 0 +} + // stepWindow is a change FTW did not make, and how long FTW may take to // answer it: until from + extraS + its lever's time + 2 s for the meter and // the tick + 3 s. @@ -241,6 +295,13 @@ const ( evRestartDwellS = 35.0 // control/ev_fuse_budget.go evRestartDwell tickS = 2.0 // the rig's site.control_interval_s plannerFloorPct = 10.0 // Core's planner.soc_min when the config leaves it out (cmd/ftw/main.go) + + // R8: how long a battery FTW does not dispatch may hold a forced mode, + // and the Core timers that may delay it after the inverter returns. + inverterDriver = "sungrow" // the rig's inverter driver (stack_test.go) + forcedBudgetS = 10.0 + reconnectBackoffMaxS = 60.0 // modbus/client.go reconnectBackoffMax + defaultRetryMaxS = 30.0 // drivers/registry.go defaultRetryMax; 0 once #1915 (C11) lands ) // dispatchW is Core's battery order on r, + charge; false when Core @@ -683,26 +744,37 @@ func judgeInvariants(d *runData) (metrics, []criterion) { } m["raises_on_a_stale_meter"] = raises - // 8: a battery FTW does not dispatch is back in its own mode within - // 10 s: no forced charge or discharge left in the inverter's register - // once the inverter and Core are both up again (R8, C1). The 10 s - // count starts when both are back. - forcedS, run8 := 0.0, 0.0 + // 8: a battery FTW does not dispatch runs in its own mode (R8, C1). The + // inverter's register may hold a forced charge or discharge FTW does + // not dispatch for forcedBudgetS. After an inverter outage or a Core + // restart the count starts once Core has the inverter back (backFrom). + // While Core is stopped with SIGTERM its driver cleanup writes the + // default, so a running inverter gets the same 10 s from the stop. One + // that comes back while Core is down has its outage's allowance. + backFrom := d.inverterBackFrom(inverterOut, down) + forcedS, pastS, run8, from8 := 0.0, 0.0, 0.0, 0.0 for i, r := range run { w, ok := dispatchW(r) - stray := r.Status != nil && (!ok || math.Abs(w) <= 50) && math.Abs(batOrder(r)) > 50 && - !within(inverterOut, r.T, 0) && !within(down, r.T, 0) - if stray && i > 0 { - run8 += r.T - run[i-1].T - forcedS = math.Max(forcedS, run8) - if run8 > 10 { - note("forced battery", r.T) - } - } else { + stray := math.Abs(batOrder(r)) > 50 && !within(inverterOut, r.T, 0) + if !within(down, r.T, 0) { + stray = stray && r.Status != nil && (!ok || math.Abs(w) <= 50) + } + if !stray || i == 0 { run8 = 0 + continue + } + if run8 == 0 { + from8 = run[i-1].T + } + run8 += r.T - run[i-1].T + forcedS = math.Max(forcedS, run8) + if r.T > math.Max(from8, backFrom(r.T))+forcedBudgetS { + pastS += r.T - run[i-1].T + note("forced battery", r.T) } } m["battery_forced_without_dispatch_s"] = round(forcedS, 1) + m["battery_forced_past_allowance_s"] = round(pastS, 1) // The first second an invariant broke, from the start, per invariant: // a kept seed is cut a minute after the one it failed on. firsts := map[string]float64{} @@ -719,7 +791,7 @@ func judgeInvariants(d *runData) (metrics, []criterion) { crit("no offer above the charger's rating", over == 0), crit("no discharge asked at the battery's floor", floorS <= 10), crit("no raise on a stale meter", raises == 0), - crit("no forced battery mode FTW does not dispatch", forcedS <= 10), + crit("no forced battery mode FTW does not dispatch", pastS == 0), } } diff --git a/go/test/site/random_unit_test.go b/go/test/site/random_unit_test.go index ea502de39..1dfee8ea0 100644 --- a/go/test/site/random_unit_test.go +++ b/go/test/site/random_unit_test.go @@ -359,38 +359,123 @@ func TestInvariantsLeverAndFaults(t *testing.T) { } } -// R8, C1: once the inverter and Core are back, a battery FTW does not -// dispatch leaves its forced mode within 10 s. +// forcedCase is an R8 run: the inverter's register holds a forced command +// from the start until forcedTo, and FTW dispatches the battery from +// dispatchFrom (0: never). An outage of the inverter (offline: it stops; a +// silence: it runs but does not answer) or a Core restart may come first. +type forcedCase struct { + name string + outage string // "offline", "silence" or "" + outFrom float64 // the outage, from outFrom + outFor float64 // for outFor + readAt float64 // Core's polls of the inverter fail from outFrom until here + stop, up float64 // a Core restart (SIGTERM at stop), if up > 0 + cmd float64 // 187 discharge, 170 charge + setW float64 + forcedTo float64 + dispatchFrom float64 + want bool +} + +func (c forcedCase) run() *runData { + // FTW dispatched the battery until the outage or the stop. + lead := c.outFrom + if c.up > 0 && (c.outage == "" || c.stop < lead) { + lead = c.stop + } + d := randomRun(1201, 16, 5, func(i int, r *sample) { + t := float64(i) + if t < c.forcedTo { + r.Bat.EMSMode, r.Bat.EMSCmd, r.Bat.EMSSetW = 2, c.cmd, c.setW + } else { + r.Bat.EMSCmd = 204 + } + if c.outage == "offline" && t >= c.outFrom && t < c.outFrom+c.outFor { + off := false + r.Bat.Running = &off + } + if c.up > 0 && t >= c.stop && t <= c.up { + return // Core is down: no reading + } + errs := 0 + if c.outage != "" && t >= c.outFrom && t < c.readAt { + errs = 1 + int(t-c.outFrom)/2 + } + r.Status = &ftwStatus{Drivers: map[string]ftwDriver{inverterDriver: {Status: "ok", ConsecutiveErrors: &errs}}} + if t < lead || (c.dispatchFrom > 0 && t >= c.dispatchFrom) { + r.Status.Dispatch = []ftwDispatch{{Driver: inverterDriver, TargetW: -c.setW}} + } + }) + if c.outage != "" { + d.events = []houseEventLine{{T: c.outFrom, Event: houseEvent{Do: c.outage, Device: "inverter", ForS: c.outFor}}} + } + if c.up > 0 { + d.allMarks = []markLine{{T: c.stop, Label: "ftw:restart"}, {T: c.up, Label: "ftw:up"}} + } + return d +} + +// R8, C1, on the shapes of the nightly seeds on master 83c2b24c. Once Core +// has the inverter back, a battery FTW does not dispatch leaves its forced +// mode within 10 s, plus the registry's 30 s default retry until #1915 +// (C11) lands. The Modbus reconnect backoff between the inverter's return +// and Core's first good read is Core's to take. func TestInvariantsNoForcedBatteryWithoutDispatch(t *testing.T) { const forced = "no forced battery mode FTW does not dispatch" - for _, c := range []struct { - name string - forcedS int // the register holds 187 / 3 kW until here - orderW float64 // FTW's dispatch; 0: none - down bool // Core, not the inverter, is away from 10 to 40 s - want bool - }{ - {"8 s after the inverter is back", 48, 0, false, true}, - {"30 s after the inverter is back", 70, 0, false, false}, - {"FTW dispatches the discharge", 120, -3000, false, true}, - {"8 s after Core is back", 48, 0, true, true}, - {"30 s after Core is back", 70, 0, true, false}, + for _, c := range []forcedCase{ + // 207533: offline 576.6-644.7 s, Core reads 9 s later, 187 / + // 2668 W until FTW dispatches again 236 s after the return (C1). + {name: "207533: held until Core's 5 min retry", outage: "offline", outFrom: 100, outFor: 68, readAt: 177, + cmd: 187, setW: 2668, forcedTo: 1201, dispatchFrom: 404, want: false}, + // 207535: the same with a forced charge, 170 / 2632 W, for 235 s. + {name: "207535: a forced charge held", outage: "offline", outFrom: 100, outFor: 69, readAt: 177, + cmd: 170, setW: 2632, forcedTo: 404, want: false}, + // 207537: the inverter runs but does not answer for 46.4 s; Core + // reads 19.8 s after it answers again, the discharge holds 264 s. + {name: "207537: held after a silence", outage: "silence", outFrom: 100, outFor: 46, readAt: 166, + cmd: 187, setW: 3854, forcedTo: 1201, dispatchFrom: 410, want: false}, + // 207538: Core reads 2 s after the return; held to the run's end. + {name: "207538: held to the end", outage: "offline", outFrom: 100, outFor: 39, readAt: 141, + cmd: 187, setW: 4674, forcedTo: 1201, want: false}, + // The same outages once C1 is fixed: the default goes out within + // 10 s of Core's first good read. + {name: "default 8 s after Core reads again", outage: "offline", outFrom: 100, outFor: 68, readAt: 177, + cmd: 187, setW: 2668, forcedTo: 185, want: true}, + // 207539: offline 508.5-590.0 s, the Modbus backoff holds Core's + // first good read back 55.7 s, and FTW dispatches 20 s after it. + // Inside Core's timers; it fails once #1915 (C11) lands. + {name: "207539: Core's reconnect backoff and default retry", outage: "offline", outFrom: 100, outFor: 82, readAt: 238, + cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: 258, want: true}, + {name: "past the registry's 30 s retry", outage: "offline", outFrom: 100, outFor: 82, readAt: 238, + cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: 285, want: false}, + // Core never reads the inverter again: the count starts 60 s after + // the return all the same. + {name: "Core never reads it again", outage: "offline", outFrom: 100, outFor: 82, readAt: 1201, + cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: 290, want: false}, + {name: "FTW dispatches the discharge", cmd: 187, setW: 3000, forcedTo: 1201, dispatchFrom: 1, want: true}, + {name: "forced with no dispatch and no outage", cmd: 187, setW: 3000, forcedTo: 1201, dispatchFrom: 500, want: false}, + // Item 10, 207538's restart at 625.5 s: SIGTERM, the driver's + // cleanup writes 204 within a second, Core is back 14.6 s later + // and dispatches 6 s after that. + {name: "207538: SIGTERM restart writes the default", stop: 100, up: 115, cmd: 187, setW: 868, + forcedTo: 101, dispatchFrom: 121, want: true}, + // The same stop as SIGKILL: no cleanup, the discharge runs on + // while Core is down. + {name: "restart without cleanup", stop: 100, up: 115, cmd: 187, setW: 868, + forcedTo: 121, dispatchFrom: 121, want: false}, + // 207530: Core stops while the inverter is offline (888.7-916.4 + // s), so no cleanup can land. The inverter is back 2.8 s before + // Core (here 13 s), and the default lands as Core starts. + {name: "207530: restart while the inverter is offline", outage: "offline", outFrom: 90, outFor: 12, readAt: 116, + stop: 100, up: 115, cmd: 170, setW: 2983, forcedTo: 116, dispatchFrom: 125, want: true}, + // A Core that starts fresh has no default retry to wait for: 10 s + // from its first good read, not the 40 s an outage gets. + {name: "forced 8 s after Core starts", stop: 100, up: 101, cmd: 187, setW: 868, + forcedTo: 1201, dispatchFrom: 110, want: true}, + {name: "forced 23 s after Core starts", stop: 100, up: 101, cmd: 187, setW: 868, + forcedTo: 1201, dispatchFrom: 125, want: false}, } { - d := randomRun(121, 16, 5, func(i int, r *sample) { - r.Status = &ftwStatus{} - if c.orderW != 0 { - r.Status.Dispatch = []ftwDispatch{{Driver: "sungrow", TargetW: c.orderW}} - } - if i < c.forcedS { - r.Bat.EMSMode, r.Bat.EMSCmd, r.Bat.EMSSetW = 2, 187, 3000 - } - }) - if c.down { - d.allMarks = []markLine{{T: 10, Label: "ftw:restart"}, {T: 40, Label: "ftw:up"}} - } else { - d.events = []houseEventLine{{T: 10, Event: houseEvent{Do: "offline", Device: "inverter", ForS: 30}}} - } - if got := judged(d)[forced]; got != c.want { + if got := judged(c.run())[forced]; got != c.want { t.Errorf("%s: %q = %v, want %v", c.name, forced, got, c.want) } } From 8e2719ac9068dd5384bfeabb75b601d8c4e44228 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Tue, 27 Oct 2026 20:56:26 +0100 Subject: [PATCH 2/3] test(site): R8 records the seconds Core's timers excuse Review of #1922. The judge forgives the Modbus reconnect backoff (up to 60 s) and the registry's default retry (up to 30 s) after a return, while the triage line is 10 s from the return. result.json now records those seconds as battery_forced_in_core_timers_s, so a pass such as 207539's (123 s) shows what was forgiven. TestForcedAllowancesMatchCore reads both timers from Core's source and fails once the registry retries the default on the first good poll (#1915, C11) while defaultRetryMaxS is still 30. Two R8 cases pin the judge's clock: an inverter that is back while Core is down counts from Core's first read, and a second outage counts from its own return. Co-Authored-By: Claude Opus 5.5 --- go/test/site/random_test.go | 27 +++++--- go/test/site/random_unit_test.go | 113 +++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 10 deletions(-) diff --git a/go/test/site/random_test.go b/go/test/site/random_test.go index 1f07cfa98..757ef01c5 100644 --- a/go/test/site/random_test.go +++ b/go/test/site/random_test.go @@ -225,15 +225,15 @@ func within(ws []eventWindow, t, afterS float64) bool { return false } -// inverterBackFrom gives, for a time t, when the R8 count may start after -// the latest return before t: the inverter's from an outage (inverterOut) -// or Core's from a restart (down). Core has the inverter back at its first +// inverterBackFrom gives, for a time t, the latest return before t (at): +// the inverter's from an outage (inverterOut) or Core's from a restart +// (down), and when the R8 count may start after it (from). Core has the inverter back at its first // good read of it (no consecutive errors), which the Modbus reconnect // backoff may hold back up to reconnectBackoffMaxS. After an outage, a // default that failed while the inverter was away waits on the registry's // retry timer, up to defaultRetryMaxS more, until #1915 (C11) sends it on // the first good poll. A Core that starts fresh has no such wait. -func (d *runData) inverterBackFrom(inverterOut, down []eventWindow) func(t float64) float64 { +func (d *runData) inverterBackFrom(inverterOut, down []eventWindow) func(t float64) (at, from float64) { type back struct { at, from float64 } @@ -257,14 +257,14 @@ func (d *runData) inverterBackFrom(inverterOut, down []eventWindow) func(t float for _, w := range down { add(w.to, false) } - return func(t float64) float64 { - at, from := math.Inf(-1), math.Inf(-1) + return func(t float64) (at, from float64) { + at, from = math.Inf(-1), math.Inf(-1) for _, b := range backs { if b.at <= t && b.at >= at { at, from = b.at, b.from } } - return from + return at, from } } @@ -750,9 +750,11 @@ func judgeInvariants(d *runData) (metrics, []criterion) { // restart the count starts once Core has the inverter back (backFrom). // While Core is stopped with SIGTERM its driver cleanup writes the // default, so a running inverter gets the same 10 s from the stop. One - // that comes back while Core is down has its outage's allowance. + // that comes back while Core is down has its outage's allowance. The + // triage's line is 10 s from the return itself; the seconds past that + // line that Core's timers excuse are recorded, not hidden. backFrom := d.inverterBackFrom(inverterOut, down) - forcedS, pastS, run8, from8 := 0.0, 0.0, 0.0, 0.0 + forcedS, pastS, timersS, run8, from8 := 0.0, 0.0, 0.0, 0.0, 0.0 for i, r := range run { w, ok := dispatchW(r) stray := math.Abs(batOrder(r)) > 50 && !within(inverterOut, r.T, 0) @@ -768,13 +770,18 @@ func judgeInvariants(d *runData) (metrics, []criterion) { } run8 += r.T - run[i-1].T forcedS = math.Max(forcedS, run8) - if r.T > math.Max(from8, backFrom(r.T))+forcedBudgetS { + at, from := backFrom(r.T) + switch { + case r.T > math.Max(from8, from)+forcedBudgetS: pastS += r.T - run[i-1].T note("forced battery", r.T) + case r.T > math.Max(from8, at)+forcedBudgetS: + timersS += r.T - run[i-1].T } } m["battery_forced_without_dispatch_s"] = round(forcedS, 1) m["battery_forced_past_allowance_s"] = round(pastS, 1) + m["battery_forced_in_core_timers_s"] = round(timersS, 1) // The first second an invariant broke, from the start, per invariant: // a kept seed is cut a minute after the one it failed on. firsts := map[string]float64{} diff --git a/go/test/site/random_unit_test.go b/go/test/site/random_unit_test.go index 1dfee8ea0..f06d4ef96 100644 --- a/go/test/site/random_unit_test.go +++ b/go/test/site/random_unit_test.go @@ -7,6 +7,9 @@ import ( "fmt" "os" "path/filepath" + "regexp" + "strconv" + "strings" "testing" ) @@ -474,6 +477,12 @@ func TestInvariantsNoForcedBatteryWithoutDispatch(t *testing.T) { forcedTo: 1201, dispatchFrom: 110, want: true}, {name: "forced 23 s after Core starts", stop: 100, up: 101, cmd: 187, setW: 868, forcedTo: 1201, dispatchFrom: 125, want: false}, + // The inverter is back at 102 while Core, stopped at 100 during the + // outage, stays down until 150. Core's first read is at 151 and the + // default lands 7 s later. While Core is down nothing reads the + // inverter, so the count cannot start before 151. + {name: "inverter back while Core is down for 48 s", outage: "offline", outFrom: 90, outFor: 12, readAt: 151, + stop: 100, up: 150, cmd: 187, setW: 3000, forcedTo: 158, dispatchFrom: 170, want: true}, } { if got := judged(c.run())[forced]; got != c.want { t.Errorf("%s: %q = %v, want %v", c.name, forced, got, c.want) @@ -481,6 +490,110 @@ func TestInvariantsNoForcedBatteryWithoutDispatch(t *testing.T) { } } +// R8 with two inverter outages: the count after the second starts at its +// own return and Core's first read after it, not the first outage's. +func TestInvariantsForcedBatteryCountsFromTheLatestReturn(t *testing.T) { + const forced = "no forced battery mode FTW does not dispatch" + // Outage 1 100-120, Core reads at 125 and dispatches 130-300. Outage 2 + // 300-340, Core reads at 380 and the default lands at 388. + run := func(forcedTo float64) *runData { + d := randomRun(601, 16, 5, func(i int, r *sample) { + tt := float64(i) + if tt < forcedTo { + r.Bat.EMSMode, r.Bat.EMSCmd, r.Bat.EMSSetW = 2, 187, 3000 + } else { + r.Bat.EMSCmd = 204 + } + if (tt >= 100 && tt < 120) || (tt >= 300 && tt < 340) { + off := false + r.Bat.Running = &off + } + errs := 0 + if (tt >= 100 && tt < 125) || (tt >= 300 && tt < 380) { + errs = 3 + } + r.Status = &ftwStatus{Drivers: map[string]ftwDriver{inverterDriver: {Status: "ok", ConsecutiveErrors: &errs}}} + if tt < 100 || (tt >= 130 && tt < 300) { + r.Status.Dispatch = []ftwDispatch{{Driver: inverterDriver, TargetW: -3000}} + } + }) + d.events = []houseEventLine{ + {T: 100, Event: houseEvent{Do: "offline", Device: "inverter", ForS: 20}}, + {T: 300, Event: houseEvent{Do: "offline", Device: "inverter", ForS: 40}}, + } + return d + } + if !judged(run(388))[forced] { + t.Errorf("default 8 s after Core reads again after the second outage: got fail, want pass") + } + if judged(run(1201))[forced] { + t.Errorf("forced to the end after the second outage: got pass, want fail") + } +} + +// R8 forgives the seconds Core's reconnect backoff and default retry hold +// its first write after a return (reconnectBackoffMaxS, defaultRetryMaxS). +// The triage line is 10 s from the return, so the judge records what it +// forgave beyond that line instead of hiding it. +func TestInvariantsRecordForcedSecondsInsideCoreTimers(t *testing.T) { + // 207539's shape: back at 182, Core reads at 238 and dispatches at 245, + // inside the backoff alone. Forced 183-244 with no dispatch; past + // 182+10 that is 193-244. + c := forcedCase{outage: "offline", outFrom: 100, outFor: 82, readAt: 238, + cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: 245} + m, _ := judgeInvariants(c.run()) + if in, past := m["battery_forced_in_core_timers_s"], m["battery_forced_past_allowance_s"]; in != 52.0 || past != 0.0 { + t.Errorf("in Core's timers %v s, past the allowance %v s; want 52 and 0", in, past) + } + // Forced to 299: from 193 to the allowance's end inside the timers, + // the rest past them. + c.dispatchFrom = 300 + m, _ = judgeInvariants(c.run()) + end := 238 + defaultRetryMaxS + forcedBudgetS + if in, past := m["battery_forced_in_core_timers_s"], m["battery_forced_past_allowance_s"]; in != end-192 || past != 299-end { + t.Errorf("in Core's timers %v s, past the allowance %v s; want %v and %v", in, past, end-192, 299-end) + } +} + +// R8's allowances are Core's own timers. Read them from Core's source so +// they cannot drift. Once the registry retries a failed default on the +// first good poll after an outage (#1915, C11), defaultRetryMaxS must be +// 0: from then on the 30 s the judge gives today is no longer Core's. +func TestForcedAllowancesMatchCore(t *testing.T) { + src := func(p string) string { + b, err := os.ReadFile(filepath.Join("..", "..", "internal", p)) + if err != nil { + t.Fatal(err) + } + return string(b) + } + seconds := func(code, name string) float64 { + m := regexp.MustCompile(name + `\s*=\s*(\d+)\s*\*\s*time\.Second`).FindStringSubmatch(code) + if m == nil { + t.Fatalf("no %s = N * time.Second in Core's source", name) + } + n, _ := strconv.Atoi(m[1]) + return float64(n) + } + if got := seconds(src("modbus/client.go"), "reconnectBackoffMax"); got != reconnectBackoffMaxS { + t.Errorf("reconnectBackoffMaxS = %v, Core's reconnectBackoffMax is %v s", reconnectBackoffMaxS, got) + } + registry := src("drivers/registry.go") + // Today the registry sends the default on a failure and on its retry + // timer: two calls. #1915 adds a third, on the first good poll. + calls := strings.Count(registry, "attemptDefault()") + switch { + case calls == 2 && defaultRetryMaxS != seconds(registry, "defaultRetryMax"): + t.Errorf("defaultRetryMaxS = %v, Core's defaultRetryMax is %v s", defaultRetryMaxS, seconds(registry, "defaultRetryMax")) + case calls > 2 && defaultRetryMaxS != 0: + t.Errorf("Core's registry calls attemptDefault() in %d places, not 2. If it now retries "+ + "the default on the first good poll after an outage (#1915, C11), set defaultRetryMaxS "+ + "to 0 and turn the R8 cases that lean on it to fail. Otherwise update this check.", calls) + case calls < 2: + t.Errorf("Core's registry calls attemptDefault() in %d places; update this check", calls) + } +} + // TestJudge also judges the random seeds a results directory holds, from // the plan each run wrote. func TestRecordedRandomFindsSeedRuns(t *testing.T) { From 04151611f43a60fda2c241936639e29f258c21fd Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Wed, 28 Oct 2026 09:29:24 +0100 Subject: [PATCH 3/3] test(site): R8 reads Core's reconnect backoff from source The judge's reconnectBackoffMaxS was a fixed 60 s, checked against modbus/client.go. #1930 cuts Core's cap to 10 s, so the check would fail whichever of the two lands second. The judge now reads the cap from the source the rig builds Core from, and the 207539 shapes are set relative to it. The unit tests pass with 60 s and with #1930's 10 s applied. Co-Authored-By: Claude Opus 5.5 --- go/test/site/random_test.go | 30 ++++++++++++++++++++--- go/test/site/random_unit_test.go | 42 ++++++++++++++++++-------------- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/go/test/site/random_test.go b/go/test/site/random_test.go index d30d9979e..6c0e9c3af 100644 --- a/go/test/site/random_test.go +++ b/go/test/site/random_test.go @@ -9,7 +9,9 @@ import ( "math/rand" "os" "path/filepath" + "regexp" "sort" + "strconv" "strings" "syscall" "time" @@ -310,12 +312,32 @@ const ( // R8: how long a battery FTW does not dispatch may hold a forced mode, // and the Core timers that may delay it after the inverter returns. - inverterDriver = "sungrow" // the rig's inverter driver (stack_test.go) - forcedBudgetS = 10.0 - reconnectBackoffMaxS = 60.0 // modbus/client.go reconnectBackoffMax - defaultRetryMaxS = 30.0 // drivers/registry.go defaultRetryMax; 0 once #1915 (C11) lands + inverterDriver = "sungrow" // the rig's inverter driver (stack_test.go) + forcedBudgetS = 10.0 + defaultRetryMaxS = 30.0 // drivers/registry.go defaultRetryMax; 0 once #1915 (C11) lands ) +// reconnectBackoffMaxS is Core's Modbus reconnect backoff cap +// (modbus/client.go reconnectBackoffMax), read from the source the rig +// builds Core from, so R8 follows Core when the cap changes (60 s, then +// 10 s with #1930). +var reconnectBackoffMaxS = coreSeconds("modbus/client.go", "reconnectBackoffMax") + +// coreSeconds reads "name = N * time.Second" from a file under go/internal. +// It panics when the line is gone: the judge has no number to fall back on. +func coreSeconds(file, name string) float64 { + b, err := os.ReadFile(filepath.Join("..", "..", "internal", file)) + if err != nil { + panic(err) + } + m := regexp.MustCompile(name + `\s*=\s*(\d+)\s*\*\s*time\.Second`).FindSubmatch(b) + if m == nil { + panic(fmt.Sprintf("no %s = N * time.Second in Core's %s", name, file)) + } + n, _ := strconv.Atoi(string(m[1])) + return float64(n) +} + // dispatchW is Core's battery order on r, + charge; false when Core // dispatched no battery. func dispatchW(r *sample) (float64, bool) { diff --git a/go/test/site/random_unit_test.go b/go/test/site/random_unit_test.go index 57a97b28e..1c7b8c36b 100644 --- a/go/test/site/random_unit_test.go +++ b/go/test/site/random_unit_test.go @@ -445,6 +445,7 @@ func (c forcedCase) run() *runData { // and Core's first good read is Core's to take. func TestInvariantsNoForcedBatteryWithoutDispatch(t *testing.T) { const forced = "no forced battery mode FTW does not dispatch" + read207539 := 182 + reconnectBackoffMaxS - 4 for _, c := range []forcedCase{ // 207533: offline 576.6-644.7 s, Core reads 9 s later, 187 / // 2668 W until FTW dispatches again 236 s after the return (C1). @@ -464,15 +465,16 @@ func TestInvariantsNoForcedBatteryWithoutDispatch(t *testing.T) { // 10 s of Core's first good read. {name: "default 8 s after Core reads again", outage: "offline", outFrom: 100, outFor: 68, readAt: 177, cmd: 187, setW: 2668, forcedTo: 185, want: true}, - // 207539: offline 508.5-590.0 s, the Modbus backoff holds Core's - // first good read back 55.7 s, and FTW dispatches 20 s after it. + // 207539: offline 508.5-590.0 s, the Modbus backoff (60 s then) + // holds Core's first good read back 55.7 s, and FTW dispatches 20 s + // after it. Here the read comes 4 s before the backoff's cap ends. // Inside Core's timers; it fails once #1915 (C11) lands. - {name: "207539: Core's reconnect backoff and default retry", outage: "offline", outFrom: 100, outFor: 82, readAt: 238, - cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: 258, want: true}, - {name: "past the registry's 30 s retry", outage: "offline", outFrom: 100, outFor: 82, readAt: 238, - cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: 285, want: false}, - // Core never reads the inverter again: the count starts 60 s after - // the return all the same. + {name: "207539: Core's reconnect backoff and default retry", outage: "offline", outFrom: 100, outFor: 82, readAt: read207539, + cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: read207539 + 20, want: true}, + {name: "past the registry's 30 s retry", outage: "offline", outFrom: 100, outFor: 82, readAt: read207539, + cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: read207539 + 47, want: false}, + // Core never reads the inverter again: the count starts at the + // backoff's cap after the return all the same. {name: "Core never reads it again", outage: "offline", outFrom: 100, outFor: 82, readAt: 1201, cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: 290, want: false}, {name: "FTW dispatches the discharge", cmd: 187, setW: 3000, forcedTo: 1201, dispatchFrom: 1, want: true}, @@ -556,20 +558,22 @@ func TestInvariantsForcedBatteryCountsFromTheLatestReturn(t *testing.T) { // The triage line is 10 s from the return, so the judge records what it // forgave beyond that line instead of hiding it. func TestInvariantsRecordForcedSecondsInsideCoreTimers(t *testing.T) { - // 207539's shape: back at 182, Core reads at 238 and dispatches at 245, - // inside the backoff alone. Forced 183-244 with no dispatch; past - // 182+10 that is 193-244. - c := forcedCase{outage: "offline", outFrom: 100, outFor: 82, readAt: 238, - cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: 245} + // 207539's shape: back at 182, Core reads 4 s before the backoff's cap + // ends (238 with 60 s) and dispatches 7 s later, inside the backoff + // alone. Forced with no dispatch until then; past 182+10 that is from + // 193 (52 s with 60 s). + read := 182 + reconnectBackoffMaxS - 4 + c := forcedCase{outage: "offline", outFrom: 100, outFor: 82, readAt: read, + cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: read + 7} m, _ := judgeInvariants(c.run()) - if in, past := m["battery_forced_in_core_timers_s"], m["battery_forced_past_allowance_s"]; in != 52.0 || past != 0.0 { - t.Errorf("in Core's timers %v s, past the allowance %v s; want 52 and 0", in, past) + if in, past, want := m["battery_forced_in_core_timers_s"], m["battery_forced_past_allowance_s"], read+6-192; in != want || past != 0.0 { + t.Errorf("in Core's timers %v s, past the allowance %v s; want %v and 0", in, past, want) } // Forced to 299: from 193 to the allowance's end inside the timers, // the rest past them. c.dispatchFrom = 300 m, _ = judgeInvariants(c.run()) - end := 238 + defaultRetryMaxS + forcedBudgetS + end := read + defaultRetryMaxS + forcedBudgetS if in, past := m["battery_forced_in_core_timers_s"], m["battery_forced_past_allowance_s"]; in != end-192 || past != 299-end { t.Errorf("in Core's timers %v s, past the allowance %v s; want %v and %v", in, past, end-192, 299-end) } @@ -595,8 +599,10 @@ func TestForcedAllowancesMatchCore(t *testing.T) { n, _ := strconv.Atoi(m[1]) return float64(n) } - if got := seconds(src("modbus/client.go"), "reconnectBackoffMax"); got != reconnectBackoffMaxS { - t.Errorf("reconnectBackoffMaxS = %v, Core's reconnectBackoffMax is %v s", reconnectBackoffMaxS, got) + // reconnectBackoffMaxS is read from Core's source; the 207539 shapes + // below need a backoff that can hold Core's first read back past 4 s. + if reconnectBackoffMaxS < 5 { + t.Errorf("Core's reconnectBackoffMax is %v s; rework the 207539 shapes", reconnectBackoffMaxS) } registry := src("drivers/registry.go") // Today the registry sends the default on a failure and on its retry