diff --git a/go/test/site/evaluate_test.go b/go/test/site/evaluate_test.go index 70a65fb06..7a2f93145 100644 --- a/go/test/site/evaluate_test.go +++ b/go/test/site/evaluate_test.go @@ -70,8 +70,9 @@ type ftwDispatch struct { } type ftwDriver struct { - Status string `json:"status"` - EVSessionWh *float64 `json:"ev_session_wh"` // a charger's session energy + Status string `json:"status"` + EVSessionWh *float64 `json:"ev_session_wh"` // a charger's session energy + ConsecutiveErrors *int `json:"consecutive_errors"` // failed polls in a row; 0 once a poll reads } type ftwLoadpoint struct { diff --git a/go/test/site/random_test.go b/go/test/site/random_test.go index e2855ab73..062214b29 100644 --- a/go/test/site/random_test.go +++ b/go/test/site/random_test.go @@ -9,7 +9,9 @@ import ( "math/rand" "os" "path/filepath" + "regexp" "sort" + "strconv" "strings" "syscall" "time" @@ -225,6 +227,60 @@ func within(ws []eventWindow, t, afterS float64) bool { return false } +// inverterBackFrom gives, for a time t, the latest return before t (at): +// the inverter's from an outage (inverterOut) or Core's from a restart +// (down), and when the R8 count may start after it (from). Core has the inverter back at its first +// good read of it (no consecutive errors), which the Modbus reconnect +// backoff may hold back up to reconnectBackoffMaxS. After an outage, a +// default that failed while the inverter was away waits on the registry's +// retry timer, up to defaultRetryMaxS more, until #1915 (C11) sends it on +// the first good poll. A Core that starts fresh has no such wait. +func (d *runData) inverterBackFrom(inverterOut, down []eventWindow) func(t float64) (at, from float64) { + type back struct { + at, from float64 + } + var backs []back + add := func(at float64, outage bool) { + from := at + reconnectBackoffMaxS + for _, r := range d.samples { + if r.T >= at && r.T < from && readsInverter(r) { + from = r.T + break + } + } + if outage { + from += defaultRetryMaxS + } + backs = append(backs, back{at, from}) + } + for _, w := range inverterOut { + add(w.to, true) + } + for _, w := range down { + add(w.to, false) + } + return func(t float64) (at, from float64) { + at, from = math.Inf(-1), math.Inf(-1) + for _, b := range backs { + if b.at <= t && b.at >= at { + at, from = b.at, b.from + } + } + return at, from + } +} + +// readsInverter is whether Core's last poll of the inverter succeeded. A +// status without the driver's error count says nothing, and counts as a +// read. +func readsInverter(r *sample) bool { + if r.Status == nil { + return false + } + dr, ok := r.Status.Drivers[inverterDriver] + return !ok || dr.ConsecutiveErrors == nil || *dr.ConsecutiveErrors == 0 +} + // during is whether t falls in one of ws, up to but not at its end: at its // end the device is back. func during(ws []eventWindow, t float64) bool { @@ -253,8 +309,35 @@ const ( evRestartDwellS = 35.0 // control/ev_fuse_budget.go evRestartDwell tickS = 2.0 // the rig's site.control_interval_s plannerFloorPct = 10.0 // Core's planner.soc_min when the config leaves it out (cmd/ftw/main.go) + + // R8: how long a battery FTW does not dispatch may hold a forced mode, + // and the Core timers that may delay it after the inverter returns. + inverterDriver = "sungrow" // the rig's inverter driver (stack_test.go) + forcedBudgetS = 10.0 + defaultRetryMaxS = 30.0 // drivers/registry.go defaultRetryMax; 0 once #1915 (C11) lands ) +// reconnectBackoffMaxS is Core's Modbus reconnect backoff cap +// (modbus/client.go reconnectBackoffMax), read from the source the rig +// builds Core from, so R8 follows Core when the cap changes (60 s, then +// 10 s with #1930). +var reconnectBackoffMaxS = coreSeconds("modbus/client.go", "reconnectBackoffMax") + +// coreSeconds reads "name = N * time.Second" from a file under go/internal. +// It panics when the line is gone: the judge has no number to fall back on. +func coreSeconds(file, name string) float64 { + b, err := os.ReadFile(filepath.Join("..", "..", "internal", file)) + if err != nil { + panic(err) + } + m := regexp.MustCompile(name + `\s*=\s*(\d+)\s*\*\s*time\.Second`).FindSubmatch(b) + if m == nil { + panic(fmt.Sprintf("no %s = N * time.Second in Core's %s", name, file)) + } + n, _ := strconv.Atoi(string(m[1])) + return float64(n) +} + // dispatchW is Core's battery order on r, + charge; false when Core // dispatched no battery. func dispatchW(r *sample) (float64, bool) { @@ -813,26 +896,44 @@ func judgeInvariants(d *runData) (metrics, []criterion) { } m["raises_on_a_stale_meter"] = raises - // 8: a battery FTW does not dispatch is back in its own mode within - // 10 s: no forced charge or discharge left in the inverter's register - // once the inverter and Core are both up again (R8, C1). The 10 s - // count starts when both are back. - forcedS, run8 := 0.0, 0.0 + // 8: a battery FTW does not dispatch runs in its own mode (R8, C1). The + // inverter's register may hold a forced charge or discharge FTW does + // not dispatch for forcedBudgetS. After an inverter outage or a Core + // restart the count starts once Core has the inverter back (backFrom). + // While Core is stopped with SIGTERM its driver cleanup writes the + // default, so a running inverter gets the same 10 s from the stop. One + // that comes back while Core is down has its outage's allowance. The + // triage's line is 10 s from the return itself; the seconds past that + // line that Core's timers excuse are recorded, not hidden. + backFrom := d.inverterBackFrom(inverterOut, down) + forcedS, pastS, timersS, run8, from8 := 0.0, 0.0, 0.0, 0.0, 0.0 for i, r := range run { w, ok := dispatchW(r) - stray := r.Status != nil && (!ok || math.Abs(w) <= 50) && math.Abs(batOrder(r)) > 50 && - !within(inverterOut, r.T, 0) && !within(down, r.T, 0) - if stray && i > 0 { - run8 += r.T - run[i-1].T - forcedS = math.Max(forcedS, run8) - if run8 > 10 { - note("forced battery", r.T) - } - } else { + stray := math.Abs(batOrder(r)) > 50 && !within(inverterOut, r.T, 0) + if !within(down, r.T, 0) { + stray = stray && r.Status != nil && (!ok || math.Abs(w) <= 50) + } + if !stray || i == 0 { run8 = 0 + continue + } + if run8 == 0 { + from8 = run[i-1].T + } + run8 += r.T - run[i-1].T + forcedS = math.Max(forcedS, run8) + at, from := backFrom(r.T) + switch { + case r.T > math.Max(from8, from)+forcedBudgetS: + pastS += r.T - run[i-1].T + note("forced battery", r.T) + case r.T > math.Max(from8, at)+forcedBudgetS: + timersS += r.T - run[i-1].T } } m["battery_forced_without_dispatch_s"] = round(forcedS, 1) + m["battery_forced_past_allowance_s"] = round(pastS, 1) + m["battery_forced_in_core_timers_s"] = round(timersS, 1) // The first second an invariant broke, from the start, per invariant: // a kept seed is cut a minute after the one it failed on. firsts := map[string]float64{} @@ -849,7 +950,7 @@ func judgeInvariants(d *runData) (metrics, []criterion) { crit("no offer above the charger's rating", over == 0), crit("no discharge asked at the battery's floor", floorS <= 10), crit("no raise on a stale meter", raises == 0), - crit("no forced battery mode FTW does not dispatch", forcedS <= 10), + crit("no forced battery mode FTW does not dispatch", pastS == 0), } } diff --git a/go/test/site/random_unit_test.go b/go/test/site/random_unit_test.go index 70b0766ef..116d54032 100644 --- a/go/test/site/random_unit_test.go +++ b/go/test/site/random_unit_test.go @@ -7,6 +7,9 @@ import ( "fmt" "os" "path/filepath" + "regexp" + "strconv" + "strings" "testing" ) @@ -379,40 +382,241 @@ func TestInvariantsLeverAndFaults(t *testing.T) { } } -// R8, C1: once the inverter and Core are back, a battery FTW does not -// dispatch leaves its forced mode within 10 s. +// forcedCase is an R8 run: the inverter's register holds a forced command +// from the start until forcedTo, and FTW dispatches the battery from +// dispatchFrom (0: never). An outage of the inverter (offline: it stops; a +// silence: it runs but does not answer) or a Core restart may come first. +type forcedCase struct { + name string + outage string // "offline", "silence" or "" + outFrom float64 // the outage, from outFrom + outFor float64 // for outFor + readAt float64 // Core's polls of the inverter fail from outFrom until here + stop, up float64 // a Core restart (SIGTERM at stop), if up > 0 + cmd float64 // 187 discharge, 170 charge + setW float64 + forcedTo float64 + dispatchFrom float64 + want bool +} + +func (c forcedCase) run() *runData { + // FTW dispatched the battery until the outage or the stop. + lead := c.outFrom + if c.up > 0 && (c.outage == "" || c.stop < lead) { + lead = c.stop + } + d := randomRun(1201, 16, 5, func(i int, r *sample) { + t := float64(i) + if t < c.forcedTo { + r.Bat.EMSMode, r.Bat.EMSCmd, r.Bat.EMSSetW = 2, c.cmd, c.setW + } else { + r.Bat.EMSCmd = 204 + } + if c.outage == "offline" && t >= c.outFrom && t < c.outFrom+c.outFor { + off := false + r.Bat.Running = &off + } + if c.up > 0 && t >= c.stop && t <= c.up { + return // Core is down: no reading + } + errs := 0 + if c.outage != "" && t >= c.outFrom && t < c.readAt { + errs = 1 + int(t-c.outFrom)/2 + } + r.Status = &ftwStatus{Drivers: map[string]ftwDriver{inverterDriver: {Status: "ok", ConsecutiveErrors: &errs}}} + if t < lead || (c.dispatchFrom > 0 && t >= c.dispatchFrom) { + r.Status.Dispatch = []ftwDispatch{{Driver: inverterDriver, TargetW: -c.setW}} + } + }) + if c.outage != "" { + d.events = []houseEventLine{{T: c.outFrom, Event: houseEvent{Do: c.outage, Device: "inverter", ForS: c.outFor}}} + } + if c.up > 0 { + d.allMarks = []markLine{{T: c.stop, Label: "ftw:restart"}, {T: c.up, Label: "ftw:up"}} + } + return d +} + +// R8, C1, on the shapes of the nightly seeds on master 83c2b24c. Once Core +// has the inverter back, a battery FTW does not dispatch leaves its forced +// mode within 10 s, plus the registry's 30 s default retry until #1915 +// (C11) lands. The Modbus reconnect backoff between the inverter's return +// and Core's first good read is Core's to take. func TestInvariantsNoForcedBatteryWithoutDispatch(t *testing.T) { const forced = "no forced battery mode FTW does not dispatch" - for _, c := range []struct { - name string - forcedS int // the register holds 187 / 3 kW until here - orderW float64 // FTW's dispatch; 0: none - down bool // Core, not the inverter, is away from 10 to 40 s - want bool - }{ - {"8 s after the inverter is back", 48, 0, false, true}, - {"30 s after the inverter is back", 70, 0, false, false}, - {"FTW dispatches the discharge", 120, -3000, false, true}, - {"8 s after Core is back", 48, 0, true, true}, - {"30 s after Core is back", 70, 0, true, false}, + read207539 := 182 + reconnectBackoffMaxS - 4 + for _, c := range []forcedCase{ + // 207533: offline 576.6-644.7 s, Core reads 9 s later, 187 / + // 2668 W until FTW dispatches again 236 s after the return (C1). + {name: "207533: held until Core's 5 min retry", outage: "offline", outFrom: 100, outFor: 68, readAt: 177, + cmd: 187, setW: 2668, forcedTo: 1201, dispatchFrom: 404, want: false}, + // 207535: the same with a forced charge, 170 / 2632 W, for 235 s. + {name: "207535: a forced charge held", outage: "offline", outFrom: 100, outFor: 69, readAt: 177, + cmd: 170, setW: 2632, forcedTo: 404, want: false}, + // 207537: the inverter runs but does not answer for 46.4 s; Core + // reads 19.8 s after it answers again, the discharge holds 264 s. + {name: "207537: held after a silence", outage: "silence", outFrom: 100, outFor: 46, readAt: 166, + cmd: 187, setW: 3854, forcedTo: 1201, dispatchFrom: 410, want: false}, + // 207538: Core reads 2 s after the return; held to the run's end. + {name: "207538: held to the end", outage: "offline", outFrom: 100, outFor: 39, readAt: 141, + cmd: 187, setW: 4674, forcedTo: 1201, want: false}, + // The same outages once C1 is fixed: the default goes out within + // 10 s of Core's first good read. + {name: "default 8 s after Core reads again", outage: "offline", outFrom: 100, outFor: 68, readAt: 177, + cmd: 187, setW: 2668, forcedTo: 185, want: true}, + // 207539: offline 508.5-590.0 s, the Modbus backoff (60 s then) + // holds Core's first good read back 55.7 s, and FTW dispatches 20 s + // after it. Here the read comes 4 s before the backoff's cap ends. + // Inside Core's timers; it fails once #1915 (C11) lands. + {name: "207539: Core's reconnect backoff and default retry", outage: "offline", outFrom: 100, outFor: 82, readAt: read207539, + cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: read207539 + 20, want: true}, + {name: "past the registry's 30 s retry", outage: "offline", outFrom: 100, outFor: 82, readAt: read207539, + cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: read207539 + 47, want: false}, + // Core never reads the inverter again: the count starts at the + // backoff's cap after the return all the same. + {name: "Core never reads it again", outage: "offline", outFrom: 100, outFor: 82, readAt: 1201, + cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: 290, want: false}, + {name: "FTW dispatches the discharge", cmd: 187, setW: 3000, forcedTo: 1201, dispatchFrom: 1, want: true}, + {name: "forced with no dispatch and no outage", cmd: 187, setW: 3000, forcedTo: 1201, dispatchFrom: 500, want: false}, + // Item 10, 207538's restart at 625.5 s: SIGTERM, the driver's + // cleanup writes 204 within a second, Core is back 14.6 s later + // and dispatches 6 s after that. + {name: "207538: SIGTERM restart writes the default", stop: 100, up: 115, cmd: 187, setW: 868, + forcedTo: 101, dispatchFrom: 121, want: true}, + // The same stop as SIGKILL: no cleanup, the discharge runs on + // while Core is down. + {name: "restart without cleanup", stop: 100, up: 115, cmd: 187, setW: 868, + forcedTo: 121, dispatchFrom: 121, want: false}, + // 207530: Core stops while the inverter is offline (888.7-916.4 + // s), so no cleanup can land. The inverter is back 2.8 s before + // Core (here 13 s), and the default lands as Core starts. + {name: "207530: restart while the inverter is offline", outage: "offline", outFrom: 90, outFor: 12, readAt: 116, + stop: 100, up: 115, cmd: 170, setW: 2983, forcedTo: 116, dispatchFrom: 125, want: true}, + // A Core that starts fresh has no default retry to wait for: 10 s + // from its first good read, not the 40 s an outage gets. + {name: "forced 8 s after Core starts", stop: 100, up: 101, cmd: 187, setW: 868, + forcedTo: 1201, dispatchFrom: 110, want: true}, + {name: "forced 23 s after Core starts", stop: 100, up: 101, cmd: 187, setW: 868, + forcedTo: 1201, dispatchFrom: 125, want: false}, + // The inverter is back at 102 while Core, stopped at 100 during the + // outage, stays down until 150. Core's first read is at 151 and the + // default lands 7 s later. While Core is down nothing reads the + // inverter, so the count cannot start before 151. + {name: "inverter back while Core is down for 48 s", outage: "offline", outFrom: 90, outFor: 12, readAt: 151, + stop: 100, up: 150, cmd: 187, setW: 3000, forcedTo: 158, dispatchFrom: 170, want: true}, } { - d := randomRun(121, 16, 5, func(i int, r *sample) { - r.Status = &ftwStatus{} - if c.orderW != 0 { - r.Status.Dispatch = []ftwDispatch{{Driver: "sungrow", TargetW: c.orderW}} - } - if i < c.forcedS { + if got := judged(c.run())[forced]; got != c.want { + t.Errorf("%s: %q = %v, want %v", c.name, forced, got, c.want) + } + } +} + +// R8 with two inverter outages: the count after the second starts at its +// own return and Core's first read after it, not the first outage's. +func TestInvariantsForcedBatteryCountsFromTheLatestReturn(t *testing.T) { + const forced = "no forced battery mode FTW does not dispatch" + // Outage 1 100-120, Core reads at 125 and dispatches 130-300. Outage 2 + // 300-340, Core reads at 380 and the default lands at 388. + run := func(forcedTo float64) *runData { + d := randomRun(601, 16, 5, func(i int, r *sample) { + tt := float64(i) + if tt < forcedTo { r.Bat.EMSMode, r.Bat.EMSCmd, r.Bat.EMSSetW = 2, 187, 3000 + } else { + r.Bat.EMSCmd = 204 + } + if (tt >= 100 && tt < 120) || (tt >= 300 && tt < 340) { + off := false + r.Bat.Running = &off + } + errs := 0 + if (tt >= 100 && tt < 125) || (tt >= 300 && tt < 380) { + errs = 3 + } + r.Status = &ftwStatus{Drivers: map[string]ftwDriver{inverterDriver: {Status: "ok", ConsecutiveErrors: &errs}}} + if tt < 100 || (tt >= 130 && tt < 300) { + r.Status.Dispatch = []ftwDispatch{{Driver: inverterDriver, TargetW: -3000}} } }) - if c.down { - d.allMarks = []markLine{{T: 10, Label: "ftw:restart"}, {T: 40, Label: "ftw:up"}} - } else { - d.events = []houseEventLine{{T: 10, Event: houseEvent{Do: "offline", Device: "inverter", ForS: 30}}} + d.events = []houseEventLine{ + {T: 100, Event: houseEvent{Do: "offline", Device: "inverter", ForS: 20}}, + {T: 300, Event: houseEvent{Do: "offline", Device: "inverter", ForS: 40}}, } - if got := judged(d)[forced]; got != c.want { - t.Errorf("%s: %q = %v, want %v", c.name, forced, got, c.want) + return d + } + if !judged(run(388))[forced] { + t.Errorf("default 8 s after Core reads again after the second outage: got fail, want pass") + } + if judged(run(1201))[forced] { + t.Errorf("forced to the end after the second outage: got pass, want fail") + } +} + +// R8 forgives the seconds Core's reconnect backoff and default retry hold +// its first write after a return (reconnectBackoffMaxS, defaultRetryMaxS). +// The triage line is 10 s from the return, so the judge records what it +// forgave beyond that line instead of hiding it. +func TestInvariantsRecordForcedSecondsInsideCoreTimers(t *testing.T) { + // 207539's shape: back at 182, Core reads 4 s before the backoff's cap + // ends (238 with 60 s) and dispatches 7 s later, inside the backoff + // alone. Forced with no dispatch until then; past 182+10 that is from + // 193 (52 s with 60 s). + read := 182 + reconnectBackoffMaxS - 4 + c := forcedCase{outage: "offline", outFrom: 100, outFor: 82, readAt: read, + cmd: 187, setW: 814, forcedTo: 1201, dispatchFrom: read + 7} + m, _ := judgeInvariants(c.run()) + if in, past, want := m["battery_forced_in_core_timers_s"], m["battery_forced_past_allowance_s"], read+6-192; in != want || past != 0.0 { + t.Errorf("in Core's timers %v s, past the allowance %v s; want %v and 0", in, past, want) + } + // Forced to 299: from 193 to the allowance's end inside the timers, + // the rest past them. + c.dispatchFrom = 300 + m, _ = judgeInvariants(c.run()) + end := read + defaultRetryMaxS + forcedBudgetS + if in, past := m["battery_forced_in_core_timers_s"], m["battery_forced_past_allowance_s"]; in != end-192 || past != 299-end { + t.Errorf("in Core's timers %v s, past the allowance %v s; want %v and %v", in, past, end-192, 299-end) + } +} + +// R8's allowances are Core's own timers. Read them from Core's source so +// they cannot drift. Once the registry retries a failed default on the +// first good poll after an outage (#1915, C11), defaultRetryMaxS must be +// 0: from then on the 30 s the judge gives today is no longer Core's. +func TestForcedAllowancesMatchCore(t *testing.T) { + src := func(p string) string { + b, err := os.ReadFile(filepath.Join("..", "..", "internal", p)) + if err != nil { + t.Fatal(err) + } + return string(b) + } + seconds := func(code, name string) float64 { + m := regexp.MustCompile(name + `\s*=\s*(\d+)\s*\*\s*time\.Second`).FindStringSubmatch(code) + if m == nil { + t.Fatalf("no %s = N * time.Second in Core's source", name) } + n, _ := strconv.Atoi(m[1]) + return float64(n) + } + // reconnectBackoffMaxS is read from Core's source; the 207539 shapes + // below need a backoff that can hold Core's first read back past 4 s. + if reconnectBackoffMaxS < 5 { + t.Errorf("Core's reconnectBackoffMax is %v s; rework the 207539 shapes", reconnectBackoffMaxS) + } + registry := src("drivers/registry.go") + // Today the registry sends the default on a failure and on its retry + // timer: two calls. #1915 adds a third, on the first good poll. + calls := strings.Count(registry, "attemptDefault()") + switch { + case calls == 2 && defaultRetryMaxS != seconds(registry, "defaultRetryMax"): + t.Errorf("defaultRetryMaxS = %v, Core's defaultRetryMax is %v s", defaultRetryMaxS, seconds(registry, "defaultRetryMax")) + case calls > 2 && defaultRetryMaxS != 0: + t.Errorf("Core's registry calls attemptDefault() in %d places, not 2. If it now retries "+ + "the default on the first good poll after an outage (#1915, C11), set defaultRetryMaxS "+ + "to 0 and turn the R8 cases that lean on it to fail. Otherwise update this check.", calls) + case calls < 2: + t.Errorf("Core's registry calls attemptDefault() in %d places; update this check", calls) } }