diff --git a/.changeset/c11-default-on-fresh-poll.md b/.changeset/c11-default-on-fresh-poll.md new file mode 100644 index 000000000..31b9e1638 --- /dev/null +++ b/.changeset/c11-default-on-fresh-poll.md @@ -0,0 +1,9 @@ +--- +"ftw": patch +--- + +Send a driver's autonomous default on the device's first good poll after an +outage, when a default failed while the device was away. Before, the retry +waited for its backoff timer (up to 30 s), so the device could keep running +FTW's last forced battery command after it came back. FTW retries once per +return, not on every poll. diff --git a/go/internal/drivers/registry.go b/go/internal/drivers/registry.go index fe9baf83d..8fb6dc8e1 100644 --- a/go/internal/drivers/registry.go +++ b/go/internal/drivers/registry.go @@ -297,6 +297,10 @@ type runningDriver struct { evPauseRevision uint64 evPauseCycleID uint64 evPausePending bool + // Default-recovery backoff bounds; zero means defaultRetryInitial and + // defaultRetryMax. Tests set them. + retryInitial time.Duration + retryMax time.Duration // Poll loop coordination cmdCh chan driverCmd defaultCh chan driverCmd @@ -894,7 +898,17 @@ func (r *Registry) runLoop(rd *runningDriver) { } var recoveryTimer *time.Timer var recoveryC <-chan time.Time - retryDelay := defaultRetryInitial + retryInitial, retryMax := rd.retryInitial, rd.retryMax + if retryInitial <= 0 { + retryInitial = defaultRetryInitial + } + if retryMax <= 0 { + retryMax = defaultRetryMax + } + retryDelay := retryInitial + // pollFailed holds while the device's reads fail. The first good poll + // after it is the device's return. + pollFailed := false clearRecoveryTimer := func() { if recoveryTimer != nil { if !recoveryTimer.Stop() { @@ -905,12 +919,12 @@ func (r *Registry) runLoop(rd *runningDriver) { } } recoveryC = nil - retryDelay = defaultRetryInitial + retryDelay = retryInitial } scheduleRecovery := func() { d := retryDelay if d <= 0 { - d = defaultRetryInitial + d = retryInitial } if recoveryTimer == nil { recoveryTimer = time.NewTimer(d) @@ -925,10 +939,10 @@ func (r *Registry) runLoop(rd *runningDriver) { } recoveryC = recoveryTimer.C rd.markDefaultRecoveryPending() - if retryDelay < defaultRetryMax { + if retryDelay < retryMax { retryDelay *= 2 - if retryDelay > defaultRetryMax { - retryDelay = defaultRetryMax + if retryDelay > retryMax { + retryDelay = retryMax } } } @@ -1180,6 +1194,8 @@ func (r *Registry) runLoop(rd *runningDriver) { pollCtx, finishPoll := rd.beginCommand(ctx) returned, err := rd.driver.Poll(pollCtx) finishPoll() + returnedFromOutage := err == nil && pollFailed + pollFailed = err != nil if r.tel != nil { r.tel.CountPoll(err != nil) } @@ -1207,6 +1223,17 @@ func (r *Registry) runLoop(rd *runningDriver) { // Re-arm at the interval the poll returned, or the standing one. interval = rd.env.nextPollInterval(returned) timer.Reset(interval) + // A default that failed while the device was away is retried + // on its first good poll, not when the backoff (up to 30 s) + // next allows. Once per return: a device that reads but still + // refuses waits for the backoff again. + if returnedFromOutage && recoveryC != nil { + if err := attemptDefault(); err != nil { + slog.Error("driver default failed on its first good poll after an outage; control remains blocked", "name", rd.cfg.Name, "err", err) + } else { + slog.Info("driver default confirmed on its first good poll after an outage", "name", rd.cfg.Name) + } + } case <-recoveryC: if err := attemptDefault(); err != nil { slog.Error("driver default recovery failed; control remains blocked", "name", rd.cfg.Name, "err", err) diff --git a/go/internal/drivers/registry_default_on_return_test.go b/go/internal/drivers/registry_default_on_return_test.go new file mode 100644 index 000000000..6038533a9 --- /dev/null +++ b/go/internal/drivers/registry_default_on_return_test.go @@ -0,0 +1,206 @@ +package drivers + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/config" + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// outageRuntime is a device that can go away. While down, polls and +// defaults fail. While refusing, defaults fail but polls succeed. +type outageRuntime struct { + env *HostEnv + + mu sync.Mutex + down bool + refusing bool + events []string +} + +func (r *outageRuntime) Init(context.Context, []byte) error { return nil } +func (r *outageRuntime) Poll(context.Context) (time.Duration, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.down { + r.events = append(r.events, "poll-fail") + return 0, errors.New("modbus reconnect backoff active") + } + r.events = append(r.events, "poll-ok") + return 0, nil +} +func (r *outageRuntime) Command(context.Context, []byte) error { return nil } +func (r *outageRuntime) DefaultMode(context.Context) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.down || r.refusing { + r.events = append(r.events, "default-fail") + return errors.New("default write refused") + } + r.events = append(r.events, "default-ok") + return nil +} +func (r *outageRuntime) Cleanup(context.Context) error { return nil } +func (r *outageRuntime) Discard() {} +func (r *outageRuntime) Env() *HostEnv { return r.env } + +func (r *outageRuntime) set(down, refusing bool) { + r.mu.Lock() + r.down, r.refusing = down, refusing + r.mu.Unlock() +} + +func (r *outageRuntime) snapshot() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.events...) +} + +func (r *outageRuntime) count(event string) int { + n := 0 + for _, e := range r.snapshot() { + if e == event { + n++ + } + } + return n +} + +// waitFor polls until the runtime has recorded n events of a kind. +func (r *outageRuntime) waitFor(t *testing.T, event string, n int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for r.count(event) < n { + if time.Now().After(deadline) { + t.Fatalf("waited for %d %q, events = %v", n, event, r.snapshot()) + } + time.Sleep(time.Millisecond) + } +} + +// startOutageDriver runs a driver that polls every 10 ms and whose own +// default-recovery timer never fires within the test, so only the poll +// path can send a default. +func startOutageDriver(t *testing.T) (*Registry, *runningDriver, *outageRuntime) { + t.Helper() + tel := telemetry.NewStore() + r := NewRegistry(tel) + runtime := &outageRuntime{env: NewHostEnv("outage", tel)} + runtime.env.PollIntervalMS = 10 + lifecycleCtx, lifecycleCancel := context.WithCancel(context.Background()) + rd := &runningDriver{ + driver: runtime, + env: runtime.env, + cfg: config.Driver{Name: "outage"}, + generation: 1, + defaultConfirmed: true, + lifecycleCtx: lifecycleCtx, + lifecycleCancel: lifecycleCancel, + cmdCh: make(chan driverCmd, 8), + defaultCh: make(chan driverCmd, 1), + stop: make(chan bool, 1), + done: make(chan struct{}), + retryInitial: time.Hour, + retryMax: time.Hour, + } + r.rec[rd.cfg.Name] = rd + go r.runLoop(rd) + t.Cleanup(func() { r.remove(rd.cfg.Name, true) }) + return r, rd, runtime +} + +// failDefaultDuringOutage takes the device away and has the watchdog's +// default fail against it, leaving recovery pending. +func failDefaultDuringOutage(t *testing.T, r *Registry, runtime *outageRuntime) { + t.Helper() + runtime.set(true, false) + runtime.waitFor(t, "poll-fail", runtime.count("poll-fail")+1) + if err := r.SendDefault(context.Background(), "outage"); err == nil { + t.Fatal("SendDefault during outage succeeded, want refusal") + } + status, _ := r.ControlStatus("outage") + if !status.Blocked || !status.RecoveryPending || status.DefaultConfirmed { + t.Fatalf("status during outage = %+v, want blocked with recovery pending", status) + } + runtime.waitFor(t, "poll-fail", runtime.count("poll-fail")+2) +} + +// eventAfterFirstGoodPoll returns the event that followed the first +// poll-ok recorded after index from. +func eventAfterFirstGoodPoll(events []string, from int) string { + for i := from; i < len(events)-1; i++ { + if events[i] == "poll-ok" { + return events[i+1] + } + } + return "" +} + +func TestDefaultSentOnFirstGoodPollAfterOutage(t *testing.T) { + r, _, runtime := startOutageDriver(t) + failDefaultDuringOutage(t, r, runtime) + returnedAt := len(runtime.snapshot()) + + runtime.set(false, false) + runtime.waitFor(t, "default-ok", 1) + if got := eventAfterFirstGoodPoll(runtime.snapshot(), returnedAt); got != "default-ok" { + t.Fatalf("event after first good poll = %q, want default-ok; events = %v", got, runtime.snapshot()) + } + okPolls := runtime.count("poll-ok") + runtime.waitFor(t, "poll-ok", okPolls+5) + if n := runtime.count("default-ok") + runtime.count("default-fail"); n != 2 { + t.Fatalf("default calls = %d, want the failed one and one on return; events = %v", n, runtime.snapshot()) + } + status, _ := r.ControlStatus("outage") + if status.Blocked || !status.DefaultConfirmed || status.RecoveryPending { + t.Fatalf("status after return = %+v, want default confirmed", status) + } +} + +func TestDefaultRetriedOncePerReturnNotPerPoll(t *testing.T) { + r, _, runtime := startOutageDriver(t) + failDefaultDuringOutage(t, r, runtime) + + // The device reads again but still refuses the default: one retry on + // the return, then the backoff timer owns it. + runtime.set(false, true) + runtime.waitFor(t, "default-fail", 2) + okPolls := runtime.count("poll-ok") + runtime.waitFor(t, "poll-ok", okPolls+10) + if n := runtime.count("default-fail"); n != 2 { + t.Fatalf("failed defaults = %d, want one during the outage and one on return; events = %v", n, runtime.snapshot()) + } + status, _ := r.ControlStatus("outage") + if !status.Blocked || !status.RecoveryPending { + t.Fatalf("status while refusing = %+v, want still blocked", status) + } + + // A second outage is a second return, so the default goes out again. + runtime.set(true, false) + runtime.waitFor(t, "poll-fail", runtime.count("poll-fail")+2) + returnedAt := len(runtime.snapshot()) + runtime.set(false, false) + runtime.waitFor(t, "default-ok", 1) + if got := eventAfterFirstGoodPoll(runtime.snapshot(), returnedAt); got != "default-ok" { + t.Fatalf("event after second return = %q, want default-ok; events = %v", got, runtime.snapshot()) + } + status, _ = r.ControlStatus("outage") + if status.Blocked || !status.DefaultConfirmed || status.RecoveryPending { + t.Fatalf("status after second return = %+v, want default confirmed", status) + } +} + +func TestGoodPollWithoutPendingRecoverySendsNoDefault(t *testing.T) { + _, _, runtime := startOutageDriver(t) + runtime.set(true, false) + runtime.waitFor(t, "poll-fail", 2) + runtime.set(false, false) + runtime.waitFor(t, "poll-ok", runtime.count("poll-ok")+3) + if n := runtime.count("default-ok") + runtime.count("default-fail"); n != 0 { + t.Fatalf("default calls = %d, want none when no default was pending; events = %v", n, runtime.snapshot()) + } +}