fix: raise the meta busy-retry ceiling and quiesce the network when the stopped flip fails

hypervisor/hibernate_test.go

@@ -1,6 +1,7 @@
 package hypervisor
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
@@ -12,6 +13,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cocoonstack/cocoon/meta"
 	"github.com/cocoonstack/cocoon/types"
 	"github.com/cocoonstack/cocoon/utils"
 )
@@ -110,6 +112,41 @@ func TestHibernateSequenceTerminateFailureMarksError(t *testing.T) {
 	}
 }
 
+func TestHibernateSequenceQuiescesWhenTheStoppedFlipFails(t *testing.T) {
+	b, id := newHibernateTestVM(t)
+	if err := b.dbUpdate(t.Context(), func(idx *VMIndex) error {
+		idx.VMs[id].NetSetup = types.NetSetup{
+			NetBackend:     types.BackendCNI,
+			NetworkConfigs: []*types.NetworkConfig{{}},
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("seed network: %v", err)
+	}
+	quiesced := 0
+	b.SetNetwork(stubNetwork{quiesce: func(context.Context, *types.VM) error {
+		quiesced++
+		return nil
+	}})
+	store := b.Meta
+	spec := hibernateStubSpec(&hibernateCalls{})
+	spec.Terminate = func(*VMRecord, *http.Client, int) error {
+		b.Meta = busyStore{store}
+		return nil
+	}
+
+	err := b.HibernateSequence(t.Context(), id, spec, func(_ *types.SnapshotConfig, srcDir string) error {
+		return os.RemoveAll(srcDir)
+	})
+	b.Meta = store
+	if err != nil {
+		t.Fatalf("HibernateSequence: %v", err)
+	}
+	if quiesced != 1 {
+		t.Fatalf("quiesced = %d, want 1 after the stopped flip failed", quiesced)
+	}
+}
+
 type hibernateCalls struct {
 	resumed    bool
 	terminated bool
@@ -172,3 +209,9 @@ func hibernateStubSpec(calls *hibernateCalls) HibernateSpec {
 		Terminate: func(*VMRecord, *http.Client, int) error { calls.terminated = true; return nil },
 	}
 }
+
+type busyStore struct{ meta.Store }
+
+func (busyStore) Update(context.Context, meta.Scope, meta.CommitMode, meta.UpdateFunc) error {
+	return meta.ErrBusy
+}

hypervisor/network.go

@@ -27,10 +27,10 @@ func (b *Backend) RecoverNetwork(ctx context.Context, rec *VMRecord) error {
 }
 
 // Under the caller's ops lock the committed generation is rec's, plus one exactly when the stop just landed its transition.
-func (b *Backend) quiesceAfterStop(ctx context.Context, id string, rec *VMRecord, transitioned bool) {
-	gen, pending := rec.TransitionGeneration, rec.QuiescePending
+func (b *Backend) quiesceAfterStop(ctx context.Context, id string, rec *VMRecord, wentDown, transitioned bool) {
+	gen, pending := rec.TransitionGeneration, rec.QuiescePending || (wentDown && needsQuiesce(rec))
 	if transitioned {
-		gen, pending = gen+1, needsQuiesce(rec)
+		gen++
 	}
 	if !pending {
 		return

hypervisor/snapshot.go

@@ -151,7 +151,7 @@ func (b *Backend) HibernateSequence(ctx context.Context, ref string, spec Hibern
 	if uErr != nil {
 		logger.Warnf(ctx, "mark stopped %s: %v", vmID, uErr)
 	}
-	b.quiesceAfterStop(ctx, vmID, &rec, uErr == nil)
+	b.quiesceAfterStop(ctx, vmID, &rec, true, uErr == nil)
 	return nil
 }
 

hypervisor/stop.go

@@ -67,7 +67,7 @@ func (b *Backend) StopOneLocked(ctx context.Context, id string, spec StopSpec) e
 		}
 	}
 	// Still under the caller's ops lock: an idle TAP's TC redirect storms softirqs until its host NICs go down (#130).
-	b.quiesceAfterStop(ctx, id, &rec, transitioned)
+	b.quiesceAfterStop(ctx, id, &rec, !settled, transitioned)
 	return nil
 }
 

hypervisor/stop_test.go

@@ -219,6 +219,38 @@ func TestStopStaleStoppedRecordWithLiveVMMStillTransitions(t *testing.T) {
 	}
 }
 
+func TestStopQuiescesWhenTheStoppedFlipFails(t *testing.T) {
+	b, id := newHibernateTestVM(t)
+	ctx := t.Context()
+	if err := b.dbUpdate(ctx, func(idx *VMIndex) error {
+		idx.VMs[id].NetSetup = types.NetSetup{
+			NetBackend:     types.BackendCNI,
+			NetworkConfigs: []*types.NetworkConfig{{}},
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("seed network: %v", err)
+	}
+	quiesced := 0
+	b.SetNetwork(stubNetwork{quiesce: func(context.Context, *types.VM) error {
+		quiesced++
+		return nil
+	}})
+	store := b.Meta
+
+	err := b.StopOneLocked(ctx, id, StopSpec{Shutdown: func(ctx context.Context, _ *VMRecord, sockPath string, pid int) error {
+		b.Meta = busyStore{store}
+		return utils.TerminateProcess(ctx, pid, b.Conf.BinaryName(), sockPath, time.Second)
+	}})
+	b.Meta = store
+	if err != nil {
+		t.Fatalf("StopOneLocked: %v", err)
+	}
+	if quiesced != 1 {
+		t.Fatalf("quiesced = %d, want 1 after the stopped flip failed", quiesced)
+	}
+}
+
 func TestStopFailureKeepsThePlacementOfALiveVMM(t *testing.T) {
 	b, id := newHibernateTestVM(t)
 	ctx := t.Context()

meta/sqlite/store.go

@@ -34,7 +34,7 @@ const (
 	ManifestName = "meta-convert.manifest"
 
 	// busyRetryCeiling bounds the BEGIN IMMEDIATE retry loop; busyRetryPause caps the jittered pause between retries (§4).
-	busyRetryCeiling   = 5 * time.Second
+	busyRetryCeiling   = 30 * time.Second
 	busyRetryPause     = 2 * time.Millisecond
 	slowTxnWarn        = 500 * time.Millisecond
 	checkpointInterval = time.Second