From bc6d19b55c5a3861ea6d63e631cc8fc17abbd98b Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 5 Oct 2026 16:56:49 +0800 Subject: [PATCH 1/2] fix(meta/sqlite): raise the writer busy-retry ceiling to 30s Concurrent memory dumps (vm hibernate, snapshot save) on the same filesystem as meta.db make the occasional WAL fsync slow (p90 ~160ms), and the writer queue behind it starves past 5s. The failure lands after the dump is already captured, so the work is thrown away and a hibernating VM is resumed. At 64 concurrent hibernates on bare metal, 5s failed 19-22 of 64 per round; 30s failed 0 of 448 operations across hibernate and save storms up to 128-way, with throughput unchanged (disk-bound). Max observed wait was 11s at 64-way and 21s at 128-way. --- meta/sqlite/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/sqlite/store.go b/meta/sqlite/store.go index 7eaa22c1..92763c60 100644 --- a/meta/sqlite/store.go +++ b/meta/sqlite/store.go @@ -34,7 +34,7 @@ const ( ManifestName = "meta-convert.manifest" // busyRetryCeiling bounds the BEGIN IMMEDIATE retry loop; busyRetryPause caps the jittered pause between retries (ยง4). - busyRetryCeiling = 5 * time.Second + busyRetryCeiling = 30 * time.Second busyRetryPause = 2 * time.Millisecond slowTxnWarn = 500 * time.Millisecond checkpointInterval = time.Second From 65b15d8537a81194bd1f1b98689d41ea8d10a870 Mon Sep 17 00:00:00 2001 From: CMGS Date: Mon, 5 Oct 2026 16:56:49 +0800 Subject: [PATCH 2/2] fix(hypervisor): stop and hibernate quiesce the network when the stopped flip fails After a confirmed VMM exit the Stopped flip is warn-and-continue, but a failed flip also skipped the network quiesce: the record still reads Running with QuiescePending unset, so a CNI VM kept its host plumbing up with a dead TAP (the #130 softirq precondition) until a daemon reconcile or the next start/restore. The quiesce now keys on the VM having gone down, and the flip outcome only fences the generation. --- hypervisor/hibernate_test.go | 43 ++++++++++++++++++++++++++++++++++++ hypervisor/network.go | 6 ++--- hypervisor/snapshot.go | 2 +- hypervisor/stop.go | 2 +- hypervisor/stop_test.go | 32 +++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 5 deletions(-) diff --git a/hypervisor/hibernate_test.go b/hypervisor/hibernate_test.go index 987d64df..29504302 100644 --- a/hypervisor/hibernate_test.go +++ b/hypervisor/hibernate_test.go @@ -1,6 +1,7 @@ package hypervisor import ( + "context" "errors" "fmt" "net/http" @@ -12,6 +13,7 @@ import ( "testing" "time" + "github.com/cocoonstack/cocoon/meta" "github.com/cocoonstack/cocoon/types" "github.com/cocoonstack/cocoon/utils" ) @@ -110,6 +112,41 @@ func TestHibernateSequenceTerminateFailureMarksError(t *testing.T) { } } +func TestHibernateSequenceQuiescesWhenTheStoppedFlipFails(t *testing.T) { + b, id := newHibernateTestVM(t) + if err := b.dbUpdate(t.Context(), func(idx *VMIndex) error { + idx.VMs[id].NetSetup = types.NetSetup{ + NetBackend: types.BackendCNI, + NetworkConfigs: []*types.NetworkConfig{{}}, + } + return nil + }); err != nil { + t.Fatalf("seed network: %v", err) + } + quiesced := 0 + b.SetNetwork(stubNetwork{quiesce: func(context.Context, *types.VM) error { + quiesced++ + return nil + }}) + store := b.Meta + spec := hibernateStubSpec(&hibernateCalls{}) + spec.Terminate = func(*VMRecord, *http.Client, int) error { + b.Meta = busyStore{store} + return nil + } + + err := b.HibernateSequence(t.Context(), id, spec, func(_ *types.SnapshotConfig, srcDir string) error { + return os.RemoveAll(srcDir) + }) + b.Meta = store + if err != nil { + t.Fatalf("HibernateSequence: %v", err) + } + if quiesced != 1 { + t.Fatalf("quiesced = %d, want 1 after the stopped flip failed", quiesced) + } +} + type hibernateCalls struct { resumed bool terminated bool @@ -172,3 +209,9 @@ func hibernateStubSpec(calls *hibernateCalls) HibernateSpec { Terminate: func(*VMRecord, *http.Client, int) error { calls.terminated = true; return nil }, } } + +type busyStore struct{ meta.Store } + +func (busyStore) Update(context.Context, meta.Scope, meta.CommitMode, meta.UpdateFunc) error { + return meta.ErrBusy +} diff --git a/hypervisor/network.go b/hypervisor/network.go index 10b06970..b692388a 100644 --- a/hypervisor/network.go +++ b/hypervisor/network.go @@ -27,10 +27,10 @@ func (b *Backend) RecoverNetwork(ctx context.Context, rec *VMRecord) error { } // Under the caller's ops lock the committed generation is rec's, plus one exactly when the stop just landed its transition. -func (b *Backend) quiesceAfterStop(ctx context.Context, id string, rec *VMRecord, transitioned bool) { - gen, pending := rec.TransitionGeneration, rec.QuiescePending +func (b *Backend) quiesceAfterStop(ctx context.Context, id string, rec *VMRecord, wentDown, transitioned bool) { + gen, pending := rec.TransitionGeneration, rec.QuiescePending || (wentDown && needsQuiesce(rec)) if transitioned { - gen, pending = gen+1, needsQuiesce(rec) + gen++ } if !pending { return diff --git a/hypervisor/snapshot.go b/hypervisor/snapshot.go index 15ca49a6..d2716f73 100644 --- a/hypervisor/snapshot.go +++ b/hypervisor/snapshot.go @@ -151,7 +151,7 @@ func (b *Backend) HibernateSequence(ctx context.Context, ref string, spec Hibern if uErr != nil { logger.Warnf(ctx, "mark stopped %s: %v", vmID, uErr) } - b.quiesceAfterStop(ctx, vmID, &rec, uErr == nil) + b.quiesceAfterStop(ctx, vmID, &rec, true, uErr == nil) return nil } diff --git a/hypervisor/stop.go b/hypervisor/stop.go index 41e2cd45..0171e042 100644 --- a/hypervisor/stop.go +++ b/hypervisor/stop.go @@ -67,7 +67,7 @@ func (b *Backend) StopOneLocked(ctx context.Context, id string, spec StopSpec) e } } // Still under the caller's ops lock: an idle TAP's TC redirect storms softirqs until its host NICs go down (#130). - b.quiesceAfterStop(ctx, id, &rec, transitioned) + b.quiesceAfterStop(ctx, id, &rec, !settled, transitioned) return nil } diff --git a/hypervisor/stop_test.go b/hypervisor/stop_test.go index bab52dd6..2857ba00 100644 --- a/hypervisor/stop_test.go +++ b/hypervisor/stop_test.go @@ -219,6 +219,38 @@ func TestStopStaleStoppedRecordWithLiveVMMStillTransitions(t *testing.T) { } } +func TestStopQuiescesWhenTheStoppedFlipFails(t *testing.T) { + b, id := newHibernateTestVM(t) + ctx := t.Context() + if err := b.dbUpdate(ctx, func(idx *VMIndex) error { + idx.VMs[id].NetSetup = types.NetSetup{ + NetBackend: types.BackendCNI, + NetworkConfigs: []*types.NetworkConfig{{}}, + } + return nil + }); err != nil { + t.Fatalf("seed network: %v", err) + } + quiesced := 0 + b.SetNetwork(stubNetwork{quiesce: func(context.Context, *types.VM) error { + quiesced++ + return nil + }}) + store := b.Meta + + err := b.StopOneLocked(ctx, id, StopSpec{Shutdown: func(ctx context.Context, _ *VMRecord, sockPath string, pid int) error { + b.Meta = busyStore{store} + return utils.TerminateProcess(ctx, pid, b.Conf.BinaryName(), sockPath, time.Second) + }}) + b.Meta = store + if err != nil { + t.Fatalf("StopOneLocked: %v", err) + } + if quiesced != 1 { + t.Fatalf("quiesced = %d, want 1 after the stopped flip failed", quiesced) + } +} + func TestStopFailureKeepsThePlacementOfALiveVMM(t *testing.T) { b, id := newHibernateTestVM(t) ctx := t.Context()