From 65616b991f2d604b3088598c7a78c847393cb750 Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 8 Nov 2026 16:16:37 +0900 Subject: [PATCH 1/3] Share one ICMP echo ID allocator between probers and tracers Add internal/icmpid, a process-wide allocator with one random 16-bit base, a counter that skips 0 and IDs still held, and Release. The ICMP prober and the path tracer's ICMP transport now draw their echo IDs from it instead of private counters, so live probers and tracers in one process never share an ID, and tracer restarts cannot wrap into IDs of running probers. IDs are released when the socket is closed. --- internal/icmpid/icmpid.go | 95 ++++++++++++++++++++++ internal/icmpid/icmpid_test.go | 121 ++++++++++++++++++++++++++++ internal/prober/icmp.go | 43 ++++------ internal/prober/icmp_naming_test.go | 42 +++++++--- internal/trace/packet_test.go | 31 +++++-- internal/trace/transport_icmp.go | 45 ++++------- 6 files changed, 301 insertions(+), 76 deletions(-) create mode 100644 internal/icmpid/icmpid.go create mode 100644 internal/icmpid/icmpid_test.go diff --git a/internal/icmpid/icmpid.go b/internal/icmpid/icmpid.go new file mode 100644 index 0000000..131a361 --- /dev/null +++ b/internal/icmpid/icmpid.go @@ -0,0 +1,95 @@ +// Package icmpid allocates ICMP echo identifiers shared by every ICMP user +// in the process: the ICMP probers (internal/prober) and the path tracers +// (internal/trace). +// +// Raw ICMP sockets receive the echo replies of every socket on the host, so +// the echo ID is what tells the replies of one prober or tracer apart from +// another's. Drawing all IDs from one allocator guarantees that the IDs held +// at the same time within a process are distinct. +// +// IDs count up from a random per-process base, so concurrent mping processes +// rarely pick the same IDs (PID-derived IDs overlap for neighbouring PIDs). +// ID 0 is never returned. Released IDs become available again, but only +// after the counter wraps around the 16-bit space, so a just-released ID is +// not handed out again while late replies to it may still arrive. +package icmpid + +import ( + "math/rand/v2" + "sync" +) + +// Allocator hands out distinct non-zero 16-bit echo IDs. The zero value is +// not usable; create one with New. +type Allocator struct { + mu sync.Mutex + base uint16 + next uint32 // offset from base of the next candidate + used map[uint16]struct{} +} + +// New returns an allocator whose IDs count up from base. +func New(base uint16) *Allocator { + return &Allocator{base: base, used: make(map[uint16]struct{})} +} + +var global = New(uint16(rand.Uint32())) + +// Acquire returns an ID from the process-wide allocator. See +// Allocator.Acquire. +func Acquire() uint16 { return global.Acquire() } + +// Release returns id to the process-wide allocator. See Allocator.Release. +func Release(id uint16) { global.Release(id) } + +// InUse reports whether id is held in the process-wide allocator. +func InUse(id uint16) bool { return global.InUse(id) } + +// Acquire returns a non-zero ID that no other caller holds. The counter +// walks the 16-bit space from the base and wraps around, skipping 0 and the +// IDs still held, so up to 65535 IDs can be held at once. If all of them +// are held (far more than a process creates), the next counter value is +// returned anyway and may duplicate a held ID. +func (a *Allocator) Acquire() uint16 { + a.mu.Lock() + defer a.mu.Unlock() + var fallback uint16 + for range 1 << 16 { + id := a.base + uint16(a.next) + a.next++ + if id == 0 { + continue + } + if fallback == 0 { + fallback = id + } + if _, held := a.used[id]; !held { + a.used[id] = struct{}{} + return id + } + } + return fallback +} + +// Release marks id as no longer held, once its socket is closed. Releasing +// an ID that is not held is a no-op. +func (a *Allocator) Release(id uint16) { + a.mu.Lock() + defer a.mu.Unlock() + delete(a.used, id) +} + +// InUse reports whether id is currently held. +func (a *Allocator) InUse(id uint16) bool { + a.mu.Lock() + defer a.mu.Unlock() + _, held := a.used[id] + return held +} + +// Held returns the number of IDs currently held. +func (a *Allocator) Held() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.used) +} diff --git a/internal/icmpid/icmpid_test.go b/internal/icmpid/icmpid_test.go new file mode 100644 index 0000000..1470064 --- /dev/null +++ b/internal/icmpid/icmpid_test.go @@ -0,0 +1,121 @@ +package icmpid + +import ( + "sync" + "testing" +) + +// IDs are never 0, even when the counter passes through it. +func TestAcquireNeverZero(t *testing.T) { + a := New(0xfff0) + for range 100 { + id := a.Acquire() + if id == 0 { + t.Fatal("Acquire() returned 0") + } + a.Release(id) + } + if id := New(0).Acquire(); id == 0 { + t.Fatal("Acquire() with base 0 returned 0") + } +} + +// Every non-zero ID can be held at once, each exactly once. +func TestAcquireDistinctFullSpace(t *testing.T) { + a := New(0x1234) + seen := make(map[uint16]bool) + for range 0xffff { + id := a.Acquire() + if id == 0 || seen[id] { + t.Fatalf("Acquire() = %#x: zero or duplicate", id) + } + seen[id] = true + } + if a.Held() != 0xffff { + t.Fatalf("Held() = %d, want 65535", a.Held()) + } + // Exhausted: an ID is still returned (documented duplicate), never 0 + if id := a.Acquire(); id == 0 { + t.Fatal("Acquire() on a full allocator returned 0") + } +} + +// A long-lived holder (a prober) keeps its ID while short-lived holders +// (tracers restarted on every target change) churn through and wrap around +// the whole space many times. +func TestWrapSkipsHeldIDs(t *testing.T) { + a := New(0xabcd) + prober := a.Acquire() + other := a.Acquire() + for range 5 * 0x10000 { + id := a.Acquire() + if id == prober || id == other || id == 0 { + t.Fatalf("Acquire() = %#x, collides with a held ID", id) + } + a.Release(id) + } + if a.Held() != 2 { + t.Fatalf("Held() = %d, want 2", a.Held()) + } +} + +// A released ID is not reused before the counter wraps around. +func TestReleaseNotReusedImmediately(t *testing.T) { + a := New(1) + id := a.Acquire() + a.Release(id) + for range 1000 { + if got := a.Acquire(); got == id { + t.Fatalf("released ID %#x reused before wraparound", id) + } + } + a.Release(0x4242) // not held: no-op +} + +// Concurrent acquirers get distinct IDs; released ones are not reused +// before the wraparound either (run with -race). +func TestAcquireConcurrent(t *testing.T) { + a := New(0x8000) + const workers, per = 16, 1000 + ids := make(chan uint16, workers*per) + var wg sync.WaitGroup + for range workers { + wg.Go(func() { + for i := range per { + id := a.Acquire() + ids <- id + if i%2 == 0 { + a.Release(id) + } + } + }) + } + wg.Wait() + close(ids) + seen := make(map[uint16]bool) + for id := range ids { + if id == 0 || seen[id] { + t.Fatalf("Acquire() = %#x: zero or duplicate", id) + } + seen[id] = true + } + if a.Held() != workers*per/2 { + t.Fatalf("Held() = %d, want %d", a.Held(), workers*per/2) + } +} + +// The process-wide functions share one allocator. +func TestGlobal(t *testing.T) { + a, b := Acquire(), Acquire() + if a == 0 || b == 0 || a == b { + t.Fatalf("Acquire() = %#x, %#x", a, b) + } + if !InUse(a) || !InUse(b) { + t.Fatal("acquired IDs not in use") + } + Release(a) + Release(b) + if InUse(a) || InUse(b) { + t.Fatal("released IDs still in use") + } +} diff --git a/internal/prober/icmp.go b/internal/prober/icmp.go index 4436961..d9560fc 100644 --- a/internal/prober/icmp.go +++ b/internal/prober/icmp.go @@ -5,17 +5,17 @@ import ( "errors" "fmt" "maps" - "math/rand/v2" "net" "slices" "strings" "sync" - "sync/atomic" "time" "golang.org/x/net/icmp" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" + + "github.com/servak/mping/internal/icmpid" ) const ( @@ -114,36 +114,18 @@ func NewICMPProber(t ProbeType, cfg *ICMPConfig, prefix string) (*ICMPProber, er return newICMPProber(t, c, []byte(cfg.Body), prefix), nil } -// echoIDs counts the ICMP probers created in this process, and echoIDBase -// is a random per-process start for their echo IDs. -var ( - echoIDs atomic.Uint32 - echoIDBase = uint16(rand.Uint32()) -) - -// nextEchoID returns the echo ID for a new prober. A raw ICMP socket -// receives the echo replies of every socket on the host, and the probers -// in one process ping with the same sequence numbers, so the echo ID is -// what tells their replies apart: each prober gets its own, counting up -// from a random per-process base. The random base keeps concurrent mping -// processes apart (PID-derived IDs overlap for neighbouring PIDs), and the -// path tracer (internal/trace) draws its echo IDs from its own random base -// too, so the two rarely meet. ID 0 is skipped; the counter wraps after -// 65535 probers, far more than one process creates. +// newICMPProber returns a prober on c with its own echo ID. The echo ID +// tells the replies of this prober apart from those of the other +// probers and path tracers of the process, which ping with overlapping +// sequence numbers: a raw ICMP socket receives the echo replies of every +// socket on the host. IDs come from the process-wide allocator in +// internal/icmpid, shared with internal/trace, and are released when the +// socket is closed. // // The probers only use raw sockets. Datagram ICMP sockets (Linux // "ping" sockets) would need different matching: the kernel rewrites the // echo ID to the socket's port and delivers each socket only its own // replies, so the ID check in recvPkts would have to be skipped there. -func nextEchoID() int { - for { - id := echoIDBase + uint16(echoIDs.Add(1)-1) - if id != 0 { - return int(id) - } - } -} - func newICMPProber(t ProbeType, c packetConn, body []byte, prefix string) *ICMPProber { return &ICMPProber{ version: t, @@ -151,7 +133,7 @@ func newICMPProber(t ProbeType, c packetConn, body []byte, prefix string) *ICMPP c: c, tables: make(map[runTime]map[string]bool), targets: make(map[string]icmpTarget), - runID: nextEchoID(), + runID: int(icmpid.Acquire()), runCnt: 0, body: body, loop: newRunLoop(), @@ -539,7 +521,10 @@ func (p *ICMPProber) recvPkts(r chan *Event) { } func (p *ICMPProber) closeConn() { - p.closeOnce.Do(func() { _ = p.c.Close() }) + p.closeOnce.Do(func() { + _ = p.c.Close() + icmpid.Release(uint16(p.runID)) + }) } // Start sends one echo request per target and round until Stop. Requests diff --git a/internal/prober/icmp_naming_test.go b/internal/prober/icmp_naming_test.go index 515a22d..4bda161 100644 --- a/internal/prober/icmp_naming_test.go +++ b/internal/prober/icmp_naming_test.go @@ -4,21 +4,43 @@ import ( "net/url" "testing" + "github.com/servak/mping/internal/icmpid" "github.com/servak/mping/internal/trace" ) -// Echo IDs of the probers in one process are distinct and never 0. -func TestNextEchoIDDistinct(t *testing.T) { - seen := make(map[int]bool) - for range 1000 { - id := nextEchoID() - if id <= 0 || id > 0xffff { - t.Fatalf("nextEchoID() = %d, out of range", id) +// Probers and path tracers draw their echo IDs from one process-wide +// allocator: the IDs held at once are distinct and never 0, also while +// tracers churn (they restart on every target change), and a stopped +// prober releases its ID. +func TestEchoIDsDisjointFromTracer(t *testing.T) { + var probers []*ICMPProber + held := make(map[uint16]bool) + for i := range 200 { + p := newICMPProber(ICMPV4, newFakeICMPConn(), nil, "icmpv4") + probers = append(probers, p) + id := uint16(p.runID) + if id == 0 || p.runID > 0xffff || held[id] { + t.Fatalf("prober echo ID %d: zero, out of range or duplicate", p.runID) } - if seen[id] { - t.Fatalf("nextEchoID() returned %d twice", id) + held[id] = true + // a tracer starting and stopping between probers (trace uses + // icmpid.Acquire / Release for its transports) + tr := icmpid.Acquire() + if tr == 0 || held[tr] { + t.Fatalf("tracer echo ID %#x: zero or held by a prober", tr) + } + if i%2 == 0 { + icmpid.Release(tr) + } else { + held[tr] = true + defer icmpid.Release(tr) + } + } + for _, p := range probers { + p.Stop() + if icmpid.InUse(uint16(p.runID)) { + t.Fatalf("echo ID %d still held after Stop", p.runID) } - seen[id] = true } } diff --git a/internal/trace/packet_test.go b/internal/trace/packet_test.go index 2995f61..94aa9db 100644 --- a/internal/trace/packet_test.go +++ b/internal/trace/packet_test.go @@ -8,6 +8,8 @@ import ( "golang.org/x/net/icmp" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" + + "github.com/servak/mping/internal/icmpid" ) func echoRequest(t *testing.T, v6 bool, id, seq int) []byte { @@ -126,16 +128,29 @@ func TestParseReplyRejects(t *testing.T) { } } -func TestEchoIDAvoidsProberID(t *testing.T) { - pid := 0x12345 - if id := echoID(0x2344, 1, pid); id == uint16(pid&0xffff) { - t.Errorf("echo ID %#x equals the ICMP prober ID", id) +// A tracer's echo ID comes from the process-wide allocator shared with the +// ICMP probers: it is held while the transport is open, other allocations +// (probers, other tracers) never get it, and closing releases it once. +func TestICMPTransportEchoIDShared(t *testing.T) { + tp, err := newICMPTransport(&net.IPAddr{IP: net.IPv4(127, 0, 0, 1)}, false) + if err != nil { + t.Skipf("no ICMP socket: %v", err) + } + if tp.id == 0 || !icmpid.InUse(tp.id) { + t.Fatalf("echo ID %#x not held in the shared allocator", tp.id) + } + for range 1000 { + id := icmpid.Acquire() + if id == tp.id { + t.Fatalf("shared allocator handed out the tracer ID %#x", id) + } + defer icmpid.Release(id) } - // IDs follow the random base, not the pid: a tracer of one process - // must not take the prober ID of the next one (pid+1) - if a, b := echoID(0x1000, 1, pid), echoID(0x1000, 1, pid+1); a != b { - t.Errorf("echo ID depends on the pid: %#x / %#x", a, b) + _ = tp.close() + if icmpid.InUse(tp.id) { + t.Fatalf("echo ID %#x still held after close", tp.id) } + _ = tp.close() // closing twice must not release an ID reacquired since } // The echo ID decides whether a reply is ours. Replies from, or quoting, an diff --git a/internal/trace/transport_icmp.go b/internal/trace/transport_icmp.go index 7709e21..3d92de2 100644 --- a/internal/trace/transport_icmp.go +++ b/internal/trace/transport_icmp.go @@ -2,17 +2,16 @@ package trace import ( "fmt" - "math/rand/v2" "net" - "os" "runtime" "sync" - "sync/atomic" "time" "golang.org/x/net/icmp" "golang.org/x/net/ipv4" "golang.org/x/net/ipv6" + + "github.com/servak/mping/internal/icmpid" ) // transport sends probes and delivers the matching ICMP replies. The probe @@ -28,31 +27,13 @@ type transport interface { privileged() bool } -// Echo IDs are a random per-process base plus a counter: unique per tracer -// within the process, and unrelated to the pid, so that they do not follow -// the pid&0xffff IDs of the ICMP probers of this or another mping process -// (pids are often consecutive). -var ( - idBase = uint16(rand.Uint32()) - idCounter atomic.Uint32 -) - -// nextEchoID returns the echo ID for a new tracer, never pid&0xffff (the ID -// of this process' ICMP prober) -func nextEchoID() uint16 { - return echoID(idBase, idCounter.Add(1), os.Getpid()) -} - -func echoID(base uint16, n uint32, pid int) uint16 { - id := base + uint16(n) - if id == uint16(pid&0xffff) { - id++ - } - return id -} - // icmpTransport traces with ICMP echo requests, like mtr's default mode. The // echo ID is fixed per tracer, so every probe is one flow for ECMP hashing. +// It comes from the process-wide allocator in internal/icmpid, shared with +// the ICMP probers, so that no two live tracers or probers of the process +// use the same ID (raw sockets see every echo reply on the host); it is +// released when the transport is closed, as tracers restart on every target +// change. type icmpTransport struct { v6 bool raw bool @@ -61,14 +42,15 @@ type icmpTransport struct { id uint16 checkID bool - writeMu sync.Mutex // TTL is a socket option, so set+write must be atomic + writeMu sync.Mutex // TTL is a socket option, so set+write must be atomic + closeOnce sync.Once } // newICMPTransport prefers a raw socket and falls back to an unprivileged // datagram ICMP socket; on macOS the fallback still receives Time Exceeded // messages, on Linux only the destination may be visible without privileges. func newICMPTransport(dst *net.IPAddr, v6 bool) (*icmpTransport, error) { - t := &icmpTransport{v6: v6, id: nextEchoID()} + t := &icmpTransport{v6: v6, id: icmpid.Acquire()} rawNet, dgramNet, laddr := "ip4:icmp", "udp4", "0.0.0.0" if v6 { rawNet, dgramNet, laddr = "ip6:ipv6-icmp", "udp6", "::" @@ -84,6 +66,7 @@ func newICMPTransport(dst *net.IPAddr, v6 bool) (*icmpTransport, error) { // replies per socket, so only check the ID elsewhere (e.g. macOS). t.checkID = runtime.GOOS != "linux" } else { + icmpid.Release(t.id) return nil, fmt.Errorf("cannot open ICMP socket (%v; %v): run as root or grant cap_net_raw", err, err2) } return t, nil @@ -91,7 +74,11 @@ func newICMPTransport(dst *net.IPAddr, v6 bool) (*icmpTransport, error) { func (t *icmpTransport) privileged() bool { return t.raw } -func (t *icmpTransport) close() error { return t.conn.Close() } +func (t *icmpTransport) close() error { + err := t.conn.Close() + t.closeOnce.Do(func() { icmpid.Release(t.id) }) + return err +} func (t *icmpTransport) send(ttl int, seq uint16) error { msg := icmp.Message{Body: &icmp.Echo{ID: int(t.id), Seq: int(seq), Data: []byte("mping-trace")}} From 58aafff7faf1e979f2e62f1b7b0b31ce4ea63f05 Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 8 Nov 2026 16:21:13 +0900 Subject: [PATCH 2/3] Replace ICMP echo ID counter with leases and time-based quarantine --- internal/icmpid/export_test.go | 8 + internal/icmpid/icmpid.go | 125 +++++++++------- internal/icmpid/icmpid_test.go | 197 ++++++++++++++++++------- internal/prober/icmp.go | 15 +- internal/prober/icmp_isolation_test.go | 8 +- internal/prober/icmp_lifecycle_test.go | 4 +- internal/prober/icmp_naming_test.go | 30 ++-- internal/trace/packet_test.go | 43 ++++-- internal/trace/transport_icmp.go | 15 +- 9 files changed, 302 insertions(+), 143 deletions(-) create mode 100644 internal/icmpid/export_test.go diff --git a/internal/icmpid/export_test.go b/internal/icmpid/export_test.go new file mode 100644 index 0000000..9b97828 --- /dev/null +++ b/internal/icmpid/export_test.go @@ -0,0 +1,8 @@ +package icmpid + +// Held returns the number of IDs currently held. +func (a *Allocator) Held() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.held) +} diff --git a/internal/icmpid/icmpid.go b/internal/icmpid/icmpid.go index 131a361..d052861 100644 --- a/internal/icmpid/icmpid.go +++ b/internal/icmpid/icmpid.go @@ -7,89 +7,106 @@ // another's. Drawing all IDs from one allocator guarantees that the IDs held // at the same time within a process are distinct. // -// IDs count up from a random per-process base, so concurrent mping processes -// rarely pick the same IDs (PID-derived IDs overlap for neighbouring PIDs). -// ID 0 is never returned. Released IDs become available again, but only -// after the counter wraps around the 16-bit space, so a just-released ID is -// not handed out again while late replies to it may still arrive. +// IDs are handed out by a cursor that starts at a random per-process base, so +// concurrent mping processes rarely pick the same IDs (PID-derived IDs +// overlap for neighbouring PIDs). ID 0 is never returned. A released ID is +// quarantined for a while before it can be handed out again, so that late +// replies addressed to its previous owner are not taken for the new owner's. package icmpid import ( + "errors" "math/rand/v2" "sync" + "time" ) +// DefaultQuarantine is how long a released ID stays unavailable. +const DefaultQuarantine = 10 * time.Second + +// ErrExhausted is returned by Acquire when every ID is held or quarantined. +var ErrExhausted = errors.New("icmpid: no free ICMP echo ID (all held or recently released)") + // Allocator hands out distinct non-zero 16-bit echo IDs. The zero value is // not usable; create one with New. type Allocator struct { - mu sync.Mutex - base uint16 - next uint32 // offset from base of the next candidate - used map[uint16]struct{} + mu sync.Mutex + quarantine time.Duration + now func() time.Time + cursor uint16 + held map[uint16]struct{} + released map[uint16]time.Time // quarantined IDs and when they were released } -// New returns an allocator whose IDs count up from base. -func New(base uint16) *Allocator { - return &Allocator{base: base, used: make(map[uint16]struct{})} +// New returns an allocator whose cursor starts at base and which keeps +// released IDs out of circulation for quarantine. A nil now means time.Now. +func New(base uint16, quarantine time.Duration, now func() time.Time) *Allocator { + if now == nil { + now = time.Now + } + return &Allocator{ + quarantine: quarantine, + now: now, + cursor: base, + held: make(map[uint16]struct{}), + released: make(map[uint16]time.Time), + } } -var global = New(uint16(rand.Uint32())) +var global = New(uint16(rand.Uint32()), DefaultQuarantine, nil) -// Acquire returns an ID from the process-wide allocator. See -// Allocator.Acquire. -func Acquire() uint16 { return global.Acquire() } +// Acquire takes an ID from the process-wide allocator. See Allocator.Acquire. +func Acquire() (*Lease, error) { return global.Acquire() } -// Release returns id to the process-wide allocator. See Allocator.Release. -func Release(id uint16) { global.Release(id) } +// Lease is the ownership of one echo ID. Only the lease can release the ID, +// and releasing is idempotent, so a double Release never affects a later +// owner of the same ID. +type Lease struct { + a *Allocator + id uint16 + once sync.Once +} + +// ID returns the echo ID the lease holds. +func (l *Lease) ID() uint16 { return l.id } -// InUse reports whether id is held in the process-wide allocator. -func InUse(id uint16) bool { return global.InUse(id) } +// Release gives the ID back; it then stays quarantined before it can be +// reissued. Only the first call has an effect. +func (l *Lease) Release() { + l.once.Do(func() { l.a.release(l.id) }) +} -// Acquire returns a non-zero ID that no other caller holds. The counter -// walks the 16-bit space from the base and wraps around, skipping 0 and the -// IDs still held, so up to 65535 IDs can be held at once. If all of them -// are held (far more than a process creates), the next counter value is -// returned anyway and may duplicate a held ID. -func (a *Allocator) Acquire() uint16 { +// Acquire returns a lease on a non-zero ID that no one holds and that was not +// released within the quarantine. The cursor walks the 16-bit space from the +// base and wraps around. If no ID qualifies it returns ErrExhausted. +func (a *Allocator) Acquire() (*Lease, error) { a.mu.Lock() defer a.mu.Unlock() - var fallback uint16 + now := a.now() for range 1 << 16 { - id := a.base + uint16(a.next) - a.next++ + id := a.cursor + a.cursor++ if id == 0 { continue } - if fallback == 0 { - fallback = id + if _, ok := a.held[id]; ok { + continue } - if _, held := a.used[id]; !held { - a.used[id] = struct{}{} - return id + if at, ok := a.released[id]; ok { + if now.Sub(at) < a.quarantine { + continue + } + delete(a.released, id) } + a.held[id] = struct{}{} + return &Lease{a: a, id: id}, nil } - return fallback -} - -// Release marks id as no longer held, once its socket is closed. Releasing -// an ID that is not held is a no-op. -func (a *Allocator) Release(id uint16) { - a.mu.Lock() - defer a.mu.Unlock() - delete(a.used, id) -} - -// InUse reports whether id is currently held. -func (a *Allocator) InUse(id uint16) bool { - a.mu.Lock() - defer a.mu.Unlock() - _, held := a.used[id] - return held + return nil, ErrExhausted } -// Held returns the number of IDs currently held. -func (a *Allocator) Held() int { +func (a *Allocator) release(id uint16) { a.mu.Lock() defer a.mu.Unlock() - return len(a.used) + delete(a.held, id) + a.released[id] = a.now() } diff --git a/internal/icmpid/icmpid_test.go b/internal/icmpid/icmpid_test.go index 1470064..0e821f8 100644 --- a/internal/icmpid/icmpid_test.go +++ b/internal/icmpid/icmpid_test.go @@ -1,91 +1,185 @@ package icmpid import ( + "errors" "sync" "testing" + "time" ) -// IDs are never 0, even when the counter passes through it. +type fakeClock struct{ t time.Time } + +func (c *fakeClock) now() time.Time { return c.t } +func (c *fakeClock) advance(d time.Duration) { c.t = c.t.Add(d) } + +func mustAcquire(t *testing.T, a *Allocator) *Lease { + t.Helper() + l, err := a.Acquire() + if err != nil { + t.Fatal(err) + } + return l +} + +// IDs are never 0, even when the cursor passes through it. func TestAcquireNeverZero(t *testing.T) { - a := New(0xfff0) + a := New(0xfff0, 0, nil) for range 100 { - id := a.Acquire() - if id == 0 { + l := mustAcquire(t, a) + if l.ID() == 0 { t.Fatal("Acquire() returned 0") } - a.Release(id) + l.Release() } - if id := New(0).Acquire(); id == 0 { + if l := mustAcquire(t, New(0, 0, nil)); l.ID() == 0 { t.Fatal("Acquire() with base 0 returned 0") } } -// Every non-zero ID can be held at once, each exactly once. -func TestAcquireDistinctFullSpace(t *testing.T) { - a := New(0x1234) +// Every non-zero ID can be held at once, each exactly once; then Acquire +// fails instead of duplicating, and succeeds again after a release. +func TestAcquireFullSpaceThenExhausted(t *testing.T) { + a := New(0x1234, 0, nil) seen := make(map[uint16]bool) + var leases []*Lease for range 0xffff { - id := a.Acquire() - if id == 0 || seen[id] { - t.Fatalf("Acquire() = %#x: zero or duplicate", id) + l := mustAcquire(t, a) + if l.ID() == 0 || seen[l.ID()] { + t.Fatalf("Acquire() = %#x: zero or duplicate", l.ID()) } - seen[id] = true + seen[l.ID()] = true + leases = append(leases, l) } if a.Held() != 0xffff { t.Fatalf("Held() = %d, want 65535", a.Held()) } - // Exhausted: an ID is still returned (documented duplicate), never 0 - if id := a.Acquire(); id == 0 { - t.Fatal("Acquire() on a full allocator returned 0") + if _, err := a.Acquire(); !errors.Is(err, ErrExhausted) { + t.Fatalf("Acquire() on a full allocator: err = %v, want ErrExhausted", err) } + leases[7].Release() + if l := mustAcquire(t, a); l.ID() != leases[7].ID() { + t.Fatalf("Acquire() = %#x, want the released %#x", l.ID(), leases[7].ID()) + } +} + +// Quarantined IDs count as unavailable for exhaustion too. +func TestExhaustedByQuarantine(t *testing.T) { + clk := &fakeClock{t: time.Unix(1000, 0)} + a := New(1, time.Second, clk.now) + var leases []*Lease + for range 0xffff { + leases = append(leases, mustAcquire(t, a)) + } + for _, l := range leases { + l.Release() + } + if _, err := a.Acquire(); !errors.Is(err, ErrExhausted) { + t.Fatalf("err = %v, want ErrExhausted while all IDs are quarantined", err) + } + clk.advance(time.Second) + mustAcquire(t, a) } -// A long-lived holder (a prober) keeps its ID while short-lived holders -// (tracers restarted on every target change) churn through and wrap around -// the whole space many times. +// Long-lived holders keep their IDs while short-lived holders churn through +// and wrap around the whole space many times. func TestWrapSkipsHeldIDs(t *testing.T) { - a := New(0xabcd) - prober := a.Acquire() - other := a.Acquire() + a := New(0xabcd, 0, nil) + prober, other := mustAcquire(t, a), mustAcquire(t, a) for range 5 * 0x10000 { - id := a.Acquire() - if id == prober || id == other || id == 0 { - t.Fatalf("Acquire() = %#x, collides with a held ID", id) + l := mustAcquire(t, a) + if l.ID() == prober.ID() || l.ID() == other.ID() || l.ID() == 0 { + t.Fatalf("Acquire() = %#x, collides with a held ID", l.ID()) } - a.Release(id) + l.Release() } if a.Held() != 2 { t.Fatalf("Held() = %d, want 2", a.Held()) } } -// A released ID is not reused before the counter wraps around. -func TestReleaseNotReusedImmediately(t *testing.T) { - a := New(1) - id := a.Acquire() - a.Release(id) - for range 1000 { - if got := a.Acquire(); got == id { - t.Fatalf("released ID %#x reused before wraparound", id) +// A released ID is not reissued within the quarantine, and is after it. +func TestQuarantine(t *testing.T) { + clk := &fakeClock{t: time.Unix(1000, 0)} + a := New(1, 10*time.Second, clk.now) + l := mustAcquire(t, a) + id := l.ID() + l.Release() + // Hold every other ID: the only candidate left is the quarantined one. + for range 0xfffe { + if got := mustAcquire(t, a); got.ID() == id { + t.Fatalf("released ID %#x reissued within quarantine", id) + } + } + clk.advance(9 * time.Second) + if _, err := a.Acquire(); !errors.Is(err, ErrExhausted) { + t.Fatalf("err = %v, want ErrExhausted within quarantine", err) + } + clk.advance(time.Second) + if got := mustAcquire(t, a); got.ID() != id { + t.Fatalf("Acquire() = %#x, want %#x after quarantine", got.ID(), id) + } +} + +// Reviewer repro: an ID held across a full wrap and then released must not be +// handed out by the very next Acquire. +func TestHeldAcrossWrapNotReissued(t *testing.T) { + clk := &fakeClock{t: time.Unix(1000, 0)} + a := New(1, 10*time.Second, clk.now) + p := mustAcquire(t, a) + for range 0xffff - 1 { + mustAcquire(t, a).Release() + clk.advance(time.Millisecond) + } + p.Release() + for range 3 { + if got := mustAcquire(t, a); got.ID() == p.ID() { + t.Fatalf("released ID %#x reissued within quarantine", got.ID()) + } + } +} + +// A second Release on a lease is a no-op: it must not release the ID again +// for a later owner. +func TestDoubleReleaseKeepsNewOwner(t *testing.T) { + a := New(1, 0, nil) + first := mustAcquire(t, a) + first.Release() + // Hold everything: with no quarantine the first ID goes to a new owner. + var second *Lease + for range 0xffff { + if l := mustAcquire(t, a); l.ID() == first.ID() { + second = l } } - a.Release(0x4242) // not held: no-op + if second == nil { + t.Fatal("ID not reissued") + } + first.Release() + if a.Held() != 0xffff { + t.Fatalf("stale Release changed Held() to %d", a.Held()) + } + if _, err := a.Acquire(); !errors.Is(err, ErrExhausted) { + t.Fatalf("live lease %#x was reissued after a stale Release: err = %v", second.ID(), err) + } } -// Concurrent acquirers get distinct IDs; released ones are not reused -// before the wraparound either (run with -race). +// Concurrent acquirers get distinct IDs (run with -race). func TestAcquireConcurrent(t *testing.T) { - a := New(0x8000) + a := New(0x8000, DefaultQuarantine, nil) const workers, per = 16, 1000 ids := make(chan uint16, workers*per) var wg sync.WaitGroup for range workers { wg.Go(func() { for i := range per { - id := a.Acquire() - ids <- id + l, err := a.Acquire() + if err != nil { + t.Error(err) + return + } + ids <- l.ID() if i%2 == 0 { - a.Release(id) + l.Release() } } }) @@ -104,18 +198,19 @@ func TestAcquireConcurrent(t *testing.T) { } } -// The process-wide functions share one allocator. +// The process-wide Acquire shares one allocator. func TestGlobal(t *testing.T) { - a, b := Acquire(), Acquire() - if a == 0 || b == 0 || a == b { - t.Fatalf("Acquire() = %#x, %#x", a, b) + a, err := Acquire() + if err != nil { + t.Fatal(err) } - if !InUse(a) || !InUse(b) { - t.Fatal("acquired IDs not in use") + b, err := Acquire() + if err != nil { + t.Fatal(err) } - Release(a) - Release(b) - if InUse(a) || InUse(b) { - t.Fatal("released IDs still in use") + defer a.Release() + defer b.Release() + if a.ID() == 0 || b.ID() == 0 || a.ID() == b.ID() { + t.Fatalf("Acquire() = %#x, %#x", a.ID(), b.ID()) } } diff --git a/internal/prober/icmp.go b/internal/prober/icmp.go index d9560fc..40e3d34 100644 --- a/internal/prober/icmp.go +++ b/internal/prober/icmp.go @@ -33,6 +33,7 @@ type ( timeout time.Duration runCnt int runID int // echo ID, unique per prober in the process + lease *icmpid.Lease // owns runID until the socket is closed tables map[runTime]map[string]bool // guarded by mu mu sync.Mutex loop *runLoop @@ -111,7 +112,12 @@ func NewICMPProber(t ProbeType, cfg *ICMPConfig, prefix string) (*ICMPProber, er return nil, err } } - return newICMPProber(t, c, []byte(cfg.Body), prefix), nil + lease, err := icmpid.Acquire() + if err != nil { + _ = c.Close() + return nil, fmt.Errorf("cannot allocate ICMP echo ID: %w", err) + } + return newICMPProber(t, c, []byte(cfg.Body), prefix, lease), nil } // newICMPProber returns a prober on c with its own echo ID. The echo ID @@ -126,14 +132,15 @@ func NewICMPProber(t ProbeType, cfg *ICMPConfig, prefix string) (*ICMPProber, er // "ping" sockets) would need different matching: the kernel rewrites the // echo ID to the socket's port and delivers each socket only its own // replies, so the ID check in recvPkts would have to be skipped there. -func newICMPProber(t ProbeType, c packetConn, body []byte, prefix string) *ICMPProber { +func newICMPProber(t ProbeType, c packetConn, body []byte, prefix string, lease *icmpid.Lease) *ICMPProber { return &ICMPProber{ version: t, prefix: prefix, c: c, tables: make(map[runTime]map[string]bool), targets: make(map[string]icmpTarget), - runID: int(icmpid.Acquire()), + runID: int(lease.ID()), + lease: lease, runCnt: 0, body: body, loop: newRunLoop(), @@ -523,7 +530,7 @@ func (p *ICMPProber) recvPkts(r chan *Event) { func (p *ICMPProber) closeConn() { p.closeOnce.Do(func() { _ = p.c.Close() - icmpid.Release(uint16(p.runID)) + p.lease.Release() }) } diff --git a/internal/prober/icmp_isolation_test.go b/internal/prober/icmp_isolation_test.go index 9dc9be2..a1995a6 100644 --- a/internal/prober/icmp_isolation_test.go +++ b/internal/prober/icmp_isolation_test.go @@ -47,11 +47,11 @@ func TestICMPProbersDoNotShareReplies(t *testing.T) { lost.noReply = true // e.g. TTL=1: the requests never reach the target ok := host.newConn() - pLost := newICMPProber(ICMPV4, lost, []byte("x"), "ttl1") + pLost := newTestProber(t, ICMPV4, lost, []byte("x"), "ttl1") if err := pLost.Accept("ttl1://127.0.0.1"); err != nil { t.Fatal(err) } - pOK := newICMPProber(ICMPV4, ok, []byte("x"), "icmpv4") + pOK := newTestProber(t, ICMPV4, ok, []byte("x"), "icmpv4") if err := pOK.Accept("icmpv4://127.0.0.1"); err != nil { t.Fatal(err) } @@ -87,7 +87,7 @@ func TestICMPProberKeys(t *testing.T) { } for _, tc := range cases { t.Run(tc.target, func(t *testing.T) { - p := newICMPProber(ICMPV4, newFakeICMPConn(), nil, tc.prefix) + p := newTestProber(t, ICMPV4, newFakeICMPConn(), nil, tc.prefix) if err := p.Accept(tc.target); err != nil { t.Fatal(err) } @@ -105,7 +105,7 @@ func TestICMPProberKeys(t *testing.T) { // A reply arriving after the timeout is a TIMEOUT, reported once, never a // SUCCESS with an RTT above the timeout. func TestICMPProberLateReply(t *testing.T) { - p := newICMPProber(ICMPV4, newFakeICMPConn(), nil, "icmpv4") + p := newTestProber(t, ICMPV4, newFakeICMPConn(), nil, "icmpv4") if err := p.Accept("icmpv4://127.0.0.1"); err != nil { t.Fatal(err) } diff --git a/internal/prober/icmp_lifecycle_test.go b/internal/prober/icmp_lifecycle_test.go index 69fdfd2..5beff41 100644 --- a/internal/prober/icmp_lifecycle_test.go +++ b/internal/prober/icmp_lifecycle_test.go @@ -131,7 +131,7 @@ func (c *fakeICMPConn) isClosed() bool { func newFakeICMPProber(t *testing.T, conn *fakeICMPConn) *ICMPProber { t.Helper() - p := newICMPProber(ICMPV4, conn, []byte("fake"), "icmpv4") + p := newTestProber(t, ICMPV4, conn, []byte("fake"), "icmpv4") if err := p.Accept("icmpv4://127.0.0.1"); err != nil { t.Fatal(err) } @@ -209,7 +209,7 @@ func TestICMPProberStopBeforeStart(t *testing.T) { // The table helpers return their events instead of sending them, so the // prober never blocks on the channel while holding its mutex. func TestICMPProberTableEvents(t *testing.T) { - p := newICMPProber(ICMPV4, newFakeICMPConn(), nil, "icmpv4") + p := newTestProber(t, ICMPV4, newFakeICMPConn(), nil, "icmpv4") p.targets["127.0.0.1"] = icmpTarget{key: "127.0.0.1", name: "127.0.0.1"} p.targets["127.0.0.2"] = icmpTarget{key: "127.0.0.2", name: "localhost(127.0.0.2)"} p.timeout = time.Hour diff --git a/internal/prober/icmp_naming_test.go b/internal/prober/icmp_naming_test.go index 4bda161..96c2828 100644 --- a/internal/prober/icmp_naming_test.go +++ b/internal/prober/icmp_naming_test.go @@ -16,7 +16,7 @@ func TestEchoIDsDisjointFromTracer(t *testing.T) { var probers []*ICMPProber held := make(map[uint16]bool) for i := range 200 { - p := newICMPProber(ICMPV4, newFakeICMPConn(), nil, "icmpv4") + p := newTestProber(t, ICMPV4, newFakeICMPConn(), nil, "icmpv4") probers = append(probers, p) id := uint16(p.runID) if id == 0 || p.runID > 0xffff || held[id] { @@ -25,29 +25,30 @@ func TestEchoIDsDisjointFromTracer(t *testing.T) { held[id] = true // a tracer starting and stopping between probers (trace uses // icmpid.Acquire / Release for its transports) - tr := icmpid.Acquire() + lease, err := icmpid.Acquire() + if err != nil { + t.Fatal(err) + } + tr := lease.ID() if tr == 0 || held[tr] { t.Fatalf("tracer echo ID %#x: zero or held by a prober", tr) } if i%2 == 0 { - icmpid.Release(tr) + lease.Release() } else { held[tr] = true - defer icmpid.Release(tr) + defer lease.Release() } } for _, p := range probers { p.Stop() - if icmpid.InUse(uint16(p.runID)) { - t.Fatalf("echo ID %d still held after Stop", p.runID) - } } } // A custom ICMPv6 prober brackets bare addresses in its keys and names, so // that URL-style parsing (as by the path tracer) finds the whole address. func TestICMPCustomV6NameParses(t *testing.T) { - p := newICMPProber(ICMPV6, newFakeICMPConn(), nil, "myv6") + p := newTestProber(t, ICMPV6, newFakeICMPConn(), nil, "myv6") if err := p.Accept("myv6://2001:db8::1"); err != nil { t.Fatal(err) } @@ -72,7 +73,7 @@ func TestICMPCustomDefaultPlainTargets(t *testing.T) { "icmpv4": {Probe: ICMPV4, ICMP: &ICMPConfig{}}, } pm := NewProbeManager(config, "ping").(*probeManager) - p := newICMPProber(ICMPV4, newFakeICMPConn(), nil, "ping") + p := newTestProber(t, ICMPV4, newFakeICMPConn(), nil, "ping") pm.probers["ping"] = p if err := pm.AddTargets("127.0.0.1", "ping://127.0.0.2", "ping:127.0.0.3"); err != nil { t.Fatal(err) @@ -88,3 +89,14 @@ func TestICMPCustomDefaultPlainTargets(t *testing.T) { } } } + +// newTestProber builds a prober with an echo ID from the process-wide +// allocator. +func newTestProber(t testing.TB, typ ProbeType, c packetConn, body []byte, prefix string) *ICMPProber { + t.Helper() + lease, err := icmpid.Acquire() + if err != nil { + t.Fatal(err) + } + return newICMPProber(typ, c, body, prefix, lease) +} diff --git a/internal/trace/packet_test.go b/internal/trace/packet_test.go index 94aa9db..d959904 100644 --- a/internal/trace/packet_test.go +++ b/internal/trace/packet_test.go @@ -128,29 +128,42 @@ func TestParseReplyRejects(t *testing.T) { } } -// A tracer's echo ID comes from the process-wide allocator shared with the -// ICMP probers: it is held while the transport is open, other allocations -// (probers, other tracers) never get it, and closing releases it once. -func TestICMPTransportEchoIDShared(t *testing.T) { +// A tracer's echo ID is leased from the shared allocator for as long as the +// transport is open; closing releases it, and closing twice must not release +// the lease of a later owner of the same ID. +func TestICMPTransportEchoIDLease(t *testing.T) { + alloc := icmpid.New(1, 0, nil) + old := acquireID + acquireID = alloc.Acquire + defer func() { acquireID = old }() + tp, err := newICMPTransport(&net.IPAddr{IP: net.IPv4(127, 0, 0, 1)}, false) if err != nil { t.Skipf("no ICMP socket: %v", err) } - if tp.id == 0 || !icmpid.InUse(tp.id) { - t.Fatalf("echo ID %#x not held in the shared allocator", tp.id) + if tp.id == 0 || tp.lease == nil || tp.lease.ID() != tp.id { + t.Fatalf("echo ID %#x not backed by a lease", tp.id) } - for range 1000 { - id := icmpid.Acquire() - if id == tp.id { - t.Fatalf("shared allocator handed out the tracer ID %#x", id) + _ = tp.close() + // No quarantine: walking the whole space hands the closed ID to a new + // owner (the last ID acquired before exhaustion). + var owner *icmpid.Lease + for { + l, err := alloc.Acquire() + if err != nil { + break + } + if l.ID() == tp.id { + owner = l } - defer icmpid.Release(id) } - _ = tp.close() - if icmpid.InUse(tp.id) { - t.Fatalf("echo ID %#x still held after close", tp.id) + if owner == nil { + t.Fatalf("echo ID %#x not reacquirable after close", tp.id) + } + _ = tp.close() // second close must not release the new owner's lease + if _, err := alloc.Acquire(); err == nil { + t.Fatalf("second close released the new owner's ID %#x", tp.id) } - _ = tp.close() // closing twice must not release an ID reacquired since } // The echo ID decides whether a reply is ours. Replies from, or quoting, an diff --git a/internal/trace/transport_icmp.go b/internal/trace/transport_icmp.go index 3d92de2..44d144e 100644 --- a/internal/trace/transport_icmp.go +++ b/internal/trace/transport_icmp.go @@ -40,17 +40,24 @@ type icmpTransport struct { conn *icmp.PacketConn dstAddr net.Addr id uint16 + lease *icmpid.Lease checkID bool writeMu sync.Mutex // TTL is a socket option, so set+write must be atomic - closeOnce sync.Once } +// acquireID takes the echo ID; tests replace it to use a local allocator. +var acquireID = icmpid.Acquire + // newICMPTransport prefers a raw socket and falls back to an unprivileged // datagram ICMP socket; on macOS the fallback still receives Time Exceeded // messages, on Linux only the destination may be visible without privileges. func newICMPTransport(dst *net.IPAddr, v6 bool) (*icmpTransport, error) { - t := &icmpTransport{v6: v6, id: icmpid.Acquire()} + lease, err := acquireID() + if err != nil { + return nil, fmt.Errorf("cannot allocate ICMP echo ID: %w", err) + } + t := &icmpTransport{v6: v6, id: lease.ID(), lease: lease} rawNet, dgramNet, laddr := "ip4:icmp", "udp4", "0.0.0.0" if v6 { rawNet, dgramNet, laddr = "ip6:ipv6-icmp", "udp6", "::" @@ -66,7 +73,7 @@ func newICMPTransport(dst *net.IPAddr, v6 bool) (*icmpTransport, error) { // replies per socket, so only check the ID elsewhere (e.g. macOS). t.checkID = runtime.GOOS != "linux" } else { - icmpid.Release(t.id) + lease.Release() return nil, fmt.Errorf("cannot open ICMP socket (%v; %v): run as root or grant cap_net_raw", err, err2) } return t, nil @@ -76,7 +83,7 @@ func (t *icmpTransport) privileged() bool { return t.raw } func (t *icmpTransport) close() error { err := t.conn.Close() - t.closeOnce.Do(func() { icmpid.Release(t.id) }) + t.lease.Release() return err } From e53eeb2f331377592fb44373333243889d071b00 Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 8 Nov 2026 16:21:20 +0900 Subject: [PATCH 3/3] Format transport_icmp.go --- internal/trace/transport_icmp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/trace/transport_icmp.go b/internal/trace/transport_icmp.go index 44d144e..321e269 100644 --- a/internal/trace/transport_icmp.go +++ b/internal/trace/transport_icmp.go @@ -43,7 +43,7 @@ type icmpTransport struct { lease *icmpid.Lease checkID bool - writeMu sync.Mutex // TTL is a socket option, so set+write must be atomic + writeMu sync.Mutex // TTL is a socket option, so set+write must be atomic } // acquireID takes the echo ID; tests replace it to use a local allocator.