From 77550454a0140cbf4cab9a1c85c00330bc00d6d5 Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 15 Nov 2026 09:31:15 +0900 Subject: [PATCH 1/9] feat(prober): record ICMP reply TTL and HTTP remote address / TLS certificate Read echo replies through the ipv4/ipv6 PacketConns with the TTL / hop limit control message enabled and store it as ICMPDetails.ReplyTTL (0 when unknown, e.g. on Windows where control messages are unsupported). Add EstimateHops to estimate the return path hop count. Capture the remote address of the final HTTP response's connection via httptrace and a summary of the leaf certificate from resp.TLS. --- internal/prober/conn_info_test.go | 156 +++++++++++++++++++++++++ internal/prober/details.go | 19 +++ internal/prober/http.go | 36 +++++- internal/prober/icmp.go | 96 ++++++++++++++- internal/prober/icmp_isolation_test.go | 2 +- internal/prober/icmp_lifecycle_test.go | 8 +- 6 files changed, 307 insertions(+), 10 deletions(-) create mode 100644 internal/prober/conn_info_test.go diff --git a/internal/prober/conn_info_test.go b/internal/prober/conn_info_test.go new file mode 100644 index 0000000..af79d28 --- /dev/null +++ b/internal/prober/conn_info_test.go @@ -0,0 +1,156 @@ +package prober + +import ( + "fmt" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestEstimateHops(t *testing.T) { + tests := []struct { + ttl int + hops int + wantOK bool + }{ + {0, 0, false}, + {-1, 0, false}, + {256, 0, false}, + {64, 0, true}, + {54, 10, true}, + {1, 63, true}, + {65, 63, true}, + {128, 0, true}, + {117, 11, true}, + {129, 126, true}, + {255, 0, true}, + {243, 12, true}, + } + for _, tt := range tests { + hops, ok := EstimateHops(tt.ttl) + if hops != tt.hops || ok != tt.wantOK { + t.Errorf("EstimateHops(%d) = %d, %v; want %d, %v", tt.ttl, hops, ok, tt.hops, tt.wantOK) + } + } +} + +// ttlFakeConn reports a fixed TTL for every packet, like icmpConn does with +// control messages enabled. +type ttlFakeConn struct { + *fakeICMPConn + ttl int +} + +func (c *ttlFakeConn) ReadFromTTL(b []byte) (int, int, net.Addr, error) { + n, addr, err := c.ReadFrom(b) + return n, c.ttl, addr, err +} + +func icmpReplyTTL(t *testing.T, conn packetConn) int { + t.Helper() + p := newTestProber(t, ICMPV4, conn, []byte("fake"), "icmpv4") + if err := p.Accept("icmpv4://127.0.0.1"); err != nil { + t.Fatal(err) + } + r := make(chan *Event, 100) + done := make(chan error, 1) + go func() { done <- p.Start(r, time.Hour, time.Second) }() + defer func() { + p.Stop() + <-done + }() + for { + select { + case e := <-r: + if e.Result != SUCCESS { + continue + } + if e.Details == nil || e.Details.ICMP == nil { + t.Fatal("no ICMP details") + } + return e.Details.ICMP.ReplyTTL + case <-time.After(3 * time.Second): + t.Fatal("no reply") + } + } +} + +func TestICMPProberReplyTTL(t *testing.T) { + if got := icmpReplyTTL(t, &ttlFakeConn{fakeICMPConn: newFakeICMPConn(), ttl: 54}); got != 54 { + t.Errorf("ReplyTTL = %d, want 54", got) + } + // A connection without TTL support leaves it unknown + if got := icmpReplyTTL(t, newFakeICMPConn()); got != 0 { + t.Errorf("ReplyTTL without TTL support = %d, want 0", got) + } +} + +func TestHTTPProberConnInfoTLS(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "ok") + })) + defer srv.Close() + host := strings.TrimPrefix(srv.URL, "https://") + cert := srv.Certificate() + + for _, keepAlive := range []bool{false, true} { + t.Run(fmt.Sprintf("keep_alive=%v", keepAlive), func(t *testing.T) { + p := NewHTTPProber(HTTPS, &HTTPConfig{KeepAlive: keepAlive}, "https") + defer p.Stop() + // Twice, so that keep_alive reuses the connection + for i := range 2 { + e := probeOnce(t, p, "https://"+host) + if e.Result != SUCCESS { + t.Fatalf("probe %d: result = %v (%s)", i, e.Result, e.Message) + } + h := e.Details.HTTP + if h.RemoteAddr != host { + t.Errorf("probe %d: RemoteAddr = %q, want %q", i, h.RemoteAddr, host) + } + if h.TLS == nil { + t.Fatalf("probe %d: no TLS details", i) + } + if !strings.HasPrefix(h.TLS.Version, "TLS 1.") { + t.Errorf("Version = %q", h.TLS.Version) + } + if !h.TLS.NotAfter.Equal(cert.NotAfter) { + t.Errorf("NotAfter = %v, want %v", h.TLS.NotAfter, cert.NotAfter) + } + wantIssuer := cert.Issuer.CommonName + if wantIssuer == "" { + wantIssuer = cert.Issuer.String() + } + if h.TLS.Issuer != wantIssuer || h.TLS.Issuer == "" { + t.Errorf("Issuer = %q, want %q", h.TLS.Issuer, wantIssuer) + } + if h.TLS.SubjectCN != cert.Subject.CommonName { + t.Errorf("SubjectCN = %q, want %q", h.TLS.SubjectCN, cert.Subject.CommonName) + } + } + }) + } +} + +func TestHTTPProberConnInfoRedirect(t *testing.T) { + final := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "ok") + })) + defer final.Close() + first := httptest.NewServer(http.RedirectHandler(final.URL+"/x", http.StatusFound)) + defer first.Close() + + p := NewHTTPProber(HTTP, &HTTPConfig{}, "http") + e := probeOnce(t, p, "http://"+strings.TrimPrefix(first.URL, "http://")) + if e.Result != SUCCESS { + t.Fatalf("result = %v (%s)", e.Result, e.Message) + } + if want := strings.TrimPrefix(final.URL, "http://"); e.Details.HTTP.RemoteAddr != want { + t.Errorf("RemoteAddr = %q, want final server %q", e.Details.HTTP.RemoteAddr, want) + } + if e.Details.HTTP.TLS != nil { + t.Errorf("plain HTTP has TLS details: %+v", e.Details.HTTP.TLS) + } +} diff --git a/internal/prober/details.go b/internal/prober/details.go index 0c0eb86..f16b795 100644 --- a/internal/prober/details.go +++ b/internal/prober/details.go @@ -1,5 +1,7 @@ package prober +import "time" + // Probe detail information (TargetIP and TCPDetails removed) type ProbeDetails struct { ProbeType string `json:"probe_type"` @@ -19,6 +21,9 @@ type ICMPDetails struct { ICMPCode int `json:"icmp_code"` Checksum uint16 `json:"checksum"` Payload string `json:"payload"` // Actual payload content with length limit + // ReplyTTL is the TTL (IPv4) or hop limit (IPv6) of the echo reply as + // received, 0 when unknown (e.g. on Windows). See EstimateHops. + ReplyTTL int `json:"reply_ttl,omitempty"` } type HTTPDetails struct { @@ -28,6 +33,20 @@ type HTTPDetails struct { SizeTruncated bool `json:"size_truncated,omitempty"` Headers map[string]string `json:"headers,omitempty"` Redirects []string `json:"redirects,omitempty"` + // RemoteAddr is the IP:port of the connection that served the final + // response (after redirects). + RemoteAddr string `json:"remote_addr,omitempty"` + // TLS describes the final response's TLS connection; nil for plain HTTP. + TLS *TLSDetails `json:"tls,omitempty"` +} + +// TLSDetails summarizes a TLS connection and its leaf certificate. It is +// recorded whether or not the certificate was verified. +type TLSDetails struct { + Version string `json:"version"` // e.g. "TLS 1.3" + SubjectCN string `json:"subject_cn,omitempty"` // leaf certificate subject CN + Issuer string `json:"issuer,omitempty"` // issuer CN, or the full issuer name without a CN + NotAfter time.Time `json:"not_after,omitzero"` // leaf certificate expiry } type DNSDetails struct { diff --git a/internal/prober/http.go b/internal/prober/http.go index fc35ac7..bf57d53 100644 --- a/internal/prober/http.go +++ b/internal/prober/http.go @@ -8,6 +8,7 @@ import ( "io" "net" "net/http" + "net/http/httptrace" "net/url" "slices" "strings" @@ -210,7 +211,19 @@ func (p *HTTPProber) probe(r chan *Event, target string) { fail(err) return } - req = req.WithContext(context.WithValue(req.Context(), origHostKey{}, req.URL.Host)) + // GotConn runs for every request of the redirect chain (redirects + // inherit the context), so remote ends up as the final response's + // connection. It also reports reused keep-alive connections. + var remote string + trace := &httptrace.ClientTrace{ + GotConn: func(info httptrace.GotConnInfo) { + if info.Conn != nil && info.Conn.RemoteAddr() != nil { + remote = info.Conn.RemoteAddr().String() + } + }, + } + ctx := context.WithValue(req.Context(), origHostKey{}, req.URL.Host) + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) resp, err := p.client.Do(req) if err != nil { failErr(err) @@ -255,6 +268,8 @@ func (p *HTTPProber) probe(r chan *Event, target string) { SizeTruncated: sizeTruncated, Headers: headers, Redirects: redirects, + RemoteAddr: remote, + TLS: tlsDetails(resp.TLS), }, } @@ -262,6 +277,25 @@ func (p *HTTPProber) probe(r chan *Event, target string) { } } +// tlsDetails summarizes the TLS state of a response, nil for plain HTTP. +// The peer certificates are present even when verification is skipped. +func tlsDetails(cs *tls.ConnectionState) *TLSDetails { + if cs == nil { + return nil + } + d := &TLSDetails{Version: tls.VersionName(cs.Version)} + if len(cs.PeerCertificates) > 0 { + leaf := cs.PeerCertificates[0] + d.SubjectCN = leaf.Subject.CommonName + d.Issuer = leaf.Issuer.CommonName + if d.Issuer == "" { + d.Issuer = leaf.Issuer.String() + } + d.NotAfter = leaf.NotAfter + } + return d +} + // readBody reads up to maxBodyBytes of the body for expect_body matching and // determines the real response size. With a known Content-Length that is the // size; otherwise the remainder is read and discarded, up to maxCountBytes, diff --git a/internal/prober/icmp.go b/internal/prober/icmp.go index 40e3d34..1fd0a39 100644 --- a/internal/prober/icmp.go +++ b/internal/prober/icmp.go @@ -68,8 +68,88 @@ type ( WriteTo(b []byte, dst net.Addr) (int, error) Close() error } + + // ttlPacketConn is implemented by connections that also report the + // TTL (IPv4) or hop limit (IPv6) of each received packet. recvPkts uses + // it when available; ttl is 0 when unknown. + ttlPacketConn interface { + ReadFromTTL(b []byte) (n int, ttl int, src net.Addr, err error) + } + + // icmpConn wraps *icmp.PacketConn to read the reply TTL / hop limit + // through the ipv4/ipv6 control messages. Where those are not supported + // (Windows), cm is false and reads fall back to plain ReadFrom. + icmpConn struct { + *icmp.PacketConn + cm bool + } ) +// newICMPConn enables the TTL / hop limit control message on c. Failure is +// not an error: the prober still works, only without reply TTLs. +func newICMPConn(c *icmp.PacketConn, t ProbeType) *icmpConn { + ic := &icmpConn{PacketConn: c} + // Request only the TTL / hop limit flag: on darwin, combining it with + // FlagSrc is rejected by setsockopt (EINVAL) for some sockets. + if t == ICMPV4 { + if p := c.IPv4PacketConn(); p != nil { + ic.cm = p.SetControlMessage(ipv4.FlagTTL, true) == nil + } + } else if p := c.IPv6PacketConn(); p != nil { + ic.cm = p.SetControlMessage(ipv6.FlagHopLimit, true) == nil + } + return ic +} + +// ReadFromTTL reads an ICMP message like ReadFrom and returns the TTL or +// hop limit of the packet that carried it, 0 when unknown. The ipv4/ipv6 +// PacketConns strip the IP header like icmp.PacketConn.ReadFrom does, so +// b holds the ICMP message either way. +func (c *icmpConn) ReadFromTTL(b []byte) (int, int, net.Addr, error) { + if c.cm { + if p := c.IPv4PacketConn(); p != nil { + n, cm, src, err := p.ReadFrom(b) + ttl := 0 + if cm != nil { + ttl = cm.TTL + } + return n, ttl, src, err + } + if p := c.IPv6PacketConn(); p != nil { + n, cm, src, err := p.ReadFrom(b) + ttl := 0 + if cm != nil { + ttl = cm.HopLimit + } + return n, ttl, src, err + } + } + n, src, err := c.ReadFrom(b) + return n, 0, src, err +} + +// EstimateHops estimates the number of hops on the RETURN path (responder +// to us) from the TTL / hop limit of a reply. The responder's initial TTL +// is guessed as the smallest of the common defaults 64, 128 and 255 that is +// not below replyTTL, so hops = initial - replyTTL. ok is false for an +// unknown (<= 0) or out-of-range TTL. +// +// This is only an estimate: a responder may use another initial TTL, and +// the forward path (us to responder) can differ from the return path. The +// forward path length cannot be known from an echo reply; only a +// traceroute shows it. +func EstimateHops(replyTTL int) (hops int, ok bool) { + if replyTTL <= 0 || replyTTL > 255 { + return 0, false + } + for _, initial := range []int{64, 128, 255} { + if replyTTL <= initial { + return initial - replyTTL, true + } + } + return 0, false +} + // Validate validates the ICMP configuration func (cfg *ICMPConfig) Validate() error { // Basic validation - could be extended with more checks @@ -117,7 +197,7 @@ func NewICMPProber(t ProbeType, cfg *ICMPConfig, prefix string) (*ICMPProber, er _ = c.Close() return nil, fmt.Errorf("cannot allocate ICMP echo ID: %w", err) } - return newICMPProber(t, c, []byte(cfg.Body), prefix, lease), nil + return newICMPProber(t, newICMPConn(c, t), []byte(cfg.Body), prefix, lease), nil } // newICMPProber returns a prober on c with its own echo ID. The echo ID @@ -265,7 +345,7 @@ func (p *ICMPProber) answered(rt runTime, table map[string]bool) { // success marks the reply from addr to round runCnt and returns its event, // or nil for a duplicate or unexpected reply. Events are returned rather than // sent so that p.mu is never held while blocking on the event channel. -func (p *ICMPProber) success(runCnt int, addr string, payload icmp.Message, packetData []byte, packetSize int) *Event { +func (p *ICMPProber) success(runCnt int, addr string, payload icmp.Message, packetData []byte, packetSize, ttl int) *Event { p.mu.Lock() defer p.mu.Unlock() for k, table := range p.tables { @@ -291,6 +371,7 @@ func (p *ICMPProber) success(runCnt int, addr string, payload icmp.Message, pack ProbeType: string(p.version), ICMP: p.extractICMPDetails(runCnt, addr, payload, packetData, packetSize), } + details.ICMP.ReplyTTL = ttl return successEvent(key, displayName, k.sentTime, elapse, details) } return nil @@ -499,8 +580,15 @@ func (p *ICMPProber) recvPkts(r chan *Event) { proto = ipv6.ICMPTypeEchoReply.Protocol() } pktbuf := make([]byte, maxPacketSize) - for { + read := func() (int, int, net.Addr, error) { n, addr, err := p.c.ReadFrom(pktbuf) + return n, 0, addr, err + } + if tc, ok := p.c.(ttlPacketConn); ok { + read = func() (int, int, net.Addr, error) { return tc.ReadFromTTL(pktbuf) } + } + for { + n, ttl, addr, err := read() if err != nil { if errors.Is(err, net.ErrClosed) { return // closed by Start or Stop @@ -521,7 +609,7 @@ func (p *ICMPProber) recvPkts(r chan *Event) { if rm.Code == 0 { switch rm.Type { case ipv4.ICMPTypeEchoReply, ipv6.ICMPTypeEchoReply: - emit(r, p.success(int(seq), addr.String(), *rm, pktbuf[:n], n)) + emit(r, p.success(int(seq), addr.String(), *rm, pktbuf[:n], n, ttl)) } } } diff --git a/internal/prober/icmp_isolation_test.go b/internal/prober/icmp_isolation_test.go index a1995a6..4074e07 100644 --- a/internal/prober/icmp_isolation_test.go +++ b/internal/prober/icmp_isolation_test.go @@ -112,7 +112,7 @@ func TestICMPProberLateReply(t *testing.T) { p.timeout = 50 * time.Millisecond p.addTable(1, time.Now().Add(-100*time.Millisecond)) msg := icmp.Message{Type: ipv4.ICMPTypeEchoReply, Body: &icmp.Echo{}} - e := p.success(1, "127.0.0.1", msg, nil, 0) + e := p.success(1, "127.0.0.1", msg, nil, 0, 0) if e == nil || e.Result != TIMEOUT { t.Fatalf("late reply: success() = %+v, want TIMEOUT", e) } diff --git a/internal/prober/icmp_lifecycle_test.go b/internal/prober/icmp_lifecycle_test.go index 5beff41..dd2ac9f 100644 --- a/internal/prober/icmp_lifecycle_test.go +++ b/internal/prober/icmp_lifecycle_test.go @@ -216,16 +216,16 @@ func TestICMPProberTableEvents(t *testing.T) { p.addTable(1, time.Now().Add(-time.Millisecond)) msg := icmp.Message{Type: ipv4.ICMPTypeEchoReply, Body: &icmp.Echo{}} - if e := p.success(1, "127.0.0.1", msg, nil, 0); e == nil || e.Result != SUCCESS || e.Rtt <= 0 { + if e := p.success(1, "127.0.0.1", msg, nil, 0, 0); e == nil || e.Result != SUCCESS || e.Rtt <= 0 { t.Errorf("success() = %+v, want SUCCESS", e) } - if e := p.success(1, "127.0.0.1", msg, nil, 0); e != nil { + if e := p.success(1, "127.0.0.1", msg, nil, 0, 0); e != nil { t.Errorf("duplicate reply: success() = %+v, want nil", e) } - if e := p.success(2, "127.0.0.1", msg, nil, 0); e != nil { + if e := p.success(2, "127.0.0.1", msg, nil, 0, 0); e != nil { t.Errorf("unknown round: success() = %+v, want nil", e) } - if e := p.success(1, "192.0.2.1", msg, nil, 0); e != nil { + if e := p.success(1, "192.0.2.1", msg, nil, 0, 0); e != nil { t.Errorf("unknown address: success() = %+v, want nil", e) } if events, pending, next := p.checkTimeout(); len(events) != 0 || pending != 1 || next.IsZero() { From 72309e39ee6a644d9e8edc8e9efee9e7f6d6360c Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 15 Nov 2026 09:32:20 +0900 Subject: [PATCH 2/9] feat(stats): track reply TTL range and changes per host Keep the last, minimum and maximum reply TTL of each host, and the most recent 20 TTL changes (with a total count) between consecutive successful replies. Changes are detected on the outage tracker's settled, send-ordered result stream, so late replies cannot fake a change; reset clears them. --- internal/stats/interface.go | 3 + internal/stats/manager.go | 12 +++- internal/stats/metric.go | 14 ++++ internal/stats/outage.go | 17 ++++- internal/stats/ttl.go | 85 ++++++++++++++++++++++ internal/stats/ttl_test.go | 140 ++++++++++++++++++++++++++++++++++++ 6 files changed, 267 insertions(+), 4 deletions(-) create mode 100644 internal/stats/ttl.go create mode 100644 internal/stats/ttl_test.go diff --git a/internal/stats/interface.go b/internal/stats/interface.go index bf3cd6e..759b783 100644 --- a/internal/stats/interface.go +++ b/internal/stats/interface.go @@ -35,6 +35,9 @@ type Metrics interface { GetOutages() []Outage GetLastOutage() (Outage, bool) GetOutageSummary() OutageSummary + + // GetReplyTTL returns the ICMP reply TTL / hop limit statistics + GetReplyTTL() ReplyTTLStats } // MetricsProvider provides external API for metrics access diff --git a/internal/stats/manager.go b/internal/stats/manager.go index b622c42..c9d4c98 100644 --- a/internal/stats/manager.go +++ b/internal/stats/manager.go @@ -111,10 +111,20 @@ func (mm *metricsManager) SuccessWithDetails(host string, rtt time.Duration, sen Details: details, }) } - m.outage.add(sentTime, true, "", time.Now()) + ttl := replyTTL(details) + m.ttl.add(ttl, sentTime) + m.outage.addResult(probeResult{sentTime: sentTime, success: true, ttl: ttl}, time.Now()) mm.mu.Unlock() } +// replyTTL returns the ICMP reply TTL of details, 0 if unknown +func replyTTL(details *prober.ProbeDetails) int { + if details == nil || details.ICMP == nil { + return 0 + } + return details.ICMP.ReplyTTL +} + // Register failure for host func (mm *metricsManager) Failed(host string, sentTime time.Time, msg string) { m := mm.getMetrics(host) diff --git a/internal/stats/metric.go b/internal/stats/metric.go index 8842935..028723e 100644 --- a/internal/stats/metric.go +++ b/internal/stats/metric.go @@ -56,6 +56,7 @@ type metrics struct { LastFailDetail string history *TargetHistory // 履歴情報 outage *outageTracker + ttl ttlRange resetAt time.Time // results of probes sent before this are stale } @@ -113,6 +114,7 @@ func (m *metrics) Reset() { m.LastFailTime = time.Time{} m.LastSuccTime = time.Time{} m.LastFailDetail = "" + m.ttl = ttlRange{} if m.history != nil { m.history.Clear() } @@ -256,6 +258,18 @@ func (m *metrics) GetOutageSummary() OutageSummary { // GetRTTPercentiles returns RTT percentiles (0-100, nearest-rank method) // over the successful probes kept in history. +// GetReplyTTL returns the reply TTL / hop limit statistics. Last, Min and +// Max are up to date; Changes only include results applied in send order, +// so they may lag behind by the settle time. +func (m *metrics) GetReplyTTL() ReplyTTLStats { + s := ReplyTTLStats{Last: m.ttl.last, Min: m.ttl.min, Max: m.ttl.max} + if m.outage != nil { + s.Changes = append([]TTLChange(nil), m.outage.ttl.changes...) + s.ChangeCount = m.outage.ttl.count + } + return s +} + func (m *metrics) GetRTTPercentiles(ps ...float64) []time.Duration { res := make([]time.Duration, len(ps)) if m.history == nil { diff --git a/internal/stats/outage.go b/internal/stats/outage.go index cd7e4d9..0512260 100644 --- a/internal/stats/outage.go +++ b/internal/stats/outage.go @@ -54,6 +54,7 @@ type probeResult struct { sentTime time.Time success bool err string + ttl int // reply TTL / hop limit of a success, 0 if unknown } // outageTracker detects outages for one target. @@ -61,6 +62,7 @@ type probeResult struct { // Probe results can arrive out of send order (e.g. a timeout is reported after // a later probe already succeeded), so results are held for `settle` and // applied in send-time order once no earlier result can still arrive. +// Reply TTL changes are detected on the same ordered stream. type outageTracker struct { settle time.Duration @@ -75,6 +77,8 @@ type outageTracker struct { count int total time.Duration longest time.Duration + + ttl ttlChangeTracker } func newOutageTracker(settle time.Duration) *outageTracker { @@ -83,10 +87,15 @@ func newOutageTracker(settle time.Duration) *outageTracker { // add records a probe result and applies every result that has settled func (t *outageTracker) add(sentTime time.Time, success bool, err string, now time.Time) { - if sentTime.IsZero() { - sentTime = now + t.addResult(probeResult{sentTime: sentTime, success: success, err: err}, now) +} + +// addResult is add with the full result, including the reply TTL +func (t *outageTracker) addResult(r probeResult, now time.Time) { + if r.sentTime.IsZero() { + r.sentTime = now } - r := probeResult{sentTime: sentTime, success: success, err: err} + sentTime := r.sentTime i := sort.Search(len(t.pending), func(i int) bool { return t.pending[i].sentTime.After(sentTime) }) @@ -124,6 +133,7 @@ func (t *outageTracker) flushUntil(deadline time.Time) { } func (t *outageTracker) apply(r probeResult) { + t.ttl.apply(r) switch { case r.success: t.lostRun = t.lostRun[:0] @@ -197,6 +207,7 @@ func (t *outageTracker) clone() *outageTracker { c := *t c.pending = append([]probeResult(nil), t.pending...) c.lostRun = append([]probeResult(nil), t.lostRun...) + c.ttl = t.ttl.clone() // closed is append-only and trimming reallocates it, so the clone can // share the backing array as long as its capacity is capped. c.closed = t.closed[:len(t.closed):len(t.closed)] diff --git a/internal/stats/ttl.go b/internal/stats/ttl.go new file mode 100644 index 0000000..41bb3fe --- /dev/null +++ b/internal/stats/ttl.go @@ -0,0 +1,85 @@ +package stats + +import "time" + +// maxTTLChanges is the number of recent reply TTL changes kept per target. +// ReplyTTLStats.ChangeCount keeps counting beyond it. +const maxTTLChanges = 20 + +// TTLChange is a change of the reply TTL (IPv4) / hop limit (IPv6) between +// two consecutive successful replies, in send order. +type TTLChange struct { + Time time.Time // send time of the probe whose reply had the new TTL + Old int + New int +} + +// HopsDelta returns the change of the estimated return path length: Old - +// New, positive when the reply crossed more hops. It assumes the responder's +// initial TTL did not change. +func (c TTLChange) HopsDelta() int { + return c.Old - c.New +} + +// ReplyTTLStats describes the reply TTLs / hop limits of a target. All +// values are 0 (and Changes empty) when no reply carried a known TTL. +type ReplyTTLStats struct { + Last int // TTL of the reply to the most recently sent probe + Min int + Max int + // Changes holds the most recent changes (at most maxTTLChanges), + // oldest first; ChangeCount counts all of them. + Changes []TTLChange + ChangeCount int +} + +// ttlRange tracks the last, minimum and maximum reply TTL. It is updated +// as results arrive: newest send time wins for Last, like LastRTT. +type ttlRange struct { + last int + lastSent time.Time + min, max int +} + +func (r *ttlRange) add(ttl int, sentTime time.Time) { + if ttl <= 0 { + return + } + if r.last == 0 || !sentTime.Before(r.lastSent) { + r.last, r.lastSent = ttl, sentTime + } + if r.min == 0 || ttl < r.min { + r.min = ttl + } + if ttl > r.max { + r.max = ttl + } +} + +// ttlChangeTracker detects TTL changes between consecutive successful +// replies. It is fed by outageTracker, which applies results in send order +// once they have settled, so a late reply cannot fake a change. +type ttlChangeTracker struct { + prev int // TTL of the previous successful reply with a known TTL + changes []TTLChange + count int +} + +func (t *ttlChangeTracker) apply(r probeResult) { + if !r.success || r.ttl <= 0 { + return + } + if t.prev != 0 && r.ttl != t.prev { + t.changes = append(t.changes, TTLChange{Time: r.sentTime, Old: t.prev, New: r.ttl}) + t.count++ + if len(t.changes) > maxTTLChanges { + t.changes = append([]TTLChange(nil), t.changes[len(t.changes)-maxTTLChanges:]...) + } + } + t.prev = r.ttl +} + +func (t ttlChangeTracker) clone() ttlChangeTracker { + t.changes = append([]TTLChange(nil), t.changes...) + return t +} diff --git a/internal/stats/ttl_test.go b/internal/stats/ttl_test.go new file mode 100644 index 0000000..5e9ab03 --- /dev/null +++ b/internal/stats/ttl_test.go @@ -0,0 +1,140 @@ +package stats + +import ( + "testing" + "time" + + "github.com/servak/mping/internal/prober" +) + +func icmpTTLEvent(sent time.Time, ttl int) *prober.Event { + return &prober.Event{ + Key: "h", DisplayName: "h", Result: prober.SUCCESS, SentTime: sent, Rtt: time.Millisecond, + Details: &prober.ProbeDetails{ProbeType: "icmpv4", ICMP: &prober.ICMPDetails{ReplyTTL: ttl}}, + } +} + +func feedEvents(mm MetricsManager, events ...*prober.Event) { + ch := make(chan *prober.Event, len(events)) + for _, e := range events { + ch <- e + } + close(ch) + <-mm.Subscribe(ch) +} + +func replyTTLOf(t *testing.T, mm MetricsManager) ReplyTTLStats { + t.Helper() + ms := mm.SortBy(Host, true) + if len(ms) != 1 { + t.Fatalf("%d metrics, want 1", len(ms)) + } + return ms[0].GetReplyTTL() +} + +func TestReplyTTLTracking(t *testing.T) { + mm := NewMetricsManager() + feedEvents(mm, + icmpTTLEvent(at(0), 54), + icmpTTLEvent(at(1), 54), + icmpTTLEvent(at(2), 0), // unknown: ignored + &prober.Event{Key: "h", Result: prober.TIMEOUT, SentTime: at(3), Message: "timeout"}, + icmpTTLEvent(at(4), 52), + icmpTTLEvent(at(5), 52), + icmpTTLEvent(at(6), 0), + icmpTTLEvent(at(7), 54), + ) + s := replyTTLOf(t, mm) + if s.Last != 54 || s.Min != 52 || s.Max != 54 { + t.Errorf("last/min/max = %d/%d/%d, want 54/52/54", s.Last, s.Min, s.Max) + } + want := []TTLChange{{Time: at(4), Old: 54, New: 52}, {Time: at(7), Old: 52, New: 54}} + if s.ChangeCount != 2 || len(s.Changes) != 2 { + t.Fatalf("changes = %+v (count %d), want %+v", s.Changes, s.ChangeCount, want) + } + for i, c := range s.Changes { + if !c.Time.Equal(want[i].Time) || c.Old != want[i].Old || c.New != want[i].New { + t.Errorf("change %d = %+v, want %+v", i, c, want[i]) + } + } + if d := s.Changes[0].HopsDelta(); d != 2 { + t.Errorf("HopsDelta = %d, want 2", d) + } +} + +func TestReplyTTLNoData(t *testing.T) { + mm := NewMetricsManager() + feedEvents(mm, icmpTTLEvent(at(0), 0), icmpTTLEvent(at(1), 0)) + if s := replyTTLOf(t, mm); s.Last != 0 || s.Min != 0 || s.Max != 0 || s.ChangeCount != 0 || len(s.Changes) != 0 { + t.Errorf("stats without TTLs = %+v", s) + } +} + +func TestReplyTTLChangesCapped(t *testing.T) { + mm := NewMetricsManager() + var events []*prober.Event + n := maxTTLChanges + 5 + for i := 0; i <= n; i++ { + events = append(events, icmpTTLEvent(at(i), 50+i%2)) + } + feedEvents(mm, events...) + s := replyTTLOf(t, mm) + if s.ChangeCount != n { + t.Errorf("ChangeCount = %d, want %d", s.ChangeCount, n) + } + if len(s.Changes) != maxTTLChanges { + t.Fatalf("kept %d changes, want %d", len(s.Changes), maxTTLChanges) + } + if last := s.Changes[len(s.Changes)-1]; !last.Time.Equal(at(n)) { + t.Errorf("newest change at %v, want %v", last.Time, at(n)) + } + if first := s.Changes[0]; !first.Time.Equal(at(n - maxTTLChanges + 1)) { + t.Errorf("oldest kept change at %v, want %v", first.Time, at(n-maxTTLChanges+1)) + } +} + +// Results that arrive out of send order are applied in send order, so a +// late reply does not produce spurious changes. +func TestReplyTTLOutOfOrder(t *testing.T) { + mm := NewMetricsManagerWithOptions(Options{SettleTime: time.Hour}) + feedEvents(mm, + icmpTTLEvent(at(0), 54), + icmpTTLEvent(at(2), 52), + icmpTTLEvent(at(1), 54), // late + ) + s := replyTTLOf(t, mm) + if s.Last != 52 { + t.Errorf("Last = %d, want 52 (newest send time)", s.Last) + } + if s.ChangeCount != 1 || len(s.Changes) != 1 || !s.Changes[0].Time.Equal(at(2)) { + t.Errorf("changes = %+v (count %d), want one at %v", s.Changes, s.ChangeCount, at(2)) + } +} + +func TestReplyTTLReset(t *testing.T) { + mm := NewMetricsManager() + now := time.Now() + feedEvents(mm, icmpTTLEvent(now.Add(-2*time.Second), 54), icmpTTLEvent(now.Add(-time.Second), 52)) + mm.ResetAllMetrics() + // A result of a probe sent before the reset is dropped + feedEvents(mm, icmpTTLEvent(now.Add(-500*time.Millisecond), 60)) + if s := replyTTLOf(t, mm); s.Last != 0 || s.Min != 0 || s.Max != 0 || s.ChangeCount != 0 || len(s.Changes) != 0 { + t.Errorf("after reset = %+v, want empty", s) + } + // The first reply after the reset is not a change + feedEvents(mm, icmpTTLEvent(time.Now(), 50)) + if s := replyTTLOf(t, mm); s.Last != 50 || s.Min != 50 || s.Max != 50 || s.ChangeCount != 0 { + t.Errorf("after reset and one reply = %+v", s) + } +} + +// Snapshots must not share the change list with the live metrics. +func TestReplyTTLSnapshotIsolated(t *testing.T) { + mm := NewMetricsManager() + feedEvents(mm, icmpTTLEvent(at(0), 54), icmpTTLEvent(at(1), 52)) + before := replyTTLOf(t, mm) + feedEvents(mm, icmpTTLEvent(at(2), 54)) + if len(before.Changes) != 1 || before.Changes[0].New != 52 { + t.Errorf("earlier snapshot changed: %+v", before.Changes) + } +} From 06eb74b5c59e96e9f1c0d1054cc4e91415bb2e0e Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 15 Nov 2026 09:33:29 +0900 Subject: [PATCH 3/9] feat(tui): show reply TTL, TTL changes and HTTP connection info in host details The host detail panel shows the reply TTL with the estimated return path hop count, the observed range and the recent TTL changes, and for HTTP(S) targets the remote address, TLS version and leaf certificate with its expiry (yellow within 30 days, red within 7 days or expired). --- internal/ui/tui/panels/conn_info.go | 148 +++++++++++++++++++++++ internal/ui/tui/panels/conn_info_test.go | 121 ++++++++++++++++++ internal/ui/tui/panels/format.go | 4 + 3 files changed, 273 insertions(+) create mode 100644 internal/ui/tui/panels/conn_info.go create mode 100644 internal/ui/tui/panels/conn_info_test.go diff --git a/internal/ui/tui/panels/conn_info.go b/internal/ui/tui/panels/conn_info.go new file mode 100644 index 0000000..585ac61 --- /dev/null +++ b/internal/ui/tui/panels/conn_info.go @@ -0,0 +1,148 @@ +package panels + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/rivo/tview" + + "github.com/servak/mping/internal/prober" + "github.com/servak/mping/internal/stats" + "github.com/servak/mping/internal/ui/shared" +) + +// maxDetailTTLChanges is the number of recent TTL changes listed in host details +const maxDetailTTLChanges = 10 + +// formatConnInfo renders what is known about the path and the peer of a +// host: the ICMP reply TTL and its changes, and the remote address and TLS +// certificate of the latest HTTP response. It returns "" when nothing is +// known, e.g. on Windows (no reply TTL) or for other probe types. +func formatConnInfo(metric stats.Metrics, theme *shared.Theme, now time.Time) string { + var sections []string + if s := formatReplyTTL(metric.GetReplyTTL(), theme); s != "" { + sections = append(sections, s) + } + if s := formatHTTPConn(latestHTTPDetails(metric), theme, now); s != "" { + sections = append(sections, s) + } + return strings.Join(sections, "\n\n") +} + +// formatReplyTTL renders "Reply TTL: 54 (≈10 hops, est.), range 52–54" +// followed by the recent TTL changes, newest first. +func formatReplyTTL(s stats.ReplyTTLStats, theme *shared.Theme) string { + if s.Last <= 0 { + return "" + } + value := fmt.Sprintf("%d", s.Last) + if hops, ok := prober.EstimateHops(s.Last); ok { + value += fmt.Sprintf(" (≈%d %s, est.)", hops, plural(hops, "hop", "hops")) + } + if s.Min != s.Max { + value += fmt.Sprintf(", [%s]range %d–%d[%s]", theme.Warning, s.Min, s.Max, theme.Primary) + } + var sb strings.Builder + sb.WriteString(formatDetailRows([]detailRow{{"Reply TTL", theme.Accent, value}}, theme)) + if s.ChangeCount == 0 { + return sb.String() + } + fmt.Fprintf(&sb, "\n[%s]TTL Changes:[%s] %d", theme.Accent, theme.Primary, s.ChangeCount) + shown := 0 + for i := len(s.Changes) - 1; i >= 0 && shown < maxDetailTTLChanges; i-- { + c := s.Changes[i] + d := c.HopsDelta() + fmt.Fprintf(&sb, "\n[%s]%s[%s] [%s]TTL %d→%d (%+d %s)[%s]", + theme.Timestamp, shared.TimeFormater(c.Time), theme.Primary, + theme.Warning, c.Old, c.New, d, plural(d, "hop", "hops"), theme.Primary) + shown++ + } + if more := s.ChangeCount - shown; more > 0 { + fmt.Fprintf(&sb, "\n[%s]... %d more[%s]", theme.Secondary, more, theme.Primary) + } + return sb.String() +} + +// latestHTTPDetails returns the HTTP details of the most recently sent +// successful probe that carries connection info, or nil. +func latestHTTPDetails(metric stats.Metrics) *prober.HTTPDetails { + var ( + best *prober.HTTPDetails + bestAt time.Time + ) + for _, e := range metric.GetRecentHistory(math.MaxInt) { + if !e.Success || e.Details == nil || e.Details.HTTP == nil { + continue + } + h := e.Details.HTTP + if h.RemoteAddr == "" && h.TLS == nil { + continue + } + if best == nil || e.Timestamp.After(bestAt) { + best, bestAt = h, e.Timestamp + } + } + return best +} + +// formatHTTPConn renders the remote address and the TLS certificate +func formatHTTPConn(h *prober.HTTPDetails, theme *shared.Theme, now time.Time) string { + if h == nil { + return "" + } + var rows []detailRow + if h.RemoteAddr != "" { + rows = append(rows, detailRow{"Remote", theme.Accent, tview.Escape(h.RemoteAddr)}) + } + if t := h.TLS; t != nil { + if t.Version != "" { + rows = append(rows, detailRow{"TLS", theme.Accent, tview.Escape(t.Version)}) + } + var parts []string + if t.SubjectCN != "" { + parts = append(parts, "CN "+tview.Escape(t.SubjectCN)) + } + if t.Issuer != "" { + parts = append(parts, "Issuer "+tview.Escape(t.Issuer)) + } + if !t.NotAfter.IsZero() { + parts = append(parts, formatCertExpiry(t.NotAfter, now, theme)) + } + if len(parts) > 0 { + rows = append(rows, detailRow{"Certificate", theme.Accent, strings.Join(parts, " / ")}) + } + } + return formatDetailRows(rows, theme) +} + +// formatCertExpiry renders "expires 2027-01-02 (in 48d)": red when expired +// or within 7 days, yellow within 30 days. +func formatCertExpiry(notAfter, now time.Time, theme *shared.Theme) string { + left := notAfter.Sub(now) + var rel string + switch { + case left <= 0: + rel = "expired" + case left < 24*time.Hour: + rel = fmt.Sprintf("in %dh", int(left.Hours())) + default: + rel = fmt.Sprintf("in %dd", int(left.Hours()/24)) + } + s := fmt.Sprintf("expires %s (%s)", notAfter.UTC().Format("2006-01-02"), rel) + switch { + case left < 7*24*time.Hour: + return fmt.Sprintf("[%s]%s[%s]", theme.Error, s, theme.Primary) + case left < 30*24*time.Hour: + return fmt.Sprintf("[%s]%s[%s]", theme.Warning, s, theme.Primary) + } + return s +} + +func plural(n int, one, many string) string { + if n == 1 || n == -1 { + return one + } + return many +} diff --git a/internal/ui/tui/panels/conn_info_test.go b/internal/ui/tui/panels/conn_info_test.go new file mode 100644 index 0000000..16ef7c4 --- /dev/null +++ b/internal/ui/tui/panels/conn_info_test.go @@ -0,0 +1,121 @@ +package panels + +import ( + "strings" + "testing" + "time" + + "github.com/rivo/tview" + + "github.com/servak/mping/internal/prober" + "github.com/servak/mping/internal/stats" + "github.com/servak/mping/internal/stats/statstest" + "github.com/servak/mping/internal/ui/shared" +) + +// plainText strips the tview color tags +func plainText(s string) string { + tv := tview.NewTextView().SetDynamicColors(true) + tv.SetText(s) + return tv.GetText(true) +} + +func onlyMetrics(t *testing.T, mm stats.MetricsManager) stats.Metrics { + t.Helper() + ms := mm.SortBy(stats.Host, true) + if len(ms) != 1 { + t.Fatalf("%d metrics, want 1", len(ms)) + } + return ms[0] +} + +func TestFormatConnInfoReplyTTL(t *testing.T) { + theme := shared.PredefinedThemes["dark"] + ts := time.Date(2026, 1, 1, 9, 12, 0, 0, time.Local) + ev := func(i, ttl int) *prober.Event { + return &prober.Event{Key: "h", DisplayName: "h", Result: prober.SUCCESS, SentTime: ts.Add(time.Duration(i) * time.Second), Rtt: time.Millisecond, + Details: &prober.ProbeDetails{ProbeType: "icmpv4", ICMP: &prober.ICMPDetails{ReplyTTL: ttl}}} + } + mm := statstest.FromEvents(t, ev(0, 54), ev(3, 52), ev(4, 53)) + got := plainText(formatConnInfo(onlyMetrics(t, mm), &theme, ts)) + for _, want := range []string{ + "Reply TTL: 53 (≈11 hops, est.), range 52–54", + "TTL Changes: 2", + "09:12:04 TTL 52→53 (-1 hop)", + "09:12:03 TTL 54→52 (+2 hops)", + } { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } + if i, j := strings.Index(got, "09:12:04"), strings.Index(got, "09:12:03"); i > j { + t.Errorf("changes not newest first:\n%s", got) + } + + // Truncated list shows how many more there are + var events []*prober.Event + for i := 0; i <= 15; i++ { + events = append(events, ev(i, 50+i%2)) + } + got = plainText(formatConnInfo(onlyMetrics(t, statstest.FromEvents(t, events...)), &theme, ts)) + if !strings.Contains(got, "TTL Changes: 15") || !strings.Contains(got, "... 5 more") { + t.Errorf("truncation not shown:\n%s", got) + } + + // Constant TTL: no range, no changes + got = plainText(formatConnInfo(onlyMetrics(t, statstest.FromEvents(t, ev(0, 64), ev(1, 64))), &theme, ts)) + if got != "Reply TTL: 64 (≈0 hops, est.)" { + t.Errorf("constant TTL = %q", got) + } + + // Unknown TTL (e.g. Windows): nothing at all + if got := formatConnInfo(onlyMetrics(t, statstest.FromEvents(t, ev(0, 0))), &theme, ts); got != "" { + t.Errorf("unknown TTL = %q, want empty", got) + } +} + +func TestFormatConnInfoHTTP(t *testing.T) { + theme := shared.PredefinedThemes["dark"] + now := time.Date(2026, 11, 15, 0, 0, 0, 0, time.UTC) + ev := func(i int, h *prober.HTTPDetails) *prober.Event { + return &prober.Event{Key: "h", DisplayName: "h", Result: prober.SUCCESS, SentTime: now.Add(time.Duration(i) * time.Second), Rtt: time.Millisecond, + Details: &prober.ProbeDetails{ProbeType: "https", HTTP: h}} + } + tlsd := &prober.TLSDetails{Version: "TLS 1.3", SubjectCN: "example.com", Issuer: "R3", NotAfter: now.Add(48 * 24 * time.Hour)} + mm := statstest.FromEvents(t, + ev(0, &prober.HTTPDetails{StatusCode: 200, RemoteAddr: "192.0.2.9:443"}), + ev(1, &prober.HTTPDetails{StatusCode: 200, RemoteAddr: "1.1.1.1:443", TLS: tlsd}), + ev(2, &prober.HTTPDetails{StatusCode: 200}), // no connection info + ) + raw := formatConnInfo(onlyMetrics(t, mm), &theme, now) + got := plainText(raw) + for _, want := range []string{ + "Remote: 1.1.1.1:443", + "TLS: TLS 1.3", + "Certificate: CN example.com / Issuer R3 / expires 2027-01-02 (in 48d)", + } { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } + if strings.Contains(raw, "["+theme.Warning+"]expires") || strings.Contains(raw, "["+theme.Error+"]expires") { + t.Errorf("48 days left must not be highlighted: %q", raw) + } + + tests := []struct { + left time.Duration + color string + rel string + }{ + {20 * 24 * time.Hour, theme.Warning, "(in 20d)"}, + {3 * 24 * time.Hour, theme.Error, "(in 3d)"}, + {5 * time.Hour, theme.Error, "(in 5h)"}, + {-time.Hour, theme.Error, "(expired)"}, + } + for _, tt := range tests { + s := formatCertExpiry(now.Add(tt.left), now, &theme) + if !strings.HasPrefix(s, "["+tt.color+"]expires") || !strings.Contains(s, tt.rel) { + t.Errorf("expiry in %v = %q, want color %s and %s", tt.left, s, tt.color, tt.rel) + } + } +} diff --git a/internal/ui/tui/panels/format.go b/internal/ui/tui/panels/format.go index e5d9c77..35f7149 100644 --- a/internal/ui/tui/panels/format.go +++ b/internal/ui/tui/panels/format.go @@ -72,6 +72,10 @@ func FormatHostDetail(metric stats.Metrics, theme *shared.Theme) string { {"Last Error", accent, tview.Escape(metric.GetLastFailDetail())}, }, theme) + if conn := formatConnInfo(metric, theme, time.Now()); conn != "" { + basicInfo += "\n\n" + conn + } + basicInfo += "\n\n" + FormatOutages(metric, theme) // Add history section From a237732da17f60f044263f18758c99984ecbb97e Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 15 Nov 2026 09:34:23 +0900 Subject: [PATCH 4/9] docs: document reply TTL and connection info in host details --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index 77d050f..65a8a6b 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,28 @@ The host list has a **History** column with the last 20 probes (oldest on the le Latency spikes and loss bursts are visible at a glance. The host details and JSON/CSV output include RTT p50/p95/p99 over the last 100 probes. +## Path and Peer Changes + +During maintenance, a route change, a DNS / load balancer switchover or a certificate replacement often keeps the target reachable but changes *what* answers. The TUI host detail panel (`v`) shows the facts that reveal this: + +``` +Reply TTL: 52 (≈12 hops, est.), range 52–54 +TTL Changes: 1 +09:12:03 TTL 54→52 (+2 hops) + +Remote: 1.1.1.1:443 +TLS: TLS 1.3 +Certificate: CN example.com / Issuer R3 / expires 2027-01-02 (in 48d) +``` + +- **ICMP**: the TTL (IPv4) / hop limit (IPv6) of the last echo reply, the hop count estimated from it, the range seen so far and the most recent TTL changes (`+2 hops` = the reply crossed two more hops). A change is a successful reply whose TTL differs from the previous successful reply's. +- **HTTP(S)**: the IP:port of the connection that served the final response (after redirects) and the TLS version and leaf certificate (subject CN, issuer, expiry; yellow within 30 days, red within 7 days or expired). The certificate is shown even with `skip_verify` (the default). + +Caveats: +- The hop count is an **estimate of the return path** (responder to mping): the responder's initial TTL is guessed as the smallest of 64, 128 and 255 that fits. The forward path can differ (asymmetric routing) and cannot be seen in an echo reply; use the path view (`p`) for that. +- A change to a path of the same length (e.g. another ECMP branch) does not change the TTL; it only shows in the path view. +- Windows does not report the reply TTL, so the ICMP lines are not shown there. + ## Target Expansion mping supports powerful target expansion syntax for monitoring large numbers of hosts efficiently: From 4c2fdb124dcd40c88873e546184db5689f51a7db Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 15 Nov 2026 09:40:20 +0900 Subject: [PATCH 5/9] fix(prober): attach connection info to rejected HTTP responses Unexpected status / body failures (and body read errors) now carry the HTTP details with the remote address and TLS info, so a switchover to a peer that answers 503 is visible. --- internal/prober/conn_info_test.go | 30 ++++++++++++++ internal/prober/http.go | 66 ++++++++++++++++--------------- 2 files changed, 65 insertions(+), 31 deletions(-) diff --git a/internal/prober/conn_info_test.go b/internal/prober/conn_info_test.go index af79d28..01434d0 100644 --- a/internal/prober/conn_info_test.go +++ b/internal/prober/conn_info_test.go @@ -134,6 +134,36 @@ func TestHTTPProberConnInfoTLS(t *testing.T) { } } +// A response that is received but rejected still reports its peer, so a +// switchover to a peer answering 503 is visible. +func TestHTTPProberConnInfoOnRejectedResponse(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + fmt.Fprint(w, "down") + })) + defer srv.Close() + host := strings.TrimPrefix(srv.URL, "https://") + + for name, cfg := range map[string]*HTTPConfig{ + "status": {ExpectCodes: "200"}, + "body": {ExpectBody: "ok"}, + } { + t.Run(name, func(t *testing.T) { + e := probeOnce(t, NewHTTPProber(HTTPS, cfg, "https"), "https://"+host) + if e.Result != FAILED { + t.Fatalf("result = %v (%s), want FAILED", e.Result, e.Message) + } + if e.Details == nil || e.Details.HTTP == nil { + t.Fatal("rejected response without HTTP details") + } + h := e.Details.HTTP + if h.RemoteAddr != host || h.TLS == nil || h.StatusCode != http.StatusServiceUnavailable { + t.Errorf("details = %+v", h) + } + }) + } +} + func TestHTTPProberConnInfoRedirect(t *testing.T) { final := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, "ok") diff --git a/internal/prober/http.go b/internal/prober/http.go index bf57d53..97a0138 100644 --- a/internal/prober/http.go +++ b/internal/prober/http.go @@ -192,8 +192,13 @@ func (p *HTTPProber) resolveURL(target string) (scheme, rest string) { func (p *HTTPProber) probe(r chan *Event, target string) { now := time.Now() r <- sentEvent(target, target, now) + // details is set once a response has been received, so that failures + // with a response (unexpected status or body) still report the peer. + var details *ProbeDetails fail := func(err error) { - r <- failureEvent(target, target, FAILED, now, time.Since(now), err.Error()) + e := failureEvent(target, target, FAILED, now, time.Since(now), err.Error()) + e.Details = details + r <- e } // Convert target to actual HTTP URL @@ -201,7 +206,9 @@ func (p *HTTPProber) probe(r chan *Event, target string) { actualURL := scheme + "://" + rest failErr := func(err error) { if classifyErr(err) == TIMEOUT { - r <- failureEvent(target, target, TIMEOUT, now, time.Since(now), "timeout") + e := failureEvent(target, target, TIMEOUT, now, time.Since(now), "timeout") + e.Details = details + r <- e } else { fail(err) } @@ -231,50 +238,47 @@ func (p *HTTPProber) probe(r chan *Event, target string) { } defer resp.Body.Close() + details = &ProbeDetails{ + ProbeType: scheme, + HTTP: &HTTPDetails{ + StatusCode: resp.StatusCode, + RemoteAddr: remote, + TLS: tlsDetails(resp.TLS), + }, + } body, size, sizeTruncated, bodyTruncated, err := readBody(resp) if err != nil { failErr(err) return } + details.HTTP.ResponseSize, details.HTTP.SizeTruncated = size, sizeTruncated if !p.isExpectedStatusCode(resp.StatusCode) { fail(fmt.Errorf("unexpected status code: %d", resp.StatusCode)) - } else if p.config.ExpectBody != "" && p.config.ExpectBody != strings.TrimRight(string(body), "\n") { + return + } + if p.config.ExpectBody != "" && p.config.ExpectBody != strings.TrimRight(string(body), "\n") { msg := "invalid body" if bodyTruncated { msg += " (response body truncated at 1 MiB)" } fail(errors.New(msg)) - } else { - // Create HTTP detail information - headers := make(map[string]string) - for key, values := range resp.Header { - if len(values) > 0 { - headers[key] = values[0] // Get only the first value - } - } - - // Compare against the URL actually requested (not the target, which - // may carry a custom prefix) to detect redirects. - var redirects []string - if finalURL := resp.Request.URL.String(); finalURL != actualURL { - redirects = append(redirects, finalURL) - } + return + } - details := &ProbeDetails{ - ProbeType: scheme, - HTTP: &HTTPDetails{ - StatusCode: resp.StatusCode, - ResponseSize: size, - SizeTruncated: sizeTruncated, - Headers: headers, - Redirects: redirects, - RemoteAddr: remote, - TLS: tlsDetails(resp.TLS), - }, + headers := make(map[string]string) + for key, values := range resp.Header { + if len(values) > 0 { + headers[key] = values[0] // Get only the first value } - - r <- successEvent(target, target, now, time.Since(now), details) } + // Compare against the URL actually requested (not the target, which + // may carry a custom prefix) to detect redirects. + var redirects []string + if finalURL := resp.Request.URL.String(); finalURL != actualURL { + redirects = append(redirects, finalURL) + } + details.HTTP.Headers, details.HTTP.Redirects = headers, redirects + r <- successEvent(target, target, now, time.Since(now), details) } // tlsDetails summarizes the TLS state of a response, nil for plain HTTP. From 0cf70d5f60eb9b0c061bc8ee251e3fd1372c3765 Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 15 Nov 2026 09:40:20 +0900 Subject: [PATCH 6/9] fix(stats): keep latest HTTP peer and compute TTL range over recent replies Track the remote address / TLS info of the latest HTTP(S) response per host, successful or not (newest send time wins, reset clears it), and expose it via GetConnInfo. Compute the reply TTL last/min/max over the history window instead of all time, so one transient TTL does not keep the range forever. Move GetReplyTTL out of GetRTTPercentiles' doc comment. --- internal/stats/history.go | 8 +++++ internal/stats/interface.go | 2 ++ internal/stats/manager.go | 19 +++++++----- internal/stats/metric.go | 49 ++++++++++++++++++++--------- internal/stats/ttl.go | 62 ++++++++++++++++++++----------------- internal/stats/ttl_test.go | 59 +++++++++++++++++++++++++++++++++++ 6 files changed, 150 insertions(+), 49 deletions(-) diff --git a/internal/stats/history.go b/internal/stats/history.go index 11e6a94..42ce752 100644 --- a/internal/stats/history.go +++ b/internal/stats/history.go @@ -103,6 +103,14 @@ func (th *TargetHistory) GetRecentEntries(n int) []HistoryEntry { return result } +// eachNewest calls fn for every entry, newest first, without copying them. +// fn must not modify the entries. +func (th *TargetHistory) eachNewest(fn func(*HistoryEntry)) { + for i := 0; i < th.count; i++ { + fn(&th.entries[(th.index-1-i+th.size)%th.size]) + } +} + // GetEntriesSince retrieves entries since the specified time func (th *TargetHistory) GetEntriesSince(since time.Time) []HistoryEntry { if th.count == 0 { diff --git a/internal/stats/interface.go b/internal/stats/interface.go index 759b783..0cbbb27 100644 --- a/internal/stats/interface.go +++ b/internal/stats/interface.go @@ -38,6 +38,8 @@ type Metrics interface { // GetReplyTTL returns the ICMP reply TTL / hop limit statistics GetReplyTTL() ReplyTTLStats + // GetConnInfo returns the peer of the latest HTTP(S) response + GetConnInfo() (ConnInfo, bool) } // MetricsProvider provides external API for metrics access diff --git a/internal/stats/manager.go b/internal/stats/manager.go index c9d4c98..c2741bd 100644 --- a/internal/stats/manager.go +++ b/internal/stats/manager.go @@ -111,9 +111,8 @@ func (mm *metricsManager) SuccessWithDetails(host string, rtt time.Duration, sen Details: details, }) } - ttl := replyTTL(details) - m.ttl.add(ttl, sentTime) - m.outage.addResult(probeResult{sentTime: sentTime, success: true, ttl: ttl}, time.Now()) + m.conn.add(sentTime, details) + m.outage.addResult(probeResult{sentTime: sentTime, success: true, ttl: replyTTL(details)}, time.Now()) mm.mu.Unlock() } @@ -127,6 +126,13 @@ func replyTTL(details *prober.ProbeDetails) int { // Register failure for host func (mm *metricsManager) Failed(host string, sentTime time.Time, msg string) { + mm.FailedWithDetails(host, sentTime, msg, nil) +} + +// FailedWithDetails registers a failure for host. details, if set, come +// from a response that was received but rejected (e.g. an unexpected HTTP +// status); only its connection info is kept. +func (mm *metricsManager) FailedWithDetails(host string, sentTime time.Time, msg string, details *prober.ProbeDetails) { m := mm.getMetrics(host) mm.mu.Lock() @@ -144,6 +150,7 @@ func (mm *metricsManager) Failed(host string, sentTime time.Time, msg string) { Error: msg, }) } + m.conn.add(sentTime, details) m.outage.add(sentTime, false, msg, time.Now()) mm.mu.Unlock() @@ -181,10 +188,8 @@ func (mm *metricsManager) Subscribe(res <-chan *prober.Event) <-chan struct{} { mm.Sent(r.Key, r.SentTime) case prober.SUCCESS: mm.SuccessWithDetails(r.Key, r.Rtt, r.SentTime, r.Details) - case prober.TIMEOUT: - mm.Failed(r.Key, r.SentTime, r.Message) - case prober.FAILED: - mm.Failed(r.Key, r.SentTime, r.Message) + case prober.TIMEOUT, prober.FAILED: + mm.FailedWithDetails(r.Key, r.SentTime, r.Message, r.Details) } } }() diff --git a/internal/stats/metric.go b/internal/stats/metric.go index 028723e..15df2be 100644 --- a/internal/stats/metric.go +++ b/internal/stats/metric.go @@ -56,7 +56,7 @@ type metrics struct { LastFailDetail string history *TargetHistory // 履歴情報 outage *outageTracker - ttl ttlRange + conn ConnInfo // latest HTTP(S) peer; zero SentTime if none resetAt time.Time // results of probes sent before this are stale } @@ -114,7 +114,7 @@ func (m *metrics) Reset() { m.LastFailTime = time.Time{} m.LastSuccTime = time.Time{} m.LastFailDetail = "" - m.ttl = ttlRange{} + m.conn = ConnInfo{} if m.history != nil { m.history.Clear() } @@ -258,18 +258,6 @@ func (m *metrics) GetOutageSummary() OutageSummary { // GetRTTPercentiles returns RTT percentiles (0-100, nearest-rank method) // over the successful probes kept in history. -// GetReplyTTL returns the reply TTL / hop limit statistics. Last, Min and -// Max are up to date; Changes only include results applied in send order, -// so they may lag behind by the settle time. -func (m *metrics) GetReplyTTL() ReplyTTLStats { - s := ReplyTTLStats{Last: m.ttl.last, Min: m.ttl.min, Max: m.ttl.max} - if m.outage != nil { - s.Changes = append([]TTLChange(nil), m.outage.ttl.changes...) - s.ChangeCount = m.outage.ttl.count - } - return s -} - func (m *metrics) GetRTTPercentiles(ps ...float64) []time.Duration { res := make([]time.Duration, len(ps)) if m.history == nil { @@ -293,6 +281,39 @@ func (m *metrics) GetRTTPercentiles(ps ...float64) []time.Duration { return res } +// GetReplyTTL returns the reply TTL / hop limit statistics. Last, Min and +// Max cover the successful replies kept in history (the last HistorySize +// probes), so the range reflects the current path, not old events. +// Changes only include results applied in send order, so they may lag +// behind by the settle time. +func (m *metrics) GetReplyTTL() ReplyTTLStats { + var s ReplyTTLStats + if m.history != nil { + m.history.eachNewest(func(e *HistoryEntry) { + if !e.Success || e.Details == nil || e.Details.ICMP == nil || e.Details.ICMP.ReplyTTL <= 0 { + return + } + ttl := e.Details.ICMP.ReplyTTL + if s.Last == 0 { + s.Last, s.Min, s.Max = ttl, ttl, ttl + return + } + s.Min, s.Max = min(s.Min, ttl), max(s.Max, ttl) + }) + } + if m.outage != nil { + s.Changes = append([]TTLChange(nil), m.outage.ttl.changes...) + s.ChangeCount = m.outage.ttl.count + } + return s +} + +// GetConnInfo returns the peer of the latest response from an HTTP(S) +// target, whether or not the probe succeeded; ok is false if none. +func (m *metrics) GetConnInfo() (ConnInfo, bool) { + return m.conn, !m.conn.SentTime.IsZero() +} + func (m *metrics) GetSuccessRateInPeriod(duration time.Duration) float64 { if m.history == nil { return 0.0 diff --git a/internal/stats/ttl.go b/internal/stats/ttl.go index 41bb3fe..ee62107 100644 --- a/internal/stats/ttl.go +++ b/internal/stats/ttl.go @@ -1,6 +1,10 @@ package stats -import "time" +import ( + "time" + + "github.com/servak/mping/internal/prober" +) // maxTTLChanges is the number of recent reply TTL changes kept per target. // ReplyTTLStats.ChangeCount keeps counting beyond it. @@ -21,41 +25,19 @@ func (c TTLChange) HopsDelta() int { return c.Old - c.New } -// ReplyTTLStats describes the reply TTLs / hop limits of a target. All -// values are 0 (and Changes empty) when no reply carried a known TTL. +// ReplyTTLStats describes the reply TTLs / hop limits of a target. Last, +// Min and Max are 0 when no recent reply carried a known TTL. type ReplyTTLStats struct { Last int // TTL of the reply to the most recently sent probe - Min int - Max int + // Min and Max cover the replies kept in history (recent probes only) + Min int + Max int // Changes holds the most recent changes (at most maxTTLChanges), // oldest first; ChangeCount counts all of them. Changes []TTLChange ChangeCount int } -// ttlRange tracks the last, minimum and maximum reply TTL. It is updated -// as results arrive: newest send time wins for Last, like LastRTT. -type ttlRange struct { - last int - lastSent time.Time - min, max int -} - -func (r *ttlRange) add(ttl int, sentTime time.Time) { - if ttl <= 0 { - return - } - if r.last == 0 || !sentTime.Before(r.lastSent) { - r.last, r.lastSent = ttl, sentTime - } - if r.min == 0 || ttl < r.min { - r.min = ttl - } - if ttl > r.max { - r.max = ttl - } -} - // ttlChangeTracker detects TTL changes between consecutive successful // replies. It is fed by outageTracker, which applies results in send order // once they have settled, so a late reply cannot fake a change. @@ -83,3 +65,27 @@ func (t ttlChangeTracker) clone() ttlChangeTracker { t.changes = append([]TTLChange(nil), t.changes...) return t } + +// ConnInfo is the peer of the latest response received from an HTTP(S) +// target, successful or not (e.g. a 503 during a switchover). +type ConnInfo struct { + SentTime time.Time // send time of the probe that got the response + RemoteAddr string + TLS *prober.TLSDetails // nil for plain HTTP; shared, read-only +} + +// add records the connection info of details if it is newer (by send +// time, ties go to the later result, like LastRTT). +func (c *ConnInfo) add(sentTime time.Time, details *prober.ProbeDetails) { + if details == nil || details.HTTP == nil { + return + } + h := details.HTTP + if h.RemoteAddr == "" && h.TLS == nil { + return + } + if sentTime.Before(c.SentTime) { + return + } + *c = ConnInfo{SentTime: sentTime, RemoteAddr: h.RemoteAddr, TLS: h.TLS} +} diff --git a/internal/stats/ttl_test.go b/internal/stats/ttl_test.go index 5e9ab03..3435b66 100644 --- a/internal/stats/ttl_test.go +++ b/internal/stats/ttl_test.go @@ -138,3 +138,62 @@ func TestReplyTTLSnapshotIsolated(t *testing.T) { t.Errorf("earlier snapshot changed: %+v", before.Changes) } } + +// The range covers the replies kept in history, so an old transient TTL +// drops out of it. +func TestReplyTTLRangeIsRecent(t *testing.T) { + mm := NewMetricsManagerWithOptions(Options{HistorySize: 5}) + events := []*prober.Event{icmpTTLEvent(at(0), 60)} + for i := 1; i <= 5; i++ { + events = append(events, icmpTTLEvent(at(i), 54)) + } + feedEvents(mm, events[:3]...) + if s := replyTTLOf(t, mm); s.Min != 54 || s.Max != 60 { + t.Errorf("min/max with the transient TTL in history = %d/%d, want 54/60", s.Min, s.Max) + } + feedEvents(mm, events[3:]...) + s := replyTTLOf(t, mm) + if s.Last != 54 || s.Min != 54 || s.Max != 54 { + t.Errorf("last/min/max = %d/%d/%d, want 54/54/54", s.Last, s.Min, s.Max) + } + if s.ChangeCount != 1 { + t.Errorf("ChangeCount = %d, want 1 (changes outlive the range window)", s.ChangeCount) + } +} + +// httpEvent is an HTTP(S) result; remote/tls empty means no response +func httpEvent(sent time.Time, success bool, remote string, tls *prober.TLSDetails) *prober.Event { + e := &prober.Event{Key: "h", DisplayName: "h", Result: prober.SUCCESS, SentTime: sent} + if !success { + e.Result, e.Message = prober.FAILED, "unexpected status code: 503" + } + if remote != "" || tls != nil { + e.Details = &prober.ProbeDetails{ProbeType: "https", HTTP: &prober.HTTPDetails{StatusCode: 200, RemoteAddr: remote, TLS: tls}} + } + return e +} + +func TestConnInfo(t *testing.T) { + mm := NewMetricsManager() + connOf := func() (ConnInfo, bool) { return mm.SortBy(Host, true)[0].GetConnInfo() } + feedEvents(mm, httpEvent(at(0), true, "192.0.2.1:443", &prober.TLSDetails{SubjectCN: "old"})) + ci, ok := connOf() + if !ok || ci.RemoteAddr != "192.0.2.1:443" || ci.TLS == nil || ci.TLS.SubjectCN != "old" { + t.Fatalf("after success: %+v, %v", ci, ok) + } + // A rejected response (e.g. 503 from the new peer) updates it; a + // failure without a response and a late older result do not. + feedEvents(mm, + httpEvent(at(2), false, "192.0.2.2:443", nil), + httpEvent(at(3), false, "", nil), + httpEvent(at(1), true, "192.0.2.1:443", nil), + ) + ci, ok = connOf() + if !ok || ci.RemoteAddr != "192.0.2.2:443" || ci.TLS != nil || !ci.SentTime.Equal(at(2)) { + t.Errorf("after rejected response: %+v, %v", ci, ok) + } + mm.ResetAllMetrics() + if _, ok := connOf(); ok { + t.Error("reset did not clear the connection info") + } +} From 1d8448a023e271efdd349dff60c6fb44046ad78f Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 15 Nov 2026 09:40:20 +0900 Subject: [PATCH 7/9] refactor(tui): read HTTP connection info from stats instead of scanning history --- internal/ui/tui/panels/conn_info.go | 35 ++++-------------------- internal/ui/tui/panels/conn_info_test.go | 9 ++++++ 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/internal/ui/tui/panels/conn_info.go b/internal/ui/tui/panels/conn_info.go index 585ac61..fec92ec 100644 --- a/internal/ui/tui/panels/conn_info.go +++ b/internal/ui/tui/panels/conn_info.go @@ -2,7 +2,6 @@ package panels import ( "fmt" - "math" "strings" "time" @@ -25,8 +24,8 @@ func formatConnInfo(metric stats.Metrics, theme *shared.Theme, now time.Time) st if s := formatReplyTTL(metric.GetReplyTTL(), theme); s != "" { sections = append(sections, s) } - if s := formatHTTPConn(latestHTTPDetails(metric), theme, now); s != "" { - sections = append(sections, s) + if ci, ok := metric.GetConnInfo(); ok { + sections = append(sections, formatHTTPConn(ci, theme, now)) } return strings.Join(sections, "\n\n") } @@ -65,33 +64,9 @@ func formatReplyTTL(s stats.ReplyTTLStats, theme *shared.Theme) string { return sb.String() } -// latestHTTPDetails returns the HTTP details of the most recently sent -// successful probe that carries connection info, or nil. -func latestHTTPDetails(metric stats.Metrics) *prober.HTTPDetails { - var ( - best *prober.HTTPDetails - bestAt time.Time - ) - for _, e := range metric.GetRecentHistory(math.MaxInt) { - if !e.Success || e.Details == nil || e.Details.HTTP == nil { - continue - } - h := e.Details.HTTP - if h.RemoteAddr == "" && h.TLS == nil { - continue - } - if best == nil || e.Timestamp.After(bestAt) { - best, bestAt = h, e.Timestamp - } - } - return best -} - -// formatHTTPConn renders the remote address and the TLS certificate -func formatHTTPConn(h *prober.HTTPDetails, theme *shared.Theme, now time.Time) string { - if h == nil { - return "" - } +// formatHTTPConn renders the remote address and the TLS certificate of the +// latest HTTP(S) response, which may have been a rejected one (e.g. 503) +func formatHTTPConn(h stats.ConnInfo, theme *shared.Theme, now time.Time) string { var rows []detailRow if h.RemoteAddr != "" { rows = append(rows, detailRow{"Remote", theme.Accent, tview.Escape(h.RemoteAddr)}) diff --git a/internal/ui/tui/panels/conn_info_test.go b/internal/ui/tui/panels/conn_info_test.go index 16ef7c4..8422d6d 100644 --- a/internal/ui/tui/panels/conn_info_test.go +++ b/internal/ui/tui/panels/conn_info_test.go @@ -102,6 +102,15 @@ func TestFormatConnInfoHTTP(t *testing.T) { t.Errorf("48 days left must not be highlighted: %q", raw) } + // A rejected response (503 from a new peer) replaces the shown peer + rejected := ev(3, &prober.HTTPDetails{StatusCode: 503, RemoteAddr: "198.51.100.7:443"}) + rejected.Result, rejected.Message = prober.FAILED, "unexpected status code: 503" + statstest.Feed(t, mm, rejected) + got = plainText(formatConnInfo(onlyMetrics(t, mm), &theme, now)) + if !strings.Contains(got, "Remote: 198.51.100.7:443") || strings.Contains(got, "Certificate") { + t.Errorf("after a rejected response:\n%s", got) + } + tests := []struct { left time.Duration color string From 9e3712aff9bf3e0c6dd11dc99a9463d48deef67c Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 15 Nov 2026 09:40:21 +0900 Subject: [PATCH 8/9] test(prober): check reply TTL on the raw ICMP socket path Also document the recent-window TTL range and that rejected HTTP responses update the shown peer. --- README.md | 4 ++-- internal/prober/icmp_integration_test.go | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 65a8a6b..4173c75 100644 --- a/README.md +++ b/README.md @@ -195,8 +195,8 @@ TLS: TLS 1.3 Certificate: CN example.com / Issuer R3 / expires 2027-01-02 (in 48d) ``` -- **ICMP**: the TTL (IPv4) / hop limit (IPv6) of the last echo reply, the hop count estimated from it, the range seen so far and the most recent TTL changes (`+2 hops` = the reply crossed two more hops). A change is a successful reply whose TTL differs from the previous successful reply's. -- **HTTP(S)**: the IP:port of the connection that served the final response (after redirects) and the TLS version and leaf certificate (subject CN, issuer, expiry; yellow within 30 days, red within 7 days or expired). The certificate is shown even with `skip_verify` (the default). +- **ICMP**: the TTL (IPv4) / hop limit (IPv6) of the last echo reply, the hop count estimated from it, the range over the last 100 probes (shown when replies currently arrive with different TTLs, e.g. over several paths) and the most recent TTL changes since start or reset (`+2 hops` = the reply crossed two more hops). A change is a successful reply whose TTL differs from the previous successful reply's. +- **HTTP(S)**: the IP:port of the connection that served the latest final response (after redirects), including responses rejected by `expect_codes` / `expect_body` such as a 503 from a new peer, and the TLS version and leaf certificate (subject CN, issuer, expiry; yellow within 30 days, red within 7 days or expired). The certificate is shown even with `skip_verify` (the default). Caveats: - The hop count is an **estimate of the return path** (responder to mping): the responder's initial TTL is guessed as the smallest of 64, 128 and 255 that fits. The forward path can differ (asymmetric routing) and cannot be seen in an echo reply; use the path view (`p`) for that. diff --git a/internal/prober/icmp_integration_test.go b/internal/prober/icmp_integration_test.go index 62eca80..6aa29a5 100644 --- a/internal/prober/icmp_integration_test.go +++ b/internal/prober/icmp_integration_test.go @@ -56,6 +56,11 @@ func TestIntegrationICMPProberLifecycle(t *testing.T) { if e.Details.ICMP.Payload != "mping-it" { t.Errorf("payload = %q, want %q", e.Details.ICMP.Payload, "mping-it") } + // The real socket must report the reply TTL / hop limit, + // except on Windows, which has no control messages. + if runtime.GOOS != "windows" && e.Details.ICMP.ReplyTTL <= 0 { + t.Errorf("ReplyTTL = %d, want > 0", e.Details.ICMP.ReplyTTL) + } } if goroutineRunning("(*ICMPProber).recvPkts") { t.Error("receive goroutine still running after Stop") From a2ff030716a3b797e61c924e9ba82a7d923f4be2 Mon Sep 17 00:00:00 2001 From: Kakuya Ando Date: Sun, 15 Nov 2026 09:52:39 +0900 Subject: [PATCH 9/9] fix(stats): record a TTL change only after 3 consecutive replies with the new TTL Replies alternating between ECMP paths of unequal length flooded the change list. A different TTL now becomes a candidate and is recorded as a change (timed at its first reply) once it is seen on 3 consecutive successful replies; a reply with the confirmed TTL drops the candidate. Failures and unknown TTLs neither advance nor clear it. The recent-history range still shows the alternation. --- README.md | 2 +- internal/stats/ttl.go | 63 +++++++--- internal/stats/ttl_test.go | 141 ++++++++++++++++------- internal/ui/tui/panels/conn_info_test.go | 11 +- 4 files changed, 151 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 4173c75..1f43440 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ TLS: TLS 1.3 Certificate: CN example.com / Issuer R3 / expires 2027-01-02 (in 48d) ``` -- **ICMP**: the TTL (IPv4) / hop limit (IPv6) of the last echo reply, the hop count estimated from it, the range over the last 100 probes (shown when replies currently arrive with different TTLs, e.g. over several paths) and the most recent TTL changes since start or reset (`+2 hops` = the reply crossed two more hops). A change is a successful reply whose TTL differs from the previous successful reply's. +- **ICMP**: the TTL (IPv4) / hop limit (IPv6) of the last echo reply, the hop count estimated from it, the range over the last 100 probes (shown when replies currently arrive with different TTLs, e.g. over several paths) and the most recent TTL changes since start or reset (`+2 hops` = the reply crossed two more hops). A change is recorded once a new TTL is seen on 3 consecutive successful replies, timed at the first of them; replies alternating between ECMP paths of unequal length show up as a range instead of a stream of changes. - **HTTP(S)**: the IP:port of the connection that served the latest final response (after redirects), including responses rejected by `expect_codes` / `expect_body` such as a 503 from a new peer, and the TLS version and leaf certificate (subject CN, issuer, expiry; yellow within 30 days, red within 7 days or expired). The certificate is shown even with `skip_verify` (the default). Caveats: diff --git a/internal/stats/ttl.go b/internal/stats/ttl.go index ee62107..f672ba6 100644 --- a/internal/stats/ttl.go +++ b/internal/stats/ttl.go @@ -10,10 +10,11 @@ import ( // ReplyTTLStats.ChangeCount keeps counting beyond it. const maxTTLChanges = 20 -// TTLChange is a change of the reply TTL (IPv4) / hop limit (IPv6) between -// two consecutive successful replies, in send order. +// TTLChange is a confirmed change of the reply TTL (IPv4) / hop limit +// (IPv6): the new TTL was seen on ttlChangeMinReplies consecutive +// successful replies. type TTLChange struct { - Time time.Time // send time of the probe whose reply had the new TTL + Time time.Time // send time of the first reply with the new TTL Old int New int } @@ -38,27 +39,57 @@ type ReplyTTLStats struct { ChangeCount int } -// ttlChangeTracker detects TTL changes between consecutive successful -// replies. It is fed by outageTracker, which applies results in send order -// once they have settled, so a late reply cannot fake a change. +// ttlChangeMinReplies is the number of consecutive successful replies a +// new TTL must be seen on before it is recorded as a change (like the 3 +// consecutive losses that make an outage). Replies alternating between +// unequal ECMP paths therefore show up as a range, not as a flood of +// changes. +const ttlChangeMinReplies = 3 + +// ttlChangeTracker detects TTL changes between successful replies. It is +// fed by outageTracker, which applies results in send order once they have +// settled, so a late reply cannot fake a change. +// +// prev is the confirmed TTL. A reply with another TTL starts (or extends) +// a candidate; once the candidate's TTL has been seen on +// ttlChangeMinReplies consecutive replies it is recorded as a change at the +// send time of its first reply. A reply with the confirmed TTL drops the +// candidate. Failures and replies with an unknown TTL change nothing. type ttlChangeTracker struct { - prev int // TTL of the previous successful reply with a known TTL - changes []TTLChange - count int + prev int // confirmed TTL; 0 until the first reply with a known TTL + candTTL int // candidate TTL; 0 if none + candSent time.Time + candRun int + changes []TTLChange + count int } func (t *ttlChangeTracker) apply(r probeResult) { if !r.success || r.ttl <= 0 { return } - if t.prev != 0 && r.ttl != t.prev { - t.changes = append(t.changes, TTLChange{Time: r.sentTime, Old: t.prev, New: r.ttl}) - t.count++ - if len(t.changes) > maxTTLChanges { - t.changes = append([]TTLChange(nil), t.changes[len(t.changes)-maxTTLChanges:]...) - } + switch { + case t.prev == 0: + t.prev = r.ttl // the first known TTL is not a change + return + case r.ttl == t.prev: + t.candTTL, t.candRun = 0, 0 + return + case r.ttl == t.candTTL: + t.candRun++ + default: + t.candTTL, t.candSent, t.candRun = r.ttl, r.sentTime, 1 + } + if t.candRun < ttlChangeMinReplies { + return + } + t.changes = append(t.changes, TTLChange{Time: t.candSent, Old: t.prev, New: t.candTTL}) + t.count++ + if len(t.changes) > maxTTLChanges { + t.changes = append([]TTLChange(nil), t.changes[len(t.changes)-maxTTLChanges:]...) } - t.prev = r.ttl + t.prev = t.candTTL + t.candTTL, t.candRun = 0, 0 } func (t ttlChangeTracker) clone() ttlChangeTracker { diff --git a/internal/stats/ttl_test.go b/internal/stats/ttl_test.go index 3435b66..223b898 100644 --- a/internal/stats/ttl_test.go +++ b/internal/stats/ttl_test.go @@ -32,24 +32,19 @@ func replyTTLOf(t *testing.T, mm MetricsManager) ReplyTTLStats { return ms[0].GetReplyTTL() } -func TestReplyTTLTracking(t *testing.T) { - mm := NewMetricsManager() - feedEvents(mm, - icmpTTLEvent(at(0), 54), - icmpTTLEvent(at(1), 54), - icmpTTLEvent(at(2), 0), // unknown: ignored - &prober.Event{Key: "h", Result: prober.TIMEOUT, SentTime: at(3), Message: "timeout"}, - icmpTTLEvent(at(4), 52), - icmpTTLEvent(at(5), 52), - icmpTTLEvent(at(6), 0), - icmpTTLEvent(at(7), 54), - ) - s := replyTTLOf(t, mm) - if s.Last != 54 || s.Min != 52 || s.Max != 54 { - t.Errorf("last/min/max = %d/%d/%d, want 54/52/54", s.Last, s.Min, s.Max) - } - want := []TTLChange{{Time: at(4), Old: 54, New: 52}, {Time: at(7), Old: 52, New: 54}} - if s.ChangeCount != 2 || len(s.Changes) != 2 { +// ttlSeq returns successful replies with the given TTLs, one per second +// from at(start) +func ttlSeq(start int, ttls ...int) []*prober.Event { + events := make([]*prober.Event, len(ttls)) + for i, ttl := range ttls { + events[i] = icmpTTLEvent(at(start+i), ttl) + } + return events +} + +func checkChanges(t *testing.T, s ReplyTTLStats, want ...TTLChange) { + t.Helper() + if s.ChangeCount != len(want) || len(s.Changes) != len(want) { t.Fatalf("changes = %+v (count %d), want %+v", s.Changes, s.ChangeCount, want) } for i, c := range s.Changes { @@ -57,11 +52,55 @@ func TestReplyTTLTracking(t *testing.T) { t.Errorf("change %d = %+v, want %+v", i, c, want[i]) } } +} + +func TestReplyTTLTracking(t *testing.T) { + mm := NewMetricsManager() + events := ttlSeq(0, 54, 54, 52) + events = append(events, + icmpTTLEvent(at(3), 0), // unknown: neither advances nor clears the candidate + &prober.Event{Key: "h", Result: prober.TIMEOUT, SentTime: at(4), Message: "timeout"}, + ) + events = append(events, ttlSeq(5, 52, 52, 54, 54, 54)...) + feedEvents(mm, events...) + s := replyTTLOf(t, mm) + if s.Last != 54 || s.Min != 52 || s.Max != 54 { + t.Errorf("last/min/max = %d/%d/%d, want 54/52/54", s.Last, s.Min, s.Max) + } + checkChanges(t, s, TTLChange{Time: at(2), Old: 54, New: 52}, TTLChange{Time: at(7), Old: 52, New: 54}) if d := s.Changes[0].HopsDelta(); d != 2 { t.Errorf("HopsDelta = %d, want 2", d) } } +func TestReplyTTLChangeNeedsConsecutiveReplies(t *testing.T) { + tests := []struct { + name string + ttls []int + want []TTLChange + }{ + {"alternating ECMP paths", []int{54, 53, 54, 53, 54, 53, 54, 53}, nil}, + {"persistent change", []int{54, 54, 52, 52, 52}, []TTLChange{{Time: at(2), Old: 54, New: 52}}}, + {"interrupted candidate", []int{54, 52, 52, 54, 52, 52, 52}, []TTLChange{{Time: at(4), Old: 54, New: 52}}}, + {"two short runs of the new TTL", []int{54, 52, 52, 54, 52, 52}, nil}, + {"another candidate replaces it", []int{54, 52, 52, 50, 50, 50}, []TTLChange{{Time: at(3), Old: 54, New: 50}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mm := NewMetricsManager() + feedEvents(mm, ttlSeq(0, tt.ttls...)...) + s := replyTTLOf(t, mm) + checkChanges(t, s, tt.want...) + }) + } + // Alternation stays visible as a range + mm := NewMetricsManager() + feedEvents(mm, ttlSeq(0, 54, 53, 54, 53)...) + if s := replyTTLOf(t, mm); s.Min != 53 || s.Max != 54 { + t.Errorf("alternating min/max = %d/%d, want 53/54", s.Min, s.Max) + } +} + func TestReplyTTLNoData(t *testing.T) { mm := NewMetricsManager() feedEvents(mm, icmpTTLEvent(at(0), 0), icmpTTLEvent(at(1), 0)) @@ -72,12 +111,16 @@ func TestReplyTTLNoData(t *testing.T) { func TestReplyTTLChangesCapped(t *testing.T) { mm := NewMetricsManager() - var events []*prober.Event + // Blocks of ttlChangeMinReplies replies, alternating 50 and 51: every + // block after the first is a change at its first reply. n := maxTTLChanges + 5 - for i := 0; i <= n; i++ { - events = append(events, icmpTTLEvent(at(i), 50+i%2)) + var ttls []int + for k := 0; k <= n; k++ { + for range ttlChangeMinReplies { + ttls = append(ttls, 50+k%2) + } } - feedEvents(mm, events...) + feedEvents(mm, ttlSeq(0, ttls...)...) s := replyTTLOf(t, mm) if s.ChangeCount != n { t.Errorf("ChangeCount = %d, want %d", s.ChangeCount, n) @@ -85,36 +128,48 @@ func TestReplyTTLChangesCapped(t *testing.T) { if len(s.Changes) != maxTTLChanges { t.Fatalf("kept %d changes, want %d", len(s.Changes), maxTTLChanges) } - if last := s.Changes[len(s.Changes)-1]; !last.Time.Equal(at(n)) { - t.Errorf("newest change at %v, want %v", last.Time, at(n)) + m := ttlChangeMinReplies + if last := s.Changes[len(s.Changes)-1]; !last.Time.Equal(at(m * n)) { + t.Errorf("newest change at %v, want %v", last.Time, at(m*n)) } - if first := s.Changes[0]; !first.Time.Equal(at(n - maxTTLChanges + 1)) { - t.Errorf("oldest kept change at %v, want %v", first.Time, at(n-maxTTLChanges+1)) + if first := s.Changes[0]; !first.Time.Equal(at(m * (n - maxTTLChanges + 1))) { + t.Errorf("oldest kept change at %v, want %v", first.Time, at(m*(n-maxTTLChanges+1))) } } -// Results that arrive out of send order are applied in send order, so a -// late reply does not produce spurious changes. +// Results that arrive out of send order are applied in send order: the +// late 54 interrupts the run of 52s, so no change is recorded (in arrival +// order the 52s would have been consecutive). func TestReplyTTLOutOfOrder(t *testing.T) { + // Recent send times, so that nothing settles before the channel closes + base := time.Now() + sec := func(n int) time.Time { return base.Add(time.Duration(n) * time.Second) } mm := NewMetricsManagerWithOptions(Options{SettleTime: time.Hour}) feedEvents(mm, - icmpTTLEvent(at(0), 54), - icmpTTLEvent(at(2), 52), - icmpTTLEvent(at(1), 54), // late + icmpTTLEvent(sec(0), 54), + icmpTTLEvent(sec(1), 52), + icmpTTLEvent(sec(2), 52), + icmpTTLEvent(sec(4), 52), + icmpTTLEvent(sec(3), 54), // late ) s := replyTTLOf(t, mm) if s.Last != 52 { t.Errorf("Last = %d, want 52 (newest send time)", s.Last) } - if s.ChangeCount != 1 || len(s.Changes) != 1 || !s.Changes[0].Time.Equal(at(2)) { - t.Errorf("changes = %+v (count %d), want one at %v", s.Changes, s.ChangeCount, at(2)) - } + checkChanges(t, s) } func TestReplyTTLReset(t *testing.T) { mm := NewMetricsManager() now := time.Now() - feedEvents(mm, icmpTTLEvent(now.Add(-2*time.Second), 54), icmpTTLEvent(now.Add(-time.Second), 52)) + var events []*prober.Event + for i, ttl := range []int{54, 52, 52, 52} { + events = append(events, icmpTTLEvent(now.Add(time.Duration(i-5)*time.Second), ttl)) + } + feedEvents(mm, events...) + if s := replyTTLOf(t, mm); s.ChangeCount != 1 { + t.Fatalf("before reset: ChangeCount = %d, want 1", s.ChangeCount) + } mm.ResetAllMetrics() // A result of a probe sent before the reset is dropped feedEvents(mm, icmpTTLEvent(now.Add(-500*time.Millisecond), 60)) @@ -131,22 +186,22 @@ func TestReplyTTLReset(t *testing.T) { // Snapshots must not share the change list with the live metrics. func TestReplyTTLSnapshotIsolated(t *testing.T) { mm := NewMetricsManager() - feedEvents(mm, icmpTTLEvent(at(0), 54), icmpTTLEvent(at(1), 52)) + feedEvents(mm, ttlSeq(0, 54, 52, 52, 52)...) before := replyTTLOf(t, mm) - feedEvents(mm, icmpTTLEvent(at(2), 54)) + feedEvents(mm, ttlSeq(4, 54, 54, 54)...) if len(before.Changes) != 1 || before.Changes[0].New != 52 { t.Errorf("earlier snapshot changed: %+v", before.Changes) } + if s := replyTTLOf(t, mm); s.ChangeCount != 2 { + t.Errorf("ChangeCount = %d, want 2", s.ChangeCount) + } } // The range covers the replies kept in history, so an old transient TTL // drops out of it. func TestReplyTTLRangeIsRecent(t *testing.T) { mm := NewMetricsManagerWithOptions(Options{HistorySize: 5}) - events := []*prober.Event{icmpTTLEvent(at(0), 60)} - for i := 1; i <= 5; i++ { - events = append(events, icmpTTLEvent(at(i), 54)) - } + events := ttlSeq(0, 60, 54, 54, 54, 54, 54) feedEvents(mm, events[:3]...) if s := replyTTLOf(t, mm); s.Min != 54 || s.Max != 60 { t.Errorf("min/max with the transient TTL in history = %d/%d, want 54/60", s.Min, s.Max) @@ -156,9 +211,7 @@ func TestReplyTTLRangeIsRecent(t *testing.T) { if s.Last != 54 || s.Min != 54 || s.Max != 54 { t.Errorf("last/min/max = %d/%d/%d, want 54/54/54", s.Last, s.Min, s.Max) } - if s.ChangeCount != 1 { - t.Errorf("ChangeCount = %d, want 1 (changes outlive the range window)", s.ChangeCount) - } + checkChanges(t, s, TTLChange{Time: at(1), Old: 60, New: 54}) // changes outlive the range window } // httpEvent is an HTTP(S) result; remote/tls empty means no response diff --git a/internal/ui/tui/panels/conn_info_test.go b/internal/ui/tui/panels/conn_info_test.go index 8422d6d..3519bd9 100644 --- a/internal/ui/tui/panels/conn_info_test.go +++ b/internal/ui/tui/panels/conn_info_test.go @@ -36,26 +36,27 @@ func TestFormatConnInfoReplyTTL(t *testing.T) { return &prober.Event{Key: "h", DisplayName: "h", Result: prober.SUCCESS, SentTime: ts.Add(time.Duration(i) * time.Second), Rtt: time.Millisecond, Details: &prober.ProbeDetails{ProbeType: "icmpv4", ICMP: &prober.ICMPDetails{ReplyTTL: ttl}}} } - mm := statstest.FromEvents(t, ev(0, 54), ev(3, 52), ev(4, 53)) + // A change is recorded once the new TTL is seen on 3 consecutive replies + mm := statstest.FromEvents(t, ev(0, 54), ev(3, 52), ev(4, 52), ev(5, 52), ev(6, 53), ev(7, 53), ev(8, 53)) got := plainText(formatConnInfo(onlyMetrics(t, mm), &theme, ts)) for _, want := range []string{ "Reply TTL: 53 (≈11 hops, est.), range 52–54", "TTL Changes: 2", - "09:12:04 TTL 52→53 (-1 hop)", + "09:12:06 TTL 52→53 (-1 hop)", "09:12:03 TTL 54→52 (+2 hops)", } { if !strings.Contains(got, want) { t.Errorf("missing %q in:\n%s", want, got) } } - if i, j := strings.Index(got, "09:12:04"), strings.Index(got, "09:12:03"); i > j { + if i, j := strings.Index(got, "09:12:06"), strings.Index(got, "09:12:03"); i > j { t.Errorf("changes not newest first:\n%s", got) } // Truncated list shows how many more there are var events []*prober.Event - for i := 0; i <= 15; i++ { - events = append(events, ev(i, 50+i%2)) + for i := 0; i < 16*3; i++ { + events = append(events, ev(i, 50+(i/3)%2)) } got = plainText(formatConnInfo(onlyMetrics(t, statstest.FromEvents(t, events...)), &theme, ts)) if !strings.Contains(got, "TTL Changes: 15") || !strings.Contains(got, "... 5 more") {