diff --git a/port/nat/ipnat.go b/port/nat/ipnat.go index 71a968c2..8bdf8829 100644 --- a/port/nat/ipnat.go +++ b/port/nat/ipnat.go @@ -33,6 +33,16 @@ const ( osNATTCPDialTimeout = 5 * time.Second // osNATMaxSegment is the maximum TCP payload that fits in one DDP-carried IP packet. osNATMaxSegment = 546 // max TCP payload: 586 (DDP) - 20 (IP) - 20 (TCP) + // osNATTCPInitialRTO is the first retransmission timeout for data or FIN sent toward the Mac. + osNATTCPInitialRTO = time.Second + // osNATTCPMaxRTO caps the exponential backoff of the retransmission timeout. + osNATTCPMaxRTO = 16 * time.Second + // osNATTCPMaxRetries is how often the oldest unacknowledged segment is resent before the flow is reset. + osNATTCPMaxRetries = 8 + // osNATTCPDupAckThreshold is the duplicate ACK count that triggers a fast retransmit (RFC 5681). + osNATTCPDupAckThreshold = 3 + // osNATTCPCloseLinger bounds how long a flow waits, after sending its FIN, for the Mac to close its side. + osNATTCPCloseLinger = time.Minute ) // osFlowKey identifies a UDP or TCP flow by 5-tuple. @@ -89,6 +99,19 @@ type tcpFwdFlow struct { windowAdv chan struct{} // windowAdv is signaled when macAck or macWindow advances. done chan struct{} // done is closed when the flow is terminated. doneOnce sync.Once // doneOnce ensures done is only closed once. + + ourISN uint32 // ourISN is the sequence number of our SYN-ACK, kept to resend it. + unacked []byte // unacked holds the data sent toward the Mac from macAck up to ourSeq. + finSent bool // finSent is set once our FIN is sent; it takes sequence number ourSeq-1. + finAcked bool // finAcked is set once the Mac has acknowledged our FIN. + macFIN bool // macFIN is set once the Mac's FIN has been accepted. + dupAcks int // dupAcks counts duplicate ACKs from the Mac for the oldest unacked byte. + rto time.Duration // rto is the current retransmission timeout. + retries int // retries counts resends of the oldest unacknowledged segment. + rtxTimer *time.Timer // rtxTimer resends the oldest segment while data or FIN is unacknowledged. + rtxGen uint64 // rtxGen tells a stale rtxTimer callback from the current one. + recovery bool // recovery is set from a resend until everything sent before it is acknowledged. + recover uint32 // recover is ourSeq when recovery began (RFC 6582). } // closeConn closes the flow's done channel once and then closes the host connection. @@ -99,6 +122,56 @@ func (f *tcpFwdFlow) closeConn() { } } +// outstanding reports whether data or a FIN sent toward the Mac is still +// unacknowledged. The caller holds f.mu. +func (f *tcpFwdFlow) outstanding() bool { + return len(f.unacked) > 0 || (f.finSent && !f.finAcked) +} + +// ackTo advances macAck to ack, which lies in (macAck, ourSeq], drops the +// acknowledged data and resets the retransmission state. The caller holds f.mu. +func (f *tcpFwdFlow) ackTo(ack uint32) { + acked := int(ack - f.macAck) + if f.finSent && ack == f.ourSeq { + f.finAcked = true + acked-- // the FIN's sequence number carries no data + } + if acked >= len(f.unacked) { + f.unacked = nil + } else { + f.unacked = f.unacked[acked:] + } + f.macAck = ack + f.dupAcks = 0 + f.retries = 0 + f.rto = osNATTCPInitialRTO + f.stopRetransmit() +} + +// oldestSegment returns the oldest unacknowledged segment toward the Mac, to +// be resent: up to one MSS of data from macAck, or else the FIN. It starts +// loss recovery if none is under way. The caller holds f.mu. +func (f *tcpFwdFlow) oldestSegment() (seq uint32, flags byte, data []byte) { + if !f.recovery { + f.recovery = true + f.recover = f.ourSeq + } + if len(f.unacked) > 0 { + m := min(len(f.unacked), int(f.mss)) + return f.macAck, 0x18, append([]byte(nil), f.unacked[:m]...) // PSH+ACK + } + return f.ourSeq - 1, 0x11, nil // FIN+ACK +} + +// stopRetransmit cancels a pending retransmission. The caller holds f.mu. +func (f *tcpFwdFlow) stopRetransmit() { + if f.rtxTimer != nil { + f.rtxTimer.Stop() + f.rtxTimer = nil + } + f.rtxGen++ +} + // OSNAT forwards off-subnet Mac IP traffic through the host OS network stack. // Each protocol uses real OS sockets so the host's own IP is the NAT source, // avoiding the routing problem that occurs when the MacIP subnet differs from @@ -398,8 +471,11 @@ func (n *OSNAT) handleTCP(pkt []byte, ihl int, atNet uint16, atNode uint8) { if flags&flagSYN != 0 && flags&flagACK == 0 { // New connection n.tcpMu.Lock() - if _, exists := n.tcpFlows[key]; exists { + if flow, exists := n.tcpFlows[key]; exists { n.tcpMu.Unlock() + if flow != nil { + n.resendTCPSYNACK(flow, seq) + } return } n.tcpFlows[key] = nil // mark as connecting @@ -450,24 +526,60 @@ func (n *OSNAT) handleTCP(pkt []byte, ihl int, atNet uint16, atNode uint8) { return } + hasFIN := flags&flagFIN != 0 + var resend bool + var rtxSeq uint32 + var rtxFlags byte + var rtxData []byte + flow.mu.Lock() flow.expiry = time.Now().Add(osNATTCPTimeout) - if len(payload) > 0 { - flow.macSeq += uint32(len(payload)) - } - hasFIN := flags&flagFIN != 0 - if hasFIN { - flow.macSeq++ + // Accept only the next in-order bytes from the Mac. A retransmit of data + // already written to the server is trimmed or dropped, and out-of-order + // data is dropped, so the server sees each byte once and in order. The + // ACK sent below tells the Mac what we expect next. + var accepted []byte + finAccepted := false + if off := int32(flow.macSeq - seq); (len(payload) > 0 || hasFIN) && off >= 0 && int(off) <= len(payload) { + accepted = payload[off:] + finAccepted = hasFIN && !flow.macFIN + flow.macSeq += uint32(len(accepted)) + if finAccepted { + flow.macSeq++ + flow.macFIN = true + } } - ack := flow.macSeq - ourSeq := flow.ourSeq if flags&flagACK != 0 { macAck := binary.BigEndian.Uint32(pkt[ihl+8 : ihl+12]) - if int32(macAck-flow.macAck) > 0 { - flow.macAck = macAck + window := binary.BigEndian.Uint16(pkt[ihl+14 : ihl+16]) + switch { + case int32(macAck-flow.macAck) > 0 && int32(macAck-flow.ourSeq) <= 0: + flow.ackTo(macAck) + if flow.recovery && int32(macAck-flow.recover) < 0 { + // Partial ACK during recovery: the next segment was lost + // too (the Mac keeps no out-of-order data), so resend it now + // rather than after another timeout (RFC 6582). + resend = true + rtxSeq, rtxFlags, rtxData = flow.oldestSegment() + } else { + flow.recovery = false + } + if flow.outstanding() { + n.armRetransmit(flow) + } + case macAck == flow.macAck && len(payload) == 0 && !hasFIN && window == flow.macWindow && flow.outstanding(): + // Duplicate ACK: the Mac is missing the segment at macAck (RFC 5681 fast retransmit). + flow.dupAcks++ + if flow.dupAcks == osNATTCPDupAckThreshold { + resend = true + rtxSeq, rtxFlags, rtxData = flow.oldestSegment() + } } - flow.macWindow = binary.BigEndian.Uint16(pkt[ihl+14 : ihl+16]) + flow.macWindow = window } + ack := flow.macSeq + ourSeq := flow.ourSeq + closed := flow.macFIN && flow.finAcked flow.mu.Unlock() if flags&flagACK != 0 { select { @@ -475,13 +587,17 @@ func (n *OSNAT) handleTCP(pkt []byte, ihl int, atNet uint16, atNode uint8) { default: } } + if resend { + netlog.Debug("macip-osnat: TCP retransmit seq %d (%d bytes) after ACK", rtxSeq, len(rtxData)) + n.sendTCPSegment(flow, rtxSeq, ack, rtxFlags, rtxData) + } - if len(payload) > 0 { - if _, err := flow.conn.Write(payload); err != nil { + if len(accepted) > 0 { + if _, err := flow.conn.Write(accepted); err != nil { netlog.Debug("macip-osnat: TCP write: %v", err) } } - if hasFIN { + if finAccepted { if tc, ok := flow.conn.(*net.TCPConn); ok { _ = tc.CloseWrite() } @@ -489,6 +605,71 @@ func (n *OSNAT) handleTCP(pkt []byte, ihl int, atNet uint16, atNode uint8) { if len(payload) > 0 || hasFIN { n.sendTCPSegment(flow, ourSeq, ack, 0x10, nil) // ACK } + if closed { + // Both sides have closed and every FIN is acknowledged. + flow.closeConn() + } +} + +// resendTCPSYNACK answers a retransmitted SYN for an existing flow: the Mac +// did not get our SYN-ACK, so it is sent again. +func (n *OSNAT) resendTCPSYNACK(flow *tcpFwdFlow, macISN uint32) { + flow.mu.Lock() + again := macISN+1 == flow.macSeq + ourISN := flow.ourISN + flow.mu.Unlock() + if again { + n.sendTCPSYNACK(flow, ourISN) + } +} + +// armRetransmit starts the retransmission timer unless it is already +// running. The caller holds flow.mu. +func (n *OSNAT) armRetransmit(flow *tcpFwdFlow) { + if flow.rtxTimer != nil { + return + } + flow.rtxGen++ + gen := flow.rtxGen + flow.rtxTimer = time.AfterFunc(flow.rto, func() { n.tcpRetransmitTimeout(flow, gen) }) +} + +// tcpRetransmitTimeout resends the oldest unacknowledged segment toward the +// Mac and backs off the timeout. After osNATTCPMaxRetries resends the flow is +// reset. +func (n *OSNAT) tcpRetransmitTimeout(flow *tcpFwdFlow, gen uint64) { + flow.mu.Lock() + if gen != flow.rtxGen { + flow.mu.Unlock() + return + } + flow.rtxTimer = nil + select { + case <-flow.done: + flow.mu.Unlock() + return + default: + } + if !flow.outstanding() { + flow.mu.Unlock() + return + } + ack := flow.macSeq + flow.retries++ + if flow.retries > osNATTCPMaxRetries { + seq := flow.ourSeq + flow.mu.Unlock() + netlog.Debug("macip-osnat: TCP no ACK after %d resends, reset", osNATTCPMaxRetries) + n.sendTCPSegment(flow, seq, ack, 0x14, nil) // RST+ACK + flow.closeConn() + return + } + seq, flags, data := flow.oldestSegment() + flow.rto = min(2*flow.rto, osNATTCPMaxRTO) + n.armRetransmit(flow) + flow.mu.Unlock() + netlog.Debug("macip-osnat: TCP retransmit seq %d (%d bytes), try %d", seq, len(data), flow.retries) + n.sendTCPSegment(flow, seq, ack, flags, data) } // tcpConnect dials the remote server and initializes host-side state for a new TCP flow. @@ -510,11 +691,13 @@ func (n *OSNAT) tcpConnect(key osFlowKey, macISN uint32, mss uint16, synWindow u atNet: atNet, atNode: atNode, clientIP: clientIP, serverIP: serverIPb, clientPort: clientPort, serverPort: serverPort, + ourISN: ourISN, ourSeq: ourISN + 1, macSeq: macISN + 1, macAck: ourISN + 1, // optimistic: assume Mac will ACK our SYN-ACK macWindow: synWindow, mss: mss, + rto: osNATTCPInitialRTO, expiry: time.Now().Add(osNATTCPTimeout), windowAdv: make(chan struct{}, 1), done: make(chan struct{}), @@ -579,20 +762,38 @@ func (n *OSNAT) tcpServerReadLoop(key osFlowKey, flow *tcpFwdFlow) { seq := flow.ourSeq ack := flow.macSeq flow.ourSeq += uint32(len(chunk)) + flow.unacked = append(flow.unacked, chunk...) + n.armRetransmit(flow) flow.mu.Unlock() n.sendTCPSegment(flow, seq, ack, 0x18, chunk) // PSH+ACK } } if err != nil { + select { + case <-flow.done: + return // closed by the Mac's RST, a give-up or Close + default: + } flow.mu.Lock() seq := flow.ourSeq ack := flow.macSeq - flow.ourSeq++ + graceful := errors.Is(err, io.EOF) + if graceful { + flow.ourSeq++ + flow.finSent = true + n.armRetransmit(flow) + } flow.mu.Unlock() - if errors.Is(err, io.EOF) { - n.sendTCPSegment(flow, seq, ack, 0x11, nil) // FIN+ACK — graceful close - } else { + if !graceful { n.sendTCPSegment(flow, seq, ack, 0x14, nil) // RST+ACK — abortive close + return + } + n.sendTCPSegment(flow, seq, ack, 0x11, nil) // FIN+ACK — graceful close + // Keep the flow until the Mac has acknowledged our FIN (it is + // resent until then) and closed its own side, so its FIN is ACKed. + select { + case <-flow.done: + case <-time.After(osNATTCPCloseLinger): } return } diff --git a/port/nat/ipnat_test.go b/port/nat/ipnat_test.go new file mode 100644 index 00000000..7ead53e9 --- /dev/null +++ b/port/nat/ipnat_test.go @@ -0,0 +1,270 @@ +package nat + +import ( + "bytes" + "encoding/binary" + "io" + "net" + "sync" + "testing" + "time" + + "github.com/ObsoleteMadness/ClassicStack/protocol/ddp" + "github.com/ObsoleteMadness/ClassicStack/service" +) + +// captureRouter records the datagrams OSNAT routes toward the Mac. Only Route +// is used by the NAT; the embedded interface leaves the rest unimplemented. +type captureRouter struct { + service.Router + mu sync.Mutex + segs []tcpSeg +} + +// tcpSeg is a decoded TCP segment sent toward the Mac. +type tcpSeg struct { + seq, ack uint32 + flags byte + data []byte +} + +func (r *captureRouter) Route(d ddp.Datagram, _ bool) error { + pkt := d.Data + ihl := int(pkt[0]&0xf) * 4 + tcp := pkt[ihl:] + off := int(tcp[12]>>4) * 4 + r.mu.Lock() + r.segs = append(r.segs, tcpSeg{ + seq: binary.BigEndian.Uint32(tcp[4:8]), + ack: binary.BigEndian.Uint32(tcp[8:12]), + flags: tcp[13], + data: append([]byte(nil), tcp[off:]...), + }) + r.mu.Unlock() + return nil +} + +// waitSeg waits for the n-th segment (0-based) routed toward the Mac. +func (r *captureRouter) waitSeg(t *testing.T, n int) tcpSeg { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + r.mu.Lock() + if len(r.segs) > n { + s := r.segs[n] + r.mu.Unlock() + return s + } + r.mu.Unlock() + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("segment %d never sent", n) + return tcpSeg{} +} + +func (r *captureRouter) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.segs) +} + +var ( + testClientIP = [4]byte{192, 168, 100, 2} + testServerIP = [4]byte{10, 0, 0, 1} +) + +const ( + testMacISN = 1000 + testOurISN = 5000 + testWindow = 1092 +) + +// newTestFlow returns an OSNAT with one established flow whose host side is +// one end of a pipe; the other end is returned as the "server". +func newTestFlow(t *testing.T) (*OSNAT, *captureRouter, *tcpFwdFlow, osFlowKey, net.Conn) { + t.Helper() + r := &captureRouter{} + n := &OSNAT{router: r, socket: 72, ddpType: 22, tcpFlows: make(map[osFlowKey]*tcpFwdFlow), stop: make(chan struct{})} + local, server := net.Pipe() + t.Cleanup(func() { _ = server.Close(); _ = local.Close() }) + flow := &tcpFwdFlow{ + conn: local, + clientIP: testClientIP, serverIP: testServerIP, + clientPort: 1025, serverPort: 70, + ourISN: testOurISN, + ourSeq: testOurISN + 1, + macSeq: testMacISN + 1, + macAck: testOurISN + 1, + macWindow: testWindow, + mss: osNATMaxSegment, + rto: 20 * time.Millisecond, + expiry: time.Now().Add(osNATTCPTimeout), + windowAdv: make(chan struct{}, 1), + done: make(chan struct{}), + } + key := osFlowKey{6, testClientIP, 1025, testServerIP, 70} + n.tcpFlows[key] = flow + return n, r, flow, key, server +} + +// macSegment builds an IP packet carrying a TCP segment from the Mac. +func macSegment(seq, ack uint32, flags byte, data []byte) []byte { + hdr := make([]byte, 20+len(data)) + binary.BigEndian.PutUint16(hdr[0:2], 1025) + binary.BigEndian.PutUint16(hdr[2:4], 70) + binary.BigEndian.PutUint32(hdr[4:8], seq) + binary.BigEndian.PutUint32(hdr[8:12], ack) + hdr[12] = 0x50 + hdr[13] = flags + binary.BigEndian.PutUint16(hdr[14:16], testWindow) + copy(hdr[20:], data) + return BuildIPv4Packet(testClientIP[:], testServerIP[:], 6, hdr) +} + +// readServer collects what the NAT writes to the server side. +func readServer(server net.Conn) <-chan []byte { + out := make(chan []byte, 1) + go func() { + b, _ := io.ReadAll(server) + out <- b + }() + return out +} + +func TestHandleTCP_RetransmitFromMacWrittenOnce(t *testing.T) { + n, r, flow, _, server := newTestFlow(t) + got := readServer(server) + + n.Forward(macSegment(testMacISN+1, testOurISN+1, 0x18, []byte("abc")), 0, 0) + n.Forward(macSegment(testMacISN+1, testOurISN+1, 0x18, []byte("abc")), 0, 0) // retransmit + n.Forward(macSegment(testMacISN+1, testOurISN+1, 0x18, []byte("abcdef")), 0, 0) // overlap + n.Forward(macSegment(testMacISN+20, testOurISN+1, 0x18, []byte("zzz")), 0, 0) // out of order + for i, want := range []uint32{testMacISN + 4, testMacISN + 4, testMacISN + 7, testMacISN + 7} { + if s := r.waitSeg(t, i); s.ack != want || s.flags != 0x10 { + t.Errorf("ACK %d: ack %d flags %#x, want ack %d", i, s.ack, s.flags, want) + } + } + _ = flow.conn.Close() + if b := <-got; !bytes.Equal(b, []byte("abcdef")) { + t.Errorf("server got %q, want %q", b, "abcdef") + } +} + +func TestTCP_FastRetransmitOnDuplicateACKs(t *testing.T) { + n, r, flow, key, server := newTestFlow(t) + flow.rto = time.Hour // only the duplicate ACKs may trigger the resend + go n.tcpServerReadLoop(key, flow) + + if _, err := server.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + first := r.waitSeg(t, 0) + if first.seq != testOurISN+1 || string(first.data) != "hello" { + t.Fatalf("first segment seq %d %q", first.seq, first.data) + } + for range osNATTCPDupAckThreshold { + n.Forward(macSegment(testMacISN+1, testOurISN+1, 0x10, nil), 0, 0) + } + resend := r.waitSeg(t, 1) + if resend.seq != first.seq || string(resend.data) != "hello" { + t.Errorf("resend seq %d %q, want seq %d %q", resend.seq, resend.data, first.seq, "hello") + } + flow.closeConn() +} + +func TestTCP_RetransmitTimeoutThenACK(t *testing.T) { + n, r, flow, key, server := newTestFlow(t) + go n.tcpServerReadLoop(key, flow) + + if _, err := server.Write([]byte("data")); err != nil { + t.Fatal(err) + } + r.waitSeg(t, 0) + resend := r.waitSeg(t, 1) // after the 20 ms RTO + if resend.seq != testOurISN+1 || string(resend.data) != "data" { + t.Errorf("resend seq %d %q", resend.seq, resend.data) + } + n.Forward(macSegment(testMacISN+1, testOurISN+5, 0x10, nil), 0, 0) + flow.mu.Lock() + if len(flow.unacked) != 0 || flow.rtxTimer != nil { + t.Errorf("after ACK: %d bytes unacked, timer %v", len(flow.unacked), flow.rtxTimer != nil) + } + flow.mu.Unlock() + before := r.count() + time.Sleep(100 * time.Millisecond) + if r.count() != before { + t.Errorf("resent after everything was acknowledged") + } + flow.closeConn() +} + +func TestTCP_FINResentAndMacFINAcked(t *testing.T) { + n, r, flow, key, server := newTestFlow(t) + exited := make(chan struct{}) + go func() { n.tcpServerReadLoop(key, flow); close(exited) }() + + _ = server.Close() // server EOF + fin := r.waitSeg(t, 0) + if fin.flags != 0x11 || fin.seq != testOurISN+1 { + t.Fatalf("FIN flags %#x seq %d", fin.flags, fin.seq) + } + if again := r.waitSeg(t, 1); again.flags != 0x11 || again.seq != fin.seq { + t.Errorf("FIN resend flags %#x seq %d", again.flags, again.seq) + } + // The Mac acknowledges our FIN and sends its own. + n.Forward(macSegment(testMacISN+1, testOurISN+2, 0x11, nil), 0, 0) + select { + case <-exited: + case <-time.After(2 * time.Second): + t.Fatal("flow not closed after both FINs") + } + r.mu.Lock() + last := r.segs[len(r.segs)-1] + r.mu.Unlock() + if last.flags != 0x10 || last.ack != testMacISN+2 { + t.Errorf("Mac's FIN answered with flags %#x ack %d, want ACK %d", last.flags, last.ack, testMacISN+2) + } + n.tcpMu.Lock() + _, still := n.tcpFlows[key] + n.tcpMu.Unlock() + if still { + t.Error("flow still in the table") + } +} + +func TestTCP_RetransmittedSYNGetsSYNACK(t *testing.T) { + n, r, _, _, _ := newTestFlow(t) + n.Forward(macSegment(testMacISN, 0, 0x02, nil), 0, 0) + s := r.waitSeg(t, 0) + if s.flags != 0x12 || s.seq != testOurISN || s.ack != testMacISN+1 { + t.Errorf("SYN-ACK flags %#x seq %d ack %d", s.flags, s.seq, s.ack) + } +} + +func TestTCP_PartialACKResendsNextSegment(t *testing.T) { + n, r, flow, key, server := newTestFlow(t) + go n.tcpServerReadLoop(key, flow) + + // Two full segments fill the Mac's 1092-byte window; say both are lost. + if _, err := server.Write(bytes.Repeat([]byte("x"), 2*osNATMaxSegment)); err != nil { + t.Fatal(err) + } + r.waitSeg(t, 1) + if s := r.waitSeg(t, 2); s.seq != testOurISN+1 { // timeout resend of the first + t.Fatalf("timeout resend seq %d", s.seq) + } + // The Mac acknowledges the first segment only: the second must be resent + // at once, not after another (now 1 s) timeout. + acked := time.Now() + n.Forward(macSegment(testMacISN+1, testOurISN+1+osNATMaxSegment, 0x10, nil), 0, 0) + for i := 3; ; i++ { + s := r.waitSeg(t, i) + if s.seq == testOurISN+1+osNATMaxSegment { + if d := time.Since(acked); d > 500*time.Millisecond { + t.Errorf("second segment resent %v after the partial ACK", d) + } + break + } + } + flow.closeConn() +}