diff --git a/vmnetsock/adapter.go b/vmnetsock/adapter.go index af89a24a..ede056aa 100644 --- a/vmnetsock/adapter.go +++ b/vmnetsock/adapter.go @@ -52,7 +52,13 @@ const ( // With TSO enabled, the adapter tracks TCP handshakes and splits complete-checksum // TCP frames using the peer's MSS and the interface MTU. It also converts standard // Virtio TCPv4 and TCPv6 GSO, including ECN, using gso_size as a payload limit. -// GSO conversion requires TSO on iface. UDP GSO is not supported. +// Virtio UDP_L4 GSO is split into complete UDP datagrams at gso_size boundaries, +// with a shorter final datagram when needed. It requires no TCP handshake. +// GSO conversion requires TSO on iface for the aggregate input and output buffers. +// Aggregate input is bounded by iface.MaxPacketSize(); each output must fit MTU. +// UDP gso_size plus IP and UDP headers must fit MTU. +// UDP supports fixed IPv4/IPv6 headers without fragments, extensions, or VLANs. +// Legacy UFO (IP fragmentation), UDP RSC, and UDP ECN GSO modifiers are unsupported. // TCP RSC normalization also requires TSO. It splits checksum-validated packets // and recalculates their checksums, including when vmnet supplies no GSO size. // RSC metadata that reports duplicate ACKs is not supported. @@ -72,7 +78,7 @@ func WithVirtioHeaderConversion() Option { } } -// WithPeerVirtioHeaderNormalization completes partial checksums and splits TCP +// WithPeerVirtioHeaderNormalization completes partial checksums and splits TCP/UDP // GSO received from the socket peer before sending packets to iface. Both ends // exchange 12-byte little-endian Virtio network headers. Start iface with // vmnet.WithInterfaceVirtioHeader(true). Normalized packets have zeroed headers. @@ -82,9 +88,13 @@ func WithVirtioHeaderConversion() Option { // from the socket peer require TSO and a supported TCP packet. Close returns // the forwarding error for unsupported RSC metadata, including duplicate ACKs. // -// Checksum completion works without TSO. TCPv4 and TCPv6 GSO, including ECN, -// require TSO on iface. Output is limited by gso_size, the interface MTU, and -// the iface-side receiver's advertised MSS. UDP GSO is not supported. +// Checksum completion works without TSO. GSO requires TSO on iface for the +// aggregate input and output buffers. TCPv4 and TCPv6 GSO support ECN, with output +// limited by gso_size, the interface MTU, and the iface-side receiver's advertised +// MSS. UDP_L4 GSO preserves gso_size datagram boundaries without a handshake. +// UDP gso_size plus IP and UDP headers must fit MTU. Aggregate input is bounded by +// iface.MaxPacketSize(). UDP has the packet layout and metadata restrictions of +// [WithVirtioHeaderConversion], including no legacy UFO support. // TCP RSC uses the same MTU and MSS limits and recalculates checksums from the // packet instead of interpreting its coalescing counters as checksum offsets. // @@ -353,7 +363,7 @@ func (s *adapterState) forwardToPeer(ctx context.Context, iface *vmnet.Interface continue } if config.mode == packetEthernet && s.tcp != nil { - if err := s.forwardTCPPackets(packets, peerFrames, false, writePackets); err != nil { + if err := s.forwardVirtioPackets(packets, peerFrames, false, writePackets); err != nil { return err } continue @@ -392,7 +402,7 @@ func (s *adapterState) normalizePeerPackets(packets, output [][]byte, writePacke } } if s.tcp != nil { - return s.forwardTCPPackets(packets, output, true, writePackets) + return s.forwardVirtioPackets(packets, output, true, writePackets) } for _, packet := range packets { if _, err := virtioEthernetPacket(packet); err != nil { @@ -415,9 +425,9 @@ func (s *adapterState) forwardObservedPackets(packets [][]byte, writePackets fun return writePackets(packets) } -// forwardTCPPackets copies each borrowed segment before its scratch buffer is reused. -// Flush full batches so a large frame with a small MSS does not grow the batch. -func (s *adapterState) forwardTCPPackets(packets, peerFrames [][]byte, virtioOutput bool, writePackets func([][]byte) error) error { +// forwardVirtioPackets copies each borrowed segment before its scratch buffer is reused. +// Flush full batches so a large GSO frame does not grow the batch. +func (s *adapterState) forwardVirtioPackets(packets, peerFrames [][]byte, virtioOutput bool, writePackets func([][]byte) error) error { prefix := 0 if virtioOutput { prefix = virtioHeaderSize @@ -442,6 +452,12 @@ func (s *adapterState) forwardTCPPackets(packets, peerFrames [][]byte, virtioOut return nil } for _, frame := range packets { + if len(frame) >= 2 && frame[1]&^virtioGSOECN == virtioGSOUDPL4 { + if err := forwardVirtioUDPGSO(frame, s.tcp.mtu, emit); err != nil { + return err + } + continue + } if err := s.tcp.forwardVirtio(frame, time.Now(), emit); err != nil { return err } diff --git a/vmnetsock/checksum.go b/vmnetsock/checksum.go new file mode 100644 index 00000000..6e7f2978 --- /dev/null +++ b/vmnetsock/checksum.go @@ -0,0 +1,22 @@ +package vmnetsock + +import "encoding/binary" + +func packetSum(data []byte, sum uint64) uint64 { + for len(data) >= 2 { + sum += uint64(binary.BigEndian.Uint16(data)) + data = data[2:] + } + if len(data) > 0 { + sum += uint64(data[0]) << 8 + } + return sum +} + +func packetChecksum(data []byte, sum uint64) uint16 { + sum = packetSum(data, sum) + for sum>>16 != 0 { + sum = sum&0xffff + sum>>16 + } + return ^uint16(sum) +} diff --git a/vmnetsock/gso_test.go b/vmnetsock/gso_test.go index 6c66793e..f2316b32 100644 --- a/vmnetsock/gso_test.go +++ b/vmnetsock/gso_test.go @@ -94,7 +94,7 @@ func TestVirtioTCPGSO(t *testing.T) { var sizes []int var joined []byte batches := 0 - err := state.forwardTCPPackets([][]byte{packet}, buffers, false, func(frames [][]byte) error { + err := state.forwardVirtioPackets([][]byte{packet}, buffers, false, func(frames [][]byte) error { batches++ if len(frames) > 2 { t.Fatal("output exceeded the available batch buffers") @@ -168,7 +168,7 @@ func TestVirtioTCPGSOInvalidMetadata(t *testing.T) { {"wrong checksum start", func(p []byte) { p[6]++ }}, {"wrong checksum offset", func(p []byte) { p[8]++ }}, {"UFO", func(p []byte) { p[1] = 3 }}, - {"UDP L4", func(p []byte) { p[1] = 5 }}, + {"UDP L4 with TCP payload", func(p []byte) { p[1] = virtioGSOUDPL4 }}, {"ECN alone", func(p []byte) { p[1] = 0x80 }}, {"unknown GSO", func(p []byte) { p[1] = 0x41 }}, } { @@ -177,7 +177,7 @@ func TestVirtioTCPGSOInvalidMetadata(t *testing.T) { packet := gsoTestPacket(false, false, 701, 250) test.edit(packet) written := 0 - err := state.forwardTCPPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { + err := state.forwardVirtioPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { written += len(frames) return nil }) @@ -189,7 +189,7 @@ func TestVirtioTCPGSOInvalidMetadata(t *testing.T) { state := gsoTestState(t, false, 1460) valid := gsoTestPacket(false, false, 701, 250) for _, packet := range [][]byte{nil, valid[:1], valid[:25], valid[:26], gsoTestPacket(false, false, 0, 250)} { - if err := state.forwardTCPPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func([][]byte) error { + if err := state.forwardVirtioPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func([][]byte) error { t.Fatal("short or empty GSO emitted data") return nil }); err == nil { @@ -199,7 +199,7 @@ func TestVirtioTCPGSOInvalidMetadata(t *testing.T) { state = &adapterState{ tcp: newTCPSegmentation(1500), } - if err := state.forwardTCPPackets([][]byte{gsoTestPacket(false, false, 701, 250)}, [][]byte{make([]byte, 1514)}, false, func([][]byte) error { + if err := state.forwardVirtioPackets([][]byte{gsoTestPacket(false, false, 701, 250)}, [][]byte{make([]byte, 1514)}, false, func([][]byte) error { t.Fatal("unobserved connection emitted data") return nil }); err == nil { @@ -211,7 +211,7 @@ func TestVirtioTCPGSOWriteError(t *testing.T) { state := gsoTestState(t, false, 1460) want := errors.New("peer closed") calls := 0 - err := state.forwardTCPPackets([][]byte{gsoTestPacket(false, false, 2101, 60)}, [][]byte{make([]byte, 1514), make([]byte, 1514)}, false, func([][]byte) error { + err := state.forwardVirtioPackets([][]byte{gsoTestPacket(false, false, 2101, 60)}, [][]byte{make([]byte, 1514), make([]byte, 1514)}, false, func([][]byte) error { calls++ return want }) @@ -238,7 +238,7 @@ func TestVirtioTCPGSOFrameLength(t *testing.T) { length := binary.BigEndian.Uint16(packet[lengthOffset:]) binary.BigEndian.PutUint16(packet[lengthOffset:], length-64) written := 0 - err := state.forwardTCPPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { + err := state.forwardVirtioPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { written += len(frames) return nil }) @@ -258,7 +258,7 @@ func TestVirtioTCPGSOFrameLength(t *testing.T) { binary.LittleEndian.PutUint16(packet[6:], 34) binary.LittleEndian.PutUint16(packet[8:], 16) written := 0 - err := state.forwardTCPPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { + err := state.forwardVirtioPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { for _, frame := range frames { written++ if len(frame) != 55 || frame[54] != 20 || wireSum(frame[14:34]) != 65535 { diff --git a/vmnetsock/rsc_test.go b/vmnetsock/rsc_test.go index 5d9df281..ce9e1046 100644 --- a/vmnetsock/rsc_test.go +++ b/vmnetsock/rsc_test.go @@ -107,7 +107,7 @@ func TestVirtioTCPRSCRejectsUnsupportedMetadata(t *testing.T) { state := gsoTestState(t, false, 1460) packet := rscTestPacket(false, 0, 37, 0) test.edit(packet) - err := state.forwardTCPPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func([][]byte) error { + err := state.forwardVirtioPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func([][]byte) error { t.Fatal("unsupported RSC metadata emitted a packet") return nil }) @@ -123,7 +123,7 @@ func TestVirtioTCPRSCAcknowledgmentAndErrors(t *testing.T) { packet := rscTestPacket(false, 0, 0, 0) packet[virtioHeaderSize+ethernetHeaderSize+ipv4HeaderSize+tcpFlagsOffset] = 0x10 calls := 0 - if err := state.forwardTCPPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { + if err := state.forwardVirtioPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { calls++ if len(frames) != 1 || len(frames[0]) != 66 { t.Fatalf("ACK-only RSC output = %d frames", len(frames)) @@ -140,7 +140,7 @@ func TestVirtioTCPRSCAcknowledgmentAndErrors(t *testing.T) { } packet = rscTestPacket(false, 0, 3001, 0) state = &adapterState{tcp: newTCPSegmentation(1500)} - if err := state.forwardTCPPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func([][]byte) error { + if err := state.forwardVirtioPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func([][]byte) error { t.Fatal("unknown connection emitted RSC data") return nil }); err == nil { @@ -149,7 +149,7 @@ func TestVirtioTCPRSCAcknowledgmentAndErrors(t *testing.T) { state = gsoTestState(t, false, 1460) want := errors.New("receiver closed") calls = 0 - err := state.forwardTCPPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func([][]byte) error { + err := state.forwardVirtioPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func([][]byte) error { calls++ return want }) @@ -165,7 +165,7 @@ func TestVirtioRSCNonTCPIsUnchanged(t *testing.T) { binary.LittleEndian.PutUint16(packet[virtioRSCSegmentsOffset:], 65535) want := bytes.Clone(packet[virtioHeaderSize:]) calls := 0 - if err := state.forwardTCPPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { + if err := state.forwardVirtioPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { calls++ if len(frames) != 1 || !bytes.Equal(frames[0], want) { t.Fatal("non-TCP RSC Ethernet bytes changed") diff --git a/vmnetsock/tcp.go b/vmnetsock/tcp.go index 220fc84f..23166c52 100644 --- a/vmnetsock/tcp.go +++ b/vmnetsock/tcp.go @@ -529,7 +529,7 @@ func (p *tcpPacket) updateSegmentHeaders(ip []byte, segmentIndex int) { id := binary.BigEndian.Uint16(p.ip[ipv4IdentificationOffset:]) + uint16(segmentIndex) binary.BigEndian.PutUint16(ip[ipv4IdentificationOffset:], id) binary.BigEndian.PutUint16(ip[ipv4ChecksumOffset:], 0) - binary.BigEndian.PutUint16(ip[ipv4ChecksumOffset:], tcpChecksum(ip[:ipv4HeaderSize], 0)) + binary.BigEndian.PutUint16(ip[ipv4ChecksumOffset:], packetChecksum(ip[:ipv4HeaderSize], 0)) } else { binary.BigEndian.PutUint16(ip[ipv6PayloadLengthOffset:], uint16(len(tcp))) } @@ -538,28 +538,9 @@ func (p *tcpPacket) updateSegmentHeaders(ip []byte, segmentIndex int) { binary.BigEndian.PutUint16(tcp[tcpChecksumOffset:], 0) sum := uint64(ipProtocolTCP + len(tcp)) if p.version == ipv4Version { - sum = tcpSum(ip[ipv4SourceOffset:ipv4HeaderSize], sum) + sum = packetSum(ip[ipv4SourceOffset:ipv4HeaderSize], sum) } else { - sum = tcpSum(ip[ipv6SourceOffset:ipv6HeaderSize], sum) + sum = packetSum(ip[ipv6SourceOffset:ipv6HeaderSize], sum) } - binary.BigEndian.PutUint16(tcp[tcpChecksumOffset:], tcpChecksum(tcp, sum)) -} - -func tcpSum(data []byte, sum uint64) uint64 { - for len(data) >= 2 { - sum += uint64(binary.BigEndian.Uint16(data)) - data = data[2:] - } - if len(data) > 0 { - sum += uint64(data[0]) << 8 - } - return sum -} - -func tcpChecksum(data []byte, sum uint64) uint16 { - sum = tcpSum(data, sum) - for sum>>16 != 0 { - sum = sum&0xffff + sum>>16 - } - return ^uint16(sum) + binary.BigEndian.PutUint16(tcp[tcpChecksumOffset:], packetChecksum(tcp, sum)) } diff --git a/vmnetsock/tcp_batch_test.go b/vmnetsock/tcp_batch_test.go index 93f38bb7..f4bbfb1f 100644 --- a/vmnetsock/tcp_batch_test.go +++ b/vmnetsock/tcp_batch_test.go @@ -30,7 +30,7 @@ func TestTCPBatchCopiesSegments(t *testing.T) { peerFrames := [][]byte{make([]byte, 1514), make([]byte, 1514)} var batchSizes []int var received []byte - err := state.forwardTCPPackets([][]byte{packet}, peerFrames, false, func(frames [][]byte) error { + err := state.forwardVirtioPackets([][]byte{packet}, peerFrames, false, func(frames [][]byte) error { batchSizes = append(batchSizes, len(frames)) for _, frame := range frames { if len(frame) < 54 { diff --git a/vmnetsock/udp.go b/vmnetsock/udp.go new file mode 100644 index 00000000..15faccb4 --- /dev/null +++ b/vmnetsock/udp.go @@ -0,0 +1,145 @@ +package vmnetsock + +import ( + "encoding/binary" + "fmt" +) + +// UDP's header and checksum layout follow RFC 768. +// https://www.rfc-editor.org/rfc/rfc768 +const ( + ipProtocolUDP = 17 + udpHeaderSize = 8 + udpLengthOffset = 4 + udpChecksumOffset = 6 +) + +type udpPacket struct { + frame []byte + ip []byte + udp []byte + version int + ipHeader int +} + +// The returned slices borrow frame. +func parseUDPPacket(frame []byte) (udpPacket, error) { + if len(frame) < ethernetHeaderSize { + return udpPacket{}, fmt.Errorf("short Ethernet frame") + } + var p udpPacket + var err error + switch binary.BigEndian.Uint16(frame[etherTypeOffset:]) { + case etherTypeIPv4: + p.ip, err = parseIPv4UDP(frame[ethernetHeaderSize:]) + p.version, p.ipHeader = ipv4Version, ipv4HeaderSize + case etherTypeIPv6: + p.ip, err = parseIPv6UDP(frame[ethernetHeaderSize:]) + p.version, p.ipHeader = ipv6Version, ipv6HeaderSize + default: + return udpPacket{}, fmt.Errorf("UDP GSO requires untagged IPv4 or IPv6") + } + if err != nil { + return udpPacket{}, err + } + // IP length excludes Ethernet padding, which must not enter a UDP datagram. + p.frame = frame[:ethernetHeaderSize+len(p.ip)] + p.udp = p.ip[p.ipHeader:] + if int(binary.BigEndian.Uint16(p.udp[udpLengthOffset:])) != len(p.udp) { + return udpPacket{}, fmt.Errorf("UDP length does not match the IP payload") + } + if len(p.udp) == udpHeaderSize { + return udpPacket{}, fmt.Errorf("UDP GSO requires UDP payload") + } + padding := len(frame) - len(p.frame) + if padding > max(0, ethernetMinimumFrameSize-len(p.frame)) { + return udpPacket{}, fmt.Errorf("UDP GSO frame length does not match the IP length") + } + return p, nil +} + +func parseIPv4UDP(ip []byte) ([]byte, error) { + if len(ip) < ipv4HeaderSize || ip[0]>>4 != ipv4Version { + return nil, fmt.Errorf("UDP GSO requires a fixed IPv4 header") + } + headerSize := int(ip[0]&0x0f) * 4 + if headerSize != ipv4HeaderSize { + return nil, fmt.Errorf("UDP GSO requires a fixed IPv4 header") + } + length := int(binary.BigEndian.Uint16(ip[ipv4TotalLengthOffset:])) + if length < ipv4HeaderSize+udpHeaderSize || length > len(ip) { + return nil, fmt.Errorf("invalid IPv4 UDP length") + } + fragments := binary.BigEndian.Uint16(ip[ipv4FragmentFlagsOffset:]) + if ip[ipv4ProtocolOffset] != ipProtocolUDP || fragments&(ipv4MoreFragments|ipv4FragmentOffsetMask) != 0 { + return nil, fmt.Errorf("UDP GSO requires unfragmented UDP") + } + return ip[:length], nil +} + +func parseIPv6UDP(ip []byte) ([]byte, error) { + if len(ip) < ipv6HeaderSize || ip[0]>>4 != ipv6Version { + return nil, fmt.Errorf("invalid IPv6 header") + } + length := ipv6HeaderSize + int(binary.BigEndian.Uint16(ip[ipv6PayloadLengthOffset:])) + if length < ipv6HeaderSize+udpHeaderSize || length > len(ip) { + return nil, fmt.Errorf("invalid IPv6 UDP length") + } + if ip[ipv6NextHeaderOffset] != ipProtocolUDP { + return nil, fmt.Errorf("UDP GSO requires UDP without IPv6 extensions") + } + return ip[:length], nil +} + +// emitSegments reuses its scratch frame after each callback. A consumer retaining +// a frame past the callback must copy it. Each output is a complete UDP datagram, +// even when size exceeds the remaining payload length. +func (p *udpPacket) emitSegments(size int, emit func([]byte) error) error { + headers := ethernetHeaderSize + p.ipHeader + udpHeaderSize + buffer := make([]byte, headers+size) + payload := p.udp[udpHeaderSize:] + for offset := 0; offset < len(payload); offset += size { + end := min(offset+size, len(payload)) + frame := buffer[:headers+end-offset] + copy(frame, p.frame[:headers]) + copy(frame[headers:], payload[offset:end]) + ip := frame[ethernetHeaderSize:] + p.updateIPHeader(ip, offset/size) + p.updateUDPHeader(ip) + if err := emit(frame); err != nil { + return err + } + } + return nil +} + +func (p *udpPacket) updateIPHeader(ip []byte, segmentIndex int) { + if p.version == ipv6Version { + binary.BigEndian.PutUint16(ip[ipv6PayloadLengthOffset:], uint16(len(ip)-ipv6HeaderSize)) + return + } + binary.BigEndian.PutUint16(ip[ipv4TotalLengthOffset:], uint16(len(ip))) + id := binary.BigEndian.Uint16(p.ip[ipv4IdentificationOffset:]) + uint16(segmentIndex) + binary.BigEndian.PutUint16(ip[ipv4IdentificationOffset:], id) + binary.BigEndian.PutUint16(ip[ipv4ChecksumOffset:], 0) + binary.BigEndian.PutUint16(ip[ipv4ChecksumOffset:], packetChecksum(ip[:ipv4HeaderSize], 0)) +} + +func (p *udpPacket) updateUDPHeader(ip []byte) { + udp := ip[p.ipHeader:] + binary.BigEndian.PutUint16(udp[udpLengthOffset:], uint16(len(udp))) + binary.BigEndian.PutUint16(udp[udpChecksumOffset:], 0) + // The UDP length also appears in the checksum pseudo-header. + sum := uint64(ipProtocolUDP + len(udp)) + if p.version == ipv4Version { + sum = packetSum(ip[ipv4SourceOffset:ipv4HeaderSize], sum) + } else { + sum = packetSum(ip[ipv6SourceOffset:ipv6HeaderSize], sum) + } + checksum := packetChecksum(udp, sum) + // RFC 768 encodes a computed zero as all ones. + if checksum == 0 { + checksum = 0xffff + } + binary.BigEndian.PutUint16(udp[udpChecksumOffset:], checksum) +} diff --git a/vmnetsock/udp_test.go b/vmnetsock/udp_test.go new file mode 100644 index 00000000..bf1a071d --- /dev/null +++ b/vmnetsock/udp_test.go @@ -0,0 +1,347 @@ +package vmnetsock + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "slices" + "testing" +) + +const ( + udpTestIPStart = virtioHeaderSize + ethernetHeaderSize + udpTestIPv4UDPStart = udpTestIPStart + ipv4HeaderSize + udpTestIPv6UDPStart = udpTestIPStart + ipv6HeaderSize + udpTestIPv4PayloadStart = udpTestIPv4UDPStart + udpHeaderSize +) + +func udpGSOPacket(v6 bool, payload, size int) []byte { + ipSize := 20 + if v6 { + ipSize = 40 + } + p := make([]byte, udpTestIPStart+ipSize+udpHeaderSize+payload) + p[0], p[1] = virtioNeedsChecksum, virtioGSOUDPL4 + binary.LittleEndian.PutUint16(p[2:], 65535) // hdr_len is only a hint. + binary.LittleEndian.PutUint16(p[virtioGSOSizeOffset:], uint16(size)) + binary.LittleEndian.PutUint16(p[virtioChecksumStartOffset:], uint16(14+ipSize)) + binary.LittleEndian.PutUint16(p[virtioChecksumFieldOffset:], udpChecksumOffset) + binary.LittleEndian.PutUint16(p[10:], 65535) + frame := p[virtioHeaderSize:] + copy(frame, []byte{2, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 1}) + ip := frame[ethernetHeaderSize:] + if v6 { + binary.BigEndian.PutUint16(frame[12:], 0x86dd) + ip[0], ip[1], ip[2], ip[3] = 0x6a, 0xb1, 0x23, 0x45 + binary.BigEndian.PutUint16(ip[4:], uint16(8+payload)) + ip[6], ip[7] = 17, 63 + copy(ip[8:24], []byte{0x20, 1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) + copy(ip[24:40], []byte{0x20, 1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}) + } else { + binary.BigEndian.PutUint16(frame[12:], 0x0800) + ip[0], ip[1], ip[8], ip[9] = 0x45, 0xab, 63, 17 + binary.BigEndian.PutUint16(ip[2:], uint16(28+payload)) + binary.BigEndian.PutUint16(ip[4:], 65535) + binary.BigEndian.PutUint16(ip[6:], 0x4000) + copy(ip[12:20], []byte{192, 0, 2, 1, 192, 0, 2, 2}) + binary.BigEndian.PutUint16(ip[10:], ^uint16(wireSum(ip[:20]))) + } + udp := ip[ipSize:] + binary.BigEndian.PutUint16(udp, 40000) + binary.BigEndian.PutUint16(udp[2:], 9000) + binary.BigEndian.PutUint16(udp[udpLengthOffset:], uint16(8+payload)) + binary.BigEndian.PutUint16(udp[udpChecksumOffset:], 0x1234) + for i := range udp[udpHeaderSize:] { + udp[udpHeaderSize+i] = byte(i*29 + 7) + } + return p +} + +func udpWirePseudo(v6 bool, ip []byte, length int) []byte { + if v6 { + p := bytes.Clone(ip[8:40]) + return append(p, 0, 0, byte(length>>8), byte(length), 0, 0, 0, 17) + } + p := bytes.Clone(ip[12:20]) + return append(p, 0, 17, byte(length>>8), byte(length)) +} + +func checkUDPWire(t *testing.T, frame, input []byte, v6 bool, index, size int) { + t.Helper() + ipSize := 20 + if v6 { + ipSize = 40 + } + if len(frame) != 14+ipSize+8+size { + t.Fatalf("datagram %d frame length = %d, want %d", index, len(frame), 14+ipSize+8+size) + } + if !bytes.Equal(frame[:14], input[virtioHeaderSize:udpTestIPStart]) { + t.Fatal("Ethernet addresses or type changed") + } + ip, udp := frame[14:], frame[14+ipSize:] + originalIP := input[udpTestIPStart:] + if v6 { + if binary.BigEndian.Uint16(ip[4:]) != uint16(8+size) || !bytes.Equal(ip[:4], originalIP[:4]) || !bytes.Equal(ip[6:40], originalIP[6:40]) { + t.Fatal("IPv6 length, traffic class, flow label, addresses or hop limit changed") + } + } else { + if binary.BigEndian.Uint16(ip[2:]) != uint16(28+size) || binary.BigEndian.Uint16(ip[4:]) != uint16(65535+index) || wireSum(ip[:20]) != 65535 { + t.Fatal("invalid IPv4 length, ID or checksum") + } + if !bytes.Equal(ip[:2], originalIP[:2]) || !bytes.Equal(ip[6:10], originalIP[6:10]) || !bytes.Equal(ip[12:20], originalIP[12:20]) { + t.Fatal("IPv4 flags, traffic class, protocol, addresses or TTL changed") + } + } + if !bytes.Equal(udp[:4], originalIP[ipSize:ipSize+4]) || binary.BigEndian.Uint16(udp[udpLengthOffset:]) != uint16(8+size) { + t.Fatal("UDP ports or length changed") + } + if binary.BigEndian.Uint16(udp[udpChecksumOffset:]) == 0 || wireSum(append(udpWirePseudo(v6, ip, len(udp)), udp...)) != 65535 { + t.Fatal("invalid UDP checksum") + } +} + +func TestVirtioUDPGSO(t *testing.T) { + for _, test := range []struct { + name string + v6 bool + payload int + size int + segments []int + }{ + {"IPv4 exact boundary", false, 500, 250, []int{250, 250}}, + {"IPv6 exact boundary", true, 500, 250, []int{250, 250}}, + {"IPv4 odd and short last", false, 503, 251, []int{251, 251, 1}}, + {"IPv6 odd and short last", true, 503, 251, []int{251, 251, 1}}, + {"IPv4 single", false, 37, 250, []int{37}}, + {"IPv6 single", true, 37, 250, []int{37}}, + {"IPv4 MTU boundary", false, 2945, 1472, []int{1472, 1472, 1}}, + {"IPv6 MTU boundary", true, 2905, 1452, []int{1452, 1452, 1}}, + } { + for _, peer := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/peer=%v", test.name, peer), func(t *testing.T) { + state := &adapterState{tcp: newTCPSegmentation(1500)} + packet := udpGSOPacket(test.v6, test.payload, test.size) + original := bytes.Clone(packet) + ipSize := 20 + if test.v6 { + ipSize = 40 + } + var sizes []int + var joined []byte + write := func(frames [][]byte) error { + for _, frame := range frames { + if peer { + if !bytes.Equal(frame[:12], make([]byte, 12)) { + t.Fatal("normalized Virtio header is not zero") + } + frame = frame[12:] + } + if len(sizes) >= len(test.segments) { + t.Fatal("extra UDP datagram") + } + checkUDPWire(t, frame, original, test.v6, len(sizes), test.segments[len(sizes)]) + sizes = append(sizes, len(frame)-14-ipSize-8) + joined = append(joined, frame[14+ipSize+8:]...) + } + return nil + } + buffers := [][]byte{bytes.Repeat([]byte{0xa5}, 1526), bytes.Repeat([]byte{0xa5}, 1526)} + var err error + if peer { + err = state.normalizePeerPackets([][]byte{packet}, buffers, write) + } else { + err = state.forwardVirtioPackets([][]byte{packet}, buffers, false, write) + } + if err != nil { + t.Fatal(err) + } + if !slices.Equal(sizes, test.segments) || !bytes.Equal(joined, original[12+14+ipSize+8:]) { + t.Fatalf("datagram lengths = %v, want %v; payload changed = %v", sizes, test.segments, !bytes.Equal(joined, original[12+14+ipSize+8:])) + } + if !bytes.Equal(packet, original) { + t.Fatal("UDP GSO changed its borrowed input") + } + }) + } + } +} + +func TestVirtioUDPGSORejectsInvalidInput(t *testing.T) { + for _, test := range []struct { + name string + v6 bool + edit func([]byte) []byte + }{ + {"missing checksum flag", false, func(p []byte) []byte { p[0] = 0; return p }}, + {"RSC", false, func(p []byte) []byte { p[0] |= 4; return p }}, + {"ECN modifier", false, func(p []byte) []byte { p[1] = 0x85; return p }}, + {"UFO is not UDP L4", false, func(p []byte) []byte { p[1] = 3; return p }}, + {"wrong checksum start", false, func(p []byte) []byte { p[virtioChecksumStartOffset]++; return p }}, + {"wrong checksum offset", false, func(p []byte) []byte { p[virtioChecksumFieldOffset]++; return p }}, + {"zero segment size", false, func(p []byte) []byte { binary.LittleEndian.PutUint16(p[virtioGSOSizeOffset:], 0); return p }}, + {"IPv4 size exceeds MTU", false, func(p []byte) []byte { binary.LittleEndian.PutUint16(p[virtioGSOSizeOffset:], 1473); return p }}, + {"IPv6 size exceeds MTU", true, func(p []byte) []byte { binary.LittleEndian.PutUint16(p[virtioGSOSizeOffset:], 1453); return p }}, + {"IPv4 UDP length too short", false, func(p []byte) []byte { + binary.BigEndian.PutUint16(p[udpTestIPv4UDPStart+udpLengthOffset:], 258) + return p + }}, + {"IPv6 UDP length too long", true, func(p []byte) []byte { + binary.BigEndian.PutUint16(p[udpTestIPv6UDPStart+udpLengthOffset:], 260) + return p + }}, + {"IPv4 empty payload", false, func([]byte) []byte { return udpGSOPacket(false, 0, 100) }}, + {"IPv6 empty payload", true, func([]byte) []byte { return udpGSOPacket(true, 0, 100) }}, + {"IPv4 length exceeds frame", false, func(p []byte) []byte { p[udpTestIPStart+ipv4TotalLengthOffset+1]++; return p }}, + {"IPv6 length exceeds frame", true, func(p []byte) []byte { p[udpTestIPStart+ipv6PayloadLengthOffset+1]++; return p }}, + {"IPv4 trailing bytes", false, func(p []byte) []byte { return append(p, 0) }}, + {"IPv6 trailing bytes", true, func(p []byte) []byte { return append(p, 0) }}, + {"IPv4 zero length", false, func(p []byte) []byte { + binary.BigEndian.PutUint16(p[udpTestIPStart+ipv4TotalLengthOffset:], 0) + return p + }}, + {"IPv6 zero length", true, func(p []byte) []byte { + binary.BigEndian.PutUint16(p[udpTestIPStart+ipv6PayloadLengthOffset:], 0) + return p + }}, + {"IPv4 options", false, func(p []byte) []byte { p[udpTestIPStart] = 0x46; return p }}, + {"IPv4 more fragments", false, func(p []byte) []byte { p[udpTestIPStart+ipv4FragmentFlagsOffset] |= 0x20; return p }}, + {"IPv4 fragment offset", false, func(p []byte) []byte { p[udpTestIPStart+ipv4FragmentFlagsOffset+1] = 1; return p }}, + {"IPv6 hop by hop", true, func(p []byte) []byte { p[udpTestIPStart+ipv6NextHeaderOffset] = 0; return p }}, + {"IPv6 routing", true, func(p []byte) []byte { p[udpTestIPStart+ipv6NextHeaderOffset] = 43; return p }}, + {"IPv6 fragment", true, func(p []byte) []byte { p[udpTestIPStart+ipv6NextHeaderOffset] = 44; return p }}, + {"IPv6 authentication", true, func(p []byte) []byte { p[udpTestIPStart+ipv6NextHeaderOffset] = 51; return p }}, + {"IPv6 destination options", true, func(p []byte) []byte { p[udpTestIPStart+ipv6NextHeaderOffset] = 60; return p }}, + {"VLAN", false, func(p []byte) []byte { + binary.BigEndian.PutUint16(p[virtioHeaderSize+etherTypeOffset:], 0x8100) + return p + }}, + {"QinQ", false, func(p []byte) []byte { + binary.BigEndian.PutUint16(p[virtioHeaderSize+etherTypeOffset:], 0x88a8) + return p + }}, + {"TCP payload", false, func(p []byte) []byte { p[udpTestIPStart+ipv4ProtocolOffset] = 6; return p }}, + {"short Virtio header", false, func(p []byte) []byte { return p[:11] }}, + {"short UDP header", false, func(p []byte) []byte { return p[:udpTestIPv4UDPStart+udpHeaderSize-1] }}, + } { + t.Run(test.name, func(t *testing.T) { + state := &adapterState{tcp: newTCPSegmentation(1500)} + packet := test.edit(udpGSOPacket(test.v6, 251, 100)) + written := 0 + err := state.forwardVirtioPackets([][]byte{packet}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { + written += len(frames) + return nil + }) + if err == nil || written != 0 { + t.Fatalf("invalid input: error = %v, written = %d", err, written) + } + }) + } + state := &adapterState{} + if err := state.normalizePeerPackets([][]byte{udpGSOPacket(false, 251, 100)}, nil, func([][]byte) error { + t.Fatal("UDP GSO without TSO emitted packets") + return nil + }); err == nil { + t.Fatal("UDP GSO without TSO was accepted") + } +} + +func TestVirtioUDPGSOZeroChecksum(t *testing.T) { + for _, v6 := range []bool{false, true} { + t.Run(fmt.Sprintf("IPv6=%v", v6), func(t *testing.T) { + p := udpGSOPacket(v6, 2, 2) + ipSize := 20 + if v6 { + ipSize = 40 + } + ip, udp := p[udpTestIPStart:], p[udpTestIPStart+ipSize:] + clear(udp[udpChecksumOffset:]) // A partial checksum seed of zero is valid. + binary.BigEndian.PutUint16(udp[udpHeaderSize:], ^uint16(wireSum(append(udpWirePseudo(v6, ip, 10), udp...)))) + if wireSum(append(udpWirePseudo(v6, ip, 10), udp...)) != 65535 { + t.Fatal("test packet does not require a zero checksum") + } + state := &adapterState{tcp: newTCPSegmentation(1500)} + written := 0 + err := state.forwardVirtioPackets([][]byte{p}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { + for _, frame := range frames { + written++ + checkUDPWire(t, frame, p, v6, 0, 2) + if binary.BigEndian.Uint16(frame[ethernetHeaderSize+ipSize+udpChecksumOffset:]) != 65535 { + t.Fatal("UDP checksum zero was not encoded as 0xffff") + } + } + return nil + }) + if err != nil || written != 1 { + t.Fatalf("zero checksum: error = %v, written = %d", err, written) + } + }) + } +} + +func TestVirtioUDPGSOPadding(t *testing.T) { + p := udpGSOPacket(false, 1, 1) + p = append(p, bytes.Repeat([]byte{0xa5}, 17)...) + state := &adapterState{tcp: newTCPSegmentation(1500)} + written := 0 + err := state.forwardVirtioPackets([][]byte{p}, [][]byte{make([]byte, 1514)}, false, func(frames [][]byte) error { + for _, frame := range frames { + written++ + checkUDPWire(t, frame, p, false, 0, 1) + if frame[42] != 7 { + t.Fatal("Ethernet padding entered the UDP payload") + } + } + return nil + }) + if err != nil || written != 1 { + t.Fatalf("padded UDP GSO: error = %v, written = %d", err, written) + } + if err := state.forwardVirtioPackets([][]byte{append(p, 0)}, [][]byte{make([]byte, 1514)}, false, func([][]byte) error { + t.Fatal("excess padding emitted a datagram") + return nil + }); err == nil { + t.Fatal("padding past the Ethernet minimum was accepted") + } +} + +func TestVirtioUDPGSOBatchesAndWriteError(t *testing.T) { + packet := udpGSOPacket(false, 2177, 64) + want := bytes.Clone(packet[udpTestIPv4PayloadStart:]) + state := &adapterState{tcp: newTCPSegmentation(1500)} + buffers := make([][]byte, 32) + for i := range buffers { + buffers[i] = bytes.Repeat([]byte{0xa5}, 1526) + } + var batches []int + var joined []byte + count := 0 + err := state.normalizePeerPackets([][]byte{packet}, buffers, func(frames [][]byte) error { + batches = append(batches, len(frames)) + for _, frame := range frames { + if !bytes.Equal(frame[:12], make([]byte, 12)) { + t.Fatal("stale output Virtio metadata") + } + size := 64 + if count == 34 { + size = 1 + } + checkUDPWire(t, frame[12:], packet, false, count, size) + joined = append(joined, frame[udpTestIPv4PayloadStart:]...) + count++ + } + return nil + }) + if err != nil || !slices.Equal(batches, []int{32, 3}) || !bytes.Equal(joined, want) { + t.Fatalf("UDP batches: error = %v, sizes = %v, payload equal = %v", err, batches, bytes.Equal(joined, want)) + } + wantErr := errors.New("peer closed") + calls := 0 + err = state.normalizePeerPackets([][]byte{packet}, buffers, func([][]byte) error { + calls++ + return wantErr + }) + if !errors.Is(err, wantErr) || calls != 1 { + t.Fatalf("UDP write error = %v, calls = %d", err, calls) + } +} diff --git a/vmnetsock/virtio.go b/vmnetsock/virtio.go index 85399865..5a43311c 100644 --- a/vmnetsock/virtio.go +++ b/vmnetsock/virtio.go @@ -16,10 +16,15 @@ const ( virtioRSCInfo byte = 4 virtioGSOTCPv4 byte = 1 virtioGSOTCPv6 byte = 4 + virtioGSOUDPL4 byte = 5 virtioGSOECN byte = 0x80 ) const ( + virtioGSOSizeOffset = 4 + virtioChecksumStartOffset = 6 + virtioChecksumFieldOffset = 8 + virtioRSCSegmentsOffset = 6 virtioRSCDuplicateACKsOffset = 8 ) @@ -44,8 +49,8 @@ func virtioEthernetPacket(packet []byte) ([]byte, error) { if flags&virtioRSCInfo != 0 { return nil, fmt.Errorf("cannot complete virtio checksum with RSC metadata") } - start := int(binary.LittleEndian.Uint16(packet[6:8])) - offset := int(binary.LittleEndian.Uint16(packet[8:10])) + start := int(binary.LittleEndian.Uint16(packet[virtioChecksumStartOffset:])) + offset := int(binary.LittleEndian.Uint16(packet[virtioChecksumFieldOffset:])) if start+offset+2 > len(frame) { return nil, fmt.Errorf("virtio checksum field lies outside the Ethernet frame") } @@ -73,6 +78,34 @@ func virtioEthernetPacket(packet []byte) ([]byte, error) { return frame, nil } +func forwardVirtioUDPGSO(packet []byte, mtu int, emit func([]byte) error) error { + if len(packet) < virtioHeaderSize+ethernetHeaderSize { + return fmt.Errorf("virtio packet is shorter than its header and Ethernet frame") + } + if packet[1] != virtioGSOUDPL4 || packet[0]&virtioRSCInfo != 0 { + return fmt.Errorf("unsupported virtio UDP GSO metadata") + } + if packet[0]&virtioNeedsChecksum == 0 { + return fmt.Errorf("virtio UDP GSO requires partial checksum metadata") + } + p, err := parseUDPPacket(packet[virtioHeaderSize:]) + if err != nil { + return err + } + start := int(binary.LittleEndian.Uint16(packet[virtioChecksumStartOffset:])) + offset := int(binary.LittleEndian.Uint16(packet[virtioChecksumFieldOffset:])) + if start != ethernetHeaderSize+p.ipHeader || offset != udpChecksumOffset { + return fmt.Errorf("virtio UDP GSO checksum metadata does not match the UDP header") + } + size := int(binary.LittleEndian.Uint16(packet[virtioGSOSizeOffset:])) + // UDP L4 GSO describes datagram boundaries. Reducing size to fit the MTU + // would change those boundaries, unlike a TCP stream's segmentation. + if size == 0 || size+p.ipHeader+udpHeaderSize > mtu { + return fmt.Errorf("virtio UDP GSO segment size does not fit the MTU") + } + return p.emitSegments(size, emit) +} + func (s *tcpSegmentation) forwardVirtio(packet []byte, now time.Time, emit func([]byte) error) error { if len(packet) < virtioHeaderSize+ethernetHeaderSize { return fmt.Errorf("virtio packet is shorter than its header and Ethernet frame") @@ -99,7 +132,7 @@ func (s *tcpSegmentation) forwardVirtio(packet []byte, now time.Time, emit func( default: return fmt.Errorf("unsupported virtio GSO type %#x", packet[1]) } - size := int(binary.LittleEndian.Uint16(packet[4:6])) + size := int(binary.LittleEndian.Uint16(packet[virtioGSOSizeOffset:])) if expectedVersion != 0 && size == 0 { return fmt.Errorf("virtio TCP GSO segment size is zero") } @@ -147,8 +180,8 @@ func (s *tcpSegmentation) forwardVirtio(packet []byte, now time.Time, emit func( return fmt.Errorf("virtio GSO type does not match the IP version") } if !rsc { - start := int(binary.LittleEndian.Uint16(packet[6:8])) - offset := int(binary.LittleEndian.Uint16(packet[8:10])) + start := int(binary.LittleEndian.Uint16(packet[virtioChecksumStartOffset:])) + offset := int(binary.LittleEndian.Uint16(packet[virtioChecksumFieldOffset:])) if start != ethernetHeaderSize+p.ipHeader || offset != tcpChecksumOffset { return fmt.Errorf("virtio TCP GSO checksum metadata does not match the TCP header") }