From ccc71f9d6bbd43077012f6f08d09e70171ba556c Mon Sep 17 00:00:00 2001 From: Hardik Harti Date: Thu, 22 Oct 2026 23:02:30 +0000 Subject: [PATCH] Support multi-frame VXLAN header encapsulation and decapsulation --- dataplane/forwarding/protocol/handler.go | 8 + dataplane/forwarding/protocol/ip/ip.go | 3 +- dataplane/forwarding/protocol/packet.go | 58 +++++- .../forwarding/protocol/packet/tunnel_test.go | 180 ++++++++++++++++++ dataplane/forwarding/protocol/udp/udp.go | 4 +- 5 files changed, 240 insertions(+), 13 deletions(-) diff --git a/dataplane/forwarding/protocol/handler.go b/dataplane/forwarding/protocol/handler.go index 6b7e8070e..f453d6317 100644 --- a/dataplane/forwarding/protocol/handler.go +++ b/dataplane/forwarding/protocol/handler.go @@ -167,6 +167,14 @@ func (d *Desc) EnvelopeDesc() *Desc { return d.prev } +// EnvelopeField returns the value of id in the encapsulating header (d.prev). +func (d *Desc) EnvelopeField(id fwdpacket.FieldID) ([]byte, error) { + if d.prev == nil { + return nil, fmt.Errorf("protocol: EnvelopeField %v failed, no envelope", id) + } + return d.prev.handler.Field(id) +} + // Dirty returns true if the header is dirty. func (d *Desc) Dirty() bool { return d.dirty diff --git a/dataplane/forwarding/protocol/ip/ip.go b/dataplane/forwarding/protocol/ip/ip.go index 4bf2ef692..b31722bd3 100644 --- a/dataplane/forwarding/protocol/ip/ip.go +++ b/dataplane/forwarding/protocol/ip/ip.go @@ -245,7 +245,7 @@ func (ip *IP) Rebuild() error { // add adds an IP header to an empty L3. // Note that the L3 cannot contain a stand-alone GRE header. -func add(id fwdpb.PacketHeaderId, _ *protocol.Desc) (protocol.Handler, error) { +func add(id fwdpb.PacketHeaderId, desc *protocol.Desc) (protocol.Handler, error) { var h header switch id { case fwdpb.PacketHeaderId_PACKET_HEADER_ID_IP4: @@ -259,6 +259,7 @@ func add(id fwdpb.PacketHeaderId, _ *protocol.Desc) (protocol.Handler, error) { } return &IP{ headers: []header{h}, + desc: desc, }, nil } diff --git a/dataplane/forwarding/protocol/packet.go b/dataplane/forwarding/protocol/packet.go index d33eef800..5ea0f7e21 100644 --- a/dataplane/forwarding/protocol/packet.go +++ b/dataplane/forwarding/protocol/packet.go @@ -386,13 +386,44 @@ func (p *Packet) FirstDesc() *Desc { return nil } +// outermost returns the descriptor in the outermost frame belonging to group, +// or nil if the outermost frame does not contain that group. +func (p *Packet) outermost(group fwdpb.PacketHeaderGroup) *Desc { + targetPos := GroupAttr[group].Position + for d := p.FirstDesc(); d != nil; d = d.next { + if d.group == group { + return d + } + // Because headers within a single frame always have strictly increasing + // Position (0 -> 1 -> 3 -> 4 -> 5), as soon as d hits a header whose + // Position > targetPos (for example, we are looking for L3 (Position 3), + // and we encounter UDP (Position 4) or VXLAN (Position 5)), we know the + // outermost frame does not have that header yet. + if GroupAttr[d.group].Position > targetPos { + return nil + } + } + return nil +} + +// innermost returns the innermost descriptor in the packet belonging to group. +func (p *Packet) innermost(group fwdpb.PacketHeaderGroup) *Desc { + var found *Desc + for d := p.FirstDesc(); d != nil; d = d.next { + if d.group == group { + found = d + } + } + return found +} + // Decap removes the header specified by id. The header is marked as dirty. func (p *Packet) Decap(id fwdpb.PacketHeaderId) error { attr, ok := HeaderAttr[id] if !ok { return fmt.Errorf("Decap %v failed for packet %v, unknown header", id, p) } - header := p.headers[attr.Group] + header := p.outermost(attr.Group) if header == nil { return fmt.Errorf("Decap %v failed for packet %v, non existing header %v", id, p, attr.Group) } @@ -409,7 +440,9 @@ func (p *Packet) Decap(id fwdpb.PacketHeaderId) error { if header.prev != nil { header.prev.next = header.next } - p.headers[attr.Group] = nil + if p.headers[attr.Group] == header { + p.headers[attr.Group] = p.innermost(attr.Group) + } } p.rebuildHeaders(header.prev) return nil @@ -423,9 +456,8 @@ func (p *Packet) Encap(id fwdpb.PacketHeaderId) error { if !ok { return fmt.Errorf("Encap header %v failed for packet %v, unknown header", id, p) } - - // Add the encap to an existing Header in the group. - if header := p.headers[attr.Group]; header != nil { + // Handles VxLAN case when PACKET_HEADER_GROUP_PAYLOAD is being encapped + if header := p.outermost(attr.Group); header != nil && attr.Group != fwdpb.PacketHeaderGroup_PACKET_HEADER_GROUP_PAYLOAD { if err := header.handler.Modify(id); err != nil { return fmt.Errorf("Encap header %v failed for packet %v, err %v", id, p, err) } @@ -438,15 +470,19 @@ func (p *Packet) Encap(id fwdpb.PacketHeaderId) error { if attr.Add == nil { return fmt.Errorf("Encap header %v failed for packet %v, cannot add header", id, p) } - - // Find the position of the new Header in the frame and add it. group := GroupAttr[attr.Group] if group.Position <= 0 { return fmt.Errorf("Encap %v failed for packet %v, cannot add header at position %v", id, p, group.Position) } var prev *Desc - for pos := group.Position - 1; pos >= 0 && prev == nil; pos-- { - prev = p.headers[Sequence[pos]] + if attr.Group == fwdpb.PacketHeaderGroup_PACKET_HEADER_GROUP_PAYLOAD { + // Encapsulating a payload-group tunnel header (e.g. VXLAN) starts a new + // outer frame in front of the current frame. + prev = p.FirstDesc() + } else { + for pos := group.Position - 1; pos >= 0 && prev == nil; pos-- { + prev = p.outermost(Sequence[pos]) + } } if prev == nil { return fmt.Errorf("Encap %v failed for packet %v, malformed packet", id, p) @@ -466,7 +502,9 @@ func (p *Packet) Encap(id fwdpb.PacketHeaderId) error { prev.next.prev = header } prev.next = header - p.headers[attr.Group] = header + if p.headers[attr.Group] == nil { + p.headers[attr.Group] = header + } header.MarkDirty(true) p.rebuildHeaders(header) return nil diff --git a/dataplane/forwarding/protocol/packet/tunnel_test.go b/dataplane/forwarding/protocol/packet/tunnel_test.go index 772d31c0a..6cd837b0f 100644 --- a/dataplane/forwarding/protocol/packet/tunnel_test.go +++ b/dataplane/forwarding/protocol/packet/tunnel_test.go @@ -1414,3 +1414,183 @@ func TestVXLANPacketParsing(t *testing.T) { } } +func TestVXLANPacketEncapDecap(t *testing.T) { + innerEthernet := []byte{ + 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, // dst mac + 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xfe, // src mac + 0x08, 0x00, // ethertype ipv4 + } + innerIP := []byte{ + 0x45, 0x01, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x40, 0xff, 0xf5, 0x6b, + 0xc0, 0xa8, 0x01, 0x64, // src ip: 192.168.1.100 + 0xc0, 0xa8, 0x01, 0xc8, // dst ip: 192.168.1.200 + 0x00, 0x00, + } + + origFrame := append(append([]byte(nil), innerEthernet...), innerIP...) + p, err := protocol.NewPacket(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET, frame.NewFrame(append([]byte(nil), origFrame...))) + if err != nil { + t.Fatalf("NewPacket failed: %v", err) + } + + // Perform VXLAN encap sequence matching SAI NextHop + TunnelEncap pipeline: + // 1. Encap VXLAN, UDP, outer IP4 + for _, hid := range []fwdpb.PacketHeaderId{ + fwdpb.PacketHeaderId_PACKET_HEADER_ID_VXLAN, + fwdpb.PacketHeaderId_PACKET_HEADER_ID_UDP, + fwdpb.PacketHeaderId_PACKET_HEADER_ID_IP4, + } { + if err := p.Encap(hid); err != nil { + t.Fatalf("Encap(%v) failed: %v", hid, err) + } + } + + // 2. Set VNI, inner ETHER_MAC_SRC (from inner ETHER_MAC_DST), inner ETHER_MAC_DST (TunnelMac) + if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_VXLAN_VNI, 0), fwdpacket.OpSet, []byte{0x00, 0x4e, 0x21}); err != nil { + t.Fatalf("Update(VXLAN_VNI) failed: %v", err) + } + innerDstMAC, err := p.Field(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_DST, 0)) + if err != nil { + t.Fatalf("Field(ETHER_MAC_DST) failed: %v", err) + } + if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_SRC, 0), fwdpacket.OpSet, innerDstMAC); err != nil { + t.Fatalf("Update(inner ETHER_MAC_SRC) failed: %v", err) + } + if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_DST, 0), fwdpacket.OpSet, []byte{0x00, 0xaa, 0xbb, 0xcc, 0xdd, 0xee}); err != nil { + t.Fatalf("Update(inner ETHER_MAC_DST) failed: %v", err) + } + + // 3. Simulate outputTable: Decap(ETHERNET) when no outer L2 header exists + // must fail without removing Inner Ethernet, and Encap(ETHERNET) then adds + // the outer Ethernet header. + if err := p.Decap(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET); err == nil { + t.Fatalf("Decap(ETHERNET) without outer L2 header succeeded unexpectedly") + } + if err := p.Encap(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET); err != nil { + t.Fatalf("Encap(ETHERNET) failed: %v", err) + } + + // 4. Populate outer IP, UDP, and Ethernet fields + if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_SRC, 0), fwdpacket.OpSet, []byte{10, 0, 1, 1}); err != nil { + t.Fatalf("Update(outer IP_ADDR_SRC) failed: %v", err) + } + if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, 0), fwdpacket.OpSet, []byte{10, 0, 1, 2}); err != nil { + t.Fatalf("Update(outer IP_ADDR_DST) failed: %v", err) + } + innerTTL, err := p.Field(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_HOP, 1)) + if err != nil { + t.Fatalf("Field(inner IP_HOP) failed: %v", err) + } + if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_HOP, 0), fwdpacket.OpSet, innerTTL); err != nil { + t.Fatalf("Update(outer IP_HOP) failed: %v", err) + } + if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_L4_PORT_DST, 0), fwdpacket.OpSet, []byte{0x12, 0xb5}); err != nil { + t.Fatalf("Update(L4_PORT_DST) failed: %v", err) + } + if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_SRC, 0), fwdpacket.OpSet, []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x66}); err != nil { + t.Fatalf("Update(outer ETHER_MAC_SRC) failed: %v", err) + } + if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_DST, 0), fwdpacket.OpSet, []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}); err != nil { + t.Fatalf("Update(outer ETHER_MAC_DST) failed: %v", err) + } + + // Query UDP source port before Frame() to verify on-demand entropy computation. + sportBeforeRebuild, err := p.Field(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_L4_PORT_SRC, 0)) + if err != nil { + t.Fatalf("Field(L4_PORT_SRC) before Frame() failed: %v", err) + } + sportVal := int(sportBeforeRebuild[0])<<8 | int(sportBeforeRebuild[1]) + if sportVal < 49152 || sportVal > 65535 { + t.Errorf("entropy UDP source port before Frame() = %d, want in [49152, 65535]", sportVal) + } + + // Rebuild frame and reparse to verify wire format. + encapBytes := p.Frame() + reparsed, err := protocol.NewPacket(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET, frame.NewFrame(append([]byte(nil), encapBytes...))) + if err != nil { + t.Fatalf("NewPacket(encapBytes) failed: %v", err) + } + + wantFields := []struct { + name string + id fwdpacket.FieldID + want []byte + }{ + {"Outer MAC Dst", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_DST, 0), []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}}, + {"Outer MAC Src", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_SRC, 0), []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x66}}, + {"Outer EtherType", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_TYPE, 0), []byte{0x08, 0x00}}, + {"Outer IP Src", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_SRC, 0), []byte{10, 0, 1, 1}}, + {"Outer IP Dst", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, 0), []byte{10, 0, 1, 2}}, + {"Outer IP TTL", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_HOP, 0), []byte{0x40}}, + {"Outer IP Proto", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_PROTO, 0), []byte{17}}, + {"Outer UDP Sport", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_L4_PORT_SRC, 0), sportBeforeRebuild}, + {"Outer UDP Dport", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_L4_PORT_DST, 0), []byte{0x12, 0xb5}}, + {"VXLAN VNI", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_VXLAN_VNI, 0), []byte{0x00, 0x4e, 0x21}}, + {"Inner MAC Dst", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_DST, 1), []byte{0x00, 0xaa, 0xbb, 0xcc, 0xdd, 0xee}}, + {"Inner MAC Src", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_SRC, 1), []byte{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}}, + {"Inner IP Src", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_SRC, 1), []byte{192, 168, 1, 100}}, + {"Inner IP Dst", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, 1), []byte{192, 168, 1, 200}}, + } + for _, tc := range wantFields { + got, err := reparsed.Field(tc.id) + if err != nil { + t.Fatalf("%s: Field(%v) failed: %v", tc.name, tc.id, err) + } + if diff := cmp.Diff(got, tc.want); diff != "" { + t.Errorf("%s: diff (-got +want):\n%s", tc.name, diff) + } + } + + // Now test VXLAN Decap on reparsed packet (matching SAI termination + L2 rewrite). + for _, hid := range []fwdpb.PacketHeaderId{ + fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET, + fwdpb.PacketHeaderId_PACKET_HEADER_ID_IP4, + fwdpb.PacketHeaderId_PACKET_HEADER_ID_UDP, + fwdpb.PacketHeaderId_PACKET_HEADER_ID_VXLAN, + } { + if err := reparsed.Decap(hid); err != nil { + t.Fatalf("Decap(%v) failed: %v", hid, err) + } + } + + wantDecap := append([]byte(nil), + 0x00, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, + 0x08, 0x00, + ) + wantDecap = append(wantDecap, innerIP...) + if diff := cmp.Diff(reparsed.Frame(), wantDecap); diff != "" { + t.Errorf("Decap frame diff (-got +want):\n%s", diff) + } + + // Also verify IPv6 VXLAN outer encapsulation computes UDP checksum over outer IPv6 addresses. + p6, err := protocol.NewPacket(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET, frame.NewFrame(append([]byte(nil), origFrame...))) + if err != nil { + t.Fatalf("NewPacket(p6) failed: %v", err) + } + for _, hid := range []fwdpb.PacketHeaderId{ + fwdpb.PacketHeaderId_PACKET_HEADER_ID_VXLAN, + fwdpb.PacketHeaderId_PACKET_HEADER_ID_UDP, + fwdpb.PacketHeaderId_PACKET_HEADER_ID_IP6, + fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET, + } { + if err := p6.Encap(hid); err != nil { + t.Fatalf("p6.Encap(%v) failed: %v", hid, err) + } + } + if err := p6.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_SRC, 0), fwdpacket.OpSet, []byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}); err != nil { + t.Fatalf("p6.Update(IP_ADDR_SRC) failed: %v", err) + } + if err := p6.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, 0), fwdpacket.OpSet, []byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}); err != nil { + t.Fatalf("p6.Update(IP_ADDR_DST) failed: %v", err) + } + if err := p6.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_L4_PORT_DST, 0), fwdpacket.OpSet, []byte{0x12, 0xb5}); err != nil { + t.Fatalf("p6.Update(L4_PORT_DST) failed: %v", err) + } + f6 := p6.Frame() + // Outer Ethernet (14B) + Outer IPv6 (40B) + UDP (8B: checksum at offset 54+6=60..62) + if len(f6) < 62 || (f6[60] == 0 && f6[61] == 0) { + t.Errorf("IPv6 VXLAN outer UDP checksum was not computed (got %x)", f6[60:62]) + } +} + diff --git a/dataplane/forwarding/protocol/udp/udp.go b/dataplane/forwarding/protocol/udp/udp.go index d763d2bde..f6619345d 100644 --- a/dataplane/forwarding/protocol/udp/udp.go +++ b/dataplane/forwarding/protocol/udp/udp.go @@ -179,11 +179,11 @@ func (udp *UDP) Rebuild() error { var sum csum16.Sum _, _ = sum.Write(udp.Header()) _, _ = sum.Write(udp.desc.Payload()) - if f, err = udp.desc.Packet.Field(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_SRC, fwdpacket.LastField)); err != nil { + if f, err = udp.desc.EnvelopeField(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_SRC, fwdpacket.LastField)); err != nil { return fmt.Errorf("udp: Rebuild failed: %v", err) } _, _ = sum.Write(f) - if f, err = udp.desc.Packet.Field(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, fwdpacket.LastField)); err != nil { + if f, err = udp.desc.EnvelopeField(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, fwdpacket.LastField)); err != nil { return fmt.Errorf("udp: Rebuild failed: %v", err) } _, _ = sum.Write(f)