Skip to content

Support multi-frame VXLAN header encapsulation and decapsulation #787

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sonikajindal merged 1 commit into from
Nov 7, 2026
Merged

Support multi-frame VXLAN header encapsulation and decapsulation #787

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions dataplane/forwarding/protocol/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,14 @@ func (d *Desc) EnvelopeDesc() *Desc {
return d.prev
}

// EnvelopeField returns the value of id in the encapsulating header (d.prev).
func (d *Desc) EnvelopeField(id fwdpacket.FieldID) ([]byte, error) {
if d.prev == nil {
return nil, fmt.Errorf("protocol: EnvelopeField %v failed, no envelope", id)
}
return d.prev.handler.Field(id)
}

// Dirty returns true if the header is dirty.
func (d *Desc) Dirty() bool {
return d.dirty
Expand Down
3 changes: 2 additions & 1 deletion dataplane/forwarding/protocol/ip/ip.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ func (ip *IP) Rebuild() error {

// add adds an IP header to an empty L3.
// Note that the L3 cannot contain a stand-alone GRE header.
func add(id fwdpb.PacketHeaderId, _ *protocol.Desc) (protocol.Handler, error) {
func add(id fwdpb.PacketHeaderId, desc *protocol.Desc) (protocol.Handler, error) {
var h header
switch id {
case fwdpb.PacketHeaderId_PACKET_HEADER_ID_IP4:
Expand All @@ -259,6 +259,7 @@ func add(id fwdpb.PacketHeaderId, _ *protocol.Desc) (protocol.Handler, error) {
}
return &IP{
headers: []header{h},
desc: desc,
}, nil
}

Expand Down
58 changes: 48 additions & 10 deletions dataplane/forwarding/protocol/packet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,13 +386,44 @@ func (p *Packet) FirstDesc() *Desc {
return nil
}

// outermost returns the descriptor in the outermost frame belonging to group,
// or nil if the outermost frame does not contain that group.
func (p *Packet) outermost(group fwdpb.PacketHeaderGroup) *Desc {
targetPos := GroupAttr[group].Position
for d := p.FirstDesc(); d != nil; d = d.next {
if d.group == group {
return d
}
// Because headers within a single frame always have strictly increasing
// Position (0 -> 1 -> 3 -> 4 -> 5), as soon as d hits a header whose
// Position > targetPos (for example, we are looking for L3 (Position 3),
// and we encounter UDP (Position 4) or VXLAN (Position 5)), we know the
// outermost frame does not have that header yet.
if GroupAttr[d.group].Position > targetPos {
return nil
}
}
return nil
}

// innermost returns the innermost descriptor in the packet belonging to group.
func (p *Packet) innermost(group fwdpb.PacketHeaderGroup) *Desc {
var found *Desc
for d := p.FirstDesc(); d != nil; d = d.next {
if d.group == group {
found = d
}
}
return found
}

// Decap removes the header specified by id. The header is marked as dirty.
func (p *Packet) Decap(id fwdpb.PacketHeaderId) error {
attr, ok := HeaderAttr[id]
if !ok {
return fmt.Errorf("Decap %v failed for packet %v, unknown header", id, p)
}
header := p.headers[attr.Group]
header := p.outermost(attr.Group)
if header == nil {
return fmt.Errorf("Decap %v failed for packet %v, non existing header %v", id, p, attr.Group)
}
Expand All @@ -409,7 +440,9 @@ func (p *Packet) Decap(id fwdpb.PacketHeaderId) error {
if header.prev != nil {
header.prev.next = header.next
}
p.headers[attr.Group] = nil
if p.headers[attr.Group] == header {
p.headers[attr.Group] = p.innermost(attr.Group)
}
}
p.rebuildHeaders(header.prev)
return nil
Expand All @@ -423,9 +456,8 @@ func (p *Packet) Encap(id fwdpb.PacketHeaderId) error {
if !ok {
return fmt.Errorf("Encap header %v failed for packet %v, unknown header", id, p)
}

// Add the encap to an existing Header in the group.
if header := p.headers[attr.Group]; header != nil {
// Handles VxLAN case when PACKET_HEADER_GROUP_PAYLOAD is being encapped
if header := p.outermost(attr.Group); header != nil && attr.Group != fwdpb.PacketHeaderGroup_PACKET_HEADER_GROUP_PAYLOAD {
if err := header.handler.Modify(id); err != nil {
return fmt.Errorf("Encap header %v failed for packet %v, err %v", id, p, err)
}
Expand All @@ -438,15 +470,19 @@ func (p *Packet) Encap(id fwdpb.PacketHeaderId) error {
if attr.Add == nil {
return fmt.Errorf("Encap header %v failed for packet %v, cannot add header", id, p)
}

// Find the position of the new Header in the frame and add it.
group := GroupAttr[attr.Group]
if group.Position <= 0 {
return fmt.Errorf("Encap %v failed for packet %v, cannot add header at position %v", id, p, group.Position)
}
var prev *Desc
for pos := group.Position - 1; pos >= 0 && prev == nil; pos-- {
prev = p.headers[Sequence[pos]]
if attr.Group == fwdpb.PacketHeaderGroup_PACKET_HEADER_GROUP_PAYLOAD {
// Encapsulating a payload-group tunnel header (e.g. VXLAN) starts a new
// outer frame in front of the current frame.
prev = p.FirstDesc()
} else {
for pos := group.Position - 1; pos >= 0 && prev == nil; pos-- {
prev = p.outermost(Sequence[pos])
}
}
if prev == nil {
return fmt.Errorf("Encap %v failed for packet %v, malformed packet", id, p)
Expand All @@ -466,7 +502,9 @@ func (p *Packet) Encap(id fwdpb.PacketHeaderId) error {
prev.next.prev = header
}
prev.next = header
p.headers[attr.Group] = header
if p.headers[attr.Group] == nil {
p.headers[attr.Group] = header
}
header.MarkDirty(true)
p.rebuildHeaders(header)
return nil
Expand Down
180 changes: 180 additions & 0 deletions dataplane/forwarding/protocol/packet/tunnel_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1414,3 +1414,183 @@ func TestVXLANPacketParsing(t *testing.T) {
}
}

func TestVXLANPacketEncapDecap(t *testing.T) {
innerEthernet := []byte{
0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, // dst mac
0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xfe, // src mac
0x08, 0x00, // ethertype ipv4
}
innerIP := []byte{
0x45, 0x01, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x40, 0xff, 0xf5, 0x6b,
0xc0, 0xa8, 0x01, 0x64, // src ip: 192.168.1.100
0xc0, 0xa8, 0x01, 0xc8, // dst ip: 192.168.1.200
0x00, 0x00,
}

origFrame := append(append([]byte(nil), innerEthernet...), innerIP...)
p, err := protocol.NewPacket(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET, frame.NewFrame(append([]byte(nil), origFrame...)))
if err != nil {
t.Fatalf("NewPacket failed: %v", err)
}

// Perform VXLAN encap sequence matching SAI NextHop + TunnelEncap pipeline:
// 1. Encap VXLAN, UDP, outer IP4
for _, hid := range []fwdpb.PacketHeaderId{
fwdpb.PacketHeaderId_PACKET_HEADER_ID_VXLAN,
fwdpb.PacketHeaderId_PACKET_HEADER_ID_UDP,
fwdpb.PacketHeaderId_PACKET_HEADER_ID_IP4,
} {
if err := p.Encap(hid); err != nil {
t.Fatalf("Encap(%v) failed: %v", hid, err)
}
}

// 2. Set VNI, inner ETHER_MAC_SRC (from inner ETHER_MAC_DST), inner ETHER_MAC_DST (TunnelMac)
if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_VXLAN_VNI, 0), fwdpacket.OpSet, []byte{0x00, 0x4e, 0x21}); err != nil {
t.Fatalf("Update(VXLAN_VNI) failed: %v", err)
}
innerDstMAC, err := p.Field(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_DST, 0))
if err != nil {
t.Fatalf("Field(ETHER_MAC_DST) failed: %v", err)
}
if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_SRC, 0), fwdpacket.OpSet, innerDstMAC); err != nil {
t.Fatalf("Update(inner ETHER_MAC_SRC) failed: %v", err)
}
if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_DST, 0), fwdpacket.OpSet, []byte{0x00, 0xaa, 0xbb, 0xcc, 0xdd, 0xee}); err != nil {
t.Fatalf("Update(inner ETHER_MAC_DST) failed: %v", err)
}

// 3. Simulate outputTable: Decap(ETHERNET) when no outer L2 header exists
// must fail without removing Inner Ethernet, and Encap(ETHERNET) then adds
// the outer Ethernet header.
if err := p.Decap(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET); err == nil {
t.Fatalf("Decap(ETHERNET) without outer L2 header succeeded unexpectedly")
}
if err := p.Encap(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET); err != nil {
t.Fatalf("Encap(ETHERNET) failed: %v", err)
}

// 4. Populate outer IP, UDP, and Ethernet fields
if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_SRC, 0), fwdpacket.OpSet, []byte{10, 0, 1, 1}); err != nil {
t.Fatalf("Update(outer IP_ADDR_SRC) failed: %v", err)
}
if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, 0), fwdpacket.OpSet, []byte{10, 0, 1, 2}); err != nil {
t.Fatalf("Update(outer IP_ADDR_DST) failed: %v", err)
}
innerTTL, err := p.Field(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_HOP, 1))
if err != nil {
t.Fatalf("Field(inner IP_HOP) failed: %v", err)
}
if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_HOP, 0), fwdpacket.OpSet, innerTTL); err != nil {
t.Fatalf("Update(outer IP_HOP) failed: %v", err)
}
if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_L4_PORT_DST, 0), fwdpacket.OpSet, []byte{0x12, 0xb5}); err != nil {
t.Fatalf("Update(L4_PORT_DST) failed: %v", err)
}
if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_SRC, 0), fwdpacket.OpSet, []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x66}); err != nil {
t.Fatalf("Update(outer ETHER_MAC_SRC) failed: %v", err)
}
if err := p.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_DST, 0), fwdpacket.OpSet, []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}); err != nil {
t.Fatalf("Update(outer ETHER_MAC_DST) failed: %v", err)
}

// Query UDP source port before Frame() to verify on-demand entropy computation.
sportBeforeRebuild, err := p.Field(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_L4_PORT_SRC, 0))
if err != nil {
t.Fatalf("Field(L4_PORT_SRC) before Frame() failed: %v", err)
}
sportVal := int(sportBeforeRebuild[0])<<8 | int(sportBeforeRebuild[1])
if sportVal < 49152 || sportVal > 65535 {
t.Errorf("entropy UDP source port before Frame() = %d, want in [49152, 65535]", sportVal)
}

// Rebuild frame and reparse to verify wire format.
encapBytes := p.Frame()
reparsed, err := protocol.NewPacket(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET, frame.NewFrame(append([]byte(nil), encapBytes...)))
if err != nil {
t.Fatalf("NewPacket(encapBytes) failed: %v", err)
}

wantFields := []struct {
name string
id fwdpacket.FieldID
want []byte
}{
{"Outer MAC Dst", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_DST, 0), []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55}},
{"Outer MAC Src", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_SRC, 0), []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x66}},
{"Outer EtherType", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_TYPE, 0), []byte{0x08, 0x00}},
{"Outer IP Src", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_SRC, 0), []byte{10, 0, 1, 1}},
{"Outer IP Dst", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, 0), []byte{10, 0, 1, 2}},
{"Outer IP TTL", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_HOP, 0), []byte{0x40}},
{"Outer IP Proto", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_PROTO, 0), []byte{17}},
{"Outer UDP Sport", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_L4_PORT_SRC, 0), sportBeforeRebuild},
{"Outer UDP Dport", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_L4_PORT_DST, 0), []byte{0x12, 0xb5}},
{"VXLAN VNI", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_VXLAN_VNI, 0), []byte{0x00, 0x4e, 0x21}},
{"Inner MAC Dst", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_DST, 1), []byte{0x00, 0xaa, 0xbb, 0xcc, 0xdd, 0xee}},
{"Inner MAC Src", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_ETHER_MAC_SRC, 1), []byte{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}},
{"Inner IP Src", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_SRC, 1), []byte{192, 168, 1, 100}},
{"Inner IP Dst", fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, 1), []byte{192, 168, 1, 200}},
}
for _, tc := range wantFields {
got, err := reparsed.Field(tc.id)
if err != nil {
t.Fatalf("%s: Field(%v) failed: %v", tc.name, tc.id, err)
}
if diff := cmp.Diff(got, tc.want); diff != "" {
t.Errorf("%s: diff (-got +want):\n%s", tc.name, diff)
}
}

// Now test VXLAN Decap on reparsed packet (matching SAI termination + L2 rewrite).
for _, hid := range []fwdpb.PacketHeaderId{
fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET,
fwdpb.PacketHeaderId_PACKET_HEADER_ID_IP4,
fwdpb.PacketHeaderId_PACKET_HEADER_ID_UDP,
fwdpb.PacketHeaderId_PACKET_HEADER_ID_VXLAN,
} {
if err := reparsed.Decap(hid); err != nil {
t.Fatalf("Decap(%v) failed: %v", hid, err)
}
}

wantDecap := append([]byte(nil),
0x00, 0xaa, 0xbb, 0xcc, 0xdd, 0xee,
0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
0x08, 0x00,
)
wantDecap = append(wantDecap, innerIP...)
if diff := cmp.Diff(reparsed.Frame(), wantDecap); diff != "" {
t.Errorf("Decap frame diff (-got +want):\n%s", diff)
}

// Also verify IPv6 VXLAN outer encapsulation computes UDP checksum over outer IPv6 addresses.
p6, err := protocol.NewPacket(fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET, frame.NewFrame(append([]byte(nil), origFrame...)))
if err != nil {
t.Fatalf("NewPacket(p6) failed: %v", err)
}
for _, hid := range []fwdpb.PacketHeaderId{
fwdpb.PacketHeaderId_PACKET_HEADER_ID_VXLAN,
fwdpb.PacketHeaderId_PACKET_HEADER_ID_UDP,
fwdpb.PacketHeaderId_PACKET_HEADER_ID_IP6,
fwdpb.PacketHeaderId_PACKET_HEADER_ID_ETHERNET,
} {
if err := p6.Encap(hid); err != nil {
t.Fatalf("p6.Encap(%v) failed: %v", hid, err)
}
}
if err := p6.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_SRC, 0), fwdpacket.OpSet, []byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}); err != nil {
t.Fatalf("p6.Update(IP_ADDR_SRC) failed: %v", err)
}
if err := p6.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, 0), fwdpacket.OpSet, []byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}); err != nil {
t.Fatalf("p6.Update(IP_ADDR_DST) failed: %v", err)
}
if err := p6.Update(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_L4_PORT_DST, 0), fwdpacket.OpSet, []byte{0x12, 0xb5}); err != nil {
t.Fatalf("p6.Update(L4_PORT_DST) failed: %v", err)
}
f6 := p6.Frame()
// Outer Ethernet (14B) + Outer IPv6 (40B) + UDP (8B: checksum at offset 54+6=60..62)
if len(f6) < 62 || (f6[60] == 0 && f6[61] == 0) {
t.Errorf("IPv6 VXLAN outer UDP checksum was not computed (got %x)", f6[60:62])
}
}

4 changes: 2 additions & 2 deletions dataplane/forwarding/protocol/udp/udp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,11 +179,11 @@ func (udp *UDP) Rebuild() error {
var sum csum16.Sum
_, _ = sum.Write(udp.Header())
_, _ = sum.Write(udp.desc.Payload())
if f, err = udp.desc.Packet.Field(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_SRC, fwdpacket.LastField)); err != nil {
if f, err = udp.desc.EnvelopeField(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_SRC, fwdpacket.LastField)); err != nil {
return fmt.Errorf("udp: Rebuild failed: %v", err)
}
_, _ = sum.Write(f)
if f, err = udp.desc.Packet.Field(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, fwdpacket.LastField)); err != nil {
if f, err = udp.desc.EnvelopeField(fwdpacket.NewFieldIDFromNum(fwdpb.PacketFieldNum_PACKET_FIELD_NUM_IP_ADDR_DST, fwdpacket.LastField)); err != nil {
return fmt.Errorf("udp: Rebuild failed: %v", err)
}
_, _ = sum.Write(f)
Expand Down
Loading