diff --git a/cmd/urunc/create_libcontainer.go b/cmd/urunc/create_libcontainer.go index 8778387a4..c441ebaa3 100644 --- a/cmd/urunc/create_libcontainer.go +++ b/cmd/urunc/create_libcontainer.go @@ -133,6 +133,10 @@ func buildMonitorArgs(id string, logFormat string) []string { // pivot: /proc is always mounted in the monitor rootfs, so this re-execs the // sealed urunc binary without urunc having to be copied into that rootfs. func monitorProcess(cmd *cli.Command, u *unikontainers.Unikontainer, readyFile *os.File) (*libcontainer.Process, error) { + caps, err := u.MonitorCapabilities() + if err != nil { + return nil, err + } process := &libcontainer.Process{ Args: buildMonitorArgs(u.State.ID, cmd.String("log-format")), Env: os.Environ(), @@ -141,7 +145,7 @@ func monitorProcess(cmd *cli.Command, u *unikontainers.Unikontainer, readyFile * Cwd: "/", // TODO set uid, gid and additional groups here Init: true, - Capabilities: unikontainers.MonitorCapabilities(u.Spec), + Capabilities: caps, LogLevel: strconv.Itoa(int(logrus.GetLevel())), // The ready pipe's write end. It must be the only/first ExtraFile so the // monitor inherits it at unikontainers.ReadyPipeFD: diff --git a/docs/hypervisor-support.md b/docs/hypervisor-support.md index b2439c9c6..db32d08da 100644 --- a/docs/hypervisor-support.md +++ b/docs/hypervisor-support.md @@ -335,14 +335,33 @@ leaves the size to `hluk`, which gives each guest runtime the memory it is tested with (e.g. 256 MiB for Python, 1536 MiB for the agent image). [Hyperlight](https://github.com/hyperlight-dev/hyperlight) does not provide any -VirtIO devices, nor a tap-based network interface. As a result, networking is -not available for unikernels running on top of Hyperlight and, for the time -being, initramfs is the only supported storage option. Neither virtio-block nor -devmapper-backed block storage can be used. - -Upstream `hluk` can preopen host directories for the guest and forward POSIX -file operations to them, as well as forward the sockets of the guest to the -host. We plan to add support for these options. +VirtIO devices, nor a tap-based network interface. Instead, `hluk` forwards the +socket calls of the guest to real sockets that it opens itself. Since `hluk` +runs inside the network namespace of the container, the guest uses that +namespace as it is: the address of the pod, its routes, and any +NetworkPolicy that applies to it. For that reason, `urunc` does not create a +tap device for this monitor, nor does it redirect the traffic of the +container's interface or give the monitor `CAP_NET_ADMIN` and `/dev/net/tun`, +and it starts `hluk` with networking enabled, both +outbound (`--net`) and inbound (`--port=all`). The network namespace remains the +boundary, as for any other container. A container started with no network has +no interface in its namespace, and `urunc` then leaves the guest to the default +of `hluk`, which is no network at all. A deployment can also limit the ports +the guest may serve on, as a second boundary inside the namespace, with the +`com.urunc.unikernel.hyperlightPorts` annotation; without it the guest serves +on any port, as a container does. The `resolv.conf` of the container is +mounted into the monitor and installed in the guest as well (`--resolv-conf`), +both when it boots and when it resumes from a snapshot, so that cluster names +resolve inside the guest. The guest binds ports as the user of the container: +a container that runs as a non-root user needs +`net.ipv4.ip_unprivileged_port_start=0` in its network namespace to serve on a +port below 1024 (Docker sets it; on Kubernetes, set the sysctl in the pod), +since the monitor keeps no capability across the switch to that user. + +For the time being, initramfs is the only supported storage option. Neither +virtio-block nor devmapper-backed block storage can be used. Upstream `hluk` +can also preopen host directories for the guest and forward POSIX file +operations to them. We plan to add support for this option. Rootfs images for every guest runtime that `hluk` supports (Python, Node.js, .NET, Go, Rust, C, Bash and more) are published under diff --git a/docs/package/index.md b/docs/package/index.md index cafba952f..de9e6273b 100644 --- a/docs/package/index.md +++ b/docs/package/index.md @@ -74,6 +74,10 @@ Apart from the above, `urunc` accepts the following optional annotations: - `com.urunc.unikernel.mountRootfs`: A boolean value that if it is `true`, requests from `urunc` to mount the container's image rootfs in the unikernel (either as a block device or through shared-fs). +- `com.urunc.unikernel.hyperlightPorts`: The ports a `hyperlight-unikraft` + guest may serve on, as a comma-separated list of ports and `LOW-HIGH` ranges + (e.g. `8080,9000-9100`). Without it the guest serves on any port, as a + container does. Only supported by `hyperlight-unikraft`. Due to the fact that [Docker](https://www.docker.com/) and some high-level container runtimes do not pass the image annotations to the underlying container diff --git a/pkg/network/network.go b/pkg/network/network.go index cbc73c707..034d82232 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -33,6 +33,10 @@ const ( var netlog = logrus.WithField("subsystem", "network") +// ErrNoContainerIface reports a network namespace with no usable interface, +// as in a container started with no network. +var ErrNoContainerIface = errors.New("no suitable network interface found in namespace") + type UnikernelNetworkInfo struct { TapDevice string EthDevice Interface @@ -403,7 +407,18 @@ func discoverContainerIface() (netlink.Link, error) { } netlog.Debugf("skipping interface %s: no default route found", attrs.Name) } - return nil, errors.New("no suitable network interface found in namespace") + return nil, ErrNoContainerIface +} + +// ContainerIface describes the interface of the container in the current +// network namespace, without touching it, for a monitor that gets no tap: +// ErrNoContainerIface when there is none. +func ContainerIface() (Interface, error) { + link, err := discoverContainerIface() + if err != nil { + return Interface{}, err + } + return getInterfaceInfo(link.Attrs().Name) } func deleteAllQDiscs(device netlink.Link) error { diff --git a/pkg/unikontainers/annotations.go b/pkg/unikontainers/annotations.go index 4037ecebd..8eb367b57 100644 --- a/pkg/unikontainers/annotations.go +++ b/pkg/unikontainers/annotations.go @@ -40,20 +40,21 @@ import ( // Urunc specific annotations // Always keep it in sync with the struct UnikernelConfig struct const ( - annotUruncPrefix = "com.urunc.unikernel." - annotType = "com.urunc.unikernel.unikernelType" - annotVersion = "com.urunc.unikernel.unikernelVersion" - annotBinary = "com.urunc.unikernel.binary" - annotHypervisor = "com.urunc.unikernel.hypervisor" - annotInitrd = "com.urunc.unikernel.initrd" - annotSnapshot = "com.urunc.unikernel.snapshot" - annotBlock = "com.urunc.unikernel.block" - annotBlockMntPoint = "com.urunc.unikernel.blkMntPoint" - annotMountRootfs = "com.urunc.unikernel.mountRootfs" - annotNetDev = "com.urunc.unikernel.solo5NetDev" - annotBlkDev = "com.urunc.unikernel.solo5BlkDev" - annotVAccel = "com.urunc.unikernel.vAccel" - annotRPCAddress = "com.urunc.unikernel.RPCAddress" + annotUruncPrefix = "com.urunc.unikernel." + annotType = "com.urunc.unikernel.unikernelType" + annotVersion = "com.urunc.unikernel.unikernelVersion" + annotBinary = "com.urunc.unikernel.binary" + annotHypervisor = "com.urunc.unikernel.hypervisor" + annotInitrd = "com.urunc.unikernel.initrd" + annotSnapshot = "com.urunc.unikernel.snapshot" + annotBlock = "com.urunc.unikernel.block" + annotBlockMntPoint = "com.urunc.unikernel.blkMntPoint" + annotMountRootfs = "com.urunc.unikernel.mountRootfs" + annotNetDev = "com.urunc.unikernel.solo5NetDev" + annotBlkDev = "com.urunc.unikernel.solo5BlkDev" + annotVAccel = "com.urunc.unikernel.vAccel" + annotRPCAddress = "com.urunc.unikernel.RPCAddress" + annotHyperlightPorts = "com.urunc.unikernel.hyperlightPorts" ) // Annotations that other runtimes set and urunc only reads. @@ -105,6 +106,7 @@ type UnikernelConfig struct { MountRootfs string `json:"com.urunc.unikernel.mountRootfs"` NetDev string `json:"com.urunc.unikernel.solo5NetDev,omitempty"` BlkDev string `json:"com.urunc.unikernel.solo5BlkDev,omitempty"` + HyperlightPorts string `json:"com.urunc.unikernel.hyperlightPorts,omitempty"` // The vAccel annotations are deliberately not part of urunc.json, since their // values are runtime specific and therefore we should only reach them // through the annotations of the spec. @@ -116,6 +118,10 @@ type UnikernelConfig struct { // identifier for Solo5 guests when received from the annotations. var solo5DevNameRe = regexp.MustCompile(`^[A-Za-z0-9]{1,67}$`) +// hyperlightPortsRe constrains the ports a hyperlight-unikraft guest may serve +// on to a comma-separated list of ports and LOW-HIGH ranges. +var hyperlightPortsRe = regexp.MustCompile(`^[0-9]{1,5}(-[0-9]{1,5})?(,[0-9]{1,5}(-[0-9]{1,5})?)*$`) + var ErrNotUnikernel = errors.New("this is not a unikernel container") // validate checks if the mandatory configuration fields are present. It @@ -181,6 +187,13 @@ func GetUnikernelConfig(bundleDir string, spec *specs.Spec) (*UnikernelConfig, e // configured through urunc.json can still use vAccel jsonConf.VAccel = spec.Annotations[annotVAccel] jsonConf.RPCAddress = spec.Annotations[annotRPCAddress] + // The ports a hyperlight-unikraft guest may serve on are a choice of the + // deployment first, and of the image second: an annotation of the spec + // replaces whatever urunc.json says, and validateValues() checks it as it + // does for an image configured through the spec. + if ports := spec.Annotations[annotHyperlightPorts]; ports != "" { + jsonConf.HyperlightPorts = ports + } return jsonConf, nil } @@ -198,6 +211,7 @@ func getConfigFromSpec(spec *specs.Spec) *UnikernelConfig { MountRootfs := spec.Annotations[annotMountRootfs] netDev := spec.Annotations[annotNetDev] blkDev := spec.Annotations[annotBlkDev] + hyperlightPorts := spec.Annotations[annotHyperlightPorts] vAccel := spec.Annotations[annotVAccel] rpcAddress := spec.Annotations[annotRPCAddress] uniklog.WithFields(logrus.Fields{ @@ -212,6 +226,7 @@ func getConfigFromSpec(spec *specs.Spec) *UnikernelConfig { "mountRootfs": MountRootfs, "netDev": netDev, "blkDev": blkDev, + "hyperlightPorts": hyperlightPorts, "vAccel": vAccel, "rpcAddress": rpcAddress, }).WithField("source", "spec").Debug("urunc annotations") @@ -228,6 +243,7 @@ func getConfigFromSpec(spec *specs.Spec) *UnikernelConfig { MountRootfs: MountRootfs, NetDev: netDev, BlkDev: blkDev, + HyperlightPorts: hyperlightPorts, VAccel: vAccel, RPCAddress: rpcAddress, } @@ -271,6 +287,7 @@ func getConfigFromJSON(jsonFilePath string) (*UnikernelConfig, error) { "mountRootfs": tryDecode(conf.MountRootfs), "netDev": tryDecode(conf.NetDev), "blkDev": tryDecode(conf.BlkDev), + "hyperlightPorts": tryDecode(conf.HyperlightPorts), }).WithField("source", uruncJSONFilename).Debug("urunc annotations") return &conf, nil @@ -352,6 +369,12 @@ func (c *UnikernelConfig) decode() error { } c.BlkDev = string(decoded) + decoded, err = base64.StdEncoding.DecodeString(c.HyperlightPorts) + if err != nil { + return fmt.Errorf("failed to decode hyperlightPorts: %v", err) + } + c.HyperlightPorts = string(decoded) + return nil } @@ -391,6 +414,9 @@ func (c *UnikernelConfig) Map() map[string]string { if c.BlkDev != "" { myMap[annotBlkDev] = c.BlkDev } + if c.HyperlightPorts != "" { + myMap[annotHyperlightPorts] = c.HyperlightPorts + } if c.VAccel != "" { myMap[annotVAccel] = c.VAccel } @@ -471,6 +497,11 @@ func (c *UnikernelConfig) validateValues() error { return err } + err = validateHyperlightPorts(monitor, c.HyperlightPorts) + if err != nil { + return err + } + if c.VAccel == "" { if c.RPCAddress != "" { return fmt.Errorf("%s is set, but %s is not", annotRPCAddress, annotVAccel) @@ -517,6 +548,44 @@ func validateSolo5DevName(key string, val string) error { return nil } +// validateHyperlightPorts verifies that val lists the ports a +// hyperlight-unikraft guest may serve on: ports and LOW-HIGH ranges within +// 1-65535, separated by commas. +// An empty value is accepted, since the guest then serves on any port, as a +// container does. No other monitor can limit the ports of a guest. +func validateHyperlightPorts(monitor hypervisors.VmmType, val string) error { + if val == "" { + return nil + } + + if monitor != hypervisors.HyperlightVmm { + return fmt.Errorf("%s is only supported by %s", annotHyperlightPorts, hypervisors.HyperlightVmm) + } + + if len(val) > maxAnnotationValueLen { + return fmt.Errorf("%s value is longer than %d bytes", annotHyperlightPorts, maxAnnotationValueLen) + } + + if !hyperlightPortsRe.MatchString(val) { + return fmt.Errorf("%s must match %s, got %q", annotHyperlightPorts, hyperlightPortsRe.String(), val) + } + + for _, item := range strings.Split(val, ",") { + lowStr, highStr, isRange := strings.Cut(item, "-") + if !isRange { + highStr = lowStr + } + // The regexp already limited both to digits. + low, _ := strconv.Atoi(lowStr) + high, _ := strconv.Atoi(highStr) + if low < 1 || high > 65535 || low > high { + return fmt.Errorf("%s must list ports within 1-65535 and ranges from low to high, got %q", annotHyperlightPorts, item) + } + } + + return nil +} + // validateAnnotationPathClean verifies that val looks like a filepath with // simple plain characters and is also clean. func validateAnnotationPathClean(key string, val string, rejectRoot bool) error { diff --git a/pkg/unikontainers/annotations_test.go b/pkg/unikontainers/annotations_test.go index c952453e2..91bff5584 100644 --- a/pkg/unikontainers/annotations_test.go +++ b/pkg/unikontainers/annotations_test.go @@ -17,6 +17,7 @@ package unikontainers import ( "encoding/base64" "encoding/json" + "maps" "os" "path/filepath" "strings" @@ -33,16 +34,17 @@ func TestGetConfigFromSpec(t *testing.T) { t.Parallel() spec := &specs.Spec{ Annotations: map[string]string{ - annotType: "type1", - annotBinary: "binary1", - annotHypervisor: "hypervisor1", - annotInitrd: "initrd1", - annotSnapshot: "snapshot1", - annotBlock: "block1", - annotBlockMntPoint: "point1", - annotMountRootfs: "true", - annotNetDev: "management", - annotBlkDev: "database", + annotType: "type1", + annotBinary: "binary1", + annotHypervisor: "hypervisor1", + annotInitrd: "initrd1", + annotSnapshot: "snapshot1", + annotBlock: "block1", + annotBlockMntPoint: "point1", + annotMountRootfs: "true", + annotNetDev: "management", + annotBlkDev: "database", + annotHyperlightPorts: "8080", }, } @@ -57,6 +59,7 @@ func TestGetConfigFromSpec(t *testing.T) { MountRootfs: "true", NetDev: "management", BlkDev: "database", + HyperlightPorts: "8080", } config := getConfigFromSpec(spec) @@ -192,6 +195,7 @@ func TestDecode(t *testing.T) { encodedBinary := base64.StdEncoding.EncodeToString([]byte("testBinary")) encodedInitrd := base64.StdEncoding.EncodeToString([]byte("testInitrd")) encodedSnapshot := base64.StdEncoding.EncodeToString([]byte("testSnapshot")) + encodedPorts := base64.StdEncoding.EncodeToString([]byte("8080,9000-9100")) config := &UnikernelConfig{ Hypervisor: encodedHypervisor, @@ -199,6 +203,7 @@ func TestDecode(t *testing.T) { UnikernelBinary: encodedBinary, Initrd: encodedInitrd, Snapshot: encodedSnapshot, + HyperlightPorts: encodedPorts, } // Call the decode method @@ -211,6 +216,7 @@ func TestDecode(t *testing.T) { assert.Equal(t, "testBinary", config.UnikernelBinary) assert.Equal(t, "testInitrd", config.Initrd) assert.Equal(t, "testSnapshot", config.Snapshot) + assert.Equal(t, "8080,9000-9100", config.HyperlightPorts) }) t.Run("decode invalid base64", func(t *testing.T) { @@ -246,22 +252,24 @@ func TestMap(t *testing.T) { MountRootfs: "false", NetDev: "netdev_value", BlkDev: "blkdev_value", + HyperlightPorts: "443", VAccel: "vsock", RPCAddress: "vsock://2:1234", } expectedMap := map[string]string{ - annotType: "type_value", - annotHypervisor: "hypervisor_value", - annotBinary: "binary_value", - annotInitrd: "initrd_value", - annotSnapshot: "snapshot_value", - annotBlock: "block_value", - annotBlockMntPoint: "point_value", - annotMountRootfs: "false", - annotNetDev: "netdev_value", - annotBlkDev: "blkdev_value", - annotVAccel: "vsock", - annotRPCAddress: "vsock://2:1234", + annotType: "type_value", + annotHypervisor: "hypervisor_value", + annotBinary: "binary_value", + annotInitrd: "initrd_value", + annotSnapshot: "snapshot_value", + annotBlock: "block_value", + annotBlockMntPoint: "point_value", + annotMountRootfs: "false", + annotNetDev: "netdev_value", + annotBlkDev: "blkdev_value", + annotHyperlightPorts: "443", + annotVAccel: "vsock", + annotRPCAddress: "vsock://2:1234", } resultMap := config.Map() assert.Equal(t, expectedMap, resultMap) @@ -653,6 +661,75 @@ func TestGetUnikernelConfigVAccelFromJSONFallback(t *testing.T) { assert.Equal(t, "vsock://2:1234", conf.RPCAddress, "Expected the rpc address to be read from the spec") } +// GetUnikernelConfig must read the ports annotation from the spec when the +// rest of the config comes from urunc.json, as a docker image's does: the +// ports are the deployment's choice, so a runtime supplied --annotation +// replaces what the image says, and the image's value stands without one. +func TestGetUnikernelConfigHyperlightPortsFromJSONFallback(t *testing.T) { + t.Parallel() + + writeJSONBundle := func(t *testing.T, ports string) string { + t.Helper() + jsonConf := map[string]string{ + annotType: base64.StdEncoding.EncodeToString([]byte(unikernels.UnikraftUnikernel)), + annotHypervisor: base64.StdEncoding.EncodeToString([]byte(hypervisors.HyperlightVmm)), + annotInitrd: base64.StdEncoding.EncodeToString([]byte("/unikernel/initrd.cpio")), + } + if ports != "" { + jsonConf[annotHyperlightPorts] = base64.StdEncoding.EncodeToString([]byte(ports)) + } + configData, err := json.Marshal(jsonConf) + assert.NoError(t, err) + + bundle := t.TempDir() + rootfsDir := filepath.Join(bundle, rootfsDirName) + err = os.Mkdir(rootfsDir, 0755) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(rootfsDir, uruncJSONFilename), configData, 0600) + assert.NoError(t, err) + return bundle + } + + t.Run("the spec annotation replaces the ports of urunc.json", func(t *testing.T) { + t.Parallel() + bundle := writeJSONBundle(t, "8080") + spec := &specs.Spec{ + Root: &specs.Root{Path: rootfsDirName}, + Annotations: map[string]string{annotHyperlightPorts: "9000-9100"}, + } + + conf, err := GetUnikernelConfig(bundle, spec) + assert.NoError(t, err) + assert.Equal(t, "9000-9100", conf.HyperlightPorts, "Expected the ports to be read from the spec") + }) + + t.Run("the ports of urunc.json stand without a spec annotation", func(t *testing.T) { + t.Parallel() + bundle := writeJSONBundle(t, "8080") + spec := &specs.Spec{ + Root: &specs.Root{Path: rootfsDirName}, + Annotations: map[string]string{}, + } + + conf, err := GetUnikernelConfig(bundle, spec) + assert.NoError(t, err) + assert.Equal(t, "8080", conf.HyperlightPorts, "Expected the ports to be read from urunc.json") + }) + + t.Run("no ports anywhere leaves the guest to any port", func(t *testing.T) { + t.Parallel() + bundle := writeJSONBundle(t, "") + spec := &specs.Spec{ + Root: &specs.Root{Path: rootfsDirName}, + Annotations: map[string]string{}, + } + + conf, err := GetUnikernelConfig(bundle, spec) + assert.NoError(t, err) + assert.Empty(t, conf.HyperlightPorts, "Expected no ports restriction") + }) +} + // writeBundle creates a bundle directory with a config.json holding annots. func writeBundle(t *testing.T, annots map[string]string) string { t.Helper() @@ -931,3 +1008,55 @@ func TestValidateValuesSnapshot(t *testing.T) { }) } } + +// The ports annotation lists the ports a hyperlight-unikraft guest may serve +// on, and no other monitor can enforce it. +func TestValidateValuesHyperlightPorts(t *testing.T) { + t.Parallel() + + hyperlight := map[string]string{ + annotType: unikernels.UnikraftUnikernel, + annotHypervisor: string(hypervisors.HyperlightVmm), + annotInitrd: "/unikernel/initrd.cpio", + } + qemu := map[string]string{ + annotType: unikernels.UnikraftUnikernel, + annotHypervisor: string(hypervisors.QemuVmm), + annotBinary: "/unikernel/app", + } + testCases := []struct { + name string + base map[string]string + ports string + wantErr bool + }{ + {name: "a port", base: hyperlight, ports: "8080"}, + {name: "ports and ranges", base: hyperlight, ports: "80,443,8000-8100"}, + {name: "the whole range", base: hyperlight, ports: "1-65535"}, + {name: "unset on qemu", base: qemu, ports: ""}, + {name: "port zero is rejected", base: hyperlight, ports: "0", wantErr: true}, + {name: "a port above 65535 is rejected", base: hyperlight, ports: "65536", wantErr: true}, + {name: "a range from high to low is rejected", base: hyperlight, ports: "8100-8000", wantErr: true}, + {name: "a trailing comma is rejected", base: hyperlight, ports: "80,", wantErr: true}, + {name: "all is not a value", base: hyperlight, ports: "all", wantErr: true}, + {name: "none is not a value", base: hyperlight, ports: "none", wantErr: true}, + {name: "a word is rejected", base: hyperlight, ports: "http", wantErr: true}, + {name: "a value on qemu is rejected", base: qemu, ports: "8080", wantErr: true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + annots := maps.Clone(tc.base) + if tc.ports != "" { + annots[annotHyperlightPorts] = tc.ports + } + err := validateAnnots(annots) + if tc.wantErr { + assert.ErrorContains(t, err, annotHyperlightPorts, "Expected error to mention the annotation") + return + } + assert.NoError(t, err) + }) + } +} diff --git a/pkg/unikontainers/hypervisors/cloud_hypervisor.go b/pkg/unikontainers/hypervisors/cloud_hypervisor.go index 936355423..b6a972f8b 100644 --- a/pkg/unikontainers/hypervisors/cloud_hypervisor.go +++ b/pkg/unikontainers/hypervisors/cloud_hypervisor.go @@ -49,6 +49,11 @@ func (ch *CloudHypervisor) UsesKVM() bool { return true } +// UsesTUNTAP reports that the guest gets its network through a tap device. +func (ch *CloudHypervisor) UsesTUNTAP() bool { + return true +} + // SupportsControlSocket reports that Cloud Hypervisor exposes a control socket // (its REST API socket). func (ch *CloudHypervisor) SupportsControlSocket() bool { diff --git a/pkg/unikontainers/hypervisors/firecracker.go b/pkg/unikontainers/hypervisors/firecracker.go index 5add98256..5ab588652 100644 --- a/pkg/unikontainers/hypervisors/firecracker.go +++ b/pkg/unikontainers/hypervisors/firecracker.go @@ -94,6 +94,11 @@ func (fc *Firecracker) UsesKVM() bool { return true } +// UsesTUNTAP reports that the guest gets its network through a tap device. +func (fc *Firecracker) UsesTUNTAP() bool { + return true +} + // SupportsSharedfs returns a bool value depending on the monitor support for shared-fs func (fc *Firecracker) SupportsSharedfs(_ string) bool { return false diff --git a/pkg/unikontainers/hypervisors/hedge.go b/pkg/unikontainers/hypervisors/hedge.go index ed2c6fe6c..a148a0701 100644 --- a/pkg/unikontainers/hypervisors/hedge.go +++ b/pkg/unikontainers/hypervisors/hedge.go @@ -46,6 +46,11 @@ func (h *Hedge) UsesKVM() bool { return true } +// UsesTUNTAP reports that the guest gets its network through a tap device. +func (h *Hedge) UsesTUNTAP() bool { + return true +} + // SupportsSharedfs returns a bool value depending on the monitor support for shared-fs func (h *Hedge) SupportsSharedfs(_ string) bool { return false diff --git a/pkg/unikontainers/hypervisors/hvt.go b/pkg/unikontainers/hypervisors/hvt.go index 4e761605e..3bbba247f 100644 --- a/pkg/unikontainers/hypervisors/hvt.go +++ b/pkg/unikontainers/hypervisors/hvt.go @@ -135,6 +135,11 @@ func (h *HVT) UsesKVM() bool { return true } +// UsesTUNTAP reports that the guest gets its network through a tap device. +func (h *HVT) UsesTUNTAP() bool { + return true +} + // SupportsSharedfs returns a bool value depending on the monitor support for shared-fs func (h *HVT) SupportsSharedfs(_ string) bool { return false diff --git a/pkg/unikontainers/hypervisors/hyperlight.go b/pkg/unikontainers/hypervisors/hyperlight.go index a593d559d..4adf0e138 100644 --- a/pkg/unikontainers/hypervisors/hyperlight.go +++ b/pkg/unikontainers/hypervisors/hyperlight.go @@ -17,6 +17,7 @@ package hypervisors import ( "errors" "strconv" + "strings" "github.com/urunc-dev/urunc/pkg/unikontainers/types" "golang.org/x/sys/unix" @@ -54,6 +55,51 @@ func (h *Hyperlight) SupportsControlSocket() bool { return false } +// UsesTUNTAP reports that hluk needs no tap: it serves the guest's sockets +// itself, each guest socket call becoming a call on a real socket hluk opens, +// so the guest shares the pod's network namespace, its address, routes and +// NetworkPolicy, with no tap in between. +func (h *Hyperlight) UsesTUNTAP() bool { + return false +} + +// netArgs opens the network of the container's namespace to the guest. +// hluk's default is no network at all, and --port alone only lets the guest +// serve: it can then connect nowhere, not even to a DNS server. A container +// expects the whole network of its namespace, outbound and inbound, so the +// guest gets both, with no hluk policy on top: the namespace and whatever +// NetworkPolicy applies to it stay the boundary, as they are for any other +// container. A container started with no network has no interface in its +// namespace, and the guest is then left to hluk's default, as a second +// boundary. A deployment can ask for another one, inside the namespace, by +// listing the ports the guest may serve on. The guest also gets the +// container's resolver configuration: the rootfs, or a snapshot of a guest +// booted elsewhere, carries its own, which knows nothing of the cluster's DNS. +func netArgs(args types.ExecArgs) []string { + if args.Net.IP == "" { + return nil + } + cmdArgs := append([]string{"--net"}, portArgs(args.Ports)...) + if args.ResolvConf { + cmdArgs = append(cmdArgs, "--resolv-conf="+types.ResolvConfPath) + } + return cmdArgs +} + +// portArgs limits the ports the guest may serve on, as the ports annotation +// asks: any port without it, and otherwise each port or LOW-HIGH range +// listed, which urunc has already validated, as a --port of its own. +func portArgs(ports string) []string { + if ports == "" { + return []string{"--port=all"} + } + var cmdArgs []string + for _, port := range strings.Split(ports, ",") { + cmdArgs = append(cmdArgs, "--port="+port) + } + return cmdArgs +} + // Path returns the path to the hluk binary. func (h *Hyperlight) Path() string { return h.binaryPath @@ -82,6 +128,7 @@ func (h *Hyperlight) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel) // no kernel, initrd or scratch memory with it, only what to run in it. if args.SnapshotPath != "" { cmdArgs := []string{h.binaryPath, "snapshot", "run", args.SnapshotPath} + cmdArgs = append(cmdArgs, netArgs(args)...) cmdArgs = append(cmdArgs, extraMonArgs.OtherArgs...) return cmdArgs, nil } @@ -102,6 +149,7 @@ func (h *Hyperlight) BuildExecCmd(args types.ExecArgs, ukernel types.Unikernel) if memMiB := bytesToMiB(args.MemSizeB); memMiB > 0 { cmdArgs = append(cmdArgs, "--scratch-mb", strconv.FormatUint(memMiB, 10)) } + cmdArgs = append(cmdArgs, netArgs(args)...) cmdArgs = append(cmdArgs, extraMonArgs.OtherArgs...) return cmdArgs, nil diff --git a/pkg/unikontainers/hypervisors/hyperlight_test.go b/pkg/unikontainers/hypervisors/hyperlight_test.go index 88a69d0aa..591bdf964 100644 --- a/pkg/unikontainers/hypervisors/hyperlight_test.go +++ b/pkg/unikontainers/hypervisors/hyperlight_test.go @@ -99,6 +99,64 @@ func TestHyperlightBuildExecCmd(t *testing.T) { unikernel: &fakeUnikernel{}, expected: []string{"/usr/local/bin/hluk", "snapshot", "run", "/unikernel/snapshot"}, }, + { + name: "the network of the container is opened to a booted guest", + args: types.ExecArgs{ + InitrdPath: "/unikernel/initrd.cpio", + Net: types.NetDevParams{IP: "10.244.1.7"}, + }, + unikernel: &fakeUnikernel{monitorCli: types.MonitorCliArgs{OtherArgs: []string{"--guest-exec=/entrypoint.py"}}}, + expected: []string{"/usr/local/bin/hluk", "run", "--initrd", "/unikernel/initrd.cpio", "--net", "--port=all", "--guest-exec=/entrypoint.py"}, + }, + { + name: "the network of the container is opened to a resumed guest", + args: types.ExecArgs{ + SnapshotPath: "/unikernel/snapshot", + Net: types.NetDevParams{IP: "10.244.1.7"}, + }, + unikernel: &fakeUnikernel{}, + expected: []string{"/usr/local/bin/hluk", "snapshot", "run", "/unikernel/snapshot", "--net", "--port=all"}, + }, + { + name: "the container's resolv.conf is installed in a networked guest", + args: types.ExecArgs{ + SnapshotPath: "/unikernel/snapshot", + Net: types.NetDevParams{IP: "10.244.1.7"}, + ResolvConf: true, + }, + unikernel: &fakeUnikernel{}, + expected: []string{"/usr/local/bin/hluk", "snapshot", "run", "/unikernel/snapshot", "--net", "--port=all", "--resolv-conf=/etc/resolv.conf"}, + }, + { + name: "the ports annotation limits what a booted guest serves on", + args: types.ExecArgs{ + InitrdPath: "/unikernel/initrd.cpio", + Net: types.NetDevParams{IP: "10.244.1.7"}, + Ports: "8080,9000-9100", + }, + unikernel: &fakeUnikernel{}, + expected: []string{"/usr/local/bin/hluk", "run", "--initrd", "/unikernel/initrd.cpio", "--net", "--port=8080", "--port=9000-9100"}, + }, + { + name: "the ports annotation limits what a resumed guest serves on", + args: types.ExecArgs{ + SnapshotPath: "/unikernel/snapshot", + Net: types.NetDevParams{IP: "10.244.1.7"}, + Ports: "8080", + ResolvConf: true, + }, + unikernel: &fakeUnikernel{}, + expected: []string{"/usr/local/bin/hluk", "snapshot", "run", "/unikernel/snapshot", "--net", "--port=8080", "--resolv-conf=/etc/resolv.conf"}, + }, + { + name: "a container without a network leaves the guest to hluk's default of none", + args: types.ExecArgs{ + InitrdPath: "/unikernel/initrd.cpio", + ResolvConf: true, + }, + unikernel: &fakeUnikernel{}, + expected: []string{"/usr/local/bin/hluk", "run", "--initrd", "/unikernel/initrd.cpio"}, + }, { name: "no initrd is an error", args: types.ExecArgs{ @@ -124,3 +182,11 @@ func TestHyperlightBuildExecCmd(t *testing.T) { }) } } + +func TestHyperlightUsesTUNTAP(t *testing.T) { + t.Parallel() + assert.False(t, (&Hyperlight{}).UsesTUNTAP(), "hyperlight-unikraft opens the guest's sockets itself, with no tap") + for _, tap := range []types.VMM{&Qemu{}, &Firecracker{}, &CloudHypervisor{}, &HVT{}, &SPT{}, &Hedge{}} { + assert.True(t, tap.UsesTUNTAP(), "%T gets a tap", tap) + } +} diff --git a/pkg/unikontainers/hypervisors/qemu.go b/pkg/unikontainers/hypervisors/qemu.go index 8c051a5da..5fc07e165 100644 --- a/pkg/unikontainers/hypervisors/qemu.go +++ b/pkg/unikontainers/hypervisors/qemu.go @@ -51,6 +51,11 @@ func (q *Qemu) UsesKVM() bool { return true } +// UsesTUNTAP reports that the guest gets its network through a tap device. +func (q *Qemu) UsesTUNTAP() bool { + return true +} + // SupportsSharedfs returns a bool value depending on the monitor support for shared-fs func (q *Qemu) SupportsSharedfs(_ string) bool { return true diff --git a/pkg/unikontainers/hypervisors/spt.go b/pkg/unikontainers/hypervisors/spt.go index a7cfa84a1..80b622e78 100644 --- a/pkg/unikontainers/hypervisors/spt.go +++ b/pkg/unikontainers/hypervisors/spt.go @@ -45,6 +45,11 @@ func (s *SPT) UsesKVM() bool { return false } +// UsesTUNTAP reports that the guest gets its network through a tap device. +func (s *SPT) UsesTUNTAP() bool { + return true +} + // SupportsSharedfs returns a bool value depending on the monitor support for shared-fs func (s *SPT) SupportsSharedfs(_ string) bool { return false diff --git a/pkg/unikontainers/libcontainer.go b/pkg/unikontainers/libcontainer.go index 93d0cc8a0..649923b8a 100644 --- a/pkg/unikontainers/libcontainer.go +++ b/pkg/unikontainers/libcontainer.go @@ -15,6 +15,7 @@ package unikontainers import ( + "fmt" "slices" "github.com/opencontainers/runc/libcontainer/configs" @@ -49,16 +50,30 @@ func monitorOCISpec(spec *specs.Spec, monRes monitorResources, rootfsParams type return &monSpec } -// MonitorCapabilities returns the capability set the monitor process requires +// MonitorCapabilities returns the capability set the monitor process of the +// container requires; see monitorCapabilities. +func (u *Unikontainer) MonitorCapabilities() (*configs.Capabilities, error) { + monRes, err := loadMonitorResources(u.BaseDir) + if err != nil { + return nil, fmt.Errorf("failed to load monitor resources: %w", err) + } + return monitorCapabilities(u.Spec, monRes.TUNTAP), nil +} + +// monitorCapabilities returns the capability set the monitor process requires // to finalize the execution environment for the monitor. Currently, it consists -// of the container's own set plus CAP_NET_ADMIN in all five sets. +// of the container's own set plus CAP_NET_ADMIN in all five sets for a monitor +// that creates a tap. A monitor without one gets the container's own set. // TODO: Narrow down to the extremely necessary capabilities. -func MonitorCapabilities(spec *specs.Spec) *configs.Capabilities { +func monitorCapabilities(spec *specs.Spec, tuntap bool) *configs.Capabilities { caps := specCapabilities(spec) if caps == nil { uniklog.Warn("The container declares no capabilities. The monitor inherits urunc's own set.") return nil } + if !tuntap { + return caps + } for _, capab := range monCaps { caps.Bounding = withCap(caps.Bounding, capab) diff --git a/pkg/unikontainers/libcontainer_test.go b/pkg/unikontainers/libcontainer_test.go index 7f4676994..40abd2ebc 100644 --- a/pkg/unikontainers/libcontainer_test.go +++ b/pkg/unikontainers/libcontainer_test.go @@ -263,7 +263,7 @@ func TestMonitorCapabilities(t *testing.T) { t.Parallel() spec := testSpec() - caps := MonitorCapabilities(spec) + caps := monitorCapabilities(spec, true) require.NotNil(t, caps) for name, set := range map[string][]string{ @@ -286,7 +286,7 @@ func TestMonitorCapabilities(t *testing.T) { spec := testSpec() spec.Process.Capabilities.Bounding = []string{monCaps[0]} - caps := MonitorCapabilities(spec) + caps := monitorCapabilities(spec, true) require.NotNil(t, caps) assert.Equal(t, []string{monCaps[0]}, caps.Bounding) @@ -297,6 +297,18 @@ func TestMonitorCapabilities(t *testing.T) { spec := testSpec() spec.Process.Capabilities = nil - assert.Nil(t, MonitorCapabilities(spec)) + assert.Nil(t, monitorCapabilities(spec, true)) + }) + + t.Run("a monitor without a tap keeps the container's own set", func(t *testing.T) { + t.Parallel() + spec := testSpec() + + caps := monitorCapabilities(spec, false) + + require.NotNil(t, caps) + assert.NotContains(t, caps.Bounding, monCaps[0]) + assert.NotContains(t, caps.Ambient, monCaps[0]) + assert.Contains(t, caps.Bounding, "CAP_CHOWN") }) } diff --git a/pkg/unikontainers/monitor.go b/pkg/unikontainers/monitor.go index 84aa64119..0f138f0b4 100644 --- a/pkg/unikontainers/monitor.go +++ b/pkg/unikontainers/monitor.go @@ -83,7 +83,7 @@ func runMonitor(metrics m.Writer, ms monitorSpec) error { return err } - netArgs, err := SetupNet(ms.NetworkType, ms.User.UID, ms.User.GID) + netArgs, err := SetupNet(vmm.UsesTUNTAP(), ms.NetworkType, ms.User.UID, ms.User.GID) if err != nil { return fmt.Errorf("failed to setup network: %w", err) } diff --git a/pkg/unikontainers/rootfs.go b/pkg/unikontainers/rootfs.go index f7d732262..1021a7bfa 100644 --- a/pkg/unikontainers/rootfs.go +++ b/pkg/unikontainers/rootfs.go @@ -415,8 +415,9 @@ func changeRoot(rootfsDir string, pivot bool) error { } // getMonitorDevices returns the devices which are needed for the execution of -// the monitor process -func getMonitorDevices(needsKVM bool) ([]specs.LinuxDevice, error) { +// the monitor process. needsTUN is false for a monitor that never creates a +// tap device. +func getMonitorDevices(needsKVM bool, needsTUN bool) ([]specs.LinuxDevice, error) { nullDev, err := deviceFromHost("/dev/null") if err != nil { return nil, fmt.Errorf("could not get host device /dev/null: %w", err) @@ -426,21 +427,25 @@ func getMonitorDevices(needsKVM bool) ([]specs.LinuxDevice, error) { return nil, fmt.Errorf("could not get host device /dev/urandom: %w", err) } devices := []specs.LinuxDevice{nullDev, randomDev} - // The tun device is always included because in urunc create we can not know - // if there will be a virtual ethernet device or not, since the CNI hooks - // have not executed yet. Therefore, the decision about whether it is actually - // created in the monitor rootfs is decided in Exec, which checks the - // network configuration. However, if the host does not have the "/dev/net/tun" - // device then simply skip it and fail later if the container requires network - tunDev, err := deviceFromHost("/dev/net/tun") - if err != nil { - if errors.Is(err, unix.ENOENT) { - uniklog.Warnf("could not find /dev/net/tun in host, skipping it") + // A monitor that opens the guest's sockets itself never creates a tap, so + // it gets no tun device. For the others it is always included, because in + // urunc create we can not know if there will be a virtual ethernet device + // or not, since the CNI hooks have not executed yet. Therefore, the + // decision about whether it is actually created in the monitor rootfs is + // decided in Exec, which checks the network configuration. However, if the + // host does not have the "/dev/net/tun" device then simply skip it and fail + // later if the container requires network + if needsTUN { + tunDev, err := deviceFromHost("/dev/net/tun") + if err != nil { + if errors.Is(err, unix.ENOENT) { + uniklog.Warnf("could not find /dev/net/tun in host, skipping it") + } else { + return nil, fmt.Errorf("could not get host device /dev/net/tun: %w", err) + } } else { - return nil, fmt.Errorf("could not get host device /dev/net/tun: %w", err) + devices = append(devices, tunDev) } - } else { - devices = append(devices, tunDev) } if needsKVM { diff --git a/pkg/unikontainers/rootfs_test.go b/pkg/unikontainers/rootfs_test.go index 811c01399..e0f8fc91a 100644 --- a/pkg/unikontainers/rootfs_test.go +++ b/pkg/unikontainers/rootfs_test.go @@ -18,6 +18,7 @@ import ( "os" "testing" + "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" @@ -216,3 +217,28 @@ func TestNoRootfsGetMounts(t *testing.T) { assert.NotContains(t, mounts[0].Options, "ro") }) } + +// A monitor that creates no tap gets no /dev/net/tun in its rootfs. +func TestGetMonitorDevicesTUN(t *testing.T) { + t.Parallel() + + paths := func(devs []specs.LinuxDevice) []string { + var p []string + for _, d := range devs { + p = append(p, d.Path) + } + return p + } + + devs, err := getMonitorDevices(false, false) + require.NoError(t, err) + assert.NotContains(t, paths(devs), "/dev/net/tun") + assert.Contains(t, paths(devs), "/dev/null") + + if _, err := os.Stat("/dev/net/tun"); err != nil { + t.Skip("no /dev/net/tun on this host") + } + devs, err = getMonitorDevices(false, true) + require.NoError(t, err) + assert.Contains(t, paths(devs), "/dev/net/tun") +} diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index 83933375e..be18c76f1 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -17,6 +17,10 @@ package types import "golang.org/x/sys/unix" +// ResolvConfPath is where the resolv.conf of the container is mounted in the +// monitor rootfs, for a monitor that installs it in the guest. +const ResolvConfPath = "/etc/resolv.conf" + type Unikernel interface { Init(UnikernelParams) error CommandString() (string, error) @@ -41,11 +45,21 @@ type VMM interface { Signal(int, unix.Signal) error Path() string UsesKVM() bool + // UsesTUNTAP reports whether the guest gets its network through a tap + // device, which urunc creates in the container's network namespace and + // redirects the traffic of the container's interface to. A monitor that + // opens the guest's sockets itself, in that namespace, reports false, and + // urunc leaves the interface untouched: redirecting its traffic to a tap + // would take it away from those sockets. + UsesTUNTAP() bool SupportsSharedfs(string) bool SupportsControlSocket() bool Ok() error } +// NetDevParams describes the network of the container: its interface, and +// the tap device that the traffic of the interface is redirected to when the +// monitor uses one. A container started with no network leaves it empty. type NetDevParams struct { IP string // The veth device IP Mask string // The veth device mask @@ -111,6 +125,8 @@ type ExecArgs struct { Command string // The unikernel's command line Seccomp bool // Enable or disable seccomp filters for the VMM MemSizeB uint64 // The size of the memory provided to the VM in bytes + ResolvConf bool // The container's resolv.conf is mounted at ResolvConfPath in the monitor rootfs, for a monitor that installs it in the guest + Ports string // The ports the guest may serve on, as the annotation of the monitor lists them; "" for any VCPUs uint // The number of vCPUs to allocate UnikernelPath string // The path of the unikernel inside rootfs InitrdPath string // The path to the initrd of the unikernel diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 9456cf49e..27efaf6bd 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -25,6 +25,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "strings" "syscall" @@ -220,7 +221,7 @@ func (u *Unikontainer) InitialSetup() error { return fmt.Errorf("pre setup step for rootfs failed: %w", err) } - monRes, err := getMonitorResources(rfsBuilder, rootfsParams, vmm, u.UruncCfg.Monitors[vmmType].DataPath) + monRes, err := getMonitorResources(rfsBuilder, rootfsParams, vmm, u.UruncCfg.Monitors[vmmType].DataPath, u.Spec.Mounts) if err != nil { return err } @@ -280,11 +281,54 @@ func (u *Unikontainer) SetRunningState() error { return u.saveContainerState() } -// SetupNet creates the sandbox's network device (tap) in the current network -// namespace and returns its parameters; uid and gid own the tap device. -func SetupNet(networkType string, uid, gid uint32) (types.NetDevParams, error) { - uniklog.WithField("network type", networkType).Debug("Retrieved network type") +// monitorResolvConfMount returns the container's own resolv.conf mount, to +// apply in the monitor rootfs as well, for a monitor that serves the guest's +// sockets itself: it installs the whole file in the guest, search domains +// included, so that cluster names resolve as they do in a container, and the +// guest follows the engine's file as a container does. The monitor only +// reads it, so it gets it read-only. The other monitors get none, and neither +// does a container without a resolv.conf. +func monitorResolvConfMount(vmm types.VMM, cntrMounts []specs.Mount) (specs.Mount, bool) { + if vmm.UsesTUNTAP() { + return specs.Mount{}, false + } + m, ok := resolvConfMount(cntrMounts) + if !ok { + return specs.Mount{}, false + } + m.Destination = types.ResolvConfPath + if !slices.Contains(m.Options, "ro") { + m.Options = append(slices.Clone(m.Options), "ro") + } + return m, true +} + +// SetupNet gives the guest its network. A monitor on a tap gets one created in +// the current network namespace, owned by uid and gid, with the traffic of the +// container's interface redirected to it. Any other monitor opens the guest's +// sockets in that namespace itself, so the interface is left as it is and only +// described, for the monitor to know whether the container has a network at +// all: one started with none gets empty parameters. +func SetupNet(tuntap bool, networkType string, uid, gid uint32) (types.NetDevParams, error) { netArgs := types.NetDevParams{} + if !tuntap { + iface, err := network.ContainerIface() + if errors.Is(err, network.ErrNoContainerIface) { + uniklog.Debug("no interface in the network namespace of the container, the guest gets no network") + return netArgs, nil + } + if err != nil { + return netArgs, fmt.Errorf("failed to describe the container's interface: %w", err) + } + netArgs.IP = iface.IP + netArgs.Mask = iface.Mask + netArgs.Gateway = iface.DefaultGateway + netArgs.MAC = iface.MAC + netArgs.MTU = iface.MTU + return netArgs, nil + } + + uniklog.WithField("network type", networkType).Debug("Retrieved network type") netManager, err := network.NewNetworkManager(networkType) if err != nil { return netArgs, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err) @@ -386,7 +430,7 @@ func ChooseRootfs(bundle, specRoot string, annot map[string]string, cfg *UruncCo // getMonitorResources collects every mount and device required for the monitor's // execution environment, plus the block parameters passed to the guest. -func getMonitorResources(rfs rootfsBuilder, rootfsParams types.RootfsParams, vmm types.VMM, monitorDataPath string) (monitorResources, error) { +func getMonitorResources(rfs rootfsBuilder, rootfsParams types.RootfsParams, vmm types.VMM, monitorDataPath string, cntrMounts []specs.Mount) (monitorResources, error) { var res monitorResources var err error @@ -399,8 +443,14 @@ func getMonitorResources(rfs rootfsBuilder, rootfsParams types.RootfsParams, vmm return res, fmt.Errorf("failed to get mounts for monitor rootfs: %w", err) } res.Mounts = append(res.Mounts, rootfsMounts...) + if m, ok := monitorResolvConfMount(vmm, cntrMounts); ok { + res.Mounts = append(res.Mounts, m) + } - res.Devices, err = getMonitorDevices(vmm.UsesKVM()) + // Whether the monitor gets a tap is decided here, once, for its devices + // and for its capabilities. + res.TUNTAP = vmm.UsesTUNTAP() + res.Devices, err = getMonitorDevices(vmm.UsesKVM(), res.TUNTAP) if err != nil { return res, fmt.Errorf("failed to get host devices for monitor: %w", err) } @@ -536,6 +586,7 @@ func (u *Unikontainer) buildMonitorSpec(rootfsParams types.RootfsParams, monRes VCPUs: uint(defaultVCPUs), SocketPath: socketPath, Environment: os.Environ(), + Ports: u.State.Annotations[annotHyperlightPorts], } // Check if container is set to unconfined -- disable seccomp @@ -569,10 +620,14 @@ func (u *Unikontainer) buildMonitorSpec(rootfsParams types.RootfsParams, monRes mSpec.ExecArgs = vmmArgs mSpec.GuestParams = guest mSpec.PreStartCmd = monRes.PreStartCmd - // Resolve the guest DNS server once, here in the builder shared by both the - // libcontainer and non-libcontainer paths, where the container mount - // sources are available. - mSpec.DNSServer = getDNSServer(u.Spec.Mounts) + // A monitor without a tap hands the container's resolv.conf, mounted in + // its rootfs, to the guest. The others get the nameserver out of it, + // resolved once here in the builder shared by both the libcontainer and + // non-libcontainer paths, where the container mount sources are available. + _, mSpec.ExecArgs.ResolvConf = resolvConfMount(monRes.Mounts) + if !mSpec.ExecArgs.ResolvConf { + mSpec.DNSServer = getDNSServer(u.Spec.Mounts) + } return mSpec } @@ -652,13 +707,13 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { } // handle network - netArgs, err := SetupNet(u.getNetworkType(), u.Spec.Process.User.UID, u.Spec.Process.User.GID) + netArgs, err := SetupNet(vmm.UsesTUNTAP(), u.getNetworkType(), u.Spec.Process.User.UID, u.Spec.Process.User.GID) if err != nil { uniklog.Errorf("failed to setup network: %v", err) return err } metrics.Capture(m.TS16) - withTUNTAP := netArgs.IP != "" + withTUNTAP := netArgs.TapDev != "" // SetupNet does not resolve DNS; carry the server resolved at spec build. netArgs.DNSServer = ms.DNSServer unikernelParams.Net = netArgs diff --git a/pkg/unikontainers/unikontainers_test.go b/pkg/unikontainers/unikontainers_test.go index daace4eed..c09b12d14 100644 --- a/pkg/unikontainers/unikontainers_test.go +++ b/pkg/unikontainers/unikontainers_test.go @@ -176,3 +176,28 @@ func TestExecuteHooksSequential(t *testing.T) { require.NoError(t, err) assert.Equal(t, "12", string(out)) } + +// noTapVMM is a monitor that opens the guest's sockets itself, with no tap. +type noTapVMM struct{ types.VMM } + +func (noTapVMM) UsesTUNTAP() bool { return false } + +// Only a monitor that serves the guest's sockets itself gets the container's +// own resolv.conf mount in its rootfs, read-only, and only when there is one. +func TestMonitorResolvConfMount(t *testing.T) { + t.Parallel() + mounts := []specs.Mount{{Type: "bind", Destination: "/etc/resolv.conf/", Source: "/run/engine/resolv.conf", Options: []string{"rbind", "rprivate"}}} + + m, ok := monitorResolvConfMount(noTapVMM{}, mounts) + require.True(t, ok) + assert.Equal(t, "bind", m.Type) + assert.Equal(t, "/run/engine/resolv.conf", m.Source) + assert.Equal(t, types.ResolvConfPath, m.Destination) + assert.Equal(t, []string{"rbind", "rprivate", "ro"}, m.Options) + assert.Equal(t, []string{"rbind", "rprivate"}, mounts[0].Options, "the container's mount is left as it is") + + _, ok = monitorResolvConfMount(noTapVMM{}, nil) + assert.False(t, ok, "a container without a resolv.conf mounts none") + _, ok = monitorResolvConfMount(&hypervisors.Qemu{}, mounts) + assert.False(t, ok, "a tap-based monitor mounts none") +} diff --git a/pkg/unikontainers/utils.go b/pkg/unikontainers/utils.go index 5fa170c6b..2c035166a 100644 --- a/pkg/unikontainers/utils.go +++ b/pkg/unikontainers/utils.go @@ -46,12 +46,13 @@ const ( ) // monitorResources holds the chosen guest rootfs params, the mounts and devices -// that must be replicated inside the monitor's rootfs, and the block parameters -// the guest needs. +// that must be replicated inside the monitor's rootfs, whether the monitor gets +// a tap device, and the block parameters the guest needs. type monitorResources struct { Rootfs types.RootfsParams `json:"rootfs"` Mounts []specs.Mount `json:"mounts"` Devices []specs.LinuxDevice `json:"devices"` + TUNTAP bool `json:"tuntap"` BlockArgs []types.BlockDevParams `json:"blockArgs"` Sharedfs types.SharedfsParams `json:"sharedfs"` PreStartCmd []string `json:"preStartCmd,omitempty"` @@ -377,17 +378,24 @@ func executeHook(hook specs.Hook, state []byte) error { return nil } -func getDNSServer(mounts []specs.Mount) string { - resolvConf := "" - for _, mount := range mounts { - if filepath.Clean(mount.Destination) == "/etc/resolv.conf" { - resolvConf = mount.Source - break +// resolvConfMount returns the mount of the container's /etc/resolv.conf, +// or false when there is none. Mounts apply in order, so the last one is the +// file the container sees. +func resolvConfMount(mounts []specs.Mount) (specs.Mount, bool) { + for i := len(mounts) - 1; i >= 0; i-- { + if filepath.Clean(mounts[i].Destination) == "/etc/resolv.conf" { + return mounts[i], true } } - if resolvConf == "" { + return specs.Mount{}, false +} + +func getDNSServer(mounts []specs.Mount) string { + m, ok := resolvConfMount(mounts) + if !ok { return "" } + resolvConf := m.Source data, err := os.ReadFile(resolvConf) if err != nil { diff --git a/pkg/unikontainers/utils_test.go b/pkg/unikontainers/utils_test.go index 3e8d57c0a..13beb18e7 100644 --- a/pkg/unikontainers/utils_test.go +++ b/pkg/unikontainers/utils_test.go @@ -417,3 +417,19 @@ func TestGetDNSServer(t *testing.T) { assert.Equal(t, "", getDNSServer(mounts)) }) } + +// OCI mounts apply in order, so the last /etc/resolv.conf mount is the file +// the container sees. +func TestResolvConfMount(t *testing.T) { + t.Parallel() + mounts := []specs.Mount{ + {Destination: "/etc/resolv.conf", Source: "/engine/resolv.conf"}, + {Destination: "/etc/hosts", Source: "/engine/hosts"}, + {Destination: "/etc/resolv.conf/", Source: "/user/resolv.conf"}, + } + m, ok := resolvConfMount(mounts) + assert.True(t, ok) + assert.Equal(t, "/user/resolv.conf", m.Source) + _, ok = resolvConfMount(mounts[1:2]) + assert.False(t, ok) +} diff --git a/tests/e2e/test_cases.go b/tests/e2e/test_cases.go index 86908422a..7475498a9 100644 --- a/tests/e2e/test_cases.go +++ b/tests/e2e/test_cases.go @@ -521,6 +521,22 @@ func nerdctlTestCases() []containerTestArgs { Skippable: false, TestFunc: pingTest, }, + { + Image: "docker.io/urunc/hello-hyperlight-unikraft:v0.18.0", + Name: "Hyperlight-unikraft-http", + Devmapper: false, + Seccomp: true, + UID: 0, + GID: 0, + Groups: []int64{}, + Memory: "", + Cli: "/usr/local/lib/python3.12/http/server.py 8000", + Volumes: []containerVolume{}, + StaticNet: false, + SideContainers: []string{}, + Skippable: false, + TestFunc: httpGetTest, + }, } } @@ -1322,5 +1338,21 @@ func dockerTestCases() []containerTestArgs { Skippable: false, TestFunc: pingTest, }, + { + Image: "docker.io/urunc/hello-hyperlight-unikraft:v0.18.0", + Name: "Hyperlight-unikraft-http", + Devmapper: false, + Seccomp: true, + UID: 0, + GID: 0, + Groups: []int64{}, + Memory: "", + Cli: "/usr/local/lib/python3.12/http/server.py 8000", + Volumes: []containerVolume{}, + StaticNet: false, + SideContainers: []string{}, + Skippable: false, + TestFunc: httpGetTest, + }, } } diff --git a/tests/e2e/test_functions.go b/tests/e2e/test_functions.go index d92cdd699..4399c0622 100644 --- a/tests/e2e/test_functions.go +++ b/tests/e2e/test_functions.go @@ -340,6 +340,32 @@ func pingTest(tool testTool) error { return nil } +// httpGetTest fetches the root of the HTTP server that the guest serves on +// port 8000, at the address of the container, from the host. +func httpGetTest(tool testTool) error { + extractedIPAddr, err := tool.inspectPAndGet("ip") + if err != nil && err != errToolDoesNotSupport { + return fmt.Errorf("Failed to extract container IP: %v", err) + } else if err == errToolDoesNotSupport { + extractedIPAddr, err = tool.inspectCAndGet("IPAddress") + if err != nil { + return fmt.Errorf("Failed to extract container IP: %v", err) + } + } + url := fmt.Sprintf("http://%s:8000/", extractedIPAddr) + // The guest may still be starting when the container reports running. + cmd := exec.Command("curl", "-sSf", "-m", "10", "--retry", "5", "--retry-delay", "1", "--retry-all-errors", url) //nolint:gosec + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("Failed to GET %s: %v\n%s", url, err, output) + } + if len(output) == 0 { + return fmt.Errorf("Empty response from %s", url) + } + + return nil +} + func httpStaticNetTest(tool testTool) (err error) { extractedPid, err := tool.inspectCAndGet("pid") if err != nil {