Merge External PR: Merge main-pr1135 into main

cmd/urunc/create_libcontainer.go

@@ -133,6 +133,10 @@ func buildMonitorArgs(id string, logFormat string) []string {
 // pivot: /proc is always mounted in the monitor rootfs, so this re-execs the
 // sealed urunc binary without urunc having to be copied into that rootfs.
 func monitorProcess(cmd *cli.Command, u *unikontainers.Unikontainer, readyFile *os.File) (*libcontainer.Process, error) {
+	caps, err := u.MonitorCapabilities()
+	if err != nil {
+		return nil, err
+	}
 	process := &libcontainer.Process{
 		Args: buildMonitorArgs(u.State.ID, cmd.String("log-format")),
 		Env:  os.Environ(),
@@ -141,7 +145,7 @@ func monitorProcess(cmd *cli.Command, u *unikontainers.Unikontainer, readyFile *
 		Cwd: "/",
 		// TODO set uid, gid and additional groups here
 		Init:         true,
-		Capabilities: unikontainers.MonitorCapabilities(u.Spec),
+		Capabilities: caps,
 		LogLevel:     strconv.Itoa(int(logrus.GetLevel())),
 		// The ready pipe's write end. It must be the only/first ExtraFile so the
 		// monitor inherits it at unikontainers.ReadyPipeFD:

docs/hypervisor-support.md

@@ -335,14 +335,33 @@ leaves the size to `hluk`, which gives each guest runtime the memory it is
 tested with (e.g. 256 MiB for Python, 1536 MiB for the agent image).
 
 [Hyperlight](https://github.com/hyperlight-dev/hyperlight) does not provide any
-VirtIO devices, nor a tap-based network interface. As a result, networking is
-not available for unikernels running on top of Hyperlight and, for the time
-being, initramfs is the only supported storage option. Neither virtio-block nor
-devmapper-backed block storage can be used.
-
-Upstream `hluk` can preopen host directories for the guest and forward POSIX
-file operations to them, as well as forward the sockets of the guest to the
-host. We plan to add support for these options.
+VirtIO devices, nor a tap-based network interface. Instead, `hluk` forwards the
+socket calls of the guest to real sockets that it opens itself. Since `hluk`
+runs inside the network namespace of the container, the guest uses that
+namespace as it is: the address of the pod, its routes, and any
+NetworkPolicy that applies to it. For that reason, `urunc` does not create a
+tap device for this monitor, nor does it redirect the traffic of the
+container's interface or give the monitor `CAP_NET_ADMIN` and `/dev/net/tun`,
+and it starts `hluk` with networking enabled, both
+outbound (`--net`) and inbound (`--port=all`). The network namespace remains the
+boundary, as for any other container. A container started with no network has
+no interface in its namespace, and `urunc` then leaves the guest to the default
+of `hluk`, which is no network at all. A deployment can also limit the ports
+the guest may serve on, as a second boundary inside the namespace, with the
+`com.urunc.unikernel.hyperlightPorts` annotation; without it the guest serves
+on any port, as a container does. The `resolv.conf` of the container is
+mounted into the monitor and installed in the guest as well (`--resolv-conf`),
+both when it boots and when it resumes from a snapshot, so that cluster names
+resolve inside the guest. The guest binds ports as the user of the container:
+a container that runs as a non-root user needs
+`net.ipv4.ip_unprivileged_port_start=0` in its network namespace to serve on a
+port below 1024 (Docker sets it; on Kubernetes, set the sysctl in the pod),
+since the monitor keeps no capability across the switch to that user.
+
+For the time being, initramfs is the only supported storage option. Neither
+virtio-block nor devmapper-backed block storage can be used. Upstream `hluk`
+can also preopen host directories for the guest and forward POSIX file
+operations to them. We plan to add support for this option.
 
 Rootfs images for every guest runtime that `hluk` supports (Python, Node.js,
 .NET, Go, Rust, C, Bash and more) are published under

docs/package/index.md

@@ -74,6 +74,10 @@ Apart from the above, `urunc` accepts the following optional annotations:
 - `com.urunc.unikernel.mountRootfs`: A boolean value that if it is `true`,
   requests from `urunc` to mount the container's image rootfs in the unikernel
   (either as a block device or through shared-fs).
+- `com.urunc.unikernel.hyperlightPorts`: The ports a `hyperlight-unikraft`
+  guest may serve on, as a comma-separated list of ports and `LOW-HIGH` ranges
+  (e.g. `8080,9000-9100`). Without it the guest serves on any port, as a
+  container does. Only supported by `hyperlight-unikraft`.
 
 Due to the fact that [Docker](https://www.docker.com/) and some high-level
 container runtimes do not pass the image annotations to the underlying container

pkg/network/network.go

@@ -33,6 +33,10 @@ const (
 
 var netlog = logrus.WithField("subsystem", "network")
 
+// ErrNoContainerIface reports a network namespace with no usable interface,
+// as in a container started with no network.
+var ErrNoContainerIface = errors.New("no suitable network interface found in namespace")
+
 type UnikernelNetworkInfo struct {
 	TapDevice string
 	EthDevice Interface
@@ -403,7 +407,18 @@ func discoverContainerIface() (netlink.Link, error) {
 		}
 		netlog.Debugf("skipping interface %s: no default route found", attrs.Name)
 	}
-	return nil, errors.New("no suitable network interface found in namespace")
+	return nil, ErrNoContainerIface
+}
+
+// ContainerIface describes the interface of the container in the current
+// network namespace, without touching it, for a monitor that gets no tap:
+// ErrNoContainerIface when there is none.
+func ContainerIface() (Interface, error) {
+	link, err := discoverContainerIface()
+	if err != nil {
+		return Interface{}, err
+	}
+	return getInterfaceInfo(link.Attrs().Name)
 }
 
 func deleteAllQDiscs(device netlink.Link) error {

pkg/unikontainers/annotations.go

@@ -40,20 +40,21 @@ import (
 // Urunc specific annotations
 // Always keep it in sync with the struct UnikernelConfig struct
 const (
-	annotUruncPrefix   = "com.urunc.unikernel."
-	annotType          = "com.urunc.unikernel.unikernelType"
-	annotVersion       = "com.urunc.unikernel.unikernelVersion"
-	annotBinary        = "com.urunc.unikernel.binary"
-	annotHypervisor    = "com.urunc.unikernel.hypervisor"
-	annotInitrd        = "com.urunc.unikernel.initrd"
-	annotSnapshot      = "com.urunc.unikernel.snapshot"
-	annotBlock         = "com.urunc.unikernel.block"
-	annotBlockMntPoint = "com.urunc.unikernel.blkMntPoint"
-	annotMountRootfs   = "com.urunc.unikernel.mountRootfs"
-	annotNetDev        = "com.urunc.unikernel.solo5NetDev"
-	annotBlkDev        = "com.urunc.unikernel.solo5BlkDev"
-	annotVAccel        = "com.urunc.unikernel.vAccel"
-	annotRPCAddress    = "com.urunc.unikernel.RPCAddress"
+	annotUruncPrefix     = "com.urunc.unikernel."
+	annotType            = "com.urunc.unikernel.unikernelType"
+	annotVersion         = "com.urunc.unikernel.unikernelVersion"
+	annotBinary          = "com.urunc.unikernel.binary"
+	annotHypervisor      = "com.urunc.unikernel.hypervisor"
+	annotInitrd          = "com.urunc.unikernel.initrd"
+	annotSnapshot        = "com.urunc.unikernel.snapshot"
+	annotBlock           = "com.urunc.unikernel.block"
+	annotBlockMntPoint   = "com.urunc.unikernel.blkMntPoint"
+	annotMountRootfs     = "com.urunc.unikernel.mountRootfs"
+	annotNetDev          = "com.urunc.unikernel.solo5NetDev"
+	annotBlkDev          = "com.urunc.unikernel.solo5BlkDev"
+	annotVAccel          = "com.urunc.unikernel.vAccel"
+	annotRPCAddress      = "com.urunc.unikernel.RPCAddress"
+	annotHyperlightPorts = "com.urunc.unikernel.hyperlightPorts"
 )
 
 // Annotations that other runtimes set and urunc only reads.
@@ -105,6 +106,7 @@ type UnikernelConfig struct {
 	MountRootfs      string `json:"com.urunc.unikernel.mountRootfs"`
 	NetDev           string `json:"com.urunc.unikernel.solo5NetDev,omitempty"`
 	BlkDev           string `json:"com.urunc.unikernel.solo5BlkDev,omitempty"`
+	HyperlightPorts  string `json:"com.urunc.unikernel.hyperlightPorts,omitempty"`
 	// The vAccel annotations are deliberately not part of urunc.json, since their
 	// values are runtime specific and therefore we should only reach them
 	// through the annotations of the spec.
@@ -116,6 +118,10 @@ type UnikernelConfig struct {
 // identifier for Solo5 guests when received from the annotations.
 var solo5DevNameRe = regexp.MustCompile(`^[A-Za-z0-9]{1,67}$`)
 
+// hyperlightPortsRe constrains the ports a hyperlight-unikraft guest may serve
+// on to a comma-separated list of ports and LOW-HIGH ranges.
+var hyperlightPortsRe = regexp.MustCompile(`^[0-9]{1,5}(-[0-9]{1,5})?(,[0-9]{1,5}(-[0-9]{1,5})?)*$`)
+
 var ErrNotUnikernel = errors.New("this is not a unikernel container")
 
 // validate checks if the mandatory configuration fields are present. It
@@ -181,6 +187,13 @@ func GetUnikernelConfig(bundleDir string, spec *specs.Spec) (*UnikernelConfig, e
 	// configured through urunc.json can still use vAccel
 	jsonConf.VAccel = spec.Annotations[annotVAccel]
 	jsonConf.RPCAddress = spec.Annotations[annotRPCAddress]
+	// The ports a hyperlight-unikraft guest may serve on are a choice of the
+	// deployment first, and of the image second: an annotation of the spec
+	// replaces whatever urunc.json says, and validateValues() checks it as it
+	// does for an image configured through the spec.
+	if ports := spec.Annotations[annotHyperlightPorts]; ports != "" {
+		jsonConf.HyperlightPorts = ports
+	}
 
 	return jsonConf, nil
 }
@@ -198,6 +211,7 @@ func getConfigFromSpec(spec *specs.Spec) *UnikernelConfig {
 	MountRootfs := spec.Annotations[annotMountRootfs]
 	netDev := spec.Annotations[annotNetDev]
 	blkDev := spec.Annotations[annotBlkDev]
+	hyperlightPorts := spec.Annotations[annotHyperlightPorts]
 	vAccel := spec.Annotations[annotVAccel]
 	rpcAddress := spec.Annotations[annotRPCAddress]
 	uniklog.WithFields(logrus.Fields{
@@ -212,6 +226,7 @@ func getConfigFromSpec(spec *specs.Spec) *UnikernelConfig {
 		"mountRootfs":      MountRootfs,
 		"netDev":           netDev,
 		"blkDev":           blkDev,
+		"hyperlightPorts":  hyperlightPorts,
 		"vAccel":           vAccel,
 		"rpcAddress":       rpcAddress,
 	}).WithField("source", "spec").Debug("urunc annotations")
@@ -228,6 +243,7 @@ func getConfigFromSpec(spec *specs.Spec) *UnikernelConfig {
 		MountRootfs:      MountRootfs,
 		NetDev:           netDev,
 		BlkDev:           blkDev,
+		HyperlightPorts:  hyperlightPorts,
 		VAccel:           vAccel,
 		RPCAddress:       rpcAddress,
 	}
@@ -271,6 +287,7 @@ func getConfigFromJSON(jsonFilePath string) (*UnikernelConfig, error) {
 		"mountRootfs":      tryDecode(conf.MountRootfs),
 		"netDev":           tryDecode(conf.NetDev),
 		"blkDev":           tryDecode(conf.BlkDev),
+		"hyperlightPorts":  tryDecode(conf.HyperlightPorts),
 	}).WithField("source", uruncJSONFilename).Debug("urunc annotations")
 
 	return &conf, nil
@@ -352,6 +369,12 @@ func (c *UnikernelConfig) decode() error {
 	}
 	c.BlkDev = string(decoded)
 
+	decoded, err = base64.StdEncoding.DecodeString(c.HyperlightPorts)
+	if err != nil {
+		return fmt.Errorf("failed to decode hyperlightPorts: %v", err)
+	}
+	c.HyperlightPorts = string(decoded)
+
 	return nil
 }
 
@@ -391,6 +414,9 @@ func (c *UnikernelConfig) Map() map[string]string {
 	if c.BlkDev != "" {
 		myMap[annotBlkDev] = c.BlkDev
 	}
+	if c.HyperlightPorts != "" {
+		myMap[annotHyperlightPorts] = c.HyperlightPorts
+	}
 	if c.VAccel != "" {
 		myMap[annotVAccel] = c.VAccel
 	}
@@ -471,6 +497,11 @@ func (c *UnikernelConfig) validateValues() error {
 		return err
 	}
 
+	err = validateHyperlightPorts(monitor, c.HyperlightPorts)
+	if err != nil {
+		return err
+	}
+
 	if c.VAccel == "" {
 		if c.RPCAddress != "" {
 			return fmt.Errorf("%s is set, but %s is not", annotRPCAddress, annotVAccel)
@@ -517,6 +548,44 @@ func validateSolo5DevName(key string, val string) error {
 	return nil
 }
 
+// validateHyperlightPorts verifies that val lists the ports a
+// hyperlight-unikraft guest may serve on: ports and LOW-HIGH ranges within
+// 1-65535, separated by commas.
+// An empty value is accepted, since the guest then serves on any port, as a
+// container does. No other monitor can limit the ports of a guest.
+func validateHyperlightPorts(monitor hypervisors.VmmType, val string) error {
+	if val == "" {
+		return nil
+	}
+
+	if monitor != hypervisors.HyperlightVmm {
+		return fmt.Errorf("%s is only supported by %s", annotHyperlightPorts, hypervisors.HyperlightVmm)
+	}
+
+	if len(val) > maxAnnotationValueLen {
+		return fmt.Errorf("%s value is longer than %d bytes", annotHyperlightPorts, maxAnnotationValueLen)
+	}
+
+	if !hyperlightPortsRe.MatchString(val) {
+		return fmt.Errorf("%s must match %s, got %q", annotHyperlightPorts, hyperlightPortsRe.String(), val)
+	}
+
+	for _, item := range strings.Split(val, ",") {
+		lowStr, highStr, isRange := strings.Cut(item, "-")
+		if !isRange {
+			highStr = lowStr
+		}
+		// The regexp already limited both to digits.
+		low, _ := strconv.Atoi(lowStr)
+		high, _ := strconv.Atoi(highStr)
+		if low < 1 || high > 65535 || low > high {
+			return fmt.Errorf("%s must list ports within 1-65535 and ranges from low to high, got %q", annotHyperlightPorts, item)
+		}
+	}
+
+	return nil
+}
+
 // validateAnnotationPathClean verifies that val looks like a filepath with
 // simple plain characters and is also clean.
 func validateAnnotationPathClean(key string, val string, rejectRoot bool) error {