diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml new file mode 100644 index 00000000..ab166887 --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml @@ -0,0 +1,75 @@ +apiVersion: hypershift.openshift.io/v1beta1 +kind: HostedCluster +metadata: + name: 'tenant-a' + namespace: 'clusters' + labels: + "cluster.open-cluster-management.io/clusterset": 'default' +spec: + configuration: + ingress: + appsDomain: apps.tenant-a.coe.muc.redhat.com # (1) + domain: '' + loadBalancer: + platform: + type: '' + channel: fast-4.21 + etcd: + managed: + storage: + persistentVolume: + size: 8Gi + type: PersistentVolume + managementType: Managed + release: + image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi + pullSecret: + name: pullsecret-cluster-tenant-a + sshKey: + name: sshkey-cluster-tenant-a + networking: + clusterNetwork: + - cidr: 10.132.0.0/14 + serviceNetwork: + - cidr: 172.31.0.0/16 + networkType: OVNKubernetes + controllerAvailabilityPolicy: SingleReplica + infrastructureAvailabilityPolicy: SingleReplica + platform: + type: KubeVirt + kubevirt: + baseDomainPassthrough: false + infraID: 'tenant-a' + services: + - service: APIServer + servicePublishingStrategy: + type: LoadBalancer + loadBalancer: + hostname: api.tenant-a.coe.muc.redhat.com # (2) + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: + hostname: oauth.tenant-a.coe.muc.redhat.com # (3) + - service: OIDC + servicePublishingStrategy: + type: Route + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: + hostname: konnectivity.tenant-a.coe.muc.redhat.com # (4) + - service: Ignition + servicePublishingStrategy: + type: Route + route: + hostname: ignition.tenant-a.coe.muc.redhat.com # (5) + + operatorConfiguration: # (6) + ingressOperator: + endpointPublishingStrategy: + type: HostNetwork + hostNetwork: + httpPort: 80 + httpsPort: 443 + protocol: TCP diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml new file mode 100644 index 00000000..5a08146b --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml @@ -0,0 +1,27 @@ +apiVersion: hypershift.openshift.io/v1beta1 +kind: NodePool +metadata: + name: 'tenant-a' + namespace: 'clusters' +spec: + arch: amd64 + clusterName: 'tenant-a' + replicas: 2 + management: + autoRepair: false + upgradeType: Replace + platform: + type: KubeVirt + kubevirt: + compute: + cores: 2 + memory: 8Gi + rootVolume: + type: Persistent + persistent: + size: 32Gi + additionalNetworks: + - name: cudn-localnet1-2003 # (1) + attachDefaultNetwork: false + release: + image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/cudn-localnet1-2003.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/cudn-localnet1-2003.yaml index ec0630fd..b322d886 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/cudn-localnet1-2003.yaml +++ b/content/cluster-installation/hosted-control-plane/tenant-network/cudn-localnet1-2003.yaml @@ -5,10 +5,8 @@ metadata: spec: namespaceSelector: matchExpressions: - - key: kubernetes.io/metadata.name - operator: In - values: - - default + - key: hypershift.openshift.io/hosted-control-plane + operator: Exists network: localnet: ipam: diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/index.md b/content/cluster-installation/hosted-control-plane/tenant-network/index.md index eb468e95..27afcd9a 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/index.md +++ b/content/cluster-installation/hosted-control-plane/tenant-network/index.md @@ -24,6 +24,12 @@ Challenge: running a hosted cluster in a different tenant network segment or VLA ![](overview.drawio){ page="Page-1" } +!!! warning "LoadBalancer Services in hosted clusters get IPs from the hub" + + There is currently **no supported way to disable or constrain this**. If the hub cluster supports Kubernetes Services of type `LoadBalancer` (e.g. via MetalLB, a cloud provider CCM, or any other implementation), any such Service created **inside the hosted cluster** will be fulfilled by the hub's load-balancer implementation. This means workloads in the tenant network can inadvertently obtain IP addresses from the hub's network. + + Tracking: [RFE-7742](https://redhat.atlassian.net/browse/RFE-7742) + An hosted cluster can devide into two parts: **control plane** and **data plan aka worker nodes**. For there parts there different technics to place it into a tenant network: ## Exposing hosted control plane into tenant network @@ -69,10 +75,24 @@ Worker nodes (VM's) of the hosted cluster are straightforward: attach them to th ??? example "VyOS config commands" + Boot and install VyOS from ISO: + + Don't forget to `commit` and `save` changes. With `compare` you can see your chages. + + Configuration active after reboot is stored `/opt/vyatta/etc/config/config.boot` (recommened) or `/opt/vyatta/etc/config.boot.default` deppend on your settings during iso installation. + ```shell --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt" ``` + FYI: `show dhcp server leases` shows you all dhcpd leases + +??? example "Tunneling the network via sshuttle" + + ```shell + sshuttle -r vyos@ 192.168.203.0/24 + ``` + ### Deployment sequence (reference) Three external load balancers appear in this write-up; keep their roles distinct: @@ -128,75 +148,24 @@ oauth.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111 ignition.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111 ``` +### Apply PullSecret, SSH-Key + +```shell +oc create secret docker-registry pullsecret-cluster-tenant-a \ + -n clusters \ + --from-file=${HOME}/redhat-pull-secret.json +``` + +```shell +oc create secret generic sshkey-cluster-tenant-a \ + -n clusters \ + --from-file=id_rsa.pub=${HOME}/.ssh/id_ed25519.pub +``` + ### Apply `HostedCluster` and `NodePool` -```yaml hl_lines="11 43-66" title="HostedCluster" -apiVersion: hypershift.openshift.io/v1beta1 -kind: HostedCluster -metadata: - name: 'tenant-a' - namespace: 'clusters' - labels: - "cluster.open-cluster-management.io/clusterset": 'default' -spec: - configuration: - ingress: - appsDomain: apps.tenant-a.coe.muc.redhat.com # (1) - domain: '' - loadBalancer: - platform: - type: '' - channel: fast-4.21 - etcd: - managed: - storage: - persistentVolume: - size: 8Gi - type: PersistentVolume - managementType: Managed - release: - image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi - pullSecret: - name: pullsecret-cluster-tenant-a - sshKey: - name: sshkey-cluster-tenant-a - networking: - clusterNetwork: - - cidr: 10.132.0.0/14 - serviceNetwork: - - cidr: 172.31.0.0/16 - networkType: OVNKubernetes - controllerAvailabilityPolicy: SingleReplica - infrastructureAvailabilityPolicy: SingleReplica - platform: - type: KubeVirt - kubevirt: - baseDomainPassthrough: false - infraID: 'tenant-a' - services: - - service: APIServer - servicePublishingStrategy: - type: LoadBalancer - loadBalancer: - hostname: api.tenant-a.coe.muc.redhat.com # (2) - - service: OAuthServer - servicePublishingStrategy: - type: Route - route: - hostname: oauth.tenant-a.coe.muc.redhat.com # (3) - - service: OIDC - servicePublishingStrategy: - type: Route - - service: Konnectivity - servicePublishingStrategy: - type: Route - route: - hostname: konnectivity.tenant-a.coe.muc.redhat.com # (4) - - service: Ignition - servicePublishingStrategy: - type: Route - route: - hostname: ignition.tenant-a.coe.muc.redhat.com # (5) +```yaml hl_lines="11 48 53 61 66 68" title="HostedCluster" +--8<-- "content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml" ``` 1. `appsDomain`: resolve names under `apps.tenant-a.coe.muc.redhat.com` to **`ingress-lb`** (hosted cluster ingress), not the hub shard. @@ -204,40 +173,15 @@ spec: 3. OAuth `route.hostname`: resolve to **`ingress-shared-lb`** (hub dedicated shard). 4. Konnectivity `route.hostname`: resolve to **`ingress-shared-lb`**. 5. Ignition `route.hostname`: resolve to **`ingress-shared-lb`**. +6. Switch from Kubernetes Type LoadBalancer to just `HostNetwork` as `endpointPublishingStrategy` ```yaml hl_lines="24-26" title="NodePool" -apiVersion: hypershift.openshift.io/v1beta1 -kind: NodePool -metadata: - name: 'tenant-a' - namespace: 'clusters' -spec: - arch: amd64 - clusterName: 'tenant-a' - replicas: 2 - management: - autoRepair: false - upgradeType: Replace - platform: - type: KubeVirt - kubevirt: - compute: - cores: 2 - memory: 8Gi - rootVolume: - type: Persistent - persistent: - size: 32Gi - additionalNetworks: - - name: default/cudn-localnet1-2003 # (1) - attachDefaultNetwork: false - release: - image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi +--8<-- "content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml" ``` -1. Attach NodePool VMs to the tenant segment using a user-defined network (UDN) `localnet` attachment (`default/cudn-localnet1-2003` in this lab). +1. Attach NodePool VMs to the tenant segment using a user-defined network (UDN) `localnet` attachment (`cudn-localnet1-2003` in this lab). -??? example "ClusterUserDefinedNetwork for `default/cudn-localnet1-2003`" +??? example "ClusterUserDefinedNetwork for `cudn-localnet1-2003`" ```yaml --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/cudn-localnet1-2003.yaml" @@ -245,6 +189,10 @@ spec: ### Deploy external load balancer for the Hosted-Cluster API (`api-lb`) +!!! info "NodePort ports are allocated randomly" + + The HAProxy backends point to NodePort high ports that are assigned dynamically. If the Service is recreated, ports may change and HAProxy must be reconfigured. See [NodePort chicken-and-egg problem](#nodeport-chicken-and-egg-problem) for a discussion and potential solutions. + Use an RHEL 9 virtual machine with HAProxy. * Install HAProxy: `dnf install haproxy` @@ -285,21 +233,78 @@ Add DNS record: *.apps.tenant-a.coe.muc.redhat.com. IN A 192.168.203. ``` -## Open topics +## LoadBalancer Services in hosted clusters get IPs from the hub -* Disable or constrain cloud provider integration so that Kubernetes `LoadBalancer` Service requests for the hosted cluster are not satisfied by the hub cluster cloud integration unless that is intentional. -* WebUI bug: ACM shows `https://console-openshift-console.apps.tenant-a.apps.ocp5.stormshift.coe.muc.redhat.com/` for the console, but the URL should be `https://console-openshift-console.apps.tenant-a.coe.muc.redhat.com/`. -* Add custom endpoint publishing strategy -* Find a solution for the NodePort chicken-and-egg problem of the external API load balancer -* Improve ClusterUserDefinedNetwork with following selector: +When the hub cluster supports Kubernetes Services of type `LoadBalancer` (e.g. via MetalLB, a cloud provider CCM, or any other implementation), any such Service created inside the hosted cluster is fulfilled by the **hub cluster's** load-balancer implementation. The hosted cluster's control plane runs on the hub and inherits its configuration. - ```yaml - namespaceSelector: - matchExpressions: - - key: hypershift.openshift.io/hosted-control-plane - operator: Exists +This means a tenant workload can unintentionally allocate an IP address from the hub's infrastructure network simply by creating a `LoadBalancer` Service. + +**There is currently no supported way to disable this behaviour.** Tracking: [RFE-7742](https://redhat.atlassian.net/browse/RFE-7742) + +### Reproducing the problem + +??? example "Deployment for a Kubernetes Service type LoadBalancer" + + ```shell + % oc project service-type-loadbalancer + % oc apply -k 'https://github.com/openshift-examples/kustomize.git/components/simple-https?ref=2026-07-20' + % oc apply -f - <