From 5eccbd20b02596e0b7028a9a5d7689d90c55c651 Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Tue, 28 Jul 2026 10:46:46 +0200 Subject: [PATCH 01/10] work-in-progress --- .../HostedCluster.tenant-a.yaml | 69 +++++++++++ .../tenant-network/NodePool.tenant-a.yaml | 27 ++++ .../tenant-network/index.md | 117 ++++-------------- 3 files changed, 119 insertions(+), 94 deletions(-) create mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml create mode 100644 content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml new file mode 100644 index 00000000..2a89b7a4 --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml @@ -0,0 +1,69 @@ +apiVersion: hypershift.openshift.io/v1beta1 +kind: HostedCluster +metadata: + name: 'tenant-a' + namespace: 'clusters' + labels: + "cluster.open-cluster-management.io/clusterset": 'default' +spec: + controllerManager: + extraArgs: + controllers: ["-service"] # (6) + configuration: + ingress: + appsDomain: apps.tenant-a.coe.muc.redhat.com # (1) + domain: '' + loadBalancer: + platform: + type: '' + channel: fast-4.21 + etcd: + managed: + storage: + persistentVolume: + size: 8Gi + type: PersistentVolume + managementType: Managed + release: + image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi + pullSecret: + name: pullsecret-cluster-tenant-a + sshKey: + name: sshkey-cluster-tenant-a + networking: + clusterNetwork: + - cidr: 10.132.0.0/14 + serviceNetwork: + - cidr: 172.31.0.0/16 + networkType: OVNKubernetes + controllerAvailabilityPolicy: SingleReplica + infrastructureAvailabilityPolicy: SingleReplica + platform: + type: KubeVirt + kubevirt: + baseDomainPassthrough: false + infraID: 'tenant-a' + services: + - service: APIServer + servicePublishingStrategy: + type: LoadBalancer + loadBalancer: + hostname: api.tenant-a.coe.muc.redhat.com # (2) + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: + hostname: oauth.tenant-a.coe.muc.redhat.com # (3) + - service: OIDC + servicePublishingStrategy: + type: Route + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: + hostname: konnectivity.tenant-a.coe.muc.redhat.com # (4) + - service: Ignition + servicePublishingStrategy: + type: Route + route: + hostname: ignition.tenant-a.coe.muc.redhat.com # (5) \ No newline at end of file diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml new file mode 100644 index 00000000..c925173e --- /dev/null +++ b/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml @@ -0,0 +1,27 @@ +apiVersion: hypershift.openshift.io/v1beta1 +kind: NodePool +metadata: + name: 'tenant-a' + namespace: 'clusters' +spec: + arch: amd64 + clusterName: 'tenant-a' + replicas: 2 + management: + autoRepair: false + upgradeType: Replace + platform: + type: KubeVirt + kubevirt: + compute: + cores: 2 + memory: 8Gi + rootVolume: + type: Persistent + persistent: + size: 32Gi + additionalNetworks: + - name: default/cudn-localnet1-2003 # (1) + attachDefaultNetwork: false + release: + image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi \ No newline at end of file diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/index.md b/content/cluster-installation/hosted-control-plane/tenant-network/index.md index eb468e95..61a8fcdc 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/index.md +++ b/content/cluster-installation/hosted-control-plane/tenant-network/index.md @@ -128,111 +128,40 @@ oauth.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111 ignition.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111 ``` +### Apply PullSecret, SSH-Key + +```shell +oc create secret docker-registry pullsecret-cluster-tenant-a \ + -n clusters \ + --from-file=${HOME}/redhat-pull-secret.json +``` + +```shell +oc create secret generic sshkey-cluster-tenant-a \ + -n clusters \ + --from-file=id_rsa.pub=${HOME}/.ssh/id_ed25519.pub +``` + ### Apply `HostedCluster` and `NodePool` -```yaml hl_lines="11 43-66" title="HostedCluster" -apiVersion: hypershift.openshift.io/v1beta1 -kind: HostedCluster -metadata: - name: 'tenant-a' - namespace: 'clusters' - labels: - "cluster.open-cluster-management.io/clusterset": 'default' -spec: - configuration: - ingress: - appsDomain: apps.tenant-a.coe.muc.redhat.com # (1) - domain: '' - loadBalancer: - platform: - type: '' - channel: fast-4.21 - etcd: - managed: - storage: - persistentVolume: - size: 8Gi - type: PersistentVolume - managementType: Managed - release: - image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi - pullSecret: - name: pullsecret-cluster-tenant-a - sshKey: - name: sshkey-cluster-tenant-a - networking: - clusterNetwork: - - cidr: 10.132.0.0/14 - serviceNetwork: - - cidr: 172.31.0.0/16 - networkType: OVNKubernetes - controllerAvailabilityPolicy: SingleReplica - infrastructureAvailabilityPolicy: SingleReplica - platform: - type: KubeVirt - kubevirt: - baseDomainPassthrough: false - infraID: 'tenant-a' - services: - - service: APIServer - servicePublishingStrategy: - type: LoadBalancer - loadBalancer: - hostname: api.tenant-a.coe.muc.redhat.com # (2) - - service: OAuthServer - servicePublishingStrategy: - type: Route - route: - hostname: oauth.tenant-a.coe.muc.redhat.com # (3) - - service: OIDC - servicePublishingStrategy: - type: Route - - service: Konnectivity - servicePublishingStrategy: - type: Route - route: - hostname: konnectivity.tenant-a.coe.muc.redhat.com # (4) - - service: Ignition - servicePublishingStrategy: - type: Route - route: - hostname: ignition.tenant-a.coe.muc.redhat.com # (5) +```yaml hl_lines="9-11 14 46-69" title="HostedCluster" +--8<-- "content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml" ``` +OSDOCS-19432 + +https://hypershift.pages.dev/contribute/add-a-capability/#background-openshift-capabilities +=> + 1. `appsDomain`: resolve names under `apps.tenant-a.coe.muc.redhat.com` to **`ingress-lb`** (hosted cluster ingress), not the hub shard. 2. API server `loadBalancer.hostname`: resolve to **`api-lb`**, which forwards to the `APIServer` publishing target on the hub. 3. OAuth `route.hostname`: resolve to **`ingress-shared-lb`** (hub dedicated shard). 4. Konnectivity `route.hostname`: resolve to **`ingress-shared-lb`**. 5. Ignition `route.hostname`: resolve to **`ingress-shared-lb`**. +6. Explicitly exclude the service controller, to avoid Kubernetes Services type LoadBalancer requestes are forwarded from Hosted Cluster to Hub Cluster. ```yaml hl_lines="24-26" title="NodePool" -apiVersion: hypershift.openshift.io/v1beta1 -kind: NodePool -metadata: - name: 'tenant-a' - namespace: 'clusters' -spec: - arch: amd64 - clusterName: 'tenant-a' - replicas: 2 - management: - autoRepair: false - upgradeType: Replace - platform: - type: KubeVirt - kubevirt: - compute: - cores: 2 - memory: 8Gi - rootVolume: - type: Persistent - persistent: - size: 32Gi - additionalNetworks: - - name: default/cudn-localnet1-2003 # (1) - attachDefaultNetwork: false - release: - image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi +--8<-- "content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml" ``` 1. Attach NodePool VMs to the tenant segment using a user-defined network (UDN) `localnet` attachment (`default/cudn-localnet1-2003` in this lab). From 494430e992ff141b85b6fb2d98d93ea7b5da6c40 Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Fri, 14 Aug 2026 12:07:58 +0200 Subject: [PATCH 02/10] Improve clusteruserdefine network --- .../tenant-network/NodePool.tenant-a.yaml | 2 +- .../tenant-network/cudn-localnet1-2003.yaml | 6 ++---- .../hosted-control-plane/tenant-network/index.md | 8 -------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml index c925173e..2a7396b1 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml +++ b/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml @@ -21,7 +21,7 @@ spec: persistent: size: 32Gi additionalNetworks: - - name: default/cudn-localnet1-2003 # (1) + - name: cudn-localnet1-2003 # (1) attachDefaultNetwork: false release: image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi \ No newline at end of file diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/cudn-localnet1-2003.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/cudn-localnet1-2003.yaml index ec0630fd..b322d886 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/cudn-localnet1-2003.yaml +++ b/content/cluster-installation/hosted-control-plane/tenant-network/cudn-localnet1-2003.yaml @@ -5,10 +5,8 @@ metadata: spec: namespaceSelector: matchExpressions: - - key: kubernetes.io/metadata.name - operator: In - values: - - default + - key: hypershift.openshift.io/hosted-control-plane + operator: Exists network: localnet: ipam: diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/index.md b/content/cluster-installation/hosted-control-plane/tenant-network/index.md index 61a8fcdc..eec35414 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/index.md +++ b/content/cluster-installation/hosted-control-plane/tenant-network/index.md @@ -220,14 +220,6 @@ Add DNS record: * WebUI bug: ACM shows `https://console-openshift-console.apps.tenant-a.apps.ocp5.stormshift.coe.muc.redhat.com/` for the console, but the URL should be `https://console-openshift-console.apps.tenant-a.coe.muc.redhat.com/`. * Add custom endpoint publishing strategy * Find a solution for the NodePort chicken-and-egg problem of the external API load balancer -* Improve ClusterUserDefinedNetwork with following selector: - - ```yaml - namespaceSelector: - matchExpressions: - - key: hypershift.openshift.io/hosted-control-plane - operator: Exists - ``` ## Verions From 5d10f85be797e3b2acfe99a284edb4ebe9aa75ff Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Fri, 14 Aug 2026 12:08:49 +0200 Subject: [PATCH 03/10] Improve VyOS commands --- .../HostedCluster.tenant-a.yaml | 3 --- .../tenant-network/index.md | 20 +++++++++----- .../tenant-network/vyos-router-2003.txt | 27 ++++++++++++++----- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml index 2a89b7a4..4f46368e 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml +++ b/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml @@ -6,9 +6,6 @@ metadata: labels: "cluster.open-cluster-management.io/clusterset": 'default' spec: - controllerManager: - extraArgs: - controllers: ["-service"] # (6) configuration: ingress: appsDomain: apps.tenant-a.coe.muc.redhat.com # (1) diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/index.md b/content/cluster-installation/hosted-control-plane/tenant-network/index.md index eec35414..73701cd7 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/index.md +++ b/content/cluster-installation/hosted-control-plane/tenant-network/index.md @@ -69,10 +69,24 @@ Worker nodes (VM's) of the hosted cluster are straightforward: attach them to th ??? example "VyOS config commands" + Boot and install VyOS from ISO: + + Don't forget to `commit` and `save` changes. With `compare` you can see your chages. + + Configuration active after reboot is stored `/opt/vyatta/etc/config/config.boot` (recommened) or `/opt/vyatta/etc/config.boot.default` deppend on your settings during iso installation. + ```shell --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt" ``` + FYI: `show dhcp server leases` shows you all dhcpd leases + +??? example "Tunneling the network via sshuttle" + + ```shell + sshuttle -r vyos@ 192.168.203.0/24 + ``` + ### Deployment sequence (reference) Three external load balancers appear in this write-up; keep their roles distinct: @@ -148,17 +162,11 @@ oc create secret generic sshkey-cluster-tenant-a \ --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml" ``` -OSDOCS-19432 - -https://hypershift.pages.dev/contribute/add-a-capability/#background-openshift-capabilities -=> - 1. `appsDomain`: resolve names under `apps.tenant-a.coe.muc.redhat.com` to **`ingress-lb`** (hosted cluster ingress), not the hub shard. 2. API server `loadBalancer.hostname`: resolve to **`api-lb`**, which forwards to the `APIServer` publishing target on the hub. 3. OAuth `route.hostname`: resolve to **`ingress-shared-lb`** (hub dedicated shard). 4. Konnectivity `route.hostname`: resolve to **`ingress-shared-lb`**. 5. Ignition `route.hostname`: resolve to **`ingress-shared-lb`**. -6. Explicitly exclude the service controller, to avoid Kubernetes Services type LoadBalancer requestes are forwarded from Hosted Cluster to Hub Cluster. ```yaml hl_lines="24-26" title="NodePool" --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml" diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt b/content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt index 5e09bc44..e7081a41 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt +++ b/content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt @@ -1,3 +1,19 @@ +configure + +set interfaces ethernet eth0 address 'dhcp' +set interfaces ethernet eth1 address '192.168.203.1/24' + +set service ssh +set system host-name 'router-2003' +set system name-server '10.32.96.1' +set system name-server '10.32.96.31' + +# Recommend to commit,save and reboot at this point +commit +save +exit +reboot + set firewall group address-group ALLOWED-IPS address '10.32.96.1' set firewall group address-group ALLOWED-IPS address '10.32.96.31' set firewall group address-group ALLOWED-IPS address '10.32.111.254' @@ -8,9 +24,6 @@ set firewall ipv4 forward filter rule 50 action 'drop' set firewall ipv4 forward filter rule 50 description 'Drop enire coe lab' set firewall ipv4 forward filter rule 50 destination address '10.32.96.0/20' -set interfaces ethernet eth0 address 'dhcp' -set interfaces ethernet eth1 address '192.168.203.1/24' - set nat source rule 100 outbound-interface name 'eth0' set nat source rule 100 source address '192.168.203.0/24' set nat source rule 100 translation address 'masquerade' @@ -21,7 +34,7 @@ set service dhcp-server shared-network-name coe-2003 subnet 192.168.203.0/24 opt set service dhcp-server shared-network-name coe-2003 subnet 192.168.203.0/24 range 1 start '192.168.203.100' set service dhcp-server shared-network-name coe-2003 subnet 192.168.203.0/24 range 1 stop '192.168.203.200' set service dhcp-server shared-network-name coe-2003 subnet 192.168.203.0/24 subnet-id '1' -set service ssh -set system host-name 'router-2003' -set system name-server '10.32.96.1' -set system name-server '10.32.96.31' + +# Don't forget to commit and write +commit +save \ No newline at end of file From e353575f939be6d3a68e190b53d7fb333cc144cc Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Fri, 14 Aug 2026 12:27:10 +0200 Subject: [PATCH 04/10] Add more information --- .../tenant-network/index.md | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/index.md b/content/cluster-installation/hosted-control-plane/tenant-network/index.md index 73701cd7..56913e1e 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/index.md +++ b/content/cluster-installation/hosted-control-plane/tenant-network/index.md @@ -225,9 +225,46 @@ Add DNS record: ## Open topics * Disable or constrain cloud provider integration so that Kubernetes `LoadBalancer` Service requests for the hosted cluster are not satisfied by the hub cluster cloud integration unless that is intentional. + + + ??? example Deployment for an kubernetes service type loadbalancer" + + ```shell + % oc project service-type-loadbalancer + % oc apply -k 'https://github.com/openshift-examples/kustomize.git/components/simple-https?ref=2026-07-20' + % oc apply -f - < * Add custom endpoint publishing strategy * Find a solution for the NodePort chicken-and-egg problem of the external API load balancer + Potential solutions to expose into external networks: + * [F5 BIG-IP (Container Ingress Services - CIS)](https://clouddocs.f5.com/containers/latest/) + * NetScaler (formerly Citrix ADC) + * A10 Networks (Thunder ADC) + * Kemp (Progress) LoadMaster + * LoxiLB ## Verions From 9c219859fca6f4cf0507e744164edd96a25050ee Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Fri, 14 Aug 2026 12:36:54 +0200 Subject: [PATCH 05/10] Improve k8s srv type lb --- .../tenant-network/index.md | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/index.md b/content/cluster-installation/hosted-control-plane/tenant-network/index.md index 56913e1e..cab5807f 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/index.md +++ b/content/cluster-installation/hosted-control-plane/tenant-network/index.md @@ -24,6 +24,12 @@ Challenge: running a hosted cluster in a different tenant network segment or VLA ![](overview.drawio){ page="Page-1" } +!!! warning "LoadBalancer Services in hosted clusters get IPs from the hub" + + There is currently **no supported way to disable or constrain this**. If the hub cluster supports Kubernetes Services of type `LoadBalancer` (e.g. via MetalLB, a cloud provider CCM, or any other implementation), any such Service created **inside the hosted cluster** will be fulfilled by the hub's load-balancer implementation. This means workloads in the tenant network can inadvertently obtain IP addresses from the hub's network. + + Tracking: [RFE-7742](https://redhat.atlassian.net/browse/RFE-7742) + An hosted cluster can devide into two parts: **control plane** and **data plan aka worker nodes**. For there parts there different technics to place it into a tenant network: ## Exposing hosted control plane into tenant network @@ -222,12 +228,17 @@ Add DNS record: *.apps.tenant-a.coe.muc.redhat.com. IN A 192.168.203. ``` -## Open topics +## LoadBalancer Services in hosted clusters get IPs from the hub + +When the hub cluster supports Kubernetes Services of type `LoadBalancer` (e.g. via MetalLB, a cloud provider CCM, or any other implementation), any such Service created inside the hosted cluster is fulfilled by the **hub cluster's** load-balancer implementation. The hosted cluster's control plane runs on the hub and inherits its configuration. + +This means a tenant workload can unintentionally allocate an IP address from the hub's infrastructure network simply by creating a `LoadBalancer` Service. -* Disable or constrain cloud provider integration so that Kubernetes `LoadBalancer` Service requests for the hosted cluster are not satisfied by the hub cluster cloud integration unless that is intentional. - +**There is currently no supported way to disable this behaviour.** Tracking: [RFE-7742](https://redhat.atlassian.net/browse/RFE-7742) - ??? example Deployment for an kubernetes service type loadbalancer" +### Reproducing the problem + +??? example "Deployment for a Kubernetes Service type LoadBalancer" ```shell % oc project service-type-loadbalancer @@ -236,28 +247,34 @@ Add DNS record: apiVersion: v1 kind: Service metadata: - name: simple-https-lb + name: simple-https-lb spec: - ports: - - name: http + ports: + - name: http port: 8080 protocol: TCP targetPort: 8080 - - name: https + - name: https port: 8443 protocol: TCP targetPort: 8443 - selector: + selector: app: simple-https deployment: simple-https - sessionAffinity: None - type: LoadBalancer + sessionAffinity: None + type: LoadBalancer EOF ``` + The Service will receive an `EXTERNAL-IP` from the hub's cloud provider—**not** from the tenant network. + +## Open topics + * WebUI bug: ACM shows `https://console-openshift-console.apps.tenant-a.apps.ocp5.stormshift.coe.muc.redhat.com/` for the console, but the URL should be `https://console-openshift-console.apps.tenant-a.coe.muc.redhat.com/`. + * Add custom endpoint publishing strategy + * Find a solution for the NodePort chicken-and-egg problem of the external API load balancer Potential solutions to expose into external networks: * [F5 BIG-IP (Container Ingress Services - CIS)](https://clouddocs.f5.com/containers/latest/) From bf76361c893a7b059324d3bb21a60d410e016711 Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Fri, 14 Aug 2026 12:38:08 +0200 Subject: [PATCH 06/10] Fix typo --- .../hosted-control-plane/tenant-network/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/index.md b/content/cluster-installation/hosted-control-plane/tenant-network/index.md index cab5807f..6c1c66c2 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/index.md +++ b/content/cluster-installation/hosted-control-plane/tenant-network/index.md @@ -178,9 +178,9 @@ oc create secret generic sshkey-cluster-tenant-a \ --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml" ``` -1. Attach NodePool VMs to the tenant segment using a user-defined network (UDN) `localnet` attachment (`default/cudn-localnet1-2003` in this lab). +1. Attach NodePool VMs to the tenant segment using a user-defined network (UDN) `localnet` attachment (`cudn-localnet1-2003` in this lab). -??? example "ClusterUserDefinedNetwork for `default/cudn-localnet1-2003`" +??? example "ClusterUserDefinedNetwork for `cudn-localnet1-2003`" ```yaml --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/cudn-localnet1-2003.yaml" From c9949025d98507d205e952c33f7e5b33ee4adbd0 Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Fri, 14 Aug 2026 12:43:18 +0200 Subject: [PATCH 07/10] Added NodePort chicken-and-egg problem --- .../tenant-network/index.md | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/index.md b/content/cluster-installation/hosted-control-plane/tenant-network/index.md index 6c1c66c2..66949bdd 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/index.md +++ b/content/cluster-installation/hosted-control-plane/tenant-network/index.md @@ -188,6 +188,10 @@ oc create secret generic sshkey-cluster-tenant-a \ ### Deploy external load balancer for the Hosted-Cluster API (`api-lb`) +!!! info "NodePort ports are allocated randomly" + + The HAProxy backends point to NodePort high ports that are assigned dynamically. If the Service is recreated, ports may change and HAProxy must be reconfigured. See [NodePort chicken-and-egg problem](#nodeport-chicken-and-egg-problem) for a discussion and potential solutions. + Use an RHEL 9 virtual machine with HAProxy. * Install HAProxy: `dnf install haproxy` @@ -268,6 +272,32 @@ This means a tenant workload can unintentionally allocate an IP address from the The Service will receive an `EXTERNAL-IP` from the hub's cloud provider—**not** from the tenant network. +## NodePort chicken-and-egg problem + +The external HAProxy load balancers (`api-lb`, `ingress-shared-lb`, `ingress-lb`) in this setup forward traffic to Kubernetes Services of type `NodePort`. The problem: NodePort allocates a **random high port** (default range 30000–32767) that is only known after the Service is created. Every time the Service is recreated or the port changes, the HAProxy configuration must be updated manually. + +This creates a chicken-and-egg situation especially for the API load balancer: you need the NodePort to configure HAProxy, but the hosted cluster needs a working API endpoint to become healthy. + +### Why an external LoadBalancer controller would solve this + +An external load balancer that **integrates with Kubernetes** (i.e. watches Service objects and configures itself automatically) eliminates the manual NodePort tracking entirely. Such a controller: + +* Reacts to Service creation/updates and provisions the correct backend configuration automatically +* Can expose Services into **separate network segments** (tenant networks) with its own VIP pool +* Removes the need for static HAProxy configs that break when ports change + +### Potential solutions + +| Product | Notes | +|---------|-------| +| [F5 BIG-IP (Container Ingress Services - CIS)](https://clouddocs.f5.com/containers/latest/) | Watches Kubernetes Services, provisions VIPs and pool members on BIG-IP | +| NetScaler (formerly Citrix ADC) | Kubernetes integration via Citrix Ingress Controller | +| A10 Networks (Thunder ADC) | Thunder Kubernetes Connector | +| Kemp (Progress) LoadMaster | Kubernetes integration available | +| LoxiLB | Open-source, cloud-native LB with Kubernetes Service support | + +Any of these can watch the hub cluster for the relevant Services and automatically configure load balancing into the tenant network—without manual NodePort tracking or HAProxy reconfiguration. + ## Open topics * WebUI bug: ACM shows `https://console-openshift-console.apps.tenant-a.apps.ocp5.stormshift.coe.muc.redhat.com/` for the console, but the URL should be `https://console-openshift-console.apps.tenant-a.coe.muc.redhat.com/`. @@ -275,14 +305,6 @@ This means a tenant workload can unintentionally allocate an IP address from the * Add custom endpoint publishing strategy -* Find a solution for the NodePort chicken-and-egg problem of the external API load balancer - Potential solutions to expose into external networks: - * [F5 BIG-IP (Container Ingress Services - CIS)](https://clouddocs.f5.com/containers/latest/) - * NetScaler (formerly Citrix ADC) - * A10 Networks (Thunder ADC) - * Kemp (Progress) LoadMaster - * LoxiLB - ## Verions Tested with: From f96fe409598cdc6b64bd4d24d43e1b339f31f5e0 Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Fri, 14 Aug 2026 13:03:07 +0200 Subject: [PATCH 08/10] Added endpointPublishingStrategy --- .../tenant-network/HostedCluster.tenant-a.yaml | 11 ++++++++++- .../hosted-control-plane/tenant-network/index.md | 5 ++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml index 4f46368e..c589c49d 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml +++ b/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml @@ -63,4 +63,13 @@ spec: servicePublishingStrategy: type: Route route: - hostname: ignition.tenant-a.coe.muc.redhat.com # (5) \ No newline at end of file + hostname: ignition.tenant-a.coe.muc.redhat.com # (5) + + operatorConfiguration # (6) + ingressOperator: + endpointPublishingStrategy + type: HostNetwork + hostNetwork: + httpPort: 80 + httpsPort: 443 + protocol: TCP diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/index.md b/content/cluster-installation/hosted-control-plane/tenant-network/index.md index 66949bdd..123e8b40 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/index.md +++ b/content/cluster-installation/hosted-control-plane/tenant-network/index.md @@ -164,7 +164,7 @@ oc create secret generic sshkey-cluster-tenant-a \ ### Apply `HostedCluster` and `NodePool` -```yaml hl_lines="9-11 14 46-69" title="HostedCluster" +```yaml hl_lines="11 48 53 61 66 68" title="HostedCluster" --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml" ``` @@ -173,6 +173,7 @@ oc create secret generic sshkey-cluster-tenant-a \ 3. OAuth `route.hostname`: resolve to **`ingress-shared-lb`** (hub dedicated shard). 4. Konnectivity `route.hostname`: resolve to **`ingress-shared-lb`**. 5. Ignition `route.hostname`: resolve to **`ingress-shared-lb`**. +6. Switch from Kubernetes Type LoadBalancer to just `HostNetwork` as `endpointPublishingStrategy` ```yaml hl_lines="24-26" title="NodePool" --8<-- "content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml" @@ -303,8 +304,6 @@ Any of these can watch the hub cluster for the relevant Services and automatical * WebUI bug: ACM shows `https://console-openshift-console.apps.tenant-a.apps.ocp5.stormshift.coe.muc.redhat.com/` for the console, but the URL should be `https://console-openshift-console.apps.tenant-a.coe.muc.redhat.com/`. -* Add custom endpoint publishing strategy - ## Verions Tested with: From 66a386bda9176e1832d27031039a99da9286fbed Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Fri, 14 Aug 2026 13:04:55 +0200 Subject: [PATCH 09/10] Improve last open issuie --- .../hosted-control-plane/tenant-network/index.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/index.md b/content/cluster-installation/hosted-control-plane/tenant-network/index.md index 123e8b40..27afcd9a 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/index.md +++ b/content/cluster-installation/hosted-control-plane/tenant-network/index.md @@ -299,10 +299,11 @@ An external load balancer that **integrates with Kubernetes** (i.e. watches Serv Any of these can watch the hub cluster for the relevant Services and automatically configure load balancing into the tenant network—without manual NodePort tracking or HAProxy reconfiguration. -## Open topics +## ACM console URL bug -* WebUI bug: ACM shows `https://console-openshift-console.apps.tenant-a.apps.ocp5.stormshift.coe.muc.redhat.com/` for the console, but the URL should be `https://console-openshift-console.apps.tenant-a.coe.muc.redhat.com/`. - +ACM displays an incorrect console URL for the hosted cluster. It shows `https://console-openshift-console.apps.tenant-a.apps.ocp5.stormshift.coe.muc.redhat.com/` instead of the correct `https://console-openshift-console.apps.tenant-a.coe.muc.redhat.com/`. The extra `.apps.ocp5.stormshift` segment comes from the hub's ingress domain being appended. + +Tracking: [OCPBUGS-105612](https://redhat.atlassian.net/browse/OCPBUGS-105612) ## Verions From 582b70379569f0a83e172181b2e116bc0ac94a4c Mon Sep 17 00:00:00 2001 From: Robert Bohne Date: Fri, 14 Aug 2026 13:06:47 +0200 Subject: [PATCH 10/10] Fix linter --- .../tenant-network/HostedCluster.tenant-a.yaml | 6 +++--- .../tenant-network/NodePool.tenant-a.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml index c589c49d..ab166887 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml +++ b/content/cluster-installation/hosted-control-plane/tenant-network/HostedCluster.tenant-a.yaml @@ -8,7 +8,7 @@ metadata: spec: configuration: ingress: - appsDomain: apps.tenant-a.coe.muc.redhat.com # (1) + appsDomain: apps.tenant-a.coe.muc.redhat.com # (1) domain: '' loadBalancer: platform: @@ -65,9 +65,9 @@ spec: route: hostname: ignition.tenant-a.coe.muc.redhat.com # (5) - operatorConfiguration # (6) + operatorConfiguration: # (6) ingressOperator: - endpointPublishingStrategy + endpointPublishingStrategy: type: HostNetwork hostNetwork: httpPort: 80 diff --git a/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml b/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml index 2a7396b1..5a08146b 100644 --- a/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml +++ b/content/cluster-installation/hosted-control-plane/tenant-network/NodePool.tenant-a.yaml @@ -21,7 +21,7 @@ spec: persistent: size: 32Gi additionalNetworks: - - name: cudn-localnet1-2003 # (1) + - name: cudn-localnet1-2003 # (1) attachDefaultNetwork: false release: - image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi \ No newline at end of file + image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi