Skip to content

Commit 3021f21

Browse files
LEDfanLEDfan
committed
Fix #29480: remove dependency on Skipper
1 parent dae73dc commit 3021f21

16 files changed

Lines changed: 225 additions & 640 deletions

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/Operator.kt‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,24 @@ package eu.openanalytics.shinyproxyoperator
2222

2323
import com.fasterxml.jackson.module.kotlin.registerKotlinModule
2424
import eu.openanalytics.shinyproxyoperator.controller.IReconcileListener
25+
import eu.openanalytics.shinyproxyoperator.controller.IRecyclableChecker
2526
import eu.openanalytics.shinyproxyoperator.controller.PodRetriever
27+
import eu.openanalytics.shinyproxyoperator.controller.RecyclableChecker
2628
import eu.openanalytics.shinyproxyoperator.controller.ResourceListener
2729
import eu.openanalytics.shinyproxyoperator.controller.ResourceRetriever
2830
import eu.openanalytics.shinyproxyoperator.controller.ShinyProxyController
2931
import eu.openanalytics.shinyproxyoperator.controller.ShinyProxyEvent
3032
import eu.openanalytics.shinyproxyoperator.controller.ShinyProxyListener
3133
import eu.openanalytics.shinyproxyoperator.crd.ShinyProxy
32-
import eu.openanalytics.shinyproxyoperator.ingress.skipper.IngressController
34+
import eu.openanalytics.shinyproxyoperator.controller.IngressController
3335
import io.fabric8.kubernetes.api.model.ConfigMap
3436
import io.fabric8.kubernetes.api.model.ConfigMapList
3537
import io.fabric8.kubernetes.api.model.Service
3638
import io.fabric8.kubernetes.api.model.ServiceList
3739
import io.fabric8.kubernetes.api.model.apps.ReplicaSet
3840
import io.fabric8.kubernetes.api.model.apps.ReplicaSetList
41+
import io.fabric8.kubernetes.api.model.networking.v1.Ingress
42+
import io.fabric8.kubernetes.api.model.networking.v1.IngressList
3943
import io.fabric8.kubernetes.client.DefaultKubernetesClient
4044
import io.fabric8.kubernetes.client.KubernetesClientException
4145
import io.fabric8.kubernetes.client.NamespacedKubernetesClient
@@ -62,7 +66,8 @@ class Operator(client: NamespacedKubernetesClient? = null,
6266
probeFailureThreshold: Int? = null,
6367
probeTimeout: Int? = null,
6468
startupProbeInitialDelay: Int? = null,
65-
logLevel: Level? = null) {
69+
logLevel: Level? = null,
70+
recyclableChecker: IRecyclableChecker? = null) {
6671

6772
private val logger = KotlinLogging.logger {}
6873
private val client: NamespacedKubernetesClient
@@ -77,11 +82,13 @@ class Operator(client: NamespacedKubernetesClient? = null,
7782

7883
val podRetriever: PodRetriever
7984
private val shinyProxyClient: ShinyProxyClient
85+
private val recyclableChecker: IRecyclableChecker
8086

8187
private val shinyProxyListener: ShinyProxyListener
8288
private val replicaSetListener: ResourceListener<ReplicaSet, ReplicaSetList, RollableScalableResource<ReplicaSet>>
8389
private val serviceListener: ResourceListener<Service, ServiceList, ServiceResource<Service>>
8490
private val configMapListener: ResourceListener<ConfigMap, ConfigMapList, Resource<ConfigMap>>
91+
private val ingressListener: ResourceListener<Ingress, IngressList, Resource<Ingress>>
8592
private val ingressController: IngressController
8693

8794
private val channel = Channel<ShinyProxyEvent>(10000)
@@ -145,24 +152,27 @@ class Operator(client: NamespacedKubernetesClient? = null,
145152

146153
shinyProxyListener = ShinyProxyListener(sendChannel, this.shinyProxyClient)
147154
podRetriever = PodRetriever(this.client)
155+
this.recyclableChecker = recyclableChecker ?: RecyclableChecker(podRetriever)
148156

149157
if (this.mode == Mode.CLUSTERED) {
150158
replicaSetListener = ResourceListener(sendChannel, this.client.inAnyNamespace().apps().replicaSets())
151159
serviceListener = ResourceListener(sendChannel, this.client.inAnyNamespace().services())
152160
configMapListener = ResourceListener(sendChannel, this.client.inAnyNamespace().configMaps())
153-
ingressController = IngressController(sendChannel, this.client.inAnyNamespace(), this.client.inAnyNamespace().network().v1().ingresses())
161+
ingressListener = ResourceListener(sendChannel, this.client.inAnyNamespace().network().v1().ingresses())
162+
ingressController = IngressController(this.client.inAnyNamespace().network().v1().ingresses())
154163
} else {
155164
replicaSetListener = ResourceListener(sendChannel, this.client.inNamespace(namespace).apps().replicaSets())
156165
serviceListener = ResourceListener(sendChannel, this.client.inNamespace(namespace).services())
157166
configMapListener = ResourceListener(sendChannel, this.client.inNamespace(namespace).configMaps())
158-
ingressController = IngressController(sendChannel, this.client.inNamespace(namespace), this.client.inNamespace(namespace).network().v1().ingresses())
167+
ingressListener = ResourceListener(sendChannel, this.client.inNamespace(namespace).network().v1().ingresses())
168+
ingressController = IngressController(this.client.inNamespace(namespace).network().v1().ingresses())
159169
}
160170
}
161171

162172
/**
163173
* Controllers
164174
*/
165-
val shinyProxyController = ShinyProxyController(channel, this.client, shinyProxyClient, ingressController, podRetriever, reconcileListener)
175+
val shinyProxyController = ShinyProxyController(channel, this.client, shinyProxyClient, ingressController, reconcileListener, this.recyclableChecker)
166176

167177
private fun _checkCrdExists(name: String, shortName: String) {
168178
try {
@@ -194,14 +204,13 @@ class Operator(client: NamespacedKubernetesClient? = null,
194204
logger.info { "Starting background processes of ShinyProxy Operator" }
195205

196206
_checkCrdExists("shinyproxies.openanalytics.eu", "ShinyProxy")
197-
_checkCrdExists("routegroups.zalando.org", "RouteGroup")
198207

199208
try {
200209
val shinyProxyLister = Lister(shinyProxyListener.start())
201210
val replicaSetLister = Lister(replicaSetListener.start(shinyProxyLister))
202211
val serviceLister = Lister(serviceListener.start(shinyProxyLister))
203212
val configMapLister = Lister(configMapListener.start(shinyProxyLister))
204-
val ingressLister = Lister(ingressController.start(shinyProxyLister))
213+
val ingressLister = Lister(ingressListener.start(shinyProxyLister))
205214
val resourceRetriever = ResourceRetriever(replicaSetLister, configMapLister, serviceLister, ingressLister)
206215
return resourceRetriever to shinyProxyLister
207216
} catch (e: KubernetesClientException) {
@@ -228,7 +237,7 @@ class Operator(client: NamespacedKubernetesClient? = null,
228237
replicaSetListener.stop()
229238
serviceListener.stop()
230239
configMapListener.stop()
231-
ingressController.stop()
240+
ingressListener.stop()
232241
}
233242

234243
companion object {

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/IngressFactory.kt‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/**
2+
* ShinyProxy-Operator
3+
*
4+
* Copyright (C) 2021-2022 Open Analytics
5+
*
6+
* ===========================================================================
7+
*
8+
* This program is free software: you can redistribute it and/or modify
9+
* it under the terms of the Apache License as published by
10+
* The Apache Software Foundation, either version 2 of the License, or
11+
* (at your option) any later version.
12+
*
13+
* This program is distributed in the hope that it will be useful,
14+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
15+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16+
* Apache License for more details.
17+
*
18+
* You should have received a copy of the Apache License
19+
* along with this program. If not, see <http://www.apache.org/licenses/>
20+
*/
21+
package eu.openanalytics.shinyproxyoperator.components
22+
23+
import eu.openanalytics.shinyproxyoperator.crd.ShinyProxy
24+
import eu.openanalytics.shinyproxyoperator.crd.ShinyProxyInstance
25+
import io.fabric8.kubernetes.api.model.networking.v1.HTTPIngressPath
26+
import io.fabric8.kubernetes.api.model.networking.v1.HTTPIngressPathBuilder
27+
import io.fabric8.kubernetes.api.model.networking.v1.Ingress
28+
import io.fabric8.kubernetes.api.model.networking.v1.IngressBuilder
29+
import io.fabric8.kubernetes.api.model.networking.v1.IngressList
30+
import io.fabric8.kubernetes.client.dsl.MixedOperation
31+
import io.fabric8.kubernetes.client.dsl.Resource
32+
import mu.KotlinLogging
33+
34+
class IngressFactory(private val kubeClient: MixedOperation<Ingress, IngressList, Resource<Ingress>>) {
35+
36+
private val logger = KotlinLogging.logger {}
37+
38+
fun createOrReplaceIngress(shinyProxy: ShinyProxy, latestInstance: ShinyProxyInstance) {
39+
val labels = LabelFactory.labelsForShinyProxy(shinyProxy).toMutableMap()
40+
labels[LabelFactory.LATEST_INSTANCE_LABEL] = latestInstance.hashOfSpec
41+
labels[LabelFactory.INSTANCE_LABEL] = latestInstance.hashOfSpec
42+
43+
//@formatter:off
44+
val ingressDefinition = IngressBuilder()
45+
.withNewMetadata()
46+
.withName(ResourceNameFactory.createNameForIngress(shinyProxy))
47+
.withLabels<String, String>(labels)
48+
.addNewOwnerReference()
49+
.withController(true)
50+
.withKind("ShinyProxy")
51+
.withApiVersion("openanalytics.eu/v1")
52+
.withName(shinyProxy.metadata.name)
53+
.withUid(shinyProxy.metadata.uid)
54+
.endOwnerReference()
55+
.endMetadata()
56+
.withNewSpec()
57+
.withIngressClassName("nginx")
58+
.addNewRule()
59+
.withHost(shinyProxy.fqdn)
60+
.withNewHttp()
61+
.addAllToPaths(createPaths(shinyProxy, latestInstance))
62+
.endHttp()
63+
.endRule()
64+
.endSpec()
65+
.build()
66+
//@formatter:on
67+
68+
val createdIngress = kubeClient.inNamespace(shinyProxy.metadata.namespace).resource(ingressDefinition).createOrReplace()
69+
logger.debug { "${shinyProxy.logPrefix()} [Component/Ingress] Created ${createdIngress.metadata.name}" }
70+
}
71+
72+
private fun createPaths(shinyProxy: ShinyProxy, latestInstance: ShinyProxyInstance): ArrayList<HTTPIngressPath> {
73+
val res = arrayListOf(createPathV1(shinyProxy, shinyProxy.subPath, latestInstance))
74+
75+
for (instance in shinyProxy.status.instances) {
76+
val path = shinyProxy.subPath + instance.hashOfSpec + "/"
77+
res.add(createPathV1(shinyProxy, path, instance))
78+
}
79+
80+
return res
81+
}
82+
83+
private fun createPathV1(shinyProxy: ShinyProxy, path: String, shinyProxyInstance: ShinyProxyInstance): HTTPIngressPath {
84+
//@formatter:off
85+
val builder = HTTPIngressPathBuilder()
86+
.withPathType("Prefix")
87+
.withPath(path)
88+
.withNewBackend()
89+
.withNewService()
90+
.withName(ResourceNameFactory.createNameForService(shinyProxy, shinyProxyInstance))
91+
.withNewPort()
92+
.withNumber(80)
93+
.endPort()
94+
.endService()
95+
.endBackend()
96+
//@formatter:on
97+
98+
return builder.build()
99+
}
100+
101+
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/LabelFactory.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ object LabelFactory {
5353
const val APP_LABEL_VALUE = "shinyproxy"
5454
const val NAME_LABEL = "openanalytics.eu/sp-resource-name"
5555
const val INSTANCE_LABEL = "openanalytics.eu/sp-instance"
56+
const val LATEST_INSTANCE_LABEL = "openanalytics.eu/sp-latest-instance"
5657
const val PROXIED_APP = "openanalytics.eu/sp-proxied-app"
57-
const val INGRESS_IS_LATEST = "openanalytics.eu/ingress-is-latest"
5858

5959
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/ResourceNameFactory.kt‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,8 @@ object ResourceNameFactory {
4646
return "sp-${shinyProxy.metadata.name}-rs-${shinyProxyInstance.hashOfSpec}".take(KUBE_RESOURCE_NAME_MAX_LENGTH)
4747
}
4848

49-
fun createNameForIngress(shinyProxy: ShinyProxy, routeName: String, shinyProxyInstance: ShinyProxyInstance): String {
50-
return listOf("sp", shinyProxy.metadata.name, "ing", routeName, shinyProxyInstance.hashOfSpec).filter { it.isNotEmpty() }.joinToString("-").take(KUBE_RESOURCE_NAME_MAX_LENGTH)
51-
}
52-
53-
fun createNameForMetadataIngress(shinyProxy: ShinyProxy): String {
54-
return "sp-${shinyProxy.metadata.name}-ing-metadata".take(KUBE_RESOURCE_NAME_MAX_LENGTH)
49+
fun createNameForIngress(shinyProxy: ShinyProxy): String {
50+
return "sp-${shinyProxy.metadata.name}-ing".take(KUBE_RESOURCE_NAME_MAX_LENGTH)
5551
}
5652

5753
}

src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/IIngressController.kt renamed to src/main/kotlin/eu/openanalytics/shinyproxyoperator/controller/IRecyclableChecker.kt

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,13 @@
1818
* You should have received a copy of the Apache License
1919
* along with this program. If not, see <http://www.apache.org/licenses/>
2020
*/
21-
package eu.openanalytics.shinyproxyoperator.ingress
21+
package eu.openanalytics.shinyproxyoperator.controller
2222

23-
import eu.openanalytics.shinyproxyoperator.controller.ResourceRetriever
2423
import eu.openanalytics.shinyproxyoperator.crd.ShinyProxy
2524
import eu.openanalytics.shinyproxyoperator.crd.ShinyProxyInstance
2625

27-
interface IIngressController {
26+
interface IRecyclableChecker {
2827

29-
fun reconcile(resourceRetriever: ResourceRetriever, shinyProxy: ShinyProxy)
28+
fun isInstanceRecyclable(shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance): Boolean
3029

31-
fun onRemoveInstance(resourceRetriever: ResourceRetriever, shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance)
32-
33-
fun stop()
34-
fun reconcileMetadataEndpoint(shinyProxy: ShinyProxy, force: Boolean = false)
3530
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/controller/IngressController.kt‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/**
2+
* ShinyProxy-Operator
3+
*
4+
* Copyright (C) 2021-2022 Open Analytics
5+
*
6+
* ===========================================================================
7+
*
8+
* This program is free software: you can redistribute it and/or modify
9+
* it under the terms of the Apache License as published by
10+
* The Apache Software Foundation, either version 2 of the License, or
11+
* (at your option) any later version.
12+
*
13+
* This program is distributed in the hope that it will be useful,
14+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
15+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16+
* Apache License for more details.
17+
*
18+
* You should have received a copy of the Apache License
19+
* along with this program. If not, see <http://www.apache.org/licenses/>
20+
*/
21+
package eu.openanalytics.shinyproxyoperator.controller
22+
23+
import eu.openanalytics.shinyproxyoperator.components.LabelFactory
24+
import eu.openanalytics.shinyproxyoperator.crd.ShinyProxy
25+
import eu.openanalytics.shinyproxyoperator.crd.ShinyProxyInstance
26+
import eu.openanalytics.shinyproxyoperator.components.IngressFactory
27+
import io.fabric8.kubernetes.api.model.apps.ReplicaSet
28+
import io.fabric8.kubernetes.api.model.networking.v1.Ingress
29+
import io.fabric8.kubernetes.api.model.networking.v1.IngressList
30+
import io.fabric8.kubernetes.client.dsl.MixedOperation
31+
import io.fabric8.kubernetes.client.dsl.Resource
32+
import io.fabric8.kubernetes.client.readiness.Readiness
33+
import mu.KotlinLogging
34+
35+
class IngressController(
36+
ingressClient: MixedOperation<Ingress, IngressList, Resource<Ingress>>
37+
) {
38+
39+
private val logger = KotlinLogging.logger {}
40+
private val ingressFactory = IngressFactory(ingressClient)
41+
42+
fun reconcile(resourceRetriever: ResourceRetriever, shinyProxy: ShinyProxy) {
43+
reconcileLatestInstance(resourceRetriever, shinyProxy)
44+
}
45+
46+
fun onRemoveInstance(resourceRetriever: ResourceRetriever, shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance) {
47+
reconcileLatestInstance(resourceRetriever, shinyProxy)
48+
}
49+
50+
private fun reconcileLatestInstance(resourceRetriever: ResourceRetriever, shinyProxy: ShinyProxy) {
51+
val latestInstance = shinyProxy.status.latestInstance() ?: return
52+
53+
val ingresses = resourceRetriever.getIngressByLabels(LabelFactory.labelsForShinyProxy(shinyProxy), shinyProxy.metadata.namespace)
54+
val mustBeUpdated = ingresses.isEmpty()
55+
|| ingresses[0].metadata?.labels?.get(LabelFactory.LATEST_INSTANCE_LABEL) != latestInstance.hashOfSpec
56+
|| ingresses[0].spec?.rules?.get(0)?.http?.paths?.size != shinyProxy.status.instances.size + 1
57+
58+
if (mustBeUpdated) {
59+
val replicaSet = getReplicaSet(resourceRetriever, shinyProxy, latestInstance)
60+
if (replicaSet == null) {
61+
logger.warn { "${shinyProxy.logPrefix(latestInstance)} [Component/Ingress] Cannot reconcile Ingress since it has no ReplicaSet - probably this resource is being deleted" }
62+
return
63+
}
64+
if (!Readiness.getInstance().isReady(replicaSet)) {
65+
logger.warn { "${shinyProxy.logPrefix(latestInstance)} [Component/Ingress] Cannot reconcile Ingress since the corresponding ReplicaSet is not ready yet - it is probably being created" }
66+
return
67+
}
68+
// ReplicaSet exists and is ready -> time to create ingress
69+
// By only creating the ingress now, we ensure no 502 bad gateways are generated
70+
logger.debug { "${shinyProxy.logPrefix(latestInstance)} [Component/Ingress] Reconciling" }
71+
ingressFactory.createOrReplaceIngress(shinyProxy, latestInstance)
72+
}
73+
}
74+
75+
private fun getReplicaSet(resourceRetriever: ResourceRetriever, shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance): ReplicaSet? {
76+
val replicaSets = resourceRetriever.getReplicaSetByLabels(LabelFactory.labelsForShinyProxyInstance(shinyProxy, shinyProxyInstance), shinyProxy.metadata.namespace)
77+
if (replicaSets.isEmpty()) {
78+
return null
79+
}
80+
return replicaSets[0]
81+
}
82+
83+
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/controller/RecyclableChecker.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import java.util.concurrent.TimeUnit
3434

3535
class RecyclableChecker(
3636
private val podRetriever: PodRetriever,
37-
) {
37+
) : IRecyclableChecker {
3838

3939
private val logger = KotlinLogging.logger {}
4040
private val client: OkHttpClient = OkHttpClient.Builder()
@@ -46,7 +46,7 @@ class RecyclableChecker(
4646

4747
data class Response(@JsonProperty("isRecyclable") val isRecyclable: Boolean, @JsonProperty("activeConnections") val activeConnections: Int)
4848

49-
fun isInstanceRecyclable(shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance): Boolean {
49+
override fun isInstanceRecyclable(shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance): Boolean {
5050
val pods = podRetriever.getShinyProxyPods(shinyProxy, shinyProxyInstance)
5151

5252
for (pod in pods) {

0 commit comments

Comments
 (0)