Skip to content

Commit 852558c

Browse files
LEDfanLEDfan
committed
Ref #27576: add metadata endpoint, fixes and update tests
1 parent bc8c3d2 commit 852558c

13 files changed

Lines changed: 411 additions & 231 deletions

File tree

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/Operator.kt‎

Lines changed: 35 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -150,12 +150,12 @@ class Operator(client: NamespacedKubernetesClient? = null,
150150
replicaSetListener = ResourceListener(sendChannel, this.client.inAnyNamespace().apps().replicaSets())
151151
serviceListener = ResourceListener(sendChannel, this.client.inAnyNamespace().services())
152152
configMapListener = ResourceListener(sendChannel, this.client.inAnyNamespace().configMaps())
153-
ingressController = IngressController(sendChannel, this.client, this.client.inAnyNamespace().network().v1().ingresses())
153+
ingressController = IngressController(sendChannel, this.client.inAnyNamespace(), this.client.inAnyNamespace().network().ingress())
154154
} else {
155155
replicaSetListener = ResourceListener(sendChannel, this.client.inNamespace(namespace).apps().replicaSets())
156156
serviceListener = ResourceListener(sendChannel, this.client.inNamespace(namespace).services())
157157
configMapListener = ResourceListener(sendChannel, this.client.inNamespace(namespace).configMaps())
158-
ingressController = IngressController(sendChannel, this.client, this.client.inNamespace(namespace).network().v1().ingresses())
158+
ingressController = IngressController(sendChannel, this.client.inNamespace(namespace), this.client.inNamespace(namespace).network().ingress())
159159
}
160160
}
161161

@@ -164,39 +164,58 @@ class Operator(client: NamespacedKubernetesClient? = null,
164164
*/
165165
val shinyProxyController = ShinyProxyController(channel, this.client, shinyProxyClient, ingressController, podRetriever, reconcileListener)
166166

167-
fun prepare(): Pair<ResourceRetriever, Lister<ShinyProxy>> {
168-
logger.info { "Starting background processes of ShinyProxy Operator" }
167+
private fun _checkCrdExists(name: String, shortName: String) {
169168
try {
170-
if (client.apiextensions().v1().customResourceDefinitions().withName("shinyproxies.openanalytics.eu").get() == null) {
169+
if (client.apiextensions().v1().customResourceDefinitions().withName(name).get() == null) {
171170
println()
172171
println()
173-
println("ERROR: the CustomResourceDefinition (CRD) of the Operator does not exist!")
174-
println("The name of the CRD is 'shinyproxies.openanalytics.eu'")
172+
println("ERROR: the CustomResourceDefinition (CRD) '${shortName}' does not exist!")
173+
println("The name of the CRD is '${name}'")
175174
println("Create the CRD first, before starting the operator")
176175
println()
177176
println("Exiting in 10 seconds because of the above error")
178-
Thread.sleep(10000) // sleep 10 seconds to make it easier to find this error by an sysadmin
177+
Thread.sleep(10000) // sleep 10 seconds to make it easier to find this error by a sysadmin
179178

180179
exitProcess(2)
181180
}
182181
} catch (e: KubernetesClientException) {
183182
println()
184183
println()
185-
println("Warning: could not check whether ShinyProxy CRD exits.")
184+
println("Warning: could not check whether $shortName CRD exits.")
186185
println("This is normal when the ServiceAccount of the operator does not have permission to access CRDs (at cluster scope).")
187186
println("If you get an unexpected error after this message, make sure that the CRD exists.")
188187
println()
189188
println()
190189
}
191190

192-
val shinyProxyLister = Lister(shinyProxyListener.start())
193-
val replicaSetLister = Lister(replicaSetListener.start(shinyProxyLister))
194-
val serviceLister = Lister(serviceListener.start(shinyProxyLister))
195-
val configMapLister = Lister(configMapListener.start(shinyProxyLister))
196-
val ingressLister = Lister(ingressController.start(shinyProxyLister))
197-
val resourceRetriever = ResourceRetriever(replicaSetLister, configMapLister, serviceLister, ingressLister)
191+
}
192+
193+
fun prepare(): Pair<ResourceRetriever, Lister<ShinyProxy>> {
194+
logger.info { "Starting background processes of ShinyProxy Operator" }
198195

199-
return resourceRetriever to shinyProxyLister
196+
_checkCrdExists("shinyproxies.openanalytics.eu", "ShinyProxy")
197+
_checkCrdExists("routegroups.zalando.org", "RouteGroup")
198+
199+
try {
200+
val shinyProxyLister = Lister(shinyProxyListener.start())
201+
val replicaSetLister = Lister(replicaSetListener.start(shinyProxyLister))
202+
val serviceLister = Lister(serviceListener.start(shinyProxyLister))
203+
val configMapLister = Lister(configMapListener.start(shinyProxyLister))
204+
val ingressLister = Lister(ingressController.start(shinyProxyLister))
205+
val resourceRetriever = ResourceRetriever(replicaSetLister, configMapLister, serviceLister, ingressLister)
206+
return resourceRetriever to shinyProxyLister
207+
} catch (e: KubernetesClientException) {
208+
println()
209+
println()
210+
println("Error during starting up. Please check if all CRDs exists (see above).")
211+
println("Exiting in 10 seconds because of the above error")
212+
println()
213+
e.printStackTrace()
214+
println()
215+
println()
216+
Thread.sleep(10000) // sleep 10 seconds to make it easier to find this error by a sysadmin
217+
exitProcess(3)
218+
}
200219
}
201220

202221
suspend fun run(resourceRetriever: ResourceRetriever, shinyProxyLister: Lister<ShinyProxy>) {

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/LabelFactory.kt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,13 @@ object LabelFactory {
4242
)
4343
}
4444

45+
fun labelsForShinyProxy(shinyProxy: ShinyProxy): Map<String, String> {
46+
return mapOf(
47+
APP_LABEL to APP_LABEL_VALUE,
48+
NAME_LABEL to shinyProxy.metadata.name
49+
)
50+
}
51+
4552
const val APP_LABEL = "app"
4653
const val APP_LABEL_VALUE = "shinyproxy"
4754
const val NAME_LABEL = "openanalytics.eu/sp-resource-name"

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/ResourceNameFactory.kt‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,11 @@ object ResourceNameFactory {
4747
}
4848

4949
fun createNameForIngress(shinyProxy: ShinyProxy, routeName: String, shinyProxyInstance: ShinyProxyInstance): String {
50-
return "sp-${shinyProxy.metadata.name}-ing-${routeName}-${shinyProxyInstance.hashOfSpec}".take(KUBE_RESOURCE_NAME_MAX_LENGTH)
50+
return listOf("sp", shinyProxy.metadata.name, "ing", routeName, shinyProxyInstance.hashOfSpec).filter { it.isNotEmpty() }.joinToString("-").take(KUBE_RESOURCE_NAME_MAX_LENGTH)
51+
}
52+
53+
fun createNameForMetadataIngress(shinyProxy: ShinyProxy): String {
54+
return "sp-${shinyProxy.metadata.name}-ing-metadata".take(KUBE_RESOURCE_NAME_MAX_LENGTH)
5155
}
5256

5357
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/controller/ShinyProxyController.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ class ShinyProxyController(private val channel: Channel<ShinyProxyEvent>,
172172
logger.debug { "${shinyProxy.logPrefix()} Trying to update status (attempt ${i}/5)" }
173173
tryUpdateStatus()
174174
logger.debug { "${shinyProxy.logPrefix()} Status successfully updated" }
175+
ingressController.reconcileMetadataEndpoint(refreshShinyProxy(shinyProxy), true)
175176
return
176177
} catch (e: KubernetesClientException) {
177178
logger.warn(e) { "${shinyProxy.logPrefix()} Update of status not succeeded (attempt ${i}/5)" }

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/IIngressController.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,5 @@ interface IIngressController {
3131
fun onRemoveInstance(resourceRetriever: ResourceRetriever, shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance)
3232

3333
fun stop()
34+
fun reconcileMetadataEndpoint(shinyProxy: ShinyProxy, force: Boolean = false)
3435
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressController.kt‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ package eu.openanalytics.shinyproxyoperator.ingress.skipper
2222

2323
import eu.openanalytics.shinyproxyoperator.components.LabelFactory
2424
import eu.openanalytics.shinyproxyoperator.components.LabelFactory.INGRESS_IS_LATEST
25+
import eu.openanalytics.shinyproxyoperator.components.ResourceNameFactory
2526
import eu.openanalytics.shinyproxyoperator.controller.ResourceRetriever
2627
import eu.openanalytics.shinyproxyoperator.controller.ShinyProxyEvent
2728
import eu.openanalytics.shinyproxyoperator.crd.ShinyProxy
@@ -30,7 +31,7 @@ import eu.openanalytics.shinyproxyoperator.ingress.IIngressController
3031
import io.fabric8.kubernetes.api.model.apps.ReplicaSet
3132
import io.fabric8.kubernetes.api.model.networking.v1.Ingress
3233
import io.fabric8.kubernetes.api.model.networking.v1.IngressList
33-
import io.fabric8.kubernetes.client.KubernetesClient
34+
import io.fabric8.kubernetes.client.NamespacedKubernetesClient
3435
import io.fabric8.kubernetes.client.dsl.MixedOperation
3536
import io.fabric8.kubernetes.client.dsl.Resource
3637
import io.fabric8.kubernetes.client.informers.cache.Indexer
@@ -41,7 +42,7 @@ import mu.KotlinLogging
4142

4243
class IngressController(
4344
channel: SendChannel<ShinyProxyEvent>,
44-
private val kubernetesClient: KubernetesClient,
45+
private val kubernetesClient: NamespacedKubernetesClient,
4546
ingressClient: MixedOperation<Ingress, IngressList, Resource<Ingress>>
4647
) : IIngressController {
4748

@@ -50,9 +51,12 @@ class IngressController(
5051

5152
// Note: do not move this to the DiContainer since it is a Skipper-specific implementation
5253
private val ingressListener = IngressListener(channel, kubernetesClient, ingressClient)
53-
54+
private val routeGroupClient = kubernetesClient.resources(RouteGroup::class.java)
55+
private val metadataIngressFactory = MetadataRouteGroupFactory(routeGroupClient)
56+
private val routeGroupListener = RouteGroupListener(this, routeGroupClient)
5457

5558
fun start(shinyProxyLister: Lister<ShinyProxy>): Indexer<Ingress> {
59+
routeGroupListener.start(shinyProxyLister)
5660
return ingressListener.start(shinyProxyLister)
5761
}
5862

@@ -69,6 +73,7 @@ class IngressController(
6973
if (failed) {
7074
throw RuntimeException("One or more ingresses failed to reconcile")
7175
}
76+
reconcileMetadataEndpoint(shinyProxy,false)
7277
}
7378

7479
override fun onRemoveInstance(resourceRetriever: ResourceRetriever, shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance) {
@@ -84,11 +89,13 @@ class IngressController(
8489
private fun reconcileSingleInstance(resourceRetriever: ResourceRetriever, shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance) {
8590
val ingresses = resourceRetriever.getIngressByLabels(LabelFactory.labelsForShinyProxyInstance(shinyProxy, shinyProxyInstance), shinyProxy.metadata.namespace)
8691

87-
val mustBeUpdated = if (ingresses.isEmpty()) {
92+
val mustBeUpdated = if (ingresses.size < 3) {
8893
true
8994
} else {
9095
// if the label indicating this is the latest is different from the actual state -> reconcile
91-
ingresses[0].metadata.labels[INGRESS_IS_LATEST]?.toBoolean() != shinyProxyInstance.isLatestInstance
96+
ingresses[0].metadata.labels[INGRESS_IS_LATEST]?.toBoolean() != shinyProxyInstance.isLatestInstance ||
97+
ingresses[1].metadata.labels[INGRESS_IS_LATEST]?.toBoolean() != shinyProxyInstance.isLatestInstance ||
98+
ingresses[2].metadata.labels[INGRESS_IS_LATEST]?.toBoolean() != shinyProxyInstance.isLatestInstance
9299
}
93100

94101
if (mustBeUpdated) {
@@ -116,4 +123,11 @@ class IngressController(
116123
return replicaSets[0]
117124
}
118125

126+
override fun reconcileMetadataEndpoint(shinyProxy: ShinyProxy, force: Boolean) {
127+
val existingObject = routeGroupClient.inNamespace(shinyProxy.metadata.namespace).withName(ResourceNameFactory.createNameForMetadataIngress(shinyProxy)).get()
128+
if (existingObject == null || force) {
129+
metadataIngressFactory.createOrReplaceRouteGroup(shinyProxy)
130+
}
131+
}
132+
119133
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressFactory.kt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,8 @@ class IngressFactory(private val kubeClient: KubernetesClient) {
8484
return if (isLatest) {
8585
mapOf(
8686
"" to createRoute(true, hashOfSpec, shinyProxy, "True()"),
87-
"cookie-override" to createRoute(true, hashOfSpec, shinyProxy, """Cookie("sp-instance-override", "$hashOfSpec") && Weight(20)""""),
88-
"query-override" to createRoute(true, hashOfSpec, shinyProxy, """QueryParam("sp_instance_override", "$hashOfSpec") && Weight(20)""""),
87+
"cookie-override" to createRoute(false, hashOfSpec, shinyProxy, """Cookie("sp-instance-override", "$hashOfSpec") && Weight(20)"""),
88+
"query-override" to createRoute(false, hashOfSpec, shinyProxy, """QueryParam("sp_instance_override", "$hashOfSpec") && Weight(20)"""),
8989
)
9090
} else {
9191
mapOf(
@@ -96,7 +96,7 @@ class IngressFactory(private val kubeClient: KubernetesClient) {
9696
}
9797
}
9898

99-
private fun createRoute(isLatest: Boolean, hashOfSpec: String, shinyProxy: ShinyProxy, predicate: String): Map<String, String> {
99+
private fun createRoute(isDefaultRoute: Boolean, hashOfSpec: String, shinyProxy: ShinyProxy, predicate: String): Map<String, String> {
100100
val security = if (Operator.getOperatorInstance().disableSecureCookies) {
101101
""
102102
} else {
@@ -114,14 +114,14 @@ class IngressFactory(private val kubeClient: KubernetesClient) {
114114
"""setRequestHeader("X-ShinyProxy-Instance", "$hashOfSpec")""" +
115115
""" -> """ +
116116
"""setRequestHeader("X-ShinyProxy-Latest-Instance", "${shinyProxy.hashOfCurrentSpec}")""" +
117-
if (isLatest) {
117+
if (isDefaultRoute) {
118118
""" -> """ +
119119
"""appendResponseHeader("Set-Cookie", "sp-instance=$hashOfSpec; $security Path=$cookiePath")"""
120120
} else {
121121
""
122122
} +
123123
""" -> """ +
124-
"""appendResponseHeader("Set-Cookie", "sp-latest-instance=${shinyProxy.hashOfCurrentSpec}; $security Path=$cookiePath")"""
124+
"""appendResponseHeader("Set-Cookie", "sp-latest-instance=${shinyProxy.hashOfCurrentSpec}; $security Path=$cookiePath")""",
125125
)
126126

127127
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressListener.kt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,6 @@ class IngressListener(private val channel: SendChannel<ShinyProxyEvent>,
7777

7878
private suspend fun enqueueResource(shinyProxyLister: Lister<ShinyProxy>, trigger: String, resource: Ingress) {
7979
val replicaSetOwnerReference = getShinyProxyOwnerRefByKind(resource, "ReplicaSet") ?: return
80-
// TODO namespace
8180
val replicaSet = kubernetesClient.apps().replicaSets().inNamespace(resource.metadata.namespace).withName(replicaSetOwnerReference.name).get()
8281
if (replicaSet == null) {
8382
logger.warn { "[${resource.kind}] [${resource.metadata.namespace}/${resource.metadata.name}] Cannot find owner (ReplicaSet) for this resource - probably the resource is being deleted" }

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/MetadataRouteGroupFactory.kt‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/**
2+
* ShinyProxy-Operator
3+
*
4+
* Copyright (C) 2021 Open Analytics
5+
*
6+
* ===========================================================================
7+
*
8+
* This program is free software: you can redistribute it and/or modify
9+
* it under the terms of the Apache License as published by
10+
* The Apache Software Foundation, either version 2 of the License, or
11+
* (at your option) any later version.
12+
*
13+
* This program is distributed in the hope that it will be useful,
14+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
15+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16+
* Apache License for more details.
17+
*
18+
* You should have received a copy of the Apache License
19+
* along with this program. If not, see <http://www.apache.org/licenses/>
20+
*/
21+
package eu.openanalytics.shinyproxyoperator.ingress.skipper
22+
23+
import com.fasterxml.jackson.databind.ObjectMapper
24+
import eu.openanalytics.shinyproxyoperator.components.LabelFactory
25+
import eu.openanalytics.shinyproxyoperator.components.ResourceNameFactory
26+
import eu.openanalytics.shinyproxyoperator.crd.ShinyProxy
27+
import io.fabric8.kubernetes.api.model.KubernetesResourceList
28+
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder
29+
import io.fabric8.kubernetes.client.dsl.MixedOperation
30+
import io.fabric8.kubernetes.client.dsl.Resource
31+
import mu.KotlinLogging
32+
33+
34+
class MetadataRouteGroupFactory(private val routeGroupClient: MixedOperation<RouteGroup, KubernetesResourceList<RouteGroup>, Resource<RouteGroup>>) {
35+
36+
private val logger = KotlinLogging.logger {}
37+
private val objectMapper = ObjectMapper()
38+
39+
fun createOrReplaceRouteGroup(shinyProxy: ShinyProxy) {
40+
val metadata = objectMapper.writeValueAsString(mapOf("instances" to shinyProxy.status.instances)).replace("\"", "\\\"")
41+
42+
val path = if (shinyProxy.subPath != "") {
43+
shinyProxy.subPath + "/operator/metadata"
44+
} else {
45+
"/operator/metadata"
46+
}
47+
48+
val routeGroupSpec = RouteGroupSpec(
49+
hosts = listOf(shinyProxy.fqdn),
50+
backends = listOf(Backend("shunt", "shunt")),
51+
defaultBackends = listOf(BackendName("shunt")),
52+
routes = listOf(
53+
Route(
54+
pathSubtree = path,
55+
filters = listOf(
56+
"""setResponseHeader("Content-Type","application/json")""",
57+
"""inlineContent("$metadata")""",
58+
"""status(200)"""
59+
),
60+
backends = listOf(BackendName("shunt"))
61+
)
62+
)
63+
)
64+
65+
//@formatter:off
66+
val routeGroup = RouteGroup()
67+
routeGroup.spec = routeGroupSpec
68+
routeGroup.metadata = ObjectMetaBuilder()
69+
.withNamespace(shinyProxy.metadata.namespace)
70+
.withName(ResourceNameFactory.createNameForMetadataIngress(shinyProxy))
71+
.withLabels<String, String>(LabelFactory.labelsForShinyProxy(shinyProxy))
72+
.addNewOwnerReference()
73+
.withController(true)
74+
.withKind("ShinyProxy")
75+
.withApiVersion("openanalytics.eu/v1")
76+
.withName(shinyProxy.metadata.name)
77+
.withUid(shinyProxy.metadata.uid)
78+
.endOwnerReference()
79+
.build()
80+
//@formatter:on
81+
82+
val createdRouteGroup = routeGroupClient.inNamespace(shinyProxy.metadata.namespace).createOrReplace(routeGroup)
83+
logger.info { "[Component/RouteGroup] Created ${createdRouteGroup.metadata.name}" }
84+
}
85+
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/RouteGroup.kt‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/**
2+
* ShinyProxy-Operator
3+
*
4+
* Copyright (C) 2021 Open Analytics
5+
*
6+
* ===========================================================================
7+
*
8+
* This program is free software: you can redistribute it and/or modify
9+
* it under the terms of the Apache License as published by
10+
* The Apache Software Foundation, either version 2 of the License, or
11+
* (at your option) any later version.
12+
*
13+
* This program is distributed in the hope that it will be useful,
14+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
15+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16+
* Apache License for more details.
17+
*
18+
* You should have received a copy of the Apache License
19+
* along with this program. If not, see <http://www.apache.org/licenses/>
20+
*/
21+
package eu.openanalytics.shinyproxyoperator.ingress.skipper
22+
23+
import io.fabric8.kubernetes.api.model.Namespaced
24+
import io.fabric8.kubernetes.client.CustomResource
25+
import io.fabric8.kubernetes.model.annotation.Group
26+
import io.fabric8.kubernetes.model.annotation.Version
27+
28+
class RouteGroupStatus()
29+
30+
data class Backend(val name: String, val type: String)
31+
data class BackendName(val backendName: String)
32+
data class Route(val pathSubtree: String, val filters: List<String>, val backends: List<BackendName>)
33+
data class RouteGroupSpec(val hosts: List<String>, val backends: List<Backend>, val defaultBackends: List<BackendName>, val routes: List<Route>)
34+
35+
// TODO create tests
36+
@Version("v1")
37+
@Group("zalando.org")
38+
class RouteGroup: CustomResource<RouteGroupSpec, RouteGroupStatus>(), Namespaced {
39+
40+
}
41+

0 commit comments

Comments
 (0)