Skip to content

Commit 10596ac

Browse files
LEDfanLEDfan
committed
Fix #30475: add anti affinity
1 parent b60334a commit 10596ac

5 files changed

Lines changed: 276 additions & 0 deletions

File tree

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/PodTemplateSpecFactory.kt‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ class PodTemplateSpecFactory {
112112
.withPeriodSeconds(5)
113113
.endStartupProbe()
114114
.endContainer()
115+
.withAffinity(createAffinity(shinyProxy, shinyProxyInstance))
115116
.withVolumes(VolumeBuilder()
116117
.withName("config-volume")
117118
.withConfigMap(ConfigMapVolumeSourceBuilder()
@@ -125,5 +126,38 @@ class PodTemplateSpecFactory {
125126
return podTemplatePatcher.patch(template, shinyProxy.parsedKubernetesPodTemplateSpecPatches)
126127
}
127128

129+
private fun createAffinity(shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance): Affinity {
130+
if (shinyProxy.antiAffinityRequired) {
131+
//@formatter:off
132+
return AffinityBuilder()
133+
.withNewPodAntiAffinity()
134+
.addNewRequiredDuringSchedulingIgnoredDuringExecution()
135+
.withNewLabelSelector()
136+
.withMatchLabels<String, String>(LabelFactory.labelsForShinyProxyInstance(shinyProxy, shinyProxyInstance))
137+
.endLabelSelector()
138+
.withTopologyKey(shinyProxy.antiAffinityTopologyKey)
139+
.endRequiredDuringSchedulingIgnoredDuringExecution()
140+
.endPodAntiAffinity()
141+
.build()
142+
//@formatter:on
143+
} else {
144+
//@formatter:off
145+
return AffinityBuilder()
146+
.withNewPodAntiAffinity()
147+
.addNewPreferredDuringSchedulingIgnoredDuringExecution()
148+
.withWeight(1)
149+
.withNewPodAffinityTerm()
150+
.withNewLabelSelector()
151+
.withMatchLabels<String, String>(LabelFactory.labelsForShinyProxyInstance(shinyProxy, shinyProxyInstance))
152+
.endLabelSelector()
153+
.withTopologyKey(shinyProxy.antiAffinityTopologyKey)
154+
.endPodAffinityTerm()
155+
.endPreferredDuringSchedulingIgnoredDuringExecution()
156+
.endPodAntiAffinity()
157+
.build()
158+
//@formatter:on
159+
}
160+
}
161+
128162

129163
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/crd/ShinyProxy.kt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,21 @@ class ShinyProxy : CustomResource<JsonNode, ShinyProxyStatus>(), Namespaced {
161161
return@lazy "${metadata.name}-${metadata.namespace}"
162162
}
163163

164+
@get:JsonIgnore
165+
val antiAffinityTopologyKey: String by lazy {
166+
if (spec.get("antiAffinityTopologyKey")?.isTextual == true) {
167+
return@lazy spec.get("antiAffinityTopologyKey").textValue()
168+
}
169+
return@lazy "kubernetes.io/hostname"
170+
}
171+
172+
@get:JsonIgnore
173+
val antiAffinityRequired: Boolean by lazy {
174+
if (spec.get("antiAffinityRequired")?.isBoolean == true) {
175+
return@lazy spec.get("antiAffinityRequired").booleanValue()
176+
}
177+
return@lazy false
178+
}
164179

165180
fun logPrefix(shinyProxyInstance: ShinyProxyInstance): String {
166181
return "[${metadata.namespace}/${metadata.name}/${shinyProxyInstance.hashOfSpec}]"

‎src/test/kotlin/eu/openanalytics/shinyproxyoperator/MainIntegrationTest.kt‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
*/
2121
package eu.openanalytics.shinyproxyoperator
2222

23+
import eu.openanalytics.shinyproxyoperator.components.LabelFactory
2324
import eu.openanalytics.shinyproxyoperator.controller.ShinyProxyEvent
2425
import eu.openanalytics.shinyproxyoperator.controller.ShinyProxyEventType
2526
import eu.openanalytics.shinyproxyoperator.helpers.IntegrationTestBase
@@ -36,6 +37,7 @@ import org.junit.jupiter.api.assertThrows
3637
import kotlin.test.assertEquals
3738
import kotlin.test.assertFalse
3839
import kotlin.test.assertNotNull
40+
import kotlin.test.assertNull
3941
import kotlin.test.assertTrue
4042

4143
class MainIntegrationTest : IntegrationTestBase() {
@@ -904,4 +906,159 @@ class MainIntegrationTest : IntegrationTestBase() {
904906
spTestInstance.assertInstanceIsCorrect()
905907
job.cancel()
906908
}
909+
910+
@Test
911+
fun `operator should have correct antiAffinity defaults`() =
912+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener, _ ->
913+
// 1. create a SP instance
914+
val spTestInstance = ShinyProxyTestInstance(
915+
namespace,
916+
namespacedClient,
917+
shinyProxyClient,
918+
"simple_config.yaml",
919+
reconcileListener
920+
)
921+
val sp = spTestInstance.create()
922+
923+
val (resourceRetriever, shinyProxyLister) = operator.prepare()
924+
925+
// 3. start the operator and let it do it's work
926+
val job = GlobalScope.launch {
927+
operator.run(resourceRetriever, shinyProxyLister)
928+
}
929+
930+
// 4. wait until instance is created
931+
spTestInstance.waitForOneReconcile()
932+
933+
// 5. assert correctness
934+
spTestInstance.assertInstanceIsCorrect()
935+
936+
// 6. check antiAffinity rules
937+
val replicaSets = stableClient.inNamespace(namespace).apps().replicaSets().list().items
938+
assertEquals(1, replicaSets.size)
939+
val replicaSet = replicaSets.firstOrNull { it.metadata.labels[LabelFactory.INSTANCE_LABEL] == spTestInstance.hash }
940+
assertNotNull(replicaSet)
941+
942+
assertNotNull(replicaSet.spec.template.spec.affinity)
943+
assertNotNull(replicaSet.spec.template.spec.affinity.podAntiAffinity)
944+
assertNull(replicaSet.spec.template.spec.affinity.podAffinity)
945+
assertNull(replicaSet.spec.template.spec.affinity.nodeAffinity)
946+
assertEquals(0, replicaSet.spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution.size)
947+
assertNotNull(replicaSet.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution)
948+
assertEquals(1, replicaSet.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.size)
949+
val rule = replicaSet.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0]
950+
assertEquals(1, rule.weight)
951+
assertEquals("kubernetes.io/hostname", rule.podAffinityTerm.topologyKey)
952+
assertEquals(mapOf(
953+
LabelFactory.APP_LABEL to LabelFactory.APP_LABEL_VALUE,
954+
LabelFactory.REALM_ID_LABEL to sp.realmId,
955+
LabelFactory.INSTANCE_LABEL to spTestInstance.hash
956+
), rule.podAffinityTerm.labelSelector.matchLabels)
957+
958+
job.cancel()
959+
}
960+
961+
@Test
962+
fun `operator should have correct antiAffinity when required is true`() =
963+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener, _ ->
964+
// 1. create a SP instance
965+
val spTestInstance = ShinyProxyTestInstance(
966+
namespace,
967+
namespacedClient,
968+
shinyProxyClient,
969+
"affinity_required.yaml",
970+
reconcileListener
971+
)
972+
val sp = spTestInstance.create()
973+
974+
val (resourceRetriever, shinyProxyLister) = operator.prepare()
975+
976+
// 3. start the operator and let it do it's work
977+
val job = GlobalScope.launch {
978+
operator.run(resourceRetriever, shinyProxyLister)
979+
}
980+
981+
// 4. wait until instance is created
982+
spTestInstance.waitForOneReconcile()
983+
984+
// 5. assert correctness
985+
spTestInstance.assertInstanceIsCorrect()
986+
987+
// 6. check antiAffinity rules
988+
val replicaSets = stableClient.inNamespace(namespace).apps().replicaSets().list().items
989+
assertEquals(1, replicaSets.size)
990+
val replicaSet = replicaSets.firstOrNull { it.metadata.labels[LabelFactory.INSTANCE_LABEL] == spTestInstance.hash }
991+
assertNotNull(replicaSet)
992+
993+
assertNotNull(replicaSet.spec.template.spec.affinity)
994+
assertNotNull(replicaSet.spec.template.spec.affinity.podAntiAffinity)
995+
assertNull(replicaSet.spec.template.spec.affinity.podAffinity)
996+
assertNull(replicaSet.spec.template.spec.affinity.nodeAffinity)
997+
assertEquals(0, replicaSet.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.size)
998+
assertEquals(1, replicaSet.spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution.size)
999+
assertNotNull(replicaSet.spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution)
1000+
val rule = replicaSet.spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0]
1001+
assertEquals("kubernetes.io/hostname", rule.topologyKey)
1002+
assertEquals(mapOf(
1003+
LabelFactory.APP_LABEL to LabelFactory.APP_LABEL_VALUE,
1004+
LabelFactory.REALM_ID_LABEL to sp.realmId,
1005+
LabelFactory.INSTANCE_LABEL to spTestInstance.hash
1006+
), rule.labelSelector.matchLabels)
1007+
1008+
job.cancel()
1009+
}
1010+
1011+
1012+
@Test
1013+
fun `operator should have correct antiAffinity when topologyKey is set`() =
1014+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener, _ ->
1015+
// 1. create a SP instance
1016+
val spTestInstance = ShinyProxyTestInstance(
1017+
namespace,
1018+
namespacedClient,
1019+
shinyProxyClient,
1020+
"affinity_toplogykey.yaml",
1021+
reconcileListener
1022+
)
1023+
val sp = spTestInstance.create()
1024+
1025+
val (resourceRetriever, shinyProxyLister) = operator.prepare()
1026+
1027+
// 3. start the operator and let it do it's work
1028+
val job = GlobalScope.launch {
1029+
operator.run(resourceRetriever, shinyProxyLister)
1030+
}
1031+
1032+
// 4. wait until instance is created
1033+
spTestInstance.waitForOneReconcile()
1034+
1035+
// 5. assert correctness
1036+
spTestInstance.assertInstanceIsCorrect()
1037+
1038+
// 6. check antiAffinity rules
1039+
val replicaSets = stableClient.inNamespace(namespace).apps().replicaSets().list().items
1040+
assertEquals(1, replicaSets.size)
1041+
val replicaSet = replicaSets.firstOrNull { it.metadata.labels[LabelFactory.INSTANCE_LABEL] == spTestInstance.hash }
1042+
assertNotNull(replicaSet)
1043+
1044+
assertNotNull(replicaSet.spec.template.spec.affinity)
1045+
assertNotNull(replicaSet.spec.template.spec.affinity.podAntiAffinity)
1046+
assertNull(replicaSet.spec.template.spec.affinity.podAffinity)
1047+
assertNull(replicaSet.spec.template.spec.affinity.nodeAffinity)
1048+
assertEquals(0, replicaSet.spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution.size)
1049+
assertNotNull(replicaSet.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution)
1050+
assertEquals(1, replicaSet.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.size)
1051+
val rule = replicaSet.spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0]
1052+
assertEquals(1, rule.weight)
1053+
assertEquals("example.com/custom-topology-key", rule.podAffinityTerm.topologyKey)
1054+
assertEquals(mapOf(
1055+
LabelFactory.APP_LABEL to LabelFactory.APP_LABEL_VALUE,
1056+
LabelFactory.REALM_ID_LABEL to sp.realmId,
1057+
LabelFactory.INSTANCE_LABEL to spTestInstance.hash
1058+
), rule.podAffinityTerm.labelSelector.matchLabels)
1059+
1060+
job.cancel()
1061+
}
1062+
1063+
9071064
}

‎src/test/resources/configs/affinity_required.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
apiVersion: openanalytics.eu/v1
2+
kind: ShinyProxy
3+
metadata:
4+
name: example-shinyproxy
5+
spec:
6+
antiAffinityTopologyKey: example.com/custom-topology-key
7+
fqdn: itest.local
8+
image: openanalytics/shinyproxy:2.6.1
9+
proxy:
10+
title: Open Analytics Shiny Proxy
11+
logoUrl: http://www.openanalytics.eu/sites/www.openanalytics.eu/themes/oa/logo.png
12+
landingPage: /
13+
heartbeatRate: 10000
14+
heartbeatTimeout: 60000
15+
port: 8080
16+
authentication: simple
17+
containerBackend: kubernetes
18+
kubernetes:
19+
namespace: itest
20+
users:
21+
- name: demo
22+
password: demo
23+
groups: scientists
24+
- name: demo2
25+
password: demo2
26+
groups: mathematicians
27+
specs:
28+
- id: 01_hello
29+
displayName: Hello Application
30+
description: Application which demonstrates the basics of a Shiny app
31+
containerCmd: [ "R", "-e", "shinyproxy::run_01_hello()" ]
32+
containerImage: openanalytics/shinyproxy-demo
33+
- id: 06_tabsets
34+
container-cmd: [ "R", "-e", "shinyproxy::run_06_tabsets()" ]
35+
container-image: openanalytics/shinyproxy-demo

‎src/test/resources/configs/required_affinity.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
apiVersion: openanalytics.eu/v1
2+
kind: ShinyProxy
3+
metadata:
4+
name: example-shinyproxy
5+
spec:
6+
antiAffinityRequired: true
7+
fqdn: itest.local
8+
image: openanalytics/shinyproxy:2.6.1
9+
proxy:
10+
title: Open Analytics Shiny Proxy
11+
logoUrl: http://www.openanalytics.eu/sites/www.openanalytics.eu/themes/oa/logo.png
12+
landingPage: /
13+
heartbeatRate: 10000
14+
heartbeatTimeout: 60000
15+
port: 8080
16+
authentication: simple
17+
containerBackend: kubernetes
18+
kubernetes:
19+
namespace: itest
20+
users:
21+
- name: demo
22+
password: demo
23+
groups: scientists
24+
- name: demo2
25+
password: demo2
26+
groups: mathematicians
27+
specs:
28+
- id: 01_hello
29+
displayName: Hello Application
30+
description: Application which demonstrates the basics of a Shiny app
31+
containerCmd: [ "R", "-e", "shinyproxy::run_01_hello()" ]
32+
containerImage: openanalytics/shinyproxy-demo
33+
- id: 06_tabsets
34+
container-cmd: [ "R", "-e", "shinyproxy::run_06_tabsets()" ]
35+
container-image: openanalytics/shinyproxy-demo

0 commit comments

Comments
 (0)