Skip to content

Commit 2d676dd

Browse files
LEDfanLEDfan
committed
Introduce Chaos testing for Kube API interaction
1 parent e0e0e07 commit 2d676dd

3 files changed

Lines changed: 146 additions & 65 deletions

File tree

‎src/test/kotlin/eu/openanalytics/shinyproxyoperator/MainIntegrationTest.kt‎

Lines changed: 46 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@ class MainIntegrationTest : IntegrationTestBase() {
4747

4848
@Test
4949
fun `ingress should not be created before ReplicaSet is ready`() =
50-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
50+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener ->
51+
if (chaosEnabled) return@setup // this test depends on timings and therefore it does not work with chaos enabled
5152
// 1. create a SP instance
5253
val spTestInstance = ShinyProxyTestInstance(
5354
namespace,
@@ -67,7 +68,7 @@ class MainIntegrationTest : IntegrationTestBase() {
6768
// let the operator handle one event
6869
operator.shinyProxyController.receiveAndHandleEvent()
6970

70-
val replicaSets = namespacedClient.apps().replicaSets().list().items
71+
val replicaSets = stableClient.apps().replicaSets().list().items
7172
if (replicaSets.size == 0) {
7273
// if replicaset is not created -> continue handling events
7374
continue
@@ -79,10 +80,10 @@ class MainIntegrationTest : IntegrationTestBase() {
7980
if (!Readiness.isReady(replicaSet)) {
8081
// replicaset not ready
8182
// -> Service should not yet be created
82-
assertEquals(0, namespacedClient.services().list().items.size)
83+
assertEquals(0, stableClient.services().list().items.size)
8384

8485
// -> Ingress should not yet be created
85-
assertEquals(0, namespacedClient.network().ingresses().list().items.size)
86+
assertEquals(0, stableClient.network().ingresses().list().items.size)
8687

8788
// -> Latest marker should not yet be set
8889
val sp = spTestInstance.retrieveInstance()
@@ -112,7 +113,7 @@ class MainIntegrationTest : IntegrationTestBase() {
112113

113114
@Test
114115
fun `simple test namespaces`() =
115-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
116+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, _, operator, reconcileListener ->
116117
// 1. create a SP instance
117118
val spTestInstance = ShinyProxyTestInstance(
118119
namespace,
@@ -139,7 +140,8 @@ class MainIntegrationTest : IntegrationTestBase() {
139140

140141
@Test
141142
fun `operator should re-create removed resources`() =
142-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
143+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener ->
144+
if (chaosEnabled) return@setup // this test depends on timings and therefore it does not work with chaos enabled
143145
// 1. create a SP instance
144146
val spTestInstance = ShinyProxyTestInstance(
145147
namespace,
@@ -165,7 +167,7 @@ class MainIntegrationTest : IntegrationTestBase() {
165167

166168
// 5. Delete Replicaset -> reconcile -> assert it is still ok
167169
executeAsyncAfter100ms {
168-
namespacedClient.apps().replicaSets()
170+
stableClient.apps().replicaSets()
169171
.withName("sp-${sp.metadata.name}-rs-${spTestInstance.hash}".take(63)).delete()
170172
logger.info { "Deleted ReplicaSet" }
171173
}
@@ -175,7 +177,7 @@ class MainIntegrationTest : IntegrationTestBase() {
175177

176178
// 6. Delete ConfigMap -> reconcile -> assert it is still ok
177179
executeAsyncAfter100ms {
178-
namespacedClient.configMaps().withName("sp-${sp.metadata.name}-cm-${spTestInstance.hash}".take(63))
180+
stableClient.configMaps().withName("sp-${sp.metadata.name}-cm-${spTestInstance.hash}".take(63))
179181
.delete()
180182
logger.info { "Deleted ConfigMap" }
181183
}
@@ -185,7 +187,7 @@ class MainIntegrationTest : IntegrationTestBase() {
185187

186188
// 7. Delete Service -> reconcile -> assert it is still ok
187189
executeAsyncAfter100ms {
188-
namespacedClient.services().withName("sp-${sp.metadata.name}-svc-${spTestInstance.hash}".take(63))
190+
stableClient.services().withName("sp-${sp.metadata.name}-svc-${spTestInstance.hash}".take(63))
189191
.delete()
190192
logger.info { "Deleted Service" }
191193
}
@@ -195,7 +197,7 @@ class MainIntegrationTest : IntegrationTestBase() {
195197

196198
// 8. Delete Ingress -> reconcile -> assert it is still ok
197199
executeAsyncAfter100ms {
198-
namespacedClient.network().ingress()
200+
stableClient.network().ingress()
199201
.withName("sp-${sp.metadata.name}-ing-${spTestInstance.hash}".take(63)).delete()
200202
logger.info { "Deleted Ingress" }
201203
}
@@ -209,11 +211,11 @@ class MainIntegrationTest : IntegrationTestBase() {
209211

210212
@Test
211213
fun `sp in other namespaced should be ignored when using namespaced mode`() =
212-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
214+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener ->
213215
// 1. create a SP instance in other namespace
214216
val spTestInstance = ShinyProxyTestInstance(
215217
"itest-2",
216-
namespacedClient.inNamespace("itest-2"),
218+
stableClient.inNamespace("itest-2"),
217219
shinyProxyClient,
218220
"simple_config.yaml",
219221
reconcileListener
@@ -230,14 +232,14 @@ class MainIntegrationTest : IntegrationTestBase() {
230232
delay(20000)
231233

232234
// assert that there are no ReplicaSets created
233-
assertEquals(0, namespacedClient.apps().replicaSets().list().items.size)
235+
assertEquals(0, stableClient.apps().replicaSets().list().items.size)
234236

235237
job.cancel()
236238
}
237239

238240
@Test
239241
fun `simple test with PodTemplateSpec patches`() =
240-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
242+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener ->
241243
// 1. create a SP instance in other namespace
242244
val spTestInstance = ShinyProxyTestInstance(
243245
namespace,
@@ -269,7 +271,7 @@ class MainIntegrationTest : IntegrationTestBase() {
269271
spTestInstance.assertConfigMapIsCorrect(sp)
270272

271273
// check replicaset
272-
val replicaSets = namespacedClient.inNamespace(namespace).apps().replicaSets().list().items
274+
val replicaSets = stableClient.inNamespace(namespace).apps().replicaSets().list().items
273275
assertEquals(1, replicaSets.size)
274276
val replicaSet = replicaSets[0]
275277
val templateSpec = replicaSet.spec.template.spec
@@ -310,7 +312,8 @@ class MainIntegrationTest : IntegrationTestBase() {
310312

311313
@Test
312314
fun `update without apps running`() =
313-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
315+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, _, operator, reconcileListener ->
316+
if (chaosEnabled) return@setup // this test depends on timings and therefore it does not work with chaos enabled
314317
// 1. create a SP instance
315318
val spTestInstanceOriginal = ShinyProxyTestInstance(
316319
namespace,
@@ -364,7 +367,8 @@ class MainIntegrationTest : IntegrationTestBase() {
364367

365368
@Test
366369
fun `update with apps running`() =
367-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
370+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener ->
371+
if (chaosEnabled) return@setup // this test depends on timings and therefore it does not work with chaos enabled
368372
// 1. create a SP instance
369373
val spTestInstanceOriginal = ShinyProxyTestInstance(
370374
namespace,
@@ -414,11 +418,11 @@ class MainIntegrationTest : IntegrationTestBase() {
414418
spTestInstanceUpdated.assertInstanceIsCorrect(2, true)
415419

416420
// 9. kill app on older instance
417-
namespacedClient.pods().delete(getPodsForInstance(spTestInstanceOriginal.hash)?.items)
421+
stableClient.pods().delete(getPodsForInstance(spTestInstanceOriginal.hash)?.items)
418422

419423
// 10. wait for delete to happen
420424
while (getPodsForInstance(spTestInstanceOriginal.hash)?.items?.isNotEmpty() == true
421-
|| namespacedClient.pods().withName("sp-${spUpdated.metadata.name}-pod-${spUpdated.hashOfCurrentSpec}")
425+
|| stableClient.pods().withName("sp-${spUpdated.metadata.name}-pod-${spUpdated.hashOfCurrentSpec}")
422426
.get() != null
423427
) {
424428
delay(1000)
@@ -440,7 +444,7 @@ class MainIntegrationTest : IntegrationTestBase() {
440444

441445
@Test
442446
fun `apps running in different namespaces`() =
443-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
447+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, _, operator, reconcileListener ->
444448
// 1. create a SP instance
445449
val spTestInstance = ShinyProxyTestInstance(
446450
namespace,
@@ -481,7 +485,7 @@ class MainIntegrationTest : IntegrationTestBase() {
481485

482486
@Test
483487
fun `simple test clustered`() =
484-
setup(Mode.CLUSTERED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
488+
setup(Mode.CLUSTERED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener ->
485489
// 1. create a SP instance
486490
val spTestInstance = ShinyProxyTestInstance(
487491
namespace,
@@ -507,7 +511,7 @@ class MainIntegrationTest : IntegrationTestBase() {
507511
// 5. create instance in other namespace
508512
val spTestInstance2 = ShinyProxyTestInstance(
509513
"itest-2",
510-
namespacedClient.inNamespace("itest-2"),
514+
stableClient.inNamespace("itest-2"),
511515
shinyProxyClient,
512516
"simple_config.yaml",
513517
reconcileListener
@@ -525,7 +529,7 @@ class MainIntegrationTest : IntegrationTestBase() {
525529

526530
@Test
527531
fun `configuration with subpath not ending in slash`() =
528-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
532+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener ->
529533
// 1. create a SP instance
530534
val spTestInstance = ShinyProxyTestInstance(
531535
namespace,
@@ -546,14 +550,14 @@ class MainIntegrationTest : IntegrationTestBase() {
546550
spTestInstance.waitForOneReconcile()
547551

548552
// 4. assert correctness
549-
assertCorrectnessWithSubPath(spTestInstance, namespacedClient, namespace)
553+
assertCorrectnessWithSubPath(spTestInstance, stableClient, namespace)
550554

551555
job.cancel()
552556
}
553557

554558
@Test
555559
fun `configuration with subpath ending in slash`() =
556-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
560+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener ->
557561
// 1. create a SP instance
558562
val spTestInstance = ShinyProxyTestInstance(
559563
namespace,
@@ -574,7 +578,7 @@ class MainIntegrationTest : IntegrationTestBase() {
574578
spTestInstance.waitForOneReconcile()
575579

576580
// 4. assert correctness
577-
assertCorrectnessWithSubPath(spTestInstance, namespacedClient, namespace)
581+
assertCorrectnessWithSubPath(spTestInstance, stableClient, namespace)
578582

579583
job.cancel()
580584
}
@@ -703,7 +707,7 @@ class MainIntegrationTest : IntegrationTestBase() {
703707
fun `simple test namespaces non-secure cookie`() = setup(
704708
Mode.NAMESPACED,
705709
disableSecureCookies = true
706-
) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
710+
) { namespace, shinyProxyClient, namespacedClient, _, operator, reconcileListener ->
707711
// 1. create a SP instance
708712
val spTestInstance = ShinyProxyTestInstance(
709713
namespace,
@@ -764,7 +768,8 @@ class MainIntegrationTest : IntegrationTestBase() {
764768
*/
765769
@Test
766770
fun `reconcile of old instance should not update latestermarker and therefore delete old instance`() =
767-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
771+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, _, operator, reconcileListener ->
772+
if (chaosEnabled) return@setup // this test depends on timings and therefore it does not work with chaos enabled
768773
// 1. create a SP instance
769774
val spTestInstanceOriginal = ShinyProxyTestInstance(
770775
namespace,
@@ -850,7 +855,8 @@ class MainIntegrationTest : IntegrationTestBase() {
850855
fun `may no re-create instance after remove`() = setup(
851856
Mode.NAMESPACED,
852857
disableSecureCookies = true
853-
) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
858+
) { namespace, shinyProxyClient, namespacedClient, _, operator, reconcileListener ->
859+
if (chaosEnabled) return@setup // this test depends on timings and therefore it does not work with chaos enabled
854860
// 1. create a SP instance
855861
val spTestInstance = ShinyProxyTestInstance(
856862
namespace,
@@ -881,7 +887,7 @@ class MainIntegrationTest : IntegrationTestBase() {
881887
GlobalScope.launch {
882888
repeat(10) {
883889
delay(10)
884-
logger.debug { "Trying to trigger bug, by triggering renoncile with old status" }
890+
logger.debug { "Trying to trigger bug, by triggering reconcile with old status" }
885891
operator.shinyProxyController.reconcileSingleShinyProxyInstance(sp, instance)
886892
}
887893
}
@@ -900,7 +906,7 @@ class MainIntegrationTest : IntegrationTestBase() {
900906
fun `restore old config version`() =
901907
// idea of test: launch instance A, update config to get instance B, and the update config again
902908
// using the same config as A, resulting in instance A' (which is the same instance as A, as A was never removed!)
903-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
909+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener ->
904910
// 1. create a SP instance
905911
val instanceA = ShinyProxyTestInstance(
906912
namespace,
@@ -964,7 +970,7 @@ class MainIntegrationTest : IntegrationTestBase() {
964970
instanceAPrime.waitForOneReconcile()
965971

966972
// 11. wait for delete to happen
967-
while (namespacedClient.pods().withName(ResourceNameFactory.createNameForReplicaSet(spB, spB.status.instances[0])).get() != null) {
973+
while (stableClient.pods().withName(ResourceNameFactory.createNameForReplicaSet(spB, spB.status.instances[0])).get() != null) {
968974
delay(1000)
969975
logger.debug { "Pod still exists!" }
970976
}
@@ -986,7 +992,8 @@ class MainIntegrationTest : IntegrationTestBase() {
986992

987993
// see #25154
988994
@Test
989-
fun `latest marker and ingress should be created in a single, atomic step`() = setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
995+
fun `latest marker and ingress should be created in a single, atomic step`() = setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener ->
996+
if (chaosEnabled) return@setup // this test depends on timings and therefore it does not work with chaos enabled
990997
// 1. create a SP instance
991998
val spTestInstance = ShinyProxyTestInstance(
992999
namespace,
@@ -1005,7 +1012,7 @@ class MainIntegrationTest : IntegrationTestBase() {
10051012
// let the operator handle one event
10061013
operator.shinyProxyController.receiveAndHandleEvent()
10071014

1008-
val replicaSets = namespacedClient.apps().replicaSets().list().items
1015+
val replicaSets = stableClient.apps().replicaSets().list().items
10091016
if (replicaSets.size == 0) {
10101017
// if replicaset is not created -> continue handling events
10111018
continue
@@ -1022,7 +1029,7 @@ class MainIntegrationTest : IntegrationTestBase() {
10221029

10231030
// 4. check state
10241031
// A) when the ReplicaSet is ready, the service should already exist
1025-
val services = namespacedClient.services().list().items
1032+
val services = stableClient.services().list().items
10261033
assertEquals(1, services.size)
10271034

10281035
// B) at this point the latestMarker should not be set yet
@@ -1032,7 +1039,7 @@ class MainIntegrationTest : IntegrationTestBase() {
10321039
assertEquals(false, sp.status.instances[0].isLatestInstance)
10331040

10341041
// C) at this point the ingress should not exist yet
1035-
var ingresses = namespacedClient.inNamespace(namespace).network().ingresses().list().items
1042+
var ingresses = stableClient.inNamespace(namespace).network().ingresses().list().items
10361043
assertEquals(0, ingresses.size)
10371044

10381045
// Test starts here:
@@ -1048,7 +1055,7 @@ class MainIntegrationTest : IntegrationTestBase() {
10481055
assertEquals(true, sp.status.instances[0].isLatestInstance)
10491056

10501057
// B) at this point the ingress should exist
1051-
ingresses = namespacedClient.inNamespace(namespace).network().ingresses().list().items
1058+
ingresses = stableClient.inNamespace(namespace).network().ingresses().list().items
10521059
assertEquals(1, ingresses.size)
10531060

10541061
}
@@ -1057,9 +1064,9 @@ class MainIntegrationTest : IntegrationTestBase() {
10571064
// see #25161
10581065
@Test
10591066
fun `operator should properly handle 409 conflicts by replacing the resource`() =
1060-
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, operator, reconcileListener ->
1067+
setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener ->
10611068
// 1. create conflicting resources
1062-
namespacedClient.load(this.javaClass.getResourceAsStream("/configs/conflict.yaml")).createOrReplace()
1069+
stableClient.load(this.javaClass.getResourceAsStream("/configs/conflict.yaml")).createOrReplace()
10631070

10641071
// 2. create a SP instance
10651072
val spTestInstance = ShinyProxyTestInstance(

0 commit comments

Comments
 (0)