Ref #29447: add more tests for pre-initalization

LEDfan · LEDfan · commit 57f71d361e21 · 2024-03-01T09:31:43.000+01:00
diff --git a/src/test/java/eu/openanalytics/containerproxy/test/helpers/ShinyProxyClient.java b/src/test/java/eu/openanalytics/containerproxy/test/helpers/ShinyProxyClient.java
@@ -24,6 +24,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.openanalytics.containerproxy.model.runtime.ProxyStatus;
 import eu.openanalytics.containerproxy.util.Retrying;
+import okhttp3.Call;
 import okhttp3.MediaType;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
@@ -233,4 +234,7 @@ public String getBaseUrl() {
         return baseUrl;
     }
 
+    public Call newCall(Request request) {
+        return client.newCall(request);
+    }
 }
diff --git a/src/test/java/eu/openanalytics/containerproxy/test/helpers/ShinyProxyInstance.java b/src/test/java/eu/openanalytics/containerproxy/test/helpers/ShinyProxyInstance.java
@@ -52,6 +52,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
+import java.util.concurrent.atomic.AtomicReference;
 
 
 public class ShinyProxyInstance implements AutoCloseable {
@@ -98,10 +99,24 @@ public ShinyProxyInstance(String configFileName, int port, String usernameAndPas
 
             client = new ShinyProxyClient(usernameAndPassword, port);
 
+            AtomicReference<Throwable> exception = new AtomicReference<>();
             thread = new Thread(() -> app = application.run());
+            thread.setUncaughtExceptionHandler((thread, ex) -> {
+                exception.set(ex);
+            });
             thread.start();
 
-            boolean available = Retrying.retry((c, m) -> client.checkAlive(), 60_000, "ShinyProxyInstance available", 1, true);
+            boolean available = Retrying.retry((c, m) -> {
+                if (exception.get() != null) {
+                    logger.warn("Exception during startup");
+                    return true;
+                }
+                return client.checkAlive();
+            }, 60_000, "ShinyProxyInstance available", 1, true);
+            Throwable ex = exception.get();
+            if (ex != null) {
+                throw ex;
+            }
             if (!available) {
                 throw new TestHelperException("ShinyProxy did not become available!");
             } else {
@@ -152,7 +167,7 @@ public void enableCleanup() {
     }
 
     public void stopAllApps() {
-        for (Proxy proxy: proxyService.getAllProxies()) {
+        for (Proxy proxy : proxyService.getAllProxies()) {
             proxyService.stopProxy(null, proxy, true).run();
         }
     }
diff --git a/src/test/java/eu/openanalytics/containerproxy/test/helpers/TestProxySharingScaler.java b/src/test/java/eu/openanalytics/containerproxy/test/helpers/TestProxySharingScaler.java
@@ -55,4 +55,8 @@ public IDelegateProxyStore getDelegateProxyStore() {
         return delegateProxyStore;
     }
 
+    public ISeatStore getSeatStore() {
+        return seatStore;
+    }
+
 }
diff --git a/src/test/java/eu/openanalytics/containerproxy/test/proxy/TestPreInitialization.java b/src/test/java/eu/openanalytics/containerproxy/test/proxy/TestPreInitialization.java
@@ -20,22 +20,36 @@
  */
 package eu.openanalytics.containerproxy.test.proxy;
 
+import com.google.common.base.Throwables;
 import eu.openanalytics.containerproxy.backend.dispatcher.proxysharing.IDelegateProxyStore;
 import eu.openanalytics.containerproxy.backend.dispatcher.proxysharing.ProxySharingScaler;
+import eu.openanalytics.containerproxy.backend.dispatcher.proxysharing.Seat;
 import eu.openanalytics.containerproxy.backend.dispatcher.proxysharing.SeatIdKey;
 import eu.openanalytics.containerproxy.backend.dispatcher.proxysharing.store.DelegateProxy;
+import eu.openanalytics.containerproxy.backend.dispatcher.proxysharing.store.DelegateProxyStatus;
+import eu.openanalytics.containerproxy.backend.dispatcher.proxysharing.store.ISeatStore;
+import eu.openanalytics.containerproxy.model.runtime.Container;
 import eu.openanalytics.containerproxy.model.runtime.Proxy;
 import eu.openanalytics.containerproxy.model.runtime.runtimevalues.PublicPathKey;
 import eu.openanalytics.containerproxy.model.runtime.runtimevalues.TargetIdKey;
 import eu.openanalytics.containerproxy.test.helpers.ContainerSetup;
 import eu.openanalytics.containerproxy.test.helpers.ShinyProxyInstance;
+import eu.openanalytics.containerproxy.test.helpers.TestHelperException;
 import eu.openanalytics.containerproxy.test.helpers.TestProxySharingScaler;
 import eu.openanalytics.containerproxy.util.Retrying;
+import okhttp3.Request;
+import okhttp3.Response;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
+import org.mandas.docker.client.DockerClient;
+import org.mandas.docker.client.builder.jersey.JerseyDockerClientBuilder;
+import org.mandas.docker.client.exceptions.DockerCertificateException;
+import org.mandas.docker.client.exceptions.DockerException;
 
+import java.io.IOException;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -57,26 +71,40 @@ private static Stream<Arguments> backends() {
     @MethodSource("backends")
     public void simpleTest(String backend, Map<String, String> properties) {
         try (ContainerSetup containerSetup = new ContainerSetup(backend)) {
-            try (ShinyProxyInstance inst = new ShinyProxyInstance("application-test-pre-initialization.yml", properties, true)) {
-                ProxySharingScaler proxySharingScaler = inst.getBean("proxySharingScaler_simpleTest", ProxySharingScaler.class);
+            try (ShinyProxyInstance inst = new ShinyProxyInstance("application-test-pre-initialization-1.yml", properties, true)) {
+                TestProxySharingScaler proxySharingScaler = inst.getBean("proxySharingScaler_myApp", TestProxySharingScaler.class);
+                IDelegateProxyStore delegateProxyStore = proxySharingScaler.getDelegateProxyStore();
+                ISeatStore seatStore = proxySharingScaler.getSeatStore();
                 inst.enableCleanup();
-                String id = inst.client.startProxy("simpleTest");
+                String id = inst.client.startProxy("myApp");
                 Proxy proxy = inst.proxyService.getProxy(id);
                 inst.client.testProxyReachable(proxy.getTargetId());
 
                 // target id should be different from proxy id
                 Assertions.assertNotEquals(proxy.getTargetId(), proxy.getId());
                 Assertions.assertEquals("/api/route/" + proxy.getTargetId() + "/", proxy.getRuntimeValue(PublicPathKey.inst));
                 Assertions.assertEquals(proxy.getTargetId(), proxy.getRuntimeValue(TargetIdKey.inst));
-                Assertions.assertNotNull(proxy.getRuntimeValue(SeatIdKey.inst));
 
+                // check DelegateProxy
+                DelegateProxy delegateProxy = delegateProxyStore.getDelegateProxy(proxy.getTargetId());
+                Assertions.assertNotNull(delegateProxy);
+                Assertions.assertEquals(1, delegateProxy.getSeatIds().size());
+                Assertions.assertEquals(proxy.getRuntimeValue(SeatIdKey.inst), delegateProxy.getSeatIds().stream().findFirst().get());
+
+                // check seat
+                Seat seat = seatStore.getSeat(proxy.getRuntimeValue(SeatIdKey.inst));
+                Assertions.assertEquals(proxy.getRuntimeValue(SeatIdKey.inst), seat.getId());
+                Assertions.assertEquals(delegateProxy.getProxy().getId(), seat.getDelegateProxyId());
+                Assertions.assertEquals(proxy.getId(), seat.getDelegatingProxyId());
+
+                // should have scaled-up
                 waitUntilNoPendingSeats(proxySharingScaler);
                 Assertions.assertEquals(1, proxySharingScaler.getNumClaimedSeats());
                 Assertions.assertEquals(1, proxySharingScaler.getNumUnclaimedSeats());
 
                 // start an additional app
                 Instant start = Instant.now();
-                String id2 = inst.client.startProxy("simpleTest");
+                String id2 = inst.client.startProxy("myApp");
                 Proxy proxy2 = inst.proxyService.getProxy(id2);
                 inst.client.testProxyReachable(proxy2.getTargetId());
 
@@ -113,9 +141,9 @@ public void simpleTest(String backend, Map<String, String> properties) {
     public void testSetPublicPathPrefix(String backend, Map<String, String> properties) {
         ProxySharingScaler.setPublicPathPrefix("/my/custom/path/");
         try (ContainerSetup containerSetup = new ContainerSetup(backend)) {
-            try (ShinyProxyInstance inst = new ShinyProxyInstance("application-test-pre-initialization.yml", properties, true)) {
+            try (ShinyProxyInstance inst = new ShinyProxyInstance("application-test-pre-initialization-1.yml", properties, true)) {
                 inst.enableCleanup();
-                String id = inst.client.startProxy("simpleTest");
+                String id = inst.client.startProxy("myApp");
                 Proxy proxy = inst.proxyService.getProxy(id);
                 inst.client.testProxyReachable(proxy.getTargetId());
 
@@ -134,13 +162,13 @@ public void testSetPublicPathPrefix(String backend, Map<String, String> properti
     @MethodSource("backends")
     public void testNoContainerReUse(String backend, Map<String, String> properties) {
         try (ContainerSetup containerSetup = new ContainerSetup(backend)) {
-            try (ShinyProxyInstance inst = new ShinyProxyInstance("application-test-pre-initialization.yml", properties, true)) {
+            try (ShinyProxyInstance inst = new ShinyProxyInstance("application-test-pre-initialization-2.yml", properties, true)) {
                 inst.enableCleanup();
-                TestProxySharingScaler proxySharingScaler = inst.getBean("proxySharingScaler_noReUse", TestProxySharingScaler.class);
+                TestProxySharingScaler proxySharingScaler = inst.getBean("proxySharingScaler_myApp", TestProxySharingScaler.class);
                 proxySharingScaler.disableCleanup();
                 IDelegateProxyStore delegateProxyStore = proxySharingScaler.getDelegateProxyStore();
 
-                String id = inst.client.startProxy("noReUse");
+                String id = inst.client.startProxy("myApp");
                 Proxy proxy = inst.proxyService.getProxy(id);
                 inst.client.testProxyReachable(proxy.getTargetId());
 
@@ -173,9 +201,66 @@ public void testNoContainerReUse(String backend, Map<String, String> properties)
         }
     }
 
+    @ParameterizedTest
+    @MethodSource("backends")
+    public void testReUseWithMultipleSeats(String backend, Map<String, String> properties) {
+        TestHelperException ex = Assertions.assertThrows(TestHelperException.class,
+            () -> new ShinyProxyInstance("application-test-pre-initialization-3.yml", properties, true),
+            "Provided parameter values are not allowed");
+
+        Throwable rootCause = Throwables.getRootCause(ex);
+        Assertions.assertInstanceOf(IllegalStateException.class, rootCause);
+        Assertions.assertEquals("Spec myApp is invalid: when allow-container-re-use is disabled, seatsPerContainer must be exactly 1", rootCause.getMessage());
+    }
+
+    @Test
+    public void testDelegateProxyCrashed() throws DockerCertificateException, DockerException, InterruptedException, IOException {
+        DockerClient dockerClient = new JerseyDockerClientBuilder().fromEnv().build();
+        try (ContainerSetup containerSetup = new ContainerSetup("docker")) {
+            try (ShinyProxyInstance inst = new ShinyProxyInstance("application-test-pre-initialization-1.yml", Map.of("proxy.container-backend", "docker"), true)) {
+                inst.enableCleanup();
+                String id = inst.client.startProxy("myApp");
+                Proxy proxy = inst.proxyService.getProxy(id);
+                inst.client.testProxyReachable(proxy.getTargetId());
+
+                // delete underlying container
+                TestProxySharingScaler proxySharingScaler = inst.getBean("proxySharingScaler_myApp", TestProxySharingScaler.class);
+                proxySharingScaler.disableCleanup();
+                IDelegateProxyStore delegateProxyStore = proxySharingScaler.getDelegateProxyStore();
+                DelegateProxy delegateProxy = delegateProxyStore.getDelegateProxy(proxy.getTargetId());
+                Container container = delegateProxy.getProxy().getContainer(0);
+                dockerClient.removeContainer(container.getId(), DockerClient.RemoveContainerParam.forceKill());
+
+                // try to access proxy
+                Request request = new Request.Builder()
+                    .get()
+                    .url(inst.client.getBaseUrl() + "/api/route/" + proxy.getTargetId() + "/")
+                    .build();
+
+                try (Response response = inst.client.newCall(request).execute()) {
+                    Assertions.assertEquals(503, response.code());
+                    Assertions.assertEquals("{\"status\":\"fail\",\"data\":\"app_crashed\"}", response.body().string());
+                }
+
+                // DelegateProxy should have no seats and marked for removal
+                delegateProxy = delegateProxyStore.getDelegateProxy(proxy.getTargetId());
+                Assertions.assertEquals(DelegateProxyStatus.ToRemove, delegateProxy.getDelegateProxyStatus());
+                Assertions.assertTrue( delegateProxy.getSeatIds().isEmpty());
+
+                // enable cleanup
+                proxySharingScaler.enableCleanup();
+
+                // old proxy should get cleaned up
+                waitUntilNumberOfDelegateProxies(proxySharingScaler, 1);
+                DelegateProxy newDelegateProxy = delegateProxyStore.getAllDelegateProxies().stream().findFirst().get();
+                // should be a different proxy, old one should have been removed
+                Assertions.assertNotEquals(newDelegateProxy.getProxy().getId(), delegateProxy.getProxy().getId());
+                Assertions.assertNotEquals(newDelegateProxy.getProxy().getTargetId(), delegateProxy.getProxy().getTargetId());
+                Assertions.assertEquals(1, newDelegateProxy.getSeatIds().size());
+            }
+        }
+    }
 
-    // TODO test re-use with more than one seats
-    // TODO test crashed
     // TODO test scaleDownDelay
     // TODO test minimumSeatsAvailable
     // TODO test config change
diff --git a/src/test/resources/application-test-pre-initialization-1.yml b/src/test/resources/application-test-pre-initialization-1.yml
@@ -0,0 +1,34 @@
+spring:
+  session:
+    store-type: none
+  data:
+    redis:
+      repositories:
+        enabled: false
+proxy:
+  authentication: simple
+  heartbeat-timeout: -1
+  default-stop-proxy-on-logout: false
+
+  users:
+    - name: demo
+      password: demo
+      groups:
+        - group1
+        - group2
+    - name: demo2
+      password: demo2
+
+  docker:
+    url: http://localhost:2375
+
+
+  specs:
+    - id: myApp
+      minimum-seats-available: 1
+      container-specs:
+        - image: "openanalytics/shinyproxy-integration-test-app"
+          cmd: [ "R", "-e", "shinyproxy::run_01_hello()" ]
+          port-mapping:
+            - name: default
+              port: 3838
diff --git a/src/test/resources/application-test-pre-initialization-2.yml b/src/test/resources/application-test-pre-initialization-2.yml
@@ -24,15 +24,7 @@ proxy:
 
 
   specs:
-    - id: simpleTest
-      minimum-seats-available: 1
-      container-specs:
-        - image: "openanalytics/shinyproxy-integration-test-app"
-          cmd: [ "R", "-e", "shinyproxy::run_01_hello()" ]
-          port-mapping:
-            - name: default
-              port: 3838
-    - id: noReUse
+    - id: myApp
       minimum-seats-available: 1
       allow-container-re-use: false
       container-specs:
diff --git a/src/test/resources/application-test-pre-initialization-3.yml b/src/test/resources/application-test-pre-initialization-3.yml
@@ -0,0 +1,36 @@
+spring:
+  session:
+    store-type: none
+  data:
+    redis:
+      repositories:
+        enabled: false
+proxy:
+  authentication: simple
+  heartbeat-timeout: -1
+  default-stop-proxy-on-logout: false
+
+  users:
+    - name: demo
+      password: demo
+      groups:
+        - group1
+        - group2
+    - name: demo2
+      password: demo2
+
+  docker:
+    url: http://localhost:2375
+
+
+  specs:
+    - id: myApp
+      minimum-seats-available: 1
+      seats-per-container: 2
+      allow-container-re-use: false
+      container-specs:
+        - image: "openanalytics/shinyproxy-integration-test-app"
+          cmd: [ "R", "-e", "shinyproxy::run_01_hello()" ]
+          port-mapping:
+            - name: default
+              port: 3838

Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,7 @@`
`24`	`24`	`import com.fasterxml.jackson.databind.ObjectMapper;`
`25`	`25`	`import eu.openanalytics.containerproxy.model.runtime.ProxyStatus;`
`26`	`26`	`import eu.openanalytics.containerproxy.util.Retrying;`
	`27`	`+import okhttp3.Call;`
`27`	`28`	`import okhttp3.MediaType;`
`28`	`29`	`import okhttp3.OkHttpClient;`
`29`	`30`	`import okhttp3.Request;`
`@@ -233,4 +234,7 @@ public String getBaseUrl() {`
`233`	`234`	`return baseUrl;`
`234`	`235`	`}`
`235`	`236`
	`237`	`+ public Call newCall(Request request) {`
	`238`	`+ return client.newCall(request);`
	`239`	`+ }`
`236`	`240`	`}`
Original file line number	Diff line number	Diff line change
`@@ -55,4 +55,8 @@ public IDelegateProxyStore getDelegateProxyStore() {`
`55`	`55`	`return delegateProxyStore;`
`56`	`56`	`}`
`57`	`57`
	`58`	`+ public ISeatStore getSeatStore() {`
	`59`	`+ return seatStore;`
	`60`	`+ }`
	`61`	`+`
`58`	`62`	`}`