Ref #33873: check health of docker swarm container

Author: LEDfan

src/main/java/eu/openanalytics/containerproxy/backend/docker/AbstractDockerBackend.java

@@ -20,6 +20,8 @@
  */
 package eu.openanalytics.containerproxy.backend.docker;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.openanalytics.containerproxy.ContainerProxyException;
 import eu.openanalytics.containerproxy.backend.AbstractContainerBackend;
 import eu.openanalytics.containerproxy.model.runtime.Container;
@@ -31,9 +33,12 @@
 import org.mandas.docker.client.DockerCertificates;
 import org.mandas.docker.client.DockerClient;
 import org.mandas.docker.client.LogStream;
+import org.mandas.docker.client.ObjectMapperProvider;
 import org.mandas.docker.client.builder.jersey.JerseyDockerClientBuilder;
 import org.mandas.docker.client.exceptions.DockerCertificateException;
 import org.mandas.docker.client.exceptions.DockerException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.inject.Inject;
 import java.io.IOException;
@@ -64,6 +69,8 @@ public abstract class AbstractDockerBackend extends AbstractContainerBackend {
     protected Integer portRangeTo;
 
     private final ScheduledExecutorService releasePortExecutor = Executors.newSingleThreadScheduledExecutor();
+    private final ObjectMapper objectMapper = new ObjectMapper();
+    private final Logger logger = LoggerFactory.getLogger(getClass());
 
     @Override
     public void initialize() throws ContainerProxyException {
@@ -144,4 +151,13 @@ protected void releasePort(String ownerId) {
         releasePortExecutor.schedule(() -> portAllocator.release(ownerId), 10, TimeUnit.SECONDS);
     }
 
+    protected String toJson(Object object) {
+        try {
+            return objectMapper.writeValueAsString(object);
+        } catch (JsonProcessingException e) {
+            logger.warn("Error while generating json", e);
+            return null;
+        }
+    }
+
 }

src/main/java/eu/openanalytics/containerproxy/backend/docker/DockerEngineBackend.java

@@ -20,8 +20,6 @@
  */
 package eu.openanalytics.containerproxy.backend.docker;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.openanalytics.containerproxy.ContainerFailedToStartException;
 import eu.openanalytics.containerproxy.event.NewProxyEvent;
 import eu.openanalytics.containerproxy.model.runtime.Container;
@@ -315,15 +313,14 @@ public boolean isProxyHealthy(Proxy proxy) {
                 ContainerInfo info = dockerClient.inspectContainer(container.getId());
                 ContainerState state = info.state();
                 if (!state.running() || !state.status().equals("running")) {
-                    ObjectMapper objectMapper = new ObjectMapper();
-                    slog.warn(proxy, "Docker container failed: container not running, state reported by docker: " + objectMapper.writeValueAsString(state));
+                    slog.warn(proxy, "Docker container failed: container not running, state reported by docker: " + toJson(state));
                     return false;
                 }
                 return true;
             } catch (ContainerNotFoundException e) {
                 slog.warn(proxy, "Docker container failed: container does not exist");
                 return false;
-            } catch (DockerException | InterruptedException | JsonProcessingException e) {
+            } catch (DockerException | InterruptedException e) {
                 throw new RuntimeException(e);
             }
         }

src/main/java/eu/openanalytics/containerproxy/backend/docker/DockerSwarmBackend.java

@@ -44,6 +44,7 @@
 import org.mandas.docker.client.LogStream;
 import org.mandas.docker.client.exceptions.DockerException;
 import org.mandas.docker.client.exceptions.ServiceNotFoundException;
+import org.mandas.docker.client.exceptions.TaskNotFoundException;
 import org.mandas.docker.client.messages.RegistryAuth;
 import org.mandas.docker.client.messages.mount.Mount;
 import org.mandas.docker.client.messages.swarm.DnsConfig;
@@ -53,6 +54,7 @@
 import org.mandas.docker.client.messages.swarm.Reservations;
 import org.mandas.docker.client.messages.swarm.ResourceRequirements;
 import org.mandas.docker.client.messages.swarm.Resources;
+import org.mandas.docker.client.messages.swarm.RestartPolicy;
 import org.mandas.docker.client.messages.swarm.SecretBind;
 import org.mandas.docker.client.messages.swarm.SecretFile;
 import org.mandas.docker.client.messages.swarm.Service;
@@ -314,7 +316,7 @@ public List<ExistingContainerInfo> scanExistingContainers() throws Exception {
         ArrayList<ExistingContainerInfo> containers = new ArrayList<>();
 
         for (Service service : dockerClient.listServices()) {
-           org.mandas.docker.client.messages.swarm.ContainerSpec containerSpec = service.spec().taskTemplate().containerSpec();
+            org.mandas.docker.client.messages.swarm.ContainerSpec containerSpec = service.spec().taskTemplate().containerSpec();
 
             if (containerSpec == null) {
                 continue;
@@ -359,7 +361,34 @@ public List<ExistingContainerInfo> scanExistingContainers() throws Exception {
 
     @Override
     public boolean isProxyHealthy(Proxy proxy) {
-        return true; // TODO
+        for (Container container : proxy.getContainers()) {
+            try {
+                BackendContainerName serviceId = container.getRuntimeObjectOrNull(BackendContainerNameKey.inst);
+                if (serviceId == null) {
+                    slog.warn(proxy, "Docker Swarm container failed: no service id found");
+                    return false;
+                }
+                dockerClient.inspectService(serviceId.getName()); // check service exists
+                List<Task> serviceTask = dockerClient.listTasks(Task.Criteria.builder().serviceName(serviceId.getName()).build());
+                if (serviceTask.isEmpty()) {
+                    slog.warn(proxy, "Docker Swarm container failed: service does not exist");
+                    return false;
+                }
+                Task task = serviceTask.get(0);
+                if (!task.status().state().equals("running")) {
+                    slog.warn(proxy, "Docker Swarm container failed: container not running, state reported by docker: " + toJson(task.status()));
+                    return false;
+                }
+                return true;
+            } catch (ServiceNotFoundException e) {
+                slog.warn(proxy, "Docker Swarm container failed: service does not exist");
+                return false;
+            } catch (DockerException | InterruptedException e) {
+                slog.warn(proxy, e, "Failed to check Docker Swarm container health");
+                return false;
+            }
+        }
+        return true;
     }
 
     @Override