Skip to content

Commit 693ccc5

Browse files
LEDfanLEDfan
committed
Merge pull request 'Fix #17096: support secrets in Docker swarm' (#59) from feature/17096/swarm_secrets into develop
2 parents 7c9e195 + 2907e85 commit 693ccc5

8 files changed

Lines changed: 531 additions & 16 deletions

File tree

‎pom.xml‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -240,14 +240,29 @@
240240
</dependency>
241241
<!-- Jersey, a dependency of docker-client with wonky version constraints -->
242242
<dependency>
243-
<groupId>org.glassfish.jersey.inject</groupId>
244-
<artifactId>jersey-hk2</artifactId>
245-
<version>2.26</version>
243+
<groupId>org.glassfish.jersey.core</groupId>
244+
<artifactId>jersey-client</artifactId>
245+
<version>2.22</version>
246246
</dependency>
247247
<dependency>
248-
<groupId>org.glassfish.jersey.bundles.repackaged</groupId>
249-
<artifactId>jersey-guava</artifactId>
250-
<version>2.26-b03</version>
248+
<groupId>org.glassfish.jersey.core</groupId>
249+
<artifactId>jersey-common</artifactId>
250+
<version>2.22</version>
251+
</dependency>
252+
<dependency>
253+
<groupId>org.glassfish.jersey.connectors</groupId>
254+
<artifactId>jersey-apache-connector</artifactId>
255+
<version>2.22</version>
256+
</dependency>
257+
<dependency>
258+
<groupId>org.glassfish.jersey.media</groupId>
259+
<artifactId>jersey-media-json-jackson</artifactId>
260+
<version>2.22</version>
261+
</dependency>
262+
<dependency>
263+
<groupId>org.glassfish.hk2</groupId>
264+
<artifactId>hk2-api</artifactId>
265+
<version>2.4.0-b31</version>
251266
</dependency>
252267

253268
<!-- MonetDB, for gathering usage stats (optional) -->

‎src/main/java/eu/openanalytics/containerproxy/backend/docker/DockerEngineBackend.java‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,16 @@
2222

2323
import com.spotify.docker.client.DockerClient.ListContainersParam;
2424
import com.spotify.docker.client.DockerClient.RemoveContainerParam;
25+
import com.spotify.docker.client.exceptions.DockerException;
26+
import com.spotify.docker.client.exceptions.NotFoundException;
2527
import com.spotify.docker.client.messages.Container.PortMapping;
2628
import com.spotify.docker.client.messages.ContainerConfig;
2729
import com.spotify.docker.client.messages.ContainerCreation;
2830
import com.spotify.docker.client.messages.ContainerInfo;
2931
import com.spotify.docker.client.messages.HostConfig;
3032
import com.spotify.docker.client.messages.HostConfig.Builder;
3133
import com.spotify.docker.client.messages.PortBinding;
34+
import com.spotify.docker.client.messages.RegistryAuth;
3235
import eu.openanalytics.containerproxy.model.runtime.Container;
3336
import eu.openanalytics.containerproxy.model.runtime.ExistingContainerInfo;
3437
import eu.openanalytics.containerproxy.model.runtime.Proxy;
@@ -37,6 +40,8 @@
3740
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.RuntimeValueKey;
3841
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.UserIdKey;
3942
import eu.openanalytics.containerproxy.model.spec.ContainerSpec;
43+
import org.slf4j.Logger;
44+
import org.slf4j.LoggerFactory;
4045

4146
import java.net.URI;
4247
import java.net.URL;
@@ -49,10 +54,27 @@
4954

5055
public class DockerEngineBackend extends AbstractDockerBackend {
5156

57+
private static final String PROPERTY_IMG_PULL_POLICY = "image-pull-policy";
58+
59+
private ImagePullPolicy imagePullPolicy;
60+
private final Logger logger = LoggerFactory.getLogger(getClass());
61+
62+
@Override
63+
public void initialize() {
64+
super.initialize();
65+
imagePullPolicy = environment.getProperty(getPropertyPrefix() + PROPERTY_IMG_PULL_POLICY, ImagePullPolicy.class, ImagePullPolicy.IfNotPresent);
66+
}
67+
5268
@Override
5369
protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Exception {
5470
Builder hostConfigBuilder = HostConfig.builder();
55-
71+
72+
if (imagePullPolicy == ImagePullPolicy.Always
73+
|| (imagePullPolicy == ImagePullPolicy.IfNotPresent && !isImagePresent(spec))) {
74+
logger.info("Pulling image {}", spec.getImage());
75+
pullImage(spec);
76+
}
77+
5678
Map<String, List<PortBinding>> portBindings = new HashMap<>();
5779
if (isUseInternalNetwork()) {
5880
// In internal networking mode, we can access container ports directly, no need to bind on host.
@@ -203,6 +225,36 @@ public List<ExistingContainerInfo> scanExistingContainers() throws Exception {
203225

204226
return containers;
205227
}
206-
228+
229+
private boolean isImagePresent(ContainerSpec spec) throws DockerException, InterruptedException {
230+
try {
231+
dockerClient.inspectImage(spec.getImage());
232+
return true;
233+
} catch (NotFoundException ex) {
234+
return false;
235+
}
236+
}
237+
238+
private void pullImage(ContainerSpec spec) throws DockerException, InterruptedException {
239+
if (spec.getDockerRegistryDomain() != null
240+
&& spec.getDockerRegistryUsername() != null
241+
&& spec.getDockerRegistryPassword() != null) {
242+
243+
RegistryAuth registryAuth = RegistryAuth.builder()
244+
.serverAddress(spec.getDockerRegistryDomain())
245+
.username(spec.getDockerRegistryUsername())
246+
.password(spec.getDockerRegistryPassword())
247+
.build();
248+
dockerClient.pull(spec.getImage(), registryAuth, message -> {});
249+
} else {
250+
dockerClient.pull(spec.getImage(), message -> {});
251+
}
252+
}
253+
254+
public enum ImagePullPolicy {
255+
Never,
256+
Always,
257+
IfNotPresent
258+
}
207259

208260
}

‎src/main/java/eu/openanalytics/containerproxy/backend/docker/DockerSwarmBackend.java‎

Lines changed: 82 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,18 @@
2121
package eu.openanalytics.containerproxy.backend.docker;
2222

2323
import com.spotify.docker.client.DockerClient;
24+
import com.spotify.docker.client.exceptions.DockerException;
25+
import com.spotify.docker.client.messages.RegistryAuth;
2426
import com.spotify.docker.client.messages.mount.Mount;
2527
import com.spotify.docker.client.messages.swarm.DnsConfig;
2628
import com.spotify.docker.client.messages.swarm.EndpointSpec;
2729
import com.spotify.docker.client.messages.swarm.NetworkAttachmentConfig;
2830
import com.spotify.docker.client.messages.swarm.PortConfig;
31+
import com.spotify.docker.client.messages.swarm.ResourceRequirements;
32+
import com.spotify.docker.client.messages.swarm.Resources;
33+
import com.spotify.docker.client.messages.swarm.Secret;
34+
import com.spotify.docker.client.messages.swarm.SecretBind;
35+
import com.spotify.docker.client.messages.swarm.SecretFile;
2936
import com.spotify.docker.client.messages.swarm.Service;
3037
import com.spotify.docker.client.messages.swarm.ServiceSpec;
3138
import com.spotify.docker.client.messages.swarm.Task;
@@ -39,6 +46,7 @@
3946
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.RuntimeValueKey;
4047
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.UserIdKey;
4148
import eu.openanalytics.containerproxy.model.spec.ContainerSpec;
49+
import eu.openanalytics.containerproxy.model.spec.DockerSwarmSecret;
4250
import eu.openanalytics.containerproxy.util.Retrying;
4351

4452
import java.net.URI;
@@ -84,6 +92,11 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
8492
}
8593
}
8694

95+
List<SecretBind> secretBinds = new ArrayList<>();
96+
for (DockerSwarmSecret secret : spec.getDockerSwarmSecrets()) {
97+
secretBinds.add(convertSecret(secret));
98+
}
99+
87100
com.spotify.docker.client.messages.swarm.ContainerSpec containerSpec =
88101
com.spotify.docker.client.messages.swarm.ContainerSpec.builder()
89102
.image(spec.getImage())
@@ -92,6 +105,7 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
92105
.env(convertEnv(buildEnv(spec, proxy)))
93106
.dnsConfig(DnsConfig.builder().nameServers(spec.getDns()).build())
94107
.mounts(mounts)
108+
.secrets(secretBinds)
95109
.build();
96110

97111
NetworkAttachmentConfig[] networks = Arrays
@@ -104,12 +118,35 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
104118
networks[networks.length - 1] = NetworkAttachmentConfig.builder().target(spec.getNetwork()).build();
105119
}
106120

121+
Resources.Builder reservationsBuilder = Resources.builder();
122+
// reservations are used by the Docker swarm scheduler
123+
if (spec.getCpuRequest() != null) {
124+
// note: 1 CPU = 1 * 10e8 nanoCpu -> equivalent to --cpus option
125+
reservationsBuilder.nanoCpus((long) (Double.parseDouble(spec.getCpuRequest()) * 10e8));
126+
}
127+
if (spec.getMemoryRequest() != null) {
128+
reservationsBuilder.memoryBytes(memoryToBytes(spec.getMemoryRequest()));
129+
}
130+
131+
Resources.Builder limitsBuilder = Resources.builder();
132+
if (spec.getCpuLimit() != null) {
133+
// note: 1 CPU = 1 * 10e8 nanoCpu -> equivalent to --cpus option
134+
limitsBuilder.nanoCpus((long) (Double.parseDouble(spec.getCpuLimit()) * 10e8));
135+
}
136+
if (spec.getMemoryLimit() != null) {
137+
limitsBuilder.memoryBytes(memoryToBytes(spec.getMemoryLimit()));
138+
}
139+
107140
String serviceName = "sp-service-" + UUID.randomUUID().toString();
108141
ServiceSpec.Builder serviceSpecBuilder = ServiceSpec.builder()
109142
.networks(networks)
110143
.name(serviceName)
111144
.taskTemplate(TaskSpec.builder()
112145
.containerSpec(containerSpec)
146+
.resources(ResourceRequirements.builder()
147+
.reservations(reservationsBuilder.build())
148+
.limits(limitsBuilder.build())
149+
.build())
113150
.build());
114151

115152
List<PortConfig> portsToPublish = new ArrayList<>();
@@ -124,7 +161,21 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
124161
serviceSpecBuilder.endpointSpec(EndpointSpec.builder().ports(portsToPublish).build());
125162
}
126163

127-
String serviceId = dockerClient.createService(serviceSpecBuilder.build()).id();
164+
String serviceId;
165+
if (spec.getDockerRegistryDomain() != null
166+
&& spec.getDockerRegistryUsername() != null
167+
&& spec.getDockerRegistryPassword() != null) {
168+
169+
RegistryAuth registryAuth = RegistryAuth.builder()
170+
.serverAddress(spec.getDockerRegistryDomain())
171+
.username(spec.getDockerRegistryUsername())
172+
.password(spec.getDockerRegistryPassword())
173+
.build();
174+
serviceId = dockerClient.createService(serviceSpecBuilder.build(), registryAuth).id();
175+
} else {
176+
serviceId = dockerClient.createService(serviceSpecBuilder.build()).id();
177+
}
178+
128179
container.getParameters().put(PARAM_SERVICE_ID, serviceId);
129180

130181
// Give the service some time to start up and launch a container.
@@ -133,7 +184,9 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
133184
Task serviceTask = dockerClient
134185
.listTasks(Task.Criteria.builder().serviceName(serviceName).build())
135186
.stream().findAny().orElseThrow(() -> new IllegalStateException("Swarm service has no tasks"));
136-
container.setId(serviceTask.status().containerStatus().containerId());
187+
if (serviceTask.status().containerStatus() != null) {
188+
container.setId(serviceTask.status().containerStatus().containerId());
189+
}
137190
} catch (Exception e) {
138191
throw new RuntimeException("Failed to inspect swarm service tasks", e);
139192
}
@@ -181,7 +234,33 @@ protected URI calculateTarget(Container container, int containerPort, int servic
181234
return new URI(String.format("%s://%s:%s%s", targetProtocol, targetHostName, targetPort, targetPath));
182235
}
183236

184-
@Override
237+
private SecretBind convertSecret(DockerSwarmSecret secret) throws DockerException, InterruptedException {
238+
if (secret.getName() == null) {
239+
throw new IllegalArgumentException("No name for a Docker swarm secret provided");
240+
}
241+
return SecretBind.builder()
242+
.secretName(secret.getName())
243+
.secretId(getSecretId(secret.getName()))
244+
.file(
245+
SecretFile.builder()
246+
.name(Optional.ofNullable(secret.getTarget()).orElse(secret.getName()))
247+
.gid(Optional.ofNullable(secret.getGid()).orElse("0"))
248+
.uid(Optional.ofNullable(secret.getUid()).orElse("0"))
249+
.mode(Long.parseLong(Optional.ofNullable(secret.getMode()).orElse("444"), 8))
250+
.build()
251+
)
252+
.build();
253+
254+
}
255+
256+
private String getSecretId(String secretName) throws DockerException, InterruptedException {
257+
return dockerClient.listSecrets().stream()
258+
.filter(it -> it.secretSpec().name().equals(secretName))
259+
.findFirst()
260+
.map(Secret::id).orElseThrow(() -> new IllegalArgumentException("Secret not found!"));
261+
}
262+
263+
@Override
185264
protected void doStopProxy(Proxy proxy) throws Exception {
186265
for (Container container: proxy.getContainers()) {
187266
String serviceId = (String) container.getParameters().get(PARAM_SERVICE_ID);

‎src/main/java/eu/openanalytics/containerproxy/model/spec/ContainerSpec.java‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,10 @@
2020
*/
2121
package eu.openanalytics.containerproxy.model.spec;
2222

23+
import java.util.ArrayList;
2324
import java.util.Arrays;
2425
import java.util.HashMap;
26+
import java.util.List;
2527
import java.util.Map;
2628

2729
public class ContainerSpec {
@@ -43,6 +45,10 @@ public class ContainerSpec {
4345
private String targetPath;
4446
private Map<String, String> labels = new HashMap<>();
4547
private Map<String, String> settings = new HashMap<>();
48+
private List<DockerSwarmSecret> dockerSwarmSecrets = new ArrayList();
49+
private String dockerRegistryDomain;
50+
private String dockerRegistryUsername;
51+
private String dockerRegistryPassword;
4652

4753
public String getImage() {
4854
return image;
@@ -153,6 +159,38 @@ public void setTargetPath(String targetPath) {
153159
this.targetPath = targetPath;
154160
}
155161

162+
public List<DockerSwarmSecret> getDockerSwarmSecrets() {
163+
return dockerSwarmSecrets;
164+
}
165+
166+
public void setDockerSwarmSecrets(List<DockerSwarmSecret> dockerSwarmSecrets) {
167+
this.dockerSwarmSecrets = dockerSwarmSecrets;
168+
}
169+
170+
public String getDockerRegistryDomain() {
171+
return dockerRegistryDomain;
172+
}
173+
174+
public void setDockerRegistryDomain(String dockerRegistryDomain) {
175+
this.dockerRegistryDomain = dockerRegistryDomain;
176+
}
177+
178+
public String getDockerRegistryUsername() {
179+
return dockerRegistryUsername;
180+
}
181+
182+
public void setDockerRegistryUsername(String dockerRegistryUsername) {
183+
this.dockerRegistryUsername = dockerRegistryUsername;
184+
}
185+
186+
public String getDockerRegistryPassword() {
187+
return dockerRegistryPassword;
188+
}
189+
190+
public void setDockerRegistryPassword(String dockerRegistryPassword) {
191+
this.dockerRegistryPassword = dockerRegistryPassword;
192+
}
193+
156194
public void copy(ContainerSpec target) {
157195
target.setImage(image);
158196
if (cmd != null) target.setCmd(Arrays.copyOf(cmd, cmd.length));
@@ -182,7 +220,13 @@ public void copy(ContainerSpec target) {
182220
if (target.getSettings() == null) target.setSettings(new HashMap<>());
183221
target.getSettings().putAll(settings);
184222
}
223+
if (dockerSwarmSecrets != null) {
224+
if (target.getDockerSwarmSecrets() == null) target.setDockerSwarmSecrets(new ArrayList<>());
225+
target.getDockerSwarmSecrets().addAll(dockerSwarmSecrets);
226+
}
227+
target.setDockerRegistryDomain(dockerRegistryDomain);
228+
target.setDockerRegistryUsername(dockerRegistryUsername);
229+
target.setDockerRegistryPassword(dockerRegistryPassword);
185230
target.setTargetPath(targetPath);
186231
}
187-
188232
}

0 commit comments

Comments
 (0)