Skip to content

Commit 1bcd08a

Browse files
fmichielssenfmichielssen
committed
Merge pull request 'Add support for SpEL to Kubernetes patches and additional manifests' (#16) from feature/23437 into develop
+1
2 parents 0f1768a + 422d620 commit 1bcd08a

4 files changed

Lines changed: 143 additions & 44 deletions

File tree

‎src/main/java/eu/openanalytics/containerproxy/backend/kubernetes/KubernetesBackend.java‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,17 +39,20 @@
3939
import java.util.UUID;
4040
import java.util.function.BiConsumer;
4141
import java.util.stream.Collectors;
42+
import javax.json.JsonPatch;
4243

4344
import javax.inject.Inject;
4445

4546
import org.apache.commons.io.IOUtils;
4647

4748
import com.fasterxml.jackson.core.JsonParseException;
49+
import com.fasterxml.jackson.core.JsonProcessingException;
4850
import com.fasterxml.jackson.databind.JsonMappingException;
4951
import com.fasterxml.jackson.databind.MapperFeature;
5052
import com.fasterxml.jackson.databind.ObjectMapper;
5153
import com.fasterxml.jackson.databind.SerializationFeature;
5254
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
55+
import com.fasterxml.jackson.datatype.jsr353.JSR353Module;
5356
import com.google.common.base.Charsets;
5457
import com.google.common.base.Splitter;
5558

@@ -59,6 +62,8 @@
5962
import eu.openanalytics.containerproxy.model.runtime.Container;
6063
import eu.openanalytics.containerproxy.model.runtime.Proxy;
6164
import eu.openanalytics.containerproxy.model.spec.ContainerSpec;
65+
import eu.openanalytics.containerproxy.spec.expression.SpecExpressionContext;
66+
import eu.openanalytics.containerproxy.spec.expression.SpecExpressionResolver;
6267
import eu.openanalytics.containerproxy.util.Retrying;
6368
import io.fabric8.kubernetes.api.model.ContainerBuilder;
6469
import io.fabric8.kubernetes.api.model.ContainerPort;
@@ -258,8 +263,10 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
258263
podSpec.setNodeSelector(Splitter.on(",").withKeyValueSeparator("=").split(nodeSelectorString));
259264
}
260265

266+
JsonPatch patch = readPatchFromSpec(spec, proxy);
267+
261268
Pod startupPod = podBuilder.withSpec(podSpec).build();
262-
Pod patchedPod = podPatcher.patchWithDebug(startupPod, proxy.getSpec().getKubernetesPodPatchAsJsonpatch());
269+
Pod patchedPod = podPatcher.patchWithDebug(startupPod, patch);
263270
final String effectiveKubeNamespace = patchedPod.getMetadata().getNamespace(); // use the namespace of the patched Pod, in case the patch changes the namespace.
264271
container.getParameters().put(PARAM_NAMESPACE, effectiveKubeNamespace);
265272

@@ -333,6 +340,20 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
333340
return container;
334341
}
335342

343+
private JsonPatch readPatchFromSpec(ContainerSpec containerSpec, Proxy proxy) throws JsonMappingException, JsonProcessingException {
344+
String patchAsString = proxy.getSpec().getKubernetesPodPatch();
345+
if (patchAsString == null) {
346+
return null;
347+
}
348+
349+
// resolve expressions
350+
SpecExpressionContext context = SpecExpressionContext.create(containerSpec, proxy, proxy.getSpec());
351+
String expressionAwarePatch = expressionResolver.evaluateToString(patchAsString, context);
352+
353+
ObjectMapper yamlReader = new ObjectMapper(new YAMLFactory());
354+
yamlReader.registerModule(new JSR353Module());
355+
return yamlReader.readValue(expressionAwarePatch, JsonPatch.class);
356+
}
336357

337358
/**
338359
* Creates the extra manifests/resources defined in the ProxySpec.
@@ -353,11 +374,14 @@ private void createAdditionalManifstes(Proxy proxy, String namespace) {
353374
* parameter will be used.
354375
*/
355376
private List<HasMetadata> getAdditionManifestsAsObjects(Proxy proxy, String namespace) {
377+
SpecExpressionContext context = SpecExpressionContext.create(proxy, proxy.getSpec());
378+
356379
ArrayList<HasMetadata> result = new ArrayList<HasMetadata>();
357380
for (String manifest : proxy.getSpec().getKubernetesAdditionalManifests()) {
358-
HasMetadata object = Serialization.unmarshal(new ByteArrayInputStream(manifest.getBytes())); // used to determine whether the manifest has specified a namespace
381+
String expressionManifest = expressionResolver.evaluateToString(manifest, context);
382+
HasMetadata object = Serialization.unmarshal(new ByteArrayInputStream(expressionManifest.getBytes())); // used to determine whether the manifest has specified a namespace
359383

360-
HasMetadata fullObject = kubeClient.load(new ByteArrayInputStream(manifest.getBytes())).get().get(0);
384+
HasMetadata fullObject = kubeClient.load(new ByteArrayInputStream(expressionManifest.getBytes())).get().get(0);
361385
if (object.getMetadata().getNamespace() == null) {
362386
// the load method (in some cases) automatically sets a namepsace when no namespace is provided
363387
// therefore we overwrite this namespace with the namsepace of the pod.

‎src/main/java/eu/openanalytics/containerproxy/model/spec/ProxySpec.java‎

Lines changed: 3 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -20,23 +20,12 @@
2020
*/
2121
package eu.openanalytics.containerproxy.model.spec;
2222

23-
import java.io.IOException;
2423
import java.util.ArrayList;
2524
import java.util.HashMap;
2625
import java.util.List;
2726
import java.util.Map;
2827
import java.util.stream.Collectors;
2928

30-
import javax.json.JsonPatch;
31-
import javax.json.JsonValue;
32-
33-
import com.fasterxml.jackson.annotation.JsonIgnore;
34-
import com.fasterxml.jackson.core.JsonParseException;
35-
import com.fasterxml.jackson.databind.JsonMappingException;
36-
import com.fasterxml.jackson.databind.ObjectMapper;
37-
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
38-
import com.fasterxml.jackson.datatype.jsr353.JSR353Module;
39-
4029
public class ProxySpec {
4130

4231
private String id;
@@ -50,7 +39,7 @@ public class ProxySpec {
5039

5140
private Map<String, String> settings = new HashMap<>();
5241

53-
private JsonPatch kubernetesPodPatches;
42+
private String kubernetesPodPatches;
5443
private List<String> kubernetesAdditionalManifests = new ArrayList<>();
5544

5645
public ProxySpec() {
@@ -124,36 +113,11 @@ public void setSettings(Map<String, String> settings) {
124113
/**
125114
* Returns the Kubernetes Pod Patch as JsonValue (i.e. array) for nice representation in API requests.
126115
*/
127-
public JsonValue getKubernetesPodPatch() {
128-
if (this.kubernetesPodPatches == null) {
129-
return null;
130-
} else {
131-
return kubernetesPodPatches.toJsonArray();
132-
}
133-
}
134-
135-
/**
136-
* Returns the Kubernetes Pod Patch as a JsonPatch, so it can be directly be used to patch the spec.
137-
* Should not be returned by API responses.
138-
*/
139-
@JsonIgnore
140-
public JsonPatch getKubernetesPodPatchAsJsonpatch() {
116+
public String getKubernetesPodPatch() {
141117
return kubernetesPodPatches;
142118
}
143119

144-
public void setKubernetesPodPatches(String kubernetesPodPatches) throws JsonParseException, JsonMappingException, IOException {
145-
try {
146-
// convert the raw YAML string into a JsonPatch
147-
ObjectMapper yamlReader = new ObjectMapper(new YAMLFactory());
148-
yamlReader.registerModule(new JSR353Module());
149-
this.kubernetesPodPatches = yamlReader.readValue(kubernetesPodPatches, JsonPatch.class);
150-
} catch (Exception exception) {
151-
exception.printStackTrace(); // log the exception for easier debugging
152-
throw exception;
153-
}
154-
}
155-
156-
private void setKubernetesPodPatches(JsonPatch kubernetesPodPatches) {
120+
public void setKubernetesPodPatches(String kubernetesPodPatches) {
157121
this.kubernetesPodPatches = kubernetesPodPatches;
158122
}
159123

@@ -201,7 +165,6 @@ public void copy(ProxySpec target) {
201165

202166

203167
if (kubernetesPodPatches != null) {
204-
// JsonPatch is an immutable object
205168
target.setKubernetesPodPatches(kubernetesPodPatches);
206169
}
207170

‎src/test/java/eu/openanalytics/containerproxy/test/proxy/TestIntegrationOnKube.java‎

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import eu.openanalytics.containerproxy.model.runtime.Proxy;
6161
import eu.openanalytics.containerproxy.model.spec.ProxySpec;
6262
import eu.openanalytics.containerproxy.service.ProxyService;
63+
import eu.openanalytics.containerproxy.service.UserService;
6364
import eu.openanalytics.containerproxy.test.proxy.TestIntegrationOnKube.TestConfiguration;
6465
import eu.openanalytics.containerproxy.util.ProxyMappingManager;
6566
import io.fabric8.kubernetes.api.model.ContainerStatus;
@@ -77,6 +78,7 @@
7778
import io.fabric8.kubernetes.api.model.Service;
7879
import io.fabric8.kubernetes.api.model.ServiceAccountBuilder;
7980
import io.fabric8.kubernetes.api.model.ServiceList;
81+
import io.fabric8.kubernetes.api.model.Volume;
8082
import io.fabric8.kubernetes.api.model.VolumeMount;
8183
import io.fabric8.kubernetes.client.KubernetesClient;
8284

@@ -248,7 +250,7 @@ public void launchProxyWithEnv() throws Exception {
248250
List<EnvVar> envList = pod.getSpec().getContainers().get(0).getEnv();
249251
Map<String, EnvVar> env = envList.stream().collect(Collectors.toMap(EnvVar::getName, e -> e));
250252
assertTrue(env.containsKey("SHINYPROXY_USERNAME"));
251-
assertEquals("null", env.get("SHINYPROXY_USERNAME").getValue()); // value is a String "null"
253+
assertEquals("jack", env.get("SHINYPROXY_USERNAME").getValue()); // value is a String "null"
252254
assertTrue(env.containsKey("SHINYPROXY_USERGROUPS"));
253255
assertEquals(null, env.get("SHINYPROXY_USERGROUPS").getValue());
254256
assertTrue(env.containsKey("VAR1"));
@@ -672,6 +674,64 @@ public void launchProxyWithAdditionalManifestsOfWhichOneAlreadyExists() throws E
672674
}
673675
}
674676

677+
678+
/**
679+
* Tests the use of Spring Epxression in kubernetes patches and additional manifests.
680+
*/
681+
@Test
682+
public void launchProxyWithExpressionInPatchAndManifests() throws Exception {
683+
String specId = environment.getProperty("proxy.specs[9].id");
684+
685+
ProxySpec baseSpec = proxyService.findProxySpec(s -> s.getId().equals(specId), true);
686+
ProxySpec spec = proxyService.resolveProxySpec(baseSpec, null, null);
687+
Proxy proxy = proxyService.startProxy(spec, true);
688+
String containerId = proxy.getContainers().get(0).getId();
689+
690+
PodList podList = client.pods().inNamespace(session.getNamespace()).list();
691+
assertEquals(1, podList.getItems().size());
692+
Pod pod = podList.getItems().get(0);
693+
assertEquals("Running", pod.getStatus().getPhase());
694+
assertEquals(session.getNamespace(), pod.getMetadata().getNamespace());
695+
assertEquals("sp-pod-" + containerId, pod.getMetadata().getName());
696+
assertEquals(1, pod.getStatus().getContainerStatuses().size());
697+
ContainerStatus container = pod.getStatus().getContainerStatuses().get(0);
698+
assertEquals(true, container.getReady());
699+
assertEquals("openanalytics/shinyproxy-demo:latest", container.getImage());
700+
701+
702+
// check env variables
703+
List<EnvVar> envList = pod.getSpec().getContainers().get(0).getEnv();
704+
Map<String, EnvVar> env = envList.stream().collect(Collectors.toMap(EnvVar::getName, e -> e));
705+
assertTrue(env.containsKey("CUSTOM_USERNAME"));
706+
assertTrue(env.containsKey("PROXY_ID"));
707+
assertEquals("jack", env.get("CUSTOM_USERNAME").getValue());
708+
assertEquals(proxy.getId(), env.get("PROXY_ID").getValue());
709+
710+
PersistentVolumeClaimList claimList = client.persistentVolumeClaims().inNamespace(session.getNamespace()).list();
711+
assertEquals(1, claimList.getItems().size());
712+
PersistentVolumeClaim claim = claimList.getItems().get(0);
713+
assertEquals(session.getNamespace(), claim.getMetadata().getNamespace());
714+
assertEquals("home-dir-pvc-jack", claim.getMetadata().getName());
715+
716+
// check volume mount
717+
Volume volume = pod.getSpec().getVolumes().get(0);
718+
assertEquals("home-dir-pvc-jack", volume.getName());
719+
assertEquals("home-dir-pvc-jack", volume.getPersistentVolumeClaim().getClaimName());
720+
721+
proxyService.stopProxy(proxy, false, true);
722+
723+
// Give Kube the time to clean
724+
Thread.sleep(2000);
725+
726+
// all pods should be deleted
727+
podList = client.pods().inNamespace(session.getNamespace()).list();
728+
assertEquals(0, podList.getItems().size());
729+
// all additional manifests should be deleted
730+
assertEquals(0, client.persistentVolumeClaims().inNamespace(session.getNamespace()).list().getItems().size());
731+
732+
assertEquals(0, proxyService.getProxies(null, true).size());
733+
}
734+
675735
private final String overridenNamespace = "it-b9fa0a24-overriden";
676736

677737
private void createOverridenNamespace() throws InterruptedException {
@@ -695,6 +755,12 @@ private void deleteOverridenNamespace() throws InterruptedException {
695755
}
696756
}
697757

758+
public static class MockedUserService extends UserService {
759+
public String getCurrentUserId() {
760+
return "jack";
761+
}
762+
}
763+
698764
public static class TestConfiguration {
699765
@Bean
700766
@Primary
@@ -708,6 +774,12 @@ public AbstractFactoryBean<IContainerBackend> backendFactory() {
708774
return new TestContainerBackendFactory();
709775
}
710776

777+
@Bean
778+
@Primary
779+
public UserService mockedUserService() {
780+
return new MockedUserService();
781+
}
782+
711783
}
712784

713785
public static class TestContainerBackendFactory extends AbstractFactoryBean<IContainerBackend>

‎src/test/resources/application-test.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,46 @@ proxy:
149149
data:
150150
password: cGFzc3dvcmQ=
151151
152+
- id: 01_hello_manifests_espression
153+
container-specs:
154+
- image: "openanalytics/shinyproxy-demo"
155+
cmd: ["R", "-e", "shinyproxy::run_01_hello()"]
156+
port-mapping:
157+
default: 3838
158+
kubernetes-pod-patches: |
159+
- op: add
160+
path: /spec/containers/0/env/-
161+
value:
162+
name: CUSTOM_USERNAME
163+
value: "#{proxy.userId}"
164+
- op: add
165+
path: /spec/containers/0/env/-
166+
value:
167+
name: PROXY_ID
168+
value: "#{proxy.id}"
169+
- op: add
170+
path: /spec/volumes
171+
value:
172+
- name: "home-dir-pvc-#{proxy.userId}"
173+
persistentVolumeClaim:
174+
claimName: "home-dir-pvc-#{proxy.userId}"
175+
- op: add
176+
path: /spec/containers/0/volumeMounts
177+
value:
178+
- mountPath: "/home/#{proxy.userId}"
179+
name: "home-dir-pvc-#{proxy.userId}"
180+
kubernetes-additional-manifests:
181+
- |
182+
apiVersion: v1
183+
kind: PersistentVolumeClaim
184+
metadata:
185+
name: "home-dir-pvc-#{proxy.userId}"
186+
spec:
187+
accessModes:
188+
- ReadWriteOnce
189+
resources:
190+
requests:
191+
storage: 5Gi
152192
- id: 02_hello
153193
container-specs:
154194
- image: "openanalytics/shinyproxy-demo"

0 commit comments

Comments
 (0)