Skip to content

Commit 99f3d6a

Browse files
LEDfanLEDfan
committed
Fix #31309: add ECS integration tests
1 parent 943fd08 commit 99f3d6a

14 files changed

Lines changed: 676 additions & 28 deletions

‎src/main/java/eu/openanalytics/containerproxy/backend/ecs/EcsBackend.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -111,10 +111,13 @@ public class EcsBackend extends AbstractContainerBackend {
111111
public void initialize() {
112112
super.initialize();
113113

114-
Region region = Region.of(getProperty(PROPERTY_REGION));
114+
String region = getProperty(PROPERTY_REGION);
115+
if (region == null) {
116+
throw new IllegalStateException("Error in configuration of ECS backend: proxy.ecs.region not set");
117+
}
115118

116119
ecsClient = EcsClient.builder()
117-
.region(region)
120+
.region(Region.of(region))
118121
.build();
119122

120123
cluster = getProperty(PROPERTY_CLUSTER);
@@ -127,14 +130,22 @@ public void initialize() {
127130
cloudWatchRegion = environment.getProperty("proxy.ecs.cloud-watch-region", String.class, getProperty(PROPERTY_REGION));
128131
cloudWatchStreamPrefix = environment.getProperty("proxy.ecs.cloud-watch-stream-prefix", String.class, "ecs");
129132

130-
if (subnets.isEmpty()) {
133+
if (cluster == null) {
134+
throw new IllegalStateException("Error in configuration of ECS backend: proxy.ecs.cluster not set to name of cluster");
135+
}
136+
137+
if (subnets == null || subnets.isEmpty()) {
131138
throw new IllegalStateException("Error in configuration of ECS backend: need at least one subnet in proxy.ecs.subnets");
132139
}
133140

134-
if (securityGroups.isEmpty()) {
141+
if (securityGroups == null || securityGroups.isEmpty()) {
135142
throw new IllegalStateException("Error in configuration of ECS backend: need at least one security group in proxy.ecs.security-groups");
136143
}
137144

145+
if (isPrivileged()) {
146+
throw new IllegalStateException("Error in configuration of ECS backend: config has 'privileged: true' configured, this is not supported by ECS fargated");
147+
}
148+
138149
for (ProxySpec spec : proxySpecProvider.getSpecs()) {
139150
ContainerSpec containerSpec = spec.getContainerSpecs().get(0);
140151
if (!containerSpec.getMemoryRequest().isOriginalValuePresent()) {
@@ -152,6 +163,9 @@ public void initialize() {
152163
if (containerSpec.isPrivileged()) {
153164
throw new IllegalStateException(String.format("Error in configuration of specs: spec with id '%s' has 'privileged: true' configured, this is not supported by ECS fargate", spec.getId()));
154165
}
166+
if (containerSpec.getDns().isOriginalValuePresent()) {
167+
throw new IllegalStateException(String.format("Error in configuration of specs: spec with id '%s' has 'dns' configured, this is not supported by ECS fargate", spec.getId()));
168+
}
155169
}
156170
}
157171

@@ -277,7 +291,6 @@ private String getTaskDefinition(Authentication user, ContainerSpec spec, EcsSpe
277291
.containerDefinitions(ContainerDefinition.builder()
278292
.name("sp-container-" + containerId)
279293
.image(spec.getImage().getValue())
280-
.dnsServers(spec.getDns().getValueOrNull())
281294
.command(spec.getCmd().getValueOrNull())
282295
.environment(env)
283296
.stopTimeout(2)

‎src/test/java/eu/openanalytics/containerproxy/test/proxy/TestIntegrationOnEcs.java‎

Lines changed: 263 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,26 +20,282 @@
2020
*/
2121
package eu.openanalytics.containerproxy.test.proxy;
2222

23+
import com.google.common.base.Throwables;
24+
import eu.openanalytics.containerproxy.model.runtime.Proxy;
25+
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.BackendContainerNameKey;
2326
import eu.openanalytics.containerproxy.test.helpers.ContainerSetup;
2427
import eu.openanalytics.containerproxy.test.helpers.ShinyProxyInstance;
25-
import org.junit.jupiter.api.AfterAll;
28+
import eu.openanalytics.containerproxy.test.helpers.TestHelperException;
29+
import org.junit.jupiter.api.Assertions;
30+
import org.junit.jupiter.api.Assumptions;
2631
import org.junit.jupiter.api.Test;
32+
import org.slf4j.Logger;
33+
import org.slf4j.LoggerFactory;
34+
import software.amazon.awssdk.core.exception.SdkClientException;
35+
import software.amazon.awssdk.regions.Region;
36+
import software.amazon.awssdk.services.ecs.EcsClient;
37+
import software.amazon.awssdk.services.ecs.model.ContainerDefinition;
38+
import software.amazon.awssdk.services.ecs.model.KeyValuePair;
39+
import software.amazon.awssdk.services.ecs.model.LaunchType;
40+
import software.amazon.awssdk.services.ecs.model.LogConfiguration;
41+
import software.amazon.awssdk.services.ecs.model.LogDriver;
42+
import software.amazon.awssdk.services.ecs.model.Tag;
43+
import software.amazon.awssdk.services.ecs.model.Task;
44+
import software.amazon.awssdk.services.ecs.model.TaskDefinition;
45+
import software.amazon.awssdk.services.sts.StsClient;
2746

28-
import java.util.HashMap;
47+
import java.util.List;
48+
import java.util.Map;
49+
import java.util.stream.Collectors;
2950

3051
public class TestIntegrationOnEcs {
3152

32-
private static final ShinyProxyInstance inst = new ShinyProxyInstance("application-test-ecs.yml", new HashMap<>());
53+
private final Logger logger = LoggerFactory.getLogger(getClass());
54+
private static EcsClient ecsClient;
55+
private static String cluster;
3356

34-
@AfterAll
35-
public static void afterAll() {
36-
inst.close();
57+
private EcsClient getEcsClient() {
58+
if (ecsClient == null) {
59+
Region region = Region.of(System.getenv("ITEST_ECS_REGION"));
60+
cluster = System.getenv("ITEST_ECS_NAME");
61+
62+
ecsClient = EcsClient.builder()
63+
.region(region)
64+
.build();
65+
}
66+
return ecsClient;
3767
}
3868

69+
3970
@Test
4071
public void launchProxy() {
72+
Assumptions.assumeTrue(checkAwsCredentials(), "Skipping ECS tests");
4173
try (ContainerSetup containerSetup = new ContainerSetup("ecs")) {
42-
String id = inst.client.startProxy("01_hello");
74+
try (ShinyProxyInstance inst = new ShinyProxyInstance("application-test-ecs.yml", Map.of(), true)) {
75+
inst.enableCleanup();
76+
// launch a proxy on ECS
77+
String id = inst.client.startProxy("01_hello");
78+
Proxy proxy = inst.proxyService.getProxy(id);
79+
inst.client.testProxyReachable(id);
80+
81+
Task task = getTask(proxy);
82+
83+
// get tags
84+
Map<String, String> tags = getTags(task);
85+
Assertions.assertEquals("demo", tags.get("openanalytics.eu/sp-user-id"));
86+
Assertions.assertEquals("01_hello", tags.get("openanalytics.eu/sp-spec-id"));
87+
Assertions.assertEquals("true", tags.get("openanalytics.eu/sp-proxied-app"));
88+
Assertions.assertEquals(proxy.getId(), tags.get("openanalytics.eu/sp-proxy-id"));
89+
Assertions.assertEquals("myvalue", tags.get("valid-label"));
90+
Assertions.assertFalse(tags.containsKey("invalid-label"));
91+
Assertions.assertFalse(tags.containsKey("invalid-label2"));
92+
93+
Assertions.assertEquals("1024", task.cpu());
94+
Assertions.assertEquals("2048", task.memory());
95+
Assertions.assertEquals(21, task.ephemeralStorage().sizeInGiB());
96+
Assertions.assertEquals(false, task.enableExecuteCommand());
97+
Assertions.assertEquals(LaunchType.FARGATE, task.launchType());
98+
99+
TaskDefinition taskDefinition = getTaskDefinition(task);
100+
101+
Assertions.assertNull(taskDefinition.deregisteredAt());
102+
Assertions.assertNull(taskDefinition.executionRoleArn());
103+
Assertions.assertNull(taskDefinition.taskRoleArn());
104+
Assertions.assertTrue(taskDefinition.volumes().isEmpty());
105+
Assertions.assertEquals(1, taskDefinition.revision());
106+
Assertions.assertEquals(1, taskDefinition.containerDefinitions().size());
107+
ContainerDefinition containerDefinition = taskDefinition.containerDefinitions().get(0);
108+
Assertions.assertEquals(List.of("R", "-e", "shinyproxy::run_01_hello()"), containerDefinition.command());
109+
Assertions.assertEquals(List.of(), containerDefinition.dnsServers());
110+
Map<String, String> dockerLabels = containerDefinition.dockerLabels();
111+
Assertions.assertTrue(dockerLabels.size() > 15);
112+
Assertions.assertEquals("demo", dockerLabels.get("openanalytics.eu/sp-user-id"));
113+
Assertions.assertEquals("01_hello", dockerLabels.get("openanalytics.eu/sp-spec-id"));
114+
115+
Map<String, String> environment = containerDefinition.environment().stream().collect(Collectors.toMap(KeyValuePair::name, KeyValuePair::value));
116+
Assertions.assertEquals(3, environment.size());
117+
Assertions.assertEquals("demo", environment.get("SHINYPROXY_USERNAME"));
118+
Assertions.assertEquals("", environment.get("SHINYPROXY_USERGROUPS"));
119+
Assertions.assertEquals("/api/route/" + proxy.getId(), environment.get("SHINYPROXY_PUBLIC_PATH")); // TODO no ending slash?
120+
Assertions.assertEquals("openanalytics/shinyproxy-integration-test-app", containerDefinition.image());
121+
Assertions.assertNull(containerDefinition.privileged()); // fargate does not support privileged
122+
Assertions.assertNull(containerDefinition.hostname());
123+
Assertions.assertNull(containerDefinition.logConfiguration());
124+
125+
inst.client.stopProxy(id);
126+
127+
taskDefinition = getTaskDefinition(task);
128+
Assertions.assertNotNull(taskDefinition.deregisteredAt());
129+
}
130+
}
131+
}
132+
133+
@Test
134+
public void launchProxyWithRoles() {
135+
Assumptions.assumeTrue(checkAwsCredentials(), "Skipping ECS tests");
136+
try (ContainerSetup containerSetup = new ContainerSetup("ecs")) {
137+
try (ShinyProxyInstance inst = new ShinyProxyInstance("application-test-ecs.yml", Map.of("proxy.ecs.enable-cloudwatch", "true"), true)) {
138+
inst.enableCleanup();
139+
// launch a proxy on ECS
140+
String id = inst.client.startProxy("01_hello_roles");
141+
Proxy proxy = inst.proxyService.getProxy(id);
142+
inst.client.testProxyReachable(id);
143+
144+
Task task = getTask(proxy);
145+
146+
Assertions.assertEquals(50, task.ephemeralStorage().sizeInGiB());
147+
Assertions.assertEquals(true, task.enableExecuteCommand());
148+
149+
TaskDefinition taskDefinition = getTaskDefinition(task);
150+
151+
Assertions.assertEquals(System.getenv("ITEST_ECS_EXECUTION_ROLE"), taskDefinition.executionRoleArn());
152+
Assertions.assertEquals(System.getenv("ITEST_ECS_TASK_ROLE"), taskDefinition.taskRoleArn());
153+
154+
ContainerDefinition containerDefinition = taskDefinition.containerDefinitions().get(0);
155+
LogConfiguration logConfiguration = containerDefinition.logConfiguration();
156+
Assertions.assertNotNull(logConfiguration);
157+
Assertions.assertEquals(LogDriver.AWSLOGS, logConfiguration.logDriver());
158+
Assertions.assertEquals("/ecs/sp-" + proxy.getId(), logConfiguration.options().get("awslogs-group"));
159+
Assertions.assertEquals(System.getenv("ITEST_ECS_REGION"), logConfiguration.options().get("awslogs-region"));
160+
Assertions.assertEquals("true", logConfiguration.options().get("awslogs-create-group"));
161+
Assertions.assertEquals("ecs", logConfiguration.options().get("awslogs-stream-prefix"));
162+
163+
inst.client.stopProxy(id);
164+
165+
taskDefinition = getTaskDefinition(task);
166+
Assertions.assertNotNull(taskDefinition.deregisteredAt());
167+
}
168+
}
169+
}
170+
171+
@Test
172+
public void testInvalidConfig1() {
173+
TestHelperException ex = Assertions.assertThrows(TestHelperException.class, () -> new ShinyProxyInstance("application-test-ecs-invalid-1.yml"));
174+
175+
Throwable rootCause = Throwables.getRootCause(ex);
176+
Assertions.assertInstanceOf(IllegalStateException.class, rootCause);
177+
Assertions.assertEquals("Error in configuration of ECS backend: need at least one subnet in proxy.ecs.subnets", rootCause.getMessage());
178+
}
179+
180+
@Test
181+
public void testInvalidConfig2() {
182+
TestHelperException ex = Assertions.assertThrows(TestHelperException.class, () -> new ShinyProxyInstance("application-test-ecs-invalid-2.yml"));
183+
184+
Throwable rootCause = Throwables.getRootCause(ex);
185+
Assertions.assertInstanceOf(IllegalStateException.class, rootCause);
186+
Assertions.assertEquals("Error in configuration of ECS backend: proxy.ecs.region not set", rootCause.getMessage());
187+
}
188+
189+
@Test
190+
public void testInvalidConfig3() {
191+
TestHelperException ex = Assertions.assertThrows(TestHelperException.class, () -> new ShinyProxyInstance("application-test-ecs-invalid-3.yml"));
192+
193+
Throwable rootCause = Throwables.getRootCause(ex);
194+
Assertions.assertInstanceOf(IllegalStateException.class, rootCause);
195+
Assertions.assertEquals("Error in configuration of ECS backend: proxy.ecs.cluster not set to name of cluster", rootCause.getMessage());
196+
}
197+
198+
@Test
199+
public void testInvalidConfig4() {
200+
TestHelperException ex = Assertions.assertThrows(TestHelperException.class, () -> new ShinyProxyInstance("application-test-ecs-invalid-4.yml"));
201+
202+
Throwable rootCause = Throwables.getRootCause(ex);
203+
Assertions.assertInstanceOf(IllegalStateException.class, rootCause);
204+
Assertions.assertEquals("Error in configuration of ECS backend: config has 'privileged: true' configured, this is not supported by ECS fargated", rootCause.getMessage());
205+
}
206+
207+
@Test
208+
public void testInvalidConfig5() {
209+
TestHelperException ex = Assertions.assertThrows(TestHelperException.class, () -> new ShinyProxyInstance("application-test-ecs-invalid-5.yml"));
210+
211+
Throwable rootCause = Throwables.getRootCause(ex);
212+
Assertions.assertInstanceOf(IllegalStateException.class, rootCause);
213+
Assertions.assertEquals("Error in configuration of specs: spec with id '01_hello' has non 'memory-request' configured, this is required for running on ECS fargate", rootCause.getMessage());
214+
}
215+
216+
@Test
217+
public void testInvalidConfig6() {
218+
TestHelperException ex = Assertions.assertThrows(TestHelperException.class, () -> new ShinyProxyInstance("application-test-ecs-invalid-6.yml"));
219+
220+
Throwable rootCause = Throwables.getRootCause(ex);
221+
Assertions.assertInstanceOf(IllegalStateException.class, rootCause);
222+
Assertions.assertEquals("Error in configuration of specs: spec with id '01_hello' has non 'cpu-request' configured, this is required for running on ECS fargate", rootCause.getMessage());
223+
}
224+
225+
@Test
226+
public void testInvalidConfig7() {
227+
TestHelperException ex = Assertions.assertThrows(TestHelperException.class, () -> new ShinyProxyInstance("application-test-ecs-invalid-7.yml"));
228+
229+
Throwable rootCause = Throwables.getRootCause(ex);
230+
Assertions.assertInstanceOf(IllegalStateException.class, rootCause);
231+
Assertions.assertEquals("Error in configuration of specs: spec with id '01_hello' has 'cpu-limit' configured, this is not supported by ECS fargate", rootCause.getMessage());
232+
}
233+
234+
@Test
235+
public void testInvalidConfig8() {
236+
TestHelperException ex = Assertions.assertThrows(TestHelperException.class, () -> new ShinyProxyInstance("application-test-ecs-invalid-8.yml"));
237+
238+
Throwable rootCause = Throwables.getRootCause(ex);
239+
Assertions.assertInstanceOf(IllegalStateException.class, rootCause);
240+
Assertions.assertEquals("Error in configuration of specs: spec with id '01_hello' has 'memory-limit' configured, this is not supported by ECS fargate", rootCause.getMessage());
241+
}
242+
243+
@Test
244+
public void testInvalidConfig9() {
245+
TestHelperException ex = Assertions.assertThrows(TestHelperException.class, () -> new ShinyProxyInstance("application-test-ecs-invalid-9.yml"));
246+
247+
Throwable rootCause = Throwables.getRootCause(ex);
248+
Assertions.assertInstanceOf(IllegalStateException.class, rootCause);
249+
Assertions.assertEquals("Error in configuration of specs: spec with id '01_hello' has 'privileged: true' configured, this is not supported by ECS fargate", rootCause.getMessage());
250+
}
251+
252+
@Test
253+
public void testInvalidConfig10() {
254+
TestHelperException ex = Assertions.assertThrows(TestHelperException.class, () -> new ShinyProxyInstance("application-test-ecs-invalid-10.yml"));
255+
256+
Throwable rootCause = Throwables.getRootCause(ex);
257+
Assertions.assertInstanceOf(IllegalStateException.class, rootCause);
258+
Assertions.assertEquals("Error in configuration of specs: spec with id '01_hello' has 'dns' configured, this is not supported by ECS fargate", rootCause.getMessage());
259+
}
260+
261+
private Task getTask(Proxy proxy) {
262+
String taskArn = proxy.getContainers().get(0).getRuntimeValue(BackendContainerNameKey.inst);
263+
List<Task> tasks = getEcsClient().describeTasks(builder -> builder.cluster(cluster).tasks(taskArn)).tasks();
264+
Assertions.assertEquals(1, tasks.size());
265+
return tasks.get(0);
266+
}
267+
268+
private Map<String, String> getTags(Task task) {
269+
return getEcsClient().listTagsForResource(b -> b.resourceArn(task.taskArn()))
270+
.tags()
271+
.stream()
272+
.collect(Collectors.toMap(Tag::key, Tag::value));
273+
}
274+
275+
private TaskDefinition getTaskDefinition(Task task) {
276+
return getEcsClient().describeTaskDefinition(builder -> builder.taskDefinition(task.taskDefinitionArn())).taskDefinition();
277+
}
278+
279+
private boolean requireEnvVar(String name) {
280+
if (System.getenv(name) == null) {
281+
logger.info("Env var {} missing, skipping ECS Tests", name);
282+
return false;
283+
}
284+
return true;
285+
}
286+
287+
private boolean checkAwsCredentials() {
288+
try (StsClient client = StsClient.create()) {
289+
client.getCallerIdentity();
290+
291+
return requireEnvVar("ITEST_ECS_NAME")
292+
&& requireEnvVar("ITEST_ECS_REGION")
293+
&& requireEnvVar("ITEST_ECS_SECURITY_GROUPS")
294+
&& requireEnvVar("ITEST_ECS_SUBNETS")
295+
&& requireEnvVar("ITEST_ECS_TASK_ROLE")
296+
&& requireEnvVar("ITEST_ECS_EXECUTION_ROLE");
297+
} catch (SdkClientException ex) {
298+
return false;
43299
}
44300
}
45301

‎src/test/java/eu/openanalytics/containerproxy/test/proxy/TestIntegrationProxySharing.java‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -207,12 +207,9 @@ public void testNoContainerReUse(String backend, Map<String, String> properties)
207207
}
208208
}
209209

210-
@ParameterizedTest
211-
@MethodSource("backends")
212-
public void testReUseWithMultipleSeats(String backend, Map<String, String> properties) {
213-
TestHelperException ex = Assertions.assertThrows(TestHelperException.class,
214-
() -> new ShinyProxyInstance("application-test-pre-initialization-3.yml", properties, true),
215-
"Provided parameter values are not allowed");
210+
@Test
211+
public void testReUseWithMultipleSeats() {
212+
TestHelperException ex = Assertions.assertThrows(TestHelperException.class, () -> new ShinyProxyInstance("application-test-pre-initialization-3.yml"));
216213

217214
Throwable rootCause = Throwables.getRootCause(ex);
218215
Assertions.assertInstanceOf(IllegalStateException.class, rootCause);

0 commit comments

Comments
 (0)