Skip to content

Commit cdc4368

Browse files
LEDfanLEDfan
committed
Ref #27090: add more Prometheus timings metrics
1 parent 4c6c497 commit cdc4368

14 files changed

Lines changed: 302 additions & 17 deletions

File tree

‎src/main/java/eu/openanalytics/containerproxy/backend/AbstractContainerBackend.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import eu.openanalytics.containerproxy.model.spec.ContainerSpec;
4040
import eu.openanalytics.containerproxy.service.AppRecoveryService;
4141
import eu.openanalytics.containerproxy.service.IdentifierService;
42+
import eu.openanalytics.containerproxy.service.ProxyStatusService;
4243
import eu.openanalytics.containerproxy.service.UserService;
4344
import eu.openanalytics.containerproxy.spec.expression.ExpressionAwareContainerSpec;
4445
import eu.openanalytics.containerproxy.spec.expression.SpecExpressionResolver;
@@ -99,6 +100,11 @@ public abstract class AbstractContainerBackend implements IContainerBackend {
99100
@Inject
100101
protected IdentifierService identifierService;
101102

103+
@Inject
104+
@Lazy
105+
// Note: lazy needed to work around early initialization conflict
106+
protected ProxyStatusService proxyStatusService;
107+
102108
@Inject
103109
@Lazy
104110
// Note: lazy to prevent cyclic dependencies
@@ -128,11 +134,13 @@ public SuccessOrFailure<Proxy> startProxy(Proxy proxy) throws ContainerProxyExce
128134

129135
if (!testStrategy.testProxy(proxy)) {
130136
stopProxy(proxy);
137+
proxyStatusService.applicationStartupFailed(proxy);
131138
return SuccessOrFailure.createFailure(proxy, "Container did not respond in time");
132139
}
133140

134141
proxy.setStartupTimestamp(System.currentTimeMillis());
135142
proxy.setStatus(ProxyStatus.Up);
143+
proxyStatusService.proxyStarted(proxy);
136144

137145
return SuccessOrFailure.createSuccess(proxy);
138146
} catch (Throwable t) {

‎src/main/java/eu/openanalytics/containerproxy/backend/docker/DockerEngineBackend.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
131131
Container container = new Container();
132132
container.setSpec(spec);
133133
container.setId(containerCreation.id());
134+
container.setIndex(spec.getIndex());
134135

135136
// Calculate proxy routes for all configured ports.
136137
for (String mappingKey: spec.getPortMapping().keySet()) {

‎src/main/java/eu/openanalytics/containerproxy/backend/docker/DockerSwarmBackend.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ public void initialize() throws ContainerProxyException {
7878
protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Exception {
7979
Container container = new Container();
8080
container.setSpec(spec);
81+
container.setIndex(spec.getIndex());
8182

8283
Mount[] mounts = null;
8384
if (spec.getVolumes() != null) mounts = Arrays.stream(spec.getVolumes())

‎src/main/java/eu/openanalytics/containerproxy/backend/kubernetes/KubernetesBackend.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,7 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
159159
Container container = new Container();
160160
container.setSpec(spec);
161161
container.setId(UUID.randomUUID().toString());
162+
container.setIndex(spec.getIndex());
162163

163164
String kubeNamespace = getProperty(PROPERTY_NAMESPACE, DEFAULT_NAMESPACE);
164165
String apiVersion = getProperty(PROPERTY_API_VERSION, DEFAULT_API_VERSION);
@@ -281,9 +282,13 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
281282
final String effectiveKubeNamespace = patchedPod.getMetadata().getNamespace(); // use the namespace of the patched Pod, in case the patch changes the namespace.
282283
container.getParameters().put(PARAM_NAMESPACE, effectiveKubeNamespace);
283284

284-
// create additional manifests -> use the effective (i.e. patched) namespace if no namespace is provided
285+
// create additional manifests -> use the effective f(i.e. patched) namespace if no namespace is provided
285286
createAdditionalManifests(proxy, effectiveKubeNamespace);
286287

288+
// tell the status service we are starting the pod/container
289+
proxyStatusService.containerStarting(proxy, container);
290+
291+
// create and start the pod
287292
Pod startedPod = kubeClient.pods().inNamespace(effectiveKubeNamespace).create(patchedPod);
288293

289294
int totalWaitMs = Integer.parseInt(environment.getProperty("proxy.kubernetes.pod-wait-time", "60000"));
@@ -300,8 +305,11 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
300305
Pod pod = kubeClient.resource(startedPod).fromServer().get();
301306
container.getParameters().put(PARAM_POD, pod);
302307
proxy.getContainers().add(container);
308+
309+
proxyStatusService.containerStartupFailed(proxy, container);
303310
throw new ContainerProxyException("Container did not become ready in time");
304311
}
312+
proxyStatusService.containerStarted(proxy, container);
305313
Pod pod = kubeClient.resource(startedPod).fromServer().get();
306314

307315
Service service = null;

‎src/main/java/eu/openanalytics/containerproxy/event/ProxyStartEvent.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,15 @@
2626

2727
public class ProxyStartEvent extends ApplicationEvent {
2828

29+
private final String proxyId;
2930
private final String userId;
3031
private final String specId;
31-
private final Duration startupTime;
3232

33-
public ProxyStartEvent(Object source, String userId, String specId, Duration startupTime) {
33+
public ProxyStartEvent(Object source, String proxyId, String userId, String specId) {
3434
super(source);
35+
this.proxyId = proxyId;
3536
this.userId = userId;
3637
this.specId = specId;
37-
this.startupTime = startupTime;
3838
}
3939

4040
public String getUserId() {
@@ -45,7 +45,7 @@ public String getSpecId() {
4545
return specId;
4646
}
4747

48-
public Duration getStartupTime() {
49-
return startupTime;
48+
public String getProxyId() {
49+
return proxyId;
5050
}
5151
}

‎src/main/java/eu/openanalytics/containerproxy/model/runtime/Container.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@
2929

3030
public class Container {
3131

32+
/**
33+
* Index in the array of ContainerSpecs of the ProxySpec.
34+
*/
35+
private Integer index;
3236
private String id;
3337
private ContainerSpec spec;
3438

@@ -37,12 +41,15 @@ public class Container {
3741
public String getId() {
3842
return id;
3943
}
44+
4045
public void setId(String id) {
4146
this.id = id;
4247
}
48+
4349
public ContainerSpec getSpec() {
4450
return spec;
4551
}
52+
4653
public void setSpec(ContainerSpec spec) {
4754
this.spec = spec;
4855
}
@@ -51,8 +58,16 @@ public void setSpec(ContainerSpec spec) {
5158
public Map<String, Object> getParameters() {
5259
return parameters;
5360
}
61+
5462
public void setParameters(Map<String, Object> parameters) {
5563
this.parameters = parameters;
5664
}
5765

66+
public Integer getIndex() {
67+
return index;
68+
}
69+
70+
public void setIndex(Integer index) {
71+
this.index = index;
72+
}
5873
}

‎src/main/java/eu/openanalytics/containerproxy/model/runtime/ProxyStartupLog.java‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
package eu.openanalytics.containerproxy.model.runtime;
2+
3+
import java.time.Duration;
4+
import java.time.LocalDateTime;
5+
import java.util.HashMap;
6+
import java.util.Map;
7+
import java.util.Objects;
8+
import java.util.Optional;
9+
10+
public class ProxyStartupLog {
11+
12+
private final StartupStep createProxy = new StartupStep();
13+
private final Map<Integer, StartupStep> scheduleContainer = new HashMap<>();
14+
private final Map<Integer, StartupStep> pullImage = new HashMap<>();
15+
private final Map<Integer, StartupStep> startContainer = new HashMap<>();
16+
private final Map<Integer, StartupStep> startApplication = new HashMap<>();
17+
18+
public StartupStep getCreateProxy() {
19+
return createProxy;
20+
}
21+
22+
public StartupStep getScheduleContainer(Integer containerIdx) {
23+
Objects.requireNonNull(containerIdx, "ContainerIdx may not be null");
24+
return scheduleContainer.computeIfAbsent(containerIdx, s -> new StartupStep());
25+
}
26+
27+
public StartupStep getPullImage(Integer containerIdx) {
28+
Objects.requireNonNull(containerIdx, "ContainerIdx may not be null");
29+
return pullImage.computeIfAbsent(containerIdx, s -> new StartupStep());
30+
}
31+
32+
public StartupStep getStartContainer(Integer containerIdx) {
33+
Objects.requireNonNull(containerIdx, "ContainerIdx may not be null");
34+
return startContainer.computeIfAbsent(containerIdx, s -> new StartupStep());
35+
}
36+
37+
public StartupStep getStartApplication(Integer containerIdx) {
38+
Objects.requireNonNull(containerIdx, "ContainerIdx may not be null");
39+
return startApplication.computeIfAbsent(containerIdx, s -> new StartupStep());
40+
}
41+
42+
public Map<Integer, StartupStep> getStartContainer() {
43+
return startContainer;
44+
}
45+
46+
public Map<Integer, StartupStep> getStartApplication() {
47+
return startApplication;
48+
}
49+
50+
public static class StartupStep {
51+
52+
private LocalDateTime startTime = null;
53+
private LocalDateTime endTime = null;
54+
private StartupStepState state = StartupStepState.NOT_EXECUTED;
55+
56+
public void stepStarted() {
57+
if (state != StartupStepState.NOT_EXECUTED) {
58+
throw new IllegalStateException("Cannot start step if it's already started!");
59+
}
60+
startTime = LocalDateTime.now();
61+
state = StartupStepState.STARTED;
62+
}
63+
64+
public void stepSucceeded() {
65+
if (state != StartupStepState.STARTED) {
66+
throw new IllegalStateException("Cannot finish (with success) step if it is not yet started or already completed");
67+
}
68+
endTime = LocalDateTime.now();
69+
state = StartupStepState.SUCCESS;
70+
}
71+
72+
public void stepFailed() {
73+
if (state != StartupStepState.STARTED) {
74+
throw new IllegalStateException("Cannot finish (with failure) step if it is not yet started or already completed");
75+
}
76+
endTime = LocalDateTime.now();
77+
state = StartupStepState.FAILURE;
78+
}
79+
80+
public LocalDateTime getStartTime() {
81+
return startTime;
82+
}
83+
84+
public LocalDateTime getEndTime() {
85+
return endTime;
86+
}
87+
88+
public Optional<Duration> getStepDuration() {
89+
if (endTime == null || startTime == null) {
90+
return Optional.empty();
91+
}
92+
return Optional.of(Duration.between(startTime, endTime));
93+
}
94+
95+
public StartupStepState getState() {
96+
return state;
97+
}
98+
99+
}
100+
101+
public enum StartupStepState {
102+
NOT_EXECUTED,
103+
STARTED,
104+
SUCCESS,
105+
FAILURE
106+
}
107+
108+
}

‎src/main/java/eu/openanalytics/containerproxy/model/spec/ContainerSpec.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,16 @@
2121
package eu.openanalytics.containerproxy.model.spec;
2222

2323
import java.util.ArrayList;
24-
import java.util.Arrays;
2524
import java.util.HashMap;
2625
import java.util.List;
2726
import java.util.Map;
2827

2928
public class ContainerSpec {
3029

30+
/**
31+
* Index in the array of ContainerSpecs of the ProxySpec.
32+
*/
33+
private Integer index;
3134
private String image;
3235
private String[] cmd;
3336
private Map<String, String> env;
@@ -182,4 +185,11 @@ public void setDockerRegistryPassword(String dockerRegistryPassword) {
182185
this.dockerRegistryPassword = dockerRegistryPassword;
183186
}
184187

188+
public Integer getIndex() {
189+
return index;
190+
}
191+
192+
public void setIndex(Integer index) {
193+
this.index = index;
194+
}
185195
}

‎src/main/java/eu/openanalytics/containerproxy/model/spec/ProxySpec.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,9 @@ public ContainerSpec getContainerSpec(String image) {
103103

104104
public void setContainerSpecs(List<ContainerSpec> containerSpecs) {
105105
this.containerSpecs = containerSpecs;
106+
for (int i = 0; i < this.containerSpecs.size(); i++) {
107+
this.containerSpecs.get(i).setIndex(i);
108+
}
106109
}
107110

108111
/**

‎src/main/java/eu/openanalytics/containerproxy/service/AppRecoveryService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,8 @@ public void recoverRunningApps() throws Exception {
123123
Proxy proxy = proxies.get(proxyId);
124124
Container container = new Container();
125125
container.setId(containerInfo.getContainerId());
126+
// TODO
127+
// container.setIndex(spec.getIndex());
126128
container.setParameters(containerInfo.getParameters());
127129
ContainerSpec containerSpec = proxy.getSpec().getContainerSpec(containerInfo.getImage());
128130
if (containerSpec == null) {

0 commit comments

Comments
 (0)