Skip to content

Commit 237fe44

Browse files
LEDfanLEDfan
committed
Fix #34125: add appInfo and delegate_app_info metrics
1 parent 23ddc8a commit 237fe44

11 files changed

Lines changed: 290 additions & 41 deletions

File tree

‎src/main/java/eu/openanalytics/containerproxy/backend/dispatcher/proxysharing/ProxySharingDispatcher.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import eu.openanalytics.containerproxy.model.runtime.ProxyStartupLog;
3636
import eu.openanalytics.containerproxy.model.runtime.ProxyStatus;
3737
import eu.openanalytics.containerproxy.model.runtime.ProxyStopReason;
38+
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.BackendContainerNameKey;
3839
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.PublicPathKey;
3940
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.RuntimeValue;
4041
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.RuntimeValueKeyRegistry;
@@ -175,7 +176,10 @@ public Proxy startProxy(Authentication user, Proxy proxy, ProxySpec spec, ProxyS
175176
resultProxy.addRuntimeValue(new RuntimeValue(TargetIdKey.inst, delegateProxy.getId()), true);
176177
resultProxy.addRuntimeValue(new RuntimeValue(SeatIdKey.inst, seat.getId()), true);
177178

178-
Container resultContainer = proxy.getContainer(0).toBuilder().id(UUID.randomUUID().toString()).build();
179+
Container resultContainer = proxy.getContainer(0).toBuilder()
180+
.id(UUID.randomUUID().toString())
181+
.addRuntimeValue(new RuntimeValue(BackendContainerNameKey.inst, delegateProxy.getContainers().get(0).getRuntimeObjectOrNull(BackendContainerNameKey.inst)), true)
182+
.build();
179183
resultProxy.updateContainer(resultContainer);
180184

181185
return resultProxy.build();

‎src/main/java/eu/openanalytics/containerproxy/backend/dispatcher/proxysharing/ProxySharingMicrometer.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,20 +20,33 @@
2020
*/
2121
package eu.openanalytics.containerproxy.backend.dispatcher.proxysharing;
2222

23+
import eu.openanalytics.containerproxy.backend.dispatcher.proxysharing.store.DelegateProxy;
24+
import eu.openanalytics.containerproxy.model.runtime.Proxy;
25+
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.BackendContainerName;
26+
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.BackendContainerNameKey;
2327
import eu.openanalytics.containerproxy.stat.IStatCollector;
28+
import io.micrometer.core.instrument.Gauge;
2429
import io.micrometer.core.instrument.MeterRegistry;
2530
import io.micrometer.core.instrument.Tags;
31+
import io.micrometer.core.instrument.search.MeterNotFoundException;
2632
import org.springframework.beans.factory.annotation.Autowired;
2733

2834
import javax.annotation.PostConstruct;
2935
import javax.inject.Inject;
3036
import java.time.Duration;
3137
import java.util.ArrayList;
38+
import java.util.HashMap;
3239
import java.util.List;
40+
import java.util.Map;
41+
import java.util.Timer;
42+
import java.util.TimerTask;
3343
import java.util.function.ToDoubleFunction;
44+
import java.util.stream.Collectors;
3445

3546
public class ProxySharingMicrometer implements IStatCollector {
3647

48+
private static final int CACHE_UPDATE_INTERVAL = 20 * 1000; // update cache every 20 seconds
49+
3750
@Inject
3851
private MeterRegistry registry;
3952

@@ -71,12 +84,68 @@ public void init() {
7184
registry.gauge("seats_claimed", Tags.of("spec.id", specId), scaler, wrapHandleNull(ProxySharingScaler::getNumClaimedSeats));
7285
registry.gauge("seats_creating", Tags.of("spec.id", specId), scaler, wrapHandleNull(ProxySharingScaler::getNumPendingSeats));
7386
}
87+
new Timer().schedule(new TimerTask() {
88+
@Override
89+
public void run() {
90+
updateDelegateAppInfo();
91+
}
92+
}, 0, CACHE_UPDATE_INTERVAL);
7493
}
7594

7695
public void registerSeatWaitTime(String specId, Duration time) {
7796
registry.timer("seats_wait_time", "spec.id", specId).record(time);
7897
}
7998

99+
private void updateDelegateAppInfo() {
100+
Map<String, Gauge> existingGauges = getDelegateAppInfoGauges();
101+
for (ProxySharingScaler scaler : proxySharingScalers) {
102+
String specId = scaler.getSpec().getId();
103+
for (DelegateProxy delegateProxy : scaler.getAllDelegateProxies()) {
104+
Proxy proxy = delegateProxy.getProxy();
105+
if (existingGauges.remove(proxy.getId()) != null) {
106+
// gauge already exists, no need to re-create
107+
continue;
108+
}
109+
110+
BackendContainerName backendContainerName = getBackendContainerName(proxy);
111+
if (backendContainerName == null) {
112+
// container not fully ready, will be registered later
113+
continue;
114+
}
115+
116+
registry.gauge("delegate_app_info",
117+
Tags.of(
118+
"spec.id", specId,
119+
"proxy.id", proxy.getId(),
120+
"proxy.created.timestamp", Long.toString(proxy.getCreatedTimestamp()),
121+
"resource.id", backendContainerName.getName(),
122+
"proxy.namespace", backendContainerName.getNamespace()),
123+
1
124+
);
125+
}
126+
}
127+
for (Gauge gauge : existingGauges.values()) {
128+
// the proxy of this gauge no longer exists -> remove the gauge
129+
registry.remove(gauge);
130+
}
131+
}
132+
133+
private Map<String, Gauge> getDelegateAppInfoGauges() {
134+
try {
135+
return new HashMap<>(registry.get("delegate_app_info").gauges().stream()
136+
.collect(Collectors.toMap(g -> g.getId().getTag("proxy.id"), g -> g)));
137+
} catch (MeterNotFoundException ignored) {
138+
return new HashMap<>();
139+
}
140+
}
141+
142+
private BackendContainerName getBackendContainerName(Proxy proxy) {
143+
if (!proxy.getContainers().isEmpty()) {
144+
return proxy.getContainers().get(0).getRuntimeObjectOrNull(BackendContainerNameKey.inst);
145+
}
146+
return null;
147+
}
148+
80149
@FunctionalInterface
81150
private interface ToLongFunction<T> {
82151
Long applyAsDouble(T var1);

‎src/main/java/eu/openanalytics/containerproxy/backend/dispatcher/proxysharing/ProxySharingScaler.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@
7171
import org.springframework.scheduling.annotation.Scheduled;
7272

7373
import javax.annotation.PostConstruct;
74-
import javax.annotation.PreDestroy;
7574
import javax.inject.Inject;
7675
import java.time.Duration;
7776
import java.time.Instant;
@@ -228,6 +227,10 @@ public void onRemoveDelegateProxiesEvent(RemoveDelegateProxiesEvent event) {
228227
}
229228
}
230229

230+
public Collection<DelegateProxy> getAllDelegateProxies() {
231+
return delegateProxyStore.getAllDelegateProxies();
232+
}
233+
231234
/**
232235
* Processes the SeatReleasedEvent, should only process one event a a time (i.e. using the event loop),
233236
* since it modifies the Delegateproxy.

‎src/main/java/eu/openanalytics/containerproxy/backend/docker/DockerEngineBackend.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import eu.openanalytics.containerproxy.model.runtime.PortMappings;
2727
import eu.openanalytics.containerproxy.model.runtime.Proxy;
2828
import eu.openanalytics.containerproxy.model.runtime.ProxyStartupLog;
29+
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.BackendContainerName;
2930
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.BackendContainerNameKey;
3031
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.ContainerImageKey;
3132
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.InstanceIdKey;
@@ -191,7 +192,7 @@ public Proxy startContainer(Authentication user, Container initialContainer, Con
191192
}
192193

193194
dockerClient.startContainer(containerCreation.id());
194-
rContainerBuilder.addRuntimeValue(new RuntimeValue(BackendContainerNameKey.inst, containerName), false);
195+
rContainerBuilder.addRuntimeValue(new RuntimeValue(BackendContainerNameKey.inst, new BackendContainerName(containerName)), false);
195196
proxyStartupLogBuilder.containerStarted(initialContainer.getIndex());
196197

197198
Container rContainer = rContainerBuilder.build();
@@ -275,7 +276,7 @@ public List<ExistingContainerInfo> scanExistingContainers() throws Exception {
275276
continue;
276277
}
277278
runtimeValues.put(ContainerImageKey.inst, new RuntimeValue(ContainerImageKey.inst, container.image()));
278-
runtimeValues.put(BackendContainerNameKey.inst, new RuntimeValue(BackendContainerNameKey.inst, container.id()));
279+
runtimeValues.put(BackendContainerNameKey.inst, new RuntimeValue(BackendContainerNameKey.inst, new BackendContainerName(container.id())));
279280

280281
// add ports to PortAllocator (even if we don't recover the proxy)
281282
for (org.mandas.docker.client.messages.Container.PortMapping portMapping : container.ports()) {

‎src/main/java/eu/openanalytics/containerproxy/backend/docker/DockerSwarmBackend.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import eu.openanalytics.containerproxy.model.runtime.PortMappings;
2828
import eu.openanalytics.containerproxy.model.runtime.Proxy;
2929
import eu.openanalytics.containerproxy.model.runtime.ProxyStartupLog;
30+
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.BackendContainerName;
3031
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.BackendContainerNameKey;
3132
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.ContainerImageKey;
3233
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.InstanceIdKey;
@@ -211,7 +212,7 @@ public Proxy startContainer(Authentication user, Container initialContainer, Con
211212

212213
// tell the status service we are starting the container
213214
proxyStartupLogBuilder.startingContainer(initialContainer.getIndex());
214-
rContainerBuilder.addRuntimeValue(new RuntimeValue(BackendContainerNameKey.inst, serviceName), false);
215+
rContainerBuilder.addRuntimeValue(new RuntimeValue(BackendContainerNameKey.inst, new BackendContainerName(serviceName)), false);
215216

216217
// Give the service some time to start up and launch a container.
217218
boolean containerFound = Retrying.retry((currentAttempt, maxAttempts) -> {
@@ -294,10 +295,10 @@ private String getSecretId(String secretName) throws DockerException, Interrupte
294295
@Override
295296
protected void doStopProxy(Proxy proxy) throws Exception {
296297
for (Container container : proxy.getContainers()) {
297-
String serviceId = container.getRuntimeObjectOrNull(BackendContainerNameKey.inst);
298+
BackendContainerName serviceId = container.getRuntimeObjectOrNull(BackendContainerNameKey.inst);
298299
if (serviceId != null) {
299300
try {
300-
dockerClient.removeService(serviceId);
301+
dockerClient.removeService(serviceId.getName());
301302
} catch (ServiceNotFoundException e) {
302303
// ignore, service is already removed
303304
}
@@ -328,7 +329,7 @@ public List<ExistingContainerInfo> scanExistingContainers() throws Exception {
328329
continue;
329330
}
330331
runtimeValues.put(ContainerImageKey.inst, new RuntimeValue(ContainerImageKey.inst, containersInService.get(0).image()));
331-
runtimeValues.put(BackendContainerNameKey.inst, new RuntimeValue(BackendContainerNameKey.inst, service.id()));
332+
runtimeValues.put(BackendContainerNameKey.inst, new RuntimeValue(BackendContainerNameKey.inst, new BackendContainerName(service.id())));
332333

333334
String containerInstanceId = runtimeValues.get(InstanceIdKey.inst).getObject();
334335
if (!appRecoveryService.canRecoverProxy(containerInstanceId)) {
@@ -363,11 +364,11 @@ public boolean isProxyHealthy(Proxy proxy) {
363364
public BiConsumer<OutputStream, OutputStream> getOutputAttacher(Proxy proxy) {
364365
Container container = getPrimaryContainer(proxy);
365366
if (container == null) return null;
366-
String serviceId = container.getRuntimeObjectOrNull(BackendContainerNameKey.inst);
367+
BackendContainerName serviceId = container.getRuntimeObjectOrNull(BackendContainerNameKey.inst);
367368

368369
return (stdOut, stdErr) -> {
369370
try {
370-
LogStream logStream = dockerClient.serviceLogs(serviceId, DockerClient.LogsParam.follow(), DockerClient.LogsParam.stdout(), DockerClient.LogsParam.stderr());
371+
LogStream logStream = dockerClient.serviceLogs(serviceId.getName(), DockerClient.LogsParam.follow(), DockerClient.LogsParam.stdout(), DockerClient.LogsParam.stderr());
371372
logStream.attach(stdOut, stdErr);
372373
} catch (ClosedChannelException ignored) {
373374
} catch (IOException | InterruptedException | DockerException e) {

‎src/main/java/eu/openanalytics/containerproxy/backend/ecs/EcsBackend.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import eu.openanalytics.containerproxy.model.runtime.PortMappings;
2828
import eu.openanalytics.containerproxy.model.runtime.Proxy;
2929
import eu.openanalytics.containerproxy.model.runtime.ProxyStartupLog;
30+
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.BackendContainerName;
3031
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.BackendContainerNameKey;
3132
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.ContainerImageKey;
3233
import eu.openanalytics.containerproxy.model.runtime.runtimevalues.PortMappingsKey;
@@ -243,7 +244,7 @@ public Proxy startContainer(Authentication user, Container initialContainer, Con
243244
proxyStartupLogBuilder.containerStarted(initialContainer.getIndex());
244245

245246
String image = ecsClient.describeTasks(builder -> builder.cluster(cluster).tasks(taskArn)).tasks().get(0).containers().get(0).image();
246-
rContainerBuilder.addRuntimeValue(new RuntimeValue(BackendContainerNameKey.inst, taskArn), false);
247+
rContainerBuilder.addRuntimeValue(new RuntimeValue(BackendContainerNameKey.inst, new BackendContainerName(taskArn)), false);
247248
rContainerBuilder.addRuntimeValue(new RuntimeValue(ContainerImageKey.inst, image), false);
248249

249250
if (!serviceReady) {
@@ -478,11 +479,11 @@ private Optional<Task> getTask(Container container) {
478479
}
479480

480481
private Optional<String> getTaskInfo(Container container) {
481-
String taskId = container.getRuntimeObjectOrNull(BackendContainerNameKey.inst);
482+
BackendContainerName taskId = container.getRuntimeObjectOrNull(BackendContainerNameKey.inst);
482483
if (taskId == null) {
483484
return Optional.empty();
484485
}
485-
return Optional.of(taskId);
486+
return Optional.of(taskId.getName());
486487
}
487488

488489
private Optional<Task> getTask(String taskInfo) {

0 commit comments

Comments
 (0)