Skip to content

Commit 1071011

Browse files
LEDfanLEDfan
committed
Fix #34870: keep log files of ShinyProxy
1 parent dcc1520 commit 1071011

9 files changed

Lines changed: 201 additions & 16 deletions

File tree

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/FileManager.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,4 +79,10 @@ class FileManager {
7979
path.deleteRecursively()
8080
}
8181

82+
fun isDirectoryEmpty(path: Path): Boolean {
83+
Files.newDirectoryStream(path).use { dirStream ->
84+
return !dirStream.iterator().hasNext()
85+
}
86+
}
87+
8288
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/controller/ShinyProxyController.kt‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,11 @@ class ShinyProxyController(
253253
}
254254

255255
private fun instanceFailure(shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance, message: String?) {
256-
logger.info { "${logPrefix(shinyProxyInstance)} Instance failed to start up" }
256+
if (message != null) {
257+
logger.info { "${logPrefix(shinyProxyInstance)} Instance failed to start up, output: $message" }
258+
} else {
259+
logger.info { "${logPrefix(shinyProxyInstance)} Instance failed to start up" }
260+
}
257261
eventController.createInstanceFailed(shinyProxyInstance, message)
258262
}
259263

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CraneConfig.kt‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
*/
2121
package eu.openanalytics.shinyproxyoperator.impl.docker
2222

23+
import com.fasterxml.jackson.databind.DeserializationFeature
2324
import com.fasterxml.jackson.databind.JsonNode
2425
import com.fasterxml.jackson.databind.ObjectMapper
2526
import com.fasterxml.jackson.databind.SerializationFeature
@@ -45,6 +46,7 @@ import org.mandas.docker.client.messages.ContainerConfig
4546
import org.mandas.docker.client.messages.HostConfig
4647
import java.nio.file.Path
4748
import java.util.concurrent.ConcurrentHashMap
49+
import kotlin.io.path.createDirectories
4850
import kotlin.io.path.exists
4951
import kotlin.io.path.readText
5052

@@ -61,10 +63,11 @@ class CraneConfig(private val dockerClient: DockerClient,
6163
private val logger = KotlinLogging.logger { }
6264
private val scope = CoroutineScope(Dispatchers.Default)
6365
private val deletedContainers = ConcurrentHashMap.newKeySet<String>()
64-
private val craneReadyChecker = CraneReadyChecker()
66+
private val craneReadyChecker = CraneReadyChecker(dockerClient, dataDir)
6567

6668
init {
6769
yamlMapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true)
70+
yamlMapper.enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY)
6871
}
6972

7073

@@ -80,7 +83,7 @@ class CraneConfig(private val dockerClient: DockerClient,
8083
return
8184
}
8285

83-
val spec = yamlMapper.readValue<JsonNode>(config)
86+
val spec = parseConfig(shinyProxy, config) ?: return
8487
val hash = hash(spec)
8588
var containerId = dockerActions.getCraneContainer(shinyProxy, hash, deletedContainers.toList())?.id()
8689
if (containerId == null) {
@@ -92,6 +95,8 @@ class CraneConfig(private val dockerClient: DockerClient,
9295
val containerName = "sp-crane-${shinyProxyInstance.realmId}-${hash}-${suffix}"
9396

9497
val dir = dataDir.resolve(containerName)
98+
val logsDir = dataDir.resolve("logs").resolve(containerName)
99+
logsDir.createDirectories()
95100
withContext(Dispatchers.IO) {
96101
fileManager.createDirectories(dir)
97102
fileManager.writeFile(
@@ -114,7 +119,11 @@ class CraneConfig(private val dockerClient: DockerClient,
114119
.from(dir.resolve("generated.yml").toString())
115120
.to("/opt/crane/generated.yml")
116121
.readOnly(true)
117-
.build()
122+
.build(),
123+
HostConfig.Bind.builder()
124+
.from(logsDir.toString())
125+
.to("/opt/crane/logs")
126+
.build(),
118127
)
119128

120129
val mount = getMount(spec)
@@ -145,8 +154,9 @@ class CraneConfig(private val dockerClient: DockerClient,
145154

146155
if (containerId != null) {
147156
val ip = getIp(containerId, shinyProxy) ?: return
148-
craneReadyChecker.add(ip, shinyProxy.realmId, hash)
149-
if (craneReadyChecker.isReady(shinyProxy.realmId, hash) == CraneReadyChecker.TaskStatus.READY) {
157+
craneReadyChecker.add(ip, shinyProxy.realmId, hash, containerId)
158+
val status = craneReadyChecker.isReady(shinyProxy.realmId, hash)
159+
if (status == CraneReadyChecker.TaskStatus.READY) {
150160
logger.info { "${logPrefix(shinyProxyInstance)} [Crane] Container ready" }
151161
caddyConfig.addCraneServer(shinyProxy.realmId, ip, getSubPath(spec))
152162
caddyConfig.reconcile()
@@ -156,6 +166,15 @@ class CraneConfig(private val dockerClient: DockerClient,
156166
}
157167
}
158168

169+
private fun parseConfig(shinyProxy: ShinyProxy, config: String): JsonNode? {
170+
try {
171+
return yamlMapper.readValue<JsonNode>(config)
172+
} catch (e: Exception) {
173+
logger.warn(e) { "${logPrefix(shinyProxy)} Failed to parse crane config" }
174+
return null
175+
}
176+
}
177+
159178
suspend fun remove(realmId: String) {
160179
val containers = dockerActions.getCraneContainers(realmId)
161180
for (container in containers) {
@@ -187,7 +206,7 @@ class CraneConfig(private val dockerClient: DockerClient,
187206
logger.warn { "${logPrefix(shinyProxy)} [Crane] No hash label found for container" }
188207
return
189208
}
190-
craneReadyChecker.add(ip, shinyProxy.realmId, hashOfContainer)
209+
craneReadyChecker.add(ip, shinyProxy.realmId, hashOfContainer, container.id())
191210
caddyConfig.addCraneServer(shinyProxy.realmId, ip, getSubPath(spec))
192211
} catch (e: Exception) {
193212
logger.warn(e) { "Error while initializing CraneConfig" }
@@ -241,6 +260,11 @@ class CraneConfig(private val dockerClient: DockerClient,
241260
)
242261
)
243262
),
263+
"logging" to hashMapOf<String, Any>(
264+
"file" to mapOf(
265+
"name" to "/opt/crane/logs/crane.log"
266+
)
267+
)
244268
)
245269
return yamlMapper.writeValueAsString(config)
246270
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/CraneReadyChecker.kt‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,13 @@ import kotlinx.coroutines.async
3232
import kotlinx.coroutines.delay
3333
import okhttp3.OkHttpClient
3434
import okhttp3.Request
35+
import org.mandas.docker.client.DockerClient
3536
import java.io.IOException
37+
import java.nio.file.Path
3638
import java.util.concurrent.ConcurrentHashMap
3739
import java.util.concurrent.TimeUnit
3840

39-
class CraneReadyChecker {
41+
class CraneReadyChecker(private val dockerClient: DockerClient, private val dataDir: Path) {
4042

4143
private val scope = CoroutineScope(Dispatchers.Default)
4244
private val tasks = ConcurrentHashMap<TaskKey, Deferred<TaskStatus>>()
@@ -54,10 +56,10 @@ class CraneReadyChecker {
5456
private const val MAX_CHECKS = 24
5557
}
5658

57-
fun add(ip: String, realmId: String, hashOfSpec: String) {
59+
fun add(ip: String, realmId: String, hashOfSpec: String, containerId: String) {
5860
tasks.computeIfAbsent(TaskKey(realmId, hashOfSpec)) {
5961
scope.async {
60-
checkInstance(ip, realmId)
62+
checkInstance(ip, realmId, containerId)
6163
}
6264
}
6365
}
@@ -74,7 +76,7 @@ class CraneReadyChecker {
7476
tasks.values.forEach { it.cancel() }
7577
}
7678

77-
private suspend fun checkInstance(ip: String, realmId: String): TaskStatus {
79+
private suspend fun checkInstance(ip: String, realmId: String, containerId: String): TaskStatus {
7880
for (checks in 0..MAX_CHECKS) {
7981
val resp = checkServer(ip)
8082
if (resp != null && resp.status.equals("up", ignoreCase = true)) {
@@ -84,7 +86,14 @@ class CraneReadyChecker {
8486
logger.info { "${logPrefix(realmId)} [Crane] Not ready yet (${checks}/${MAX_CHECKS})" }
8587
delay(5_000)
8688
}
87-
logger.info { "${logPrefix(realmId)} [Crane] Failed (${MAX_CHECKS}/${MAX_CHECKS})" }
89+
val containerName = dockerClient.inspectContainer(containerId)?.name()?.drop(1)
90+
val message = if (containerName != null) {
91+
val path = dataDir.resolve("logs").resolve(containerName).resolve("crane.log").toAbsolutePath().toString()
92+
"full log file available at '${path}'"
93+
} else {
94+
null
95+
}
96+
logger.info { "${logPrefix(realmId)} [Crane] Failed (${MAX_CHECKS}/${MAX_CHECKS}): $message" }
8897
return TaskStatus.FAILED
8998
}
9099

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/DockerOrchestrator.kt‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ import java.time.ZoneOffset
5858
import java.time.temporal.ChronoUnit
5959
import java.util.*
6060
import java.util.regex.Pattern
61+
import kotlin.io.path.createDirectories
6162

6263
class DockerOrchestrator(channel: Channel<ShinyProxyEvent>,
6364
config: Config,
@@ -85,6 +86,7 @@ class DockerOrchestrator(channel: Channel<ShinyProxyEvent>,
8586
.disable(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS)
8687
private val craneConfig: CraneConfig
8788
private val persistentState = PersistentState(dataDir)
89+
private val logFilesCleaner: LogFilesCleaner
8890

8991
init {
9092
objectMapper.registerKotlinModule()
@@ -99,6 +101,7 @@ class DockerOrchestrator(channel: Channel<ShinyProxyEvent>,
99101
redisConfig = RedisConfig(dockerClient, dockerActions, dataDir, config)
100102
craneConfig = CraneConfig(dockerClient, dockerActions, dataDir, inputDir, redisConfig, caddyConfig, persistentState)
101103
monitoringConfig = MonitoringConfig(dockerClient, dockerActions, dataDir, caddyConfig, config)
104+
logFilesCleaner = LogFilesCleaner(dataDir.resolve("logs"), fileManager, dockerActions)
102105
fileManager.createDirectories(dataDir)
103106
eventWriter = FileWriter(dataDir.resolve("events.json").toFile())
104107
}
@@ -195,6 +198,8 @@ class DockerOrchestrator(channel: Channel<ShinyProxyEvent>,
195198
val containerName = "sp-${shinyProxyInstance.realmId}-${shinyProxyInstance.hashOfSpec}-${shinyProxyInstance.revision}-${suffix}"
196199

197200
val dir = dataDir.resolve(containerName)
201+
val logsDir = dataDir.resolve("logs").resolve(containerName)
202+
logsDir.createDirectories()
198203
withContext(Dispatchers.IO) {
199204
fileManager.createDirectories(dir)
200205
fileManager.writeFile(
@@ -212,6 +217,7 @@ class DockerOrchestrator(channel: Channel<ShinyProxyEvent>,
212217
}
213218

214219
copyTemplates(shinyProxy, dir)
220+
fileManager.createDirectories(dir.resolve("logs"))
215221

216222
val hostConfigBuilder = HostConfig.builder()
217223
.networkMode(SHARED_NETWORK_NAME)
@@ -236,6 +242,10 @@ class DockerOrchestrator(channel: Channel<ShinyProxyEvent>,
236242
.to("/opt/shinyproxy/templates")
237243
.readOnly(true)
238244
.build(),
245+
HostConfig.Bind.builder()
246+
.from(logsDir.toString())
247+
.to("/opt/shinyproxy/logs")
248+
.build(),
239249
HostConfig.Bind.builder()
240250
.from(dir.resolve("termination-log").toString())
241251
.to("/dev/termination-log")
@@ -436,11 +446,13 @@ class DockerOrchestrator(channel: Channel<ShinyProxyEvent>,
436446
caddyConfig.addShinyProxy(shinyProxy, latestInstance) // not reconcile yet, wait for all ShinyProxy servers to be added
437447
craneConfig.init(shinyProxy)
438448
}
449+
logFilesCleaner.init()
439450
}
440451

441452
fun stop() {
442453
shinyProxyReadyChecker.stop()
443454
craneConfig.stop()
455+
logFilesCleaner.stop()
444456
}
445457

446458
private fun generateConfig(shinyProxy: ShinyProxy, networkName: String): String {
@@ -464,6 +476,11 @@ class DockerOrchestrator(channel: Channel<ShinyProxyEvent>,
464476
if (monitoringConfig.isEnabled()) {
465477
put("monitoring", mapOf("grafana-url" to monitoringConfig.grafanaConfig.getGrafanaUrl(shinyProxy)))
466478
}
479+
},
480+
"logging" to buildMap {
481+
put("file", buildMap {
482+
put("name", "/opt/shinyproxy/logs/shinyproxy.log")
483+
})
467484
}
468485
)
469486
return objectMapper.writeValueAsString(config)

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/LogFilesCleaner.kt‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
/*
2+
* ShinyProxy-Operator
3+
*
4+
* Copyright (C) 2021-2025 Open Analytics
5+
*
6+
* ===========================================================================
7+
*
8+
* This program is free software: you can redistribute it and/or modify
9+
* it under the terms of the Apache License as published by
10+
* The Apache Software Foundation, either version 2 of the License, or
11+
* (at your option) any later version.
12+
*
13+
* This program is distributed in the hope that it will be useful,
14+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
15+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16+
* Apache License for more details.
17+
*
18+
* You should have received a copy of the Apache License
19+
* along with this program. If not, see <http://www.apache.org/licenses/>
20+
*/
21+
package eu.openanalytics.shinyproxyoperator.impl.docker
22+
23+
import eu.openanalytics.shinyproxyoperator.FileManager
24+
import io.github.oshai.kotlinlogging.KotlinLogging
25+
import kotlinx.coroutines.runBlocking
26+
import java.io.IOException
27+
import java.nio.file.FileVisitResult
28+
import java.nio.file.FileVisitor
29+
import java.nio.file.Files
30+
import java.nio.file.Path
31+
import java.nio.file.attribute.BasicFileAttributes
32+
import java.time.Duration
33+
import java.time.Instant
34+
import java.util.*
35+
import kotlin.concurrent.timer
36+
37+
class LogFilesCleaner(private val path: Path, private val fileManager: FileManager, private val dockerActions: DockerActions) {
38+
39+
private val logger = KotlinLogging.logger {}
40+
private var timer: Timer? = null
41+
42+
fun init() {
43+
timer = timer(period = 60 * 60_000L, initialDelay = 60_000L) {
44+
runBlocking {
45+
try {
46+
cleanupDirectory(path, Duration.ofDays(7))
47+
} catch (e: Exception) {
48+
logger.warn(e) { "Failed to cleanup logs directory" }
49+
}
50+
}
51+
}
52+
}
53+
54+
private fun cleanupDirectory(path: Path, maxAge: Duration) {
55+
val now = Instant.now()
56+
Files.walkFileTree(path, object : FileVisitor<Path> {
57+
override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
58+
if (isUsedByContainer(dir)) {
59+
return FileVisitResult.SKIP_SUBTREE
60+
}
61+
return FileVisitResult.CONTINUE
62+
}
63+
64+
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
65+
try {
66+
val age = Duration.between(attrs.lastModifiedTime().toInstant(), now)
67+
if (age > maxAge) {
68+
logger.debug { "Deleting: $file" }
69+
Files.delete(file)
70+
}
71+
} catch (e: IOException) {
72+
logger.warn(e) { "Failed to delete file: $file" }
73+
}
74+
return FileVisitResult.CONTINUE
75+
}
76+
77+
override fun visitFileFailed(file: Path, exc: IOException): FileVisitResult {
78+
return FileVisitResult.CONTINUE
79+
}
80+
81+
override fun postVisitDirectory(dir: Path, exc: IOException?): FileVisitResult {
82+
try {
83+
if (fileManager.isDirectoryEmpty(dir)) {
84+
logger.debug { "Deleting: $dir" }
85+
Files.delete(dir)
86+
}
87+
} catch (e: IOException) {
88+
logger.warn(e) { "Failed to delete empty directory: $dir" }
89+
}
90+
return FileVisitResult.CONTINUE
91+
}
92+
})
93+
}
94+
95+
/**
96+
* Checks if the given path is currently in use as a container.
97+
* Assumes the directory name starts with 'sp-' and corresponds to the name of the container.
98+
*/
99+
private fun isUsedByContainer(path: Path): Boolean {
100+
if (path.fileName.toString().startsWith("sp-")) {
101+
try {
102+
if (dockerActions.getContainerByName(path.fileName.toString()) != null) {
103+
return true
104+
}
105+
} catch (_: Exception) {
106+
107+
}
108+
}
109+
return false
110+
}
111+
112+
113+
fun stop() {
114+
timer?.cancel()
115+
}
116+
117+
}

‎src/main/kotlin/eu/openanalytics/shinyproxyoperator/impl/docker/ShinyProxyReadyChecker.kt‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,13 @@ class ShinyProxyReadyChecker(private val channel: Channel<ShinyProxyEvent>, priv
100100
val status = check.await()
101101
if (!failed && status == TaskStatus.FAILED) {
102102
failed = true
103-
val message = readTerminationMessage(container)
103+
val containerName = container.name()
104+
val message = if (containerName != null) {
105+
val path = dataDir.resolve("logs").resolve(containerName).resolve("shinyproxy.log")
106+
"Full log file available at '${path}', last output: " + readTerminationMessage(container)
107+
} else {
108+
readTerminationMessage(container)
109+
}
104110
channel.send(ShinyProxyEvent(ShinyProxyEventType.FAILURE, shinyProxyInstance.realmId, shinyProxyInstance.name, shinyProxyInstance.namespace, shinyProxyInstance.hashOfSpec, message = message))
105111
}
106112
}

0 commit comments

Comments
 (0)