feat(service): replace/refactor internal repository cache (#3534)

Author: Panaetius

cache-cleanup-job/Dockerfile

@@ -0,0 +1,12 @@
+# Docker image for core-svc cronjob
+FROM alpine:3.18.2
+RUN apk add --no-cache ca-certificates=20230506-r0 curl=8.1.2-r0 bash=5.2.15-r5 && rm -rf /var/cache/apk/*
+RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" &&\
+    chmod +x ./kubectl &&\
+    mv ./kubectl /usr/bin/kubectl
+
+RUN mkdir /code
+WORKDIR /code
+COPY cleanup.sh /code/
+
+ENTRYPOINT ["/bin/bash", "/code/cleanup.sh"]

cache-cleanup-job/README.md

@@ -0,0 +1,6 @@
+# Core Service cache cleanup image
+Small image to be used for the cache cleanup CronJob for the core service.
+
+Loops through endpoint slices and call the cleanup endpoint on each core-svc instance.
+
+Push as `renku/renku-core-cleanup:<version>` to use

cache-cleanup-job/cleanup.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+
+core_version=$1
+namespace=$2
+
+mapfile -t -d " " pod_ips < <(kubectl -n "$namespace" get pods --selector="app.kubernetes.io/name=core" --selector="app.kubernetes.io/deploymentVersion=$core_version" -o=jsonpath="{.items[*].status.podIP}" )
+
+success=true
+
+for pod_ip in "${pod_ips[@]}"
+do
+    echo "Calling http://$pod_ip:8080/renku/cache.cleanup"
+    if curl "http://$pod_ip:8080/renku/cache.cleanup" ; then
+        :
+    else
+        echo "Cleanup failed for pod $pod_ip with status $?">&2
+        success=false
+    fi
+done
+
+if ! $success; then
+    exit 1;
+fi

conftest.py

@@ -58,7 +58,6 @@
     "tests.service.fixtures.service_integration",
     "tests.service.fixtures.service_jobs",
     "tests.service.fixtures.service_projects",
-    "tests.service.fixtures.service_scheduler",
 ]
 
 INCLUDE_FIXTURES = GLOBAL_FIXTURE_LOCATIONS + CORE_FIXTURE_LOCATIONS + CLI_FIXTURE_LOCATIONS + SERVICE_FIXTURE_LOCATIONS

docker-compose.yml

@@ -87,19 +87,6 @@ services:
       - traefik.http.routers.swagger.rule=PathPrefix(`/api/docs`)
       - traefik.http.services.my-service.loadbalancer.server.port=8080
 
-  renku-scheduler:
-    build:
-      context: .
-      dockerfile: Dockerfile
-      args:
-        BUILD_CORE_SERVICE: 1
-    command: ["service", "scheduler"]
-    depends_on:
-      - redis
-    networks:
-      - net
-    env_file: .env
-
   renku-worker:
     build:
       context: .

helm-chart/renku-core/templates/cache-cleanup-job.yaml

@@ -0,0 +1,27 @@
+{{- range $version := .Values.versions }}
+{{ if ne $version.name "v9"}}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "renku-core.fullname" $ }}-cleanup-{{ $version.name }}
+  labels:
+    app.kubernetes.io/deploymentVersion: {{ $version.name }}
+spec:
+  schedule: "*/5 * * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: {{ include "renku-core.fullname" $ }}-cache-cleanup-{{ $version.name }}
+            image: renku/renku-core-cleanup:v1
+            imagePullPolicy: IfNotPresent
+            args:
+            - {{ $version.name | quote}}
+            - {{ $.Release.Namespace }}
+          restartPolicy: OnFailure
+          serviceAccountName: {{ include "renku-core.fullname" $ }}-cleanup
+{{ end }}
+{{ end }}

helm-chart/renku-core/templates/cronjob-serviceaccount.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "renku-core.fullname" $ }}-cleanup
+  labels:
+{{ include "renku-core.labels" $ | indent 4 }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "renku-core.fullname" $ }}-cleanup
+  labels:
+{{ include "renku-core.labels" $ | indent 4 }}
+rules:
+- apiGroups:
+    - ""
+  resources:
+    - pods
+  verbs:
+    - get
+    - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "renku-core.fullname" $ }}-cleanup
+  labels:
+{{ include "renku-core.labels" $ | indent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "renku-core.fullname" $ }}-cleanup
+subjects:
+- kind: ServiceAccount
+  name: {{ include "renku-core.fullname" $ }}-cleanup
+  namespace: {{ $.Release.Namespace }}

helm-chart/renku-core/templates/deployment.yaml

@@ -279,48 +279,6 @@ spec:
             - name: shared-volume
               mountPath: {{ $.Values.cacheDirectory }}
             {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
-          resources:
-            {{- toYaml $.Values.resources.managementWorkers | nindent 12 }}
-
-        - name: {{ $.Chart.Name }}-scheduler
-          image: "{{ $version.image.repository }}:{{ $version.image.tag }}"
-          imagePullPolicy: {{ $version.image.pullPolicy }}
-          securityContext:
-            {{- toYaml $.Values.securityContext | nindent 12 }}
-          args: ["service", "scheduler"]
-          env:
-            - name: REDIS_HOST
-              value: {{ $.Values.global.redis.host | quote }}
-            - name: REDIS_PORT
-              value: {{ $.Values.global.redis.port | quote }}
-            - name: REDIS_DATABASE
-              value: {{ $.Values.global.redis.dbIndex.coreService | quote }}
-            - name: REDIS_IS_SENTINEL
-              value: {{ $.Values.global.redis.sentinel.enabled | quote }}
-            - name: REDIS_MASTER_SET
-              value: {{ $.Values.global.redis.sentinel.masterSet | quote }}
-            - name: REDIS_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ $.Values.global.redis.existingSecret }}
-                  key: {{ $.Values.global.redis.existingSecretPasswordKey }}
-            - name: REDIS_NAMESPACE
-              value: {{ $version.name }}
-            - name: CACHE_DIR
-              value: {{ $.Values.cacheDirectory | quote }}
-            - name: RENKU_SVC_CLEANUP_INTERVAL
-              value: {{ $.Values.cleanupInterval | quote }}
-            - name: SENTRY_ENABLED
-              value: {{ $.Values.sentry.enabled | quote }}
-            - name: SENTRY_DSN
-              value: {{ $.Values.sentry.dsn }}
-            - name: SENTRY_SAMPLE_RATE
-              value: {{ $.Values.sentry.sampleRate | quote }}
-            - name: SENTRY_ENV
-              value: {{ $.Values.sentry.environment }}
-            {{- include "certificates.env.python" $ | nindent 12 }}
-          volumeMounts:
-            {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
           resources:
             {{- toYaml $.Values.resources.scheduler | nindent 12 }}
       {{- with $.Values.nodeSelector }}