Add envoy proxy for per-beamline ingress routing

Charts/tiled/templates/NOTES.txt

@@ -1,22 +1,11 @@
 1. Get the application URL by running these commands:
 {{- if .Values.ingress.enabled }}
 {{- range $host := .Values.ingress.hosts }}
-  {{- range .paths }}
-  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- range $.Values.beamlines }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host }}/api/v1/search/{{ .name }}
   {{- end }}
 {{- end }}
-{{- else if contains "NodePort" .Values.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "tiled.fullname" . }})
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo http://$NODE_IP:$NODE_PORT
-{{- else if contains "LoadBalancer" .Values.service.type }}
-     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-           You can watch its status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "tiled.fullname" . }}'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "tiled.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo http://$SERVICE_IP:{{ .Values.service.port }}
-{{- else if contains "ClusterIP" .Values.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "tiled.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+{{- else }}
   echo "Visit http://127.0.0.1:8080 to use your application"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+  kubectl --namespace {{ .Release.Namespace }} port-forward svc/{{ include "tiled.fullname" . }}-envoy 8080:{{ .Values.envoy.service.port }}
 {{- end }}

Charts/tiled/templates/_helpers.tpl

@@ -60,3 +60,15 @@ Create the name of the service account to use
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{/*
+Name of a beamline's Deployment and Service. Called with (dict "root" $ "beamline" .).
+Fails rather than truncates, so two beamlines can never collide on one name.
+*/}}
+{{- define "tiled.beamlineFullname" -}}
+{{- $name := printf "%s-%s" (include "tiled.fullname" .root) .beamline.name }}
+{{- if gt (len $name) 63 }}
+{{- fail (printf "%q is longer than 63 characters, shorten the release or beamline name" $name) }}
+{{- end }}
+{{- $name }}
+{{- end }}

Charts/tiled/templates/deployment.yaml

@@ -1,67 +1,73 @@
+{{- range .Values.beamlines }}
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ include "tiled.fullname" . }}
+  name: {{ include "tiled.beamlineFullname" (dict "root" $ "beamline" .) }}
   labels:
-    {{- include "tiled.labels" . | nindent 4 }}
+    {{- include "tiled.labels" $ | nindent 4 }}
+    tiled.io/beamline: {{ .name }}
 spec:
-  {{- if not .Values.autoscaling.enabled }}
-  replicas: {{ .Values.replicaCount }}
+  {{- if not $.Values.autoscaling.enabled }}
+  replicas: {{ $.Values.replicaCount }}
   {{- end }}
   selector:
     matchLabels:
-      {{- include "tiled.selectorLabels" . | nindent 6 }}
+      {{- include "tiled.selectorLabels" $ | nindent 6 }}
+      tiled.io/beamline: {{ .name }}
   template:
     metadata:
       annotations:
-        {{- if .Values.rollOnConfigChange }}
-        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- if $.Values.rollOnConfigChange }}
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") $ | sha256sum }}
         {{- end }}
-        {{- with .Values.podAnnotations }}
+        {{- with $.Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
       labels:
-        {{- include "tiled.labels" . | nindent 8 }}
-        {{- with .Values.podLabels }}
+        {{- include "tiled.labels" $ | nindent 8 }}
+        tiled.io/beamline: {{ .name }}
+        {{- with $.Values.podLabels }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
     spec:
-      {{- with .Values.imagePullSecrets }}
+      {{- with $.Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      serviceAccountName: {{ include "tiled.serviceAccountName" . }}
+      serviceAccountName: {{ include "tiled.serviceAccountName" $ }}
+      {{- $beamlineSC := .securityContext | default dict }}
       securityContext:
-        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+        {{- toYaml (merge (pick $beamlineSC "supplementalGroups") $.Values.podSecurityContext) | nindent 8 }}
       containers:
-        - name: {{ .Chart.Name }}
+        - name: {{ $.Chart.Name }}
           securityContext:
-            {{- toYaml .Values.securityContext | nindent 12 }}
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          {{- with .Values.command }}
+            {{- toYaml (merge (omit $beamlineSC "supplementalGroups") $.Values.securityContext) | nindent 12 }}
+          image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag | default $.Chart.AppVersion }}"
+          imagePullPolicy: {{ $.Values.image.pullPolicy }}
+          {{- with $.Values.command }}
           command:
             {{- toYaml . | nindent 12 }}
           {{- end }}
-          {{- with .Values.args }}
+          {{- with $.Values.args }}
           args:
             {{- toYaml . | nindent 12 }}
           {{- end }}
           ports:
             - name: http
-              containerPort: {{ .Values.service.port }}
+              containerPort: 8000
               protocol: TCP
           livenessProbe:
-            {{- toYaml .Values.livenessProbe | nindent 12 }}
+            {{- toYaml (merge (.livenessProbe | default dict) $.Values.livenessProbe) | nindent 12 }}
           readinessProbe:
-            {{- toYaml .Values.readinessProbe | nindent 12 }}
+            {{- toYaml (merge (.readinessProbe | default dict) $.Values.readinessProbe) | nindent 12 }}
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
+            {{- toYaml $.Values.resources | nindent 12 }}
+          {{- with concat $.Values.volumeMounts (.volumeMounts | default list) }}
           volumeMounts:
-          {{- with .Values.volumeMounts }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
-          {{- with .Values.extraEnvVars }}
+          {{- with concat $.Values.extraEnvVars (.extraEnvVars | default list) }}
           env:
             {{- toYaml . | nindent 12 }}
           {{- end }}
@@ -70,19 +76,20 @@ spec:
           projected:
             sources:
               - configMap:
-                  name: {{ include "tiled.fullname" . }}-config
-      {{- with .Values.volumes }}
+                  name: {{ include "tiled.fullname" $ }}-config
+      {{- with concat $.Values.volumes (.volumes | default list) }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.nodeSelector }}
+      {{- with $.Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.affinity }}
+      {{- with $.Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.tolerations }}
+      {{- with $.Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+{{- end }}

Charts/tiled/templates/envoy-configmap.yaml

@@ -0,0 +1,97 @@
+{{- if .Values.envoy.enabled }}
+{{- /* A bad config stops envoy starting and takes the whole ingress down, so fail the render instead */}}
+{{- $names := list }}
+{{- range .Values.beamlines }}{{ $names = append $names .name }}{{ end }}
+{{- if not (has .Values.envoy.defaultBeamline $names) }}
+{{- fail (printf "envoy.defaultBeamline %q must be one of beamlines %v" .Values.envoy.defaultBeamline $names) }}
+{{- end }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "tiled.fullname" . }}-envoy-config
+  labels:
+    {{- include "tiled.labels" . | nindent 4 }}
+    app.kubernetes.io/component: envoy-proxy
+data:
+  envoy.yaml: |
+    # Only reachable from inside the pod (kubectl exec), so other pods can't
+    # e.g. POST /quitquitquit
+    admin:
+      address:
+        socket_address: { address: 127.0.0.1, port_value: {{ .Values.envoy.adminPort }} }
+    # The beamline regexes list every tiled endpoint, which is over RE2's default limit
+    layered_runtime:
+      layers:
+        - name: static
+          static_layer:
+            re2.max_program_size.error_level: 1000
+    static_resources:
+      listeners:
+        - name: listener_http
+          address:
+            socket_address: { address: 0.0.0.0, port_value: {{ .Values.envoy.listenerPort }} }
+          filter_chains:
+            - filters:
+                - name: envoy.filters.network.http_connection_manager
+                  typed_config:
+                    "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                    stat_prefix: ingress_http
+                    normalize_path: true
+                    merge_slashes: true
+                    # For the websockets of /api/v1/stream/single/...
+                    upgrade_configs:
+                      - upgrade_type: websocket
+                    access_log:
+                      - name: envoy.access_loggers.stdout
+                        typed_config:
+                          "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
+                    route_config:
+                      name: local_route
+                      virtual_hosts:
+                        - name: tiled_backend
+                          domains: ["*"]
+                          routes:
+                            - match:
+                                path: /envoy-ready
+                              direct_response:
+                                status: 200
+                            {{- range .Values.beamlines }}
+                            {{- if ne .name $.Values.envoy.defaultBeamline }}
+                            # The beamline is the first segment of the tiled path,
+                            # e.g. /api/v1/search/{{ .name }}/... or /zarr/v2/{{ .name }}/...
+                            - match:
+                                safe_regex:
+                                  regex: {{ printf "^/(%s)/%s(/.*)?$" (join "|" $.Values.envoy.pathPrefixes) (regexQuoteMeta .name) | quote }}
+                              route:
+                                cluster: {{ .name }}_cluster
+                                # No timeout: large array/asset reads and slow searches
+                                # can take longer than envoy's 15s default
+                                timeout: 0s
+                            {{- end }}
+                            {{- end }}
+                            - match:
+                                prefix: "/"
+                              route:
+                                cluster: {{ .Values.envoy.defaultBeamline }}_cluster
+                                timeout: 0s
+                    http_filters:
+                      - name: envoy.filters.http.router
+                        typed_config:
+                          "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+      clusters:
+        {{- range .Values.beamlines }}
+        - name: {{ .name }}_cluster
+          connect_timeout: 5s
+          type: STRICT_DNS
+          lb_policy: ROUND_ROBIN
+          load_assignment:
+            cluster_name: {{ .name }}_cluster
+            endpoints:
+              - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          address: {{ include "tiled.beamlineFullname" (dict "root" $ "beamline" .) }}.{{ $.Release.Namespace }}.svc.cluster.local
+                          port_value: {{ $.Values.service.port }}
+        {{- end }}
+{{- end }}

Charts/tiled/templates/envoy-deployment.yaml

@@ -0,0 +1,76 @@
+{{- if .Values.envoy.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "tiled.fullname" . }}-envoy
+  labels:
+    {{- include "tiled.labels" . | nindent 4 }}
+    app.kubernetes.io/component: envoy-proxy
+spec:
+  replicas: {{ .Values.envoy.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "tiled.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: envoy-proxy
+  template:
+    metadata:
+      labels:
+        {{- include "tiled.labels" . | nindent 8 }}
+        app.kubernetes.io/component: envoy-proxy
+      {{- if .Values.rollOnConfigChange }}
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/envoy-configmap.yaml") . | sha256sum }}
+      {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.envoy.podSecurityContext | nindent 8 }}
+      containers:
+        - name: envoy
+          image: "{{ .Values.envoy.image.repository }}:{{ .Values.envoy.image.tag }}"
+          imagePullPolicy: {{ .Values.envoy.image.pullPolicy }}
+          securityContext:
+            {{- toYaml .Values.envoy.securityContext | nindent 12 }}
+          command:
+            - envoy
+          args:
+            - -c
+            - /etc/envoy/envoy.yaml
+          ports:
+            - name: http
+              containerPort: {{ .Values.envoy.listenerPort }}
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /envoy-ready
+              port: http
+          livenessProbe:
+            httpGet:
+              path: /envoy-ready
+              port: http
+          volumeMounts:
+            - name: envoy-config
+              mountPath: /etc/envoy
+              readOnly: true
+          resources:
+            {{- toYaml .Values.envoy.resources | nindent 12 }}
+      volumes:
+        - name: envoy-config
+          configMap:
+            name: {{ include "tiled.fullname" . }}-envoy-config
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

Charts/tiled/templates/envoy-pdb.yaml

@@ -0,0 +1,15 @@
+{{- if and .Values.envoy.enabled (gt (int .Values.envoy.replicaCount) 1) }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "tiled.fullname" . }}-envoy
+  labels:
+    {{- include "tiled.labels" . | nindent 4 }}
+    app.kubernetes.io/component: envoy-proxy
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      {{- include "tiled.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: envoy-proxy
+{{- end }}

Charts/tiled/templates/envoy-service.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.envoy.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "tiled.fullname" . }}-envoy
+  labels:
+    {{- include "tiled.labels" . | nindent 4 }}
+    app.kubernetes.io/component: envoy-proxy
+spec:
+  type: {{ .Values.envoy.service.type }}
+  ports:
+    - port: {{ .Values.envoy.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "tiled.selectorLabels" . | nindent 4 }}
+    app.kubernetes.io/component: envoy-proxy
+{{- end }}