Feature: also send logs to VictoriaLogs

.github/workflows/deploy.yaml

@@ -148,5 +148,6 @@ jobs:
                 --detach \
                 --restart on-failure \
                 --env OPENSEARCH_URL=vpc-yieldseeker-logs-vzis2bk6an4gq654xu6fynrnby.eu-west-1.es.amazonaws.com \
+                --env VICTORIALOGS_HOST=10.18.1.50 \
                 --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \
                 ${DOCKER_IMAGE}

fluentbit/Dockerfile

@@ -1,6 +1,12 @@
 FROM fluent/fluent-bit:5.1.1
 
+# NOTE(krishan711): the fluent-bit image has no shell, so add a static busybox for entrypoint.sh to pick the outputs
+COPY --from=busybox:1.37.0-musl /bin/busybox /bin/busybox
+RUN ["/bin/busybox", "--install", "-s", "/bin"]
+
 COPY *.yaml /fluent-bit/etc/
 COPY *.lua /fluent-bit/etc/
+COPY outputs /fluent-bit/etc/outputs
+COPY entrypoint.sh /fluent-bit/entrypoint.sh
 
-CMD ["/fluent-bit/bin/fluent-bit", "--config=/fluent-bit/etc/fluent-bit.yaml"]
+ENTRYPOINT ["/fluent-bit/entrypoint.sh"]

fluentbit/entrypoint.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+set -e
+
+config=/fluent-bit/etc/fluent-bit.generated.yaml
+cp /fluent-bit/etc/fluent-bit.yaml "$config"
+echo "  outputs:" >> "$config"
+if [ -n "$OPENSEARCH_URL" ]; then
+    cat /fluent-bit/etc/outputs/opensearch.yaml >> "$config"
+    echo "Sending logs to OpenSearch at $OPENSEARCH_URL"
+fi
+if [ -n "$VICTORIALOGS_HOST" ]; then
+    cat /fluent-bit/etc/outputs/victorialogs.yaml >> "$config"
+    echo "Sending logs to VictoriaLogs at $VICTORIALOGS_HOST"
+fi
+if [ -z "$OPENSEARCH_URL" ] && [ -z "$VICTORIALOGS_HOST" ]; then
+    echo "Error: set OPENSEARCH_URL and/or VICTORIALOGS_HOST"
+    exit 1
+fi
+
+exec /fluent-bit/bin/fluent-bit --config="$config"

fluentbit/fluent-bit.yaml

@@ -20,14 +20,6 @@ pipeline:
       tag: docker_logs
       multiline.parser: docker
       path_key: containerLogPath
-  outputs:
-    - name: opensearch
-      match: "*"
-      host: ${OPENSEARCH_URL}
-      port: 443
-      tls: "On"
-      logstash_format: "On"
-      suppress_type_name: "On"
   # # Testing only
   # inputs:
   #   - name: dummy

fluentbit/outputs/opensearch.yaml

@@ -0,0 +1,7 @@
+    - name: opensearch
+      match: "*"
+      host: ${OPENSEARCH_URL}
+      port: 443
+      tls: "On"
+      logstash_format: "On"
+      suppress_type_name: "On"

fluentbit/outputs/victorialogs.yaml

@@ -0,0 +1,9 @@
+    - name: http
+      match: "*"
+      host: ${VICTORIALOGS_HOST}
+      port: 9428
+      uri: /insert/jsonline?_stream_fields=name,containerId&_msg_field=message,log&_time_field=date
+      format: json_lines
+      json_date_key: date
+      json_date_format: iso8601
+      compress: gzip