diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index edaf19d..6bf61a7 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -148,5 +148,6 @@ jobs: --detach \ --restart on-failure \ --env OPENSEARCH_URL=vpc-yieldseeker-logs-vzis2bk6an4gq654xu6fynrnby.eu-west-1.es.amazonaws.com \ + --env VICTORIALOGS_HOST=10.18.1.50 \ --volume /var/lib/docker/containers:/var/lib/docker/containers:ro \ ${DOCKER_IMAGE} diff --git a/fluentbit/Dockerfile b/fluentbit/Dockerfile index 856adeb..b1e568d 100644 --- a/fluentbit/Dockerfile +++ b/fluentbit/Dockerfile @@ -1,6 +1,12 @@ FROM fluent/fluent-bit:5.1.1 +# NOTE(krishan711): the fluent-bit image has no shell, so add a static busybox for entrypoint.sh to pick the outputs +COPY --from=busybox:1.37.0-musl /bin/busybox /bin/busybox +RUN ["/bin/busybox", "--install", "-s", "/bin"] + COPY *.yaml /fluent-bit/etc/ COPY *.lua /fluent-bit/etc/ +COPY outputs /fluent-bit/etc/outputs +COPY entrypoint.sh /fluent-bit/entrypoint.sh -CMD ["/fluent-bit/bin/fluent-bit", "--config=/fluent-bit/etc/fluent-bit.yaml"] +ENTRYPOINT ["/fluent-bit/entrypoint.sh"] diff --git a/fluentbit/entrypoint.sh b/fluentbit/entrypoint.sh new file mode 100755 index 0000000..1adbdef --- /dev/null +++ b/fluentbit/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -e + +config=/fluent-bit/etc/fluent-bit.generated.yaml +cp /fluent-bit/etc/fluent-bit.yaml "$config" +echo " outputs:" >> "$config" +if [ -n "$OPENSEARCH_URL" ]; then + cat /fluent-bit/etc/outputs/opensearch.yaml >> "$config" + echo "Sending logs to OpenSearch at $OPENSEARCH_URL" +fi +if [ -n "$VICTORIALOGS_HOST" ]; then + cat /fluent-bit/etc/outputs/victorialogs.yaml >> "$config" + echo "Sending logs to VictoriaLogs at $VICTORIALOGS_HOST" +fi +if [ -z "$OPENSEARCH_URL" ] && [ -z "$VICTORIALOGS_HOST" ]; then + echo "Error: set OPENSEARCH_URL and/or VICTORIALOGS_HOST" + exit 1 +fi + +exec /fluent-bit/bin/fluent-bit --config="$config" diff --git a/fluentbit/fluent-bit.yaml b/fluentbit/fluent-bit.yaml index f02f8cc..94b2881 100644 --- a/fluentbit/fluent-bit.yaml +++ b/fluentbit/fluent-bit.yaml @@ -20,14 +20,6 @@ pipeline: tag: docker_logs multiline.parser: docker path_key: containerLogPath - outputs: - - name: opensearch - match: "*" - host: ${OPENSEARCH_URL} - port: 443 - tls: "On" - logstash_format: "On" - suppress_type_name: "On" # # Testing only # inputs: # - name: dummy diff --git a/fluentbit/outputs/opensearch.yaml b/fluentbit/outputs/opensearch.yaml new file mode 100644 index 0000000..f2c77eb --- /dev/null +++ b/fluentbit/outputs/opensearch.yaml @@ -0,0 +1,7 @@ + - name: opensearch + match: "*" + host: ${OPENSEARCH_URL} + port: 443 + tls: "On" + logstash_format: "On" + suppress_type_name: "On" diff --git a/fluentbit/outputs/victorialogs.yaml b/fluentbit/outputs/victorialogs.yaml new file mode 100644 index 0000000..2b771db --- /dev/null +++ b/fluentbit/outputs/victorialogs.yaml @@ -0,0 +1,9 @@ + - name: http + match: "*" + host: ${VICTORIALOGS_HOST} + port: 9428 + uri: /insert/jsonline?_stream_fields=name,containerId&_msg_field=message,log&_time_field=date + format: json_lines + json_date_key: date + json_date_format: iso8601 + compress: gzip