Run current OATS consumer suites

.mise/tasks/oats-tests.sh

@@ -10,7 +10,20 @@ set -euo pipefail
   :opentelemetry-examples-logging-k8s-stdout-otlp-json:assemble \
   :opentelemetry-examples-spring-declarative-configuration:bootJar
 
-oats -timeout 5m doc-snippets/extensions-minimal/oats/
-oats -timeout 5m javaagent-declarative-configuration/oats/
-oats -timeout 5m logging-k8s-stdout-otlp-json/
-oats -timeout 5m spring-declarative-configuration/oats/
+workdir="$(mktemp -d)"
+trap 'rm -rf "$workdir"' EXIT
+export GOBIN="$workdir/bin"
+mkdir -p "$GOBIN"
+
+# renovate: datasource=github-releases depName=gcx packageName=grafana/gcx
+export GCX_VERSION=v0.4.0
+go install "github.com/grafana/gcx/cmd/gcx@${GCX_VERSION}"
+
+git clone --depth 1 --branch v2 https://github.com/grafana/oats "$workdir/oats-src"
+GOWORK=off go -C "$workdir/oats-src" build -o "$workdir/oats" ./cmd/v2
+
+"$workdir/oats" \
+  --config oats.toml \
+  --gcx "$GOBIN/gcx" \
+  --no-cache \
+  --timeout=10m

doc-snippets/extensions-minimal/oats/oats.yaml

@@ -1,21 +1,23 @@
-# OATS is an acceptance testing framework for OpenTelemetry - https://github.com/grafana/oats
-oats-schema-version: 2
-
-docker-compose:
-  files:
-    - ./docker-compose.yml
-
+oats: 2
+name: extensions minimal traces custom processor attribute
+fixture:
+  type: compose
+  template: lgtm
+  compose_file: ./docker-compose.yml
+interval: 5s
+seed:
+  type: app
 input:
   - path: /hello
-
-interval: 5000ms
-
 expected:
   traces:
     - traceql: '{ span.http.route = "/hello" }'
-      equals: "GET /hello"
-      attributes:
-        http.request.method: "GET"
-        http.route: "/hello"
-        # This custom attribute is added by MySpanProcessor
-        custom.processor: "active"
+      match_spans:
+        - name: GET /hello
+          attributes:
+            - key: http.request.method
+              value: GET
+            - key: http.route
+              value: /hello
+            - key: custom.processor
+              value: active

javaagent-declarative-configuration/oats/oats.yaml

@@ -1,31 +1,26 @@
-# OATS is an acceptance testing framework for OpenTelemetry - https://github.com/grafana/oats
-oats-schema-version: 2
-
-docker-compose:
-  files:
-    - ./docker-compose.yml
-
+oats: 2
+name: javaagent declarative config excludes actuator health traces
+fixture:
+  type: compose
+  template: lgtm
+  compose_file: ./docker-compose.yml
+seed:
+  type: app
 input:
-  # This endpoint should be traced normally
   - path: /api/example
-  # This endpoint should NOT be traced (excluded by declarative config)
   - path: /actuator/health
-  # This endpoint makes an outgoing client call, exercising the peer service mapping
   - path: /api/remote
-
 expected:
   traces:
-    # Verify that /api/example creates a trace with SERVER span
     - traceql: '{ span.http.route = "/api/example" }'
-      equals: "GET /api/example"
-      attributes:
-        http.request.method: "GET"
-        http.route: "/api/example"
+      match_spans:
+        - name: GET /api/example
+          attributes:
+            - key: http.request.method
+              value: GET
+            - key: http.route
+              value: /api/example
     - traceql: '{ span.http.route = "/actuator/health" }'
-      count:
-        max: 0
-    # Verify the outgoing client span is tagged with peer.service via the
-    # instrumentation.general.peer.service_mapping declarative config
+      absent: true
     - traceql: '{ span.peer.service = "example-backend" }'
-      count:
-        min: 1
+      count: '>= 1'

logging-k8s-stdout-otlp-json/k8s/lgtm.yaml

@@ -30,6 +30,10 @@ spec:
       protocol: TCP
       port: 3200
       targetPort: 3200
+    - name: pyroscope # needed for automated tests
+      protocol: TCP
+      port: 4040
+      targetPort: 4040
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -55,6 +59,7 @@ spec:
             - containerPort: 9090 # needed for automated tests
             - containerPort: 3100 # needed for automated tests
             - containerPort: 3200 # needed for automated tests
+            - containerPort: 4040 # needed for automated tests
           readinessProbe:
             exec:
               command:

logging-k8s-stdout-otlp-json/oats.yaml

@@ -1,53 +1,34 @@
-# OATS is an acceptance testing framework for OpenTelemetry - https://github.com/grafana/oats
-oats-schema-version: 2
-
-kubernetes:
-  dir: k8s
-  app-service: dice
-  app-docker-file: Dockerfile
-  app-docker-tag: dice:1.1-SNAPSHOT
-  app-docker-port: 8080
-input:
-  - path: /rolldice
+oats: 2
+name: logging k8s stdout otlp json emits structured logs
+fixture:
+  type: k3d
+  k8s_dir: k8s
+  app_service: dice
+  app_docker_file: Dockerfile
+  app_docker_tag: dice:1.1-SNAPSHOT
+  app_port: 8080
+seed:
+  type: app
 expected:
-  logs:
-    - logql: '{service_name="dice"} |~ `.*Anonymous player is rolling the dice:.*`'
-      regexp: 'Anonymous player is rolling the dice: \d+' # uses formatted message
-    - logql: '{service_name="dice"} |~ `.*simulating an error.*`'
-      equals: 'Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed: java.lang.RuntimeException: simulating an error] with root cause'
-      attributes:
-        deployment_environment: staging
-        exception_message: "simulating an error"
-        exception_type: "java.lang.RuntimeException"
-        scope_name: "org.apache.catalina.core.ContainerBase.[Tomcat].[localhost].[/].[dispatcherServlet]"
-        service_name: dice
-        service_namespace: shop
-        service_version: 1.1
-        severity_number: 17
-        severity_text: SEVERE
-      attribute-regexp:
-        flags: ".*" # from loki
-        detected_level: ".*" # from loki
-        observed_timestamp: ".*" # from loki
-        # thread_name: ".*" # thread name is missing when there is an exception - has nothing to do with stdout logging
-        span_id: ".*"
-        trace_id: ".*"
-        host_arch: ".*"
-        host_name: ".*"
-        os_description: ".*"
-        os_type: ".*"
-        os_version: ".*"
-        process_command_args: ".*"
-        process_executable_path: ".*"
-        process_pid: ".*"
-        process_runtime_description: ".*"
-        process_runtime_name: ".*"
-        process_runtime_version: ".*"
-        service_instance_id: ".*"
-        telemetry_distro_name: ".*"
-        telemetry_distro_version: ".*"
-        telemetry_sdk_language: ".*"
-        telemetry_sdk_name: ".*"
-        telemetry_sdk_version: ".*"
-        exception_stacktrace: "java\\.lang\\.RuntimeException: simulating an error\n\tat io\\.opentelemetry\\.example\\.RollController\\.index\\(RollController\\.java:21\\)\n\t.*\n"
-
+  custom-checks:
+    - script: |
+        set -eu
+        ok=0
+        err=0
+        i=0
+        while [ "$i" -lt 20 ]; do
+          code=$(curl -s -o /dev/null -w '%{http_code}' http://localhost:8080/rolldice || true)
+          [ "$code" = "200" ] && ok=1 || true
+          [ "$code" = "500" ] && err=1 || true
+          if [ "$ok" = 1 ] && [ "$err" = 1 ]; then
+            break
+          fi
+          i=$((i+1))
+          sleep 1
+        done
+        [ "$ok" = 1 ]
+        [ "$err" = 1 ]
+        logs=$(kubectl logs deployment/dice --tail=200)
+        printf '%s\n' "$logs" | grep -Eq 'Anonymous player is rolling the dice: [0-9]+'
+        printf '%s\n' "$logs" | grep -Fq 'Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed: java.lang.RuntimeException: simulating an error] with root cause'
+        printf '%s\n' "$logs" | grep -Fq 'java.lang.RuntimeException: simulating an error'

oats.toml

@@ -0,0 +1,18 @@
+[meta]
+version = 2
+
+[[suite]]
+name = "extensions-minimal"
+cases = ["doc-snippets/extensions-minimal/oats/oats.yaml"]
+
+[[suite]]
+name = "javaagent-declarative-configuration"
+cases = ["javaagent-declarative-configuration/oats/oats.yaml"]
+
+[[suite]]
+name = "spring-declarative-configuration"
+cases = ["spring-declarative-configuration/oats/oats.yaml"]
+
+[[suite]]
+name = "logging-k8s-stdout-otlp-json"
+cases = ["logging-k8s-stdout-otlp-json/oats.yaml"]

spring-declarative-configuration/oats/oats.yaml

@@ -1,24 +1,23 @@
-# OATS is an acceptance testing framework for OpenTelemetry - https://github.com/grafana/oats
-oats-schema-version: 2
-
-docker-compose:
-  files:
-    - ./docker-compose.yml
-
+oats: 2
+name: spring declarative config excludes actuator health traces
+fixture:
+  type: compose
+  template: lgtm
+  compose_file: ./docker-compose.yml
+seed:
+  type: app
 input:
-  # This endpoint should be traced normally
   - path: /api/example
-  # This endpoint should NOT be traced (excluded by declarative config)
   - path: /actuator/health
-
 expected:
   traces:
-    # Verify that /api/example creates a trace with SERVER span
     - traceql: '{ span.http.route = "/api/example" }'
-      equals: "GET /api/example"
-      attributes:
-        http.request.method: "GET"
-        http.route: "/api/example"
+      match_spans:
+        - name: GET /api/example
+          attributes:
+            - key: http.request.method
+              value: GET
+            - key: http.route
+              value: /api/example
     - traceql: '{ span.http.route = "/actuator/health" }'
-      count:
-        max: 0
+      absent: true