From 5fdc9163d64518b74273ba8476a7494b077ce91d Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 9 Oct 2026 03:49:26 -0700 Subject: [PATCH 1/2] feat(observability): backend metrics + tracing primitives Build on the logging foundation (PR 1) with the backend emit path: workflow lifecycle metrics and a run-level setup trace span, plus the tracing/metrics primitives they use. This is PR 2 of the observability stack. Primitives (common/observability): - TexeraTracer: lazy accessor for the process tracer off GlobalOpenTelemetry. - SpanAttrs: typed AttributeKey constants so span attributes use standard keys rather than ad hoc strings. - WorkflowMetrics: the OTel instruments (start/completion/failure/cancellation counters and run-duration histogram) keyed by workflow kind. - TraceparentValidator: validates W3C traceparent headers before use. Emit path (amber): - WorkflowMetricsRecorder: single owner of the metric instruments. init() wires them once; onStart stamps a run's start; onStateChange records terminal counters and duration exactly once on the first transition into a terminal state (idempotent, safe to call on every transition). - WorkflowService.initExecutionService runs inside a run-level setup span so setup-path logs carry its trace id. The span covers only the synchronous setup; synchronous setup failures are recorded on it in the catch block, and the span is ended in finally. The async errorHandler deliberately does not touch the span: it is invoked after setup returns and the span has ended, so the failure is surfaced through the metadata store instead. - ExecutionStateStore.updateWorkflowState is the single chokepoint that feeds every state transition to WorkflowMetricsRecorder.onStateChange. - ComputingUnitMaster initializes the recorder at startup. All observability sources live under common/observability (the module from PR 1); the tracing/metrics classes are not duplicated into common/config. Tests: WorkflowMetricsSpec, SpanAttrsSpec, and TraceparentValidatorSpec, plus TexeraTracerSpec (the tracer accessor and current-context passthrough) and WorkflowMetricsRecorderSpec (the state-to-metric mapping against an in-memory meter: starts increment the start counter, COMPLETED and FAILED record a completion and a duration sample, KILLED records a cancellation with no duration, and non-terminal or already-terminal transitions record nothing). Review follow-ups addressed: span attributes use SpanAttrs keys instead of plain strings; the error handler no longer records onto a span that may have already ended (documented and handled via the metadata store). Stacked on PR 1 (obs/pr1/foundations). --- .../texera/web/ComputingUnitMaster.scala | 1 + .../WorkflowMetricsRecorder.scala | 107 ++++++++ .../service/WorkflowExecutionService.scala | 10 +- .../texera/web/service/WorkflowService.scala | 249 ++++++++++-------- .../web/storage/ExecutionStateStore.scala | 4 + .../WorkflowMetricsRecorderSpec.scala | 169 ++++++++++++ .../WorkflowExecutionServiceSpec.scala | 7 +- .../texera/observability/SpanAttrs.scala | 62 +++++ .../texera/observability/TexeraTracer.scala | 61 +++++ .../observability/TraceparentValidator.scala | 98 +++++++ .../observability/WorkflowMetrics.scala | 243 +++++++++++++++++ .../texera/observability/SpanAttrsSpec.scala | 105 ++++++++ .../observability/TexeraTracerSpec.scala | 56 ++++ .../TraceparentValidatorSpec.scala | 124 +++++++++ .../observability/WorkflowMetricsSpec.scala | 195 ++++++++++++++ 15 files changed, 1384 insertions(+), 107 deletions(-) create mode 100644 amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala create mode 100644 amber/src/test/scala/org/apache/texera/web/observability/WorkflowMetricsRecorderSpec.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/SpanAttrs.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/TexeraTracer.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala create mode 100644 common/observability/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/TexeraTracerSpec.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala create mode 100644 common/observability/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala diff --git a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala index b5bdcbf5573..d85f127b740 100644 --- a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala +++ b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala @@ -139,6 +139,7 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with override def run(configuration: Configuration, environment: Environment): Unit = { org.apache.texera.observability.OtelInit.init("computing-unit-master") + org.apache.texera.web.observability.WorkflowMetricsRecorder.init() ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() SqlServer.initConnection( diff --git a/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala new file mode 100644 index 00000000000..0f9ab2401aa --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.core.virtualidentity.ExecutionIdentity +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.{ + COMPLETED, + FAILED, + KILLED, + PAUSED, + PAUSING, + RESUMING, + RUNNING +} +import org.apache.texera.observability.WorkflowMetrics +import org.apache.texera.observability.WorkflowMetrics.WorkflowKind +import org.apache.texera.web.service.WorkflowService + +import java.util.concurrent.ConcurrentHashMap + +/** + * Drives [[WorkflowMetrics]] from amber's execution lifecycle. The metric + * instruments live in common/config; this object is the single place + * that records them, so the lifecycle code only needs one-line calls. + * + * - start / terminal counters and the duration histogram are recorded + * from [[onStart]] and [[onStateChange]]; + * - the always-polled `texera.workflow.active` gauge is sourced from the + * live WorkflowService registry via the supplier registered in [[init]]. + */ +object WorkflowMetricsRecorder extends LazyLogging { + + // Start time + kind per in-flight run, so a terminal transition can emit + // a duration and attribute the outcome. Keyed by the execution identity. + private val inFlight = new ConcurrentHashMap[ExecutionIdentity, (Long, WorkflowKind)]() + + private val ActiveStates: Set[WorkflowAggregatedState] = Set(RUNNING, PAUSING, PAUSED, RESUMING) + private val TerminalStates: Set[WorkflowAggregatedState] = Set(COMPLETED, FAILED, KILLED) + + /** Register the active-executions gauge supplier and bind the instruments. + * Call once at startup, after OtelInit.init. The supplier is polled on + * every metric collection, so it must never throw. + */ + def init(): Unit = { + WorkflowMetrics.setActiveExecutionsSupplier(() => + try { + WorkflowService.getAllWorkflowServices.iterator + .flatMap(s => Option(s.executionService.getValue)) + .map(_.executionStateStore.metadataStore.getState.state) + .count(ActiveStates.contains) + .toLong + } catch { + case _: Throwable => 0L + } + ) + WorkflowMetrics.ensureBound() + } + + /** Record that a run started. */ + def onStart( + executionId: ExecutionIdentity, + kind: WorkflowKind = WorkflowKind.Interactive + ): Unit = { + inFlight.put(executionId, (System.currentTimeMillis(), kind)) + WorkflowMetrics.recordStart(kind) + } + + /** Record terminal counters + duration exactly once, on the first + * transition from a non-terminal into a terminal state. Safe to call on + * every state change; non-terminal and repeat-terminal calls are no-ops. + */ + def onStateChange( + executionId: ExecutionIdentity, + oldState: WorkflowAggregatedState, + newState: WorkflowAggregatedState + ): Unit = { + if (!TerminalStates.contains(newState) || TerminalStates.contains(oldState)) return + val entry = Option(inFlight.remove(executionId)) + val kind = entry.map(_._2).getOrElse(WorkflowKind.Interactive) + val durationSec = entry.map(e => (System.currentTimeMillis() - e._1) / 1000.0).getOrElse(0.0) + newState match { + case COMPLETED => WorkflowMetrics.recordCompletion(kind, durationSec) + case FAILED => WorkflowMetrics.recordFailure(kind, durationSec) + case KILLED => WorkflowMetrics.recordCancellation(kind) + case _ => () + } + } +} diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala index 4c8eb209b28..1c921fcacbc 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala @@ -108,7 +108,12 @@ class WorkflowExecutionService( var executionRuntimeService: ExecutionRuntimeService = _ var executionConsoleService: ExecutionConsoleService = _ - def executeWorkflow(): Unit = { + // Returns Some(err) when setup fails synchronously (e.g. compilation) after the + // failure has already been surfaced to the UI via errorHandler, so the caller can + // mark its still-open setup span; None once the run has started, after which + // runtime failures arrive asynchronously through errorHandler and belong to later + // execution work rather than this setup. + def executeWorkflow(): Option[Throwable] = { try { val compilationResult = new WorkflowCompiler(workflowContext) .compile(request.logicalPlan, CompilationErrorHandling.Strict) @@ -118,7 +123,7 @@ class WorkflowExecutionService( // stop here: `workflow` is still null, so falling through would NPE // below and mask the reported compilation error errorHandler(err) - return + return Some(err) } client = ComputingUnitMaster.createAmberRuntime( @@ -172,6 +177,7 @@ class WorkflowExecutionService( } ) ) + None } override def unsubscribeAll(): Unit = { diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index 8ec650ba715..54056c4caea 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -21,9 +21,11 @@ package org.apache.texera.web.service import com.google.protobuf.timestamp.Timestamp import com.typesafe.scalalogging.LazyLogging +import io.opentelemetry.api.trace.StatusCode import io.reactivex.rxjava3.disposables.{CompositeDisposable, Disposable} import io.reactivex.rxjava3.subjects.BehaviorSubject import org.apache.texera.common.config.{ApplicationConfig, StorageConfig} +import org.apache.texera.observability.{SpanAttrs, TexeraTracer} import org.apache.texera.dao.SqlServer import org.apache.texera.dao.jooq.generated.Tables.USER_WAREHOUSE import org.apache.texera.amber.core.WorkflowRuntimeException @@ -229,126 +231,165 @@ class WorkflowService( userOpt: Option[User], sessionUri: URI ): Unit = { + val span = TexeraTracer.tracer + .spanBuilder("WorkflowService.initExecutionService") + .setAttribute(SpanAttrs.WorkflowId, Long.box(workflowId.id)) + .startSpan() + val scope = span.makeCurrent() + try { - val (uidOpt, userEmailOpt) = userOpt.map(user => (user.getUid, user.getEmail)).unzip + val (uidOpt, userEmailOpt) = userOpt.map(user => (user.getUid, user.getEmail)).unzip - // Validate before touching the execution already in flight: a request that is - // going to be refused must not take the running one's subscriptions with it. - // uid is NOT NULL in the DB; fail early here rather than letting the insert fail downstream. - val uid = uidOpt.getOrElse( - throw new IllegalArgumentException( - "Cannot start execution: a user id (uid) is required but none was provided." + // Validate before touching the execution already in flight: a request that is + // going to be refused must not take the running one's subscriptions with it. + // uid is NOT NULL in the DB; fail early here rather than letting the insert fail downstream. + val uid = uidOpt.getOrElse( + throw new IllegalArgumentException( + "Cannot start execution: a user id (uid) is required but none was provided." + ) ) - ) - val warehouseName = WorkflowService.resolveLakekeeperWarehouseName(req.warehouseId, uid) + val warehouseName = WorkflowService.resolveLakekeeperWarehouseName(req.warehouseId, uid) - if (executionService.hasValue) { - executionService.getValue.unsubscribeAll() - } - - val workflowContext: WorkflowContext = createWorkflowContext() - workflowContext.warehouse = warehouseName - var coordinatorConf = CoordinatorConfig.default - - // clean up results from previous run - val previousExecutionId = - WorkflowExecutionService.getLatestExecutionId(workflowId, req.computingUnitId) - previousExecutionId.foreach(eid => { - clearExecutionResources(eid) - }) // TODO: change this behavior after enabling cache. - - workflowContext.executionId = ExecutionsMetadataPersistService.insertNewExecution( - workflowContext.workflowId, - uid, - req.executionName, - convertToJson(req.engineVersion), - req.computingUnitId, - req.warehouseId - ) + if (executionService.hasValue) { + executionService.getValue.unsubscribeAll() + } - if (ApplicationConfig.faultToleranceLogRootFolder.isDefined) { - val writeLocation = ApplicationConfig.faultToleranceLogRootFolder.get.resolve( - s"${workflowContext.workflowId}/${workflowContext.executionId}/" + val workflowContext: WorkflowContext = createWorkflowContext() + workflowContext.warehouse = warehouseName + var coordinatorConf = CoordinatorConfig.default + + // clean up results from previous run + val previousExecutionId = + WorkflowExecutionService.getLatestExecutionId(workflowId, req.computingUnitId) + previousExecutionId.foreach(eid => { + clearExecutionResources(eid) + }) // TODO: change this behavior after enabling cache. + + workflowContext.executionId = ExecutionsMetadataPersistService.insertNewExecution( + workflowContext.workflowId, + uid, + req.executionName, + convertToJson(req.engineVersion), + req.computingUnitId, + req.warehouseId ) - ExecutionsMetadataPersistService.tryUpdateExistingExecution(workflowContext.executionId) { - execution => execution.setLogLocation(writeLocation.toString) + span.setAttribute(SpanAttrs.ExecutionId, Long.box(workflowContext.executionId.id)) + // A run has started: record the start counter and stamp its start time. + org.apache.texera.web.observability.WorkflowMetricsRecorder + .onStart(workflowContext.executionId) + + if (ApplicationConfig.faultToleranceLogRootFolder.isDefined) { + val writeLocation = ApplicationConfig.faultToleranceLogRootFolder.get.resolve( + s"${workflowContext.workflowId}/${workflowContext.executionId}/" + ) + ExecutionsMetadataPersistService.tryUpdateExistingExecution(workflowContext.executionId) { + execution => execution.setLogLocation(writeLocation.toString) + } + coordinatorConf = coordinatorConf.copy(faultToleranceConfOpt = + Some(FaultToleranceConfig(writeTo = writeLocation)) + ) } - coordinatorConf = coordinatorConf.copy(faultToleranceConfOpt = - Some(FaultToleranceConfig(writeTo = writeLocation)) - ) - } - if (req.replayFromExecution.isDefined) { - val replayInfo = req.replayFromExecution.get - ExecutionsMetadataPersistService - .tryGetExistingExecution(ExecutionIdentity(replayInfo.eid)) - .foreach { execution => - val readLocation = new URI(execution.getLogLocation) - coordinatorConf = coordinatorConf.copy(stateRestoreConfOpt = - Some( - StateRestoreConfig( - readFrom = readLocation, - replayDestination = EmbeddedControlMessageIdentity(replayInfo.interaction) + if (req.replayFromExecution.isDefined) { + val replayInfo = req.replayFromExecution.get + ExecutionsMetadataPersistService + .tryGetExistingExecution(ExecutionIdentity(replayInfo.eid)) + .foreach { execution => + val readLocation = new URI(execution.getLogLocation) + coordinatorConf = coordinatorConf.copy(stateRestoreConfOpt = + Some( + StateRestoreConfig( + readFrom = readLocation, + replayDestination = EmbeddedControlMessageIdentity(replayInfo.interaction) + ) ) ) - ) - } - } + } + } - val executionStateStore = new ExecutionStateStore() - // assign execution id to find the execution from DB in case the constructor fails. - executionStateStore.metadataStore.updateState(state => - state.withExecutionId(workflowContext.executionId) - ) - val errorHandler: Throwable => Unit = { t => - { - val fromActorOpt = t match { - case ex: WorkflowRuntimeException => - ex.relatedWorkerId - case other => - None + val executionStateStore = new ExecutionStateStore() + // assign execution id to find the execution from DB in case the constructor fails. + executionStateStore.metadataStore.updateState(state => + state.withExecutionId(workflowContext.executionId) + ) + val errorHandler: Throwable => Unit = { t => + { + val fromActorOpt = t match { + case ex: WorkflowRuntimeException => + ex.relatedWorkerId + case other => + None + } + val (operatorId, workerId) = getOperatorFromActorIdOpt(fromActorOpt) + logger.error("error during execution", t) + // Do NOT touch `span` here: this handler is passed into + // WorkflowExecutionService and invoked asynchronously (runtime, + // websocket, startWorkflow callbacks) after initExecutionService has + // returned and the setup span has already ended, so recording onto it + // would be a silent no-op. The failure is surfaced via the metadata + // store below; setup-span errors are recorded in the catch block. + executionStateStore.statsStore.updateState(stats => + stats.withEndTimeStamp(System.currentTimeMillis()) + ) + executionStateStore.metadataStore.updateState { metadataStore => + updateWorkflowState(FAILED, metadataStore).addFatalErrors( + WorkflowFatalError( + EXECUTION_FAILURE, + Timestamp(Instant.now), + t.toString, + getStackTraceWithAllCauses(t), + operatorId, + workerId + ) + ) + } } - val (operatorId, workerId) = getOperatorFromActorIdOpt(fromActorOpt) - logger.error("error during execution", t) - executionStateStore.statsStore.updateState(stats => - stats.withEndTimeStamp(System.currentTimeMillis()) + } + // WorkflowExecutionService construction does no external work and cannot + // throw; it registers its error/state diff handler up front. Once published + // via `executionService.onNext`, any failure in `executeWorkflow()` is + // recorded by `errorHandler` into the metadata store, whose handler emits a + // WorkflowErrorEvent that `connectToExecution` forwards. + try { + val execution = new WorkflowExecutionService( + coordinatorConf, + workflowContext, + resultService, + req, + executionStateStore, + errorHandler, + userEmailOpt, + sessionUri ) - executionStateStore.metadataStore.updateState { metadataStore => - updateWorkflowState(FAILED, metadataStore).addFatalErrors( - WorkflowFatalError( - EXECUTION_FAILURE, - Timestamp(Instant.now), - t.toString, - getStackTraceWithAllCauses(t), - operatorId, - workerId - ) - ) + lifeCycleManager.registerCleanUpOnStateChange(executionStateStore) + executionService.onNext(execution) + // A synchronous setup failure (e.g. compilation) is surfaced to the UI by + // the run's errorHandler and signalled back here; record it on the still-open + // setup span so error-trace searches find it. None means the run started, after + // which runtime failures arrive asynchronously through errorHandler and belong + // to later execution work, not this setup span. + execution.executeWorkflow().foreach { setupErr => + span.recordException(setupErr) + span.setStatus(StatusCode.ERROR) } + } catch { + case e: Throwable => + // Setup threw before the run's errorHandler could record it; mark the + // open setup span and surface the failure to the UI. + span.recordException(e) + span.setStatus(StatusCode.ERROR) + errorHandler(e) } - } - // WorkflowExecutionService construction does no external work and cannot - // throw; it registers its error/state diff handler up front. Once published - // via `executionService.onNext`, any failure in `executeWorkflow()` is - // recorded by `errorHandler` into the metadata store, whose handler emits a - // WorkflowErrorEvent that `connectToExecution` forwards. - try { - val execution = new WorkflowExecutionService( - coordinatorConf, - workflowContext, - resultService, - req, - executionStateStore, - errorHandler, - userEmailOpt, - sessionUri - ) - lifeCycleManager.registerCleanUpOnStateChange(executionStateStore) - executionService.onNext(execution) - execution.executeWorkflow() + } catch { - case e: Throwable => errorHandler(e) + case t: Throwable => + // Synchronous setup failure (before the run's own errorHandler is wired). + span.recordException(t) + span.setStatus(StatusCode.ERROR) + throw t + } finally { + scope.close() + span.end() } - } def convertToJson(frontendVersion: String): String = { diff --git a/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala b/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala index 654acbbefd1..bc97b39d185 100644 --- a/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala +++ b/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala @@ -27,6 +27,7 @@ import org.apache.texera.amber.engine.common.executionruntimestate.{ ExecutionMetadataStore, ExecutionStatsStore } +import org.apache.texera.web.observability.WorkflowMetricsRecorder import org.apache.texera.web.service.ExecutionsMetadataPersistService import java.sql.Timestamp @@ -44,6 +45,9 @@ object ExecutionStateStore { execution.setStatus(maptoStatusCode(state)) execution.setLastUpdateTime(new Timestamp(System.currentTimeMillis())) } + // Single chokepoint for every state transition: emit lifecycle metrics + // once on the first transition into a terminal state. + WorkflowMetricsRecorder.onStateChange(metadataStore.executionId, metadataStore.state, state) metadataStore.withState(state) } } diff --git a/amber/src/test/scala/org/apache/texera/web/observability/WorkflowMetricsRecorderSpec.scala b/amber/src/test/scala/org/apache/texera/web/observability/WorkflowMetricsRecorderSpec.scala new file mode 100644 index 00000000000..4c45c126c6a --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/web/observability/WorkflowMetricsRecorderSpec.scala @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability + +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.common.CompletableResultCode +import io.opentelemetry.sdk.metrics.{InstrumentType, SdkMeterProvider} +import io.opentelemetry.sdk.metrics.`export`.{MetricExporter, PeriodicMetricReader} +import io.opentelemetry.sdk.metrics.data.{AggregationTemporality, MetricData} +import org.apache.texera.amber.core.virtualidentity.ExecutionIdentity +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState +import org.apache.texera.observability.WorkflowMetrics.WorkflowKind +import org.scalatest.BeforeAndAfterAll +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.Duration +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong +import scala.jdk.CollectionConverters._ + +/** + * Exercises the execution-state to metric mapping in WorkflowMetricsRecorder + * against a real in-memory meter. WorkflowMetrics.bindForTest is package + * private to common/observability, so we drive the binding the production way: + * install an SDK into GlobalOpenTelemetry and let ensureBound pick it up. + */ +class WorkflowMetricsRecorderSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll { + + private val exporter = new CapturingMetricExporter + private var provider: SdkMeterProvider = _ + private val nextId = new AtomicLong(1000) + private def newId(): ExecutionIdentity = ExecutionIdentity(nextId.incrementAndGet()) + + override def beforeAll(): Unit = { + GlobalOpenTelemetry.resetForTest() + val reader = PeriodicMetricReader.builder(exporter).setInterval(Duration.ofHours(1)).build() + provider = SdkMeterProvider.builder().registerMetricReader(reader).build() + GlobalOpenTelemetry.set(OpenTelemetrySdk.builder().setMeterProvider(provider).build()) + } + + override def afterAll(): Unit = { + if (provider != null) provider.close() + GlobalOpenTelemetry.resetForTest() + } + + // Cumulative total of a counter, summed over all attribute points. + private def counter(name: String): Long = { + provider.forceFlush().join(5, TimeUnit.SECONDS) + exporter.latest.asScala + .find(_.getName == name) + .map(_.getLongSumData.getPoints.asScala.map(_.getValue.longValue()).sum) + .getOrElse(0L) + } + + // Cumulative sample count of a histogram, summed over all attribute points. + private def histogramCount(name: String): Long = { + provider.forceFlush().join(5, TimeUnit.SECONDS) + exporter.latest.asScala + .find(_.getName == name) + .map(_.getHistogramData.getPoints.asScala.map(_.getCount).sum) + .getOrElse(0L) + } + + "onStart" should "increment texera.workflow.starts" in { + val before = counter("texera.workflow.starts") + WorkflowMetricsRecorder.onStart(newId(), WorkflowKind.Interactive) + counter("texera.workflow.starts") shouldBe before + 1 + } + + "onStateChange into COMPLETED" should "record a completion and a duration sample" in { + val id = newId() + WorkflowMetricsRecorder.onStart(id, WorkflowKind.Interactive) + val beforeCompletions = counter("texera.workflow.completions") + val beforeDurations = histogramCount("texera.workflow.duration") + WorkflowMetricsRecorder.onStateChange( + id, + WorkflowAggregatedState.RUNNING, + WorkflowAggregatedState.COMPLETED + ) + counter("texera.workflow.completions") shouldBe beforeCompletions + 1 + histogramCount("texera.workflow.duration") shouldBe beforeDurations + 1 + } + + "onStateChange into FAILED" should "record a completion (outcome=failure) and a duration sample" in { + val id = newId() + WorkflowMetricsRecorder.onStart(id, WorkflowKind.Scheduled) + val beforeCompletions = counter("texera.workflow.completions") + val beforeDurations = histogramCount("texera.workflow.duration") + WorkflowMetricsRecorder.onStateChange( + id, + WorkflowAggregatedState.RUNNING, + WorkflowAggregatedState.FAILED + ) + counter("texera.workflow.completions") shouldBe beforeCompletions + 1 + histogramCount("texera.workflow.duration") shouldBe beforeDurations + 1 + } + + "onStateChange into KILLED" should "record a cancellation and no duration sample" in { + val id = newId() + WorkflowMetricsRecorder.onStart(id, WorkflowKind.Interactive) + val beforeCancellations = counter("texera.workflow.cancellations") + val beforeDurations = histogramCount("texera.workflow.duration") + WorkflowMetricsRecorder.onStateChange( + id, + WorkflowAggregatedState.RUNNING, + WorkflowAggregatedState.KILLED + ) + counter("texera.workflow.cancellations") shouldBe beforeCancellations + 1 + histogramCount("texera.workflow.duration") shouldBe beforeDurations + } + + "onStateChange to a non-terminal state" should "record nothing" in { + val id = newId() + WorkflowMetricsRecorder.onStart(id, WorkflowKind.Interactive) + val before = counter("texera.workflow.completions") + WorkflowMetricsRecorder.onStateChange( + id, + WorkflowAggregatedState.RUNNING, + WorkflowAggregatedState.PAUSED + ) + counter("texera.workflow.completions") shouldBe before + } + + "onStateChange whose old state is already terminal" should "record nothing (dedup)" in { + val id = newId() + WorkflowMetricsRecorder.onStart(id, WorkflowKind.Interactive) + val before = counter("texera.workflow.completions") + WorkflowMetricsRecorder.onStateChange( + id, + WorkflowAggregatedState.COMPLETED, + WorkflowAggregatedState.FAILED + ) + counter("texera.workflow.completions") shouldBe before + } +} + +/** Captures the most recent metric export so a test can read cumulative values. */ +private class CapturingMetricExporter extends MetricExporter { + @volatile private var captured: java.util.Collection[MetricData] = + java.util.Collections.emptyList() + def latest: java.util.Collection[MetricData] = captured + override def `export`(metrics: java.util.Collection[MetricData]): CompletableResultCode = { + captured = new java.util.ArrayList[MetricData](metrics) + CompletableResultCode.ofSuccess() + } + override def flush(): CompletableResultCode = CompletableResultCode.ofSuccess() + override def shutdown(): CompletableResultCode = CompletableResultCode.ofSuccess() + override def getAggregationTemporality(instrumentType: InstrumentType): AggregationTemporality = + AggregationTemporality.CUMULATIVE +} diff --git a/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala b/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala index abf2bb039c5..53c63a9f25d 100644 --- a/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala @@ -430,7 +430,12 @@ class WorkflowExecutionServiceSpec // null `workflow` on the way in, and throws out of `executeWorkflow` -- which is also what // keeps a "unit" test out of the live-runtime half on whatever actor system the shared JVM // holds. - noException should be thrownBy service.executeWorkflow() + var setupFailure: Option[Throwable] = None + noException should be thrownBy { setupFailure = service.executeWorkflow() } + + // The compilation failure is returned (not thrown) so the caller can record it on the + // still-open setup span, on top of reaching the UI through errorHandler. + setupFailure.map(_.getMessage).getOrElse("") should include("No file selected") errors should have size 1 errors.head.getMessage should include("No file selected") diff --git a/common/observability/src/main/scala/org/apache/texera/observability/SpanAttrs.scala b/common/observability/src/main/scala/org/apache/texera/observability/SpanAttrs.scala new file mode 100644 index 00000000000..c818f69bc16 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/SpanAttrs.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.api.common.AttributeKey + +/** + * Standard Texera span-attribute keys plus a free-text sanitizer. + * + * These are the shared label keys so every callsite tags spans with the + * same names. Set them with the standard OTel API at the callsite, e.g. + * ``spanBuilder.setAttribute(SpanAttrs.WorkflowId, id)`` or + * ``span.setAttribute(SpanAttrs.WorkflowId, id)``. Run any free-text value + * through [[sanitizeFreeText]] first to strip CRLF and cap its length. + */ +object SpanAttrs { + + /** Maximum length for free-text span attribute values. */ + val FreeTextMaxLen: Int = 256 + + // ---- Standard Texera correlation labels ------------------------------ + + val WorkflowId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.workflow.id") + val ExecutionId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.execution.id") + val ProjectId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.project.id") + val UserId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.user.id") + val OperatorId: AttributeKey[String] = AttributeKey.stringKey("texera.operator.id") + val OperatorName: AttributeKey[String] = AttributeKey.stringKey("texera.operator.name") + val Outcome: AttributeKey[String] = AttributeKey.stringKey("texera.outcome") + + // ---- Pure helpers (exposed for testing) ------------------------------ + + /** + * Strip CR/LF and other C0 control characters from a free-text + * value, then cap at [[FreeTextMaxLen]]. Returns null for + * null/empty input (caller skips the setAttribute call). + */ + def sanitizeFreeText(value: String): String = { + if (value == null || value.isEmpty) return null + val stripped = value.filter(c => c >= 0x20 && c != 0x7f) + if (stripped.isEmpty) null + else if (stripped.length <= FreeTextMaxLen) stripped + else stripped.substring(0, FreeTextMaxLen) + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/TexeraTracer.scala b/common/observability/src/main/scala/org/apache/texera/observability/TexeraTracer.scala new file mode 100644 index 00000000000..38e1c36b620 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/TexeraTracer.scala @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context + +/** + * Accessor for the Texera OTel tracer. + * + * The only thing this adds over calling ``GlobalOpenTelemetry.getTracer`` + * directly is a single instrumentation scope name (``org.apache.texera``), + * so every Texera-produced span shows up under one logical scope in the + * backend, separable from anything emitted by transitive libraries. + * + * Start and end spans with the standard OTel API at the callsite (see the + * OpenTelemetry Java demo for the recommended pattern): + * + * {{{ + * val span = TexeraTracer.tracer.spanBuilder("MyClass.myMethod").startSpan() + * val scope = span.makeCurrent() + * try { ... } catch { + * case t: Throwable => span.recordException(t); span.setStatus(ERROR); throw t + * } finally { scope.close(); span.end() } + * }}} + * + * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns a + * no-op tracer, so calling this is safe at any time. + */ +object TexeraTracer { + + private val InstrumentationScope = "org.apache.texera" + + def tracer: Tracer = GlobalOpenTelemetry.getTracer(InstrumentationScope) + + /** + * Snapshot the current OTel ``Context`` so async callbacks can + * re-attach it via ``Context.makeCurrent`` later. Useful at the + * Scala to Python boundary where the calling thread is not the + * receiving thread. + */ + def currentContext: Context = Context.current() +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala b/common/observability/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala new file mode 100644 index 00000000000..f5ea4d6cfa4 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +/** + * Pure validators for W3C Trace Context headers crossing the + * Scala↔Python boundary. The single rule: if the inbound bytes do + * not match the strict regex, we discard the value and start a fresh + * trace. We never echo a rejected value back into a span, log, or + * error message. + * + * Spec reference: https://www.w3.org/TR/trace-context/ + * + * The regexes here intentionally do NOT use any context-sensitive + * grouping or backreferences, keeps the validators safe against + * pathological inputs (ReDoS) and trivially fast. + */ +object TraceparentValidator { + + /** W3C traceparent format: `---`. + * We accept only version `00` (the only published version) with the + * canonical 32-hex / 16-hex / 2-hex layout. All hex lowercase per + * spec, uppercase is invalid. + */ + private val TraceparentPattern = + "^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$".r.pattern + + /** Bounded tracestate length. Spec recommends ≤512 chars. We are + * stricter to remove a small DoS surface, an attacker can't send + * a 1 MiB tracestate to balloon downstream context allocations. + */ + val MaxTracestateLength: Int = 512 + + /** Validate a traceparent header. Returns the input unchanged on + * success, None on any failure (rejected, caller starts a fresh + * trace). Null and empty are silent failures. + */ + def validateTraceparent(header: String): Option[String] = { + if (header == null || header.isEmpty) return None + // Trace-id and parent-id must not be all-zero per spec, an + // all-zero ID is a sentinel for "no value" and MUST be rejected. + if (!TraceparentPattern.matcher(header).matches()) return None + val parts = header.split('-') + val traceId = parts(1) + val spanId = parts(2) + if (isAllZero(traceId) || isAllZero(spanId)) return None + Some(header) + } + + /** Validate a tracestate header. Spec: comma-separated list of + * key=value pairs, ASCII-printable only, total length capped. + * Returns the input unchanged on success, None on rejection. + */ + def validateTracestate(header: String): Option[String] = { + if (header == null || header.isEmpty) return None + if (header.length > MaxTracestateLength) return None + if (!isAsciiPrintable(header)) return None + Some(header) + } + + private def isAllZero(s: String): Boolean = { + var i = 0 + while (i < s.length) { + if (s.charAt(i) != '0') return false + i += 1 + } + true + } + + private def isAsciiPrintable(s: String): Boolean = { + var i = 0 + while (i < s.length) { + val c = s.charAt(i) + // Allow printable ASCII range (0x20-0x7E). Tab and CR/LF are + // rejected, a tracestate must not span lines. + if (c < 0x20 || c > 0x7e) return false + i += 1 + } + true + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala b/common/observability/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala new file mode 100644 index 00000000000..84cd2d0e8f5 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import com.typesafe.scalalogging.LazyLogging +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.common.{AttributeKey, Attributes} +import io.opentelemetry.api.metrics.Meter + +/** + * Strongly-typed facade for the workflow-execution metrics cluster. + * + * This facade pattern is deliberate here because these instruments are a + * small, fixed, correlated set with strict cardinality rules, and it is + * worth centralizing that control in one place. It is NOT the default + * pattern for metrics in general: most call sites should just call the OTel + * meter API directly next to the business logic, e.g. + * + * {{{ + * private val meter = GlobalOpenTelemetry.getMeter("org.apache.texera") + * private val requests = meter.counterBuilder("myfeature.requests").build() + * // in the handler: + * requests.add(1, Attributes.of(AttributeKey.stringKey("route"), route)) + * }}} + * + * Only reach for a facade like this one when the metrics are complex or + * need centralized, standardized control. Do not copy it by default. + * + * Cardinality safety here is enforced by the API surface, not by + * documentation: there is no public method that accepts an arbitrary + * string as a label key or value. The only labels that ever land on + * an instrument are the two enums [[Outcome]] and [[WorkflowKind]], + * each restricted to a fixed set. ``workflow.id`` / ``execution.id`` + * are deliberately NOT metric labels: per-execution detail belongs + * in traces and logs, joined on ``trace_id`` at query time. + * + * Histogram bucket bounds are hard-coded constants so they can't be + * coerced by request input. The OTel SDK applies its own default + * attribute-value-length cap to anything that does slip through. + */ +object WorkflowMetrics extends LazyLogging { + + /** Outcome enum, the only mutable label on lifecycle counters. */ + sealed abstract class Outcome(val name: String) + object Outcome { + case object Success extends Outcome("success") + case object Failure extends Outcome("failure") + case object Cancelled extends Outcome("cancelled") + } + + /** Workflow kind enum. Distinguishes interactive vs. scheduled + * workflows for the dashboard's basic split, extend deliberately. + */ + sealed abstract class WorkflowKind(val name: String) + object WorkflowKind { + case object Interactive extends WorkflowKind("interactive") + case object Scheduled extends WorkflowKind("scheduled") + case object Unknown extends WorkflowKind("unknown") + } + + private val OutcomeKey: AttributeKey[String] = AttributeKey.stringKey("texera.outcome") + private val WorkflowKindKey: AttributeKey[String] = AttributeKey.stringKey("texera.workflow.kind") + + /** Histogram bucket bounds in seconds. Hard-coded, constants so + * request input can't reshape the histogram. Range covers + * fast (<1s) to long (>1h) workflows. + */ + private val DurationBuckets: java.util.List[java.lang.Double] = { + val builder = new java.util.ArrayList[java.lang.Double]() + Seq(0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0, 600.0, 1800.0, 3600.0) + .foreach(b => builder.add(java.lang.Double.valueOf(b))) + builder + } + + private val InstrumentationScope = "org.apache.texera" + + // Instruments are lazy + memoised. The first call after SDK init + // builds them against the active GlobalOpenTelemetry meter; once + // built they hold the meter instance, so a later GlobalOpenTelemetry + // reset (in tests) wouldn't be visible here, see [[resetForTest]]. + @volatile private var _starts: io.opentelemetry.api.metrics.LongCounter = _ + @volatile private var _completions: io.opentelemetry.api.metrics.LongCounter = _ + @volatile private var _cancellations: io.opentelemetry.api.metrics.LongCounter = _ + // `texera.workflow.active` is an OBSERVABLE gauge, not a manual up/down + // counter. A manual +1/-1 counter leaks whenever a run starts but its + // terminal event never fires (the controller is killed, the process + // restarts mid-run, or the +1 and -1 are split across the engine and web + // tiers and one side is missed), the gauge then drifts upward forever and + // the dashboard shows phantom "active" executions. An observable gauge + // instead reports the TRUE in-progress count from the live execution + // registry on every collection, so it cannot leak. The count is supplied + // by the host process via [[setActiveExecutionsSupplier]]. + @volatile private var _active: io.opentelemetry.api.metrics.ObservableLongGauge = _ + @volatile private var _duration: io.opentelemetry.api.metrics.DoubleHistogram = _ + + // Supplier of the current in-progress execution count. Defaults to 0 until + // the host process registers the real source (so a process that never + // registers reports a flat 0 rather than a wrong number). Read by the gauge + // callback on every metric collection. + @volatile private var activeExecutionsSupplier: () => Long = () => 0L + + /** Register the authoritative source of "currently active executions". + * The supplier is polled on every metric collection, so the gauge always + * reflects ground truth and can never leak. Called once at process + * startup (e.g. by ComputingUnitMaster). + */ + def setActiveExecutionsSupplier(supplier: () => Long): Unit = + synchronized { + activeExecutionsSupplier = supplier + ensureBound() + } + + /** Bind instruments to the current global meter. Idempotent, the + * first call wins; later calls are no-ops. Tests can call + * [[bindForTest]] with an explicit Meter, then [[resetForTest]] to + * rebind. + */ + def ensureBound(): Unit = + synchronized { + if (_starts == null) bind(GlobalOpenTelemetry.getMeter(InstrumentationScope)) + } + + private[observability] def bindForTest(meter: Meter): Unit = + synchronized { + bind(meter) + } + + private[observability] def resetForTest(): Unit = + synchronized { + _starts = null + _completions = null + _cancellations = null + // The observable gauge registered a collection callback, close it so the + // previous test's meter provider stops being polled after it's discarded. + if (_active != null) _active.close() + _active = null + _duration = null + } + + private def bind(meter: Meter): Unit = { + _starts = meter + .counterBuilder("texera.workflow.starts") + .setDescription("Number of workflow executions started.") + .build() + // Completions carry texera.outcome={success|failure}. Both success + // and self-terminating failures land here so the success/failure-rate + // queries (non-success ÷ all completions) have a denominator that + // means "runs that finished on their own". User-initiated kills are + // NOT completions, see _cancellations. + _completions = meter + .counterBuilder("texera.workflow.completions") + .setDescription("Number of workflow executions that ran to completion (success or failure).") + .build() + // Cancellations (user kills) are tracked separately so they decrement + // the active gauge without polluting the success/failure-rate + // denominator. Deliberately label-free apart from workflow.kind. + _cancellations = meter + .counterBuilder("texera.workflow.cancellations") + .setDescription("Number of workflow executions cancelled/killed before finishing.") + .build() + // Observable gauge: the callback runs on each collection and reports the + // live in-progress count, so the value cannot leak. No per-execution + // labels (cardinality-safe); the dashboard queries sum(texera_workflow_active). + _active = meter + .gaugeBuilder("texera.workflow.active") + .ofLongs() + .setDescription( + "Number of workflow executions currently in progress (observed from the live registry)." + ) + .buildWithCallback(obs => obs.record(activeExecutionsSupplier())) + _duration = meter + .histogramBuilder("texera.workflow.duration") + .setDescription("End-to-end duration of a workflow execution.") + .setUnit("s") + .setExplicitBucketBoundariesAdvice(DurationBuckets) + .build() + logger.info(s"Texera metric instruments bound to meter scope '$InstrumentationScope'") + } + + // ---- Public emitters, typed, no untyped escape hatch -------------- + + def recordStart(kind: WorkflowKind): Unit = { + ensureBound() + logger.debug(s"metric: workflow started (kind=${kind.name}), starts +1") + _starts.add(1L, Attributes.of(WorkflowKindKey, kind.name)) + // `active` is no longer mutated here, it is an observable gauge sourced + // from the live execution registry (see setActiveExecutionsSupplier). + } + + def recordCompletion(kind: WorkflowKind, durationSec: Double): Unit = { + ensureBound() + logger.debug( + f"metric: workflow completed successfully (kind=${kind.name}, ${durationSec}%.1fs), completions +1" + ) + val attrs = Attributes.of(OutcomeKey, Outcome.Success.name, WorkflowKindKey, kind.name) + _completions.add(1L, attrs) + _duration.record(durationSec, attrs) + } + + def recordFailure(kind: WorkflowKind, durationSec: Double): Unit = { + ensureBound() + logger.debug( + f"metric: workflow failed (kind=${kind.name}, ${durationSec}%.1fs), completions +1 (outcome=failure)" + ) + // A failure is a completion with outcome=failure, it shares the + // completions counter (and duration histogram) with successes so the + // failure-rate query has both numerator and denominator. + val attrs = Attributes.of(OutcomeKey, Outcome.Failure.name, WorkflowKindKey, kind.name) + _completions.add(1L, attrs) + _duration.record(durationSec, attrs) + } + + /** A user-initiated kill/cancel. Bumps a dedicated counter. Deliberately + * NOT recorded as a completion: a cancelled run never finished on its own, + * so it must not drag down the success rate. No duration is recorded for + * the same reason, a killed run's wall-clock time is not a real runtime + * and would skew the duration percentiles. (The active gauge is observed + * from the live registry and needs no decrement here.) + */ + def recordCancellation(kind: WorkflowKind): Unit = { + ensureBound() + logger.debug(s"metric: workflow cancelled (kind=${kind.name}), cancellations +1") + val attrs = Attributes.of(WorkflowKindKey, kind.name) + _cancellations.add(1L, attrs) + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala new file mode 100644 index 00000000000..8fabc602a8e --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.`export`.SimpleSpanProcessor +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class SpanAttrsSpec extends AnyFlatSpec with Matchers { + + // ----- sanitizeFreeText: pure ---------------------------------------- + + "sanitizeFreeText" should "return null for null / empty input" in { + SpanAttrs.sanitizeFreeText(null) shouldBe null + SpanAttrs.sanitizeFreeText("") shouldBe null + } + + it should "strip CR/LF and other control characters" in { + SpanAttrs.sanitizeFreeText("hello\r\nworld") shouldBe "helloworld" + SpanAttrs.sanitizeFreeText("a\u0000b\u0001c\u001fd\u007fe") shouldBe "abcde" + } + + it should "preserve printable text including spaces" in { + SpanAttrs.sanitizeFreeText("my-op v1 (alpha)") shouldBe "my-op v1 (alpha)" + } + + it should "cap at FreeTextMaxLen" in { + val long = "x" * (SpanAttrs.FreeTextMaxLen * 4) + val out = SpanAttrs.sanitizeFreeText(long) + out.length shouldBe SpanAttrs.FreeTextMaxLen + } + + it should "return null when stripping reduces the input to empty" in { + SpanAttrs.sanitizeFreeText("\r\n\u0000\u0001") shouldBe null + } + + // ----- standard label keys land via the plain OTel API --------------- + + "the standard label keys" should "set correctly through the OTel builder API" in { + val (exporter, tracer) = newTracer() + val sb = tracer.spanBuilder("workflow.execute") + sb.setAttribute(SpanAttrs.WorkflowId, java.lang.Long.valueOf(42L)) + sb.setAttribute(SpanAttrs.ExecutionId, java.lang.Long.valueOf(7L)) + sb.setAttribute(SpanAttrs.OperatorName, SpanAttrs.sanitizeFreeText("good\r\nFAKE LINE")) + sb.startSpan().end() + + val attrs = exporter.getFinishedSpanItems.asScala.head.getAttributes + attrs.get(SpanAttrs.WorkflowId) shouldBe 42L + attrs.get(SpanAttrs.ExecutionId) shouldBe 7L + attrs.get(SpanAttrs.OperatorName) shouldBe "goodFAKE LINE" + } + + it should "expose stable key names so callsites share one label vocabulary" in { + SpanAttrs.WorkflowId.getKey shouldBe "texera.workflow.id" + SpanAttrs.ExecutionId.getKey shouldBe "texera.execution.id" + } + + // ----- attribute-count cap is the OTel SDK default (128) ------------- + + it should "respect the SDK's per-span attribute cap of 128" in { + val (exporter, tracer) = newTracer() + val sb = tracer.spanBuilder("many") + // Set 200 distinct keys; SDK should drop everything past the + // default cap of 128 without crashing the test. + (0 until 200).foreach { i => + sb.setAttribute(s"k$i", s"v$i") + } + sb.startSpan().end() + + val attrs = exporter.getFinishedSpanItems.asScala.head.getAttributes + attrs.size should be <= 128 + } + + private def newTracer(): (InMemorySpanExporter, io.opentelemetry.api.trace.Tracer) = { + val exporter = InMemorySpanExporter.create() + val tp = SdkTracerProvider + .builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build() + val sdk = OpenTelemetrySdk.builder().setTracerProvider(tp).build() + (exporter, sdk.getTracer("test")) + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/TexeraTracerSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/TexeraTracerSpec.scala new file mode 100644 index 00000000000..77d5b3ab0f9 --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/TexeraTracerSpec.scala @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.context.Context +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class TexeraTracerSpec extends AnyFlatSpec with Matchers { + + "TexeraTracer.tracer" should "return a tracer that can build and start a span" in { + val tracer = TexeraTracer.tracer + tracer should not be null + // Without an installed SDK this is a no-op tracer, but it must still + // hand back a usable span rather than null so callsites never NPE. + val span = tracer.spanBuilder("texera.tracer.spec").startSpan() + span should not be null + span.end() + } + + "TexeraTracer.tracer" should "be stable across calls (same instrumentation scope)" in { + TexeraTracer.tracer should not be null + TexeraTracer.tracer should not be null + } + + "TexeraTracer.currentContext" should "return the active context" in { + TexeraTracer.currentContext shouldBe Context.current() + } + + it should "observe a value made current via the OpenTelemetry context" in { + val key = io.opentelemetry.context.ContextKey.named[String]("texera.tracer.spec.key") + val scope = Context.current().`with`(key, "v").makeCurrent() + try { + TexeraTracer.currentContext.get(key) shouldBe "v" + } finally { + scope.close() + } + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala new file mode 100644 index 00000000000..6bf74d99b38 --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class TraceparentValidatorSpec extends AnyFlatSpec with Matchers { + + // ----- traceparent: positive paths ----------------------------------- + + "validateTraceparent" should "accept a canonical W3C traceparent" in { + val good = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" + TraceparentValidator.validateTraceparent(good) shouldBe Some(good) + } + + it should "accept sampled=0 flag" in { + val good = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-00" + TraceparentValidator.validateTraceparent(good) shouldBe Some(good) + } + + // ----- traceparent: negative paths ----------------------------------- + + it should "reject null and empty input" in { + TraceparentValidator.validateTraceparent(null) shouldBe None + TraceparentValidator.validateTraceparent("") shouldBe None + } + + it should "reject path-traversal-style spoofing attempts" in { + TraceparentValidator.validateTraceparent("../../etc/passwd") shouldBe None + TraceparentValidator.validateTraceparent( + "00-../../../etc/passwd-b7ad6b7169203331-01" + ) shouldBe None + } + + it should "reject the wrong version byte" in { + // Only version 00 is published. 01, ff, etc. are invalid. + val bad01 = "01-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" + val badff = "ff-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" + TraceparentValidator.validateTraceparent(bad01) shouldBe None + TraceparentValidator.validateTraceparent(badff) shouldBe None + } + + it should "reject all-zero trace-id (sentinel for no-value)" in { + val zero = "00-00000000000000000000000000000000-b7ad6b7169203331-01" + TraceparentValidator.validateTraceparent(zero) shouldBe None + } + + it should "reject all-zero span-id (sentinel for no-value)" in { + val zero = "00-0af7651916cd43dd8448eb211c80319c-0000000000000000-01" + TraceparentValidator.validateTraceparent(zero) shouldBe None + } + + it should "reject UPPERCASE hex (spec requires lowercase)" in { + val upper = "00-0AF7651916CD43DD8448EB211C80319C-b7ad6b7169203331-01" + TraceparentValidator.validateTraceparent(upper) shouldBe None + } + + it should "reject non-hex characters in trace-id" in { + val bad = "00-XYZ7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" + TraceparentValidator.validateTraceparent(bad) shouldBe None + } + + it should "reject wrong-length segments" in { + // trace-id one char short + val short = "00-af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" + // span-id one char long + val long = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b71692033311-01" + TraceparentValidator.validateTraceparent(short) shouldBe None + TraceparentValidator.validateTraceparent(long) shouldBe None + } + + it should "reject embedded CRLF (header injection)" in { + val crlf = + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01\r\nX-Forge: yes" + TraceparentValidator.validateTraceparent(crlf) shouldBe None + } + + // ----- tracestate ---------------------------------------------------- + + "validateTracestate" should "accept a canonical tracestate" in { + val ts = "vendor1=hello,vendor2=world" + TraceparentValidator.validateTracestate(ts) shouldBe Some(ts) + } + + it should "reject null and empty" in { + TraceparentValidator.validateTracestate(null) shouldBe None + TraceparentValidator.validateTracestate("") shouldBe None + } + + it should "reject tracestate longer than the cap" in { + val oversize = "a=" + ("b" * (TraceparentValidator.MaxTracestateLength + 1)) + TraceparentValidator.validateTracestate(oversize) shouldBe None + } + + it should "reject non-ASCII / control characters" in { + TraceparentValidator.validateTracestate("vendor=hello\u00ffworld") shouldBe None + TraceparentValidator.validateTracestate("vendor=hello\r\nworld") shouldBe None + TraceparentValidator.validateTracestate("vendor=hello\u0000world") shouldBe None + } + + it should "accept tracestate at exactly the cap" in { + val atCap = "k=" + ("v" * (TraceparentValidator.MaxTracestateLength - 2)) + atCap.length shouldBe TraceparentValidator.MaxTracestateLength + TraceparentValidator.validateTracestate(atCap) shouldBe Some(atCap) + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala new file mode 100644 index 00000000000..17c9c4c74dd --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.data.MetricData +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class WorkflowMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + private var reader: InMemoryMetricReader = _ + private var provider: SdkMeterProvider = _ + + override def beforeEach(): Unit = { + reader = InMemoryMetricReader.create() + provider = SdkMeterProvider.builder().registerMetricReader(reader).build() + WorkflowMetrics.resetForTest() + WorkflowMetrics.bindForTest(provider.get("org.apache.texera")) + } + + override def afterEach(): Unit = { + WorkflowMetrics.resetForTest() + provider.close() + } + + private def collectAll(): Map[String, MetricData] = { + reader.collectAllMetrics().asScala.map(m => m.getName -> m).toMap + } + + // ----- positive: lifecycle emissions ---------------------------------- + + "WorkflowMetrics" should "increment workflow.starts on recordStart" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + + val metrics = collectAll() + metrics.keySet should contain("texera.workflow.starts") + + val starts = metrics("texera.workflow.starts").getLongSumData.getPoints.asScala.head + starts.getValue shouldBe 1L + // recordStart no longer mutates `active` — that gauge is observed from the + // live registry (see "report the active-execution count…" below). + } + + it should "report the active-execution count from the registered supplier, never a manual counter" in { + // The whole point of the fix: `active` is an observable gauge sourced from + // ground truth, so it cannot leak. Drive it with a stub supplier and + // confirm the gauge reports exactly what the supplier returns — regardless + // of how many starts/completions were recorded. + @volatile var live = 3L + WorkflowMetrics.setActiveExecutionsSupplier(() => live) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + + val first = collectAll()("texera.workflow.active").getLongGaugeData.getPoints.asScala.head + first.getValue shouldBe 3L + + // It tracks the live source on the next collection — a manual +1/-1 + // counter could never drop like this without an explicit decrement. + live = 0L + val second = collectAll()("texera.workflow.active").getLongGaugeData.getPoints.asScala.head + second.getValue shouldBe 0L + } + + it should "record a completion and duration sample on recordCompletion" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = 2.5) + + val metrics = collectAll() + metrics.keySet should contain allOf ( + "texera.workflow.completions", + "texera.workflow.duration" + ) + + metrics( + "texera.workflow.completions" + ).getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + + val histogram = metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head + histogram.getCount shouldBe 1L + histogram.getSum shouldBe 2.5 + } + + it should "record a failure as a non-success completion (so failure-rate queries work)" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Scheduled) + WorkflowMetrics.recordFailure(WorkflowMetrics.WorkflowKind.Scheduled, durationSec = 12.0) + + val metrics = collectAll() + // A failure shares the completions counter with successes — the + // failure-rate query divides non-success completions by all + // completions, so failures must live here (not in a separate + // series the query never reads). + val completion = metrics("texera.workflow.completions").getLongSumData.getPoints.asScala.head + completion.getValue shouldBe 1L + completion.getAttributes.asMap.asScala.map { + case (k, v) => k.getKey -> v.toString + } should contain( + "texera.outcome" -> "failure" + ) + metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head.getSum shouldBe 12.0 + // The orphan counter the old wiring used must be gone. + metrics.keySet should not contain "texera.workflow.failures" + } + + it should "record a cancellation that is not a completion" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCancellation(WorkflowMetrics.WorkflowKind.Interactive) + + val metrics = collectAll() + metrics( + "texera.workflow.cancellations" + ).getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + // A kill is not a completion and records no duration: it must not + // drag down the success rate nor skew the duration percentiles. + metrics.keySet should not contain "texera.workflow.completions" + metrics.keySet should not contain "texera.workflow.duration" + } + + // ----- security: cardinality safety ----------------------------------- + + it should "only emit the texera.outcome and texera.workflow.kind labels" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = 1.0) + + val attrKeys = collectAll().values.flatMap { md => + val pointSet = md.getType.name() match { + case "HISTOGRAM" => md.getHistogramData.getPoints.asScala + case "LONG_GAUGE" => md.getLongGaugeData.getPoints.asScala + case _ => md.getLongSumData.getPoints.asScala + } + pointSet.flatMap(_.getAttributes.asMap.keySet.asScala.map(_.getKey)) + }.toSet + + attrKeys.foreach { key => + Set("texera.outcome", "texera.workflow.kind") should contain(key) + } + attrKeys should not contain "texera.workflow.id" + attrKeys should not contain "texera.execution.id" + } + + it should "expose no public API to attach an arbitrary string label" in { + // The class has only typed emitters whose attribute set is + // hard-coded. This test is intentionally a compile-time check + // disguised as a runtime one — if a future contributor adds an + // untyped public method like recordStart(attrs: Attributes), + // this assertion still passes but the design intent is broken. + // Make the intent explicit: + val methodNames = classOf[WorkflowMetrics.type].getDeclaredMethods + .map(_.getName) + .toSet + methodNames should contain allOf ("recordStart", "recordCompletion", "recordFailure") + methodNames should not contain "recordWithAttributes" + } + + // ----- histogram buckets are constants -------------------------------- + + it should "use the hard-coded explicit bucket boundaries for duration" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + // Hit a few bucket bounds. + Seq(0.05, 0.6, 7.0, 65.0, 400.0).foreach { d => + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = d) + } + + val histogram = collectAll()("texera.workflow.duration").getHistogramData + val point = histogram.getPoints.asScala.head + point.getCount shouldBe 5L + // The point exposes the SDK-configured boundaries; we don't + // assert exact values here (would couple the test to the impl) + // but we do assert there ARE explicit boundaries — anything + // empty would mean the .setExplicitBucketBoundariesAdvice call + // was lost during a refactor. + point.getBoundaries.size should be > 0 + } +} From 722ec3d47cabcca1d11b3b5628fb085d3db2ae4f Mon Sep 17 00:00:00 2001 From: Matthew Ball Date: Fri, 9 Oct 2026 11:00:24 -0700 Subject: [PATCH 2/2] feat(observability): deployment (collector, Grafana/Loki, Parca) Add the local deployment wiring that receives, stores, and visualizes the telemetry emitted by PR 2: an OpenTelemetry Collector, per-signal backends, and a Grafana UI, for both the single-node docker-compose stack and the Kubernetes Helm chart. Infrastructure only; the application runs unchanged whether or not these services are started. This is PR 3 of the observability stack. Single-node compose (bin/single-node, bin/observability): - otel-collector as the single OTLP ingress, routing logs to Loki over OTLP HTTP, metrics to VictoriaMetrics via prometheusremotewrite, and traces to Jaeger over OTLP; the collector config caps OTLP message size and bounds memory via memory_limiter. - Grafana as the dashboard/UI, pre-provisioned (bin/observability/grafana/provisioning) with datasources for Loki, VictoriaMetrics (Prometheus-compatible query API), and Jaeger, so operators get log search, metric dashboards, and trace views without a Texera-built UI. - Loki as the log store (bin/observability/loki/config.yaml, OTLP ingest on /otlp), VictoriaMetrics as the metric store, Jaeger as the trace store, and Parca server plus eBPF agent for continuous profiling (bin/observability/parca). - A one-shot victoriametrics-init container chowns the metrics volume to VictoriaMetrics's non-root uid before it starts, so a freshly created (root-owned) named volume does not crash-loop the read-only VM container; it is idempotent on an existing volume. - Each signal is its own compose profile and is OFF by default: COMPOSE_PROFILES is empty, so observability is opt-in, and Grafana has its own observability-grafana profile. - Every observability host port, including the Grafana UI, binds to loopback (127.0.0.1); nothing is published on 0.0.0.0. Emission is gated by OTEL_SDK_DISABLED (default disabled), so nothing is emitted unless an operator opts in alongside the collector profile. Kubernetes Helm chart (bin/k8s): - Templates for the collector, Grafana, Loki, Jaeger, and VictoriaMetrics under templates/base/observability, plus values.yaml knobs and an install.sh helper. Grafana is a ClusterIP service with its datasources provisioned from a ConfigMap. - Service deployments gain the OTLP endpoint wiring. Licensing: the Collector, VictoriaMetrics, Jaeger, and Parca are Apache-2.0 and form the shipped reference stack. Grafana and Loki are AGPLv3 optional operator-run components, documented as such in bin/observability/README.md and not bundled in any Texera release artifact. Docs: bin/observability/README.md describes the stack, how to reach Grafana at http://localhost:3000 with the pre-provisioned datasources, and how to point an external Grafana or OpenSearch at the same endpoints instead. Tests: ObservabilityComposeSpec and ParcaConfigSpec are string-level smoke tests over the compose, collector, and Parca config. They pin the image versions, assert the opt-in default and loopback binding for every port, and check the collector routes logs to Loki, metrics to VictoriaMetrics, and traces to Jaeger. bin/observability/smoke_test.py is a runtime routing check meant to run against a live stack: it pushes a log, a metric, and a trace per service through the collector over OTLP and asserts each reaches its correct backend (Loki, VictoriaMetrics, Jaeger) attributed to the emitting service, plus an isolation check that one service's data is not mis-attributed to another; it exits non-zero on a routing regression. Stacked on PR 2 (obs/pr2/backend-emit). --- .../workflows/create-release-candidate.yml | 11 +- .licenserc.yaml | 2 + bin/k8s/Chart.lock | 21 ++ bin/k8s/install.sh | 56 ++++ bin/k8s/templates/base/_helpers.tpl | 98 +++++++ .../access-control-service-deployment.yaml | 1 + .../config-service-deployment.yaml | 1 + .../file-service/file-service-deployment.yaml | 3 +- .../templates/base/observability/grafana.yaml | 125 +++++++++ .../templates/base/observability/jaeger.yaml | 81 ++++++ .../templates/base/observability/loki.yaml | 136 ++++++++++ .../otel-collector-configmap.yaml | 109 ++++++++ .../base/observability/otel-collector.yaml | 95 +++++++ .../base/observability/victoriametrics.yaml | 112 ++++++++ .../base/webserver/webserver-deployment.yaml | 3 +- ...workflow-compiling-service-deployment.yaml | 3 +- ...low-computing-unit-manager-deployment.yaml | 3 +- bin/k8s/values.yaml | 81 ++++++ bin/local-dev/docker-compose.override.yml | 8 + bin/observability/README.md | 131 +++++++++ bin/observability/docker-compose.yml | 39 +++ .../provisioning/datasources/datasources.yaml | 51 ++++ bin/observability/loki/config.yaml | 84 ++++++ bin/observability/otel-collector/config.yaml | 132 +++++++++ bin/observability/parca/README.md | 72 +++++ bin/observability/parca/parca-agent.env | 60 +++++ bin/observability/parca/parca.yaml | 42 +++ bin/observability/smoke_test.py | 207 +++++++++++++++ bin/single-node/.env | 65 ++++- bin/single-node/docker-compose.override.yml | 29 ++ bin/single-node/docker-compose.yml | 250 +++++++++++++++++- .../ObservabilityComposeSpec.scala | 210 +++++++++++++++ .../observability/ParcaConfigSpec.scala | 129 +++++++++ .../ComputingUnitManagingResource.scala | 42 ++- .../ComputingUnitManagingResourceSpec.scala | 37 +++ 35 files changed, 2511 insertions(+), 18 deletions(-) create mode 100644 bin/k8s/Chart.lock create mode 100755 bin/k8s/install.sh create mode 100644 bin/k8s/templates/base/observability/grafana.yaml create mode 100644 bin/k8s/templates/base/observability/jaeger.yaml create mode 100644 bin/k8s/templates/base/observability/loki.yaml create mode 100644 bin/k8s/templates/base/observability/otel-collector-configmap.yaml create mode 100644 bin/k8s/templates/base/observability/otel-collector.yaml create mode 100644 bin/k8s/templates/base/observability/victoriametrics.yaml create mode 100644 bin/observability/README.md create mode 100644 bin/observability/docker-compose.yml create mode 100644 bin/observability/grafana/provisioning/datasources/datasources.yaml create mode 100644 bin/observability/loki/config.yaml create mode 100644 bin/observability/otel-collector/config.yaml create mode 100644 bin/observability/parca/README.md create mode 100644 bin/observability/parca/parca-agent.env create mode 100644 bin/observability/parca/parca.yaml create mode 100644 bin/observability/smoke_test.py create mode 100644 bin/single-node/docker-compose.override.yml create mode 100644 common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala create mode 100644 common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala diff --git a/.github/workflows/create-release-candidate.yml b/.github/workflows/create-release-candidate.yml index 1e12d228083..d73b1df6dda 100644 --- a/.github/workflows/create-release-candidate.yml +++ b/.github/workflows/create-release-candidate.yml @@ -174,7 +174,7 @@ jobs: # Export the single-node directory from the tagged source mkdir -p "$TEMP_DIR/_raw" - git archive --format=tar "$TAG_NAME" -- bin/single-node/ sql/ NOTICE licenses/LICENSE-BSD-3-Clause.txt | tar -x -C "$TEMP_DIR/_raw" + git archive --format=tar "$TAG_NAME" -- bin/single-node/ bin/observability/ sql/ NOTICE licenses/LICENSE-BSD-3-Clause.txt | tar -x -C "$TEMP_DIR/_raw" # Copy deployment files cp "$TEMP_DIR/_raw/bin/single-node/docker-compose.yml" "$BUNDLE_DIR/" @@ -187,10 +187,19 @@ jobs: cp "$TEMP_DIR/_raw/licenses/LICENSE-BSD-3-Clause.txt" "$BUNDLE_DIR/licenses/" cp -r "$TEMP_DIR/_raw/sql" "$BUNDLE_DIR/" + # Observability configs (OTel collector + Parca) referenced by the + # opt-in observability compose profiles. Shipped next to the compose + # file so the ../observability mounts resolve in the flattened bundle. + cp -r "$TEMP_DIR/_raw/bin/observability" "$BUNDLE_DIR/observability" + # Patch the SQL mount path for the self-contained bundle layout # In the repo it's ../../sql (relative to bin/single-node/), in the bundle it's ./sql sed -i 's|../../sql|./sql|g' "$BUNDLE_DIR/docker-compose.yml" + # Patch the observability mount paths: ../observability (relative to + # bin/single-node/ in the repo) becomes ./observability in the bundle. + sed -i 's|../observability|./observability|g' "$BUNDLE_DIR/docker-compose.yml" + # Generate a release-pinned .env file with the version tag # Start from the source .env and ensure IMAGE_REGISTRY and IMAGE_TAG are set cp "$TEMP_DIR/_raw/bin/single-node/.env" "$BUNDLE_DIR/.env" diff --git a/.licenserc.yaml b/.licenserc.yaml index cbef454e403..0cebbbb3ca5 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -37,6 +37,8 @@ header: - '**/.env.example' - '**/.prettierrc' - '**/bun.lock' + # Helm's generated dependency lock file (no header; like yarn.lock/bun.lock) + - '**/Chart.lock' # Third-party code with MIT license - see LICENSE file for attribution - 'common/workflow-operator/src/main/scala/com/kjetland/**' # TypeFox monaco-languageclient derived files (MIT License) diff --git a/bin/k8s/Chart.lock b/bin/k8s/Chart.lock new file mode 100644 index 00000000000..af90dd0b29b --- /dev/null +++ b/bin/k8s/Chart.lock @@ -0,0 +1,21 @@ +dependencies: +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 16.5.6 +- name: minio + repository: https://charts.bitnami.com/bitnami + version: 15.0.7 +- name: lakefs + repository: https://charts.lakefs.io + version: 1.8.1 +- name: gateway-helm + repository: oci://docker.io/envoyproxy + version: 1.6.3 +- name: lakekeeper + repository: https://lakekeeper.github.io/lakekeeper-charts/ + version: 0.9.0 +- name: metrics-server + repository: https://kubernetes-sigs.github.io/metrics-server/ + version: 3.12.2 +digest: sha256:031184824e3bd3e03883a3debe64459f462610e4f5bd7c147f03fd05333b169e +generated: "2026-07-11T11:09:44.800500493-07:00" diff --git a/bin/k8s/install.sh b/bin/k8s/install.sh new file mode 100755 index 00000000000..d1ed9b55e6e --- /dev/null +++ b/bin/k8s/install.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# One-command install/upgrade of the Texera Helm chart. +# +# `helm install` on its own fails on a fresh checkout because the subchart +# dependencies (postgresql, minio, lakefs, envoy-gateway, lakekeeper, +# metrics-server) are declared in Chart.yaml but not vendored into charts/. +# This wrapper fetches them first (idempotent), then upgrades-or-installs. +# +# Usage (run from anywhere): +# bin/k8s/install.sh # app only +# bin/k8s/install.sh --set observability.enabled=true # app + observability +# RELEASE=texera NAMESPACE=texera bin/k8s/install.sh --set observability.enabled=true +# +# Any extra args are forwarded verbatim to `helm upgrade --install`, so +# --set / -f values.override.yaml / --namespace etc. all work. + +set -euo pipefail + +CHART_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RELEASE="${RELEASE:-texera}" +NAMESPACE="${NAMESPACE:-texera}" + +command -v helm >/dev/null 2>&1 || { echo "helm not found on PATH" >&2; exit 1; } + +# Fetch/refresh subcharts into charts/. `dependency build` uses Chart.lock when +# present (reproducible); it falls back to `dependency update` if there is no +# lock yet. Both are safe to re-run. +echo "==> Resolving chart dependencies" +if [ -f "${CHART_DIR}/Chart.lock" ]; then + helm dependency build "${CHART_DIR}" +else + helm dependency update "${CHART_DIR}" +fi + +echo "==> helm upgrade --install ${RELEASE} (namespace: ${NAMESPACE})" +exec helm upgrade --install "${RELEASE}" "${CHART_DIR}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + "$@" diff --git a/bin/k8s/templates/base/_helpers.tpl b/bin/k8s/templates/base/_helpers.tpl index fe25aab1b72..8f310823851 100644 --- a/bin/k8s/templates/base/_helpers.tpl +++ b/bin/k8s/templates/base/_helpers.tpl @@ -55,6 +55,26 @@ services fall back to the in-cluster RustFS Service and its auto-generated {{- if .Values.storage.s3.endpoint -}}secret-access-key{{- else -}}RUSTFS_SECRET_KEY{{- end -}} {{- end -}} +{{/* +Observability emission env for a Scala service pod. + +Renders OTEL_EXPORTER_OTLP_ENDPOINT (pointing at the in-cluster OTel +Collector Service) plus TEXERA_OTEL_ALLOWED_HOSTS. OtelInit validates the +endpoint host against an allowlist whose default is localhost only, so the +collector's Service name must be added explicitly or the SDK rejects it and +emits nothing. Renders nothing unless both the observability stack and the +collector are enabled, so the default install is unchanged. Include inside a +container's `env:` list, e.g. `{{- include "texera.observability.env" . | nindent 12 }}`. +*/}} +{{- define "texera.observability.env" -}} +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://{{ .Release.Name }}-otel-collector:{{ .Values.observability.collector.grpcPort }}" +- name: TEXERA_OTEL_ALLOWED_HOSTS + value: "{{ .Release.Name }}-otel-collector" +{{- end }} +{{- end -}} + {{/* The audience the mounter's service-account tokens are bound to. Fixed rather than configurable: it is one half of a credential contract between access-control-service and the @@ -83,3 +103,81 @@ append the uid to it, so a bare value would render "http://jupyter/7". * {{- printf "/%s" (trimAll "/" .Values.jupyterPool.basePath) -}} {{- end -}} +{{/* Grafana datasource provisioning body. Kept as a helper so the Deployment's +checksum/config annotation can hash just this content; hashing the template that +holds the annotation would make it reference itself and recurse. */}} +{{- define "texera.observability.grafanaDatasources" -}} +apiVersion: 1 +datasources: + # Metrics: VictoriaMetrics speaks the Prometheus query API. Default. + - name: Metrics + type: prometheus + uid: metrics + access: proxy + url: http://{{ .Release.Name }}-victoriametrics:{{ .Values.observability.metrics.port }} + isDefault: true + editable: false +{{- if .Values.observability.logs.enabled }} + # Logs: Loki, LogQL over OTLP-ingested logs. + - name: Loki + type: loki + uid: loki + access: proxy + url: http://{{ .Release.Name }}-loki:{{ .Values.observability.logs.port }} + editable: false +{{- end }} +{{- if .Values.observability.traces.enabled }} + # Traces: Jaeger query API / UI backend. + - name: Jaeger + type: jaeger + uid: jaeger + access: proxy + url: http://{{ .Release.Name }}-jaeger:{{ .Values.observability.traces.queryPort }} + editable: false +{{- end }} +{{- end -}} + +{{/* Loki server config body. A helper for the same reason as the Grafana one, +and so the checksum/config annotation hashes the config instead of its template. +Retention is enforced here: the compactor deletes chunks older than +observability.logs.retentionPeriod, which Loki otherwise ignores. */}} +{{- define "texera.observability.lokiConfig" -}} +# No multi-tenancy: one implicit tenant for the in-cluster stack. +auth_enabled: false +server: + http_listen_port: {{ .Values.observability.logs.port }} + grpc_listen_port: 9096 +common: + instance_addr: 127.0.0.1 + # Every component's data path defaults under this prefix, the PVC at /loki. + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + # Required for OTLP ingest (OTLP attributes -> structured metadata). + allow_structured_metadata: true + reject_old_samples: false + # Age after which log chunks are eligible for deletion by the compactor. + retention_period: {{ .Values.observability.logs.retentionPeriod }} +compactor: + working_directory: /loki/compactor + # Without this the retention_period above is parsed but never enforced, + # so logs accumulate until the volume fills. + retention_enabled: true + delete_request_store: filesystem +{{- end -}} diff --git a/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml b/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml index 99419d1f549..be518a09717 100644 --- a/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml +++ b/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml @@ -89,6 +89,7 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} livenessProbe: httpGet: path: /api/healthcheck diff --git a/bin/k8s/templates/base/config-service/config-service-deployment.yaml b/bin/k8s/templates/base/config-service/config-service-deployment.yaml index 7b86369f256..54fd1e99594 100644 --- a/bin/k8s/templates/base/config-service/config-service-deployment.yaml +++ b/bin/k8s/templates/base/config-service/config-service-deployment.yaml @@ -59,6 +59,7 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} livenessProbe: httpGet: path: /api/healthcheck diff --git a/bin/k8s/templates/base/file-service/file-service-deployment.yaml b/bin/k8s/templates/base/file-service/file-service-deployment.yaml index b60eddf11b5..e70f2d32135 100644 --- a/bin/k8s/templates/base/file-service/file-service-deployment.yaml +++ b/bin/k8s/templates/base/file-service/file-service-deployment.yaml @@ -82,4 +82,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/observability/grafana.yaml b/bin/k8s/templates/base/observability/grafana.yaml new file mode 100644 index 00000000000..36b1e05fe3e --- /dev/null +++ b/bin/k8s/templates/base/observability/grafana.yaml @@ -0,0 +1,125 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.grafana.enabled }} +# Grafana: dashboard/UI, k8s counterpart of the docker-compose grafana +# service. Reads the Apache-2.0 backends (Loki, VictoriaMetrics, Jaeger) +# as pre-provisioned datasources via the ConfigMap below. Grafana is +# AGPLv3, so it is an optional operator-run component, not part of any +# Texera release artifact (see bin/observability/README.md). Only this +# ASF-licensed template lives in the tree. The Service is ClusterIP (no +# Ingress), matching the rest of the stack; expose it via your own +# Ingress / port-forward to reach the UI. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-grafana-datasources + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-grafana +data: + datasources.yaml: | + {{- include "texera.observability.grafanaDatasources" . | nindent 4 }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-grafana + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-grafana +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-grafana + template: + metadata: + labels: + app: {{ .Release.Name }}-grafana + annotations: + # Roll the pod when the provisioned datasources change. + checksum/config: {{ include "texera.observability.grafanaDatasources" . | sha256sum }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 472 + fsGroup: 472 + containers: + - name: grafana + image: {{ .Values.observability.grafana.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + env: + # Admin login. Override for any environment reachable beyond + # an internal port-forward. + - name: GF_SECURITY_ADMIN_USER + value: {{ .Values.observability.grafana.adminUser | quote }} + - name: GF_SECURITY_ADMIN_PASSWORD + value: {{ .Values.observability.grafana.adminPassword | quote }} + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "false" + - name: GF_USERS_ALLOW_SIGN_UP + value: "false" + ports: + - name: http + containerPort: {{ .Values.observability.grafana.port }} + volumeMounts: + - name: datasources + mountPath: /etc/grafana/provisioning/datasources + readOnly: true + - name: data + mountPath: /var/lib/grafana + livenessProbe: + httpGet: + path: /api/health + port: {{ .Values.observability.grafana.port }} + initialDelaySeconds: 15 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.grafana.resources | nindent 12 }} + volumes: + - name: datasources + configMap: + name: {{ .Release.Name }}-grafana-datasources + # Grafana's SQLite state. emptyDir: datasources are re-provisioned + # on every start, so no PVC is required for the reference stack. + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-grafana + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-grafana +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-grafana + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.grafana.port }} + targetPort: {{ .Values.observability.grafana.port }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/jaeger.yaml b/bin/k8s/templates/base/observability/jaeger.yaml new file mode 100644 index 00000000000..2821a977736 --- /dev/null +++ b/bin/k8s/templates/base/observability/jaeger.yaml @@ -0,0 +1,81 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.traces.enabled }} +# Jaeger v2: traces backend + UI, k8s counterpart of the docker-compose +# jaeger service. v2 accepts OTLP natively (the collector exports here on +# otlpPort). In-memory storage: a restart wipes traces, same as the +# single-node compose. Swap in a persistent storage backend for +# multi-node / production use. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-jaeger + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-jaeger +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-jaeger + template: + metadata: + labels: + app: {{ .Release.Name }}-jaeger + spec: + securityContext: + runAsNonRoot: true + containers: + - name: jaeger + image: {{ .Values.observability.traces.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: query + containerPort: {{ .Values.observability.traces.queryPort }} + - name: otlp-grpc + containerPort: {{ .Values.observability.traces.otlpPort }} + resources: + {{- toYaml .Values.observability.traces.resources | nindent 12 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-jaeger + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-jaeger +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-jaeger + ports: + - name: query + protocol: TCP + port: {{ .Values.observability.traces.queryPort }} + targetPort: {{ .Values.observability.traces.queryPort }} + - name: otlp-grpc + protocol: TCP + port: {{ .Values.observability.traces.otlpPort }} + targetPort: {{ .Values.observability.traces.otlpPort }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/loki.yaml b/bin/k8s/templates/base/observability/loki.yaml new file mode 100644 index 00000000000..d26ca944e28 --- /dev/null +++ b/bin/k8s/templates/base/observability/loki.yaml @@ -0,0 +1,136 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.logs.enabled }} +# Loki: log store, k8s counterpart of the docker-compose loki service. +# OTLP ingest (/otlp) + LogQL query API on one port. Loki is AGPLv3, so +# it is an optional operator-run component, not part of any Texera release +# artifact (see bin/observability/README.md). Only this ASF-licensed +# template lives in the tree. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-loki-config + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-loki +data: + config.yaml: | + {{- include "texera.observability.lokiConfig" . | nindent 4 }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-loki-data + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-loki +spec: + accessModes: ["ReadWriteOnce"] + {{- if .Values.observability.logs.storageClass }} + storageClassName: {{ .Values.observability.logs.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.observability.logs.storage }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-loki + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-loki +spec: + replicas: 1 + # Single writer bound to one RWO volume; never run two at once. + strategy: + type: Recreate + selector: + matchLabels: + app: {{ .Release.Name }}-loki + template: + metadata: + labels: + app: {{ .Release.Name }}-loki + annotations: + # Roll the pod when the rendered config changes. + checksum/config: {{ include "texera.observability.lokiConfig" . | sha256sum }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + containers: + - name: loki + image: {{ .Values.observability.logs.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: ["-config.file=/etc/loki/config.yaml"] + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: http + containerPort: {{ .Values.observability.logs.port }} + volumeMounts: + - name: config + mountPath: /etc/loki + readOnly: true + - name: data + mountPath: /loki + # Read-only root filesystem, so scratch space is an emptyDir. + - name: tmp + mountPath: /tmp + livenessProbe: + httpGet: + path: /ready + port: {{ .Values.observability.logs.port }} + initialDelaySeconds: 15 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.logs.resources | nindent 12 }} + volumes: + - name: config + configMap: + name: {{ .Release.Name }}-loki-config + - name: data + persistentVolumeClaim: + claimName: {{ .Release.Name }}-loki-data + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-loki + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-loki +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-loki + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.logs.port }} + targetPort: {{ .Values.observability.logs.port }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/otel-collector-configmap.yaml b/bin/k8s/templates/base/observability/otel-collector-configmap.yaml new file mode 100644 index 00000000000..4a2cd2deee1 --- /dev/null +++ b/bin/k8s/templates/base/observability/otel-collector-configmap.yaml @@ -0,0 +1,109 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +# OpenTelemetry Collector config, k8s counterpart of +# bin/observability/otel-collector/config.yaml. Same pipeline topology; +# exporters point at the in-cluster backend Services instead of the +# docker-compose service names. Only the pipelines for enabled signals +# are rendered, so disabling a signal drops its exporter cleanly. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-otel-collector-config + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +data: + config.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:{{ .Values.observability.collector.grpcPort }} + max_recv_msg_size_mib: 4 + http: + endpoint: 0.0.0.0:{{ .Values.observability.collector.httpPort }} + + processors: + batch: + send_batch_size: 1024 + timeout: 5s + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + + exporters: + {{- if .Values.observability.logs.enabled }} + # Loki accepts OTLP at /otlp (Loki 3.x maps it to /otlp/v1/logs). + otlphttp/loki: + endpoint: http://{{ .Release.Name }}-loki:{{ .Values.observability.logs.port }}/otlp + compression: gzip + timeout: 10s + tls: + insecure: true + {{- end }} + {{- if .Values.observability.metrics.enabled }} + prometheusremotewrite: + endpoint: http://{{ .Release.Name }}-victoriametrics:{{ .Values.observability.metrics.port }}/api/v1/write + tls: + insecure: true + remote_write_queue: + queue_size: 1000 + num_consumers: 4 + {{- end }} + {{- if .Values.observability.traces.enabled }} + otlp/jaeger: + endpoint: {{ .Release.Name }}-jaeger:{{ .Values.observability.traces.otlpPort }} + tls: + insecure: true + {{- end }} + + service: + pipelines: + {{- if .Values.observability.logs.enabled }} + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + {{- end }} + {{- if .Values.observability.metrics.enabled }} + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + {{- end }} + {{- if .Values.observability.traces.enabled }} + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/jaeger] + {{- end }} + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + logs: + level: info +{{- end }} diff --git a/bin/k8s/templates/base/observability/otel-collector.yaml b/bin/k8s/templates/base/observability/otel-collector.yaml new file mode 100644 index 00000000000..89c537317d8 --- /dev/null +++ b/bin/k8s/templates/base/observability/otel-collector.yaml @@ -0,0 +1,95 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +# OpenTelemetry Collector: the single OTLP ingress for all signals, +# k8s counterpart of the docker-compose otel-collector service. The +# receiver Ports are exposed only on the ClusterIP Service (no Ingress), +# mirroring the loopback-only posture of the compose deployment. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-otel-collector + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-otel-collector + template: + metadata: + labels: + app: {{ .Release.Name }}-otel-collector + annotations: + # Roll the pod when the rendered config changes. + checksum/config: {{ include (print $.Template.BasePath "/base/observability/otel-collector-configmap.yaml") . | sha256sum }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + containers: + - name: otel-collector + image: {{ .Values.observability.collector.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: ["--config=/etc/otelcol-contrib/config.yaml"] + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: otlp-grpc + containerPort: {{ .Values.observability.collector.grpcPort }} + - name: otlp-http + containerPort: {{ .Values.observability.collector.httpPort }} + volumeMounts: + - name: config + mountPath: /etc/otelcol-contrib + readOnly: true + resources: + {{- toYaml .Values.observability.collector.resources | nindent 12 }} + volumes: + - name: config + configMap: + name: {{ .Release.Name }}-otel-collector-config +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-otel-collector + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-otel-collector + ports: + - name: otlp-grpc + protocol: TCP + port: {{ .Values.observability.collector.grpcPort }} + targetPort: {{ .Values.observability.collector.grpcPort }} + - name: otlp-http + protocol: TCP + port: {{ .Values.observability.collector.httpPort }} + targetPort: {{ .Values.observability.collector.httpPort }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/victoriametrics.yaml b/bin/k8s/templates/base/observability/victoriametrics.yaml new file mode 100644 index 00000000000..5a6a1ea97b8 --- /dev/null +++ b/bin/k8s/templates/base/observability/victoriametrics.yaml @@ -0,0 +1,112 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.metrics.enabled }} +# VictoriaMetrics: metrics store, k8s counterpart of the docker-compose +# victoriametrics service. Accepts Prometheus remote-write from the +# collector; MetricsQL query API on the same port. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-victoriametrics-data + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + accessModes: ["ReadWriteOnce"] + {{- if .Values.observability.metrics.storageClass }} + storageClassName: {{ .Values.observability.metrics.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.observability.metrics.storage }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-victoriametrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + replicas: 1 + # Single writer bound to one RWO volume; never run two at once. + strategy: + type: Recreate + selector: + matchLabels: + app: {{ .Release.Name }}-victoriametrics + template: + metadata: + labels: + app: {{ .Release.Name }}-victoriametrics + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10003 + fsGroup: 10003 + containers: + - name: victoriametrics + image: {{ .Values.observability.metrics.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: + - "-storageDataPath=/data" + - "-retentionPeriod={{ .Values.observability.metrics.retentionPeriod }}" + - "-httpListenAddr=:{{ .Values.observability.metrics.port }}" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: http + containerPort: {{ .Values.observability.metrics.port }} + volumeMounts: + - name: data + mountPath: /data + livenessProbe: + httpGet: + path: /health + port: {{ .Values.observability.metrics.port }} + initialDelaySeconds: 10 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.metrics.resources | nindent 12 }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ .Release.Name }}-victoriametrics-data +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-victoriametrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-victoriametrics + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.metrics.port }} + targetPort: {{ .Values.observability.metrics.port }} +{{- end }} diff --git a/bin/k8s/templates/base/webserver/webserver-deployment.yaml b/bin/k8s/templates/base/webserver/webserver-deployment.yaml index 99a58870957..f71c2211903 100644 --- a/bin/k8s/templates/base/webserver/webserver-deployment.yaml +++ b/bin/k8s/templates/base/webserver/webserver-deployment.yaml @@ -77,4 +77,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml index a538ce4a86f..0bc09731390 100644 --- a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml +++ b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml @@ -69,4 +69,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml index 1f6ef3f99b3..163a0e8e840 100644 --- a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml +++ b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml @@ -151,4 +151,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index 4be79adf73a..4c6f43a9755 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -634,3 +634,84 @@ envoy-gateway: extensionApis: enableBackend: true enableEnvoyPatchPolicy: true + +# Observability stack (hand-rolled, mirrors bin/observability + the +# single-node docker-compose profiles). Everything is gated behind +# `enabled` (default off) so the standard install is unchanged. When +# enabled, the Scala services emit OTLP to the in-cluster collector, +# which fans logs/metrics/traces to the backends below. All Services are +# ClusterIP only (no Ingress), matching the loopback-only posture of the +# compose deployment. Profiling (Parca) is intentionally not included: it +# needs a privileged eBPF DaemonSet and is a separate change. +observability: + # Master switch for the whole stack. + enabled: false + + # OpenTelemetry Collector: the single OTLP ingress. Required whenever + # any signal below is enabled (it is what the services emit to). + collector: + enabled: true + image: otel/opentelemetry-collector-contrib:0.153.0 + grpcPort: 4317 + httpPort: 4318 + resources: + limits: + memory: 768Mi + cpu: "1" + + # Loki: log store (OTLP ingest at /otlp + LogQL). AGPLv3, so it is an + # optional operator-run component, not part of any Texera release + # artifact (see bin/observability/README.md). + logs: + enabled: true + image: grafana/loki:3.2.0 + port: 3100 + retentionPeriod: 30d + storage: 10Gi + storageClass: local-path + resources: + limits: + memory: 1Gi + cpu: "1" + + # VictoriaMetrics: metrics store (Prometheus remote-write + MetricsQL). + metrics: + enabled: true + image: victoriametrics/victoria-metrics:v1.144.0 + port: 8428 + retentionPeriod: 90d + storage: 10Gi + storageClass: local-path + resources: + limits: + memory: 1Gi + cpu: "1" + + # Jaeger v2: traces backend + UI. In-memory storage (restart wipes + # traces); swap in persistent storage for production. + traces: + enabled: true + image: jaegertracing/jaeger:2.18.0 + queryPort: 16686 + otlpPort: 4317 + resources: + limits: + memory: 1Gi + cpu: "1" + + # Grafana: dashboard/UI reading Loki + VictoriaMetrics + Jaeger as + # pre-provisioned datasources. AGPLv3, so it is an optional operator-run + # component, not part of any Texera release artifact (see + # bin/observability/README.md). ClusterIP only; expose via your own + # Ingress or `kubectl port-forward`. Override the admin credentials for + # any environment reachable beyond an internal port-forward. + grafana: + enabled: true + image: grafana/grafana:11.2.0 + port: 3000 + adminUser: admin + adminPassword: texera + resources: + limits: + memory: 512Mi + cpu: "0.5" diff --git a/bin/local-dev/docker-compose.override.yml b/bin/local-dev/docker-compose.override.yml index 3b053a85700..bde7cbd37e5 100644 --- a/bin/local-dev/docker-compose.override.yml +++ b/bin/local-dev/docker-compose.override.yml @@ -22,6 +22,14 @@ # Node services *natively* on the host, so we expose each backend port here. services: + # A host-native backend (sbt / IntelliJ) emits OTLP to the collector that + # runs inside compose, so publish the collector's receiver ports on the host + # loopback. Only takes effect when the observability-collector profile is on. + otel-collector: + ports: + - "127.0.0.1:4317:4317" + - "127.0.0.1:4318:4318" + postgres: ports: - "5432:5432" diff --git a/bin/observability/README.md b/bin/observability/README.md new file mode 100644 index 00000000000..04ff0a832ea --- /dev/null +++ b/bin/observability/README.md @@ -0,0 +1,131 @@ + + +# Texera Observability Stack + +Texera's backend services are instrumented with [OpenTelemetry](https://opentelemetry.io/) +and export OTLP (logs, metrics, and traces). This directory is an **optional, +bundled reference stack** that an operator can start alongside a single-node +deployment to collect and visualize that telemetry. It is opt-in: nothing here +runs unless you enable the observability compose profiles (see below), and +Texera emits no telemetry unless `OTEL_SDK_DISABLED=false` is set. + +## Topology + +A single OpenTelemetry Collector is the only OTLP ingress. It fans each signal +out to a dedicated backend, and Grafana sits on top as the query UI: + +``` + Texera services + │ OTLP/gRPC (4317) or OTLP/HTTP (4318) + ▼ + otel-collector + ┌──────────────┬──────┴────────┬───────────────┐ + OTLP │ remote-write OTLP │ │ (eBPF agent push) + ▼ ▼ ▼ ▼ + Loki VictoriaMetrics Jaeger Parca + (logs) (metrics) (traces) (profiles) + │ │ │ + └──────────────┼────────────────┘ + ▼ + Grafana + (reads Loki + VictoriaMetrics + Jaeger) +``` + +- **Loki** stores logs; the collector exports OTLP to `http://loki:3100/otlp`. +- **VictoriaMetrics** stores metrics; the collector exports Prometheus + remote-write to `http://victoriametrics:8428/api/v1/write`. +- **Jaeger** stores traces; the collector exports OTLP to `jaeger:4317`. +- **Parca** stores continuous profiles, pushed by its eBPF agent (not read by + Grafana; use the Parca UI on `:7070`). +- **Grafana** reads Loki, VictoriaMetrics (via the Prometheus query API), and + Jaeger as pre-provisioned datasources. + +## Starting it + +The stack is driven by docker-compose profiles. Enable the signals you want in +`bin/single-node/.env` via `COMPOSE_PROFILES`, then bring the deployment up: + +``` +# bin/single-node/.env +COMPOSE_PROFILES=observability-collector,observability-logs,observability-metrics,observability-traces,observability-grafana +``` + +| Profile | Service(s) | +|---------------------------|-----------------------------------------| +| `observability-collector` | otel-collector (OTLP ingress; required) | +| `observability-logs` | loki | +| `observability-metrics` | victoriametrics | +| `observability-traces` | jaeger | +| `observability-profiles` | parca + privileged eBPF parca-agent | +| `observability-grafana` | grafana | + +Also set `OTEL_SDK_DISABLED=false` so the services actually emit telemetry, then: + +``` +cd bin/single-node && docker compose up -d +# or the dedicated entry point: +cd bin/observability && docker compose up -d +``` + +Reach **Grafana at http://localhost:3000**. The default login is +`admin` / `texera` (set via `GRAFANA_ADMIN_USER` / `GRAFANA_ADMIN_PASSWORD` in +`.env`); change it for anything reachable beyond localhost. + +### Pre-provisioned datasources + +Grafana comes with three datasources already wired up +(`grafana/provisioning/datasources/datasources.yaml`), so you can start querying +immediately: + +- **Metrics** (default) - VictoriaMetrics at `http://victoriametrics:8428`, + queried through the Prometheus API. +- **Loki** - logs at `http://loki:3100`. +- **Jaeger** - traces at `http://jaeger:16686`. + +The backend query ports are also published on `127.0.0.1` for ad-hoc `curl` +debugging (Loki `3100`, VictoriaMetrics `8428`, Jaeger `16686`, Parca `7070`). + +## Using your own stack + +Nothing here is mandatory. The Collector speaks standard OTLP and +Prometheus remote-write, so an operator can point their own tooling at the same +endpoints instead of running the bundled backends: + +- Point an **external Grafana** at the bundled Loki / VictoriaMetrics / Jaeger, + or swap the backends entirely. +- Send logs to **OpenSearch** (or any OTLP-capable log store) by changing the + collector's logs exporter in `otel-collector/config.yaml`. +- Replace the metrics or traces exporters with any Prometheus-remote-write or + OTLP-compatible destination. +- Or drop the collector and have an external collector scrape/receive directly; + the services only need an OTLP endpoint via `OTEL_EXPORTER_OTLP_ENDPOINT`. + +## Licensing + +> **NOTE:** **Grafana and Loki are licensed under the AGPLv3.** They are +> included here only as an **optional, operator-run** convenience and are **NOT +> bundled into, or distributed as part of, any Apache Texera release artifact**. +> An operator who runs them does so on their own; their images are pulled at +> deploy time, not shipped by the project. +> +> The rest of the stack - the **OpenTelemetry Collector, VictoriaMetrics, +> Jaeger, and Parca** - is **Apache-2.0** licensed, consistent with Texera +> itself. Only Apache-2.0 configuration files (collector, Loki, Grafana +> datasources, Parca) live in this repository. diff --git a/bin/observability/docker-compose.yml b/bin/observability/docker-compose.yml new file mode 100644 index 00000000000..adce75a4a40 --- /dev/null +++ b/bin/observability/docker-compose.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Dedicated entry point for the observability stack, so it can be started like +# the other dev targets (e.g. file-service's own compose) instead of reaching +# into the single-node deployment file. +# +# It does NOT redefine the services: it `include`s the single-node compose, the +# single source of truth for every container definition. The observability +# services there are gated behind compose profiles, so they are selected via +# COMPOSE_PROFILES + the service list (see start_app.sh `up_obs`). The other +# single-node services are not profile-gated, so they are simply not named and +# stay down. +# +# Relative volume paths inside the included file (e.g. the collector/parca +# configs under ../observability/) are resolved relative to the included file's +# own directory, so they keep working unchanged. +# +# The project name matches single-node so this manages the SAME containers +# whether obs is brought up here or as part of the full single-node deployment +# (the containers use fixed container_names, so a different project would clash). +name: texera-single-node + +include: + - ../single-node/docker-compose.yml diff --git a/bin/observability/grafana/provisioning/datasources/datasources.yaml b/bin/observability/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 00000000000..b4cd9c09461 --- /dev/null +++ b/bin/observability/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Grafana datasource provisioning for the Texera bundled observability +# stack. Grafana (AGPLv3) reads the three Apache-2.0 query backends by +# their docker-compose bridge-network service names. This file is only +# ASF-licensed configuration; Grafana itself is an optional operator-run +# component and is NOT shipped in any Texera release artifact (see +# bin/observability/README.md). +apiVersion: 1 + +datasources: + # Metrics: VictoriaMetrics speaks the Prometheus query API, so it is + # added as a prometheus-type datasource. Marked default. + - name: Metrics + type: prometheus + uid: metrics + access: proxy + url: http://victoriametrics:8428 + isDefault: true + editable: false + + # Logs: Loki, LogQL over OTLP-ingested logs. + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki:3100 + editable: false + + # Traces: Jaeger query API / UI backend. + - name: Jaeger + type: jaeger + uid: jaeger + access: proxy + url: http://jaeger:16686 + editable: false diff --git a/bin/observability/loki/config.yaml b/bin/observability/loki/config.yaml new file mode 100644 index 00000000000..5d695805e77 --- /dev/null +++ b/bin/observability/loki/config.yaml @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Loki configuration for the Texera bundled observability stack. +# +# Bundled image: grafana/loki:3.2.0 (AGPLv3). Loki is an optional, +# operator-run component; it is NOT shipped in any Texera release +# artifact (see bin/observability/README.md). Only this ASF-licensed +# config file lives in the tree. +# +# This is a minimal single-binary, filesystem-backed deployment for +# single-node development. Replace the filesystem storage with an +# object-storage backend (S3/GCS/Azure) before any multi-node or +# production use. + +# No multi-tenancy: a single-node dev stack has one implicit tenant, +# so requests need no X-Scope-OrgID header. +auth_enabled: false + +server: + # LogQL query API + OTLP ingest (/otlp/v1/logs) are served here. + http_listen_port: 3100 + # Internal gRPC; not published to the host. + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + # Every component's working/data path defaults under this prefix, + # which is the loki_data volume mounted at /loki in docker-compose. + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + # Single binary, single replica: no distributed ring. + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + # tsdb index + filesystem object store is the current (v13) layout. + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + # Required for OTLP ingest: OTLP log attributes are written as + # structured metadata. Enabled by default in Loki 3.x; kept explicit. + allow_structured_metadata: true + # Drop out-of-order rejections for a low-volume dev stack. + reject_old_samples: false + # Age after which log chunks are eligible for deletion by the compactor. + retention_period: 30d + +compactor: + working_directory: /loki/compactor + # Without this the retention_period above is parsed but never enforced, + # so logs accumulate until the loki_data volume fills. + retention_enabled: true + delete_request_store: filesystem + +# OTLP ingest needs no extra config: Loki 3.x serves it at +# http://loki:3100/otlp out of the box. The collector exports there +# (see bin/observability/otel-collector/config.yaml). diff --git a/bin/observability/otel-collector/config.yaml b/bin/observability/otel-collector/config.yaml new file mode 100644 index 00000000000..8cbee85742d --- /dev/null +++ b/bin/observability/otel-collector/config.yaml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# OpenTelemetry Collector configuration for the Texera bundled +# observability stack. +# +# Bundled image: otel/opentelemetry-collector-contrib:0.153.0 +# (Apache-2.0). The contrib distribution is needed for the +# prometheusremotewrite exporter that fans metrics to VictoriaMetrics. +# +# Pipeline topology: +# +# Texera services +# │ OTLP/gRPC (4317) or OTLP/HTTP (4318) +# ▼ +# otel-collector +# ├── logs ── OTLP ──────────────────► loki:3100/otlp +# ├── metrics ── prometheusremotewrite ─► victoriametrics:8428 +# └── traces ── OTLP ──────────────────► jaeger:4317 +# +# Network: every receiver and exporter is reachable only on the +# compose bridge network. No host-port mapping for the receivers in +# docker-compose.yml, that keeps OTLP ingest cluster-local, which +# is the security posture the PR plan calls for. +# +# Hostnames below are docker-compose service names, they resolve via +# docker's embedded DNS inside the network. + +receivers: + otlp: + protocols: + grpc: + # 0.0.0.0 binds inside the container only; the compose service + # does NOT publish this port to the host. Reachable only via + # the texera-single-node bridge network. + endpoint: 0.0.0.0:4317 + # 4 MiB cap on a single OTLP message, anyone trying to spam + # the collector with a huge payload gets a clean reject. + max_recv_msg_size_mib: 4 + http: + endpoint: 0.0.0.0:4318 + +processors: + # batch is the upstream-recommended first processor. Caps memory + # per export by batching by either size or timeout, whichever hits + # first. Defaults are sensible for a single-node deployment. + batch: + send_batch_size: 1024 + timeout: 5s + + # memory_limiter prevents the collector OOMing under a sudden burst + # of telemetry. We set a soft limit, not a hard percentage, so the + # numbers are reviewable. + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + +exporters: + # Loki accepts OTLP for logs natively at /otlp (Loki 3.x maps it to + # /otlp/v1/logs). OTLP resource attributes become stream labels / + # structured metadata. Verify the endpoint on every version bump. + otlphttp/loki: + endpoint: http://loki:3100/otlp + compression: gzip + timeout: 10s + # tls is disabled for the compose bridge network. Any deploy + # that opens these ports beyond the bridge must enable TLS here. + tls: + insecure: true + + # VictoriaMetrics consumes Prometheus remote-write natively. The + # prometheusremotewrite exporter is the upstream-recommended way + # to bridge OTel metrics into Prom-ecosystem stores. + prometheusremotewrite: + endpoint: http://victoriametrics:8428/api/v1/write + tls: + insecure: true + # Keep the remote-write batch bounded so a metric burst can't + # become an oversized HTTP request. + remote_write_queue: + queue_size: 1000 + num_consumers: 4 + + # Jaeger v2 accepts OTLP directly, no jaeger exporter type needed. + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + +service: + pipelines: + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/jaeger] + + # Collector telemetry binds to loopback inside the container only + # so we can scrape /metrics from a sidecar if needed without + # exposing the collector's own metrics to the host. + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + logs: + level: info diff --git a/bin/observability/parca/README.md b/bin/observability/parca/README.md new file mode 100644 index 00000000000..d799ba5e7aa --- /dev/null +++ b/bin/observability/parca/README.md @@ -0,0 +1,72 @@ + + +# Parca profiles + +Configuration for the **profiles** signal in the Texera observability +stack: the Parca server and its eBPF agent. The compose services that +run them are in `bin/single-node/docker-compose.yml`; this directory +holds only their configuration. + +Both components are Apache-2.0 (see +[`docs/observability/LICENSING.md`](../../../docs/observability/LICENSING.md)): + +| File | Component | Image | +|---|---|---| +| `parca.yaml` | Parca server v0.28.0 | `ghcr.io/parca-dev/parca:v0.28.0` | +| `parca-agent.env` | Parca eBPF agent v0.47.1 | `ghcr.io/parca-dev/parca-agent:v0.47.1` | + +## Host requirements + +The agent uses eBPF to sample stack traces, which constrains the host: + +- **Linux only.** eBPF is a Linux kernel feature; the agent cannot run + on macOS or Windows. The rest of the stack (logs, metrics, traces) + is cross-platform. +- **Privileged container.** The agent needs `CAP_SYS_ADMIN`-class + permissions to load eBPF programs and open the perf-event facility, + so the compose service runs `privileged: true` and bind-mounts + `/sys/kernel/debug`, `/proc`, and `/sys` read-only. +- **Outbound only.** The agent writes to the bundled Parca server on + `parca:7070` over plain gRPC on the docker bridge network and opens + no other ports. Any deploy that exposes those ports to a wider + network must add TLS at the compose/k8s layer. + +## Opt-in + +Profiling is off by default. It is enabled per host by selecting the +`profiles` profile before `docker compose up`; see `bin/single-node/.env` +for the flags. Enable it only after reviewing the privilege and scope +notes below. + +## What gets profiled + +The agent profiles **every process on the host**, not only Texera's. +It is not scoped to Texera's containers; doing that would require +configuring the agent against specific cgroups, which is not done +here. The static labels in `parca-agent.env` (`deployment=texera`, +`cluster=local`) only let a query filter which profiles are *shown*, +not what is *collected*. Treat enabling the agent as profiling the +whole machine. + +Profiles are not labeled with `workflow.id` / `execution.id` (unbounded +identifiers that would blow up Parca's cardinality). The agent samples +stack traces and does not read application-level OpenTelemetry trace +ids, so profiles cannot be joined to individual traces by `trace_id`; +they correlate only by the coarse labels above and by time. diff --git a/bin/observability/parca/parca-agent.env b/bin/observability/parca/parca-agent.env new file mode 100644 index 00000000000..bfd61dde79f --- /dev/null +++ b/bin/observability/parca/parca-agent.env @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Parca eBPF agent environment. +# +# Bundled image: ghcr.io/parca-dev/parca-agent:v0.47.1 (Apache-2.0). +# +# Consumed by the docker-compose parca-agent service via env_file:. +# The agent reads its config from CLI args; staging them here keeps +# them version-controlled and reusable across compose and Helm. +# +# Deploy requirements (Linux-only, privileged): +# * The agent uses eBPF and must run on a Linux host kernel. +# * Requires CAP_SYS_ADMIN-class permissions to load eBPF programs. +# The compose service runs the agent container with +# `privileged: true` and mounts /sys/kernel/debug, /proc, /sys +# read-only. macOS / Windows developers must leave the +# observability-profiles profile out of COMPOSE_PROFILES (see README.md). +# +# Labels: +# * node: host identifier — set by the compose layer to the docker +# host's hostname. Overridable. +# * metadata-external-labels: static labels attached to every +# profile (deployment=texera;cluster=local). Coarse fields only; +# no workflow/execution id (cardinality DoS in Parca storage). + +PARCA_AGENT_REMOTE_STORE_ADDRESS=parca:7070 +PARCA_AGENT_REMOTE_STORE_INSECURE=true + +# Per-host identification. Compose can override at deploy time: +# `--node=${HOSTNAME:-texera-dev}` +PARCA_AGENT_NODE=texera-dev + +# Static labels attached to every profile this agent emits. +# Must NOT include workflow.id / execution.id — cardinality DoS in +# Parca's storage. Keep to coarse fields only. +PARCA_AGENT_METADATA_EXTERNAL_LABELS=deployment=texera;cluster=local + +# CPU sample rate. 19 Hz is upstream default and a sensible balance +# between overhead and resolution; we keep it explicit so an +# accidental upstream change doesn't silently shift it. +PARCA_AGENT_PROFILING_CPU_SAMPLING_FREQUENCY=19 + +# Log level — info by default; switch to warn in production to keep +# the agent quiet. +PARCA_AGENT_LOG_LEVEL=info diff --git a/bin/observability/parca/parca.yaml b/bin/observability/parca/parca.yaml new file mode 100644 index 00000000000..0af9e88a393 --- /dev/null +++ b/bin/observability/parca/parca.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Parca server configuration. +# +# This is the minimal config the Texera bundled observability stack +# uses. We deliberately keep scrape_configs empty — profiles arrive +# from the Parca eBPF agent (push model) rather than pull-mode scraping. +# Bundled image: ghcr.io/parca-dev/parca:v0.28.0 (Apache-2.0). +# +# Storage is intentionally filesystem-backed for the single-node +# docker-compose deployment. Replace with an object-storage backend +# (S3/GCS/Azure) before any multi-node or production use. + +object_storage: + bucket: + # FILESYSTEM is the simplest backend for single-node development. + # Data lives inside the container at this path; mount a volume + # in docker-compose to persist across container restarts. + type: "FILESYSTEM" + config: + directory: "/var/lib/parca" + +# scrape_configs is intentionally empty: the Texera deployment pushes +# profiles from the parca-agent via OTLP/gRPC. Adding scrape targets +# here would generate significantly more data than agent-based +# profiling per the upstream recommendation. +scrape_configs: [] diff --git a/bin/observability/smoke_test.py b/bin/observability/smoke_test.py new file mode 100644 index 00000000000..b109496bd7f --- /dev/null +++ b/bin/observability/smoke_test.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# End-to-end routing check for the bundled observability stack. +# +# Emits one log, one metric, and one trace span (each tagged with a test +# service name and a unique run id) at the OpenTelemetry Collector over +# OTLP/HTTP, exactly the way OtelInit's exporters do, then asserts that: +# +# logs land in Loki (not elsewhere) +# metrics land in VictoriaMetrics (not elsewhere) +# traces land in Jaeger (not elsewhere) +# +# and that each signal is attributed to the emitting service. This proves +# the Collector routes each signal type to the right backend per service. +# +# Run against a live stack (OTLP ports published on loopback by the dev +# override). Exit code 0 = all routes correct, 1 = a routing failure. +# +# Usage: python3 bin/observability/smoke_test.py [--service NAME] + +import argparse +import json +import sys +import time +import urllib.parse +import urllib.request +import uuid + +COLLECTOR = "http://127.0.0.1:4318" # OTLP/HTTP receiver +LOKI = "http://127.0.0.1:3100" # logs backend +VICTORIAMETRICS = "http://127.0.0.1:8428" # metrics backend +JAEGER = "http://127.0.0.1:16686" # traces backend + +TIMEOUT_S = 60 # how long to wait for a signal to become queryable +POLL_S = 3 + + +def _post(url, payload): + req = urllib.request.Request( + url, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=10) as r: + return r.status + + +def _get(url): + with urllib.request.urlopen(url, timeout=10) as r: + return json.load(r) + + +def _attr(key, value): + return {"key": key, "value": {"stringValue": value}} + + +def emit(service, run_id): + now_ns = str(time.time_ns()) + resource = {"attributes": [_attr("service.name", service)]} + + # LOG -> logs pipeline -> Loki + _post(f"{COLLECTOR}/v1/logs", {"resourceLogs": [{ + "resource": resource, + "scopeLogs": [{"logRecords": [{ + "timeUnixNano": now_ns, + "severityText": "INFO", + "severityNumber": 9, + "body": {"stringValue": f"smoke-test log {run_id}"}, + }]}], + }]}) + + # METRIC -> metrics pipeline -> VictoriaMetrics + metric_name = f"obs_smoke_test_{run_id}" + _post(f"{COLLECTOR}/v1/metrics", {"resourceMetrics": [{ + "resource": resource, + "scopeMetrics": [{"metrics": [{ + "name": metric_name, + "gauge": {"dataPoints": [{ + "asDouble": 1, + "timeUnixNano": now_ns, + "attributes": [_attr("run", run_id)], + }]}, + }]}], + }]}) + + # TRACE -> traces pipeline -> Jaeger + trace_id = (uuid.uuid4().hex + uuid.uuid4().hex)[:32] + span_id = uuid.uuid4().hex[:16] + _post(f"{COLLECTOR}/v1/traces", {"resourceSpans": [{ + "resource": resource, + "scopeSpans": [{"spans": [{ + "traceId": trace_id, + "spanId": span_id, + "name": f"smoke.test.{run_id}", + "kind": 1, + "startTimeUnixNano": now_ns, + "endTimeUnixNano": str(int(now_ns) + 1_000_000), + }]}], + }]}) + return metric_name + + +def _poll(check): + deadline = time.time() + TIMEOUT_S + while time.time() < deadline: + try: + if check(): + return True + except Exception: + pass + time.sleep(POLL_S) + return False + + +def loki_has(service, run_id): + q = urllib.parse.quote(f'{{service_name="{service}"}}') + end = time.time_ns() + start = end - 10 * 60 * 1_000_000_000 + url = f"{LOKI}/loki/api/v1/query_range?query={q}&start={start}&end={end}&limit=500" + res = _get(url)["data"]["result"] + return any(run_id in v[1] for s in res for v in s["values"]) + + +def vm_has(metric_name): + url = f"{VICTORIAMETRICS}/api/v1/query?query={urllib.parse.quote(metric_name)}" + return len(_get(url)["data"]["result"]) > 0 + + +def jaeger_has(service, run_id): + svcs = _get(f"{JAEGER}/api/services").get("data") or [] + if service not in svcs: + return False + url = f"{JAEGER}/api/traces?service={urllib.parse.quote(service)}&limit=50&lookback=1h" + data = _get(url).get("data") or [] + return any(run_id in s.get("operationName", "") for t in data for s in t["spans"]) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "--services", + default="obs-smoke-a,obs-smoke-b", + help="comma-separated service names to emit as (default tests two)", + ) + args = ap.parse_args() + services = [s.strip() for s in args.services.split(",") if s.strip()] + + # Emit a log + metric + trace for every service, each with its own run id. + emitted = {} + for svc in services: + run_id = uuid.uuid4().hex[:12] + metric_name = emit(svc, run_id) + emitted[svc] = (run_id, metric_name) + print(f"emitted service={svc} run_id={run_id}") + + results = [] + for svc, (run_id, metric_name) in emitted.items(): + checks = [ + (f"{svc}: logs -> Loki", lambda s=svc, r=run_id: loki_has(s, r)), + (f"{svc}: metrics -> VictoriaMetrics", lambda m=metric_name: vm_has(m)), + (f"{svc}: traces -> Jaeger", lambda s=svc, r=run_id: jaeger_has(s, r)), + ] + for label, check in checks: + ok = _poll(check) + results.append(ok) + print(f" {'PASS' if ok else 'FAIL'} {label}") + + # Isolation: one service's log marker must NOT appear under another service. + if len(services) >= 2: + a, b = services[0], services[1] + a_run = emitted[a][0] + leaked = False + try: + leaked = loki_has(b, a_run) # a's marker should not be under b + except Exception: + leaked = False + ok = not leaked + results.append(ok) + print(f" {'PASS' if ok else 'FAIL'} isolation: {a} logs not attributed to {b}") + + if all(results): + print("RESULT: PASS (each signal routed to the correct backend, per service)") + return 0 + print("RESULT: FAIL (a signal was mis-routed or mis-attributed)") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/single-node/.env b/bin/single-node/.env index 40098c54c01..4fcfe17d32e 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -101,17 +101,60 @@ TEXERA_DASHBOARD_SERVICE_ENDPOINT=http://dashboard-service:8080 WORKFLOW_COMPILING_SERVICE_ENDPOINT=http://workflow-compiling-service:9090 WORKFLOW_EXECUTION_SERVICE_ENDPOINT=http://workflow-runtime-coordinator-service:8085 -# OpenTelemetry (observability). Disabled by default; the SDK stays inert until -# OTEL_SDK_DISABLED=false. Point the endpoint at a reachable OTLP collector -# (http/https only) before enabling. -OTEL_SDK_DISABLED=true -OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4317 -# Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service. -OTEL_RESOURCE_ATTRIBUTES= -# Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default). -TEXERA_OTEL_ALLOWED_HOSTS= -# Metric export interval in ms. -OTEL_METRIC_EXPORT_INTERVAL=30000 +# ============================================================================ +# Observability +# ============================================================================ +# Each signal is a separate docker-compose profile and is OFF by default: +# COMPOSE_PROFILES is empty, so `docker compose up` runs only core Texera. +# Observability is opt-in, for operators who want it; enable it by listing +# the profiles you want (docker compose reads COMPOSE_PROFILES natively): +# COMPOSE_PROFILES=observability-collector,observability-logs,observability-metrics,observability-traces,observability-grafana +# +# Profiles: +# observability-collector otel-collector (OTLP ingress; the others need it) +# observability-logs loki (AGPLv3; see bin/observability/README.md) +# observability-metrics victoriametrics +# observability-traces jaeger +# observability-profiles parca + privileged eBPF parca-agent +# observability-grafana grafana UI (AGPLv3; reads loki/victoriametrics/jaeger) +# +# Grafana and Loki are AGPLv3 optional operator-run components; they are not +# bundled into any Texera release artifact. The collector, VictoriaMetrics, +# Jaeger, and Parca are Apache-2.0. +# +# observability-profiles is Linux-only, privileged, and profiles every +# process on the host (see bin/observability/parca/README.md). Add it only +# deliberately, after reading that file's privilege and scope notes. +COMPOSE_PROFILES= + +# Query endpoints the dashboard gateway calls to read each signal. The +# defaults in observability-gateway.conf are host-local (127.0.0.1), which +# is what a natively-run backend (sbt / IntelliJ) needs. Inside this compose +# the gateway runs in a container, so it must reach the backends by their +# bridge-network service names, these overrides do that. +TEXERA_OBS_LOGS_URL=http://loki:3100 +TEXERA_OBS_METRICS_URL=http://victoriametrics:8428 +TEXERA_OBS_TRACES_URL=http://jaeger:16686 +TEXERA_OBS_PROFILES_URL=http://parca:7070 + +# Grafana UI (observability-grafana profile). The port is bound to +# loopback (127.0.0.1) like the backends, so the UI is reached at +# http://localhost:3000. The admin login defaults below are weak dev +# defaults; override them for any deployment reachable beyond localhost. +GRAFANA_PORT=3000 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=texera + +# Emission side: the Scala services (via OtelInit) push OTLP to the bundled +# collector over the bridge network. Emission is controlled explicitly by +# OTEL_SDK_DISABLED, which defaults to disabled, so no service emits unless +# you opt in by setting OTEL_SDK_DISABLED=false (do that together with the +# observability-collector profile). The endpoint and allowed-hosts below only +# take effect once emission is enabled: OtelInit validates the endpoint host +# against an allowlist (localhost only by default), so the collector's compose +# service name must be allowlisted explicitly. +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +TEXERA_OTEL_ALLOWED_HOSTS=otel-collector # Notebook migration tool # Toggles the tool in the GUI; the migration itself needs an LLM key above. diff --git a/bin/single-node/docker-compose.override.yml b/bin/single-node/docker-compose.override.yml new file mode 100644 index 00000000000..4ed7f5708fc --- /dev/null +++ b/bin/single-node/docker-compose.override.yml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Local-dev override, auto-loaded by `docker compose` when run from this +# directory. It publishes the OTel collector's OTLP receiver ports on the +# host loopback so a backend running outside docker (sbt / IntelliJ) can +# emit telemetry to the collector. The released bundle does not ship this +# file, so the base compose stays network-clean in a deploy. +name: texera-single-node + +services: + otel-collector: + ports: + - "127.0.0.1:4317:4317" + - "127.0.0.1:4318:4318" diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index d09cbc09a3d..d0752c2ca40 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -590,6 +590,247 @@ services: command: > sh -c 'apk add --no-cache curl jq bash > /dev/null 2>&1 && bash /examples/load-examples.sh' + # ======================================================================== + # Observability stack: collector + logs, metrics, traces, profiles. + # + # Each signal is a separate compose profile; enable them via + # COMPOSE_PROFILES in bin/single-node/.env. Every port binds to + # loopback (127.0.0.1) or stays on the bridge network; no 0.0.0.0 + # host bindings. + # ======================================================================== + + # OpenTelemetry Collector, the single OTLP ingress for all three + # signals. Reads bin/observability/otel-collector/config.yaml. + # Apache-2.0; pinned to the contrib distribution. + otel-collector: + image: otel/opentelemetry-collector-contrib:0.153.0 + container_name: texera-otel-collector + profiles: [observability-collector] + restart: always + user: "10001:10001" + read_only: true + security_opt: + - no-new-privileges:true + volumes: + - ../observability/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml:ro,z + command: ["--config=/etc/otelcol-contrib/config.yaml"] + # In-compose services reach the OTLP receivers over the bridge + # network as otel-collector:4317/:4318, so the base file publishes + # no host ports. For a backend running outside docker (sbt / + # IntelliJ), bin/local-dev/docker-compose.override.yml, which the + # local-dev launcher loads, publishes them on the host loopback; the + # released bundle omits that override. + deploy: + resources: + limits: + memory: 768M + cpus: "1.0" + + # Loki (AGPLv3) log store. OTLP ingest at /otlp, LogQL query API on + # 3100. AGPLv3, so it is an optional operator-run component and is NOT + # shipped in any Texera release artifact (see bin/observability/README.md). + # Reads bin/observability/loki/config.yaml. + loki: + image: grafana/loki:3.2.0 + container_name: texera-loki + profiles: [observability-logs] + restart: always + user: "10001:10001" + read_only: true + security_opt: + - no-new-privileges:true + volumes: + - ../observability/loki/config.yaml:/etc/loki/config.yaml:ro,z + - loki_data:/loki + # Read-only root filesystem, so Loki's scratch space is an in-memory + # tmpfs; all durable state lives on the loki_data volume under /loki. + tmpfs: + - /tmp + command: ["-config.file=/etc/loki/config.yaml"] + # Loopback-only host binding so an operator can curl the query + # API from the host for ad-hoc debugging without exposing it to + # the network. + ports: + - "127.0.0.1:3100:3100" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Grafana (AGPLv3) dashboard/UI. Reads the Apache-2.0 backends below + # (Loki, VictoriaMetrics, Jaeger) as pre-provisioned datasources. The UI + # is meant to be opened by a human, but like the backends its port is + # bound to loopback only (http://localhost:3000). AGPLv3, so it is an + # optional operator-run component and is NOT shipped in any Texera + # release artifact (see bin/observability/README.md). + grafana: + image: grafana/grafana:11.2.0 + container_name: texera-grafana + profiles: [observability-grafana] + restart: always + security_opt: + - no-new-privileges:true + depends_on: + - loki + - victoriametrics + - jaeger + environment: + # Admin login. Defaults are weak dev defaults; override in .env for + # any deployment reachable beyond localhost. + - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-texera} + # No anonymous access: the UI requires the admin login above. + - GF_AUTH_ANONYMOUS_ENABLED=false + # Users cannot self-register. + - GF_USERS_ALLOW_SIGN_UP=false + volumes: + # Datasource provisioning (Loki + VictoriaMetrics + Jaeger), read-only. + - ../observability/grafana/provisioning:/etc/grafana/provisioning:ro,z + - grafana_data:/var/lib/grafana + # Loopback-only host binding, like the backends: a human opens + # http://localhost:3000, and the UI is never reachable from the + # network by default. + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" + + # VictoriaMetrics, Apache-2.0 metrics store. Accepts Prometheus + # remote-write from the collector at /api/v1/write, MetricsQL on + # /api/v1/query. + # Fix /data ownership before VictoriaMetrics starts. VM runs non-root + # (uid 10003) with a read-only rootfs, but a freshly created named volume + # is owned by root, so without this VM cannot create its lock file and + # crash-loops. Idempotent on an existing volume. + victoriametrics-init: + image: alpine:3.19 + container_name: texera-victoriametrics-init + profiles: [observability-metrics] + restart: "no" + user: "0:0" + entrypoint: ["/bin/sh", "-c"] + command: + - chown -R 10003:10003 /data + volumes: + - victoriametrics_data:/data + + victoriametrics: + image: victoriametrics/victoria-metrics:v1.144.0 + container_name: texera-victoriametrics + profiles: [observability-metrics] + restart: always + user: "10003:10003" + depends_on: + victoriametrics-init: + condition: service_completed_successfully + read_only: true + security_opt: + - no-new-privileges:true + command: + - "-storageDataPath=/data" + - "-retentionPeriod=90d" + - "-httpListenAddr=:8428" + volumes: + - victoriametrics_data:/data + ports: + - "127.0.0.1:8428:8428" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Jaeger v2, Apache-2.0 traces backend + UI. v2 accepts OTLP + # natively. The query API lives at :16686, OTLP ingest at :4317 + # (kept inside the bridge network only, only the collector talks + # to it). + jaeger: + image: jaegertracing/jaeger:2.18.0 + container_name: texera-jaeger + profiles: [observability-traces] + restart: always + security_opt: + - no-new-privileges:true + # In-memory storage for the single-node deployment, restarts + # wipe traces. Any deploy needing trace persistence must swap + # in Cassandra or OpenSearch-backed storage here. + ports: + - "127.0.0.1:16686:16686" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Parca server (Apache-2.0) profiles backend. Reads + # bin/observability/parca/parca.yaml. FILESYSTEM storage. + parca: + image: ghcr.io/parca-dev/parca:v0.28.0 + container_name: texera-parca + profiles: [observability-profiles] + restart: always + security_opt: + - no-new-privileges:true + volumes: + - ../observability/parca/parca.yaml:/parca.yaml:ro,z + - parca_data:/var/lib/parca + command: + - "/parca" + - "--config-path=/parca.yaml" + ports: + - "127.0.0.1:7070:7070" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Parca eBPF agent (Apache-2.0). Linux-only, privileged: it is the + # only observability service needing elevated permissions, and it + # profiles every process on the host, not just Texera's. Keep it + # opt-in and read bin/observability/parca/README.md for the privilege + # and scope implications before enabling it. + parca-agent: + image: ghcr.io/parca-dev/parca-agent:v0.47.1 + container_name: texera-parca-agent + profiles: [observability-profiles] + restart: always + depends_on: + parca: + condition: service_started + env_file: + - ../observability/parca/parca-agent.env + # eBPF requires CAP_SYS_ADMIN-class privileges. macOS / Windows + # developers cannot run this, they should drop the + # observability-profiles profile from COMPOSE_PROFILES. + privileged: true + pid: "host" + # Bind-mount kernel state read-only, the agent reads /proc and + # /sys for stack-trace symbolization but cannot write to either. + volumes: + - /sys/kernel/debug:/sys/kernel/debug:ro + - /proc:/host/proc:ro + - /sys:/host/sys:ro + # parca-agent ships as a distroless image (no /bin/sh), so flags + # are passed directly. env_file values above are kept as the + # source of truth, edit both if you change a value. + command: + - "--remote-store-address=parca:7070" + - "--remote-store-insecure=true" + - "--node=texera-dev" + - "--metadata-external-labels=deployment=texera;cluster=local" + - "--profiling-cpu-sampling-frequency=19" + - "--log-level=info" + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" + networks: default: name: texera-single-node @@ -598,4 +839,11 @@ networks: volumes: rustfs_data: postgres_data: - workflow_result_data: \ No newline at end of file + workflow_result_data: + # Observability storage. Each backend gets its own named volume so + # an operator can drop one signal's history (e.g. logs) without + # touching the others. + loki_data: + grafana_data: + victoriametrics_data: + parca_data: \ No newline at end of file diff --git a/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala new file mode 100644 index 00000000000..4f263eb08d5 --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} + +/** + * Smoke tests for the single-node docker-compose + collector config. + * Same design as [[ParcaConfigSpec]]: string-level assertions, no YAML + * parser, because the goal is to catch typos and licence-pin drift, + * not to validate the upstream schemas. + */ +class ObservabilityComposeSpec extends AnyFlatSpec with Matchers { + + private def resolveBundled(relative: String): Path = { + var dir = Paths.get("").toAbsolutePath + var hops = 0 + while (hops < 10) { + val candidate = dir.resolve(relative) + if (Files.exists(candidate)) return candidate + val parent = dir.getParent + if (parent == null) return candidate + dir = parent + hops += 1 + } + Paths.get(relative) + } + + private def read(p: Path): String = + new String(Files.readAllBytes(p), StandardCharsets.UTF_8) + + // ----- bundled paths -------------------------------------------------- + + private val compose = resolveBundled("bin/single-node/docker-compose.yml") + private val envFile = resolveBundled("bin/single-node/.env") + private val overrideFile = resolveBundled("bin/single-node/docker-compose.override.yml") + private val collector = resolveBundled("bin/observability/otel-collector/config.yaml") + + // ----- docker-compose: pinned images --------------------------------- + + "docker-compose.yml" should "exist" in { + Files.exists(compose) shouldBe true + } + + it should "pin the OSS observability image tags from LICENSING.md" in { + val text = read(compose) + text should include("otel/opentelemetry-collector-contrib:0.153.0") + text should include("grafana/loki:3.2.0") + text should include("grafana/grafana:11.2.0") + text should include("victoriametrics/victoria-metrics:v1.144.0") + text should include("jaegertracing/jaeger:2.18.0") + text should include("ghcr.io/parca-dev/parca:v0.28.0") + text should include("ghcr.io/parca-dev/parca-agent:v0.47.1") + } + + it should "NOT reference the VictoriaMetrics enterprise images" in { + // Tripwire: '-enterprise' images are not Apache-2.0. Texera ships + // OSS only, so anyone editing the tags must be reminded of that. + val text = read(compose) + text should not include "-enterprise" + } + + it should "bind every observability host port to loopback (127.0.0.1)" in { + val text = read(compose) + // The backend query/UI endpoints we expose to the host MUST use + // the 127.0.0.1: prefix. Any "0.0.0.0:3100" or naked "3100:3100" + // would publish to all interfaces, a misconfig that would expose + // the data store to the network. (Grafana's own loopback binding is + // checked separately below, since its port is a variable.) + Seq("3100", "8428", "16686", "7070").foreach { port => + val pattern = s""""127.0.0.1:$port:$port"""" + withClue(s"port $port should be loopback-only: ") { + text should include(pattern) + } + } + } + + it should "bind the Grafana UI to loopback too (not all interfaces)" in { + val text = read(compose) + // Grafana publishes a configurable port but still only on loopback, + // so the UI is reachable at http://localhost:3000 and never from the + // network by default. + text should include("127.0.0.1:${GRAFANA_PORT:-3000}:3000") + text should not include "0.0.0.0:3000" + } + + it should "publish the OTel collector OTLP receivers only in the dev override, on loopback" in { + // The base compose publishes no OTLP host ports (deploy-clean); the + // dev override exposes 127.0.0.1:4317-4318 so a backend running + // outside docker (sbt / IntelliJ) can emit telemetry. The hard + // requirement is that the receiver is NEVER bound on a non-loopback + // address, otherwise OTLP would be reachable from the LAN. + val base = read(compose) + base should not include "4317:4317" + base should not include "4318:4318" + val dev = read(overrideFile) + dev should include("\"127.0.0.1:4317:4317\"") + dev should include("\"127.0.0.1:4318:4318\"") + dev should not include "0.0.0.0:4317" + dev should not include "0.0.0.0:4318" + dev should not include "\"4317:4317\"" + dev should not include "\"4318:4318\"" + } + + it should "only mark the parca-agent privileged (eBPF needs CAP_SYS_ADMIN)" in { + // privileged: true is dangerous; we want it on exactly one + // service. If a future contributor copies the agent block as a + // template for another service, this test trips. + val text = read(compose) + val priv = "privileged: true".r.findAllIn(text).length + priv shouldBe 1 + } + + it should "give every observability backend a profile so it can be disabled" in { + val text = read(compose) + Seq( + "observability-collector", + "observability-logs", + "observability-grafana", + "observability-metrics", + "observability-traces", + "observability-profiles" + ).foreach { profile => + text should include(profile) + } + } + + it should "set a memory limit on every observability service" in { + // The new section lives below the "Observability stack" banner. + val text = read(compose).split("Observability stack").last + // Seven services, each gets a `deploy.resources.limits.memory:` line. + val memoryLimits = "memory:".r.findAllIn(text).length + memoryLimits should be >= 7 + } + + // ----- .env defaults ------------------------------------------------- + + ".env" should "default COMPOSE_PROFILES to empty so observability is opt-in" in { + val text = read(envFile) + // The active (non-comment) COMPOSE_PROFILES line must be empty: no + // observability profile runs unless the operator opts in. + val activeLine = text + .split('\n') + .map(_.trim) + .find(l => !l.startsWith("#") && l.startsWith("COMPOSE_PROFILES=")) + withClue("an active COMPOSE_PROFILES= line should exist: ")(activeLine shouldBe defined) + activeLine.get shouldBe "COMPOSE_PROFILES=" + } + + // ----- otel-collector config ----------------------------------------- + + "otel-collector/config.yaml" should "exist and declare all three signal pipelines" in { + Files.exists(collector) shouldBe true + val text = read(collector) + // The Service block defines exactly three pipelines. + text should include("logs:") + text should include("metrics:") + text should include("traces:") + } + + it should "route metrics via prometheusremotewrite to VictoriaMetrics" in { + val text = read(collector) + text should include("prometheusremotewrite") + text should include("victoriametrics:8428") + } + + it should "route logs to Loki over OTLP HTTP" in { + val text = read(collector) + text should include("otlphttp/loki") + text should include("loki:3100") + } + + it should "route traces to Jaeger over OTLP gRPC" in { + val text = read(collector) + text should include("otlp/jaeger") + text should include("jaeger:4317") + } + + it should "cap incoming OTLP message size (DoS guard)" in { + val text = read(collector) + text should include("max_recv_msg_size_mib") + } + + it should "configure memory_limiter to bound collector memory" in { + val text = read(collector) + text should include("memory_limiter") + text should include("limit_mib") + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala new file mode 100644 index 00000000000..42e29c6d75d --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} + +/** + * Smoke test for the bundled Parca configuration files. + * + * Intentionally lightweight: we are guarding against accidental + * deletion / emptying / tag drift, not validating Parca's schema. + * The real config validation happens when the agent starts up + * inside its container — but a unit-level smoke test catches typos + * before a developer pushes them. + */ +class ParcaConfigSpec extends AnyFlatSpec with Matchers { + + // sbt runs tests with the module dir as CWD, but a developer who + // runs `sbt test` from the project root has a different CWD. Walk + // upwards until we find the file, bounded so a missing file fails + // loudly rather than infinite-looping. + private def resolveBundled(relative: String): Path = { + var dir = Paths.get("").toAbsolutePath + var hops = 0 + while (hops < 10) { + val candidate = dir.resolve(relative) + if (Files.exists(candidate)) return candidate + val parent = dir.getParent + if (parent == null) return candidate + dir = parent + hops += 1 + } + Paths.get(relative) // will fail the existence assert below + } + + private val parcaYaml = resolveBundled("bin/observability/parca/parca.yaml") + private val agentEnv = resolveBundled("bin/observability/parca/parca-agent.env") + private val readme = resolveBundled("bin/observability/parca/README.md") + + // ----- parca.yaml ----------------------------------------------------- + + "parca.yaml" should "exist in bin/observability/parca/" in { + Files.exists(parcaYaml) shouldBe true + } + + it should "declare object_storage with the FILESYSTEM bucket type" in { + val text = new String(Files.readAllBytes(parcaYaml), StandardCharsets.UTF_8) + text should include("object_storage") + text should include("FILESYSTEM") + text should include("/var/lib/parca") + } + + it should "include the ASF license header" in { + val text = new String(Files.readAllBytes(parcaYaml), StandardCharsets.UTF_8) + text should include("Apache License, Version 2.0") + } + + // ----- parca-agent.env ------------------------------------------------ + + "parca-agent.env" should "exist and pin the bundled image version reference" in { + Files.exists(agentEnv) shouldBe true + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("v0.47.1") + } + + it should "point the agent at the bundled Parca server hostname" in { + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("PARCA_AGENT_REMOTE_STORE_ADDRESS=parca:7070") + } + + it should "carry the deployment label so the gateway can filter Texera processes" in { + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("deployment=texera") + } + + it should "NOT include high-cardinality labels (workflow.id / execution.id)" in { + // Tripwire: a future contributor might be tempted to add + // workflow.id as a static label. That blows up Parca storage. + // This assertion makes the design intent enforceable. Comments + // are skipped — they're allowed (and required) to explain the + // rule. + val configLines = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8).linesIterator + .map(_.trim) + .filter(line => line.nonEmpty && !line.startsWith("#")) + .toSeq + configLines.foreach { line => + line should not include "workflow.id" + line should not include "workflow_id" + line should not include "execution.id" + line should not include "execution_id" + } + } + + // ----- README --------------------------------------------------------- + + "README.md" should "exist and document that profiling is opt-in / off by default" in { + Files.exists(readme) shouldBe true + val text = new String(Files.readAllBytes(readme), StandardCharsets.UTF_8).toLowerCase + text should include("opt-in") + text should include("off by default") + } + + it should "document the Linux-only / privileged-container requirement" in { + val text = new String(Files.readAllBytes(readme), StandardCharsets.UTF_8) + text.toLowerCase should include("linux") + text.toLowerCase should include("privileged") + } +} diff --git a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/resource/ComputingUnitManagingResource.scala b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/resource/ComputingUnitManagingResource.scala index a9326e14a44..dd3701080d2 100644 --- a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/resource/ComputingUnitManagingResource.scala +++ b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/resource/ComputingUnitManagingResource.scala @@ -385,6 +385,46 @@ object ComputingUnitManagingResource { lookup(name).map(_.trim).filter(_.nonEmpty).map(name -> _) }.toMap + // Forwarded to a created unit so it emits to the same collector as the rest of + // the deployment. service.instance.id is NOT here: it is built per unit below. + private val observabilityComputingUnitEnvNames: Seq[String] = Seq( + EnvironmentalVariable.ENV_OTEL_SDK_DISABLED, + EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT, + EnvironmentalVariable.ENV_TEXERA_OTEL_ALLOWED_HOSTS, + EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL + ) + + /** + * Observability environment for a created unit. Without this a unit runs with + * the SDK-disabled default even when the manager has telemetry enabled, so a + * worker failure leaves no unit logs, metrics, or traces to investigate. The + * unit inherits the manager's collector endpoint and related settings, but is + * given its own service.instance.id (service.name stays shared) so units with + * the same name do not collapse into one metric series; an instance id the + * manager itself carries is dropped rather than copied. Returns nothing when no + * collector endpoint is configured, leaving the unit at its default-off state. + */ + private[resource] def observabilityComputingUnitEnv( + cuid: Int, + lookup: String => Option[String] + ): Map[String, String] = { + val get = (name: String) => lookup(name).map(_.trim).filter(_.nonEmpty) + if (get(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT).isEmpty) { + Map.empty + } else { + val forwarded = + observabilityComputingUnitEnvNames.flatMap(name => get(name).map(name -> _)).toMap + val kept = get(EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES) + .getOrElse("") + .split(",") + .map(_.trim) + .filter(_.nonEmpty) + .filterNot(_.toLowerCase.startsWith("service.instance.id=")) + val resourceAttrs = (kept :+ s"service.instance.id=computing-unit-$cuid").mkString(",") + forwarded + (EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES -> resourceAttrs) + } + } + // Environment variables passed to the created computing unit(pod) private lazy val computingUnitEnvironmentVariables: Map[String, Any] = icebergEnvironmentVariables ++ Map( @@ -733,7 +773,7 @@ class ComputingUnitManagingResource { computingUnitEnvironmentVariables ++ Map( EnvironmentalVariable.ENV_USER_JWT_TOKEN -> userToken, EnvironmentalVariable.ENV_JAVA_OPTS -> s"-Xmx${param.jvmMemorySize}" - ), + ) ++ observabilityComputingUnitEnv(cuid, EnvironmentalVariable.get), Some(param.shmSize), curatedImage ) diff --git a/computing-unit-managing-service/src/test/scala/org/apache/texera/service/resource/ComputingUnitManagingResourceSpec.scala b/computing-unit-managing-service/src/test/scala/org/apache/texera/service/resource/ComputingUnitManagingResourceSpec.scala index 34b3c996cb3..fffecb0aea9 100644 --- a/computing-unit-managing-service/src/test/scala/org/apache/texera/service/resource/ComputingUnitManagingResourceSpec.scala +++ b/computing-unit-managing-service/src/test/scala/org/apache/texera/service/resource/ComputingUnitManagingResourceSpec.scala @@ -563,4 +563,41 @@ class ComputingUnitManagingResourceSpec ComputingUnitManagingResource.optionalComputingUnitEnv(_ => Some(" ")) shouldBe empty } + "observabilityComputingUnitEnv" should "forward collector settings and give the unit its own instance id when enabled" in { + val enabled = Map( + EnvironmentalVariable.ENV_OTEL_SDK_DISABLED -> "false", + EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT -> "http://texera-otel-collector:4317", + EnvironmentalVariable.ENV_TEXERA_OTEL_ALLOWED_HOSTS -> "texera-otel-collector", + EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL -> "30000" + ) + val env = ComputingUnitManagingResource.observabilityComputingUnitEnv(7, enabled.get) + env(EnvironmentalVariable.ENV_OTEL_SDK_DISABLED) shouldBe "false" + env(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT) shouldBe + "http://texera-otel-collector:4317" + env(EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES) shouldBe + "service.instance.id=computing-unit-7" + } + + // Each unit must be distinguishable, so a unit never inherits the manager's instance id. + it should "replace an inherited service.instance.id with the unit's own, keeping other attributes" in { + val lookup: String => Option[String] = Map( + EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT -> "http://texera-otel-collector:4317", + EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES -> + "deployment.environment=staging,service.instance.id=manager,service.namespace=texera" + ).get + val attrs = ComputingUnitManagingResource + .observabilityComputingUnitEnv(8, lookup)(EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES) + attrs should include("deployment.environment=staging") + attrs should include("service.namespace=texera") + attrs should include("service.instance.id=computing-unit-8") + attrs should not include "service.instance.id=manager" + } + + // Telemetry stays opt-in: with no collector endpoint the unit is left default-off. + it should "forward nothing when no collector endpoint is configured" in { + ComputingUnitManagingResource.observabilityComputingUnitEnv(1, _ => None) shouldBe empty + val blankEndpoint = Map(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT -> " ") + ComputingUnitManagingResource.observabilityComputingUnitEnv(1, blankEndpoint.get) shouldBe empty + } + }