diff --git a/.github/workflows/create-release-candidate.yml b/.github/workflows/create-release-candidate.yml index 1e12d228083..d73b1df6dda 100644 --- a/.github/workflows/create-release-candidate.yml +++ b/.github/workflows/create-release-candidate.yml @@ -174,7 +174,7 @@ jobs: # Export the single-node directory from the tagged source mkdir -p "$TEMP_DIR/_raw" - git archive --format=tar "$TAG_NAME" -- bin/single-node/ sql/ NOTICE licenses/LICENSE-BSD-3-Clause.txt | tar -x -C "$TEMP_DIR/_raw" + git archive --format=tar "$TAG_NAME" -- bin/single-node/ bin/observability/ sql/ NOTICE licenses/LICENSE-BSD-3-Clause.txt | tar -x -C "$TEMP_DIR/_raw" # Copy deployment files cp "$TEMP_DIR/_raw/bin/single-node/docker-compose.yml" "$BUNDLE_DIR/" @@ -187,10 +187,19 @@ jobs: cp "$TEMP_DIR/_raw/licenses/LICENSE-BSD-3-Clause.txt" "$BUNDLE_DIR/licenses/" cp -r "$TEMP_DIR/_raw/sql" "$BUNDLE_DIR/" + # Observability configs (OTel collector + Parca) referenced by the + # opt-in observability compose profiles. Shipped next to the compose + # file so the ../observability mounts resolve in the flattened bundle. + cp -r "$TEMP_DIR/_raw/bin/observability" "$BUNDLE_DIR/observability" + # Patch the SQL mount path for the self-contained bundle layout # In the repo it's ../../sql (relative to bin/single-node/), in the bundle it's ./sql sed -i 's|../../sql|./sql|g' "$BUNDLE_DIR/docker-compose.yml" + # Patch the observability mount paths: ../observability (relative to + # bin/single-node/ in the repo) becomes ./observability in the bundle. + sed -i 's|../observability|./observability|g' "$BUNDLE_DIR/docker-compose.yml" + # Generate a release-pinned .env file with the version tag # Start from the source .env and ensure IMAGE_REGISTRY and IMAGE_TAG are set cp "$TEMP_DIR/_raw/bin/single-node/.env" "$BUNDLE_DIR/.env" diff --git a/.licenserc.yaml b/.licenserc.yaml index cbef454e403..0cebbbb3ca5 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -37,6 +37,8 @@ header: - '**/.env.example' - '**/.prettierrc' - '**/bun.lock' + # Helm's generated dependency lock file (no header; like yarn.lock/bun.lock) + - '**/Chart.lock' # Third-party code with MIT license - see LICENSE file for attribution - 'common/workflow-operator/src/main/scala/com/kjetland/**' # TypeFox monaco-languageclient derived files (MIT License) diff --git a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala index b5bdcbf5573..d85f127b740 100644 --- a/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala +++ b/amber/src/main/scala/org/apache/texera/web/ComputingUnitMaster.scala @@ -139,6 +139,7 @@ class ComputingUnitMaster extends io.dropwizard.Application[Configuration] with override def run(configuration: Configuration, environment: Environment): Unit = { org.apache.texera.observability.OtelInit.init("computing-unit-master") + org.apache.texera.web.observability.WorkflowMetricsRecorder.init() ObjectMapperUtils.warmupObjectMapperForOperatorsSerde() SqlServer.initConnection( diff --git a/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala new file mode 100644 index 00000000000..0f9ab2401aa --- /dev/null +++ b/amber/src/main/scala/org/apache/texera/web/observability/WorkflowMetricsRecorder.scala @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability + +import com.typesafe.scalalogging.LazyLogging +import org.apache.texera.amber.core.virtualidentity.ExecutionIdentity +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState.{ + COMPLETED, + FAILED, + KILLED, + PAUSED, + PAUSING, + RESUMING, + RUNNING +} +import org.apache.texera.observability.WorkflowMetrics +import org.apache.texera.observability.WorkflowMetrics.WorkflowKind +import org.apache.texera.web.service.WorkflowService + +import java.util.concurrent.ConcurrentHashMap + +/** + * Drives [[WorkflowMetrics]] from amber's execution lifecycle. The metric + * instruments live in common/config; this object is the single place + * that records them, so the lifecycle code only needs one-line calls. + * + * - start / terminal counters and the duration histogram are recorded + * from [[onStart]] and [[onStateChange]]; + * - the always-polled `texera.workflow.active` gauge is sourced from the + * live WorkflowService registry via the supplier registered in [[init]]. + */ +object WorkflowMetricsRecorder extends LazyLogging { + + // Start time + kind per in-flight run, so a terminal transition can emit + // a duration and attribute the outcome. Keyed by the execution identity. + private val inFlight = new ConcurrentHashMap[ExecutionIdentity, (Long, WorkflowKind)]() + + private val ActiveStates: Set[WorkflowAggregatedState] = Set(RUNNING, PAUSING, PAUSED, RESUMING) + private val TerminalStates: Set[WorkflowAggregatedState] = Set(COMPLETED, FAILED, KILLED) + + /** Register the active-executions gauge supplier and bind the instruments. + * Call once at startup, after OtelInit.init. The supplier is polled on + * every metric collection, so it must never throw. + */ + def init(): Unit = { + WorkflowMetrics.setActiveExecutionsSupplier(() => + try { + WorkflowService.getAllWorkflowServices.iterator + .flatMap(s => Option(s.executionService.getValue)) + .map(_.executionStateStore.metadataStore.getState.state) + .count(ActiveStates.contains) + .toLong + } catch { + case _: Throwable => 0L + } + ) + WorkflowMetrics.ensureBound() + } + + /** Record that a run started. */ + def onStart( + executionId: ExecutionIdentity, + kind: WorkflowKind = WorkflowKind.Interactive + ): Unit = { + inFlight.put(executionId, (System.currentTimeMillis(), kind)) + WorkflowMetrics.recordStart(kind) + } + + /** Record terminal counters + duration exactly once, on the first + * transition from a non-terminal into a terminal state. Safe to call on + * every state change; non-terminal and repeat-terminal calls are no-ops. + */ + def onStateChange( + executionId: ExecutionIdentity, + oldState: WorkflowAggregatedState, + newState: WorkflowAggregatedState + ): Unit = { + if (!TerminalStates.contains(newState) || TerminalStates.contains(oldState)) return + val entry = Option(inFlight.remove(executionId)) + val kind = entry.map(_._2).getOrElse(WorkflowKind.Interactive) + val durationSec = entry.map(e => (System.currentTimeMillis() - e._1) / 1000.0).getOrElse(0.0) + newState match { + case COMPLETED => WorkflowMetrics.recordCompletion(kind, durationSec) + case FAILED => WorkflowMetrics.recordFailure(kind, durationSec) + case KILLED => WorkflowMetrics.recordCancellation(kind) + case _ => () + } + } +} diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala index 4c8eb209b28..1c921fcacbc 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowExecutionService.scala @@ -108,7 +108,12 @@ class WorkflowExecutionService( var executionRuntimeService: ExecutionRuntimeService = _ var executionConsoleService: ExecutionConsoleService = _ - def executeWorkflow(): Unit = { + // Returns Some(err) when setup fails synchronously (e.g. compilation) after the + // failure has already been surfaced to the UI via errorHandler, so the caller can + // mark its still-open setup span; None once the run has started, after which + // runtime failures arrive asynchronously through errorHandler and belong to later + // execution work rather than this setup. + def executeWorkflow(): Option[Throwable] = { try { val compilationResult = new WorkflowCompiler(workflowContext) .compile(request.logicalPlan, CompilationErrorHandling.Strict) @@ -118,7 +123,7 @@ class WorkflowExecutionService( // stop here: `workflow` is still null, so falling through would NPE // below and mask the reported compilation error errorHandler(err) - return + return Some(err) } client = ComputingUnitMaster.createAmberRuntime( @@ -172,6 +177,7 @@ class WorkflowExecutionService( } ) ) + None } override def unsubscribeAll(): Unit = { diff --git a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala index 8ec650ba715..54056c4caea 100644 --- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala +++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala @@ -21,9 +21,11 @@ package org.apache.texera.web.service import com.google.protobuf.timestamp.Timestamp import com.typesafe.scalalogging.LazyLogging +import io.opentelemetry.api.trace.StatusCode import io.reactivex.rxjava3.disposables.{CompositeDisposable, Disposable} import io.reactivex.rxjava3.subjects.BehaviorSubject import org.apache.texera.common.config.{ApplicationConfig, StorageConfig} +import org.apache.texera.observability.{SpanAttrs, TexeraTracer} import org.apache.texera.dao.SqlServer import org.apache.texera.dao.jooq.generated.Tables.USER_WAREHOUSE import org.apache.texera.amber.core.WorkflowRuntimeException @@ -229,126 +231,165 @@ class WorkflowService( userOpt: Option[User], sessionUri: URI ): Unit = { + val span = TexeraTracer.tracer + .spanBuilder("WorkflowService.initExecutionService") + .setAttribute(SpanAttrs.WorkflowId, Long.box(workflowId.id)) + .startSpan() + val scope = span.makeCurrent() + try { - val (uidOpt, userEmailOpt) = userOpt.map(user => (user.getUid, user.getEmail)).unzip + val (uidOpt, userEmailOpt) = userOpt.map(user => (user.getUid, user.getEmail)).unzip - // Validate before touching the execution already in flight: a request that is - // going to be refused must not take the running one's subscriptions with it. - // uid is NOT NULL in the DB; fail early here rather than letting the insert fail downstream. - val uid = uidOpt.getOrElse( - throw new IllegalArgumentException( - "Cannot start execution: a user id (uid) is required but none was provided." + // Validate before touching the execution already in flight: a request that is + // going to be refused must not take the running one's subscriptions with it. + // uid is NOT NULL in the DB; fail early here rather than letting the insert fail downstream. + val uid = uidOpt.getOrElse( + throw new IllegalArgumentException( + "Cannot start execution: a user id (uid) is required but none was provided." + ) ) - ) - val warehouseName = WorkflowService.resolveLakekeeperWarehouseName(req.warehouseId, uid) + val warehouseName = WorkflowService.resolveLakekeeperWarehouseName(req.warehouseId, uid) - if (executionService.hasValue) { - executionService.getValue.unsubscribeAll() - } - - val workflowContext: WorkflowContext = createWorkflowContext() - workflowContext.warehouse = warehouseName - var coordinatorConf = CoordinatorConfig.default - - // clean up results from previous run - val previousExecutionId = - WorkflowExecutionService.getLatestExecutionId(workflowId, req.computingUnitId) - previousExecutionId.foreach(eid => { - clearExecutionResources(eid) - }) // TODO: change this behavior after enabling cache. - - workflowContext.executionId = ExecutionsMetadataPersistService.insertNewExecution( - workflowContext.workflowId, - uid, - req.executionName, - convertToJson(req.engineVersion), - req.computingUnitId, - req.warehouseId - ) + if (executionService.hasValue) { + executionService.getValue.unsubscribeAll() + } - if (ApplicationConfig.faultToleranceLogRootFolder.isDefined) { - val writeLocation = ApplicationConfig.faultToleranceLogRootFolder.get.resolve( - s"${workflowContext.workflowId}/${workflowContext.executionId}/" + val workflowContext: WorkflowContext = createWorkflowContext() + workflowContext.warehouse = warehouseName + var coordinatorConf = CoordinatorConfig.default + + // clean up results from previous run + val previousExecutionId = + WorkflowExecutionService.getLatestExecutionId(workflowId, req.computingUnitId) + previousExecutionId.foreach(eid => { + clearExecutionResources(eid) + }) // TODO: change this behavior after enabling cache. + + workflowContext.executionId = ExecutionsMetadataPersistService.insertNewExecution( + workflowContext.workflowId, + uid, + req.executionName, + convertToJson(req.engineVersion), + req.computingUnitId, + req.warehouseId ) - ExecutionsMetadataPersistService.tryUpdateExistingExecution(workflowContext.executionId) { - execution => execution.setLogLocation(writeLocation.toString) + span.setAttribute(SpanAttrs.ExecutionId, Long.box(workflowContext.executionId.id)) + // A run has started: record the start counter and stamp its start time. + org.apache.texera.web.observability.WorkflowMetricsRecorder + .onStart(workflowContext.executionId) + + if (ApplicationConfig.faultToleranceLogRootFolder.isDefined) { + val writeLocation = ApplicationConfig.faultToleranceLogRootFolder.get.resolve( + s"${workflowContext.workflowId}/${workflowContext.executionId}/" + ) + ExecutionsMetadataPersistService.tryUpdateExistingExecution(workflowContext.executionId) { + execution => execution.setLogLocation(writeLocation.toString) + } + coordinatorConf = coordinatorConf.copy(faultToleranceConfOpt = + Some(FaultToleranceConfig(writeTo = writeLocation)) + ) } - coordinatorConf = coordinatorConf.copy(faultToleranceConfOpt = - Some(FaultToleranceConfig(writeTo = writeLocation)) - ) - } - if (req.replayFromExecution.isDefined) { - val replayInfo = req.replayFromExecution.get - ExecutionsMetadataPersistService - .tryGetExistingExecution(ExecutionIdentity(replayInfo.eid)) - .foreach { execution => - val readLocation = new URI(execution.getLogLocation) - coordinatorConf = coordinatorConf.copy(stateRestoreConfOpt = - Some( - StateRestoreConfig( - readFrom = readLocation, - replayDestination = EmbeddedControlMessageIdentity(replayInfo.interaction) + if (req.replayFromExecution.isDefined) { + val replayInfo = req.replayFromExecution.get + ExecutionsMetadataPersistService + .tryGetExistingExecution(ExecutionIdentity(replayInfo.eid)) + .foreach { execution => + val readLocation = new URI(execution.getLogLocation) + coordinatorConf = coordinatorConf.copy(stateRestoreConfOpt = + Some( + StateRestoreConfig( + readFrom = readLocation, + replayDestination = EmbeddedControlMessageIdentity(replayInfo.interaction) + ) ) ) - ) - } - } + } + } - val executionStateStore = new ExecutionStateStore() - // assign execution id to find the execution from DB in case the constructor fails. - executionStateStore.metadataStore.updateState(state => - state.withExecutionId(workflowContext.executionId) - ) - val errorHandler: Throwable => Unit = { t => - { - val fromActorOpt = t match { - case ex: WorkflowRuntimeException => - ex.relatedWorkerId - case other => - None + val executionStateStore = new ExecutionStateStore() + // assign execution id to find the execution from DB in case the constructor fails. + executionStateStore.metadataStore.updateState(state => + state.withExecutionId(workflowContext.executionId) + ) + val errorHandler: Throwable => Unit = { t => + { + val fromActorOpt = t match { + case ex: WorkflowRuntimeException => + ex.relatedWorkerId + case other => + None + } + val (operatorId, workerId) = getOperatorFromActorIdOpt(fromActorOpt) + logger.error("error during execution", t) + // Do NOT touch `span` here: this handler is passed into + // WorkflowExecutionService and invoked asynchronously (runtime, + // websocket, startWorkflow callbacks) after initExecutionService has + // returned and the setup span has already ended, so recording onto it + // would be a silent no-op. The failure is surfaced via the metadata + // store below; setup-span errors are recorded in the catch block. + executionStateStore.statsStore.updateState(stats => + stats.withEndTimeStamp(System.currentTimeMillis()) + ) + executionStateStore.metadataStore.updateState { metadataStore => + updateWorkflowState(FAILED, metadataStore).addFatalErrors( + WorkflowFatalError( + EXECUTION_FAILURE, + Timestamp(Instant.now), + t.toString, + getStackTraceWithAllCauses(t), + operatorId, + workerId + ) + ) + } } - val (operatorId, workerId) = getOperatorFromActorIdOpt(fromActorOpt) - logger.error("error during execution", t) - executionStateStore.statsStore.updateState(stats => - stats.withEndTimeStamp(System.currentTimeMillis()) + } + // WorkflowExecutionService construction does no external work and cannot + // throw; it registers its error/state diff handler up front. Once published + // via `executionService.onNext`, any failure in `executeWorkflow()` is + // recorded by `errorHandler` into the metadata store, whose handler emits a + // WorkflowErrorEvent that `connectToExecution` forwards. + try { + val execution = new WorkflowExecutionService( + coordinatorConf, + workflowContext, + resultService, + req, + executionStateStore, + errorHandler, + userEmailOpt, + sessionUri ) - executionStateStore.metadataStore.updateState { metadataStore => - updateWorkflowState(FAILED, metadataStore).addFatalErrors( - WorkflowFatalError( - EXECUTION_FAILURE, - Timestamp(Instant.now), - t.toString, - getStackTraceWithAllCauses(t), - operatorId, - workerId - ) - ) + lifeCycleManager.registerCleanUpOnStateChange(executionStateStore) + executionService.onNext(execution) + // A synchronous setup failure (e.g. compilation) is surfaced to the UI by + // the run's errorHandler and signalled back here; record it on the still-open + // setup span so error-trace searches find it. None means the run started, after + // which runtime failures arrive asynchronously through errorHandler and belong + // to later execution work, not this setup span. + execution.executeWorkflow().foreach { setupErr => + span.recordException(setupErr) + span.setStatus(StatusCode.ERROR) } + } catch { + case e: Throwable => + // Setup threw before the run's errorHandler could record it; mark the + // open setup span and surface the failure to the UI. + span.recordException(e) + span.setStatus(StatusCode.ERROR) + errorHandler(e) } - } - // WorkflowExecutionService construction does no external work and cannot - // throw; it registers its error/state diff handler up front. Once published - // via `executionService.onNext`, any failure in `executeWorkflow()` is - // recorded by `errorHandler` into the metadata store, whose handler emits a - // WorkflowErrorEvent that `connectToExecution` forwards. - try { - val execution = new WorkflowExecutionService( - coordinatorConf, - workflowContext, - resultService, - req, - executionStateStore, - errorHandler, - userEmailOpt, - sessionUri - ) - lifeCycleManager.registerCleanUpOnStateChange(executionStateStore) - executionService.onNext(execution) - execution.executeWorkflow() + } catch { - case e: Throwable => errorHandler(e) + case t: Throwable => + // Synchronous setup failure (before the run's own errorHandler is wired). + span.recordException(t) + span.setStatus(StatusCode.ERROR) + throw t + } finally { + scope.close() + span.end() } - } def convertToJson(frontendVersion: String): String = { diff --git a/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala b/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala index 654acbbefd1..bc97b39d185 100644 --- a/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala +++ b/amber/src/main/scala/org/apache/texera/web/storage/ExecutionStateStore.scala @@ -27,6 +27,7 @@ import org.apache.texera.amber.engine.common.executionruntimestate.{ ExecutionMetadataStore, ExecutionStatsStore } +import org.apache.texera.web.observability.WorkflowMetricsRecorder import org.apache.texera.web.service.ExecutionsMetadataPersistService import java.sql.Timestamp @@ -44,6 +45,9 @@ object ExecutionStateStore { execution.setStatus(maptoStatusCode(state)) execution.setLastUpdateTime(new Timestamp(System.currentTimeMillis())) } + // Single chokepoint for every state transition: emit lifecycle metrics + // once on the first transition into a terminal state. + WorkflowMetricsRecorder.onStateChange(metadataStore.executionId, metadataStore.state, state) metadataStore.withState(state) } } diff --git a/amber/src/test/scala/org/apache/texera/web/observability/WorkflowMetricsRecorderSpec.scala b/amber/src/test/scala/org/apache/texera/web/observability/WorkflowMetricsRecorderSpec.scala new file mode 100644 index 00000000000..4c45c126c6a --- /dev/null +++ b/amber/src/test/scala/org/apache/texera/web/observability/WorkflowMetricsRecorderSpec.scala @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.web.observability + +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.common.CompletableResultCode +import io.opentelemetry.sdk.metrics.{InstrumentType, SdkMeterProvider} +import io.opentelemetry.sdk.metrics.`export`.{MetricExporter, PeriodicMetricReader} +import io.opentelemetry.sdk.metrics.data.{AggregationTemporality, MetricData} +import org.apache.texera.amber.core.virtualidentity.ExecutionIdentity +import org.apache.texera.amber.engine.architecture.rpc.controlreturns.WorkflowAggregatedState +import org.apache.texera.observability.WorkflowMetrics.WorkflowKind +import org.scalatest.BeforeAndAfterAll +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.time.Duration +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong +import scala.jdk.CollectionConverters._ + +/** + * Exercises the execution-state to metric mapping in WorkflowMetricsRecorder + * against a real in-memory meter. WorkflowMetrics.bindForTest is package + * private to common/observability, so we drive the binding the production way: + * install an SDK into GlobalOpenTelemetry and let ensureBound pick it up. + */ +class WorkflowMetricsRecorderSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll { + + private val exporter = new CapturingMetricExporter + private var provider: SdkMeterProvider = _ + private val nextId = new AtomicLong(1000) + private def newId(): ExecutionIdentity = ExecutionIdentity(nextId.incrementAndGet()) + + override def beforeAll(): Unit = { + GlobalOpenTelemetry.resetForTest() + val reader = PeriodicMetricReader.builder(exporter).setInterval(Duration.ofHours(1)).build() + provider = SdkMeterProvider.builder().registerMetricReader(reader).build() + GlobalOpenTelemetry.set(OpenTelemetrySdk.builder().setMeterProvider(provider).build()) + } + + override def afterAll(): Unit = { + if (provider != null) provider.close() + GlobalOpenTelemetry.resetForTest() + } + + // Cumulative total of a counter, summed over all attribute points. + private def counter(name: String): Long = { + provider.forceFlush().join(5, TimeUnit.SECONDS) + exporter.latest.asScala + .find(_.getName == name) + .map(_.getLongSumData.getPoints.asScala.map(_.getValue.longValue()).sum) + .getOrElse(0L) + } + + // Cumulative sample count of a histogram, summed over all attribute points. + private def histogramCount(name: String): Long = { + provider.forceFlush().join(5, TimeUnit.SECONDS) + exporter.latest.asScala + .find(_.getName == name) + .map(_.getHistogramData.getPoints.asScala.map(_.getCount).sum) + .getOrElse(0L) + } + + "onStart" should "increment texera.workflow.starts" in { + val before = counter("texera.workflow.starts") + WorkflowMetricsRecorder.onStart(newId(), WorkflowKind.Interactive) + counter("texera.workflow.starts") shouldBe before + 1 + } + + "onStateChange into COMPLETED" should "record a completion and a duration sample" in { + val id = newId() + WorkflowMetricsRecorder.onStart(id, WorkflowKind.Interactive) + val beforeCompletions = counter("texera.workflow.completions") + val beforeDurations = histogramCount("texera.workflow.duration") + WorkflowMetricsRecorder.onStateChange( + id, + WorkflowAggregatedState.RUNNING, + WorkflowAggregatedState.COMPLETED + ) + counter("texera.workflow.completions") shouldBe beforeCompletions + 1 + histogramCount("texera.workflow.duration") shouldBe beforeDurations + 1 + } + + "onStateChange into FAILED" should "record a completion (outcome=failure) and a duration sample" in { + val id = newId() + WorkflowMetricsRecorder.onStart(id, WorkflowKind.Scheduled) + val beforeCompletions = counter("texera.workflow.completions") + val beforeDurations = histogramCount("texera.workflow.duration") + WorkflowMetricsRecorder.onStateChange( + id, + WorkflowAggregatedState.RUNNING, + WorkflowAggregatedState.FAILED + ) + counter("texera.workflow.completions") shouldBe beforeCompletions + 1 + histogramCount("texera.workflow.duration") shouldBe beforeDurations + 1 + } + + "onStateChange into KILLED" should "record a cancellation and no duration sample" in { + val id = newId() + WorkflowMetricsRecorder.onStart(id, WorkflowKind.Interactive) + val beforeCancellations = counter("texera.workflow.cancellations") + val beforeDurations = histogramCount("texera.workflow.duration") + WorkflowMetricsRecorder.onStateChange( + id, + WorkflowAggregatedState.RUNNING, + WorkflowAggregatedState.KILLED + ) + counter("texera.workflow.cancellations") shouldBe beforeCancellations + 1 + histogramCount("texera.workflow.duration") shouldBe beforeDurations + } + + "onStateChange to a non-terminal state" should "record nothing" in { + val id = newId() + WorkflowMetricsRecorder.onStart(id, WorkflowKind.Interactive) + val before = counter("texera.workflow.completions") + WorkflowMetricsRecorder.onStateChange( + id, + WorkflowAggregatedState.RUNNING, + WorkflowAggregatedState.PAUSED + ) + counter("texera.workflow.completions") shouldBe before + } + + "onStateChange whose old state is already terminal" should "record nothing (dedup)" in { + val id = newId() + WorkflowMetricsRecorder.onStart(id, WorkflowKind.Interactive) + val before = counter("texera.workflow.completions") + WorkflowMetricsRecorder.onStateChange( + id, + WorkflowAggregatedState.COMPLETED, + WorkflowAggregatedState.FAILED + ) + counter("texera.workflow.completions") shouldBe before + } +} + +/** Captures the most recent metric export so a test can read cumulative values. */ +private class CapturingMetricExporter extends MetricExporter { + @volatile private var captured: java.util.Collection[MetricData] = + java.util.Collections.emptyList() + def latest: java.util.Collection[MetricData] = captured + override def `export`(metrics: java.util.Collection[MetricData]): CompletableResultCode = { + captured = new java.util.ArrayList[MetricData](metrics) + CompletableResultCode.ofSuccess() + } + override def flush(): CompletableResultCode = CompletableResultCode.ofSuccess() + override def shutdown(): CompletableResultCode = CompletableResultCode.ofSuccess() + override def getAggregationTemporality(instrumentType: InstrumentType): AggregationTemporality = + AggregationTemporality.CUMULATIVE +} diff --git a/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala b/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala index abf2bb039c5..53c63a9f25d 100644 --- a/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala +++ b/amber/src/test/scala/org/apache/texera/web/service/WorkflowExecutionServiceSpec.scala @@ -430,7 +430,12 @@ class WorkflowExecutionServiceSpec // null `workflow` on the way in, and throws out of `executeWorkflow` -- which is also what // keeps a "unit" test out of the live-runtime half on whatever actor system the shared JVM // holds. - noException should be thrownBy service.executeWorkflow() + var setupFailure: Option[Throwable] = None + noException should be thrownBy { setupFailure = service.executeWorkflow() } + + // The compilation failure is returned (not thrown) so the caller can record it on the + // still-open setup span, on top of reaching the UI through errorHandler. + setupFailure.map(_.getMessage).getOrElse("") should include("No file selected") errors should have size 1 errors.head.getMessage should include("No file selected") diff --git a/bin/k8s/Chart.lock b/bin/k8s/Chart.lock new file mode 100644 index 00000000000..af90dd0b29b --- /dev/null +++ b/bin/k8s/Chart.lock @@ -0,0 +1,21 @@ +dependencies: +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 16.5.6 +- name: minio + repository: https://charts.bitnami.com/bitnami + version: 15.0.7 +- name: lakefs + repository: https://charts.lakefs.io + version: 1.8.1 +- name: gateway-helm + repository: oci://docker.io/envoyproxy + version: 1.6.3 +- name: lakekeeper + repository: https://lakekeeper.github.io/lakekeeper-charts/ + version: 0.9.0 +- name: metrics-server + repository: https://kubernetes-sigs.github.io/metrics-server/ + version: 3.12.2 +digest: sha256:031184824e3bd3e03883a3debe64459f462610e4f5bd7c147f03fd05333b169e +generated: "2026-07-11T11:09:44.800500493-07:00" diff --git a/bin/k8s/install.sh b/bin/k8s/install.sh new file mode 100755 index 00000000000..d1ed9b55e6e --- /dev/null +++ b/bin/k8s/install.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# One-command install/upgrade of the Texera Helm chart. +# +# `helm install` on its own fails on a fresh checkout because the subchart +# dependencies (postgresql, minio, lakefs, envoy-gateway, lakekeeper, +# metrics-server) are declared in Chart.yaml but not vendored into charts/. +# This wrapper fetches them first (idempotent), then upgrades-or-installs. +# +# Usage (run from anywhere): +# bin/k8s/install.sh # app only +# bin/k8s/install.sh --set observability.enabled=true # app + observability +# RELEASE=texera NAMESPACE=texera bin/k8s/install.sh --set observability.enabled=true +# +# Any extra args are forwarded verbatim to `helm upgrade --install`, so +# --set / -f values.override.yaml / --namespace etc. all work. + +set -euo pipefail + +CHART_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RELEASE="${RELEASE:-texera}" +NAMESPACE="${NAMESPACE:-texera}" + +command -v helm >/dev/null 2>&1 || { echo "helm not found on PATH" >&2; exit 1; } + +# Fetch/refresh subcharts into charts/. `dependency build` uses Chart.lock when +# present (reproducible); it falls back to `dependency update` if there is no +# lock yet. Both are safe to re-run. +echo "==> Resolving chart dependencies" +if [ -f "${CHART_DIR}/Chart.lock" ]; then + helm dependency build "${CHART_DIR}" +else + helm dependency update "${CHART_DIR}" +fi + +echo "==> helm upgrade --install ${RELEASE} (namespace: ${NAMESPACE})" +exec helm upgrade --install "${RELEASE}" "${CHART_DIR}" \ + --namespace "${NAMESPACE}" \ + --create-namespace \ + "$@" diff --git a/bin/k8s/templates/base/_helpers.tpl b/bin/k8s/templates/base/_helpers.tpl index fe25aab1b72..8f310823851 100644 --- a/bin/k8s/templates/base/_helpers.tpl +++ b/bin/k8s/templates/base/_helpers.tpl @@ -55,6 +55,26 @@ services fall back to the in-cluster RustFS Service and its auto-generated {{- if .Values.storage.s3.endpoint -}}secret-access-key{{- else -}}RUSTFS_SECRET_KEY{{- end -}} {{- end -}} +{{/* +Observability emission env for a Scala service pod. + +Renders OTEL_EXPORTER_OTLP_ENDPOINT (pointing at the in-cluster OTel +Collector Service) plus TEXERA_OTEL_ALLOWED_HOSTS. OtelInit validates the +endpoint host against an allowlist whose default is localhost only, so the +collector's Service name must be added explicitly or the SDK rejects it and +emits nothing. Renders nothing unless both the observability stack and the +collector are enabled, so the default install is unchanged. Include inside a +container's `env:` list, e.g. `{{- include "texera.observability.env" . | nindent 12 }}`. +*/}} +{{- define "texera.observability.env" -}} +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://{{ .Release.Name }}-otel-collector:{{ .Values.observability.collector.grpcPort }}" +- name: TEXERA_OTEL_ALLOWED_HOSTS + value: "{{ .Release.Name }}-otel-collector" +{{- end }} +{{- end -}} + {{/* The audience the mounter's service-account tokens are bound to. Fixed rather than configurable: it is one half of a credential contract between access-control-service and the @@ -83,3 +103,81 @@ append the uid to it, so a bare value would render "http://jupyter/7". * {{- printf "/%s" (trimAll "/" .Values.jupyterPool.basePath) -}} {{- end -}} +{{/* Grafana datasource provisioning body. Kept as a helper so the Deployment's +checksum/config annotation can hash just this content; hashing the template that +holds the annotation would make it reference itself and recurse. */}} +{{- define "texera.observability.grafanaDatasources" -}} +apiVersion: 1 +datasources: + # Metrics: VictoriaMetrics speaks the Prometheus query API. Default. + - name: Metrics + type: prometheus + uid: metrics + access: proxy + url: http://{{ .Release.Name }}-victoriametrics:{{ .Values.observability.metrics.port }} + isDefault: true + editable: false +{{- if .Values.observability.logs.enabled }} + # Logs: Loki, LogQL over OTLP-ingested logs. + - name: Loki + type: loki + uid: loki + access: proxy + url: http://{{ .Release.Name }}-loki:{{ .Values.observability.logs.port }} + editable: false +{{- end }} +{{- if .Values.observability.traces.enabled }} + # Traces: Jaeger query API / UI backend. + - name: Jaeger + type: jaeger + uid: jaeger + access: proxy + url: http://{{ .Release.Name }}-jaeger:{{ .Values.observability.traces.queryPort }} + editable: false +{{- end }} +{{- end -}} + +{{/* Loki server config body. A helper for the same reason as the Grafana one, +and so the checksum/config annotation hashes the config instead of its template. +Retention is enforced here: the compactor deletes chunks older than +observability.logs.retentionPeriod, which Loki otherwise ignores. */}} +{{- define "texera.observability.lokiConfig" -}} +# No multi-tenancy: one implicit tenant for the in-cluster stack. +auth_enabled: false +server: + http_listen_port: {{ .Values.observability.logs.port }} + grpc_listen_port: 9096 +common: + instance_addr: 127.0.0.1 + # Every component's data path defaults under this prefix, the PVC at /loki. + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + # Required for OTLP ingest (OTLP attributes -> structured metadata). + allow_structured_metadata: true + reject_old_samples: false + # Age after which log chunks are eligible for deletion by the compactor. + retention_period: {{ .Values.observability.logs.retentionPeriod }} +compactor: + working_directory: /loki/compactor + # Without this the retention_period above is parsed but never enforced, + # so logs accumulate until the volume fills. + retention_enabled: true + delete_request_store: filesystem +{{- end -}} diff --git a/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml b/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml index 99419d1f549..be518a09717 100644 --- a/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml +++ b/bin/k8s/templates/base/access-control-service/access-control-service-deployment.yaml @@ -89,6 +89,7 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} livenessProbe: httpGet: path: /api/healthcheck diff --git a/bin/k8s/templates/base/config-service/config-service-deployment.yaml b/bin/k8s/templates/base/config-service/config-service-deployment.yaml index 7b86369f256..54fd1e99594 100644 --- a/bin/k8s/templates/base/config-service/config-service-deployment.yaml +++ b/bin/k8s/templates/base/config-service/config-service-deployment.yaml @@ -59,6 +59,7 @@ spec: - name: {{ .name }} value: "{{ .value }}" {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} livenessProbe: httpGet: path: /api/healthcheck diff --git a/bin/k8s/templates/base/file-service/file-service-deployment.yaml b/bin/k8s/templates/base/file-service/file-service-deployment.yaml index b60eddf11b5..e70f2d32135 100644 --- a/bin/k8s/templates/base/file-service/file-service-deployment.yaml +++ b/bin/k8s/templates/base/file-service/file-service-deployment.yaml @@ -82,4 +82,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/observability/grafana.yaml b/bin/k8s/templates/base/observability/grafana.yaml new file mode 100644 index 00000000000..36b1e05fe3e --- /dev/null +++ b/bin/k8s/templates/base/observability/grafana.yaml @@ -0,0 +1,125 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.grafana.enabled }} +# Grafana: dashboard/UI, k8s counterpart of the docker-compose grafana +# service. Reads the Apache-2.0 backends (Loki, VictoriaMetrics, Jaeger) +# as pre-provisioned datasources via the ConfigMap below. Grafana is +# AGPLv3, so it is an optional operator-run component, not part of any +# Texera release artifact (see bin/observability/README.md). Only this +# ASF-licensed template lives in the tree. The Service is ClusterIP (no +# Ingress), matching the rest of the stack; expose it via your own +# Ingress / port-forward to reach the UI. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-grafana-datasources + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-grafana +data: + datasources.yaml: | + {{- include "texera.observability.grafanaDatasources" . | nindent 4 }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-grafana + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-grafana +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-grafana + template: + metadata: + labels: + app: {{ .Release.Name }}-grafana + annotations: + # Roll the pod when the provisioned datasources change. + checksum/config: {{ include "texera.observability.grafanaDatasources" . | sha256sum }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 472 + fsGroup: 472 + containers: + - name: grafana + image: {{ .Values.observability.grafana.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + env: + # Admin login. Override for any environment reachable beyond + # an internal port-forward. + - name: GF_SECURITY_ADMIN_USER + value: {{ .Values.observability.grafana.adminUser | quote }} + - name: GF_SECURITY_ADMIN_PASSWORD + value: {{ .Values.observability.grafana.adminPassword | quote }} + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "false" + - name: GF_USERS_ALLOW_SIGN_UP + value: "false" + ports: + - name: http + containerPort: {{ .Values.observability.grafana.port }} + volumeMounts: + - name: datasources + mountPath: /etc/grafana/provisioning/datasources + readOnly: true + - name: data + mountPath: /var/lib/grafana + livenessProbe: + httpGet: + path: /api/health + port: {{ .Values.observability.grafana.port }} + initialDelaySeconds: 15 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.grafana.resources | nindent 12 }} + volumes: + - name: datasources + configMap: + name: {{ .Release.Name }}-grafana-datasources + # Grafana's SQLite state. emptyDir: datasources are re-provisioned + # on every start, so no PVC is required for the reference stack. + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-grafana + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-grafana +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-grafana + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.grafana.port }} + targetPort: {{ .Values.observability.grafana.port }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/jaeger.yaml b/bin/k8s/templates/base/observability/jaeger.yaml new file mode 100644 index 00000000000..2821a977736 --- /dev/null +++ b/bin/k8s/templates/base/observability/jaeger.yaml @@ -0,0 +1,81 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.traces.enabled }} +# Jaeger v2: traces backend + UI, k8s counterpart of the docker-compose +# jaeger service. v2 accepts OTLP natively (the collector exports here on +# otlpPort). In-memory storage: a restart wipes traces, same as the +# single-node compose. Swap in a persistent storage backend for +# multi-node / production use. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-jaeger + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-jaeger +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-jaeger + template: + metadata: + labels: + app: {{ .Release.Name }}-jaeger + spec: + securityContext: + runAsNonRoot: true + containers: + - name: jaeger + image: {{ .Values.observability.traces.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: query + containerPort: {{ .Values.observability.traces.queryPort }} + - name: otlp-grpc + containerPort: {{ .Values.observability.traces.otlpPort }} + resources: + {{- toYaml .Values.observability.traces.resources | nindent 12 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-jaeger + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-jaeger +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-jaeger + ports: + - name: query + protocol: TCP + port: {{ .Values.observability.traces.queryPort }} + targetPort: {{ .Values.observability.traces.queryPort }} + - name: otlp-grpc + protocol: TCP + port: {{ .Values.observability.traces.otlpPort }} + targetPort: {{ .Values.observability.traces.otlpPort }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/loki.yaml b/bin/k8s/templates/base/observability/loki.yaml new file mode 100644 index 00000000000..d26ca944e28 --- /dev/null +++ b/bin/k8s/templates/base/observability/loki.yaml @@ -0,0 +1,136 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.logs.enabled }} +# Loki: log store, k8s counterpart of the docker-compose loki service. +# OTLP ingest (/otlp) + LogQL query API on one port. Loki is AGPLv3, so +# it is an optional operator-run component, not part of any Texera release +# artifact (see bin/observability/README.md). Only this ASF-licensed +# template lives in the tree. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-loki-config + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-loki +data: + config.yaml: | + {{- include "texera.observability.lokiConfig" . | nindent 4 }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-loki-data + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-loki +spec: + accessModes: ["ReadWriteOnce"] + {{- if .Values.observability.logs.storageClass }} + storageClassName: {{ .Values.observability.logs.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.observability.logs.storage }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-loki + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-loki +spec: + replicas: 1 + # Single writer bound to one RWO volume; never run two at once. + strategy: + type: Recreate + selector: + matchLabels: + app: {{ .Release.Name }}-loki + template: + metadata: + labels: + app: {{ .Release.Name }}-loki + annotations: + # Roll the pod when the rendered config changes. + checksum/config: {{ include "texera.observability.lokiConfig" . | sha256sum }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + containers: + - name: loki + image: {{ .Values.observability.logs.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: ["-config.file=/etc/loki/config.yaml"] + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: http + containerPort: {{ .Values.observability.logs.port }} + volumeMounts: + - name: config + mountPath: /etc/loki + readOnly: true + - name: data + mountPath: /loki + # Read-only root filesystem, so scratch space is an emptyDir. + - name: tmp + mountPath: /tmp + livenessProbe: + httpGet: + path: /ready + port: {{ .Values.observability.logs.port }} + initialDelaySeconds: 15 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.logs.resources | nindent 12 }} + volumes: + - name: config + configMap: + name: {{ .Release.Name }}-loki-config + - name: data + persistentVolumeClaim: + claimName: {{ .Release.Name }}-loki-data + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-loki + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-loki +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-loki + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.logs.port }} + targetPort: {{ .Values.observability.logs.port }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/otel-collector-configmap.yaml b/bin/k8s/templates/base/observability/otel-collector-configmap.yaml new file mode 100644 index 00000000000..4a2cd2deee1 --- /dev/null +++ b/bin/k8s/templates/base/observability/otel-collector-configmap.yaml @@ -0,0 +1,109 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +# OpenTelemetry Collector config, k8s counterpart of +# bin/observability/otel-collector/config.yaml. Same pipeline topology; +# exporters point at the in-cluster backend Services instead of the +# docker-compose service names. Only the pipelines for enabled signals +# are rendered, so disabling a signal drops its exporter cleanly. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-otel-collector-config + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +data: + config.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:{{ .Values.observability.collector.grpcPort }} + max_recv_msg_size_mib: 4 + http: + endpoint: 0.0.0.0:{{ .Values.observability.collector.httpPort }} + + processors: + batch: + send_batch_size: 1024 + timeout: 5s + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + + exporters: + {{- if .Values.observability.logs.enabled }} + # Loki accepts OTLP at /otlp (Loki 3.x maps it to /otlp/v1/logs). + otlphttp/loki: + endpoint: http://{{ .Release.Name }}-loki:{{ .Values.observability.logs.port }}/otlp + compression: gzip + timeout: 10s + tls: + insecure: true + {{- end }} + {{- if .Values.observability.metrics.enabled }} + prometheusremotewrite: + endpoint: http://{{ .Release.Name }}-victoriametrics:{{ .Values.observability.metrics.port }}/api/v1/write + tls: + insecure: true + remote_write_queue: + queue_size: 1000 + num_consumers: 4 + {{- end }} + {{- if .Values.observability.traces.enabled }} + otlp/jaeger: + endpoint: {{ .Release.Name }}-jaeger:{{ .Values.observability.traces.otlpPort }} + tls: + insecure: true + {{- end }} + + service: + pipelines: + {{- if .Values.observability.logs.enabled }} + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + {{- end }} + {{- if .Values.observability.metrics.enabled }} + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + {{- end }} + {{- if .Values.observability.traces.enabled }} + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/jaeger] + {{- end }} + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + logs: + level: info +{{- end }} diff --git a/bin/k8s/templates/base/observability/otel-collector.yaml b/bin/k8s/templates/base/observability/otel-collector.yaml new file mode 100644 index 00000000000..89c537317d8 --- /dev/null +++ b/bin/k8s/templates/base/observability/otel-collector.yaml @@ -0,0 +1,95 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.collector.enabled }} +# OpenTelemetry Collector: the single OTLP ingress for all signals, +# k8s counterpart of the docker-compose otel-collector service. The +# receiver Ports are exposed only on the ClusterIP Service (no Ingress), +# mirroring the loopback-only posture of the compose deployment. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-otel-collector + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-otel-collector + template: + metadata: + labels: + app: {{ .Release.Name }}-otel-collector + annotations: + # Roll the pod when the rendered config changes. + checksum/config: {{ include (print $.Template.BasePath "/base/observability/otel-collector-configmap.yaml") . | sha256sum }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + containers: + - name: otel-collector + image: {{ .Values.observability.collector.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: ["--config=/etc/otelcol-contrib/config.yaml"] + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: otlp-grpc + containerPort: {{ .Values.observability.collector.grpcPort }} + - name: otlp-http + containerPort: {{ .Values.observability.collector.httpPort }} + volumeMounts: + - name: config + mountPath: /etc/otelcol-contrib + readOnly: true + resources: + {{- toYaml .Values.observability.collector.resources | nindent 12 }} + volumes: + - name: config + configMap: + name: {{ .Release.Name }}-otel-collector-config +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-otel-collector + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-otel-collector +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-otel-collector + ports: + - name: otlp-grpc + protocol: TCP + port: {{ .Values.observability.collector.grpcPort }} + targetPort: {{ .Values.observability.collector.grpcPort }} + - name: otlp-http + protocol: TCP + port: {{ .Values.observability.collector.httpPort }} + targetPort: {{ .Values.observability.collector.httpPort }} +{{- end }} diff --git a/bin/k8s/templates/base/observability/victoriametrics.yaml b/bin/k8s/templates/base/observability/victoriametrics.yaml new file mode 100644 index 00000000000..5a6a1ea97b8 --- /dev/null +++ b/bin/k8s/templates/base/observability/victoriametrics.yaml @@ -0,0 +1,112 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/}} + +{{- if and .Values.observability.enabled .Values.observability.metrics.enabled }} +# VictoriaMetrics: metrics store, k8s counterpart of the docker-compose +# victoriametrics service. Accepts Prometheus remote-write from the +# collector; MetricsQL query API on the same port. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-victoriametrics-data + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + accessModes: ["ReadWriteOnce"] + {{- if .Values.observability.metrics.storageClass }} + storageClassName: {{ .Values.observability.metrics.storageClass }} + {{- end }} + resources: + requests: + storage: {{ .Values.observability.metrics.storage }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-victoriametrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + replicas: 1 + # Single writer bound to one RWO volume; never run two at once. + strategy: + type: Recreate + selector: + matchLabels: + app: {{ .Release.Name }}-victoriametrics + template: + metadata: + labels: + app: {{ .Release.Name }}-victoriametrics + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10003 + fsGroup: 10003 + containers: + - name: victoriametrics + image: {{ .Values.observability.metrics.image }} + imagePullPolicy: {{ .Values.texeraImages.pullPolicy }} + args: + - "-storageDataPath=/data" + - "-retentionPeriod={{ .Values.observability.metrics.retentionPeriod }}" + - "-httpListenAddr=:{{ .Values.observability.metrics.port }}" + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + ports: + - name: http + containerPort: {{ .Values.observability.metrics.port }} + volumeMounts: + - name: data + mountPath: /data + livenessProbe: + httpGet: + path: /health + port: {{ .Values.observability.metrics.port }} + initialDelaySeconds: 10 + periodSeconds: 15 + resources: + {{- toYaml .Values.observability.metrics.resources | nindent 12 }} + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ .Release.Name }}-victoriametrics-data +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-victoriametrics + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Release.Name }}-victoriametrics +spec: + type: ClusterIP + selector: + app: {{ .Release.Name }}-victoriametrics + ports: + - name: http + protocol: TCP + port: {{ .Values.observability.metrics.port }} + targetPort: {{ .Values.observability.metrics.port }} +{{- end }} diff --git a/bin/k8s/templates/base/webserver/webserver-deployment.yaml b/bin/k8s/templates/base/webserver/webserver-deployment.yaml index 99a58870957..f71c2211903 100644 --- a/bin/k8s/templates/base/webserver/webserver-deployment.yaml +++ b/bin/k8s/templates/base/webserver/webserver-deployment.yaml @@ -77,4 +77,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml index a538ce4a86f..0bc09731390 100644 --- a/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml +++ b/bin/k8s/templates/base/workflow-compiling-service/workflow-compiling-service-deployment.yaml @@ -69,4 +69,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml index 1f6ef3f99b3..163a0e8e840 100644 --- a/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml +++ b/bin/k8s/templates/base/workflow-computing-unit-manager/workflow-computing-unit-manager-deployment.yaml @@ -151,4 +151,5 @@ spec: {{- range .Values.texeraEnvVars }} - name: {{ .name }} value: "{{ .value }}" - {{- end }} \ No newline at end of file + {{- end }} + {{- include "texera.observability.env" . | nindent 12 }} \ No newline at end of file diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml index 4be79adf73a..4c6f43a9755 100644 --- a/bin/k8s/values.yaml +++ b/bin/k8s/values.yaml @@ -634,3 +634,84 @@ envoy-gateway: extensionApis: enableBackend: true enableEnvoyPatchPolicy: true + +# Observability stack (hand-rolled, mirrors bin/observability + the +# single-node docker-compose profiles). Everything is gated behind +# `enabled` (default off) so the standard install is unchanged. When +# enabled, the Scala services emit OTLP to the in-cluster collector, +# which fans logs/metrics/traces to the backends below. All Services are +# ClusterIP only (no Ingress), matching the loopback-only posture of the +# compose deployment. Profiling (Parca) is intentionally not included: it +# needs a privileged eBPF DaemonSet and is a separate change. +observability: + # Master switch for the whole stack. + enabled: false + + # OpenTelemetry Collector: the single OTLP ingress. Required whenever + # any signal below is enabled (it is what the services emit to). + collector: + enabled: true + image: otel/opentelemetry-collector-contrib:0.153.0 + grpcPort: 4317 + httpPort: 4318 + resources: + limits: + memory: 768Mi + cpu: "1" + + # Loki: log store (OTLP ingest at /otlp + LogQL). AGPLv3, so it is an + # optional operator-run component, not part of any Texera release + # artifact (see bin/observability/README.md). + logs: + enabled: true + image: grafana/loki:3.2.0 + port: 3100 + retentionPeriod: 30d + storage: 10Gi + storageClass: local-path + resources: + limits: + memory: 1Gi + cpu: "1" + + # VictoriaMetrics: metrics store (Prometheus remote-write + MetricsQL). + metrics: + enabled: true + image: victoriametrics/victoria-metrics:v1.144.0 + port: 8428 + retentionPeriod: 90d + storage: 10Gi + storageClass: local-path + resources: + limits: + memory: 1Gi + cpu: "1" + + # Jaeger v2: traces backend + UI. In-memory storage (restart wipes + # traces); swap in persistent storage for production. + traces: + enabled: true + image: jaegertracing/jaeger:2.18.0 + queryPort: 16686 + otlpPort: 4317 + resources: + limits: + memory: 1Gi + cpu: "1" + + # Grafana: dashboard/UI reading Loki + VictoriaMetrics + Jaeger as + # pre-provisioned datasources. AGPLv3, so it is an optional operator-run + # component, not part of any Texera release artifact (see + # bin/observability/README.md). ClusterIP only; expose via your own + # Ingress or `kubectl port-forward`. Override the admin credentials for + # any environment reachable beyond an internal port-forward. + grafana: + enabled: true + image: grafana/grafana:11.2.0 + port: 3000 + adminUser: admin + adminPassword: texera + resources: + limits: + memory: 512Mi + cpu: "0.5" diff --git a/bin/local-dev/docker-compose.override.yml b/bin/local-dev/docker-compose.override.yml index 3b053a85700..bde7cbd37e5 100644 --- a/bin/local-dev/docker-compose.override.yml +++ b/bin/local-dev/docker-compose.override.yml @@ -22,6 +22,14 @@ # Node services *natively* on the host, so we expose each backend port here. services: + # A host-native backend (sbt / IntelliJ) emits OTLP to the collector that + # runs inside compose, so publish the collector's receiver ports on the host + # loopback. Only takes effect when the observability-collector profile is on. + otel-collector: + ports: + - "127.0.0.1:4317:4317" + - "127.0.0.1:4318:4318" + postgres: ports: - "5432:5432" diff --git a/bin/observability/README.md b/bin/observability/README.md new file mode 100644 index 00000000000..04ff0a832ea --- /dev/null +++ b/bin/observability/README.md @@ -0,0 +1,131 @@ + + +# Texera Observability Stack + +Texera's backend services are instrumented with [OpenTelemetry](https://opentelemetry.io/) +and export OTLP (logs, metrics, and traces). This directory is an **optional, +bundled reference stack** that an operator can start alongside a single-node +deployment to collect and visualize that telemetry. It is opt-in: nothing here +runs unless you enable the observability compose profiles (see below), and +Texera emits no telemetry unless `OTEL_SDK_DISABLED=false` is set. + +## Topology + +A single OpenTelemetry Collector is the only OTLP ingress. It fans each signal +out to a dedicated backend, and Grafana sits on top as the query UI: + +``` + Texera services + │ OTLP/gRPC (4317) or OTLP/HTTP (4318) + ▼ + otel-collector + ┌──────────────┬──────┴────────┬───────────────┐ + OTLP │ remote-write OTLP │ │ (eBPF agent push) + ▼ ▼ ▼ ▼ + Loki VictoriaMetrics Jaeger Parca + (logs) (metrics) (traces) (profiles) + │ │ │ + └──────────────┼────────────────┘ + ▼ + Grafana + (reads Loki + VictoriaMetrics + Jaeger) +``` + +- **Loki** stores logs; the collector exports OTLP to `http://loki:3100/otlp`. +- **VictoriaMetrics** stores metrics; the collector exports Prometheus + remote-write to `http://victoriametrics:8428/api/v1/write`. +- **Jaeger** stores traces; the collector exports OTLP to `jaeger:4317`. +- **Parca** stores continuous profiles, pushed by its eBPF agent (not read by + Grafana; use the Parca UI on `:7070`). +- **Grafana** reads Loki, VictoriaMetrics (via the Prometheus query API), and + Jaeger as pre-provisioned datasources. + +## Starting it + +The stack is driven by docker-compose profiles. Enable the signals you want in +`bin/single-node/.env` via `COMPOSE_PROFILES`, then bring the deployment up: + +``` +# bin/single-node/.env +COMPOSE_PROFILES=observability-collector,observability-logs,observability-metrics,observability-traces,observability-grafana +``` + +| Profile | Service(s) | +|---------------------------|-----------------------------------------| +| `observability-collector` | otel-collector (OTLP ingress; required) | +| `observability-logs` | loki | +| `observability-metrics` | victoriametrics | +| `observability-traces` | jaeger | +| `observability-profiles` | parca + privileged eBPF parca-agent | +| `observability-grafana` | grafana | + +Also set `OTEL_SDK_DISABLED=false` so the services actually emit telemetry, then: + +``` +cd bin/single-node && docker compose up -d +# or the dedicated entry point: +cd bin/observability && docker compose up -d +``` + +Reach **Grafana at http://localhost:3000**. The default login is +`admin` / `texera` (set via `GRAFANA_ADMIN_USER` / `GRAFANA_ADMIN_PASSWORD` in +`.env`); change it for anything reachable beyond localhost. + +### Pre-provisioned datasources + +Grafana comes with three datasources already wired up +(`grafana/provisioning/datasources/datasources.yaml`), so you can start querying +immediately: + +- **Metrics** (default) - VictoriaMetrics at `http://victoriametrics:8428`, + queried through the Prometheus API. +- **Loki** - logs at `http://loki:3100`. +- **Jaeger** - traces at `http://jaeger:16686`. + +The backend query ports are also published on `127.0.0.1` for ad-hoc `curl` +debugging (Loki `3100`, VictoriaMetrics `8428`, Jaeger `16686`, Parca `7070`). + +## Using your own stack + +Nothing here is mandatory. The Collector speaks standard OTLP and +Prometheus remote-write, so an operator can point their own tooling at the same +endpoints instead of running the bundled backends: + +- Point an **external Grafana** at the bundled Loki / VictoriaMetrics / Jaeger, + or swap the backends entirely. +- Send logs to **OpenSearch** (or any OTLP-capable log store) by changing the + collector's logs exporter in `otel-collector/config.yaml`. +- Replace the metrics or traces exporters with any Prometheus-remote-write or + OTLP-compatible destination. +- Or drop the collector and have an external collector scrape/receive directly; + the services only need an OTLP endpoint via `OTEL_EXPORTER_OTLP_ENDPOINT`. + +## Licensing + +> **NOTE:** **Grafana and Loki are licensed under the AGPLv3.** They are +> included here only as an **optional, operator-run** convenience and are **NOT +> bundled into, or distributed as part of, any Apache Texera release artifact**. +> An operator who runs them does so on their own; their images are pulled at +> deploy time, not shipped by the project. +> +> The rest of the stack - the **OpenTelemetry Collector, VictoriaMetrics, +> Jaeger, and Parca** - is **Apache-2.0** licensed, consistent with Texera +> itself. Only Apache-2.0 configuration files (collector, Loki, Grafana +> datasources, Parca) live in this repository. diff --git a/bin/observability/docker-compose.yml b/bin/observability/docker-compose.yml new file mode 100644 index 00000000000..adce75a4a40 --- /dev/null +++ b/bin/observability/docker-compose.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Dedicated entry point for the observability stack, so it can be started like +# the other dev targets (e.g. file-service's own compose) instead of reaching +# into the single-node deployment file. +# +# It does NOT redefine the services: it `include`s the single-node compose, the +# single source of truth for every container definition. The observability +# services there are gated behind compose profiles, so they are selected via +# COMPOSE_PROFILES + the service list (see start_app.sh `up_obs`). The other +# single-node services are not profile-gated, so they are simply not named and +# stay down. +# +# Relative volume paths inside the included file (e.g. the collector/parca +# configs under ../observability/) are resolved relative to the included file's +# own directory, so they keep working unchanged. +# +# The project name matches single-node so this manages the SAME containers +# whether obs is brought up here or as part of the full single-node deployment +# (the containers use fixed container_names, so a different project would clash). +name: texera-single-node + +include: + - ../single-node/docker-compose.yml diff --git a/bin/observability/grafana/provisioning/datasources/datasources.yaml b/bin/observability/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 00000000000..b4cd9c09461 --- /dev/null +++ b/bin/observability/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Grafana datasource provisioning for the Texera bundled observability +# stack. Grafana (AGPLv3) reads the three Apache-2.0 query backends by +# their docker-compose bridge-network service names. This file is only +# ASF-licensed configuration; Grafana itself is an optional operator-run +# component and is NOT shipped in any Texera release artifact (see +# bin/observability/README.md). +apiVersion: 1 + +datasources: + # Metrics: VictoriaMetrics speaks the Prometheus query API, so it is + # added as a prometheus-type datasource. Marked default. + - name: Metrics + type: prometheus + uid: metrics + access: proxy + url: http://victoriametrics:8428 + isDefault: true + editable: false + + # Logs: Loki, LogQL over OTLP-ingested logs. + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki:3100 + editable: false + + # Traces: Jaeger query API / UI backend. + - name: Jaeger + type: jaeger + uid: jaeger + access: proxy + url: http://jaeger:16686 + editable: false diff --git a/bin/observability/loki/config.yaml b/bin/observability/loki/config.yaml new file mode 100644 index 00000000000..5d695805e77 --- /dev/null +++ b/bin/observability/loki/config.yaml @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Loki configuration for the Texera bundled observability stack. +# +# Bundled image: grafana/loki:3.2.0 (AGPLv3). Loki is an optional, +# operator-run component; it is NOT shipped in any Texera release +# artifact (see bin/observability/README.md). Only this ASF-licensed +# config file lives in the tree. +# +# This is a minimal single-binary, filesystem-backed deployment for +# single-node development. Replace the filesystem storage with an +# object-storage backend (S3/GCS/Azure) before any multi-node or +# production use. + +# No multi-tenancy: a single-node dev stack has one implicit tenant, +# so requests need no X-Scope-OrgID header. +auth_enabled: false + +server: + # LogQL query API + OTLP ingest (/otlp/v1/logs) are served here. + http_listen_port: 3100 + # Internal gRPC; not published to the host. + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + # Every component's working/data path defaults under this prefix, + # which is the loki_data volume mounted at /loki in docker-compose. + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + # Single binary, single replica: no distributed ring. + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + # tsdb index + filesystem object store is the current (v13) layout. + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + # Required for OTLP ingest: OTLP log attributes are written as + # structured metadata. Enabled by default in Loki 3.x; kept explicit. + allow_structured_metadata: true + # Drop out-of-order rejections for a low-volume dev stack. + reject_old_samples: false + # Age after which log chunks are eligible for deletion by the compactor. + retention_period: 30d + +compactor: + working_directory: /loki/compactor + # Without this the retention_period above is parsed but never enforced, + # so logs accumulate until the loki_data volume fills. + retention_enabled: true + delete_request_store: filesystem + +# OTLP ingest needs no extra config: Loki 3.x serves it at +# http://loki:3100/otlp out of the box. The collector exports there +# (see bin/observability/otel-collector/config.yaml). diff --git a/bin/observability/otel-collector/config.yaml b/bin/observability/otel-collector/config.yaml new file mode 100644 index 00000000000..8cbee85742d --- /dev/null +++ b/bin/observability/otel-collector/config.yaml @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# OpenTelemetry Collector configuration for the Texera bundled +# observability stack. +# +# Bundled image: otel/opentelemetry-collector-contrib:0.153.0 +# (Apache-2.0). The contrib distribution is needed for the +# prometheusremotewrite exporter that fans metrics to VictoriaMetrics. +# +# Pipeline topology: +# +# Texera services +# │ OTLP/gRPC (4317) or OTLP/HTTP (4318) +# ▼ +# otel-collector +# ├── logs ── OTLP ──────────────────► loki:3100/otlp +# ├── metrics ── prometheusremotewrite ─► victoriametrics:8428 +# └── traces ── OTLP ──────────────────► jaeger:4317 +# +# Network: every receiver and exporter is reachable only on the +# compose bridge network. No host-port mapping for the receivers in +# docker-compose.yml, that keeps OTLP ingest cluster-local, which +# is the security posture the PR plan calls for. +# +# Hostnames below are docker-compose service names, they resolve via +# docker's embedded DNS inside the network. + +receivers: + otlp: + protocols: + grpc: + # 0.0.0.0 binds inside the container only; the compose service + # does NOT publish this port to the host. Reachable only via + # the texera-single-node bridge network. + endpoint: 0.0.0.0:4317 + # 4 MiB cap on a single OTLP message, anyone trying to spam + # the collector with a huge payload gets a clean reject. + max_recv_msg_size_mib: 4 + http: + endpoint: 0.0.0.0:4318 + +processors: + # batch is the upstream-recommended first processor. Caps memory + # per export by batching by either size or timeout, whichever hits + # first. Defaults are sensible for a single-node deployment. + batch: + send_batch_size: 1024 + timeout: 5s + + # memory_limiter prevents the collector OOMing under a sudden burst + # of telemetry. We set a soft limit, not a hard percentage, so the + # numbers are reviewable. + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + +exporters: + # Loki accepts OTLP for logs natively at /otlp (Loki 3.x maps it to + # /otlp/v1/logs). OTLP resource attributes become stream labels / + # structured metadata. Verify the endpoint on every version bump. + otlphttp/loki: + endpoint: http://loki:3100/otlp + compression: gzip + timeout: 10s + # tls is disabled for the compose bridge network. Any deploy + # that opens these ports beyond the bridge must enable TLS here. + tls: + insecure: true + + # VictoriaMetrics consumes Prometheus remote-write natively. The + # prometheusremotewrite exporter is the upstream-recommended way + # to bridge OTel metrics into Prom-ecosystem stores. + prometheusremotewrite: + endpoint: http://victoriametrics:8428/api/v1/write + tls: + insecure: true + # Keep the remote-write batch bounded so a metric burst can't + # become an oversized HTTP request. + remote_write_queue: + queue_size: 1000 + num_consumers: 4 + + # Jaeger v2 accepts OTLP directly, no jaeger exporter type needed. + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + +service: + pipelines: + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/jaeger] + + # Collector telemetry binds to loopback inside the container only + # so we can scrape /metrics from a sidecar if needed without + # exposing the collector's own metrics to the host. + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: 8888 + logs: + level: info diff --git a/bin/observability/parca/README.md b/bin/observability/parca/README.md new file mode 100644 index 00000000000..d799ba5e7aa --- /dev/null +++ b/bin/observability/parca/README.md @@ -0,0 +1,72 @@ + + +# Parca profiles + +Configuration for the **profiles** signal in the Texera observability +stack: the Parca server and its eBPF agent. The compose services that +run them are in `bin/single-node/docker-compose.yml`; this directory +holds only their configuration. + +Both components are Apache-2.0 (see +[`docs/observability/LICENSING.md`](../../../docs/observability/LICENSING.md)): + +| File | Component | Image | +|---|---|---| +| `parca.yaml` | Parca server v0.28.0 | `ghcr.io/parca-dev/parca:v0.28.0` | +| `parca-agent.env` | Parca eBPF agent v0.47.1 | `ghcr.io/parca-dev/parca-agent:v0.47.1` | + +## Host requirements + +The agent uses eBPF to sample stack traces, which constrains the host: + +- **Linux only.** eBPF is a Linux kernel feature; the agent cannot run + on macOS or Windows. The rest of the stack (logs, metrics, traces) + is cross-platform. +- **Privileged container.** The agent needs `CAP_SYS_ADMIN`-class + permissions to load eBPF programs and open the perf-event facility, + so the compose service runs `privileged: true` and bind-mounts + `/sys/kernel/debug`, `/proc`, and `/sys` read-only. +- **Outbound only.** The agent writes to the bundled Parca server on + `parca:7070` over plain gRPC on the docker bridge network and opens + no other ports. Any deploy that exposes those ports to a wider + network must add TLS at the compose/k8s layer. + +## Opt-in + +Profiling is off by default. It is enabled per host by selecting the +`profiles` profile before `docker compose up`; see `bin/single-node/.env` +for the flags. Enable it only after reviewing the privilege and scope +notes below. + +## What gets profiled + +The agent profiles **every process on the host**, not only Texera's. +It is not scoped to Texera's containers; doing that would require +configuring the agent against specific cgroups, which is not done +here. The static labels in `parca-agent.env` (`deployment=texera`, +`cluster=local`) only let a query filter which profiles are *shown*, +not what is *collected*. Treat enabling the agent as profiling the +whole machine. + +Profiles are not labeled with `workflow.id` / `execution.id` (unbounded +identifiers that would blow up Parca's cardinality). The agent samples +stack traces and does not read application-level OpenTelemetry trace +ids, so profiles cannot be joined to individual traces by `trace_id`; +they correlate only by the coarse labels above and by time. diff --git a/bin/observability/parca/parca-agent.env b/bin/observability/parca/parca-agent.env new file mode 100644 index 00000000000..bfd61dde79f --- /dev/null +++ b/bin/observability/parca/parca-agent.env @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Parca eBPF agent environment. +# +# Bundled image: ghcr.io/parca-dev/parca-agent:v0.47.1 (Apache-2.0). +# +# Consumed by the docker-compose parca-agent service via env_file:. +# The agent reads its config from CLI args; staging them here keeps +# them version-controlled and reusable across compose and Helm. +# +# Deploy requirements (Linux-only, privileged): +# * The agent uses eBPF and must run on a Linux host kernel. +# * Requires CAP_SYS_ADMIN-class permissions to load eBPF programs. +# The compose service runs the agent container with +# `privileged: true` and mounts /sys/kernel/debug, /proc, /sys +# read-only. macOS / Windows developers must leave the +# observability-profiles profile out of COMPOSE_PROFILES (see README.md). +# +# Labels: +# * node: host identifier — set by the compose layer to the docker +# host's hostname. Overridable. +# * metadata-external-labels: static labels attached to every +# profile (deployment=texera;cluster=local). Coarse fields only; +# no workflow/execution id (cardinality DoS in Parca storage). + +PARCA_AGENT_REMOTE_STORE_ADDRESS=parca:7070 +PARCA_AGENT_REMOTE_STORE_INSECURE=true + +# Per-host identification. Compose can override at deploy time: +# `--node=${HOSTNAME:-texera-dev}` +PARCA_AGENT_NODE=texera-dev + +# Static labels attached to every profile this agent emits. +# Must NOT include workflow.id / execution.id — cardinality DoS in +# Parca's storage. Keep to coarse fields only. +PARCA_AGENT_METADATA_EXTERNAL_LABELS=deployment=texera;cluster=local + +# CPU sample rate. 19 Hz is upstream default and a sensible balance +# between overhead and resolution; we keep it explicit so an +# accidental upstream change doesn't silently shift it. +PARCA_AGENT_PROFILING_CPU_SAMPLING_FREQUENCY=19 + +# Log level — info by default; switch to warn in production to keep +# the agent quiet. +PARCA_AGENT_LOG_LEVEL=info diff --git a/bin/observability/parca/parca.yaml b/bin/observability/parca/parca.yaml new file mode 100644 index 00000000000..0af9e88a393 --- /dev/null +++ b/bin/observability/parca/parca.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Parca server configuration. +# +# This is the minimal config the Texera bundled observability stack +# uses. We deliberately keep scrape_configs empty — profiles arrive +# from the Parca eBPF agent (push model) rather than pull-mode scraping. +# Bundled image: ghcr.io/parca-dev/parca:v0.28.0 (Apache-2.0). +# +# Storage is intentionally filesystem-backed for the single-node +# docker-compose deployment. Replace with an object-storage backend +# (S3/GCS/Azure) before any multi-node or production use. + +object_storage: + bucket: + # FILESYSTEM is the simplest backend for single-node development. + # Data lives inside the container at this path; mount a volume + # in docker-compose to persist across container restarts. + type: "FILESYSTEM" + config: + directory: "/var/lib/parca" + +# scrape_configs is intentionally empty: the Texera deployment pushes +# profiles from the parca-agent via OTLP/gRPC. Adding scrape targets +# here would generate significantly more data than agent-based +# profiling per the upstream recommendation. +scrape_configs: [] diff --git a/bin/observability/smoke_test.py b/bin/observability/smoke_test.py new file mode 100644 index 00000000000..b109496bd7f --- /dev/null +++ b/bin/observability/smoke_test.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# End-to-end routing check for the bundled observability stack. +# +# Emits one log, one metric, and one trace span (each tagged with a test +# service name and a unique run id) at the OpenTelemetry Collector over +# OTLP/HTTP, exactly the way OtelInit's exporters do, then asserts that: +# +# logs land in Loki (not elsewhere) +# metrics land in VictoriaMetrics (not elsewhere) +# traces land in Jaeger (not elsewhere) +# +# and that each signal is attributed to the emitting service. This proves +# the Collector routes each signal type to the right backend per service. +# +# Run against a live stack (OTLP ports published on loopback by the dev +# override). Exit code 0 = all routes correct, 1 = a routing failure. +# +# Usage: python3 bin/observability/smoke_test.py [--service NAME] + +import argparse +import json +import sys +import time +import urllib.parse +import urllib.request +import uuid + +COLLECTOR = "http://127.0.0.1:4318" # OTLP/HTTP receiver +LOKI = "http://127.0.0.1:3100" # logs backend +VICTORIAMETRICS = "http://127.0.0.1:8428" # metrics backend +JAEGER = "http://127.0.0.1:16686" # traces backend + +TIMEOUT_S = 60 # how long to wait for a signal to become queryable +POLL_S = 3 + + +def _post(url, payload): + req = urllib.request.Request( + url, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=10) as r: + return r.status + + +def _get(url): + with urllib.request.urlopen(url, timeout=10) as r: + return json.load(r) + + +def _attr(key, value): + return {"key": key, "value": {"stringValue": value}} + + +def emit(service, run_id): + now_ns = str(time.time_ns()) + resource = {"attributes": [_attr("service.name", service)]} + + # LOG -> logs pipeline -> Loki + _post(f"{COLLECTOR}/v1/logs", {"resourceLogs": [{ + "resource": resource, + "scopeLogs": [{"logRecords": [{ + "timeUnixNano": now_ns, + "severityText": "INFO", + "severityNumber": 9, + "body": {"stringValue": f"smoke-test log {run_id}"}, + }]}], + }]}) + + # METRIC -> metrics pipeline -> VictoriaMetrics + metric_name = f"obs_smoke_test_{run_id}" + _post(f"{COLLECTOR}/v1/metrics", {"resourceMetrics": [{ + "resource": resource, + "scopeMetrics": [{"metrics": [{ + "name": metric_name, + "gauge": {"dataPoints": [{ + "asDouble": 1, + "timeUnixNano": now_ns, + "attributes": [_attr("run", run_id)], + }]}, + }]}], + }]}) + + # TRACE -> traces pipeline -> Jaeger + trace_id = (uuid.uuid4().hex + uuid.uuid4().hex)[:32] + span_id = uuid.uuid4().hex[:16] + _post(f"{COLLECTOR}/v1/traces", {"resourceSpans": [{ + "resource": resource, + "scopeSpans": [{"spans": [{ + "traceId": trace_id, + "spanId": span_id, + "name": f"smoke.test.{run_id}", + "kind": 1, + "startTimeUnixNano": now_ns, + "endTimeUnixNano": str(int(now_ns) + 1_000_000), + }]}], + }]}) + return metric_name + + +def _poll(check): + deadline = time.time() + TIMEOUT_S + while time.time() < deadline: + try: + if check(): + return True + except Exception: + pass + time.sleep(POLL_S) + return False + + +def loki_has(service, run_id): + q = urllib.parse.quote(f'{{service_name="{service}"}}') + end = time.time_ns() + start = end - 10 * 60 * 1_000_000_000 + url = f"{LOKI}/loki/api/v1/query_range?query={q}&start={start}&end={end}&limit=500" + res = _get(url)["data"]["result"] + return any(run_id in v[1] for s in res for v in s["values"]) + + +def vm_has(metric_name): + url = f"{VICTORIAMETRICS}/api/v1/query?query={urllib.parse.quote(metric_name)}" + return len(_get(url)["data"]["result"]) > 0 + + +def jaeger_has(service, run_id): + svcs = _get(f"{JAEGER}/api/services").get("data") or [] + if service not in svcs: + return False + url = f"{JAEGER}/api/traces?service={urllib.parse.quote(service)}&limit=50&lookback=1h" + data = _get(url).get("data") or [] + return any(run_id in s.get("operationName", "") for t in data for s in t["spans"]) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "--services", + default="obs-smoke-a,obs-smoke-b", + help="comma-separated service names to emit as (default tests two)", + ) + args = ap.parse_args() + services = [s.strip() for s in args.services.split(",") if s.strip()] + + # Emit a log + metric + trace for every service, each with its own run id. + emitted = {} + for svc in services: + run_id = uuid.uuid4().hex[:12] + metric_name = emit(svc, run_id) + emitted[svc] = (run_id, metric_name) + print(f"emitted service={svc} run_id={run_id}") + + results = [] + for svc, (run_id, metric_name) in emitted.items(): + checks = [ + (f"{svc}: logs -> Loki", lambda s=svc, r=run_id: loki_has(s, r)), + (f"{svc}: metrics -> VictoriaMetrics", lambda m=metric_name: vm_has(m)), + (f"{svc}: traces -> Jaeger", lambda s=svc, r=run_id: jaeger_has(s, r)), + ] + for label, check in checks: + ok = _poll(check) + results.append(ok) + print(f" {'PASS' if ok else 'FAIL'} {label}") + + # Isolation: one service's log marker must NOT appear under another service. + if len(services) >= 2: + a, b = services[0], services[1] + a_run = emitted[a][0] + leaked = False + try: + leaked = loki_has(b, a_run) # a's marker should not be under b + except Exception: + leaked = False + ok = not leaked + results.append(ok) + print(f" {'PASS' if ok else 'FAIL'} isolation: {a} logs not attributed to {b}") + + if all(results): + print("RESULT: PASS (each signal routed to the correct backend, per service)") + return 0 + print("RESULT: FAIL (a signal was mis-routed or mis-attributed)") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/single-node/.env b/bin/single-node/.env index 40098c54c01..4fcfe17d32e 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -101,17 +101,60 @@ TEXERA_DASHBOARD_SERVICE_ENDPOINT=http://dashboard-service:8080 WORKFLOW_COMPILING_SERVICE_ENDPOINT=http://workflow-compiling-service:9090 WORKFLOW_EXECUTION_SERVICE_ENDPOINT=http://workflow-runtime-coordinator-service:8085 -# OpenTelemetry (observability). Disabled by default; the SDK stays inert until -# OTEL_SDK_DISABLED=false. Point the endpoint at a reachable OTLP collector -# (http/https only) before enabling. -OTEL_SDK_DISABLED=true -OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4317 -# Comma-separated resource attributes (k1=v1,k2=v2); service.name is set per service. -OTEL_RESOURCE_ATTRIBUTES= -# Comma-separated extra hosts allowed for the OTLP endpoint (loopback allowed by default). -TEXERA_OTEL_ALLOWED_HOSTS= -# Metric export interval in ms. -OTEL_METRIC_EXPORT_INTERVAL=30000 +# ============================================================================ +# Observability +# ============================================================================ +# Each signal is a separate docker-compose profile and is OFF by default: +# COMPOSE_PROFILES is empty, so `docker compose up` runs only core Texera. +# Observability is opt-in, for operators who want it; enable it by listing +# the profiles you want (docker compose reads COMPOSE_PROFILES natively): +# COMPOSE_PROFILES=observability-collector,observability-logs,observability-metrics,observability-traces,observability-grafana +# +# Profiles: +# observability-collector otel-collector (OTLP ingress; the others need it) +# observability-logs loki (AGPLv3; see bin/observability/README.md) +# observability-metrics victoriametrics +# observability-traces jaeger +# observability-profiles parca + privileged eBPF parca-agent +# observability-grafana grafana UI (AGPLv3; reads loki/victoriametrics/jaeger) +# +# Grafana and Loki are AGPLv3 optional operator-run components; they are not +# bundled into any Texera release artifact. The collector, VictoriaMetrics, +# Jaeger, and Parca are Apache-2.0. +# +# observability-profiles is Linux-only, privileged, and profiles every +# process on the host (see bin/observability/parca/README.md). Add it only +# deliberately, after reading that file's privilege and scope notes. +COMPOSE_PROFILES= + +# Query endpoints the dashboard gateway calls to read each signal. The +# defaults in observability-gateway.conf are host-local (127.0.0.1), which +# is what a natively-run backend (sbt / IntelliJ) needs. Inside this compose +# the gateway runs in a container, so it must reach the backends by their +# bridge-network service names, these overrides do that. +TEXERA_OBS_LOGS_URL=http://loki:3100 +TEXERA_OBS_METRICS_URL=http://victoriametrics:8428 +TEXERA_OBS_TRACES_URL=http://jaeger:16686 +TEXERA_OBS_PROFILES_URL=http://parca:7070 + +# Grafana UI (observability-grafana profile). The port is bound to +# loopback (127.0.0.1) like the backends, so the UI is reached at +# http://localhost:3000. The admin login defaults below are weak dev +# defaults; override them for any deployment reachable beyond localhost. +GRAFANA_PORT=3000 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=texera + +# Emission side: the Scala services (via OtelInit) push OTLP to the bundled +# collector over the bridge network. Emission is controlled explicitly by +# OTEL_SDK_DISABLED, which defaults to disabled, so no service emits unless +# you opt in by setting OTEL_SDK_DISABLED=false (do that together with the +# observability-collector profile). The endpoint and allowed-hosts below only +# take effect once emission is enabled: OtelInit validates the endpoint host +# against an allowlist (localhost only by default), so the collector's compose +# service name must be allowlisted explicitly. +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +TEXERA_OTEL_ALLOWED_HOSTS=otel-collector # Notebook migration tool # Toggles the tool in the GUI; the migration itself needs an LLM key above. diff --git a/bin/single-node/docker-compose.override.yml b/bin/single-node/docker-compose.override.yml new file mode 100644 index 00000000000..4ed7f5708fc --- /dev/null +++ b/bin/single-node/docker-compose.override.yml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Local-dev override, auto-loaded by `docker compose` when run from this +# directory. It publishes the OTel collector's OTLP receiver ports on the +# host loopback so a backend running outside docker (sbt / IntelliJ) can +# emit telemetry to the collector. The released bundle does not ship this +# file, so the base compose stays network-clean in a deploy. +name: texera-single-node + +services: + otel-collector: + ports: + - "127.0.0.1:4317:4317" + - "127.0.0.1:4318:4318" diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index d09cbc09a3d..d0752c2ca40 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -590,6 +590,247 @@ services: command: > sh -c 'apk add --no-cache curl jq bash > /dev/null 2>&1 && bash /examples/load-examples.sh' + # ======================================================================== + # Observability stack: collector + logs, metrics, traces, profiles. + # + # Each signal is a separate compose profile; enable them via + # COMPOSE_PROFILES in bin/single-node/.env. Every port binds to + # loopback (127.0.0.1) or stays on the bridge network; no 0.0.0.0 + # host bindings. + # ======================================================================== + + # OpenTelemetry Collector, the single OTLP ingress for all three + # signals. Reads bin/observability/otel-collector/config.yaml. + # Apache-2.0; pinned to the contrib distribution. + otel-collector: + image: otel/opentelemetry-collector-contrib:0.153.0 + container_name: texera-otel-collector + profiles: [observability-collector] + restart: always + user: "10001:10001" + read_only: true + security_opt: + - no-new-privileges:true + volumes: + - ../observability/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml:ro,z + command: ["--config=/etc/otelcol-contrib/config.yaml"] + # In-compose services reach the OTLP receivers over the bridge + # network as otel-collector:4317/:4318, so the base file publishes + # no host ports. For a backend running outside docker (sbt / + # IntelliJ), bin/local-dev/docker-compose.override.yml, which the + # local-dev launcher loads, publishes them on the host loopback; the + # released bundle omits that override. + deploy: + resources: + limits: + memory: 768M + cpus: "1.0" + + # Loki (AGPLv3) log store. OTLP ingest at /otlp, LogQL query API on + # 3100. AGPLv3, so it is an optional operator-run component and is NOT + # shipped in any Texera release artifact (see bin/observability/README.md). + # Reads bin/observability/loki/config.yaml. + loki: + image: grafana/loki:3.2.0 + container_name: texera-loki + profiles: [observability-logs] + restart: always + user: "10001:10001" + read_only: true + security_opt: + - no-new-privileges:true + volumes: + - ../observability/loki/config.yaml:/etc/loki/config.yaml:ro,z + - loki_data:/loki + # Read-only root filesystem, so Loki's scratch space is an in-memory + # tmpfs; all durable state lives on the loki_data volume under /loki. + tmpfs: + - /tmp + command: ["-config.file=/etc/loki/config.yaml"] + # Loopback-only host binding so an operator can curl the query + # API from the host for ad-hoc debugging without exposing it to + # the network. + ports: + - "127.0.0.1:3100:3100" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Grafana (AGPLv3) dashboard/UI. Reads the Apache-2.0 backends below + # (Loki, VictoriaMetrics, Jaeger) as pre-provisioned datasources. The UI + # is meant to be opened by a human, but like the backends its port is + # bound to loopback only (http://localhost:3000). AGPLv3, so it is an + # optional operator-run component and is NOT shipped in any Texera + # release artifact (see bin/observability/README.md). + grafana: + image: grafana/grafana:11.2.0 + container_name: texera-grafana + profiles: [observability-grafana] + restart: always + security_opt: + - no-new-privileges:true + depends_on: + - loki + - victoriametrics + - jaeger + environment: + # Admin login. Defaults are weak dev defaults; override in .env for + # any deployment reachable beyond localhost. + - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-texera} + # No anonymous access: the UI requires the admin login above. + - GF_AUTH_ANONYMOUS_ENABLED=false + # Users cannot self-register. + - GF_USERS_ALLOW_SIGN_UP=false + volumes: + # Datasource provisioning (Loki + VictoriaMetrics + Jaeger), read-only. + - ../observability/grafana/provisioning:/etc/grafana/provisioning:ro,z + - grafana_data:/var/lib/grafana + # Loopback-only host binding, like the backends: a human opens + # http://localhost:3000, and the UI is never reachable from the + # network by default. + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" + + # VictoriaMetrics, Apache-2.0 metrics store. Accepts Prometheus + # remote-write from the collector at /api/v1/write, MetricsQL on + # /api/v1/query. + # Fix /data ownership before VictoriaMetrics starts. VM runs non-root + # (uid 10003) with a read-only rootfs, but a freshly created named volume + # is owned by root, so without this VM cannot create its lock file and + # crash-loops. Idempotent on an existing volume. + victoriametrics-init: + image: alpine:3.19 + container_name: texera-victoriametrics-init + profiles: [observability-metrics] + restart: "no" + user: "0:0" + entrypoint: ["/bin/sh", "-c"] + command: + - chown -R 10003:10003 /data + volumes: + - victoriametrics_data:/data + + victoriametrics: + image: victoriametrics/victoria-metrics:v1.144.0 + container_name: texera-victoriametrics + profiles: [observability-metrics] + restart: always + user: "10003:10003" + depends_on: + victoriametrics-init: + condition: service_completed_successfully + read_only: true + security_opt: + - no-new-privileges:true + command: + - "-storageDataPath=/data" + - "-retentionPeriod=90d" + - "-httpListenAddr=:8428" + volumes: + - victoriametrics_data:/data + ports: + - "127.0.0.1:8428:8428" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Jaeger v2, Apache-2.0 traces backend + UI. v2 accepts OTLP + # natively. The query API lives at :16686, OTLP ingest at :4317 + # (kept inside the bridge network only, only the collector talks + # to it). + jaeger: + image: jaegertracing/jaeger:2.18.0 + container_name: texera-jaeger + profiles: [observability-traces] + restart: always + security_opt: + - no-new-privileges:true + # In-memory storage for the single-node deployment, restarts + # wipe traces. Any deploy needing trace persistence must swap + # in Cassandra or OpenSearch-backed storage here. + ports: + - "127.0.0.1:16686:16686" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Parca server (Apache-2.0) profiles backend. Reads + # bin/observability/parca/parca.yaml. FILESYSTEM storage. + parca: + image: ghcr.io/parca-dev/parca:v0.28.0 + container_name: texera-parca + profiles: [observability-profiles] + restart: always + security_opt: + - no-new-privileges:true + volumes: + - ../observability/parca/parca.yaml:/parca.yaml:ro,z + - parca_data:/var/lib/parca + command: + - "/parca" + - "--config-path=/parca.yaml" + ports: + - "127.0.0.1:7070:7070" + deploy: + resources: + limits: + memory: 1G + cpus: "1.0" + + # Parca eBPF agent (Apache-2.0). Linux-only, privileged: it is the + # only observability service needing elevated permissions, and it + # profiles every process on the host, not just Texera's. Keep it + # opt-in and read bin/observability/parca/README.md for the privilege + # and scope implications before enabling it. + parca-agent: + image: ghcr.io/parca-dev/parca-agent:v0.47.1 + container_name: texera-parca-agent + profiles: [observability-profiles] + restart: always + depends_on: + parca: + condition: service_started + env_file: + - ../observability/parca/parca-agent.env + # eBPF requires CAP_SYS_ADMIN-class privileges. macOS / Windows + # developers cannot run this, they should drop the + # observability-profiles profile from COMPOSE_PROFILES. + privileged: true + pid: "host" + # Bind-mount kernel state read-only, the agent reads /proc and + # /sys for stack-trace symbolization but cannot write to either. + volumes: + - /sys/kernel/debug:/sys/kernel/debug:ro + - /proc:/host/proc:ro + - /sys:/host/sys:ro + # parca-agent ships as a distroless image (no /bin/sh), so flags + # are passed directly. env_file values above are kept as the + # source of truth, edit both if you change a value. + command: + - "--remote-store-address=parca:7070" + - "--remote-store-insecure=true" + - "--node=texera-dev" + - "--metadata-external-labels=deployment=texera;cluster=local" + - "--profiling-cpu-sampling-frequency=19" + - "--log-level=info" + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" + networks: default: name: texera-single-node @@ -598,4 +839,11 @@ networks: volumes: rustfs_data: postgres_data: - workflow_result_data: \ No newline at end of file + workflow_result_data: + # Observability storage. Each backend gets its own named volume so + # an operator can drop one signal's history (e.g. logs) without + # touching the others. + loki_data: + grafana_data: + victoriametrics_data: + parca_data: \ No newline at end of file diff --git a/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala new file mode 100644 index 00000000000..4f263eb08d5 --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/ObservabilityComposeSpec.scala @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} + +/** + * Smoke tests for the single-node docker-compose + collector config. + * Same design as [[ParcaConfigSpec]]: string-level assertions, no YAML + * parser, because the goal is to catch typos and licence-pin drift, + * not to validate the upstream schemas. + */ +class ObservabilityComposeSpec extends AnyFlatSpec with Matchers { + + private def resolveBundled(relative: String): Path = { + var dir = Paths.get("").toAbsolutePath + var hops = 0 + while (hops < 10) { + val candidate = dir.resolve(relative) + if (Files.exists(candidate)) return candidate + val parent = dir.getParent + if (parent == null) return candidate + dir = parent + hops += 1 + } + Paths.get(relative) + } + + private def read(p: Path): String = + new String(Files.readAllBytes(p), StandardCharsets.UTF_8) + + // ----- bundled paths -------------------------------------------------- + + private val compose = resolveBundled("bin/single-node/docker-compose.yml") + private val envFile = resolveBundled("bin/single-node/.env") + private val overrideFile = resolveBundled("bin/single-node/docker-compose.override.yml") + private val collector = resolveBundled("bin/observability/otel-collector/config.yaml") + + // ----- docker-compose: pinned images --------------------------------- + + "docker-compose.yml" should "exist" in { + Files.exists(compose) shouldBe true + } + + it should "pin the OSS observability image tags from LICENSING.md" in { + val text = read(compose) + text should include("otel/opentelemetry-collector-contrib:0.153.0") + text should include("grafana/loki:3.2.0") + text should include("grafana/grafana:11.2.0") + text should include("victoriametrics/victoria-metrics:v1.144.0") + text should include("jaegertracing/jaeger:2.18.0") + text should include("ghcr.io/parca-dev/parca:v0.28.0") + text should include("ghcr.io/parca-dev/parca-agent:v0.47.1") + } + + it should "NOT reference the VictoriaMetrics enterprise images" in { + // Tripwire: '-enterprise' images are not Apache-2.0. Texera ships + // OSS only, so anyone editing the tags must be reminded of that. + val text = read(compose) + text should not include "-enterprise" + } + + it should "bind every observability host port to loopback (127.0.0.1)" in { + val text = read(compose) + // The backend query/UI endpoints we expose to the host MUST use + // the 127.0.0.1: prefix. Any "0.0.0.0:3100" or naked "3100:3100" + // would publish to all interfaces, a misconfig that would expose + // the data store to the network. (Grafana's own loopback binding is + // checked separately below, since its port is a variable.) + Seq("3100", "8428", "16686", "7070").foreach { port => + val pattern = s""""127.0.0.1:$port:$port"""" + withClue(s"port $port should be loopback-only: ") { + text should include(pattern) + } + } + } + + it should "bind the Grafana UI to loopback too (not all interfaces)" in { + val text = read(compose) + // Grafana publishes a configurable port but still only on loopback, + // so the UI is reachable at http://localhost:3000 and never from the + // network by default. + text should include("127.0.0.1:${GRAFANA_PORT:-3000}:3000") + text should not include "0.0.0.0:3000" + } + + it should "publish the OTel collector OTLP receivers only in the dev override, on loopback" in { + // The base compose publishes no OTLP host ports (deploy-clean); the + // dev override exposes 127.0.0.1:4317-4318 so a backend running + // outside docker (sbt / IntelliJ) can emit telemetry. The hard + // requirement is that the receiver is NEVER bound on a non-loopback + // address, otherwise OTLP would be reachable from the LAN. + val base = read(compose) + base should not include "4317:4317" + base should not include "4318:4318" + val dev = read(overrideFile) + dev should include("\"127.0.0.1:4317:4317\"") + dev should include("\"127.0.0.1:4318:4318\"") + dev should not include "0.0.0.0:4317" + dev should not include "0.0.0.0:4318" + dev should not include "\"4317:4317\"" + dev should not include "\"4318:4318\"" + } + + it should "only mark the parca-agent privileged (eBPF needs CAP_SYS_ADMIN)" in { + // privileged: true is dangerous; we want it on exactly one + // service. If a future contributor copies the agent block as a + // template for another service, this test trips. + val text = read(compose) + val priv = "privileged: true".r.findAllIn(text).length + priv shouldBe 1 + } + + it should "give every observability backend a profile so it can be disabled" in { + val text = read(compose) + Seq( + "observability-collector", + "observability-logs", + "observability-grafana", + "observability-metrics", + "observability-traces", + "observability-profiles" + ).foreach { profile => + text should include(profile) + } + } + + it should "set a memory limit on every observability service" in { + // The new section lives below the "Observability stack" banner. + val text = read(compose).split("Observability stack").last + // Seven services, each gets a `deploy.resources.limits.memory:` line. + val memoryLimits = "memory:".r.findAllIn(text).length + memoryLimits should be >= 7 + } + + // ----- .env defaults ------------------------------------------------- + + ".env" should "default COMPOSE_PROFILES to empty so observability is opt-in" in { + val text = read(envFile) + // The active (non-comment) COMPOSE_PROFILES line must be empty: no + // observability profile runs unless the operator opts in. + val activeLine = text + .split('\n') + .map(_.trim) + .find(l => !l.startsWith("#") && l.startsWith("COMPOSE_PROFILES=")) + withClue("an active COMPOSE_PROFILES= line should exist: ")(activeLine shouldBe defined) + activeLine.get shouldBe "COMPOSE_PROFILES=" + } + + // ----- otel-collector config ----------------------------------------- + + "otel-collector/config.yaml" should "exist and declare all three signal pipelines" in { + Files.exists(collector) shouldBe true + val text = read(collector) + // The Service block defines exactly three pipelines. + text should include("logs:") + text should include("metrics:") + text should include("traces:") + } + + it should "route metrics via prometheusremotewrite to VictoriaMetrics" in { + val text = read(collector) + text should include("prometheusremotewrite") + text should include("victoriametrics:8428") + } + + it should "route logs to Loki over OTLP HTTP" in { + val text = read(collector) + text should include("otlphttp/loki") + text should include("loki:3100") + } + + it should "route traces to Jaeger over OTLP gRPC" in { + val text = read(collector) + text should include("otlp/jaeger") + text should include("jaeger:4317") + } + + it should "cap incoming OTLP message size (DoS guard)" in { + val text = read(collector) + text should include("max_recv_msg_size_mib") + } + + it should "configure memory_limiter to bound collector memory" in { + val text = read(collector) + text should include("memory_limiter") + text should include("limit_mib") + } +} diff --git a/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala new file mode 100644 index 00000000000..42e29c6d75d --- /dev/null +++ b/common/config/src/test/scala/org/apache/texera/observability/ParcaConfigSpec.scala @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} + +/** + * Smoke test for the bundled Parca configuration files. + * + * Intentionally lightweight: we are guarding against accidental + * deletion / emptying / tag drift, not validating Parca's schema. + * The real config validation happens when the agent starts up + * inside its container — but a unit-level smoke test catches typos + * before a developer pushes them. + */ +class ParcaConfigSpec extends AnyFlatSpec with Matchers { + + // sbt runs tests with the module dir as CWD, but a developer who + // runs `sbt test` from the project root has a different CWD. Walk + // upwards until we find the file, bounded so a missing file fails + // loudly rather than infinite-looping. + private def resolveBundled(relative: String): Path = { + var dir = Paths.get("").toAbsolutePath + var hops = 0 + while (hops < 10) { + val candidate = dir.resolve(relative) + if (Files.exists(candidate)) return candidate + val parent = dir.getParent + if (parent == null) return candidate + dir = parent + hops += 1 + } + Paths.get(relative) // will fail the existence assert below + } + + private val parcaYaml = resolveBundled("bin/observability/parca/parca.yaml") + private val agentEnv = resolveBundled("bin/observability/parca/parca-agent.env") + private val readme = resolveBundled("bin/observability/parca/README.md") + + // ----- parca.yaml ----------------------------------------------------- + + "parca.yaml" should "exist in bin/observability/parca/" in { + Files.exists(parcaYaml) shouldBe true + } + + it should "declare object_storage with the FILESYSTEM bucket type" in { + val text = new String(Files.readAllBytes(parcaYaml), StandardCharsets.UTF_8) + text should include("object_storage") + text should include("FILESYSTEM") + text should include("/var/lib/parca") + } + + it should "include the ASF license header" in { + val text = new String(Files.readAllBytes(parcaYaml), StandardCharsets.UTF_8) + text should include("Apache License, Version 2.0") + } + + // ----- parca-agent.env ------------------------------------------------ + + "parca-agent.env" should "exist and pin the bundled image version reference" in { + Files.exists(agentEnv) shouldBe true + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("v0.47.1") + } + + it should "point the agent at the bundled Parca server hostname" in { + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("PARCA_AGENT_REMOTE_STORE_ADDRESS=parca:7070") + } + + it should "carry the deployment label so the gateway can filter Texera processes" in { + val text = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8) + text should include("deployment=texera") + } + + it should "NOT include high-cardinality labels (workflow.id / execution.id)" in { + // Tripwire: a future contributor might be tempted to add + // workflow.id as a static label. That blows up Parca storage. + // This assertion makes the design intent enforceable. Comments + // are skipped — they're allowed (and required) to explain the + // rule. + val configLines = new String(Files.readAllBytes(agentEnv), StandardCharsets.UTF_8).linesIterator + .map(_.trim) + .filter(line => line.nonEmpty && !line.startsWith("#")) + .toSeq + configLines.foreach { line => + line should not include "workflow.id" + line should not include "workflow_id" + line should not include "execution.id" + line should not include "execution_id" + } + } + + // ----- README --------------------------------------------------------- + + "README.md" should "exist and document that profiling is opt-in / off by default" in { + Files.exists(readme) shouldBe true + val text = new String(Files.readAllBytes(readme), StandardCharsets.UTF_8).toLowerCase + text should include("opt-in") + text should include("off by default") + } + + it should "document the Linux-only / privileged-container requirement" in { + val text = new String(Files.readAllBytes(readme), StandardCharsets.UTF_8) + text.toLowerCase should include("linux") + text.toLowerCase should include("privileged") + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/SpanAttrs.scala b/common/observability/src/main/scala/org/apache/texera/observability/SpanAttrs.scala new file mode 100644 index 00000000000..c818f69bc16 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/SpanAttrs.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.api.common.AttributeKey + +/** + * Standard Texera span-attribute keys plus a free-text sanitizer. + * + * These are the shared label keys so every callsite tags spans with the + * same names. Set them with the standard OTel API at the callsite, e.g. + * ``spanBuilder.setAttribute(SpanAttrs.WorkflowId, id)`` or + * ``span.setAttribute(SpanAttrs.WorkflowId, id)``. Run any free-text value + * through [[sanitizeFreeText]] first to strip CRLF and cap its length. + */ +object SpanAttrs { + + /** Maximum length for free-text span attribute values. */ + val FreeTextMaxLen: Int = 256 + + // ---- Standard Texera correlation labels ------------------------------ + + val WorkflowId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.workflow.id") + val ExecutionId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.execution.id") + val ProjectId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.project.id") + val UserId: AttributeKey[java.lang.Long] = AttributeKey.longKey("texera.user.id") + val OperatorId: AttributeKey[String] = AttributeKey.stringKey("texera.operator.id") + val OperatorName: AttributeKey[String] = AttributeKey.stringKey("texera.operator.name") + val Outcome: AttributeKey[String] = AttributeKey.stringKey("texera.outcome") + + // ---- Pure helpers (exposed for testing) ------------------------------ + + /** + * Strip CR/LF and other C0 control characters from a free-text + * value, then cap at [[FreeTextMaxLen]]. Returns null for + * null/empty input (caller skips the setAttribute call). + */ + def sanitizeFreeText(value: String): String = { + if (value == null || value.isEmpty) return null + val stripped = value.filter(c => c >= 0x20 && c != 0x7f) + if (stripped.isEmpty) null + else if (stripped.length <= FreeTextMaxLen) stripped + else stripped.substring(0, FreeTextMaxLen) + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/TexeraTracer.scala b/common/observability/src/main/scala/org/apache/texera/observability/TexeraTracer.scala new file mode 100644 index 00000000000..38e1c36b620 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/TexeraTracer.scala @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context + +/** + * Accessor for the Texera OTel tracer. + * + * The only thing this adds over calling ``GlobalOpenTelemetry.getTracer`` + * directly is a single instrumentation scope name (``org.apache.texera``), + * so every Texera-produced span shows up under one logical scope in the + * backend, separable from anything emitted by transitive libraries. + * + * Start and end spans with the standard OTel API at the callsite (see the + * OpenTelemetry Java demo for the recommended pattern): + * + * {{{ + * val span = TexeraTracer.tracer.spanBuilder("MyClass.myMethod").startSpan() + * val scope = span.makeCurrent() + * try { ... } catch { + * case t: Throwable => span.recordException(t); span.setStatus(ERROR); throw t + * } finally { scope.close(); span.end() } + * }}} + * + * When the SDK is disabled, ``GlobalOpenTelemetry.getTracer`` returns a + * no-op tracer, so calling this is safe at any time. + */ +object TexeraTracer { + + private val InstrumentationScope = "org.apache.texera" + + def tracer: Tracer = GlobalOpenTelemetry.getTracer(InstrumentationScope) + + /** + * Snapshot the current OTel ``Context`` so async callbacks can + * re-attach it via ``Context.makeCurrent`` later. Useful at the + * Scala to Python boundary where the calling thread is not the + * receiving thread. + */ + def currentContext: Context = Context.current() +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala b/common/observability/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala new file mode 100644 index 00000000000..f5ea4d6cfa4 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/TraceparentValidator.scala @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +/** + * Pure validators for W3C Trace Context headers crossing the + * Scala↔Python boundary. The single rule: if the inbound bytes do + * not match the strict regex, we discard the value and start a fresh + * trace. We never echo a rejected value back into a span, log, or + * error message. + * + * Spec reference: https://www.w3.org/TR/trace-context/ + * + * The regexes here intentionally do NOT use any context-sensitive + * grouping or backreferences, keeps the validators safe against + * pathological inputs (ReDoS) and trivially fast. + */ +object TraceparentValidator { + + /** W3C traceparent format: `---`. + * We accept only version `00` (the only published version) with the + * canonical 32-hex / 16-hex / 2-hex layout. All hex lowercase per + * spec, uppercase is invalid. + */ + private val TraceparentPattern = + "^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$".r.pattern + + /** Bounded tracestate length. Spec recommends ≤512 chars. We are + * stricter to remove a small DoS surface, an attacker can't send + * a 1 MiB tracestate to balloon downstream context allocations. + */ + val MaxTracestateLength: Int = 512 + + /** Validate a traceparent header. Returns the input unchanged on + * success, None on any failure (rejected, caller starts a fresh + * trace). Null and empty are silent failures. + */ + def validateTraceparent(header: String): Option[String] = { + if (header == null || header.isEmpty) return None + // Trace-id and parent-id must not be all-zero per spec, an + // all-zero ID is a sentinel for "no value" and MUST be rejected. + if (!TraceparentPattern.matcher(header).matches()) return None + val parts = header.split('-') + val traceId = parts(1) + val spanId = parts(2) + if (isAllZero(traceId) || isAllZero(spanId)) return None + Some(header) + } + + /** Validate a tracestate header. Spec: comma-separated list of + * key=value pairs, ASCII-printable only, total length capped. + * Returns the input unchanged on success, None on rejection. + */ + def validateTracestate(header: String): Option[String] = { + if (header == null || header.isEmpty) return None + if (header.length > MaxTracestateLength) return None + if (!isAsciiPrintable(header)) return None + Some(header) + } + + private def isAllZero(s: String): Boolean = { + var i = 0 + while (i < s.length) { + if (s.charAt(i) != '0') return false + i += 1 + } + true + } + + private def isAsciiPrintable(s: String): Boolean = { + var i = 0 + while (i < s.length) { + val c = s.charAt(i) + // Allow printable ASCII range (0x20-0x7E). Tab and CR/LF are + // rejected, a tracestate must not span lines. + if (c < 0x20 || c > 0x7e) return false + i += 1 + } + true + } +} diff --git a/common/observability/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala b/common/observability/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala new file mode 100644 index 00000000000..84cd2d0e8f5 --- /dev/null +++ b/common/observability/src/main/scala/org/apache/texera/observability/WorkflowMetrics.scala @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import com.typesafe.scalalogging.LazyLogging +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.common.{AttributeKey, Attributes} +import io.opentelemetry.api.metrics.Meter + +/** + * Strongly-typed facade for the workflow-execution metrics cluster. + * + * This facade pattern is deliberate here because these instruments are a + * small, fixed, correlated set with strict cardinality rules, and it is + * worth centralizing that control in one place. It is NOT the default + * pattern for metrics in general: most call sites should just call the OTel + * meter API directly next to the business logic, e.g. + * + * {{{ + * private val meter = GlobalOpenTelemetry.getMeter("org.apache.texera") + * private val requests = meter.counterBuilder("myfeature.requests").build() + * // in the handler: + * requests.add(1, Attributes.of(AttributeKey.stringKey("route"), route)) + * }}} + * + * Only reach for a facade like this one when the metrics are complex or + * need centralized, standardized control. Do not copy it by default. + * + * Cardinality safety here is enforced by the API surface, not by + * documentation: there is no public method that accepts an arbitrary + * string as a label key or value. The only labels that ever land on + * an instrument are the two enums [[Outcome]] and [[WorkflowKind]], + * each restricted to a fixed set. ``workflow.id`` / ``execution.id`` + * are deliberately NOT metric labels: per-execution detail belongs + * in traces and logs, joined on ``trace_id`` at query time. + * + * Histogram bucket bounds are hard-coded constants so they can't be + * coerced by request input. The OTel SDK applies its own default + * attribute-value-length cap to anything that does slip through. + */ +object WorkflowMetrics extends LazyLogging { + + /** Outcome enum, the only mutable label on lifecycle counters. */ + sealed abstract class Outcome(val name: String) + object Outcome { + case object Success extends Outcome("success") + case object Failure extends Outcome("failure") + case object Cancelled extends Outcome("cancelled") + } + + /** Workflow kind enum. Distinguishes interactive vs. scheduled + * workflows for the dashboard's basic split, extend deliberately. + */ + sealed abstract class WorkflowKind(val name: String) + object WorkflowKind { + case object Interactive extends WorkflowKind("interactive") + case object Scheduled extends WorkflowKind("scheduled") + case object Unknown extends WorkflowKind("unknown") + } + + private val OutcomeKey: AttributeKey[String] = AttributeKey.stringKey("texera.outcome") + private val WorkflowKindKey: AttributeKey[String] = AttributeKey.stringKey("texera.workflow.kind") + + /** Histogram bucket bounds in seconds. Hard-coded, constants so + * request input can't reshape the histogram. Range covers + * fast (<1s) to long (>1h) workflows. + */ + private val DurationBuckets: java.util.List[java.lang.Double] = { + val builder = new java.util.ArrayList[java.lang.Double]() + Seq(0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0, 600.0, 1800.0, 3600.0) + .foreach(b => builder.add(java.lang.Double.valueOf(b))) + builder + } + + private val InstrumentationScope = "org.apache.texera" + + // Instruments are lazy + memoised. The first call after SDK init + // builds them against the active GlobalOpenTelemetry meter; once + // built they hold the meter instance, so a later GlobalOpenTelemetry + // reset (in tests) wouldn't be visible here, see [[resetForTest]]. + @volatile private var _starts: io.opentelemetry.api.metrics.LongCounter = _ + @volatile private var _completions: io.opentelemetry.api.metrics.LongCounter = _ + @volatile private var _cancellations: io.opentelemetry.api.metrics.LongCounter = _ + // `texera.workflow.active` is an OBSERVABLE gauge, not a manual up/down + // counter. A manual +1/-1 counter leaks whenever a run starts but its + // terminal event never fires (the controller is killed, the process + // restarts mid-run, or the +1 and -1 are split across the engine and web + // tiers and one side is missed), the gauge then drifts upward forever and + // the dashboard shows phantom "active" executions. An observable gauge + // instead reports the TRUE in-progress count from the live execution + // registry on every collection, so it cannot leak. The count is supplied + // by the host process via [[setActiveExecutionsSupplier]]. + @volatile private var _active: io.opentelemetry.api.metrics.ObservableLongGauge = _ + @volatile private var _duration: io.opentelemetry.api.metrics.DoubleHistogram = _ + + // Supplier of the current in-progress execution count. Defaults to 0 until + // the host process registers the real source (so a process that never + // registers reports a flat 0 rather than a wrong number). Read by the gauge + // callback on every metric collection. + @volatile private var activeExecutionsSupplier: () => Long = () => 0L + + /** Register the authoritative source of "currently active executions". + * The supplier is polled on every metric collection, so the gauge always + * reflects ground truth and can never leak. Called once at process + * startup (e.g. by ComputingUnitMaster). + */ + def setActiveExecutionsSupplier(supplier: () => Long): Unit = + synchronized { + activeExecutionsSupplier = supplier + ensureBound() + } + + /** Bind instruments to the current global meter. Idempotent, the + * first call wins; later calls are no-ops. Tests can call + * [[bindForTest]] with an explicit Meter, then [[resetForTest]] to + * rebind. + */ + def ensureBound(): Unit = + synchronized { + if (_starts == null) bind(GlobalOpenTelemetry.getMeter(InstrumentationScope)) + } + + private[observability] def bindForTest(meter: Meter): Unit = + synchronized { + bind(meter) + } + + private[observability] def resetForTest(): Unit = + synchronized { + _starts = null + _completions = null + _cancellations = null + // The observable gauge registered a collection callback, close it so the + // previous test's meter provider stops being polled after it's discarded. + if (_active != null) _active.close() + _active = null + _duration = null + } + + private def bind(meter: Meter): Unit = { + _starts = meter + .counterBuilder("texera.workflow.starts") + .setDescription("Number of workflow executions started.") + .build() + // Completions carry texera.outcome={success|failure}. Both success + // and self-terminating failures land here so the success/failure-rate + // queries (non-success ÷ all completions) have a denominator that + // means "runs that finished on their own". User-initiated kills are + // NOT completions, see _cancellations. + _completions = meter + .counterBuilder("texera.workflow.completions") + .setDescription("Number of workflow executions that ran to completion (success or failure).") + .build() + // Cancellations (user kills) are tracked separately so they decrement + // the active gauge without polluting the success/failure-rate + // denominator. Deliberately label-free apart from workflow.kind. + _cancellations = meter + .counterBuilder("texera.workflow.cancellations") + .setDescription("Number of workflow executions cancelled/killed before finishing.") + .build() + // Observable gauge: the callback runs on each collection and reports the + // live in-progress count, so the value cannot leak. No per-execution + // labels (cardinality-safe); the dashboard queries sum(texera_workflow_active). + _active = meter + .gaugeBuilder("texera.workflow.active") + .ofLongs() + .setDescription( + "Number of workflow executions currently in progress (observed from the live registry)." + ) + .buildWithCallback(obs => obs.record(activeExecutionsSupplier())) + _duration = meter + .histogramBuilder("texera.workflow.duration") + .setDescription("End-to-end duration of a workflow execution.") + .setUnit("s") + .setExplicitBucketBoundariesAdvice(DurationBuckets) + .build() + logger.info(s"Texera metric instruments bound to meter scope '$InstrumentationScope'") + } + + // ---- Public emitters, typed, no untyped escape hatch -------------- + + def recordStart(kind: WorkflowKind): Unit = { + ensureBound() + logger.debug(s"metric: workflow started (kind=${kind.name}), starts +1") + _starts.add(1L, Attributes.of(WorkflowKindKey, kind.name)) + // `active` is no longer mutated here, it is an observable gauge sourced + // from the live execution registry (see setActiveExecutionsSupplier). + } + + def recordCompletion(kind: WorkflowKind, durationSec: Double): Unit = { + ensureBound() + logger.debug( + f"metric: workflow completed successfully (kind=${kind.name}, ${durationSec}%.1fs), completions +1" + ) + val attrs = Attributes.of(OutcomeKey, Outcome.Success.name, WorkflowKindKey, kind.name) + _completions.add(1L, attrs) + _duration.record(durationSec, attrs) + } + + def recordFailure(kind: WorkflowKind, durationSec: Double): Unit = { + ensureBound() + logger.debug( + f"metric: workflow failed (kind=${kind.name}, ${durationSec}%.1fs), completions +1 (outcome=failure)" + ) + // A failure is a completion with outcome=failure, it shares the + // completions counter (and duration histogram) with successes so the + // failure-rate query has both numerator and denominator. + val attrs = Attributes.of(OutcomeKey, Outcome.Failure.name, WorkflowKindKey, kind.name) + _completions.add(1L, attrs) + _duration.record(durationSec, attrs) + } + + /** A user-initiated kill/cancel. Bumps a dedicated counter. Deliberately + * NOT recorded as a completion: a cancelled run never finished on its own, + * so it must not drag down the success rate. No duration is recorded for + * the same reason, a killed run's wall-clock time is not a real runtime + * and would skew the duration percentiles. (The active gauge is observed + * from the live registry and needs no decrement here.) + */ + def recordCancellation(kind: WorkflowKind): Unit = { + ensureBound() + logger.debug(s"metric: workflow cancelled (kind=${kind.name}), cancellations +1") + val attrs = Attributes.of(WorkflowKindKey, kind.name) + _cancellations.add(1L, attrs) + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala new file mode 100644 index 00000000000..8fabc602a8e --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/SpanAttrsSpec.scala @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.`export`.SimpleSpanProcessor +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class SpanAttrsSpec extends AnyFlatSpec with Matchers { + + // ----- sanitizeFreeText: pure ---------------------------------------- + + "sanitizeFreeText" should "return null for null / empty input" in { + SpanAttrs.sanitizeFreeText(null) shouldBe null + SpanAttrs.sanitizeFreeText("") shouldBe null + } + + it should "strip CR/LF and other control characters" in { + SpanAttrs.sanitizeFreeText("hello\r\nworld") shouldBe "helloworld" + SpanAttrs.sanitizeFreeText("a\u0000b\u0001c\u001fd\u007fe") shouldBe "abcde" + } + + it should "preserve printable text including spaces" in { + SpanAttrs.sanitizeFreeText("my-op v1 (alpha)") shouldBe "my-op v1 (alpha)" + } + + it should "cap at FreeTextMaxLen" in { + val long = "x" * (SpanAttrs.FreeTextMaxLen * 4) + val out = SpanAttrs.sanitizeFreeText(long) + out.length shouldBe SpanAttrs.FreeTextMaxLen + } + + it should "return null when stripping reduces the input to empty" in { + SpanAttrs.sanitizeFreeText("\r\n\u0000\u0001") shouldBe null + } + + // ----- standard label keys land via the plain OTel API --------------- + + "the standard label keys" should "set correctly through the OTel builder API" in { + val (exporter, tracer) = newTracer() + val sb = tracer.spanBuilder("workflow.execute") + sb.setAttribute(SpanAttrs.WorkflowId, java.lang.Long.valueOf(42L)) + sb.setAttribute(SpanAttrs.ExecutionId, java.lang.Long.valueOf(7L)) + sb.setAttribute(SpanAttrs.OperatorName, SpanAttrs.sanitizeFreeText("good\r\nFAKE LINE")) + sb.startSpan().end() + + val attrs = exporter.getFinishedSpanItems.asScala.head.getAttributes + attrs.get(SpanAttrs.WorkflowId) shouldBe 42L + attrs.get(SpanAttrs.ExecutionId) shouldBe 7L + attrs.get(SpanAttrs.OperatorName) shouldBe "goodFAKE LINE" + } + + it should "expose stable key names so callsites share one label vocabulary" in { + SpanAttrs.WorkflowId.getKey shouldBe "texera.workflow.id" + SpanAttrs.ExecutionId.getKey shouldBe "texera.execution.id" + } + + // ----- attribute-count cap is the OTel SDK default (128) ------------- + + it should "respect the SDK's per-span attribute cap of 128" in { + val (exporter, tracer) = newTracer() + val sb = tracer.spanBuilder("many") + // Set 200 distinct keys; SDK should drop everything past the + // default cap of 128 without crashing the test. + (0 until 200).foreach { i => + sb.setAttribute(s"k$i", s"v$i") + } + sb.startSpan().end() + + val attrs = exporter.getFinishedSpanItems.asScala.head.getAttributes + attrs.size should be <= 128 + } + + private def newTracer(): (InMemorySpanExporter, io.opentelemetry.api.trace.Tracer) = { + val exporter = InMemorySpanExporter.create() + val tp = SdkTracerProvider + .builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build() + val sdk = OpenTelemetrySdk.builder().setTracerProvider(tp).build() + (exporter, sdk.getTracer("test")) + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/TexeraTracerSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/TexeraTracerSpec.scala new file mode 100644 index 00000000000..77d5b3ab0f9 --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/TexeraTracerSpec.scala @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.context.Context +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class TexeraTracerSpec extends AnyFlatSpec with Matchers { + + "TexeraTracer.tracer" should "return a tracer that can build and start a span" in { + val tracer = TexeraTracer.tracer + tracer should not be null + // Without an installed SDK this is a no-op tracer, but it must still + // hand back a usable span rather than null so callsites never NPE. + val span = tracer.spanBuilder("texera.tracer.spec").startSpan() + span should not be null + span.end() + } + + "TexeraTracer.tracer" should "be stable across calls (same instrumentation scope)" in { + TexeraTracer.tracer should not be null + TexeraTracer.tracer should not be null + } + + "TexeraTracer.currentContext" should "return the active context" in { + TexeraTracer.currentContext shouldBe Context.current() + } + + it should "observe a value made current via the OpenTelemetry context" in { + val key = io.opentelemetry.context.ContextKey.named[String]("texera.tracer.spec.key") + val scope = Context.current().`with`(key, "v").makeCurrent() + try { + TexeraTracer.currentContext.get(key) shouldBe "v" + } finally { + scope.close() + } + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala new file mode 100644 index 00000000000..6bf74d99b38 --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/TraceparentValidatorSpec.scala @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class TraceparentValidatorSpec extends AnyFlatSpec with Matchers { + + // ----- traceparent: positive paths ----------------------------------- + + "validateTraceparent" should "accept a canonical W3C traceparent" in { + val good = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" + TraceparentValidator.validateTraceparent(good) shouldBe Some(good) + } + + it should "accept sampled=0 flag" in { + val good = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-00" + TraceparentValidator.validateTraceparent(good) shouldBe Some(good) + } + + // ----- traceparent: negative paths ----------------------------------- + + it should "reject null and empty input" in { + TraceparentValidator.validateTraceparent(null) shouldBe None + TraceparentValidator.validateTraceparent("") shouldBe None + } + + it should "reject path-traversal-style spoofing attempts" in { + TraceparentValidator.validateTraceparent("../../etc/passwd") shouldBe None + TraceparentValidator.validateTraceparent( + "00-../../../etc/passwd-b7ad6b7169203331-01" + ) shouldBe None + } + + it should "reject the wrong version byte" in { + // Only version 00 is published. 01, ff, etc. are invalid. + val bad01 = "01-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" + val badff = "ff-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" + TraceparentValidator.validateTraceparent(bad01) shouldBe None + TraceparentValidator.validateTraceparent(badff) shouldBe None + } + + it should "reject all-zero trace-id (sentinel for no-value)" in { + val zero = "00-00000000000000000000000000000000-b7ad6b7169203331-01" + TraceparentValidator.validateTraceparent(zero) shouldBe None + } + + it should "reject all-zero span-id (sentinel for no-value)" in { + val zero = "00-0af7651916cd43dd8448eb211c80319c-0000000000000000-01" + TraceparentValidator.validateTraceparent(zero) shouldBe None + } + + it should "reject UPPERCASE hex (spec requires lowercase)" in { + val upper = "00-0AF7651916CD43DD8448EB211C80319C-b7ad6b7169203331-01" + TraceparentValidator.validateTraceparent(upper) shouldBe None + } + + it should "reject non-hex characters in trace-id" in { + val bad = "00-XYZ7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" + TraceparentValidator.validateTraceparent(bad) shouldBe None + } + + it should "reject wrong-length segments" in { + // trace-id one char short + val short = "00-af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" + // span-id one char long + val long = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b71692033311-01" + TraceparentValidator.validateTraceparent(short) shouldBe None + TraceparentValidator.validateTraceparent(long) shouldBe None + } + + it should "reject embedded CRLF (header injection)" in { + val crlf = + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01\r\nX-Forge: yes" + TraceparentValidator.validateTraceparent(crlf) shouldBe None + } + + // ----- tracestate ---------------------------------------------------- + + "validateTracestate" should "accept a canonical tracestate" in { + val ts = "vendor1=hello,vendor2=world" + TraceparentValidator.validateTracestate(ts) shouldBe Some(ts) + } + + it should "reject null and empty" in { + TraceparentValidator.validateTracestate(null) shouldBe None + TraceparentValidator.validateTracestate("") shouldBe None + } + + it should "reject tracestate longer than the cap" in { + val oversize = "a=" + ("b" * (TraceparentValidator.MaxTracestateLength + 1)) + TraceparentValidator.validateTracestate(oversize) shouldBe None + } + + it should "reject non-ASCII / control characters" in { + TraceparentValidator.validateTracestate("vendor=hello\u00ffworld") shouldBe None + TraceparentValidator.validateTracestate("vendor=hello\r\nworld") shouldBe None + TraceparentValidator.validateTracestate("vendor=hello\u0000world") shouldBe None + } + + it should "accept tracestate at exactly the cap" in { + val atCap = "k=" + ("v" * (TraceparentValidator.MaxTracestateLength - 2)) + atCap.length shouldBe TraceparentValidator.MaxTracestateLength + TraceparentValidator.validateTracestate(atCap) shouldBe Some(atCap) + } +} diff --git a/common/observability/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala b/common/observability/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala new file mode 100644 index 00000000000..17c9c4c74dd --- /dev/null +++ b/common/observability/src/test/scala/org/apache/texera/observability/WorkflowMetricsSpec.scala @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.observability + +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.data.MetricData +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters._ + +class WorkflowMetricsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + private var reader: InMemoryMetricReader = _ + private var provider: SdkMeterProvider = _ + + override def beforeEach(): Unit = { + reader = InMemoryMetricReader.create() + provider = SdkMeterProvider.builder().registerMetricReader(reader).build() + WorkflowMetrics.resetForTest() + WorkflowMetrics.bindForTest(provider.get("org.apache.texera")) + } + + override def afterEach(): Unit = { + WorkflowMetrics.resetForTest() + provider.close() + } + + private def collectAll(): Map[String, MetricData] = { + reader.collectAllMetrics().asScala.map(m => m.getName -> m).toMap + } + + // ----- positive: lifecycle emissions ---------------------------------- + + "WorkflowMetrics" should "increment workflow.starts on recordStart" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + + val metrics = collectAll() + metrics.keySet should contain("texera.workflow.starts") + + val starts = metrics("texera.workflow.starts").getLongSumData.getPoints.asScala.head + starts.getValue shouldBe 1L + // recordStart no longer mutates `active` — that gauge is observed from the + // live registry (see "report the active-execution count…" below). + } + + it should "report the active-execution count from the registered supplier, never a manual counter" in { + // The whole point of the fix: `active` is an observable gauge sourced from + // ground truth, so it cannot leak. Drive it with a stub supplier and + // confirm the gauge reports exactly what the supplier returns — regardless + // of how many starts/completions were recorded. + @volatile var live = 3L + WorkflowMetrics.setActiveExecutionsSupplier(() => live) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + + val first = collectAll()("texera.workflow.active").getLongGaugeData.getPoints.asScala.head + first.getValue shouldBe 3L + + // It tracks the live source on the next collection — a manual +1/-1 + // counter could never drop like this without an explicit decrement. + live = 0L + val second = collectAll()("texera.workflow.active").getLongGaugeData.getPoints.asScala.head + second.getValue shouldBe 0L + } + + it should "record a completion and duration sample on recordCompletion" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = 2.5) + + val metrics = collectAll() + metrics.keySet should contain allOf ( + "texera.workflow.completions", + "texera.workflow.duration" + ) + + metrics( + "texera.workflow.completions" + ).getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + + val histogram = metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head + histogram.getCount shouldBe 1L + histogram.getSum shouldBe 2.5 + } + + it should "record a failure as a non-success completion (so failure-rate queries work)" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Scheduled) + WorkflowMetrics.recordFailure(WorkflowMetrics.WorkflowKind.Scheduled, durationSec = 12.0) + + val metrics = collectAll() + // A failure shares the completions counter with successes — the + // failure-rate query divides non-success completions by all + // completions, so failures must live here (not in a separate + // series the query never reads). + val completion = metrics("texera.workflow.completions").getLongSumData.getPoints.asScala.head + completion.getValue shouldBe 1L + completion.getAttributes.asMap.asScala.map { + case (k, v) => k.getKey -> v.toString + } should contain( + "texera.outcome" -> "failure" + ) + metrics("texera.workflow.duration").getHistogramData.getPoints.asScala.head.getSum shouldBe 12.0 + // The orphan counter the old wiring used must be gone. + metrics.keySet should not contain "texera.workflow.failures" + } + + it should "record a cancellation that is not a completion" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCancellation(WorkflowMetrics.WorkflowKind.Interactive) + + val metrics = collectAll() + metrics( + "texera.workflow.cancellations" + ).getLongSumData.getPoints.asScala.head.getValue shouldBe 1L + // A kill is not a completion and records no duration: it must not + // drag down the success rate nor skew the duration percentiles. + metrics.keySet should not contain "texera.workflow.completions" + metrics.keySet should not contain "texera.workflow.duration" + } + + // ----- security: cardinality safety ----------------------------------- + + it should "only emit the texera.outcome and texera.workflow.kind labels" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = 1.0) + + val attrKeys = collectAll().values.flatMap { md => + val pointSet = md.getType.name() match { + case "HISTOGRAM" => md.getHistogramData.getPoints.asScala + case "LONG_GAUGE" => md.getLongGaugeData.getPoints.asScala + case _ => md.getLongSumData.getPoints.asScala + } + pointSet.flatMap(_.getAttributes.asMap.keySet.asScala.map(_.getKey)) + }.toSet + + attrKeys.foreach { key => + Set("texera.outcome", "texera.workflow.kind") should contain(key) + } + attrKeys should not contain "texera.workflow.id" + attrKeys should not contain "texera.execution.id" + } + + it should "expose no public API to attach an arbitrary string label" in { + // The class has only typed emitters whose attribute set is + // hard-coded. This test is intentionally a compile-time check + // disguised as a runtime one — if a future contributor adds an + // untyped public method like recordStart(attrs: Attributes), + // this assertion still passes but the design intent is broken. + // Make the intent explicit: + val methodNames = classOf[WorkflowMetrics.type].getDeclaredMethods + .map(_.getName) + .toSet + methodNames should contain allOf ("recordStart", "recordCompletion", "recordFailure") + methodNames should not contain "recordWithAttributes" + } + + // ----- histogram buckets are constants -------------------------------- + + it should "use the hard-coded explicit bucket boundaries for duration" in { + WorkflowMetrics.recordStart(WorkflowMetrics.WorkflowKind.Interactive) + // Hit a few bucket bounds. + Seq(0.05, 0.6, 7.0, 65.0, 400.0).foreach { d => + WorkflowMetrics.recordCompletion(WorkflowMetrics.WorkflowKind.Interactive, durationSec = d) + } + + val histogram = collectAll()("texera.workflow.duration").getHistogramData + val point = histogram.getPoints.asScala.head + point.getCount shouldBe 5L + // The point exposes the SDK-configured boundaries; we don't + // assert exact values here (would couple the test to the impl) + // but we do assert there ARE explicit boundaries — anything + // empty would mean the .setExplicitBucketBoundariesAdvice call + // was lost during a refactor. + point.getBoundaries.size should be > 0 + } +} diff --git a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/resource/ComputingUnitManagingResource.scala b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/resource/ComputingUnitManagingResource.scala index a9326e14a44..dd3701080d2 100644 --- a/computing-unit-managing-service/src/main/scala/org/apache/texera/service/resource/ComputingUnitManagingResource.scala +++ b/computing-unit-managing-service/src/main/scala/org/apache/texera/service/resource/ComputingUnitManagingResource.scala @@ -385,6 +385,46 @@ object ComputingUnitManagingResource { lookup(name).map(_.trim).filter(_.nonEmpty).map(name -> _) }.toMap + // Forwarded to a created unit so it emits to the same collector as the rest of + // the deployment. service.instance.id is NOT here: it is built per unit below. + private val observabilityComputingUnitEnvNames: Seq[String] = Seq( + EnvironmentalVariable.ENV_OTEL_SDK_DISABLED, + EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT, + EnvironmentalVariable.ENV_TEXERA_OTEL_ALLOWED_HOSTS, + EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL + ) + + /** + * Observability environment for a created unit. Without this a unit runs with + * the SDK-disabled default even when the manager has telemetry enabled, so a + * worker failure leaves no unit logs, metrics, or traces to investigate. The + * unit inherits the manager's collector endpoint and related settings, but is + * given its own service.instance.id (service.name stays shared) so units with + * the same name do not collapse into one metric series; an instance id the + * manager itself carries is dropped rather than copied. Returns nothing when no + * collector endpoint is configured, leaving the unit at its default-off state. + */ + private[resource] def observabilityComputingUnitEnv( + cuid: Int, + lookup: String => Option[String] + ): Map[String, String] = { + val get = (name: String) => lookup(name).map(_.trim).filter(_.nonEmpty) + if (get(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT).isEmpty) { + Map.empty + } else { + val forwarded = + observabilityComputingUnitEnvNames.flatMap(name => get(name).map(name -> _)).toMap + val kept = get(EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES) + .getOrElse("") + .split(",") + .map(_.trim) + .filter(_.nonEmpty) + .filterNot(_.toLowerCase.startsWith("service.instance.id=")) + val resourceAttrs = (kept :+ s"service.instance.id=computing-unit-$cuid").mkString(",") + forwarded + (EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES -> resourceAttrs) + } + } + // Environment variables passed to the created computing unit(pod) private lazy val computingUnitEnvironmentVariables: Map[String, Any] = icebergEnvironmentVariables ++ Map( @@ -733,7 +773,7 @@ class ComputingUnitManagingResource { computingUnitEnvironmentVariables ++ Map( EnvironmentalVariable.ENV_USER_JWT_TOKEN -> userToken, EnvironmentalVariable.ENV_JAVA_OPTS -> s"-Xmx${param.jvmMemorySize}" - ), + ) ++ observabilityComputingUnitEnv(cuid, EnvironmentalVariable.get), Some(param.shmSize), curatedImage ) diff --git a/computing-unit-managing-service/src/test/scala/org/apache/texera/service/resource/ComputingUnitManagingResourceSpec.scala b/computing-unit-managing-service/src/test/scala/org/apache/texera/service/resource/ComputingUnitManagingResourceSpec.scala index 34b3c996cb3..fffecb0aea9 100644 --- a/computing-unit-managing-service/src/test/scala/org/apache/texera/service/resource/ComputingUnitManagingResourceSpec.scala +++ b/computing-unit-managing-service/src/test/scala/org/apache/texera/service/resource/ComputingUnitManagingResourceSpec.scala @@ -563,4 +563,41 @@ class ComputingUnitManagingResourceSpec ComputingUnitManagingResource.optionalComputingUnitEnv(_ => Some(" ")) shouldBe empty } + "observabilityComputingUnitEnv" should "forward collector settings and give the unit its own instance id when enabled" in { + val enabled = Map( + EnvironmentalVariable.ENV_OTEL_SDK_DISABLED -> "false", + EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT -> "http://texera-otel-collector:4317", + EnvironmentalVariable.ENV_TEXERA_OTEL_ALLOWED_HOSTS -> "texera-otel-collector", + EnvironmentalVariable.ENV_OTEL_METRIC_EXPORT_INTERVAL -> "30000" + ) + val env = ComputingUnitManagingResource.observabilityComputingUnitEnv(7, enabled.get) + env(EnvironmentalVariable.ENV_OTEL_SDK_DISABLED) shouldBe "false" + env(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT) shouldBe + "http://texera-otel-collector:4317" + env(EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES) shouldBe + "service.instance.id=computing-unit-7" + } + + // Each unit must be distinguishable, so a unit never inherits the manager's instance id. + it should "replace an inherited service.instance.id with the unit's own, keeping other attributes" in { + val lookup: String => Option[String] = Map( + EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT -> "http://texera-otel-collector:4317", + EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES -> + "deployment.environment=staging,service.instance.id=manager,service.namespace=texera" + ).get + val attrs = ComputingUnitManagingResource + .observabilityComputingUnitEnv(8, lookup)(EnvironmentalVariable.ENV_OTEL_RESOURCE_ATTRIBUTES) + attrs should include("deployment.environment=staging") + attrs should include("service.namespace=texera") + attrs should include("service.instance.id=computing-unit-8") + attrs should not include "service.instance.id=manager" + } + + // Telemetry stays opt-in: with no collector endpoint the unit is left default-off. + it should "forward nothing when no collector endpoint is configured" in { + ComputingUnitManagingResource.observabilityComputingUnitEnv(1, _ => None) shouldBe empty + val blankEndpoint = Map(EnvironmentalVariable.ENV_OTEL_EXPORTER_OTLP_ENDPOINT -> " ") + ComputingUnitManagingResource.observabilityComputingUnitEnv(1, blankEndpoint.get) shouldBe empty + } + }