From fcbfad04bbd30d972ba99662ea43eff0f9e5969c Mon Sep 17 00:00:00 2001 From: Alex Cruise Date: Thu, 30 Jul 2026 12:18:04 -0700 Subject: [PATCH 1/4] KNOX-3402 [wip]: Spark Connect gateway support --- LICENSE | 37 + .../build-tools/checkstyle/suppressions.xml | 3 + .../resources/build-tools/spotbugs-filter.xml | 6 + .../federation/jwt}/JWTValidator.java | 3 +- gateway-release/pom.xml | 4 + .../apache/knox/gateway/GatewayMessages.java | 9 + .../apache/knox/gateway/GatewayServer.java | 50 + .../config/impl/GatewayConfigImpl.java | 99 ++ .../webshell/WebshellWebSocketAdapter.java | 2 +- .../websockets/GatewayWebsocketHandler.java | 1 + .../websockets/JWTValidatorFactory.java | 1 + .../WebshellWebsocketAdapterTest.java | 2 +- .../GatewayWebsocketHandlerTest.java | 1 + .../gateway/websockets/JWTValidatorTest.java | 1 + .../services/sparkconnect/1.0.0/service.xml | 43 + gateway-service-sparkconnect/pom.xml | 183 +++ .../knox/gateway/grpc/AclAuthorizer.java | 176 +++ .../knox/gateway/grpc/AuditInterceptor.java | 122 ++ .../knox/gateway/grpc/AuthenticatedUser.java | 48 + .../grpc/AuthenticationInterceptor.java | 89 ++ .../grpc/AuthorizationInterceptor.java | 156 ++ .../gateway/grpc/BackendChannelCache.java | 172 +++ .../gateway/grpc/BackendChannelProvider.java | 39 + .../gateway/grpc/BackendHeaderRewriter.java | 82 + .../gateway/grpc/ByteArrayMarshaller.java | 67 + .../knox/gateway/grpc/GrpcCallContext.java | 147 ++ .../gateway/grpc/GrpcGatewayListener.java | 320 ++++ .../gateway/grpc/GrpcGatewayMessages.java | 84 + .../gateway/grpc/GrpcListenerSettings.java | 125 ++ .../knox/gateway/grpc/GrpcMetadataKeys.java | 45 + .../knox/gateway/grpc/HeaderRewriter.java | 39 + .../knox/gateway/grpc/InterceptorChain.java | 59 + .../knox/gateway/grpc/MapFilterConfig.java | 65 + .../knox/gateway/grpc/MessageInterceptor.java | 66 + .../grpc/PassthroughHandlerRegistry.java | 87 ++ .../knox/gateway/grpc/ProxyCallHandler.java | 235 +++ .../knox/gateway/grpc/RoutingInterceptor.java | 115 ++ .../knox/gateway/grpc/TokenAuthenticator.java | 196 +++ .../sparkconnect/AddArtifactsGuard.java | 90 ++ .../sparkconnect/ReservedConfigGuard.java | 123 ++ .../sparkconnect/SparkConnectListener.java | 167 ++ .../SparkConnectMessageInterceptor.java | 164 ++ .../src/main/proto/spark/connect/base.proto | 1367 +++++++++++++++++ .../main/proto/spark/connect/catalog.proto | 324 ++++ .../main/proto/spark/connect/commands.proto | 560 +++++++ .../src/main/proto/spark/connect/common.proto | 179 +++ .../proto/spark/connect/expressions.proto | 557 +++++++ .../src/main/proto/spark/connect/ml.proto | 147 ++ .../main/proto/spark/connect/ml_common.proto | 64 + .../main/proto/spark/connect/pipelines.proto | 385 +++++ .../main/proto/spark/connect/relations.proto | 1309 ++++++++++++++++ .../src/main/proto/spark/connect/types.proto | 227 +++ ...che.knox.gateway.protocol.ProtocolListener | 18 + .../knox/gateway/grpc/AclAuthorizerTest.java | 153 ++ .../sparkconnect/AddArtifactsGuardTest.java | 89 ++ .../sparkconnect/ReservedConfigGuardTest.java | 110 ++ .../SparkConnectMessageInterceptorTest.java | 209 +++ .../SparkConnectProxyIntegrationTest.java | 351 +++++ .../knox/gateway/GatewayTestConfig.java | 106 ++ .../knox/gateway/config/GatewayConfig.java | 113 ++ .../gateway/protocol/ProtocolListener.java | 94 ++ knox-site/docs/spark-connect-support.md | 233 +++ knox-site/mkdocs.yml | 1 + pom.xml | 22 + 64 files changed, 10137 insertions(+), 4 deletions(-) rename {gateway-server/src/main/java/org/apache/knox/gateway/websockets => gateway-provider-security-jwt/src/main/java/org/apache/knox/gateway/provider/federation/jwt}/JWTValidator.java (98%) create mode 100644 gateway-service-definitions/src/main/resources/services/sparkconnect/1.0.0/service.xml create mode 100644 gateway-service-sparkconnect/pom.xml create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AclAuthorizer.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuditInterceptor.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticatedUser.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthorizationInterceptor.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelCache.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelProvider.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ByteArrayMarshaller.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcCallContext.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayListener.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/HeaderRewriter.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/InterceptorChain.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MapFilterConfig.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptor.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/PassthroughHandlerRegistry.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ProxyCallHandler.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/TokenAuthenticator.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuard.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuard.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectListener.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptor.java create mode 100644 gateway-service-sparkconnect/src/main/proto/spark/connect/base.proto create mode 100644 gateway-service-sparkconnect/src/main/proto/spark/connect/catalog.proto create mode 100644 gateway-service-sparkconnect/src/main/proto/spark/connect/commands.proto create mode 100644 gateway-service-sparkconnect/src/main/proto/spark/connect/common.proto create mode 100644 gateway-service-sparkconnect/src/main/proto/spark/connect/expressions.proto create mode 100644 gateway-service-sparkconnect/src/main/proto/spark/connect/ml.proto create mode 100644 gateway-service-sparkconnect/src/main/proto/spark/connect/ml_common.proto create mode 100644 gateway-service-sparkconnect/src/main/proto/spark/connect/pipelines.proto create mode 100644 gateway-service-sparkconnect/src/main/proto/spark/connect/relations.proto create mode 100644 gateway-service-sparkconnect/src/main/proto/spark/connect/types.proto create mode 100644 gateway-service-sparkconnect/src/main/resources/META-INF/services/org.apache.knox.gateway.protocol.ProtocolListener create mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AclAuthorizerTest.java create mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuardTest.java create mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuardTest.java create mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptorTest.java create mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectProxyIntegrationTest.java create mode 100644 gateway-spi/src/main/java/org/apache/knox/gateway/protocol/ProtocolListener.java create mode 100644 knox-site/docs/spark-connect-support.md diff --git a/LICENSE b/LICENSE index 1a7827712d..eb40047c15 100644 --- a/LICENSE +++ b/LICENSE @@ -1379,3 +1379,40 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------ +Protocol Buffers License (BSD 3-clause) (from Spark Connect support) +------------------------------------------------------------------------------ + +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/build-tools/src/main/resources/build-tools/checkstyle/suppressions.xml b/build-tools/src/main/resources/build-tools/checkstyle/suppressions.xml index 43f7677259..f01a5e1fa1 100644 --- a/build-tools/src/main/resources/build-tools/checkstyle/suppressions.xml +++ b/build-tools/src/main/resources/build-tools/checkstyle/suppressions.xml @@ -24,4 +24,7 @@ limitations under the License. + + + \ No newline at end of file diff --git a/build-tools/src/main/resources/build-tools/spotbugs-filter.xml b/build-tools/src/main/resources/build-tools/spotbugs-filter.xml index fb4d7857d4..bb137395cd 100644 --- a/build-tools/src/main/resources/build-tools/spotbugs-filter.xml +++ b/build-tools/src/main/resources/build-tools/spotbugs-filter.xml @@ -85,4 +85,10 @@ limitations under the License. + + + + + diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/websockets/JWTValidator.java b/gateway-provider-security-jwt/src/main/java/org/apache/knox/gateway/provider/federation/jwt/JWTValidator.java similarity index 98% rename from gateway-server/src/main/java/org/apache/knox/gateway/websockets/JWTValidator.java rename to gateway-provider-security-jwt/src/main/java/org/apache/knox/gateway/provider/federation/jwt/JWTValidator.java index 34fd050664..6990cb5172 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/websockets/JWTValidator.java +++ b/gateway-provider-security-jwt/src/main/java/org/apache/knox/gateway/provider/federation/jwt/JWTValidator.java @@ -15,11 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.knox.gateway.websockets; +package org.apache.knox.gateway.provider.federation.jwt; import com.nimbusds.jose.JWSHeader; import org.apache.knox.gateway.i18n.messages.MessagesFactory; -import org.apache.knox.gateway.provider.federation.jwt.JWTMessages; import org.apache.knox.gateway.provider.federation.jwt.filter.SignatureVerificationCache; import org.apache.knox.gateway.services.security.token.JWTokenAuthority; import org.apache.knox.gateway.services.security.token.TokenMetadata; diff --git a/gateway-release/pom.xml b/gateway-release/pom.xml index 7df1c6c224..06bb8d0e4f 100644 --- a/gateway-release/pom.xml +++ b/gateway-release/pom.xml @@ -524,5 +524,9 @@ org.apache.knox gateway-service-restcatalog + + org.apache.knox + gateway-service-sparkconnect + diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java index d4639b0114..03bffeafca 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java @@ -57,6 +57,15 @@ public interface GatewayMessages { @Message( level = MessageLevel.INFO, text = "Failed to stopped gateway." ) void failedToStopGateway(@StackTrace( level = MessageLevel.INFO ) Exception e); + @Message( level = MessageLevel.INFO, text = "Started the {0} protocol listener on port {1}." ) + void startedProtocolListener( String name, String port ); + + @Message( level = MessageLevel.FATAL, text = "Failed to start the {0} protocol listener: {1}" ) + void failedToStartProtocolListener( String name, @StackTrace( level = MessageLevel.FATAL ) Exception e ); + + @Message( level = MessageLevel.WARN, text = "Failed to stop the {0} protocol listener: {1}" ) + void failedToStopProtocolListener( String name, @StackTrace( level = MessageLevel.WARN ) Exception e ); + @Message( level = MessageLevel.INFO, text = "Loading configuration resource {0}" ) void loadingConfigurationResource( String res ); diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java index c5db37cc9c..14cb7b9196 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java @@ -38,6 +38,7 @@ import org.apache.knox.gateway.filter.PortMappingHelperHandler; import org.apache.knox.gateway.i18n.messages.MessagesFactory; import org.apache.knox.gateway.i18n.resources.ResourcesFactory; +import org.apache.knox.gateway.protocol.ProtocolListener; import org.apache.knox.gateway.services.GatewayServices; import org.apache.knox.gateway.services.ServiceType; import org.apache.knox.gateway.services.registry.ServiceDefinitionRegistry; @@ -166,6 +167,13 @@ public class GatewayServer { private AtomicBoolean stopped = new AtomicBoolean(false); private GatewayStatusService gatewayStatusService; + /** + * Listeners for protocols the servlet pipeline cannot carry, each on its own + * port. Discovered with ServiceLoader so the server keeps no compile-time + * dependency on their transport libraries. + */ + private final List protocolListeners = new ArrayList<>(); + private final Set inactiveTopologies = new HashSet<>(); public static void main( String[] args ) { @@ -765,6 +773,10 @@ private synchronized void start() throws Exception { cleanupTopologyDeployments(); + // Started after Jetty and after topologies are deployed, so a listener can + // resolve backends from the service registry as it comes up. + startProtocolListeners(); + // Start the topology monitor. monitor.startMonitor(); @@ -799,6 +811,41 @@ void createJetty() throws IOException, CertificateException, NoSuchAlgorithmExce } } + /** + * Starts every enabled protocol listener on the classpath. + *

+ * A listener that fails to start fails the gateway, the same as a Jetty + * connector would: a deployment that asked for a listener and silently did not + * get one is worse than one that refuses to come up. + */ + private void startProtocolListeners() throws Exception { + for (ProtocolListener listener : ServiceLoader.load(ProtocolListener.class)) { + if (!listener.isEnabled(config)) { + continue; + } + try { + listener.start(config, services); + protocolListeners.add(listener); + log.startedProtocolListener(listener.getName(), convertPortToString(listener.getPort())); + } catch (Exception e) { + log.failedToStartProtocolListener(listener.getName(), e); + throw e; + } + } + } + + private void stopProtocolListeners() { + for (ProtocolListener listener : protocolListeners) { + try { + listener.stop(); + } catch (Exception e) { + // One listener refusing to stop must not keep the rest of the gateway up. + log.failedToStopProtocolListener(listener.getName(), e); + } + } + protocolListeners.clear(); + } + private void handleHadoopXmlResources() { final HadoopXmlResourceParser hadoopXmlResourceParser = new HadoopXmlResourceParser(config); final HadoopXmlResourceMonitor hadoopXmlResourceMonitor = new HadoopXmlResourceMonitor(config, hadoopXmlResourceParser); @@ -811,6 +858,9 @@ public synchronized void stop() throws Exception { log.stoppingGateway(); services.stop(); monitor.stopMonitor(); + // Drain before Jetty stops: long-lived streams get a bounded window to + // finish rather than being cut the moment shutdown begins. + stopProtocolListeners(); jetty.stop(); jetty.join(); log.stoppedGateway(); diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java index 061518537d..08dd19f251 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java @@ -164,6 +164,21 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { public static final String WEBSOCKET_IDLE_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".websocket.idle.timeout"; public static final String WEBSOCKET_MAX_WAIT_BUFFER_COUNT = GATEWAY_CONFIG_FILE_PREFIX + ".websocket.max.wait.buffer.count"; + /* @since 3.0.0 Spark Connect (gRPC) listener config variables */ + public static final String SPARKCONNECT_FEATURE_ENABLED = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.enabled"; + public static final String SPARKCONNECT_PORT = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.port"; + public static final String SPARKCONNECT_DEFAULT_TOPOLOGY = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.default.topology"; + public static final String SPARKCONNECT_MAX_MESSAGE_SIZE = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.max.message.size"; + public static final String SPARKCONNECT_PERMIT_KEEPALIVE_TIME = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.permit.keepalive.time"; + public static final String SPARKCONNECT_PERMIT_KEEPALIVE_WITHOUT_CALLS = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.permit.keepalive.without.calls"; + public static final String SPARKCONNECT_MAX_CONCURRENT_CALLS_PER_CONNECTION = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.max.concurrent.calls.per.connection"; + public static final String SPARKCONNECT_CHANNEL_IDLE_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.channel.idle.timeout"; + public static final String SPARKCONNECT_DRAIN_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.drain.timeout"; + public static final String SPARKCONNECT_BACKEND_TOKEN_ALIAS = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.backend.token.alias"; + public static final String SPARKCONNECT_ADD_ARTIFACTS_MODE = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.add.artifacts.mode"; + public static final String SPARKCONNECT_ADD_ARTIFACTS_ALLOWED_USERS = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.add.artifacts.allowed.users"; + public static final String SPARKCONNECT_RESERVED_CONFIG_PREFIX = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.reserved.config.prefix"; + /* @since 2.0.0 WebShell config variables */ public static final String WEBSHELL_FEATURE_ENABLED = GATEWAY_CONFIG_FILE_PREFIX + ".webshell.feature.enabled"; @@ -227,6 +242,21 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { public static final int DEFAULT_WEBSOCKET_IDLE_TIMEOUT = 300000; public static final int DEFAULT_WEBSOCKET_MAX_WAIT_BUFFER_COUNT = 100; + /* Spark Connect defaults */ + public static final boolean DEFAULT_SPARKCONNECT_FEATURE_ENABLED = false; + /* The port the Spark Connect server itself listens on; clients default to it too. */ + public static final int DEFAULT_SPARKCONNECT_PORT = 15002; + /* Matches Spark's own 128 MB default. */ + public static final int DEFAULT_SPARKCONNECT_MAX_MESSAGE_SIZE = 134217728; + /* grpc-java's server-side floor; clients ping every 60s by default. */ + public static final long DEFAULT_SPARKCONNECT_PERMIT_KEEPALIVE_TIME = 10000L; + public static final boolean DEFAULT_SPARKCONNECT_PERMIT_KEEPALIVE_WITHOUT_CALLS = true; + public static final int DEFAULT_SPARKCONNECT_MAX_CONCURRENT_CALLS_PER_CONNECTION = 1000; + public static final long DEFAULT_SPARKCONNECT_CHANNEL_IDLE_TIMEOUT = 1800000L; + public static final long DEFAULT_SPARKCONNECT_DRAIN_TIMEOUT = 30000L; + public static final String DEFAULT_SPARKCONNECT_ADD_ARTIFACTS_MODE = "ALLOW"; + public static final String DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX = "knox."; + public static final boolean DEFAULT_WEBSHELL_FEATURE_ENABLED = false; public static final boolean DEFAULT_WEBSHELL_AUDIT_LOGGING_ENABLED = false; public static final int DEFAULT_WEBSHELL_MAX_CONCURRENT_SESSIONS = 3; @@ -1114,6 +1144,75 @@ public int getWebsocketMaxWaitBufferCount() { return getInt( WEBSOCKET_MAX_WAIT_BUFFER_COUNT, DEFAULT_WEBSOCKET_MAX_WAIT_BUFFER_COUNT); } + @Override + public boolean isSparkConnectEnabled() { + return getBoolean(SPARKCONNECT_FEATURE_ENABLED, DEFAULT_SPARKCONNECT_FEATURE_ENABLED); + } + + @Override + public int getSparkConnectPort() { + return getInt(SPARKCONNECT_PORT, DEFAULT_SPARKCONNECT_PORT); + } + + @Override + public String getSparkConnectDefaultTopology() { + return get(SPARKCONNECT_DEFAULT_TOPOLOGY); + } + + @Override + public int getSparkConnectMaxMessageSize() { + return getInt(SPARKCONNECT_MAX_MESSAGE_SIZE, DEFAULT_SPARKCONNECT_MAX_MESSAGE_SIZE); + } + + @Override + public long getSparkConnectPermitKeepAliveTime() { + return getLong(SPARKCONNECT_PERMIT_KEEPALIVE_TIME, DEFAULT_SPARKCONNECT_PERMIT_KEEPALIVE_TIME); + } + + @Override + public boolean isSparkConnectPermitKeepAliveWithoutCalls() { + return getBoolean(SPARKCONNECT_PERMIT_KEEPALIVE_WITHOUT_CALLS, DEFAULT_SPARKCONNECT_PERMIT_KEEPALIVE_WITHOUT_CALLS); + } + + @Override + public int getSparkConnectMaxConcurrentCallsPerConnection() { + return getInt(SPARKCONNECT_MAX_CONCURRENT_CALLS_PER_CONNECTION, DEFAULT_SPARKCONNECT_MAX_CONCURRENT_CALLS_PER_CONNECTION); + } + + @Override + public long getSparkConnectChannelIdleTimeout() { + return getLong(SPARKCONNECT_CHANNEL_IDLE_TIMEOUT, DEFAULT_SPARKCONNECT_CHANNEL_IDLE_TIMEOUT); + } + + @Override + public long getSparkConnectDrainTimeout() { + return getLong(SPARKCONNECT_DRAIN_TIMEOUT, DEFAULT_SPARKCONNECT_DRAIN_TIMEOUT); + } + + @Override + public String getSparkConnectBackendTokenAlias() { + return get(SPARKCONNECT_BACKEND_TOKEN_ALIAS); + } + + @Override + public String getSparkConnectAddArtifactsMode() { + return get(SPARKCONNECT_ADD_ARTIFACTS_MODE, DEFAULT_SPARKCONNECT_ADD_ARTIFACTS_MODE); + } + + @Override + public List getSparkConnectAddArtifactsAllowedUsers() { + final String value = get(SPARKCONNECT_ADD_ARTIFACTS_ALLOWED_USERS); + if (value == null || value.trim().isEmpty()) { + return Collections.emptyList(); + } + return Arrays.asList(value.trim().split("\\s*,\\s*")); + } + + @Override + public String getSparkConnectReservedConfigPrefix() { + return get(SPARKCONNECT_RESERVED_CONFIG_PREFIX, DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX); + } + @Override public Map getGatewayPortMappings() { diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/webshell/WebshellWebSocketAdapter.java b/gateway-server/src/main/java/org/apache/knox/gateway/webshell/WebshellWebSocketAdapter.java index 44fb8d1a6b..6e6404b025 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/webshell/WebshellWebSocketAdapter.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/webshell/WebshellWebSocketAdapter.java @@ -33,7 +33,7 @@ import org.apache.knox.gateway.audit.log4j.audit.AuditConstants; import org.apache.knox.gateway.config.GatewayConfig; import org.apache.knox.gateway.services.security.token.UnknownTokenException; -import org.apache.knox.gateway.websockets.JWTValidator; +import org.apache.knox.gateway.provider.federation.jwt.JWTValidator; import org.apache.knox.gateway.websockets.ProxyWebSocketAdapter; import org.eclipse.jetty.websocket.api.Session; diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/websockets/GatewayWebsocketHandler.java b/gateway-server/src/main/java/org/apache/knox/gateway/websockets/GatewayWebsocketHandler.java index f275ee9eeb..0295178557 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/websockets/GatewayWebsocketHandler.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/websockets/GatewayWebsocketHandler.java @@ -20,6 +20,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.knox.gateway.config.GatewayConfig; import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.provider.federation.jwt.JWTValidator; import org.apache.knox.gateway.services.GatewayServices; import org.apache.knox.gateway.services.ServiceType; import org.apache.knox.gateway.services.registry.ServiceDefEntry; diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/websockets/JWTValidatorFactory.java b/gateway-server/src/main/java/org/apache/knox/gateway/websockets/JWTValidatorFactory.java index 9cb93bd9a2..65cf9752ee 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/websockets/JWTValidatorFactory.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/websockets/JWTValidatorFactory.java @@ -20,6 +20,7 @@ import org.apache.knox.gateway.config.GatewayConfig; import org.apache.knox.gateway.i18n.messages.MessagesFactory; import org.apache.knox.gateway.provider.federation.jwt.JWTMessages; +import org.apache.knox.gateway.provider.federation.jwt.JWTValidator; import org.apache.knox.gateway.provider.federation.jwt.filter.SignatureVerificationCache; import org.apache.knox.gateway.services.GatewayServices; import org.apache.knox.gateway.services.ServiceType; diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/webshell/WebshellWebsocketAdapterTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/webshell/WebshellWebsocketAdapterTest.java index 61e4446ff8..7dec1f35ba 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/webshell/WebshellWebsocketAdapterTest.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/webshell/WebshellWebsocketAdapterTest.java @@ -25,7 +25,7 @@ import org.apache.knox.gateway.config.GatewayConfig; import org.apache.knox.gateway.i18n.messages.MessagesFactory; import org.apache.knox.gateway.provider.federation.jwt.JWTMessages; -import org.apache.knox.gateway.websockets.JWTValidator; +import org.apache.knox.gateway.provider.federation.jwt.JWTValidator; import org.apache.knox.gateway.websockets.WebsocketLogMessages; import org.easymock.EasyMock; import org.easymock.EasyMockSupport; diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/websockets/GatewayWebsocketHandlerTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/websockets/GatewayWebsocketHandlerTest.java index 47331b8071..9ff38b1e5d 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/websockets/GatewayWebsocketHandlerTest.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/websockets/GatewayWebsocketHandlerTest.java @@ -26,6 +26,7 @@ import org.apache.knox.gateway.i18n.GatewaySpiMessages; import org.apache.knox.gateway.i18n.messages.MessagesFactory; import org.apache.knox.gateway.provider.federation.jwt.JWTMessages; +import org.apache.knox.gateway.provider.federation.jwt.JWTValidator; import org.apache.knox.gateway.services.GatewayServices; import org.apache.knox.gateway.webshell.WebshellWebSocketAdapter; import org.easymock.EasyMock; diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/websockets/JWTValidatorTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/websockets/JWTValidatorTest.java index e79bea6d7c..f539c4e7ea 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/websockets/JWTValidatorTest.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/websockets/JWTValidatorTest.java @@ -26,6 +26,7 @@ import com.nimbusds.jwt.SignedJWT; import org.apache.commons.codec.binary.Base64; import org.apache.knox.gateway.config.GatewayConfig; +import org.apache.knox.gateway.provider.federation.jwt.JWTValidator; import org.apache.knox.gateway.provider.federation.jwt.filter.SignatureVerificationCache; import org.apache.knox.gateway.services.GatewayServices; import org.apache.knox.gateway.services.ServiceType; diff --git a/gateway-service-definitions/src/main/resources/services/sparkconnect/1.0.0/service.xml b/gateway-service-definitions/src/main/resources/services/sparkconnect/1.0.0/service.xml new file mode 100644 index 0000000000..5a101dbf02 --- /dev/null +++ b/gateway-service-definitions/src/main/resources/services/sparkconnect/1.0.0/service.xml @@ -0,0 +1,43 @@ + + + + + + API + /sparkconnect + Spark Connect + Apache Spark Connect gRPC endpoint, proxied by the Knox Spark Connect listener on its own port. + + diff --git a/gateway-service-sparkconnect/pom.xml b/gateway-service-sparkconnect/pom.xml new file mode 100644 index 0000000000..48cdbdbcda --- /dev/null +++ b/gateway-service-sparkconnect/pom.xml @@ -0,0 +1,183 @@ + + + + 4.0.0 + + org.apache.knox + gateway + 3.0.0-SNAPSHOT + + + gateway-service-sparkconnect + gateway-service-sparkconnect + Spark Connect (gRPC) gateway listener for Apache Knox + + + + org.apache.knox + gateway-spi + compile + + + org.apache.knox + gateway-i18n + + + + org.apache.knox + gateway-util-common + + + + org.apache.knox + gateway-provider-security-jwt + + + + javax.servlet + javax.servlet-api + provided + + + + org.apache.knox + gateway-provider-security-authz-acls + + + + io.grpc + grpc-api + + + io.grpc + grpc-stub + + + io.grpc + grpc-protobuf + + + + io.grpc + grpc-netty-shaded + + + com.google.protobuf + protobuf-java + + + + com.google.guava + guava + + + org.apache.commons + commons-lang3 + + + + io.grpc + grpc-inprocess + test + + + io.grpc + grpc-testing + test + + + junit + junit + test + + + org.easymock + easymock + test + + + org.hamcrest + hamcrest + test + + + org.apache.knox + gateway-test-utils + test + + + + + + + + kr.motd.maven + os-maven-plugin + ${os-maven-plugin.version} + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + + + + + compile + compile-custom + + + + + + org.apache.maven.plugins + maven-pmd-plugin + + + + ${project.build.directory}/generated-sources/protobuf/java + ${project.build.directory}/generated-sources/protobuf/grpc-java + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + ${project.build.sourceDirectory} + + + + + + diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AclAuthorizer.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AclAuthorizer.java new file mode 100644 index 0000000000..bbf3adf0bc --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AclAuthorizer.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.knox.gateway.filter.AclParser; +import org.apache.knox.gateway.filter.InvalidACLException; + +/** + * Evaluates a topology's {@code AclsAuthz} ACLs for a gRPC call. + *

+ * Knox's authorization responsibility on this path is deliberately one question: + * may this user use this service in this topology at all? Fine-grained + * authorization — databases, tables, columns, row filters, masking — belongs to + * Ranger policy evaluated inside the Spark Connect server against the identity + * Knox asserts, and is not something a gateway can usefully duplicate. + *

+ * The syntax and semantics are the servlet provider's, down to sharing its + * {@link AclParser}: {@code users;groups;ipaddresses}, an {@code AND}/{@code OR} + * processing mode, {@code *} wildcards, and the {@code KNOX_ADMIN_USERS} / + * {@code KNOX_ADMIN_GROUPS} placeholders. Operators should not have to learn a + * second ACL dialect because the transport changed. + */ +public class AclAuthorizer { + + private static final String ACL_SUFFIX = ".acl"; + private static final String ACL_MODE_SUFFIX = ".acl.mode"; + private static final String DEFAULT_ACL_MODE = "AND"; + private static final String KNOX_ADMIN_USERS_PLACEHOLDER = "KNOX_ADMIN_USERS"; + private static final String KNOX_ADMIN_GROUPS_PLACEHOLDER = "KNOX_ADMIN_GROUPS"; + + private final AclParser parser = new AclParser(); + private final String aclProcessingMode; + private final Set adminUsers; + private final Set adminGroups; + private final boolean unrestricted; + + /** + * Builds an authorizer for one resource role from a topology's provider + * parameters. + * + * @param resourceRole the service role the ACLs apply to, e.g. {@code SPARKCONNECT} + * @param providerParams the {@code AclsAuthz} provider parameters, or null if the + * topology declares no such provider + * @param knoxAdminUsers comma-separated admin users from gateway configuration + * @param knoxAdminGroups comma-separated admin groups from gateway configuration + * @throws InvalidACLException if a configured ACL is malformed + */ + public AclAuthorizer(String resourceRole, + Map providerParams, + String knoxAdminUsers, + String knoxAdminGroups) throws InvalidACLException { + // Provider params become filter params lowercased on the servlet path, and + // the filter looks them up that way; match it so the same topology XML works. + final Map params = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + if (providerParams != null) { + params.putAll(providerParams); + } + + String mode = params.get(resourceRole + ACL_MODE_SUFFIX); + if (mode == null) { + mode = params.get("acl.mode"); + } + this.aclProcessingMode = mode == null ? DEFAULT_ACL_MODE : mode.toUpperCase(Locale.ROOT); + + final String acls = params.get(resourceRole + ACL_SUFFIX); + parser.parseAcls(resourceRole, acls); + + this.adminUsers = split(knoxAdminUsers); + this.adminGroups = split(knoxAdminGroups); + + // No ACLs configured at all means no restrictions, matching the servlet + // provider: a topology that never mentions this role does not silently deny. + this.unrestricted = parser.users.isEmpty() && parser.groups.isEmpty() + && parser.ipv.getIPAddresses().isEmpty(); + } + + private static Set split(String csv) { + if (csv == null || csv.trim().isEmpty()) { + return Collections.emptySet(); + } + return new HashSet<>(Arrays.asList(csv.trim().split("\\s*,\\s*"))); + } + + /** + * Decides whether a call is permitted. + * + * @param user the authenticated principal + * @param groups the principal's groups, possibly empty + * @param remoteAddress the client's IP address, or null if unavailable + * @return true if the call may proceed + */ + public boolean isPermitted(String user, Set groups, String remoteAddress) { + if (unrestricted) { + return true; + } + + boolean userAccess = checkUser(user); + boolean groupAccess = checkGroups(groups); + boolean ipAccess = remoteAddress != null && parser.ipv.validateIpAddress(remoteAddress); + + if ("OR".equals(aclProcessingMode)) { + // Under OR, a wildcard has to read as "not a reason to grant" — otherwise a + // single '*' in any position would admit everyone. + if (parser.anyUser) { + userAccess = false; + } + if (parser.anyGroup) { + groupAccess = false; + } + if (parser.ipv.allowsAnyIP()) { + ipAccess = false; + } + return userAccess || groupAccess || ipAccess; + } + if ("AND".equals(aclProcessingMode)) { + return userAccess && groupAccess && ipAccess; + } + return false; + } + + private boolean checkUser(String user) { + if (user == null) { + return false; + } + if (parser.anyUser) { + return true; + } + if (parser.users.contains(user)) { + return true; + } + return parser.users.contains(KNOX_ADMIN_USERS_PLACEHOLDER) && adminUsers.contains(user); + } + + private boolean checkGroups(Set groups) { + if (groups == null || groups.isEmpty()) { + // A subject with no groups can still satisfy an AND policy whose group + // position is a wildcard, e.g. '*;*;127.0.0.*'. + return parser.anyGroup && "AND".equals(aclProcessingMode); + } + if (parser.anyGroup) { + return true; + } + for (String group : groups) { + if (parser.groups.contains(group)) { + return true; + } + if (parser.groups.contains(KNOX_ADMIN_GROUPS_PLACEHOLDER) && adminGroups.contains(group)) { + return true; + } + } + return false; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuditInterceptor.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuditInterceptor.java new file mode 100644 index 0000000000..5358ca7942 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuditInterceptor.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.concurrent.TimeUnit; + +import org.apache.knox.gateway.audit.api.Action; +import org.apache.knox.gateway.audit.api.ActionOutcome; +import org.apache.knox.gateway.audit.api.AuditService; +import org.apache.knox.gateway.audit.api.AuditServiceFactory; +import org.apache.knox.gateway.audit.api.Auditor; +import org.apache.knox.gateway.audit.api.ResourceType; +import org.apache.knox.gateway.audit.log4j.audit.AuditConstants; + +import io.grpc.Context; +import io.grpc.Contexts; +import io.grpc.ForwardingServerCall; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * Creates the per-call context and writes one audit record per RPC. + *

+ * This is the outermost interceptor. It runs first so that every call — including + * ones rejected for a bad token or a failed ACL check — gets a record, and it + * observes the outcome last, once the inner interceptors have filled in whatever + * they resolved. The record therefore reports the principal and topology even on + * paths where the call never reached a backend. + *

+ * A shared mutable {@link GrpcCallContext} is what makes that possible: gRPC + * context values set by an inner interceptor are not visible to an outer one, so + * the state the inner stages establish has to live in an object this interceptor + * created and attached before delegating. + */ +public class AuditInterceptor implements ServerInterceptor { + + private static final AuditService AUDIT_SERVICE = AuditServiceFactory.getAuditService(); + private static final Auditor AUDITOR = AuditServiceFactory.getAuditService() + .getAuditor(AuditConstants.DEFAULT_AUDITOR_NAME, + AuditConstants.KNOX_SERVICE_NAME, + AuditConstants.KNOX_COMPONENT_NAME); + + @Override + public ServerCall.Listener interceptCall(ServerCall call, + Metadata headers, + ServerCallHandler next) { + final String method = call.getMethodDescriptor().getFullMethodName(); + final GrpcCallContext callContext = new GrpcCallContext( + method, + call.getAuthority(), + AuthorizationInterceptor.remoteAddressOf(call), + System.nanoTime()); + + final ServerCall auditedCall = + new ForwardingServerCall.SimpleForwardingServerCall(call) { + @Override + public void close(Status status, Metadata trailers) { + try { + audit(callContext, status); + } finally { + super.close(status, trailers); + } + } + }; + + final Context grpcContext = Context.current().withValue(GrpcCallContext.KEY, callContext); + return Contexts.interceptCall(grpcContext, auditedCall, headers, next); + } + + private void audit(GrpcCallContext callContext, Status status) { + final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - callContext.getStartNanos()); + final String outcome = status.isOk() ? ActionOutcome.SUCCESS : ActionOutcome.FAILURE; + + final StringBuilder message = new StringBuilder(160); + message.append("status=").append(status.getCode()) + .append(", topology=").append(nullSafe(callContext.getTopology())) + .append(", backend=").append(nullSafe(callContext.getBackendUrl())) + .append(", remoteAddress=").append(nullSafe(callContext.getRemoteAddress())) + .append(", authority=").append(nullSafe(callContext.getAuthority())) + .append(", durationMs=").append(millis); + // Populated only on the proto-aware path; a byte-level proxy cannot know them. + if (callContext.getSessionId() != null) { + message.append(", sessionId=").append(callContext.getSessionId()); + } + if (callContext.getOperationId() != null) { + message.append(", operationId=").append(callContext.getOperationId()); + } + + AUDIT_SERVICE.createContext(); + try { + if (callContext.getPrincipal() != null) { + AUDIT_SERVICE.getContext().setUsername(callContext.getPrincipal()); + } + AUDITOR.audit(Action.ACCESS, callContext.getMethodName(), ResourceType.URI, outcome, + message.toString()); + } finally { + AUDIT_SERVICE.detachContext(); + } + } + + private static String nullSafe(String value) { + return value == null ? "-" : value; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticatedUser.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticatedUser.java new file mode 100644 index 0000000000..81ee2081fe --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticatedUser.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.Collections; +import java.util.Set; + +/** + * The identity a validated bearer token established. + *

+ * Groups come from the token's {@code knox.groups} claim when the deployment + * configures {@code knoxtoken} to embed them. That keeps authorization decisions + * free of a per-RPC group lookup, which suits a credential that is already a + * point-in-time delegation of the user's identity. + */ +public class AuthenticatedUser { + + private final String principal; + private final Set groups; + + public AuthenticatedUser(String principal, Set groups) { + this.principal = principal; + this.groups = groups == null ? Collections.emptySet() : Collections.unmodifiableSet(groups); + } + + public String getPrincipal() { + return principal; + } + + public Set getGroups() { + return groups; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java new file mode 100644 index 0000000000..9ecd8bc118 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import org.apache.knox.gateway.i18n.messages.MessagesFactory; + +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * Rejects any call that does not present a valid Knox bearer token. + *

+ * Authentication happens before a backend channel is opened, so an + * unauthenticated request never reaches Spark — which matters because the OSS + * Spark Connect server has essentially no authentication of its own and assumes + * a fronting proxy provides it. + *

+ * Tokens are checked when an RPC starts and not again while it runs. A + * multi-hour {@code ExecutePlan} is therefore not severed the moment its token + * expires; the next RPC fails instead. Cutting off long queries at expiry would + * punish precisely the workloads Spark Connect exists to serve, and Spark's own + * session timeout still bounds how long a session survives. + */ +public class AuthenticationInterceptor implements ServerInterceptor { + + private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); + + private final TokenAuthenticator authenticator; + + public AuthenticationInterceptor(TokenAuthenticator authenticator) { + this.authenticator = authenticator; + } + + @Override + public ServerCall.Listener interceptCall(ServerCall call, + Metadata headers, + ServerCallHandler next) { + final String method = call.getMethodDescriptor().getFullMethodName(); + final String header = headers.get(GrpcMetadataKeys.AUTHORIZATION); + + if (header == null || !header.regionMatches(true, 0, GrpcMetadataKeys.BEARER_PREFIX, 0, + GrpcMetadataKeys.BEARER_PREFIX.length())) { + return reject(call, method, "no bearer token presented"); + } + + final String serializedToken = header.substring(GrpcMetadataKeys.BEARER_PREFIX.length()).trim(); + final AuthenticatedUser user; + try { + user = authenticator.authenticate(serializedToken); + } catch (TokenAuthenticator.AuthenticationException e) { + return reject(call, method, e.getMessage()); + } + + final GrpcCallContext callContext = GrpcCallContext.current(); + if (callContext != null) { + callContext.setPrincipal(user.getPrincipal()); + callContext.setGroups(user.getGroups()); + } + return next.startCall(call, headers); + } + + private ServerCall.Listener reject(ServerCall call, + String method, + String reason) { + LOG.authenticationFailed(method, reason); + // The description is deliberately generic: distinguishing "expired" from + // "bad signature" tells an attacker which tokens are real. + call.close(Status.UNAUTHENTICATED.withDescription("Invalid or missing bearer token"), new Metadata()); + return new ServerCall.Listener() { }; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthorizationInterceptor.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthorizationInterceptor.java new file mode 100644 index 0000000000..1549ff556e --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthorizationInterceptor.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.knox.gateway.config.GatewayConfig; +import org.apache.knox.gateway.filter.InvalidACLException; +import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.services.GatewayServices; +import org.apache.knox.gateway.services.ServiceType; +import org.apache.knox.gateway.services.topology.TopologyService; +import org.apache.knox.gateway.topology.Provider; +import org.apache.knox.gateway.topology.Topology; + +import io.grpc.Grpc; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * Applies the coarse "may this user use this service in this topology" check, + * after authentication and before any backend connection is opened. + *

+ * The servlet {@code AclsAuthz} filter cannot run here — there is no filter + * chain on a gRPC call — so this reads the same provider configuration directly + * and evaluates it with the same parser. A topology that declares no ACLs for + * the role is unrestricted, as on the servlet path. + */ +public class AuthorizationInterceptor implements ServerInterceptor { + + private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); + + private static final String AUTHZ_PROVIDER_ROLE = "authorization"; + private static final String ACLS_AUTHZ_PROVIDER_NAME = "AclsAuthz"; + + private final GatewayConfig config; + private final GatewayServices services; + private final String resourceRole; + /** + * Authorizers are derived from topology configuration, which changes only on + * redeploy, so they are cached per topology rather than rebuilt per RPC. The + * cache is cleared when topologies are reloaded. + */ + private final Map authorizers = new ConcurrentHashMap<>(); + + public AuthorizationInterceptor(GatewayConfig config, GatewayServices services, String resourceRole) { + this.config = config; + this.services = services; + this.resourceRole = resourceRole; + } + + /** Drops cached ACLs so a redeployed topology takes effect. */ + public void invalidate() { + authorizers.clear(); + } + + @Override + public ServerCall.Listener interceptCall(ServerCall call, + Metadata headers, + ServerCallHandler next) { + final GrpcCallContext callContext = GrpcCallContext.current(); + final String method = call.getMethodDescriptor().getFullMethodName(); + final String topology = callContext == null ? null : callContext.getTopology(); + final String user = callContext == null ? null : callContext.getPrincipal(); + + if (topology == null || user == null) { + // Routing and authentication run first; reaching here without either means + // the chain was assembled wrongly. Deny rather than guess. + return reject(call, method, user, topology, "call reached authorization without an identity or topology"); + } + + final AclAuthorizer authorizer; + try { + authorizer = authorizers.computeIfAbsent(topology, this::buildAuthorizer); + } catch (InvalidAclConfigurationException e) { + return reject(call, method, user, topology, e.getMessage()); + } + + if (!authorizer.isPermitted(user, callContext.getGroups(), callContext.getRemoteAddress())) { + return reject(call, method, user, topology, "denied by the topology ACLs for " + resourceRole); + } + return next.startCall(call, headers); + } + + private AclAuthorizer buildAuthorizer(String topologyName) { + final TopologyService topologyService = services.getService(ServiceType.TOPOLOGY_SERVICE); + Map providerParams = null; + if (topologyService != null) { + for (Topology topology : topologyService.getTopologies()) { + if (topologyName.equals(topology.getName())) { + final Provider provider = topology.getProvider(AUTHZ_PROVIDER_ROLE, ACLS_AUTHZ_PROVIDER_NAME); + if (provider != null && provider.isEnabled()) { + providerParams = provider.getParams(); + } + break; + } + } + } + try { + return new AclAuthorizer(resourceRole, providerParams, + config.getKnoxAdminUsers(), config.getKnoxAdminGroups()); + } catch (InvalidACLException e) { + throw new InvalidAclConfigurationException( + "Topology " + topologyName + " has malformed ACLs for " + resourceRole, e); + } + } + + static String remoteAddressOf(ServerCall call) { + final SocketAddress address = call.getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR); + if (address instanceof InetSocketAddress) { + final InetSocketAddress inet = (InetSocketAddress) address; + return inet.getAddress() == null ? inet.getHostString() : inet.getAddress().getHostAddress(); + } + return address == null ? null : address.toString(); + } + + private ServerCall.Listener reject(ServerCall call, + String method, + String user, + String topology, + String reason) { + LOG.authorizationFailed(method, user, topology, reason); + call.close(Status.PERMISSION_DENIED.withDescription("Not permitted to use " + resourceRole), new Metadata()); + return new ServerCall.Listener() { }; + } + + /** Wraps {@link InvalidACLException} so it can escape a {@code computeIfAbsent} mapping function. */ + static class InvalidAclConfigurationException extends RuntimeException { + private static final long serialVersionUID = 1L; + + InvalidAclConfigurationException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelCache.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelCache.java new file mode 100644 index 0000000000..a84f7bd223 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelCache.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.net.URI; +import java.net.URISyntaxException; +import java.security.KeyStore; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.TrustManagerFactory; + +import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.services.GatewayServices; +import org.apache.knox.gateway.services.ServiceType; +import org.apache.knox.gateway.services.security.KeystoreService; + +import io.grpc.ManagedChannel; +import io.grpc.Status; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.NegotiationType; +import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; + +/** + * Keeps one {@link ManagedChannel} per backend URL, shared by every call routed + * to that backend. + *

+ * gRPC channels multiplex concurrent calls over a pooled HTTP/2 connection and + * are designed to be long-lived, so creating one per RPC would be both slower + * and wasteful of connections. Channels go idle on their own after + * {@code gateway.sparkconnect.channel.idle.timeout} and reconnect transparently + * when used again, so a cached entry for an unused backend costs nothing. + */ +public class BackendChannelCache { + + private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); + + private static final String PLAINTEXT_SCHEME = "grpc"; + private static final String TLS_SCHEME = "grpcs"; + + private final GrpcListenerSettings settings; + private final GatewayServices services; + private final Map channels = new ConcurrentHashMap<>(); + + public BackendChannelCache(GrpcListenerSettings settings, GatewayServices services) { + this.settings = settings; + this.services = services; + } + + /** + * Returns the shared channel for the given backend URL, creating it if this is + * the first call to that backend. + * + * @param backendUrl a {@code grpc://host:port} or {@code grpcs://host:port} URL + * @return the channel for that backend + * @throws io.grpc.StatusRuntimeException if the URL is unusable or TLS cannot be set up + */ + public ManagedChannel getChannel(String backendUrl) { + return channels.computeIfAbsent(backendUrl, this::createChannel); + } + + private ManagedChannel createChannel(String backendUrl) { + final URI uri = parse(backendUrl); + final String scheme = uri.getScheme() == null ? "" : uri.getScheme().toLowerCase(Locale.ROOT); + final String host = uri.getHost(); + final int port = uri.getPort(); + + if (host == null || port < 0) { + throw Status.FAILED_PRECONDITION + .withDescription("Spark Connect backend URL must include a host and port: " + backendUrl) + .asRuntimeException(); + } + + final NettyChannelBuilder builder = NettyChannelBuilder.forAddress(host, port) + .maxInboundMessageSize(settings.getMaxMessageSize()) + .idleTimeout(settings.getChannelIdleTimeoutMillis(), TimeUnit.MILLISECONDS); + + if (TLS_SCHEME.equals(scheme)) { + builder.negotiationType(NegotiationType.TLS); + try { + builder.sslContext(GrpcSslContexts.forClient().trustManager(backendTrustManagers()).build()); + } catch (Exception e) { + LOG.failedToBuildBackendTls(backendUrl, e); + throw Status.UNAVAILABLE + .withDescription("Cannot establish TLS to the Spark Connect backend") + .withCause(e) + .asRuntimeException(); + } + } else if (PLAINTEXT_SCHEME.equals(scheme)) { + builder.negotiationType(NegotiationType.PLAINTEXT); + } else { + throw Status.FAILED_PRECONDITION + .withDescription("Spark Connect backend URL scheme must be grpc:// or grpcs://, got: " + backendUrl) + .asRuntimeException(); + } + + LOG.openedBackendChannel(backendUrl); + return builder.build(); + } + + /** + * Trust material for the backend leg: the HTTP client truststore if the + * deployment configured one, otherwise the gateway keystore. This is the same + * fallback the WebSocket handler applies, so a deployment that already trusts + * its backends over {@code wss://} needs no extra configuration here. + */ + private TrustManagerFactory backendTrustManagers() throws Exception { + final KeystoreService keystoreService = services.getService(ServiceType.KEYSTORE_SERVICE); + KeyStore truststore = keystoreService.getTruststoreForHttpClient(); + if (truststore == null) { + truststore = keystoreService.getKeystoreForGateway(); + } + final TrustManagerFactory factory = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + factory.init(truststore); + return factory; + } + + private static URI parse(String backendUrl) { + try { + return new URI(backendUrl); + } catch (URISyntaxException e) { + throw Status.FAILED_PRECONDITION + .withDescription("Malformed Spark Connect backend URL: " + backendUrl) + .withCause(e) + .asRuntimeException(); + } + } + + /** + * Shuts every cached channel down, waiting up to the given deadline in total + * for in-flight calls to finish. + * + * @param timeoutMillis total time to wait for all channels to terminate + */ + public void shutdown(long timeoutMillis) { + final long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + for (Map.Entry entry : channels.entrySet()) { + entry.getValue().shutdown(); + } + for (Map.Entry entry : channels.entrySet()) { + final long remaining = deadline - System.nanoTime(); + try { + if (remaining <= 0 || !entry.getValue().awaitTermination(remaining, TimeUnit.NANOSECONDS)) { + entry.getValue().shutdownNow(); + } + } catch (InterruptedException e) { + entry.getValue().shutdownNow(); + Thread.currentThread().interrupt(); + } + LOG.closedBackendChannel(entry.getKey()); + } + channels.clear(); + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelProvider.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelProvider.java new file mode 100644 index 0000000000..0c8f11cb6b --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelProvider.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import io.grpc.Channel; + +/** + * Supplies the backend channel for the call in flight. + *

+ * Implementations read the backend the routing interceptor resolved into the + * current {@link GrpcCallContext}, so the proxy handler itself never needs to + * know how topologies map to backends. + */ +@FunctionalInterface +public interface BackendChannelProvider { + + /** + * Returns the channel for the current call's backend. + * + * @return a channel to the backend + * @throws io.grpc.StatusRuntimeException if no backend can be resolved + */ + Channel getChannel(); +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java new file mode 100644 index 0000000000..c6f9116d97 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.services.security.AliasService; +import org.apache.knox.gateway.services.security.AliasServiceException; + +import io.grpc.Metadata; + +/** + * Replaces the client's credentials with Knox's own on the backend leg. + *

+ * The client's bearer token proves the user's identity to Knox and has no + * meaning beyond it, so it is removed rather than forwarded. In its place, if + * the backend is configured with a pre-shared token + * ({@code spark.connect.authenticate.token}), Knox presents that. Besides + * authenticating the gateway to Spark, it closes the hole where a client with + * network reachability to the backend port could simply bypass the gateway + * altogether — network restrictions should prevent that too, but a credential + * the client does not hold makes it structural rather than topological. + *

+ * Knox-internal routing metadata is dropped for the same reason: it was + * addressed to the gateway, and the backend has no use for it. + */ +public class BackendHeaderRewriter implements HeaderRewriter { + + private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); + + private final String backendAuthorization; + + /** + * @param aliasService used to resolve the backend token; may be null when no + * alias is configured + * @param backendTokenAlias the alias holding the backend's pre-shared token, or + * null if the backend requires no token + */ + public BackendHeaderRewriter(AliasService aliasService, String backendTokenAlias) { + this.backendAuthorization = resolveBackendToken(aliasService, backendTokenAlias); + } + + private static String resolveBackendToken(AliasService aliasService, String alias) { + if (aliasService == null || alias == null || alias.trim().isEmpty()) { + return null; + } + try { + final char[] token = aliasService.getPasswordFromAliasForGateway(alias); + if (token == null || token.length == 0) { + LOG.missingBackendTokenAlias(alias); + return null; + } + return GrpcMetadataKeys.BEARER_PREFIX + new String(token); + } catch (AliasServiceException e) { + LOG.missingBackendTokenAlias(alias); + return null; + } + } + + @Override + public void rewrite(Metadata headers) { + headers.removeAll(GrpcMetadataKeys.AUTHORIZATION); + headers.removeAll(GrpcMetadataKeys.TOPOLOGY); + if (backendAuthorization != null) { + headers.put(GrpcMetadataKeys.AUTHORIZATION, backendAuthorization); + } + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ByteArrayMarshaller.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ByteArrayMarshaller.java new file mode 100644 index 0000000000..e836591dfc --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ByteArrayMarshaller.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +import io.grpc.MethodDescriptor; +import io.grpc.Status; + +/** + * Passes message bodies through as opaque bytes. + *

+ * With this marshaller the gateway can relay a call whose message types it has + * no generated classes for, which is what makes the fallback path work for + * methods outside the vendored protos — a newer client calling an RPC added + * after this build still gets proxied rather than rejected. + */ +public final class ByteArrayMarshaller implements MethodDescriptor.Marshaller { + + public static final ByteArrayMarshaller INSTANCE = new ByteArrayMarshaller(); + + private ByteArrayMarshaller() { + } + + @Override + public InputStream stream(byte[] value) { + return new ByteArrayInputStream(value); + } + + @Override + public byte[] parse(InputStream stream) { + try { + // grpc hands over the complete message, so a single drain is enough and the + // inbound size limit has already been applied by the transport. + return readAll(stream); + } catch (IOException e) { + throw Status.INTERNAL.withDescription("Failed to read gRPC message").withCause(e).asRuntimeException(); + } + } + + private static byte[] readAll(InputStream stream) throws IOException { + final java.io.ByteArrayOutputStream buffer = new java.io.ByteArrayOutputStream(); + final byte[] chunk = new byte[8192]; + int read; + while ((read = stream.read(chunk)) != -1) { + buffer.write(chunk, 0, read); + } + return buffer.toByteArray(); + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcCallContext.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcCallContext.java new file mode 100644 index 0000000000..8cab010a06 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcCallContext.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.Collections; +import java.util.Set; + +import io.grpc.Context; + +/** + * Per-call state shared across the interceptor chain and the proxy handler. + *

+ * The instance is created by the outermost interceptor and attached to the gRPC + * {@link Context}, then filled in as the call descends the chain: authentication + * sets the principal and groups, routing sets the topology and backend. Because + * every interceptor mutates one object rather than layering new context values, + * the audit interceptor — which wraps the call from outside — can still report + * what the inner interceptors resolved, including on the paths where they + * rejected the call. + *

+ * Instances are confined to a single call. gRPC may invoke listener callbacks on + * different threads, so the fields are volatile; they are written once during + * interceptor descent and only read afterwards. + */ +// volatile, not synchronized: gRPC dispatches a call's listener callbacks across +// threads, and these fields are written once during interceptor descent and read +// afterwards. A lock would serialise readers for no benefit. +@SuppressWarnings("PMD.AvoidUsingVolatile") +public class GrpcCallContext { + + public static final Context.Key KEY = Context.key("KnoxGrpcCallContext"); + + private final String methodName; + private final String authority; + private final String remoteAddress; + private final long startNanos; + + private volatile String principal; + private volatile Set groups = Collections.emptySet(); + private volatile String topology; + private volatile String backendUrl; + private volatile String sessionId; + private volatile String operationId; + + public GrpcCallContext(String methodName, String authority, String remoteAddress, long startNanos) { + this.methodName = methodName; + this.authority = authority; + this.remoteAddress = remoteAddress; + this.startNanos = startNanos; + } + + /** + * Returns the call context attached to the current gRPC context, or null when + * called outside a proxied call. + * + * @return the current call context, or null + */ + public static GrpcCallContext current() { + return KEY.get(); + } + + public String getMethodName() { + return methodName; + } + + public String getAuthority() { + return authority; + } + + public String getRemoteAddress() { + return remoteAddress; + } + + public long getStartNanos() { + return startNanos; + } + + public String getPrincipal() { + return principal; + } + + public void setPrincipal(String principal) { + this.principal = principal; + } + + public Set getGroups() { + return groups; + } + + public void setGroups(Set groups) { + this.groups = groups == null ? Collections.emptySet() : Collections.unmodifiableSet(groups); + } + + public String getTopology() { + return topology; + } + + public void setTopology(String topology) { + this.topology = topology; + } + + public String getBackendUrl() { + return backendUrl; + } + + public void setBackendUrl(String backendUrl) { + this.backendUrl = backendUrl; + } + + /** + * The Spark Connect session this call belongs to, once a request message has + * been parsed. Null on the generic byte-level path, which never looks inside + * messages. + * + * @return the session id, or null + */ + public String getSessionId() { + return sessionId; + } + + public void setSessionId(String sessionId) { + this.sessionId = sessionId; + } + + public String getOperationId() { + return operationId; + } + + public void setOperationId(String operationId) { + this.operationId = operationId; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayListener.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayListener.java new file mode 100644 index 0000000000..dd527e26c9 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayListener.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.security.Key; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.Certificate; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.KeyManagerFactory; + +import org.apache.knox.gateway.config.GatewayConfig; +import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.protocol.ProtocolListener; +import org.apache.knox.gateway.services.GatewayServices; +import org.apache.knox.gateway.services.ServiceType; +import org.apache.knox.gateway.services.security.AliasService; +import org.apache.knox.gateway.services.security.KeystoreService; + +import io.grpc.Server; +import io.grpc.ServerInterceptor; +import io.grpc.ServerMethodDefinition; +import io.grpc.ServerServiceDefinition; +import io.grpc.Status; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder; + +/** + * A gRPC listener: a Netty server on its own port, wired to Knox's identity, + * token, topology and audit services. + *

+ * It is a separate socket rather than a route on the gateway's existing + * connectors because gRPC requires HTTP/2 negotiated over ALPN, and Knox's Jetty + * connectors are HTTP/1.1 only. Beyond the transport, the servlet pipeline could + * not carry these calls anyway: Servlet 3.1 has no trailer API, and gRPC puts + * {@code grpc-status} — and, for Spark Connect, structured error details — in + * trailers. + * + *

Why this is abstract

+ * Almost everything a gRPC gateway needs is protocol-agnostic: the transport, + * TLS from the gateway identity, bearer authentication, coarse authorization, + * topology routing, backend channel caching, auditing, graceful drain, and the + * byte-level relay itself. Only message-body handling — identity assertion and + * per-RPC gating — needs to know what is being proxied. + *

+ * Keeping that split explicit means a generic gRPC gateway would later be a + * configuration-and-documentation exercise rather than an engineering one. But + * Knox does not offer one today, and this class is deliberately + * not a way to get one: it is abstract, there is no configuration property that + * selects a listener generically, and the only concrete subclass is the Spark + * Connect one. A generic offering would need its own service-to-role mapping, + * default-deny posture, and an honest account of what byte-level proxying cannot + * enforce — none of which is in scope here. + *

+ * Subclasses supply four things: what to call the listener, which Knox service + * role backs it, the typed handlers, and which proto services may fall back to + * byte-level relay. + */ +// volatile: the lifecycle fields are written by the thread calling start/stop and +// read by request threads, so they need visibility but not mutual exclusion. +@SuppressWarnings("PMD.AvoidUsingVolatile") +public abstract class GrpcGatewayListener implements ProtocolListener { + + private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); + + private volatile Server server; + private volatile BackendChannelCache channelCache; + private volatile AuthorizationInterceptor authorizationInterceptor; + private volatile GrpcListenerSettings settings; + + /** + * The Knox service role backing this listener, e.g. {@code SPARKCONNECT}. + * Topologies declare a service with this role, and its ACLs are keyed on it. + * + * @return the service role + */ + protected abstract String getServiceRole(); + + /** + * Reads this listener's transport settings from gateway configuration. + * Subclasses own this because they own the configuration properties; the + * template deliberately does not read {@code GatewayConfig} for transport + * limits itself. + * + * @param config the gateway configuration + * @return the settings to build the server with + */ + protected abstract GrpcListenerSettings createSettings(GatewayConfig config); + + /** + * Builds the typed service definition whose handlers may inspect and rewrite + * message bodies. + * + * @param channels supplies the backend channel for the call in flight + * @param headers rewrites metadata for the backend leg + * @return the service definition to register + */ + protected abstract ServerServiceDefinition bindService(BackendChannelProvider channels, + HeaderRewriter headers); + + /** + * Fully qualified proto service names whose unregistered methods may still be + * relayed as opaque bytes. Anything not named here is answered + * {@code UNIMPLEMENTED}, so this is a closed list rather than an opt-out. + * + * @return the proto service names eligible for byte-level passthrough + */ + protected abstract Set getPassthroughServiceNames(); + + @Override + public void start(GatewayConfig config, GatewayServices services) throws Exception { + final GrpcListenerSettings listenerSettings = createSettings(config); + this.settings = listenerSettings; + + final BackendChannelCache channels = new BackendChannelCache(listenerSettings, services); + this.channelCache = channels; + + final BackendChannelProvider channelProvider = () -> { + final GrpcCallContext callContext = GrpcCallContext.current(); + if (callContext == null || callContext.getBackendUrl() == null) { + throw Status.UNAVAILABLE.withDescription("No backend resolved for this call").asRuntimeException(); + } + return channels.getChannel(callContext.getBackendUrl()); + }; + + final AliasService aliasService = services.getService(ServiceType.ALIAS_SERVICE); + final HeaderRewriter headerRewriter = + new BackendHeaderRewriter(aliasService, listenerSettings.getBackendTokenAlias()); + + this.authorizationInterceptor = new AuthorizationInterceptor(config, services, getServiceRole()); + + // Order is load-bearing: audit wraps everything so even rejected calls are + // recorded, then identity, then topology selection, then the ACL check that + // depends on both having succeeded. + final List interceptors = Arrays.asList( + new AuditInterceptor(), + new AuthenticationInterceptor(new TokenAuthenticator(config, services)), + new RoutingInterceptor(config, services, getServiceRole()), + authorizationInterceptor); + + final NettyServerBuilder builder = NettyServerBuilder.forPort(listenerSettings.getPort()) + .maxInboundMessageSize(listenerSettings.getMaxMessageSize()) + .maxConcurrentCallsPerConnection(listenerSettings.getMaxConcurrentCallsPerConnection()) + .permitKeepAliveTime(listenerSettings.getPermitKeepAliveTimeMillis(), TimeUnit.MILLISECONDS) + .permitKeepAliveWithoutCalls(listenerSettings.isPermitKeepAliveWithoutCalls()); + + if (config.isSSLEnabled()) { + builder.sslContext(buildServerSslContext(config, services)); + } else { + // A client that sets token= forces use_ssl=true, so this is really a test + // and development posture; say so rather than let it pass silently. + LOG.listenerTlsDisabled(listenerSettings.getName()); + } + + builder.addService(intercept(bindService(channelProvider, headerRewriter), interceptors)); + + final ProxyCallHandler passthroughHandler = + new ProxyCallHandler<>(channelProvider, MessageInterceptor.passthrough(), headerRewriter); + builder.fallbackHandlerRegistry(new PassthroughHandlerRegistry( + getPassthroughServiceNames(), + InterceptorChain.intercept(passthroughHandler, interceptors))); + + try { + this.server = builder.build().start(); + } catch (Exception e) { + LOG.failedToStartListener(listenerSettings.getName(), e); + channels.shutdown(0L); + this.channelCache = null; + throw e; + } + LOG.startedListener(listenerSettings.getName(), getPort()); + } + + /** + * Applies the interceptor chain to every method of a service definition. The + * chain is composed by hand rather than through {@code ServerInterceptors} so + * the ordering established above is preserved exactly. + */ + private static ServerServiceDefinition intercept(ServerServiceDefinition service, + List interceptors) { + final ServerServiceDefinition.Builder builder = + ServerServiceDefinition.builder(service.getServiceDescriptor()); + for (ServerMethodDefinition method : service.getMethods()) { + builder.addMethod(wrap(method, interceptors)); + } + return builder.build(); + } + + private static ServerMethodDefinition wrap( + ServerMethodDefinition method, List interceptors) { + return ServerMethodDefinition.create( + method.getMethodDescriptor(), + InterceptorChain.intercept(method.getServerCallHandler(), interceptors)); + } + + /** + * Builds the server's TLS context from the gateway identity — the same key + * material Jetty presents — so a deployment has one certificate to manage, not + * two. + *

+ * The identity is copied into a single-entry keystore before building the key + * manager, so the configured alias is the one presented even when the gateway + * keystore holds other entries. + */ + private SslContext buildServerSslContext(GatewayConfig config, GatewayServices services) + throws Exception { + try { + final KeystoreService keystoreService = services.getService(ServiceType.KEYSTORE_SERVICE); + final AliasService aliasService = services.getService(ServiceType.ALIAS_SERVICE); + + final String alias = config.getIdentityKeyAlias(); + final char[] passphrase = aliasService.getGatewayIdentityPassphrase(); + final KeyStore gatewayKeystore = keystoreService.getKeystoreForGateway(); + if (gatewayKeystore == null) { + throw new IllegalStateException("The gateway identity keystore is not available"); + } + + final Key key = gatewayKeystore.getKey(alias, passphrase); + final Certificate[] chain = gatewayKeystore.getCertificateChain(alias); + if (!(key instanceof PrivateKey) || chain == null || chain.length == 0) { + throw new IllegalStateException( + "The gateway identity keystore has no usable key entry for alias " + alias); + } + + final KeyStore identity = KeyStore.getInstance("PKCS12"); + identity.load(null, null); + identity.setKeyEntry(alias, key, passphrase, chain); + + final KeyManagerFactory keyManagers = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagers.init(identity, passphrase); + + // GrpcSslContexts applies the ALPN and cipher requirements of the HTTP/2 + // profile gRPC mandates. + return GrpcSslContexts.configure(SslContextBuilder.forServer(keyManagers)).build(); + } catch (Exception e) { + LOG.failedToBuildServerTls(getName(), e); + throw e; + } + } + + /** + * Stops accepting new calls and lets in-flight ones finish, up to the + * configured drain timeout. + *

+ * Long-lived streams are severed if they outlast the drain. That is survivable + * by design: Spark Connect clients already retry through + * {@code ReattachExecute}, which exists precisely because a connection can drop + * mid-query. + */ + @Override + public void stop() { + final Server current = server; + if (current == null) { + return; + } + final GrpcListenerSettings listenerSettings = settings; + final long drainTimeoutMillis = + listenerSettings == null ? 0L : listenerSettings.getDrainTimeoutMillis(); + LOG.stoppingListener(getName(), drainTimeoutMillis); + current.shutdown(); + try { + if (!current.awaitTermination(drainTimeoutMillis, TimeUnit.MILLISECONDS)) { + LOG.drainTimedOut(getName(), drainTimeoutMillis); + current.shutdownNow(); + } + } catch (InterruptedException e) { + current.shutdownNow(); + Thread.currentThread().interrupt(); + } finally { + server = null; + final BackendChannelCache channels = channelCache; + if (channels != null) { + channels.shutdown(drainTimeoutMillis); + channelCache = null; + } + LOG.stoppedListener(getName()); + } + } + + /** Drops cached topology ACLs so a redeployed topology takes effect. */ + public void reload() { + final AuthorizationInterceptor interceptor = authorizationInterceptor; + if (interceptor != null) { + interceptor.invalidate(); + } + } + + @Override + public int getPort() { + final Server current = server; + return current == null ? -1 : current.getPort(); + } + + /** The settings this listener started with, or null before {@code start}. */ + protected GrpcListenerSettings getSettings() { + return settings; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java new file mode 100644 index 0000000000..1ae172f780 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import org.apache.knox.gateway.i18n.messages.Message; +import org.apache.knox.gateway.i18n.messages.MessageLevel; +import org.apache.knox.gateway.i18n.messages.Messages; +import org.apache.knox.gateway.i18n.messages.StackTrace; + +/** + * Logging for the gRPC gateway listener. + * + * @since 3.0.0 + */ +@Messages(logger = "org.apache.knox.gateway.grpc") +public interface GrpcGatewayMessages { + + @Message(level = MessageLevel.INFO, text = "Started {0} gRPC listener on port {1}") + void startedListener(String name, int port); + + @Message(level = MessageLevel.INFO, text = "Stopping {0} gRPC listener, draining for up to {1} ms") + void stoppingListener(String name, long drainTimeoutMillis); + + @Message(level = MessageLevel.WARN, + text = "The {0} gRPC listener did not drain within {1} ms; terminating in-flight calls") + void drainTimedOut(String name, long drainTimeoutMillis); + + @Message(level = MessageLevel.INFO, text = "Stopped {0} gRPC listener") + void stoppedListener(String name); + + @Message(level = MessageLevel.ERROR, text = "Failed to start the {0} gRPC listener") + void failedToStartListener(String name, @StackTrace(level = MessageLevel.ERROR) Exception e); + + @Message(level = MessageLevel.WARN, text = "Rejected unauthenticated gRPC call to {0}: {1}") + void authenticationFailed(String method, String reason); + + @Message(level = MessageLevel.WARN, + text = "Denied gRPC call to {0} for user {1} in topology {2}: {3}") + void authorizationFailed(String method, String user, String topology, String reason); + + @Message(level = MessageLevel.WARN, text = "Cannot route gRPC call to {0}: {1}") + void routingFailed(String method, String reason); + + @Message(level = MessageLevel.DEBUG, + text = "Routing gRPC call to {0} for user {1} to topology {2} backend {3}") + void routingCall(String method, String user, String topology, String backend); + + @Message(level = MessageLevel.DEBUG, text = "Opened backend gRPC channel to {0}") + void openedBackendChannel(String backend); + + @Message(level = MessageLevel.DEBUG, text = "Closed backend gRPC channel to {0}") + void closedBackendChannel(String backend); + + @Message(level = MessageLevel.ERROR, text = "Failed to build TLS context for the {0} gRPC listener") + void failedToBuildServerTls(String name, @StackTrace(level = MessageLevel.ERROR) Exception e); + + @Message(level = MessageLevel.ERROR, text = "Failed to build TLS context for backend {0}") + void failedToBuildBackendTls(String backend, @StackTrace(level = MessageLevel.ERROR) Exception e); + + @Message(level = MessageLevel.WARN, + text = "The {0} gRPC listener is running without TLS; bearer tokens will cross the network in clear text") + void listenerTlsDisabled(String name); + + @Message(level = MessageLevel.WARN, text = "Could not resolve the backend token alias {0}") + void missingBackendTokenAlias(String alias); + + @Message(level = MessageLevel.DEBUG, text = "{0}") + void debugLog(String message); +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java new file mode 100644 index 0000000000..8791701fc7 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +/** + * Transport and lifecycle settings for a {@link GrpcGatewayListener}. + *

+ * These are deliberately plain values rather than reads against + * {@code GatewayConfig}. The listener is meant to be reusable for any gRPC + * service, so the layer that knows which {@code gateway.*} properties apply — + * currently the Spark Connect plugin — is the layer that reads them. + *

+ * The limits here are the listener's DoS surface. A new socket accepting 128 MB + * messages on long-lived streams needs message-size, stream-count and + * keepalive-abuse bounds configured from the start, not added after the first + * incident. + */ +public class GrpcListenerSettings { + + private String name = "grpc"; + private int port; + private int maxMessageSize = 134217728; + private int maxConcurrentCallsPerConnection = 1000; + private long permitKeepAliveTimeMillis = 10000L; + private boolean permitKeepAliveWithoutCalls = true; + private long channelIdleTimeoutMillis = 1800000L; + private long drainTimeoutMillis = 30000L; + private String backendTokenAlias; + + public String getName() { + return name; + } + + public GrpcListenerSettings name(String value) { + this.name = value; + return this; + } + + public int getPort() { + return port; + } + + public GrpcListenerSettings port(int value) { + this.port = value; + return this; + } + + public int getMaxMessageSize() { + return maxMessageSize; + } + + public GrpcListenerSettings maxMessageSize(int value) { + this.maxMessageSize = value; + return this; + } + + public int getMaxConcurrentCallsPerConnection() { + return maxConcurrentCallsPerConnection; + } + + public GrpcListenerSettings maxConcurrentCallsPerConnection(int value) { + this.maxConcurrentCallsPerConnection = value; + return this; + } + + public long getPermitKeepAliveTimeMillis() { + return permitKeepAliveTimeMillis; + } + + public GrpcListenerSettings permitKeepAliveTimeMillis(long value) { + this.permitKeepAliveTimeMillis = value; + return this; + } + + public boolean isPermitKeepAliveWithoutCalls() { + return permitKeepAliveWithoutCalls; + } + + public GrpcListenerSettings permitKeepAliveWithoutCalls(boolean value) { + this.permitKeepAliveWithoutCalls = value; + return this; + } + + public long getChannelIdleTimeoutMillis() { + return channelIdleTimeoutMillis; + } + + public GrpcListenerSettings channelIdleTimeoutMillis(long value) { + this.channelIdleTimeoutMillis = value; + return this; + } + + public long getDrainTimeoutMillis() { + return drainTimeoutMillis; + } + + public GrpcListenerSettings drainTimeoutMillis(long value) { + this.drainTimeoutMillis = value; + return this; + } + + public String getBackendTokenAlias() { + return backendTokenAlias; + } + + public GrpcListenerSettings backendTokenAlias(String value) { + this.backendTokenAlias = value; + return this; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java new file mode 100644 index 0000000000..b71c06bfbe --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import io.grpc.Metadata; + +/** + * Call metadata keys the gateway reads or writes. + *

+ * Everything here is expressible in a vanilla {@code sc://} connection string. + * The {@code token=} parameter becomes {@link #AUTHORIZATION}, and any parameter + * the client does not recognise — {@code knox-topology=analytics}, say — is sent + * verbatim as metadata on every request. That is what lets clients select a + * topology despite gRPC forbidding a path component in the connection URL. + */ +public final class GrpcMetadataKeys { + + /** Carries the Knox-issued bearer token, set by the client's {@code token=} parameter. */ + public static final Metadata.Key AUTHORIZATION = + Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER); + + /** Selects the topology, set by a {@code knox-topology=} connection-string parameter. */ + public static final Metadata.Key TOPOLOGY = + Metadata.Key.of("knox-topology", Metadata.ASCII_STRING_MARSHALLER); + + public static final String BEARER_PREFIX = "Bearer "; + + private GrpcMetadataKeys() { + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/HeaderRewriter.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/HeaderRewriter.java new file mode 100644 index 0000000000..90b1ce82a0 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/HeaderRewriter.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import io.grpc.Metadata; + +/** + * Adjusts call metadata in place before it is forwarded to the backend. + *

+ * The two legs have separate credentials: the client's bearer token + * authenticates the user to Knox and must not travel further, while the backend + * gets Knox's own pre-shared token if one is configured. Knox-internal routing + * metadata is dropped here too. + */ +@FunctionalInterface +public interface HeaderRewriter { + + /** + * Rewrites the metadata that will be sent to the backend. + * + * @param headers the client's call metadata, modified in place + */ + void rewrite(Metadata headers); +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/InterceptorChain.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/InterceptorChain.java new file mode 100644 index 0000000000..584e365e56 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/InterceptorChain.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.List; + +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; + +/** + * Wraps a call handler in an explicitly ordered interceptor chain. + *

+ * Order is a correctness property here, not a preference: routing must have + * chosen a topology before ACLs for that topology can be evaluated, and + * authentication must have established a principal before either. Rather than + * depend on the registration-order semantics of a builder, the chain is composed + * directly so the ordering is visible at the call site and cannot drift. + */ +public final class InterceptorChain { + + private InterceptorChain() { + } + + /** + * Returns a handler that applies the interceptors in list order, so the first + * element sees the call first and closes it last. + * + * @param handler the innermost handler + * @param interceptors the interceptors, outermost first + * @param the request message type + * @param the response message type + * @return the wrapped handler + */ + public static ServerCallHandler intercept( + ServerCallHandler handler, List interceptors) { + ServerCallHandler result = handler; + for (int i = interceptors.size() - 1; i >= 0; i--) { + final ServerInterceptor interceptor = interceptors.get(i); + final ServerCallHandler next = result; + result = (call, headers) -> interceptor.interceptCall(call, headers, next); + } + return result; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MapFilterConfig.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MapFilterConfig.java new file mode 100644 index 0000000000..3b05656c3e --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MapFilterConfig.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.Collections; +import java.util.Enumeration; +import java.util.Map; + +import javax.servlet.FilterConfig; +import javax.servlet.ServletContext; + +/** + * A map-backed {@link FilterConfig} for reusing servlet-configured collaborators + * off the servlet path. + *

+ * {@code SignatureVerificationCache} takes its settings from a + * {@code FilterConfig}, but the gRPC listener has no filter chain to get one + * from; its settings come from topology provider parameters instead. The + * WebSocket listener solves the same problem the same way. + */ +public class MapFilterConfig implements FilterConfig { + + private final String name; + private final Map params; + + public MapFilterConfig(String name, Map params) { + this.name = name; + this.params = params == null ? Collections.emptyMap() : params; + } + + @Override + public String getFilterName() { + return name; + } + + @Override + public ServletContext getServletContext() { + return null; + } + + @Override + public String getInitParameter(String key) { + return params.get(key); + } + + @Override + public Enumeration getInitParameterNames() { + return Collections.enumeration(params.keySet()); + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptor.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptor.java new file mode 100644 index 0000000000..8e8ac420e3 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptor.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import io.grpc.StatusRuntimeException; + +/** + * Inspects and optionally rewrites each request message on its way to the + * backend. + *

+ * This is the seam between the generic gRPC core and a protocol-aware plugin. + * The core never parses message bodies; everything that needs to — identity + * assertion, per-RPC gating, reserved-key protection — is expressed here. A + * generic byte-level proxy simply uses {@link #PASSTHROUGH}. + * + * @param the request message type + */ +@FunctionalInterface +public interface MessageInterceptor { + + /** + * A message interceptor that forwards every message unchanged. This is what + * makes the byte-level path a pure pipe. + */ + MessageInterceptor PASSTHROUGH = message -> message; + + /** + * Returns the message to forward to the backend, which may be the argument + * itself or a rewritten copy. + *

+ * Throwing {@link StatusRuntimeException} rejects the call with that status; + * the proxy closes the client call and cancels the backend call. This is how + * per-RPC gating denies a request without the backend ever seeing it. + * + * @param message the message received from the client + * @return the message to send to the backend + * @throws StatusRuntimeException to reject the call + */ + T intercept(T message); + + /** + * Returns the passthrough interceptor, typed for the caller's message type. + * + * @param the request message type + * @return an interceptor that forwards every message unchanged + */ + @SuppressWarnings("unchecked") + static MessageInterceptor passthrough() { + return (MessageInterceptor) PASSTHROUGH; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/PassthroughHandlerRegistry.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/PassthroughHandlerRegistry.java new file mode 100644 index 0000000000..8ff6cae50e --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/PassthroughHandlerRegistry.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.Locale; +import java.util.Set; + +import org.apache.knox.gateway.i18n.messages.MessagesFactory; + +import io.grpc.HandlerRegistry; +import io.grpc.MethodDescriptor; +import io.grpc.ServerCallHandler; +import io.grpc.ServerMethodDefinition; + +/** + * Handles calls to methods that have no typed handler, relaying them as opaque + * bytes. + *

+ * This exists so that proto skew degrades gracefully. When a client calls an RPC + * this build has no generated classes for — an addition in a newer Spark line, + * typically — the call is still authenticated, authorized, routed and audited; + * only the message-body handling is skipped, because there is nothing to inspect + * with. Without it, such a call would fail outright at the gateway even though + * the backend could serve it. + *

+ * The trade-off is explicit: no identity assertion happens on this path, since + * rewriting {@code user_context} requires parsing the message. Passthrough is + * therefore restricted to a configured set of proto services, and default-denies + * everything else — a gateway that forwarded arbitrary unknown services without + * being asked to would be a very different, and much weaker, security posture. + */ +public class PassthroughHandlerRegistry extends HandlerRegistry { + + private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); + + private final Set allowedServices; + private final ServerCallHandler handler; + + /** + * @param allowedServices fully qualified proto service names whose unknown + * methods may be relayed, e.g. {@code spark.connect.SparkConnectService} + * @param handler the proxy handler to relay with + */ + public PassthroughHandlerRegistry(Set allowedServices, ServerCallHandler handler) { + this.allowedServices = allowedServices; + this.handler = handler; + } + + @Override + public ServerMethodDefinition lookupMethod(String methodName, String authority) { + final String serviceName = MethodDescriptor.extractFullServiceName(methodName); + if (serviceName == null || !allowedServices.contains(serviceName)) { + // Returning null makes grpc answer UNIMPLEMENTED, which is also what a real + // server says about a method it does not have — so this reveals nothing + // about what the gateway is fronting. + return null; + } + + LOG.debugLog(String.format(Locale.ROOT, + "Relaying %s as opaque bytes; no typed handler is registered for it", methodName)); + + final MethodDescriptor descriptor = MethodDescriptor.newBuilder() + // UNKNOWN keeps grpc from assuming a message count in either direction, + // so unary and streaming methods alike relay correctly. + .setType(MethodDescriptor.MethodType.UNKNOWN) + .setFullMethodName(methodName) + .setRequestMarshaller(ByteArrayMarshaller.INSTANCE) + .setResponseMarshaller(ByteArrayMarshaller.INSTANCE) + .build(); + return ServerMethodDefinition.create(descriptor, handler); + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ProxyCallHandler.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ProxyCallHandler.java new file mode 100644 index 0000000000..c1507e9ed9 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ProxyCallHandler.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.concurrent.atomic.AtomicBoolean; + +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; + +/** + * Pipes one gRPC call through to a backend, relaying messages, headers, status + * and trailers in both directions. + *

+ * All four RPC shapes collapse into this one handler. Unary, server-streaming, + * client-streaming and bidirectional calls differ only in how many messages flow + * each way, which the listener callbacks below express naturally — so there is + * no need for a handler per shape, and Spark Connect's long-lived + * {@code ExecutePlan} streams work by the same code path as a unary + * {@code Config}. + *

+ * Backend {@link Status} and trailers are relayed verbatim. This matters more + * than it might appear: gRPC carries {@code grpc-status} in trailers, and Spark + * Connect packs {@code google.rpc.ErrorInfo} in there too, so anything that + * interprets or drops trailers breaks error reporting wholesale. + *

+ * Flow control is explicit in both directions: a message is only requested from + * one side once the other side has accepted the previous one, so a slow + * {@code sc://} client cannot make the gateway buffer an unbounded number of + * Arrow batches. This follows the flow-control structure of the upstream + * grpc-java {@code GrpcProxy} example. + * + * @param the request message type + * @param the response message type + */ +public class ProxyCallHandler implements ServerCallHandler { + + private final BackendChannelProvider channelProvider; + private final MessageInterceptor messageInterceptor; + private final HeaderRewriter headerRewriter; + + public ProxyCallHandler(BackendChannelProvider channelProvider, + MessageInterceptor messageInterceptor, + HeaderRewriter headerRewriter) { + this.channelProvider = channelProvider; + this.messageInterceptor = messageInterceptor; + this.headerRewriter = headerRewriter; + } + + @Override + public ServerCall.Listener startCall(ServerCall serverCall, Metadata headers) { + final Channel channel; + try { + channel = channelProvider.getChannel(); + } catch (StatusRuntimeException e) { + serverCall.close(e.getStatus(), e.getTrailers() == null ? new Metadata() : e.getTrailers()); + return new ServerCall.Listener() { }; + } catch (Exception e) { + serverCall.close(Status.UNAVAILABLE.withDescription("No backend available").withCause(e), new Metadata()); + return new ServerCall.Listener() { }; + } + + headerRewriter.rewrite(headers); + + final ClientCall clientCall = + channel.newCall(serverCall.getMethodDescriptor(), CallOptions.DEFAULT); + final CallProxy proxy = new CallProxy(serverCall, clientCall); + clientCall.start(proxy.clientCallListener, headers); + + // Prime both directions with a single outstanding message; each side then + // requests the next only when the other has taken the previous one. + serverCall.request(1); + clientCall.request(1); + return proxy.serverCallListener; + } + + /** + * Holds the two halves of the relay and the shared close latch. The client + * half forwards requests to the backend; the server half forwards responses + * back. + */ + private final class CallProxy { + + private final RequestProxy serverCallListener; + private final ResponseProxy clientCallListener; + /** + * A {@link ServerCall} may only be closed once, and both halves can race to + * close it: the backend can fail at the same moment a message interceptor + * rejects a request. Whoever wins reports the status. + */ + private final AtomicBoolean closed = new AtomicBoolean(); + + CallProxy(ServerCall serverCall, ClientCall clientCall) { + this.serverCallListener = new RequestProxy(clientCall); + this.clientCallListener = new ResponseProxy(serverCall); + } + + private void closeServerCall(ServerCall serverCall, Status status, Metadata trailers) { + if (closed.compareAndSet(false, true)) { + serverCall.close(status, trailers); + } + } + + /** Relays the client's request stream to the backend. */ + private final class RequestProxy extends ServerCall.Listener { + + private final ClientCall clientCall; + /** Guarded by {@code this}: a request is owed once the backend is writable again. */ + private boolean needToRequest; + + RequestProxy(ClientCall clientCall) { + this.clientCall = clientCall; + } + + @Override + public void onCancel() { + clientCall.cancel("Cancelled by client", null); + } + + @Override + public void onHalfClose() { + clientCall.halfClose(); + } + + @Override + public void onMessage(ReqT message) { + final ReqT forwarded; + try { + forwarded = messageInterceptor.intercept(message); + } catch (StatusRuntimeException e) { + // A gating decision, e.g. AddArtifacts denied or a write to a reserved + // config key. Reject without the backend ever seeing the message. + // + // Close the client first, then cancel the backend. Cancelling first + // would race: the backend's own onClose fires with CANCELLED — on a + // direct executor, synchronously — and would win the close latch, so + // the caller would see CANCELLED instead of why they were denied. + closeServerCall(clientCallListener.serverCall, e.getStatus(), + e.getTrailers() == null ? new Metadata() : e.getTrailers()); + clientCall.cancel(e.getStatus().getDescription(), e); + return; + } + + clientCall.sendMessage(forwarded); + synchronized (this) { + if (clientCall.isReady()) { + clientCallListener.serverCall.request(1); + } else { + // The backend is not writable; wait for onClientReady rather than + // pulling more from the client and buffering it here. + needToRequest = true; + } + } + } + + @Override + public void onReady() { + clientCallListener.onServerReady(); + } + + synchronized void onClientReady() { + if (needToRequest) { + clientCallListener.serverCall.request(1); + needToRequest = false; + } + } + } + + /** Relays the backend's response stream to the client. */ + private final class ResponseProxy extends ClientCall.Listener { + + private final ServerCall serverCall; + /** Guarded by {@code this}: a request is owed once the client is writable again. */ + private boolean needToRequest; + + ResponseProxy(ServerCall serverCall) { + this.serverCall = serverCall; + } + + @Override + public void onClose(Status status, Metadata trailers) { + closeServerCall(serverCall, status, trailers); + } + + @Override + public void onHeaders(Metadata headers) { + serverCall.sendHeaders(headers); + } + + @Override + public void onMessage(RespT message) { + serverCall.sendMessage(message); + synchronized (this) { + if (serverCall.isReady()) { + serverCallListener.clientCall.request(1); + } else { + needToRequest = true; + } + } + } + + @Override + public void onReady() { + serverCallListener.onClientReady(); + } + + synchronized void onServerReady() { + if (needToRequest) { + serverCallListener.clientCall.request(1); + needToRequest = false; + } + } + } + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java new file mode 100644 index 0000000000..ea0d8b7804 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import org.apache.commons.lang3.StringUtils; +import org.apache.knox.gateway.config.GatewayConfig; +import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.services.GatewayServices; +import org.apache.knox.gateway.services.ServiceType; +import org.apache.knox.gateway.services.registry.ServiceRegistry; + +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * Selects the topology for a call and resolves its backend. + *

+ * Knox normally routes on {@code /gateway/{topology}/{service}}, which is not + * available here: the Spark Connect connection string forbids a path component, + * and gRPC fixes request paths at {@code /pkg.Service/Method}. The topology + * therefore has to come from something else a vanilla client can send. Two + * discriminators are supported: + *

    + *
  1. a {@code knox-topology} metadata entry, which the client supplies as an + * extra {@code sc://} connection-string parameter;
    2. + *
  2. the configured default topology, for the single-topology case.
    3. + *
+ * Because the client chooses in case 1, this is only a routing decision, not an + * authorization one — the coarse ACL check downstream still gates whether the + * user may use the topology they asked for. + *

+ * Backend lookup then goes through the ordinary registry + * ({@code ServiceRegistry.lookupServiceURL}), so a Spark Connect backend is + * declared in topology XML like any other service. The registry treats service + * URLs as opaque strings, which is why a {@code grpc://} URL needs no special + * handling — the same property that already lets {@code ws://} URLs through. + */ +public class RoutingInterceptor implements ServerInterceptor { + + private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); + + private final GatewayConfig config; + private final GatewayServices services; + private final String serviceRole; + + public RoutingInterceptor(GatewayConfig config, GatewayServices services, String serviceRole) { + this.config = config; + this.services = services; + this.serviceRole = serviceRole; + } + + @Override + public ServerCall.Listener interceptCall(ServerCall call, + Metadata headers, + ServerCallHandler next) { + final String method = call.getMethodDescriptor().getFullMethodName(); + + final String topology = resolveTopology(headers); + if (StringUtils.isBlank(topology)) { + return reject(call, method, Status.UNIMPLEMENTED, + "no topology selected; set a knox-topology connection parameter or configure " + + "gateway.sparkconnect.default.topology"); + } + + final ServiceRegistry registry = services.getService(ServiceType.SERVICE_REGISTRY_SERVICE); + final String backendUrl = registry == null ? null : registry.lookupServiceURL(topology, serviceRole); + if (StringUtils.isBlank(backendUrl)) { + return reject(call, method, Status.UNAVAILABLE, + "topology " + topology + " declares no " + serviceRole + " service"); + } + + final GrpcCallContext callContext = GrpcCallContext.current(); + if (callContext != null) { + callContext.setTopology(topology); + callContext.setBackendUrl(backendUrl); + LOG.routingCall(method, callContext.getPrincipal(), topology, backendUrl); + } + return next.startCall(call, headers); + } + + private String resolveTopology(Metadata headers) { + final String requested = headers.get(GrpcMetadataKeys.TOPOLOGY); + if (StringUtils.isNotBlank(requested)) { + return requested.trim(); + } + return config.getSparkConnectDefaultTopology(); + } + + private ServerCall.Listener reject(ServerCall call, + String method, + Status status, + String reason) { + LOG.routingFailed(method, reason); + call.close(status.withDescription(reason), new Metadata()); + return new ServerCall.Listener() { }; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/TokenAuthenticator.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/TokenAuthenticator.java new file mode 100644 index 0000000000..bf8a4fd7be --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/TokenAuthenticator.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.security.interfaces.RSAPublicKey; +import java.text.ParseException; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +import javax.servlet.ServletException; + +import org.apache.knox.gateway.config.GatewayConfig; +import org.apache.knox.gateway.provider.federation.jwt.JWTValidator; +import org.apache.knox.gateway.provider.federation.jwt.filter.SignatureVerificationCache; +import org.apache.knox.gateway.services.GatewayServices; +import org.apache.knox.gateway.services.ServiceType; +import org.apache.knox.gateway.services.security.token.TokenStateService; +import org.apache.knox.gateway.services.security.token.impl.JWT; +import org.apache.knox.gateway.services.security.token.impl.JWTToken; +import org.apache.knox.gateway.services.topology.TopologyService; +import org.apache.knox.gateway.topology.Service; +import org.apache.knox.gateway.topology.Topology; +import org.apache.knox.gateway.util.CertificateUtils; + +/** + * Validates the bearer token a gRPC client presents, using the same machinery as + * the WebSocket listener. + *

+ * Bearer tokens are the whole of the credential vocabulary here, because that is + * all a vanilla Spark Connect client can carry: the {@code sc://} connection + * string offers a {@code token=} parameter, static metadata and TLS, and gRPC + * has no challenge-response step for SPNEGO to hook into. In a Kerberos + * deployment the user still authenticates with Kerberos — to the {@code + * knoxtoken} API, over HTTPS — and the resulting JWT acts as the delegation + * credential on the data path, exactly as delegation tokens do for HDFS. + *

+ * Validation covers issuer, expiry, not-before, signature and — when server + * managed token state is on — revocation, so an administrator can kill one + * long-running client's access without touching the principal. + */ +public class TokenAuthenticator { + + private static final String KNOXSSO_TOPOLOGY = "knoxsso"; + private static final String KNOXSSO_ROLE = "KNOXSSO"; + private static final String KNOXTOKEN_ROLE = "KNOXTOKEN"; + private static final String JWT_EXPECTED_ISSUER = "jwt.expected.issuer"; + private static final String JWT_EXPECTED_SIGALG = "jwt.expected.sigalg"; + private static final String SSO_VERIFICATION_PEM = "sso.token.verification.pem"; + /** Names the signature-verification cache; not a topology lookup. */ + private static final String CACHE_NAME = "sparkconnect"; + + private final GatewayConfig config; + private final GatewayServices services; + + public TokenAuthenticator(GatewayConfig config, GatewayServices services) { + this.config = config; + this.services = services; + } + + /** + * Validates a serialized JWT and returns the identity it establishes. + * + * @param serializedToken the bearer token from the call metadata + * @return the authenticated user + * @throws AuthenticationException if the token is malformed, expired, revoked, + * from an unexpected issuer, or fails signature verification + */ + public AuthenticatedUser authenticate(String serializedToken) throws AuthenticationException { + final JWT token; + try { + token = new JWTToken(serializedToken); + } catch (ParseException e) { + throw new AuthenticationException("Bearer token is not a well-formed JWT", e); + } + + final Map params = tokenProviderParams(); + final JWTValidator validator = new JWTValidator( + token, + services.getService(ServiceType.TOKEN_SERVICE), + SignatureVerificationCache.getInstance(CACHE_NAME, new MapFilterConfig(CACHE_NAME, params))); + + if (params.containsKey(SSO_VERIFICATION_PEM)) { + try { + final RSAPublicKey publicKey = CertificateUtils.parseRSAPublicKey(params.get(SSO_VERIFICATION_PEM)); + validator.setPublicKey(publicKey); + } catch (ServletException e) { + throw new AuthenticationException("Cannot parse the configured token verification key", e); + } + } + if (params.containsKey(JWT_EXPECTED_ISSUER)) { + validator.setExpectedIssuer(params.get(JWT_EXPECTED_ISSUER)); + } + if (params.containsKey(JWT_EXPECTED_SIGALG)) { + validator.setExpectedSigAlg(params.get(JWT_EXPECTED_SIGALG)); + } + if (isServerManagedTokenStateEnabled(params.get(TokenStateService.CONFIG_SERVER_MANAGED))) { + validator.setTokenStateService(services.getService(ServiceType.TOKEN_STATE_SERVICE)); + } + + if (!validator.validate()) { + throw new AuthenticationException("Bearer token failed validation"); + } + + final String principal = validator.getUsername(); + if (principal == null || principal.isEmpty()) { + throw new AuthenticationException("Bearer token carries no subject"); + } + return new AuthenticatedUser(principal, groupsFrom(token)); + } + + /** + * Reads the group claim the token issuer embedded, if any. A token minted + * without groups simply yields none, which the ACL check treats as "member of + * nothing" rather than an error. + */ + private static Set groupsFrom(JWT token) { + final Object claim = token.getClaimAsObject(JWTToken.KNOX_GROUPS_CLAIM); + if (claim == null) { + return Collections.emptySet(); + } + final Set groups = new LinkedHashSet<>(); + if (claim instanceof Collection) { + for (Object group : (Collection) claim) { + if (group != null) { + groups.add(String.valueOf(group)); + } + } + } else { + groups.add(String.valueOf(claim)); + } + return groups; + } + + /** + * Collects the token provider's parameters so this listener validates tokens + * on the same terms the servlet path does. The topology that issues tokens is + * preferred; a deployment that only runs KnoxSSO falls back to that. + */ + private Map tokenProviderParams() { + final TopologyService topologyService = services.getService(ServiceType.TOPOLOGY_SERVICE); + if (topologyService == null) { + return Collections.emptyMap(); + } + + Map ssoParams = null; + for (Topology topology : topologyService.getTopologies()) { + for (Service service : topology.getServices()) { + if (KNOXTOKEN_ROLE.equals(service.getRole())) { + return service.getParams(); + } + if (KNOXSSO_ROLE.equals(service.getRole()) && KNOXSSO_TOPOLOGY.equals(topology.getName())) { + ssoParams = service.getParams(); + } + } + } + return ssoParams == null ? Collections.emptyMap() : ssoParams; + } + + private boolean isServerManagedTokenStateEnabled(String providerParamValue) { + if (providerParamValue == null || providerParamValue.isEmpty()) { + return config != null && config.isServerManagedTokenStateEnabled(); + } + return Boolean.parseBoolean(providerParamValue); + } + + /** Signals that a call's credential did not establish an identity. */ + public static class AuthenticationException extends Exception { + private static final long serialVersionUID = 1L; + + public AuthenticationException(String message) { + super(message); + } + + public AuthenticationException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuard.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuard.java new file mode 100644 index 0000000000..6f3666b9b6 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuard.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.sparkconnect; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; + +import org.apache.knox.gateway.sparkconnect.SparkConnectMessageInterceptor.RequestGuard; + +import com.google.protobuf.Message; + +import io.grpc.Status; + +/** + * Controls who may upload artifacts through {@code AddArtifacts}. + *

+ * This is defense in depth, and it is worth being clear why it cannot be more + * than that. A shared Spark Connect server runs as one principal, and any + * user-supplied code — an uploaded jar, or an inline Python or Scala UDF + * embedded in a plan — executes inside that JVM with that principal's storage + * credentials. Such code can read data directly, bypassing any plan-level policy + * check, and could subvert an in-JVM authorization plugin. Session-scoped + * artifact classloaders isolate sessions from each other, not from the + * application's own privileges. + *

+ * So blocking artifact upload shrinks the attack surface; it does not create a + * boundary, because inline UDFs remain a path to the same capability. This is a + * property of plan-level enforcement in general rather than something the + * gateway introduces. Deployments needing a hard boundary want per-user backends + * instead. + */ +public class AddArtifactsGuard implements RequestGuard { + + /** Every user may upload artifacts. */ + public static final String MODE_ALLOW = "ALLOW"; + /** No user may upload artifacts. */ + public static final String MODE_DENY = "DENY"; + /** Only explicitly listed users may upload artifacts. */ + public static final String MODE_ALLOW_LISTED_USERS = "ALLOW_LISTED_USERS"; + + private final String mode; + private final Set allowedUsers; + + public AddArtifactsGuard(String mode, Collection allowedUsers) { + this.mode = mode == null ? MODE_ALLOW : mode.trim().toUpperCase(Locale.ROOT); + this.allowedUsers = allowedUsers == null + ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(allowedUsers)); + } + + /** + * Whether this guard would reject every call, letting the caller skip + * per-message work entirely. + * + * @return true if no user may upload artifacts + */ + public boolean deniesEveryone() { + return MODE_DENY.equals(mode); + } + + @Override + public void check(Message request, String principal) { + if (MODE_ALLOW.equals(mode)) { + return; + } + if (MODE_ALLOW_LISTED_USERS.equals(mode) && allowedUsers.contains(principal)) { + return; + } + throw Status.PERMISSION_DENIED + .withDescription("Uploading artifacts through Spark Connect is not permitted for this user") + .asRuntimeException(); + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuard.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuard.java new file mode 100644 index 0000000000..b7b89ef7cd --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuard.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.sparkconnect; + +import java.util.Locale; + +import org.apache.knox.gateway.sparkconnect.SparkConnectMessageInterceptor.RequestGuard; + +import com.google.protobuf.Message; + +import io.grpc.Status; + +/** + * Refuses client writes to session-configuration keys reserved for Knox. + *

+ * A deployment may publish the authenticated identity into the Spark session as + * a configuration entry, which downstream authorization then reads. If a client + * could overwrite that entry it could assume any identity it liked, so + * {@code Set} and {@code Unset} on the reserved prefix are rejected outright. + *

+ * This covers the structured path completely and cheaply, because the keys are + * named fields in the {@code Config} RPC. It does not cover + * {@code SET reserved.key=...} issued as SQL inside {@code ExecutePlan}, which + * would need plan-text inspection and would still be best-effort. That gap is + * the argument for the stronger server-side arrangement, where an interceptor in + * the Spark application recomputes the identity from {@code user_context} on + * every request: a value derived per request cannot be overwritten by a session + * {@code SET} at all. + */ +public class ReservedConfigGuard implements RequestGuard { + + private static final String OPERATION_FIELD = "operation"; + private static final String SET_FIELD = "set"; + private static final String UNSET_FIELD = "unset"; + private static final String PAIRS_FIELD = "pairs"; + private static final String KEYS_FIELD = "keys"; + private static final String KEY_FIELD = "key"; + + private final String reservedPrefix; + + public ReservedConfigGuard(String reservedPrefix) { + this.reservedPrefix = reservedPrefix == null ? "" : reservedPrefix.toLowerCase(Locale.ROOT); + } + + @Override + public void check(Message request, String principal) { + if (reservedPrefix.isEmpty()) { + return; + } + final Message operation = childMessage(request, OPERATION_FIELD); + if (operation == null) { + return; + } + + final Message set = childMessage(operation, SET_FIELD); + if (set != null) { + final com.google.protobuf.Descriptors.FieldDescriptor pairs = + set.getDescriptorForType().findFieldByName(PAIRS_FIELD); + if (pairs != null) { + final int count = set.getRepeatedFieldCount(pairs); + for (int i = 0; i < count; i++) { + final Message pair = (Message) set.getRepeatedField(pairs, i); + final com.google.protobuf.Descriptors.FieldDescriptor key = + pair.getDescriptorForType().findFieldByName(KEY_FIELD); + if (key != null) { + reject(String.valueOf(pair.getField(key))); + } + } + } + } + + final Message unset = childMessage(operation, UNSET_FIELD); + if (unset != null) { + final com.google.protobuf.Descriptors.FieldDescriptor keys = + unset.getDescriptorForType().findFieldByName(KEYS_FIELD); + if (keys != null) { + final int count = unset.getRepeatedFieldCount(keys); + for (int i = 0; i < count; i++) { + reject(String.valueOf(unset.getRepeatedField(keys, i))); + } + } + } + } + + private void reject(String key) { + if (key != null && key.toLowerCase(Locale.ROOT).startsWith(reservedPrefix)) { + throw Status.PERMISSION_DENIED + .withDescription("Session configuration keys beginning with '" + reservedPrefix + + "' are reserved by the gateway and cannot be set or unset by clients") + .asRuntimeException(); + } + } + + /** + * Returns a singular message-valued field only when it is actually present, so + * an absent branch of the {@code op_type} oneof does not read as an empty + * {@code Set}. + */ + private static Message childMessage(Message parent, String fieldName) { + final com.google.protobuf.Descriptors.FieldDescriptor field = + parent.getDescriptorForType().findFieldByName(fieldName); + if (field == null || !parent.hasField(field)) { + return null; + } + final Object value = parent.getField(field); + return value instanceof Message ? (Message) value : null; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectListener.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectListener.java new file mode 100644 index 0000000000..a0f6a665fc --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectListener.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.sparkconnect; + +import java.util.Collections; +import java.util.Set; + +import org.apache.knox.gateway.config.GatewayConfig; +import org.apache.knox.gateway.grpc.BackendChannelProvider; +import org.apache.knox.gateway.grpc.GrpcGatewayListener; +import org.apache.knox.gateway.grpc.GrpcListenerSettings; +import org.apache.knox.gateway.grpc.HeaderRewriter; +import org.apache.knox.gateway.grpc.MessageInterceptor; +import org.apache.knox.gateway.grpc.ProxyCallHandler; + +import com.google.protobuf.Message; + +import io.grpc.MethodDescriptor; +import io.grpc.ServerMethodDefinition; +import io.grpc.ServerServiceDefinition; + +import org.apache.spark.connect.proto.SparkConnectServiceGrpc; + +/** + * Fronts a Spark Connect server, the one concrete listener the gRPC template + * offers. + *

+ * Spark Connect is worth fronting because its server has essentially no + * authentication or authorization of its own — the project assumes a proxy + * supplies them — while Knox already fronts the surfaces around it. What Knox + * adds is authentication at the edge, an identity the client cannot forge, an + * audit trail, and topology-based routing. + *

+ * Clients need no code changes and no plugins. A vanilla connection string + * carries everything required: + * 

+ * sc://knox-host:15002/;use_ssl=true;token=<knox-jwt>;knox-topology=analytics
+ *
+ * {@code token=} becomes a standard bearer header (and forces TLS on), and any + * parameter the client does not recognise — {@code knox-topology} here — is sent + * as call metadata, which is what makes topology selection possible despite gRPC + * forbidding a path in the URL. + */ +// volatile: the message-level policy is captured at start-up and read by request +// threads thereafter. +@SuppressWarnings("PMD.AvoidUsingVolatile") +public class SparkConnectListener extends GrpcGatewayListener { + + private static final String LISTENER_NAME = "SparkConnect"; + private static final String SERVICE_ROLE = "SPARKCONNECT"; + private static final String PROTO_SERVICE_NAME = "spark.connect.SparkConnectService"; + + private static final String CONFIG_METHOD = "Config"; + private static final String ADD_ARTIFACTS_METHOD = "AddArtifacts"; + + private volatile String reservedConfigPrefix; + private volatile AddArtifactsGuard addArtifactsGuard; + + @Override + public String getName() { + return LISTENER_NAME; + } + + @Override + public boolean isEnabled(GatewayConfig config) { + return config.isSparkConnectEnabled(); + } + + @Override + protected String getServiceRole() { + return SERVICE_ROLE; + } + + @Override + protected GrpcListenerSettings createSettings(GatewayConfig config) { + // Capture the message-level policy at start-up, alongside the transport + // settings, so a call never has to consult configuration mid-stream. + this.reservedConfigPrefix = config.getSparkConnectReservedConfigPrefix(); + this.addArtifactsGuard = new AddArtifactsGuard( + config.getSparkConnectAddArtifactsMode(), + config.getSparkConnectAddArtifactsAllowedUsers()); + + return new GrpcListenerSettings() + .name(LISTENER_NAME) + .port(config.getSparkConnectPort()) + .maxMessageSize(config.getSparkConnectMaxMessageSize()) + .maxConcurrentCallsPerConnection(config.getSparkConnectMaxConcurrentCallsPerConnection()) + .permitKeepAliveTimeMillis(config.getSparkConnectPermitKeepAliveTime()) + .permitKeepAliveWithoutCalls(config.isSparkConnectPermitKeepAliveWithoutCalls()) + .channelIdleTimeoutMillis(config.getSparkConnectChannelIdleTimeout()) + .drainTimeoutMillis(config.getSparkConnectDrainTimeout()) + .backendTokenAlias(config.getSparkConnectBackendTokenAlias()); + } + + @Override + protected Set getPassthroughServiceNames() { + // Only methods of the Spark Connect service itself may fall back to a + // byte-level relay, and only when this build has no typed handler for them — + // which is how a client from a newer Spark line still gets proxied. + return Collections.singleton(PROTO_SERVICE_NAME); + } + + /** + * Registers a proxy handler for every method of {@code SparkConnectService}. + *

+ * There is one handler implementation rather than one per RPC shape. Unary, + * server-streaming and client-streaming calls differ only in message counts, + * which the relay handles uniformly, so the ten-odd methods need no bespoke + * code — just the right request interceptor each. + */ + @Override + protected ServerServiceDefinition bindService(BackendChannelProvider channels, + HeaderRewriter headers) { + final ServerServiceDefinition.Builder builder = + ServerServiceDefinition.builder(SparkConnectServiceGrpc.getServiceDescriptor()); + for (MethodDescriptor method : SparkConnectServiceGrpc.getServiceDescriptor().getMethods()) { + builder.addMethod(proxyMethod(method, channels, headers)); + } + return builder.build(); + } + + /** + * Erasure lets one relay serve every method: the generated marshallers still + * parse each message into its concrete type, and the interceptor only touches + * fields it looks up by name on the descriptor. + */ + @SuppressWarnings("unchecked") + private ServerMethodDefinition proxyMethod(MethodDescriptor method, + BackendChannelProvider channels, + HeaderRewriter headers) { + final MethodDescriptor descriptor = (MethodDescriptor) method; + final MessageInterceptor interceptor = + interceptorFor(bareMethodName(descriptor.getFullMethodName())); + return ServerMethodDefinition.create(descriptor, + new ProxyCallHandler<>(channels, interceptor, headers)); + } + + private MessageInterceptor interceptorFor(String methodName) { + if (CONFIG_METHOD.equals(methodName)) { + return new SparkConnectMessageInterceptor(new ReservedConfigGuard(reservedConfigPrefix)); + } + if (ADD_ARTIFACTS_METHOD.equals(methodName)) { + return new SparkConnectMessageInterceptor(addArtifactsGuard); + } + return new SparkConnectMessageInterceptor(null); + } + + private static String bareMethodName(String fullMethodName) { + final int separator = fullMethodName.lastIndexOf('/'); + return separator < 0 ? fullMethodName : fullMethodName.substring(separator + 1); + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptor.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptor.java new file mode 100644 index 0000000000..b27063bdf4 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptor.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.sparkconnect; + +import org.apache.knox.gateway.grpc.GrpcCallContext; +import org.apache.knox.gateway.grpc.MessageInterceptor; + +import com.google.protobuf.Descriptors.FieldDescriptor; +import com.google.protobuf.Message; + +import io.grpc.Status; + +/** + * Asserts the authenticated identity onto every request, and applies the + * per-RPC gating switches. + *

+ * Identity assertion is the reason this gateway parses messages at all. Spark + * Connect trusts a client-asserted {@code user_context.user_id}: the + * client simply states who it is, and the server believes it. Overwriting that + * field with the principal Knox authenticated closes a real spoofing hole, and + * it is something no byte-level proxy — or L4 passthrough, or generic sidecar — + * could do. + *

+ * Be precise about what it buys, though. On the server, {@code user_id} keys the + * session cache ({@code SessionKey(userId, sessionId)}) and appears in logs and + * events. It is not propagated into Spark's {@code CurrentUserContext}, + * so {@code current_user()} in SQL still reports the Spark application's own + * user. What assertion guarantees is session isolation between users and a + * trustworthy audit trail — not storage-level enforcement, which needs either + * per-user backends or a server-side component that bridges this field into the + * session. + */ +public class SparkConnectMessageInterceptor implements MessageInterceptor { + + private static final String USER_CONTEXT_FIELD = "user_context"; + private static final String SESSION_ID_FIELD = "session_id"; + private static final String OPERATION_ID_FIELD = "operation_id"; + private static final String USER_ID_FIELD = "user_id"; + private static final String USER_NAME_FIELD = "user_name"; + + private final RequestGuard guard; + + /** + * @param guard an extra check for this RPC, or null when identity assertion is + * all that applies + */ + public SparkConnectMessageInterceptor(RequestGuard guard) { + this.guard = guard; + } + + @Override + public Message intercept(Message message) { + final GrpcCallContext callContext = GrpcCallContext.current(); + final String principal = callContext == null ? null : callContext.getPrincipal(); + if (principal == null) { + // The authentication interceptor runs before any handler, so this cannot + // happen unless the chain was assembled wrongly. Fail rather than forward a + // request carrying whatever identity the client claimed. + throw Status.INTERNAL + .withDescription("No authenticated principal available for identity assertion") + .asRuntimeException(); + } + + recordCallDetails(message, callContext); + if (guard != null) { + guard.check(message, principal); + } + return assertIdentity(message, principal); + } + + /** + * Copies the session and operation identifiers into the call context so audit + * records can name the session a call belongs to. Every Spark Connect request + * carries {@code session_id}; only some carry {@code operation_id}. + */ + private static void recordCallDetails(Message message, GrpcCallContext callContext) { + if (callContext == null) { + return; + } + final FieldDescriptor sessionField = + message.getDescriptorForType().findFieldByName(SESSION_ID_FIELD); + if (sessionField != null) { + final Object sessionId = message.getField(sessionField); + if (sessionId instanceof String && !((String) sessionId).isEmpty()) { + callContext.setSessionId((String) sessionId); + } + } + final FieldDescriptor operationField = + message.getDescriptorForType().findFieldByName(OPERATION_ID_FIELD); + if (operationField != null && message.hasField(operationField)) { + final Object operationId = message.getField(operationField); + if (operationId instanceof String && !((String) operationId).isEmpty()) { + callContext.setOperationId((String) operationId); + } + } + } + + /** + * Replaces the client-supplied identity with the authenticated one. + *

+ * This works the same way for all twelve RPCs because every Spark Connect + * request message carries {@code UserContext user_context = 2} in the same + * position — so the rewrite is driven off the descriptor rather than written + * out once per message type. Fields the gateway does not touch, including ones + * from a newer Spark than these vendored protos describe, survive: protobuf + * retains unknown fields across a parse and re-serialize. + */ + @SuppressWarnings("unchecked") + static T assertIdentity(T message, String principal) { + final FieldDescriptor userContextField = + message.getDescriptorForType().findFieldByName(USER_CONTEXT_FIELD); + if (userContextField == null) { + return message; + } + + final Message userContext = (Message) message.getField(userContextField); + final FieldDescriptor userIdField = + userContext.getDescriptorForType().findFieldByName(USER_ID_FIELD); + final FieldDescriptor userNameField = + userContext.getDescriptorForType().findFieldByName(USER_NAME_FIELD); + + final Message.Builder userContextBuilder = userContext.toBuilder(); + if (userIdField != null) { + userContextBuilder.setField(userIdField, principal); + } + // The client's user_name is overwritten too: it is purely descriptive on the + // server, but leaving a self-asserted value would put a name Knox never + // verified into Spark's logs next to the id it did. + if (userNameField != null) { + userContextBuilder.setField(userNameField, principal); + } + + return (T) message.toBuilder() + .setField(userContextField, userContextBuilder.build()) + .build(); + } + + /** An additional per-RPC check applied before a request is forwarded. */ + @FunctionalInterface + public interface RequestGuard { + + /** + * @param message the request message + * @param principal the authenticated principal + * @throws io.grpc.StatusRuntimeException to reject the call + */ + void check(Message message, String principal); + } +} diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/base.proto b/gateway-service-sparkconnect/src/main/proto/spark/connect/base.proto new file mode 100644 index 0000000000..c7247129f1 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/proto/spark/connect/base.proto @@ -0,0 +1,1367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +import "google/protobuf/any.proto"; +import "spark/connect/commands.proto"; +import "spark/connect/common.proto"; +import "spark/connect/expressions.proto"; +import "spark/connect/relations.proto"; +import "spark/connect/types.proto"; +import "spark/connect/ml.proto"; +import "spark/connect/pipelines.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// A [[Plan]] is the structure that carries the runtime information for the execution from the +// client to the server. A [[Plan]] can be one of the following: +// - [[Relation]]: a reference to the underlying logical plan. +// - [[Command]]: used to execute commands on the server. +// - [[CompressedOperation]]: a compressed representation of either a Relation or a Command. +message Plan { + oneof op_type { + Relation root = 1; + Command command = 2; + CompressedOperation compressed_operation = 3; + } + + message CompressedOperation { + bytes data = 1; + OpType op_type = 2; + CompressionCodec compression_codec = 3; + + enum OpType { + OP_TYPE_UNSPECIFIED = 0; + OP_TYPE_RELATION = 1; + OP_TYPE_COMMAND = 2; + } + } +} + +// Compression codec for plan compression. +enum CompressionCodec { + COMPRESSION_CODEC_UNSPECIFIED = 0; + COMPRESSION_CODEC_ZSTD = 1; +} + +// User Context is used to refer to one particular user session that is executing +// queries in the backend. +message UserContext { + string user_id = 1; + string user_name = 2; + + // To extend the existing user context message that is used to identify incoming requests, + // Spark Connect leverages the Any protobuf type that can be used to inject arbitrary other + // messages into this message. Extensions are stored as a `repeated` type to be able to + // handle multiple active extensions. + repeated google.protobuf.Any extensions = 999; +} + +// Request to perform plan analyze, optionally to explain the plan. +message AnalyzePlanRequest { + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 17; + + // (Required) User context + UserContext user_context = 2; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + oneof analyze { + Schema schema = 4; + Explain explain = 5; + TreeString tree_string = 6; + IsLocal is_local = 7; + IsStreaming is_streaming = 8; + InputFiles input_files = 9; + SparkVersion spark_version = 10; + DDLParse ddl_parse = 11; + SameSemantics same_semantics = 12; + SemanticHash semantic_hash = 13; + Persist persist = 14; + Unpersist unpersist = 15; + GetStorageLevel get_storage_level = 16; + JsonToDDL json_to_ddl = 18; + } + + message Schema { + // (Required) The logical plan to be analyzed. + Plan plan = 1; + } + + // Explains the input plan based on a configurable mode. + message Explain { + // (Required) The logical plan to be analyzed. + Plan plan = 1; + + // (Required) For analyzePlan rpc calls, configure the mode to explain plan in strings. + ExplainMode explain_mode = 2; + + // Plan explanation mode. + enum ExplainMode { + EXPLAIN_MODE_UNSPECIFIED = 0; + + // Generates only physical plan. + EXPLAIN_MODE_SIMPLE = 1; + + // Generates parsed logical plan, analyzed logical plan, optimized logical plan and physical plan. + // Parsed Logical plan is a unresolved plan that extracted from the query. Analyzed logical plans + // transforms which translates unresolvedAttribute and unresolvedRelation into fully typed objects. + // The optimized logical plan transforms through a set of optimization rules, resulting in the + // physical plan. + EXPLAIN_MODE_EXTENDED = 2; + + // Generates code for the statement, if any and a physical plan. + EXPLAIN_MODE_CODEGEN = 3; + + // If plan node statistics are available, generates a logical plan and also the statistics. + EXPLAIN_MODE_COST = 4; + + // Generates a physical plan outline and also node details. + EXPLAIN_MODE_FORMATTED = 5; + } + } + + message TreeString { + // (Required) The logical plan to be analyzed. + Plan plan = 1; + + // (Optional) Max level of the schema. + optional int32 level = 2; + } + + message IsLocal { + // (Required) The logical plan to be analyzed. + Plan plan = 1; + } + + message IsStreaming { + // (Required) The logical plan to be analyzed. + Plan plan = 1; + } + + message InputFiles { + // (Required) The logical plan to be analyzed. + Plan plan = 1; + } + + message SparkVersion { } + + message DDLParse { + // (Required) The DDL formatted string to be parsed. + string ddl_string = 1; + } + + + // Returns `true` when the logical query plans are equal and therefore return same results. + message SameSemantics { + // (Required) The plan to be compared. + Plan target_plan = 1; + + // (Required) The other plan to be compared. + Plan other_plan = 2; + } + + message SemanticHash { + // (Required) The logical plan to get a hashCode. + Plan plan = 1; + } + + message Persist { + // (Required) The logical plan to persist. + Relation relation = 1; + + // (Optional) The storage level. + optional StorageLevel storage_level = 2; + } + + message Unpersist { + // (Required) The logical plan to unpersist. + Relation relation = 1; + + // (Optional) Whether to block until all blocks are deleted. + optional bool blocking = 2; + } + + message GetStorageLevel { + // (Required) The logical plan to get the storage level. + Relation relation = 1; + } + + message JsonToDDL { + // (Required) The JSON formatted string to be converted to DDL. + string json_string = 1; + } +} + +// Response to performing analysis of the query. Contains relevant metadata to be able to +// reason about the performance. +// Next ID: 16 +message AnalyzePlanResponse { + string session_id = 1; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 15; + + oneof result { + Schema schema = 2; + Explain explain = 3; + TreeString tree_string = 4; + IsLocal is_local = 5; + IsStreaming is_streaming = 6; + InputFiles input_files = 7; + SparkVersion spark_version = 8; + DDLParse ddl_parse = 9; + SameSemantics same_semantics = 10; + SemanticHash semantic_hash = 11; + Persist persist = 12; + Unpersist unpersist = 13; + GetStorageLevel get_storage_level = 14; + JsonToDDL json_to_ddl = 16; + } + + message Schema { + DataType schema = 1; + } + + message Explain { + string explain_string = 1; + } + + message TreeString { + string tree_string = 1; + } + + message IsLocal { + bool is_local = 1; + } + + message IsStreaming { + bool is_streaming = 1; + } + + message InputFiles { + // A best-effort snapshot of the files that compose this Dataset + repeated string files = 1; + } + + message SparkVersion { + string version = 1; + } + + message DDLParse { + DataType parsed = 1; + } + + message SameSemantics { + bool result = 1; + } + + message SemanticHash { + int32 result = 1; + } + + message Persist { } + + message Unpersist { } + + message GetStorageLevel { + // (Required) The StorageLevel as a result of get_storage_level request. + StorageLevel storage_level = 1; + } + + message JsonToDDL { + string ddl_string = 1; + } +} + +// A request to be executed by the service. +message ExecutePlanRequest { + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 8; + + // (Required) User context + // + // user_context.user_id and session+id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // (Optional) + // Provide an id for this request. If not provided, it will be generated by the server. + // It is returned in every ExecutePlanResponse.operation_id of the ExecutePlan response stream. + // The id must be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + optional string operation_id = 6; + + // (Required) The logical plan to be executed / analyzed. + Plan plan = 3; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 4; + + // Repeated element for options that can be passed to the request. This element is currently + // unused but allows to pass in an extension value used for arbitrary options. + repeated RequestOption request_options = 5; + + message RequestOption { + oneof request_option { + ReattachOptions reattach_options = 1; + ResultChunkingOptions result_chunking_options = 2; + // Extension type for request options + google.protobuf.Any extension = 999; + } + } + + // Tags to tag the given execution with. + // Tags cannot contain ',' character and cannot be empty strings. + // Used by Interrupt with interrupt.tag. + repeated string tags = 7; +} + +// The response of a query, can be one or more for each request. Responses belonging to the +// same input query, carry the same `session_id`. +// Next ID: 17 +message ExecutePlanResponse { + string session_id = 1; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 15; + + // Identifies the ExecutePlan execution. + // If set by the client in ExecutePlanRequest.operationId, that value is returned. + // Otherwise generated by the server. + // It is an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string operation_id = 12; + + // Identified the response in the stream. + // The id is an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string response_id = 13; + + // Union type for the different response messages. + oneof response_type { + ArrowBatch arrow_batch = 2; + + // Special case for executing SQL commands. + SqlCommandResult sql_command_result = 5; + + // Response for a streaming query. + WriteStreamOperationStartResult write_stream_operation_start_result = 8; + + // Response for commands on a streaming query. + StreamingQueryCommandResult streaming_query_command_result = 9; + + // Response for 'SparkContext.resources'. + GetResourcesCommandResult get_resources_command_result = 10; + + // Response for commands on the streaming query manager. + StreamingQueryManagerCommandResult streaming_query_manager_command_result = 11; + + // Response for commands on the client side streaming query listener. + StreamingQueryListenerEventsResult streaming_query_listener_events_result = 16; + + // Response type informing if the stream is complete in reattachable execution. + ResultComplete result_complete = 14; + + // Response for command that creates ResourceProfile. + CreateResourceProfileCommandResult create_resource_profile_command_result = 17; + + // (Optional) Intermediate query progress reports. + ExecutionProgress execution_progress = 18; + + // Response for command that checkpoints a DataFrame. + CheckpointCommandResult checkpoint_command_result = 19; + + // ML command response + MlCommandResult ml_command_result = 20; + + // Response containing pipeline event that is streamed back to the client during a pipeline run + PipelineEventResult pipeline_event_result = 21; + + // Pipeline command response + PipelineCommandResult pipeline_command_result = 22; + + // A signal from the server to the client to execute the query function for a flow, and to + // register its result with the server. + PipelineQueryFunctionExecutionSignal pipeline_query_function_execution_signal = 23; + + // Support arbitrary result objects. + google.protobuf.Any extension = 999; + } + + // Metrics for the query execution. Typically, this field is only present in the last + // batch of results and then represent the overall state of the query execution. + Metrics metrics = 4; + + // The metrics observed during the execution of the query plan. + repeated ObservedMetrics observed_metrics = 6; + + // (Optional) The Spark schema. This field is available when `collect` is called. + DataType schema = 7; + + // A SQL command returns an opaque Relation that can be directly used as input for the next + // call. + message SqlCommandResult { + Relation relation = 1; + } + + // Batch results of metrics. + message ArrowBatch { + // Count rows in `data`. Must match the number of rows inside `data`. + int64 row_count = 1; + // Serialized Arrow data. + bytes data = 2; + + // If set, row offset of the start of this ArrowBatch in execution results. + optional int64 start_offset = 3; + + // Index of this chunk in the batch if chunking is enabled. The index starts from 0. + optional int64 chunk_index = 4; + + // Total number of chunks in this batch if chunking is enabled. + // It is missing when chunking is disabled - the batch is returned whole + // and client will treat this response as the batch. + optional int64 num_chunks_in_batch = 5; + } + + message Metrics { + + repeated MetricObject metrics = 1; + + message MetricObject { + string name = 1; + int64 plan_id = 2; + int64 parent = 3; + map execution_metrics = 4; + } + + message MetricValue { + string name = 1; + int64 value = 2; + string metric_type = 3; + } + } + + message ObservedMetrics { + string name = 1; + repeated Expression.Literal values = 2; + repeated string keys = 3; + int64 plan_id = 4; + // (Optional) The index of the root error in errors. + // The field will not be set if there are no errors. + optional int32 root_error_idx = 5; + // A list of errors that occurred while collecting the observed metrics. + // If the length is 0, it means no errors occurred. + repeated FetchErrorDetailsResponse.Error errors = 6; + } + + message ResultComplete { + // If present, in a reattachable execution this means that after server sends onComplete, + // the execution is complete. If the server sends onComplete without sending a ResultComplete, + // it means that there is more, and the client should use ReattachExecute RPC to continue. + } + + // This message is used to communicate progress about the query progress during the execution. + message ExecutionProgress { + // Captures the progress of each individual stage. + repeated StageInfo stages = 1; + + // Captures the currently in progress tasks. + int64 num_inflight_tasks = 2; + + message StageInfo { + int64 stage_id = 1; + int64 num_tasks = 2; + int64 num_completed_tasks = 3; + int64 input_bytes_read = 4; + bool done = 5; + } + } +} + +// The key-value pair for the config request and response. +message KeyValue { + // (Required) The key. + string key = 1; + // (Optional) The value. + optional string value = 2; +} + +// Request to update or fetch the configurations. +message ConfigRequest { + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 8; + + // (Required) User context + UserContext user_context = 2; + + // (Required) The operation for the config. + Operation operation = 3; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 4; + + message Operation { + oneof op_type { + Set set = 1; + Get get = 2; + GetWithDefault get_with_default = 3; + GetOption get_option = 4; + GetAll get_all = 5; + Unset unset = 6; + IsModifiable is_modifiable = 7; + } + } + + message Set { + // (Required) The config key-value pairs to set. + repeated KeyValue pairs = 1; + + // (Optional) Whether to ignore failures. + optional bool silent = 2; + } + + message Get { + // (Required) The config keys to get. + repeated string keys = 1; + } + + message GetWithDefault { + // (Required) The config key-value pairs to get. The value will be used as the default value. + repeated KeyValue pairs = 1; + } + + message GetOption { + // (Required) The config keys to get optionally. + repeated string keys = 1; + } + + message GetAll { + // (Optional) The prefix of the config key to get. + optional string prefix = 1; + } + + message Unset { + // (Required) The config keys to unset. + repeated string keys = 1; + } + + message IsModifiable { + // (Required) The config keys to check the config is modifiable. + repeated string keys = 1; + } +} + +// Response to the config request. +// Next ID: 5 +message ConfigResponse { + string session_id = 1; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 4; + + // (Optional) The result key-value pairs. + // + // Available when the operation is 'Get', 'GetWithDefault', 'GetOption', 'GetAll'. + // Also available for the operation 'IsModifiable' with boolean string "true" and "false". + repeated KeyValue pairs = 2; + + // (Optional) + // + // Warning messages for deprecated or unsupported configurations. + repeated string warnings = 3; +} + +// Request to transfer client-local artifacts. +message AddArtifactsRequest { + + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // User context + UserContext user_context = 2; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 7; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 6; + + // A chunk of an Artifact. + message ArtifactChunk { + // Data chunk. + bytes data = 1; + // CRC to allow server to verify integrity of the chunk. + int64 crc = 2; + } + + // An artifact that is contained in a single `ArtifactChunk`. + // Generally, this message represents tiny artifacts such as REPL-generated class files. + message SingleChunkArtifact { + // The name of the artifact is expected in the form of a "Relative Path" that is made up of a + // sequence of directories and the final file element. + // Examples of "Relative Path"s: "jars/test.jar", "classes/xyz.class", "abc.xyz", "a/b/X.jar". + // The server is expected to maintain the hierarchy of files as defined by their name. (i.e + // The relative path of the file on the server's filesystem will be the same as the name of + // the provided artifact) + string name = 1; + // A single data chunk. + ArtifactChunk data = 2; + } + + // A number of `SingleChunkArtifact` batched into a single RPC. + message Batch { + repeated SingleChunkArtifact artifacts = 1; + } + + // Signals the beginning/start of a chunked artifact. + // A large artifact is transferred through a payload of `BeginChunkedArtifact` followed by a + // sequence of `ArtifactChunk`s. + message BeginChunkedArtifact { + // Name of the artifact undergoing chunking. Follows the same conventions as the `name` in + // the `Artifact` message. + string name = 1; + // Total size of the artifact in bytes. + int64 total_bytes = 2; + // Number of chunks the artifact is split into. + // This includes the `initial_chunk`. + int64 num_chunks = 3; + // The first/initial chunk. + ArtifactChunk initial_chunk = 4; + } + + // The payload is either a batch of artifacts or a partial chunk of a large artifact. + oneof payload { + Batch batch = 3; + // The metadata and the initial chunk of a large artifact chunked into multiple requests. + // The server side is notified about the total size of the large artifact as well as the + // number of chunks to expect. + BeginChunkedArtifact begin_chunk = 4; + // A chunk of an artifact excluding metadata. This can be any chunk of a large artifact + // excluding the first chunk (which is included in `BeginChunkedArtifact`). + ArtifactChunk chunk = 5; + } +} + +// Response to adding an artifact. Contains relevant metadata to verify successful transfer of +// artifact(s). +// Next ID: 4 +message AddArtifactsResponse { + // Session id in which the AddArtifact was running. + string session_id = 2; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 3; + + // The list of artifact(s) seen by the server. + repeated ArtifactSummary artifacts = 1; + + // Metadata of an artifact. + message ArtifactSummary { + string name = 1; + // Whether the CRC (Cyclic Redundancy Check) is successful on server verification. + // The server discards any artifact that fails the CRC. + // If false, the client may choose to resend the artifact specified by `name`. + bool is_crc_successful = 2; + } +} + +// Request to get current statuses of artifacts at the server side. +message ArtifactStatusesRequest { + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 5; + + // User context + UserContext user_context = 2; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + // The name of the artifact is expected in the form of a "Relative Path" that is made up of a + // sequence of directories and the final file element. + // Examples of "Relative Path"s: "jars/test.jar", "classes/xyz.class", "abc.xyz", "a/b/X.jar". + // The server is expected to maintain the hierarchy of files as defined by their name. (i.e + // The relative path of the file on the server's filesystem will be the same as the name of + // the provided artifact) + repeated string names = 4; +} + +// Response to checking artifact statuses. +// Next ID: 4 +message ArtifactStatusesResponse { + // Session id in which the ArtifactStatus was running. + string session_id = 2; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 3; + // A map of artifact names to their statuses. + map statuses = 1; + + message ArtifactStatus { + // Exists or not particular artifact at the server. + bool exists = 1; + } +} + +message InterruptRequest { + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 7; + + // (Required) User context + UserContext user_context = 2; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + // (Required) The type of interrupt to execute. + InterruptType interrupt_type = 4; + + enum InterruptType { + INTERRUPT_TYPE_UNSPECIFIED = 0; + + // Interrupt all running executions within the session with the provided session_id. + INTERRUPT_TYPE_ALL = 1; + + // Interrupt all running executions within the session with the provided operation_tag. + INTERRUPT_TYPE_TAG = 2; + + // Interrupt the running execution within the session with the provided operation_id. + INTERRUPT_TYPE_OPERATION_ID = 3; + } + + oneof interrupt { + // if interrupt_tag == INTERRUPT_TYPE_TAG, interrupt operation with this tag. + string operation_tag = 5; + + // if interrupt_tag == INTERRUPT_TYPE_OPERATION_ID, interrupt operation with this operation_id. + string operation_id = 6; + } +} + +// Next ID: 4 +message InterruptResponse { + // Session id in which the interrupt was running. + string session_id = 1; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 3; + + // Operation ids of the executions which were interrupted. + repeated string interrupted_ids = 2; + +} + +message ReattachOptions { + // If true, the request can be reattached to using ReattachExecute. + // ReattachExecute can be used either if the stream broke with a GRPC network error, + // or if the server closed the stream without sending a response with StreamStatus.complete=true. + // The server will keep a buffer of responses in case a response is lost, and + // ReattachExecute needs to back-track. + // + // If false, the execution response stream will will not be reattachable, and all responses are + // immediately released by the server after being sent. + bool reattachable = 1; +} + +message ResultChunkingOptions { + // Although Arrow results are split into batches with a size limit according to estimation, the + // size of the batches is not guaranteed to be less than the limit, especially when a single row + // is larger than the limit, in which case the server will fail to split it further into smaller + // batches. As a result, the client may encounter a gRPC error stating “Received message larger + // than max” when a batch is too large. + // If allow_arrow_batch_chunking=true, the server will split large Arrow batches into smaller chunks, + // and the client is expected to handle the chunked Arrow batches. + // + // If false, the server will not chunk large Arrow batches. + bool allow_arrow_batch_chunking = 1; + + // Optional preferred Arrow batch size in bytes for the server to use when sending Arrow results. + // The server will attempt to use this size if it is set and within the valid range + // ([1KB, max batch size on server]). Otherwise, the server's maximum batch size is used. + optional int64 preferred_arrow_chunk_size = 2; +} + +message ReattachExecuteRequest { + // (Required) + // + // The session_id of the request to reattach to. + // This must be an id of existing session. + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 6; + + // (Required) User context + // + // user_context.user_id and session+id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // (Required) + // Provide an id of the request to reattach to. + // This must be an id of existing operation. + string operation_id = 3; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 4; + + // (Optional) + // Last already processed response id from the response stream. + // After reattach, server will resume the response stream after that response. + // If not specified, server will restart the stream from the start. + // + // Note: server controls the amount of responses that it buffers and it may drop responses, + // that are far behind the latest returned response, so this can't be used to arbitrarily + // scroll back the cursor. If the response is no longer available, this will result in an error. + optional string last_response_id = 5; +} + +message ReleaseExecuteRequest { + // (Required) + // + // The session_id of the request to reattach to. + // This must be an id of existing session. + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 7; + + // (Required) User context + // + // user_context.user_id and session+id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // (Required) + // Provide an id of the request to reattach to. + // This must be an id of existing operation. + string operation_id = 3; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 4; + + // Release and close operation completely. + // This will also interrupt the query if it is running execution, and wait for it to be torn down. + message ReleaseAll {} + + // Release all responses from the operation response stream up to and including + // the response with the given by response_id. + // While server determines by itself how much of a buffer of responses to keep, client providing + // explicit release calls will help reduce resource consumption. + // Noop if response_id not found in cached responses. + message ReleaseUntil { + string response_id = 1; + } + + oneof release { + ReleaseAll release_all = 5; + ReleaseUntil release_until = 6; + } +} + +// Next ID: 4 +message ReleaseExecuteResponse { + // Session id in which the release was running. + string session_id = 1; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 3; + + // Operation id of the operation on which the release executed. + // If the operation couldn't be found (because e.g. it was concurrently released), will be unset. + // Otherwise, it will be equal to the operation_id from request. + optional string operation_id = 2; +} + +message ReleaseSessionRequest { + // (Required) + // + // The session_id of the request to reattach to. + // This must be an id of existing session. + string session_id = 1; + + // (Required) User context + // + // user_context.user_id and session+id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + // Signals the server to allow the client to reconnect to the session after it is released. + // + // By default, the server tombstones the session upon release, preventing reconnections and + // fully cleaning the session state. + // + // If this flag is set to true, the server may permit the client to reconnect to the session + // post-release, even if the session state has been cleaned. This can result in missing state, + // such as Temporary Views, Temporary UDFs, or the Current Catalog, in the reconnected session. + // + // Use this option sparingly and only when the client fully understands the implications of + // reconnecting to a released session. The client must ensure that any queries executed do not + // rely on the session state prior to its release. + bool allow_reconnect = 4; +} + +// Next ID: 3 +message ReleaseSessionResponse { + // Session id of the session on which the release executed. + string session_id = 1; + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 2; +} + +message FetchErrorDetailsRequest { + + // (Required) + // The session_id specifies a Spark session for a user identified by user_context.user_id. + // The id should be a UUID string of the format `00112233-4455-6677-8899-aabbccddeeff`. + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 5; + + // User context + UserContext user_context = 2; + + // (Required) + // The id of the error. + string error_id = 3; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 4; +} + +// Next ID: 5 +message FetchErrorDetailsResponse { + + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 3; + + string session_id = 4; + + // The index of the root error in errors. The field will not be set if the error is not found. + optional int32 root_error_idx = 1; + + // A list of errors. + repeated Error errors = 2; + + message StackTraceElement { + // The fully qualified name of the class containing the execution point. + string declaring_class = 1; + + // The name of the method containing the execution point. + string method_name = 2; + + // The name of the file containing the execution point. + optional string file_name = 3; + + // The line number of the source line containing the execution point. + int32 line_number = 4; + } + + // QueryContext defines the schema for the query context of a SparkThrowable. + // It helps users understand where the error occurs while executing queries. + message QueryContext { + // The type of this query context. + enum ContextType { + SQL = 0; + DATAFRAME = 1; + } + ContextType context_type = 10; + + // The object type of the query which throws the exception. + // If the exception is directly from the main query, it should be an empty string. + // Otherwise, it should be the exact object type in upper case. For example, a "VIEW". + string object_type = 1; + + // The object name of the query which throws the exception. + // If the exception is directly from the main query, it should be an empty string. + // Otherwise, it should be the object name. For example, a view name "V1". + string object_name = 2; + + // The starting index in the query text which throws the exception. The index starts from 0. + int32 start_index = 3; + + // The stopping index in the query which throws the exception. The index starts from 0. + int32 stop_index = 4; + + // The corresponding fragment of the query which throws the exception. + string fragment = 5; + + // The user code (call site of the API) that caused throwing the exception. + string call_site = 6; + + // Summary of the exception cause. + string summary = 7; + } + + // SparkThrowable defines the schema for SparkThrowable exceptions. + message SparkThrowable { + // Succinct, human-readable, unique, and consistent representation of the error category. + optional string error_class = 1; + + // The message parameters for the error framework. + map message_parameters = 2; + + // The query context of a SparkThrowable. + repeated QueryContext query_contexts = 3; + + // Portable error identifier across SQL engines + // If null, error class or SQLSTATE is not set. + optional string sql_state = 4; + + // Additional information if the error was caused by a breaking change. + optional BreakingChangeInfo breaking_change_info = 5; + } + + // BreakingChangeInfo defines the schema for breaking change information. + message BreakingChangeInfo { + // A message explaining how the user can migrate their job to work + // with the breaking change. + repeated string migration_message = 1; + + // A spark config flag that can be used to mitigate the breaking change. + optional MitigationConfig mitigation_config = 2; + + // If true, the breaking change should be inspected manually. + // If false, the spark job should be retried by setting the mitigationConfig. + optional bool needs_audit = 3; + } + + // MitigationConfig defines a spark config flag that can be used to mitigate a breaking change. + message MitigationConfig { + // The spark config key. + string key = 1; + + // The spark config value that mitigates the breaking change. + string value = 2; + } + + // Error defines the schema for the representing exception. + message Error { + // The fully qualified names of the exception class and its parent classes. + repeated string error_type_hierarchy = 1; + + // The detailed message of the exception. + string message = 2; + + // The stackTrace of the exception. It will be set + // if the SQLConf spark.sql.connect.serverStacktrace.enabled is true. + repeated StackTraceElement stack_trace = 3; + + // The index of the cause error in errors. + optional int32 cause_idx = 4; + + // The structured data of a SparkThrowable exception. + optional SparkThrowable spark_throwable = 5; + } +} + +message CheckpointCommandResult { + // (Required) The logical plan checkpointed. + CachedRemoteRelation relation = 1; +} + +message CloneSessionRequest { + // (Required) + // + // The session_id specifies a spark session for a user id (which is specified + // by user_context.user_id). The session_id is set by the client to be able to + // collate streaming responses from different queries within the dedicated session. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 5; + + // (Required) User context + // + // user_context.user_id and session_id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + // (Optional) + // The session_id for the new cloned session. If not provided, a new UUID will be generated. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + optional string new_session_id = 4; +} + +// Next ID: 5 +message CloneSessionResponse { + // Session id of the original session that was cloned. + string session_id = 1; + + // Server-side generated idempotency key that the client can use to assert that the server side + // session (parent of the cloned session) has not changed. + string server_side_session_id = 2; + + // Session id of the new cloned session. + string new_session_id = 3; + + // Server-side session ID of the new cloned session. + string new_server_side_session_id = 4; +} + +// Next ID: 6 +message GetStatusRequest { + // (Required) + // + // The session_id specifies a Spark session for a user identified by user_context.user_id. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Required) + // + // user_context.user_id and session_id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // (Optional) + // + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 4; + + // (Optional) + // + // Get status of operations in the session. + optional OperationStatusRequest operation_status = 5; + + // Extension point for custom status request types. + repeated google.protobuf.Any extensions = 999; + + message OperationStatusRequest { + // Get status of operations with these operation_ids. + // If unset or empty, returns status of all operations in the session. + repeated string operation_ids = 1; + + // Extension point for custom operation-level status requests. + repeated google.protobuf.Any extensions = 999; + } +} + +// Next ID: 4 +message GetStatusResponse { + // Session id of the session for which the status was requested. + string session_id = 1; + + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 2; + + // Status information about requested operations. + repeated OperationStatus operation_statuses = 3; + + // Extension point for custom status response types. + repeated google.protobuf.Any extensions = 999; + + // Status information for a single operation. + message OperationStatus { + // The operation_id of the operation. + string operation_id = 1; + + // The current status of the operation. + OperationState state = 2; + + // Extension point for custom operation-level status fields. + repeated google.protobuf.Any extensions = 999; + + enum OperationState { + OPERATION_STATE_UNSPECIFIED = 0; + OPERATION_STATE_UNKNOWN = 1; + OPERATION_STATE_RUNNING = 2; + OPERATION_STATE_TERMINATING = 3; + OPERATION_STATE_SUCCEEDED = 4; + OPERATION_STATE_FAILED = 5; + OPERATION_STATE_CANCELLED = 6; + } + } +} + +// Main interface for the SparkConnect service. +service SparkConnectService { + + // Executes a request that contains the query and returns a stream of [[Response]]. + // + // It is guaranteed that there is at least one ARROW batch returned even if the result set is empty. + rpc ExecutePlan(ExecutePlanRequest) returns (stream ExecutePlanResponse) {} + + // Analyzes a query and returns a [[AnalyzeResponse]] containing metadata about the query. + rpc AnalyzePlan(AnalyzePlanRequest) returns (AnalyzePlanResponse) {} + + // Update or fetch the configurations and returns a [[ConfigResponse]] containing the result. + rpc Config(ConfigRequest) returns (ConfigResponse) {} + + // Add artifacts to the session and returns a [[AddArtifactsResponse]] containing metadata about + // the added artifacts. + rpc AddArtifacts(stream AddArtifactsRequest) returns (AddArtifactsResponse) {} + + // Check statuses of artifacts in the session and returns them in a [[ArtifactStatusesResponse]] + rpc ArtifactStatus(ArtifactStatusesRequest) returns (ArtifactStatusesResponse) {} + + // Interrupts running executions + rpc Interrupt(InterruptRequest) returns (InterruptResponse) {} + + // Reattach to an existing reattachable execution. + // The ExecutePlan must have been started with ReattachOptions.reattachable=true. + // If the ExecutePlanResponse stream ends without a ResultComplete message, there is more to + // continue. If there is a ResultComplete, the client should use ReleaseExecute with + rpc ReattachExecute(ReattachExecuteRequest) returns (stream ExecutePlanResponse) {} + + // Release an reattachable execution, or parts thereof. + // The ExecutePlan must have been started with ReattachOptions.reattachable=true. + // Non reattachable executions are released automatically and immediately after the ExecutePlan + // RPC and ReleaseExecute may not be used. + rpc ReleaseExecute(ReleaseExecuteRequest) returns (ReleaseExecuteResponse) {} + + // Release a session. + // All the executions in the session will be released. Any further requests for the session with + // that session_id for the given user_id will fail. If the session didn't exist or was already + // released, this is a noop. + rpc ReleaseSession(ReleaseSessionRequest) returns (ReleaseSessionResponse) {} + + // FetchErrorDetails retrieves the matched exception with details based on a provided error id. + rpc FetchErrorDetails(FetchErrorDetailsRequest) returns (FetchErrorDetailsResponse) {} + + // Create a clone of a Spark Connect session on the server side. The server-side session + // is cloned with all its current state (SQL configurations, temporary views, registered + // functions, catalog state) copied over to a new independent session. The cloned session + // is isolated from the source session - any subsequent changes to either session's + // server-side state will not be reflected in the other. + // + // The request can optionally specify a custom session ID for the cloned session (must be + // a valid UUID). If not provided, a new UUID will be generated automatically. + rpc CloneSession(CloneSessionRequest) returns (CloneSessionResponse) {} + + // Get status information of different types. + rpc GetStatus(GetStatusRequest) returns (GetStatusResponse) {} +} diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/catalog.proto b/gateway-service-sparkconnect/src/main/proto/spark/connect/catalog.proto new file mode 100644 index 0000000000..b5341d16eb --- /dev/null +++ b/gateway-service-sparkconnect/src/main/proto/spark/connect/catalog.proto @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +import "spark/connect/common.proto"; +import "spark/connect/types.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// Catalog messages are marked as unstable. +message Catalog { + oneof cat_type { + CurrentDatabase current_database = 1; + SetCurrentDatabase set_current_database = 2; + ListDatabases list_databases = 3; + ListTables list_tables = 4; + ListFunctions list_functions = 5; + ListColumns list_columns = 6; + GetDatabase get_database = 7; + GetTable get_table = 8; + GetFunction get_function = 9; + DatabaseExists database_exists = 10; + TableExists table_exists = 11; + FunctionExists function_exists = 12; + CreateExternalTable create_external_table = 13; + CreateTable create_table = 14; + DropTempView drop_temp_view = 15; + DropGlobalTempView drop_global_temp_view = 16; + RecoverPartitions recover_partitions = 17; + IsCached is_cached = 18; + CacheTable cache_table = 19; + UncacheTable uncache_table = 20; + ClearCache clear_cache = 21; + RefreshTable refresh_table = 22; + RefreshByPath refresh_by_path = 23; + CurrentCatalog current_catalog = 24; + SetCurrentCatalog set_current_catalog = 25; + ListCatalogs list_catalogs = 26; + DropTable drop_table = 27; + DropView drop_view = 28; + CreateDatabase create_database = 29; + DropDatabase drop_database = 30; + ListPartitions list_partitions = 31; + ListViews list_views = 32; + GetTableProperties get_table_properties = 33; + GetCreateTableString get_create_table_string = 34; + TruncateTable truncate_table = 35; + AnalyzeTable analyze_table = 36; + } +} + +// See `spark.catalog.currentDatabase` +message CurrentDatabase { } + +// See `spark.catalog.setCurrentDatabase` +message SetCurrentDatabase { + // (Required) + string db_name = 1; +} + +// See `spark.catalog.listDatabases` +message ListDatabases { + // (Optional) The pattern that the database name needs to match + optional string pattern = 1; +} + +// See `spark.catalog.listTables` +message ListTables { + // (Optional) + optional string db_name = 1; + // (Optional) The pattern that the table name needs to match + optional string pattern = 2; +} + +// See `spark.catalog.listFunctions` +message ListFunctions { + // (Optional) + optional string db_name = 1; + // (Optional) The pattern that the function name needs to match + optional string pattern = 2; +} + +// See `spark.catalog.listColumns` +message ListColumns { + // (Required) + string table_name = 1; + // (Optional) + optional string db_name = 2; +} + +// See `spark.catalog.getDatabase` +message GetDatabase { + // (Required) + string db_name = 1; +} + +// See `spark.catalog.getTable` +message GetTable { + // (Required) + string table_name = 1; + // (Optional) + optional string db_name = 2; +} + +// See `spark.catalog.getFunction` +message GetFunction { + // (Required) + string function_name = 1; + // (Optional) + optional string db_name = 2; +} + +// See `spark.catalog.databaseExists` +message DatabaseExists { + // (Required) + string db_name = 1; +} + +// See `spark.catalog.tableExists` +message TableExists { + // (Required) + string table_name = 1; + // (Optional) + optional string db_name = 2; +} + +// See `spark.catalog.functionExists` +message FunctionExists { + // (Required) + string function_name = 1; + // (Optional) + optional string db_name = 2; +} + +// See `spark.catalog.createExternalTable` +message CreateExternalTable { + // (Required) + string table_name = 1; + // (Optional) + optional string path = 2; + // (Optional) + optional string source = 3; + // (Optional) + optional DataType schema = 4; + // Options could be empty for valid data source format. + // The map key is case insensitive. + map options = 5; +} + +// See `spark.catalog.createTable` +message CreateTable { + // (Required) + string table_name = 1; + // (Optional) + optional string path = 2; + // (Optional) + optional string source = 3; + // (Optional) + optional string description = 4; + // (Optional) + optional DataType schema = 5; + // Options could be empty for valid data source format. + // The map key is case insensitive. + map options = 6; +} + +// See `spark.catalog.dropTempView` +message DropTempView { + // (Required) + string view_name = 1; +} + +// See `spark.catalog.dropGlobalTempView` +message DropGlobalTempView { + // (Required) + string view_name = 1; +} + +// See `spark.catalog.recoverPartitions` +message RecoverPartitions { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.isCached` +message IsCached { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.cacheTable` +message CacheTable { + // (Required) + string table_name = 1; + + // (Optional) + optional StorageLevel storage_level = 2; +} + +// See `spark.catalog.uncacheTable` +message UncacheTable { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.clearCache` +message ClearCache { } + +// See `spark.catalog.refreshTable` +message RefreshTable { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.refreshByPath` +message RefreshByPath { + // (Required) + string path = 1; +} + +// See `spark.catalog.currentCatalog` +message CurrentCatalog { } + +// See `spark.catalog.setCurrentCatalog` +message SetCurrentCatalog { + // (Required) + string catalog_name = 1; +} + +// See `spark.catalog.listCatalogs` +message ListCatalogs { + // (Optional) The pattern that the catalog name needs to match + optional string pattern = 1; +} + +// See `spark.catalog.dropTable` +message DropTable { + // (Required) + string table_name = 1; + bool if_exists = 2; + bool purge = 3; +} + +// See `spark.catalog.dropView` +message DropView { + // (Required) + string view_name = 1; + bool if_exists = 2; +} + +// See `spark.catalog.createDatabase` +message CreateDatabase { + // (Required) + string db_name = 1; + bool if_not_exists = 2; + map properties = 3; +} + +// See `spark.catalog.dropDatabase` +message DropDatabase { + // (Required) + string db_name = 1; + bool if_exists = 2; + bool cascade = 3; +} + +// See `spark.catalog.listPartitions` +message ListPartitions { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.listViews` +message ListViews { + // (Optional) + optional string db_name = 1; + // (Optional) The pattern that the view name needs to match + optional string pattern = 2; +} + +// See `spark.catalog.getTableProperties` +message GetTableProperties { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.getCreateTableString` +message GetCreateTableString { + // (Required) + string table_name = 1; + bool as_serde = 2; +} + +// See `spark.catalog.truncateTable` +message TruncateTable { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.analyzeTable` +message AnalyzeTable { + // (Required) + string table_name = 1; + bool no_scan = 2; +} diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/commands.proto b/gateway-service-sparkconnect/src/main/proto/spark/connect/commands.proto new file mode 100644 index 0000000000..dcf5aff236 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/proto/spark/connect/commands.proto @@ -0,0 +1,560 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +import "google/protobuf/any.proto"; +import "spark/connect/common.proto"; +import "spark/connect/expressions.proto"; +import "spark/connect/relations.proto"; +import "spark/connect/ml.proto"; +import "spark/connect/pipelines.proto"; + +package spark.connect; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// A [[Command]] is an operation that is executed by the server that does not directly consume or +// produce a relational result. +message Command { + oneof command_type { + CommonInlineUserDefinedFunction register_function = 1; + WriteOperation write_operation = 2; + CreateDataFrameViewCommand create_dataframe_view = 3; + WriteOperationV2 write_operation_v2 = 4; + SqlCommand sql_command = 5; + WriteStreamOperationStart write_stream_operation_start = 6; + StreamingQueryCommand streaming_query_command = 7; + GetResourcesCommand get_resources_command = 8; + StreamingQueryManagerCommand streaming_query_manager_command = 9; + CommonInlineUserDefinedTableFunction register_table_function = 10; + StreamingQueryListenerBusCommand streaming_query_listener_bus_command = 11; + CommonInlineUserDefinedDataSource register_data_source = 12; + CreateResourceProfileCommand create_resource_profile_command = 13; + CheckpointCommand checkpoint_command = 14; + RemoveCachedRemoteRelationCommand remove_cached_remote_relation_command = 15; + MergeIntoTableCommand merge_into_table_command = 16; + MlCommand ml_command = 17; + ExecuteExternalCommand execute_external_command = 18; + PipelineCommand pipeline_command = 19; + + // This field is used to mark extensions to the protocol. When plugins generate arbitrary + // Commands they can add them here. During the planning the correct resolution is done. + google.protobuf.Any extension = 999; + + } +} + +// A SQL Command is used to trigger the eager evaluation of SQL commands in Spark. +// +// When the SQL provide as part of the message is a command it will be immediately evaluated +// and the result will be collected and returned as part of a LocalRelation. If the result is +// not a command, the operation will simply return a SQL Relation. This allows the client to be +// almost oblivious to the server-side behavior. +message SqlCommand { + // (Required) SQL Query. + string sql = 1 [deprecated=true]; + + // (Optional) A map of parameter names to literal expressions. + map args = 2 [deprecated=true]; + + // (Optional) A sequence of literal expressions for positional parameters in the SQL query text. + repeated Expression.Literal pos_args = 3 [deprecated=true]; + + // (Optional) A map of parameter names to expressions. + // It cannot coexist with `pos_arguments`. + map named_arguments = 4 [deprecated=true]; + + // (Optional) A sequence of expressions for positional parameters in the SQL query text. + // It cannot coexist with `named_arguments`. + repeated Expression pos_arguments = 5 [deprecated=true]; + + // (Optional) The relation that this SQL command will be built on. + Relation input = 6; +} + +// A command that can create DataFrame global temp view or local temp view. +message CreateDataFrameViewCommand { + // (Required) The relation that this view will be built on. + Relation input = 1; + + // (Required) View name. + string name = 2; + + // (Required) Whether this is global temp view or local temp view. + bool is_global = 3; + + // (Required) + // + // If true, and if the view already exists, updates it; if false, and if the view + // already exists, throws exception. + bool replace = 4; +} + +// As writes are not directly handled during analysis and planning, they are modeled as commands. +message WriteOperation { + // (Required) The output of the `input` relation will be persisted according to the options. + Relation input = 1; + + // (Optional) Format value according to the Spark documentation. Examples are: text, parquet, delta. + optional string source = 2; + + // (Optional) + // + // The destination of the write operation can be either a path or a table. + // If the destination is neither a path nor a table, such as jdbc and noop, + // the `save_type` should not be set. + oneof save_type { + string path = 3; + SaveTable table = 4; + } + + // (Required) the save mode. + SaveMode mode = 5; + + // (Optional) List of columns to sort the output by. + repeated string sort_column_names = 6; + + // (Optional) List of columns for partitioning. + repeated string partitioning_columns = 7; + + // (Optional) Bucketing specification. Bucketing must set the number of buckets and the columns + // to bucket by. + BucketBy bucket_by = 8; + + // (Optional) A list of configuration options. + map options = 9; + + // (Optional) Columns used for clustering the table. + repeated string clustering_columns = 10; + + // (Optional) Whether schema evolution is enabled for the write. + bool with_schema_evolution = 11; + + message SaveTable { + // (Required) The table name. + string table_name = 1; + // (Required) The method to be called to write to the table. + TableSaveMethod save_method = 2; + + enum TableSaveMethod { + TABLE_SAVE_METHOD_UNSPECIFIED = 0; + TABLE_SAVE_METHOD_SAVE_AS_TABLE = 1; + TABLE_SAVE_METHOD_INSERT_INTO = 2; + } + } + + message BucketBy { + repeated string bucket_column_names = 1; + int32 num_buckets = 2; + } + + enum SaveMode { + SAVE_MODE_UNSPECIFIED = 0; + SAVE_MODE_APPEND = 1; + SAVE_MODE_OVERWRITE = 2; + SAVE_MODE_ERROR_IF_EXISTS = 3; + SAVE_MODE_IGNORE = 4; + } +} + +// As writes are not directly handled during analysis and planning, they are modeled as commands. +message WriteOperationV2 { + // (Required) The output of the `input` relation will be persisted according to the options. + Relation input = 1; + + // (Required) The destination of the write operation must be either a path or a table. + string table_name = 2; + + // (Optional) A provider for the underlying output data source. Spark's default catalog supports + // "parquet", "json", etc. + optional string provider = 3; + + // (Optional) List of columns for partitioning for output table created by `create`, + // `createOrReplace`, or `replace` + repeated Expression partitioning_columns = 4; + + // (Optional) A list of configuration options. + map options = 5; + + // (Optional) A list of table properties. + map table_properties = 6; + + // (Required) Write mode. + Mode mode = 7; + + enum Mode { + MODE_UNSPECIFIED = 0; + MODE_CREATE = 1; + MODE_OVERWRITE = 2; + MODE_OVERWRITE_PARTITIONS = 3; + MODE_APPEND = 4; + MODE_REPLACE = 5; + MODE_CREATE_OR_REPLACE = 6; + } + + // (Optional) A condition for overwrite saving mode + Expression overwrite_condition = 8; + + // (Optional) Columns used for clustering the table. + repeated string clustering_columns = 9; + + // (Optional) Whether schema evolution is enabled for the write. + bool with_schema_evolution = 10; +} + +// Starts write stream operation as streaming query. Query ID and Run ID of the streaming +// query are returned. +message WriteStreamOperationStart { + + // (Required) The output of the `input` streaming relation will be written. + Relation input = 1; + + // The following fields directly map to API for DataStreamWriter(). + // Consult API documentation unless explicitly documented here. + + string format = 2; + map options = 3; + repeated string partitioning_column_names = 4; + + oneof trigger { + string processing_time_interval = 5; + bool available_now = 6; + bool once = 7; + string continuous_checkpoint_interval = 8; + string real_time_batch_duration = 100; + } + + string output_mode = 9; + string query_name = 10; + + // The destination is optional. When set, it can be a path or a table name. + oneof sink_destination { + string path = 11; + string table_name = 12; + } + + StreamingForeachFunction foreach_writer = 13; + StreamingForeachFunction foreach_batch = 14; + + // (Optional) Columns used for clustering the table. + repeated string clustering_column_names = 15; +} + +message StreamingForeachFunction { + oneof function { + PythonUDF python_function = 1; + ScalarScalaUDF scala_function = 2; + } +} + +message WriteStreamOperationStartResult { + + // (Required) Query instance. See `StreamingQueryInstanceId`. + StreamingQueryInstanceId query_id = 1; + + // An optional query name. + string name = 2; + + // Optional query started event if there is any listener registered on the client side. + optional string query_started_event_json = 3; + + // TODO: How do we indicate errors? + // TODO: Consider adding status, last progress etc here. +} + +// A tuple that uniquely identifies an instance of streaming query run. It consists of `id` that +// persists across the streaming runs and `run_id` that changes between each run of the +// streaming query that resumes from the checkpoint. +message StreamingQueryInstanceId { + + // (Required) The unique id of this query that persists across restarts from checkpoint data. + // That is, this id is generated when a query is started for the first time, and + // will be the same every time it is restarted from checkpoint data. + string id = 1; + + // (Required) The unique id of this run of the query. That is, every start/restart of a query + // will generate a unique run_id. Therefore, every time a query is restarted from + // checkpoint, it will have the same `id` but different `run_id`s. + string run_id = 2; +} + +// Commands for a streaming query. +message StreamingQueryCommand { + + // (Required) Query instance. See `StreamingQueryInstanceId`. + StreamingQueryInstanceId query_id = 1; + + // See documentation for the corresponding API method in StreamingQuery. + oneof command { + // status() API. + bool status = 2; + // lastProgress() API. + bool last_progress = 3; + // recentProgress() API. + bool recent_progress = 4; + // stop() API. Stops the query. + bool stop = 5; + // processAllAvailable() API. Waits till all the available data is processed + bool process_all_available = 6; + // explain() API. Returns logical and physical plans. + ExplainCommand explain = 7; + // exception() API. Returns the exception in the query if any. + bool exception = 8; + // awaitTermination() API. Waits for the termination of the query. + AwaitTerminationCommand await_termination = 9; + } + + message ExplainCommand { + // TODO: Consider reusing Explain from AnalyzePlanRequest message. + // We can not do this right now since it base.proto imports this file. + bool extended = 1; + } + + message AwaitTerminationCommand { + optional int64 timeout_ms = 2; + } +} + +// Response for commands on a streaming query. +message StreamingQueryCommandResult { + // (Required) Query instance id. See `StreamingQueryInstanceId`. + StreamingQueryInstanceId query_id = 1; + + oneof result_type { + StatusResult status = 2; + RecentProgressResult recent_progress = 3; + ExplainResult explain = 4; + ExceptionResult exception = 5; + AwaitTerminationResult await_termination = 6; + } + + message StatusResult { + // See documentation for these Scala 'StreamingQueryStatus' struct + string status_message = 1; + bool is_data_available = 2; + bool is_trigger_active = 3; + bool is_active = 4; + } + + message RecentProgressResult { + // Progress reports as an array of json strings. + repeated string recent_progress_json = 5; + } + + message ExplainResult { + // Logical and physical plans as string + string result = 1; + } + + message ExceptionResult { + // (Optional) Exception message as string, maps to the return value of original + // StreamingQueryException's toString method + optional string exception_message = 1; + // (Optional) Exception error class as string + optional string error_class = 2; + // (Optional) Exception stack trace as string + optional string stack_trace = 3; + } + + message AwaitTerminationResult { + bool terminated = 1; + } +} + +// Commands for the streaming query manager. +message StreamingQueryManagerCommand { + + // See documentation for the corresponding API method in StreamingQueryManager. + oneof command { + // active() API, returns a list of active queries. + bool active = 1; + // get() API, returns the StreamingQuery identified by id. + string get_query = 2; + // awaitAnyTermination() API, wait until any query terminates or timeout. + AwaitAnyTerminationCommand await_any_termination = 3; + // resetTerminated() API. + bool reset_terminated = 4; + // addListener API. + StreamingQueryListenerCommand add_listener = 5; + // removeListener API. + StreamingQueryListenerCommand remove_listener = 6; + // listListeners() API, returns a list of streaming query listeners. + bool list_listeners = 7; + } + + message AwaitAnyTerminationCommand { + // (Optional) The waiting time in milliseconds to wait for any query to terminate. + optional int64 timeout_ms = 1; + } + + message StreamingQueryListenerCommand { + bytes listener_payload = 1; + optional PythonUDF python_listener_payload = 2; + string id = 3; + } +} + +// Response for commands on the streaming query manager. +message StreamingQueryManagerCommandResult { + oneof result_type { + ActiveResult active = 1; + StreamingQueryInstance query = 2; + AwaitAnyTerminationResult await_any_termination = 3; + bool reset_terminated = 4; + bool add_listener = 5; + bool remove_listener = 6; + ListStreamingQueryListenerResult list_listeners = 7; + } + + message ActiveResult { + repeated StreamingQueryInstance active_queries = 1; + } + + message StreamingQueryInstance { + // (Required) The id and runId of this query. + StreamingQueryInstanceId id = 1; + // (Optional) The name of this query. + optional string name = 2; + } + + message AwaitAnyTerminationResult { + bool terminated = 1; + } + + message StreamingQueryListenerInstance { + bytes listener_payload = 1; + } + + message ListStreamingQueryListenerResult { + // (Required) Reference IDs of listener instances. + repeated string listener_ids = 1; + } +} + +// The protocol for client-side StreamingQueryListener. +// This command will only be set when either the first listener is added to the client, or the last +// listener is removed from the client. +// The add_listener_bus_listener command will only be set true in the first case. +// The remove_listener_bus_listener command will only be set true in the second case. +message StreamingQueryListenerBusCommand { + oneof command { + bool add_listener_bus_listener = 1; + bool remove_listener_bus_listener = 2; + } +} + +// The enum used for client side streaming query listener event +// There is no QueryStartedEvent defined here, +// it is added as a field in WriteStreamOperationStartResult +enum StreamingQueryEventType { + QUERY_PROGRESS_UNSPECIFIED = 0; + QUERY_PROGRESS_EVENT = 1; + QUERY_TERMINATED_EVENT = 2; + QUERY_IDLE_EVENT = 3; +} + +// The protocol for the returned events in the long-running response channel. +message StreamingQueryListenerEvent { + // (Required) The json serialized event, all StreamingQueryListener events have a json method + string event_json = 1; + // (Required) Query event type used by client to decide how to deserialize the event_json + StreamingQueryEventType event_type = 2; +} + +message StreamingQueryListenerEventsResult { + repeated StreamingQueryListenerEvent events = 1; + optional bool listener_bus_listener_added = 2; +} + +// Command to get the output of 'SparkContext.resources' +message GetResourcesCommand { } + +// Response for command 'GetResourcesCommand'. +message GetResourcesCommandResult { + map resources = 1; +} + +// Command to create ResourceProfile +message CreateResourceProfileCommand { + // (Required) The ResourceProfile to be built on the server-side. + ResourceProfile profile = 1; +} + +// Response for command 'CreateResourceProfileCommand'. +message CreateResourceProfileCommandResult { + // (Required) Server-side generated resource profile id. + int32 profile_id = 1; +} + +// Command to remove `CashedRemoteRelation` +message RemoveCachedRemoteRelationCommand { + // (Required) The remote to be related + CachedRemoteRelation relation = 1; +} + +message CheckpointCommand { + // (Required) The logical plan to checkpoint. + Relation relation = 1; + + // (Required) Locally checkpoint using a local temporary + // directory in Spark Connect server (Spark Driver) + bool local = 2; + + // (Required) Whether to checkpoint this dataframe immediately. + bool eager = 3; + + // (Optional) For local checkpoint, the storage level to use. + optional StorageLevel storage_level = 4; +} + +message MergeIntoTableCommand { + // (Required) The name of the target table. + string target_table_name = 1; + + // (Required) The relation of the source table. + Relation source_table_plan = 2; + + // (Required) The condition to match the source and target. + Expression merge_condition = 3; + + // (Optional) The actions to be taken when the condition is matched. + repeated Expression match_actions = 4; + + // (Optional) The actions to be taken when the condition is not matched. + repeated Expression not_matched_actions = 5; + + // (Optional) The actions to be taken when the condition is not matched by source. + repeated Expression not_matched_by_source_actions = 6; + + // (Required) Whether to enable schema evolution. + bool with_schema_evolution = 7; +} + +// Execute an arbitrary string command inside an external execution engine +message ExecuteExternalCommand { + // (Required) The class name of the runner that implements `ExternalCommandRunner` + string runner = 1; + + // (Required) The target command to be executed. + string command = 2; + + // (Optional) The options for the runner. + map options = 3; +} diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/common.proto b/gateway-service-sparkconnect/src/main/proto/spark/connect/common.proto new file mode 100644 index 0000000000..c5470538c1 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/proto/spark/connect/common.proto @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// StorageLevel for persisting Datasets/Tables. +message StorageLevel { + // (Required) Whether the cache should use disk or not. + bool use_disk = 1; + // (Required) Whether the cache should use memory or not. + bool use_memory = 2; + // (Required) Whether the cache should use off-heap or not. + bool use_off_heap = 3; + // (Required) Whether the cached data is deserialized or not. + bool deserialized = 4; + // (Required) The number of replicas. + int32 replication = 5; +} + + +// ResourceInformation to hold information about a type of Resource. +// The corresponding class is 'org.apache.spark.resource.ResourceInformation' +message ResourceInformation { + // (Required) The name of the resource + string name = 1; + // (Required) An array of strings describing the addresses of the resource. + repeated string addresses = 2; +} + +// An executor resource request. +message ExecutorResourceRequest { + // (Required) resource name. + string resource_name = 1; + + // (Required) resource amount requesting. + int64 amount = 2; + + // Optional script used to discover the resources. + optional string discovery_script = 3; + + // Optional vendor, required for some cluster managers. + optional string vendor = 4; +} + +// A task resource request. +message TaskResourceRequest { + // (Required) resource name. + string resource_name = 1; + + // (Required) resource amount requesting as a double to support fractional + // resource requests. + double amount = 2; +} + +message ResourceProfile { + // (Optional) Resource requests for executors. Mapped from the resource name + // (e.g., cores, memory, CPU) to its specific request. + map executor_resources = 1; + + // (Optional) Resource requests for tasks. Mapped from the resource name + // (e.g., cores, memory, CPU) to its specific request. + map task_resources = 2; +} + +message Origin { + // (Required) Indicate the origin type. + oneof function { + PythonOrigin python_origin = 1; + JvmOrigin jvm_origin = 2; + } +} + +message PythonOrigin { + // (Required) Name of the origin, for example, the name of the function + string fragment = 1; + + // (Required) Callsite to show to end users, for example, stacktrace. + string call_site = 2; +} + +message JvmOrigin { + // (Optional) Line number in the source file. + optional int32 line = 1; + + // (Optional) Start position in the source file. + optional int32 start_position = 2; + + // (Optional) Start index in the source file. + optional int32 start_index = 3; + + // (Optional) Stop index in the source file. + optional int32 stop_index = 4; + + // (Optional) SQL text. + optional string sql_text = 5; + + // (Optional) Object type. + optional string object_type = 6; + + // (Optional) Object name. + optional string object_name = 7; + + // (Optional) Stack trace. + repeated StackTraceElement stack_trace = 8; +} + +// A message to hold a [[java.lang.StackTraceElement]]. +message StackTraceElement { + // (Optional) Class loader name + optional string class_loader_name = 1; + + // (Optional) Module name + optional string module_name = 2; + + // (Optional) Module version + optional string module_version = 3; + + // (Required) Declaring class + string declaring_class = 4; + + // (Required) Method name + string method_name = 5; + + // (Optional) File name + optional string file_name = 6; + + // (Required) Line number + int32 line_number = 7; +} + +message ResolvedIdentifier { + string catalog_name = 1; + repeated string namespace = 2; + string table_name = 3; +} + +message Bools { + repeated bool values = 1; +} + +message Ints { + repeated int32 values = 1; +} + +message Longs { + repeated int64 values = 1; +} + +message Floats { + repeated float values = 1; +} + +message Doubles { + repeated double values = 1; +} + +message Strings { + repeated string values = 1; +} diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/expressions.proto b/gateway-service-sparkconnect/src/main/proto/spark/connect/expressions.proto new file mode 100644 index 0000000000..f74c5af117 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/proto/spark/connect/expressions.proto @@ -0,0 +1,557 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +import "google/protobuf/any.proto"; +import "spark/connect/types.proto"; +import "spark/connect/common.proto"; + +package spark.connect; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// Expression used to refer to fields, functions and similar. This can be used everywhere +// expressions in SQL appear. +message Expression { + + ExpressionCommon common = 18; + oneof expr_type { + Literal literal = 1; + UnresolvedAttribute unresolved_attribute = 2; + UnresolvedFunction unresolved_function = 3; + ExpressionString expression_string = 4; + UnresolvedStar unresolved_star = 5; + Alias alias = 6; + Cast cast = 7; + UnresolvedRegex unresolved_regex = 8; + SortOrder sort_order = 9; + LambdaFunction lambda_function = 10; + Window window = 11; + UnresolvedExtractValue unresolved_extract_value = 12; + UpdateFields update_fields = 13; + UnresolvedNamedLambdaVariable unresolved_named_lambda_variable = 14; + CommonInlineUserDefinedFunction common_inline_user_defined_function = 15; + CallFunction call_function = 16; + NamedArgumentExpression named_argument_expression = 17; + MergeAction merge_action = 19; + TypedAggregateExpression typed_aggregate_expression = 20; + SubqueryExpression subquery_expression = 21; + DirectShufflePartitionID direct_shuffle_partition_id = 22; + + // This field is used to mark extensions to the protocol. When plugins generate arbitrary + // relations they can add them here. During the planning the correct resolution is done. + google.protobuf.Any extension = 999; + } + + + // Expression for the OVER clause or WINDOW clause. + message Window { + + // (Required) The window function. + Expression window_function = 1; + + // (Optional) The way that input rows are partitioned. + repeated Expression partition_spec = 2; + + // (Optional) Ordering of rows in a partition. + repeated SortOrder order_spec = 3; + + // (Optional) Window frame in a partition. + // + // If not set, it will be treated as 'UnspecifiedFrame'. + WindowFrame frame_spec = 4; + + // The window frame + message WindowFrame { + + // (Required) The type of the frame. + FrameType frame_type = 1; + + // (Required) The lower bound of the frame. + FrameBoundary lower = 2; + + // (Required) The upper bound of the frame. + FrameBoundary upper = 3; + + enum FrameType { + FRAME_TYPE_UNDEFINED = 0; + + // RowFrame treats rows in a partition individually. + FRAME_TYPE_ROW = 1; + + // RangeFrame treats rows in a partition as groups of peers. + // All rows having the same 'ORDER BY' ordering are considered as peers. + FRAME_TYPE_RANGE = 2; + } + + message FrameBoundary { + oneof boundary { + // CURRENT ROW boundary + bool current_row = 1; + + // UNBOUNDED boundary. + // For lower bound, it will be converted to 'UnboundedPreceding'. + // for upper bound, it will be converted to 'UnboundedFollowing'. + bool unbounded = 2; + + // This is an expression for future proofing. We are expecting literals on the server side. + Expression value = 3; + } + } + } + } + + // SortOrder is used to specify the data ordering, it is normally used in Sort and Window. + // It is an unevaluable expression and cannot be evaluated, so can not be used in Projection. + message SortOrder { + // (Required) The expression to be sorted. + Expression child = 1; + + // (Required) The sort direction, should be ASCENDING or DESCENDING. + SortDirection direction = 2; + + // (Required) How to deal with NULLs, should be NULLS_FIRST or NULLS_LAST. + NullOrdering null_ordering = 3; + + enum SortDirection { + SORT_DIRECTION_UNSPECIFIED = 0; + SORT_DIRECTION_ASCENDING = 1; + SORT_DIRECTION_DESCENDING = 2; + } + + enum NullOrdering { + SORT_NULLS_UNSPECIFIED = 0; + SORT_NULLS_FIRST = 1; + SORT_NULLS_LAST = 2; + } + } + + // Expression that takes a partition ID value and passes it through directly for use in + // shuffle partitioning. This is used with RepartitionByExpression to allow users to + // directly specify target partition IDs. + message DirectShufflePartitionID { + // (Required) The expression that evaluates to the partition ID. + Expression child = 1; + } + + message Cast { + // (Required) the expression to be casted. + Expression expr = 1; + + // (Required) the data type that the expr to be casted to. + oneof cast_to_type { + DataType type = 2; + // If this is set, Server will use Catalyst parser to parse this string to DataType. + string type_str = 3; + } + + // (Optional) The expression evaluation mode. + EvalMode eval_mode = 4; + + enum EvalMode { + EVAL_MODE_UNSPECIFIED = 0; + EVAL_MODE_LEGACY = 1; + EVAL_MODE_ANSI = 2; + EVAL_MODE_TRY = 3; + } + } + + message Literal { + oneof literal_type { + DataType null = 1; + bytes binary = 2; + bool boolean = 3; + + int32 byte = 4; + int32 short = 5; + int32 integer = 6; + int64 long = 7; + float float = 10; + double double = 11; + Decimal decimal = 12; + + string string = 13; + + // Date in units of days since the UNIX epoch. + int32 date = 16; + // Timestamp in units of microseconds since the UNIX epoch. + int64 timestamp = 17; + // Timestamp in units of microseconds since the UNIX epoch (without timezone information). + int64 timestamp_ntz = 18; + + CalendarInterval calendar_interval = 19; + int32 year_month_interval = 20; + int64 day_time_interval = 21; + Array array = 22; + Map map = 23; + Struct struct = 24; + + SpecializedArray specialized_array = 25; + Time time = 26; + } + + // Reserved for Geometry and Geography. + reserved 27, 28; + + // Data type information for the literal. + // This field is required only in the root literal message for null values or + // for data types (e.g., array, map, or struct) with non-trivial information. + // If the data_type field is not set at the root level, the data type will be + // inferred or retrieved from the deprecated data type fields using best efforts. + DataType data_type = 100; + + message Decimal { + // the string representation. + string value = 1; + // The maximum number of digits allowed in the value. + // the maximum precision is 38. + optional int32 precision = 2; + // declared scale of decimal literal + optional int32 scale = 3; + } + + message CalendarInterval { + int32 months = 1; + int32 days = 2; + int64 microseconds = 3; + } + + message Array { + // (Deprecated) The element type of the array. + // + // This field is deprecated since Spark 4.1+. Use data_type field instead. + DataType element_type = 1 [deprecated = true]; + + // The literal values that make up the array elements. + repeated Literal elements = 2; + } + + message Map { + // (Deprecated) The key type of the map. + // + // This field is deprecated since Spark 4.1+. Use data_type field instead. + DataType key_type = 1 [deprecated = true]; + + // (Deprecated) The value type of the map. + // + // This field is deprecated since Spark 4.1+ and should only be set + // if the data_type field is not set. Use data_type field instead. + DataType value_type = 2 [deprecated = true]; + + // The literal keys that make up the map. + repeated Literal keys = 3; + + // The literal values that make up the map. + repeated Literal values = 4; + } + + message Struct { + // (Deprecated) The type of the struct. + // + // This field is deprecated since Spark 4.1+ because using DataType as the type of a struct + // is ambiguous. Use data_type field instead. + DataType struct_type = 1 [deprecated = true]; + + // The literal values that make up the struct elements. + repeated Literal elements = 2; + } + + message SpecializedArray { + oneof value_type { + Bools bools = 1; + Ints ints = 2; + Longs longs = 3; + Floats floats = 4; + Doubles doubles = 5; + Strings strings = 6; + } + } + + message Time { + int64 nano = 1; + // The precision of this time, if omitted, uses the default value of MICROS_PRECISION. + optional int32 precision = 2; + } + } + + // An unresolved attribute that is not explicitly bound to a specific column, but the column + // is resolved during analysis by name. + message UnresolvedAttribute { + // (Required) An identifier that will be parsed by Catalyst parser. This should follow the + // Spark SQL identifier syntax. + string unparsed_identifier = 1; + + // (Optional) The id of corresponding connect plan. + optional int64 plan_id = 2; + + // (Optional) The requested column is a metadata column. + optional bool is_metadata_column = 3; + } + + // An unresolved function is not explicitly bound to one explicit function, but the function + // is resolved during analysis following Sparks name resolution rules. + message UnresolvedFunction { + // (Required) name (or unparsed name for user defined function) for the unresolved function. + string function_name = 1; + + // (Optional) Function arguments. Empty arguments are allowed. + repeated Expression arguments = 2; + + // (Required) Indicate if this function should be applied on distinct values. + bool is_distinct = 3; + + // (Required) Indicate if this is a user defined function. + // + // When it is not a user defined function, Connect will use the function name directly. + // When it is a user defined function, Connect will parse the function name first. + bool is_user_defined_function = 4; + + // (Optional) Indicate if this function is defined in the internal function registry. + // If not set, the server will try to look up the function in the internal function registry + // and decide appropriately. + optional bool is_internal = 5; + } + + // Expression as string. + message ExpressionString { + // (Required) A SQL expression that will be parsed by Catalyst parser. + string expression = 1; + } + + // UnresolvedStar is used to expand all the fields of a relation or struct. + message UnresolvedStar { + + // (Optional) The target of the expansion. + // + // If set, it should end with '.*' and will be parsed by 'parseAttributeName' + // in the server side. + optional string unparsed_target = 1; + + // (Optional) The id of corresponding connect plan. + optional int64 plan_id = 2; + } + + // Represents all of the input attributes to a given relational operator, for example in + // "SELECT `(id)?+.+` FROM ...". + message UnresolvedRegex { + // (Required) The column name used to extract column with regex. + string col_name = 1; + + // (Optional) The id of corresponding connect plan. + optional int64 plan_id = 2; + } + + // Extracts a value or values from an Expression + message UnresolvedExtractValue { + // (Required) The expression to extract value from, can be + // Map, Array, Struct or array of Structs. + Expression child = 1; + + // (Required) The expression to describe the extraction, can be + // key of Map, index of Array, field name of Struct. + Expression extraction = 2; + } + + // Add, replace or drop a field of `StructType` expression by name. + message UpdateFields { + // (Required) The struct expression. + Expression struct_expression = 1; + + // (Required) The field name. + string field_name = 2; + + // (Optional) The expression to add or replace. + // + // When not set, it means this field will be dropped. + Expression value_expression = 3; + } + + message Alias { + // (Required) The expression that alias will be added on. + Expression expr = 1; + + // (Required) a list of name parts for the alias. + // + // Scalar columns only has one name that presents. + repeated string name = 2; + + // (Optional) Alias metadata expressed as a JSON map. + optional string metadata = 3; + } + + message LambdaFunction { + // (Required) The lambda function. + // + // The function body should use 'UnresolvedAttribute' as arguments, the sever side will + // replace 'UnresolvedAttribute' with 'UnresolvedNamedLambdaVariable'. + Expression function = 1; + + // (Required) Function variables. Must contains 1 ~ 3 variables. + repeated Expression.UnresolvedNamedLambdaVariable arguments = 2; + } + + message UnresolvedNamedLambdaVariable { + + // (Required) a list of name parts for the variable. Must not be empty. + repeated string name_parts = 1; + } +} + +message ExpressionCommon { + // (Required) Keep the information of the origin for this expression such as stacktrace. + Origin origin = 1; +} + +message CommonInlineUserDefinedFunction { + // (Required) Name of the user-defined function. + string function_name = 1; + // (Optional) Indicate if the user-defined function is deterministic. + bool deterministic = 2; + // (Optional) Function arguments. Empty arguments are allowed. + repeated Expression arguments = 3; + // (Required) Indicate the function type of the user-defined function. + oneof function { + PythonUDF python_udf = 4; + ScalarScalaUDF scalar_scala_udf = 5; + JavaUDF java_udf = 6; + } + // (Required) Indicate if this function should be applied on distinct values. + bool is_distinct = 7; +} + +message PythonUDF { + // (Required) Output type of the Python UDF + DataType output_type = 1; + // (Required) EvalType of the Python UDF + int32 eval_type = 2; + // (Required) The encoded commands of the Python UDF + bytes command = 3; + // (Required) Python version being used in the client. + string python_ver = 4; + // (Optional) Additional includes for the Python UDF. + repeated string additional_includes = 5; +} + +message ScalarScalaUDF { + // (Required) Serialized JVM object containing UDF definition, input encoders and output encoder + bytes payload = 1; + // (Optional) Input type(s) of the UDF + repeated DataType inputTypes = 2; + // (Required) Output type of the UDF + DataType outputType = 3; + // (Required) True if the UDF can return null value + bool nullable = 4; + // (Required) Indicate if the UDF is an aggregate function + bool aggregate = 5; +} + +message JavaUDF { + // (Required) Fully qualified name of Java class + string class_name = 1; + + // (Optional) Output type of the Java UDF + optional DataType output_type = 2; + + // (Required) Indicate if the Java user-defined function is an aggregate function + bool aggregate = 3; +} + +message TypedAggregateExpression { + // (Required) The aggregate function object packed into bytes. + ScalarScalaUDF scalar_scala_udf = 1; +} + +message CallFunction { + // (Required) Unparsed name of the SQL function. + string function_name = 1; + + // (Optional) Function arguments. Empty arguments are allowed. + repeated Expression arguments = 2; +} + +message NamedArgumentExpression { + // (Required) The key of the named argument. + string key = 1; + + // (Required) The value expression of the named argument. + Expression value = 2; +} + +message MergeAction { + // (Required) The action type of the merge action. + ActionType action_type = 1; + + // (Optional) The condition expression of the merge action. + optional Expression condition = 2; + + // (Optional) The assignments of the merge action. Required for ActionTypes INSERT and UPDATE. + repeated Assignment assignments = 3; + + enum ActionType { + ACTION_TYPE_INVALID = 0; + ACTION_TYPE_DELETE = 1; + ACTION_TYPE_INSERT = 2; + ACTION_TYPE_INSERT_STAR = 3; + ACTION_TYPE_UPDATE = 4; + ACTION_TYPE_UPDATE_STAR = 5; + } + + message Assignment { + // (Required) The key of the assignment. + Expression key = 1; + + // (Required) The value of the assignment. + Expression value = 2; + } +} + +message SubqueryExpression { + // (Required) The ID of the corresponding connect plan. + int64 plan_id = 1; + + // (Required) The type of the subquery. + SubqueryType subquery_type = 2; + + // (Optional) Options specific to table arguments. + optional TableArgOptions table_arg_options = 3; + + // (Optional) IN subquery values. + repeated Expression in_subquery_values = 4; + + enum SubqueryType { + SUBQUERY_TYPE_UNKNOWN = 0; + SUBQUERY_TYPE_SCALAR = 1; + SUBQUERY_TYPE_EXISTS = 2; + SUBQUERY_TYPE_TABLE_ARG = 3; + SUBQUERY_TYPE_IN = 4; + } + + // Nested message for table argument options. + message TableArgOptions { + // (Optional) The way that input rows are partitioned. + repeated Expression partition_spec = 1; + + // (Optional) Ordering of rows in a partition. + repeated Expression.SortOrder order_spec = 2; + + // (Optional) Whether this is a single partition. + optional bool with_single_partition = 3; + } +} diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/ml.proto b/gateway-service-sparkconnect/src/main/proto/spark/connect/ml.proto new file mode 100644 index 0000000000..ef5c406ded --- /dev/null +++ b/gateway-service-sparkconnect/src/main/proto/spark/connect/ml.proto @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +import "spark/connect/relations.proto"; +import "spark/connect/expressions.proto"; +import "spark/connect/ml_common.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// Command for ML +message MlCommand { + oneof command { + Fit fit = 1; + Fetch fetch = 2; + Delete delete = 3; + Write write = 4; + Read read = 5; + Evaluate evaluate = 6; + CleanCache clean_cache = 7; + GetCacheInfo get_cache_info = 8; + CreateSummary create_summary = 9; + GetModelSize get_model_size = 10; + } + + // Command for estimator.fit(dataset) + message Fit { + // (Required) Estimator information (its type should be OPERATOR_TYPE_ESTIMATOR) + MlOperator estimator = 1; + // (Optional) parameters of the Estimator + optional MlParams params = 2; + // (Required) the training dataset + Relation dataset = 3; + } + + // Command to delete the cached objects which could be a model + // or summary evaluated by a model + message Delete { + repeated ObjectRef obj_refs = 1; + // if set `evict_only` to true, only evict the cached model from memory, + // but keep the offloaded model in Spark driver local disk. + optional bool evict_only = 2; + } + + // Force to clean up all the ML cached objects + message CleanCache { } + + // Get the information of all the ML cached objects + message GetCacheInfo { } + + // Command to write ML operator + message Write { + // It could be an estimator/evaluator or the cached model + oneof type { + // Estimator or evaluator + MlOperator operator = 1; + // The cached model + ObjectRef obj_ref = 2; + } + // (Optional) The parameters of operator which could be estimator/evaluator or a cached model + optional MlParams params = 3; + // (Required) Save the ML instance to the path + string path = 4; + // (Optional) Overwrites if the output path already exists. + optional bool should_overwrite = 5; + // (Optional) The options of the writer + map options = 6; + } + + // Command to load ML operator. + message Read { + // (Required) ML operator information + MlOperator operator = 1; + // (Required) Load the ML instance from the input path + string path = 2; + } + + // Command for evaluator.evaluate(dataset) + message Evaluate { + // (Required) Evaluator information (its type should be OPERATOR_TYPE_EVALUATOR) + MlOperator evaluator = 1; + // (Optional) parameters of the Evaluator + optional MlParams params = 2; + // (Required) the evaluating dataset + Relation dataset = 3; + } + + // This is for re-creating the model summary when the model summary is lost + // (model summary is lost when the model is offloaded and then loaded back) + message CreateSummary { + ObjectRef model_ref = 1; + Relation dataset = 2; + } + + // This is for query the model estimated in-memory size + message GetModelSize { + ObjectRef model_ref = 1; + } +} + +// The result of MlCommand +message MlCommandResult { + oneof result_type { + // The result of the attribute + Expression.Literal param = 1; + // Evaluate a Dataset in a model and return the cached ID of summary + string summary = 2; + // Operator information + MlOperatorInfo operator_info = 3; + } + + // Represents an operator info + message MlOperatorInfo { + oneof type { + // The cached object which could be a model or summary evaluated by a model + ObjectRef obj_ref = 1; + // Operator name + string name = 2; + } + // (Optional) the 'uid' of a ML object + // Note it is different from the 'id' of a cached object. + optional string uid = 3; + // (Optional) parameters + optional MlParams params = 4; + // (Optional) warning message generated during the ML command execution + optional string warning_message = 5; + } +} diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/ml_common.proto b/gateway-service-sparkconnect/src/main/proto/spark/connect/ml_common.proto new file mode 100644 index 0000000000..06ca4e5db6 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/proto/spark/connect/ml_common.proto @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +import "spark/connect/expressions.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// MlParams stores param settings for ML Estimator / Transformer / Evaluator +message MlParams { + // User-supplied params + map params = 1; +} + +// MLOperator represents the ML operators like (Estimator, Transformer or Evaluator) +message MlOperator { + // (Required) The qualified name of the ML operator. + string name = 1; + + // (Required) Unique id of the ML operator + string uid = 2; + + // (Required) Represents what the ML operator is + OperatorType type = 3; + + enum OperatorType { + OPERATOR_TYPE_UNSPECIFIED = 0; + // ML estimator + OPERATOR_TYPE_ESTIMATOR = 1; + // ML transformer (non-model) + OPERATOR_TYPE_TRANSFORMER = 2; + // ML evaluator + OPERATOR_TYPE_EVALUATOR = 3; + // ML model + OPERATOR_TYPE_MODEL = 4; + } +} + +// Represents a reference to the cached object which could be a model +// or summary evaluated by a model +message ObjectRef { + // (Required) The ID is used to lookup the object on the server side. + // Note it is different from the 'uid' of a ML object. + string id = 1; +} diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/pipelines.proto b/gateway-service-sparkconnect/src/main/proto/spark/connect/pipelines.proto new file mode 100644 index 0000000000..7632ec95b9 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/proto/spark/connect/pipelines.proto @@ -0,0 +1,385 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package spark.connect; + +import "google/protobuf/any.proto"; +import "google/protobuf/timestamp.proto"; +import "spark/connect/common.proto"; +import "spark/connect/expressions.proto"; +import "spark/connect/relations.proto"; +import "spark/connect/types.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// Dispatch object for pipelines commands. See each individual command for documentation. +message PipelineCommand { + oneof command_type { + CreateDataflowGraph create_dataflow_graph = 1; + DefineOutput define_output = 2; + DefineFlow define_flow = 3; + DropDataflowGraph drop_dataflow_graph = 4; + StartRun start_run = 5; + DefineSqlGraphElements define_sql_graph_elements = 6; + GetQueryFunctionExecutionSignalStream get_query_function_execution_signal_stream = 7; + DefineFlowQueryFunctionResult define_flow_query_function_result = 8; + ExecuteOutputFlows execute_output_flows = 9; + + // Reserved field for protocol extensions. + // Used to support forward-compatibility by carrying additional command types + // that are not yet defined in this version of the proto. During planning, the + // engine will resolve and dispatch the concrete command contained in this field. + google.protobuf.Any extension = 999; + } + + // Request to create a new dataflow graph. + message CreateDataflowGraph { + // The default catalog. + optional string default_catalog = 1; + + // The default database. + optional string default_database = 2; + + // SQL configurations for all flows in this graph. + map sql_conf = 5; + } + + // Drops the graph and stops any running attached flows. + message DropDataflowGraph { + // The graph to drop. + optional string dataflow_graph_id = 1; + } + + // Request to define an output: a table, a materialized view, a temporary view or a sink. + message DefineOutput { + // The graph to attach this output to. + optional string dataflow_graph_id = 1; + + // Name of the output. Can be partially or fully qualified. + optional string output_name = 2; + + // The type of the output. + optional OutputType output_type = 3; + + // Optional comment for the output. + optional string comment = 4; + + // The location in source code that this output was defined. + optional SourceCodeLocation source_code_location = 5; + + oneof details { + TableDetails table_details = 6; + SinkDetails sink_details = 7; + google.protobuf.Any extension = 999; + } + + // Metadata that's only applicable to tables and materialized views. + message TableDetails { + // Optional table properties. + map table_properties = 1; + + // Optional partition columns for the table. + repeated string partition_cols = 2; + + // The output table format for the table. + optional string format = 3; + + // Schema for the table. If unset, this will be inferred from incoming flows. + oneof schema { + spark.connect.DataType schema_data_type = 4; + string schema_string = 5; + } + + // Optional cluster columns for the table. + repeated string clustering_columns = 6; + } + + // Metadata that's only applicable to sinks. + message SinkDetails { + // Streaming write options + map options = 1; + + // Streaming write format + optional string format = 2; + } + } + + // Request to define a flow targeting a dataset. + message DefineFlow { + // The graph to attach this flow to. + optional string dataflow_graph_id = 1; + + // Name of the flow. For standalone flows, this must be a single-part name. + optional string flow_name = 2; + + // Name of the dataset this flow writes to. Can be partially or fully qualified. + optional string target_dataset_name = 3; + + // SQL configurations set when running this flow. + map sql_conf = 4; + + // Identifier for the client making the request. The server uses this to determine what flow + // evaluation request stream to dispatch evaluation requests to for this flow. + optional string client_id = 5; + + // The location in source code that this flow was defined. + optional SourceCodeLocation source_code_location = 6; + + oneof details { + WriteRelationFlowDetails relation_flow_details = 7; + AutoCdcFlowDetails auto_cdc_flow_details = 10; + google.protobuf.Any extension = 999; + } + + // A flow that is that takes the contents of a relation and writes it to the target dataset. + message WriteRelationFlowDetails { + // An unresolved relation that defines the dataset's flow. Empty if the query function + // that defines the flow cannot be analyzed at the time of flow definition. + optional spark.connect.Relation relation = 1; + } + + // Details for Auto CDC flows. + message AutoCdcFlowDetails { + // The name of the CDC source to stream from. + optional string source = 1; + + // Column(s) that uniquely identify a row in source and target data. + repeated Expression keys = 2; + + // Expression to order the source data. + optional Expression sequence_by = 3; + + // Delete condition for the merged operation. + optional Expression apply_as_deletes = 6; + + // Truncate condition for the merged operation. + optional Expression apply_as_truncates = 7; + + // Columns included in the output table. + repeated Expression column_list = 8; + + // Columns excluded from the output table. + repeated Expression except_column_list = 9; + + // SCD Type for target table. + SCDType stored_as_scd_type = 10; + + // Subset of columns to ignore null in updates. + repeated Expression ignore_null_updates_column_list = 14; + + // Subset of columns excluded from ignoring null in updates. + repeated Expression ignore_null_updates_except_column_list = 15; + + } + + // SCD Type for Auto CDC target tables. + enum SCDType { + SCD_TYPE_UNSPECIFIED = 0; + SCD_TYPE_1 = 1; + } + + // If true, define the flow as a one-time flow, such as for backfill. + // Set to true changes the flow in two ways: + // - The flow is run one time by default. If the pipeline is ran with a full refresh, + // the flow will run again. + // - The flow function must be a batch DataFrame, not a streaming DataFrame. + optional bool once = 8; + + message Response { + // Fully qualified flow name that uniquely identify a flow in the Dataflow graph. + optional string flow_name = 1; + } + } + + // Request to execute all flows for a single output (dataset or sink) remotely. + message ExecuteOutputFlows { + + // The output (table or materialized view or sink) definition. + optional DefineOutput define_output = 1; + + // The flows to execute for this table. + repeated DefineFlow define_flows = 2; + + // Whether to perform a full refresh instead of an incremental update. + optional bool full_refresh = 3; + + // Storage location for pipeline checkpoints and metadata. + optional string storage = 4; + + // Reserved field for protocol extensions. + repeated google.protobuf.Any extension = 999; + } + + // Resolves all datasets and flows and start a pipeline update. Should be called after all + // graph elements are registered. + message StartRun { + // The graph to start. + optional string dataflow_graph_id = 1; + + // List of dataset to reset and recompute. + repeated string full_refresh_selection = 2; + + // Perform a full graph reset and recompute. + optional bool full_refresh_all = 3; + + // List of dataset to update. + repeated string refresh_selection = 4; + + // If true, the run will not actually execute any flows, but will only validate the graph and + // check for any errors. This is useful for testing and validation purposes. + optional bool dry = 5; + + // storage location for pipeline checkpoints and metadata. + optional string storage = 6; + } + + // Parses the SQL file and registers all datasets and flows. + message DefineSqlGraphElements { + // The graph to attach this dataset to. + optional string dataflow_graph_id = 1; + + // The full path to the SQL file. Can be relative or absolute. + optional string sql_file_path = 2; + + // The contents of the SQL file. + optional string sql_text = 3; + } + + // Request to get the stream of query function execution signals for a graph. Responses should + // be a stream of PipelineQueryFunctionExecutionSignal messages. + message GetQueryFunctionExecutionSignalStream { + // The graph to get the query function execution signal stream for. + optional string dataflow_graph_id = 1; + + // Identifier for the client that is requesting the stream. + optional string client_id = 2; + } + + // Request from the client to update the flow function evaluation result + // for a previously un-analyzed flow. + message DefineFlowQueryFunctionResult { + // (Deprecated) The fully qualified name of the flow being updated. + // + // This field is deprecated since Spark 4.2+. Use flow_identifier field instead. + optional string flow_name = 1 [deprecated = true]; + + // The fully qualified identifier of the flow being updated. + optional ResolvedIdentifier flow_identifier = 4; + + // The ID of the graph this flow belongs to. + optional string dataflow_graph_id = 2; + + // An unresolved relation that defines the dataset's flow. + optional spark.connect.Relation relation = 3; + } +} + +// Dispatch object for pipelines command results. +message PipelineCommandResult { + oneof result_type { + CreateDataflowGraphResult create_dataflow_graph_result = 1; + DefineOutputResult define_output_result = 2; + DefineFlowResult define_flow_result = 3; + } + message CreateDataflowGraphResult { + // The ID of the created graph. + optional string dataflow_graph_id = 1; + } + message DefineOutputResult { + // Resolved identifier of the output + optional ResolvedIdentifier resolved_identifier = 1; + } + message DefineFlowResult { + // Resolved identifier of the flow + optional ResolvedIdentifier resolved_identifier = 1; + } +} + +// The type of output. +enum OutputType { + // Safe default value. Should not be used. + OUTPUT_TYPE_UNSPECIFIED = 0; + // A materialized view which is published to the catalog + MATERIALIZED_VIEW = 1; + // A table which is published to the catalog + TABLE = 2; + // A view which is not published to the catalog + TEMPORARY_VIEW = 3; + // A sink which is not published to the catalog + SINK = 4; +} + +// A response containing an event emitted during the run of a pipeline. +message PipelineEventResult { + PipelineEvent event = 1; +} + +message PipelineEvent { + // The timestamp corresponding to when the event occurred. + google.protobuf.Timestamp timestamp = 1; + // The message that should be displayed to users. + optional string message = 2; +} + +// Source code location information associated with a particular dataset or flow. +message SourceCodeLocation { + // The file that this pipeline source code was defined in. + optional string file_name = 1; + // The specific line number that this pipeline source code is located at, if applicable. + optional int32 line_number = 2; + // The path of the top-level pipeline file determined at runtime during pipeline initialization. + optional string definition_path = 3; + + // Reserved field for protocol extensions. + // Used to support forward-compatibility by carrying additional fields + // that are not yet defined in this version of the proto. During planning, the + // engine will resolve and dispatch the concrete command contained in this field. + repeated google.protobuf.Any extension = 999; +} + +// A signal from the server to the client to execute the query function for one or more flows, and +// to register their results with the server. +message PipelineQueryFunctionExecutionSignal { + // (Deprecated) The name of flows that are ready to be re-evaluated. + // + // This field is deprecated since Spark 4.2+. Use flow_identifiers field instead. + repeated string flow_names = 1 [deprecated = true]; + + // The identifier of flows that are ready to be re-evaluated + repeated ResolvedIdentifier flow_identifiers = 2; +} + +// Metadata providing context about the pipeline during Spark Connect query analysis. +message PipelineAnalysisContext { + // Unique identifier of the dataflow graph associated with this pipeline. + optional string dataflow_graph_id = 1; + // The path of the top-level pipeline file determined at runtime during pipeline initialization. + optional string definition_path = 2; + // (Deprecated) The name of the Flow involved in this analysis + // + // This field is deprecated since Spark 4.2+. Use flow_identifier field instead. + optional string flow_name = 3 [deprecated = true]; + // The identifier of the Flow involved in this analysis + optional ResolvedIdentifier flow_identifier = 4; + + // Reserved field for protocol extensions. + repeated google.protobuf.Any extension = 999; +} diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/relations.proto b/gateway-service-sparkconnect/src/main/proto/spark/connect/relations.proto new file mode 100644 index 0000000000..95cc9281d8 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/proto/spark/connect/relations.proto @@ -0,0 +1,1309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +import "google/protobuf/any.proto"; +import "spark/connect/expressions.proto"; +import "spark/connect/types.proto"; +import "spark/connect/catalog.proto"; +import "spark/connect/common.proto"; +import "spark/connect/ml_common.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// The main [[Relation]] type. Fundamentally, a relation is a typed container +// that has exactly one explicit relation type set. +// +// When adding new relation types, they have to be registered here. +message Relation { + RelationCommon common = 1; + oneof rel_type { + Read read = 2; + Project project = 3; + Filter filter = 4; + Join join = 5; + SetOperation set_op = 6; + Sort sort = 7; + Limit limit = 8; + Aggregate aggregate = 9; + SQL sql = 10; + LocalRelation local_relation = 11; + Sample sample = 12; + Offset offset = 13; + Deduplicate deduplicate = 14; + Range range = 15; + SubqueryAlias subquery_alias = 16; + Repartition repartition = 17; + ToDF to_df = 18; + WithColumnsRenamed with_columns_renamed = 19; + ShowString show_string = 20; + Drop drop = 21; + Tail tail = 22; + WithColumns with_columns = 23; + Hint hint = 24; + Unpivot unpivot = 25; + ToSchema to_schema = 26; + RepartitionByExpression repartition_by_expression = 27; + MapPartitions map_partitions = 28; + CollectMetrics collect_metrics = 29; + Parse parse = 30; + GroupMap group_map = 31; + CoGroupMap co_group_map = 32; + WithWatermark with_watermark = 33; + ApplyInPandasWithState apply_in_pandas_with_state = 34; + HtmlString html_string = 35; + CachedLocalRelation cached_local_relation = 36; + CachedRemoteRelation cached_remote_relation = 37; + CommonInlineUserDefinedTableFunction common_inline_user_defined_table_function = 38; + AsOfJoin as_of_join = 39; + CommonInlineUserDefinedDataSource common_inline_user_defined_data_source = 40; + WithRelations with_relations = 41; + Transpose transpose = 42; + UnresolvedTableValuedFunction unresolved_table_valued_function = 43; + LateralJoin lateral_join = 44; + ChunkedCachedLocalRelation chunked_cached_local_relation = 45; + RelationChanges relation_changes = 46; + NearestByJoin nearest_by_join = 47; + + // NA functions + NAFill fill_na = 90; + NADrop drop_na = 91; + NAReplace replace = 92; + + // stat functions + StatSummary summary = 100; + StatCrosstab crosstab = 101; + StatDescribe describe = 102; + StatCov cov = 103; + StatCorr corr = 104; + StatApproxQuantile approx_quantile = 105; + StatFreqItems freq_items = 106; + StatSampleBy sample_by = 107; + + // Catalog API (experimental / unstable) + Catalog catalog = 200; + + // ML relation + MlRelation ml_relation = 300; + + // This field is used to mark extensions to the protocol. When plugins generate arbitrary + // relations they can add them here. During the planning the correct resolution is done. + google.protobuf.Any extension = 998; + Unknown unknown = 999; + } +} + +// Relation to represent ML world +message MlRelation { + oneof ml_type { + Transform transform = 1; + Fetch fetch = 2; + } + // (Optional) the dataset for restoring the model summary + optional Relation model_summary_dataset = 3; + + // Relation to represent transform(input) of the operator + // which could be a cached model or a new transformer + message Transform { + oneof operator { + // Object reference + ObjectRef obj_ref = 1; + // Could be an ML transformer like VectorAssembler + MlOperator transformer = 2; + } + // the input dataframe + Relation input = 3; + // the operator specific parameters + MlParams params = 4; + } +} + +// Message for fetching attribute from object on the server side. +// Fetch can be represented as a Relation or a ML command +// Command: model.coefficients, model.summary.weightedPrecision which +// returns the final literal result +// Relation: model.summary.roc which returns a DataFrame (Relation) +message Fetch { + // (Required) reference to the object on the server side + ObjectRef obj_ref = 1; + // (Required) the calling method chains + repeated Method methods = 2; + + // Represents a method with inclusion of method name and its arguments + message Method { + // (Required) the method name + string method = 1; + // (Optional) the arguments of the method + repeated Args args = 2; + + message Args { + oneof args_type { + Expression.Literal param = 1; + Relation input = 2; + } + } + } +} + +// Used for testing purposes only. +message Unknown {} + +// Common metadata of all relations. +message RelationCommon { + // (Required) Shared relation metadata. + string source_info = 1 [deprecated=true]; + + // (Optional) A per-client globally unique id for a given connect plan. + optional int64 plan_id = 2; + + // (Optional) Keep the information of the origin for this expression such as stacktrace. + Origin origin = 3; +} + +// Relation that uses a SQL query to generate the output. +message SQL { + // (Required) The SQL query. + string query = 1; + + // (Optional) A map of parameter names to literal expressions. + map args = 2 [deprecated=true]; + + // (Optional) A sequence of literal expressions for positional parameters in the SQL query text. + repeated Expression.Literal pos_args = 3 [deprecated=true]; + + // (Optional) A map of parameter names to expressions. + // It cannot coexist with `pos_arguments`. + map named_arguments = 4; + + // (Optional) A sequence of expressions for positional parameters in the SQL query text. + // It cannot coexist with `named_arguments`. + repeated Expression pos_arguments = 5; +} + +// Relation of type [[WithRelations]]. +// +// This relation contains a root plan, and one or more references that are used by the root plan. +// There are two ways of referencing a relation, by name (through a subquery alias), or by plan_id +// (using RelationCommon.plan_id). +// +// This relation can be used to implement CTEs, describe DAGs, or to reduce tree depth. +message WithRelations { + // (Required) Plan at the root of the query tree. This plan is expected to contain one or more + // references. Those references get expanded later on by the engine. + Relation root = 1; + + // (Required) Plans referenced by the root plan. Relations in this list are also allowed to + // contain references to other relations in this list, as long they do not form cycles. + repeated Relation references = 2; +} + +// Relation that reads from a file / table or other data source. Does not have additional +// inputs. +message Read { + oneof read_type { + NamedTable named_table = 1; + DataSource data_source = 2; + } + + // (Optional) Indicates if this is a streaming read. + bool is_streaming = 3; + + message NamedTable { + // (Required) Unparsed identifier for the table. + string unparsed_identifier = 1; + + // Options for the named table. The map key is case insensitive. + map options = 2; + } + + message DataSource { + // (Optional) Supported formats include: parquet, orc, text, json, parquet, csv, avro. + // + // If not set, the value from SQL conf 'spark.sql.sources.default' will be used. + optional string format = 1; + + // (Optional) If not set, Spark will infer the schema. + // + // This schema string should be either DDL-formatted or JSON-formatted. + optional string schema = 2; + + // Options for the data source. The context of this map varies based on the + // data source format. This options could be empty for valid data source format. + // The map key is case insensitive. + map options = 3; + + // (Optional) A list of path for file-system backed data sources. + repeated string paths = 4; + + // (Optional) Condition in the where clause for each partition. + // + // This is only supported by the JDBC data source. + repeated string predicates = 5; + + // (Optional) A user-provided name for the streaming source. + // This name is used in checkpoint metadata and enables stable checkpoint locations + // for source evolution. + optional string source_name = 6; + } +} + +// Reads Change Data Capture (CDC) changes for a named table. +// +// This corresponds to the `DataFrameReader.changes()` or `DataStreamReader.changes()` API. +// CDC-specific options (startingVersion, endingVersion, startingTimestamp, endingTimestamp, +// deduplicationMode, computeUpdates, etc.) are passed in the options map. +message RelationChanges { + // (Required) Unparsed identifier for the table. + string unparsed_identifier = 1; + + // Options for the CDC query. The map key is case insensitive. + // Supported keys include: startingVersion, endingVersion, startingTimestamp, + // endingTimestamp, deduplicationMode, computeUpdates, startingBoundInclusive, + // endingBoundInclusive. + map options = 2; + + // (Optional) Indicates if this is a streaming CDC read. + bool is_streaming = 3; +} + +// Projection of a bag of expressions for a given input relation. +// +// The input relation must be specified. +// The projected expression can be an arbitrary expression. +message Project { + // (Optional) Input relation is optional for Project. + // + // For example, `SELECT ABS(-1)` is valid plan without an input plan. + Relation input = 1; + + // (Required) A Project requires at least one expression. + repeated Expression expressions = 3; +} + +// Relation that applies a boolean expression `condition` on each row of `input` to produce +// the output result. +message Filter { + // (Required) Input relation for a Filter. + Relation input = 1; + + // (Required) A Filter must have a condition expression. + Expression condition = 2; +} + +// Relation of type [[Join]]. +// +// `left` and `right` must be present. +message Join { + // (Required) Left input relation for a Join. + Relation left = 1; + + // (Required) Right input relation for a Join. + Relation right = 2; + + // (Optional) The join condition. Could be unset when `using_columns` is utilized. + // + // This field does not co-exist with using_columns. + Expression join_condition = 3; + + // (Required) The join type. + JoinType join_type = 4; + + // Optional. using_columns provides a list of columns that should present on both sides of + // the join inputs that this Join will join on. For example A JOIN B USING col_name is + // equivalent to A JOIN B on A.col_name = B.col_name. + // + // This field does not co-exist with join_condition. + repeated string using_columns = 5; + + enum JoinType { + JOIN_TYPE_UNSPECIFIED = 0; + JOIN_TYPE_INNER = 1; + JOIN_TYPE_FULL_OUTER = 2; + JOIN_TYPE_LEFT_OUTER = 3; + JOIN_TYPE_RIGHT_OUTER = 4; + JOIN_TYPE_LEFT_ANTI = 5; + JOIN_TYPE_LEFT_SEMI = 6; + JOIN_TYPE_CROSS = 7; + } + + // (Optional) Only used by joinWith. Set the left and right join data types. + optional JoinDataType join_data_type = 6; + + message JoinDataType { + // If the left data type is a struct. + bool is_left_struct = 1; + // If the right data type is a struct. + bool is_right_struct = 2; + } +} + +// Relation of type [[SetOperation]] +message SetOperation { + // (Required) Left input relation for a Set operation. + Relation left_input = 1; + + // (Required) Right input relation for a Set operation. + Relation right_input = 2; + + // (Required) The Set operation type. + SetOpType set_op_type = 3; + + // (Optional) If to remove duplicate rows. + // + // True to preserve all results. + // False to remove duplicate rows. + optional bool is_all = 4; + + // (Optional) If to perform the Set operation based on name resolution. + // + // Only UNION supports this option. + optional bool by_name = 5; + + // (Optional) If to perform the Set operation and allow missing columns. + // + // Only UNION supports this option. + optional bool allow_missing_columns = 6; + + enum SetOpType { + SET_OP_TYPE_UNSPECIFIED = 0; + SET_OP_TYPE_INTERSECT = 1; + SET_OP_TYPE_UNION = 2; + SET_OP_TYPE_EXCEPT = 3; + } +} + +// Relation of type [[Limit]] that is used to `limit` rows from the input relation. +message Limit { + // (Required) Input relation for a Limit. + Relation input = 1; + + // (Required) the limit. + int32 limit = 2; +} + +// Relation of type [[Offset]] that is used to read rows staring from the `offset` on +// the input relation. +message Offset { + // (Required) Input relation for an Offset. + Relation input = 1; + + // (Required) the limit. + int32 offset = 2; +} + +// Relation of type [[Tail]] that is used to fetch `limit` rows from the last of the input relation. +message Tail { + // (Required) Input relation for an Tail. + Relation input = 1; + + // (Required) the limit. + int32 limit = 2; +} + +// Relation of type [[Aggregate]]. +message Aggregate { + // (Required) Input relation for a RelationalGroupedDataset. + Relation input = 1; + + // (Required) How the RelationalGroupedDataset was built. + GroupType group_type = 2; + + // (Required) Expressions for grouping keys + repeated Expression grouping_expressions = 3; + + // (Required) List of values that will be translated to columns in the output DataFrame. + repeated Expression aggregate_expressions = 4; + + // (Optional) Pivots a column of the current `DataFrame` and performs the specified aggregation. + Pivot pivot = 5; + + // (Optional) List of values that will be translated to columns in the output DataFrame. + repeated GroupingSets grouping_sets = 6; + + enum GroupType { + GROUP_TYPE_UNSPECIFIED = 0; + GROUP_TYPE_GROUPBY = 1; + GROUP_TYPE_ROLLUP = 2; + GROUP_TYPE_CUBE = 3; + GROUP_TYPE_PIVOT = 4; + GROUP_TYPE_GROUPING_SETS = 5; + } + + message Pivot { + // (Required) The column to pivot + Expression col = 1; + + // (Optional) List of values that will be translated to columns in the output DataFrame. + // + // Note that if it is empty, the server side will immediately trigger a job to collect + // the distinct values of the column. + repeated Expression.Literal values = 2; + } + + message GroupingSets { + // (Required) Individual grouping set + repeated Expression grouping_set = 1; + } +} + +// Relation of type [[Sort]]. +message Sort { + // (Required) Input relation for a Sort. + Relation input = 1; + + // (Required) The ordering expressions + repeated Expression.SortOrder order = 2; + + // (Optional) if this is a global sort. + optional bool is_global = 3; +} + + +// Drop specified columns. +message Drop { + // (Required) The input relation. + Relation input = 1; + + // (Optional) columns to drop. + repeated Expression columns = 2; + + // (Optional) names of columns to drop. + repeated string column_names = 3; +} + + +// Relation of type [[Deduplicate]] which have duplicate rows removed, could consider either only +// the subset of columns or all the columns. +message Deduplicate { + // (Required) Input relation for a Deduplicate. + Relation input = 1; + + // (Optional) Deduplicate based on a list of column names. + // + // This field does not co-use with `all_columns_as_keys`. + repeated string column_names = 2; + + // (Optional) Deduplicate based on all the columns of the input relation. + // + // This field does not co-use with `column_names`. + optional bool all_columns_as_keys = 3; + + // (Optional) Deduplicate within the time range of watermark. + optional bool within_watermark = 4; +} + +// A relation that does not need to be qualified by name. +message LocalRelation { + // (Optional) Local collection data serialized into Arrow IPC streaming format which contains + // the schema of the data. + optional bytes data = 1; + + // (Optional) The schema of local data. + // It should be either a DDL-formatted type string or a JSON string. + // + // The server side will update the column names and data types according to this schema. + // If the 'data' is not provided, then this schema will be required. + optional string schema = 2; +} + +// A local relation that has been cached already. +// CachedLocalRelation doesn't support LocalRelations of size over 2GB. +message CachedLocalRelation { + // `userId` and `sessionId` fields are deleted since the server must always use the active + // session/user rather than arbitrary values provided by the client. It is never valid to access + // a local relation from a different session/user. + reserved 1, 2; + reserved "userId", "sessionId"; + + // (Required) A sha-256 hash of the serialized local relation in proto, see LocalRelation. + string hash = 3; +} + +// A local relation that has been cached already. +message ChunkedCachedLocalRelation { + // (Required) A list of sha-256 hashes for representing LocalRelation.data. + // Data is serialized in Arrow IPC streaming format, each batch is cached on the server as + // a separate artifact. Each hash represents one batch stored on the server. + // Hashes are hex-encoded strings (e.g., "a3b2c1d4..."). + repeated string dataHashes = 1; + + // (Optional) A sha-256 hash of the serialized LocalRelation.schema. + // Scala clients always provide the schema, Python clients can omit it. + // Hash is a hex-encoded string (e.g., "a3b2c1d4..."). + optional string schemaHash = 2; +} + +// Represents a remote relation that has been cached on server. +message CachedRemoteRelation { + // (Required) ID of the remote related (assigned by the service). + string relation_id = 1; +} + +// Relation of type [[Sample]] that samples a fraction of the dataset. +message Sample { + // (Required) Input relation for a Sample. + Relation input = 1; + + // (Required) lower bound. + double lower_bound = 2; + + // (Required) upper bound. + double upper_bound = 3; + + // (Optional) Whether to sample with replacement. + optional bool with_replacement = 4; + + // (Required) The random seed. + // This field is required to avoid generating mutable dataframes (see SPARK-48184 for details), + // however, still keep it 'optional' here for backward compatibility. + optional int64 seed = 5; + + // (Required) Explicitly sort the underlying plan to make the ordering deterministic or cache it. + // This flag is true when invoking `dataframe.randomSplit` to randomly splits DataFrame with the + // provided weights. Otherwise, it is false. + bool deterministic_order = 6; +} + +// Relation of type [[Range]] that generates a sequence of integers. +message Range { + // (Optional) Default value = 0 + optional int64 start = 1; + + // (Required) + int64 end = 2; + + // (Required) + int64 step = 3; + + // Optional. Default value is assigned by 1) SQL conf "spark.sql.leafNodeDefaultParallelism" if + // it is set, or 2) spark default parallelism. + optional int32 num_partitions = 4; +} + +// Relation alias. +message SubqueryAlias { + // (Required) The input relation of SubqueryAlias. + Relation input = 1; + + // (Required) The alias. + string alias = 2; + + // (Optional) Qualifier of the alias. + repeated string qualifier = 3; +} + +// Relation repartition. +message Repartition { + // (Required) The input relation of Repartition. + Relation input = 1; + + // (Required) Must be positive. + int32 num_partitions = 2; + + // (Optional) Default value is false. + optional bool shuffle = 3; +} + +// Compose the string representing rows for output. +// It will invoke 'Dataset.showString' to compute the results. +message ShowString { + // (Required) The input relation. + Relation input = 1; + + // (Required) Number of rows to show. + int32 num_rows = 2; + + // (Required) If set to more than 0, truncates strings to + // `truncate` characters and all cells will be aligned right. + int32 truncate = 3; + + // (Required) If set to true, prints output rows vertically (one line per column value). + bool vertical = 4; +} + +// Compose the string representing rows for output. +// It will invoke 'Dataset.htmlString' to compute the results. +message HtmlString { + // (Required) The input relation. + Relation input = 1; + + // (Required) Number of rows to show. + int32 num_rows = 2; + + // (Required) If set to more than 0, truncates strings to + // `truncate` characters and all cells will be aligned right. + int32 truncate = 3; +} + +// Computes specified statistics for numeric and string columns. +// It will invoke 'Dataset.summary' (same as 'StatFunctions.summary') +// to compute the results. +message StatSummary { + // (Required) The input relation. + Relation input = 1; + + // (Optional) Statistics from to be computed. + // + // Available statistics are: + // count + // mean + // stddev + // min + // max + // arbitrary approximate percentiles specified as a percentage (e.g. 75%) + // count_distinct + // approx_count_distinct + // + // If no statistics are given, this function computes 'count', 'mean', 'stddev', 'min', + // 'approximate quartiles' (percentiles at 25%, 50%, and 75%), and 'max'. + repeated string statistics = 2; +} + +// Computes basic statistics for numeric and string columns, including count, mean, stddev, min, +// and max. If no columns are given, this function computes statistics for all numerical or +// string columns. +message StatDescribe { + // (Required) The input relation. + Relation input = 1; + + // (Optional) Columns to compute statistics on. + repeated string cols = 2; +} + +// Computes a pair-wise frequency table of the given columns. Also known as a contingency table. +// It will invoke 'Dataset.stat.crosstab' (same as 'StatFunctions.crossTabulate') +// to compute the results. +message StatCrosstab { + // (Required) The input relation. + Relation input = 1; + + // (Required) The name of the first column. + // + // Distinct items will make the first item of each row. + string col1 = 2; + + // (Required) The name of the second column. + // + // Distinct items will make the column names of the DataFrame. + string col2 = 3; +} + +// Calculate the sample covariance of two numerical columns of a DataFrame. +// It will invoke 'Dataset.stat.cov' (same as 'StatFunctions.calculateCov') to compute the results. +message StatCov { + // (Required) The input relation. + Relation input = 1; + + // (Required) The name of the first column. + string col1 = 2; + + // (Required) The name of the second column. + string col2 = 3; +} + +// Calculates the correlation of two columns of a DataFrame. Currently only supports the Pearson +// Correlation Coefficient. It will invoke 'Dataset.stat.corr' (same as +// 'StatFunctions.pearsonCorrelation') to compute the results. +message StatCorr { + // (Required) The input relation. + Relation input = 1; + + // (Required) The name of the first column. + string col1 = 2; + + // (Required) The name of the second column. + string col2 = 3; + + // (Optional) Default value is 'pearson'. + // + // Currently only supports the Pearson Correlation Coefficient. + optional string method = 4; +} + +// Calculates the approximate quantiles of numerical columns of a DataFrame. +// It will invoke 'Dataset.stat.approxQuantile' (same as 'StatFunctions.approxQuantile') +// to compute the results. +message StatApproxQuantile { + // (Required) The input relation. + Relation input = 1; + + // (Required) The names of the numerical columns. + repeated string cols = 2; + + // (Required) A list of quantile probabilities. + // + // Each number must belong to [0, 1]. + // For example 0 is the minimum, 0.5 is the median, 1 is the maximum. + repeated double probabilities = 3; + + // (Required) The relative target precision to achieve (greater than or equal to 0). + // + // If set to zero, the exact quantiles are computed, which could be very expensive. + // Note that values greater than 1 are accepted but give the same result as 1. + double relative_error = 4; +} + +// Finding frequent items for columns, possibly with false positives. +// It will invoke 'Dataset.stat.freqItems' (same as 'StatFunctions.freqItems') +// to compute the results. +message StatFreqItems { + // (Required) The input relation. + Relation input = 1; + + // (Required) The names of the columns to search frequent items in. + repeated string cols = 2; + + // (Optional) The minimum frequency for an item to be considered `frequent`. + // Should be greater than 1e-4. + optional double support = 3; +} + + +// Returns a stratified sample without replacement based on the fraction +// given on each stratum. +// It will invoke 'Dataset.stat.freqItems' (same as 'StatFunctions.freqItems') +// to compute the results. +message StatSampleBy { + // (Required) The input relation. + Relation input = 1; + + // (Required) The column that defines strata. + Expression col = 2; + + // (Required) Sampling fraction for each stratum. + // + // If a stratum is not specified, we treat its fraction as zero. + repeated Fraction fractions = 3; + + // (Required) The random seed. + // This field is required to avoid generating mutable dataframes (see SPARK-48184 for details), + // however, still keep it 'optional' here for backward compatibility. + optional int64 seed = 5; + + message Fraction { + // (Required) The stratum. + Expression.Literal stratum = 1; + + // (Required) The fraction value. Must be in [0, 1]. + double fraction = 2; + } +} + + +// Replaces null values. +// It will invoke 'Dataset.na.fill' (same as 'DataFrameNaFunctions.fill') to compute the results. +// Following 3 parameter combinations are supported: +// 1, 'values' only contains 1 item, 'cols' is empty: +// replaces null values in all type-compatible columns. +// 2, 'values' only contains 1 item, 'cols' is not empty: +// replaces null values in specified columns. +// 3, 'values' contains more than 1 items, then 'cols' is required to have the same length: +// replaces each specified column with corresponding value. +message NAFill { + // (Required) The input relation. + Relation input = 1; + + // (Optional) Optional list of column names to consider. + repeated string cols = 2; + + // (Required) Values to replace null values with. + // + // Should contain at least 1 item. + // Only 4 data types are supported now: bool, long, double, string + repeated Expression.Literal values = 3; +} + + +// Drop rows containing null values. +// It will invoke 'Dataset.na.drop' (same as 'DataFrameNaFunctions.drop') to compute the results. +message NADrop { + // (Required) The input relation. + Relation input = 1; + + // (Optional) Optional list of column names to consider. + // + // When it is empty, all the columns in the input relation will be considered. + repeated string cols = 2; + + // (Optional) The minimum number of non-null and non-NaN values required to keep. + // + // When not set, it is equivalent to the number of considered columns, which means + // a row will be kept only if all columns are non-null. + // + // 'how' options ('all', 'any') can be easily converted to this field: + // - 'all' -> set 'min_non_nulls' 1; + // - 'any' -> keep 'min_non_nulls' unset; + optional int32 min_non_nulls = 3; +} + + +// Replaces old values with the corresponding values. +// It will invoke 'Dataset.na.replace' (same as 'DataFrameNaFunctions.replace') +// to compute the results. +message NAReplace { + // (Required) The input relation. + Relation input = 1; + + // (Optional) List of column names to consider. + // + // When it is empty, all the type-compatible columns in the input relation will be considered. + repeated string cols = 2; + + // (Optional) The value replacement mapping. + repeated Replacement replacements = 3; + + message Replacement { + // (Required) The old value. + // + // Only 4 data types are supported now: null, bool, double, string. + Expression.Literal old_value = 1; + + // (Required) The new value. + // + // Should be of the same data type with the old value. + Expression.Literal new_value = 2; + } +} + + +// Rename columns on the input relation by the same length of names. +message ToDF { + // (Required) The input relation of RenameColumnsBySameLengthNames. + Relation input = 1; + + // (Required) + // + // The number of columns of the input relation must be equal to the length + // of this field. If this is not true, an exception will be returned. + repeated string column_names = 2; +} + + +// Rename columns on the input relation by a map with name to name mapping. +message WithColumnsRenamed { + // (Required) The input relation. + Relation input = 1; + + + // (Optional) + // + // Renaming column names of input relation from A to B where A is the map key + // and B is the map value. This is a no-op if schema doesn't contain any A. It + // does not require that all input relation column names to present as keys. + // duplicated B are not allowed. + map rename_columns_map = 2 [deprecated=true]; + + repeated Rename renames = 3; + + message Rename { + // (Required) The existing column name. + string col_name = 1; + + // (Required) The new column name. + string new_col_name = 2; + } +} + +// Adding columns or replacing the existing columns that have the same names. +message WithColumns { + // (Required) The input relation. + Relation input = 1; + + // (Required) + // + // Given a column name, apply the corresponding expression on the column. If column + // name exists in the input relation, then replace the column. If the column name + // does not exist in the input relation, then adds it as a new column. + // + // Only one name part is expected from each Expression.Alias. + // + // An exception is thrown when duplicated names are present in the mapping. + repeated Expression.Alias aliases = 2; +} + +message WithWatermark { + + // (Required) The input relation + Relation input = 1; + + // (Required) Name of the column containing event time. + string event_time = 2; + + // (Required) + string delay_threshold = 3; +} + +// Specify a hint over a relation. Hint should have a name and optional parameters. +message Hint { + // (Required) The input relation. + Relation input = 1; + + // (Required) Hint name. + // + // Supported Join hints include BROADCAST, MERGE, SHUFFLE_HASH, SHUFFLE_REPLICATE_NL. + // + // Supported partitioning hints include COALESCE, REPARTITION, REPARTITION_BY_RANGE. + string name = 2; + + // (Optional) Hint parameters. + repeated Expression parameters = 3; +} + +// Unpivot a DataFrame from wide format to long format, optionally leaving identifier columns set. +message Unpivot { + // (Required) The input relation. + Relation input = 1; + + // (Required) Id columns. + repeated Expression ids = 2; + + // (Optional) Value columns to unpivot. + optional Values values = 3; + + // (Required) Name of the variable column. + string variable_column_name = 4; + + // (Required) Name of the value column. + string value_column_name = 5; + + message Values { + repeated Expression values = 1; + } +} + +// Transpose a DataFrame, switching rows to columns. +// Transforms the DataFrame such that the values in the specified index column +// become the new columns of the DataFrame. +message Transpose { + // (Required) The input relation. + Relation input = 1; + + // (Optional) A list of columns that will be treated as the indices. + // Only single column is supported now. + repeated Expression index_columns = 2; +} + +message UnresolvedTableValuedFunction { + // (Required) name (or unparsed name for user defined function) for the unresolved function. + string function_name = 1; + + // (Optional) Function arguments. Empty arguments are allowed. + repeated Expression arguments = 2; +} + +message ToSchema { + // (Required) The input relation. + Relation input = 1; + + // (Required) The user provided schema. + // + // The Sever side will update the dataframe with this schema. + DataType schema = 2; +} + +message RepartitionByExpression { + // (Required) The input relation. + Relation input = 1; + + // (Required) The partitioning expressions. + repeated Expression partition_exprs = 2; + + // (Optional) number of partitions, must be positive. + optional int32 num_partitions = 3; +} + +message MapPartitions { + // (Required) Input relation for a mapPartitions-equivalent API: mapInPandas, mapInArrow. + Relation input = 1; + + // (Required) Input user-defined function. + CommonInlineUserDefinedFunction func = 2; + + // (Optional) Whether to use barrier mode execution or not. + optional bool is_barrier = 3; + + // (Optional) ResourceProfile id used for the stage level scheduling. + optional int32 profile_id = 4; +} + +message GroupMap { + // (Required) Input relation for Group Map API: apply, applyInPandas. + Relation input = 1; + + // (Required) Expressions for grouping keys. + repeated Expression grouping_expressions = 2; + + // (Required) Input user-defined function. + CommonInlineUserDefinedFunction func = 3; + + // (Optional) Expressions for sorting. Only used by Scala Sorted Group Map API. + repeated Expression sorting_expressions = 4; + + // Below fields are only used by (Flat)MapGroupsWithState + // (Optional) Input relation for initial State. + Relation initial_input = 5; + + // (Optional) Expressions for grouping keys of the initial state input relation. + repeated Expression initial_grouping_expressions = 6; + + // (Optional) True if MapGroupsWithState, false if FlatMapGroupsWithState. + optional bool is_map_groups_with_state = 7; + + // (Optional) The output mode of the function. + optional string output_mode = 8; + + // (Optional) Timeout configuration for groups that do not receive data for a while. + optional string timeout_conf = 9; + + // (Optional) The schema for the grouped state. + optional DataType state_schema = 10; + + // Below fields are used by TransformWithState and TransformWithStateInPandas + // (Optional) TransformWithState related parameters. + optional TransformWithStateInfo transform_with_state_info = 11; +} + +// Additional input parameters used for TransformWithState operator. +message TransformWithStateInfo { + // (Required) Time mode string for transformWithState. + string time_mode = 1; + + // (Optional) Event time column name. + optional string event_time_column_name = 2; + + // (Optional) Schema for the output DataFrame. + // Only required used for TransformWithStateInPandas. + optional DataType output_schema = 3; +} + +message CoGroupMap { + // (Required) One input relation for CoGroup Map API - applyInPandas. + Relation input = 1; + + // Expressions for grouping keys of the first input relation. + repeated Expression input_grouping_expressions = 2; + + // (Required) The other input relation. + Relation other = 3; + + // Expressions for grouping keys of the other input relation. + repeated Expression other_grouping_expressions = 4; + + // (Required) Input user-defined function. + CommonInlineUserDefinedFunction func = 5; + + // (Optional) Expressions for sorting. Only used by Scala Sorted CoGroup Map API. + repeated Expression input_sorting_expressions = 6; + + // (Optional) Expressions for sorting. Only used by Scala Sorted CoGroup Map API. + repeated Expression other_sorting_expressions = 7; +} + +message ApplyInPandasWithState { + // (Required) Input relation for applyInPandasWithState. + Relation input = 1; + + // (Required) Expressions for grouping keys. + repeated Expression grouping_expressions = 2; + + // (Required) Input user-defined function. + CommonInlineUserDefinedFunction func = 3; + + // (Required) Schema for the output DataFrame. + string output_schema = 4; + + // (Required) Schema for the state. + string state_schema = 5; + + // (Required) The output mode of the function. + string output_mode = 6; + + // (Required) Timeout configuration for groups that do not receive data for a while. + string timeout_conf = 7; +} + +message CommonInlineUserDefinedTableFunction { + // (Required) Name of the user-defined table function. + string function_name = 1; + + // (Optional) Whether the user-defined table function is deterministic. + bool deterministic = 2; + + // (Optional) Function input arguments. Empty arguments are allowed. + repeated Expression arguments = 3; + + // (Required) Type of the user-defined table function. + oneof function { + PythonUDTF python_udtf = 4; + } +} + +message PythonUDTF { + // (Optional) Return type of the Python UDTF. + optional DataType return_type = 1; + + // (Required) EvalType of the Python UDTF. + int32 eval_type = 2; + + // (Required) The encoded commands of the Python UDTF. + bytes command = 3; + + // (Required) Python version being used in the client. + string python_ver = 4; +} + +message CommonInlineUserDefinedDataSource { + // (Required) Name of the data source. + string name = 1; + + // (Required) The data source type. + oneof data_source { + PythonDataSource python_data_source = 2; + } +} + +message PythonDataSource { + // (Required) The encoded commands of the Python data source. + bytes command = 1; + + // (Required) Python version being used in the client. + string python_ver = 2; +} + +// Collect arbitrary (named) metrics from a dataset. +message CollectMetrics { + // (Required) The input relation. + Relation input = 1; + + // (Required) Name of the metrics. + string name = 2; + + // (Required) The metric sequence. + repeated Expression metrics = 3; +} + +message Parse { + // (Required) Input relation to Parse. The input is expected to have single text column. + Relation input = 1; + // (Required) The expected format of the text. + ParseFormat format = 2; + + // (Optional) DataType representing the schema. If not set, Spark will infer the schema. + optional DataType schema = 3; + + // Options for the csv/json/xml parser. The map key is case insensitive. + map options = 4; + enum ParseFormat { + PARSE_FORMAT_UNSPECIFIED = 0; + PARSE_FORMAT_CSV = 1; + PARSE_FORMAT_JSON = 2; + PARSE_FORMAT_XML = 3; + } +} + +// Relation of type [[AsOfJoin]]. +// +// `left` and `right` must be present. +message AsOfJoin { + // (Required) Left input relation for a Join. + Relation left = 1; + + // (Required) Right input relation for a Join. + Relation right = 2; + + // (Required) Field to join on in left DataFrame + Expression left_as_of = 3; + + // (Required) Field to join on in right DataFrame + Expression right_as_of = 4; + + // (Optional) The join condition. Could be unset when `using_columns` is utilized. + // + // This field does not co-exist with using_columns. + Expression join_expr = 5; + + // Optional. using_columns provides a list of columns that should present on both sides of + // the join inputs that this Join will join on. For example A JOIN B USING col_name is + // equivalent to A JOIN B on A.col_name = B.col_name. + // + // This field does not co-exist with join_condition. + repeated string using_columns = 6; + + // (Required) The join type. + string join_type = 7; + + // (Optional) The asof tolerance within this range. + Expression tolerance = 8; + + // (Required) Whether allow matching with the same value or not. + bool allow_exact_matches = 9; + + // (Required) Whether to search for prior, subsequent, or closest matches. + string direction = 10; +} + +// Relation of type [[LateralJoin]]. +// +// `left` and `right` must be present. +message LateralJoin { + // (Required) Left input relation for a Join. + Relation left = 1; + + // (Required) Right input relation for a Join. + Relation right = 2; + + // (Optional) The join condition. + Expression join_condition = 3; + + // (Required) The join type. + Join.JoinType join_type = 4; +} + +// Relation of type [[NearestByJoin]]. +// +// For each row on the left side, returns up to `num_results` rows from the right side ranked +// by `ranking_expression`. +message NearestByJoin { + // (Required) Left (query) input relation. + Relation left = 1; + + // (Required) Right (base) input relation. + Relation right = 2; + + // (Required) Scalar expression used to rank candidate rows on the right side. + Expression ranking_expression = 3; + + // (Required) Maximum number of matches per left row. Must be between 1 and 100000. + int32 num_results = 4; + + // The following three fields use `string` (not typed enums) for parity with `AsOfJoin`, + // which models analogous fields the same way. Validation happens server-side at planning time. + + // (Required) The join type. Must be one of: "inner", "leftouter". + string join_type = 5; + + // (Required) Search algorithm contract. Must be one of: "approx", "exact". + string mode = 6; + + // (Required) Ranking direction. Must be one of: "distance", "similarity". + string direction = 7; +} diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/types.proto b/gateway-service-sparkconnect/src/main/proto/spark/connect/types.proto new file mode 100644 index 0000000000..caaa2340f9 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/proto/spark/connect/types.proto @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = 'proto3'; + +package spark.connect; + +option java_multiple_files = true; +option java_package = "org.apache.spark.connect.proto"; +option go_package = "internal/generated"; + +// This message describes the logical [[DataType]] of something. It does not carry the value +// itself but only describes it. +message DataType { + oneof kind { + NULL null = 1; + + Binary binary = 2; + + Boolean boolean = 3; + + // Numeric types + Byte byte = 4; + Short short = 5; + Integer integer = 6; + Long long = 7; + + Float float = 8; + Double double = 9; + Decimal decimal = 10; + + // String types + String string = 11; + Char char = 12; + VarChar var_char = 13; + + // Datatime types + Date date = 14; + Timestamp timestamp = 15; + TimestampNTZ timestamp_ntz = 16; + + // Interval types + CalendarInterval calendar_interval = 17; + YearMonthInterval year_month_interval = 18; + DayTimeInterval day_time_interval = 19; + + // Complex types + Array array = 20; + Struct struct = 21; + Map map = 22; + Variant variant = 25; + + // UserDefinedType + UDT udt = 23; + + // Geospatial types + Geometry geometry = 26; + + Geography geography = 27; + + // UnparsedDataType + Unparsed unparsed = 24; + + Time time = 28; + } + + message Boolean { + uint32 type_variation_reference = 1; + } + + message Byte { + uint32 type_variation_reference = 1; + } + + message Short { + uint32 type_variation_reference = 1; + } + + message Integer { + uint32 type_variation_reference = 1; + } + + message Long { + uint32 type_variation_reference = 1; + } + + message Float { + uint32 type_variation_reference = 1; + } + + message Double { + uint32 type_variation_reference = 1; + } + + message String { + uint32 type_variation_reference = 1; + string collation = 2; + } + + message Binary { + uint32 type_variation_reference = 1; + } + + message NULL { + uint32 type_variation_reference = 1; + } + + message Timestamp { + uint32 type_variation_reference = 1; + } + + message Date { + uint32 type_variation_reference = 1; + } + + message TimestampNTZ { + uint32 type_variation_reference = 1; + } + + message Time { + optional int32 precision = 1; + uint32 type_variation_reference = 2; + } + + message CalendarInterval { + uint32 type_variation_reference = 1; + } + + message YearMonthInterval { + optional int32 start_field = 1; + optional int32 end_field = 2; + uint32 type_variation_reference = 3; + } + + message DayTimeInterval { + optional int32 start_field = 1; + optional int32 end_field = 2; + uint32 type_variation_reference = 3; + } + + // Start compound types. + message Char { + int32 length = 1; + uint32 type_variation_reference = 2; + } + + message VarChar { + int32 length = 1; + uint32 type_variation_reference = 2; + } + + message Decimal { + optional int32 scale = 1; + optional int32 precision = 2; + uint32 type_variation_reference = 3; + } + + message StructField { + string name = 1; + DataType data_type = 2; + bool nullable = 3; + optional string metadata = 4; + } + + message Struct { + repeated StructField fields = 1; + uint32 type_variation_reference = 2; + } + + message Array { + DataType element_type = 1; + bool contains_null = 2; + uint32 type_variation_reference = 3; + } + + message Map { + DataType key_type = 1; + DataType value_type = 2; + bool value_contains_null = 3; + uint32 type_variation_reference = 4; + } + + message Geometry { + int32 srid = 1; + uint32 type_variation_reference = 2; + } + + message Geography { + int32 srid = 1; + uint32 type_variation_reference = 2; + } + + message Variant { + uint32 type_variation_reference = 1; + } + + message UDT { + string type = 1; + // Required for Scala/Java UDT + optional string jvm_class = 2; + // Required for Python UDT + optional string python_class = 3; + // Required for Python UDT + optional string serialized_python_class = 4; + // Required for Python UDT + optional DataType sql_type = 5; + } + + message Unparsed { + // (Required) The unparsed data type string + string data_type_string = 1; + } +} diff --git a/gateway-service-sparkconnect/src/main/resources/META-INF/services/org.apache.knox.gateway.protocol.ProtocolListener b/gateway-service-sparkconnect/src/main/resources/META-INF/services/org.apache.knox.gateway.protocol.ProtocolListener new file mode 100644 index 0000000000..aca5a0e902 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/resources/META-INF/services/org.apache.knox.gateway.protocol.ProtocolListener @@ -0,0 +1,18 @@ +########################################################################## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +########################################################################## +org.apache.knox.gateway.sparkconnect.SparkConnectListener diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AclAuthorizerTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AclAuthorizerTest.java new file mode 100644 index 0000000000..ec76d6fdb7 --- /dev/null +++ b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AclAuthorizerTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.knox.gateway.filter.InvalidACLException; + +import org.junit.Test; + +public class AclAuthorizerTest { + + private static final String ROLE = "SPARKCONNECT"; + private static final String IP = "10.0.0.5"; + + @Test + public void allowsEverythingWhenTheTopologyDeclaresNoAcls() throws Exception { + // Matching the servlet provider: a topology that never mentions the role does + // not silently deny it. + final AclAuthorizer authorizer = authorizer(Collections.emptyMap()); + assertTrue(authorizer.isPermitted("alice", groups(), IP)); + } + + @Test + public void andModeRequiresUserGroupAndAddressToMatch() throws Exception { + final AclAuthorizer authorizer = authorizer(acls("alice;analysts;10.0.0.*", "AND")); + + assertTrue(authorizer.isPermitted("alice", groups("analysts"), IP)); + assertFalse("wrong user", authorizer.isPermitted("mallory", groups("analysts"), IP)); + assertFalse("wrong group", authorizer.isPermitted("alice", groups("interns"), IP)); + assertFalse("wrong address", authorizer.isPermitted("alice", groups("analysts"), "192.168.1.1")); + } + + @Test + public void andModeAcceptsAGrouplessSubjectAgainstAGroupWildcard() throws Exception { + // A token minted without a groups claim must still satisfy '*;*;10.0.0.*'. + final AclAuthorizer authorizer = authorizer(acls("*;*;10.0.0.*", "AND")); + assertTrue(authorizer.isPermitted("alice", groups(), IP)); + } + + @Test + public void andModeRejectsAGrouplessSubjectAgainstANamedGroup() throws Exception { + final AclAuthorizer authorizer = authorizer(acls("*;analysts;*", "AND")); + assertFalse(authorizer.isPermitted("alice", groups(), IP)); + } + + @Test + public void orModeAcceptsAnySingleMatch() throws Exception { + final AclAuthorizer authorizer = authorizer(acls("alice;analysts;10.0.0.*", "OR")); + + assertTrue("user match alone", authorizer.isPermitted("alice", groups("interns"), "192.168.1.1")); + assertTrue("group match alone", authorizer.isPermitted("mallory", groups("analysts"), "192.168.1.1")); + assertTrue("address match alone", authorizer.isPermitted("mallory", groups("interns"), IP)); + assertFalse(authorizer.isPermitted("mallory", groups("interns"), "192.168.1.1")); + } + + @Test + public void orModeTreatsWildcardsAsNoReasonToGrant() throws Exception { + // Otherwise a single '*' in any position would admit everyone, which inverts + // the intent of an OR policy. + final AclAuthorizer authorizer = authorizer(acls("*;analysts;*", "OR")); + + assertTrue(authorizer.isPermitted("alice", groups("analysts"), IP)); + assertFalse(authorizer.isPermitted("alice", groups("interns"), IP)); + } + + @Test + public void resolvesTheKnoxAdminPlaceholders() throws Exception { + final AclAuthorizer users = new AclAuthorizer(ROLE, + acls("KNOX_ADMIN_USERS;*;*", "AND"), "admin,root", "wheel"); + assertTrue(users.isPermitted("admin", groups(), IP)); + assertFalse(users.isPermitted("alice", groups(), IP)); + + final AclAuthorizer adminGroups = new AclAuthorizer(ROLE, + acls("*;KNOX_ADMIN_GROUPS;*", "AND"), "admin", "wheel"); + assertTrue(adminGroups.isPermitted("alice", groups("wheel"), IP)); + assertFalse(adminGroups.isPermitted("alice", groups("interns"), IP)); + } + + @Test + public void defaultsToAndWhenNoModeIsConfigured() throws Exception { + final Map params = new HashMap<>(); + params.put(ROLE + ".acl", "alice;*;*"); + final AclAuthorizer authorizer = authorizer(params); + + assertTrue(authorizer.isPermitted("alice", groups(), IP)); + assertFalse(authorizer.isPermitted("mallory", groups(), IP)); + } + + @Test + public void readsParametersCaseInsensitively() throws Exception { + // Provider params are lowercased when they become filter params on the + // servlet path, so the same topology XML has to work here either way. + final Map params = new HashMap<>(); + params.put("sparkconnect.acl", "alice;*;*"); + final AclAuthorizer authorizer = authorizer(params); + + assertTrue(authorizer.isPermitted("alice", groups(), IP)); + assertFalse(authorizer.isPermitted("mallory", groups(), IP)); + } + + @Test + public void fallsBackToTheSharedAclMode() throws Exception { + final Map params = new HashMap<>(); + params.put(ROLE + ".acl", "alice;analysts;10.0.0.*"); + params.put("acl.mode", "OR"); + final AclAuthorizer authorizer = authorizer(params); + + assertTrue(authorizer.isPermitted("mallory", groups("analysts"), "192.168.1.1")); + } + + @Test(expected = InvalidACLException.class) + public void rejectsAMalformedAcl() throws Exception { + authorizer(acls("alice;analysts", "AND")); + } + + private static AclAuthorizer authorizer(Map params) throws InvalidACLException { + return new AclAuthorizer(ROLE, params, "", ""); + } + + private static Map acls(String acl, String mode) { + final Map params = new HashMap<>(); + params.put(ROLE + ".acl", acl); + params.put(ROLE + ".acl.mode", mode); + return params; + } + + private static Set groups(String... names) { + return new HashSet<>(java.util.Arrays.asList(names)); + } +} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuardTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuardTest.java new file mode 100644 index 0000000000..9eadfee5bd --- /dev/null +++ b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuardTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.sparkconnect; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.Arrays; +import java.util.Collections; + +import com.google.protobuf.Message; + +import io.grpc.Status; +import io.grpc.StatusRuntimeException; + +import org.apache.spark.connect.proto.AddArtifactsRequest; + +import org.junit.Test; + +public class AddArtifactsGuardTest { + + private static final Message REQUEST = AddArtifactsRequest.getDefaultInstance(); + + @Test + public void allowModePermitsEveryone() { + new AddArtifactsGuard(AddArtifactsGuard.MODE_ALLOW, Collections.emptyList()).check(REQUEST, "alice"); + } + + @Test + public void defaultsToAllowWhenUnconfigured() { + new AddArtifactsGuard(null, null).check(REQUEST, "alice"); + } + + @Test + public void denyModeRejectsEveryone() { + final AddArtifactsGuard guard = + new AddArtifactsGuard(AddArtifactsGuard.MODE_DENY, Arrays.asList("alice")); + assertTrue(guard.deniesEveryone()); + // Even a listed user is refused: DENY is not "deny except the list". + assertDenied(guard, "alice"); + } + + @Test + public void listedUsersModeAdmitsOnlyListedUsers() { + final AddArtifactsGuard guard = new AddArtifactsGuard( + AddArtifactsGuard.MODE_ALLOW_LISTED_USERS, Arrays.asList("alice", "bob")); + assertFalse(guard.deniesEveryone()); + guard.check(REQUEST, "alice"); + guard.check(REQUEST, "bob"); + assertDenied(guard, "mallory"); + } + + @Test + public void modeIsCaseAndWhitespaceInsensitive() { + new AddArtifactsGuard(" allow ", Collections.emptyList()).check(REQUEST, "alice"); + } + + @Test + public void unrecognisedModeFailsClosed() { + // A typo in configuration must not silently become "allow everyone". + assertDenied(new AddArtifactsGuard("permissive", Collections.emptyList()), "alice"); + } + + private static void assertDenied(AddArtifactsGuard guard, String principal) { + try { + guard.check(REQUEST, principal); + fail("Expected artifact upload to be denied for " + principal); + } catch (StatusRuntimeException e) { + assertEquals(Status.Code.PERMISSION_DENIED, e.getStatus().getCode()); + } + } +} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuardTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuardTest.java new file mode 100644 index 0000000000..41795ea043 --- /dev/null +++ b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuardTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.sparkconnect; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import io.grpc.Status; +import io.grpc.StatusRuntimeException; + +import org.apache.spark.connect.proto.ConfigRequest; +import org.apache.spark.connect.proto.KeyValue; + +import org.junit.Test; + +public class ReservedConfigGuardTest { + + private static final String PREFIX = "knox."; + private static final String USER = "alice"; + + private final ReservedConfigGuard guard = new ReservedConfigGuard(PREFIX); + + @Test + public void deniesSettingAReservedKey() { + // If a client could overwrite the key Knox publishes the identity into, it + // could assume any identity it liked. + assertDenied(configSet("knox.principal", "root")); + } + + @Test + public void deniesSettingAReservedKeyRegardlessOfCase() { + assertDenied(configSet("KNOX.Principal", "root")); + } + + @Test + public void deniesUnsettingAReservedKey() { + // Clearing the key is as good as overwriting it if downstream code then + // falls back to something less trustworthy. + assertDenied(ConfigRequest.newBuilder() + .setOperation(ConfigRequest.Operation.newBuilder() + .setUnset(ConfigRequest.Unset.newBuilder().addKeys("knox.principal"))) + .build()); + } + + @Test + public void deniesWhenAReservedKeyIsBuriedAmongAllowedOnes() { + assertDenied(ConfigRequest.newBuilder() + .setOperation(ConfigRequest.Operation.newBuilder() + .setSet(ConfigRequest.Set.newBuilder() + .addPairs(KeyValue.newBuilder().setKey("spark.sql.shuffle.partitions").setValue("8")) + .addPairs(KeyValue.newBuilder().setKey("knox.principal").setValue("root")))) + .build()); + } + + @Test + public void allowsOrdinarySparkSettings() { + guard.check(configSet("spark.sql.shuffle.partitions", "8"), USER); + } + + @Test + public void allowsReadingAReservedKey() { + // Reading is harmless; only writes can forge an identity. + guard.check(ConfigRequest.newBuilder() + .setOperation(ConfigRequest.Operation.newBuilder() + .setGet(ConfigRequest.Get.newBuilder().addKeys("knox.principal"))) + .build(), USER); + } + + @Test + public void allowsRequestsWithNoConfigOperation() { + guard.check(ConfigRequest.getDefaultInstance(), USER); + } + + @Test + public void doesNothingWhenNoPrefixIsReserved() { + new ReservedConfigGuard("").check(configSet("knox.principal", "root"), USER); + } + + private void assertDenied(ConfigRequest request) { + try { + guard.check(request, USER); + fail("Expected the reserved key write to be denied"); + } catch (StatusRuntimeException e) { + assertEquals(Status.Code.PERMISSION_DENIED, e.getStatus().getCode()); + } + } + + private static ConfigRequest configSet(String key, String value) { + return ConfigRequest.newBuilder() + .setOperation(ConfigRequest.Operation.newBuilder() + .setSet(ConfigRequest.Set.newBuilder() + .addPairs(KeyValue.newBuilder().setKey(key).setValue(value)))) + .build(); + } +} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptorTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptorTest.java new file mode 100644 index 0000000000..79e071420c --- /dev/null +++ b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptorTest.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.sparkconnect; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.apache.knox.gateway.grpc.GrpcCallContext; + +import com.google.protobuf.Message; + +import io.grpc.Status; +import io.grpc.StatusRuntimeException; + +import org.apache.spark.connect.proto.AnalyzePlanRequest; +import org.apache.spark.connect.proto.ConfigRequest; +import org.apache.spark.connect.proto.ExecutePlanRequest; +import org.apache.spark.connect.proto.InterruptRequest; +import org.apache.spark.connect.proto.ReattachExecuteRequest; +import org.apache.spark.connect.proto.UserContext; + +import org.junit.Test; + +public class SparkConnectMessageInterceptorTest { + + private static final String PRINCIPAL = "alice"; + private static final String SPOOFED = "root"; + + @Test + public void overwritesClientSuppliedUserId() { + final ExecutePlanRequest request = ExecutePlanRequest.newBuilder() + .setSessionId("session-1") + .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).setUserName(SPOOFED).build()) + .build(); + + final ExecutePlanRequest forwarded = (ExecutePlanRequest) intercept(request, PRINCIPAL); + + // The whole point of parsing message bodies: the client says who it is, and + // Spark believes it, so Knox replaces the claim with the identity it verified. + assertEquals(PRINCIPAL, forwarded.getUserContext().getUserId()); + assertEquals(PRINCIPAL, forwarded.getUserContext().getUserName()); + } + + @Test + public void assertsIdentityWhenClientSuppliesNoUserContext() { + final ExecutePlanRequest request = ExecutePlanRequest.newBuilder().setSessionId("session-1").build(); + + final ExecutePlanRequest forwarded = (ExecutePlanRequest) intercept(request, PRINCIPAL); + + assertEquals(PRINCIPAL, forwarded.getUserContext().getUserId()); + } + + @Test + public void assertsIdentityOnEveryRequestShape() { + // All twelve request types carry UserContext in the same field position, which + // is why one descriptor-driven rewrite covers the whole service rather than + // needing a handler per RPC. Spot-check across the RPC shapes. + final Message[] requests = { + ExecutePlanRequest.newBuilder() + .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()).build(), + AnalyzePlanRequest.newBuilder() + .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()).build(), + ConfigRequest.newBuilder() + .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()).build(), + InterruptRequest.newBuilder() + .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()).build(), + ReattachExecuteRequest.newBuilder() + .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()).build(), + }; + + for (Message request : requests) { + final Message forwarded = intercept(request, PRINCIPAL); + final UserContext userContext = (UserContext) forwarded.getField( + forwarded.getDescriptorForType().findFieldByName("user_context")); + assertEquals(request.getDescriptorForType().getName() + " kept the client's user_id", + PRINCIPAL, userContext.getUserId()); + } + } + + @Test + public void preservesEveryOtherField() { + final ExecutePlanRequest request = ExecutePlanRequest.newBuilder() + .setSessionId("session-1") + .setOperationId("operation-1") + .setClientType("pyspark") + .addTags("tag-a") + .addTags("tag-b") + .setUserContext(UserContext.newBuilder() + .setUserId(SPOOFED) + .addExtensions(com.google.protobuf.Any.newBuilder().setTypeUrl("type/x").build()) + .build()) + .build(); + + final ExecutePlanRequest forwarded = (ExecutePlanRequest) intercept(request, PRINCIPAL); + + assertEquals("session-1", forwarded.getSessionId()); + assertEquals("operation-1", forwarded.getOperationId()); + assertEquals("pyspark", forwarded.getClientType()); + assertEquals(request.getTagsList(), forwarded.getTagsList()); + // UserContext extensions belong to the client, not to the identity claim. + assertEquals(1, forwarded.getUserContext().getExtensionsCount()); + assertEquals("type/x", forwarded.getUserContext().getExtensions(0).getTypeUrl()); + } + + @Test + public void preservesFieldsUnknownToTheVendoredProtos() throws Exception { + // A newer Spark client can send fields these protos do not describe. Protobuf + // retains them as unknown fields across a parse and re-serialize, which is + // what keeps proto skew from being a breaking problem. + final ExecutePlanRequest known = ExecutePlanRequest.newBuilder() + .setSessionId("session-1") + .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()) + .build(); + final com.google.protobuf.UnknownFieldSet unknown = com.google.protobuf.UnknownFieldSet.newBuilder() + .addField(9999, com.google.protobuf.UnknownFieldSet.Field.newBuilder() + .addVarint(42L).build()) + .build(); + final ExecutePlanRequest request = known.toBuilder().setUnknownFields(unknown).build(); + + final ExecutePlanRequest forwarded = (ExecutePlanRequest) intercept(request, PRINCIPAL); + + assertEquals(PRINCIPAL, forwarded.getUserContext().getUserId()); + assertEquals(42L, forwarded.getUnknownFields().getField(9999).getVarintList().get(0).longValue()); + } + + @Test + public void recordsSessionAndOperationForAuditing() { + final GrpcCallContext callContext = newCallContext(PRINCIPAL); + final ExecutePlanRequest request = ExecutePlanRequest.newBuilder() + .setSessionId("session-7") + .setOperationId("operation-9") + .build(); + + interceptWith(callContext, request, null); + + assertEquals("session-7", callContext.getSessionId()); + assertEquals("operation-9", callContext.getOperationId()); + } + + @Test + public void refusesToForwardWithoutAnAuthenticatedPrincipal() { + final GrpcCallContext callContext = + new GrpcCallContext("m", "authority", "127.0.0.1", System.nanoTime()); + try { + interceptWith(callContext, ExecutePlanRequest.getDefaultInstance(), null); + fail("Expected the request to be rejected without a principal"); + } catch (StatusRuntimeException e) { + // Forwarding here would send the client's own identity claim through + // untouched, which is exactly the spoofing this layer exists to stop. + assertEquals(Status.Code.INTERNAL, e.getStatus().getCode()); + } + } + + @Test + public void appliesTheConfiguredGuardBeforeRewriting() { + final boolean[] guardRan = {false}; + final SparkConnectMessageInterceptor.RequestGuard guard = (message, principal) -> { + guardRan[0] = true; + assertEquals(PRINCIPAL, principal); + // The guard sees the client's message, before identity assertion. + assertNotNull(message); + }; + + interceptWith(newCallContext(PRINCIPAL), ExecutePlanRequest.getDefaultInstance(), guard); + + assertTrue("guard was not invoked", guardRan[0]); + } + + private static Message intercept(Message request, String principal) { + return interceptWith(newCallContext(principal), request, null); + } + + private static Message interceptWith(GrpcCallContext callContext, + Message request, + SparkConnectMessageInterceptor.RequestGuard guard) { + final Message[] result = new Message[1]; + // Context.run keeps StatusRuntimeException unwrapped, which the rejection + // tests assert on directly. + io.grpc.Context.current() + .withValue(GrpcCallContext.KEY, callContext) + .run(() -> result[0] = new SparkConnectMessageInterceptor(guard).intercept(request)); + return result[0]; + } + + private static GrpcCallContext newCallContext(String principal) { + final GrpcCallContext callContext = + new GrpcCallContext("spark.connect.SparkConnectService/ExecutePlan", + "knox.example.com:15002", "127.0.0.1", System.nanoTime()); + callContext.setPrincipal(principal); + return callContext; + } +} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectProxyIntegrationTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectProxyIntegrationTest.java new file mode 100644 index 0000000000..0333abfda3 --- /dev/null +++ b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectProxyIntegrationTest.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.sparkconnect; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.knox.gateway.grpc.BackendChannelProvider; +import org.apache.knox.gateway.grpc.GrpcCallContext; +import org.apache.knox.gateway.grpc.GrpcMetadataKeys; +import org.apache.knox.gateway.grpc.HeaderRewriter; +import org.apache.knox.gateway.grpc.MessageInterceptor; +import org.apache.knox.gateway.grpc.ProxyCallHandler; + +import com.google.protobuf.Message; + +import io.grpc.Context; +import io.grpc.Contexts; +import io.grpc.ManagedChannel; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.Server; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.ServerInterceptors; +import io.grpc.ServerMethodDefinition; +import io.grpc.ServerServiceDefinition; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.stub.StreamObserver; + +import org.apache.spark.connect.proto.AnalyzePlanRequest; +import org.apache.spark.connect.proto.AnalyzePlanResponse; +import org.apache.spark.connect.proto.ExecutePlanRequest; +import org.apache.spark.connect.proto.ExecutePlanResponse; +import org.apache.spark.connect.proto.SparkConnectServiceGrpc; +import org.apache.spark.connect.proto.UserContext; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +/** + * Exercises the relay against a real gRPC backend over an in-process transport: + * a stand-in Spark Connect server, the gateway's proxy handler in front of it, + * and a generated client stub calling through. + *

+ * The properties under test are the ones a hand-rolled proxy tends to get wrong + * — response streaming, verbatim trailers and status codes, and rejecting a + * gated call before the backend is touched — so they are checked end to end + * rather than against mocks. + */ +// volatile: the stub's knobs are set by the test thread and read by the gRPC +// server threads handling the call. +@SuppressWarnings("PMD.AvoidUsingVolatile") +public class SparkConnectProxyIntegrationTest { + + /** Stands in for the trailer Spark uses to carry structured error details. */ + private static final Metadata.Key ERROR_DETAILS = + Metadata.Key.of("x-spark-error-class", Metadata.ASCII_STRING_MARSHALLER); + private static final Context.Key BACKEND_HEADERS = Context.key("backendHeaders"); + private static final String PRINCIPAL = "alice"; + + /** A stalled relay is a plausible failure mode here, so fail rather than hang. */ + @Rule + public final Timeout timeout = Timeout.seconds(30); + + private Server backend; + private Server gateway; + private ManagedChannel backendChannel; + private ManagedChannel clientChannel; + private StubSparkConnect stub; + + @Before + public void setUp() throws Exception { + final String backendName = InProcessServerBuilder.generateName(); + final String gatewayName = InProcessServerBuilder.generateName(); + + stub = new StubSparkConnect(); + backend = InProcessServerBuilder.forName(backendName) + .addService(ServerInterceptors.intercept(stub, new CaptureHeaders())) + .build() + .start(); + backendChannel = InProcessChannelBuilder.forName(backendName).build(); + + gateway = InProcessServerBuilder.forName(gatewayName) + .addService(proxyService(identityAsserting())) + .build() + .start(); + clientChannel = InProcessChannelBuilder.forName(gatewayName).build(); + } + + @After + public void tearDown() { + shutdown(clientChannel); + shutdown(backendChannel); + shutdown(gateway); + shutdown(backend); + } + + private static void shutdown(ManagedChannel channel) { + if (channel != null) { + channel.shutdownNow(); + } + } + + private static void shutdown(Server server) { + if (server != null) { + server.shutdownNow(); + } + } + + private MessageInterceptor identityAsserting() { + return new SparkConnectMessageInterceptor(null); + } + + /** + * Registers a relay for every Spark Connect method, inside a context carrying + * the principal and backend the interceptor chain would normally have + * resolved. The chain itself is covered by its own tests. + */ + private ServerServiceDefinition proxyService(MessageInterceptor messageInterceptor) { + final BackendChannelProvider channels = () -> backendChannel; + final HeaderRewriter headers = metadata -> metadata.removeAll(GrpcMetadataKeys.AUTHORIZATION); + + final ServerServiceDefinition.Builder builder = + ServerServiceDefinition.builder(SparkConnectServiceGrpc.getServiceDescriptor()); + for (MethodDescriptor method : SparkConnectServiceGrpc.getServiceDescriptor().getMethods()) { + @SuppressWarnings("unchecked") + final MethodDescriptor descriptor = (MethodDescriptor) method; + final ProxyCallHandler handler = + new ProxyCallHandler<>(channels, messageInterceptor, headers); + + builder.addMethod(ServerMethodDefinition.create(descriptor, (call, metadata) -> { + final GrpcCallContext callContext = new GrpcCallContext( + descriptor.getFullMethodName(), "test", "127.0.0.1", System.nanoTime()); + callContext.setPrincipal(PRINCIPAL); + callContext.setBackendUrl("grpc://backend:15002"); + // Contexts.interceptCall, exactly as the audit interceptor uses it: it + // attaches the context to the listener callbacks too, not just to + // startCall. Request messages arrive in onMessage, well after startCall + // returns, so anything that only wrapped startCall would leave the + // handler without a principal at the moment it needs one. + return Contexts.interceptCall( + Context.current().withValue(GrpcCallContext.KEY, callContext), + call, metadata, handler); + })); + } + return builder.build(); + } + + @Test + public void relaysAUnaryCall() { + final AnalyzePlanResponse response = SparkConnectServiceGrpc.newBlockingStub(clientChannel) + .analyzePlan(AnalyzePlanRequest.newBuilder().setSessionId("s1").build()); + + assertEquals("s1", response.getSessionId()); + assertEquals(1, stub.analyzeRequests.size()); + } + + @Test + public void assertsIdentityOnTheWayThrough() { + SparkConnectServiceGrpc.newBlockingStub(clientChannel) + .analyzePlan(AnalyzePlanRequest.newBuilder() + .setSessionId("s1") + .setUserContext(UserContext.newBuilder().setUserId("root").build()) + .build()); + + // Assert on what the backend received, not on what the client sent. + assertEquals(PRINCIPAL, stub.analyzeRequests.get(0).getUserContext().getUserId()); + } + + @Test + public void stripsTheClientCredentialFromTheBackendLeg() { + SparkConnectServiceGrpc.newBlockingStub(clientChannel) + .analyzePlan(AnalyzePlanRequest.newBuilder().setSessionId("s1").build()); + + // The user's token authenticates them to Knox and has no meaning past it. + assertNotNull(stub.lastHeaders); + assertNull(stub.lastHeaders.get(GrpcMetadataKeys.AUTHORIZATION)); + } + + @Test + public void relaysEveryMessageOfAServerStream() { + stub.executeResponseCount = 25; + + final Iterator responses = SparkConnectServiceGrpc + .newBlockingStub(clientChannel) + .executePlan(ExecutePlanRequest.newBuilder().setSessionId("s1").build()); + + final List ids = new ArrayList<>(); + while (responses.hasNext()) { + ids.add(responses.next().getResponseId()); + } + // Long server streams are the normal case for ExecutePlan, not an edge case. + assertEquals(25, ids.size()); + assertEquals("response-0", ids.get(0)); + assertEquals("response-24", ids.get(24)); + } + + @Test + public void relaysBackendStatusCodeVerbatim() { + stub.failExecuteWith = Status.RESOURCE_EXHAUSTED.withDescription("query too large"); + + try { + SparkConnectServiceGrpc.newBlockingStub(clientChannel) + .executePlan(ExecutePlanRequest.newBuilder().setSessionId("s1").build()) + .next(); + fail("Expected the backend failure to surface at the client"); + } catch (StatusRuntimeException e) { + // Not remapped to INTERNAL or UNKNOWN: clients branch on these codes. + assertEquals(Status.Code.RESOURCE_EXHAUSTED, e.getStatus().getCode()); + assertEquals("query too large", e.getStatus().getDescription()); + } + } + + @Test + public void relaysBackendTrailersVerbatim() { + stub.failExecuteWith = Status.INTERNAL.withDescription("boom"); + stub.failureTrailerValue = "AnalysisException"; + + try { + SparkConnectServiceGrpc.newBlockingStub(clientChannel) + .executePlan(ExecutePlanRequest.newBuilder().setSessionId("s1").build()) + .next(); + fail("Expected the backend failure to surface at the client"); + } catch (StatusRuntimeException e) { + // Spark packs structured error details into trailers and clients read them, + // so losing trailers breaks error reporting wholesale rather than at edges. + assertNotNull("trailers were not relayed", e.getTrailers()); + assertEquals("AnalysisException", e.getTrailers().get(ERROR_DETAILS)); + } + } + + @Test + public void relaysUnimplementedForAMethodTheBackendLacks() { + stub.failExecuteWith = Status.UNIMPLEMENTED.withDescription("not in this Spark"); + + try { + SparkConnectServiceGrpc.newBlockingStub(clientChannel) + .executePlan(ExecutePlanRequest.newBuilder().setSessionId("s1").build()) + .next(); + fail("Expected UNIMPLEMENTED to surface"); + } catch (StatusRuntimeException e) { + assertEquals(Status.Code.UNIMPLEMENTED, e.getStatus().getCode()); + } + } + + @Test + public void rejectsAGatedCallWithoutContactingTheBackend() throws Exception { + final String name = InProcessServerBuilder.generateName(); + final MessageInterceptor denying = message -> { + throw Status.PERMISSION_DENIED.withDescription("nope").asRuntimeException(); + }; + + final Server gatingGateway = InProcessServerBuilder.forName(name) + .addService(proxyService(denying)).build().start(); + final ManagedChannel gatingClient = InProcessChannelBuilder.forName(name).build(); + try { + final int before = stub.analyzeRequests.size(); + try { + SparkConnectServiceGrpc.newBlockingStub(gatingClient) + .analyzePlan(AnalyzePlanRequest.newBuilder().setSessionId("s1").build()); + fail("Expected the gated call to be denied"); + } catch (StatusRuntimeException e) { + assertEquals(Status.Code.PERMISSION_DENIED, e.getStatus().getCode()); + } + // A denial that still forwarded the message would be advisory, not enforcing. + assertEquals(before, stub.analyzeRequests.size()); + } finally { + shutdown(gatingClient); + shutdown(gatingGateway); + } + } + + /** Records the metadata the backend actually received. */ + private static final class CaptureHeaders implements ServerInterceptor { + @Override + public ServerCall.Listener interceptCall(ServerCall call, + Metadata headers, + ServerCallHandler next) { + return Contexts.interceptCall( + Context.current().withValue(BACKEND_HEADERS, headers), call, headers, next); + } + } + + /** A stand-in Spark Connect server that records what it was sent. */ + private static final class StubSparkConnect + extends SparkConnectServiceGrpc.SparkConnectServiceImplBase { + + private final List analyzeRequests = new ArrayList<>(); + private volatile Metadata lastHeaders; + private volatile int executeResponseCount = 1; + private volatile Status failExecuteWith; + private volatile String failureTrailerValue; + + @Override + public void analyzePlan(AnalyzePlanRequest request, StreamObserver observer) { + analyzeRequests.add(request); + lastHeaders = BACKEND_HEADERS.get(); + observer.onNext(AnalyzePlanResponse.newBuilder().setSessionId(request.getSessionId()).build()); + observer.onCompleted(); + } + + @Override + public void executePlan(ExecutePlanRequest request, StreamObserver observer) { + lastHeaders = BACKEND_HEADERS.get(); + if (failExecuteWith != null) { + final Metadata trailers = new Metadata(); + if (failureTrailerValue != null) { + trailers.put(ERROR_DETAILS, failureTrailerValue); + } + observer.onError(failExecuteWith.asRuntimeException(trailers)); + return; + } + for (int i = 0; i < executeResponseCount; i++) { + observer.onNext(ExecutePlanResponse.newBuilder() + .setSessionId(request.getSessionId()) + .setResponseId("response-" + i) + .build()); + } + observer.onCompleted(); + } + } +} diff --git a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java index a3b3d0d9a8..813a4807b8 100644 --- a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java +++ b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java @@ -55,6 +55,23 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig { public static final int DEFAULT_WEBSHELL_MAX_CONCURRENT_SESSIONS = 3; public static final int DEFAULT_WEBSHELL_READ_BUFFER_SIZE = 1024; + /* Spark Connect defaults */ + public static final int DEFAULT_SPARKCONNECT_PORT = 15002; + public static final int DEFAULT_SPARKCONNECT_MAX_MESSAGE_SIZE = 134217728; + public static final long DEFAULT_SPARKCONNECT_PERMIT_KEEPALIVE_TIME = 10000L; + public static final int DEFAULT_SPARKCONNECT_MAX_CONCURRENT_CALLS_PER_CONNECTION = 1000; + public static final long DEFAULT_SPARKCONNECT_CHANNEL_IDLE_TIMEOUT = 1800000L; + public static final long DEFAULT_SPARKCONNECT_DRAIN_TIMEOUT = 30000L; + public static final String DEFAULT_SPARKCONNECT_ADD_ARTIFACTS_MODE = "ALLOW"; + public static final String DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX = "knox."; + + private boolean sparkConnectEnabled; + private int sparkConnectPort = DEFAULT_SPARKCONNECT_PORT; + private String sparkConnectDefaultTopology; + private String sparkConnectBackendTokenAlias; + private String sparkConnectAddArtifactsMode = DEFAULT_SPARKCONNECT_ADD_ARTIFACTS_MODE; + private List sparkConnectAddArtifactsAllowedUsers = Collections.emptyList(); + private Path gatewayHomePath = Paths.get("gateway-home"); @@ -681,6 +698,95 @@ public int getWebsocketMaxWaitBufferCount() { return DEFAULT_WEBSOCKET_MAX_WAIT_BUFFER_COUNT; } + @Override + public boolean isSparkConnectEnabled() { + return sparkConnectEnabled; + } + + public void setSparkConnectEnabled(boolean sparkConnectEnabled) { + this.sparkConnectEnabled = sparkConnectEnabled; + } + + @Override + public int getSparkConnectPort() { + return sparkConnectPort; + } + + public void setSparkConnectPort(int sparkConnectPort) { + this.sparkConnectPort = sparkConnectPort; + } + + @Override + public String getSparkConnectDefaultTopology() { + return sparkConnectDefaultTopology; + } + + public void setSparkConnectDefaultTopology(String sparkConnectDefaultTopology) { + this.sparkConnectDefaultTopology = sparkConnectDefaultTopology; + } + + @Override + public int getSparkConnectMaxMessageSize() { + return DEFAULT_SPARKCONNECT_MAX_MESSAGE_SIZE; + } + + @Override + public long getSparkConnectPermitKeepAliveTime() { + return DEFAULT_SPARKCONNECT_PERMIT_KEEPALIVE_TIME; + } + + @Override + public boolean isSparkConnectPermitKeepAliveWithoutCalls() { + return true; + } + + @Override + public int getSparkConnectMaxConcurrentCallsPerConnection() { + return DEFAULT_SPARKCONNECT_MAX_CONCURRENT_CALLS_PER_CONNECTION; + } + + @Override + public long getSparkConnectChannelIdleTimeout() { + return DEFAULT_SPARKCONNECT_CHANNEL_IDLE_TIMEOUT; + } + + @Override + public long getSparkConnectDrainTimeout() { + return DEFAULT_SPARKCONNECT_DRAIN_TIMEOUT; + } + + @Override + public String getSparkConnectBackendTokenAlias() { + return sparkConnectBackendTokenAlias; + } + + public void setSparkConnectBackendTokenAlias(String sparkConnectBackendTokenAlias) { + this.sparkConnectBackendTokenAlias = sparkConnectBackendTokenAlias; + } + + @Override + public String getSparkConnectAddArtifactsMode() { + return sparkConnectAddArtifactsMode; + } + + public void setSparkConnectAddArtifactsMode(String sparkConnectAddArtifactsMode) { + this.sparkConnectAddArtifactsMode = sparkConnectAddArtifactsMode; + } + + @Override + public List getSparkConnectAddArtifactsAllowedUsers() { + return sparkConnectAddArtifactsAllowedUsers; + } + + public void setSparkConnectAddArtifactsAllowedUsers(List allowedUsers) { + this.sparkConnectAddArtifactsAllowedUsers = allowedUsers; + } + + @Override + public String getSparkConnectReservedConfigPrefix() { + return DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX; + } + @Override public boolean isMetricsEnabled() { return false; diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java index 6b36e729bf..404f8a43c3 100644 --- a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java +++ b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java @@ -546,6 +546,119 @@ public interface GatewayConfig { */ int getWebsocketMaxWaitBufferCount(); + /** + * Returns true if the Spark Connect (gRPC) listener is enabled, else false. + * Default is false. + * @since 3.0.0 + * @return true if the Spark Connect listener should be started + */ + boolean isSparkConnectEnabled(); + + /** + * The port the Spark Connect gRPC listener binds to. This is a dedicated + * socket, separate from the gateway's Jetty connectors, because gRPC needs + * HTTP/2 with ALPN. + * @since 3.0.0 + * @return the listener port + */ + int getSparkConnectPort(); + + /** + * The topology used when no other discriminator selects one. Spark Connect + * clients cannot put a path in an {@code sc://} URL, so Knox's usual + * {@code /gateway/{topology}/{service}} routing is unavailable and the + * topology must come from elsewhere. + * @since 3.0.0 + * @return the default topology name, or null if unset + */ + String getSparkConnectDefaultTopology(); + + /** + * Maximum inbound message size in bytes, applied to both legs. Spark's own + * default is 128 MB and grpc-java materializes whole messages, so this bounds + * per-message heap. + * @since 3.0.0 + * @return max message size in bytes + */ + int getSparkConnectMaxMessageSize(); + + /** + * The minimum interval the listener will tolerate between client keepalive + * pings before treating them as abusive, in milliseconds. + * @since 3.0.0 + * @return permitted keepalive interval in milliseconds + */ + long getSparkConnectPermitKeepAliveTime(); + + /** + * Whether clients may send keepalive pings with no active calls. Spark Connect + * clients ping idle channels, so this defaults to true. + * @since 3.0.0 + * @return true if keepalives without calls are permitted + */ + boolean isSparkConnectPermitKeepAliveWithoutCalls(); + + /** + * Maximum concurrent gRPC streams per client connection. + * @since 3.0.0 + * @return max concurrent calls per connection + */ + int getSparkConnectMaxConcurrentCallsPerConnection(); + + /** + * How long an unused backend channel is kept before being shut down, in + * milliseconds. + * @since 3.0.0 + * @return backend channel idle timeout in milliseconds + */ + long getSparkConnectChannelIdleTimeout(); + + /** + * How long to let in-flight RPCs finish when the gateway is shutting down, + * in milliseconds. Long-running {@code ExecutePlan} streams are severed once + * this elapses; clients recover through their own reattach logic. + * @since 3.0.0 + * @return drain timeout in milliseconds + */ + long getSparkConnectDrainTimeout(); + + /** + * The alias holding the pre-shared token Knox presents to the Spark Connect + * backend ({@code spark.connect.authenticate.token}). Besides authenticating + * Knox to Spark, this stops clients bypassing the gateway when they have + * network reachability to the backend port. + * @since 3.0.0 + * @return the alias name, or null if the backend requires no token + */ + String getSparkConnectBackendTokenAlias(); + + /** + * Governs the {@code AddArtifacts} RPC: {@code ALLOW}, {@code DENY}, or + * {@code ALLOW_LISTED_USERS}. User-supplied jars run with the Spark + * application's own storage credentials, so this is defense in depth rather + * than an authorization boundary. + * @since 3.0.0 + * @return the gating mode + */ + String getSparkConnectAddArtifactsMode(); + + /** + * Users permitted to call {@code AddArtifacts} when the gating mode is + * {@code ALLOW_LISTED_USERS}. + * @since 3.0.0 + * @return the permitted user names; empty if none configured + */ + List getSparkConnectAddArtifactsAllowedUsers(); + + /** + * The session-configuration key prefix reserved for Knox. Clients are denied + * {@code Set}/{@code Unset} on keys under this prefix so they cannot forge the + * identity Knox publishes into the session. + * @since 3.0.0 + * @return the reserved key prefix + */ + String getSparkConnectReservedConfigPrefix(); + boolean isMetricsEnabled(); boolean isJmxMetricsReportingEnabled(); diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/protocol/ProtocolListener.java b/gateway-spi/src/main/java/org/apache/knox/gateway/protocol/ProtocolListener.java new file mode 100644 index 0000000000..2ad3037d17 --- /dev/null +++ b/gateway-spi/src/main/java/org/apache/knox/gateway/protocol/ProtocolListener.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.protocol; + +import org.apache.knox.gateway.config.GatewayConfig; +import org.apache.knox.gateway.services.GatewayServices; + +/** + * A network listener for a protocol the servlet pipeline cannot carry, running + * alongside Jetty on its own port and managed by the {@code GatewayServer} + * lifecycle. + *

+ * Implementations are discovered with {@link java.util.ServiceLoader}, so the + * gateway server does not depend on them at compile time and their transport + * libraries stay off the gateway classpath unless the providing module is + * deployed. This is what lets the Spark Connect listener bring gRPC and a shaded + * Netty along without either reaching the servlet stack. + *

+ * The WebSocket handler solves the same class of problem, but a WebSocket + * upgrade can share Jetty's HTTP/1.1 connector; protocols that cannot (gRPC + * needs HTTP/2 with ALPN, which Knox's connectors do not offer) need their own + * socket, and therefore their own lifecycle hook. + * + * @since 3.0.0 + */ +public interface ProtocolListener { + + /** + * A short name for this listener, used in startup logging and error messages. + * + * @return the listener name; never null + */ + String getName(); + + /** + * Whether the deployment has switched this listener on. Called before + * {@link #start}; a listener that returns false is never started and must not + * bind a port or allocate threads. + * + * @param config the gateway configuration + * @return true if this listener should run + */ + boolean isEnabled(GatewayConfig config); + + /** + * Bind and begin serving. Called after the gateway services have started and + * topologies have been deployed, so implementations may resolve backends from + * the service registry during startup. + * + * @param config the gateway configuration + * @param services the started gateway services + * @throws Exception if the listener cannot start; the gateway will fail to start + */ + void start(GatewayConfig config, GatewayServices services) throws Exception; + + /** + * Stop accepting new work and drain in-flight requests before forcing + * termination. Called before Jetty stops. + *

+ * How long to drain for is the implementation's own configuration to read — + * what counts as a reasonable wait depends entirely on the protocol, and a + * listener carrying multi-hour streams has different needs from one serving + * short requests. + *

+ * Implementations must return rather than throw when the drain deadline passes + * with work still in flight; failing to stop cleanly should not prevent the + * rest of the gateway from shutting down. + * + * @throws Exception if the listener cannot be stopped + */ + void stop() throws Exception; + + /** + * The port this listener is bound to, for startup logging. + * + * @return the bound port, or -1 if the listener is not running + */ + int getPort(); +} diff --git a/knox-site/docs/spark-connect-support.md b/knox-site/docs/spark-connect-support.md new file mode 100644 index 0000000000..7445d2a0dc --- /dev/null +++ b/knox-site/docs/spark-connect-support.md @@ -0,0 +1,233 @@ + + +## Spark Connect Support ## + +### Introduction ### + +Spark Connect is the decoupled client/server protocol for Apache Spark (3.4+, and +the default architecture in Spark 4). Clients — PySpark, the Scala client, Go, +Rust, or JDBC via the Spark Connect driver — talk to a Spark Connect server over +gRPC, by default on port 15002. + +The OSS Spark Connect server has essentially no built-in authentication or +authorization; the project assumes a fronting proxy provides them. Knox can now +fill that role, adding: + +- **Authentication at the edge** — Knox-issued JWTs (KnoxToken bearer tokens), + validated before anything reaches Spark. +- **Identity assertion** — Spark Connect otherwise trusts a *client-asserted* + `user_context.user_id`. Knox overwrites it with the authenticated principal. +- **Coarse authorization** — the usual topology ACLs decide who may use Spark + Connect at all. +- **Auditing** — one record per RPC: principal, topology, method, session, + outcome and duration. + +Because gRPC requires HTTP/2 with ALPN, and because `grpc-status` and Spark's +structured error details travel in HTTP trailers, this cannot run on Knox's +existing Jetty connectors. Spark Connect is served by a **dedicated listener on +its own port**, started and stopped with the gateway. + +### Configuration ### + +Spark Connect support is disabled by default. Enable it in +`/conf/gateway-site.xml`: + + + gateway.sparkconnect.enabled + true + Enable the Spark Connect (gRPC) listener. + + + gateway.sparkconnect.default.topology + analytics + Topology used when a client does not select one. + + +The listener presents the gateway's own TLS identity — the same keystore and +alias Jetty uses — whenever `ssl.enabled` is true, so there is no second +certificate to manage. + +#### All properties #### + +| Property | Default | Meaning | +|---|---|---| +| `gateway.sparkconnect.enabled` | `false` | Master switch; the listener is not started when false. | +| `gateway.sparkconnect.port` | `15002` | Port for the gRPC listener. | +| `gateway.sparkconnect.default.topology` | *(none)* | Topology used when the client sends no `knox-topology`. | +| `gateway.sparkconnect.max.message.size` | `134217728` | Maximum inbound message size in bytes, both legs. Matches Spark's 128 MB default. | +| `gateway.sparkconnect.max.concurrent.calls.per.connection` | `1000` | Maximum concurrent gRPC streams per client connection. | +| `gateway.sparkconnect.permit.keepalive.time` | `10000` | Minimum tolerated interval between client keepalive pings, in ms. | +| `gateway.sparkconnect.permit.keepalive.without.calls` | `true` | Whether clients may ping an idle channel. Spark Connect clients do. | +| `gateway.sparkconnect.channel.idle.timeout` | `1800000` | Idle time before an unused backend channel is shut down, in ms. | +| `gateway.sparkconnect.drain.timeout` | `30000` | How long in-flight RPCs get to finish at shutdown, in ms. | +| `gateway.sparkconnect.backend.token.alias` | *(none)* | Alias holding the backend's pre-shared token (see below). | +| `gateway.sparkconnect.add.artifacts.mode` | `ALLOW` | `ALLOW`, `DENY`, or `ALLOW_LISTED_USERS` for the `AddArtifacts` RPC. | +| `gateway.sparkconnect.add.artifacts.allowed.users` | *(none)* | Comma-separated users permitted when the mode is `ALLOW_LISTED_USERS`. | +| `gateway.sparkconnect.reserved.config.prefix` | `knox.` | Session-configuration key prefix clients may not `Set` or `Unset`. | + +### Topology configuration ### + +Declare the backend like any other service. The registry treats the URL as an +opaque string, so `grpc://` and `grpcs://` need no special handling: + + + SPARKCONNECT + grpc://spark-connect-host:15002 + + +Use `grpcs://` for a TLS backend; Knox verifies it against the HTTP client +truststore, falling back to the gateway keystore. + +Authorization uses the ordinary `AclsAuthz` provider syntax, keyed on the +`SPARKCONNECT` role: + + + authorization + AclsAuthz + true + + SPARKCONNECT.acl + *;analysts;* + + + +Group membership comes from the `knox.groups` claim in the token, so configure +`knoxtoken` to include groups if you intend to write group ACLs. + +### Connecting ### + +Clients need no plugins or code changes. First acquire a token — over HTTPS, +authenticating however that topology is configured: + + curl --negotiate -u : https://knox:8443/gateway/tokens/knoxtoken/api/v1/token + +Then put it in the connection string: + + sc://knox-host:15002/;use_ssl=true;token=;knox-topology=analytics + +Two details make this work. The `token=` parameter is sent as a standard +`Authorization: Bearer` header and forces TLS on. Any parameter the client does +not recognise — `knox-topology` here — is sent as gRPC metadata on every request, +which is how a topology gets selected despite gRPC forbidding a path component in +the connection URL. If you set `gateway.sparkconnect.default.topology`, the +`knox-topology` parameter can be omitted. + +### Kerberos environments ### + +Neither gRPC nor the vanilla Spark Connect clients support SPNEGO, and gRPC has no +challenge-response step for it to hook into. Kerberos therefore authenticates +*token acquisition* rather than each RPC: a `kinit`'d user or a keytab'd service +fetches a token from a `knoxtoken` topology using HadoopAuth/SPNEGO, and the JWT +carries the data path. + +This is the same trade Kerberized Hadoop already makes — nobody SPNEGOs every +HDFS block read. It is also better operationally for long-running jobs: an +administrator can revoke one token without touching the principal. + +Tokens are validated when an RPC starts, not continuously. A long-running +`ExecutePlan` is not killed when its token expires; the next RPC fails with +`UNAUTHENTICATED`. + +### Security considerations ### + +**Knox's authorization here is coarse by design.** It answers only "may this user +use Spark Connect in this topology". Database, table, column and row-level policy +must be enforced inside the Spark Connect server — for example by a Ranger-backed +plan-level plugin keyed off the identity Knox asserts. + +**Asserting `user_id` is not storage-level enforcement.** On the server, +`user_id` keys the session cache — so two users can never share a session — and +appears in logs and events. It is not propagated into Spark's +`CurrentUserContext`, so `current_user()` in SQL reports the Spark application's +own user unless a server-side component bridges it. A shared Spark Connect server +is one application running as one principal, and its storage credentials are that +principal's. + +**User-supplied code bypasses plan-level policy.** Uploaded jars and inline +Python/Scala UDFs run inside that JVM with that principal's credentials, so they +can read data directly. `gateway.sparkconnect.add.artifacts.mode` shrinks the +attack surface but does not close it, because inline UDFs reach the same +capability. This is a property of plan-level enforcement generally, not something +the gateway introduces. Deployments needing a hard boundary want per-user or +per-tenant backend instances. + +**Restrict the backend.** Knox in front of an openly reachable Spark Connect port +secures nothing. Firewall the backend so only Knox can reach it, and set Spark 4's +pre-shared token (`spark.connect.authenticate.token`), storing it as a Knox alias +and naming it in `gateway.sparkconnect.backend.token.alias`. Knox then presents it +on the backend leg — and, because it strips the client's own credential there, a +client cannot bypass the gateway even with network reachability. + +### Is this a generic gRPC gateway? ### + +No — and deliberately so. Knox proxies exactly one gRPC service, +`spark.connect.SparkConnectService`. There is no configuration property that +points this listener at an arbitrary gRPC backend, and a call to any other proto +service is answered `UNIMPLEMENTED`. + +It is worth being open about what sits behind that, because anyone reading the +source will notice it: most of this feature is not Spark-specific. The listener, +TLS from the gateway identity, bearer authentication, the coarse ACL check, +topology routing, backend channel caching, auditing, graceful drain and the relay +itself are all protocol-agnostic — the relay in particular treats messages as +opaque and collapses all four RPC shapes into one code path. Only identity +assertion and the per-RPC gating switches need to understand Spark Connect's +messages. + +The implementation keeps that split explicit: the gateway listener is an abstract +class whose protocol-aware parts are abstract methods, and the Spark Connect +listener is its only concrete subclass. That is a bet that someone will +eventually want to front a second gRPC service, and it costs little to leave the +seam in place rather than discover it later. It is **not** a commitment, and it +is not a supported extension point — the abstraction exists for the benefit of a +future change to Knox itself, not as an API to build against. + +Promoting it to a real capability would take more than flipping a switch, which +is the main reason it has not been: + +- **A service-to-role mapping.** gRPC paths are `/pkg.Service/Method`, so a + topology would need to declare which proto services map to which backend roles, + default-denying anything unmapped. +- **An honest security posture.** Byte-level proxying gives authentication, + coarse authorization, TLS and method-level audit — but no message-body + controls at all. It cannot assert identity, cannot protect a reserved config + key, and cannot record a session id. That is a materially weaker offering than + what Spark Connect gets here, and it competes much less favourably with simply + putting Envoy or nginx in front of the service. +- **A community discussion**, its own configuration surface, and its own + documentation. + +One narrow piece of byte-level relay *is* active, and it is scoped accordingly: +an RPC that belongs to `spark.connect.SparkConnectService` but has no generated +handler in this build — a method added in a newer Spark line — is forwarded as +opaque bytes rather than rejected. Such a call is still authenticated, +authorized, routed and audited; it simply does not get identity assertion, +because that requires parsing the message. This exists so version skew degrades +gracefully, not as a general passthrough. + +### Limitations ### + +- One `SPARKCONNECT` URL per topology. Spark Connect sessions are server-side + state keyed by `(user_id, session_id)` and `ReattachExecute` must reach the + backend owning the operation, so round-robin over several backends would be + wrong; session-affine routing across multiple backends is not yet implemented. +- The asserted principal is the token's subject. Identity-assertion provider + mapping rules are not applied on this path. +- A gateway restart severs active streams. Clients recover through their own + `ReattachExecute` retry logic, and shutdown drains for + `gateway.sparkconnect.drain.timeout` first. diff --git a/knox-site/mkdocs.yml b/knox-site/mkdocs.yml index f147f0937e..a8bf1d78e0 100644 --- a/knox-site/mkdocs.yml +++ b/knox-site/mkdocs.yml @@ -124,6 +124,7 @@ nav: - Admin API: admin_api.md - Monitoring API: dev-guide/knox_monitoring_api.md - Advanced Topics: + - Spark Connect Support: spark-connect-support.md - SSE Support: sse-support.md - WebSocket Support: websocket-support.md - X-Forwarded Headers: x-forwarded-headers.md diff --git a/pom.xml b/pom.xml index 369e336faa..2730bfc40d 100644 --- a/pom.xml +++ b/pom.xml @@ -125,6 +125,7 @@ gateway-service-storm gateway-service-remoteconfig gateway-service-restcatalog + gateway-service-sparkconnect gateway-service-definitions gateway-shell gateway-shell-launcher @@ -208,6 +209,9 @@ 7.7.0 1.15.1 1.0.0 + + 1.80.0 4.0.5 2.10.1 1.9.0 @@ -261,6 +265,7 @@ v22.20.0 4.12.0 5.2.2 + 1.7.1 6.5.3 42.7.11 8.0.28 @@ -268,6 +273,7 @@ 3.6.0 23.26.0.0.0 3.25.8 + 0.6.1 2.0.9 0.0.11.1 0.12.4 @@ -1289,6 +1295,11 @@ gateway-service-restcatalog ${project.version} + + org.apache.knox + gateway-service-sparkconnect + ${project.version} + org.apache.knox gateway-server @@ -1796,6 +1807,17 @@ ${protobuf.version} + + + io.grpc + grpc-bom + ${grpc.version} + pom + import + + org.apache.hadoop hadoop-auth From 9a8fec9885c2e4835bd794c16deb28da4e12cd10 Mon Sep 17 00:00:00 2001 From: Alex Cruise Date: Thu, 30 Jul 2026 15:30:56 -0700 Subject: [PATCH 2/4] KNOX-3402 [wip]: Doc update, IT, hot-reload boundaries (and fix the bugs caught by the IT) --- .github/workflows/build/Dockerfile | 2 + .../topologies/sparkconnect-restricted.xml | 48 +++ .../build/conf/topologies/sparkconnect.xml | 39 ++ .github/workflows/build/gateway-site.xml | 13 + .github/workflows/compose/docker-compose.yml | 49 +++ .../compose/sparkconnect/mock_server.py | 113 ++++++ .github/workflows/tests/requirements.txt | 4 +- .github/workflows/tests/test_spark_connect.py | 256 +++++++++++++ .../apache/knox/gateway/GatewayMessages.java | 11 + .../apache/knox/gateway/GatewayServer.java | 80 ++++ .../ApplicationDeploymentContributor.java | 4 + ...erviceDefinitionDeploymentContributor.java | 9 + .../DefaultServiceDefinitionRegistry.java | 7 + .../ProtocolListenerEnablementTest.java | 126 +++++++ ...ceDefinitionDeploymentContributorTest.java | 42 +++ gateway-service-sparkconnect/pom.xml | 6 + .../grpc/AuthenticationInterceptor.java | 7 + .../gateway/grpc/GrpcGatewayListener.java | 1 + .../gateway/grpc/GrpcGatewayMessages.java | 9 + .../knox/gateway/grpc/TokenAuthenticator.java | 14 +- .../sparkconnect/SparkConnectListener.java | 98 ++++- .../sparkconnect/SparkConnectPolicy.java | 91 +++++ .../src/main/proto/spark/connect/README.md | 98 +++++ .../grpc/AuthenticationInterceptorTest.java | 218 +++++++++++ .../AuthorizationInterceptorReloadTest.java | 216 +++++++++++ .../TopologySelectionAuthorizationTest.java | 260 +++++++++++++ .../SparkConnectListenerConfigReloadTest.java | 170 +++++++++ .../knox/gateway/GatewayTestConfig.java | 7 +- .../gateway/protocol/ProtocolListener.java | 17 + knox-site/docs/spark-connect-support.md | 347 ++++++++++++++++-- 30 files changed, 2325 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/build/conf/topologies/sparkconnect-restricted.xml create mode 100644 .github/workflows/build/conf/topologies/sparkconnect.xml create mode 100644 .github/workflows/compose/sparkconnect/mock_server.py create mode 100644 .github/workflows/tests/test_spark_connect.py create mode 100644 gateway-server/src/test/java/org/apache/knox/gateway/ProtocolListenerEnablementTest.java create mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectPolicy.java create mode 100644 gateway-service-sparkconnect/src/main/proto/spark/connect/README.md create mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AuthenticationInterceptorTest.java create mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AuthorizationInterceptorReloadTest.java create mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/TopologySelectionAuthorizationTest.java create mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectListenerConfigReloadTest.java diff --git a/.github/workflows/build/Dockerfile b/.github/workflows/build/Dockerfile index 1781ead94f..8e2c8c0b46 100644 --- a/.github/workflows/build/Dockerfile +++ b/.github/workflows/build/Dockerfile @@ -43,6 +43,8 @@ ADD .github/workflows/build/conf/topologies/health.xml /knox-runtime/conf/topolo ADD .github/workflows/build/conf/topologies/knoxldap.xml /knox-runtime/conf/topologies/knoxldap.xml ADD .github/workflows/build/conf/topologies/remoteauth.xml /knox-runtime/conf/topologies/remoteauth.xml ADD .github/workflows/build/conf/topologies/k8sauth.xml /knox-runtime/conf/topologies/k8sauth.xml +ADD .github/workflows/build/conf/topologies/sparkconnect.xml /knox-runtime/conf/topologies/sparkconnect.xml +ADD .github/workflows/build/conf/topologies/sparkconnect-restricted.xml /knox-runtime/conf/topologies/sparkconnect-restricted.xml RUN chown -R gateway /knox-runtime/ diff --git a/.github/workflows/build/conf/topologies/sparkconnect-restricted.xml b/.github/workflows/build/conf/topologies/sparkconnect-restricted.xml new file mode 100644 index 0000000000..779527d22a --- /dev/null +++ b/.github/workflows/build/conf/topologies/sparkconnect-restricted.xml @@ -0,0 +1,48 @@ + + + + + + federation + JWTProvider + true + + knox.token.use.cookie + false + + + + authorization + AclsAuthz + true + + SPARKCONNECT.acl + nobody;*;* + + + + + SPARKCONNECT + grpc://sparkconnect-mock:15002 + + diff --git a/.github/workflows/build/conf/topologies/sparkconnect.xml b/.github/workflows/build/conf/topologies/sparkconnect.xml new file mode 100644 index 0000000000..c8d5710930 --- /dev/null +++ b/.github/workflows/build/conf/topologies/sparkconnect.xml @@ -0,0 +1,39 @@ + + + + + + federation + JWTProvider + true + + knox.token.use.cookie + false + + + + + SPARKCONNECT + grpc://sparkconnect-mock:15002 + + diff --git a/.github/workflows/build/gateway-site.xml b/.github/workflows/build/gateway-site.xml index 36e884a3ef..7da2274ec4 100644 --- a/.github/workflows/build/gateway-site.xml +++ b/.github/workflows/build/gateway-site.xml @@ -212,4 +212,17 @@ limitations under the License. member + + + gateway.sparkconnect.enabled + true + + + + gateway.sparkconnect.add.artifacts.mode + DENY + + diff --git a/.github/workflows/compose/docker-compose.yml b/.github/workflows/compose/docker-compose.yml index 727bca2633..fa6ef8e9e8 100644 --- a/.github/workflows/compose/docker-compose.yml +++ b/.github/workflows/compose/docker-compose.yml @@ -85,6 +85,47 @@ services: depends_on: - k3s + # One-shot: generates Python protobuf/gRPC stubs from the same vendored + # spark/connect/*.proto files the gateway compiles against, into a volume + # shared by the mock backend and the tests. Generating rather than depending on + # pyspark keeps the images small and means a proto refresh that broke the wire + # contract would break these tests too. + sparkconnect-protos: + image: python:3.10-slim + entrypoint: + - /bin/sh + - -c + command: + - | + set -e + pip install --no-cache-dir --quiet grpcio-tools==1.60.0 + python -m grpc_tools.protoc -I/protos \ + --python_out=/out --grpc_python_out=/out \ + /protos/spark/connect/*.proto + # Generated modules import each other as spark.connect.*, so the output + # has to be an importable package. + touch /out/spark/__init__.py /out/spark/connect/__init__.py + echo 'spark connect stubs generated' + volumes: + - ../../../gateway-service-sparkconnect/src/main/proto:/protos:ro + - sparkconnect-protos:/out + + # Stands in for a Spark Connect server on a private network. Plaintext, which + # is what Knox's grpc:// backend scheme describes. + sparkconnect-mock: + image: python:3.10-slim + environment: + - PYTHONPATH=/stubs + volumes: + - ./sparkconnect:/mock:ro + - sparkconnect-protos:/stubs:ro + command: > + sh -c "pip install --no-cache-dir --quiet grpcio==1.60.0 protobuf==4.25.8 + && python /mock/mock_server.py" + depends_on: + sparkconnect-protos: + condition: service_completed_successfully + knox: image: apache/knox-dev:${IMAGE_TAG:-master} command: /gateway.sh @@ -101,14 +142,21 @@ services: condition: service_started k8s-bootstrap: condition: service_completed_successfully + sparkconnect-mock: + condition: service_started tests: image: python:3.10-slim working_dir: /tests volumes: - ../tests:/tests + - sparkconnect-protos:/stubs:ro environment: - KNOX_GATEWAY_URL=https://knox:8443/ + - KNOX_SPARKCONNECT_HOST=knox + - KNOX_SPARKCONNECT_PORT=15002 + # Generated Spark Connect stubs, shared with the mock backend. + - PYTHONPATH=/stubs command: > bash -c "pip install -r requirements.txt && pylint *.py @@ -120,3 +168,4 @@ services: volumes: k3s-output: + sparkconnect-protos: diff --git a/.github/workflows/compose/sparkconnect/mock_server.py b/.github/workflows/compose/sparkconnect/mock_server.py new file mode 100644 index 0000000000..20b0a01e86 --- /dev/null +++ b/.github/workflows/compose/sparkconnect/mock_server.py @@ -0,0 +1,113 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A stand-in Spark Connect server for the Knox integration tests. + +Running real Spark would add a gigabyte of image and a minute of startup to +test a gateway, and the gateway does not care what is behind it -- only that it +speaks `spark.connect.SparkConnectService`. So this implements just enough of +that service to make the gateway's behavior observable. + +The important trick is that the RPCs echo back what the *backend* received, +rather than returning canned data. Knox overwrites `user_context.user_id` with +the authenticated principal on its way through, and that rewrite is invisible +from the client side -- the client only knows what it sent. By reflecting the +observed identity into the response, an assertion about what Spark would have +seen becomes an ordinary assertion in the test. + +The stubs are generated at container start from the same vendored +`spark/connect/*.proto` files the gateway compiles against, so a proto refresh +that broke the wire contract would break this too. +""" + +import logging +import os +from concurrent import futures + +import grpc + +from spark.connect import base_pb2 +from spark.connect import base_pb2_grpc + +LOG = logging.getLogger("mock-spark-connect") + +# Enough responses to prove a server stream is relayed message by message rather +# than collapsed or truncated. +EXECUTE_PLAN_RESPONSE_COUNT = 5 + + +def _observed_user(request): + """The user_id the backend actually received, i.e. after Knox's rewrite.""" + return request.user_context.user_id + + +class MockSparkConnectService(base_pb2_grpc.SparkConnectServiceServicer): + """Implements the handful of RPCs the integration tests exercise.""" + + def AnalyzePlan(self, request, context): # noqa: N802 - gRPC naming + observed = _observed_user(request) + LOG.info("AnalyzePlan session=%s user_id=%s", request.session_id, observed) + # explain_string is a free-form string field, so it can carry the observed + # identity back to the test without inventing a side channel. + return base_pb2.AnalyzePlanResponse( + session_id=request.session_id, + explain=base_pb2.AnalyzePlanResponse.Explain(explain_string=observed), + ) + + def ExecutePlan(self, request, context): # noqa: N802 - gRPC naming + observed = _observed_user(request) + LOG.info("ExecutePlan session=%s user_id=%s", request.session_id, observed) + for index in range(EXECUTE_PLAN_RESPONSE_COUNT): + yield base_pb2.ExecutePlanResponse( + session_id=request.session_id, + operation_id=observed, + response_id=f"response-{index}", + ) + + def Config(self, request, context): # noqa: N802 - gRPC naming + observed = _observed_user(request) + LOG.info("Config session=%s user_id=%s", request.session_id, observed) + return base_pb2.ConfigResponse( + session_id=request.session_id, + # Echo the observed identity as a config value so the Config path can + # be asserted the same way as AnalyzePlan. + pairs=[base_pb2.KeyValue(key="knox.observed.user", value=observed)], + ) + + def AddArtifacts(self, request_iterator, context): # noqa: N802 - gRPC naming + observed = "" + count = 0 + for request in request_iterator: + observed = _observed_user(request) + count += 1 + LOG.info("AddArtifacts messages=%d user_id=%s", count, observed) + return base_pb2.AddArtifactsResponse() + + +def serve(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + port = os.environ.get("MOCK_PORT", "15002") + server = grpc.server(futures.ThreadPoolExecutor(max_workers=8)) + base_pb2_grpc.add_SparkConnectServiceServicer_to_server(MockSparkConnectService(), server) + # Plaintext: this stands in for a Spark Connect server on a private network, + # which is exactly the posture Knox's grpc:// backend scheme describes. + server.add_insecure_port(f"[::]:{port}") + server.start() + LOG.info("Mock Spark Connect server listening on %s", port) + server.wait_for_termination() + + +if __name__ == "__main__": + serve() diff --git a/.github/workflows/tests/requirements.txt b/.github/workflows/tests/requirements.txt index 736823a6a2..feee7beece 100644 --- a/.github/workflows/tests/requirements.txt +++ b/.github/workflows/tests/requirements.txt @@ -1,4 +1,6 @@ requests==2.33.0 pytest==9.0.3 pylint==4.0.5 -ldap3==2.9.1 \ No newline at end of file +ldap3==2.9.1 +grpcio==1.60.0 +protobuf==4.25.8 diff --git a/.github/workflows/tests/test_spark_connect.py b/.github/workflows/tests/test_spark_connect.py new file mode 100644 index 0000000000..6cdc3b5fd9 --- /dev/null +++ b/.github/workflows/tests/test_spark_connect.py @@ -0,0 +1,256 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Integration tests for the Spark Connect (gRPC) listener. + +These drive a real gRPC client against a running gateway, which is the only way +to cover the parts unit tests have to mock: the listener being discovered and +started, TLS from the gateway identity, real token validation, topology +deployment, and the interceptor chain in its real order. + +The backend is a stand-in Spark Connect server that echoes back the +`user_context.user_id` it received. That is what makes identity assertion +observable -- the client cannot otherwise see what Knox rewrote on the way +through. +""" + +# Protobuf message classes are created dynamically from the descriptor pool when +# the generated modules are imported, so static analysis cannot see them. +# pylint: disable=no-member + +import os +import ssl +import unittest + +import grpc +import requests +import urllib3 + +from spark.connect import base_pb2 +from spark.connect import base_pb2_grpc + +# The dev environment uses self-signed certificates throughout. +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +TOPOLOGY_METADATA_KEY = "knox-topology" +OPEN_TOPOLOGY = "sparkconnect" +RESTRICTED_TOPOLOGY = "sparkconnect-restricted" +# Present in the demo LDAP the compose environment starts. +KNOX_USER = "guest" +KNOX_PASSWORD = "guest-password" + + +def _gateway_url(): + return os.environ.get("KNOX_GATEWAY_URL", "https://localhost:8443/") + + +def _sparkconnect_endpoint(): + host = os.environ.get("KNOX_SPARKCONNECT_HOST", "localhost") + port = os.environ.get("KNOX_SPARKCONNECT_PORT", "15002") + return host, int(port) + + +def _acquire_token(): + """Gets a Knox JWT the way a user would, over HTTPS before any gRPC call. + + The knoxldap topology fronts KNOXTOKEN with a basic-auth Shiro realm over the + demo LDAP, which is the closest thing this environment has to the + authenticate-once-then-carry-a-token flow the gRPC listener expects. + """ + url = f"{_gateway_url()}gateway/knoxldap/knoxtoken/api/v1/token" + response = requests.get(url, auth=(KNOX_USER, KNOX_PASSWORD), verify=False, timeout=30) + response.raise_for_status() + return response.json()["access_token"] + + +class SparkConnectTestBase(unittest.TestCase): + """Shared channel plumbing for the Spark Connect listener tests.""" + + token = None + server_certificate = None + + @classmethod + def setUpClass(cls): + host, port = _sparkconnect_endpoint() + # The listener presents the gateway identity, which is self-signed here. + # Trust exactly that certificate rather than disabling verification, so + # the test still proves TLS is actually working. + cls.server_certificate = ssl.get_server_certificate((host, port)).encode("utf-8") + cls.token = _acquire_token() + + def _channel(self, token=None, topology=None): + host, port = _sparkconnect_endpoint() + credentials = grpc.ssl_channel_credentials(root_certificates=self.server_certificate) + # The gateway certificate is issued for its own hostname, which need not + # match the compose service name. + options = (("grpc.ssl_target_name_override", "localhost"),) + channel = grpc.secure_channel(f"{host}:{port}", credentials, options) + metadata = [] + if token is not None: + metadata.append(("authorization", f"Bearer {token}")) + if topology is not None: + metadata.append((TOPOLOGY_METADATA_KEY, topology)) + return channel, tuple(metadata) + + def _analyze(self, token=None, topology=None, session_id="itest-session", claimed_user="root"): + """Sends AnalyzePlan and returns the response, or raises RpcError.""" + channel, metadata = self._channel(token=token, topology=topology) + with channel: + stub = base_pb2_grpc.SparkConnectServiceStub(channel) + request = base_pb2.AnalyzePlanRequest(session_id=session_id) + # Claim to be someone else; Knox must overwrite this. + request.user_context.user_id = claimed_user + return stub.AnalyzePlan(request, metadata=metadata, timeout=30) + + def assert_rpc_code(self, expected, callable_obj): + """Asserts a call fails with a specific gRPC status code.""" + with self.assertRaises(grpc.RpcError) as raised: + callable_obj() + self.assertEqual(expected, raised.exception.code(), + f"expected {expected}, got {raised.exception.code()}: " + f"{raised.exception.details()}") + + +class TestSparkConnectAuthentication(SparkConnectTestBase): + """Nothing reaches the backend without a valid Knox token.""" + + def test_call_without_a_token_is_rejected(self): + """An unauthenticated call must never reach the backend.""" + self.assert_rpc_code(grpc.StatusCode.UNAUTHENTICATED, + lambda: self._analyze(token=None, topology=OPEN_TOPOLOGY)) + + def test_call_with_a_malformed_token_is_rejected(self): + """Something that is not a JWT at all is refused cleanly.""" + self.assert_rpc_code(grpc.StatusCode.UNAUTHENTICATED, + lambda: self._analyze(token="not-a-jwt", topology=OPEN_TOPOLOGY)) + + def test_call_with_a_well_formed_but_unsigned_token_is_rejected(self): + """A forged JWT is refused as UNAUTHENTICATED, not UNKNOWN.""" + # Structurally a JWT, but not one this gateway issued. + forged = ("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" + ".eyJzdWIiOiJndWVzdCIsImlzcyI6IktOT1hTU08ifQ" + ".c2lnbmF0dXJl") + self.assert_rpc_code(grpc.StatusCode.UNAUTHENTICATED, + lambda: self._analyze(token=forged, topology=OPEN_TOPOLOGY)) + + +class TestSparkConnectRouting(SparkConnectTestBase): + """Topology selection comes from client metadata and must resolve.""" + + def test_call_without_a_topology_is_rejected(self): + """With no default topology configured there is nowhere to route.""" + # No default topology is configured, so there is nowhere to route. + self.assert_rpc_code(grpc.StatusCode.UNIMPLEMENTED, + lambda: self._analyze(token=self.token, topology=None)) + + def test_call_to_an_unknown_topology_is_rejected(self): + """A topology that declares no SPARKCONNECT service is unroutable.""" + self.assert_rpc_code(grpc.StatusCode.UNAVAILABLE, + lambda: self._analyze(token=self.token, topology="no-such-topology")) + + +class TestSparkConnectAuthorization(SparkConnectTestBase): + """Naming a topology is not the same as being allowed to use it.""" + + def test_topology_acl_denies_an_authenticated_user(self): + """A valid token does not by itself grant access to a topology.""" + # Same valid token, same backend -- refused by that topology's ACL. + self.assert_rpc_code(grpc.StatusCode.PERMISSION_DENIED, + lambda: self._analyze(token=self.token, topology=RESTRICTED_TOPOLOGY)) + + def test_permitted_topology_is_reachable_with_the_same_token(self): + """The same token reaches a topology whose ACLs permit the user.""" + response = self._analyze(token=self.token, topology=OPEN_TOPOLOGY) + self.assertEqual("itest-session", response.session_id) + + +class TestSparkConnectIdentityAssertion(SparkConnectTestBase): + """The client's claimed identity is replaced with the authenticated one.""" + + def test_backend_sees_the_authenticated_principal_not_the_claim(self): + """Knox overwrites the client-supplied user_id before the backend sees it.""" + response = self._analyze(token=self.token, topology=OPEN_TOPOLOGY, claimed_user="root") + # The mock echoes back the user_id it received. + self.assertEqual(KNOX_USER, response.explain.explain_string) + + def test_claim_is_overwritten_even_when_left_empty(self): + """An absent claim is filled in rather than passed through empty.""" + response = self._analyze(token=self.token, topology=OPEN_TOPOLOGY, claimed_user="") + self.assertEqual(KNOX_USER, response.explain.explain_string) + + def test_identity_is_asserted_on_the_config_rpc_too(self): + """Identity assertion applies to every RPC, not just AnalyzePlan.""" + channel, metadata = self._channel(token=self.token, topology=OPEN_TOPOLOGY) + with channel: + stub = base_pb2_grpc.SparkConnectServiceStub(channel) + request = base_pb2.ConfigRequest(session_id="itest-config") + request.user_context.user_id = "root" + request.operation.get_all.SetInParent() + response = stub.Config(request, metadata=metadata, timeout=30) + self.assertEqual(KNOX_USER, response.pairs[0].value) + + +class TestSparkConnectStreaming(SparkConnectTestBase): + """Server streaming is relayed message by message.""" + + def test_execute_plan_relays_every_response(self): + """A server stream arrives complete and in order.""" + channel, metadata = self._channel(token=self.token, topology=OPEN_TOPOLOGY) + with channel: + stub = base_pb2_grpc.SparkConnectServiceStub(channel) + request = base_pb2.ExecutePlanRequest(session_id="itest-stream") + request.user_context.user_id = "root" + responses = list(stub.ExecutePlan(request, metadata=metadata, timeout=30)) + self.assertEqual(5, len(responses)) + self.assertEqual("response-0", responses[0].response_id) + self.assertEqual("response-4", responses[-1].response_id) + # The backend reflects the asserted identity into operation_id. + self.assertEqual(KNOX_USER, responses[0].operation_id) + + +class TestSparkConnectMessageGating(SparkConnectTestBase): + """Per-RPC gating is enforced at the gateway, before the backend.""" + + def test_add_artifacts_is_denied_when_configured_to_deny(self): + """Artifact upload gating is enforced at the gateway.""" + def upload(): + channel, metadata = self._channel(token=self.token, topology=OPEN_TOPOLOGY) + with channel: + stub = base_pb2_grpc.SparkConnectServiceStub(channel) + request = base_pb2.AddArtifactsRequest(session_id="itest-artifacts") + request.user_context.user_id = "root" + return stub.AddArtifacts(iter([request]), metadata=metadata, timeout=30) + + # gateway.sparkconnect.add.artifacts.mode is DENY in gateway-site.xml. + self.assert_rpc_code(grpc.StatusCode.PERMISSION_DENIED, upload) + + def test_reserved_config_key_cannot_be_set_by_a_client(self): + """Clients cannot write the session keys Knox reserves for itself.""" + def set_reserved(): + channel, metadata = self._channel(token=self.token, topology=OPEN_TOPOLOGY) + with channel: + stub = base_pb2_grpc.SparkConnectServiceStub(channel) + request = base_pb2.ConfigRequest(session_id="itest-reserved") + request.user_context.user_id = "root" + pair = request.operation.set.pairs.add() + pair.key = "knox.principal" + pair.value = "root" + return stub.Config(request, metadata=metadata, timeout=30) + + self.assert_rpc_code(grpc.StatusCode.PERMISSION_DENIED, set_reserved) + + +if __name__ == "__main__": + unittest.main() diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java index 03bffeafca..d9ef95b40c 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayMessages.java @@ -66,6 +66,17 @@ public interface GatewayMessages { @Message( level = MessageLevel.WARN, text = "Failed to stop the {0} protocol listener: {1}" ) void failedToStopProtocolListener( String name, @StackTrace( level = MessageLevel.WARN ) Exception e ); + @Message( level = MessageLevel.WARN, text = "Failed to reload the {0} protocol listener after a topology change: {1}" ) + void failedToReloadProtocolListener( String name, @StackTrace( level = MessageLevel.WARN ) Exception e ); + + @Message( level = MessageLevel.WARN, + text = "The {0} protocol listener is enabled in the refreshed configuration but cannot be started without a gateway restart." ) + void protocolListenerCannotBeStarted( String name ); + + @Message( level = MessageLevel.WARN, + text = "The {0} protocol listener is disabled in the refreshed configuration but cannot be stopped without a gateway restart." ) + void protocolListenerCannotBeStopped( String name ); + @Message( level = MessageLevel.INFO, text = "Loading configuration resource {0}" ) void loadingConfigurationResource( String res ); diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java index 14cb7b9196..487579683d 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java @@ -174,6 +174,12 @@ public class GatewayServer { */ private final List protocolListeners = new ArrayList<>(); + /** + * Listeners present on the classpath but switched off at startup. Held only so + * that switching one on later can be reported as needing a restart. + */ + private final List inactiveProtocolListeners = new ArrayList<>(); + private final Set inactiveTopologies = new HashSet<>(); public static void main( String[] args ) { @@ -821,22 +827,91 @@ void createJetty() throws IOException, CertificateException, NoSuchAlgorithmExce private void startProtocolListeners() throws Exception { for (ProtocolListener listener : ServiceLoader.load(ProtocolListener.class)) { if (!listener.isEnabled(config)) { + // Kept so that switching it on later can be reported rather than ignored. + inactiveProtocolListeners.add(listener); continue; } try { listener.start(config, services); protocolListeners.add(listener); + // A listener that tracks gateway configuration opts in by implementing + // GatewayConfigChangeListener; registering after a successful start keeps + // a failed listener from receiving refreshes. + if (listener instanceof GatewayConfigChangeListener) { + registerConfigChangeListener((GatewayConfigChangeListener) listener); + } log.startedProtocolListener(listener.getName(), convertPortToString(listener.getPort())); } catch (Exception e) { log.failedToStartProtocolListener(listener.getName(), e); throw e; } } + if (!protocolListeners.isEmpty() || !inactiveProtocolListeners.isEmpty()) { + registerConfigChangeListener(enablementWatcher); + } + } + + /** + * Reports an attempt to switch a protocol listener on or off in a running + * gateway. + *

+ * Whether a listener runs is decided once, at startup: an enabled one binds its + * socket, a disabled one is never started. Neither can change without a + * restart. Enablement is the gateway's decision rather than the listener's, so + * it is watched here instead of in each listener — and reported rather than + * silently ignored, because an operator who edits the property and sees nothing + * in the log has no way to tell the setting was not applied. + */ + private final GatewayConfigChangeListener enablementWatcher = + refreshed -> warnAboutEnablementChanges(refreshed, protocolListeners, inactiveProtocolListeners); + + /** + * Logs a warning for each listener whose enablement changed, in either + * direction. + * + * @param refreshed the reloaded configuration + * @param running listeners started at gateway startup + * @param inactive listeners on the classpath that were switched off at startup + * @return the names of the listeners reported, for testing + */ + static List warnAboutEnablementChanges(GatewayConfig refreshed, + List running, + List inactive) { + final List reported = new ArrayList<>(); + for (ProtocolListener listener : running) { + if (!listener.isEnabled(refreshed)) { + log.protocolListenerCannotBeStopped(listener.getName()); + reported.add(listener.getName()); + } + } + for (ProtocolListener listener : inactive) { + if (listener.isEnabled(refreshed)) { + log.protocolListenerCannotBeStarted(listener.getName()); + reported.add(listener.getName()); + } + } + return reported; + } + + private void notifyProtocolListeners() { + for (ProtocolListener listener : protocolListeners) { + try { + listener.reload(); + } catch (Exception e) { + // A listener that cannot refresh must not block the redeployment of the + // topologies the rest of the gateway serves. + log.failedToReloadProtocolListener(listener.getName(), e); + } + } } private void stopProtocolListeners() { + unregisterConfigChangeListener(enablementWatcher); for (ProtocolListener listener : protocolListeners) { try { + if (listener instanceof GatewayConfigChangeListener) { + unregisterConfigChangeListener((GatewayConfigChangeListener) listener); + } listener.stop(); } catch (Exception e) { // One listener refusing to stop must not keep the rest of the gateway up. @@ -844,6 +919,7 @@ private void stopProtocolListeners() { } } protocolListeners.clear(); + inactiveProtocolListeners.clear(); } private void handleHadoopXmlResources() { @@ -1171,6 +1247,10 @@ public void handleTopologyEvent( List events ) { handleCreateDeployment(topology, deployDir); } } + // Protocol listeners bypass the webapp redeployment that refreshes the + // servlet filter chains, so anything they cached from a topology has to + // be invalidated explicitly or an edited topology never takes effect. + notifyProtocolListeners(); } } diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/deploy/impl/ApplicationDeploymentContributor.java b/gateway-server/src/main/java/org/apache/knox/gateway/deploy/impl/ApplicationDeploymentContributor.java index 1e4d1f6a5a..034c563728 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/deploy/impl/ApplicationDeploymentContributor.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/deploy/impl/ApplicationDeploymentContributor.java @@ -162,6 +162,10 @@ private void contributeRewriteRules(DeploymentContext context) { private void contributeResources(DeploymentContext context, Service service) { Map filterParams = new HashMap<>(); List bindings = serviceDefinition.getRoutes(); + if ( bindings == null ) { + // JAXB leaves the list null when a definition declares no . + return; + } for ( Route binding : bindings ) { List filters = binding.getRewrites(); if ( filters != null && !filters.isEmpty() ) { diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/deploy/impl/ServiceDefinitionDeploymentContributor.java b/gateway-server/src/main/java/org/apache/knox/gateway/deploy/impl/ServiceDefinitionDeploymentContributor.java index 1a9a7eb36e..b2c366a60a 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/deploy/impl/ServiceDefinitionDeploymentContributor.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/deploy/impl/ServiceDefinitionDeploymentContributor.java @@ -113,6 +113,15 @@ private void contributeRewriteRules(DeploymentContext context) { private void contributeResources(DeploymentContext context, Service service) { Map filterParams = new HashMap<>(); List bindings = serviceDefinition.getRoutes(); + if ( bindings == null ) { + // A service definition need not declare routes. Services carried by a + // non-servlet listener — Spark Connect over gRPC, for instance — have no + // path for the servlet pipeline to match, and exist as definitions only so + // the role is known to the registry and to tooling. JAXB leaves the list + // null when is absent, and iterating it would fail the whole + // topology deployment, not merely this service. + return; + } for ( Route binding : bindings ) { List filters = binding.getRewrites(); if ( filters != null && !filters.isEmpty() ) { diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/services/registry/impl/DefaultServiceDefinitionRegistry.java b/gateway-server/src/main/java/org/apache/knox/gateway/services/registry/impl/DefaultServiceDefinitionRegistry.java index 44bfd4b209..62b573d743 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/services/registry/impl/DefaultServiceDefinitionRegistry.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/services/registry/impl/DefaultServiceDefinitionRegistry.java @@ -98,6 +98,13 @@ private void populateServiceDefinitions() { for (ServiceDefinition serviceDefinition : getServices()) { List routes = serviceDefinition.getRoutes(); + if (routes == null) { + // A service carried by a non-servlet listener has no path for the + // servlet pipeline to match and so declares no routes, contributing no + // URL templates here. This registry walks every definition on the + // classpath at startup, so failing on one would stop the whole gateway. + continue; + } for (Route route : routes) { try { Template template = Parser.parseTemplate(route.getPath()); diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/ProtocolListenerEnablementTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/ProtocolListenerEnablementTest.java new file mode 100644 index 0000000000..9e08a31a04 --- /dev/null +++ b/gateway-server/src/test/java/org/apache/knox/gateway/ProtocolListenerEnablementTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.knox.gateway.config.GatewayConfig; +import org.apache.knox.gateway.protocol.ProtocolListener; +import org.apache.knox.gateway.services.GatewayServices; + +import org.junit.Test; + +/** + * Whether a protocol listener runs is decided once, at startup, and cannot change + * without a restart. Toggling the property in a running gateway therefore does + * nothing — so it has to say so, or an operator has no way to tell the edit was + * not applied. + */ +public class ProtocolListenerEnablementTest { + + @Test + public void reportsAListenerSwitchedOffWhileRunning() { + final StubListener running = new StubListener("SparkConnect", false); + + final List reported = GatewayServer.warnAboutEnablementChanges( + config(), Collections.singletonList(running), Collections.emptyList()); + + assertEquals(Collections.singletonList("SparkConnect"), reported); + } + + @Test + public void reportsAListenerSwitchedOnWhileStopped() { + // The likelier mistake: an operator sets enabled=true, expects a listener, + // and gets silence. Nothing else in the gateway would mention it. + final StubListener inactive = new StubListener("SparkConnect", true); + + final List reported = GatewayServer.warnAboutEnablementChanges( + config(), Collections.emptyList(), Collections.singletonList(inactive)); + + assertEquals(Collections.singletonList("SparkConnect"), reported); + } + + @Test + public void staysQuietWhenEnablementIsUnchanged() { + final StubListener running = new StubListener("SparkConnect", true); + final StubListener inactive = new StubListener("Other", false); + + final List reported = GatewayServer.warnAboutEnablementChanges( + config(), Collections.singletonList(running), Collections.singletonList(inactive)); + + assertTrue("no warning is due when nothing changed", reported.isEmpty()); + } + + @Test + public void reportsEachChangedListenerSeparately() { + final List running = + Arrays.asList(new StubListener("A", false), new StubListener("B", true)); + final List inactive = + Arrays.asList(new StubListener("C", true), new StubListener("D", false)); + + final List reported = + GatewayServer.warnAboutEnablementChanges(config(), running, inactive); + + // A was switched off, C was switched on; B and D are unchanged. + assertEquals(Arrays.asList("A", "C"), reported); + } + + private static GatewayConfig config() { + return new GatewayTestConfig(); + } + + /** A listener that reports a fixed enablement, standing in for the config read. */ + private static final class StubListener implements ProtocolListener { + + private final String name; + private final boolean enabled; + + StubListener(String name, boolean enabled) { + this.name = name; + this.enabled = enabled; + } + + @Override + public String getName() { + return name; + } + + @Override + public boolean isEnabled(GatewayConfig config) { + return enabled; + } + + @Override + public void start(GatewayConfig config, GatewayServices services) { + } + + @Override + public void stop() { + } + + @Override + public int getPort() { + return -1; + } + } +} diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/deploy/impl/ServiceDefinitionDeploymentContributorTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/deploy/impl/ServiceDefinitionDeploymentContributorTest.java index c8e7c60d24..355fdf895e 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/deploy/impl/ServiceDefinitionDeploymentContributorTest.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/deploy/impl/ServiceDefinitionDeploymentContributorTest.java @@ -366,6 +366,48 @@ public void testServiceAttributeParameters() throws Exception { assertEquals("test2def", fparamKeyVal.get("test2")); } + /* + * A service definition need not declare routes. Services carried by a + * non-servlet listener — Spark Connect over gRPC — have no path for the servlet + * pipeline to match and exist as definitions only so the role is known to the + * registry. JAXB leaves the route list null when is absent, and + * iterating it threw, which failed the whole topology deployment rather than + * just that service. + */ + @Test + public void testServiceDefinitionWithoutRoutesContributesNothing() throws Exception { + UrlRewriteRulesDescriptor clusterRules = EasyMock.createNiceMock(UrlRewriteRulesDescriptor.class); + EasyMock.replay(clusterRules); + + ServiceDefinition svcDef = EasyMock.createNiceMock(ServiceDefinition.class); + EasyMock.expect(svcDef.getRole()).andReturn("SPARKCONNECT").anyTimes(); + // Exactly what JAXB produces for a definition with no element. + EasyMock.expect(svcDef.getRoutes()).andReturn(null).anyTimes(); + EasyMock.expect(svcDef.getDispatch()).andReturn(null).anyTimes(); + EasyMock.replay(svcDef); + + ServiceDefinitionDeploymentContributor sddc = + new ServiceDefinitionDeploymentContributor(svcDef, null); + + DeploymentContext context = EasyMock.createNiceMock(DeploymentContext.class); + EasyMock.expect(context.getDescriptor("rewrite")).andReturn(clusterRules).anyTimes(); + TestGatewayDescriptor gd = new TestGatewayDescriptor(); + EasyMock.expect(context.getGatewayDescriptor()).andReturn(gd).anyTimes(); + EasyMock.replay(context); + + Service service = EasyMock.createNiceMock(Service.class); + EasyMock.expect(service.getRole()).andReturn("SPARKCONNECT").anyTimes(); + EasyMock.replay(service); + + // Must not throw; a throw here becomes a DeploymentException and takes the + // entire topology down, including its other services. + sddc.contributeService(context, service); + + assertNotNull(gd.resources()); + assertEquals("a routeless definition should contribute no resources", + 0, gd.resources().size()); + } + private static class TestGatewayDescriptor extends GatewayDescriptorImpl { } diff --git a/gateway-service-sparkconnect/pom.xml b/gateway-service-sparkconnect/pom.xml index 48cdbdbcda..2f6ab8a356 100644 --- a/gateway-service-sparkconnect/pom.xml +++ b/gateway-service-sparkconnect/pom.xml @@ -126,6 +126,12 @@ gateway-test-utils test + + + org.apache.knox + gateway-spi-common + test + diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java index 9ecd8bc118..14fbfb142c 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java @@ -67,6 +67,13 @@ public ServerCall.Listener interceptCall(ServerCall proxyMethod(MethodDescriptor(channels, interceptor, headers)); } - private MessageInterceptor interceptorFor(String methodName) { + /** + * The guards indirect through {@link #policy} on every call rather than being + * captured here. Handlers are registered once when the server is built, so a + * guard captured at that moment could never be replaced — which is what made + * these settings silently restart-only before. + */ + // Package-private so tests can assert that a policy change reaches an + // interceptor built before the change. + MessageInterceptor interceptorFor(String methodName) { if (CONFIG_METHOD.equals(methodName)) { - return new SparkConnectMessageInterceptor(new ReservedConfigGuard(reservedConfigPrefix)); + return new SparkConnectMessageInterceptor( + (message, principal) -> policy.reservedConfigGuard().check(message, principal)); } if (ADD_ARTIFACTS_METHOD.equals(methodName)) { - return new SparkConnectMessageInterceptor(addArtifactsGuard); + return new SparkConnectMessageInterceptor( + (message, principal) -> policy.addArtifactsGuard().check(message, principal)); } return new SparkConnectMessageInterceptor(null); } + /** + * Applies a changed {@code gateway-reloadable.xml} to the controls that can + * move on a running gateway. + *

+ * Only the message-level policy is refreshed. The transport settings are built + * into the bound server and cannot change without a restart, so rather than + * accept them silently and do nothing — which looks like it worked — any + * attempt to change one is named in the log. + */ + @Override + public void onGatewayConfigChanged(GatewayConfig config) { + final SparkConnectPolicy updated = SparkConnectPolicy.from(config); + if (updated.differsFrom(policy)) { + this.policy = updated; + LOG.reloadedPolicy(LISTENER_NAME, updated.toString()); + } + warnAboutRestartOnlyChanges(config); + } + + private void warnAboutRestartOnlyChanges(GatewayConfig config) { + final GrpcListenerSettings running = getSettings(); + if (running == null) { + return; + } + final List changed = new ArrayList<>(); + if (config.getSparkConnectPort() != running.getPort()) { + changed.add("port"); + } + if (config.getSparkConnectMaxMessageSize() != running.getMaxMessageSize()) { + changed.add("max.message.size"); + } + if (config.getSparkConnectMaxConcurrentCallsPerConnection() + != running.getMaxConcurrentCallsPerConnection()) { + changed.add("max.concurrent.calls.per.connection"); + } + if (config.getSparkConnectPermitKeepAliveTime() != running.getPermitKeepAliveTimeMillis()) { + changed.add("permit.keepalive.time"); + } + if (config.isSparkConnectPermitKeepAliveWithoutCalls() != running.isPermitKeepAliveWithoutCalls()) { + changed.add("permit.keepalive.without.calls"); + } + if (config.getSparkConnectChannelIdleTimeout() != running.getChannelIdleTimeoutMillis()) { + changed.add("channel.idle.timeout"); + } + if (config.getSparkConnectDrainTimeout() != running.getDrainTimeoutMillis()) { + changed.add("drain.timeout"); + } + if (!Objects.equals(config.getSparkConnectBackendTokenAlias(), running.getBackendTokenAlias())) { + changed.add("backend.token.alias"); + } + if (!changed.isEmpty()) { + LOG.restartOnlyConfigChanged(LISTENER_NAME, String.join(", ", changed)); + } + } + private static String bareMethodName(String fullMethodName) { final int separator = fullMethodName.lastIndexOf('/'); return separator < 0 ? fullMethodName : fullMethodName.substring(separator + 1); diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectPolicy.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectPolicy.java new file mode 100644 index 0000000000..6dce42aa38 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectPolicy.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.sparkconnect; + +import java.util.List; +import java.util.Objects; + +import org.apache.knox.gateway.config.GatewayConfig; + +/** + * The message-level controls, held together so they can be replaced atomically. + *

+ * These are the settings that can change on a running gateway. The handlers + * registered at startup hold a reference to the listener rather than to a + * particular guard, and read the current policy per call, so swapping this + * object takes effect on the next RPC without rebuilding the gRPC service. + *

+ * It is one object rather than separate fields so a configuration change is seen + * whole: a call can never observe the new artifact-gating rule alongside the old + * reserved prefix. + */ +final class SparkConnectPolicy { + + private final String reservedConfigPrefix; + private final String addArtifactsMode; + private final List addArtifactsAllowedUsers; + private final ReservedConfigGuard reservedConfigGuard; + private final AddArtifactsGuard addArtifactsGuard; + + private SparkConnectPolicy(String reservedConfigPrefix, + String addArtifactsMode, + List addArtifactsAllowedUsers) { + this.reservedConfigPrefix = reservedConfigPrefix; + this.addArtifactsMode = addArtifactsMode; + this.addArtifactsAllowedUsers = addArtifactsAllowedUsers; + this.reservedConfigGuard = new ReservedConfigGuard(reservedConfigPrefix); + this.addArtifactsGuard = new AddArtifactsGuard(addArtifactsMode, addArtifactsAllowedUsers); + } + + static SparkConnectPolicy from(GatewayConfig config) { + return new SparkConnectPolicy( + config.getSparkConnectReservedConfigPrefix(), + config.getSparkConnectAddArtifactsMode(), + config.getSparkConnectAddArtifactsAllowedUsers()); + } + + ReservedConfigGuard reservedConfigGuard() { + return reservedConfigGuard; + } + + AddArtifactsGuard addArtifactsGuard() { + return addArtifactsGuard; + } + + /** + * Whether this policy differs from another, used to decide if a configuration + * change is worth logging. Compares the configured values rather than the + * derived guards, which have no meaningful equality. + * + * @param other the policy to compare against, may be null + * @return true if the two express different rules + */ + boolean differsFrom(SparkConnectPolicy other) { + return other == null + || !Objects.equals(reservedConfigPrefix, other.reservedConfigPrefix) + || !Objects.equals(addArtifactsMode, other.addArtifactsMode) + || !Objects.equals(addArtifactsAllowedUsers, other.addArtifactsAllowedUsers); + } + + @Override + public String toString() { + return "addArtifactsMode=" + addArtifactsMode + + ", addArtifactsAllowedUsers=" + addArtifactsAllowedUsers + + ", reservedConfigPrefix=" + reservedConfigPrefix; + } +} diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/README.md b/gateway-service-sparkconnect/src/main/proto/spark/connect/README.md new file mode 100644 index 0000000000..543b6b1044 --- /dev/null +++ b/gateway-service-sparkconnect/src/main/proto/spark/connect/README.md @@ -0,0 +1,98 @@ + + +# Vendored Spark Connect protocol definitions + +These `.proto` files are copied verbatim from Apache Spark. They are **not** +maintained here — do not edit them. Changes belong upstream. + +## Provenance + +| | | +|-------------|---------------------------------------------------------------------| +| Repository | | +| Path | `sql/connect/common/src/main/protobuf/spark/connect/` | +| Tag | `v4.2.0` | +| Commit | `32f7299601108917fb01920a54e084595b7b3bf8` | +| Commit date | 2026-07-11 | +| License | Apache License 2.0 (same as Knox; each file retains its ASF header) | + +Every file in this directory is byte-identical to that commit. + +## What is here, and what is not + +All ten protos that make up the `spark.connect.SparkConnectService` protocol are +vendored. Upstream also carries `example_plugins.proto`, which is **deliberately +not copied** — it is sample code for Spark's own extension mechanism and defines +nothing the gateway proxies. + +`base.proto` also imports `google/protobuf/any.proto` and +`google/protobuf/timestamp.proto`. Those are not vendored; they ship inside +`protobuf-java` and protoc resolves them from there. + +## Why vendored rather than a dependency + +Depending on `org.apache.spark:spark-connect-common` would pull Spark's whole +dependency tree onto the gateway classpath for the sake of a handful of message +definitions. Vendoring the protos and generating with protoc at build time keeps +Knox's classpath its own. `protobuf-java` is already managed in the root pom, and +the grpc-java version is pinned to the line that matches it. + +## Drift policy + +These track the newest supported Spark line. Skew is expected and mostly +harmless, because the gateway only reads and writes `user_context` and +`session_id` and otherwise round-trips messages through the generated classes. +Protobuf preserves unknown fields across a parse and re-serialize, so a field +added in a newer Spark survives the trip untouched. Requests for RPCs added after +this snapshot are relayed as opaque bytes by `PassthroughHandlerRegistry`; they +lose identity assertion but are still authenticated, authorized, routed and +audited. + +The uniformity the gateway relies on is that every `SparkConnectService` request +message carries `string session_id = 1` and `UserContext user_context = 2` in the +same positions. `SparkConnectMessageInterceptorTest` asserts this across the RPC +shapes, so a refresh that broke the assumption would fail the build rather than +silently stop asserting identity. + +## Refreshing + +Set the tag you want, re-copy, and rerun the build: + +```bash +SPARK_TAG=v4.2.0 +DEST=gateway-service-sparkconnect/src/main/proto/spark/connect +BASE=https://raw.githubusercontent.com/apache/spark/$SPARK_TAG/sql/connect/common/src/main/protobuf/spark/connect + +for f in base catalog commands common expressions ml ml_common pipelines relations types; do + curl -fsS -o "$DEST/$f.proto" "$BASE/$f.proto" +done +``` + +Then update the provenance table above with the new tag and the commit it +resolves to: + +```bash +curl -fsS "https://api.github.com/repos/apache/spark/git/ref/tags/$SPARK_TAG" \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["object"]["sha"])' +``` + +After refreshing, check whether upstream added any RPC to `SparkConnectService`. +New methods are picked up automatically — handlers are registered by iterating +the generated service descriptor, not from a handwritten list — but a new RPC +whose request message departs from the `session_id`/`user_context` shape would +need the interceptor revisited. diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AuthenticationInterceptorTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AuthenticationInterceptorTest.java new file mode 100644 index 0000000000..9d37a63ea2 --- /dev/null +++ b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AuthenticationInterceptorTest.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.Collections; + +import io.grpc.Attributes; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.Status; + +import org.junit.Test; + +/** + * Authentication has to fail closed. + *

+ * Validation reaches third-party JOSE code that signals some malformed + * credentials by throwing rather than returning false — a token signed with an + * unexpected algorithm family, for one. If that escapes, the caller gets + * {@code UNKNOWN} instead of {@code UNAUTHENTICATED}, which answers a probe + * differently from an ordinary rejection and costs a stack trace per request. + */ +public class AuthenticationInterceptorTest { + + private static final String BEARER = "Bearer some-token"; + + @Test + public void rejectsWhenValidationThrowsUnexpectedly() { + final Outcome outcome = intercept(new ThrowingAuthenticator( + new IllegalStateException("unexpected JOSE failure"))); + + assertFalse("the call must not reach the handler", outcome.proceeded); + assertEquals(Status.Code.UNAUTHENTICATED, outcome.status.getCode()); + } + + @Test + public void rejectsWithTheSameStatusAsAnOrdinaryFailure() { + // A probe must not be able to tell "malformed in an interesting way" from + // "simply wrong" by the status code. + final Status thrown = intercept(new ThrowingAuthenticator( + new IllegalArgumentException("boom"))).status; + final Status declined = intercept(new DecliningAuthenticator()).status; + + assertEquals(declined.getCode(), thrown.getCode()); + assertEquals(declined.getDescription(), thrown.getDescription()); + } + + @Test + public void rejectsWhenNoTokenIsPresented() { + final Outcome outcome = intercept(new DecliningAuthenticator(), new Metadata()); + + assertFalse(outcome.proceeded); + assertEquals(Status.Code.UNAUTHENTICATED, outcome.status.getCode()); + } + + @Test + public void rejectsANonBearerAuthorizationHeader() { + final Metadata headers = new Metadata(); + headers.put(GrpcMetadataKeys.AUTHORIZATION, "Basic dXNlcjpwYXNz"); + + final Outcome outcome = intercept(new DecliningAuthenticator(), headers); + + assertFalse(outcome.proceeded); + assertEquals(Status.Code.UNAUTHENTICATED, outcome.status.getCode()); + } + + @Test + public void admitsAValidTokenAndPublishesTheIdentity() { + final GrpcCallContext callContext = + new GrpcCallContext("m", "authority", "127.0.0.1", System.nanoTime()); + final Outcome outcome = intercept(new AcceptingAuthenticator(), bearerHeaders(), callContext); + + assertTrue("a valid token should reach the handler", outcome.proceeded); + assertNull(outcome.status); + assertEquals("alice", callContext.getPrincipal()); + assertEquals(Collections.singleton("analysts"), callContext.getGroups()); + } + + private static Metadata bearerHeaders() { + final Metadata headers = new Metadata(); + headers.put(GrpcMetadataKeys.AUTHORIZATION, BEARER); + return headers; + } + + private static Outcome intercept(TokenAuthenticator authenticator) { + return intercept(authenticator, bearerHeaders()); + } + + private static Outcome intercept(TokenAuthenticator authenticator, Metadata headers) { + return intercept(authenticator, headers, + new GrpcCallContext("m", "authority", "127.0.0.1", System.nanoTime())); + } + + private static Outcome intercept(TokenAuthenticator authenticator, + Metadata headers, + GrpcCallContext callContext) { + final RecordingServerCall call = new RecordingServerCall(); + final Outcome outcome = new Outcome(); + final ServerCallHandler next = (c, h) -> { + outcome.proceeded = true; + return new ServerCall.Listener() { }; + }; + + io.grpc.Context.current().withValue(GrpcCallContext.KEY, callContext) + .run(() -> new AuthenticationInterceptor(authenticator).interceptCall(call, headers, next)); + + outcome.status = call.closedWith; + return outcome; + } + + private static final class Outcome { + private boolean proceeded; + private Status status; + } + + /** Stands in for validation blowing up inside third-party JOSE code. */ + private static final class ThrowingAuthenticator extends TokenAuthenticator { + private final RuntimeException failure; + + ThrowingAuthenticator(RuntimeException failure) { + super(null, null); + this.failure = failure; + } + + @Override + public AuthenticatedUser authenticate(String serializedToken) { + throw failure; + } + } + + /** Stands in for an ordinary "this token is not valid" outcome. */ + private static final class DecliningAuthenticator extends TokenAuthenticator { + DecliningAuthenticator() { + super(null, null); + } + + @Override + public AuthenticatedUser authenticate(String serializedToken) throws AuthenticationException { + throw new AuthenticationException("Bearer token failed validation"); + } + } + + private static final class AcceptingAuthenticator extends TokenAuthenticator { + AcceptingAuthenticator() { + super(null, null); + } + + @Override + public AuthenticatedUser authenticate(String serializedToken) { + return new AuthenticatedUser("alice", Collections.singleton("analysts")); + } + } + + /** Captures the status a rejected call was closed with. */ + private static final class RecordingServerCall extends ServerCall { + + private Status closedWith; + + @Override + public void request(int numMessages) { + } + + @Override + public void sendHeaders(Metadata headers) { + } + + @Override + public void sendMessage(byte[] message) { + } + + @Override + public void close(Status status, Metadata trailers) { + this.closedWith = status; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public Attributes getAttributes() { + return Attributes.EMPTY; + } + + @Override + public MethodDescriptor getMethodDescriptor() { + return MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.UNKNOWN) + .setFullMethodName("spark.connect.SparkConnectService/AnalyzePlan") + .setRequestMarshaller(ByteArrayMarshaller.INSTANCE) + .setResponseMarshaller(ByteArrayMarshaller.INSTANCE) + .build(); + } + } +} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AuthorizationInterceptorReloadTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AuthorizationInterceptorReloadTest.java new file mode 100644 index 0000000000..a2a591ad5a --- /dev/null +++ b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AuthorizationInterceptorReloadTest.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.knox.gateway.config.GatewayConfig; +import org.apache.knox.gateway.services.GatewayServices; +import org.apache.knox.gateway.services.ServiceType; +import org.apache.knox.gateway.services.topology.TopologyService; +import org.apache.knox.gateway.topology.Provider; +import org.apache.knox.gateway.topology.Topology; + +import io.grpc.Attributes; +import io.grpc.Context; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.Status; + +import org.easymock.EasyMock; +import org.junit.Test; + +/** + * Knox reloads topologies from disk while running, but this listener does not go + * through the webapp redeployment that refreshes the servlet filter chains. Its + * cached authorization rules therefore have to be invalidated explicitly, or an + * administrator's ACL edit silently never takes effect. + */ +public class AuthorizationInterceptorReloadTest { + + private static final String TOPOLOGY = "analytics"; + private static final String ROLE = "SPARKCONNECT"; + + @Test + public void picksUpAnAclChangeAfterReload() { + final MutableTopologies topologies = new MutableTopologies(); + topologies.setAcl("alice;*;*"); + + final AuthorizationInterceptor interceptor = + new AuthorizationInterceptor(config(), services(topologies.asService()), ROLE); + + assertTrue("alice should be permitted by the initial ACL", permitted(interceptor, "alice")); + assertEquals("bob should not be", Status.Code.PERMISSION_DENIED, denyCode(interceptor, "bob")); + + // An administrator edits the topology; the file monitor redeploys it. + topologies.setAcl("alice,bob;*;*"); + + // Without invalidation the interceptor keeps serving the ACL it first read. + assertEquals("stale ACL should still be in effect before reload", + Status.Code.PERMISSION_DENIED, denyCode(interceptor, "bob")); + + interceptor.invalidate(); + + assertTrue("bob should be permitted once the change is picked up", + permitted(interceptor, "bob")); + assertTrue("alice should still be permitted", permitted(interceptor, "alice")); + } + + @Test + public void picksUpAnAclThatBecomesMoreRestrictive() { + final MutableTopologies topologies = new MutableTopologies(); + topologies.setAcl("*;*;*"); + + final AuthorizationInterceptor interceptor = + new AuthorizationInterceptor(config(), services(topologies.asService()), ROLE); + assertTrue(permitted(interceptor, "mallory")); + + // Revoking access matters more than granting it: this is the direction where + // a stale cache leaves someone with access they were meant to lose. + topologies.setAcl("alice;*;*"); + interceptor.invalidate(); + + assertEquals(Status.Code.PERMISSION_DENIED, denyCode(interceptor, "mallory")); + assertTrue(permitted(interceptor, "alice")); + } + + private static boolean permitted(AuthorizationInterceptor interceptor, String user) { + return outcome(interceptor, user) == null; + } + + private static Status.Code denyCode(AuthorizationInterceptor interceptor, String user) { + final Status status = outcome(interceptor, user); + return status == null ? null : status.getCode(); + } + + /** Runs one call through the interceptor; returns null when it was allowed. */ + private static Status outcome(AuthorizationInterceptor interceptor, String user) { + final GrpcCallContext callContext = new GrpcCallContext( + "spark.connect.SparkConnectService/ExecutePlan", "knox", "10.0.0.5", System.nanoTime()); + callContext.setPrincipal(user); + callContext.setTopology(TOPOLOGY); + + final RecordingServerCall call = new RecordingServerCall(); + final boolean[] proceeded = {false}; + final ServerCallHandler next = (c, h) -> { + proceeded[0] = true; + return new ServerCall.Listener() { }; + }; + + Context.current().withValue(GrpcCallContext.KEY, callContext) + .run(() -> interceptor.interceptCall(call, new Metadata(), next)); + + return proceeded[0] ? null : call.closedWith; + } + + private static GatewayConfig config() { + final GatewayConfig config = EasyMock.createNiceMock(GatewayConfig.class); + EasyMock.expect(config.getKnoxAdminUsers()).andReturn("").anyTimes(); + EasyMock.expect(config.getKnoxAdminGroups()).andReturn("").anyTimes(); + EasyMock.replay(config); + return config; + } + + private static GatewayServices services(TopologyService topologyService) { + final GatewayServices services = EasyMock.createNiceMock(GatewayServices.class); + EasyMock.expect(services.getService(ServiceType.TOPOLOGY_SERVICE)) + .andReturn(topologyService).anyTimes(); + EasyMock.replay(services); + return services; + } + + /** A single topology whose ACL parameter can be changed between calls. */ + private static final class MutableTopologies { + + private final Topology topology = new Topology(); + private final List topologies = new ArrayList<>(); + + MutableTopologies() { + topology.setName(TOPOLOGY); + topologies.add(topology); + } + + /** Stands in for an administrator editing the file and the monitor redeploying it. */ + void setAcl(String acl) { + topology.getProviders().clear(); + final Provider provider = new Provider(); + provider.setRole("authorization"); + provider.setName("AclsAuthz"); + provider.setEnabled(true); + provider.getParams().put(ROLE + ".acl", acl); + topology.addProvider(provider); + } + + TopologyService asService() { + final TopologyService service = EasyMock.createNiceMock(TopologyService.class); + // Returns the same live list, so edits are visible to any later lookup. + EasyMock.expect(service.getTopologies()).andReturn(topologies).anyTimes(); + EasyMock.replay(service); + return service; + } + } + + /** Captures the status a rejected call was closed with. */ + private static final class RecordingServerCall extends ServerCall { + + private Status closedWith; + + @Override + public void request(int numMessages) { + } + + @Override + public void sendHeaders(Metadata headers) { + } + + @Override + public void sendMessage(byte[] message) { + } + + @Override + public void close(Status status, Metadata trailers) { + this.closedWith = status; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public Attributes getAttributes() { + return Attributes.EMPTY; + } + + @Override + public MethodDescriptor getMethodDescriptor() { + return MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.UNKNOWN) + .setFullMethodName("spark.connect.SparkConnectService/ExecutePlan") + .setRequestMarshaller(ByteArrayMarshaller.INSTANCE) + .setResponseMarshaller(ByteArrayMarshaller.INSTANCE) + .build(); + } + } +} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/TopologySelectionAuthorizationTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/TopologySelectionAuthorizationTest.java new file mode 100644 index 0000000000..aca4720b91 --- /dev/null +++ b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/TopologySelectionAuthorizationTest.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import org.apache.knox.gateway.config.GatewayConfig; +import org.apache.knox.gateway.services.GatewayServices; +import org.apache.knox.gateway.services.ServiceType; +import org.apache.knox.gateway.services.topology.TopologyService; +import org.apache.knox.gateway.topology.Provider; +import org.apache.knox.gateway.topology.Topology; + +import io.grpc.Attributes; +import io.grpc.Context; +import io.grpc.Metadata; +import io.grpc.MethodDescriptor; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.Status; + +import org.easymock.EasyMock; +import org.junit.Test; + +/** + * A client picks its topology with a {@code knox-topology} connection parameter, + * so selection is a client-supplied value and cannot be trusted on its own. + *

+ * What makes it safe is that authorization runs after routing and is keyed on the + * topology that was selected: asking for a topology is not the same as being + * allowed to use it. These tests cover the cross-topology case specifically — + * one identity, several topologies, different answers. + */ +public class TopologySelectionAuthorizationTest { + + private static final String ROLE = "SPARKCONNECT"; + private static final String IP = "10.0.0.5"; + + @Test + public void aUserMayReachOneTopologyAndNotAnother() { + final Topologies topologies = new Topologies() + .with("analytics", "alice;*;*") + .with("etl", "bob;*;*"); + final AuthorizationInterceptor interceptor = interceptor(topologies); + + assertAllowed(interceptor, "analytics", "alice"); + // Alice can name 'etl' in her connection string; naming it is not reaching it. + assertDenied(interceptor, "etl", "alice"); + + assertAllowed(interceptor, "etl", "bob"); + assertDenied(interceptor, "analytics", "bob"); + } + + @Test + public void topologySelectionCanBeAuthorizedByGroup() { + final Topologies topologies = new Topologies() + .with("analytics", "*;analysts;*") + .with("etl", "*;engineers;*"); + final AuthorizationInterceptor interceptor = interceptor(topologies); + + assertAllowed(interceptor, "analytics", "alice", "analysts"); + assertDenied(interceptor, "etl", "alice", "analysts"); + + // Membership of both groups reaches both clusters. + assertAllowed(interceptor, "analytics", "carol", "analysts", "engineers"); + assertAllowed(interceptor, "etl", "carol", "analysts", "engineers"); + } + + @Test + public void eachTopologyIsEvaluatedAgainstItsOwnRules() { + // The per-topology authorizer cache must not let one topology's decision + // leak into another's. + final Topologies topologies = new Topologies() + .with("open", "*;*;*") + .with("restricted", "alice;*;*"); + final AuthorizationInterceptor interceptor = interceptor(topologies); + + assertAllowed(interceptor, "open", "mallory"); + assertDenied(interceptor, "restricted", "mallory"); + // Re-check the first, in case evaluating the second disturbed it. + assertAllowed(interceptor, "open", "mallory"); + } + + @Test + public void aTopologyWithNoAclIsReachableByAnyAuthenticatedUser() { + // Worth pinning because it is the permissive direction: declaring no ACL for + // the role does not restrict selection, it leaves the topology open to anyone + // who can authenticate. Restricting selection means setting an ACL. + final Topologies topologies = new Topologies().withNoAclProvider("wide-open"); + final AuthorizationInterceptor interceptor = interceptor(topologies); + + assertAllowed(interceptor, "wide-open", "mallory"); + } + + @Test + public void aDisabledAclProviderDoesNotRestrictSelection() { + final Topologies topologies = new Topologies().withDisabled("analytics", "alice;*;*"); + final AuthorizationInterceptor interceptor = interceptor(topologies); + + assertAllowed(interceptor, "analytics", "mallory"); + } + + private static void assertAllowed(AuthorizationInterceptor interceptor, + String topology, String user, String... groups) { + assertNull("expected " + user + " to be allowed into " + topology, + outcome(interceptor, topology, user, groups)); + } + + private static void assertDenied(AuthorizationInterceptor interceptor, + String topology, String user, String... groups) { + final Status status = outcome(interceptor, topology, user, groups); + assertEquals("expected " + user + " to be denied " + topology, + Status.Code.PERMISSION_DENIED, status == null ? null : status.getCode()); + } + + /** Runs one call; returns null when it was allowed through. */ + private static Status outcome(AuthorizationInterceptor interceptor, + String topology, String user, String... groups) { + final GrpcCallContext callContext = new GrpcCallContext( + "spark.connect.SparkConnectService/ExecutePlan", "knox", IP, System.nanoTime()); + callContext.setPrincipal(user); + callContext.setGroups(new HashSet<>(Arrays.asList(groups))); + // Set by the routing interceptor from the client's knox-topology parameter. + callContext.setTopology(topology); + + final RecordingServerCall call = new RecordingServerCall(); + final boolean[] proceeded = {false}; + final ServerCallHandler next = (c, h) -> { + proceeded[0] = true; + return new ServerCall.Listener() { }; + }; + + Context.current().withValue(GrpcCallContext.KEY, callContext) + .run(() -> interceptor.interceptCall(call, new Metadata(), next)); + + return proceeded[0] ? null : call.closedWith; + } + + private static AuthorizationInterceptor interceptor(Topologies topologies) { + final GatewayConfig config = EasyMock.createNiceMock(GatewayConfig.class); + EasyMock.expect(config.getKnoxAdminUsers()).andReturn("").anyTimes(); + EasyMock.expect(config.getKnoxAdminGroups()).andReturn("").anyTimes(); + EasyMock.replay(config); + + final TopologyService topologyService = EasyMock.createNiceMock(TopologyService.class); + EasyMock.expect(topologyService.getTopologies()).andReturn(topologies.all()).anyTimes(); + EasyMock.replay(topologyService); + + final GatewayServices services = EasyMock.createNiceMock(GatewayServices.class); + EasyMock.expect(services.getService(ServiceType.TOPOLOGY_SERVICE)) + .andReturn(topologyService).anyTimes(); + EasyMock.replay(services); + + return new AuthorizationInterceptor(config, services, ROLE); + } + + /** Builds a set of topologies with differing ACLs. */ + private static final class Topologies { + + private final List declared = new ArrayList<>(); + + Topologies with(String name, String acl) { + declared.add(topology(name, acl, true)); + return this; + } + + Topologies withDisabled(String name, String acl) { + declared.add(topology(name, acl, false)); + return this; + } + + Topologies withNoAclProvider(String name) { + final Topology topology = new Topology(); + topology.setName(name); + declared.add(topology); + return this; + } + + private static Topology topology(String name, String acl, boolean enabled) { + final Topology topology = new Topology(); + topology.setName(name); + final Provider provider = new Provider(); + provider.setRole("authorization"); + provider.setName("AclsAuthz"); + provider.setEnabled(enabled); + provider.getParams().put(ROLE + ".acl", acl); + topology.addProvider(provider); + return topology; + } + + List all() { + return Collections.unmodifiableList(declared); + } + } + + /** Captures the status a rejected call was closed with. */ + private static final class RecordingServerCall extends ServerCall { + + private Status closedWith; + + @Override + public void request(int numMessages) { + } + + @Override + public void sendHeaders(Metadata headers) { + } + + @Override + public void sendMessage(byte[] message) { + } + + @Override + public void close(Status status, Metadata trailers) { + this.closedWith = status; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public Attributes getAttributes() { + return Attributes.EMPTY; + } + + @Override + public MethodDescriptor getMethodDescriptor() { + return MethodDescriptor.newBuilder() + .setType(MethodDescriptor.MethodType.UNKNOWN) + .setFullMethodName("spark.connect.SparkConnectService/ExecutePlan") + .setRequestMarshaller(ByteArrayMarshaller.INSTANCE) + .setResponseMarshaller(ByteArrayMarshaller.INSTANCE) + .build(); + } + } +} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectListenerConfigReloadTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectListenerConfigReloadTest.java new file mode 100644 index 0000000000..646d8d2660 --- /dev/null +++ b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectListenerConfigReloadTest.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.sparkconnect; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.util.Arrays; +import java.util.Collections; + +import org.apache.knox.gateway.GatewayTestConfig; +import org.apache.knox.gateway.grpc.GrpcCallContext; +import org.apache.knox.gateway.grpc.MessageInterceptor; + +import com.google.protobuf.Message; + +import io.grpc.Context; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; + +import org.apache.spark.connect.proto.AddArtifactsRequest; +import org.apache.spark.connect.proto.ConfigRequest; +import org.apache.spark.connect.proto.KeyValue; + +import org.junit.Test; + +/** + * The message-level controls are the only Spark Connect settings that can change + * on a running gateway; the rest are built into the bound server. + *

+ * Each test obtains an interceptor before changing configuration and + * asserts on that same instance afterwards. That is the property that matters: + * handlers are registered once when the gRPC service is built, so a guard + * captured at that moment could never be replaced, and the setting would be + * silently restart-only. + */ +public class SparkConnectListenerConfigReloadTest { + + private static final String USER = "alice"; + + @Test + public void addArtifactsGatingTakesEffectWithoutRestart() { + final GatewayTestConfig config = new GatewayTestConfig(); + config.setSparkConnectAddArtifactsMode(AddArtifactsGuard.MODE_ALLOW); + final SparkConnectListener listener = started(config); + + final MessageInterceptor addArtifacts = listener.interceptorFor("AddArtifacts"); + intercept(addArtifacts, AddArtifactsRequest.getDefaultInstance()); + + config.setSparkConnectAddArtifactsMode(AddArtifactsGuard.MODE_DENY); + listener.onGatewayConfigChanged(config); + + assertDenied(addArtifacts, AddArtifactsRequest.getDefaultInstance()); + } + + @Test + public void addArtifactsAllowListTakesEffectWithoutRestart() { + final GatewayTestConfig config = new GatewayTestConfig(); + config.setSparkConnectAddArtifactsMode(AddArtifactsGuard.MODE_ALLOW_LISTED_USERS); + config.setSparkConnectAddArtifactsAllowedUsers(Collections.emptyList()); + final SparkConnectListener listener = started(config); + final MessageInterceptor addArtifacts = listener.interceptorFor("AddArtifacts"); + + assertDenied(addArtifacts, AddArtifactsRequest.getDefaultInstance()); + + config.setSparkConnectAddArtifactsAllowedUsers(Arrays.asList(USER, "bob")); + listener.onGatewayConfigChanged(config); + + intercept(addArtifacts, AddArtifactsRequest.getDefaultInstance()); + } + + @Test + public void reservedConfigPrefixTakesEffectWithoutRestart() { + final GatewayTestConfig config = new GatewayTestConfig(); + final SparkConnectListener listener = started(config); + final MessageInterceptor configRpc = listener.interceptorFor("Config"); + + // 'acme.' is not reserved under the default 'knox.' prefix. + intercept(configRpc, configSet("acme.principal", "root")); + + config.setSparkConnectReservedConfigPrefix("acme."); + listener.onGatewayConfigChanged(config); + + assertDenied(configRpc, configSet("acme.principal", "root")); + } + + @Test + public void aRestartOnlyChangeDoesNotDisturbTheMessagePolicy() { + final GatewayTestConfig config = new GatewayTestConfig(); + final SparkConnectListener listener = started(config); + final MessageInterceptor configRpc = listener.interceptorFor("Config"); + + // The port cannot be rebound on a running listener. Handling the change must + // neither throw nor quietly drop the policy that is still in force. + config.setSparkConnectPort(15099); + listener.onGatewayConfigChanged(config); + + assertDenied(configRpc, configSet("knox.principal", "root")); + } + + @Test + public void identityAssertionIsUnaffectedByPolicyChanges() { + final GatewayTestConfig config = new GatewayTestConfig(); + final SparkConnectListener listener = started(config); + final MessageInterceptor configRpc = listener.interceptorFor("Config"); + + config.setSparkConnectReservedConfigPrefix("acme."); + listener.onGatewayConfigChanged(config); + + // A permitted call still gets its identity asserted; the guard swap must not + // replace the interceptor's primary job. + final ConfigRequest forwarded = (ConfigRequest) interceptAndReturn( + configRpc, configSet("spark.sql.shuffle.partitions", "8")); + assertEquals(USER, forwarded.getUserContext().getUserId()); + } + + /** Populates the listener's policy without binding a port. */ + private static SparkConnectListener started(GatewayTestConfig config) { + final SparkConnectListener listener = new SparkConnectListener(); + listener.createSettings(config); + return listener; + } + + private static void intercept(MessageInterceptor interceptor, Message request) { + interceptAndReturn(interceptor, request); + } + + private static Message interceptAndReturn(MessageInterceptor interceptor, + Message request) { + final GrpcCallContext callContext = + new GrpcCallContext("m", "authority", "127.0.0.1", System.nanoTime()); + callContext.setPrincipal(USER); + final Message[] result = new Message[1]; + Context.current().withValue(GrpcCallContext.KEY, callContext) + .run(() -> result[0] = interceptor.intercept(request)); + return result[0]; + } + + private static void assertDenied(MessageInterceptor interceptor, Message request) { + try { + intercept(interceptor, request); + fail("Expected the request to be denied"); + } catch (StatusRuntimeException e) { + assertEquals(Status.Code.PERMISSION_DENIED, e.getStatus().getCode()); + } + } + + private static ConfigRequest configSet(String key, String value) { + return ConfigRequest.newBuilder() + .setOperation(ConfigRequest.Operation.newBuilder() + .setSet(ConfigRequest.Set.newBuilder() + .addPairs(KeyValue.newBuilder().setKey(key).setValue(value)))) + .build(); + } +} diff --git a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java index 813a4807b8..05a0caf827 100644 --- a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java +++ b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java @@ -70,6 +70,7 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig { private String sparkConnectDefaultTopology; private String sparkConnectBackendTokenAlias; private String sparkConnectAddArtifactsMode = DEFAULT_SPARKCONNECT_ADD_ARTIFACTS_MODE; + private String sparkConnectReservedConfigPrefix = DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX; private List sparkConnectAddArtifactsAllowedUsers = Collections.emptyList(); @@ -784,7 +785,11 @@ public void setSparkConnectAddArtifactsAllowedUsers(List allowedUsers) { @Override public String getSparkConnectReservedConfigPrefix() { - return DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX; + return sparkConnectReservedConfigPrefix; + } + + public void setSparkConnectReservedConfigPrefix(String sparkConnectReservedConfigPrefix) { + this.sparkConnectReservedConfigPrefix = sparkConnectReservedConfigPrefix; } @Override diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/protocol/ProtocolListener.java b/gateway-spi/src/main/java/org/apache/knox/gateway/protocol/ProtocolListener.java index 2ad3037d17..064b1c6640 100644 --- a/gateway-spi/src/main/java/org/apache/knox/gateway/protocol/ProtocolListener.java +++ b/gateway-spi/src/main/java/org/apache/knox/gateway/protocol/ProtocolListener.java @@ -91,4 +91,21 @@ public interface ProtocolListener { * @return the bound port, or -1 if the listener is not running */ int getPort(); + + /** + * Notifies the listener that topologies have been redeployed, so anything it + * derived from topology configuration must be recomputed. + *

+ * Knox reloads topologies from disk while running, and listeners on this path + * do not go through the webapp redeployment that refreshes the servlet filter + * chains. A listener that caches anything from a topology — authorization + * rules, provider parameters — will therefore keep serving stale configuration + * until it is told otherwise, which for security configuration means an + * administrator's change silently not taking effect. + *

+ * Called for every topology event, so implementations should be cheap: + * invalidate and recompute lazily rather than rebuilding here. + */ + default void reload() { + } } diff --git a/knox-site/docs/spark-connect-support.md b/knox-site/docs/spark-connect-support.md index 7445d2a0dc..4939e4837a 100644 --- a/knox-site/docs/spark-connect-support.md +++ b/knox-site/docs/spark-connect-support.md @@ -37,10 +37,60 @@ fill that role, adding: - **Auditing** — one record per RPC: principal, topology, method, session, outcome and duration. -Because gRPC requires HTTP/2 with ALPN, and because `grpc-status` and Spark's -structured error details travel in HTTP trailers, this cannot run on Knox's -existing Jetty connectors. Spark Connect is served by a **dedicated listener on -its own port**, started and stopped with the gateway. +#### Why a separate port #### + +Spark Connect does not go through Knox's servlet pipeline, and could not: gRPC +needs HTTP/2 negotiated over ALPN, which Knox's Jetty connectors do not offer; +the Servlet 3.1 API Knox targets has no way to read or write HTTP trailers, where +`grpc-status` and Spark's structured error details live; and the outbound +dispatch layer is built on a strict request/response HTTP/1.1 client, whereas +`ExecutePlan` and `ReattachExecute` are long-lived server streams and +`AddArtifacts` is a client stream. + +Routing rules it out independently. A `sc://` connection string may not contain a +path — the Spark client forbids it, to stay compatible with the gRPC standard — +and gRPC fixes request paths at `/pkg.Service/Method`. Knox's usual +`/gateway/{topology}/{service}` routing therefore has nothing to match on. + +So Spark Connect is served by a **dedicated listener on its own port**, started +and stopped with the gateway, alongside Jetty rather than inside it: + + Knox JVM + ┌─────────────────────────────────────────────┐ + sc:// ─▶│ :15002 gRPC listener (Netty) │ + grpc/h2 │ ├─ TLS (gateway identity) │ + │ ├─ audit │ + │ ├─ authentication (bearer JWT) │ grpc/h2 + │ ├─ routing (knox-topology → registry) │──▶ Spark Connect + │ ├─ authorization (topology ACLs) │ server :15002 + │ └─ relay (asserts user_context.user_id) │ + ├─────────────────────────────────────────────┤ + │ :8443 Jetty (the existing servlet gateway) │ + └─────────────────────────────────────────────┘ + +This mirrors how Knox already handles WebSockets, which likewise bypass the +topology filter chains and do their own authentication — the difference being +that a WebSocket upgrade can share Jetty's HTTP/1.1 connector, and gRPC cannot. + +### What is proxied ### + +The whole `spark.connect.SparkConnectService` surface. Every RPC is relayed with +its status and trailers passed through verbatim, and every request has its +`user_context.user_id` replaced with the authenticated principal: + +| Shape | RPCs | +|------------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| Unary | `AnalyzePlan`, `Config`, `ArtifactStatus`, `Interrupt`, `ReleaseExecute`, `ReleaseSession`, `FetchErrorDetails`, `CloneSession`, `GetStatus` | +| Server-streaming | `ExecutePlan`, `ReattachExecute` | +| Client-streaming | `AddArtifacts` | + +Flow control is honored in both directions, so a slow client cannot make the +gateway buffer an unbounded number of Arrow batches, and cancellation propagates +both ways rather than leaving orphaned executions on the backend. + +Nothing in the message bodies is rewritten apart from the identity fields. There +are no URLs or hostnames inside these protobufs, so Knox's rewrite machinery has +no role here. ### Configuration ### @@ -64,21 +114,67 @@ certificate to manage. #### All properties #### -| Property | Default | Meaning | -|---|---|---| -| `gateway.sparkconnect.enabled` | `false` | Master switch; the listener is not started when false. | -| `gateway.sparkconnect.port` | `15002` | Port for the gRPC listener. | -| `gateway.sparkconnect.default.topology` | *(none)* | Topology used when the client sends no `knox-topology`. | -| `gateway.sparkconnect.max.message.size` | `134217728` | Maximum inbound message size in bytes, both legs. Matches Spark's 128 MB default. | -| `gateway.sparkconnect.max.concurrent.calls.per.connection` | `1000` | Maximum concurrent gRPC streams per client connection. | -| `gateway.sparkconnect.permit.keepalive.time` | `10000` | Minimum tolerated interval between client keepalive pings, in ms. | -| `gateway.sparkconnect.permit.keepalive.without.calls` | `true` | Whether clients may ping an idle channel. Spark Connect clients do. | -| `gateway.sparkconnect.channel.idle.timeout` | `1800000` | Idle time before an unused backend channel is shut down, in ms. | -| `gateway.sparkconnect.drain.timeout` | `30000` | How long in-flight RPCs get to finish at shutdown, in ms. | -| `gateway.sparkconnect.backend.token.alias` | *(none)* | Alias holding the backend's pre-shared token (see below). | -| `gateway.sparkconnect.add.artifacts.mode` | `ALLOW` | `ALLOW`, `DENY`, or `ALLOW_LISTED_USERS` for the `AddArtifacts` RPC. | -| `gateway.sparkconnect.add.artifacts.allowed.users` | *(none)* | Comma-separated users permitted when the mode is `ALLOW_LISTED_USERS`. | -| `gateway.sparkconnect.reserved.config.prefix` | `knox.` | Session-configuration key prefix clients may not `Set` or `Unset`. | +| Property | Default | Meaning | +|------------------------------------------------------------|-------------|-----------------------------------------------------------------------------------| +| `gateway.sparkconnect.enabled` | `false` | Master switch; the listener is not started when false. | +| `gateway.sparkconnect.port` | `15002` | Port for the gRPC listener. | +| `gateway.sparkconnect.default.topology` | *(none)* | Topology used when the client sends no `knox-topology`. | +| `gateway.sparkconnect.max.message.size` | `134217728` | Maximum inbound message size in bytes, both legs. Matches Spark's 128 MB default. | +| `gateway.sparkconnect.max.concurrent.calls.per.connection` | `1000` | Maximum concurrent gRPC streams per client connection. | +| `gateway.sparkconnect.permit.keepalive.time` | `10000` | Minimum tolerated interval between client keepalive pings, in ms. | +| `gateway.sparkconnect.permit.keepalive.without.calls` | `true` | Whether clients may ping an idle channel. Spark Connect clients do. | +| `gateway.sparkconnect.channel.idle.timeout` | `1800000` | Idle time before an unused backend channel is shut down, in ms. | +| `gateway.sparkconnect.drain.timeout` | `30000` | How long in-flight RPCs get to finish at shutdown, in ms. | +| `gateway.sparkconnect.backend.token.alias` | *(none)* | Alias holding the backend's pre-shared token (see below). | +| `gateway.sparkconnect.add.artifacts.mode` | `ALLOW` | `ALLOW`, `DENY`, or `ALLOW_LISTED_USERS` for the `AddArtifacts` RPC. | +| `gateway.sparkconnect.add.artifacts.allowed.users` | *(none)* | Comma-separated users permitted when the mode is `ALLOW_LISTED_USERS`. | +| `gateway.sparkconnect.reserved.config.prefix` | `knox.` | Session-configuration key prefix clients may not `Set` or `Unset`. | + +#### Keeping these out of `gateway-site.xml` #### + +Knox has no `conf.d` directory, but it does load one optional extra file. The +gateway reads exactly three configuration files from `{GATEWAY_HOME}/conf`, in +this order, with later files overriding earlier ones: + + gateway-default.xml + gateway-site.xml + gateway-reloadable.xml + +`gateway-reloadable.xml` is not shipped and does not have to exist, so the +`gateway.sparkconnect.*` properties can live there instead of being merged into +`gateway-site.xml`. It is a single shared file rather than a per-feature +directory, so anything else using it has to co-exist in the same file — but it +does keep this feature's settings out of the main one. See +[Reloadable Gateway Configuration](config.md) for the general mechanism. + +Knox re-reads that file every `gateway.config.refresh.interval` milliseconds +(default 10 seconds). Some of the properties above then take effect immediately; +the rest cannot, because they are built into the bound server. + +**Applied on the next RPC, no restart:** + +- `gateway.sparkconnect.default.topology` +- `gateway.sparkconnect.add.artifacts.mode` +- `gateway.sparkconnect.add.artifacts.allowed.users` +- `gateway.sparkconnect.reserved.config.prefix` + +These are the message-level and routing controls — the ones an operator is most +likely to want to change in response to something happening. Tightening artifact +gating, or reserving a different configuration prefix, applies to the very next +call without interrupting any session. + +**Restart required:** everything else — `gateway.sparkconnect.enabled` itself, +the port, TLS, message and stream limits, keepalive settings, channel idle and +drain timeouts, and the backend token alias. Whether the listener runs at all is +decided once at startup, and the rest are fixed when the socket is bound. + +Changing a restart-only property in a running gateway does not silently do +nothing. The refreshed configuration is compared against what the gateway started +with, and a warning names what could not be applied — for the transport settings, +which properties changed; for `gateway.sparkconnect.enabled`, that the listener +cannot be started or stopped without a restart. Switching it on when it was off +at startup is reported too, which is the case most likely to be mistaken for a +malfunction: without the warning, the only symptom is a port that never opens. ### Topology configuration ### @@ -109,6 +205,100 @@ Authorization uses the ordinary `AclsAuthz` provider syntax, keyed on the Group membership comes from the `knox.groups` claim in the token, so configure `knoxtoken` to include groups if you intend to write group ACLs. +### Multiple Spark Connect clusters ### + +One topology per cluster, all served by the single listener port. A topology +declares exactly one `SPARKCONNECT` backend, so a second cluster means a second +topology: + + conf/analytics.xml -> grpc://spark-analytics:15002 + conf/etl.xml -> grpc://spark-etl:15002 + +Clients pick one with the `knox-topology` connection parameter: + + sc://knox-host:15002/;use_ssl=true;token=;knox-topology=analytics + sc://knox-host:15002/;use_ssl=true;token=;knox-topology=etl + +Both connect to the *same* Knox port and are routed to different Spark clusters. +Topology selection is per-RPC, from call metadata, so concurrent sessions from +different users — or from one user — are multiplexed over that one port onto +distinct backends. Knox keeps one pooled gRPC channel per backend URL and shares +it across all calls routed there. + +This is safe for Spark Connect's session model because the discriminator is +sticky by construction: the client sends the same `knox-topology` on every +request of the connection, so every call in a session lands on the backend that +owns it — which `ReattachExecute` requires. Nothing round-robins. + +Each topology carries its own authentication provider, ACLs and audit scope, so +"separate cluster" and "separate policy boundary" stay aligned. + +#### Authorizing which users may select which cluster #### + +Topology selection is a client-supplied value, so it is authorized rather than +trusted. Authorization runs *after* routing and is evaluated against the topology +that was selected — naming a topology in a connection string is not the same as +being allowed to use it. Give each topology its own `SPARKCONNECT.acl`: + + + + SPARKCONNECT.acl + *;analysts;* + + + + + SPARKCONNECT.acl + *;engineers;* + + +An analyst connecting with `knox-topology=etl` is refused with +`PERMISSION_DENIED` before any backend connection is made. The full ACL syntax +applies per topology — named users, named groups, IP ranges, `AND`/`OR` +processing mode, and the `KNOX_ADMIN_USERS` / `KNOX_ADMIN_GROUPS` placeholders — +so selection can be gated by user name, by group, by source address, or by a +combination. + +Two things to be deliberate about: + +- **The default is permissive.** A topology that declares no `SPARKCONNECT.acl`, + or whose `AclsAuthz` provider is disabled, is reachable by *any* authenticated + user. This matches the servlet provider's behaviour, but it means restricting + selection is something you switch on, not something you get for free. If a + cluster should be reachable by a subset of your users, it needs an ACL. +- **Group ACLs need group claims.** Groups come from the token's `knox.groups` + claim, so a `knoxtoken` deployment that does not embed groups will match no + group ACL. Where groups are unavailable, gate on user names instead. + +A user probing topologies they cannot use can tell an existing Spark Connect +topology (`PERMISSION_DENIED`) from one that does not exist or serves no +`SPARKCONNECT` service (`UNAVAILABLE`). They must already hold a valid token to +learn even that, but do not treat topology names as secrets. + +What is *not* supported is several backends **within** one topology. Spark Connect +sessions are server-side state keyed by `(user_id, session_id)`, so spreading one +topology across backends needs session-affine routing rather than any form of +load balancing; that is not implemented (see Limitations). + +#### Adding a cluster without a restart #### + +Topologies are hot-reloaded. Knox watches the topologies directory and picks up +changes within about five seconds, so dropping in a new topology file, or editing +an existing one, takes effect on a running gateway: + +- **A new topology, or a changed backend URL** — takes effect on the next RPC. + The backend is resolved from the service registry per call, and redeployment + rewrites the registry entry. +- **A changed `SPARKCONNECT.acl`** — takes effect on the next RPC after the + redeployment. The listener caches parsed ACLs per topology and drops that cache + when topologies are redeployed. +- **A deleted topology** — subsequent calls selecting it fail `UNAVAILABLE`. + Calls already in flight are not interrupted. + +Only `gateway-site.xml` properties — the port, message limits, `AddArtifacts` +mode and so on — need a gateway restart, since the listener binds its socket and +captures those settings at startup. + ### Connecting ### Clients need no plugins or code changes. First acquire a token — over HTTPS, @@ -122,7 +312,7 @@ Then put it in the connection string: Two details make this work. The `token=` parameter is sent as a standard `Authorization: Bearer` header and forces TLS on. Any parameter the client does -not recognise — `knox-topology` here — is sent as gRPC metadata on every request, +not recognize — `knox-topology` here — is sent as gRPC metadata on every request, which is how a topology gets selected despite gRPC forbidding a path component in the connection URL. If you set `gateway.sparkconnect.default.topology`, the `knox-topology` parameter can be omitted. @@ -173,6 +363,61 @@ and naming it in `gateway.sparkconnect.backend.token.alias`. Knox then presents on the backend leg — and, because it strips the client's own credential there, a client cannot bypass the gateway even with network reachability. +### Making the asserted identity usable inside Spark ### + +The deployment this was built for is a single always-on Spark Connect server +behind the firewall, running as a privileged principal, with fine-grained +authorization enforced *inside* the server — typically a plan-level plugin +evaluating Ranger policies against the identity Knox asserts. Getting that +identity from the gateway into the engine takes one more step, and it is worth +being explicit about it because the gap is easy to miss. + +**The carrier of record is `user_context.user_id`.** Knox rewrites it on every +message, it keys the server-side session cache — so two users cannot share a +session by construction — and it lands in Knox's audit records. Anything else +should be *derived* from it, never asserted independently by the client. + +**But OSS Spark does not surface it to SQL.** `user_id` is used for the session +key and for logging; it is not propagated into `CurrentUserContext`, so +`current_user()` returns the Spark application's own user. A server-side +component has to bridge it. + +The robust bridge is a gRPC `ServerInterceptor` deployed with the Spark +application and registered through `spark.connect.grpc.interceptor.classes`. +That is a static configuration, so clients cannot alter it. The interceptor reads +`user_context.user_id` on each request and publishes it — by setting +`CurrentUserContext.CURRENT_USER`, which makes `current_user()` itself correct, +and/or by writing a reserved session configuration key. Whatever consumes the +identity downstream (a Ranger plugin, say) and the bridge should agree on one +mechanism rather than each inventing its own. + +One thing to verify when building such a bridge: `CurrentUserContext` is an +`InheritableThreadLocal`, and Spark Connect runs plans on dedicated execution +threads. Confirm that a value set in the interceptor is actually visible at +analysis and optimization time; if it is not, set it from a session hook on the +execution path instead. + +A weaker alternative needing no server-side code is to have the client's session +prime a reserved configuration key. Knox does **not** do this for you — it does +not inject `Config` calls — and the approach is less trustworthy than the +interceptor for the reason below. + +**Reserved keys are protected, but only on the structured path.** Knox denies +client `Set` and `Unset` on any session configuration key beginning with +`gateway.sparkconnect.reserved.config.prefix` (default `knox.`). Those are named +fields in the `Config` RPC, so the check is exact and cheap. What Knox does *not* +screen is `SET knox.whatever=...` issued as SQL inside `ExecutePlan`, which would +require inspecting plan text and would be best-effort at best. This is the main +argument for the interceptor bridge: a value recomputed from `user_context` on +every request cannot be overwritten by a session `SET` at all, whereas a +configuration key can. + +**And code execution bypasses all of it.** See the security notes above: a +plan-level plugin lives in the same JVM as user code, which runs with the +application's credentials. Plan-level enforcement is a real control among +cooperating users, and an honest audit trail; it is not a boundary against a +determined one. + ### Is this a generic gRPC gateway? ### No — and deliberately so. Knox proxies exactly one gRPC service, @@ -180,8 +425,17 @@ No — and deliberately so. Knox proxies exactly one gRPC service, points this listener at an arbitrary gRPC backend, and a call to any other proto service is answered `UNIMPLEMENTED`. -It is worth being open about what sits behind that, because anyone reading the -source will notice it: most of this feature is not Spark-specific. The listener, +It is worth saying why that is not the slippery slope it might look like. Knox's +servlet pipeline is already a generic reverse proxy: an arbitrary REST API is +proxied with a service definition and one rewrite rule, no code, and WebSocket +proxying matches any service definition by context path. gRPC was the one +protocol class outside that coverage, because it is the one the servlet stack +cannot physically carry. This closes that gap; it does not begin a pattern of +per-protocol special cases. + +It is also worth being open about what sits behind the abstraction, because +anyone reading the source will notice it: most of this feature is not +Spark-specific. The listener, TLS from the gateway identity, bearer authentication, the coarse ACL check, topology routing, backend channel caching, auditing, graceful drain and the relay itself are all protocol-agnostic — the relay in particular treats messages as @@ -231,3 +485,52 @@ gracefully, not as a general passthrough. - A gateway restart severs active streams. Clients recover through their own `ReattachExecute` retry logic, and shutdown drains for `gateway.sparkconnect.drain.timeout` first. +- **Bearer tokens only.** Neither gRPC nor the vanilla clients can carry Kerberos + on the RPC path, and the connection string exposes no client-certificate + surface, so mutual TLS from the client is not available without a non-vanilla + `channelBuilder`. +- **No gRPC-Web.** Spark Connect clients speak native gRPC; no translation layer + is provided, so browsers cannot talk to this listener directly. +- **No per-RPC metrics yet.** Audit records cover each call; the standard gateway + metrics do not yet include gRPC counters, latencies or active-stream gauges. +- **`Config` key restrictions beyond the reserved prefix, and per-RPC allow/deny + lists**, are not implemented. `AddArtifacts` gating and reserved-prefix + protection are the only message-level controls. + +### Possible future work ### + +Recorded so the reasoning is not lost; none of this is implemented or promised. + +- **Session affinity across multiple backends** — consistent hashing on + `session_id` with an in-memory affinity map. Failover semantics would stay + honest: if a backend dies its sessions die, and Knox routes the client's *new* + session to a live backend rather than pretending the old one survived. The same + mechanism with a different stickiness key (principal or group) is also the + route to per-user or per-tenant backend instances, which is what a deployment + needing genuine storage-level isolation actually wants. +- **More topology discriminators.** Two beyond `knox-topology` metadata were + designed for but not built. A **token claim** binding a topology at issuance + would make routing an authorization property — a user could not reach a + topology their token was not minted for. **Virtual-host mapping** on the HTTP/2 + `:authority` would be invisible in the connection string and immune to clients + stripping unknown parameters, but needs DNS discipline and one certificate + covering every mapped hostname; where the platform PKI cannot issue + multi-name (SAN or wildcard) certificates, a listener per topology is the + practical alternative, since every listener then presents the same hostname and + a plain single-name certificate covers them all. +- **Identity-assertion provider mapping** on this path, so the asserted principal + can be transformed the way the servlet pipeline transforms it. + +### References ### + +- Spark Connect connection string specification — + `apache/spark: sql/connect/docs/client-connection-string.md` +- Spark Connect protocol definitions — + `apache/spark: sql/connect/common/src/main/protobuf/spark/connect/` + (vendored into `gateway-service-sparkconnect`; see the README there for the + exact revision and the refresh procedure) +- PySpark `ChannelBuilder`, for how connection-string parameters become metadata — + `apache/spark: python/pyspark/sql/connect/client/core.py` +- [SPARK-51156](https://issues.apache.org/jira/browse/SPARK-51156) — the + pre-shared backend token (`spark.connect.authenticate.token`) +- [KNOX-3402](https://issues.apache.org/jira/browse/KNOX-3402) — this feature From 991e66af09fb7848a8429a5ab9b368408e9081a0 Mon Sep 17 00:00:00 2001 From: Alex Cruise Date: Thu, 30 Jul 2026 17:40:22 -0700 Subject: [PATCH 3/4] KNOX-3402 [wip]: Topology selection metadata key is configurable --- .../config/impl/GatewayConfigImpl.java | 7 ++ .../gateway/grpc/BackendHeaderRewriter.java | 7 +- .../gateway/grpc/GrpcGatewayListener.java | 42 ++++++- .../gateway/grpc/GrpcGatewayMessages.java | 12 ++ .../gateway/grpc/GrpcListenerSettings.java | 17 +++ .../knox/gateway/grpc/GrpcMetadataKeys.java | 52 ++++++++- .../knox/gateway/grpc/RoutingInterceptor.java | 14 ++- .../sparkconnect/SparkConnectListener.java | 1 + .../gateway/grpc/GrpcMetadataKeysTest.java | 109 ++++++++++++++++++ .../knox/gateway/GatewayTestConfig.java | 11 ++ .../knox/gateway/config/GatewayConfig.java | 10 ++ 11 files changed, 270 insertions(+), 12 deletions(-) create mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/GrpcMetadataKeysTest.java diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java index 08dd19f251..f35369f1ae 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java @@ -178,6 +178,7 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { public static final String SPARKCONNECT_ADD_ARTIFACTS_MODE = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.add.artifacts.mode"; public static final String SPARKCONNECT_ADD_ARTIFACTS_ALLOWED_USERS = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.add.artifacts.allowed.users"; public static final String SPARKCONNECT_RESERVED_CONFIG_PREFIX = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.reserved.config.prefix"; + public static final String SPARKCONNECT_TOPOLOGY_METADATA_KEY = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.topology.metadata.key"; /* @since 2.0.0 WebShell config variables */ @@ -256,6 +257,7 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { public static final long DEFAULT_SPARKCONNECT_DRAIN_TIMEOUT = 30000L; public static final String DEFAULT_SPARKCONNECT_ADD_ARTIFACTS_MODE = "ALLOW"; public static final String DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX = "knox."; + public static final String DEFAULT_SPARKCONNECT_TOPOLOGY_METADATA_KEY = "knox-topology"; public static final boolean DEFAULT_WEBSHELL_FEATURE_ENABLED = false; public static final boolean DEFAULT_WEBSHELL_AUDIT_LOGGING_ENABLED = false; @@ -1213,6 +1215,11 @@ public String getSparkConnectReservedConfigPrefix() { return get(SPARKCONNECT_RESERVED_CONFIG_PREFIX, DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX); } + @Override + public String getSparkConnectTopologyMetadataKey() { + return get(SPARKCONNECT_TOPOLOGY_METADATA_KEY, DEFAULT_SPARKCONNECT_TOPOLOGY_METADATA_KEY); + } + @Override public Map getGatewayPortMappings() { diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java index c6f9116d97..8d84bc00ab 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java @@ -43,6 +43,7 @@ public class BackendHeaderRewriter implements HeaderRewriter { private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); private final String backendAuthorization; + private final Metadata.Key topologyKey; /** * @param aliasService used to resolve the backend token; may be null when no @@ -50,8 +51,10 @@ public class BackendHeaderRewriter implements HeaderRewriter { * @param backendTokenAlias the alias holding the backend's pre-shared token, or * null if the backend requires no token */ - public BackendHeaderRewriter(AliasService aliasService, String backendTokenAlias) { + public BackendHeaderRewriter(AliasService aliasService, String backendTokenAlias, + Metadata.Key topologyKey) { this.backendAuthorization = resolveBackendToken(aliasService, backendTokenAlias); + this.topologyKey = topologyKey; } private static String resolveBackendToken(AliasService aliasService, String alias) { @@ -74,7 +77,7 @@ private static String resolveBackendToken(AliasService aliasService, String alia @Override public void rewrite(Metadata headers) { headers.removeAll(GrpcMetadataKeys.AUTHORIZATION); - headers.removeAll(GrpcMetadataKeys.TOPOLOGY); + headers.removeAll(topologyKey); if (backendAuthorization != null) { headers.put(GrpcMetadataKeys.AUTHORIZATION, backendAuthorization); } diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayListener.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayListener.java index 6e26ec6325..887b3837f3 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayListener.java +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayListener.java @@ -35,7 +35,11 @@ import org.apache.knox.gateway.services.ServiceType; import org.apache.knox.gateway.services.security.AliasService; import org.apache.knox.gateway.services.security.KeystoreService; +import org.apache.knox.gateway.services.topology.TopologyService; +import org.apache.knox.gateway.topology.Service; +import org.apache.knox.gateway.topology.Topology; +import io.grpc.Metadata; import io.grpc.Server; import io.grpc.ServerInterceptor; import io.grpc.ServerMethodDefinition; @@ -144,9 +148,14 @@ public void start(GatewayConfig config, GatewayServices services) throws Excepti return channels.getChannel(callContext.getBackendUrl()); }; + // Built once, and validated here rather than on the first call: a bad key + // name should stop the gateway starting, not surprise the first user. + final Metadata.Key topologyKey = + GrpcMetadataKeys.topologyKey(listenerSettings.getTopologyMetadataKey()); + final AliasService aliasService = services.getService(ServiceType.ALIAS_SERVICE); final HeaderRewriter headerRewriter = - new BackendHeaderRewriter(aliasService, listenerSettings.getBackendTokenAlias()); + new BackendHeaderRewriter(aliasService, listenerSettings.getBackendTokenAlias(), topologyKey); this.authorizationInterceptor = new AuthorizationInterceptor(config, services, getServiceRole()); @@ -156,7 +165,7 @@ public void start(GatewayConfig config, GatewayServices services) throws Excepti final List interceptors = Arrays.asList( new AuditInterceptor(), new AuthenticationInterceptor(new TokenAuthenticator(config, services)), - new RoutingInterceptor(config, services, getServiceRole()), + new RoutingInterceptor(config, services, getServiceRole(), topologyKey), authorizationInterceptor); final NettyServerBuilder builder = NettyServerBuilder.forPort(listenerSettings.getPort()) @@ -190,6 +199,35 @@ public void start(GatewayConfig config, GatewayServices services) throws Excepti throw e; } LOG.startedListener(listenerSettings.getName(), getPort()); + warnIfNoTopologyDeclaresTheRole(listenerSettings, services); + } + + /** + * Notes, at debug level, that the listener is running with nothing to route to. + *

+ * Enabling the listener and declaring a backend are separate steps in separate + * files, so it is possible to do the first and forget the second — but it is + * equally possible to do the first deliberately and wait. A deployment that + * enables the listener as a matter of course, and adds a topology only when + * someone provisions a Spark Connect cluster, is in this state normally and + * perhaps permanently. That is why this is debug rather than a warning: it + * helps when someone is asking why calls are refused, without nagging every + * deployment that is simply waiting. + */ + private void warnIfNoTopologyDeclaresTheRole(GrpcListenerSettings listenerSettings, + GatewayServices services) { + final TopologyService topologyService = services.getService(ServiceType.TOPOLOGY_SERVICE); + if (topologyService == null) { + return; + } + for (Topology topology : topologyService.getTopologies()) { + for (Service service : topology.getServices()) { + if (getServiceRole().equals(service.getRole())) { + return; + } + } + } + LOG.noTopologyDeclaresService(listenerSettings.getName(), getServiceRole()); } /** diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java index d13388d7c6..3f69c2ea40 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java @@ -79,6 +79,18 @@ public interface GrpcGatewayMessages { @Message(level = MessageLevel.WARN, text = "Could not resolve the backend token alias {0}") void missingBackendTokenAlias(String alias); + // DEBUG, not WARN: a listener enabled ahead of any backend is a legitimate + // steady state. Deployments that switch the listener on by default and create a + // topology only when someone provisions a cluster would otherwise carry a + // warning forever, which is how warnings stop being read. The actionable signal + // for a genuine misconfiguration is the per-call rejection, which names the + // missing configuration directly. + @Message(level = MessageLevel.DEBUG, + text = "The {0} listener is running but no deployed topology declares a {1} service, " + + "so calls will be rejected until one does. Add a {1}" + + "... to a topology; topologies are picked up without a restart.") + void noTopologyDeclaresService(String name, String role); + @Message(level = MessageLevel.INFO, text = "Reloaded the {0} listener message policy: {1}") void reloadedPolicy(String name, String policy); diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java index 8791701fc7..3847e441e9 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java @@ -41,6 +41,7 @@ public class GrpcListenerSettings { private long channelIdleTimeoutMillis = 1800000L; private long drainTimeoutMillis = 30000L; private String backendTokenAlias; + private String topologyMetadataKey = GrpcMetadataKeys.DEFAULT_TOPOLOGY_KEY; public String getName() { return name; @@ -114,6 +115,22 @@ public GrpcListenerSettings drainTimeoutMillis(long value) { return this; } + /** + * The metadata entry a client uses to select a topology. It is also the + * connection-string parameter users write, so a deployment may prefer a name + * that describes the choice rather than the gateway making it. + * + * @return the metadata key name + */ + public String getTopologyMetadataKey() { + return topologyMetadataKey; + } + + public GrpcListenerSettings topologyMetadataKey(String value) { + this.topologyMetadataKey = value; + return this; + } + public String getBackendTokenAlias() { return backendTokenAlias; } diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java index b71c06bfbe..255944b1d6 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java @@ -17,6 +17,9 @@ */ package org.apache.knox.gateway.grpc; +import java.util.Locale; +import java.util.regex.Pattern; + import io.grpc.Metadata; /** @@ -34,12 +37,55 @@ public final class GrpcMetadataKeys { public static final Metadata.Key AUTHORIZATION = Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER); - /** Selects the topology, set by a {@code knox-topology=} connection-string parameter. */ - public static final Metadata.Key TOPOLOGY = - Metadata.Key.of("knox-topology", Metadata.ASCII_STRING_MARSHALLER); + /** + * The default name of the metadata entry that selects a topology. Deployments + * can rename it, since it appears verbatim in the connection strings users + * write and need not advertise which gateway is reading it. + */ + public static final String DEFAULT_TOPOLOGY_KEY = "knox-topology"; + + private static final Pattern VALID_KEY = Pattern.compile("[a-z0-9_.-]+"); + private static final String BINARY_SUFFIX = "-bin"; public static final String BEARER_PREFIX = "Bearer "; private GrpcMetadataKeys() { } + + /** + * Builds the metadata key used to select a topology. + *

+ * gRPC restricts header names to lowercase ASCII letters, digits and + * {@code -_.}, and reserves the {@code -bin} suffix for binary values. An + * invalid name would otherwise surface as an obscure failure from deep inside + * the transport, so it is rejected here with an explanation instead. + * + * @param name the configured metadata key name + * @return the metadata key to read topology selection from + * @throws IllegalArgumentException if the name is not usable as a gRPC metadata key + */ + public static Metadata.Key topologyKey(String name) { + if (name == null || name.trim().isEmpty()) { + throw new IllegalArgumentException( + "The topology metadata key name must not be empty"); + } + final String trimmed = name.trim(); + if (!trimmed.equals(trimmed.toLowerCase(Locale.ROOT))) { + throw new IllegalArgumentException( + "gRPC metadata key names are case-insensitive and must be given in lower case: " + name); + } + if (!VALID_KEY.matcher(trimmed).matches()) { + throw new IllegalArgumentException( + "The topology metadata key name may contain only a-z, 0-9, '-', '_' and '.': " + name); + } + if (trimmed.endsWith(BINARY_SUFFIX)) { + throw new IllegalArgumentException( + "gRPC reserves the '-bin' suffix for binary metadata; the topology key carries text: " + name); + } + if (AUTHORIZATION.name().equals(trimmed)) { + throw new IllegalArgumentException( + "The topology metadata key must not be 'authorization', which carries the bearer token"); + } + return Metadata.Key.of(trimmed, Metadata.ASCII_STRING_MARSHALLER); + } } diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java index ea0d8b7804..10bb52f6ba 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java +++ b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java @@ -39,7 +39,8 @@ * therefore has to come from something else a vanilla client can send. Two * discriminators are supported: *

    - *
  1. a {@code knox-topology} metadata entry, which the client supplies as an + *
  2. a metadata entry, named by configuration and {@code knox-topology} by + * default, which the client supplies as an * extra {@code sc://} connection-string parameter;
    3. *
  3. the configured default topology, for the single-topology case.
    4. *
@@ -60,11 +61,14 @@ public class RoutingInterceptor implements ServerInterceptor { private final GatewayConfig config; private final GatewayServices services; private final String serviceRole; + private final Metadata.Key topologyKey; - public RoutingInterceptor(GatewayConfig config, GatewayServices services, String serviceRole) { + public RoutingInterceptor(GatewayConfig config, GatewayServices services, String serviceRole, + Metadata.Key topologyKey) { this.config = config; this.services = services; this.serviceRole = serviceRole; + this.topologyKey = topologyKey; } @Override @@ -76,8 +80,8 @@ public ServerCall.Listener interceptCall(ServerCall ServerCall.Listener interceptCall(ServerCall + * gRPC is strict about header names, so a bad one is rejected at startup with an + * explanation rather than surfacing later from deep inside the transport. + */ +public class GrpcMetadataKeysTest { + + @Test + public void acceptsTheDefault() { + assertEquals(GrpcMetadataKeys.DEFAULT_TOPOLOGY_KEY, + GrpcMetadataKeys.topologyKey(GrpcMetadataKeys.DEFAULT_TOPOLOGY_KEY).name()); + } + + @Test + public void acceptsANeutralNameThatDoesNotMentionTheGateway() { + // The point of making this configurable. + assertEquals("cluster", GrpcMetadataKeys.topologyKey("cluster").name()); + assertEquals("workspace", GrpcMetadataKeys.topologyKey("workspace").name()); + assertEquals("x-compute-target", GrpcMetadataKeys.topologyKey("x-compute-target").name()); + } + + @Test + public void trimsSurroundingWhitespace() { + assertEquals("cluster", GrpcMetadataKeys.topologyKey(" cluster ").name()); + } + + @Test + public void rejectsAnEmptyName() { + assertRejected(null, "must not be empty"); + assertRejected("", "must not be empty"); + assertRejected(" ", "must not be empty"); + } + + @Test + public void rejectsUpperCase() { + // gRPC lower-cases header names, so an upper-case configuration value would + // never match what arrives; say so rather than silently never matching. + assertRejected("Knox-Topology", "lower case"); + } + + @Test + public void rejectsCharactersGrpcDoesNotAllowInHeaderNames() { + assertRejected("knox topology", "may contain only"); + assertRejected("knox:topology", "may contain only"); + assertRejected("knox/topology", "may contain only"); + } + + @Test + public void rejectsTheBinarySuffixReservedByGrpc() { + assertRejected("topology-bin", "-bin"); + } + + @Test + public void rejectsCollidingWithTheBearerTokenHeader() { + assertRejected("authorization", "authorization"); + } + + @Test + public void theMissingTopologyNoticeIsDebugNotAWarning() throws Exception { + // A listener enabled ahead of any backend is a legitimate steady state, and a + // deployment that provisions clusters on demand would otherwise carry a + // warning forever. Pinned so it cannot drift back to WARN unnoticed. + final Message message = GrpcGatewayMessages.class + .getMethod("noTopologyDeclaresService", String.class, String.class) + .getAnnotation(Message.class); + assertEquals(MessageLevel.DEBUG, message.level()); + } + + private static void assertRejected(String name, String expectedFragment) { + try { + GrpcMetadataKeys.topologyKey(name); + fail("Expected " + name + " to be rejected"); + } catch (IllegalArgumentException e) { + assertTrue("message should explain the problem, was: " + e.getMessage(), + e.getMessage().contains(expectedFragment)); + } + } +} diff --git a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java index 05a0caf827..6f1794436f 100644 --- a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java +++ b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java @@ -64,6 +64,7 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig { public static final long DEFAULT_SPARKCONNECT_DRAIN_TIMEOUT = 30000L; public static final String DEFAULT_SPARKCONNECT_ADD_ARTIFACTS_MODE = "ALLOW"; public static final String DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX = "knox."; + public static final String DEFAULT_SPARKCONNECT_TOPOLOGY_METADATA_KEY = "knox-topology"; private boolean sparkConnectEnabled; private int sparkConnectPort = DEFAULT_SPARKCONNECT_PORT; @@ -71,6 +72,7 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig { private String sparkConnectBackendTokenAlias; private String sparkConnectAddArtifactsMode = DEFAULT_SPARKCONNECT_ADD_ARTIFACTS_MODE; private String sparkConnectReservedConfigPrefix = DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX; + private String sparkConnectTopologyMetadataKey = DEFAULT_SPARKCONNECT_TOPOLOGY_METADATA_KEY; private List sparkConnectAddArtifactsAllowedUsers = Collections.emptyList(); @@ -792,6 +794,15 @@ public void setSparkConnectReservedConfigPrefix(String sparkConnectReservedConfi this.sparkConnectReservedConfigPrefix = sparkConnectReservedConfigPrefix; } + @Override + public String getSparkConnectTopologyMetadataKey() { + return sparkConnectTopologyMetadataKey; + } + + public void setSparkConnectTopologyMetadataKey(String sparkConnectTopologyMetadataKey) { + this.sparkConnectTopologyMetadataKey = sparkConnectTopologyMetadataKey; + } + @Override public boolean isMetricsEnabled() { return false; diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java index 404f8a43c3..47dfb53dbf 100644 --- a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java +++ b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java @@ -659,6 +659,16 @@ public interface GatewayConfig { */ String getSparkConnectReservedConfigPrefix(); + /** + * The name of the call-metadata entry a client uses to select a topology, + * which is also the connection-string parameter users write. Configurable so a + * deployment can choose a name that suits its users rather than one that names + * the gateway reading it. + * @since 3.0.0 + * @return the metadata key name + */ + String getSparkConnectTopologyMetadataKey(); + boolean isMetricsEnabled(); boolean isJmxMetricsReportingEnabled(); From 5633bb7cdf14c201a322f7c3ea5727db8dd55e94 Mon Sep 17 00:00:00 2001 From: Alex Cruise Date: Tue, 4 Aug 2026 12:04:50 -0700 Subject: [PATCH 4/4] KNOX-3402 [wip]: generic gRPC support, not Spark Connect specific --- .github/workflows/build/Dockerfile | 1 + .../conf/topologies/sparkconnect-fgac.xml | 55 ++ .github/workflows/build/gateway-site.xml | 29 +- .github/workflows/compose/docker-compose.yml | 2 +- .github/workflows/tests/test_spark_connect.py | 80 +- LICENSE | 2 +- gateway-release/pom.xml | 2 +- .../apache/knox/gateway/GatewayServer.java | 17 +- .../config/impl/GatewayConfigImpl.java | 166 ++-- .../pom.xml | 24 +- .../knox/gateway/grpc/AclAuthorizer.java | 4 +- .../knox/gateway/grpc/AuditInterceptor.java | 0 .../knox/gateway/grpc/AuthenticatedUser.java | 0 .../grpc/AuthenticationInterceptor.java | 10 +- .../grpc/AuthorizationInterceptor.java | 0 .../gateway/grpc/BackendChannelCache.java | 10 +- .../gateway/grpc/BackendChannelProvider.java | 0 .../gateway/grpc/BackendHeaderRewriter.java | 9 +- .../gateway/grpc/ByteArrayMarshaller.java | 0 .../knox/gateway/grpc/GrpcCallContext.java | 6 +- .../knox/gateway/grpc/GrpcEndpoint.java | 515 ++++++++++ .../gateway/grpc/GrpcGatewayMessages.java | 29 + .../knox/gateway/grpc/GrpcListener.java | 176 ++++ .../gateway/grpc/GrpcListenerSettings.java | 295 ++++++ .../grpc/GrpcListenerSettingsFactory.java | 215 +++++ .../knox/gateway/grpc/GrpcMetadataKeys.java | 0 .../knox/gateway/grpc/HeaderRewriter.java | 0 .../grpc/IdentityAssertingInterceptor.java | 233 +++++ .../gateway/grpc/IdentityRewritePolicy.java | 199 ++++ .../gateway/grpc/IdentityRewriteRule.java | 127 +++ .../knox/gateway/grpc/IdentitySubject.java | 73 ++ .../knox/gateway/grpc/InterceptorChain.java | 0 .../knox/gateway/grpc/MapFilterConfig.java | 0 .../knox/gateway/grpc/MessageInterceptor.java | 12 + .../grpc/MessageInterceptorFactory.java | 40 + .../gateway/grpc/MethodAccessInterceptor.java | 129 +++ .../knox/gateway/grpc/MethodAccessPolicy.java | 118 +++ .../apache/knox/gateway/grpc/ProtoWire.java | 248 +++++ .../knox/gateway/grpc/ProxyCallHandler.java | 11 +- .../gateway/grpc/ProxyHandlerRegistry.java | 98 ++ .../knox/gateway/grpc/RoutingInterceptor.java | 33 +- .../knox/gateway/grpc/TokenAuthenticator.java | 14 +- ...che.knox.gateway.protocol.ProtocolListener | 2 +- .../knox/gateway/grpc/AclAuthorizerTest.java | 0 .../grpc/AuthenticationInterceptorTest.java | 0 .../AuthorizationInterceptorReloadTest.java | 0 .../grpc/GrpcListenerSettingsFactoryTest.java | 201 ++++ .../gateway/grpc/GrpcMetadataKeysTest.java | 0 .../grpc/IdentityRewritePolicyTest.java | 237 +++++ .../gateway/grpc/MethodAccessPolicyTest.java | 102 ++ .../TopologySelectionAuthorizationTest.java | 0 .../IdentityAssertionOracleTest.java | 333 +++++++ .../sparkconnect/SparkConnectProxyTest.java | 78 +- .../src/test}/proto/spark/connect/README.md | 0 .../src/test}/proto/spark/connect/base.proto | 0 .../test}/proto/spark/connect/catalog.proto | 0 .../test}/proto/spark/connect/commands.proto | 0 .../test}/proto/spark/connect/common.proto | 0 .../proto/spark/connect/expressions.proto | 0 .../src/test}/proto/spark/connect/ml.proto | 0 .../test}/proto/spark/connect/ml_common.proto | 0 .../test}/proto/spark/connect/pipelines.proto | 0 .../test}/proto/spark/connect/relations.proto | 0 .../src/test}/proto/spark/connect/types.proto | 0 .../gateway/grpc/GrpcGatewayListener.java | 359 ------- .../gateway/grpc/GrpcListenerSettings.java | 142 --- .../grpc/PassthroughHandlerRegistry.java | 87 -- .../sparkconnect/AddArtifactsGuard.java | 90 -- .../sparkconnect/ReservedConfigGuard.java | 123 --- .../sparkconnect/SparkConnectListener.java | 242 ----- .../SparkConnectMessageInterceptor.java | 164 ---- .../sparkconnect/SparkConnectPolicy.java | 91 -- .../sparkconnect/AddArtifactsGuardTest.java | 89 -- .../sparkconnect/ReservedConfigGuardTest.java | 110 --- .../SparkConnectListenerConfigReloadTest.java | 170 ---- .../SparkConnectMessageInterceptorTest.java | 209 ----- .../knox/gateway/GatewayTestConfig.java | 177 ++-- .../knox/gateway/config/GatewayConfig.java | 181 ++-- .../gateway/protocol/ProtocolListener.java | 13 + knox-site/docs/grpc-support.md | 878 ++++++++++++++++++ knox-site/docs/spark-connect-support.md | 536 ----------- knox-site/mkdocs.yml | 2 +- pom.xml | 6 +- 83 files changed, 4862 insertions(+), 2742 deletions(-) create mode 100644 .github/workflows/build/conf/topologies/sparkconnect-fgac.xml rename {gateway-service-sparkconnect => gateway-service-grpc}/pom.xml (88%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/AclAuthorizer.java (97%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/AuditInterceptor.java (100%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/AuthenticatedUser.java (100%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java (92%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/AuthorizationInterceptor.java (100%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/BackendChannelCache.java (93%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/BackendChannelProvider.java (100%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java (91%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/ByteArrayMarshaller.java (100%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/GrpcCallContext.java (96%) create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcEndpoint.java rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java (77%) create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcListener.java create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettingsFactory.java rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java (100%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/HeaderRewriter.java (100%) create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentityAssertingInterceptor.java create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentityRewritePolicy.java create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentityRewriteRule.java create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentitySubject.java rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/InterceptorChain.java (100%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/MapFilterConfig.java (100%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptor.java (84%) create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptorFactory.java create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MethodAccessInterceptor.java create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MethodAccessPolicy.java create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ProtoWire.java rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/ProxyCallHandler.java (96%) create mode 100644 gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ProxyHandlerRegistry.java rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java (78%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/java/org/apache/knox/gateway/grpc/TokenAuthenticator.java (94%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/main/resources/META-INF/services/org.apache.knox.gateway.protocol.ProtocolListener (94%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/test/java/org/apache/knox/gateway/grpc/AclAuthorizerTest.java (100%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/test/java/org/apache/knox/gateway/grpc/AuthenticationInterceptorTest.java (100%) rename {gateway-service-sparkconnect => gateway-service-grpc}/src/test/java/org/apache/knox/gateway/grpc/AuthorizationInterceptorReloadTest.java (100%) create mode 100644 gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/GrpcListenerSettingsFactoryTest.java rename {gateway-service-sparkconnect => gateway-service-grpc}/src/test/java/org/apache/knox/gateway/grpc/GrpcMetadataKeysTest.java (100%) create mode 100644 gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/IdentityRewritePolicyTest.java create mode 100644 gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/MethodAccessPolicyTest.java rename {gateway-service-sparkconnect => gateway-service-grpc}/src/test/java/org/apache/knox/gateway/grpc/TopologySelectionAuthorizationTest.java (100%) create mode 100644 gateway-service-grpc/src/test/java/org/apache/knox/gateway/sparkconnect/IdentityAssertionOracleTest.java rename gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectProxyIntegrationTest.java => gateway-service-grpc/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectProxyTest.java (83%) rename {gateway-service-sparkconnect/src/main => gateway-service-grpc/src/test}/proto/spark/connect/README.md (100%) rename {gateway-service-sparkconnect/src/main => gateway-service-grpc/src/test}/proto/spark/connect/base.proto (100%) rename {gateway-service-sparkconnect/src/main => gateway-service-grpc/src/test}/proto/spark/connect/catalog.proto (100%) rename {gateway-service-sparkconnect/src/main => gateway-service-grpc/src/test}/proto/spark/connect/commands.proto (100%) rename {gateway-service-sparkconnect/src/main => gateway-service-grpc/src/test}/proto/spark/connect/common.proto (100%) rename {gateway-service-sparkconnect/src/main => gateway-service-grpc/src/test}/proto/spark/connect/expressions.proto (100%) rename {gateway-service-sparkconnect/src/main => gateway-service-grpc/src/test}/proto/spark/connect/ml.proto (100%) rename {gateway-service-sparkconnect/src/main => gateway-service-grpc/src/test}/proto/spark/connect/ml_common.proto (100%) rename {gateway-service-sparkconnect/src/main => gateway-service-grpc/src/test}/proto/spark/connect/pipelines.proto (100%) rename {gateway-service-sparkconnect/src/main => gateway-service-grpc/src/test}/proto/spark/connect/relations.proto (100%) rename {gateway-service-sparkconnect/src/main => gateway-service-grpc/src/test}/proto/spark/connect/types.proto (100%) delete mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayListener.java delete mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java delete mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/PassthroughHandlerRegistry.java delete mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuard.java delete mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuard.java delete mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectListener.java delete mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptor.java delete mode 100644 gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectPolicy.java delete mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuardTest.java delete mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuardTest.java delete mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectListenerConfigReloadTest.java delete mode 100644 gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptorTest.java create mode 100644 knox-site/docs/grpc-support.md delete mode 100644 knox-site/docs/spark-connect-support.md diff --git a/.github/workflows/build/Dockerfile b/.github/workflows/build/Dockerfile index 8e2c8c0b46..51df76c162 100644 --- a/.github/workflows/build/Dockerfile +++ b/.github/workflows/build/Dockerfile @@ -45,6 +45,7 @@ ADD .github/workflows/build/conf/topologies/remoteauth.xml /knox-runtime/conf/to ADD .github/workflows/build/conf/topologies/k8sauth.xml /knox-runtime/conf/topologies/k8sauth.xml ADD .github/workflows/build/conf/topologies/sparkconnect.xml /knox-runtime/conf/topologies/sparkconnect.xml ADD .github/workflows/build/conf/topologies/sparkconnect-restricted.xml /knox-runtime/conf/topologies/sparkconnect-restricted.xml +ADD .github/workflows/build/conf/topologies/sparkconnect-fgac.xml /knox-runtime/conf/topologies/sparkconnect-fgac.xml RUN chown -R gateway /knox-runtime/ diff --git a/.github/workflows/build/conf/topologies/sparkconnect-fgac.xml b/.github/workflows/build/conf/topologies/sparkconnect-fgac.xml new file mode 100644 index 0000000000..b81d284065 --- /dev/null +++ b/.github/workflows/build/conf/topologies/sparkconnect-fgac.xml @@ -0,0 +1,55 @@ + + + + + + federation + JWTProvider + true + + knox.token.use.cookie + false + + + + authorization + AclsAuthz + true + + SPARKCONNECT.acl + *;*;* + + + SPARKCONNECT.methods.deny + AddArtifacts + + + + + SPARKCONNECT + grpc://sparkconnect-mock:15002 + + diff --git a/.github/workflows/build/gateway-site.xml b/.github/workflows/build/gateway-site.xml index 7da2274ec4..301920e434 100644 --- a/.github/workflows/build/gateway-site.xml +++ b/.github/workflows/build/gateway-site.xml @@ -212,17 +212,32 @@ limitations under the License. member - + - gateway.sparkconnect.enabled + gateway.grpc.enabled true - - gateway.sparkconnect.add.artifacts.mode - DENY + gateway.grpc.service.role + SPARKCONNECT + + + gateway.grpc.proto.services + spark.connect.SparkConnectService + + + gateway.grpc.identity.rules + 2.1=principal,2.2=principal + + diff --git a/.github/workflows/compose/docker-compose.yml b/.github/workflows/compose/docker-compose.yml index fa6ef8e9e8..067fef20ef 100644 --- a/.github/workflows/compose/docker-compose.yml +++ b/.github/workflows/compose/docker-compose.yml @@ -107,7 +107,7 @@ services: touch /out/spark/__init__.py /out/spark/connect/__init__.py echo 'spark connect stubs generated' volumes: - - ../../../gateway-service-sparkconnect/src/main/proto:/protos:ro + - ../../../gateway-service-grpc/src/test/proto:/protos:ro - sparkconnect-protos:/out # Stands in for a Spark Connect server on a private network. Plaintext, which diff --git a/.github/workflows/tests/test_spark_connect.py b/.github/workflows/tests/test_spark_connect.py index 6cdc3b5fd9..5f4b9ef578 100644 --- a/.github/workflows/tests/test_spark_connect.py +++ b/.github/workflows/tests/test_spark_connect.py @@ -47,6 +47,8 @@ TOPOLOGY_METADATA_KEY = "knox-topology" OPEN_TOPOLOGY = "sparkconnect" RESTRICTED_TOPOLOGY = "sparkconnect-restricted" +# Any user may reach it, but AddArtifacts is denied by method name. +FGAC_TOPOLOGY = "sparkconnect-fgac" # Present in the demo LDAP the compose environment starts. KNOX_USER = "guest" KNOX_PASSWORD = "guest-password" @@ -220,36 +222,54 @@ def test_execute_plan_relays_every_response(self): self.assertEqual(KNOX_USER, responses[0].operation_id) -class TestSparkConnectMessageGating(SparkConnectTestBase): - """Per-RPC gating is enforced at the gateway, before the backend.""" - - def test_add_artifacts_is_denied_when_configured_to_deny(self): - """Artifact upload gating is enforced at the gateway.""" - def upload(): - channel, metadata = self._channel(token=self.token, topology=OPEN_TOPOLOGY) - with channel: - stub = base_pb2_grpc.SparkConnectServiceStub(channel) - request = base_pb2.AddArtifactsRequest(session_id="itest-artifacts") - request.user_context.user_id = "root" - return stub.AddArtifacts(iter([request]), metadata=metadata, timeout=30) - - # gateway.sparkconnect.add.artifacts.mode is DENY in gateway-site.xml. - self.assert_rpc_code(grpc.StatusCode.PERMISSION_DENIED, upload) - - def test_reserved_config_key_cannot_be_set_by_a_client(self): - """Clients cannot write the session keys Knox reserves for itself.""" - def set_reserved(): - channel, metadata = self._channel(token=self.token, topology=OPEN_TOPOLOGY) - with channel: - stub = base_pb2_grpc.SparkConnectServiceStub(channel) - request = base_pb2.ConfigRequest(session_id="itest-reserved") - request.user_context.user_id = "root" - pair = request.operation.set.pairs.add() - pair.key = "knox.principal" - pair.value = "root" - return stub.Config(request, metadata=metadata, timeout=30) - - self.assert_rpc_code(grpc.StatusCode.PERMISSION_DENIED, set_reserved) +class TestSparkConnectMethodGating(SparkConnectTestBase): + """Whole RPCs can be refused by name, which needs no message parsing.""" + + def _upload(self, topology): + """Attempts an AddArtifacts call against the given topology.""" + channel, metadata = self._channel(token=self.token, topology=topology) + with channel: + stub = base_pb2_grpc.SparkConnectServiceStub(channel) + request = base_pb2.AddArtifactsRequest(session_id="itest-artifacts") + request.user_context.user_id = "root" + return stub.AddArtifacts(iter([request]), metadata=metadata, timeout=30) + + def test_add_artifacts_is_denied_in_a_topology_that_denies_it(self): + """A topology relying on plan-level policy can refuse code upload.""" + # SPARKCONNECT.methods.deny in sparkconnect-fgac.xml; the gateway reads + # only the method name from the request path to decide this. + self.assert_rpc_code(grpc.StatusCode.PERMISSION_DENIED, + lambda: self._upload(FGAC_TOPOLOGY)) + + def test_add_artifacts_is_permitted_where_it_is_not_denied(self): + """The same user and RPC succeed in a topology with no such rule.""" + self._upload(OPEN_TOPOLOGY) + + def test_other_rpcs_still_work_in_the_denying_topology(self): + """Denying one method must not disturb the rest of the service.""" + response = self._analyze(token=self.token, topology=FGAC_TOPOLOGY) + self.assertEqual(KNOX_USER, response.explain.explain_string) + + def test_config_rpc_still_carries_the_asserted_identity(self): + """Config is relayed like any other RPC, with the identity replaced. + + Knox no longer screens session configuration keys: the gateway reads only + the identity fields, by field number, and makes no assumption about the + Config RPC's internal shape. Protecting a reserved key is the job of a + component inside the backend, which can recompute the identity per + request rather than trusting a key a client could also write. + """ + channel, metadata = self._channel(token=self.token, topology=OPEN_TOPOLOGY) + with channel: + stub = base_pb2_grpc.SparkConnectServiceStub(channel) + request = base_pb2.ConfigRequest(session_id="itest-config-identity") + request.user_context.user_id = "root" + pair = request.operation.set.pairs.add() + pair.key = "spark.sql.shuffle.partitions" + pair.value = "8" + response = stub.Config(request, metadata=metadata, timeout=30) + # The mock echoes back the user_id it received. + self.assertEqual(KNOX_USER, response.pairs[0].value) if __name__ == "__main__": diff --git a/LICENSE b/LICENSE index eb40047c15..829e828115 100644 --- a/LICENSE +++ b/LICENSE @@ -1381,7 +1381,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -Protocol Buffers License (BSD 3-clause) (from Spark Connect support) +Protocol Buffers License (BSD 3-clause) ------------------------------------------------------------------------------ Copyright 2008 Google Inc. All rights reserved. diff --git a/gateway-release/pom.xml b/gateway-release/pom.xml index 06bb8d0e4f..c978146d7a 100644 --- a/gateway-release/pom.xml +++ b/gateway-release/pom.xml @@ -526,7 +526,7 @@ org.apache.knox - gateway-service-sparkconnect + gateway-service-grpc diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java index 487579683d..ac8be7aac8 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/GatewayServer.java @@ -135,6 +135,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.apache.knox.gateway.config.impl.GatewayConfigImpl.RELOADABLE_CONFIG_FILENAME; @@ -292,7 +293,15 @@ private static synchronized void refreshGatewayConfig(GatewayConfigImpl config, config.reloadConfiguration(); log.refreshedGatewayConfig(); for (GatewayConfigChangeListener listener : configChangeListeners) { - listener.onGatewayConfigChanged(config); + try { + listener.onGatewayConfigChanged(config); + } catch (Exception e) { + // This runs on a scheduleAtFixedRate task, where an escaping + // exception cancels every future execution. One listener choking on + // a bad value must not silently stop configuration refresh for the + // whole gateway. + log.unableToReloadGatewayConfig(e); + } } } } @@ -840,7 +849,11 @@ private void startProtocolListeners() throws Exception { if (listener instanceof GatewayConfigChangeListener) { registerConfigChangeListener((GatewayConfigChangeListener) listener); } - log.startedProtocolListener(listener.getName(), convertPortToString(listener.getPort())); + // A listener may bind several ports; report all of them, since a + // deployment that configured N endpoints wants to see N came up. + log.startedProtocolListener(listener.getName(), listener.getPorts().stream() + .map(GatewayServer::convertPortToString) + .collect(Collectors.joining(", "))); } catch (Exception e) { log.failedToStartProtocolListener(listener.getName(), e); throw e; diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java index f35369f1ae..5e98a5818d 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java @@ -164,21 +164,23 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { public static final String WEBSOCKET_IDLE_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".websocket.idle.timeout"; public static final String WEBSOCKET_MAX_WAIT_BUFFER_COUNT = GATEWAY_CONFIG_FILE_PREFIX + ".websocket.max.wait.buffer.count"; - /* @since 3.0.0 Spark Connect (gRPC) listener config variables */ - public static final String SPARKCONNECT_FEATURE_ENABLED = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.enabled"; - public static final String SPARKCONNECT_PORT = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.port"; - public static final String SPARKCONNECT_DEFAULT_TOPOLOGY = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.default.topology"; - public static final String SPARKCONNECT_MAX_MESSAGE_SIZE = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.max.message.size"; - public static final String SPARKCONNECT_PERMIT_KEEPALIVE_TIME = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.permit.keepalive.time"; - public static final String SPARKCONNECT_PERMIT_KEEPALIVE_WITHOUT_CALLS = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.permit.keepalive.without.calls"; - public static final String SPARKCONNECT_MAX_CONCURRENT_CALLS_PER_CONNECTION = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.max.concurrent.calls.per.connection"; - public static final String SPARKCONNECT_CHANNEL_IDLE_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.channel.idle.timeout"; - public static final String SPARKCONNECT_DRAIN_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.drain.timeout"; - public static final String SPARKCONNECT_BACKEND_TOKEN_ALIAS = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.backend.token.alias"; - public static final String SPARKCONNECT_ADD_ARTIFACTS_MODE = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.add.artifacts.mode"; - public static final String SPARKCONNECT_ADD_ARTIFACTS_ALLOWED_USERS = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.add.artifacts.allowed.users"; - public static final String SPARKCONNECT_RESERVED_CONFIG_PREFIX = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.reserved.config.prefix"; - public static final String SPARKCONNECT_TOPOLOGY_METADATA_KEY = GATEWAY_CONFIG_FILE_PREFIX + ".sparkconnect.topology.metadata.key"; + /* @since 3.0.0 gRPC listener config variables */ + public static final String GRPC_FEATURE_ENABLED = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.enabled"; + public static final String GRPC_PORT = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.port"; + public static final String GRPC_SERVICE_ROLE = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.service.role"; + public static final String GRPC_IDENTITY_RULES = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.identity.rules"; + public static final String GRPC_IDENTITY_SCAN_LIMIT = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.identity.scan.limit"; + public static final String GRPC_DEFAULT_TOPOLOGY = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.default.topology"; + public static final String GRPC_TOPOLOGY_METADATA_KEY = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.topology.metadata.key"; + public static final String GRPC_METHODS_DENY = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.methods.deny"; + public static final String GRPC_METHODS_ALLOW = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.methods.allow"; + public static final String GRPC_MAX_MESSAGE_SIZE = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.max.message.size"; + public static final String GRPC_PERMIT_KEEPALIVE_TIME = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.permit.keepalive.time"; + public static final String GRPC_PERMIT_KEEPALIVE_WITHOUT_CALLS = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.permit.keepalive.without.calls"; + public static final String GRPC_MAX_CONCURRENT_CALLS_PER_CONNECTION = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.max.concurrent.calls.per.connection"; + public static final String GRPC_CHANNEL_IDLE_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.channel.idle.timeout"; + public static final String GRPC_DRAIN_TIMEOUT = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.drain.timeout"; + public static final String GRPC_BACKEND_TOKEN_ALIAS = GATEWAY_CONFIG_FILE_PREFIX + ".grpc.backend.token.alias"; /* @since 2.0.0 WebShell config variables */ @@ -243,21 +245,20 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { public static final int DEFAULT_WEBSOCKET_IDLE_TIMEOUT = 300000; public static final int DEFAULT_WEBSOCKET_MAX_WAIT_BUFFER_COUNT = 100; - /* Spark Connect defaults */ - public static final boolean DEFAULT_SPARKCONNECT_FEATURE_ENABLED = false; - /* The port the Spark Connect server itself listens on; clients default to it too. */ - public static final int DEFAULT_SPARKCONNECT_PORT = 15002; - /* Matches Spark's own 128 MB default. */ - public static final int DEFAULT_SPARKCONNECT_MAX_MESSAGE_SIZE = 134217728; - /* grpc-java's server-side floor; clients ping every 60s by default. */ - public static final long DEFAULT_SPARKCONNECT_PERMIT_KEEPALIVE_TIME = 10000L; - public static final boolean DEFAULT_SPARKCONNECT_PERMIT_KEEPALIVE_WITHOUT_CALLS = true; - public static final int DEFAULT_SPARKCONNECT_MAX_CONCURRENT_CALLS_PER_CONNECTION = 1000; - public static final long DEFAULT_SPARKCONNECT_CHANNEL_IDLE_TIMEOUT = 1800000L; - public static final long DEFAULT_SPARKCONNECT_DRAIN_TIMEOUT = 30000L; - public static final String DEFAULT_SPARKCONNECT_ADD_ARTIFACTS_MODE = "ALLOW"; - public static final String DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX = "knox."; - public static final String DEFAULT_SPARKCONNECT_TOPOLOGY_METADATA_KEY = "knox-topology"; + /* gRPC listener defaults. The port and identity layout come from Spark + Connect, the protocol this was first built for. */ + public static final boolean DEFAULT_GRPC_FEATURE_ENABLED = false; + public static final int DEFAULT_GRPC_PORT = 15002; + public static final String DEFAULT_GRPC_SERVICE_ROLE = "GRPC"; + /** 128 KiB; see IdentityRewritePolicy for why the rewrite is bounded at all. */ + public static final int DEFAULT_GRPC_IDENTITY_SCAN_LIMIT = 131072; + public static final int DEFAULT_GRPC_MAX_MESSAGE_SIZE = 134217728; + public static final long DEFAULT_GRPC_PERMIT_KEEPALIVE_TIME = 10000L; + public static final boolean DEFAULT_GRPC_PERMIT_KEEPALIVE_WITHOUT_CALLS = true; + public static final int DEFAULT_GRPC_MAX_CONCURRENT_CALLS_PER_CONNECTION = 1000; + public static final long DEFAULT_GRPC_CHANNEL_IDLE_TIMEOUT = 1800000L; + public static final long DEFAULT_GRPC_DRAIN_TIMEOUT = 30000L; + public static final String DEFAULT_GRPC_TOPOLOGY_METADATA_KEY = "knox-topology"; public static final boolean DEFAULT_WEBSHELL_FEATURE_ENABLED = false; public static final boolean DEFAULT_WEBSHELL_AUDIT_LOGGING_ENABLED = false; @@ -1147,77 +1148,118 @@ public int getWebsocketMaxWaitBufferCount() { } @Override - public boolean isSparkConnectEnabled() { - return getBoolean(SPARKCONNECT_FEATURE_ENABLED, DEFAULT_SPARKCONNECT_FEATURE_ENABLED); + public boolean isGrpcEnabled() { + return getBoolean(GRPC_FEATURE_ENABLED, DEFAULT_GRPC_FEATURE_ENABLED); } @Override - public int getSparkConnectPort() { - return getInt(SPARKCONNECT_PORT, DEFAULT_SPARKCONNECT_PORT); + public int getGrpcPort() { + return getInt(GRPC_PORT, DEFAULT_GRPC_PORT); } @Override - public String getSparkConnectDefaultTopology() { - return get(SPARKCONNECT_DEFAULT_TOPOLOGY); + public String getGrpcServiceRole() { + return get(GRPC_SERVICE_ROLE, DEFAULT_GRPC_SERVICE_ROLE); } @Override - public int getSparkConnectMaxMessageSize() { - return getInt(SPARKCONNECT_MAX_MESSAGE_SIZE, DEFAULT_SPARKCONNECT_MAX_MESSAGE_SIZE); + public List getGrpcListenerNames() { + final String configured = get(GRPC_LISTENER_NAMES); + if (configured == null || configured.trim().isEmpty()) { + return Collections.emptyList(); + } + final List names = new ArrayList<>(); + for (String name : configured.trim().split("\\s*,\\s*")) { + if (!name.isEmpty()) { + names.add(name); + } + } + return names; } @Override - public long getSparkConnectPermitKeepAliveTime() { - return getLong(SPARKCONNECT_PERMIT_KEEPALIVE_TIME, DEFAULT_SPARKCONNECT_PERMIT_KEEPALIVE_TIME); + public Map getGrpcListenerConfig(String listenerName) { + final Map listenerConfig = new HashMap<>(); + final String prefix = GATEWAY_CONFIG_FILE_PREFIX + ".grpc." + listenerName + "."; + for (String key : getPropertyNames()) { + if (key != null && key.startsWith(prefix)) { + final String value = get(key); + if (value != null) { + listenerConfig.put(key.substring(prefix.length()), value); + } + } + } + return listenerConfig; } @Override - public boolean isSparkConnectPermitKeepAliveWithoutCalls() { - return getBoolean(SPARKCONNECT_PERMIT_KEEPALIVE_WITHOUT_CALLS, DEFAULT_SPARKCONNECT_PERMIT_KEEPALIVE_WITHOUT_CALLS); + public String getGrpcProtoServices() { + return get(GRPC_PROTO_SERVICES); } @Override - public int getSparkConnectMaxConcurrentCallsPerConnection() { - return getInt(SPARKCONNECT_MAX_CONCURRENT_CALLS_PER_CONNECTION, DEFAULT_SPARKCONNECT_MAX_CONCURRENT_CALLS_PER_CONNECTION); + public String getGrpcIdentityRules() { + return get(GRPC_IDENTITY_RULES); } @Override - public long getSparkConnectChannelIdleTimeout() { - return getLong(SPARKCONNECT_CHANNEL_IDLE_TIMEOUT, DEFAULT_SPARKCONNECT_CHANNEL_IDLE_TIMEOUT); + public int getGrpcIdentityScanLimit() { + return getInt(GRPC_IDENTITY_SCAN_LIMIT, DEFAULT_GRPC_IDENTITY_SCAN_LIMIT); } @Override - public long getSparkConnectDrainTimeout() { - return getLong(SPARKCONNECT_DRAIN_TIMEOUT, DEFAULT_SPARKCONNECT_DRAIN_TIMEOUT); + public String getGrpcDefaultTopology() { + return get(GRPC_DEFAULT_TOPOLOGY); } @Override - public String getSparkConnectBackendTokenAlias() { - return get(SPARKCONNECT_BACKEND_TOKEN_ALIAS); + public String getGrpcTopologyMetadataKey() { + return get(GRPC_TOPOLOGY_METADATA_KEY, DEFAULT_GRPC_TOPOLOGY_METADATA_KEY); } @Override - public String getSparkConnectAddArtifactsMode() { - return get(SPARKCONNECT_ADD_ARTIFACTS_MODE, DEFAULT_SPARKCONNECT_ADD_ARTIFACTS_MODE); + public String getGrpcMethodsDeny() { + return get(GRPC_METHODS_DENY); } @Override - public List getSparkConnectAddArtifactsAllowedUsers() { - final String value = get(SPARKCONNECT_ADD_ARTIFACTS_ALLOWED_USERS); - if (value == null || value.trim().isEmpty()) { - return Collections.emptyList(); - } - return Arrays.asList(value.trim().split("\\s*,\\s*")); + public String getGrpcMethodsAllow() { + return get(GRPC_METHODS_ALLOW); + } + + @Override + public int getGrpcMaxMessageSize() { + return getInt(GRPC_MAX_MESSAGE_SIZE, DEFAULT_GRPC_MAX_MESSAGE_SIZE); + } + + @Override + public long getGrpcPermitKeepAliveTime() { + return getLong(GRPC_PERMIT_KEEPALIVE_TIME, DEFAULT_GRPC_PERMIT_KEEPALIVE_TIME); + } + + @Override + public boolean isGrpcPermitKeepAliveWithoutCalls() { + return getBoolean(GRPC_PERMIT_KEEPALIVE_WITHOUT_CALLS, DEFAULT_GRPC_PERMIT_KEEPALIVE_WITHOUT_CALLS); + } + + @Override + public int getGrpcMaxConcurrentCallsPerConnection() { + return getInt(GRPC_MAX_CONCURRENT_CALLS_PER_CONNECTION, DEFAULT_GRPC_MAX_CONCURRENT_CALLS_PER_CONNECTION); + } + + @Override + public long getGrpcChannelIdleTimeout() { + return getLong(GRPC_CHANNEL_IDLE_TIMEOUT, DEFAULT_GRPC_CHANNEL_IDLE_TIMEOUT); } @Override - public String getSparkConnectReservedConfigPrefix() { - return get(SPARKCONNECT_RESERVED_CONFIG_PREFIX, DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX); + public long getGrpcDrainTimeout() { + return getLong(GRPC_DRAIN_TIMEOUT, DEFAULT_GRPC_DRAIN_TIMEOUT); } @Override - public String getSparkConnectTopologyMetadataKey() { - return get(SPARKCONNECT_TOPOLOGY_METADATA_KEY, DEFAULT_SPARKCONNECT_TOPOLOGY_METADATA_KEY); + public String getGrpcBackendTokenAlias() { + return get(GRPC_BACKEND_TOKEN_ALIAS); } @Override diff --git a/gateway-service-sparkconnect/pom.xml b/gateway-service-grpc/pom.xml similarity index 88% rename from gateway-service-sparkconnect/pom.xml rename to gateway-service-grpc/pom.xml index 2f6ab8a356..778c7fdf18 100644 --- a/gateway-service-sparkconnect/pom.xml +++ b/gateway-service-grpc/pom.xml @@ -25,9 +25,9 @@ 3.0.0-SNAPSHOT - gateway-service-sparkconnect - gateway-service-sparkconnect - Spark Connect (gRPC) gateway listener for Apache Knox + gateway-service-grpc + gateway-service-grpc + gRPC listener for Apache Knox, proxying protobuf services without compiling against their schemas @@ -71,10 +71,12 @@ io.grpc grpc-stub + test io.grpc grpc-protobuf + test @@ -82,14 +84,17 @@ io.grpc grpc-netty-shaded + com.google.protobuf protobuf-java + test - com.google.guava guava + test org.apache.commons @@ -157,8 +162,11 @@ - compile - compile-custom + + test-compile + test-compile-custom @@ -169,8 +177,8 @@ - ${project.build.directory}/generated-sources/protobuf/java - ${project.build.directory}/generated-sources/protobuf/grpc-java + ${project.build.directory}/generated-test-sources/protobuf/java + ${project.build.directory}/generated-test-sources/protobuf/grpc-java diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AclAuthorizer.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/AclAuthorizer.java similarity index 97% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AclAuthorizer.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/AclAuthorizer.java index bbf3adf0bc..5a7bad6295 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AclAuthorizer.java +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/AclAuthorizer.java @@ -34,8 +34,8 @@ * Knox's authorization responsibility on this path is deliberately one question: * may this user use this service in this topology at all? Fine-grained * authorization — databases, tables, columns, row filters, masking — belongs to - * Ranger policy evaluated inside the Spark Connect server against the identity - * Knox asserts, and is not something a gateway can usefully duplicate. + * policy evaluated inside the backend against the identity Knox asserts, and is + * not something a gateway can usefully duplicate. *

* The syntax and semantics are the servlet provider's, down to sharing its * {@link AclParser}: {@code users;groups;ipaddresses}, an {@code AND}/{@code OR} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuditInterceptor.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/AuditInterceptor.java similarity index 100% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuditInterceptor.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/AuditInterceptor.java diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticatedUser.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/AuthenticatedUser.java similarity index 100% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticatedUser.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/AuthenticatedUser.java diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java similarity index 92% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java index 14fbfb142c..8f65cb32a1 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/AuthenticationInterceptor.java @@ -29,15 +29,15 @@ * Rejects any call that does not present a valid Knox bearer token. *

* Authentication happens before a backend channel is opened, so an - * unauthenticated request never reaches Spark — which matters because the OSS - * Spark Connect server has essentially no authentication of its own and assumes - * a fronting proxy provides it. + * unauthenticated request never reaches the backend — which matters because the + * services this fronts commonly have little or no authentication of their own + * and assume a proxy provides it. *

* Tokens are checked when an RPC starts and not again while it runs. A * multi-hour {@code ExecutePlan} is therefore not severed the moment its token * expires; the next RPC fails instead. Cutting off long queries at expiry would - * punish precisely the workloads Spark Connect exists to serve, and Spark's own - * session timeout still bounds how long a session survives. + * punish precisely the workloads these protocols exist to serve, and the + * backend's own session timeout still bounds how long a session survives. */ public class AuthenticationInterceptor implements ServerInterceptor { diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthorizationInterceptor.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/AuthorizationInterceptor.java similarity index 100% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/AuthorizationInterceptor.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/AuthorizationInterceptor.java diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelCache.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/BackendChannelCache.java similarity index 93% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelCache.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/BackendChannelCache.java index a84f7bd223..3511e1b4ce 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelCache.java +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/BackendChannelCache.java @@ -45,7 +45,7 @@ * gRPC channels multiplex concurrent calls over a pooled HTTP/2 connection and * are designed to be long-lived, so creating one per RPC would be both slower * and wasteful of connections. Channels go idle on their own after - * {@code gateway.sparkconnect.channel.idle.timeout} and reconnect transparently + * {@code gateway.grpc.channel.idle.timeout} and reconnect transparently * when used again, so a cached entry for an unused backend costs nothing. */ public class BackendChannelCache { @@ -84,7 +84,7 @@ private ManagedChannel createChannel(String backendUrl) { if (host == null || port < 0) { throw Status.FAILED_PRECONDITION - .withDescription("Spark Connect backend URL must include a host and port: " + backendUrl) + .withDescription("The backend URL must include a host and port: " + backendUrl) .asRuntimeException(); } @@ -99,7 +99,7 @@ private ManagedChannel createChannel(String backendUrl) { } catch (Exception e) { LOG.failedToBuildBackendTls(backendUrl, e); throw Status.UNAVAILABLE - .withDescription("Cannot establish TLS to the Spark Connect backend") + .withDescription("Cannot establish TLS to the backend") .withCause(e) .asRuntimeException(); } @@ -107,7 +107,7 @@ private ManagedChannel createChannel(String backendUrl) { builder.negotiationType(NegotiationType.PLAINTEXT); } else { throw Status.FAILED_PRECONDITION - .withDescription("Spark Connect backend URL scheme must be grpc:// or grpcs://, got: " + backendUrl) + .withDescription("The backend URL scheme must be grpc:// or grpcs://, got: " + backendUrl) .asRuntimeException(); } @@ -138,7 +138,7 @@ private static URI parse(String backendUrl) { return new URI(backendUrl); } catch (URISyntaxException e) { throw Status.FAILED_PRECONDITION - .withDescription("Malformed Spark Connect backend URL: " + backendUrl) + .withDescription("Malformed backend URL: " + backendUrl) .withCause(e) .asRuntimeException(); } diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelProvider.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/BackendChannelProvider.java similarity index 100% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendChannelProvider.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/BackendChannelProvider.java diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java similarity index 91% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java index 8d84bc00ab..f890c7ed2b 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/BackendHeaderRewriter.java @@ -27,11 +27,10 @@ * Replaces the client's credentials with Knox's own on the backend leg. *

* The client's bearer token proves the user's identity to Knox and has no - * meaning beyond it, so it is removed rather than forwarded. In its place, if - * the backend is configured with a pre-shared token - * ({@code spark.connect.authenticate.token}), Knox presents that. Besides - * authenticating the gateway to Spark, it closes the hole where a client with - * network reachability to the backend port could simply bypass the gateway + * meaning beyond it, so it is removed rather than forwarded. In its place, if a + * pre-shared backend token is configured, Knox presents that. Besides + * authenticating the gateway to the backend, it closes the hole where a client + * with network reachability to the backend port could simply bypass the gateway * altogether — network restrictions should prevent that too, but a credential * the client does not hold makes it structural rather than topological. *

diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ByteArrayMarshaller.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ByteArrayMarshaller.java similarity index 100% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ByteArrayMarshaller.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ByteArrayMarshaller.java diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcCallContext.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcCallContext.java similarity index 96% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcCallContext.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcCallContext.java index 8cab010a06..b330cee1c4 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcCallContext.java +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcCallContext.java @@ -123,9 +123,9 @@ public void setBackendUrl(String backendUrl) { } /** - * The Spark Connect session this call belongs to, once a request message has - * been parsed. Null on the generic byte-level path, which never looks inside - * messages. + * The session this call belongs to, where the protocol has such a notion and a + * request message has been parsed. Null when requests are relayed without + * inspection. * * @return the session id, or null */ diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcEndpoint.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcEndpoint.java new file mode 100644 index 0000000000..fdf16989de --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcEndpoint.java @@ -0,0 +1,515 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.Key; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.Certificate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.KeyManagerFactory; + +import org.apache.knox.gateway.config.GatewayConfig; +import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.services.GatewayServices; +import org.apache.knox.gateway.services.ServiceType; +import org.apache.knox.gateway.services.security.AliasService; +import org.apache.knox.gateway.services.security.KeystoreService; +import org.apache.knox.gateway.services.topology.TopologyService; +import org.apache.knox.gateway.topology.Service; +import org.apache.knox.gateway.topology.Topology; + +import io.grpc.Metadata; +import io.grpc.Server; +import io.grpc.ServerInterceptor; +import io.grpc.Status; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder; + +/** + * One gRPC listener: a Netty server on one port, with its own TLS identity, its + * own view of what to proxy, and its own backend channels. + *

+ * A gateway runs one of these per configured listener. They share the gateway's + * services — tokens, topologies, audit — and route to the same topologies; what + * distinguishes them is the socket and the certificate presented on it. See + * {@link GrpcListenerSettingsFactory} for why that separation is worth having. + */ +// volatile: lifecycle and policy fields are written by the thread calling +// start/stop or delivering a configuration change, and read by request threads. +@SuppressWarnings("PMD.AvoidUsingVolatile") +public class GrpcEndpoint { + + private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); + + private final GrpcListenerSettings settings; + + private volatile Server server; + private volatile BackendChannelCache channelCache; + private volatile AuthorizationInterceptor authorizationInterceptor; + private volatile MethodAccessInterceptor methodAccessInterceptor; + /** + * Rebuilt when configuration changes. The relay reads this per message rather + * than capturing it, so a change reaches handlers that already exist. + */ + private volatile MessageInterceptor messageInterceptor = MessageInterceptor.passthrough(); + /** Read per call, so a changed default topology applies without a restart. */ + private volatile String defaultTopology; + /** The configuration the running interceptor was built from, for change detection. */ + private volatile String identityRules; + private volatile int identityScanLimit; + + public GrpcEndpoint(GrpcListenerSettings settings) { + this.settings = settings; + this.defaultTopology = settings.getDefaultTopology(); + this.identityRules = settings.getIdentityRules(); + this.identityScanLimit = settings.getIdentityScanLimit(); + } + + public String getName() { + return settings.getName(); + } + + public GrpcListenerSettings getSettings() { + return settings; + } + + public int getPort() { + final Server current = server; + return current == null ? -1 : current.getPort(); + } + + /** + * Binds the port and begins serving. + * + * @param config the gateway configuration, for the services shared across + * listeners: token validation, admin users, keystores + * @param services the started gateway services + * @throws Exception if the listener cannot start + */ + public void start(GatewayConfig config, GatewayServices services) throws Exception { + if (settings.getProtoServices().isEmpty()) { + // Refusing to start beats binding a port that answers UNIMPLEMENTED to + // everything, which would look like a working listener. + throw new IllegalStateException("The gRPC listener '" + getName() + "' is enabled but " + + GrpcListenerSettingsFactory.propertyName(null, "proto.services") + + " names no proto service to proxy"); + } + + // Parsed here rather than on the first call: a malformed rule must stop the + // gateway starting, not silently leave identity assertion switched off. + final IdentityRewritePolicy identityPolicy = createPolicy(settings); + this.messageInterceptor = createMessageInterceptor(identityPolicy); + + final BackendChannelCache channels = new BackendChannelCache(settings, services); + this.channelCache = channels; + + final BackendChannelProvider channelProvider = () -> { + final GrpcCallContext callContext = GrpcCallContext.current(); + if (callContext == null || callContext.getBackendUrl() == null) { + throw Status.UNAVAILABLE.withDescription("No backend resolved for this call").asRuntimeException(); + } + return channels.getChannel(callContext.getBackendUrl()); + }; + + // Built once, and validated here rather than on the first call: a bad key + // name should stop the gateway starting, not surprise the first user. + final Metadata.Key topologyKey = + GrpcMetadataKeys.topologyKey(settings.getTopologyMetadataKey()); + + final AliasService aliasService = services.getService(ServiceType.ALIAS_SERVICE); + final HeaderRewriter headerRewriter = + new BackendHeaderRewriter(aliasService, settings.getBackendTokenAlias(), topologyKey); + + final String serviceRole = settings.getServiceRole(); + this.authorizationInterceptor = new AuthorizationInterceptor(config, services, serviceRole); + this.methodAccessInterceptor = new MethodAccessInterceptor(services, serviceRole, + MethodAccessPolicy.of(settings.getMethodsDeny(), settings.getMethodsAllow())); + + // Order is load-bearing: audit wraps everything so even rejected calls are + // recorded, then identity, then topology selection, then the checks that + // depend on both having succeeded. + final List interceptors = Arrays.asList( + new AuditInterceptor(), + new AuthenticationInterceptor(new TokenAuthenticator(config, services)), + new RoutingInterceptor(() -> defaultTopology, services, serviceRole, topologyKey), + authorizationInterceptor, + // Coarse method gating needs no schema: gRPC carries the method name in + // the request path. It runs last so a denial is attributable to a known + // user in a known topology. + methodAccessInterceptor); + + final NettyServerBuilder builder = NettyServerBuilder.forPort(settings.getPort()) + .maxInboundMessageSize(settings.getMaxMessageSize()) + .maxConcurrentCallsPerConnection(settings.getMaxConcurrentCallsPerConnection()) + .permitKeepAliveTime(settings.getPermitKeepAliveTimeMillis(), TimeUnit.MILLISECONDS) + .permitKeepAliveWithoutCalls(settings.isPermitKeepAliveWithoutCalls()); + + if (settings.isSslEnabled()) { + builder.sslContext(buildServerSslContext(config, services)); + } else { + // Clients that carry a bearer token generally require TLS anyway, so this + // is really a test and development posture; say so rather than let it pass. + LOG.listenerTlsDisabled(getName()); + } + + // No generated service is registered. Every call for a proxied proto service + // reaches the same byte-level relay, and the relay consults the current + // interceptor per message. + final MessageInterceptor currentInterceptor = message -> messageInterceptor.intercept(message); + builder.fallbackHandlerRegistry(new ProxyHandlerRegistry( + settings.getProtoServices(), + methodName -> currentInterceptor, + relay -> InterceptorChain.intercept( + new ProxyCallHandler<>(channelProvider, relay, headerRewriter), interceptors))); + + try { + this.server = builder.build().start(); + } catch (Exception e) { + LOG.failedToStartListener(getName(), e); + channels.shutdown(0L); + this.channelCache = null; + throw e; + } + LOG.startedListener(getName(), getPort()); + LOG.proxyingServices(getName(), String.join(", ", settings.getProtoServices()), + identityPolicy.toString()); + noteIfNoTopologyDeclaresTheRole(services, serviceRole); + } + + private static IdentityRewritePolicy createPolicy(GrpcListenerSettings settings) { + return IdentityRewritePolicy.parse(settings.getIdentityRules(), settings.getIdentityScanLimit()); + } + + /** + * Identity assertion is optional: a protocol whose requests carry no identity + * field gets a pure relay. Where rules are given, every request has each named + * field replaced with the authenticated principal. + */ + private static MessageInterceptor createMessageInterceptor(IdentityRewritePolicy policy) { + return policy.isEmpty() + ? MessageInterceptor.passthrough() + : new IdentityAssertingInterceptor(policy); + } + + /** + * Notes, at debug level, that this listener is running with nothing to route to. + *

+ * Enabling a listener and declaring a backend are separate steps in separate + * files, so it is possible to do the first and forget the second — but it is + * equally possible to do the first deliberately and wait. A deployment that + * enables a listener as a matter of course, and adds a topology only when + * someone provisions a backend, is in this state normally and perhaps + * permanently. That is why this is debug rather than a warning: it helps when + * someone is asking why calls are refused, without nagging every deployment + * that is simply waiting. + */ + private void noteIfNoTopologyDeclaresTheRole(GatewayServices services, String serviceRole) { + final TopologyService topologyService = services.getService(ServiceType.TOPOLOGY_SERVICE); + if (topologyService == null) { + return; + } + for (Topology topology : topologyService.getTopologies()) { + for (Service service : topology.getServices()) { + if (serviceRole.equals(service.getRole())) { + return; + } + } + } + LOG.noTopologyDeclaresService(getName(), serviceRole); + } + + /** + * Builds this listener's TLS context. + *

+ * By default that is the gateway identity — the same key material Jetty + * presents — so a deployment has one certificate to manage, not several. A + * listener that configures its own keystore presents that instead, which is + * what allows several listeners on one gateway to answer for several hostnames + * with plain single-name certificates. + *

+ * Either way the chosen entry is copied into a single-entry keystore before the + * key manager is built, so the configured alias is the one presented even when + * the source keystore holds others. + */ + private SslContext buildServerSslContext(GatewayConfig config, GatewayServices services) + throws Exception { + try { + final AliasService aliasService = services.getService(ServiceType.ALIAS_SERVICE); + final KeyStore source; + final String alias; + final char[] passphrase; + + if (settings.getSslKeystorePath() == null) { + final KeystoreService keystoreService = services.getService(ServiceType.KEYSTORE_SERVICE); + source = keystoreService.getKeystoreForGateway(); + if (source == null) { + throw new IllegalStateException("The gateway identity keystore is not available"); + } + alias = config.getIdentityKeyAlias(); + passphrase = aliasService.getGatewayIdentityPassphrase(); + } else { + passphrase = keystorePassphrase(aliasService); + source = loadKeystore(settings.getSslKeystorePath(), settings.getSslKeystoreType(), passphrase); + alias = keyEntryAlias(source); + } + + final Key key = source.getKey(alias, passphrase); + final Certificate[] chain = source.getCertificateChain(alias); + if (!(key instanceof PrivateKey) || chain == null || chain.length == 0) { + throw new IllegalStateException("The keystore for gRPC listener '" + getName() + + "' has no usable key entry for alias " + alias); + } + + final KeyStore identity = KeyStore.getInstance("PKCS12"); + identity.load(null, null); + identity.setKeyEntry(alias, key, passphrase, chain); + + final KeyManagerFactory keyManagers = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagers.init(identity, passphrase); + + LOG.listenerTlsIdentity(getName(), + settings.getSslKeystorePath() == null ? "the gateway identity" : settings.getSslKeystorePath(), + alias); + + // GrpcSslContexts applies the ALPN and cipher requirements of the HTTP/2 + // profile gRPC mandates. + return GrpcSslContexts.configure(SslContextBuilder.forServer(keyManagers)).build(); + } catch (Exception e) { + LOG.failedToBuildServerTls(getName(), e); + throw e; + } + } + + /** + * The keystore password, from the configured alias. Falling back to the gateway + * identity passphrase matches what the embedded LDAP server does, and covers + * the common case of a keystore provisioned alongside the gateway's own. + */ + private char[] keystorePassphrase(AliasService aliasService) throws Exception { + final String alias = settings.getSslKeystorePasswordAlias(); + if (alias == null) { + return aliasService.getGatewayIdentityPassphrase(); + } + final char[] password = aliasService.getPasswordFromAliasForGateway(alias); + if (password == null || password.length == 0) { + throw new IllegalStateException("The keystore password alias '" + alias + + "' configured for gRPC listener '" + getName() + "' resolves to nothing"); + } + return password; + } + + private KeyStore loadKeystore(String path, String type, char[] passphrase) throws Exception { + if (!Files.isReadable(Paths.get(path))) { + throw new IllegalStateException("The keystore configured for gRPC listener '" + getName() + + "' cannot be read: " + path); + } + final KeyStore keystore = KeyStore.getInstance(type); + try (InputStream in = Files.newInputStream(Paths.get(path))) { + keystore.load(in, passphrase); + } catch (IOException e) { + throw new IllegalStateException("The keystore configured for gRPC listener '" + getName() + + "' could not be loaded; check its type and password: " + path, e); + } + return keystore; + } + + /** + * The entry to present. A keystore holding exactly one key entry needs no alias + * configured, which is the usual shape of a per-listener keystore; anything + * else has to say which, because picking arbitrarily would present a + * certificate nobody chose. + */ + private String keyEntryAlias(KeyStore keystore) throws Exception { + final String configured = settings.getSslKeystoreAlias(); + if (configured != null) { + if (!keystore.containsAlias(configured)) { + throw new IllegalStateException("The keystore for gRPC listener '" + getName() + + "' holds no entry named " + configured); + } + return configured; + } + final List keyEntries = new ArrayList<>(); + final Enumeration aliases = keystore.aliases(); + while (aliases.hasMoreElements()) { + final String candidate = aliases.nextElement(); + if (keystore.isKeyEntry(candidate)) { + keyEntries.add(candidate); + } + } + if (keyEntries.size() == 1) { + return keyEntries.get(0); + } + throw new IllegalStateException("The keystore for gRPC listener '" + getName() + "' holds " + + keyEntries.size() + " key entries, so " + + GrpcListenerSettingsFactory.propertyName(getName(), "ssl.keystore.alias") + + " must name the one to present"); + } + + /** + * Stops accepting new calls and lets in-flight ones finish, up to the + * configured drain timeout. + *

+ * Long-lived streams are severed if they outlast the drain. Clients of + * streaming protocols generally recover, since such protocols usually carry + * their own reattach or retry mechanism for exactly this case. + */ + public void stop() { + final Server current = server; + if (current == null) { + return; + } + final long drainTimeoutMillis = settings.getDrainTimeoutMillis(); + LOG.stoppingListener(getName(), drainTimeoutMillis); + current.shutdown(); + try { + if (!current.awaitTermination(drainTimeoutMillis, TimeUnit.MILLISECONDS)) { + LOG.drainTimedOut(getName(), drainTimeoutMillis); + current.shutdownNow(); + } + } catch (InterruptedException e) { + current.shutdownNow(); + Thread.currentThread().interrupt(); + } finally { + server = null; + final BackendChannelCache channels = channelCache; + if (channels != null) { + channels.shutdown(drainTimeoutMillis); + channelCache = null; + } + LOG.stoppedListener(getName()); + } + } + + /** Drops cached per-topology policy so a redeployed topology takes effect. */ + public void reload() { + final AuthorizationInterceptor authz = authorizationInterceptor; + if (authz != null) { + authz.invalidate(); + } + final MethodAccessInterceptor methods = methodAccessInterceptor; + if (methods != null) { + methods.invalidate(); + } + } + + /** + * Applies a changed {@code gateway-reloadable.xml} to the controls that can + * move on a running listener. + *

+ * Only the identity rewrite and the default topology are refreshed. The + * transport settings are built into the bound server and cannot change without + * a restart, so rather than accept them silently and do nothing — which looks + * like it worked — any attempt to change one is named in the log. + * + * @param updated the settings the refreshed configuration implies for this + * listener + */ + public void onSettingsChanged(GrpcListenerSettings updated) { + if (!Objects.equals(updated.getDefaultTopology(), defaultTopology)) { + this.defaultTopology = updated.getDefaultTopology(); + LOG.reloadedPolicy(getName(), "default topology: " + + (defaultTopology == null ? "none" : defaultTopology)); + } + if (!Objects.equals(updated.getIdentityRules(), identityRules) + || updated.getIdentityScanLimit() != identityScanLimit) { + // Record the new configuration either way, so a rule that cannot be parsed + // is reported once rather than on every refresh; correcting it changes the + // value again and is picked up normally. + this.identityRules = updated.getIdentityRules(); + this.identityScanLimit = updated.getIdentityScanLimit(); + try { + final IdentityRewritePolicy policy = createPolicy(updated); + this.messageInterceptor = createMessageInterceptor(policy); + LOG.reloadedPolicy(getName(), "identity rules: " + policy); + } catch (RuntimeException e) { + // Keep the running policy. Switching identity assertion off because + // someone mistyped a rule is the one outcome worse than ignoring the + // edit, and throwing here would escape into the configuration refresh + // task and stop it running again. + LOG.invalidIdentityRules(getName(), String.valueOf(updated.getIdentityRules()), e); + } + } + warnAboutRestartOnlyChanges(updated); + } + + private void warnAboutRestartOnlyChanges(GrpcListenerSettings updated) { + final List changed = new ArrayList<>(); + if (updated.getPort() != settings.getPort()) { + changed.add("port"); + } + if (!Objects.equals(updated.getServiceRole(), settings.getServiceRole())) { + changed.add("service.role"); + } + if (!Objects.equals(updated.getProtoServices(), settings.getProtoServices())) { + changed.add("proto.services"); + } + if (updated.getMaxMessageSize() != settings.getMaxMessageSize()) { + changed.add("max.message.size"); + } + if (updated.getMaxConcurrentCallsPerConnection() != settings.getMaxConcurrentCallsPerConnection()) { + changed.add("max.concurrent.calls.per.connection"); + } + if (updated.getPermitKeepAliveTimeMillis() != settings.getPermitKeepAliveTimeMillis()) { + changed.add("permit.keepalive.time"); + } + if (updated.isPermitKeepAliveWithoutCalls() != settings.isPermitKeepAliveWithoutCalls()) { + changed.add("permit.keepalive.without.calls"); + } + if (updated.getChannelIdleTimeoutMillis() != settings.getChannelIdleTimeoutMillis()) { + changed.add("channel.idle.timeout"); + } + if (updated.getDrainTimeoutMillis() != settings.getDrainTimeoutMillis()) { + changed.add("drain.timeout"); + } + if (!Objects.equals(updated.getBackendTokenAlias(), settings.getBackendTokenAlias())) { + changed.add("backend.token.alias"); + } + if (!Objects.equals(updated.getTopologyMetadataKey(), settings.getTopologyMetadataKey())) { + changed.add("topology.metadata.key"); + } + if (updated.isSslEnabled() != settings.isSslEnabled() + || !Objects.equals(updated.getSslKeystorePath(), settings.getSslKeystorePath()) + || !Objects.equals(updated.getSslKeystoreAlias(), settings.getSslKeystoreAlias()) + || !Objects.equals(updated.getSslKeystorePasswordAlias(), settings.getSslKeystorePasswordAlias()) + || !Objects.equals(updated.getSslKeystoreType(), settings.getSslKeystoreType())) { + changed.add("ssl.*"); + } + if (!changed.isEmpty()) { + LOG.restartOnlyConfigChanged(getName(), String.join(", ", changed)); + } + } + + /** Exposed so tests can drive a configuration change without binding a port. */ + MessageInterceptor currentMessageInterceptor() { + return messageInterceptor; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java similarity index 77% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java index 3f69c2ea40..7266b4b12c 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayMessages.java @@ -33,6 +33,10 @@ public interface GrpcGatewayMessages { @Message(level = MessageLevel.INFO, text = "Started {0} gRPC listener on port {1}") void startedListener(String name, int port); + @Message(level = MessageLevel.INFO, + text = "The {0} listener is proxying [{1}]; identity rules: {2}") + void proxyingServices(String name, String protoServices, String identityRules); + @Message(level = MessageLevel.INFO, text = "Stopping {0} gRPC listener, draining for up to {1} ms") void stoppingListener(String name, long drainTimeoutMillis); @@ -91,15 +95,40 @@ public interface GrpcGatewayMessages { + "... to a topology; topologies are picked up without a restart.") void noTopologyDeclaresService(String name, String role); + @Message(level = MessageLevel.WARN, + text = "Denied gRPC call to {0} for user {1} in topology {2}: the method is not permitted there") + void methodDenied(String method, String user, String topology); + @Message(level = MessageLevel.INFO, text = "Reloaded the {0} listener message policy: {1}") void reloadedPolicy(String name, String policy); + @Message(level = MessageLevel.WARN, + text = "The {0} listener cannot apply the identity rewrite rules [{1}]; " + + "the previously configured rules remain in effect") + void invalidIdentityRules(String name, String rules, + @StackTrace(level = MessageLevel.WARN) Exception e); + @Message(level = MessageLevel.WARN, text = "The {0} listener cannot apply changes to [{1}] without a gateway restart; " + "the running values remain in effect") void restartOnlyConfigChanged(String name, String properties); + @Message(level = MessageLevel.INFO, + text = "The {0} listener presents the TLS identity from {1}, alias {2}") + void listenerTlsIdentity(String name, String keystore, String alias); + + @Message(level = MessageLevel.WARN, + text = "The {0} listeners cannot be reconfigured: [{1}] are not running listeners. " + + "Adding or removing a listener needs a gateway restart") + void listenerSetChanged(String name, String names); + + @Message(level = MessageLevel.WARN, + text = "The refreshed {0} listener configuration could not be read; " + + "the running configuration remains in effect") + void invalidListenerConfiguration(String name, + @StackTrace(level = MessageLevel.WARN) Exception e); + @Message(level = MessageLevel.DEBUG, text = "{0}") void debugLog(String message); } diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcListener.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcListener.java new file mode 100644 index 0000000000..71c499dcf9 --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcListener.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.knox.gateway.config.GatewayConfig; +import org.apache.knox.gateway.config.GatewayConfigChangeListener; +import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.protocol.ProtocolListener; +import org.apache.knox.gateway.services.GatewayServices; + +/** + * The gateway's gRPC listeners: one or more Netty servers on their own ports, + * wired to Knox's identity, token, topology and audit services. + *

+ * They are separate sockets rather than routes on the gateway's existing + * connectors because gRPC requires HTTP/2 negotiated over ALPN, and Knox's Jetty + * connectors are HTTP/1.1 only. Beyond the transport, the servlet pipeline could + * not carry these calls anyway: Servlet 3.1 has no trailer API, and gRPC puts + * {@code grpc-status} — and often structured error details — in trailers. + * + *

No schema, anywhere

+ * This compiles against no {@code .proto} file and no generated class. Calls are + * relayed as opaque bytes for whatever proto services a deployment names, and the + * one thing that needs to look inside a message — replacing the caller's claimed + * identity with the authenticated one — is done by field number on the wire. + * Field numbers are the part of a protobuf schema that cannot change without + * breaking every deployed client, so the gateway tracks no particular version of + * anything. + *

+ * What a deployment supplies is therefore configuration rather than code: which + * proto services to front, which Knox service role ties them to a topology, + * where the identity lives, and which RPCs to refuse. The protocol this was + * built for runs through the documentation, but only ever as values. + * + *

Why more than one

+ * Each listener routes to as many topologies as its clients select, so several + * listeners are not a way to separate policy — topology selection already does + * that. They exist because TLS identity is per-socket: a listener each lets a + * gateway answer for several hostnames with plain single-name certificates, + * which is the only option where the platform PKI cannot issue multi-name ones. + * A deployment that names no listeners runs exactly one, configured from the + * plain {@code gateway.grpc.*} properties. + */ +public class GrpcListener implements ProtocolListener, GatewayConfigChangeListener { + + private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); + + private final List endpoints = new ArrayList<>(); + /** Started listeners by name, so a configuration change reaches the right one. */ + private final Map byName = new ConcurrentHashMap<>(); + + @Override + public String getName() { + return "gRPC"; + } + + @Override + public boolean isEnabled(GatewayConfig config) { + return config.isGrpcEnabled(); + } + + /** + * Starts every configured listener. + *

+ * One failing stops the gateway, and the listeners already started are stopped + * first: a gateway that came up serving half the endpoints an operator + * configured would be worse than one that refused to come up at all, because + * the missing half looks like a network fault from the outside. + */ + @Override + public void start(GatewayConfig config, GatewayServices services) throws Exception { + for (GrpcListenerSettings settings : GrpcListenerSettingsFactory.create(config)) { + final GrpcEndpoint endpoint = new GrpcEndpoint(settings); + try { + endpoint.start(config, services); + } catch (Exception e) { + stop(); + throw e; + } + endpoints.add(endpoint); + byName.put(settings.getName(), endpoint); + } + } + + @Override + public void stop() { + for (GrpcEndpoint endpoint : endpoints) { + endpoint.stop(); + } + endpoints.clear(); + byName.clear(); + } + + @Override + public void reload() { + for (GrpcEndpoint endpoint : endpoints) { + endpoint.reload(); + } + } + + /** + * Hands each running listener the settings the refreshed configuration implies + * for it. + *

+ * Which listeners exist is fixed at startup, like whether the feature runs at + * all: a name added or removed in a running gateway is reported rather than + * acted on, since binding or releasing a port is exactly the kind of change an + * operator should schedule. + */ + @Override + public void onGatewayConfigChanged(GatewayConfig config) { + final List updated; + try { + updated = GrpcListenerSettingsFactory.create(config); + } catch (RuntimeException e) { + LOG.invalidListenerConfiguration(getName(), e); + return; + } + final List unknown = new ArrayList<>(); + for (GrpcListenerSettings settings : updated) { + final GrpcEndpoint endpoint = byName.get(settings.getName()); + if (endpoint == null) { + unknown.add(settings.getName()); + } else { + endpoint.onSettingsChanged(settings); + } + } + if (!unknown.isEmpty()) { + LOG.listenerSetChanged(getName(), String.join(", ", unknown)); + } + } + + /** + * @return the port of the first listener, for the gateway's startup log; see + * {@link #getPorts()} for all of them + */ + @Override + public int getPort() { + return endpoints.isEmpty() ? -1 : endpoints.get(0).getPort(); + } + + @Override + public List getPorts() { + final List ports = new ArrayList<>(endpoints.size()); + for (GrpcEndpoint endpoint : endpoints) { + ports.add(endpoint.getPort()); + } + return Collections.unmodifiableList(ports); + } + + /** Exposed so tests can inspect what a given configuration would start. */ + List currentEndpoints() { + return Collections.unmodifiableList(endpoints); + } +} diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java new file mode 100644 index 0000000000..2a52895fb2 --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.Collections; +import java.util.Set; + +/** + * Everything one gRPC listener needs in order to run: its transport limits, what + * it fronts, where the identity goes, and the TLS identity it presents. + *

+ * Plain values rather than reads against {@code GatewayConfig}, so that a + * listener can be built and tested without a gateway configuration to hand — and + * so that a gateway running several listeners has one of these per listener + * rather than each of them reaching back into shared configuration. + *

+ * The limits here are the listener's DoS surface. A new socket accepting 128 MB + * messages on long-lived streams needs message-size, stream-count and + * keepalive-abuse bounds configured from the start, not added after the first + * incident. + */ +public class GrpcListenerSettings { + + private String name = "grpc"; + private int port; + private int maxMessageSize = 134217728; + private int maxConcurrentCallsPerConnection = 1000; + private long permitKeepAliveTimeMillis = 10000L; + private boolean permitKeepAliveWithoutCalls = true; + private long channelIdleTimeoutMillis = 1800000L; + private long drainTimeoutMillis = 30000L; + private String backendTokenAlias; + private String topologyMetadataKey = GrpcMetadataKeys.DEFAULT_TOPOLOGY_KEY; + private String serviceRole = "GRPC"; + private Set protoServices = Collections.emptySet(); + private String defaultTopology; + private String identityRules; + private int identityScanLimit = 131072; + private String methodsDeny; + private String methodsAllow; + private boolean sslEnabled = true; + private String sslKeystorePath; + private String sslKeystoreType = "PKCS12"; + private String sslKeystoreAlias; + private String sslKeystorePasswordAlias; + + /** + * The Knox service role this listener resolves backends under, and the prefix + * for its ACL and method-list parameters in topology XML. + * + * @return the service role + */ + public String getServiceRole() { + return serviceRole; + } + + public GrpcListenerSettings serviceRole(String value) { + this.serviceRole = value; + return this; + } + + /** + * The fully qualified proto service names this listener fronts. Anything else + * is answered {@code UNIMPLEMENTED}. + * + * @return the proxied service names + */ + public Set getProtoServices() { + return protoServices; + } + + public GrpcListenerSettings protoServices(Set value) { + this.protoServices = value == null ? Collections.emptySet() : value; + return this; + } + + /** @return the topology to use when a client selects none, or null */ + public String getDefaultTopology() { + return defaultTopology; + } + + public GrpcListenerSettings defaultTopology(String value) { + this.defaultTopology = value; + return this; + } + + /** @return the identity rewrite rules as configured, or null for none */ + public String getIdentityRules() { + return identityRules; + } + + public GrpcListenerSettings identityRules(String value) { + this.identityRules = value; + return this; + } + + public int getIdentityScanLimit() { + return identityScanLimit; + } + + public GrpcListenerSettings identityScanLimit(int value) { + this.identityScanLimit = value; + return this; + } + + public String getMethodsDeny() { + return methodsDeny; + } + + public GrpcListenerSettings methodsDeny(String value) { + this.methodsDeny = value; + return this; + } + + public String getMethodsAllow() { + return methodsAllow; + } + + public GrpcListenerSettings methodsAllow(String value) { + this.methodsAllow = value; + return this; + } + + /** @return whether this listener presents TLS */ + public boolean isSslEnabled() { + return sslEnabled; + } + + public GrpcListenerSettings sslEnabled(boolean value) { + this.sslEnabled = value; + return this; + } + + /** + * A keystore holding this listener's own server certificate, or null to present + * the gateway identity Jetty also presents. + *

+ * Distinct key material per listener is what lets several listeners answer for + * several hostnames on one gateway, each with a plain single-name certificate. + * That matters where the platform PKI cannot issue multi-name (SAN or wildcard) + * certificates, which would otherwise be the only way to serve more than one + * name from one endpoint. + * + * @return the keystore path, or null for the gateway identity + */ + public String getSslKeystorePath() { + return sslKeystorePath; + } + + public GrpcListenerSettings sslKeystorePath(String value) { + this.sslKeystorePath = value; + return this; + } + + public String getSslKeystoreType() { + return sslKeystoreType; + } + + public GrpcListenerSettings sslKeystoreType(String value) { + this.sslKeystoreType = value; + return this; + } + + /** @return the entry within the keystore to present, or null for the sole entry */ + public String getSslKeystoreAlias() { + return sslKeystoreAlias; + } + + public GrpcListenerSettings sslKeystoreAlias(String value) { + this.sslKeystoreAlias = value; + return this; + } + + /** @return the alias holding the keystore password, or null for the gateway's */ + public String getSslKeystorePasswordAlias() { + return sslKeystorePasswordAlias; + } + + public GrpcListenerSettings sslKeystorePasswordAlias(String value) { + this.sslKeystorePasswordAlias = value; + return this; + } + + public String getName() { + return name; + } + + public GrpcListenerSettings name(String value) { + this.name = value; + return this; + } + + public int getPort() { + return port; + } + + public GrpcListenerSettings port(int value) { + this.port = value; + return this; + } + + public int getMaxMessageSize() { + return maxMessageSize; + } + + public GrpcListenerSettings maxMessageSize(int value) { + this.maxMessageSize = value; + return this; + } + + public int getMaxConcurrentCallsPerConnection() { + return maxConcurrentCallsPerConnection; + } + + public GrpcListenerSettings maxConcurrentCallsPerConnection(int value) { + this.maxConcurrentCallsPerConnection = value; + return this; + } + + public long getPermitKeepAliveTimeMillis() { + return permitKeepAliveTimeMillis; + } + + public GrpcListenerSettings permitKeepAliveTimeMillis(long value) { + this.permitKeepAliveTimeMillis = value; + return this; + } + + public boolean isPermitKeepAliveWithoutCalls() { + return permitKeepAliveWithoutCalls; + } + + public GrpcListenerSettings permitKeepAliveWithoutCalls(boolean value) { + this.permitKeepAliveWithoutCalls = value; + return this; + } + + public long getChannelIdleTimeoutMillis() { + return channelIdleTimeoutMillis; + } + + public GrpcListenerSettings channelIdleTimeoutMillis(long value) { + this.channelIdleTimeoutMillis = value; + return this; + } + + public long getDrainTimeoutMillis() { + return drainTimeoutMillis; + } + + public GrpcListenerSettings drainTimeoutMillis(long value) { + this.drainTimeoutMillis = value; + return this; + } + + /** + * The metadata entry a client uses to select a topology. It is also the + * connection-string parameter users write, so a deployment may prefer a name + * that describes the choice rather than the gateway making it. + * + * @return the metadata key name + */ + public String getTopologyMetadataKey() { + return topologyMetadataKey; + } + + public GrpcListenerSettings topologyMetadataKey(String value) { + this.topologyMetadataKey = value; + return this; + } + + public String getBackendTokenAlias() { + return backendTokenAlias; + } + + public GrpcListenerSettings backendTokenAlias(String value) { + this.backendTokenAlias = value; + return this; + } +} diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettingsFactory.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettingsFactory.java new file mode 100644 index 0000000000..2ba34f50fa --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettingsFactory.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +import org.apache.knox.gateway.config.GatewayConfig; + +/** + * Turns gateway configuration into one {@link GrpcListenerSettings} per listener. + * + *

Why more than one listener

+ * A listener is a transport endpoint, not a policy boundary: each one still + * routes to as many topologies as its clients select, exactly as a single + * listener does. What separates them is the socket and the certificate on it. + *

+ * That is worth having because TLS identity is per-socket. Serving several + * hostnames from one endpoint needs one certificate naming all of them, and a + * platform PKI that cannot issue multi-name (SAN or wildcard) certificates + * cannot produce one. Several listeners, each presenting a plain single-name + * certificate for the hostname its clients dial, is the way to serve those + * clients without that certificate. + * + *

How a listener is configured

+ * {@code gateway.grpc.listener.names} lists them. Every other property is read + * from {@code gateway.grpc..} when that listener sets it, and + * from the plain {@code gateway.grpc.} otherwise — so shared settings + * are written once and only the differences are repeated. + *

+ * Naming no listeners yields exactly one, configured entirely from the plain + * properties. That is the ordinary deployment, and it means the multi-listener + * machinery costs nothing to a gateway that does not use it. + */ +public final class GrpcListenerSettingsFactory { + + private static final Pattern VALID_NAME = Pattern.compile("[a-z0-9][a-z0-9_-]*"); + + /** + * First segments of the plain properties. A listener may not be named after + * one, because {@code gateway.grpc.identity.rules} and a listener called + * {@code identity} would occupy the same configuration namespace. + */ + private static final Set RESERVED_NAMES = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("enabled", "port", "service", "proto", "identity", "default", + "topology", "methods", "max", "permit", "channel", "drain", "backend", "ssl", + "listener"))); + + private GrpcListenerSettingsFactory() { + } + + /** + * Builds the settings for every configured listener. + * + * @param config the gateway configuration + * @return one settings object per listener, in configured order; never empty + * @throws IllegalArgumentException if a listener name is unusable, duplicated, + * or two listeners would bind the same port + */ + public static List create(GatewayConfig config) { + final List names = config.getGrpcListenerNames(); + final List settings = new ArrayList<>(); + + if (names == null || names.isEmpty()) { + settings.add(build(config, null, Collections.emptyMap())); + } else { + final Set seen = new LinkedHashSet<>(); + for (String name : names) { + final String listener = validate(name, seen); + settings.add(build(config, listener, config.getGrpcListenerConfig(listener))); + } + } + requireDistinctPorts(settings); + return Collections.unmodifiableList(settings); + } + + private static String validate(String name, Set seen) { + final String trimmed = name == null ? "" : name.trim(); + if (!VALID_NAME.matcher(trimmed).matches()) { + throw new IllegalArgumentException("A gRPC listener name may contain only a-z, 0-9, '-' and" + + " '_', and must start with a letter or digit, got: " + name); + } + if (RESERVED_NAMES.contains(trimmed)) { + throw new IllegalArgumentException("'" + trimmed + "' cannot be a gRPC listener name because" + + " gateway.grpc." + trimmed + ".* is already a configuration property"); + } + if (!seen.add(trimmed)) { + throw new IllegalArgumentException("Duplicate gRPC listener name: " + trimmed); + } + return trimmed; + } + + /** + * Two listeners on one port is a startup failure rather than a race to bind: + * whichever lost would fail with an address-in-use error naming neither of the + * listeners involved. + */ + private static void requireDistinctPorts(List settings) { + final Map byPort = new java.util.HashMap<>(); + for (GrpcListenerSettings listener : settings) { + final String other = byPort.put(listener.getPort(), listener.getName()); + if (other != null) { + throw new IllegalArgumentException("gRPC listeners '" + other + "' and '" + + listener.getName() + "' are both configured on port " + listener.getPort()); + } + } + } + + private static GrpcListenerSettings build(GatewayConfig config, String name, + Map overrides) { + final String serviceRole = string(overrides, "service.role", config.getGrpcServiceRole()); + return new GrpcListenerSettings() + // An unnamed listener is named after its service role, so a single-listener + // deployment reads in the log for the thing being fronted rather than for + // the transport. A named one uses the name the operator chose. + .name(name == null ? serviceRole : name) + .serviceRole(serviceRole) + .port(integer(overrides, "port", config.getGrpcPort())) + .protoServices(protoServices(string(overrides, "proto.services", config.getGrpcProtoServices()))) + .defaultTopology(string(overrides, "default.topology", config.getGrpcDefaultTopology())) + .topologyMetadataKey(string(overrides, "topology.metadata.key", config.getGrpcTopologyMetadataKey())) + .identityRules(string(overrides, "identity.rules", config.getGrpcIdentityRules())) + .identityScanLimit(integer(overrides, "identity.scan.limit", config.getGrpcIdentityScanLimit())) + .methodsDeny(string(overrides, "methods.deny", config.getGrpcMethodsDeny())) + .methodsAllow(string(overrides, "methods.allow", config.getGrpcMethodsAllow())) + .maxMessageSize(integer(overrides, "max.message.size", config.getGrpcMaxMessageSize())) + .maxConcurrentCallsPerConnection(integer(overrides, "max.concurrent.calls.per.connection", + config.getGrpcMaxConcurrentCallsPerConnection())) + .permitKeepAliveTimeMillis(longValue(overrides, "permit.keepalive.time", + config.getGrpcPermitKeepAliveTime())) + .permitKeepAliveWithoutCalls(bool(overrides, "permit.keepalive.without.calls", + config.isGrpcPermitKeepAliveWithoutCalls())) + .channelIdleTimeoutMillis(longValue(overrides, "channel.idle.timeout", + config.getGrpcChannelIdleTimeout())) + .drainTimeoutMillis(longValue(overrides, "drain.timeout", config.getGrpcDrainTimeout())) + .backendTokenAlias(string(overrides, "backend.token.alias", config.getGrpcBackendTokenAlias())) + .sslEnabled(bool(overrides, "ssl.enabled", config.isSSLEnabled())) + .sslKeystorePath(string(overrides, "ssl.keystore.path", null)) + .sslKeystoreType(string(overrides, "ssl.keystore.type", "PKCS12")) + .sslKeystoreAlias(string(overrides, "ssl.keystore.alias", null)) + .sslKeystorePasswordAlias(string(overrides, "ssl.keystore.password.alias", null)); + } + + static Set protoServices(String configured) { + if (configured == null || configured.trim().isEmpty()) { + return Collections.emptySet(); + } + final Set names = new LinkedHashSet<>(); + for (String name : configured.trim().split("\\s*,\\s*")) { + if (!name.isEmpty()) { + names.add(name); + } + } + return Collections.unmodifiableSet(names); + } + + private static String string(Map overrides, String key, String fallback) { + final String value = overrides.get(key); + return value == null || value.trim().isEmpty() ? fallback : value.trim(); + } + + private static int integer(Map overrides, String key, int fallback) { + final String value = string(overrides, key, null); + return value == null ? fallback : parse(key, value).intValue(); + } + + private static long longValue(Map overrides, String key, long fallback) { + final String value = string(overrides, key, null); + return value == null ? fallback : parse(key, value); + } + + private static boolean bool(Map overrides, String key, boolean fallback) { + final String value = string(overrides, key, null); + return value == null ? fallback : Boolean.parseBoolean(value); + } + + private static Long parse(String key, String value) { + try { + return Long.valueOf(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "gRPC listener property '" + key + "' must be a number, got: " + value, e); + } + } + + /** @return the name of the property a listener would set to override this one */ + static String propertyName(String listener, String property) { + return listener == null + ? "gateway.grpc." + property + : String.format(Locale.ROOT, "gateway.grpc.%s.%s", listener, property); + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java similarity index 100% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/GrpcMetadataKeys.java diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/HeaderRewriter.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/HeaderRewriter.java similarity index 100% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/HeaderRewriter.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/HeaderRewriter.java diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentityAssertingInterceptor.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentityAssertingInterceptor.java new file mode 100644 index 0000000000..2280c6efc9 --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentityAssertingInterceptor.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import io.grpc.Status; + +/** + * Replaces the caller's claimed identity with the authenticated one, working + * directly on the wire format. + *

+ * This is the whole reason the gateway looks inside messages at all. Protocols + * in this family commonly trust a client-asserted identity field: the + * client states who it is and the server believes it. Such a field typically + * keys the server-side session cache, so leaving it alone would let one caller + * collide with — or attach to — another's session simply by claiming their name. + * Overwriting it is what makes sessions isolated and the audit trail meaningful. + *

+ * No schema is needed to do it. The identity lives at field numbers named by an + * {@link IdentityRewritePolicy}, and every other byte of the message is copied + * through verbatim — including fields from a newer protocol version this build + * has never heard of, which are not merely preserved but never even decoded. + *

+ * Three cases are refused rather than forwarded: a message that cannot be + * parsed, a message whose shape contradicts the configured rules, and a message + * whose identity fields lie beyond the policy's scan limit. All three share a + * reason — if the identity cannot be replaced everywhere the rules say it lives, + * then the caller's own claim would travel on somewhere, which is precisely what + * this exists to prevent. + */ +public class IdentityAssertingInterceptor implements MessageInterceptor { + + private final IdentityRewritePolicy policy; + private final PrincipalSource principalSource; + + /** Supplies the principal for the call in flight. */ + @FunctionalInterface + public interface PrincipalSource { + String currentPrincipal(); + } + + public IdentityAssertingInterceptor(IdentityRewritePolicy policy, PrincipalSource principalSource) { + this.policy = policy; + this.principalSource = principalSource; + } + + /** Uses the principal the authentication interceptor put in the call context. */ + public IdentityAssertingInterceptor(IdentityRewritePolicy policy) { + this(policy, () -> { + final GrpcCallContext callContext = GrpcCallContext.current(); + return callContext == null ? null : callContext.getPrincipal(); + }); + } + + @Override + public byte[] intercept(byte[] message) { + final String principal = principalSource.currentPrincipal(); + if (principal == null || principal.isEmpty()) { + // Authentication runs before any handler, so this cannot happen unless the + // interceptor chain was assembled wrongly. Forwarding would send the + // client's own claim through untouched. + throw Status.INTERNAL + .withDescription("No authenticated principal available for identity assertion") + .asRuntimeException(); + } + try { + return assertIdentity(message, principal); + } catch (ProtoWire.MalformedMessageException e) { + throw Status.INVALID_ARGUMENT + .withDescription("Request message is not a well-formed protobuf message") + .withCause(e) + .asRuntimeException(); + } catch (UnassertableMessageException e) { + throw Status.INVALID_ARGUMENT + .withDescription(e.getMessage()) + .withCause(e) + .asRuntimeException(); + } + } + + /** + * Returns the message with every configured identity field replaced. + * + * @param message the request as received + * @param principal the authenticated principal + * @return the request to forward + * @throws ProtoWire.MalformedMessageException if the message cannot be parsed + * @throws UnassertableMessageException if the message's shape contradicts the + * rules, or an identity field lies beyond the scan limit + */ + public byte[] assertIdentity(byte[] message, String principal) { + if (policy.isEmpty()) { + return message; + } + return rewrite(message, policy.root(), principal, 0); + } + + /** + * Rewrites one message — the request itself, or a nested message a rule + * descends into. + * + * @param buffer the bytes of this message + * @param node the rules that apply at this depth + * @param principal the authenticated principal + * @param baseOffset where {@code buffer} begins within the request as a whole, + * so the scan limit is measured against the message the client sent + * rather than against each nested message separately + * @return the rewritten bytes + */ + private byte[] rewrite(byte[] buffer, IdentityRewritePolicy.Node node, String principal, + int baseOffset) { + final ByteArrayOutputStream out = new ByteArrayOutputStream(buffer.length + 32); + final Set rewritten = new HashSet<>(); + + int pos = 0; + while (pos < buffer.length) { + final int recordStart = pos; + final ProtoWire.Varint tag = ProtoWire.readVarint(buffer, pos); + pos = tag.end(); + final int field = ProtoWire.fieldNumber(tag.value()); + final int wire = ProtoWire.wireType(tag.value()); + final int[] bounds = ProtoWire.valueBounds(buffer, pos, wire); + + final IdentityRewritePolicy.Node child = node.child(field); + if (child == null) { + out.write(buffer, recordStart, bounds[1] - recordStart); + } else { + requireWithinScanLimit(field, baseOffset + bounds[1]); + requireLengthDelimited(field, wire, child); + if (child.isLeaf()) { + write(out, field, child, principal); + } else { + final byte[] nested = ProtoWire.slice(buffer, bounds[0], bounds[1]); + ProtoWire.writeLengthDelimited(out, field, + rewrite(nested, child, principal, baseOffset + bounds[0])); + } + // Every occurrence is rewritten, not just the first: protobuf merges + // repeats, so one left alone could override the one we asserted. + rewritten.add(field); + } + pos = bounds[1]; + } + + // A client that sent no identity at all still gets one, at whatever depth the + // rules put it; the backend must never see a request whose identity Knox did + // not put there. Appending is safe however large the message is, because + // nothing was found to be overridden by — the walk above covered every byte. + for (Map.Entry entry : node.children().entrySet()) { + if (rewritten.contains(entry.getKey())) { + continue; + } + final IdentityRewritePolicy.Node child = entry.getValue(); + if (child.isLeaf()) { + write(out, entry.getKey(), child, principal); + } else { + ProtoWire.writeLengthDelimited(out, entry.getKey(), + rewrite(new byte[0], child, principal, baseOffset)); + } + } + return out.toByteArray(); + } + + private static void write(ByteArrayOutputStream out, int field, + IdentityRewritePolicy.Node leaf, String principal) { + ProtoWire.writeLengthDelimited(out, field, + leaf.subject().resolve(principal).getBytes(StandardCharsets.UTF_8)); + } + + /** + * Refuses a field that ends beyond the scan limit. + *

+ * Measured against the end rather than the start, so the bound covers what the + * rewrite has to copy: a container beginning in the first few bytes but running + * to a hundred megabytes costs as much as one that begins late. + */ + private void requireWithinScanLimit(int field, int endOffset) { + if (endOffset > policy.getScanLimit()) { + throw new UnassertableMessageException("Identity field " + field + + " extends past the first " + policy.getScanLimit() + + " bytes of the request, so the authenticated identity cannot be asserted over it"); + } + } + + /** + * Refuses a field whose wire type contradicts the rules. A rule expects a + * string to overwrite or a message to descend into, and both are + * length-delimited; anything else means the configuration does not describe + * this protocol. Skipping it quietly would forward the caller's own claim. + */ + private static void requireLengthDelimited(int field, int wire, + IdentityRewritePolicy.Node node) { + if (wire != ProtoWire.WIRETYPE_LENGTH_DELIMITED) { + throw new UnassertableMessageException("Identity field " + field + " is a " + + (node.isLeaf() ? "value to replace" : "message to descend into") + + " but arrived with wire type " + wire + + "; the configured identity rules do not describe this message"); + } + } + + /** + * Signals a message the configured rules cannot be applied to in full. Distinct + * from malformed input: the bytes parse, but their shape and the configuration + * disagree, or the identity sits further into the request than the policy + * allows. + */ + public static class UnassertableMessageException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public UnassertableMessageException(String message) { + super(message); + } + } +} diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentityRewritePolicy.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentityRewritePolicy.java new file mode 100644 index 0000000000..4e488c4ec2 --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentityRewritePolicy.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The set of identity rewrite rules in force, compiled for one pass over a + * message. + *

+ * Zero rules is the ordinary case for a protocol that carries no identity: the + * relay is then a pure pipe. Where there are rules, they are compiled into a + * tree keyed by field number, so rules sharing a prefix — {@code 2.1} and + * {@code 2.2}, say — descend into that container once rather than once each. + * + *

The scan limit

+ * Every field a rule touches must lie wholly within the first + * {@link #getScanLimit()} bytes of the message. This bounds what identity + * assertion can be made to do: rewriting a nested field means slicing it out and + * rebuilding it, so without a limit a client could put a hundred megabytes + * inside the identity container and make the gateway copy it several times over. + *

+ * It is a rejection rather than a truncation, and that is the security-relevant + * part. Giving up on a rule that sits beyond the limit — and synthesising a + * fresh identity instead — would leave the caller's own claim in the message + * behind ours, where protobuf's last-wins merge semantics would let it take + * effect. A message we cannot fully assert over is one we must not forward. + *

+ * Messages are not otherwise constrained: a large request whose identity sits at + * the front, which is what generated serializers emit, passes regardless of its + * total size. + */ +public final class IdentityRewritePolicy { + + /** + * 128 KiB. Comfortably past any identity container a real protocol declares, + * while keeping the worst-case rewrite cost of a 128 MB message the same as + * that of a small one. + */ + public static final int DEFAULT_SCAN_LIMIT = 131072; + + private static final IdentityRewritePolicy NONE = + new IdentityRewritePolicy(Collections.emptyList(), DEFAULT_SCAN_LIMIT, new Node()); + + private final List rules; + private final int scanLimit; + private final Node root; + + private IdentityRewritePolicy(List rules, int scanLimit, Node root) { + this.rules = rules; + this.scanLimit = scanLimit; + this.root = root; + } + + /** @return a policy that rewrites nothing */ + public static IdentityRewritePolicy none() { + return NONE; + } + + /** + * Parses a comma-separated list of rules. + * + * @param configuredRules for example {@code 2.1=principal, 2.2=principal}; + * null or empty yields a policy that rewrites nothing + * @param scanLimit the maximum offset, in bytes, at which a rewritten field may + * end + * @return the compiled policy + * @throws IllegalArgumentException if a rule is malformed, if two rules + * collide, or if the scan limit is not positive + */ + public static IdentityRewritePolicy parse(String configuredRules, int scanLimit) { + if (configuredRules == null || configuredRules.trim().isEmpty()) { + return NONE; + } + if (scanLimit < 1) { + throw new IllegalArgumentException("The identity scan limit must be positive, got: " + scanLimit); + } + final List parsed = new ArrayList<>(); + final Node newRoot = new Node(); + for (String entry : configuredRules.trim().split("\\s*,\\s*")) { + if (entry.isEmpty()) { + continue; + } + final IdentityRewriteRule rule = IdentityRewriteRule.parse(entry); + add(newRoot, rule); + parsed.add(rule); + } + if (parsed.isEmpty()) { + return NONE; + } + return new IdentityRewritePolicy(Collections.unmodifiableList(parsed), scanLimit, newRoot); + } + + /** + * Inserts a rule into the tree, refusing the two ways rules can contradict each + * other: writing the same place twice, and writing a value at a field another + * rule descends through. + */ + private static void add(Node root, IdentityRewriteRule rule) { + Node current = root; + for (final int field : rule.getPath()) { + if (current.subject != null) { + throw new IllegalArgumentException("Rule " + rule + + " descends through field " + field + ", which another rule writes a value to"); + } + Node child = current.children.get(field); + if (child == null) { + child = new Node(); + current.children.put(field, child); + } + current = child; + } + if (current.subject != null || !current.children.isEmpty()) { + throw new IllegalArgumentException("Rule " + rule + " collides with an earlier rule"); + } + current.subject = rule.getSubject(); + } + + /** @return true if this policy rewrites nothing, so the relay is a pure pipe */ + public boolean isEmpty() { + return rules.isEmpty(); + } + + /** + * @return the maximum offset, in bytes, at which a rewritten field may end + */ + public int getScanLimit() { + return scanLimit; + } + + public List getRules() { + return rules; + } + + Node root() { + return root; + } + + @Override + public String toString() { + if (rules.isEmpty()) { + return "none"; + } + final StringBuilder text = new StringBuilder(48); + for (IdentityRewriteRule rule : rules) { + if (text.length() > 0) { + text.append(','); + } + text.append(rule); + } + return text.append(" (scan limit ").append(scanLimit).append(" bytes)").toString(); + } + + /** + * One field number in the compiled tree. A node either writes a value + * ({@code subject} set, a leaf) or is descended through ({@code children} + * populated) — {@link #add} refuses anything that would be both. + */ + static final class Node { + /** Insertion-ordered so synthesised fields come out in the order configured. */ + private final Map children = new LinkedHashMap<>(); + private IdentitySubject subject; + + Node child(int field) { + return children.get(field); + } + + Map children() { + return children; + } + + boolean isLeaf() { + return subject != null; + } + + IdentitySubject subject() { + return subject; + } + } +} diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentityRewriteRule.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentityRewriteRule.java new file mode 100644 index 0000000000..899cfea79b --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentityRewriteRule.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.Locale; + +/** + * One place in a request message where the authenticated identity is written, + * expressed as field numbers rather than as a schema. + *

+ * Written as {@code path=subject}, where the path is one or more protobuf field + * numbers separated by dots and the subject names what to write there. Each + * leading number is a nested message to descend into; the last is the string + * field to replace. So {@code 1=principal} replaces a top-level field, and + * {@code 2.1=principal} replaces a field one level down. + *

+ * Naming numbers rather than compiling against generated classes is what keeps + * the gateway free of any particular protocol version: a schema may gain fields, + * rename them or deprecate them, but renumbering an existing field breaks every + * deployed client, so the numbers are the stable part. + */ +public final class IdentityRewriteRule { + + /** Protobuf caps field numbers at 2^29-1. */ + private static final int MAX_FIELD_NUMBER = 536870911; + private static final int RESERVED_FROM = 19000; + private static final int RESERVED_TO = 19999; + + private final int[] path; + private final IdentitySubject subject; + + private IdentityRewriteRule(int[] path, IdentitySubject subject) { + this.path = path; + this.subject = subject; + } + + /** + * Parses one rule. + * + * @param rule {@code path=subject}, for example {@code 2.1=principal} + * @return the parsed rule + * @throws IllegalArgumentException if the rule is not a dotted list of legal + * field numbers followed by a known subject + */ + public static IdentityRewriteRule parse(String rule) { + if (rule == null || rule.trim().isEmpty()) { + throw new IllegalArgumentException("A rewrite rule must not be empty"); + } + final String trimmed = rule.trim(); + final int separator = trimmed.indexOf('='); + if (separator < 0) { + throw new IllegalArgumentException( + "A rewrite rule must be written as 'path=subject', got: " + trimmed); + } + final String pathPart = trimmed.substring(0, separator).trim(); + final IdentitySubject parsedSubject = IdentitySubject.parse(trimmed.substring(separator + 1)); + + if (pathPart.isEmpty()) { + throw new IllegalArgumentException( + "A rewrite rule must name at least one field number, got: " + trimmed); + } + final String[] parts = pathPart.split("\\.", -1); + final int[] parsedPath = new int[parts.length]; + for (int i = 0; i < parts.length; i++) { + parsedPath[i] = parseFieldNumber(parts[i], trimmed); + } + return new IdentityRewriteRule(parsedPath, parsedSubject); + } + + private static int parseFieldNumber(String value, String rule) { + final int number; + try { + number = Integer.parseInt(value.trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "A rewrite rule path must contain only field numbers, got: " + rule, e); + } + if (number < 1 || number > MAX_FIELD_NUMBER) { + throw new IllegalArgumentException( + "Field numbers must be between 1 and " + MAX_FIELD_NUMBER + ", got: " + rule); + } + if (number >= RESERVED_FROM && number <= RESERVED_TO) { + throw new IllegalArgumentException( + "Field numbers " + RESERVED_FROM + "-" + RESERVED_TO + + " are reserved by protobuf, got: " + rule); + } + return number; + } + + /** + * @return the field numbers to follow, outermost first; never empty + */ + public int[] getPath() { + return path.clone(); + } + + public IdentitySubject getSubject() { + return subject; + } + + @Override + public String toString() { + final StringBuilder text = new StringBuilder(16); + for (int i = 0; i < path.length; i++) { + if (i > 0) { + text.append('.'); + } + text.append(path[i]); + } + return text.append('=').append(subject.name().toLowerCase(Locale.ROOT)).toString(); + } +} diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentitySubject.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentitySubject.java new file mode 100644 index 0000000000..08f946000a --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/IdentitySubject.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.Arrays; +import java.util.Locale; + +/** + * Which attribute of the authenticated caller a rewrite rule writes. + *

+ * A rule names a place in the message and a subject; this is the subject half. + * Keeping it explicit is what stops the gateway assuming that two fields in the + * same container mean "id" and "display name" — a convention of one protocol + * rather than a property of protobuf. + *

+ * The vocabulary is deliberately limited to what authentication actually + * establishes. Anything a deployment wishes were assertable but that Knox does + * not know is better refused at startup than written as an empty string. + */ +public enum IdentitySubject { + + /** The authenticated principal: the subject of the validated bearer token. */ + PRINCIPAL { + @Override + public String resolve(String principal) { + return principal; + } + }; + + /** + * Returns the value to write for this subject. + * + * @param principal the authenticated principal for the call in flight + * @return the value to write + */ + public abstract String resolve(String principal); + + /** + * Parses a subject name as written in configuration. + * + * @param value the configured name, case-insensitive + * @return the subject + * @throws IllegalArgumentException if the name is not one this build knows + */ + public static IdentitySubject parse(String value) { + if (value == null || value.trim().isEmpty()) { + throw new IllegalArgumentException("A rewrite rule must name a subject, e.g. '2.1=principal'"); + } + final String normalized = value.trim().toLowerCase(Locale.ROOT); + for (IdentitySubject subject : values()) { + if (subject.name().toLowerCase(Locale.ROOT).equals(normalized)) { + return subject; + } + } + throw new IllegalArgumentException("Unknown identity subject '" + value.trim() + + "'; supported subjects are " + Arrays.toString(values()).toLowerCase(Locale.ROOT)); + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/InterceptorChain.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/InterceptorChain.java similarity index 100% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/InterceptorChain.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/InterceptorChain.java diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MapFilterConfig.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MapFilterConfig.java similarity index 100% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MapFilterConfig.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MapFilterConfig.java diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptor.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptor.java similarity index 84% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptor.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptor.java index 8e8ac420e3..cc8670c372 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptor.java +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptor.java @@ -63,4 +63,16 @@ public interface MessageInterceptor { static MessageInterceptor passthrough() { return (MessageInterceptor) PASSTHROUGH; } + + /** + * Returns an interceptor that applies this one and then the given one, so a + * per-RPC check and a rewrite that applies to everything can be composed + * without either knowing about the other. + * + * @param next applied to this interceptor's result + * @return the combined interceptor + */ + default MessageInterceptor andThen(MessageInterceptor next) { + return message -> next.intercept(intercept(message)); + } } diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptorFactory.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptorFactory.java new file mode 100644 index 0000000000..a653b9244e --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MessageInterceptorFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +/** + * Chooses what, if anything, to do to the request messages of a given RPC. + *

+ * This is the entire seam between the protocol-agnostic gateway and a + * protocol-aware plugin. The gateway relays opaque bytes and knows only method + * names; a plugin returns an interceptor for the methods it cares about and + * {@link MessageInterceptor#passthrough()} for the rest. + *

+ * Interceptors are built once per method, so anything a deployment can change + * while running must be read when a message is intercepted rather than captured + * here. + */ +@FunctionalInterface +public interface MessageInterceptorFactory { + + /** + * @param fullMethodName the gRPC method name, {@code pkg.Service/Method} + * @return the interceptor for that method's requests; never null + */ + MessageInterceptor forMethod(String fullMethodName); +} diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MethodAccessInterceptor.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MethodAccessInterceptor.java new file mode 100644 index 0000000000..c43d9c123b --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MethodAccessInterceptor.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.knox.gateway.i18n.messages.MessagesFactory; +import org.apache.knox.gateway.services.GatewayServices; +import org.apache.knox.gateway.services.ServiceType; +import org.apache.knox.gateway.services.topology.TopologyService; +import org.apache.knox.gateway.topology.Provider; +import org.apache.knox.gateway.topology.Topology; + +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * Applies per-topology method allow and deny lists, after the topology has been + * selected and the user authorized to use it. + *

+ * Being keyed on the topology is the point: the same gateway can front a cluster + * where uploading code is fine and one where it is not, and the difference is a + * parameter in the topology that already forms the policy boundary. + *

+ * Configured alongside the ACLs on the {@code AclsAuthz} provider, keyed on the + * service role: + * 

+ * <param>
+ *   <name>SPARKCONNECT.methods.deny</name>
+ *   <value>AddArtifacts</value>
+ * </param>
+ *
+ */ +public class MethodAccessInterceptor implements ServerInterceptor { + + private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); + + private static final String AUTHZ_PROVIDER_ROLE = "authorization"; + private static final String ACLS_AUTHZ_PROVIDER_NAME = "AclsAuthz"; + private static final String DENY_SUFFIX = ".methods.deny"; + private static final String ALLOW_SUFFIX = ".methods.allow"; + + private final GatewayServices services; + private final String resourceRole; + private final MethodAccessPolicy defaultPolicy; + /** Derived from topology configuration, so cached per topology and cleared on redeploy. */ + private final Map policies = new ConcurrentHashMap<>(); + + public MethodAccessInterceptor(GatewayServices services, String resourceRole, + MethodAccessPolicy defaultPolicy) { + this.services = services; + this.resourceRole = resourceRole; + this.defaultPolicy = defaultPolicy; + } + + /** Drops cached policies so a redeployed topology takes effect. */ + public void invalidate() { + policies.clear(); + } + + @Override + public ServerCall.Listener interceptCall(ServerCall call, + Metadata headers, + ServerCallHandler next) { + final String method = call.getMethodDescriptor().getFullMethodName(); + final GrpcCallContext callContext = GrpcCallContext.current(); + final String topology = callContext == null ? null : callContext.getTopology(); + + final MethodAccessPolicy policy = + topology == null ? defaultPolicy : policies.computeIfAbsent(topology, this::buildPolicy); + + if (!policy.isPermitted(method)) { + LOG.methodDenied(method, callContext == null ? null : callContext.getPrincipal(), topology); + call.close(Status.PERMISSION_DENIED + .withDescription("This RPC is not permitted in this topology"), new Metadata()); + return new ServerCall.Listener() { }; + } + return next.startCall(call, headers); + } + + private MethodAccessPolicy buildPolicy(String topologyName) { + final TopologyService topologyService = services.getService(ServiceType.TOPOLOGY_SERVICE); + if (topologyService == null) { + return defaultPolicy; + } + for (Topology topology : topologyService.getTopologies()) { + if (!topologyName.equals(topology.getName())) { + continue; + } + final Provider provider = topology.getProvider(AUTHZ_PROVIDER_ROLE, ACLS_AUTHZ_PROVIDER_NAME); + if (provider == null || !provider.isEnabled() || provider.getParams() == null) { + return defaultPolicy; + } + final Map params = provider.getParams(); + final String deny = param(params, resourceRole + DENY_SUFFIX); + final String allow = param(params, resourceRole + ALLOW_SUFFIX); + if (deny == null && allow == null) { + return defaultPolicy; + } + return MethodAccessPolicy.of(deny, allow); + } + return defaultPolicy; + } + + /** Provider params are lowercased on the servlet path; accept either spelling. */ + private static String param(Map params, String name) { + final String value = params.get(name); + return value != null ? value : params.get(name.toLowerCase(java.util.Locale.ROOT)); + } +} diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MethodAccessPolicy.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MethodAccessPolicy.java new file mode 100644 index 0000000000..f86bb46453 --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/MethodAccessPolicy.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Set; + +/** + * Allows or denies whole RPCs by name, with no knowledge of what they carry. + *

+ * gRPC puts the method in the request path — {@code /pkg.Service/Method} — so + * this needs no marshaller, no descriptor and no schema. It is the coarsest + * control the gateway offers and the only message-level one that survives + * completely intact on a byte-level proxy. + *

+ * The intended use is shutting off a capability that undermines whatever the + * backend enforces. A deployment relying on the backend's own plan-level policy + * can deny code upload outright, because uploaded code runs inside the backend + * process with that process's credentials and answers to no plan check. Be clear + * about the limit, though: denying an upload RPC does not close code execution + * where a protocol also allows inline functions inside ordinary requests. It + * shrinks the attack surface rather than drawing a boundary. + *

+ * Names may be given bare ({@code AddArtifacts}) or fully qualified + * ({@code pkg.Service/Method}); a bare name matches that method on any service. + */ +public class MethodAccessPolicy { + + private static final MethodAccessPolicy ALLOW_ALL = + new MethodAccessPolicy(Collections.emptySet(), Collections.emptySet()); + + private final Set denied; + private final Set allowed; + + private MethodAccessPolicy(Set denied, Set allowed) { + this.denied = denied; + this.allowed = allowed; + } + + public static MethodAccessPolicy allowAll() { + return ALLOW_ALL; + } + + /** + * Builds a policy from comma-separated lists. + * + * @param denyList methods to refuse, or null/empty for none + * @param allowList when non-empty, the only methods permitted; anything else is + * refused + * @return the policy + */ + public static MethodAccessPolicy of(String denyList, String allowList) { + return new MethodAccessPolicy(split(denyList), split(allowList)); + } + + private static Set split(String csv) { + if (csv == null || csv.trim().isEmpty()) { + return Collections.emptySet(); + } + final Set values = new LinkedHashSet<>(); + for (String entry : csv.trim().split("\\s*,\\s*")) { + if (!entry.isEmpty()) { + values.add(entry.toLowerCase(Locale.ROOT)); + } + } + return Collections.unmodifiableSet(values); + } + + /** + * Decides whether a call may proceed. + * + * @param fullMethodName the gRPC method name, {@code pkg.Service/Method} + * @return true if permitted + */ + public boolean isPermitted(String fullMethodName) { + if (denied.isEmpty() && allowed.isEmpty()) { + return true; + } + final String full = fullMethodName == null ? "" : fullMethodName.toLowerCase(Locale.ROOT); + final String bare = full.substring(full.lastIndexOf('/') + 1); + + if (denied.contains(full) || denied.contains(bare)) { + return false; + } + // An allow list, once given, is exhaustive: anything unnamed is refused, so + // an RPC added by a newer protocol version does not appear by default. + return allowed.isEmpty() || allowed.contains(full) || allowed.contains(bare); + } + + /** @return true if this policy permits everything */ + public boolean isUnrestricted() { + return denied.isEmpty() && allowed.isEmpty(); + } + + @Override + public String toString() { + return "deny=" + Arrays.toString(denied.toArray()) + + ", allow=" + Arrays.toString(allowed.toArray()); + } +} diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ProtoWire.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ProtoWire.java new file mode 100644 index 0000000000..869297d913 --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ProtoWire.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * The little of the protobuf wire format the gateway needs in order to find and + * replace a field without knowing the schema. + *

+ * A protobuf message is a flat sequence of records, each introduced by a tag + * holding a field number and a wire type. The wire type says how long the value + * is; the field number says which field it is. Nothing else — not the field's + * name, its declared type, nor the message it belongs to — is on the wire. That + * is what lets a proxy rewrite one field of a message it has no schema for, and + * copy every other byte through untouched. + *

+ * Field numbers are also the one thing protobuf guarantees never changes: a + * schema may add, rename or deprecate fields, but renumbering an existing one + * breaks every deployed client. Depending on a field number is therefore a much + * weaker coupling than depending on a generated class. + */ +public final class ProtoWire { + + public static final int WIRETYPE_VARINT = 0; + public static final int WIRETYPE_FIXED64 = 1; + public static final int WIRETYPE_LENGTH_DELIMITED = 2; + public static final int WIRETYPE_START_GROUP = 3; + public static final int WIRETYPE_END_GROUP = 4; + public static final int WIRETYPE_FIXED32 = 5; + + /** A varint is at most ten bytes; more than that is malformed, not merely large. */ + private static final int MAX_VARINT_BYTES = 10; + + private ProtoWire() { + } + + /** + * Signals input that is not a well-formed protobuf message. + *

+ * The gateway treats this as fatal for the call rather than forwarding the + * bytes: if a message cannot be parsed then the identity field cannot be + * replaced, and forwarding it would pass the caller's own claim through + * unaltered — the exact substitution this layer exists to prevent. + */ + public static class MalformedMessageException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public MalformedMessageException(String message) { + super(message); + } + } + + /** A decoded varint: its value, and the offset just past it. */ + public static final class Varint { + private final long value; + private final int end; + + Varint(long value, int end) { + this.value = value; + this.end = end; + } + + public long value() { + return value; + } + + public int end() { + return end; + } + } + + /** + * Reads a base-128 varint. + * + * @param buffer the message bytes + * @param position offset of the first byte of the varint + * @return the value and the offset just past it + * @throws MalformedMessageException if the varint runs off the end or is over-long + */ + public static Varint readVarint(byte[] buffer, int position) { + long result = 0; + int shift = 0; + int pos = position; + for (int read = 0; read < MAX_VARINT_BYTES; read++) { + if (pos >= buffer.length) { + throw new MalformedMessageException("varint runs past the end of the message"); + } + final int b = buffer[pos++] & 0xFF; + result |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return new Varint(result, pos); + } + shift += 7; + } + throw new MalformedMessageException("varint is longer than ten bytes"); + } + + public static void writeVarint(ByteArrayOutputStream out, long value) { + long remaining = value; + while ((remaining & ~0x7FL) != 0) { + out.write((int) ((remaining & 0x7F) | 0x80)); + remaining >>>= 7; + } + out.write((int) remaining); + } + + public static void writeTag(ByteArrayOutputStream out, int fieldNumber, int wireType) { + writeVarint(out, ((long) fieldNumber << 3) | wireType); + } + + /** + * Writes a length-delimited field: tag, byte length, then the bytes. + * + * @param out destination + * @param fieldNumber the field number to write + * @param value the field's bytes + */ + public static void writeLengthDelimited(ByteArrayOutputStream out, int fieldNumber, byte[] value) { + writeTag(out, fieldNumber, WIRETYPE_LENGTH_DELIMITED); + writeVarint(out, value.length); + out.write(value, 0, value.length); + } + + /** + * Locates the value of the record beginning at {@code position}, which must be + * just past the record's tag. + * + * @param buffer the message bytes + * @param position offset just past the tag + * @param wireType the wire type taken from the tag + * @return offsets of the value: {@code [start, end)} + * @throws MalformedMessageException on an unusable wire type or a length that + * overruns the buffer + */ + public static int[] valueBounds(byte[] buffer, int position, int wireType) { + switch (wireType) { + case WIRETYPE_VARINT: { + final Varint v = readVarint(buffer, position); + return new int[] {position, v.end()}; + } + case WIRETYPE_FIXED64: + return new int[] {position, requireWithin(buffer, position + 8)}; + case WIRETYPE_LENGTH_DELIMITED: { + final Varint length = readVarint(buffer, position); + if (length.value() < 0 || length.value() > Integer.MAX_VALUE) { + throw new MalformedMessageException("length-delimited field declares a negative or huge length"); + } + final int start = length.end(); + return new int[] {start, requireWithin(buffer, start + (int) length.value())}; + } + case WIRETYPE_FIXED32: + return new int[] {position, requireWithin(buffer, position + 4)}; + case WIRETYPE_START_GROUP: + case WIRETYPE_END_GROUP: + // Groups were removed from proto3 and no protobuf schema in use here emits them. + // Refusing is safer than skipping bytes whose extent we would have to guess. + throw new MalformedMessageException("group wire types are not supported"); + default: + throw new MalformedMessageException("unknown wire type " + wireType); + } + } + + private static int requireWithin(byte[] buffer, int end) { + if (end < 0 || end > buffer.length) { + throw new MalformedMessageException("field extends past the end of the message"); + } + return end; + } + + public static int fieldNumber(long tag) { + return (int) (tag >>> 3); + } + + public static int wireType(long tag) { + return (int) (tag & 0x7); + } + + public static byte[] slice(byte[] buffer, int from, int to) { + final byte[] copy = new byte[to - from]; + System.arraycopy(buffer, from, copy, 0, to - from); + return copy; + } + + /** + * Returns the value of the first length-delimited record with the given field + * number, for reading into a nested message or string without a schema. + * + * @param buffer the message bytes + * @param fieldNumber the field to look for + * @return the field's bytes, or null if absent + * @throws MalformedMessageException if the message is not well formed + */ + public static byte[] firstLengthDelimited(byte[] buffer, int fieldNumber) { + final List found = collect(buffer, fieldNumber, true); + return found.isEmpty() ? null : found.get(0); + } + + /** + * Returns every length-delimited record with the given field number, in order, + * as a repeated field is encoded. + * + * @param buffer the message bytes + * @param fieldNumber the field to look for + * @return the values, possibly empty + * @throws MalformedMessageException if the message is not well formed + */ + public static List allLengthDelimited(byte[] buffer, int fieldNumber) { + return collect(buffer, fieldNumber, false); + } + + private static List collect(byte[] buffer, int fieldNumber, boolean stopAtFirst) { + final List values = new ArrayList<>(); + int pos = 0; + while (pos < buffer.length) { + final Varint tag = readVarint(buffer, pos); + pos = tag.end(); + final int field = fieldNumber(tag.value()); + final int wire = wireType(tag.value()); + final int[] bounds = valueBounds(buffer, pos, wire); + if (field == fieldNumber && wire == WIRETYPE_LENGTH_DELIMITED) { + values.add(slice(buffer, bounds[0], bounds[1])); + if (stopAtFirst) { + return values; + } + } + pos = bounds[1]; + } + return values; + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ProxyCallHandler.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ProxyCallHandler.java similarity index 96% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ProxyCallHandler.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ProxyCallHandler.java index c1507e9ed9..cc31c990b3 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/ProxyCallHandler.java +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ProxyCallHandler.java @@ -35,14 +35,13 @@ * All four RPC shapes collapse into this one handler. Unary, server-streaming, * client-streaming and bidirectional calls differ only in how many messages flow * each way, which the listener callbacks below express naturally — so there is - * no need for a handler per shape, and Spark Connect's long-lived - * {@code ExecutePlan} streams work by the same code path as a unary - * {@code Config}. + * no need for a handler per shape: a long-lived server stream works by the same + * code path as a unary call. *

* Backend {@link Status} and trailers are relayed verbatim. This matters more - * than it might appear: gRPC carries {@code grpc-status} in trailers, and Spark - * Connect packs {@code google.rpc.ErrorInfo} in there too, so anything that - * interprets or drops trailers breaks error reporting wholesale. + * than it might appear: gRPC carries {@code grpc-status} in trailers, and + * protocols commonly pack structured error details in there too, so anything + * that interprets or drops trailers breaks error reporting wholesale. *

* Flow control is explicit in both directions: a message is only requested from * one side once the other side has accepted the previous one, so a slow diff --git a/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ProxyHandlerRegistry.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ProxyHandlerRegistry.java new file mode 100644 index 0000000000..78dfa39380 --- /dev/null +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/ProxyHandlerRegistry.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import io.grpc.HandlerRegistry; +import io.grpc.MethodDescriptor; +import io.grpc.ServerCallHandler; +import io.grpc.ServerMethodDefinition; + +/** + * Dispatches every call, for any method of a permitted proto service, to the + * byte-level relay. + *

+ * The gateway registers no generated service and knows no message types. A + * method is identified by its name alone — which is all gRPC puts on the wire — + * and its messages are relayed as opaque bytes. Whatever inspection a call needs + * comes from the {@link MessageInterceptorFactory}, which works on those bytes + * by field number rather than by schema. + *

+ * Two consequences worth stating. An RPC added by a newer version of the + * protocol is proxied like any other, because nothing here enumerates methods. + * And proto services not named in the permitted set are answered + * {@code UNIMPLEMENTED} — the same answer a real server gives for a method it + * does not have, so this reveals nothing about what the gateway fronts. + */ +public class ProxyHandlerRegistry extends HandlerRegistry { + + private final Set proxiedServices; + private final MessageInterceptorFactory interceptors; + private final HandlerFactory handlers; + /** Handlers are stateless per method; build each once rather than per call. */ + private final Map> methods = new ConcurrentHashMap<>(); + + /** + * Builds a fully wired handler for one method: the relay itself, plus the + * authentication, routing, authorization and audit chain around it. The + * listener supplies this so the registry needs to know nothing about backends + * or interceptor ordering. + */ + @FunctionalInterface + public interface HandlerFactory { + ServerCallHandler create(MessageInterceptor messageInterceptor); + } + + /** + * @param proxiedServices fully qualified proto service names this listener fronts + * @param interceptors chooses the per-method request handling + * @param handlers builds the relay and its interceptor chain + */ + public ProxyHandlerRegistry(Set proxiedServices, + MessageInterceptorFactory interceptors, + HandlerFactory handlers) { + this.proxiedServices = proxiedServices; + this.interceptors = interceptors; + this.handlers = handlers; + } + + @Override + public ServerMethodDefinition lookupMethod(String methodName, String authority) { + final String serviceName = MethodDescriptor.extractFullServiceName(methodName); + if (serviceName == null || !proxiedServices.contains(serviceName)) { + return null; + } + return methods.computeIfAbsent(methodName, this::createMethod); + } + + private ServerMethodDefinition createMethod(String methodName) { + final MethodDescriptor descriptor = MethodDescriptor.newBuilder() + // UNKNOWN keeps grpc from assuming a message count in either direction, so + // unary and streaming methods alike relay correctly through one handler. + .setType(MethodDescriptor.MethodType.UNKNOWN) + .setFullMethodName(methodName) + .setRequestMarshaller(ByteArrayMarshaller.INSTANCE) + .setResponseMarshaller(ByteArrayMarshaller.INSTANCE) + .build(); + return ServerMethodDefinition.create(descriptor, + handlers.create(interceptors.forMethod(methodName))); + } +} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java similarity index 78% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java index 10bb52f6ba..a0880a95be 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/RoutingInterceptor.java @@ -17,8 +17,9 @@ */ package org.apache.knox.gateway.grpc; +import java.util.function.Supplier; + import org.apache.commons.lang3.StringUtils; -import org.apache.knox.gateway.config.GatewayConfig; import org.apache.knox.gateway.i18n.messages.MessagesFactory; import org.apache.knox.gateway.services.GatewayServices; import org.apache.knox.gateway.services.ServiceType; @@ -34,10 +35,10 @@ * Selects the topology for a call and resolves its backend. *

* Knox normally routes on {@code /gateway/{topology}/{service}}, which is not - * available here: the Spark Connect connection string forbids a path component, - * and gRPC fixes request paths at {@code /pkg.Service/Method}. The topology - * therefore has to come from something else a vanilla client can send. Two - * discriminators are supported: + * available here: gRPC fixes request paths at {@code /pkg.Service/Method}, and + * connection strings for such protocols typically forbid a path component + * altogether. The topology therefore has to come from something else an + * unmodified client can send. Two discriminators are supported: *

    *
  1. a metadata entry, named by configuration and {@code knox-topology} by * default, which the client supplies as an @@ -49,8 +50,8 @@ * user may use the topology they asked for. *

    * Backend lookup then goes through the ordinary registry - * ({@code ServiceRegistry.lookupServiceURL}), so a Spark Connect backend is - * declared in topology XML like any other service. The registry treats service + * ({@code ServiceRegistry.lookupServiceURL}), so a gRPC backend is declared in + * topology XML like any other service. The registry treats service * URLs as opaque strings, which is why a {@code grpc://} URL needs no special * handling — the same property that already lets {@code ws://} URLs through. */ @@ -58,14 +59,22 @@ public class RoutingInterceptor implements ServerInterceptor { private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); - private final GatewayConfig config; + private final Supplier defaultTopology; private final GatewayServices services; private final String serviceRole; private final Metadata.Key topologyKey; - public RoutingInterceptor(GatewayConfig config, GatewayServices services, String serviceRole, - Metadata.Key topologyKey) { - this.config = config; + /** + * @param defaultTopology supplies the topology to use when a client selects + * none. Read per call rather than captured, both so a changed default + * applies without a restart and so each listener has its own. + * @param services the gateway services, for the backend registry + * @param serviceRole the role a backend is declared under + * @param topologyKey the metadata entry clients select a topology with + */ + public RoutingInterceptor(Supplier defaultTopology, GatewayServices services, + String serviceRole, Metadata.Key topologyKey) { + this.defaultTopology = defaultTopology; this.services = services; this.serviceRole = serviceRole; this.topologyKey = topologyKey; @@ -105,7 +114,7 @@ private String resolveTopology(Metadata headers) { if (StringUtils.isNotBlank(requested)) { return requested.trim(); } - return config.getSparkConnectDefaultTopology(); + return defaultTopology.get(); } private ServerCall.Listener reject(ServerCall call, diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/TokenAuthenticator.java b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/TokenAuthenticator.java similarity index 94% rename from gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/TokenAuthenticator.java rename to gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/TokenAuthenticator.java index 363ccee25a..a112bdf906 100644 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/TokenAuthenticator.java +++ b/gateway-service-grpc/src/main/java/org/apache/knox/gateway/grpc/TokenAuthenticator.java @@ -45,12 +45,12 @@ * the WebSocket listener. *

    * Bearer tokens are the whole of the credential vocabulary here, because that is - * all a vanilla Spark Connect client can carry: the {@code sc://} connection - * string offers a {@code token=} parameter, static metadata and TLS, and gRPC - * has no challenge-response step for SPNEGO to hook into. In a Kerberos - * deployment the user still authenticates with Kerberos — to the {@code - * knoxtoken} API, over HTTPS — and the resulting JWT acts as the delegation - * credential on the data path, exactly as delegation tokens do for HDFS. + * all an unmodified gRPC client can generally carry: a bearer token, static + * metadata and TLS. gRPC has no challenge-response step for SPNEGO to hook into. + * In a Kerberos deployment the user still authenticates with Kerberos — to the + * {@code knoxtoken} API, over HTTPS — and the resulting JWT acts as the + * delegation credential on the data path, exactly as delegation tokens do for + * HDFS. *

    * Validation covers issuer, expiry, not-before, signature and — when server * managed token state is on — revocation, so an administrator can kill one @@ -65,7 +65,7 @@ public class TokenAuthenticator { private static final String JWT_EXPECTED_SIGALG = "jwt.expected.sigalg"; private static final String SSO_VERIFICATION_PEM = "sso.token.verification.pem"; /** Names the signature-verification cache; not a topology lookup. */ - private static final String CACHE_NAME = "sparkconnect"; + private static final String CACHE_NAME = "grpc"; private final GatewayConfig config; private final GatewayServices services; diff --git a/gateway-service-sparkconnect/src/main/resources/META-INF/services/org.apache.knox.gateway.protocol.ProtocolListener b/gateway-service-grpc/src/main/resources/META-INF/services/org.apache.knox.gateway.protocol.ProtocolListener similarity index 94% rename from gateway-service-sparkconnect/src/main/resources/META-INF/services/org.apache.knox.gateway.protocol.ProtocolListener rename to gateway-service-grpc/src/main/resources/META-INF/services/org.apache.knox.gateway.protocol.ProtocolListener index aca5a0e902..8d08f279d5 100644 --- a/gateway-service-sparkconnect/src/main/resources/META-INF/services/org.apache.knox.gateway.protocol.ProtocolListener +++ b/gateway-service-grpc/src/main/resources/META-INF/services/org.apache.knox.gateway.protocol.ProtocolListener @@ -15,4 +15,4 @@ # See the License for the specific language governing permissions and # limitations under the License. ########################################################################## -org.apache.knox.gateway.sparkconnect.SparkConnectListener +org.apache.knox.gateway.grpc.GrpcListener diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AclAuthorizerTest.java b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/AclAuthorizerTest.java similarity index 100% rename from gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AclAuthorizerTest.java rename to gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/AclAuthorizerTest.java diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AuthenticationInterceptorTest.java b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/AuthenticationInterceptorTest.java similarity index 100% rename from gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AuthenticationInterceptorTest.java rename to gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/AuthenticationInterceptorTest.java diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AuthorizationInterceptorReloadTest.java b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/AuthorizationInterceptorReloadTest.java similarity index 100% rename from gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/AuthorizationInterceptorReloadTest.java rename to gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/AuthorizationInterceptorReloadTest.java diff --git a/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/GrpcListenerSettingsFactoryTest.java b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/GrpcListenerSettingsFactoryTest.java new file mode 100644 index 0000000000..188a3ab599 --- /dev/null +++ b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/GrpcListenerSettingsFactoryTest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.knox.gateway.GatewayTestConfig; +import org.junit.Before; +import org.junit.Test; + +/** + * How gateway configuration becomes one set of settings per listener. + *

    + * The behaviour worth pinning down is the inheritance: a deployment writes what + * every listener shares once, and repeats only what differs. Getting that wrong + * in either direction is quiet — an override that does not apply, or a shared + * value that silently does not reach a listener. + */ +public class GrpcListenerSettingsFactoryTest { + + private GatewayTestConfig config; + + @Before + public void setUp() { + config = new GatewayTestConfig(); + config.setGrpcServiceRole("SPARKCONNECT"); + config.setGrpcProtoServices("spark.connect.SparkConnectService"); + config.setGrpcIdentityRules("2.1=principal,2.2=principal"); + } + + private void listeners(String... names) { + config.setGrpcListenerNames(Arrays.asList(names)); + } + + private void set(String listener, String property, String value) { + final Map existing = + new HashMap<>(config.getGrpcListenerConfig(listener)); + existing.put(property, value); + config.setGrpcListenerConfig(listener, existing); + } + + @Test + public void namingNoListenersYieldsOneFromThePlainProperties() { + final List settings = GrpcListenerSettingsFactory.create(config); + + assertEquals(1, settings.size()); + // Named after its service role, so a single-listener deployment reads in the + // log for the thing being fronted rather than for the transport. + assertEquals("SPARKCONNECT", settings.get(0).getName()); + assertEquals("SPARKCONNECT", settings.get(0).getServiceRole()); + assertEquals("2.1=principal,2.2=principal", settings.get(0).getIdentityRules()); + assertNull("no listener configures its own keystore by default", + settings.get(0).getSslKeystorePath()); + } + + @Test + public void eachListenerInheritsWhatItDoesNotSet() { + listeners("analytics", "partner"); + set("analytics", "port", "15002"); + set("partner", "port", "15003"); + + final List settings = GrpcListenerSettingsFactory.create(config); + + assertEquals(2, settings.size()); + for (GrpcListenerSettings listener : settings) { + assertEquals("the shared service role did not reach " + listener.getName(), + "SPARKCONNECT", listener.getServiceRole()); + assertEquals("the shared identity rules did not reach " + listener.getName(), + "2.1=principal,2.2=principal", listener.getIdentityRules()); + } + assertEquals(15002, settings.get(0).getPort()); + assertEquals(15003, settings.get(1).getPort()); + } + + @Test + public void aListenerOverridesWhatItSets() { + listeners("analytics", "partner"); + set("analytics", "port", "15002"); + set("partner", "port", "15003"); + set("partner", "identity.rules", "3.1=principal"); + set("partner", "service.role", "OTHERGRPC"); + set("partner", "methods.deny", "AddArtifacts"); + + final List settings = GrpcListenerSettingsFactory.create(config); + + assertEquals("2.1=principal,2.2=principal", settings.get(0).getIdentityRules()); + assertEquals("SPARKCONNECT", settings.get(0).getServiceRole()); + assertNull(settings.get(0).getMethodsDeny()); + + assertEquals("3.1=principal", settings.get(1).getIdentityRules()); + assertEquals("OTHERGRPC", settings.get(1).getServiceRole()); + assertEquals("AddArtifacts", settings.get(1).getMethodsDeny()); + } + + @Test + public void eachListenerCanPresentItsOwnCertificate() { + // The reason several listeners exist: TLS identity is per socket, so serving + // several hostnames without a multi-name certificate needs a socket each. + listeners("analytics", "partner"); + set("analytics", "port", "15002"); + set("analytics", "ssl.keystore.path", "/opt/pki/analytics.p12"); + set("analytics", "ssl.keystore.alias", "analytics"); + set("partner", "port", "15003"); + set("partner", "ssl.keystore.path", "/opt/pki/partner.p12"); + set("partner", "ssl.keystore.password.alias", "partner_keystore_password"); + + final List settings = GrpcListenerSettingsFactory.create(config); + + assertEquals("/opt/pki/analytics.p12", settings.get(0).getSslKeystorePath()); + assertEquals("analytics", settings.get(0).getSslKeystoreAlias()); + assertNull(settings.get(0).getSslKeystorePasswordAlias()); + + assertEquals("/opt/pki/partner.p12", settings.get(1).getSslKeystorePath()); + assertNull("an alias is optional where the keystore holds one key entry", + settings.get(1).getSslKeystoreAlias()); + assertEquals("partner_keystore_password", settings.get(1).getSslKeystorePasswordAlias()); + } + + @Test + public void keystorePathsAreNeverInherited() { + // Sharing one keystore across listeners would defeat the point of having + // several, so this is the one property that has no plain-property fallback. + listeners("analytics", "partner"); + set("analytics", "port", "15002"); + set("analytics", "ssl.keystore.path", "/opt/pki/analytics.p12"); + set("partner", "port", "15003"); + + final List settings = GrpcListenerSettingsFactory.create(config); + + assertEquals("/opt/pki/analytics.p12", settings.get(0).getSslKeystorePath()); + assertNull("partner should fall back to the gateway identity, not analytics' keystore", + settings.get(1).getSslKeystorePath()); + } + + @Test + public void refusesTwoListenersOnOnePort() { + // Otherwise whichever lost the race would fail with an address-in-use error + // naming neither listener. + listeners("analytics", "partner"); + set("analytics", "port", "15002"); + set("partner", "port", "15002"); + + assertRejected("both configured on port 15002"); + } + + @Test + public void refusesANameThatCollidesWithAPlainProperty() { + // gateway.grpc.identity.rules and a listener called 'identity' would occupy + // the same namespace. + listeners("identity"); + assertRejected("already a configuration property"); + } + + @Test + public void refusesUnusableAndDuplicateNames() { + listeners("Analytics"); + assertRejected("may contain only"); + + listeners("analytics", "analytics"); + assertRejected("Duplicate"); + } + + @Test + public void refusesANonNumericNumericProperty() { + listeners("analytics"); + set("analytics", "port", "not-a-port"); + assertRejected("must be a number"); + } + + private void assertRejected(String expectedFragment) { + try { + GrpcListenerSettingsFactory.create(config); + fail("Expected the configuration to be rejected"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains(expectedFragment)); + } + } +} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/GrpcMetadataKeysTest.java b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/GrpcMetadataKeysTest.java similarity index 100% rename from gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/GrpcMetadataKeysTest.java rename to gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/GrpcMetadataKeysTest.java diff --git a/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/IdentityRewritePolicyTest.java b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/IdentityRewritePolicyTest.java new file mode 100644 index 0000000000..8091ce17ec --- /dev/null +++ b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/IdentityRewritePolicyTest.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; + +import org.junit.Test; + +/** + * Rewrite behaviour with no protocol in sight. + *

    + * The oracle test proves the wire code matches typed semantics for one real + * protocol. This one covers what the rules mechanism is supposed to do in + * general: arbitrary depth, several rules at once, synthesis of an absent path, + * and the refusals — because none of that should need a {@code .proto} file to + * exercise, and a protocol-shaped test would quietly re-import the assumptions + * this design exists to drop. + */ +public class IdentityRewritePolicyTest { + + private static final String PRINCIPAL = "alice"; + private static final String SPOOFED = "root"; + + private static byte[] message(byte[]... records) { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (byte[] record : records) { + out.write(record, 0, record.length); + } + return out.toByteArray(); + } + + private static byte[] stringField(int field, String value) { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + ProtoWire.writeLengthDelimited(out, field, value.getBytes(StandardCharsets.UTF_8)); + return out.toByteArray(); + } + + private static byte[] messageField(int field, byte[] nested) { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + ProtoWire.writeLengthDelimited(out, field, nested); + return out.toByteArray(); + } + + private static byte[] varintField(int field, long value) { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + ProtoWire.writeTag(out, field, ProtoWire.WIRETYPE_VARINT); + ProtoWire.writeVarint(out, value); + return out.toByteArray(); + } + + private static byte[] rewrite(String rules, byte[] request) { + return rewrite(rules, IdentityRewritePolicy.DEFAULT_SCAN_LIMIT, request); + } + + private static byte[] rewrite(String rules, int scanLimit, byte[] request) { + return new IdentityAssertingInterceptor(IdentityRewritePolicy.parse(rules, scanLimit)) + .assertIdentity(request, PRINCIPAL); + } + + /** Reads a string field, following a dotted path of field numbers. */ + private static String read(byte[] buffer, int... path) { + byte[] current = buffer; + for (int depth = 0; depth < path.length - 1; depth++) { + current = ProtoWire.firstLengthDelimited(current, path[depth]); + if (current == null) { + return null; + } + } + final byte[] value = ProtoWire.firstLengthDelimited(current, path[path.length - 1]); + return value == null ? null : new String(value, StandardCharsets.UTF_8); + } + + @Test + public void rewritesATopLevelField() { + // Depth one: a protocol with a flat identity field, which the container-plus- + // id shape could not express at all. + final byte[] rewritten = rewrite("1=principal", stringField(1, SPOOFED)); + assertEquals(PRINCIPAL, read(rewritten, 1)); + } + + @Test + public void rewritesArbitrarilyDeepPaths() { + final byte[] request = messageField(4, messageField(3, messageField(2, stringField(1, SPOOFED)))); + final byte[] rewritten = rewrite("4.3.2.1=principal", request); + assertEquals(PRINCIPAL, read(rewritten, 4, 3, 2, 1)); + } + + @Test + public void appliesSeveralRulesInOnePass() { + final byte[] request = messageField(2, message(stringField(1, SPOOFED), stringField(2, SPOOFED))); + final byte[] rewritten = rewrite("2.1=principal,2.2=principal", request); + assertEquals(PRINCIPAL, read(rewritten, 2, 1)); + assertEquals(PRINCIPAL, read(rewritten, 2, 2)); + } + + @Test + public void synthesisesTheWholeChainWhenTheIdentityIsAbsent() { + // Nothing to overwrite, so the container, its parent and the leaf are all + // created: the backend must never see a request whose identity Knox did not + // put there. + final byte[] rewritten = rewrite("4.3.1=principal", stringField(9, "unrelated")); + assertEquals(PRINCIPAL, read(rewritten, 4, 3, 1)); + assertEquals("unrelated", read(rewritten, 9)); + } + + @Test + public void leavesEveryOtherByteAlone() { + final byte[] untouched = message( + stringField(1, "session"), + varintField(7, 4242), + stringField(9, "trailing")); + final byte[] request = message(untouched, messageField(2, stringField(1, SPOOFED))); + + final byte[] rewritten = rewrite("2.1=principal", request); + + assertEquals("session", read(rewritten, 1)); + assertEquals("trailing", read(rewritten, 9)); + assertArrayEquals("bytes outside the identity path were not copied verbatim", + untouched, java.util.Arrays.copyOfRange(rewritten, 0, untouched.length)); + } + + @Test + public void isIdempotent() { + final byte[] request = messageField(2, message(stringField(1, SPOOFED), stringField(2, SPOOFED))); + final byte[] once = rewrite("2.1=principal,2.2=principal", request); + final byte[] twice = rewrite("2.1=principal,2.2=principal", once); + assertArrayEquals(once, twice); + } + + @Test + public void refusesAFieldWhoseWireTypeContradictsTheRules() { + // Field 2 is a varint here, so it is neither a value to replace nor a message + // to descend into. Skipping it would forward the caller's own claim. + try { + rewrite("2.1=principal", varintField(2, 1)); + fail("Expected a wire type mismatch to be refused"); + } catch (IdentityAssertingInterceptor.UnassertableMessageException e) { + assertTrue(e.getMessage(), e.getMessage().contains("do not describe this message")); + } + } + + @Test + public void refusesAnIdentityBeyondTheScanLimit() { + final StringBuilder padding = new StringBuilder(); + for (int i = 0; i < 500; i++) { + padding.append('x'); + } + final byte[] request = message( + stringField(1, padding.toString()), + messageField(2, stringField(1, SPOOFED))); + + try { + rewrite("2.1=principal", 128, request); + fail("Expected an identity beyond the scan limit to be refused"); + } catch (IdentityAssertingInterceptor.UnassertableMessageException e) { + assertTrue(e.getMessage(), e.getMessage().contains("extends past the first 128 bytes")); + } + } + + @Test + public void synthesisesBeyondTheScanLimitWhenNoIdentityIsPresent() { + // Nothing was found anywhere, so there is nothing an appended identity could + // be overridden by, and the limit has nothing to say about it. + final StringBuilder padding = new StringBuilder(); + for (int i = 0; i < 500; i++) { + padding.append('x'); + } + final byte[] rewritten = + rewrite("2.1=principal", 128, stringField(1, padding.toString())); + assertEquals(PRINCIPAL, read(rewritten, 2, 1)); + } + + @Test + public void anEmptyRuleSetRewritesNothing() { + assertTrue(IdentityRewritePolicy.parse(null, 1024).isEmpty()); + assertTrue(IdentityRewritePolicy.parse(" ", 1024).isEmpty()); + final byte[] request = stringField(1, SPOOFED); + assertArrayEquals(request, rewrite("", request)); + } + + @Test + public void rejectsRulesThatContradictEachOther() { + assertRejected("2=principal,2.1=principal", "writes a value to"); + assertRejected("2.1=principal,2.1=principal", "collides"); + } + + @Test + public void rejectsMalformedRules() { + assertRejected("2.1", "path=subject"); + assertRejected("2.1=", "must name a subject"); + assertRejected("2.1=nonsense", "Unknown identity subject"); + assertRejected("=principal", "at least one field number"); + assertRejected("2.x=principal", "only field numbers"); + assertRejected("0=principal", "between 1 and"); + assertRejected("19001=principal", "reserved by protobuf"); + } + + @Test + public void rejectsANonPositiveScanLimit() { + try { + IdentityRewritePolicy.parse("2.1=principal", 0); + fail("Expected a non-positive scan limit to be rejected"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains("must be positive")); + } + } + + private static void assertRejected(String rules, String expectedFragment) { + try { + IdentityRewritePolicy.parse(rules, IdentityRewritePolicy.DEFAULT_SCAN_LIMIT); + fail("Expected '" + rules + "' to be rejected"); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().contains(expectedFragment)); + } + } +} diff --git a/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/MethodAccessPolicyTest.java b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/MethodAccessPolicyTest.java new file mode 100644 index 0000000000..0e33a6e329 --- /dev/null +++ b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/MethodAccessPolicyTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.grpc; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Coarse authorization by method name, which needs no schema at all: gRPC puts + * the method in the request path, so this works on a purely byte-level proxy. + */ +public class MethodAccessPolicyTest { + + private static final String ADD_ARTIFACTS = "spark.connect.SparkConnectService/AddArtifacts"; + private static final String EXECUTE_PLAN = "spark.connect.SparkConnectService/ExecutePlan"; + + @Test + public void permitsEverythingWhenUnconfigured() { + assertTrue(MethodAccessPolicy.allowAll().isUnrestricted()); + assertTrue(MethodAccessPolicy.allowAll().isPermitted(ADD_ARTIFACTS)); + assertTrue(MethodAccessPolicy.of(null, null).isPermitted(ADD_ARTIFACTS)); + assertTrue(MethodAccessPolicy.of("", " ").isPermitted(ADD_ARTIFACTS)); + } + + @Test + public void deniesByBareMethodName() { + // The form an operator would most naturally write. + final MethodAccessPolicy policy = MethodAccessPolicy.of("AddArtifacts", null); + assertFalse(policy.isPermitted(ADD_ARTIFACTS)); + assertTrue(policy.isPermitted(EXECUTE_PLAN)); + } + + @Test + public void deniesByFullyQualifiedName() { + final MethodAccessPolicy policy = MethodAccessPolicy.of(ADD_ARTIFACTS, null); + assertFalse(policy.isPermitted(ADD_ARTIFACTS)); + assertTrue(policy.isPermitted(EXECUTE_PLAN)); + } + + @Test + public void matchingIsCaseInsensitive() { + final MethodAccessPolicy policy = MethodAccessPolicy.of("addartifacts", null); + assertFalse(policy.isPermitted(ADD_ARTIFACTS)); + } + + @Test + public void deniesSeveralMethods() { + final MethodAccessPolicy policy = MethodAccessPolicy.of("AddArtifacts, ArtifactStatus", null); + assertFalse(policy.isPermitted(ADD_ARTIFACTS)); + assertFalse(policy.isPermitted("spark.connect.SparkConnectService/ArtifactStatus")); + assertTrue(policy.isPermitted(EXECUTE_PLAN)); + } + + @Test + public void anAllowListIsExhaustive() { + // An RPC added by a newer protocol version must not appear by default just + // because nobody thought to deny it. + final MethodAccessPolicy policy = MethodAccessPolicy.of(null, "ExecutePlan, AnalyzePlan"); + assertTrue(policy.isPermitted(EXECUTE_PLAN)); + assertTrue(policy.isPermitted("spark.connect.SparkConnectService/AnalyzePlan")); + assertFalse(policy.isPermitted(ADD_ARTIFACTS)); + assertFalse(policy.isPermitted("spark.connect.SparkConnectService/SomeRpcFromTheFuture")); + } + + @Test + public void denyBeatsAllow() { + final MethodAccessPolicy policy = + MethodAccessPolicy.of("AddArtifacts", "AddArtifacts, ExecutePlan"); + assertFalse("an explicit denial should win", policy.isPermitted(ADD_ARTIFACTS)); + assertTrue(policy.isPermitted(EXECUTE_PLAN)); + } + + @Test + public void handlesAMethodNameWithoutAService() { + final MethodAccessPolicy policy = MethodAccessPolicy.of("AddArtifacts", null); + assertFalse(policy.isPermitted("AddArtifacts")); + assertTrue(policy.isPermitted("SomethingElse")); + } + + @Test + public void aConfiguredPolicyIsNotUnrestricted() { + assertFalse(MethodAccessPolicy.of("AddArtifacts", null).isUnrestricted()); + assertFalse(MethodAccessPolicy.of(null, "ExecutePlan").isUnrestricted()); + } +} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/TopologySelectionAuthorizationTest.java b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/TopologySelectionAuthorizationTest.java similarity index 100% rename from gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/grpc/TopologySelectionAuthorizationTest.java rename to gateway-service-grpc/src/test/java/org/apache/knox/gateway/grpc/TopologySelectionAuthorizationTest.java diff --git a/gateway-service-grpc/src/test/java/org/apache/knox/gateway/sparkconnect/IdentityAssertionOracleTest.java b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/sparkconnect/IdentityAssertionOracleTest.java new file mode 100644 index 0000000000..11bec37a0f --- /dev/null +++ b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/sparkconnect/IdentityAssertionOracleTest.java @@ -0,0 +1,333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.knox.gateway.sparkconnect; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.apache.knox.gateway.grpc.IdentityAssertingInterceptor; +import org.apache.knox.gateway.grpc.IdentityRewritePolicy; +import org.apache.knox.gateway.grpc.ProtoWire; + +import com.google.protobuf.Any; +import com.google.protobuf.ByteString; +import com.google.protobuf.Message; +import com.google.protobuf.UnknownFieldSet; + +import org.apache.spark.connect.proto.AddArtifactsRequest; +import org.apache.spark.connect.proto.AnalyzePlanRequest; +import org.apache.spark.connect.proto.CloneSessionRequest; +import org.apache.spark.connect.proto.ConfigRequest; +import org.apache.spark.connect.proto.ExecutePlanRequest; +import org.apache.spark.connect.proto.FetchErrorDetailsRequest; +import org.apache.spark.connect.proto.GetStatusRequest; +import org.apache.spark.connect.proto.InterruptRequest; +import org.apache.spark.connect.proto.KeyValue; +import org.apache.spark.connect.proto.ReattachExecuteRequest; +import org.apache.spark.connect.proto.ReleaseExecuteRequest; +import org.apache.spark.connect.proto.ReleaseSessionRequest; +import org.apache.spark.connect.proto.UserContext; + +import org.junit.Test; + +/** + * Checks the schema-free identity rewrite against generated classes. + *

    + * The gateway itself compiles against no Spark Connect protos and rewrites + * requests by field number alone. These protos exist only here, as an oracle: + * they say what the wire bytes are supposed to mean, so the hand-written wire + * code can be held to typed semantics. A Spark release that moved the fields + * this depends on would fail here rather than in production. + */ +public class IdentityAssertionOracleTest { + + private static final String PRINCIPAL = "alice"; + private static final String SPOOFED = "root"; + + private final IdentityAssertingInterceptor interceptor = + new IdentityAssertingInterceptor( + IdentityRewritePolicy.parse("2.1=principal,2.2=principal", + IdentityRewritePolicy.DEFAULT_SCAN_LIMIT)); + + private static UserContext spoofedIdentity() { + return UserContext.newBuilder() + .setUserId(SPOOFED) + .setUserName(SPOOFED) + .addExtensions(Any.newBuilder().setTypeUrl("type/keep").build()) + .build(); + } + + /** + * Rewrites the message, reparses it with the generated class, and asserts both + * that the identity was replaced and that nothing else changed. + */ + private void assertRewrite(Message original) { + final byte[] rewritten = interceptor.assertIdentity(original.toByteArray(), PRINCIPAL); + + final Message reparsed; + try { + reparsed = original.getParserForType().parseFrom(rewritten); + } catch (Exception e) { + throw new AssertionError(original.getDescriptorForType().getName() + + " did not survive the rewrite as valid protobuf", e); + } + + final UserContext identity = (UserContext) reparsed.getField( + reparsed.getDescriptorForType().findFieldByName("user_context")); + assertEquals("user_id was not asserted", PRINCIPAL, identity.getUserId()); + assertEquals("user_name was not asserted", PRINCIPAL, identity.getUserName()); + + // Blank out the identity on both sides; everything remaining must be equal. + final Message.Builder before = original.toBuilder(); + final Message.Builder after = reparsed.toBuilder(); + before.setField(before.getDescriptorForType().findFieldByName("user_context"), + UserContext.getDefaultInstance()); + after.setField(after.getDescriptorForType().findFieldByName("user_context"), + UserContext.getDefaultInstance()); + assertEquals("the rewrite changed something other than the identity", + before.build(), after.build()); + } + + @Test + public void assertsIdentityOnEveryRequestShape() { + final UserContext spoof = spoofedIdentity(); + assertRewrite(ExecutePlanRequest.newBuilder().setSessionId("s").setUserContext(spoof).build()); + assertRewrite(AnalyzePlanRequest.newBuilder().setSessionId("s").setUserContext(spoof).build()); + assertRewrite(ConfigRequest.newBuilder().setSessionId("s").setUserContext(spoof).build()); + assertRewrite(AddArtifactsRequest.newBuilder().setSessionId("s").setUserContext(spoof).build()); + assertRewrite(InterruptRequest.newBuilder().setSessionId("s").setUserContext(spoof).build()); + assertRewrite(ReattachExecuteRequest.newBuilder().setSessionId("s").setUserContext(spoof).build()); + assertRewrite(ReleaseExecuteRequest.newBuilder().setSessionId("s").setUserContext(spoof).build()); + assertRewrite(ReleaseSessionRequest.newBuilder().setSessionId("s").setUserContext(spoof).build()); + assertRewrite(FetchErrorDetailsRequest.newBuilder().setSessionId("s").setUserContext(spoof).build()); + assertRewrite(CloneSessionRequest.newBuilder().setSessionId("s").setUserContext(spoof).build()); + assertRewrite(GetStatusRequest.newBuilder().setSessionId("s").setUserContext(spoof).build()); + } + + @Test + public void preservesEveryOtherField() { + assertRewrite(ExecutePlanRequest.newBuilder() + .setSessionId("session-1") + .setOperationId("operation-1") + .setClientType("pyspark") + .addTags("tag-a") + .addTags("tag-b") + .setUserContext(spoofedIdentity()) + .build()); + } + + @Test + public void preservesExtensionsInsideTheIdentityContainer() { + final ExecutePlanRequest request = ExecutePlanRequest.newBuilder() + .setSessionId("s").setUserContext(spoofedIdentity()).build(); + + final byte[] rewritten = interceptor.assertIdentity(request.toByteArray(), PRINCIPAL); + final ExecutePlanRequest reparsed; + try { + reparsed = ExecutePlanRequest.parseFrom(rewritten); + } catch (Exception e) { + throw new AssertionError(e); + } + // Extensions belong to the client; only the identity fields are ours. + assertEquals(1, reparsed.getUserContext().getExtensionsCount()); + assertEquals("type/keep", reparsed.getUserContext().getExtensions(0).getTypeUrl()); + } + + @Test + public void synthesisesAnIdentityWhenTheClientSendsNone() { + assertRewrite(ExecutePlanRequest.newBuilder().setSessionId("s").build()); + } + + @Test + public void assertsOverAnEmptyIdentity() { + assertRewrite(ExecutePlanRequest.newBuilder().setSessionId("s") + .setUserContext(UserContext.getDefaultInstance()).build()); + } + + @Test + public void handlesAPayloadLargeEnoughToNeedMultiByteLengths() { + final StringBuilder big = new StringBuilder(); + for (int i = 0; i < 50000; i++) { + big.append("xxxx"); + } + assertRewrite(ExecutePlanRequest.newBuilder().setSessionId("s") + .setUserContext(spoofedIdentity()).setClientType(big.toString()).build()); + } + + @Test + public void preservesFieldsFromANewerProtocolVersion() { + // A field this build has never heard of. It is not merely retained -- the + // wire rewrite never decodes it, so its bytes are copied verbatim. + final UnknownFieldSet unknown = UnknownFieldSet.newBuilder() + .addField(4242, UnknownFieldSet.Field.newBuilder() + .addVarint(7) + .addLengthDelimited(ByteString.copyFromUtf8("from-the-future")) + .build()) + .build(); + final ExecutePlanRequest request = ExecutePlanRequest.newBuilder() + .setSessionId("s").setUserContext(spoofedIdentity()).build() + .toBuilder().setUnknownFields(unknown).build(); + + assertRewrite(request); + + final byte[] rewritten = interceptor.assertIdentity(request.toByteArray(), PRINCIPAL); + final ExecutePlanRequest reparsed; + try { + reparsed = ExecutePlanRequest.parseFrom(rewritten); + } catch (Exception e) { + throw new AssertionError(e); + } + assertEquals(7L, reparsed.getUnknownFields().getField(4242).getVarintList().get(0).longValue()); + assertEquals(ByteString.copyFromUtf8("from-the-future"), + reparsed.getUnknownFields().getField(4242).getLengthDelimitedList().get(0)); + } + + @Test + public void leavesConfigOperationsUntouched() { + // Config carries a nested operation the gateway knows nothing about; the + // rewrite must leave every byte of it alone. + final ConfigRequest request = ConfigRequest.newBuilder() + .setSessionId("s") + .setUserContext(spoofedIdentity()) + .setOperation(ConfigRequest.Operation.newBuilder() + .setSet(ConfigRequest.Set.newBuilder() + .addPairs(KeyValue.newBuilder().setKey("spark.sql.shuffle.partitions").setValue("8")))) + .build(); + + assertRewrite(request); + } + + @Test + public void rejectsBytesThatAreNotAProtobufMessage() { + // Truncated mid-varint. Forwarding would send the client's own claim on + // untouched, so this has to fail rather than pass through. + final byte[] truncated = {(byte) 0x92, (byte) 0x80}; + try { + interceptor.assertIdentity(truncated, PRINCIPAL); + fail("Expected malformed input to be rejected"); + } catch (ProtoWire.MalformedMessageException e) { + assertEquals("varint runs past the end of the message", e.getMessage()); + } + } + + @Test + public void isIdempotent() { + final ExecutePlanRequest request = ExecutePlanRequest.newBuilder() + .setSessionId("s").setUserContext(spoofedIdentity()).build(); + + final byte[] once = interceptor.assertIdentity(request.toByteArray(), PRINCIPAL); + final byte[] twice = interceptor.assertIdentity(once, PRINCIPAL); + + assertArrayEquals("rewriting an already-rewritten message should change nothing", once, twice); + } + + @Test + public void acceptsALargeRequestWhoseIdentityIsAtTheFront() { + // The scan limit bounds where the identity may sit, not how big a request may + // be. Generated serializers emit fields in ascending number order, so + // user_context = 2 lands near the front however large the payload after it. + final StringBuilder payload = new StringBuilder(); + for (int i = 0; i < 100000; i++) { + payload.append("xxxxxxxxxx"); + } + assertRewrite(ExecutePlanRequest.newBuilder() + .setSessionId("s") + .setUserContext(spoofedIdentity()) + .setClientType(payload.toString()) + .build()); + } + + @Test + public void refusesAnIdentityThatSitsBeyondTheScanLimit() { + // Everything before user_context pushes it past a deliberately tiny limit. + final StringBuilder prefix = new StringBuilder(); + for (int i = 0; i < 200; i++) { + prefix.append('p'); + } + final ExecutePlanRequest request = ExecutePlanRequest.newBuilder() + .setSessionId(prefix.toString()) + .setUserContext(spoofedIdentity()) + .build(); + + try { + interceptorWithScanLimit(64).assertIdentity(request.toByteArray(), PRINCIPAL); + fail("Expected an identity beyond the scan limit to be refused"); + } catch (IdentityAssertingInterceptor.UnassertableMessageException e) { + assertTrue(e.getMessage(), e.getMessage().contains("extends past the first 64 bytes")); + } + } + + @Test + public void refusesAnIdentityContainerLargerThanTheScanLimit() { + // The container starts at the very front but runs long. Measuring the limit + // against where a field ends is what bounds the copying the rewrite does. + final StringBuilder padding = new StringBuilder(); + for (int i = 0; i < 4096; i++) { + padding.append('x'); + } + final ExecutePlanRequest request = ExecutePlanRequest.newBuilder() + .setUserContext(UserContext.newBuilder() + .setUserId(SPOOFED) + .setUserName(padding.toString()) + .build()) + .build(); + + try { + interceptorWithScanLimit(1024).assertIdentity(request.toByteArray(), PRINCIPAL); + fail("Expected an oversized identity container to be refused"); + } catch (IdentityAssertingInterceptor.UnassertableMessageException e) { + assertTrue(e.getMessage(), e.getMessage().contains("extends past the first 1024 bytes")); + } + } + + @Test + public void rewritesEveryOccurrenceOfARepeatedIdentityContainer() { + // Two user_context records on the wire. Protobuf merges them, so one left + // unasserted would override the one that was. + final byte[] doubled = concat( + ExecutePlanRequest.newBuilder().setSessionId("s") + .setUserContext(spoofedIdentity()).build().toByteArray(), + ExecutePlanRequest.newBuilder() + .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()) + .build().toByteArray()); + + final byte[] rewritten = interceptor.assertIdentity(doubled, PRINCIPAL); + final ExecutePlanRequest reparsed; + try { + reparsed = ExecutePlanRequest.parseFrom(rewritten); + } catch (Exception e) { + throw new AssertionError(e); + } + assertEquals("a repeated container let a spoofed identity survive the merge", + PRINCIPAL, reparsed.getUserContext().getUserId()); + } + + private static byte[] concat(byte[] first, byte[] second) { + final byte[] joined = new byte[first.length + second.length]; + System.arraycopy(first, 0, joined, 0, first.length); + System.arraycopy(second, 0, joined, first.length, second.length); + return joined; + } + + private static IdentityAssertingInterceptor interceptorWithScanLimit(int limit) { + return new IdentityAssertingInterceptor( + IdentityRewritePolicy.parse("2.1=principal,2.2=principal", limit)); + } +} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectProxyIntegrationTest.java b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectProxyTest.java similarity index 83% rename from gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectProxyIntegrationTest.java rename to gateway-service-grpc/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectProxyTest.java index 0333abfda3..e9c92952cd 100644 --- a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectProxyIntegrationTest.java +++ b/gateway-service-grpc/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectProxyTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.fail; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -30,23 +31,22 @@ import org.apache.knox.gateway.grpc.GrpcCallContext; import org.apache.knox.gateway.grpc.GrpcMetadataKeys; import org.apache.knox.gateway.grpc.HeaderRewriter; +import org.apache.knox.gateway.grpc.IdentityAssertingInterceptor; +import org.apache.knox.gateway.grpc.IdentityRewritePolicy; import org.apache.knox.gateway.grpc.MessageInterceptor; +import org.apache.knox.gateway.grpc.ProxyHandlerRegistry; import org.apache.knox.gateway.grpc.ProxyCallHandler; -import com.google.protobuf.Message; - import io.grpc.Context; +import io.grpc.HandlerRegistry; import io.grpc.Contexts; import io.grpc.ManagedChannel; import io.grpc.Metadata; -import io.grpc.MethodDescriptor; import io.grpc.Server; import io.grpc.ServerCall; import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; import io.grpc.ServerInterceptors; -import io.grpc.ServerMethodDefinition; -import io.grpc.ServerServiceDefinition; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.inprocess.InProcessChannelBuilder; @@ -79,7 +79,7 @@ // volatile: the stub's knobs are set by the test thread and read by the gRPC // server threads handling the call. @SuppressWarnings("PMD.AvoidUsingVolatile") -public class SparkConnectProxyIntegrationTest { +public class SparkConnectProxyTest { /** Stands in for the trailer Spark uses to carry structured error details. */ private static final Metadata.Key ERROR_DETAILS = @@ -110,7 +110,7 @@ public void setUp() throws Exception { backendChannel = InProcessChannelBuilder.forName(backendName).build(); gateway = InProcessServerBuilder.forName(gatewayName) - .addService(proxyService(identityAsserting())) + .fallbackHandlerRegistry(proxyRegistry(identityAsserting())) .build() .start(); clientChannel = InProcessChannelBuilder.forName(gatewayName).build(); @@ -136,43 +136,43 @@ private static void shutdown(Server server) { } } - private MessageInterceptor identityAsserting() { - return new SparkConnectMessageInterceptor(null); + private MessageInterceptor identityAsserting() { + return new IdentityAssertingInterceptor( + IdentityRewritePolicy.parse("2.1=principal,2.2=principal", + IdentityRewritePolicy.DEFAULT_SCAN_LIMIT)); } /** - * Registers a relay for every Spark Connect method, inside a context carrying - * the principal and backend the interceptor chain would normally have - * resolved. The chain itself is covered by its own tests. + * Stands up the gateway exactly as it runs: a byte-level registry with no + * generated service registered, so the relay never sees a typed message. The + * client and the stand-in backend both use generated stubs, which is the point + * — a schema-free proxy has to be invisible to schema-aware peers. + *

    + * Calls run inside a context carrying the principal and backend the interceptor + * chain would normally have resolved; the chain itself has its own tests. */ - private ServerServiceDefinition proxyService(MessageInterceptor messageInterceptor) { + private HandlerRegistry proxyRegistry(MessageInterceptor messageInterceptor) { final BackendChannelProvider channels = () -> backendChannel; final HeaderRewriter headers = metadata -> metadata.removeAll(GrpcMetadataKeys.AUTHORIZATION); - final ServerServiceDefinition.Builder builder = - ServerServiceDefinition.builder(SparkConnectServiceGrpc.getServiceDescriptor()); - for (MethodDescriptor method : SparkConnectServiceGrpc.getServiceDescriptor().getMethods()) { - @SuppressWarnings("unchecked") - final MethodDescriptor descriptor = (MethodDescriptor) method; - final ProxyCallHandler handler = - new ProxyCallHandler<>(channels, messageInterceptor, headers); - - builder.addMethod(ServerMethodDefinition.create(descriptor, (call, metadata) -> { - final GrpcCallContext callContext = new GrpcCallContext( - descriptor.getFullMethodName(), "test", "127.0.0.1", System.nanoTime()); - callContext.setPrincipal(PRINCIPAL); - callContext.setBackendUrl("grpc://backend:15002"); - // Contexts.interceptCall, exactly as the audit interceptor uses it: it - // attaches the context to the listener callbacks too, not just to - // startCall. Request messages arrive in onMessage, well after startCall - // returns, so anything that only wrapped startCall would leave the - // handler without a principal at the moment it needs one. - return Contexts.interceptCall( - Context.current().withValue(GrpcCallContext.KEY, callContext), - call, metadata, handler); - })); - } - return builder.build(); + return new ProxyHandlerRegistry( + Collections.singleton("spark.connect.SparkConnectService"), + methodName -> messageInterceptor, + relay -> (call, metadata) -> { + final GrpcCallContext callContext = new GrpcCallContext( + call.getMethodDescriptor().getFullMethodName(), "test", "127.0.0.1", System.nanoTime()); + callContext.setPrincipal(PRINCIPAL); + callContext.setBackendUrl("grpc://backend:15002"); + // Contexts.interceptCall, exactly as the audit interceptor uses it: it + // attaches the context to the listener callbacks too, not just to + // startCall. Request messages arrive in onMessage, well after startCall + // returns, so anything that only wrapped startCall would leave the + // handler without a principal at the moment it needs one. + return Contexts.interceptCall( + Context.current().withValue(GrpcCallContext.KEY, callContext), + call, metadata, + new ProxyCallHandler<>(channels, messageInterceptor, headers)); + }); } @Test @@ -275,12 +275,12 @@ public void relaysUnimplementedForAMethodTheBackendLacks() { @Test public void rejectsAGatedCallWithoutContactingTheBackend() throws Exception { final String name = InProcessServerBuilder.generateName(); - final MessageInterceptor denying = message -> { + final MessageInterceptor denying = message -> { throw Status.PERMISSION_DENIED.withDescription("nope").asRuntimeException(); }; final Server gatingGateway = InProcessServerBuilder.forName(name) - .addService(proxyService(denying)).build().start(); + .fallbackHandlerRegistry(proxyRegistry(denying)).build().start(); final ManagedChannel gatingClient = InProcessChannelBuilder.forName(name).build(); try { final int before = stub.analyzeRequests.size(); diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/README.md b/gateway-service-grpc/src/test/proto/spark/connect/README.md similarity index 100% rename from gateway-service-sparkconnect/src/main/proto/spark/connect/README.md rename to gateway-service-grpc/src/test/proto/spark/connect/README.md diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/base.proto b/gateway-service-grpc/src/test/proto/spark/connect/base.proto similarity index 100% rename from gateway-service-sparkconnect/src/main/proto/spark/connect/base.proto rename to gateway-service-grpc/src/test/proto/spark/connect/base.proto diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/catalog.proto b/gateway-service-grpc/src/test/proto/spark/connect/catalog.proto similarity index 100% rename from gateway-service-sparkconnect/src/main/proto/spark/connect/catalog.proto rename to gateway-service-grpc/src/test/proto/spark/connect/catalog.proto diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/commands.proto b/gateway-service-grpc/src/test/proto/spark/connect/commands.proto similarity index 100% rename from gateway-service-sparkconnect/src/main/proto/spark/connect/commands.proto rename to gateway-service-grpc/src/test/proto/spark/connect/commands.proto diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/common.proto b/gateway-service-grpc/src/test/proto/spark/connect/common.proto similarity index 100% rename from gateway-service-sparkconnect/src/main/proto/spark/connect/common.proto rename to gateway-service-grpc/src/test/proto/spark/connect/common.proto diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/expressions.proto b/gateway-service-grpc/src/test/proto/spark/connect/expressions.proto similarity index 100% rename from gateway-service-sparkconnect/src/main/proto/spark/connect/expressions.proto rename to gateway-service-grpc/src/test/proto/spark/connect/expressions.proto diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/ml.proto b/gateway-service-grpc/src/test/proto/spark/connect/ml.proto similarity index 100% rename from gateway-service-sparkconnect/src/main/proto/spark/connect/ml.proto rename to gateway-service-grpc/src/test/proto/spark/connect/ml.proto diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/ml_common.proto b/gateway-service-grpc/src/test/proto/spark/connect/ml_common.proto similarity index 100% rename from gateway-service-sparkconnect/src/main/proto/spark/connect/ml_common.proto rename to gateway-service-grpc/src/test/proto/spark/connect/ml_common.proto diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/pipelines.proto b/gateway-service-grpc/src/test/proto/spark/connect/pipelines.proto similarity index 100% rename from gateway-service-sparkconnect/src/main/proto/spark/connect/pipelines.proto rename to gateway-service-grpc/src/test/proto/spark/connect/pipelines.proto diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/relations.proto b/gateway-service-grpc/src/test/proto/spark/connect/relations.proto similarity index 100% rename from gateway-service-sparkconnect/src/main/proto/spark/connect/relations.proto rename to gateway-service-grpc/src/test/proto/spark/connect/relations.proto diff --git a/gateway-service-sparkconnect/src/main/proto/spark/connect/types.proto b/gateway-service-grpc/src/test/proto/spark/connect/types.proto similarity index 100% rename from gateway-service-sparkconnect/src/main/proto/spark/connect/types.proto rename to gateway-service-grpc/src/test/proto/spark/connect/types.proto diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayListener.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayListener.java deleted file mode 100644 index 887b3837f3..0000000000 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcGatewayListener.java +++ /dev/null @@ -1,359 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.knox.gateway.grpc; - -import java.security.Key; -import java.security.KeyStore; -import java.security.PrivateKey; -import java.security.cert.Certificate; -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import javax.net.ssl.KeyManagerFactory; - -import org.apache.knox.gateway.config.GatewayConfig; -import org.apache.knox.gateway.i18n.messages.MessagesFactory; -import org.apache.knox.gateway.protocol.ProtocolListener; -import org.apache.knox.gateway.services.GatewayServices; -import org.apache.knox.gateway.services.ServiceType; -import org.apache.knox.gateway.services.security.AliasService; -import org.apache.knox.gateway.services.security.KeystoreService; -import org.apache.knox.gateway.services.topology.TopologyService; -import org.apache.knox.gateway.topology.Service; -import org.apache.knox.gateway.topology.Topology; - -import io.grpc.Metadata; -import io.grpc.Server; -import io.grpc.ServerInterceptor; -import io.grpc.ServerMethodDefinition; -import io.grpc.ServerServiceDefinition; -import io.grpc.Status; -import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; -import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; -import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; -import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder; - -/** - * A gRPC listener: a Netty server on its own port, wired to Knox's identity, - * token, topology and audit services. - *

    - * It is a separate socket rather than a route on the gateway's existing - * connectors because gRPC requires HTTP/2 negotiated over ALPN, and Knox's Jetty - * connectors are HTTP/1.1 only. Beyond the transport, the servlet pipeline could - * not carry these calls anyway: Servlet 3.1 has no trailer API, and gRPC puts - * {@code grpc-status} — and, for Spark Connect, structured error details — in - * trailers. - * - *

    Why this is abstract

    - * Almost everything a gRPC gateway needs is protocol-agnostic: the transport, - * TLS from the gateway identity, bearer authentication, coarse authorization, - * topology routing, backend channel caching, auditing, graceful drain, and the - * byte-level relay itself. Only message-body handling — identity assertion and - * per-RPC gating — needs to know what is being proxied. - *

    - * Keeping that split explicit means a generic gRPC gateway would later be a - * configuration-and-documentation exercise rather than an engineering one. But - * Knox does not offer one today, and this class is deliberately - * not a way to get one: it is abstract, there is no configuration property that - * selects a listener generically, and the only concrete subclass is the Spark - * Connect one. A generic offering would need its own service-to-role mapping, - * default-deny posture, and an honest account of what byte-level proxying cannot - * enforce — none of which is in scope here. - *

    - * Subclasses supply four things: what to call the listener, which Knox service - * role backs it, the typed handlers, and which proto services may fall back to - * byte-level relay. - */ -// volatile: the lifecycle fields are written by the thread calling start/stop and -// read by request threads, so they need visibility but not mutual exclusion. -@SuppressWarnings("PMD.AvoidUsingVolatile") -public abstract class GrpcGatewayListener implements ProtocolListener { - - private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); - - private volatile Server server; - private volatile BackendChannelCache channelCache; - private volatile AuthorizationInterceptor authorizationInterceptor; - private volatile GrpcListenerSettings settings; - - /** - * The Knox service role backing this listener, e.g. {@code SPARKCONNECT}. - * Topologies declare a service with this role, and its ACLs are keyed on it. - * - * @return the service role - */ - protected abstract String getServiceRole(); - - /** - * Reads this listener's transport settings from gateway configuration. - * Subclasses own this because they own the configuration properties; the - * template deliberately does not read {@code GatewayConfig} for transport - * limits itself. - * - * @param config the gateway configuration - * @return the settings to build the server with - */ - protected abstract GrpcListenerSettings createSettings(GatewayConfig config); - - /** - * Builds the typed service definition whose handlers may inspect and rewrite - * message bodies. - * - * @param channels supplies the backend channel for the call in flight - * @param headers rewrites metadata for the backend leg - * @return the service definition to register - */ - protected abstract ServerServiceDefinition bindService(BackendChannelProvider channels, - HeaderRewriter headers); - - /** - * Fully qualified proto service names whose unregistered methods may still be - * relayed as opaque bytes. Anything not named here is answered - * {@code UNIMPLEMENTED}, so this is a closed list rather than an opt-out. - * - * @return the proto service names eligible for byte-level passthrough - */ - protected abstract Set getPassthroughServiceNames(); - - @Override - public void start(GatewayConfig config, GatewayServices services) throws Exception { - final GrpcListenerSettings listenerSettings = createSettings(config); - this.settings = listenerSettings; - - final BackendChannelCache channels = new BackendChannelCache(listenerSettings, services); - this.channelCache = channels; - - final BackendChannelProvider channelProvider = () -> { - final GrpcCallContext callContext = GrpcCallContext.current(); - if (callContext == null || callContext.getBackendUrl() == null) { - throw Status.UNAVAILABLE.withDescription("No backend resolved for this call").asRuntimeException(); - } - return channels.getChannel(callContext.getBackendUrl()); - }; - - // Built once, and validated here rather than on the first call: a bad key - // name should stop the gateway starting, not surprise the first user. - final Metadata.Key topologyKey = - GrpcMetadataKeys.topologyKey(listenerSettings.getTopologyMetadataKey()); - - final AliasService aliasService = services.getService(ServiceType.ALIAS_SERVICE); - final HeaderRewriter headerRewriter = - new BackendHeaderRewriter(aliasService, listenerSettings.getBackendTokenAlias(), topologyKey); - - this.authorizationInterceptor = new AuthorizationInterceptor(config, services, getServiceRole()); - - // Order is load-bearing: audit wraps everything so even rejected calls are - // recorded, then identity, then topology selection, then the ACL check that - // depends on both having succeeded. - final List interceptors = Arrays.asList( - new AuditInterceptor(), - new AuthenticationInterceptor(new TokenAuthenticator(config, services)), - new RoutingInterceptor(config, services, getServiceRole(), topologyKey), - authorizationInterceptor); - - final NettyServerBuilder builder = NettyServerBuilder.forPort(listenerSettings.getPort()) - .maxInboundMessageSize(listenerSettings.getMaxMessageSize()) - .maxConcurrentCallsPerConnection(listenerSettings.getMaxConcurrentCallsPerConnection()) - .permitKeepAliveTime(listenerSettings.getPermitKeepAliveTimeMillis(), TimeUnit.MILLISECONDS) - .permitKeepAliveWithoutCalls(listenerSettings.isPermitKeepAliveWithoutCalls()); - - if (config.isSSLEnabled()) { - builder.sslContext(buildServerSslContext(config, services)); - } else { - // A client that sets token= forces use_ssl=true, so this is really a test - // and development posture; say so rather than let it pass silently. - LOG.listenerTlsDisabled(listenerSettings.getName()); - } - - builder.addService(intercept(bindService(channelProvider, headerRewriter), interceptors)); - - final ProxyCallHandler passthroughHandler = - new ProxyCallHandler<>(channelProvider, MessageInterceptor.passthrough(), headerRewriter); - builder.fallbackHandlerRegistry(new PassthroughHandlerRegistry( - getPassthroughServiceNames(), - InterceptorChain.intercept(passthroughHandler, interceptors))); - - try { - this.server = builder.build().start(); - } catch (Exception e) { - LOG.failedToStartListener(listenerSettings.getName(), e); - channels.shutdown(0L); - this.channelCache = null; - throw e; - } - LOG.startedListener(listenerSettings.getName(), getPort()); - warnIfNoTopologyDeclaresTheRole(listenerSettings, services); - } - - /** - * Notes, at debug level, that the listener is running with nothing to route to. - *

    - * Enabling the listener and declaring a backend are separate steps in separate - * files, so it is possible to do the first and forget the second — but it is - * equally possible to do the first deliberately and wait. A deployment that - * enables the listener as a matter of course, and adds a topology only when - * someone provisions a Spark Connect cluster, is in this state normally and - * perhaps permanently. That is why this is debug rather than a warning: it - * helps when someone is asking why calls are refused, without nagging every - * deployment that is simply waiting. - */ - private void warnIfNoTopologyDeclaresTheRole(GrpcListenerSettings listenerSettings, - GatewayServices services) { - final TopologyService topologyService = services.getService(ServiceType.TOPOLOGY_SERVICE); - if (topologyService == null) { - return; - } - for (Topology topology : topologyService.getTopologies()) { - for (Service service : topology.getServices()) { - if (getServiceRole().equals(service.getRole())) { - return; - } - } - } - LOG.noTopologyDeclaresService(listenerSettings.getName(), getServiceRole()); - } - - /** - * Applies the interceptor chain to every method of a service definition. The - * chain is composed by hand rather than through {@code ServerInterceptors} so - * the ordering established above is preserved exactly. - */ - private static ServerServiceDefinition intercept(ServerServiceDefinition service, - List interceptors) { - final ServerServiceDefinition.Builder builder = - ServerServiceDefinition.builder(service.getServiceDescriptor()); - for (ServerMethodDefinition method : service.getMethods()) { - builder.addMethod(wrap(method, interceptors)); - } - return builder.build(); - } - - private static ServerMethodDefinition wrap( - ServerMethodDefinition method, List interceptors) { - return ServerMethodDefinition.create( - method.getMethodDescriptor(), - InterceptorChain.intercept(method.getServerCallHandler(), interceptors)); - } - - /** - * Builds the server's TLS context from the gateway identity — the same key - * material Jetty presents — so a deployment has one certificate to manage, not - * two. - *

    - * The identity is copied into a single-entry keystore before building the key - * manager, so the configured alias is the one presented even when the gateway - * keystore holds other entries. - */ - private SslContext buildServerSslContext(GatewayConfig config, GatewayServices services) - throws Exception { - try { - final KeystoreService keystoreService = services.getService(ServiceType.KEYSTORE_SERVICE); - final AliasService aliasService = services.getService(ServiceType.ALIAS_SERVICE); - - final String alias = config.getIdentityKeyAlias(); - final char[] passphrase = aliasService.getGatewayIdentityPassphrase(); - final KeyStore gatewayKeystore = keystoreService.getKeystoreForGateway(); - if (gatewayKeystore == null) { - throw new IllegalStateException("The gateway identity keystore is not available"); - } - - final Key key = gatewayKeystore.getKey(alias, passphrase); - final Certificate[] chain = gatewayKeystore.getCertificateChain(alias); - if (!(key instanceof PrivateKey) || chain == null || chain.length == 0) { - throw new IllegalStateException( - "The gateway identity keystore has no usable key entry for alias " + alias); - } - - final KeyStore identity = KeyStore.getInstance("PKCS12"); - identity.load(null, null); - identity.setKeyEntry(alias, key, passphrase, chain); - - final KeyManagerFactory keyManagers = - KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagers.init(identity, passphrase); - - // GrpcSslContexts applies the ALPN and cipher requirements of the HTTP/2 - // profile gRPC mandates. - return GrpcSslContexts.configure(SslContextBuilder.forServer(keyManagers)).build(); - } catch (Exception e) { - LOG.failedToBuildServerTls(getName(), e); - throw e; - } - } - - /** - * Stops accepting new calls and lets in-flight ones finish, up to the - * configured drain timeout. - *

    - * Long-lived streams are severed if they outlast the drain. That is survivable - * by design: Spark Connect clients already retry through - * {@code ReattachExecute}, which exists precisely because a connection can drop - * mid-query. - */ - @Override - public void stop() { - final Server current = server; - if (current == null) { - return; - } - final GrpcListenerSettings listenerSettings = settings; - final long drainTimeoutMillis = - listenerSettings == null ? 0L : listenerSettings.getDrainTimeoutMillis(); - LOG.stoppingListener(getName(), drainTimeoutMillis); - current.shutdown(); - try { - if (!current.awaitTermination(drainTimeoutMillis, TimeUnit.MILLISECONDS)) { - LOG.drainTimedOut(getName(), drainTimeoutMillis); - current.shutdownNow(); - } - } catch (InterruptedException e) { - current.shutdownNow(); - Thread.currentThread().interrupt(); - } finally { - server = null; - final BackendChannelCache channels = channelCache; - if (channels != null) { - channels.shutdown(drainTimeoutMillis); - channelCache = null; - } - LOG.stoppedListener(getName()); - } - } - - /** Drops cached topology ACLs so a redeployed topology takes effect. */ - @Override - public void reload() { - final AuthorizationInterceptor interceptor = authorizationInterceptor; - if (interceptor != null) { - interceptor.invalidate(); - } - } - - @Override - public int getPort() { - final Server current = server; - return current == null ? -1 : current.getPort(); - } - - /** The settings this listener started with, or null before {@code start}. */ - protected GrpcListenerSettings getSettings() { - return settings; - } -} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java deleted file mode 100644 index 3847e441e9..0000000000 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/GrpcListenerSettings.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.knox.gateway.grpc; - -/** - * Transport and lifecycle settings for a {@link GrpcGatewayListener}. - *

    - * These are deliberately plain values rather than reads against - * {@code GatewayConfig}. The listener is meant to be reusable for any gRPC - * service, so the layer that knows which {@code gateway.*} properties apply — - * currently the Spark Connect plugin — is the layer that reads them. - *

    - * The limits here are the listener's DoS surface. A new socket accepting 128 MB - * messages on long-lived streams needs message-size, stream-count and - * keepalive-abuse bounds configured from the start, not added after the first - * incident. - */ -public class GrpcListenerSettings { - - private String name = "grpc"; - private int port; - private int maxMessageSize = 134217728; - private int maxConcurrentCallsPerConnection = 1000; - private long permitKeepAliveTimeMillis = 10000L; - private boolean permitKeepAliveWithoutCalls = true; - private long channelIdleTimeoutMillis = 1800000L; - private long drainTimeoutMillis = 30000L; - private String backendTokenAlias; - private String topologyMetadataKey = GrpcMetadataKeys.DEFAULT_TOPOLOGY_KEY; - - public String getName() { - return name; - } - - public GrpcListenerSettings name(String value) { - this.name = value; - return this; - } - - public int getPort() { - return port; - } - - public GrpcListenerSettings port(int value) { - this.port = value; - return this; - } - - public int getMaxMessageSize() { - return maxMessageSize; - } - - public GrpcListenerSettings maxMessageSize(int value) { - this.maxMessageSize = value; - return this; - } - - public int getMaxConcurrentCallsPerConnection() { - return maxConcurrentCallsPerConnection; - } - - public GrpcListenerSettings maxConcurrentCallsPerConnection(int value) { - this.maxConcurrentCallsPerConnection = value; - return this; - } - - public long getPermitKeepAliveTimeMillis() { - return permitKeepAliveTimeMillis; - } - - public GrpcListenerSettings permitKeepAliveTimeMillis(long value) { - this.permitKeepAliveTimeMillis = value; - return this; - } - - public boolean isPermitKeepAliveWithoutCalls() { - return permitKeepAliveWithoutCalls; - } - - public GrpcListenerSettings permitKeepAliveWithoutCalls(boolean value) { - this.permitKeepAliveWithoutCalls = value; - return this; - } - - public long getChannelIdleTimeoutMillis() { - return channelIdleTimeoutMillis; - } - - public GrpcListenerSettings channelIdleTimeoutMillis(long value) { - this.channelIdleTimeoutMillis = value; - return this; - } - - public long getDrainTimeoutMillis() { - return drainTimeoutMillis; - } - - public GrpcListenerSettings drainTimeoutMillis(long value) { - this.drainTimeoutMillis = value; - return this; - } - - /** - * The metadata entry a client uses to select a topology. It is also the - * connection-string parameter users write, so a deployment may prefer a name - * that describes the choice rather than the gateway making it. - * - * @return the metadata key name - */ - public String getTopologyMetadataKey() { - return topologyMetadataKey; - } - - public GrpcListenerSettings topologyMetadataKey(String value) { - this.topologyMetadataKey = value; - return this; - } - - public String getBackendTokenAlias() { - return backendTokenAlias; - } - - public GrpcListenerSettings backendTokenAlias(String value) { - this.backendTokenAlias = value; - return this; - } -} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/PassthroughHandlerRegistry.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/PassthroughHandlerRegistry.java deleted file mode 100644 index 8ff6cae50e..0000000000 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/grpc/PassthroughHandlerRegistry.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.knox.gateway.grpc; - -import java.util.Locale; -import java.util.Set; - -import org.apache.knox.gateway.i18n.messages.MessagesFactory; - -import io.grpc.HandlerRegistry; -import io.grpc.MethodDescriptor; -import io.grpc.ServerCallHandler; -import io.grpc.ServerMethodDefinition; - -/** - * Handles calls to methods that have no typed handler, relaying them as opaque - * bytes. - *

    - * This exists so that proto skew degrades gracefully. When a client calls an RPC - * this build has no generated classes for — an addition in a newer Spark line, - * typically — the call is still authenticated, authorized, routed and audited; - * only the message-body handling is skipped, because there is nothing to inspect - * with. Without it, such a call would fail outright at the gateway even though - * the backend could serve it. - *

    - * The trade-off is explicit: no identity assertion happens on this path, since - * rewriting {@code user_context} requires parsing the message. Passthrough is - * therefore restricted to a configured set of proto services, and default-denies - * everything else — a gateway that forwarded arbitrary unknown services without - * being asked to would be a very different, and much weaker, security posture. - */ -public class PassthroughHandlerRegistry extends HandlerRegistry { - - private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); - - private final Set allowedServices; - private final ServerCallHandler handler; - - /** - * @param allowedServices fully qualified proto service names whose unknown - * methods may be relayed, e.g. {@code spark.connect.SparkConnectService} - * @param handler the proxy handler to relay with - */ - public PassthroughHandlerRegistry(Set allowedServices, ServerCallHandler handler) { - this.allowedServices = allowedServices; - this.handler = handler; - } - - @Override - public ServerMethodDefinition lookupMethod(String methodName, String authority) { - final String serviceName = MethodDescriptor.extractFullServiceName(methodName); - if (serviceName == null || !allowedServices.contains(serviceName)) { - // Returning null makes grpc answer UNIMPLEMENTED, which is also what a real - // server says about a method it does not have — so this reveals nothing - // about what the gateway is fronting. - return null; - } - - LOG.debugLog(String.format(Locale.ROOT, - "Relaying %s as opaque bytes; no typed handler is registered for it", methodName)); - - final MethodDescriptor descriptor = MethodDescriptor.newBuilder() - // UNKNOWN keeps grpc from assuming a message count in either direction, - // so unary and streaming methods alike relay correctly. - .setType(MethodDescriptor.MethodType.UNKNOWN) - .setFullMethodName(methodName) - .setRequestMarshaller(ByteArrayMarshaller.INSTANCE) - .setResponseMarshaller(ByteArrayMarshaller.INSTANCE) - .build(); - return ServerMethodDefinition.create(descriptor, handler); - } -} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuard.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuard.java deleted file mode 100644 index 6f3666b9b6..0000000000 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuard.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.knox.gateway.sparkconnect; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Locale; -import java.util.Set; - -import org.apache.knox.gateway.sparkconnect.SparkConnectMessageInterceptor.RequestGuard; - -import com.google.protobuf.Message; - -import io.grpc.Status; - -/** - * Controls who may upload artifacts through {@code AddArtifacts}. - *

    - * This is defense in depth, and it is worth being clear why it cannot be more - * than that. A shared Spark Connect server runs as one principal, and any - * user-supplied code — an uploaded jar, or an inline Python or Scala UDF - * embedded in a plan — executes inside that JVM with that principal's storage - * credentials. Such code can read data directly, bypassing any plan-level policy - * check, and could subvert an in-JVM authorization plugin. Session-scoped - * artifact classloaders isolate sessions from each other, not from the - * application's own privileges. - *

    - * So blocking artifact upload shrinks the attack surface; it does not create a - * boundary, because inline UDFs remain a path to the same capability. This is a - * property of plan-level enforcement in general rather than something the - * gateway introduces. Deployments needing a hard boundary want per-user backends - * instead. - */ -public class AddArtifactsGuard implements RequestGuard { - - /** Every user may upload artifacts. */ - public static final String MODE_ALLOW = "ALLOW"; - /** No user may upload artifacts. */ - public static final String MODE_DENY = "DENY"; - /** Only explicitly listed users may upload artifacts. */ - public static final String MODE_ALLOW_LISTED_USERS = "ALLOW_LISTED_USERS"; - - private final String mode; - private final Set allowedUsers; - - public AddArtifactsGuard(String mode, Collection allowedUsers) { - this.mode = mode == null ? MODE_ALLOW : mode.trim().toUpperCase(Locale.ROOT); - this.allowedUsers = allowedUsers == null - ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(allowedUsers)); - } - - /** - * Whether this guard would reject every call, letting the caller skip - * per-message work entirely. - * - * @return true if no user may upload artifacts - */ - public boolean deniesEveryone() { - return MODE_DENY.equals(mode); - } - - @Override - public void check(Message request, String principal) { - if (MODE_ALLOW.equals(mode)) { - return; - } - if (MODE_ALLOW_LISTED_USERS.equals(mode) && allowedUsers.contains(principal)) { - return; - } - throw Status.PERMISSION_DENIED - .withDescription("Uploading artifacts through Spark Connect is not permitted for this user") - .asRuntimeException(); - } -} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuard.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuard.java deleted file mode 100644 index b7b89ef7cd..0000000000 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuard.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.knox.gateway.sparkconnect; - -import java.util.Locale; - -import org.apache.knox.gateway.sparkconnect.SparkConnectMessageInterceptor.RequestGuard; - -import com.google.protobuf.Message; - -import io.grpc.Status; - -/** - * Refuses client writes to session-configuration keys reserved for Knox. - *

    - * A deployment may publish the authenticated identity into the Spark session as - * a configuration entry, which downstream authorization then reads. If a client - * could overwrite that entry it could assume any identity it liked, so - * {@code Set} and {@code Unset} on the reserved prefix are rejected outright. - *

    - * This covers the structured path completely and cheaply, because the keys are - * named fields in the {@code Config} RPC. It does not cover - * {@code SET reserved.key=...} issued as SQL inside {@code ExecutePlan}, which - * would need plan-text inspection and would still be best-effort. That gap is - * the argument for the stronger server-side arrangement, where an interceptor in - * the Spark application recomputes the identity from {@code user_context} on - * every request: a value derived per request cannot be overwritten by a session - * {@code SET} at all. - */ -public class ReservedConfigGuard implements RequestGuard { - - private static final String OPERATION_FIELD = "operation"; - private static final String SET_FIELD = "set"; - private static final String UNSET_FIELD = "unset"; - private static final String PAIRS_FIELD = "pairs"; - private static final String KEYS_FIELD = "keys"; - private static final String KEY_FIELD = "key"; - - private final String reservedPrefix; - - public ReservedConfigGuard(String reservedPrefix) { - this.reservedPrefix = reservedPrefix == null ? "" : reservedPrefix.toLowerCase(Locale.ROOT); - } - - @Override - public void check(Message request, String principal) { - if (reservedPrefix.isEmpty()) { - return; - } - final Message operation = childMessage(request, OPERATION_FIELD); - if (operation == null) { - return; - } - - final Message set = childMessage(operation, SET_FIELD); - if (set != null) { - final com.google.protobuf.Descriptors.FieldDescriptor pairs = - set.getDescriptorForType().findFieldByName(PAIRS_FIELD); - if (pairs != null) { - final int count = set.getRepeatedFieldCount(pairs); - for (int i = 0; i < count; i++) { - final Message pair = (Message) set.getRepeatedField(pairs, i); - final com.google.protobuf.Descriptors.FieldDescriptor key = - pair.getDescriptorForType().findFieldByName(KEY_FIELD); - if (key != null) { - reject(String.valueOf(pair.getField(key))); - } - } - } - } - - final Message unset = childMessage(operation, UNSET_FIELD); - if (unset != null) { - final com.google.protobuf.Descriptors.FieldDescriptor keys = - unset.getDescriptorForType().findFieldByName(KEYS_FIELD); - if (keys != null) { - final int count = unset.getRepeatedFieldCount(keys); - for (int i = 0; i < count; i++) { - reject(String.valueOf(unset.getRepeatedField(keys, i))); - } - } - } - } - - private void reject(String key) { - if (key != null && key.toLowerCase(Locale.ROOT).startsWith(reservedPrefix)) { - throw Status.PERMISSION_DENIED - .withDescription("Session configuration keys beginning with '" + reservedPrefix - + "' are reserved by the gateway and cannot be set or unset by clients") - .asRuntimeException(); - } - } - - /** - * Returns a singular message-valued field only when it is actually present, so - * an absent branch of the {@code op_type} oneof does not read as an empty - * {@code Set}. - */ - private static Message childMessage(Message parent, String fieldName) { - final com.google.protobuf.Descriptors.FieldDescriptor field = - parent.getDescriptorForType().findFieldByName(fieldName); - if (field == null || !parent.hasField(field)) { - return null; - } - final Object value = parent.getField(field); - return value instanceof Message ? (Message) value : null; - } -} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectListener.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectListener.java deleted file mode 100644 index 073f27e56d..0000000000 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectListener.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.knox.gateway.sparkconnect; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Set; - -import org.apache.knox.gateway.config.GatewayConfig; -import org.apache.knox.gateway.config.GatewayConfigChangeListener; -import org.apache.knox.gateway.grpc.BackendChannelProvider; -import org.apache.knox.gateway.grpc.GrpcGatewayListener; -import org.apache.knox.gateway.grpc.GrpcGatewayMessages; -import org.apache.knox.gateway.grpc.GrpcListenerSettings; -import org.apache.knox.gateway.grpc.HeaderRewriter; -import org.apache.knox.gateway.grpc.MessageInterceptor; -import org.apache.knox.gateway.grpc.ProxyCallHandler; -import org.apache.knox.gateway.i18n.messages.MessagesFactory; - -import com.google.protobuf.Message; - -import io.grpc.MethodDescriptor; -import io.grpc.ServerMethodDefinition; -import io.grpc.ServerServiceDefinition; - -import org.apache.spark.connect.proto.SparkConnectServiceGrpc; - -/** - * Fronts a Spark Connect server, the one concrete listener the gRPC template - * offers. - *

    - * Spark Connect is worth fronting because its server has essentially no - * authentication or authorization of its own — the project assumes a proxy - * supplies them — while Knox already fronts the surfaces around it. What Knox - * adds is authentication at the edge, an identity the client cannot forge, an - * audit trail, and topology-based routing. - *

    - * Clients need no code changes and no plugins. A vanilla connection string - * carries everything required: - * 

    - * sc://knox-host:15002/;use_ssl=true;token=<knox-jwt>;knox-topology=analytics
- *
    - * {@code token=} becomes a standard bearer header (and forces TLS on), and any - * parameter the client does not recognise — {@code knox-topology} here — is sent - * as call metadata, which is what makes topology selection possible despite gRPC - * forbidding a path in the URL. - */ -// volatile: the message-level policy is captured at start-up and read by request -// threads thereafter. -@SuppressWarnings("PMD.AvoidUsingVolatile") -public class SparkConnectListener extends GrpcGatewayListener implements GatewayConfigChangeListener { - - private static final GrpcGatewayMessages LOG = MessagesFactory.get(GrpcGatewayMessages.class); - - private static final String LISTENER_NAME = "SparkConnect"; - private static final String SERVICE_ROLE = "SPARKCONNECT"; - private static final String PROTO_SERVICE_NAME = "spark.connect.SparkConnectService"; - - private static final String CONFIG_METHOD = "Config"; - private static final String ADD_ARTIFACTS_METHOD = "AddArtifacts"; - - /** - * The message-level controls, replaced wholesale when configuration changes. - * Handlers read it per call rather than capturing a guard, which is what makes - * a change take effect without rebuilding the gRPC service. - */ - private volatile SparkConnectPolicy policy; - - @Override - public String getName() { - return LISTENER_NAME; - } - - @Override - public boolean isEnabled(GatewayConfig config) { - return config.isSparkConnectEnabled(); - } - - @Override - protected String getServiceRole() { - return SERVICE_ROLE; - } - - @Override - protected GrpcListenerSettings createSettings(GatewayConfig config) { - // Message-level policy can change later; the transport settings below cannot, - // because they are built into the bound server. - this.policy = SparkConnectPolicy.from(config); - - return new GrpcListenerSettings() - .name(LISTENER_NAME) - .port(config.getSparkConnectPort()) - .maxMessageSize(config.getSparkConnectMaxMessageSize()) - .maxConcurrentCallsPerConnection(config.getSparkConnectMaxConcurrentCallsPerConnection()) - .permitKeepAliveTimeMillis(config.getSparkConnectPermitKeepAliveTime()) - .permitKeepAliveWithoutCalls(config.isSparkConnectPermitKeepAliveWithoutCalls()) - .channelIdleTimeoutMillis(config.getSparkConnectChannelIdleTimeout()) - .drainTimeoutMillis(config.getSparkConnectDrainTimeout()) - .topologyMetadataKey(config.getSparkConnectTopologyMetadataKey()) - .backendTokenAlias(config.getSparkConnectBackendTokenAlias()); - } - - @Override - protected Set getPassthroughServiceNames() { - // Only methods of the Spark Connect service itself may fall back to a - // byte-level relay, and only when this build has no typed handler for them — - // which is how a client from a newer Spark line still gets proxied. - return Collections.singleton(PROTO_SERVICE_NAME); - } - - /** - * Registers a proxy handler for every method of {@code SparkConnectService}. - *

    - * There is one handler implementation rather than one per RPC shape. Unary, - * server-streaming and client-streaming calls differ only in message counts, - * which the relay handles uniformly, so the ten-odd methods need no bespoke - * code — just the right request interceptor each. - */ - @Override - protected ServerServiceDefinition bindService(BackendChannelProvider channels, - HeaderRewriter headers) { - final ServerServiceDefinition.Builder builder = - ServerServiceDefinition.builder(SparkConnectServiceGrpc.getServiceDescriptor()); - for (MethodDescriptor method : SparkConnectServiceGrpc.getServiceDescriptor().getMethods()) { - builder.addMethod(proxyMethod(method, channels, headers)); - } - return builder.build(); - } - - /** - * Erasure lets one relay serve every method: the generated marshallers still - * parse each message into its concrete type, and the interceptor only touches - * fields it looks up by name on the descriptor. - */ - @SuppressWarnings("unchecked") - private ServerMethodDefinition proxyMethod(MethodDescriptor method, - BackendChannelProvider channels, - HeaderRewriter headers) { - final MethodDescriptor descriptor = (MethodDescriptor) method; - final MessageInterceptor interceptor = - interceptorFor(bareMethodName(descriptor.getFullMethodName())); - return ServerMethodDefinition.create(descriptor, - new ProxyCallHandler<>(channels, interceptor, headers)); - } - - /** - * The guards indirect through {@link #policy} on every call rather than being - * captured here. Handlers are registered once when the server is built, so a - * guard captured at that moment could never be replaced — which is what made - * these settings silently restart-only before. - */ - // Package-private so tests can assert that a policy change reaches an - // interceptor built before the change. - MessageInterceptor interceptorFor(String methodName) { - if (CONFIG_METHOD.equals(methodName)) { - return new SparkConnectMessageInterceptor( - (message, principal) -> policy.reservedConfigGuard().check(message, principal)); - } - if (ADD_ARTIFACTS_METHOD.equals(methodName)) { - return new SparkConnectMessageInterceptor( - (message, principal) -> policy.addArtifactsGuard().check(message, principal)); - } - return new SparkConnectMessageInterceptor(null); - } - - /** - * Applies a changed {@code gateway-reloadable.xml} to the controls that can - * move on a running gateway. - *

    - * Only the message-level policy is refreshed. The transport settings are built - * into the bound server and cannot change without a restart, so rather than - * accept them silently and do nothing — which looks like it worked — any - * attempt to change one is named in the log. - */ - @Override - public void onGatewayConfigChanged(GatewayConfig config) { - final SparkConnectPolicy updated = SparkConnectPolicy.from(config); - if (updated.differsFrom(policy)) { - this.policy = updated; - LOG.reloadedPolicy(LISTENER_NAME, updated.toString()); - } - warnAboutRestartOnlyChanges(config); - } - - private void warnAboutRestartOnlyChanges(GatewayConfig config) { - final GrpcListenerSettings running = getSettings(); - if (running == null) { - return; - } - final List changed = new ArrayList<>(); - if (config.getSparkConnectPort() != running.getPort()) { - changed.add("port"); - } - if (config.getSparkConnectMaxMessageSize() != running.getMaxMessageSize()) { - changed.add("max.message.size"); - } - if (config.getSparkConnectMaxConcurrentCallsPerConnection() - != running.getMaxConcurrentCallsPerConnection()) { - changed.add("max.concurrent.calls.per.connection"); - } - if (config.getSparkConnectPermitKeepAliveTime() != running.getPermitKeepAliveTimeMillis()) { - changed.add("permit.keepalive.time"); - } - if (config.isSparkConnectPermitKeepAliveWithoutCalls() != running.isPermitKeepAliveWithoutCalls()) { - changed.add("permit.keepalive.without.calls"); - } - if (config.getSparkConnectChannelIdleTimeout() != running.getChannelIdleTimeoutMillis()) { - changed.add("channel.idle.timeout"); - } - if (config.getSparkConnectDrainTimeout() != running.getDrainTimeoutMillis()) { - changed.add("drain.timeout"); - } - if (!Objects.equals(config.getSparkConnectBackendTokenAlias(), running.getBackendTokenAlias())) { - changed.add("backend.token.alias"); - } - if (!changed.isEmpty()) { - LOG.restartOnlyConfigChanged(LISTENER_NAME, String.join(", ", changed)); - } - } - - private static String bareMethodName(String fullMethodName) { - final int separator = fullMethodName.lastIndexOf('/'); - return separator < 0 ? fullMethodName : fullMethodName.substring(separator + 1); - } -} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptor.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptor.java deleted file mode 100644 index b27063bdf4..0000000000 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptor.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.knox.gateway.sparkconnect; - -import org.apache.knox.gateway.grpc.GrpcCallContext; -import org.apache.knox.gateway.grpc.MessageInterceptor; - -import com.google.protobuf.Descriptors.FieldDescriptor; -import com.google.protobuf.Message; - -import io.grpc.Status; - -/** - * Asserts the authenticated identity onto every request, and applies the - * per-RPC gating switches. - *

    - * Identity assertion is the reason this gateway parses messages at all. Spark - * Connect trusts a client-asserted {@code user_context.user_id}: the - * client simply states who it is, and the server believes it. Overwriting that - * field with the principal Knox authenticated closes a real spoofing hole, and - * it is something no byte-level proxy — or L4 passthrough, or generic sidecar — - * could do. - *

    - * Be precise about what it buys, though. On the server, {@code user_id} keys the - * session cache ({@code SessionKey(userId, sessionId)}) and appears in logs and - * events. It is not propagated into Spark's {@code CurrentUserContext}, - * so {@code current_user()} in SQL still reports the Spark application's own - * user. What assertion guarantees is session isolation between users and a - * trustworthy audit trail — not storage-level enforcement, which needs either - * per-user backends or a server-side component that bridges this field into the - * session. - */ -public class SparkConnectMessageInterceptor implements MessageInterceptor { - - private static final String USER_CONTEXT_FIELD = "user_context"; - private static final String SESSION_ID_FIELD = "session_id"; - private static final String OPERATION_ID_FIELD = "operation_id"; - private static final String USER_ID_FIELD = "user_id"; - private static final String USER_NAME_FIELD = "user_name"; - - private final RequestGuard guard; - - /** - * @param guard an extra check for this RPC, or null when identity assertion is - * all that applies - */ - public SparkConnectMessageInterceptor(RequestGuard guard) { - this.guard = guard; - } - - @Override - public Message intercept(Message message) { - final GrpcCallContext callContext = GrpcCallContext.current(); - final String principal = callContext == null ? null : callContext.getPrincipal(); - if (principal == null) { - // The authentication interceptor runs before any handler, so this cannot - // happen unless the chain was assembled wrongly. Fail rather than forward a - // request carrying whatever identity the client claimed. - throw Status.INTERNAL - .withDescription("No authenticated principal available for identity assertion") - .asRuntimeException(); - } - - recordCallDetails(message, callContext); - if (guard != null) { - guard.check(message, principal); - } - return assertIdentity(message, principal); - } - - /** - * Copies the session and operation identifiers into the call context so audit - * records can name the session a call belongs to. Every Spark Connect request - * carries {@code session_id}; only some carry {@code operation_id}. - */ - private static void recordCallDetails(Message message, GrpcCallContext callContext) { - if (callContext == null) { - return; - } - final FieldDescriptor sessionField = - message.getDescriptorForType().findFieldByName(SESSION_ID_FIELD); - if (sessionField != null) { - final Object sessionId = message.getField(sessionField); - if (sessionId instanceof String && !((String) sessionId).isEmpty()) { - callContext.setSessionId((String) sessionId); - } - } - final FieldDescriptor operationField = - message.getDescriptorForType().findFieldByName(OPERATION_ID_FIELD); - if (operationField != null && message.hasField(operationField)) { - final Object operationId = message.getField(operationField); - if (operationId instanceof String && !((String) operationId).isEmpty()) { - callContext.setOperationId((String) operationId); - } - } - } - - /** - * Replaces the client-supplied identity with the authenticated one. - *

    - * This works the same way for all twelve RPCs because every Spark Connect - * request message carries {@code UserContext user_context = 2} in the same - * position — so the rewrite is driven off the descriptor rather than written - * out once per message type. Fields the gateway does not touch, including ones - * from a newer Spark than these vendored protos describe, survive: protobuf - * retains unknown fields across a parse and re-serialize. - */ - @SuppressWarnings("unchecked") - static T assertIdentity(T message, String principal) { - final FieldDescriptor userContextField = - message.getDescriptorForType().findFieldByName(USER_CONTEXT_FIELD); - if (userContextField == null) { - return message; - } - - final Message userContext = (Message) message.getField(userContextField); - final FieldDescriptor userIdField = - userContext.getDescriptorForType().findFieldByName(USER_ID_FIELD); - final FieldDescriptor userNameField = - userContext.getDescriptorForType().findFieldByName(USER_NAME_FIELD); - - final Message.Builder userContextBuilder = userContext.toBuilder(); - if (userIdField != null) { - userContextBuilder.setField(userIdField, principal); - } - // The client's user_name is overwritten too: it is purely descriptive on the - // server, but leaving a self-asserted value would put a name Knox never - // verified into Spark's logs next to the id it did. - if (userNameField != null) { - userContextBuilder.setField(userNameField, principal); - } - - return (T) message.toBuilder() - .setField(userContextField, userContextBuilder.build()) - .build(); - } - - /** An additional per-RPC check applied before a request is forwarded. */ - @FunctionalInterface - public interface RequestGuard { - - /** - * @param message the request message - * @param principal the authenticated principal - * @throws io.grpc.StatusRuntimeException to reject the call - */ - void check(Message message, String principal); - } -} diff --git a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectPolicy.java b/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectPolicy.java deleted file mode 100644 index 6dce42aa38..0000000000 --- a/gateway-service-sparkconnect/src/main/java/org/apache/knox/gateway/sparkconnect/SparkConnectPolicy.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.knox.gateway.sparkconnect; - -import java.util.List; -import java.util.Objects; - -import org.apache.knox.gateway.config.GatewayConfig; - -/** - * The message-level controls, held together so they can be replaced atomically. - *

    - * These are the settings that can change on a running gateway. The handlers - * registered at startup hold a reference to the listener rather than to a - * particular guard, and read the current policy per call, so swapping this - * object takes effect on the next RPC without rebuilding the gRPC service. - *

    - * It is one object rather than separate fields so a configuration change is seen - * whole: a call can never observe the new artifact-gating rule alongside the old - * reserved prefix. - */ -final class SparkConnectPolicy { - - private final String reservedConfigPrefix; - private final String addArtifactsMode; - private final List addArtifactsAllowedUsers; - private final ReservedConfigGuard reservedConfigGuard; - private final AddArtifactsGuard addArtifactsGuard; - - private SparkConnectPolicy(String reservedConfigPrefix, - String addArtifactsMode, - List addArtifactsAllowedUsers) { - this.reservedConfigPrefix = reservedConfigPrefix; - this.addArtifactsMode = addArtifactsMode; - this.addArtifactsAllowedUsers = addArtifactsAllowedUsers; - this.reservedConfigGuard = new ReservedConfigGuard(reservedConfigPrefix); - this.addArtifactsGuard = new AddArtifactsGuard(addArtifactsMode, addArtifactsAllowedUsers); - } - - static SparkConnectPolicy from(GatewayConfig config) { - return new SparkConnectPolicy( - config.getSparkConnectReservedConfigPrefix(), - config.getSparkConnectAddArtifactsMode(), - config.getSparkConnectAddArtifactsAllowedUsers()); - } - - ReservedConfigGuard reservedConfigGuard() { - return reservedConfigGuard; - } - - AddArtifactsGuard addArtifactsGuard() { - return addArtifactsGuard; - } - - /** - * Whether this policy differs from another, used to decide if a configuration - * change is worth logging. Compares the configured values rather than the - * derived guards, which have no meaningful equality. - * - * @param other the policy to compare against, may be null - * @return true if the two express different rules - */ - boolean differsFrom(SparkConnectPolicy other) { - return other == null - || !Objects.equals(reservedConfigPrefix, other.reservedConfigPrefix) - || !Objects.equals(addArtifactsMode, other.addArtifactsMode) - || !Objects.equals(addArtifactsAllowedUsers, other.addArtifactsAllowedUsers); - } - - @Override - public String toString() { - return "addArtifactsMode=" + addArtifactsMode - + ", addArtifactsAllowedUsers=" + addArtifactsAllowedUsers - + ", reservedConfigPrefix=" + reservedConfigPrefix; - } -} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuardTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuardTest.java deleted file mode 100644 index 9eadfee5bd..0000000000 --- a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/AddArtifactsGuardTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.knox.gateway.sparkconnect; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.util.Arrays; -import java.util.Collections; - -import com.google.protobuf.Message; - -import io.grpc.Status; -import io.grpc.StatusRuntimeException; - -import org.apache.spark.connect.proto.AddArtifactsRequest; - -import org.junit.Test; - -public class AddArtifactsGuardTest { - - private static final Message REQUEST = AddArtifactsRequest.getDefaultInstance(); - - @Test - public void allowModePermitsEveryone() { - new AddArtifactsGuard(AddArtifactsGuard.MODE_ALLOW, Collections.emptyList()).check(REQUEST, "alice"); - } - - @Test - public void defaultsToAllowWhenUnconfigured() { - new AddArtifactsGuard(null, null).check(REQUEST, "alice"); - } - - @Test - public void denyModeRejectsEveryone() { - final AddArtifactsGuard guard = - new AddArtifactsGuard(AddArtifactsGuard.MODE_DENY, Arrays.asList("alice")); - assertTrue(guard.deniesEveryone()); - // Even a listed user is refused: DENY is not "deny except the list". - assertDenied(guard, "alice"); - } - - @Test - public void listedUsersModeAdmitsOnlyListedUsers() { - final AddArtifactsGuard guard = new AddArtifactsGuard( - AddArtifactsGuard.MODE_ALLOW_LISTED_USERS, Arrays.asList("alice", "bob")); - assertFalse(guard.deniesEveryone()); - guard.check(REQUEST, "alice"); - guard.check(REQUEST, "bob"); - assertDenied(guard, "mallory"); - } - - @Test - public void modeIsCaseAndWhitespaceInsensitive() { - new AddArtifactsGuard(" allow ", Collections.emptyList()).check(REQUEST, "alice"); - } - - @Test - public void unrecognisedModeFailsClosed() { - // A typo in configuration must not silently become "allow everyone". - assertDenied(new AddArtifactsGuard("permissive", Collections.emptyList()), "alice"); - } - - private static void assertDenied(AddArtifactsGuard guard, String principal) { - try { - guard.check(REQUEST, principal); - fail("Expected artifact upload to be denied for " + principal); - } catch (StatusRuntimeException e) { - assertEquals(Status.Code.PERMISSION_DENIED, e.getStatus().getCode()); - } - } -} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuardTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuardTest.java deleted file mode 100644 index 41795ea043..0000000000 --- a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/ReservedConfigGuardTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.knox.gateway.sparkconnect; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -import io.grpc.Status; -import io.grpc.StatusRuntimeException; - -import org.apache.spark.connect.proto.ConfigRequest; -import org.apache.spark.connect.proto.KeyValue; - -import org.junit.Test; - -public class ReservedConfigGuardTest { - - private static final String PREFIX = "knox."; - private static final String USER = "alice"; - - private final ReservedConfigGuard guard = new ReservedConfigGuard(PREFIX); - - @Test - public void deniesSettingAReservedKey() { - // If a client could overwrite the key Knox publishes the identity into, it - // could assume any identity it liked. - assertDenied(configSet("knox.principal", "root")); - } - - @Test - public void deniesSettingAReservedKeyRegardlessOfCase() { - assertDenied(configSet("KNOX.Principal", "root")); - } - - @Test - public void deniesUnsettingAReservedKey() { - // Clearing the key is as good as overwriting it if downstream code then - // falls back to something less trustworthy. - assertDenied(ConfigRequest.newBuilder() - .setOperation(ConfigRequest.Operation.newBuilder() - .setUnset(ConfigRequest.Unset.newBuilder().addKeys("knox.principal"))) - .build()); - } - - @Test - public void deniesWhenAReservedKeyIsBuriedAmongAllowedOnes() { - assertDenied(ConfigRequest.newBuilder() - .setOperation(ConfigRequest.Operation.newBuilder() - .setSet(ConfigRequest.Set.newBuilder() - .addPairs(KeyValue.newBuilder().setKey("spark.sql.shuffle.partitions").setValue("8")) - .addPairs(KeyValue.newBuilder().setKey("knox.principal").setValue("root")))) - .build()); - } - - @Test - public void allowsOrdinarySparkSettings() { - guard.check(configSet("spark.sql.shuffle.partitions", "8"), USER); - } - - @Test - public void allowsReadingAReservedKey() { - // Reading is harmless; only writes can forge an identity. - guard.check(ConfigRequest.newBuilder() - .setOperation(ConfigRequest.Operation.newBuilder() - .setGet(ConfigRequest.Get.newBuilder().addKeys("knox.principal"))) - .build(), USER); - } - - @Test - public void allowsRequestsWithNoConfigOperation() { - guard.check(ConfigRequest.getDefaultInstance(), USER); - } - - @Test - public void doesNothingWhenNoPrefixIsReserved() { - new ReservedConfigGuard("").check(configSet("knox.principal", "root"), USER); - } - - private void assertDenied(ConfigRequest request) { - try { - guard.check(request, USER); - fail("Expected the reserved key write to be denied"); - } catch (StatusRuntimeException e) { - assertEquals(Status.Code.PERMISSION_DENIED, e.getStatus().getCode()); - } - } - - private static ConfigRequest configSet(String key, String value) { - return ConfigRequest.newBuilder() - .setOperation(ConfigRequest.Operation.newBuilder() - .setSet(ConfigRequest.Set.newBuilder() - .addPairs(KeyValue.newBuilder().setKey(key).setValue(value)))) - .build(); - } -} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectListenerConfigReloadTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectListenerConfigReloadTest.java deleted file mode 100644 index 646d8d2660..0000000000 --- a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectListenerConfigReloadTest.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.knox.gateway.sparkconnect; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -import java.util.Arrays; -import java.util.Collections; - -import org.apache.knox.gateway.GatewayTestConfig; -import org.apache.knox.gateway.grpc.GrpcCallContext; -import org.apache.knox.gateway.grpc.MessageInterceptor; - -import com.google.protobuf.Message; - -import io.grpc.Context; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; - -import org.apache.spark.connect.proto.AddArtifactsRequest; -import org.apache.spark.connect.proto.ConfigRequest; -import org.apache.spark.connect.proto.KeyValue; - -import org.junit.Test; - -/** - * The message-level controls are the only Spark Connect settings that can change - * on a running gateway; the rest are built into the bound server. - *

    - * Each test obtains an interceptor before changing configuration and - * asserts on that same instance afterwards. That is the property that matters: - * handlers are registered once when the gRPC service is built, so a guard - * captured at that moment could never be replaced, and the setting would be - * silently restart-only. - */ -public class SparkConnectListenerConfigReloadTest { - - private static final String USER = "alice"; - - @Test - public void addArtifactsGatingTakesEffectWithoutRestart() { - final GatewayTestConfig config = new GatewayTestConfig(); - config.setSparkConnectAddArtifactsMode(AddArtifactsGuard.MODE_ALLOW); - final SparkConnectListener listener = started(config); - - final MessageInterceptor addArtifacts = listener.interceptorFor("AddArtifacts"); - intercept(addArtifacts, AddArtifactsRequest.getDefaultInstance()); - - config.setSparkConnectAddArtifactsMode(AddArtifactsGuard.MODE_DENY); - listener.onGatewayConfigChanged(config); - - assertDenied(addArtifacts, AddArtifactsRequest.getDefaultInstance()); - } - - @Test - public void addArtifactsAllowListTakesEffectWithoutRestart() { - final GatewayTestConfig config = new GatewayTestConfig(); - config.setSparkConnectAddArtifactsMode(AddArtifactsGuard.MODE_ALLOW_LISTED_USERS); - config.setSparkConnectAddArtifactsAllowedUsers(Collections.emptyList()); - final SparkConnectListener listener = started(config); - final MessageInterceptor addArtifacts = listener.interceptorFor("AddArtifacts"); - - assertDenied(addArtifacts, AddArtifactsRequest.getDefaultInstance()); - - config.setSparkConnectAddArtifactsAllowedUsers(Arrays.asList(USER, "bob")); - listener.onGatewayConfigChanged(config); - - intercept(addArtifacts, AddArtifactsRequest.getDefaultInstance()); - } - - @Test - public void reservedConfigPrefixTakesEffectWithoutRestart() { - final GatewayTestConfig config = new GatewayTestConfig(); - final SparkConnectListener listener = started(config); - final MessageInterceptor configRpc = listener.interceptorFor("Config"); - - // 'acme.' is not reserved under the default 'knox.' prefix. - intercept(configRpc, configSet("acme.principal", "root")); - - config.setSparkConnectReservedConfigPrefix("acme."); - listener.onGatewayConfigChanged(config); - - assertDenied(configRpc, configSet("acme.principal", "root")); - } - - @Test - public void aRestartOnlyChangeDoesNotDisturbTheMessagePolicy() { - final GatewayTestConfig config = new GatewayTestConfig(); - final SparkConnectListener listener = started(config); - final MessageInterceptor configRpc = listener.interceptorFor("Config"); - - // The port cannot be rebound on a running listener. Handling the change must - // neither throw nor quietly drop the policy that is still in force. - config.setSparkConnectPort(15099); - listener.onGatewayConfigChanged(config); - - assertDenied(configRpc, configSet("knox.principal", "root")); - } - - @Test - public void identityAssertionIsUnaffectedByPolicyChanges() { - final GatewayTestConfig config = new GatewayTestConfig(); - final SparkConnectListener listener = started(config); - final MessageInterceptor configRpc = listener.interceptorFor("Config"); - - config.setSparkConnectReservedConfigPrefix("acme."); - listener.onGatewayConfigChanged(config); - - // A permitted call still gets its identity asserted; the guard swap must not - // replace the interceptor's primary job. - final ConfigRequest forwarded = (ConfigRequest) interceptAndReturn( - configRpc, configSet("spark.sql.shuffle.partitions", "8")); - assertEquals(USER, forwarded.getUserContext().getUserId()); - } - - /** Populates the listener's policy without binding a port. */ - private static SparkConnectListener started(GatewayTestConfig config) { - final SparkConnectListener listener = new SparkConnectListener(); - listener.createSettings(config); - return listener; - } - - private static void intercept(MessageInterceptor interceptor, Message request) { - interceptAndReturn(interceptor, request); - } - - private static Message interceptAndReturn(MessageInterceptor interceptor, - Message request) { - final GrpcCallContext callContext = - new GrpcCallContext("m", "authority", "127.0.0.1", System.nanoTime()); - callContext.setPrincipal(USER); - final Message[] result = new Message[1]; - Context.current().withValue(GrpcCallContext.KEY, callContext) - .run(() -> result[0] = interceptor.intercept(request)); - return result[0]; - } - - private static void assertDenied(MessageInterceptor interceptor, Message request) { - try { - intercept(interceptor, request); - fail("Expected the request to be denied"); - } catch (StatusRuntimeException e) { - assertEquals(Status.Code.PERMISSION_DENIED, e.getStatus().getCode()); - } - } - - private static ConfigRequest configSet(String key, String value) { - return ConfigRequest.newBuilder() - .setOperation(ConfigRequest.Operation.newBuilder() - .setSet(ConfigRequest.Set.newBuilder() - .addPairs(KeyValue.newBuilder().setKey(key).setValue(value)))) - .build(); - } -} diff --git a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptorTest.java b/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptorTest.java deleted file mode 100644 index 79e071420c..0000000000 --- a/gateway-service-sparkconnect/src/test/java/org/apache/knox/gateway/sparkconnect/SparkConnectMessageInterceptorTest.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.knox.gateway.sparkconnect; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import org.apache.knox.gateway.grpc.GrpcCallContext; - -import com.google.protobuf.Message; - -import io.grpc.Status; -import io.grpc.StatusRuntimeException; - -import org.apache.spark.connect.proto.AnalyzePlanRequest; -import org.apache.spark.connect.proto.ConfigRequest; -import org.apache.spark.connect.proto.ExecutePlanRequest; -import org.apache.spark.connect.proto.InterruptRequest; -import org.apache.spark.connect.proto.ReattachExecuteRequest; -import org.apache.spark.connect.proto.UserContext; - -import org.junit.Test; - -public class SparkConnectMessageInterceptorTest { - - private static final String PRINCIPAL = "alice"; - private static final String SPOOFED = "root"; - - @Test - public void overwritesClientSuppliedUserId() { - final ExecutePlanRequest request = ExecutePlanRequest.newBuilder() - .setSessionId("session-1") - .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).setUserName(SPOOFED).build()) - .build(); - - final ExecutePlanRequest forwarded = (ExecutePlanRequest) intercept(request, PRINCIPAL); - - // The whole point of parsing message bodies: the client says who it is, and - // Spark believes it, so Knox replaces the claim with the identity it verified. - assertEquals(PRINCIPAL, forwarded.getUserContext().getUserId()); - assertEquals(PRINCIPAL, forwarded.getUserContext().getUserName()); - } - - @Test - public void assertsIdentityWhenClientSuppliesNoUserContext() { - final ExecutePlanRequest request = ExecutePlanRequest.newBuilder().setSessionId("session-1").build(); - - final ExecutePlanRequest forwarded = (ExecutePlanRequest) intercept(request, PRINCIPAL); - - assertEquals(PRINCIPAL, forwarded.getUserContext().getUserId()); - } - - @Test - public void assertsIdentityOnEveryRequestShape() { - // All twelve request types carry UserContext in the same field position, which - // is why one descriptor-driven rewrite covers the whole service rather than - // needing a handler per RPC. Spot-check across the RPC shapes. - final Message[] requests = { - ExecutePlanRequest.newBuilder() - .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()).build(), - AnalyzePlanRequest.newBuilder() - .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()).build(), - ConfigRequest.newBuilder() - .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()).build(), - InterruptRequest.newBuilder() - .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()).build(), - ReattachExecuteRequest.newBuilder() - .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()).build(), - }; - - for (Message request : requests) { - final Message forwarded = intercept(request, PRINCIPAL); - final UserContext userContext = (UserContext) forwarded.getField( - forwarded.getDescriptorForType().findFieldByName("user_context")); - assertEquals(request.getDescriptorForType().getName() + " kept the client's user_id", - PRINCIPAL, userContext.getUserId()); - } - } - - @Test - public void preservesEveryOtherField() { - final ExecutePlanRequest request = ExecutePlanRequest.newBuilder() - .setSessionId("session-1") - .setOperationId("operation-1") - .setClientType("pyspark") - .addTags("tag-a") - .addTags("tag-b") - .setUserContext(UserContext.newBuilder() - .setUserId(SPOOFED) - .addExtensions(com.google.protobuf.Any.newBuilder().setTypeUrl("type/x").build()) - .build()) - .build(); - - final ExecutePlanRequest forwarded = (ExecutePlanRequest) intercept(request, PRINCIPAL); - - assertEquals("session-1", forwarded.getSessionId()); - assertEquals("operation-1", forwarded.getOperationId()); - assertEquals("pyspark", forwarded.getClientType()); - assertEquals(request.getTagsList(), forwarded.getTagsList()); - // UserContext extensions belong to the client, not to the identity claim. - assertEquals(1, forwarded.getUserContext().getExtensionsCount()); - assertEquals("type/x", forwarded.getUserContext().getExtensions(0).getTypeUrl()); - } - - @Test - public void preservesFieldsUnknownToTheVendoredProtos() throws Exception { - // A newer Spark client can send fields these protos do not describe. Protobuf - // retains them as unknown fields across a parse and re-serialize, which is - // what keeps proto skew from being a breaking problem. - final ExecutePlanRequest known = ExecutePlanRequest.newBuilder() - .setSessionId("session-1") - .setUserContext(UserContext.newBuilder().setUserId(SPOOFED).build()) - .build(); - final com.google.protobuf.UnknownFieldSet unknown = com.google.protobuf.UnknownFieldSet.newBuilder() - .addField(9999, com.google.protobuf.UnknownFieldSet.Field.newBuilder() - .addVarint(42L).build()) - .build(); - final ExecutePlanRequest request = known.toBuilder().setUnknownFields(unknown).build(); - - final ExecutePlanRequest forwarded = (ExecutePlanRequest) intercept(request, PRINCIPAL); - - assertEquals(PRINCIPAL, forwarded.getUserContext().getUserId()); - assertEquals(42L, forwarded.getUnknownFields().getField(9999).getVarintList().get(0).longValue()); - } - - @Test - public void recordsSessionAndOperationForAuditing() { - final GrpcCallContext callContext = newCallContext(PRINCIPAL); - final ExecutePlanRequest request = ExecutePlanRequest.newBuilder() - .setSessionId("session-7") - .setOperationId("operation-9") - .build(); - - interceptWith(callContext, request, null); - - assertEquals("session-7", callContext.getSessionId()); - assertEquals("operation-9", callContext.getOperationId()); - } - - @Test - public void refusesToForwardWithoutAnAuthenticatedPrincipal() { - final GrpcCallContext callContext = - new GrpcCallContext("m", "authority", "127.0.0.1", System.nanoTime()); - try { - interceptWith(callContext, ExecutePlanRequest.getDefaultInstance(), null); - fail("Expected the request to be rejected without a principal"); - } catch (StatusRuntimeException e) { - // Forwarding here would send the client's own identity claim through - // untouched, which is exactly the spoofing this layer exists to stop. - assertEquals(Status.Code.INTERNAL, e.getStatus().getCode()); - } - } - - @Test - public void appliesTheConfiguredGuardBeforeRewriting() { - final boolean[] guardRan = {false}; - final SparkConnectMessageInterceptor.RequestGuard guard = (message, principal) -> { - guardRan[0] = true; - assertEquals(PRINCIPAL, principal); - // The guard sees the client's message, before identity assertion. - assertNotNull(message); - }; - - interceptWith(newCallContext(PRINCIPAL), ExecutePlanRequest.getDefaultInstance(), guard); - - assertTrue("guard was not invoked", guardRan[0]); - } - - private static Message intercept(Message request, String principal) { - return interceptWith(newCallContext(principal), request, null); - } - - private static Message interceptWith(GrpcCallContext callContext, - Message request, - SparkConnectMessageInterceptor.RequestGuard guard) { - final Message[] result = new Message[1]; - // Context.run keeps StatusRuntimeException unwrapped, which the rejection - // tests assert on directly. - io.grpc.Context.current() - .withValue(GrpcCallContext.KEY, callContext) - .run(() -> result[0] = new SparkConnectMessageInterceptor(guard).intercept(request)); - return result[0]; - } - - private static GrpcCallContext newCallContext(String principal) { - final GrpcCallContext callContext = - new GrpcCallContext("spark.connect.SparkConnectService/ExecutePlan", - "knox.example.com:15002", "127.0.0.1", System.nanoTime()); - callContext.setPrincipal(principal); - return callContext; - } -} diff --git a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java index 6f1794436f..77dee7221a 100644 --- a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java +++ b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java @@ -26,6 +26,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -55,25 +56,29 @@ public class GatewayTestConfig extends Configuration implements GatewayConfig { public static final int DEFAULT_WEBSHELL_MAX_CONCURRENT_SESSIONS = 3; public static final int DEFAULT_WEBSHELL_READ_BUFFER_SIZE = 1024; - /* Spark Connect defaults */ - public static final int DEFAULT_SPARKCONNECT_PORT = 15002; - public static final int DEFAULT_SPARKCONNECT_MAX_MESSAGE_SIZE = 134217728; - public static final long DEFAULT_SPARKCONNECT_PERMIT_KEEPALIVE_TIME = 10000L; - public static final int DEFAULT_SPARKCONNECT_MAX_CONCURRENT_CALLS_PER_CONNECTION = 1000; - public static final long DEFAULT_SPARKCONNECT_CHANNEL_IDLE_TIMEOUT = 1800000L; - public static final long DEFAULT_SPARKCONNECT_DRAIN_TIMEOUT = 30000L; - public static final String DEFAULT_SPARKCONNECT_ADD_ARTIFACTS_MODE = "ALLOW"; - public static final String DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX = "knox."; - public static final String DEFAULT_SPARKCONNECT_TOPOLOGY_METADATA_KEY = "knox-topology"; - - private boolean sparkConnectEnabled; - private int sparkConnectPort = DEFAULT_SPARKCONNECT_PORT; - private String sparkConnectDefaultTopology; - private String sparkConnectBackendTokenAlias; - private String sparkConnectAddArtifactsMode = DEFAULT_SPARKCONNECT_ADD_ARTIFACTS_MODE; - private String sparkConnectReservedConfigPrefix = DEFAULT_SPARKCONNECT_RESERVED_CONFIG_PREFIX; - private String sparkConnectTopologyMetadataKey = DEFAULT_SPARKCONNECT_TOPOLOGY_METADATA_KEY; - private List sparkConnectAddArtifactsAllowedUsers = Collections.emptyList(); + /* gRPC listener defaults */ + public static final int DEFAULT_GRPC_PORT = 15002; + public static final String DEFAULT_GRPC_SERVICE_ROLE = "GRPC"; + public static final int DEFAULT_GRPC_MAX_MESSAGE_SIZE = 134217728; + public static final long DEFAULT_GRPC_PERMIT_KEEPALIVE_TIME = 10000L; + public static final int DEFAULT_GRPC_MAX_CONCURRENT_CALLS_PER_CONNECTION = 1000; + public static final long DEFAULT_GRPC_CHANNEL_IDLE_TIMEOUT = 1800000L; + public static final long DEFAULT_GRPC_DRAIN_TIMEOUT = 30000L; + public static final String DEFAULT_GRPC_TOPOLOGY_METADATA_KEY = "knox-topology"; + + private boolean grpcEnabled; + private int grpcPort = DEFAULT_GRPC_PORT; + private String grpcServiceRole = DEFAULT_GRPC_SERVICE_ROLE; + private String grpcProtoServices; + private String grpcIdentityRules; + private List grpcListenerNames = new ArrayList<>(); + private final Map> grpcListenerConfig = new HashMap<>(); + private int grpcIdentityScanLimit = 131072; + private String grpcDefaultTopology; + private String grpcTopologyMetadataKey = DEFAULT_GRPC_TOPOLOGY_METADATA_KEY; + private String grpcMethodsDeny; + private String grpcMethodsAllow; + private String grpcBackendTokenAlias; @@ -702,105 +707,151 @@ public int getWebsocketMaxWaitBufferCount() { } @Override - public boolean isSparkConnectEnabled() { - return sparkConnectEnabled; + public boolean isGrpcEnabled() { + return grpcEnabled; } - public void setSparkConnectEnabled(boolean sparkConnectEnabled) { - this.sparkConnectEnabled = sparkConnectEnabled; + public void setGrpcEnabled(boolean grpcEnabled) { + this.grpcEnabled = grpcEnabled; } @Override - public int getSparkConnectPort() { - return sparkConnectPort; + public int getGrpcPort() { + return grpcPort; } - public void setSparkConnectPort(int sparkConnectPort) { - this.sparkConnectPort = sparkConnectPort; + public void setGrpcPort(int grpcPort) { + this.grpcPort = grpcPort; } @Override - public String getSparkConnectDefaultTopology() { - return sparkConnectDefaultTopology; + public String getGrpcServiceRole() { + return grpcServiceRole; } - public void setSparkConnectDefaultTopology(String sparkConnectDefaultTopology) { - this.sparkConnectDefaultTopology = sparkConnectDefaultTopology; + public void setGrpcServiceRole(String grpcServiceRole) { + this.grpcServiceRole = grpcServiceRole; } @Override - public int getSparkConnectMaxMessageSize() { - return DEFAULT_SPARKCONNECT_MAX_MESSAGE_SIZE; + public String getGrpcProtoServices() { + return grpcProtoServices; + } + + public void setGrpcProtoServices(String grpcProtoServices) { + this.grpcProtoServices = grpcProtoServices; } @Override - public long getSparkConnectPermitKeepAliveTime() { - return DEFAULT_SPARKCONNECT_PERMIT_KEEPALIVE_TIME; + public List getGrpcListenerNames() { + return grpcListenerNames; + } + + public void setGrpcListenerNames(List grpcListenerNames) { + this.grpcListenerNames = grpcListenerNames; } @Override - public boolean isSparkConnectPermitKeepAliveWithoutCalls() { - return true; + public Map getGrpcListenerConfig(String listenerName) { + final Map config = grpcListenerConfig.get(listenerName); + return config == null ? new HashMap<>() : config; + } + + public void setGrpcListenerConfig(String listenerName, Map config) { + this.grpcListenerConfig.put(listenerName, config); + } + + @Override + public String getGrpcIdentityRules() { + return grpcIdentityRules; + } + + public void setGrpcIdentityRules(String grpcIdentityRules) { + this.grpcIdentityRules = grpcIdentityRules; + } + + @Override + public int getGrpcIdentityScanLimit() { + return grpcIdentityScanLimit; + } + + public void setGrpcIdentityScanLimit(int grpcIdentityScanLimit) { + this.grpcIdentityScanLimit = grpcIdentityScanLimit; } @Override - public int getSparkConnectMaxConcurrentCallsPerConnection() { - return DEFAULT_SPARKCONNECT_MAX_CONCURRENT_CALLS_PER_CONNECTION; + public String getGrpcDefaultTopology() { + return grpcDefaultTopology; + } + + public void setGrpcDefaultTopology(String grpcDefaultTopology) { + this.grpcDefaultTopology = grpcDefaultTopology; } @Override - public long getSparkConnectChannelIdleTimeout() { - return DEFAULT_SPARKCONNECT_CHANNEL_IDLE_TIMEOUT; + public String getGrpcTopologyMetadataKey() { + return grpcTopologyMetadataKey; + } + + public void setGrpcTopologyMetadataKey(String grpcTopologyMetadataKey) { + this.grpcTopologyMetadataKey = grpcTopologyMetadataKey; } @Override - public long getSparkConnectDrainTimeout() { - return DEFAULT_SPARKCONNECT_DRAIN_TIMEOUT; + public String getGrpcMethodsDeny() { + return grpcMethodsDeny; + } + + public void setGrpcMethodsDeny(String grpcMethodsDeny) { + this.grpcMethodsDeny = grpcMethodsDeny; } @Override - public String getSparkConnectBackendTokenAlias() { - return sparkConnectBackendTokenAlias; + public String getGrpcMethodsAllow() { + return grpcMethodsAllow; } - public void setSparkConnectBackendTokenAlias(String sparkConnectBackendTokenAlias) { - this.sparkConnectBackendTokenAlias = sparkConnectBackendTokenAlias; + public void setGrpcMethodsAllow(String grpcMethodsAllow) { + this.grpcMethodsAllow = grpcMethodsAllow; } @Override - public String getSparkConnectAddArtifactsMode() { - return sparkConnectAddArtifactsMode; + public int getGrpcMaxMessageSize() { + return DEFAULT_GRPC_MAX_MESSAGE_SIZE; } - public void setSparkConnectAddArtifactsMode(String sparkConnectAddArtifactsMode) { - this.sparkConnectAddArtifactsMode = sparkConnectAddArtifactsMode; + @Override + public long getGrpcPermitKeepAliveTime() { + return DEFAULT_GRPC_PERMIT_KEEPALIVE_TIME; } @Override - public List getSparkConnectAddArtifactsAllowedUsers() { - return sparkConnectAddArtifactsAllowedUsers; + public boolean isGrpcPermitKeepAliveWithoutCalls() { + return true; } - public void setSparkConnectAddArtifactsAllowedUsers(List allowedUsers) { - this.sparkConnectAddArtifactsAllowedUsers = allowedUsers; + @Override + public int getGrpcMaxConcurrentCallsPerConnection() { + return DEFAULT_GRPC_MAX_CONCURRENT_CALLS_PER_CONNECTION; } @Override - public String getSparkConnectReservedConfigPrefix() { - return sparkConnectReservedConfigPrefix; + public long getGrpcChannelIdleTimeout() { + return DEFAULT_GRPC_CHANNEL_IDLE_TIMEOUT; } - public void setSparkConnectReservedConfigPrefix(String sparkConnectReservedConfigPrefix) { - this.sparkConnectReservedConfigPrefix = sparkConnectReservedConfigPrefix; + @Override + public long getGrpcDrainTimeout() { + return DEFAULT_GRPC_DRAIN_TIMEOUT; } @Override - public String getSparkConnectTopologyMetadataKey() { - return sparkConnectTopologyMetadataKey; + public String getGrpcBackendTokenAlias() { + return grpcBackendTokenAlias; } - public void setSparkConnectTopologyMetadataKey(String sparkConnectTopologyMetadataKey) { - this.sparkConnectTopologyMetadataKey = sparkConnectTopologyMetadataKey; + public void setGrpcBackendTokenAlias(String grpcBackendTokenAlias) { + this.grpcBackendTokenAlias = grpcBackendTokenAlias; } @Override diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java index 47dfb53dbf..3a2532cffa 100644 --- a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java +++ b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java @@ -546,128 +546,177 @@ public interface GatewayConfig { */ int getWebsocketMaxWaitBufferCount(); + String GRPC_PROTO_SERVICES = "gateway.grpc.proto.services"; + String GRPC_LISTENER_NAMES = "gateway.grpc.listener.names"; + /** - * Returns true if the Spark Connect (gRPC) listener is enabled, else false. - * Default is false. + * The gRPC listeners to run, each on its own port with its own TLS identity. + * Several exist to serve several hostnames from one gateway, which is what a + * platform PKI that cannot issue multi-name certificates forces; they are not a + * policy boundary, since each still routes to every topology its clients + * select. An empty list means one listener configured entirely from the plain + * {@code gateway.grpc.*} properties. * @since 3.0.0 - * @return true if the Spark Connect listener should be started + * @return the configured listener names, possibly empty; never null */ - boolean isSparkConnectEnabled(); + List getGrpcListenerNames(); /** - * The port the Spark Connect gRPC listener binds to. This is a dedicated - * socket, separate from the gateway's Jetty connectors, because gRPC needs - * HTTP/2 with ALPN. + * Properties set for one gRPC listener: everything under + * {@code gateway.grpc.{listenerName}.} with the prefix stripped. A property a + * listener does not set falls back to the plain {@code gateway.grpc.} one, so + * shared settings are written once. + * @since 3.0.0 + * @param listenerName the listener name + * @return the listener's own properties, possibly empty; never null + */ + Map getGrpcListenerConfig(String listenerName); + + /** + * Returns true if the gRPC listener is enabled, else false. Default is false. + * @since 3.0.0 + * @return true if the listener should be started + */ + boolean isGrpcEnabled(); + + /** + * The port the gRPC listener binds to. This is a dedicated socket, separate + * from the gateway's Jetty connectors, because gRPC needs HTTP/2 with ALPN. * @since 3.0.0 * @return the listener port */ - int getSparkConnectPort(); + int getGrpcPort(); /** - * The topology used when no other discriminator selects one. Spark Connect - * clients cannot put a path in an {@code sc://} URL, so Knox's usual - * {@code /gateway/{topology}/{service}} routing is unavailable and the - * topology must come from elsewhere. + * The Knox service role that ties this listener to a topology. Topologies + * declare a service with this role and its backend URL, and its ACLs are keyed + * on it. * @since 3.0.0 - * @return the default topology name, or null if unset + * @return the service role */ - String getSparkConnectDefaultTopology(); + String getGrpcServiceRole(); /** - * Maximum inbound message size in bytes, applied to both legs. Spark's own - * default is 128 MB and grpc-java materializes whole messages, so this bounds - * per-message heap. + * The fully qualified proto service names to proxy, comma separated. Calls to + * anything else are answered {@code UNIMPLEMENTED}, so this is a closed list. * @since 3.0.0 - * @return max message size in bytes + * @return the proto service names */ - int getSparkConnectMaxMessageSize(); + String getGrpcProtoServices(); /** - * The minimum interval the listener will tolerate between client keepalive - * pings before treating them as abusive, in milliseconds. + * Where the authenticated identity is written in a request: a comma-separated + * list of {@code path=subject} rules, where each path is one or more protobuf + * field numbers separated by dots, for example + * {@code 2.1=principal,2.2=principal}. Naming numbers rather than compiling + * against generated classes is what keeps the gateway independent of any + * protocol version. Empty means requests are relayed without inspection. * @since 3.0.0 - * @return permitted keepalive interval in milliseconds + * @return the identity rewrite rules, or null for none */ - long getSparkConnectPermitKeepAliveTime(); + String getGrpcIdentityRules(); /** - * Whether clients may send keepalive pings with no active calls. Spark Connect - * clients ping idle channels, so this defaults to true. + * The maximum offset, in bytes, at which an identity field being rewritten may + * end. Rewriting a nested field means slicing it out and rebuilding it, so + * without a bound a client could put an arbitrarily large payload inside the + * identity container and make the gateway copy it several times over. A + * request whose identity lies beyond the limit is refused rather than + * partially asserted. * @since 3.0.0 - * @return true if keepalives without calls are permitted + * @return the scan limit in bytes */ - boolean isSparkConnectPermitKeepAliveWithoutCalls(); + int getGrpcIdentityScanLimit(); /** - * Maximum concurrent gRPC streams per client connection. + * The topology used when no other discriminator selects one. gRPC clients + * cannot put a path in the connection URL, so Knox's usual + * {@code /gateway/{topology}/{service}} routing is unavailable and the topology + * must come from elsewhere. * @since 3.0.0 - * @return max concurrent calls per connection + * @return the default topology name, or null if unset */ - int getSparkConnectMaxConcurrentCallsPerConnection(); + String getGrpcDefaultTopology(); /** - * How long an unused backend channel is kept before being shut down, in - * milliseconds. + * The name of the call-metadata entry a client uses to select a topology, + * which is also the connection-string parameter users write. * @since 3.0.0 - * @return backend channel idle timeout in milliseconds + * @return the metadata key name */ - long getSparkConnectChannelIdleTimeout(); + String getGrpcTopologyMetadataKey(); /** - * How long to let in-flight RPCs finish when the gateway is shutting down, - * in milliseconds. Long-running {@code ExecutePlan} streams are severed once - * this elapses; clients recover through their own reattach logic. + * RPCs refused by default, by bare or fully qualified method name. gRPC carries + * the method in the request path, so this needs no knowledge of message + * contents. Topologies may override it. * @since 3.0.0 - * @return drain timeout in milliseconds + * @return a comma-separated method list, or null */ - long getSparkConnectDrainTimeout(); + String getGrpcMethodsDeny(); /** - * The alias holding the pre-shared token Knox presents to the Spark Connect - * backend ({@code spark.connect.authenticate.token}). Besides authenticating - * Knox to Spark, this stops clients bypassing the gateway when they have - * network reachability to the backend port. + * When set, the only RPCs permitted by default; anything else is refused. + * Topologies may override it. * @since 3.0.0 - * @return the alias name, or null if the backend requires no token + * @return a comma-separated method list, or null */ - String getSparkConnectBackendTokenAlias(); + String getGrpcMethodsAllow(); /** - * Governs the {@code AddArtifacts} RPC: {@code ALLOW}, {@code DENY}, or - * {@code ALLOW_LISTED_USERS}. User-supplied jars run with the Spark - * application's own storage credentials, so this is defense in depth rather - * than an authorization boundary. + * Maximum inbound message size in bytes, applied to both legs. grpc-java + * materializes whole messages, so this bounds per-message heap. * @since 3.0.0 - * @return the gating mode + * @return max message size in bytes */ - String getSparkConnectAddArtifactsMode(); + int getGrpcMaxMessageSize(); /** - * Users permitted to call {@code AddArtifacts} when the gating mode is - * {@code ALLOW_LISTED_USERS}. + * The minimum interval the listener tolerates between client keepalive pings + * before treating them as abusive, in milliseconds. * @since 3.0.0 - * @return the permitted user names; empty if none configured + * @return permitted keepalive interval in milliseconds */ - List getSparkConnectAddArtifactsAllowedUsers(); + long getGrpcPermitKeepAliveTime(); /** - * The session-configuration key prefix reserved for Knox. Clients are denied - * {@code Set}/{@code Unset} on keys under this prefix so they cannot forge the - * identity Knox publishes into the session. + * Whether clients may send keepalive pings with no active calls. * @since 3.0.0 - * @return the reserved key prefix + * @return true if keepalives without calls are permitted */ - String getSparkConnectReservedConfigPrefix(); + boolean isGrpcPermitKeepAliveWithoutCalls(); /** - * The name of the call-metadata entry a client uses to select a topology, - * which is also the connection-string parameter users write. Configurable so a - * deployment can choose a name that suits its users rather than one that names - * the gateway reading it. + * Maximum concurrent gRPC streams per client connection. * @since 3.0.0 - * @return the metadata key name + * @return max concurrent calls per connection + */ + int getGrpcMaxConcurrentCallsPerConnection(); + + /** + * How long an unused backend channel is kept before being shut down, in + * milliseconds. + * @since 3.0.0 + * @return backend channel idle timeout in milliseconds + */ + long getGrpcChannelIdleTimeout(); + + /** + * How long to let in-flight RPCs finish when the gateway is shutting down, in + * milliseconds. Long-lived streams are severed once this elapses. + * @since 3.0.0 + * @return drain timeout in milliseconds + */ + long getGrpcDrainTimeout(); + + /** + * The alias holding a pre-shared token Knox presents to the backend. Besides + * authenticating the gateway, this stops clients bypassing it when they have + * network reachability to the backend port. + * @since 3.0.0 + * @return the alias name, or null if the backend requires no token */ - String getSparkConnectTopologyMetadataKey(); + String getGrpcBackendTokenAlias(); boolean isMetricsEnabled(); diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/protocol/ProtocolListener.java b/gateway-spi/src/main/java/org/apache/knox/gateway/protocol/ProtocolListener.java index 064b1c6640..d03bb0a21c 100644 --- a/gateway-spi/src/main/java/org/apache/knox/gateway/protocol/ProtocolListener.java +++ b/gateway-spi/src/main/java/org/apache/knox/gateway/protocol/ProtocolListener.java @@ -17,6 +17,9 @@ */ package org.apache.knox.gateway.protocol; +import java.util.Collections; +import java.util.List; + import org.apache.knox.gateway.config.GatewayConfig; import org.apache.knox.gateway.services.GatewayServices; @@ -92,6 +95,16 @@ public interface ProtocolListener { */ int getPort(); + /** + * Every port this listener bound, for implementations that run more than one + * server. Reported at startup so an operator can see what came up. + * + * @return the bound ports; by default the single {@link #getPort()} + */ + default List getPorts() { + return Collections.singletonList(getPort()); + } + /** * Notifies the listener that topologies have been redeployed, so anything it * derived from topology configuration must be recomputed. diff --git a/knox-site/docs/grpc-support.md b/knox-site/docs/grpc-support.md new file mode 100644 index 0000000000..4d0722cd1e --- /dev/null +++ b/knox-site/docs/grpc-support.md @@ -0,0 +1,878 @@ + + +## gRPC Support ## + +### Introduction ### + +Knox can proxy gRPC services. A dedicated listener — or several, each on its own +port — runs alongside Jetty, terminating gRPC calls at the edge and relaying them +to a backend declared in a topology — with Knox's usual authentication, topology ACLs and +audit applied first. + +It is generic. The listener compiles against no `.proto` file and no generated +class; it relays opaque bytes for whichever proto services a deployment names. +Everything protocol-specific is a configuration value. + +That is worth stating plainly because it is a change of posture. Knox's servlet +pipeline was already a generic reverse proxy — an arbitrary REST API needs a +service definition and a rewrite rule, no code, and WebSocket proxying matches +any service definition by context path. gRPC was the one protocol class outside +that coverage, because it is the one the servlet stack cannot physically carry. +This closes the gap without starting a pattern of per-protocol special cases. + +The motivating workload was **Spark Connect**, and it runs through this page as +the worked example. Spark Connect is the decoupled client/server protocol for +Apache Spark (3.4+, and the default architecture in Spark 4); its OSS server has +essentially no built-in authentication or authorization, because the project +assumes a fronting proxy provides them. But nothing in the gateway knows what +Spark Connect is. Every Spark-specific thing below is a value in +`gateway-site.xml` or a topology file. + +#### What the gateway adds #### + +- **Authentication at the edge** — Knox-issued JWTs (KnoxToken bearer tokens), + validated before anything reaches the backend. +- **Identity assertion** — where a protocol carries a *client-asserted* identity + field, Knox overwrites it with the authenticated principal. +- **Coarse authorization** — the usual topology ACLs decide who may use the + service at all, and per-topology allow/deny lists decide which RPCs they may + call. +- **Auditing** — one record per RPC: principal, topology, backend, method, + outcome and duration. + +#### Why a separate port #### + +gRPC does not go through Knox's servlet pipeline, and could not: it needs HTTP/2 +negotiated over ALPN, which Knox's Jetty connectors do not offer; the Servlet 3.1 +API Knox targets has no way to read or write HTTP trailers, where `grpc-status` +and any structured error details live; and the outbound dispatch layer is built +on a strict request/response HTTP/1.1 client, whereas gRPC calls may stream in +either or both directions for hours. + +Routing rules it out independently. gRPC fixes request paths at +`/pkg.Service/Method`, and connection strings for such protocols commonly forbid +a path component altogether — a Spark Connect `sc://` URL may not contain one, by +rule, to stay compatible with the gRPC standard. Knox's usual +`/gateway/{topology}/{service}` routing therefore has nothing to match on. + +So gRPC is served by a **dedicated listener on its own port**, started and +stopped with the gateway, alongside Jetty rather than inside it: + + Knox JVM + ┌──────────────────────────────────────────────┐ + grpc/h2 ▶│ :15002 gRPC listener (Netty) │ + │ ├─ TLS (gateway identity, or its own) │ + │ ├─ audit (one record per RPC) │ + │ ├─ authentication (bearer JWT) │ grpc/h2 + │ ├─ routing (topology metadata → registry) │──▶ backend + │ ├─ authorization (topology ACLs) │ service + │ ├─ method allow/deny (by RPC name) │ + │ └─ relay (asserts the identity field) │ + ├──────────────────────────────────────────────┤ + │ :8443 Jetty (the existing servlet gateway) │ + └──────────────────────────────────────────────┘ + +The listener is discovered through the `ProtocolListener` service-loader +interface, so gRPC and its shaded Netty stay off the servlet classpath entirely +unless the module is deployed. This mirrors how Knox already handles WebSockets, +which likewise bypass the topology filter chains and do their own +authentication — the difference being that a WebSocket upgrade can share Jetty's +HTTP/1.1 connector, and gRPC cannot. + +### Relaying without a schema ### + +Every call for a permitted proto service reaches the same byte-level relay. No +generated service is registered, no method list is enumerated, and no message +type is known. + +**All four RPC shapes** — unary, server-streaming, client-streaming and +bidirectional — go through one handler, because they differ only in how many +messages flow each way. Backend status and trailers are relayed verbatim, which +matters more than it looks: gRPC carries `grpc-status` in trailers, and protocols +commonly pack structured error details there too, so anything that interprets or +drops trailers breaks error reporting wholesale. + +Flow control is explicit in both directions — a message is requested from one +side only once the other has accepted the previous one — so a slow client cannot +make the gateway buffer an unbounded number of response batches, and cancellation +propagates both ways rather than leaving orphaned work on the backend. + +An RPC added by a newer version of a protocol is proxied like any other, since +nothing enumerates methods. Proto services *not* named in the permitted set are +answered `UNIMPLEMENTED` — the same answer a real server gives for a method it +does not have, so this reveals nothing about what the gateway fronts. + +#### Identity assertion by field number #### + +The one thing a byte-level proxy would normally give up is the ability to touch +message contents, and that is exactly what makes this worth doing. Protocols in +this family commonly trust a *client-asserted* identity field: the client states +who it is and the server believes it. Such a field typically keys the server-side +session cache, so leaving it alone lets one caller collide with — or attach +to — another's session simply by claiming their name. That is session isolation, +not merely audit fidelity. + +It is recovered without a schema by a list of rewrite rules, each naming a place +in the message by field number and what to write there: + + + gateway.grpc.identity.rules + 2.1=principal,2.2=principal + + +A rule is `path=subject`. The path is one or more protobuf field numbers +separated by dots: each leading number is a nested message to descend into, and +the last is the string field to replace. So `1=principal` rewrites a top-level +field, and `2.1=principal` rewrites a field one level down. The subject names +what to write; `principal` — the subject of the validated bearer token — is the +only one this build supports, and an unknown one is refused at startup rather +than written as an empty string. + +The example above is Spark Connect: `user_context = 2`, holding `user_id = 1` and +`user_name = 2`, both set to the authenticated principal. Field numbers are the +part of a protobuf schema that cannot change without breaking every deployed +client, so this tracks no particular protocol version. Zero rules is the ordinary +case for a protocol that carries no identity: the relay is then a pure pipe. + +What happens per request: + +- Every field named by a rule is replaced with the authenticated principal. + *Every occurrence* of it, not just the first — protobuf merges repeated + records, so one left alone could override the one that was asserted. +- Everything else is copied byte for byte, including extensions and fields from a + newer protocol version this build has never heard of. Those are not merely + preserved; they are never decoded. +- A path that is absent is created, the whole chain of it, so the backend never + sees a request whose identity Knox did not put there. +- A message that cannot be parsed is rejected with `INVALID_ARGUMENT` rather than + forwarded. So is one whose shape contradicts the rules — a field a rule expects + to descend into that arrives as a scalar, say. Forwarding either would send the + caller's own claim through unaltered, which is what this exists to prevent. + +Rules are validated at startup: field numbers in protobuf's legal range, avoiding +the reserved 19000–19999 block, no two rules writing the same place, and none +writing a value at a field another descends through. + +#### The scan limit #### + +Rewriting a nested field means slicing it out and rebuilding it, so a client that +put a hundred megabytes inside the identity container could make the gateway copy +it several times over. Every field a rule touches must therefore lie wholly +within the first `gateway.grpc.identity.scan.limit` bytes of the request, 128 KiB +by default: + + + gateway.grpc.identity.scan.limit + 131072 + + +This bounds where the identity may sit, not how large a request may be. Generated +serializers emit fields in ascending number order, so an identity container with +a low field number lands near the front however large the payload after it — a +128 MB `AddArtifacts` chunk passes, and costs the same to rewrite as a small +message. + +A request that breaks the limit is **refused**, with `INVALID_ARGUMENT`, rather +than partially asserted. That is the security-relevant part: giving up on a rule +whose target sits beyond the limit and synthesising a fresh identity instead would +leave the caller's own claim in the message behind ours, where protobuf's +last-wins merge would let it take effect. Synthesis when a path is genuinely +absent is unaffected — the whole message was walked and nothing was found, so +there is nothing an appended identity could be overridden by. + +Raise the limit for a protocol that puts its identity late or carries an unusually +large identity container. Lower it to tighten the bound. + +Correctness is held to typed semantics by test. The Spark Connect protos are +vendored at **test scope only** and used as an oracle: they say what the wire +bytes are supposed to mean, and the hand-written wire code is checked against +generated classes across every request shape, unknown fields, absent containers +and large payloads. A protocol change that moved the fields this depends on fails +CI rather than production. Nothing generated ships in the module jar, and the +gateway needs no protobuf library at runtime. + +### Configuration ### + +The listener is disabled by default. Enable it in +`/conf/gateway-site.xml`: + + + gateway.grpc.enabled + true + + + gateway.grpc.service.role + SPARKCONNECT + The Knox service role that ties this listener to a topology. + + + gateway.grpc.proto.services + spark.connect.SparkConnectService + The proto services to proxy. Anything else gets UNIMPLEMENTED. + + + gateway.grpc.identity.rules + 2.1=principal,2.2=principal + Where the identity lives: user_context = 2, user_id = 1, user_name = 2. + + +`gateway.grpc.proto.services` is required — a listener with nothing to proxy +refuses to start, rather than binding a port that answers `UNIMPLEMENTED` to +everything and looks like it is working. + +By default a listener presents the gateway's own TLS identity — the same keystore +and alias Jetty uses — so there is no second certificate to manage. A listener +can present its own instead; see below. Running without TLS is logged as a +warning: bearer tokens would cross the network in clear text, so it is a +development posture only. + +#### Several listeners, several certificates #### + +A gateway can run more than one gRPC listener. They are **not** a policy +boundary — each still routes to as many topologies as its clients select, by the +same metadata key — so this is not an alternative to topology selection. + +What separates them is the socket, and therefore the certificate on it. Serving +several hostnames from one endpoint needs one certificate naming all of them, and +a platform PKI that cannot issue multi-name (SAN or wildcard) certificates cannot +produce one. A listener per hostname, each presenting a plain single-name +certificate, serves those clients without it: + + + gateway.grpc.listener.names + analytics, partner + + + + gateway.grpc.analytics.port + 15002 + + + gateway.grpc.analytics.ssl.keystore.path + /opt/pki/analytics.p12 + + + + gateway.grpc.partner.port + 15003 + + + gateway.grpc.partner.ssl.keystore.path + /opt/pki/partner.p12 + + +Clients then dial `sc://analytics.example.com:15002` and +`sc://partner.example.com:15003`, both resolving to the same gateway, each +validating a certificate issued for the name it asked for — and both selecting +topologies exactly as they would on a single listener. + +**Every property inherits.** A listener reads `gateway.grpc..` +where it sets one and the plain `gateway.grpc.` otherwise, so shared +settings — the service role, proto services, identity rules, message limits — are +written once and only the differences are repeated. Naming no listeners runs +exactly one, configured entirely from the plain properties, which is the ordinary +deployment and what every earlier example on this page describes. + +The TLS properties are the exception: `ssl.keystore.path`, `ssl.keystore.alias`, +`ssl.keystore.password.alias` and `ssl.keystore.type` are never inherited, since +sharing one keystore across listeners would defeat the point of having several. A +listener that sets no keystore path presents the gateway identity. + +| Property | Default | Meaning | +|------------------------------------------------------|---------------------|------------------------------------------------------------------------------| +| `gateway.grpc.listener.names` | *(none)* | Comma-separated listener names. Empty means one listener from the plain properties. | +| `gateway.grpc..` | the plain property | Any property above, set for one listener. | +| `gateway.grpc..ssl.enabled` | `ssl.enabled` | Whether this listener presents TLS. | +| `gateway.grpc..ssl.keystore.path` | *(gateway identity)*| A keystore holding this listener's server certificate. | +| `gateway.grpc..ssl.keystore.type` | `PKCS12` | Its format. | +| `gateway.grpc..ssl.keystore.alias` | *(the sole entry)* | Which entry to present. Required if the keystore holds more than one key. | +| `gateway.grpc..ssl.keystore.password.alias` | *(gateway's)* | Knox alias holding the keystore password. | + +Names may contain `a-z`, `0-9`, `-` and `_`. A name that collides with an +existing property — `identity`, `default`, `methods` and so on — is refused at +startup, as are two listeners configured on one port: the alternative is an +address-in-use error naming neither of them. + +Which listeners exist is fixed at startup, like whether the feature runs at all. +A name added or removed in `gateway-reloadable.xml` is reported in the log rather +than acted on; the per-listener properties that *can* move at runtime move for +each listener independently. + +#### All properties #### + +| Property | Default | Meaning | +|----------------------------------------------------|-----------------|-----------------------------------------------------------------------------------------------------| +| `gateway.grpc.enabled` | `false` | Master switch; the listener is not started when false. | +| `gateway.grpc.port` | `15002` | Port for the gRPC listener. The default is Spark Connect's. | +| `gateway.grpc.service.role` | `GRPC` | Knox service role tying this listener to a topology, and the prefix for its ACL and method params. | +| `gateway.grpc.proto.services` | *(none)* | Comma-separated proto service names to proxy. Required; anything else gets `UNIMPLEMENTED`. | +| `gateway.grpc.identity.rules` | *(none)* | Comma-separated `path=subject` rules placing the identity by field number. Empty means no rewrite. | +| `gateway.grpc.identity.scan.limit` | `131072` | Every field a rule rewrites must end within this many bytes of the request start. | +| `gateway.grpc.default.topology` | *(none)* | Topology used when the client selects none. | +| `gateway.grpc.topology.metadata.key` | `knox-topology` | Name of the connection-string parameter clients use to select a topology. | +| `gateway.grpc.methods.deny` | *(none)* | RPCs refused by default, by method name. Topologies may override. | +| `gateway.grpc.methods.allow` | *(none)* | When set, the only RPCs permitted by default. Topologies may override. | +| `gateway.grpc.max.message.size` | `134217728` | Maximum inbound message size in bytes, both legs. Matches Spark's 128 MB default. | +| `gateway.grpc.max.concurrent.calls.per.connection` | `1000` | Maximum concurrent gRPC streams per client connection. | +| `gateway.grpc.permit.keepalive.time` | `10000` | Minimum tolerated interval between client keepalive pings, in ms. | +| `gateway.grpc.permit.keepalive.without.calls` | `true` | Whether clients may ping an idle channel. Spark Connect clients do. | +| `gateway.grpc.channel.idle.timeout` | `1800000` | Idle time before an unused backend channel is shut down, in ms. | +| `gateway.grpc.drain.timeout` | `30000` | How long in-flight RPCs get to finish at shutdown, in ms. | +| `gateway.grpc.backend.token.alias` | *(none)* | Alias holding the backend's pre-shared token (see Security considerations). | + +The message, stream and keepalive limits are the listener's DoS surface. A new +socket accepting 128 MB messages on long-lived streams wants those bounds set +from the start, not added after the first incident. + +#### Keeping these out of `gateway-site.xml` #### + +Knox has no `conf.d` directory, but it does load one optional extra file. The +gateway reads exactly three configuration files from `{GATEWAY_HOME}/conf`, in +this order, with later files overriding earlier ones: + + gateway-default.xml + gateway-site.xml + gateway-reloadable.xml + +`gateway-reloadable.xml` is not shipped and does not have to exist, so the +`gateway.grpc.*` properties can live there instead of being merged into +`gateway-site.xml`. It is a single shared file rather than a per-feature +directory, so anything else using it has to co-exist in the same file — but it +does keep this feature's settings out of the main one. See +[Reloadable Gateway Configuration](config.md) for the general mechanism. + +Knox re-reads that file every `gateway.config.refresh.interval` milliseconds +(default 10 seconds). Two of the properties above then take effect immediately; +the rest cannot, because they are built into the bound server. + +**Applied on the next RPC, no restart:** + +- `gateway.grpc.identity.rules` +- `gateway.grpc.identity.scan.limit` +- `gateway.grpc.default.topology` + +These are the message-level and routing controls — the ones an operator is most +likely to want to change in response to something happening. They apply to the +very next call without interrupting any session. + +**Restart required:** everything else — `gateway.grpc.enabled` itself, the port, +the proto service list, the service role, the topology metadata key, TLS, the +gateway-wide method lists, message and stream limits, keepalive settings, and the +channel idle and drain timeouts. Whether the listener runs at all is decided once +at startup, and the rest are fixed when the socket is bound. + +Changing a restart-only property in a running gateway does not silently do +nothing. The refreshed configuration is compared against what the gateway started +with, and a warning names what could not be applied — for the transport settings, +which properties changed; for `gateway.grpc.enabled`, that the listener cannot be +started or stopped without a restart. Switching it on when it was off at startup +is reported too, which is the case most likely to be mistaken for a malfunction: +without the warning, the only symptom is a port that never opens. + +### Topology configuration ### + +Declare the backend like any other service, under the role named by +`gateway.grpc.service.role`. The registry treats the URL as an opaque string, so +`grpc://` and `grpcs://` need no special handling: + + + SPARKCONNECT + grpc://backend-host:15002 + + +Use `grpcs://` for a TLS backend; Knox verifies it against the HTTP client +truststore, falling back to the gateway keystore. Any other scheme, or a URL +without a host and port, is refused with `FAILED_PRECONDITION`. + +Authorization uses the ordinary `AclsAuthz` provider syntax, keyed on the same +role: + + + authorization + AclsAuthz + true + + SPARKCONNECT.acl + *;analysts;* + + + +The syntax and semantics are the servlet provider's, down to sharing its parser: +`users;groups;ipaddresses`, an `AND`/`OR` processing mode via +`SPARKCONNECT.acl.mode`, `*` wildcards, and the `KNOX_ADMIN_USERS` / +`KNOX_ADMIN_GROUPS` placeholders. Nobody should have to learn a second ACL +dialect because the transport changed. + +Group membership comes from the `knox.groups` claim in the token, so configure +`knoxtoken` to include groups if you intend to write group ACLs. + +#### Refusing individual RPCs #### + +gRPC puts the method in the request path, so allowing or denying whole RPCs by +name needs no marshaller, no descriptor and no schema. It is the coarsest control +the gateway offers and the only message-level one that survives completely intact +on a byte-level proxy. + +Configure it alongside the ACLs, on the same provider: + + + SPARKCONNECT.methods.deny + AddArtifacts + + +Names may be bare (`AddArtifacts`, matching that method on any service) or fully +qualified (`spark.connect.SparkConnectService/AddArtifacts`), and matching is +case-insensitive. `SPARKCONNECT.methods.allow` is the inverse and, once given, is +exhaustive: anything unnamed is refused, so an RPC added by a newer protocol +version does not appear by default. Deny wins over allow. A topology that sets +neither falls back to the gateway-wide `gateway.grpc.methods.*` lists. + +Being keyed on the topology is the point: the same gateway can front a cluster +where uploading code is fine and one where it is not, and the difference is a +parameter in the topology that already forms the policy boundary. The check runs +last in the chain, so a denial is attributable to a known user in a known +topology. + +Be clear about the limit, though. Denying an upload RPC does not close code +execution where a protocol also allows inline functions inside ordinary +requests — inline Python and Scala UDFs travel *inside* Spark Connect's +`ExecutePlan`. It shrinks the attack surface rather than drawing a boundary. See +Security considerations. + +### Multiple backends ### + +One topology per backend, all served by the single listener port. A topology +declares exactly one service for the role, so a second backend means a second +topology: + + conf/analytics.xml -> grpc://spark-analytics:15002 + conf/etl.xml -> grpc://spark-etl:15002 + +Clients pick one with the `knox-topology` connection parameter: + + sc://knox-host:15002/;use_ssl=true;token=;knox-topology=analytics + sc://knox-host:15002/;use_ssl=true;token=;knox-topology=etl + +Both connect to the *same* Knox port and are routed to different backends. +Topology selection is per-RPC, from call metadata, so concurrent sessions from +different users — or from one user — are multiplexed over that one port onto +distinct backends. Knox keeps one pooled gRPC channel per backend URL and shares +it across all calls routed there; channels go idle on their own and reconnect +transparently, so a cached entry for an unused backend costs nothing. + +This is safe for session-oriented protocols because the discriminator is sticky +by construction: the client sends the same value on every request of the +connection, so every call in a session lands on the backend that owns it — which +Spark Connect's `ReattachExecute` requires. Nothing round-robins. + +Each topology carries its own authentication provider, ACLs, method rules and +audit scope, so "separate backend" and "separate policy boundary" stay aligned. + +#### Authorizing which users may select which backend #### + +Topology selection is a client-supplied value, so it is authorized rather than +trusted. Authorization runs *after* routing and is evaluated against the topology +that was selected — naming a topology in a connection string is not the same as +being allowed to use it. Give each topology its own ACL: + + + + SPARKCONNECT.acl + *;analysts;* + + + + + SPARKCONNECT.acl + *;engineers;* + + +An analyst connecting with `knox-topology=etl` is refused with +`PERMISSION_DENIED` before any backend connection is made. The full ACL syntax +applies per topology, so selection can be gated by user name, by group, by source +address, or by a combination. + +Two things to be deliberate about: + +- **The default is permissive.** A topology that declares no ACL for the role, or + whose `AclsAuthz` provider is disabled, is reachable by *any* authenticated + user. This matches the servlet provider's behaviour, but it means restricting + selection is something you switch on, not something you get for free. +- **Group ACLs need group claims.** Groups come from the token's `knox.groups` + claim, so a `knoxtoken` deployment that does not embed groups will match no + group ACL. Where groups are unavailable, gate on user names instead. + +A user probing topologies they cannot use can tell an existing one +(`PERMISSION_DENIED`) from one that does not exist or declares no service for the +role (`UNAVAILABLE`). They must already hold a valid token to learn even that, but +do not treat topology names as secrets. + +What is *not* supported is several backends **within** one topology. Where +sessions are server-side state keyed on the caller's identity and a session id, +spreading one topology across backends needs session-affine routing rather than +any form of load balancing; that is not implemented (see Limitations). + +#### Adding a backend without a restart #### + +Topologies are hot-reloaded. Knox watches the topologies directory and picks up +changes within about five seconds, so dropping in a new topology file, or editing +an existing one, takes effect on a running gateway: + +- **A new topology, or a changed backend URL** — takes effect on the next RPC. + The backend is resolved from the service registry per call, and redeployment + rewrites the registry entry. +- **Changed ACLs or method rules** — take effect on the next RPC after the + redeployment. The listener caches parsed ACLs and method policies per topology + and drops both caches when topologies are redeployed. +- **A deleted topology** — subsequent calls selecting it fail `UNAVAILABLE`. + Calls already in flight are not interrupted. + +Only `gateway-site.xml` properties — the port, the proto service list, message +limits and so on — need a gateway restart, since the listener binds its socket and +captures those settings at startup. + +### Connecting ### + +Clients need no plugins or code changes. First acquire a token — over HTTPS, +authenticating however that topology is configured: + + curl --negotiate -u : https://knox:8443/gateway/tokens/knoxtoken/api/v1/token + +Then put it in the connection string. For Spark Connect: + + sc://knox-host:15002/;use_ssl=true;token=;knox-topology=analytics + +Two details make this work, and both generalize to any gRPC client that can set +static metadata. The `token=` parameter is sent as a standard +`Authorization: Bearer` header and forces TLS on. Any parameter the client does +not recognize — `knox-topology` here — is sent as gRPC metadata on every request, +which is how a topology gets selected despite gRPC forbidding a path component in +the connection URL. If you set `gateway.grpc.default.topology`, the parameter can +be omitted. + +#### Renaming the topology parameter #### + +`knox-topology` is only a default. Because the name appears verbatim in every +connection string a user writes, a deployment may prefer one that describes the +choice being made rather than the gateway making it: + + + gateway.grpc.topology.metadata.key + cluster + + +Clients then write `sc://knox-host:15002/;use_ssl=true;token=;cluster=analytics`. +The gateway strips this parameter before forwarding, so the backend never sees it +under any name. + +gRPC restricts header names to lowercase letters, digits and `-_.`, and reserves +the `-bin` suffix for binary values. A name that breaks those rules — or that +collides with `authorization` — is rejected when the gateway starts, with an +explanation, rather than failing obscurely on the first call. + +### Worked example: Spark Connect ### + +Everything Spark-specific in a working deployment. In `gateway-site.xml`: + + + gateway.grpc.enabled + true + + + gateway.grpc.service.role + SPARKCONNECT + + + gateway.grpc.proto.services + spark.connect.SparkConnectService + + + + gateway.grpc.identity.rules + 2.1=principal,2.2=principal + + +The whole `spark.connect.SparkConnectService` surface is then proxied, every RPC +relayed with its status and trailers passed through verbatim and its +`user_context` replaced: + +| Shape | RPCs | +|------------------|---------------------------------------------------------------------------------------------------------------------------------------------| +| Unary | `AnalyzePlan`, `Config`, `ArtifactStatus`, `Interrupt`, `ReleaseExecute`, `ReleaseSession`, `FetchErrorDetails`, `CloneSession`, `GetStatus` | +| Server-streaming | `ExecutePlan`, `ReattachExecute` | +| Client-streaming | `AddArtifacts` | + +That table describes Spark Connect, not the gateway: the relay never enumerates +methods, and a Spark release that adds RPCs needs no change here. Nothing in the +message bodies is rewritten apart from the identity fields — there are no URLs or +hostnames inside these protobufs, so Knox's rewrite machinery has no role. + +### What failures look like ### + +Every rejection carries a gRPC status code and a description, both of which reach +the client — PySpark surfaces them in the exception it raises. The codes are +chosen so the cause is distinguishable: + +| Situation | Status | Description | +|---|---|---| +| No token, or a token that is not valid | `UNAUTHENTICATED` | `Invalid or missing bearer token` | +| No topology selected and no default configured | `UNIMPLEMENTED` | `no topology selected; set a knox-topology connection parameter or configure a default topology` | +| Selected topology declares no service for the role, or does not exist | `UNAVAILABLE` | `topology declares no SPARKCONNECT service` | +| Topology ACLs refuse the user | `PERMISSION_DENIED` | `Not permitted to use SPARKCONNECT` | +| The RPC is denied, or absent from an allow list | `PERMISSION_DENIED` | `This RPC is not permitted in this topology` | +| A proto service the listener does not front | `UNIMPLEMENTED` | from gRPC itself | +| A request that is not well-formed protobuf, where identity assertion is on | `INVALID_ARGUMENT` | `Request message is not a well-formed protobuf message` | +| An identity field that ends beyond the scan limit | `INVALID_ARGUMENT` | `Identity field extends past the first bytes of the request...` | +| A field whose wire type contradicts the identity rules | `INVALID_ARGUMENT` | `...the configured identity rules do not describe this message` | +| Backend unreachable or its TLS cannot be established | `UNAVAILABLE` | from the backend leg | + +Authentication failures are deliberately uniform: the description does not say +whether a token was expired, revoked, or badly signed, since that would tell +someone probing which of their guesses was closest. + +If the listener is enabled but **no** topology declares a service for the +configured role, the gateway still starts and binds the port — enabling the +listener and declaring a backend are separate steps in separate files. Clients +then see the `UNIMPLEMENTED` or `UNAVAILABLE` cases above depending on what they +sent. + +This is not treated as an error, because it is a reasonable steady state: a +deployment may enable the listener as a matter of course and add a topology only +when someone provisions a backend — possibly never. The gateway notes it at +`DEBUG` rather than warning: + + DEBUG gateway.grpc - The SPARKCONNECT listener is running but no deployed + topology declares a SPARKCONNECT service, so calls will be rejected until + one does. + +Enable debug logging for `org.apache.knox.gateway.grpc` if you are investigating +why calls are being refused. Adding a topology fixes it without a restart. + +Each call also produces one audit record, whether it succeeded or was rejected — +including calls refused before any backend was contacted. The record carries the +principal, method, status code, topology, backend URL, remote address, authority +and duration. + +### Kerberos environments ### + +Neither gRPC nor typical clients support SPNEGO, and gRPC has no +challenge-response step for it to hook into. Kerberos therefore authenticates +*token acquisition* rather than each RPC: a `kinit`'d user or a keytab'd service +fetches a token from a `knoxtoken` topology using HadoopAuth/SPNEGO, and the JWT +carries the data path. + +This is the same trade Kerberized Hadoop already makes — nobody SPNEGOs every +HDFS block read. It is also better operationally for long-running jobs: an +administrator can revoke one token without touching the principal. Validation +covers issuer, expiry, not-before, signature and, when server-managed token state +is on, revocation. + +Tokens are validated when an RPC starts, not continuously. A multi-hour +`ExecutePlan` is not killed when its token expires; the next RPC fails with +`UNAUTHENTICATED`. Cutting off long queries at expiry would punish precisely the +workloads these protocols exist to serve, and the backend's own session timeout +still bounds how long a session survives. + +### What works well, and what does not ### + +**What you get for any gRPC service, with no code:** TLS from the gateway +identity, bearer authentication, coarse authorization by topology ACL, topology +routing across multiple backends, per-topology method allow/deny lists, backend +channel pooling, per-RPC audit records, graceful drain, correct flow control and +cancellation in both directions, and verbatim relay of statuses and trailers. +Streaming RPCs of every shape work, including long-lived ones. + +**What needs the protocol to cooperate:** identity assertion works where the +identity sits at constant field numbers — at any depth, but the same numbers on +every request — and where the backend actually reads it. A protocol that carries +identity somewhere structurally variable is out of reach of a field-number path: +a `oneof` wrapper whose case varies, an `Any`-boxed payload, a map entry selected +by key. Such a deployment still gets authentication, authorization, method gating +and audit, but not assertion. Nor is a rule the place to express a conditional — +the rules describe where a value goes, not when. + +**What is deliberately not offered:** anything finer than a whole RPC. Screening +particular configuration keys, inspecting query plans, rewriting arbitrary +payload fields — none of it is available, because it would mean assuming a +message shape the gateway otherwise never assumes, and re-acquiring the version +coupling this design exists to avoid. Where that matters, the right home is a +component inside the backend, which has the real schema on its classpath for +free. + +An earlier iteration of this feature did screen Spark Connect's `Config` RPC for +reserved session keys, and gated `AddArtifacts` from inside the message. Both are +gone. Artifact gating became a method-name rule, which is strictly more general; +the reserved-key guard was dropped in favour of a server-side component that +recomputes the identity per request. A key a client can write is a key a client +can forge, whereas a value derived from the asserted identity on every request +cannot be overwritten by a session setting at all. + +### Security considerations ### + +**Knox's authorization here is coarse by design.** It answers "may this user use +this service in this topology" and "may they call this RPC". Database, table, +column and row-level policy must be enforced inside the backend — for example by +a Ranger-backed plan-level plugin keyed off the identity Knox asserts. + +**Asserting the identity field is not storage-level enforcement.** On a Spark +Connect server, `user_id` keys the session cache — so two users can never share a +session — and appears in logs and events. It is not propagated into Spark's +`CurrentUserContext`, so `current_user()` in SQL reports the Spark application's +own user unless a server-side component bridges it. A shared Spark Connect server +is one application running as one principal, and its storage credentials are that +principal's. + +**User-supplied code bypasses plan-level policy.** Uploaded jars and inline +Python/Scala UDFs run inside that JVM with that principal's credentials, so they +can read data directly. Denying the upload RPC by name shrinks the attack surface +but does not close it, because inline UDFs reach the same capability. This is a +property of plan-level enforcement generally, not something the gateway +introduces. Deployments needing a hard boundary want per-user or per-tenant +backend instances. + +**Restrict the backend.** Knox in front of an openly reachable backend port +secures nothing. Firewall the backend so only Knox can reach it, and where the +backend supports a pre-shared token — Spark 4's +`spark.connect.authenticate.token`, say — store it as a Knox alias and name it in +`gateway.grpc.backend.token.alias`. Knox then presents it on the backend leg, and +strips the client's own credential there, so a client cannot bypass the gateway +even with network reachability. Knox-internal routing metadata is stripped too: +it was addressed to the gateway, and the backend has no use for it. + +### Making the asserted identity usable inside the backend ### + +The deployment this was built for is a single always-on backend behind the +firewall, running as a privileged principal, with fine-grained authorization +enforced *inside* it — typically a plan-level plugin evaluating Ranger policies +against the identity Knox asserts. Getting that identity from the gateway into +the engine takes one more step, and it is worth being explicit about it because +the gap is easy to miss. + +**The carrier of record is the identity field.** Knox rewrites it on every +message, it keys the server-side session cache — so two users cannot share a +session by construction — and it lands in Knox's audit records. Anything else +should be *derived* from it, never asserted independently by the client. + +**But OSS Spark does not surface it to SQL.** `user_id` is used for the session +key and for logging; it is not propagated into `CurrentUserContext`, so +`current_user()` returns the Spark application's own user. A server-side +component has to bridge it. + +The robust bridge is a gRPC `ServerInterceptor` deployed with the Spark +application and registered through `spark.connect.grpc.interceptor.classes`. That +is a static configuration, so clients cannot alter it. The interceptor reads +`user_context.user_id` on each request and publishes it — by setting +`CurrentUserContext.CURRENT_USER`, which makes `current_user()` itself correct, +and/or by writing a reserved session configuration key. Whatever consumes the +identity downstream (a Ranger plugin, say) and the bridge should agree on one +mechanism rather than each inventing its own. + +One thing to verify when building such a bridge: `CurrentUserContext` is an +`InheritableThreadLocal`, and Spark Connect runs plans on dedicated execution +threads. Confirm that a value set in the interceptor is actually visible at +analysis and optimization time; if it is not, set it from a session hook on the +execution path instead. + +A weaker alternative is to have the session carry the identity in a reserved +configuration key. Knox does **not** do this for you, and does not police such a +key either: the gateway reads only the identity fields, by field number, and has +no knowledge of any RPC's internal structure. Deployments that use a reserved key +anyway should have the server-side component own it rather than trusting anything +the client sends. + +**And code execution bypasses all of it.** See the security notes above: a +plan-level plugin lives in the same JVM as user code, which runs with the +application's credentials. Plan-level enforcement is a real control among +cooperating users, and an honest audit trail; it is not a boundary against a +determined one. + +### Limitations ### + +- **One backend URL per topology.** Where sessions are server-side state and a + reattach RPC must reach the backend owning the operation, round-robin over + several backends would be wrong; session-affine routing across multiple + backends is not yet implemented. +- **A listener's set is fixed at startup.** Adding or removing one, or changing + its port or TLS identity, needs a gateway restart; the change is reported + rather than silently ignored. +- **One service role and identity layout per listener.** Two protocols whose + identities sit at different field numbers need a listener each, and therefore a + port each. +- **Identity paths are constant, and identity fields are strings.** A rule names + fixed field numbers at a fixed depth; it cannot select a map entry by key, + follow a `oneof` whose case varies per request, or write a non-string field. +- **Rewritten fields must lie within the scan limit** — 128 KiB into the request + by default. A protocol that emits its identity after a large payload needs the + limit raised; requests that break it are refused, not partially asserted. +- **The asserted principal is the token's subject.** Identity-assertion provider + mapping rules are not applied on this path. +- **Bearer tokens only.** Neither gRPC nor typical clients can carry Kerberos on + the RPC path, and connection strings expose no client-certificate surface, so + mutual TLS from the client is not available without a non-vanilla + `channelBuilder`. +- **No gRPC-Web.** No translation layer is provided, so browsers cannot talk to + this listener directly. +- **No per-RPC metrics yet.** Audit records cover each call; the standard gateway + metrics do not yet include gRPC counters, latencies or active-stream gauges. +- **A gateway restart severs active streams.** Shutdown drains for + `gateway.grpc.drain.timeout` first, and clients of streaming protocols + generally recover through their own reattach or retry logic. + +### Possible future work ### + +Recorded so the reasoning is not lost; none of this is implemented or promised. + +- **Session affinity across multiple backends** — consistent hashing on a session + identifier with an in-memory affinity map. Failover semantics would stay + honest: if a backend dies its sessions die, and Knox routes the client's *new* + session to a live backend rather than pretending the old one survived. The same + mechanism with a different stickiness key (principal or group) is also the + route to per-user or per-tenant backend instances, which is what a deployment + needing genuine storage-level isolation actually wants. +- **More topology discriminators.** Two beyond metadata were designed for but not + built. A **token claim** binding a topology at issuance would make routing an + authorization property — a user could not reach a topology their token was not + minted for. **Virtual-host mapping** on the HTTP/2 `:authority` would be + invisible in the connection string and immune to clients stripping unknown + parameters, but needs DNS discipline and one certificate covering every mapped + hostname; where the platform PKI cannot issue multi-name (SAN or wildcard) + certificates, several listeners — each on its own port with its own single-name + certificate — already cover that ground, at the cost of a port per hostname + rather than one shared port. +- **Multiple listeners in one gateway**, each with its own port, role, proto + service list and identity path, for a deployment fronting more than one gRPC + protocol. +- **Identity-assertion provider mapping** on this path, so the asserted principal + can be transformed the way the servlet pipeline transforms it. +- **Passing groups to the backend as metadata.** Group membership does not fit in + a single identity field, and a server-side plugin that wants it has to look it + up itself. + +### References ### + +- Spark Connect connection string specification — + `apache/spark: sql/connect/docs/client-connection-string.md` +- Spark Connect protocol definitions — + `apache/spark: sql/connect/common/src/main/protobuf/spark/connect/` + (vendored at test scope into `gateway-service-grpc`; see the README there for + the exact revision and the refresh procedure) +- PySpark `ChannelBuilder`, for how connection-string parameters become metadata — + `apache/spark: python/pyspark/sql/connect/client/core.py` +- [SPARK-51156](https://issues.apache.org/jira/browse/SPARK-51156) — the + pre-shared backend token (`spark.connect.authenticate.token`) +- [KNOX-3402](https://issues.apache.org/jira/browse/KNOX-3402) — this feature diff --git a/knox-site/docs/spark-connect-support.md b/knox-site/docs/spark-connect-support.md deleted file mode 100644 index 4939e4837a..0000000000 --- a/knox-site/docs/spark-connect-support.md +++ /dev/null @@ -1,536 +0,0 @@ - - -## Spark Connect Support ## - -### Introduction ### - -Spark Connect is the decoupled client/server protocol for Apache Spark (3.4+, and -the default architecture in Spark 4). Clients — PySpark, the Scala client, Go, -Rust, or JDBC via the Spark Connect driver — talk to a Spark Connect server over -gRPC, by default on port 15002. - -The OSS Spark Connect server has essentially no built-in authentication or -authorization; the project assumes a fronting proxy provides them. Knox can now -fill that role, adding: - -- **Authentication at the edge** — Knox-issued JWTs (KnoxToken bearer tokens), - validated before anything reaches Spark. -- **Identity assertion** — Spark Connect otherwise trusts a *client-asserted* - `user_context.user_id`. Knox overwrites it with the authenticated principal. -- **Coarse authorization** — the usual topology ACLs decide who may use Spark - Connect at all. -- **Auditing** — one record per RPC: principal, topology, method, session, - outcome and duration. - -#### Why a separate port #### - -Spark Connect does not go through Knox's servlet pipeline, and could not: gRPC -needs HTTP/2 negotiated over ALPN, which Knox's Jetty connectors do not offer; -the Servlet 3.1 API Knox targets has no way to read or write HTTP trailers, where -`grpc-status` and Spark's structured error details live; and the outbound -dispatch layer is built on a strict request/response HTTP/1.1 client, whereas -`ExecutePlan` and `ReattachExecute` are long-lived server streams and -`AddArtifacts` is a client stream. - -Routing rules it out independently. A `sc://` connection string may not contain a -path — the Spark client forbids it, to stay compatible with the gRPC standard — -and gRPC fixes request paths at `/pkg.Service/Method`. Knox's usual -`/gateway/{topology}/{service}` routing therefore has nothing to match on. - -So Spark Connect is served by a **dedicated listener on its own port**, started -and stopped with the gateway, alongside Jetty rather than inside it: - - Knox JVM - ┌─────────────────────────────────────────────┐ - sc:// ─▶│ :15002 gRPC listener (Netty) │ - grpc/h2 │ ├─ TLS (gateway identity) │ - │ ├─ audit │ - │ ├─ authentication (bearer JWT) │ grpc/h2 - │ ├─ routing (knox-topology → registry) │──▶ Spark Connect - │ ├─ authorization (topology ACLs) │ server :15002 - │ └─ relay (asserts user_context.user_id) │ - ├─────────────────────────────────────────────┤ - │ :8443 Jetty (the existing servlet gateway) │ - └─────────────────────────────────────────────┘ - -This mirrors how Knox already handles WebSockets, which likewise bypass the -topology filter chains and do their own authentication — the difference being -that a WebSocket upgrade can share Jetty's HTTP/1.1 connector, and gRPC cannot. - -### What is proxied ### - -The whole `spark.connect.SparkConnectService` surface. Every RPC is relayed with -its status and trailers passed through verbatim, and every request has its -`user_context.user_id` replaced with the authenticated principal: - -| Shape | RPCs | -|------------------|----------------------------------------------------------------------------------------------------------------------------------------------| -| Unary | `AnalyzePlan`, `Config`, `ArtifactStatus`, `Interrupt`, `ReleaseExecute`, `ReleaseSession`, `FetchErrorDetails`, `CloneSession`, `GetStatus` | -| Server-streaming | `ExecutePlan`, `ReattachExecute` | -| Client-streaming | `AddArtifacts` | - -Flow control is honored in both directions, so a slow client cannot make the -gateway buffer an unbounded number of Arrow batches, and cancellation propagates -both ways rather than leaving orphaned executions on the backend. - -Nothing in the message bodies is rewritten apart from the identity fields. There -are no URLs or hostnames inside these protobufs, so Knox's rewrite machinery has -no role here. - -### Configuration ### - -Spark Connect support is disabled by default. Enable it in -`/conf/gateway-site.xml`: - - - gateway.sparkconnect.enabled - true - Enable the Spark Connect (gRPC) listener. - - - gateway.sparkconnect.default.topology - analytics - Topology used when a client does not select one. - - -The listener presents the gateway's own TLS identity — the same keystore and -alias Jetty uses — whenever `ssl.enabled` is true, so there is no second -certificate to manage. - -#### All properties #### - -| Property | Default | Meaning | -|------------------------------------------------------------|-------------|-----------------------------------------------------------------------------------| -| `gateway.sparkconnect.enabled` | `false` | Master switch; the listener is not started when false. | -| `gateway.sparkconnect.port` | `15002` | Port for the gRPC listener. | -| `gateway.sparkconnect.default.topology` | *(none)* | Topology used when the client sends no `knox-topology`. | -| `gateway.sparkconnect.max.message.size` | `134217728` | Maximum inbound message size in bytes, both legs. Matches Spark's 128 MB default. | -| `gateway.sparkconnect.max.concurrent.calls.per.connection` | `1000` | Maximum concurrent gRPC streams per client connection. | -| `gateway.sparkconnect.permit.keepalive.time` | `10000` | Minimum tolerated interval between client keepalive pings, in ms. | -| `gateway.sparkconnect.permit.keepalive.without.calls` | `true` | Whether clients may ping an idle channel. Spark Connect clients do. | -| `gateway.sparkconnect.channel.idle.timeout` | `1800000` | Idle time before an unused backend channel is shut down, in ms. | -| `gateway.sparkconnect.drain.timeout` | `30000` | How long in-flight RPCs get to finish at shutdown, in ms. | -| `gateway.sparkconnect.backend.token.alias` | *(none)* | Alias holding the backend's pre-shared token (see below). | -| `gateway.sparkconnect.add.artifacts.mode` | `ALLOW` | `ALLOW`, `DENY`, or `ALLOW_LISTED_USERS` for the `AddArtifacts` RPC. | -| `gateway.sparkconnect.add.artifacts.allowed.users` | *(none)* | Comma-separated users permitted when the mode is `ALLOW_LISTED_USERS`. | -| `gateway.sparkconnect.reserved.config.prefix` | `knox.` | Session-configuration key prefix clients may not `Set` or `Unset`. | - -#### Keeping these out of `gateway-site.xml` #### - -Knox has no `conf.d` directory, but it does load one optional extra file. The -gateway reads exactly three configuration files from `{GATEWAY_HOME}/conf`, in -this order, with later files overriding earlier ones: - - gateway-default.xml - gateway-site.xml - gateway-reloadable.xml - -`gateway-reloadable.xml` is not shipped and does not have to exist, so the -`gateway.sparkconnect.*` properties can live there instead of being merged into -`gateway-site.xml`. It is a single shared file rather than a per-feature -directory, so anything else using it has to co-exist in the same file — but it -does keep this feature's settings out of the main one. See -[Reloadable Gateway Configuration](config.md) for the general mechanism. - -Knox re-reads that file every `gateway.config.refresh.interval` milliseconds -(default 10 seconds). Some of the properties above then take effect immediately; -the rest cannot, because they are built into the bound server. - -**Applied on the next RPC, no restart:** - -- `gateway.sparkconnect.default.topology` -- `gateway.sparkconnect.add.artifacts.mode` -- `gateway.sparkconnect.add.artifacts.allowed.users` -- `gateway.sparkconnect.reserved.config.prefix` - -These are the message-level and routing controls — the ones an operator is most -likely to want to change in response to something happening. Tightening artifact -gating, or reserving a different configuration prefix, applies to the very next -call without interrupting any session. - -**Restart required:** everything else — `gateway.sparkconnect.enabled` itself, -the port, TLS, message and stream limits, keepalive settings, channel idle and -drain timeouts, and the backend token alias. Whether the listener runs at all is -decided once at startup, and the rest are fixed when the socket is bound. - -Changing a restart-only property in a running gateway does not silently do -nothing. The refreshed configuration is compared against what the gateway started -with, and a warning names what could not be applied — for the transport settings, -which properties changed; for `gateway.sparkconnect.enabled`, that the listener -cannot be started or stopped without a restart. Switching it on when it was off -at startup is reported too, which is the case most likely to be mistaken for a -malfunction: without the warning, the only symptom is a port that never opens. - -### Topology configuration ### - -Declare the backend like any other service. The registry treats the URL as an -opaque string, so `grpc://` and `grpcs://` need no special handling: - - - SPARKCONNECT - grpc://spark-connect-host:15002 - - -Use `grpcs://` for a TLS backend; Knox verifies it against the HTTP client -truststore, falling back to the gateway keystore. - -Authorization uses the ordinary `AclsAuthz` provider syntax, keyed on the -`SPARKCONNECT` role: - - - authorization - AclsAuthz - true - - SPARKCONNECT.acl - *;analysts;* - - - -Group membership comes from the `knox.groups` claim in the token, so configure -`knoxtoken` to include groups if you intend to write group ACLs. - -### Multiple Spark Connect clusters ### - -One topology per cluster, all served by the single listener port. A topology -declares exactly one `SPARKCONNECT` backend, so a second cluster means a second -topology: - - conf/analytics.xml -> grpc://spark-analytics:15002 - conf/etl.xml -> grpc://spark-etl:15002 - -Clients pick one with the `knox-topology` connection parameter: - - sc://knox-host:15002/;use_ssl=true;token=;knox-topology=analytics - sc://knox-host:15002/;use_ssl=true;token=;knox-topology=etl - -Both connect to the *same* Knox port and are routed to different Spark clusters. -Topology selection is per-RPC, from call metadata, so concurrent sessions from -different users — or from one user — are multiplexed over that one port onto -distinct backends. Knox keeps one pooled gRPC channel per backend URL and shares -it across all calls routed there. - -This is safe for Spark Connect's session model because the discriminator is -sticky by construction: the client sends the same `knox-topology` on every -request of the connection, so every call in a session lands on the backend that -owns it — which `ReattachExecute` requires. Nothing round-robins. - -Each topology carries its own authentication provider, ACLs and audit scope, so -"separate cluster" and "separate policy boundary" stay aligned. - -#### Authorizing which users may select which cluster #### - -Topology selection is a client-supplied value, so it is authorized rather than -trusted. Authorization runs *after* routing and is evaluated against the topology -that was selected — naming a topology in a connection string is not the same as -being allowed to use it. Give each topology its own `SPARKCONNECT.acl`: - - - - SPARKCONNECT.acl - *;analysts;* - - - - - SPARKCONNECT.acl - *;engineers;* - - -An analyst connecting with `knox-topology=etl` is refused with -`PERMISSION_DENIED` before any backend connection is made. The full ACL syntax -applies per topology — named users, named groups, IP ranges, `AND`/`OR` -processing mode, and the `KNOX_ADMIN_USERS` / `KNOX_ADMIN_GROUPS` placeholders — -so selection can be gated by user name, by group, by source address, or by a -combination. - -Two things to be deliberate about: - -- **The default is permissive.** A topology that declares no `SPARKCONNECT.acl`, - or whose `AclsAuthz` provider is disabled, is reachable by *any* authenticated - user. This matches the servlet provider's behaviour, but it means restricting - selection is something you switch on, not something you get for free. If a - cluster should be reachable by a subset of your users, it needs an ACL. -- **Group ACLs need group claims.** Groups come from the token's `knox.groups` - claim, so a `knoxtoken` deployment that does not embed groups will match no - group ACL. Where groups are unavailable, gate on user names instead. - -A user probing topologies they cannot use can tell an existing Spark Connect -topology (`PERMISSION_DENIED`) from one that does not exist or serves no -`SPARKCONNECT` service (`UNAVAILABLE`). They must already hold a valid token to -learn even that, but do not treat topology names as secrets. - -What is *not* supported is several backends **within** one topology. Spark Connect -sessions are server-side state keyed by `(user_id, session_id)`, so spreading one -topology across backends needs session-affine routing rather than any form of -load balancing; that is not implemented (see Limitations). - -#### Adding a cluster without a restart #### - -Topologies are hot-reloaded. Knox watches the topologies directory and picks up -changes within about five seconds, so dropping in a new topology file, or editing -an existing one, takes effect on a running gateway: - -- **A new topology, or a changed backend URL** — takes effect on the next RPC. - The backend is resolved from the service registry per call, and redeployment - rewrites the registry entry. -- **A changed `SPARKCONNECT.acl`** — takes effect on the next RPC after the - redeployment. The listener caches parsed ACLs per topology and drops that cache - when topologies are redeployed. -- **A deleted topology** — subsequent calls selecting it fail `UNAVAILABLE`. - Calls already in flight are not interrupted. - -Only `gateway-site.xml` properties — the port, message limits, `AddArtifacts` -mode and so on — need a gateway restart, since the listener binds its socket and -captures those settings at startup. - -### Connecting ### - -Clients need no plugins or code changes. First acquire a token — over HTTPS, -authenticating however that topology is configured: - - curl --negotiate -u : https://knox:8443/gateway/tokens/knoxtoken/api/v1/token - -Then put it in the connection string: - - sc://knox-host:15002/;use_ssl=true;token=;knox-topology=analytics - -Two details make this work. The `token=` parameter is sent as a standard -`Authorization: Bearer` header and forces TLS on. Any parameter the client does -not recognize — `knox-topology` here — is sent as gRPC metadata on every request, -which is how a topology gets selected despite gRPC forbidding a path component in -the connection URL. If you set `gateway.sparkconnect.default.topology`, the -`knox-topology` parameter can be omitted. - -### Kerberos environments ### - -Neither gRPC nor the vanilla Spark Connect clients support SPNEGO, and gRPC has no -challenge-response step for it to hook into. Kerberos therefore authenticates -*token acquisition* rather than each RPC: a `kinit`'d user or a keytab'd service -fetches a token from a `knoxtoken` topology using HadoopAuth/SPNEGO, and the JWT -carries the data path. - -This is the same trade Kerberized Hadoop already makes — nobody SPNEGOs every -HDFS block read. It is also better operationally for long-running jobs: an -administrator can revoke one token without touching the principal. - -Tokens are validated when an RPC starts, not continuously. A long-running -`ExecutePlan` is not killed when its token expires; the next RPC fails with -`UNAUTHENTICATED`. - -### Security considerations ### - -**Knox's authorization here is coarse by design.** It answers only "may this user -use Spark Connect in this topology". Database, table, column and row-level policy -must be enforced inside the Spark Connect server — for example by a Ranger-backed -plan-level plugin keyed off the identity Knox asserts. - -**Asserting `user_id` is not storage-level enforcement.** On the server, -`user_id` keys the session cache — so two users can never share a session — and -appears in logs and events. It is not propagated into Spark's -`CurrentUserContext`, so `current_user()` in SQL reports the Spark application's -own user unless a server-side component bridges it. A shared Spark Connect server -is one application running as one principal, and its storage credentials are that -principal's. - -**User-supplied code bypasses plan-level policy.** Uploaded jars and inline -Python/Scala UDFs run inside that JVM with that principal's credentials, so they -can read data directly. `gateway.sparkconnect.add.artifacts.mode` shrinks the -attack surface but does not close it, because inline UDFs reach the same -capability. This is a property of plan-level enforcement generally, not something -the gateway introduces. Deployments needing a hard boundary want per-user or -per-tenant backend instances. - -**Restrict the backend.** Knox in front of an openly reachable Spark Connect port -secures nothing. Firewall the backend so only Knox can reach it, and set Spark 4's -pre-shared token (`spark.connect.authenticate.token`), storing it as a Knox alias -and naming it in `gateway.sparkconnect.backend.token.alias`. Knox then presents it -on the backend leg — and, because it strips the client's own credential there, a -client cannot bypass the gateway even with network reachability. - -### Making the asserted identity usable inside Spark ### - -The deployment this was built for is a single always-on Spark Connect server -behind the firewall, running as a privileged principal, with fine-grained -authorization enforced *inside* the server — typically a plan-level plugin -evaluating Ranger policies against the identity Knox asserts. Getting that -identity from the gateway into the engine takes one more step, and it is worth -being explicit about it because the gap is easy to miss. - -**The carrier of record is `user_context.user_id`.** Knox rewrites it on every -message, it keys the server-side session cache — so two users cannot share a -session by construction — and it lands in Knox's audit records. Anything else -should be *derived* from it, never asserted independently by the client. - -**But OSS Spark does not surface it to SQL.** `user_id` is used for the session -key and for logging; it is not propagated into `CurrentUserContext`, so -`current_user()` returns the Spark application's own user. A server-side -component has to bridge it. - -The robust bridge is a gRPC `ServerInterceptor` deployed with the Spark -application and registered through `spark.connect.grpc.interceptor.classes`. -That is a static configuration, so clients cannot alter it. The interceptor reads -`user_context.user_id` on each request and publishes it — by setting -`CurrentUserContext.CURRENT_USER`, which makes `current_user()` itself correct, -and/or by writing a reserved session configuration key. Whatever consumes the -identity downstream (a Ranger plugin, say) and the bridge should agree on one -mechanism rather than each inventing its own. - -One thing to verify when building such a bridge: `CurrentUserContext` is an -`InheritableThreadLocal`, and Spark Connect runs plans on dedicated execution -threads. Confirm that a value set in the interceptor is actually visible at -analysis and optimization time; if it is not, set it from a session hook on the -execution path instead. - -A weaker alternative needing no server-side code is to have the client's session -prime a reserved configuration key. Knox does **not** do this for you — it does -not inject `Config` calls — and the approach is less trustworthy than the -interceptor for the reason below. - -**Reserved keys are protected, but only on the structured path.** Knox denies -client `Set` and `Unset` on any session configuration key beginning with -`gateway.sparkconnect.reserved.config.prefix` (default `knox.`). Those are named -fields in the `Config` RPC, so the check is exact and cheap. What Knox does *not* -screen is `SET knox.whatever=...` issued as SQL inside `ExecutePlan`, which would -require inspecting plan text and would be best-effort at best. This is the main -argument for the interceptor bridge: a value recomputed from `user_context` on -every request cannot be overwritten by a session `SET` at all, whereas a -configuration key can. - -**And code execution bypasses all of it.** See the security notes above: a -plan-level plugin lives in the same JVM as user code, which runs with the -application's credentials. Plan-level enforcement is a real control among -cooperating users, and an honest audit trail; it is not a boundary against a -determined one. - -### Is this a generic gRPC gateway? ### - -No — and deliberately so. Knox proxies exactly one gRPC service, -`spark.connect.SparkConnectService`. There is no configuration property that -points this listener at an arbitrary gRPC backend, and a call to any other proto -service is answered `UNIMPLEMENTED`. - -It is worth saying why that is not the slippery slope it might look like. Knox's -servlet pipeline is already a generic reverse proxy: an arbitrary REST API is -proxied with a service definition and one rewrite rule, no code, and WebSocket -proxying matches any service definition by context path. gRPC was the one -protocol class outside that coverage, because it is the one the servlet stack -cannot physically carry. This closes that gap; it does not begin a pattern of -per-protocol special cases. - -It is also worth being open about what sits behind the abstraction, because -anyone reading the source will notice it: most of this feature is not -Spark-specific. The listener, -TLS from the gateway identity, bearer authentication, the coarse ACL check, -topology routing, backend channel caching, auditing, graceful drain and the relay -itself are all protocol-agnostic — the relay in particular treats messages as -opaque and collapses all four RPC shapes into one code path. Only identity -assertion and the per-RPC gating switches need to understand Spark Connect's -messages. - -The implementation keeps that split explicit: the gateway listener is an abstract -class whose protocol-aware parts are abstract methods, and the Spark Connect -listener is its only concrete subclass. That is a bet that someone will -eventually want to front a second gRPC service, and it costs little to leave the -seam in place rather than discover it later. It is **not** a commitment, and it -is not a supported extension point — the abstraction exists for the benefit of a -future change to Knox itself, not as an API to build against. - -Promoting it to a real capability would take more than flipping a switch, which -is the main reason it has not been: - -- **A service-to-role mapping.** gRPC paths are `/pkg.Service/Method`, so a - topology would need to declare which proto services map to which backend roles, - default-denying anything unmapped. -- **An honest security posture.** Byte-level proxying gives authentication, - coarse authorization, TLS and method-level audit — but no message-body - controls at all. It cannot assert identity, cannot protect a reserved config - key, and cannot record a session id. That is a materially weaker offering than - what Spark Connect gets here, and it competes much less favourably with simply - putting Envoy or nginx in front of the service. -- **A community discussion**, its own configuration surface, and its own - documentation. - -One narrow piece of byte-level relay *is* active, and it is scoped accordingly: -an RPC that belongs to `spark.connect.SparkConnectService` but has no generated -handler in this build — a method added in a newer Spark line — is forwarded as -opaque bytes rather than rejected. Such a call is still authenticated, -authorized, routed and audited; it simply does not get identity assertion, -because that requires parsing the message. This exists so version skew degrades -gracefully, not as a general passthrough. - -### Limitations ### - -- One `SPARKCONNECT` URL per topology. Spark Connect sessions are server-side - state keyed by `(user_id, session_id)` and `ReattachExecute` must reach the - backend owning the operation, so round-robin over several backends would be - wrong; session-affine routing across multiple backends is not yet implemented. -- The asserted principal is the token's subject. Identity-assertion provider - mapping rules are not applied on this path. -- A gateway restart severs active streams. Clients recover through their own - `ReattachExecute` retry logic, and shutdown drains for - `gateway.sparkconnect.drain.timeout` first. -- **Bearer tokens only.** Neither gRPC nor the vanilla clients can carry Kerberos - on the RPC path, and the connection string exposes no client-certificate - surface, so mutual TLS from the client is not available without a non-vanilla - `channelBuilder`. -- **No gRPC-Web.** Spark Connect clients speak native gRPC; no translation layer - is provided, so browsers cannot talk to this listener directly. -- **No per-RPC metrics yet.** Audit records cover each call; the standard gateway - metrics do not yet include gRPC counters, latencies or active-stream gauges. -- **`Config` key restrictions beyond the reserved prefix, and per-RPC allow/deny - lists**, are not implemented. `AddArtifacts` gating and reserved-prefix - protection are the only message-level controls. - -### Possible future work ### - -Recorded so the reasoning is not lost; none of this is implemented or promised. - -- **Session affinity across multiple backends** — consistent hashing on - `session_id` with an in-memory affinity map. Failover semantics would stay - honest: if a backend dies its sessions die, and Knox routes the client's *new* - session to a live backend rather than pretending the old one survived. The same - mechanism with a different stickiness key (principal or group) is also the - route to per-user or per-tenant backend instances, which is what a deployment - needing genuine storage-level isolation actually wants. -- **More topology discriminators.** Two beyond `knox-topology` metadata were - designed for but not built. A **token claim** binding a topology at issuance - would make routing an authorization property — a user could not reach a - topology their token was not minted for. **Virtual-host mapping** on the HTTP/2 - `:authority` would be invisible in the connection string and immune to clients - stripping unknown parameters, but needs DNS discipline and one certificate - covering every mapped hostname; where the platform PKI cannot issue - multi-name (SAN or wildcard) certificates, a listener per topology is the - practical alternative, since every listener then presents the same hostname and - a plain single-name certificate covers them all. -- **Identity-assertion provider mapping** on this path, so the asserted principal - can be transformed the way the servlet pipeline transforms it. - -### References ### - -- Spark Connect connection string specification — - `apache/spark: sql/connect/docs/client-connection-string.md` -- Spark Connect protocol definitions — - `apache/spark: sql/connect/common/src/main/protobuf/spark/connect/` - (vendored into `gateway-service-sparkconnect`; see the README there for the - exact revision and the refresh procedure) -- PySpark `ChannelBuilder`, for how connection-string parameters become metadata — - `apache/spark: python/pyspark/sql/connect/client/core.py` -- [SPARK-51156](https://issues.apache.org/jira/browse/SPARK-51156) — the - pre-shared backend token (`spark.connect.authenticate.token`) -- [KNOX-3402](https://issues.apache.org/jira/browse/KNOX-3402) — this feature diff --git a/knox-site/mkdocs.yml b/knox-site/mkdocs.yml index a8bf1d78e0..4292f78dc1 100644 --- a/knox-site/mkdocs.yml +++ b/knox-site/mkdocs.yml @@ -124,7 +124,7 @@ nav: - Admin API: admin_api.md - Monitoring API: dev-guide/knox_monitoring_api.md - Advanced Topics: - - Spark Connect Support: spark-connect-support.md + - gRPC Support: grpc-support.md - SSE Support: sse-support.md - WebSocket Support: websocket-support.md - X-Forwarded Headers: x-forwarded-headers.md diff --git a/pom.xml b/pom.xml index 2730bfc40d..69777f2a50 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,7 @@ gateway-service-storm gateway-service-remoteconfig gateway-service-restcatalog - gateway-service-sparkconnect + gateway-service-grpc gateway-service-definitions gateway-shell gateway-shell-launcher @@ -1297,7 +1297,7 @@ org.apache.knox - gateway-service-sparkconnect + gateway-service-grpc ${project.version} @@ -1808,7 +1808,7 @@ io.grpc