diff --git a/charts/kyuubi/values.yaml b/charts/kyuubi/values.yaml index e48cb01abfe..ba2c1937cbb 100644 --- a/charts/kyuubi/values.yaml +++ b/charts/kyuubi/values.yaml @@ -77,7 +77,10 @@ rbac: rules: - apiGroups: [""] resources: ["pods"] - verbs: ["create", "list", "delete"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: [""] + resources: ["services"] + verbs: ["list", "watch"] service: # configuration of the headless service diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md index 2562f0ef94c..e23143d15b6 100644 --- a/docs/configuration/settings.md +++ b/docs/configuration/settings.md @@ -363,29 +363,29 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co ### Kubernetes -| Key | Default | Meaning | Type | Since | -|----------------------------------------------------------------------|----------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------| -| kyuubi.kubernetes.application.state.container | spark-kubernetes-driver | The container name to retrieve the application state from. | string | 1.8.1 | -| kyuubi.kubernetes.application.state.source | POD | The source to retrieve the application state from. The valid values are pod and container. When the pod is in a terminated state, the container state will be ignored, and the application state will be determined based on the pod state. If the source is container and there is container inside the pod with the name of kyuubi.kubernetes.application.state.container, the application state will be from the matched container state. Otherwise, the application state will be from the pod state. | string | 1.8.1 | -| kyuubi.kubernetes.authenticate.caCertFile | <undefined> | Path to the CA cert file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | -| kyuubi.kubernetes.authenticate.clientCertFile | <undefined> | Path to the client cert file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | -| kyuubi.kubernetes.authenticate.clientKeyFile | <undefined> | Path to the client key file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | -| kyuubi.kubernetes.authenticate.oauthToken | <undefined> | The OAuth token to use when authenticating against the Kubernetes API server. Note that unlike, the other authentication options, this must be the exact string value of the token to use for the authentication. | string | 1.7.0 | -| kyuubi.kubernetes.authenticate.oauthTokenFile | <undefined> | Path to the file containing the OAuth token to use when authenticating against the Kubernetes API server. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | -| kyuubi.kubernetes.client.initialize.list || The kubernetes client initialize list to register kubernetes resource informers during Kyuubi server startup. This ensure the Kyuubi server is promptly informed for any Kubernetes resource changes after startup. It is highly recommend to set it for multiple Kyuubi instances mode. The format is `context1:namespace1,context2:namespace2`. | seq | 1.11.0 | -| kyuubi.kubernetes.context | <undefined> | The desired context from your kubernetes config file used to configure the K8s client for interacting with the cluster. | string | 1.6.0 | -| kyuubi.kubernetes.context.allow.list || The allowed kubernetes context list, if it is empty, there is no kubernetes context limitation. | set | 1.8.0 | -| kyuubi.kubernetes.master.address | <undefined> | The internal Kubernetes master (API server) address to be used for kyuubi. | string | 1.7.0 | -| kyuubi.kubernetes.namespace | default | The namespace that will be used for running the kyuubi pods and find engines. | string | 1.7.0 | -| kyuubi.kubernetes.namespace.allow.list || The allowed kubernetes namespace list, if it is empty, there is no kubernetes namespace limitation. | set | 1.8.0 | -| kyuubi.kubernetes.spark.appUrlPattern | http://{{SPARK_DRIVER_SVC}}.{{KUBERNETES_NAMESPACE}}.svc:{{SPARK_UI_PORT}} | The pattern to generate the spark on kubernetes application UI URL. The pattern should contain placeholders for the application variables. Available placeholders are `{{SPARK_APP_ID}}`, `{{SPARK_DRIVER_SVC}}`, `{{SPARK_DRIVER_POD_IP}}`, `{{KUBERNETES_NAMESPACE}}`, `{{KUBERNETES_CONTEXT}}` and `{{SPARK_UI_PORT}}`. | string | 1.10.0 | -| kyuubi.kubernetes.spark.autoCreateFileUploadPath.enabled | false | If enabled, Kyuubi server will try to create the `spark.kubernetes.file.upload.path` with permission 777 before submitting the Spark application. | boolean | 1.11.0 | -| kyuubi.kubernetes.spark.cleanupTerminatedDriverPod.checkInterval | PT1M | Kyuubi server use guava cache as the cleanup trigger with time-based eviction, but the eviction would not happened until any get/put operation happened. This option schedule a daemon thread evict cache periodically. | duration | 1.8.1 | -| kyuubi.kubernetes.spark.cleanupTerminatedDriverPod.kind | NONE | Kyuubi server will delete the spark driver pod after the application terminates for kyuubi.kubernetes.terminatedApplicationRetainPeriod. Available options are NONE, ALL, COMPLETED and default value is None which means none of the pod will be deleted | string | 1.8.1 | -| kyuubi.kubernetes.spark.forciblyRewriteDriverPodName.enabled | false | Whether to forcibly rewrite Spark driver pod name with 'kyuubi--driver'. If disabled, Kyuubi will try to preserve the application name while satisfying K8s' pod name policy, but some vendors may have stricter pod name policies, thus the generated name may become illegal. | boolean | 1.8.1 | -| kyuubi.kubernetes.spark.forciblyRewriteExecutorPodNamePrefix.enabled | false | Whether to forcibly rewrite Spark executor pod name prefix with 'kyuubi-'. If disabled, Kyuubi will try to preserve the application name while satisfying K8s' pod name policy, but some vendors may have stricter Pod name policies, thus the generated name may become illegal. | boolean | 1.8.1 | -| kyuubi.kubernetes.terminatedApplicationRetainPeriod | PT5M | The period for which the Kyuubi server retains application information after the application terminates. | duration | 1.7.1 | -| kyuubi.kubernetes.trust.certificates | false | If set to true then client can submit to kubernetes cluster only with token | boolean | 1.7.0 | +| Key | Default | Meaning | Type | Since | +|----------------------------------------------------------------------|----------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|--------| +| kyuubi.kubernetes.application.state.container | spark-kubernetes-driver | The container name to retrieve the application state from. | string | 1.8.1 | +| kyuubi.kubernetes.application.state.source | POD | The source to retrieve the application state from. The valid values are pod and container. When the pod is in a terminated state, the container state will be ignored, and the application state will be determined based on the pod state. If the source is container and there is container inside the pod with the name of kyuubi.kubernetes.application.state.container, the application state will be from the matched container state. Otherwise, the application state will be from the pod state. | string | 1.8.1 | +| kyuubi.kubernetes.authenticate.caCertFile | <undefined> | Path to the CA cert file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | +| kyuubi.kubernetes.authenticate.clientCertFile | <undefined> | Path to the client cert file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | +| kyuubi.kubernetes.authenticate.clientKeyFile | <undefined> | Path to the client key file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | +| kyuubi.kubernetes.authenticate.oauthToken | <undefined> | The OAuth token to use when authenticating against the Kubernetes API server. Note that unlike, the other authentication options, this must be the exact string value of the token to use for the authentication. | string | 1.7.0 | +| kyuubi.kubernetes.authenticate.oauthTokenFile | <undefined> | Path to the file containing the OAuth token to use when authenticating against the Kubernetes API server. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | +| kyuubi.kubernetes.client.initialize.list || The kubernetes client initialize list to register kubernetes resource informers during Kyuubi server startup. This ensures the Kyuubi server is promptly informed for any Kubernetes resource changes after startup. It is highly recommended to set it for multiple Kyuubi instances mode. The format is `context1:namespace1,context2:namespace2`. When the list is empty and Kyuubi runs in Kubernetes, the client for the configured namespace is initialized automatically with the in-cluster configuration. | seq | 1.11.0 | +| kyuubi.kubernetes.context | <undefined> | The desired context from your kubernetes config file used to configure the K8s client for interacting with the cluster. | string | 1.6.0 | +| kyuubi.kubernetes.context.allow.list || The allowed kubernetes context list, if it is empty, there is no kubernetes context limitation. | set | 1.8.0 | +| kyuubi.kubernetes.master.address | <undefined> | The internal Kubernetes master (API server) address to be used for kyuubi. | string | 1.7.0 | +| kyuubi.kubernetes.namespace | default | The default namespace used by the Kyuubi server's Kubernetes client to discover and manage engine pods, when the engine submission does not specify one (e.g. `spark.kubernetes.namespace`). It does not control the namespace where the Kyuubi server itself runs. | string | 1.7.0 | +| kyuubi.kubernetes.namespace.allow.list || The allowed kubernetes namespace list, if it is empty, there is no kubernetes namespace limitation. | set | 1.8.0 | +| kyuubi.kubernetes.spark.appUrlPattern | http://{{SPARK_DRIVER_SVC}}.{{KUBERNETES_NAMESPACE}}.svc:{{SPARK_UI_PORT}} | The pattern to generate the spark on kubernetes application UI URL. The pattern should contain placeholders for the application variables. Available placeholders are `{{SPARK_APP_ID}}`, `{{SPARK_DRIVER_SVC}}`, `{{SPARK_DRIVER_POD_IP}}`, `{{KUBERNETES_NAMESPACE}}`, `{{KUBERNETES_CONTEXT}}` and `{{SPARK_UI_PORT}}`. | string | 1.10.0 | +| kyuubi.kubernetes.spark.autoCreateFileUploadPath.enabled | false | If enabled, Kyuubi server will try to create the `spark.kubernetes.file.upload.path` with permission 777 before submitting the Spark application. | boolean | 1.11.0 | +| kyuubi.kubernetes.spark.cleanupTerminatedDriverPod.checkInterval | PT1M | Kyuubi server use guava cache as the cleanup trigger with time-based eviction, but the eviction would not happened until any get/put operation happened. This option schedule a daemon thread evict cache periodically. | duration | 1.8.1 | +| kyuubi.kubernetes.spark.cleanupTerminatedDriverPod.kind | NONE | Kyuubi server will delete the spark driver pod after the application terminates for kyuubi.kubernetes.terminatedApplicationRetainPeriod. Available options are NONE, ALL, COMPLETED and default value is None which means none of the pod will be deleted | string | 1.8.1 | +| kyuubi.kubernetes.spark.forciblyRewriteDriverPodName.enabled | false | Whether to forcibly rewrite Spark driver pod name with 'kyuubi--driver'. If disabled, Kyuubi will try to preserve the application name while satisfying K8s' pod name policy, but some vendors may have stricter pod name policies, thus the generated name may become illegal. | boolean | 1.8.1 | +| kyuubi.kubernetes.spark.forciblyRewriteExecutorPodNamePrefix.enabled | false | Whether to forcibly rewrite Spark executor pod name prefix with 'kyuubi-'. If disabled, Kyuubi will try to preserve the application name while satisfying K8s' pod name policy, but some vendors may have stricter Pod name policies, thus the generated name may become illegal. | boolean | 1.8.1 | +| kyuubi.kubernetes.terminatedApplicationRetainPeriod | PT5M | The period for which the Kyuubi server retains application information after the application terminates. | duration | 1.7.1 | +| kyuubi.kubernetes.trust.certificates | false | If set to true then client can submit to kubernetes cluster only with token | boolean | 1.7.0 | ### Lineage diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala index 485a955a755..47d04964d6a 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala @@ -289,7 +289,7 @@ object SparkSQLEngine extends Logging { kyuubiConf.setIfMissing(FRONTEND_THRIFT_BINARY_BIND_PORT, 0) kyuubiConf.setIfMissing(HA_ZK_CONN_RETRY_POLICY, RetryPolicies.N_TIME.toString) - if (Utils.isOnK8s) { + if (Utils.isOnK8s()) { kyuubiConf.setIfMissing(FRONTEND_CONNECTION_URL_USE_HOSTNAME, false) // https://github.com/apache/kyuubi/issues/3385 @@ -463,7 +463,7 @@ object SparkSQLEngine extends Logging { private def isOnK8sClusterMode: Boolean = { // only spark driver pod will build with `SPARK_APPLICATION_ID` env. - Utils.isOnK8s && sys.env.contains("SPARK_APPLICATION_ID") + Utils.isOnK8s() && sys.env.contains("SPARK_APPLICATION_ID") } @VisibleForTesting diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala index 7989f1754f8..7dc21ccc562 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala @@ -381,7 +381,9 @@ object Utils extends Logging { def getContextOrKyuubiClassLoader: ClassLoader = Option(Thread.currentThread().getContextClassLoader).getOrElse(getKyuubiClassLoader) - def isOnK8s: Boolean = Files.exists(Paths.get("/var/run/secrets/kubernetes.io")) + def isOnK8s(env: Map[String, String] = sys.env): Boolean = + env.get("KUBERNETES_SERVICE_HOST").exists(_.nonEmpty) && + env.get("KUBERNETES_SERVICE_PORT").exists(_.nonEmpty) /** * Return a nice string representation of the exception. It will call "printStackTrace" to diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 9fff9ee32f4..30eec5fefb6 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -1406,7 +1406,10 @@ object KyuubiConf { val KUBERNETES_NAMESPACE: ConfigEntry[String] = buildConf("kyuubi.kubernetes.namespace") - .doc("The namespace that will be used for running the kyuubi pods and find engines.") + .doc("The default namespace used by the Kyuubi server's Kubernetes client to discover" + + " and manage engine pods, when the engine submission does not specify one (e.g." + + " `spark.kubernetes.namespace`). It does not control the namespace where the Kyuubi" + + " server itself runs.") .version("1.7.0") .stringConf .createWithDefault("default") @@ -1425,9 +1428,12 @@ object KyuubiConf { val KUBERNETES_CLIENT_INITIALIZE_LIST: ConfigEntry[Seq[String]] = buildConf("kyuubi.kubernetes.client.initialize.list") .doc("The kubernetes client initialize list to register kubernetes resource informers" + - " during Kyuubi server startup. This ensure the Kyuubi server is promptly informed for" + - " any Kubernetes resource changes after startup. It is highly recommend to set it for" + - " multiple Kyuubi instances mode. The format is `context1:namespace1,context2:namespace2`.") + " during Kyuubi server startup. This ensures the Kyuubi server is promptly informed for" + + " any Kubernetes resource changes after startup. It is highly recommended to set it for" + + " multiple Kyuubi instances mode. The format is" + + " `context1:namespace1,context2:namespace2`." + + " When the list is empty and Kyuubi runs in Kubernetes, the client for the configured" + + " namespace is initialized automatically with the in-cluster configuration.") .version("1.11.0") .audience(SERVER) .immutable diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala index fa7baf71cd2..4c606aed157 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala @@ -191,6 +191,17 @@ class UtilsSuite extends KyuubiFunSuite { assertResult(false)(Utils.isCommandAvailable("un_exist_cmd")) } + test("is on Kubernetes") { + val kubernetesEnv = Map( + "KUBERNETES_SERVICE_HOST" -> "kubernetes.default.svc", + "KUBERNETES_SERVICE_PORT" -> "443") + + assert(Utils.isOnK8s(kubernetesEnv)) + assert(!Utils.isOnK8s(Map.empty)) + assert(!Utils.isOnK8s(kubernetesEnv.updated("KUBERNETES_SERVICE_HOST", ""))) + assert(!Utils.isOnK8s(kubernetesEnv.updated("KUBERNETES_SERVICE_PORT", ""))) + } + test("writeToTempFile rejects illegal filenames") { val dir = Utils.createTempDir() def stream: ByteArrayInputStream = new ByteArrayInputStream("data".getBytes) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala index 0160f9077ba..0c6b430fe96 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala @@ -217,14 +217,26 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging { } private[kyuubi] def getKubernetesClientInitializeInfo( - kyuubiConf: KyuubiConf): Seq[KubernetesInfo] = { - kyuubiConf.get(KyuubiConf.KUBERNETES_CLIENT_INITIALIZE_LIST).map { init => - val (context, namespace) = init.split(":") match { - case Array(ctx, ns) => (Some(ctx).filterNot(_.isEmpty), Some(ns).filterNot(_.isEmpty)) - case Array(ctx) => (Some(ctx).filterNot(_.isEmpty), None) - case _ => (None, None) + kyuubiConf: KyuubiConf, + environment: Map[String, String] = sys.env): Seq[KubernetesInfo] = { + val configuredInitializeInfo = + kyuubiConf.get(KyuubiConf.KUBERNETES_CLIENT_INITIALIZE_LIST).map { init => + val (context, namespace) = init.split(":") match { + case Array(ctx, ns) => (Some(ctx).filterNot(_.isEmpty), Some(ns).filterNot(_.isEmpty)) + case Array(ctx) => (Some(ctx).filterNot(_.isEmpty), None) + case _ => (None, None) + } + KubernetesInfo(context, namespace) } - KubernetesInfo(context, namespace) + if (configuredInitializeInfo.nonEmpty) { + configuredInitializeInfo + } else if (Utils.isOnK8s(environment)) { + Seq(KubernetesInfo( + // Kube context is not applicable to in-cluster configuration. + None, + Some(kyuubiConf.get(KyuubiConf.KUBERNETES_NAMESPACE)))) + } else { + Nil } } diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KubernetesApplicationOperationSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KubernetesApplicationOperationSuite.scala index 61f2cd82715..cb2de52a089 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KubernetesApplicationOperationSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KubernetesApplicationOperationSuite.scala @@ -116,12 +116,25 @@ class KubernetesApplicationOperationSuite extends KyuubiFunSuite { test("get kubernetes client initialization info") { val kyuubiConf = KyuubiConf() + val kubernetesEnv = Map( + KubernetesApplicationOperation.KUBERNETES_SERVICE_HOST -> "kubernetes.default.svc", + KubernetesApplicationOperation.KUBERNETES_SERVICE_PORT -> "443") + val operation = new KubernetesApplicationOperation() + + assert(operation.getKubernetesClientInitializeInfo(kyuubiConf, Map.empty) === Nil) + assert(operation.getKubernetesClientInitializeInfo( + kyuubiConf, + kubernetesEnv.updated(KubernetesApplicationOperation.KUBERNETES_SERVICE_HOST, "")) === Nil) + + kyuubiConf.set(KyuubiConf.KUBERNETES_NAMESPACE, "kyuubi") + assert(operation.getKubernetesClientInitializeInfo(kyuubiConf, kubernetesEnv) === + Seq(KubernetesInfo(None, Some("kyuubi")))) + kyuubiConf.set( KyuubiConf.KUBERNETES_CLIENT_INITIALIZE_LIST.key, "c1:ns1,c1:ns2,c2:ns1,c2:ns2,c1:,:ns1") - val operation = new KubernetesApplicationOperation() - assert(operation.getKubernetesClientInitializeInfo(kyuubiConf) === + assert(operation.getKubernetesClientInitializeInfo(kyuubiConf, kubernetesEnv) === Array( KubernetesInfo(Some("c1"), Some("ns1")), KubernetesInfo(Some("c1"), Some("ns2")),