You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ul...@apache.org on 2022/04/21 10:15:32 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #2344] [Improvement] Add Kyuubi Server on Kubernetes with Spark Cluster mode integration test
This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 7fa04947c [KYUUBI #2344] [Improvement] Add Kyuubi Server on Kubernetes with Spark Cluster mode integration test
7fa04947c is described below
commit 7fa04947c4ea2a280648b837e134265430c7ec89
Author: zwangsheng <22...@qq.com>
AuthorDate: Thu Apr 21 18:15:24 2022 +0800
[KYUUBI #2344] [Improvement] Add Kyuubi Server on Kubernetes with Spark Cluster mode integration test
### _Why are the changes needed?_
Add Kyuubi Server on Kubernetes with Spark Cluster mode integration test
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #2409 from zwangsheng/improve/add-kyuubi-k8s-spark-cluster-test.
Closes #2344
7d20425d [zwangsheng] fix scala
bf2fafef [zwangsheng] fix scala
551aa7d7 [zwangsheng] fix java & class name
bd8a7a44 [zwangsheng] fix scala
42ba9fd0 [zwangsheng] fix scala
de4135b5 [zwangsheng] fix scala
511af18e [zwangsheng] try
Authored-by: zwangsheng <22...@qq.com>
Signed-off-by: ulysses-you <ul...@apache.org>
---
.../test/WithKyuubiServerOnKubernetes.scala | 2 +-
.../deployment/KyuubiOnKubernetesTestsSuite.scala | 86 ++++++++++++++++++----
2 files changed, 72 insertions(+), 16 deletions(-)
diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala
index 362f257a7..c9d71c8a2 100644
--- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala
@@ -24,7 +24,7 @@ import org.apache.kyuubi.config.KyuubiConf
trait WithKyuubiServerOnKubernetes extends WithKyuubiServer {
protected val kyuubiServerConf: KyuubiConf = KyuubiConf()
- protected val connectionConf: Map[String, String]
+ protected def connectionConf: Map[String, String] = Map.empty
private val miniKubernetesClient: DefaultKubernetesClient = MiniKube.getKubernetesClient
final override protected lazy val conf: KyuubiConf = {
diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
index d29990002..85690a65f 100644
--- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
@@ -17,8 +17,14 @@
package org.apache.kyuubi.kubernetes.test.deployment
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.net.NetUtils
+
+import org.apache.kyuubi.{Utils, WithSimpleDFSService}
+import org.apache.kyuubi.config.KyuubiConf.{FRONTEND_CONNECTION_URL_USE_HOSTNAME, FRONTEND_THRIFT_BINARY_BIND_HOST}
import org.apache.kyuubi.kubernetes.test.WithKyuubiServerOnKubernetes
import org.apache.kyuubi.operation.SparkQueryTests
+import org.apache.kyuubi.zookeeper.ZookeeperConf.ZK_CLIENT_PORT_ADDRESS
/**
* This test is for Kyuubi Server on Kubernetes with Spark engine local deploy-mode:
@@ -32,9 +38,25 @@ import org.apache.kyuubi.operation.SparkQueryTests
*/
class KyuubiOnKubernetesWithLocalSparkTestsSuite extends WithKyuubiServerOnKubernetes
with SparkQueryTests {
- override protected val connectionConf: Map[String, String] = Map(
- "spark.master" -> "local",
- "spark.executor.instances" -> "1")
+ override protected def connectionConf: Map[String, String] = {
+ super.connectionConf ++ Map("spark.master" -> "local", "spark.executor.instances" -> "1")
+ }
+
+ override protected def jdbcUrl: String = getJdbcUrl
+}
+
+class KyuubiOnKubernetesWithSparkTestsBase extends WithKyuubiServerOnKubernetes
+ with SparkQueryTests {
+ override protected def connectionConf: Map[String, String] = {
+ super.connectionConf ++
+ Map(
+ "spark.master" -> s"k8s://$getMiniKubeApiMaster",
+ "spark.executor.memory" -> "512M",
+ "spark.driver.memory" -> "512M",
+ "spark.kubernetes.driver.request.cores" -> "250m",
+ "spark.kubernetes.executor.request.cores" -> "250m",
+ "spark.executor.instances" -> "1")
+ }
override protected def jdbcUrl: String = getJdbcUrl
}
@@ -49,17 +71,51 @@ class KyuubiOnKubernetesWithLocalSparkTestsSuite extends WithKyuubiServerOnKuber
* | | | | | |
* ------------ ------------------------------------------------- ---------------------
*/
-class KyuubiOnKubernetesWithClientSparkOnKubernetesTestsSuite extends WithKyuubiServerOnKubernetes
- with SparkQueryTests {
- override protected val connectionConf: Map[String, String] = Map(
- "spark.master" -> s"k8s://$getMiniKubeApiMaster",
- "spark.submit.deployMode" -> "client",
- "spark.kubernetes.container.image" -> "apache/spark:v3.2.1",
- "spark.executor.memory" -> "512M",
- "spark.driver.memory" -> "512M",
- "spark.kubernetes.driver.request.cores" -> "250m",
- "spark.kubernetes.executor.request.cores" -> "250m",
- "spark.executor.instances" -> "1")
+class KyuubiOnKubernetesWithClientSparkTestsSuite
+ extends KyuubiOnKubernetesWithSparkTestsBase {
+ override protected def connectionConf: Map[String, String] = {
+ super.connectionConf ++ Map("spark.submit.deployMode" -> "client")
+ }
+}
- override protected def jdbcUrl: String = getJdbcUrl
+/**
+ * This test is for Kyuubi Server on Kubernetes with Spark engine On Kubernetes client deploy-mode:
+ *
+ * Real World Kubernetes Pod
+ * ---------- ----------------- ----------------------------- ---------------------
+ * | | JDBC | | | | | |
+ * | Client | ----> | Kyuubi Server | --> |Spark Engine (cluster mode)| --> | Spark Executors |
+ * | | | | | | | |
+ * ---------- ----------------- ----------------------------- ---------------------
+ */
+class KyuubiOnKubernetesWithClusterSparkTestsSuite
+ extends KyuubiOnKubernetesWithSparkTestsBase with WithSimpleDFSService {
+ private val localhostAddress = Utils.findLocalInetAddress.getHostAddress
+ private val driverTemplate =
+ Thread.currentThread().getContextClassLoader.getResource("driver.yml")
+
+ override val hadoopConf: Configuration = {
+ val hdfsConf: Configuration = new Configuration()
+ hdfsConf.set("dfs.namenode.rpc-bind-host", "0.0.0.0")
+ hdfsConf.set("dfs.namenode.servicerpc-bind-host", "0.0.0.0")
+ hdfsConf.set("dfs.datanode.hostname", localhostAddress)
+ hdfsConf.set("dfs.datanode.address", s"0.0.0.0:${NetUtils.getFreeSocketPort}")
+ // spark use 185 as userid in docker
+ hdfsConf.set("hadoop.proxyuser.185.groups", "*")
+ hdfsConf.set("hadoop.proxyuser.185.hosts", "*")
+ hdfsConf
+ }
+
+ override protected def connectionConf: Map[String, String] = {
+ super.connectionConf ++
+ Map(
+ "spark.submit.deployMode" -> "cluster",
+ "spark.kubernetes.file.upload.path" -> s"hdfs://$localhostAddress:$getDFSPort/spark",
+ "spark.hadoop.dfs.client.use.datanode.hostname" -> "true",
+ "spark.kubernetes.authenticate.driver.serviceAccountName" -> "spark",
+ "spark.kubernetes.driver.podTemplateFile" -> driverTemplate.getPath,
+ ZK_CLIENT_PORT_ADDRESS.key -> localhostAddress,
+ FRONTEND_CONNECTION_URL_USE_HOSTNAME.key -> "false",
+ FRONTEND_THRIFT_BINARY_BIND_HOST.key -> localhostAddress)
+ }
}