You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ul...@apache.org on 2022/04/15 06:17:06 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #2359] [Test] Build WithKyuubiServerOnKuberntes
This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 65a272f6e [KYUUBI #2359] [Test] Build WithKyuubiServerOnKuberntes
65a272f6e is described below
commit 65a272f6e44d8d773c3eaad8b3146119a5bf5d3f
Author: zwangsheng <22...@qq.com>
AuthorDate: Fri Apr 15 14:16:54 2022 +0800
[KYUUBI #2359] [Test] Build WithKyuubiServerOnKuberntes
### _Why are the changes needed?_
Refactor KyuubiOnKubernetesTestsSuite
Build WithKyuubiServerOnKuberntes like WithKyuubiServerOnYarn
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #2359 from zwangsheng/test/kyuubi_on_kubernetes.
Closes #2359
c1048c6a [zwangsheng] fix
81c98cd2 [zwangsheng] unused import
976f9091 [zwangsheng] fix
bad6720b [zwangsheng] fix
98d8995c [zwangsheng] refactor
628de557 [zwangsheng] style check
9dc6675a [zwangsheng] Build WithKyuubiServerOnKubernetes
Authored-by: zwangsheng <22...@qq.com>
Signed-off-by: ulysses-you <ul...@apache.org>
---
...te.scala => WithKyuubiServerOnKubernetes.scala} | 52 +++++++++++-----------
.../deployment/KyuubiOnKubernetesTestsSuite.scala | 38 +++-------------
2 files changed, 32 insertions(+), 58 deletions(-)
diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala
similarity index 57%
copy from integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
copy to integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala
index 4ce7fe107..f58b51a84 100644
--- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala
@@ -15,36 +15,39 @@
* limitations under the License.
*/
-package org.apache.kyuubi.kubernetes.test.deployment
+package org.apache.kyuubi.kubernetes.test
-import org.apache.kyuubi.Logging
-import org.apache.kyuubi.kubernetes.test.MiniKube
-import org.apache.kyuubi.operation.SparkQueryTests
+import io.fabric8.kubernetes.client.DefaultKubernetesClient
-/**
- * This test is for Kyuubi Server on Kubernetes with Spark engine:
- *
- * Real World Kubernetes Pod
- * ------------ -----------------------------------------------------
- * | | JDBC | |
- * | Client | ----> | Kyuubi Server ----> Spark Engine (local mode) |
- * | | | |
- * ------------ -----------------------------------------------------
- */
-class KyuubiOnKubernetesTestsSuite extends SparkQueryTests with Logging {
- private lazy val _jdbcUrl: String = {
- val kubernetesclient = MiniKube.getKubernetesClient
+import org.apache.kyuubi.WithKyuubiServer
+import org.apache.kyuubi.config.KyuubiConf
+
+trait WithKyuubiServerOnKubernetes extends WithKyuubiServer {
+ protected val kyuubiServerConf: KyuubiConf = KyuubiConf()
+ protected val connectionConf: Map[String, String]
+ private var miniKubernetesClient: DefaultKubernetesClient = _
+
+ final override protected lazy val conf: KyuubiConf = {
+ connectionConf.foreach { case (k, v) => kyuubiServerConf.set(k, v) }
+ kyuubiServerConf
+ }
+
+ override def beforeAll(): Unit = {
+ miniKubernetesClient = MiniKube.getKubernetesClient
+ super.beforeAll()
+ }
+
+ override def afterAll(): Unit = super.afterAll()
+
+ override protected def getJdbcUrl: String = {
val kyuubiServers =
- kubernetesclient
- .pods()
- .list()
- .getItems
+ miniKubernetesClient.pods().list().getItems
assert(kyuubiServers.size() == 1)
val kyuubiServer = kyuubiServers.get(0)
// Kyuubi server state should be running since mvn compile is quite slowly..
if (!"running".equalsIgnoreCase(kyuubiServer.getStatus.getPhase)) {
val log =
- kubernetesclient
+ miniKubernetesClient
.pods()
.withName(kyuubiServer.getMetadata.getName)
.getLog
@@ -56,9 +59,4 @@ class KyuubiOnKubernetesTestsSuite extends SparkQueryTests with Logging {
kyuubiServer.getSpec.getContainers.get(0).getPorts.get(0).getHostPort
s"jdbc:hive2://$kyuubiServerIp:$kyuubiServerPort/;"
}
-
- override protected def jdbcUrl: String = {
- assert(_jdbcUrl != null, "Failed to get Kyuubi server")
- _jdbcUrl
- }
}
diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
index 4ce7fe107..5d15fcf2b 100644
--- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
@@ -17,8 +17,7 @@
package org.apache.kyuubi.kubernetes.test.deployment
-import org.apache.kyuubi.Logging
-import org.apache.kyuubi.kubernetes.test.MiniKube
+import org.apache.kyuubi.kubernetes.test.WithKyuubiServerOnKubernetes
import org.apache.kyuubi.operation.SparkQueryTests
/**
@@ -31,34 +30,11 @@ import org.apache.kyuubi.operation.SparkQueryTests
* | | | |
* ------------ -----------------------------------------------------
*/
-class KyuubiOnKubernetesTestsSuite extends SparkQueryTests with Logging {
- private lazy val _jdbcUrl: String = {
- val kubernetesclient = MiniKube.getKubernetesClient
- val kyuubiServers =
- kubernetesclient
- .pods()
- .list()
- .getItems
- assert(kyuubiServers.size() == 1)
- val kyuubiServer = kyuubiServers.get(0)
- // Kyuubi server state should be running since mvn compile is quite slowly..
- if (!"running".equalsIgnoreCase(kyuubiServer.getStatus.getPhase)) {
- val log =
- kubernetesclient
- .pods()
- .withName(kyuubiServer.getMetadata.getName)
- .getLog
- throw new IllegalStateException(
- s"Kyuubi server pod state error: ${kyuubiServer.getStatus.getPhase}, log:\n$log")
- }
- val kyuubiServerIp = MiniKube.getIp
- val kyuubiServerPort =
- kyuubiServer.getSpec.getContainers.get(0).getPorts.get(0).getHostPort
- s"jdbc:hive2://$kyuubiServerIp:$kyuubiServerPort/;"
- }
+class KyuubiOnKubernetesWithLocalSparkTestsSuite extends WithKyuubiServerOnKubernetes
+ with SparkQueryTests {
+ override protected val connectionConf: Map[String, String] = Map(
+ "spark.master" -> "local",
+ "spark.executor.instances" -> "1")
- override protected def jdbcUrl: String = {
- assert(_jdbcUrl != null, "Failed to get Kyuubi server")
- _jdbcUrl
- }
+ override protected def jdbcUrl: String = getJdbcUrl
}