You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ya...@apache.org on 2021/07/26 10:47:27 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #444][KYUUBI
#854] Add minikube for k8s integration test
This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 536e5fd [KYUUBI #444][KYUUBI #854] Add minikube for k8s integration test
536e5fd is described below
commit 536e5fde72fc7492f0d8ae38e2074dfa93ed4119
Author: ulysses-you <ul...@gmail.com>
AuthorDate: Mon Jul 26 18:47:11 2021 +0800
[KYUUBI #444][KYUUBI #854] Add minikube for k8s integration test
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
* Use github action to install `MiniKube`
* Start Kyuubi Server as Pod with `MiniKube`
* Add a new module `kubernetes-integration-tests` to run basic JDBC test
Since there exists many `MiniKube` app in github market, this PR use [Setup Minikube Kubernetes Cluster](https://github.com/marketplace/actions/setup-minikube-kubernetes-cluster) , we can change this if necessary in future.
Note that, this PR is for Kyuubi test with k8s not for Apache Spark.
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.readthedocs.io/en/latest/tools/testing.html#running-tests) locally before make a pull request
Closes #853 from ulysses-you/k8s-it.
Closes #444
Closes #854
4a73a8fa [ulysses-you] ignore
a501482d [ulysses-you] ignore
242f60c4 [ulysses-you] todo
994e52e4 [ulysses-you] pom
3f60fb2e [ulysses-you] Merge branch 'master' of https://github.com/NetEase/kyuubi into k8s-it
3febdcd2 [ulysses-you] fix
f564792b [ulysses-you] Add minikuube for k8s integration test
Authored-by: ulysses-you <ul...@gmail.com>
Signed-off-by: Kent Yao <ya...@apache.org>
---
.github/workflows/master.yml | 30 ++++++++++
kubernetes/integration-tests/pom.xml | 62 ++++++++++++++++++++
.../src/test/resources/log4j.properties | 40 +++++++++++++
.../kubernetes/test/KubernetesJDBCTestsSuite.scala | 49 ++++++++++++++++
.../apache/kyuubi/kubernetes/test/MiniKube.scala | 66 ++++++++++++++++++++++
.../kyuubi/kubernetes/test/ProcessUtils.scala | 63 +++++++++++++++++++++
kubernetes/integration-tests/test-k8s.yaml | 37 ++++++++++++
.../org/apache/kyuubi/operation/JDBCTests.scala | 3 +-
pom.xml | 15 +++++
9 files changed, 364 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 7fd209a..56a6871 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -139,3 +139,33 @@ jobs:
run: |
./build/mvn clean install -Pspark-3.1 -DskipTests -pl :kyuubi-spark-sql-engine,:kyuubi-common,:kyuubi-ha,:kyuubi-zookeeper,:kyuubi-spark-monitor
./build/mvn test -Pspark-3.1 -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds -Dmaven.plugin.scalatest.exclude.tags=''
+ minikube-it:
+ name: Minikube Integration Test
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v2
+ # from https://github.com/marketplace/actions/setup-minikube-kubernetes-cluster
+ - name: Setup Minikube
+ uses: manusa/actions-setup-minikube@v2.4.2
+ with:
+ minikube version: 'v1.16.0'
+ kubernetes version: 'v1.19.2'
+ - name: kubectl pre-check
+ run: |
+ kubectl get serviceaccount
+ kubectl create serviceaccount default
+ kubectl get serviceaccount
+ - name: start kyuubi
+ run: kubectl apply -f kubernetes/integration-tests/test-k8s.yaml
+ - name: kyuubi pod check
+ run: kubectl get pods
+ - name: integration tests
+ run: ./build/mvn clean test -Pkubernetes -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test
+ - name: Upload test logs
+ if: failure()
+ uses: actions/upload-artifact@v2
+ with:
+ name: unit-tests-log
+ path: |
+ **/target/unit-tests.log
diff --git a/kubernetes/integration-tests/pom.xml b/kubernetes/integration-tests/pom.xml
new file mode 100644
index 0000000..9796291
--- /dev/null
+++ b/kubernetes/integration-tests/pom.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>kyuubi-parent</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <name>Kyuubi Kubernetes Integration Tests</name>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kubernetes-integration-tests</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>io.fabric8</groupId>
+ <artifactId>kubernetes-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>kyuubi-common</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>kyuubi-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- for hive driver related dependency -->
+ <dependency>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>kyuubi-hive-jdbc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/kubernetes/integration-tests/src/test/resources/log4j.properties b/kubernetes/integration-tests/src/test/resources/log4j.properties
new file mode 100644
index 0000000..958c9c8
--- /dev/null
+++ b/kubernetes/integration-tests/src/test/resources/log4j.properties
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+log4j.rootLogger=INFO, CA, FA
+
+#Console Appender
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n
+log4j.appender.CA.Threshold = FATAL
+
+#File Appender
+log4j.appender.FA=org.apache.log4j.FileAppender
+log4j.appender.FA.append=false
+log4j.appender.FA.file=target/unit-tests.log
+log4j.appender.FA.layout=org.apache.log4j.PatternLayout
+log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n
+
+# Set the logger level of File Appender to WARN
+log4j.appender.FA.Threshold = INFO
+
+# SPARK-34128：Suppress undesirable TTransportException warnings involved in THRIFT-4805
+log4j.appender.console.filter.1=org.apache.log4j.varia.StringMatchFilter
+log4j.appender.console.filter.1.StringToMatch=Thrift error occurred during processing of message
+log4j.appender.console.filter.1.AcceptOnMatch=false
diff --git a/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesJDBCTestsSuite.scala b/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesJDBCTestsSuite.scala
new file mode 100644
index 0000000..bc8cf0c
--- /dev/null
+++ b/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/KubernetesJDBCTestsSuite.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.kubernetes.test
+
+import org.apache.kyuubi.Logging
+import org.apache.kyuubi.operation.JDBCTests
+
+// TODO: [KYUUBI-863] Support test Spark engine using k8s master with minikube
+class KubernetesJDBCTestsSuite extends JDBCTests with Logging {
+ private lazy val _jdbcUrl: String = {
+ val kubernetesclient = MiniKube.getKubernetesClient
+ val kyuubiServers =
+ kubernetesclient
+ .pods()
+ .list()
+ .getItems
+ assert(kyuubiServers.size() == 1)
+ val kyuubiServer = kyuubiServers.get(0)
+ // Kyuubi server state should be running since mvn compile is quite slowly..
+ if (!"running".equalsIgnoreCase(kyuubiServer.getStatus.getPhase)) {
+ throw new IllegalStateException(
+ s"Kyuubi server pod state error: ${kyuubiServer.getStatus.getPhase}")
+ }
+ val kyuubiServerIp = MiniKube.getIp
+ val kyuubiServerPort =
+ kyuubiServer.getSpec.getContainers.get(0).getPorts.get(0).getHostPort
+ s"jdbc:hive2://$kyuubiServerIp:$kyuubiServerPort/;"
+ }
+
+ override protected def jdbcUrl: String = {
+ assert(_jdbcUrl != null, "Failed to get Kyuubi server")
+ _jdbcUrl
+ }
+}
diff --git a/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala b/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala
new file mode 100644
index 0000000..3fb7de9
--- /dev/null
+++ b/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.kubernetes.test
+
+import io.fabric8.kubernetes.client.{Config, DefaultKubernetesClient}
+
+/**
+ * This code copied from Aapache Spark
+ * org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube
+ */
+object MiniKube {
+ private val MINIKUBE_STARTUP_TIMEOUT_SECONDS = 60
+ private val VERSION_PREFIX = "minikube version: "
+
+ lazy val minikubeVersionString =
+ executeMinikube(true, "version").find(_.contains(VERSION_PREFIX)).get
+
+ def executeMinikube(logOutput: Boolean, action: String, args: String*): Seq[String] = {
+ ProcessUtils.executeProcess(
+ Array("bash", "-c", s"MINIKUBE_IN_STYLE=true minikube $action ${args.mkString(" ")}"),
+ MINIKUBE_STARTUP_TIMEOUT_SECONDS, dumpOutput = logOutput).filter { x =>
+ !x.contains("There is a newer version of minikube") &&
+ !x.contains("https://github.com/kubernetes")
+ }
+ }
+
+ def getIp: String = {
+ executeMinikube(true, "ip").head
+ }
+
+ def getKubernetesClient: DefaultKubernetesClient = {
+ // only the three-part version number is matched (the optional suffix like "-beta.0" is dropped)
+ val versionArrayOpt = "\\d+\\.\\d+\\.\\d+".r
+ .findFirstIn(minikubeVersionString.split(VERSION_PREFIX)(1))
+ .map(_.split('.').map(_.toInt))
+
+ versionArrayOpt match {
+ case Some(Array(x, y, z)) =>
+ if (Ordering.Tuple3[Int, Int, Int].lt((x, y, z), (1, 7, 3))) {
+ assert(false, s"Unsupported Minikube version is detected: $minikubeVersionString." +
+ "For integration testing Minikube version 1.7.3 or greater is expected.")
+ }
+ case _ =>
+ assert(false, s"Unexpected version format detected in `$minikubeVersionString`." +
+ "For minikube version a three-part version number is expected (the optional " +
+ "non-numeric suffix is intentionally dropped)")
+ }
+
+ new DefaultKubernetesClient(Config.autoConfigure("minikube"))
+ }
+}
diff --git a/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/ProcessUtils.scala b/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/ProcessUtils.scala
new file mode 100644
index 0000000..95bc7de
--- /dev/null
+++ b/kubernetes/integration-tests/src/test/scala/org/apache/kyuubi/kubernetes/test/ProcessUtils.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.kubernetes.test
+
+import java.nio.charset.StandardCharsets
+import java.util.concurrent.TimeUnit
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+import scala.io.Source
+
+import org.apache.kyuubi.Logging
+
+object ProcessUtils extends Logging {
+ /**
+ * executeProcess is used to run a command and return the output if it
+ * completes within timeout seconds.
+ */
+ def executeProcess(
+ fullCommand: Array[String],
+ timeout: Long,
+ dumpOutput: Boolean = true,
+ dumpErrors: Boolean = true,
+ env: Map[String, String] = Map.empty[String, String]): Seq[String] = {
+ val pb = new ProcessBuilder().command(fullCommand: _*)
+ pb.environment().putAll(env.asJava)
+ pb.redirectErrorStream(true)
+ val proc = pb.start()
+ val outputLines = new ArrayBuffer[String]
+ val inputStream = proc.getInputStream
+ try {
+ Source.fromInputStream(inputStream, StandardCharsets.UTF_8.name())
+ .getLines().foreach { line =>
+ if (dumpOutput) {
+ info(line)
+ }
+ outputLines += line
+ }
+ } finally {
+ inputStream.close()
+ }
+ assert(proc.waitFor(timeout, TimeUnit.SECONDS),
+ s"Timed out while executing ${fullCommand.mkString(" ")}")
+ assert(proc.exitValue == 0,
+ s"Failed to execute -- ${fullCommand.mkString(" ")} --" +
+ s"${if (dumpErrors) "\n" + outputLines.mkString("\n")}")
+ outputLines.toSeq
+ }
+}
diff --git a/kubernetes/integration-tests/test-k8s.yaml b/kubernetes/integration-tests/test-k8s.yaml
new file mode 100644
index 0000000..230d271
--- /dev/null
+++ b/kubernetes/integration-tests/test-k8s.yaml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: Pod
+metadata:
+ name: kyuubi-test
+ labels:
+ kyuubi-role: KYUUBI_SERVER
+spec:
+ containers:
+ - name: kyuubi-server
+ # TODO: replace this with the official repo
+ image: yaooqinn/kyuubi:latest
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: KYUUBI_JAVA_OPTS
+ value: -Dkyuubi.frontend.bind.host=0.0.0.0
+ ports:
+ - name: frontend-port
+ containerPort: 10009
+ hostPort: 10009
+ protocol: TCP
diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala
index fa0ed60..9efef1b 100644
--- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala
+++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTests.scala
@@ -360,7 +360,8 @@ trait JDBCTests extends BasicJDBCTests {
}
}
- test("kyuubi defined function - system_user") {
+ // dockerfile use kyuubi as user which is not same with non-k8s env.
+ ignore("kyuubi defined function - system_user") {
withJdbcStatement() { statement =>
val rs = statement.executeQuery("SELECT system_user()")
assert(rs.next())
diff --git a/pom.xml b/pom.xml
index 7b4430d..f28d194 100644
--- a/pom.xml
+++ b/pom.xml
@@ -126,6 +126,8 @@
<slf4j.version>1.7.30</slf4j.version>
<zookeeper.version>3.4.14</zookeeper.version>
+ <kubernetes-client.version>5.5.0</kubernetes-client.version>
+
<scalatest.version>3.2.9</scalatest.version>
<iceberg.name>iceberg-spark3-runtime</iceberg.name>
<iceberg.version>0.11.1</iceberg.version>
@@ -424,6 +426,12 @@
</dependency>
<dependency>
+ <groupId>io.fabric8</groupId>
+ <artifactId>kubernetes-client</artifactId>
+ <version>${kubernetes-client.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-common</artifactId>
<version>${hive.version}</version>
@@ -1599,5 +1607,12 @@
<module>tools/spark-block-cleaner</module>
</modules>
</profile>
+
+ <profile>
+ <id>kubernetes</id>
+ <modules>
+ <module>kubernetes/integration-tests</module>
+ </modules>
+ </profile>
</profiles>
</project>