You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/06/16 10:13:15 UTC
[incubator-celeborn] 01/02: [CELEBORN-105][TEST] Kubernetes Integration Test
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch branch-0.3
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
commit 24222f2a20ede7250e3db92ae1af3d643e30d23a
Author: zwangsheng <22...@qq.com>
AuthorDate: Mon Jun 5 12:11:29 2023 +0800
[CELEBORN-105][TEST] Kubernetes Integration Test
---
.github/workflows/integration.yml | 78 ++++++++++++++++++++++
pom.xml | 14 ++++
tests/kubernetes-it/pom.xml | 70 +++++++++++++++++++
.../src/test/resources/log4j2-test.xml | 37 ++++++++++
.../DeploySuite.scala | 59 ++++++++++++++++
.../WithMiniKube.scala | 29 ++++++++
6 files changed, 287 insertions(+)
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
new file mode 100644
index 000000000..6f5c4f2e8
--- /dev/null
+++ b/.github/workflows/integration.yml
@@ -0,0 +1,78 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Integration Test
+
+on:
+ push:
+ branches:
+ - main
+ - branch-*
+ pull_request:
+ branches:
+ - main
+ - branch-*
+env:
+ MINIKUBE_VERSION: v1.29.0
+ KUBERNETES_VERSION: v1.26.1
+
+jobs:
+ celeborn_integration_test:
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-java@v2
+ name: Setup JDK with Maven
+ with:
+ distribution: zulu
+ java-version: 8
+ cache: maven
+ check-latest: false
+ - name: Build Celeborn Binary
+ run:
+ ./build/make-distribution.sh -Pspark-3.3 -Pgoogle-mirror
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+ - name: Build Celeborn Docker Image
+ uses: docker/build-push-action@v2
+ with:
+ context: ./dist
+ file: ./docker/Dockerfile
+ load: true
+ tags: apache/celeborn:latest
+ - name: Setup Helm
+ uses: azure/setup-helm@v3
+ with:
+ version: v3.10.0
+ - name: Setup Minikube
+ run: |
+ # https://minikube.sigs.k8s.io/docs/start/
+ curl -LO https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64
+ sudo install minikube-linux-amd64 /usr/local/bin/minikube
+ minikube start --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --force
+ - name: Deploy Celeborn
+ run: |
+ # https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime
+ minikube image load apache/celeborn:latest
+ helm install celeborn charts/celeborn -f ./tests/kubernetes-it/docker/helm/values.yaml
+ - name: Wait Celeborn Ready
+ run: >-
+ ./build/mvn clean install
+ -Pkubernetes-it
+ -Pspark-3.3
+ -Dtest=none
+ -DwildcardSuites=org.apache.celeborn.integration.DeploySuite
diff --git a/pom.xml b/pom.xml
index 3c080daf9..85f614077 100644
--- a/pom.xml
+++ b/pom.xml
@@ -89,6 +89,7 @@
<roaringbitmap.version>0.9.32</roaringbitmap.version>
<snakeyaml.version>1.33</snakeyaml.version>
<zstd-jni.version>1.5.2-1</zstd-jni.version>
+ <kubernetes-client.version>6.7.0</kubernetes-client.version>
<shading.prefix>org.apache.celeborn.shaded</shading.prefix>
@@ -405,6 +406,12 @@
<version>${mockito-scalatest.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.fabric8</groupId>
+ <artifactId>kubernetes-client</artifactId>
+ <version>${kubernetes-client.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
@@ -1134,5 +1141,12 @@
</plugins>
</build>
</profile>
+
+ <profile>
+ <id>kubernetes-it</id>
+ <modules>
+ <module>tests/kubernetes-it</module>
+ </modules>
+ </profile>
</profiles>
</project>
diff --git a/tests/kubernetes-it/pom.xml b/tests/kubernetes-it/pom.xml
new file mode 100644
index 000000000..2efbb905d
--- /dev/null
+++ b/tests/kubernetes-it/pom.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-parent_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>kubernetes-it</artifactId>
+ <packaging>jar</packaging>
+ <name>Celeborn Kubernetes Integration Test</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-common_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-client_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-client_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-master_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-worker_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.fabric8</groupId>
+ <artifactId>kubernetes-client</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/tests/kubernetes-it/src/test/resources/log4j2-test.xml b/tests/kubernetes-it/src/test/resources/log4j2-test.xml
new file mode 100644
index 000000000..9bcf7f263
--- /dev/null
+++ b/tests/kubernetes-it/src/test/resources/log4j2-test.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<Configuration status="INFO">
+ <Appenders>
+ <Console name="stdout" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}: %m%n%ex"/>
+ <Filters>
+ <ThresholdFilter level="FATAL"/>
+ </Filters>
+ </Console>
+ <File name="file" fileName="target/unit-tests.log">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}: %m%n%ex"/>
+ </File>
+ </Appenders>
+ <Loggers>
+ <Root level="INFO">
+ <AppenderRef ref="stdout"/>
+ <AppenderRef ref="file"/>
+ </Root>
+ </Loggers>
+</Configuration>
diff --git a/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala
new file mode 100644
index 000000000..7d93d6d06
--- /dev/null
+++ b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.integration
+
+import scala.language.postfixOps
+
+import org.scalatest.concurrent.Eventually.eventually
+import org.scalatest.concurrent.Waiters.{interval, timeout}
+import org.scalatest.time.SpanSugar.convertIntToGrainOfTime
+
+import org.apache.celeborn.CelebornFunSuite
+import org.apache.celeborn.client.WithShuffleClientSuite
+
+// TODO need add shuffle client test
+class DeploySuite extends CelebornFunSuite with WithMiniKube {
+ final val masterStatefulSet =
+ kubernetesClient.apps().statefulSets().withName("celeborn-master").get()
+ assert(masterStatefulSet != null)
+ final val workerStatefulSet =
+ kubernetesClient.apps().statefulSets().withName("celeborn-worker").get()
+ assert(workerStatefulSet != null)
+
+ test("Check master started") {
+ val masterPod = kubernetesClient.pods().withName("celeborn-master-0")
+ // wait for master ready
+ eventually(timeout(5 minutes), interval(10 seconds)) {
+ val log = masterPod.getLog(true)
+ assert(log.contains("Master started."))
+ }
+
+ assert(masterPod.get().getStatus.getPhase == "Running")
+ }
+
+ test("Check worker started") {
+ val workerPod = kubernetesClient.pods().withName("celeborn-worker-0")
+ // wait for worker ready
+ eventually(timeout(5 minutes), interval(10 seconds)) {
+ val log = workerPod.getLog(true)
+ assert(log.contains("Register worker successfully."))
+ }
+
+ assert(workerPod.get().getStatus.getPhase == "Running")
+ }
+}
diff --git a/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala
new file mode 100644
index 000000000..8676d7174
--- /dev/null
+++ b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.integration
+
+import io.fabric8.kubernetes.client.{KubernetesClient, KubernetesClientBuilder}
+import io.fabric8.kubernetes.client.Config
+
+trait WithMiniKube {
+
+ lazy val kubernetesClient: KubernetesClient =
+ new KubernetesClientBuilder()
+ .withConfig(Config.autoConfigure("minikube"))
+ .build()
+}