You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by mc...@apache.org on 2018/07/18 17:01:46 UTC
spark git commit: [SPARK-24825][K8S][TEST] Kubernetes integration
tests build the whole reactor
Repository: spark
Updated Branches:
refs/heads/master ebe9e2848 -> fc0c8c971
[SPARK-24825][K8S][TEST] Kubernetes integration tests build the whole reactor
## What changes were proposed in this pull request?
Make the integration test script build all modules.
In order to not run all the non-Kubernetes integration tests in the build, support specifying tags and tag all integration tests specifically with "k8s". Supply the k8s tag in the dev/dev-run-integration-tests.sh script.
## How was this patch tested?
The build system will test this.
Author: mcheah <mc...@palantir.com>
Closes #21800 from mccheah/k8s-integration-tests-maven-fix.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fc0c8c97
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fc0c8c97
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fc0c8c97
Branch: refs/heads/master
Commit: fc0c8c97173e641b50eb7cea80c63262d5ba4180
Parents: ebe9e28
Author: mcheah <mc...@palantir.com>
Authored: Wed Jul 18 10:01:39 2018 -0700
Committer: mcheah <mc...@palantir.com>
Committed: Wed Jul 18 10:01:39 2018 -0700
----------------------------------------------------------------------
pom.xml | 3 +++
.../dev/dev-run-integration-tests.sh | 21 ++++++----------
.../k8s/integrationtest/KubernetesSuite.scala | 25 ++++++++++----------
3 files changed, 23 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fc0c8c97/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 649221d..81a53ee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -194,6 +194,7 @@
<test.java.home>${java.home}</test.java.home>
<test.exclude.tags></test.exclude.tags>
+ <test.include.tags></test.include.tags>
<!-- Package to use when relocating shaded classes. -->
<spark.shade.packageName>org.spark_project</spark.shade.packageName>
@@ -2162,6 +2163,7 @@
</systemProperties>
<failIfNoTests>false</failIfNoTests>
<excludedGroups>${test.exclude.tags}</excludedGroups>
+ <groups>${test.include.tags}</groups>
</configuration>
<executions>
<execution>
@@ -2209,6 +2211,7 @@
<test.src.tables>__not_used__</test.src.tables>
</systemProperties>
<tagsToExclude>${test.exclude.tags}</tagsToExclude>
+ <tagsToInclude>${test.include.tags}</tagsToInclude>
</configuration>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/spark/blob/fc0c8c97/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
index 3acd0f5..b28b8b8 100755
--- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
+++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
@@ -16,9 +16,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-TEST_ROOT_DIR=$(git rev-parse --show-toplevel)/resource-managers/kubernetes/integration-tests
-
-cd "${TEST_ROOT_DIR}"
+set -xo errexit
+TEST_ROOT_DIR=$(git rev-parse --show-toplevel)
DEPLOY_MODE="minikube"
IMAGE_REPO="docker.io/kubespark"
@@ -27,7 +26,7 @@ IMAGE_TAG="N/A"
SPARK_MASTER=
NAMESPACE=
SERVICE_ACCOUNT=
-INCLUDE_TAGS=
+INCLUDE_TAGS="k8s"
EXCLUDE_TAGS=
# Parse arguments
@@ -62,7 +61,7 @@ while (( "$#" )); do
shift
;;
--include-tags)
- INCLUDE_TAGS="$2"
+ INCLUDE_TAGS="k8s,$2"
shift
;;
--exclude-tags)
@@ -76,13 +75,12 @@ while (( "$#" )); do
shift
done
-cd $TEST_ROOT_DIR
-
properties=(
-Dspark.kubernetes.test.sparkTgz=$SPARK_TGZ \
-Dspark.kubernetes.test.imageTag=$IMAGE_TAG \
-Dspark.kubernetes.test.imageRepo=$IMAGE_REPO \
- -Dspark.kubernetes.test.deployMode=$DEPLOY_MODE
+ -Dspark.kubernetes.test.deployMode=$DEPLOY_MODE \
+ -Dtest.include.tags=$INCLUDE_TAGS
)
if [ -n $NAMESPACE ];
@@ -105,9 +103,4 @@ then
properties=( ${properties[@]} -Dtest.exclude.tags=$EXCLUDE_TAGS )
fi
-if [ -n $INCLUDE_TAGS ];
-then
- properties=( ${properties[@]} -Dtest.include.tags=$INCLUDE_TAGS )
-fi
-
-../../../build/mvn integration-test ${properties[@]}
+$TEST_ROOT_DIR/build/mvn integration-test -f $TEST_ROOT_DIR/pom.xml -pl resource-managers/kubernetes/integration-tests -am -Pkubernetes -Phadoop-2.7 ${properties[@]}
http://git-wip-us.apache.org/repos/asf/spark/blob/fc0c8c97/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index 774c393..daabfaa 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -23,7 +23,7 @@ import java.util.regex.Pattern
import com.google.common.io.PatternFilenameFilter
import io.fabric8.kubernetes.api.model.Pod
-import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
+import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, Tag}
import org.scalatest.concurrent.{Eventually, PatienceConfiguration}
import org.scalatest.time.{Minutes, Seconds, Span}
import scala.collection.JavaConverters._
@@ -47,6 +47,7 @@ private[spark] class KubernetesSuite extends SparkFunSuite
private var containerLocalSparkDistroExamplesJar: String = _
private var appLocator: String = _
private var driverPodName: String = _
+ private val k8sTestTag = Tag("k8s")
override def beforeAll(): Unit = {
// The scalatest-maven-plugin gives system properties that are referenced but not set null
@@ -102,22 +103,22 @@ private[spark] class KubernetesSuite extends SparkFunSuite
deleteDriverPod()
}
- test("Run SparkPi with no resources") {
+ test("Run SparkPi with no resources", k8sTestTag) {
runSparkPiAndVerifyCompletion()
}
- test("Run SparkPi with a very long application name.") {
+ test("Run SparkPi with a very long application name.", k8sTestTag) {
sparkAppConf.set("spark.app.name", "long" * 40)
runSparkPiAndVerifyCompletion()
}
- test("Use SparkLauncher.NO_RESOURCE") {
+ test("Use SparkLauncher.NO_RESOURCE", k8sTestTag) {
sparkAppConf.setJars(Seq(containerLocalSparkDistroExamplesJar))
runSparkPiAndVerifyCompletion(
appResource = SparkLauncher.NO_RESOURCE)
}
- test("Run SparkPi with a master URL without a scheme.") {
+ test("Run SparkPi with a master URL without a scheme.", k8sTestTag) {
val url = kubernetesTestComponents.kubernetesClient.getMasterUrl
val k8sMasterUrl = if (url.getPort < 0) {
s"k8s://${url.getHost}"
@@ -128,11 +129,11 @@ private[spark] class KubernetesSuite extends SparkFunSuite
runSparkPiAndVerifyCompletion()
}
- test("Run SparkPi with an argument.") {
+ test("Run SparkPi with an argument.", k8sTestTag) {
runSparkPiAndVerifyCompletion(appArgs = Array("5"))
}
- test("Run SparkPi with custom labels, annotations, and environment variables.") {
+ test("Run SparkPi with custom labels, annotations, and environment variables.", k8sTestTag) {
sparkAppConf
.set("spark.kubernetes.driver.label.label1", "label1-value")
.set("spark.kubernetes.driver.label.label2", "label2-value")
@@ -158,21 +159,21 @@ private[spark] class KubernetesSuite extends SparkFunSuite
})
}
- test("Run extraJVMOptions check on driver") {
+ test("Run extraJVMOptions check on driver", k8sTestTag) {
sparkAppConf
.set("spark.driver.extraJavaOptions", "-Dspark.test.foo=spark.test.bar")
runSparkJVMCheckAndVerifyCompletion(
expectedJVMValue = Seq("(spark.test.foo,spark.test.bar)"))
}
- test("Run SparkRemoteFileTest using a remote data file") {
+ test("Run SparkRemoteFileTest using a remote data file", k8sTestTag) {
sparkAppConf
.set("spark.files", REMOTE_PAGE_RANK_DATA_FILE)
runSparkRemoteCheckAndVerifyCompletion(
appArgs = Array(REMOTE_PAGE_RANK_FILE_NAME))
}
- test("Run PySpark on simple pi.py example") {
+ test("Run PySpark on simple pi.py example", k8sTestTag) {
sparkAppConf
.set("spark.kubernetes.container.image", s"${getTestImageRepo}/spark-py:${getTestImageTag}")
runSparkApplicationAndVerifyCompletion(
@@ -186,7 +187,7 @@ private[spark] class KubernetesSuite extends SparkFunSuite
isJVM = false)
}
- test("Run PySpark with Python2 to test a pyfiles example") {
+ test("Run PySpark with Python2 to test a pyfiles example", k8sTestTag) {
sparkAppConf
.set("spark.kubernetes.container.image", s"${getTestImageRepo}/spark-py:${getTestImageTag}")
.set("spark.kubernetes.pyspark.pythonversion", "2")
@@ -204,7 +205,7 @@ private[spark] class KubernetesSuite extends SparkFunSuite
pyFiles = Some(PYSPARK_CONTAINER_TESTS))
}
- test("Run PySpark with Python3 to test a pyfiles example") {
+ test("Run PySpark with Python3 to test a pyfiles example", k8sTestTag) {
sparkAppConf
.set("spark.kubernetes.container.image", s"${getTestImageRepo}/spark-py:${getTestImageTag}")
.set("spark.kubernetes.pyspark.pythonversion", "3")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org