You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ib...@apache.org on 2020/11/25 17:09:23 UTC
[beam] branch master updated: [BEAM-10278] Move pipeline jar tests
from job server projects to python/test-suites/portable.
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new f8e8daf [BEAM-10278] Move pipeline jar tests from job server projects to python/test-suites/portable.
new b002530 Merge pull request #13417 from ibzib/BEAM-10278
f8e8daf is described below
commit f8e8dafdf78f29e551baaf64dc612689a31610d5
Author: Kyle Weaver <kc...@google.com>
AuthorDate: Tue Nov 24 11:45:02 2020 -0800
[BEAM-10278] Move pipeline jar tests from job server projects to python/test-suites/portable.
---
.../job_PostCommit_PortableJar_Flink.groovy | 3 +-
.../job_PostCommit_PortableJar_Spark.groovy | 2 +-
runners/flink/job-server/flink_job_server.gradle | 66 ---------------------
.../test_flink_uber_jar.sh | 0
runners/portability/test_pipeline_jar.sh | 3 +-
runners/spark/job-server/build.gradle | 32 ----------
sdks/python/test-suites/portable/common.gradle | 69 +++++++++++++++++++++-
7 files changed, 73 insertions(+), 102 deletions(-)
diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy
index 7572669..856b7bf 100644
--- a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy
+++ b/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy
@@ -31,7 +31,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_PortableJar_Flink',
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:flink:1.10:job-server:testPipelineJar')
+ tasks(':sdks:python:test-suites:portable:py36:testPipelineJarFlinkRunner')
+ tasks(':sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunner')
commonJobProperties.setGradleSwitches(delegate)
}
}
diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy
index 8e4f1c5..93e58af 100644
--- a/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy
+++ b/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy
@@ -31,7 +31,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_PortableJar_Spark',
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':runners:spark:job-server:testPipelineJar')
+ tasks(':sdks:python:test-suites:portable:py37:testPipelineJarSparkRunner')
commonJobProperties.setGradleSwitches(delegate)
}
}
diff --git a/runners/flink/job-server/flink_job_server.gradle b/runners/flink/job-server/flink_job_server.gradle
index 99cd12d..130dd97 100644
--- a/runners/flink/job-server/flink_job_server.gradle
+++ b/runners/flink/job-server/flink_job_server.gradle
@@ -239,30 +239,6 @@ createCrossLanguageValidatesRunnerTask(
]
)
-def addTestJavaJarCreator(String pyVersion) {
- def pyBuildPath = pyVersion.startsWith("2") ? "2" : pyVersion.replaceAll("\\.", "")
- project.tasks.create(name: "testJavaJarCreatorPy${pyBuildPath}") {
- dependsOn shadowJar
- dependsOn ":sdks:python:container:py${pyBuildPath}:docker"
- doLast{
- exec {
- executable "sh"
- def options = [
- "--runner FlinkRunner",
- "--job_server_jar ${shadowJar.archivePath}",
- "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}",
- "--python_root_dir ${project.rootDir}/sdks/python",
- "--python_version ${pyVersion}",
- "--python_container_image ${project.docker_image_default_repo_root}/"
- + "${project.docker_image_default_repo_prefix}"
- + "python${pyVersion}_sdk:${project.sdk_version}",
- ]
- args "-c", "../../../portability/test_pipeline_jar.sh ${options.join(' ')}"
- }
- }
- }
-}
-
// miniCluster jar starts an embedded Flink cluster intended for use in testing.
task miniCluster(type: Jar, dependsOn: shadowJar) {
archiveBaseName = "${project.archivesBaseName}-mini-cluster"
@@ -281,45 +257,3 @@ task miniCluster(type: Jar, dependsOn: shadowJar) {
}
zip64 true // jar needs to contain more than 65535 files
}
-
-def addTestFlinkUberJarPy(String pyVersion, boolean saveMainSession) {
- def pyBuildPath = pyVersion.startsWith("2") ? "2" : pyVersion.replaceAll("\\.", "")
- project.tasks.create(name: "testFlinkUberJarPy${pyBuildPath}${saveMainSession ? 'saveMainSession' : ''}") {
- dependsOn miniCluster
- dependsOn shadowJar
- dependsOn ":sdks:python:container:py${pyBuildPath}:docker"
- doLast{
- exec {
- executable "sh"
- def options = [
- "--flink_job_server_jar ${shadowJar.archivePath}",
- "--flink_mini_cluster_jar ${miniCluster.archivePath}",
- "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}",
- "--python_root_dir ${project.rootDir}/sdks/python",
- "--python_version ${pyVersion}",
- "--python_container_image ${project.docker_image_default_repo_root}/"
- + "${project.docker_image_default_repo_prefix}"
- + "python${pyVersion}_sdk:${project.sdk_version}",
- ]
- if (saveMainSession) {
- options.add('--save_main_session')
- }
- args "-c", "../../job-server/test_flink_uber_jar.sh ${options.join(' ')}"
- }
- }
- }
-}
-
-["2.7", "3.5", "3.6", "3.7"].each{ pyVersion ->
- addTestJavaJarCreator(pyVersion)
- addTestFlinkUberJarPy(pyVersion, false)
- addTestFlinkUberJarPy(pyVersion, true)
-}
-
-task testPipelineJar() {
- dependsOn testJavaJarCreatorPy37
- dependsOn testFlinkUberJarPy36
- dependsOn testFlinkUberJarPy36saveMainSession
- dependsOn testFlinkUberJarPy37
- dependsOn testFlinkUberJarPy37saveMainSession
-}
diff --git a/runners/flink/job-server/test_flink_uber_jar.sh b/runners/portability/test_flink_uber_jar.sh
similarity index 100%
rename from runners/flink/job-server/test_flink_uber_jar.sh
rename to runners/portability/test_flink_uber_jar.sh
diff --git a/runners/portability/test_pipeline_jar.sh b/runners/portability/test_pipeline_jar.sh
index 308c92a..6b11bb1 100755
--- a/runners/portability/test_pipeline_jar.sh
+++ b/runners/portability/test_pipeline_jar.sh
@@ -68,6 +68,7 @@ command -v docker
docker -v
# Verify container has already been built
+echo "Checking for Docker image ${PYTHON_CONTAINER_IMAGE}"
docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
# Set up Python environment
@@ -108,7 +109,7 @@ else
fi
# Create the jar
-OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
+OUTPUT_JAR="test-pipeline-${RUNNER}-$(date +%Y%m%d-%H%M%S).jar"
(python -c "$PIPELINE_PY" \
--runner "$RUNNER" \
--"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
diff --git a/runners/spark/job-server/build.gradle b/runners/spark/job-server/build.gradle
index a653f3e..e08f8ae 100644
--- a/runners/spark/job-server/build.gradle
+++ b/runners/spark/job-server/build.gradle
@@ -236,35 +236,3 @@ createCrossLanguageValidatesRunnerTask(
"--experiments=beam_fn_api",
]
)
-
-def addTestJavaJarCreator(String pyVersion) {
- def pyBuildPath = pyVersion.startsWith("2") ? "2" : pyVersion.replaceAll("\\.", "")
- project.tasks.create(name: "testJavaJarCreatorPy${pyBuildPath}") {
- dependsOn shadowJar
- dependsOn ":sdks:python:container:py${pyBuildPath}:docker"
- doLast{
- exec {
- executable "sh"
- def options = [
- "--runner SparkRunner",
- "--job_server_jar ${shadowJar.archivePath}",
- "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}",
- "--python_root_dir ${project.rootDir}/sdks/python",
- "--python_version ${pyVersion}",
- "--python_container_image ${project.docker_image_default_repo_root}/"
- + "${project.docker_image_default_repo_prefix}"
- + "python${pyVersion}_sdk:${project.sdk_version}",
- ]
- args "-c", "../../portability/test_pipeline_jar.sh ${options.join(' ')}"
- }
- }
- }
-}
-
-["2.7", "3.5", "3.6", "3.7"].each { pyVersion ->
- addTestJavaJarCreator(pyVersion)
-}
-
-task testPipelineJar() {
- dependsOn testJavaJarCreatorPy37
-}
diff --git a/sdks/python/test-suites/portable/common.gradle b/sdks/python/test-suites/portable/common.gradle
index 0e3e956..d273996 100644
--- a/sdks/python/test-suites/portable/common.gradle
+++ b/sdks/python/test-suites/portable/common.gradle
@@ -19,7 +19,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
*/
def pythonRootDir = "${rootDir}/sdks/python"
-def pythonVersionSuffix = project.ext.pythonVersion == '2.7' ? '2' : project.ext.pythonVersion.replace('.', '')
+def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '')
ext {
pythonContainerTask = ":sdks:python:container:py${pythonVersionSuffix}:docker"
@@ -193,3 +193,70 @@ project.task("postCommitPy${pythonVersionSuffix}IT") {
}
}
}
+
+def addTestJavaJarCreator(String runner, Task jobServerJarTask) {
+ project.tasks.create(name: "testJavaJarCreator${runner}") {
+ dependsOn jobServerJarTask
+ dependsOn pythonContainerTask
+ doLast{
+ exec {
+ executable "sh"
+ def options = [
+ "--runner ${runner}",
+ "--job_server_jar ${jobServerJarTask.archivePath}",
+ "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}",
+ "--python_root_dir ${project.rootDir}/sdks/python",
+ "--python_version ${project.ext.pythonVersion}",
+ "--python_container_image ${project.docker_image_default_repo_root}/"
+ + "${project.docker_image_default_repo_prefix}"
+ + "python${project.ext.pythonVersion}_sdk:${project.sdk_version}",
+ ]
+ args "-c", "${project.rootDir}/runners/portability/test_pipeline_jar.sh ${options.join(' ')}"
+ }
+ }
+ }
+}
+
+// TODO(BEAM-11333) Update and test multiple Flink versions.
+addTestJavaJarCreator("FlinkRunner", tasks.getByPath(":runners:flink:1.10:job-server:shadowJar"))
+addTestJavaJarCreator("SparkRunner", tasks.getByPath(":runners:spark:job-server:shadowJar"))
+
+def addTestFlinkUberJar(boolean saveMainSession) {
+ project.tasks.create(name: "testUberJarFlinkRunner${saveMainSession ? 'SaveMainSession' : ''}") {
+ dependsOn ':runners:flink:1.10:job-server:shadowJar'
+ dependsOn ':runners:flink:1.10:job-server:miniCluster'
+ dependsOn pythonContainerTask
+ doLast{
+ exec {
+ executable "sh"
+ def options = [
+ "--flink_job_server_jar ${tasks.getByPath(':runners:flink:1.10:job-server:shadowJar').archivePath}",
+ "--flink_mini_cluster_jar ${tasks.getByPath(':runners:flink:1.10:job-server:miniCluster').archivePath}",
+ "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}",
+ "--python_root_dir ${project.rootDir}/sdks/python",
+ "--python_version ${project.ext.pythonVersion}",
+ "--python_container_image ${project.docker_image_default_repo_root}/"
+ + "${project.docker_image_default_repo_prefix}"
+ + "python${project.ext.pythonVersion}_sdk:${project.sdk_version}",
+ ]
+ if (saveMainSession) {
+ options.add('--save_main_session')
+ }
+ args "-c", "${project.rootDir}/runners/portability/test_flink_uber_jar.sh ${options.join(' ')}"
+ }
+ }
+ }
+}
+
+addTestFlinkUberJar(false)
+addTestFlinkUberJar(true)
+
+task testPipelineJarSparkRunner() {
+ dependsOn testJavaJarCreatorSparkRunner
+}
+
+task testPipelineJarFlinkRunner() {
+ dependsOn testJavaJarCreatorFlinkRunner
+ dependsOn testUberJarFlinkRunner
+ dependsOn testUberJarFlinkRunnerSaveMainSession
+}