You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2020/09/29 19:57:53 UTC
[beam] branch master updated: [BEAM-8024] Add JPMS E2E test (#12899)
This is an automated email from the ASF dual-hosted git repository.
kenn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 245cf2b [BEAM-8024] Add JPMS E2E test (#12899)
245cf2b is described below
commit 245cf2b8ccdca6b66269b4e473c1a8903631cb3f
Author: kileys <ki...@gmail.com>
AuthorDate: Tue Sep 29 12:57:24 2020 -0700
[BEAM-8024] Add JPMS E2E test (#12899)
---
.../jenkins/job_PostCommit_Java_Jpms_Java11.groovy | 49 ++++++++
.../src/main/resources/beam/checkstyle.xml | 7 ++
sdks/java/testing/jpms-tests/build.gradle | 56 +++++++++
.../jpms-tests/src/main/java/module-info.java | 22 ++++
.../org/apache/beam/sdk/jpmstests/WordCount.java | 137 +++++++++++++++++++++
.../apache/beam/sdk/jpmstests/package-info.java | 19 +++
.../java/org/apache/beam/sdk/jpmstests/JpmsIT.java | 68 ++++++++++
settings.gradle | 5 +-
8 files changed, 361 insertions(+), 2 deletions(-)
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Jpms_Java11.groovy
new file mode 100644
index 0000000..7849148
--- /dev/null
+++ b/.test-infra/jenkins/job_PostCommit_Java_Jpms_Java11.groovy
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CommonJobProperties as commonJobProperties
+import PostcommitJobBuilder
+
+
+// This job runs the Java postcommit tests, including the suite of integration
+// tests.
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Jpms_Java11', 'Run Jpms Java 11 PostCommit',
+ 'JPMS Java 11 Post Commit Tests', this) {
+
+ description('Runs JPMS tests using the Java 11 SDK.')
+
+ // Set common parameters.
+ commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
+
+ // Publish all test results to Jenkins
+ publishers {
+ archiveJunit('**/build/test-results/**/*.xml')
+ }
+
+ // Gradle goals for this job.
+ steps {
+ gradle {
+ rootBuildScriptDir(commonJobProperties.checkoutDir)
+ tasks(':sdks:java:testing:jpms-tests:integrationTest')
+ commonJobProperties.setGradleSwitches(delegate)
+ switches("-Dorg.gradle.java.home=${commonJobProperties.JAVA_11_HOME}")
+ // Specify maven home on Jenkins, needed by Maven archetype integration tests.
+ switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4')
+ }
+ }
+ }
diff --git a/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml b/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
index bdcee75..c168453 100644
--- a/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/checkstyle.xml
@@ -42,6 +42,13 @@ page at http://checkstyle.sourceforge.net/config.html -->
<property name="fileNamePattern" value=".*Tests\.java$" />
</module>
+ <!-- Skip check on unsupported file : module-info.java
+ https://checkstyle.sourceforge.io/config_filefilters.html
+ -->
+ <module name="BeforeExecutionExclusionFileFilter">
+ <property name="fileNamePattern" value="module\-info\.java$"/>
+ </module>
+
<!-- Check that every module has a package-info.java -->
<module name="JavadocPackage"/>
diff --git a/sdks/java/testing/jpms-tests/build.gradle b/sdks/java/testing/jpms-tests/build.gradle
new file mode 100644
index 0000000..5e707ee
--- /dev/null
+++ b/sdks/java/testing/jpms-tests/build.gradle
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+plugins {
+ id 'org.apache.beam.module'
+}
+javaVersion="1.11"
+applyJavaNature(
+ enableChecker:false,
+ exportJavadoc: false,
+ publish: false,
+ disableLintWarnings: ['requires-transitive-automatic', 'requires-automatic']
+)
+provideIntegrationTestingDependencies()
+enableJavaPerformanceTesting()
+
+description = "Apache Beam :: SDKs :: Java :: Testing :: JPMS Tests"
+ext.summary = "E2E test for Java 9 modules"
+
+dependencies {
+ compile project(path: ":sdks:java:core", configuration: "shadow")
+ compile project(path: ":runners:direct-java")
+ compile project(path: ":sdks:java:extensions:google-cloud-platform-core")
+
+ testCompile library.java.junit
+ testCompile library.java.hamcrest_core
+}
+
+// Activate module support
+// https://docs.gradle.org/current/samples/sample_java_modules_multi_project.html
+plugins.withType(JavaPlugin).configureEach{
+ java {
+ modularity.inferModulePath = true
+ }
+}
+
+// JPMS requires JDK > 8
+project.tasks.each {
+ it.onlyIf {
+ JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0
+ }
+}
diff --git a/sdks/java/testing/jpms-tests/src/main/java/module-info.java b/sdks/java/testing/jpms-tests/src/main/java/module-info.java
new file mode 100644
index 0000000..0f439b7
--- /dev/null
+++ b/sdks/java/testing/jpms-tests/src/main/java/module-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+module org.apache.beam.sdk.jpmstests {
+ exports org.apache.beam.sdk.jpmstests;
+
+ requires transitive org.apache.beam.sdk;
+}
diff --git a/sdks/java/testing/jpms-tests/src/main/java/org/apache/beam/sdk/jpmstests/WordCount.java b/sdks/java/testing/jpms-tests/src/main/java/org/apache/beam/sdk/jpmstests/WordCount.java
new file mode 100644
index 0000000..7bdf1f2
--- /dev/null
+++ b/sdks/java/testing/jpms-tests/src/main/java/org/apache/beam/sdk/jpmstests/WordCount.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.jpmstests;
+
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.options.Default;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+
+/** Duplicated from beam-examples-java to avoid dependency. */
+public class WordCount {
+
+ /**
+ * Concept #2: You can make your pipeline code less verbose by defining your DoFns statically out-
+ * of-line. This DoFn tokenizes lines of text into individual words; we pass it to a ParDo in the
+ * pipeline.
+ */
+ @SuppressWarnings("StringSplitter")
+ static class ExtractWordsFn extends DoFn<String, String> {
+ private final Counter emptyLines = Metrics.counter(ExtractWordsFn.class, "emptyLines");
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ if (c.element().trim().isEmpty()) {
+ emptyLines.inc();
+ }
+
+ // Split the line into words.
+ String[] words = c.element().split("[^\\p{L}]+");
+
+ // Output each word encountered into the output PCollection.
+ for (String word : words) {
+ if (!word.isEmpty()) {
+ c.output(word);
+ }
+ }
+ }
+ }
+
+ /** A SimpleFunction that converts a Word and Count into a printable string. */
+ public static class FormatAsTextFn extends SimpleFunction<KV<String, Long>, String> {
+ @Override
+ public String apply(KV<String, Long> input) {
+ return input.getKey() + ": " + input.getValue();
+ }
+ }
+
+ /**
+ * A PTransform that converts a PCollection containing lines of text into a PCollection of
+ * formatted word counts.
+ *
+ * <p>Concept #3: This is a custom composite transform that bundles two transforms (ParDo and
+ * Count) as a reusable PTransform subclass. Using composite transforms allows for easy reuse,
+ * modular testing, and an improved monitoring experience.
+ */
+ public static class CountWords
+ extends PTransform<PCollection<String>, PCollection<KV<String, Long>>> {
+ @Override
+ public PCollection<KV<String, Long>> expand(PCollection<String> lines) {
+
+ // Convert lines of text into individual words.
+ PCollection<String> words = lines.apply(ParDo.of(new ExtractWordsFn()));
+
+ // Count the number of times each word occurs.
+ return words.apply(Count.perElement());
+ }
+ }
+
+ /**
+ * Options supported by {@link WordCount}.
+ *
+ * <p>Concept #4: Defining your own configuration options. Here, you can add your own arguments to
+ * be processed by the command-line parser, and specify default values for them. You can then
+ * access the options values in your pipeline code.
+ *
+ * <p>Inherits standard configuration options.
+ */
+ public interface WordCountOptions extends PipelineOptions {
+ @Description("Path of the file to read from")
+ @Default.String("gs://beam-samples/shakespeare/kinglear.txt")
+ String getInputFile();
+
+ void setInputFile(String value);
+
+ @Description("Path of the file to write to")
+ String getOutput();
+
+ void setOutput(String value);
+ }
+
+ static void runWordCount(WordCountOptions options) {
+ Pipeline p = Pipeline.create(options);
+
+ // Concepts #2 and #3: Our pipeline applies the composite CountWords transform, and passes the
+ // static FormatAsTextFn() to the ParDo transform.
+ p.apply("ReadLines", TextIO.read().from(options.getInputFile()))
+ .apply(new CountWords())
+ .apply(MapElements.via(new FormatAsTextFn()))
+ .apply("WriteCounts", TextIO.write().to(options.getOutput()));
+
+ p.run().waitUntilFinish();
+ }
+
+ public static void main(String[] args) {
+ WordCountOptions options =
+ PipelineOptionsFactory.fromArgs(args).withValidation().as(WordCountOptions.class);
+
+ runWordCount(options);
+ }
+}
diff --git a/sdks/java/testing/jpms-tests/src/main/java/org/apache/beam/sdk/jpmstests/package-info.java b/sdks/java/testing/jpms-tests/src/main/java/org/apache/beam/sdk/jpmstests/package-info.java
new file mode 100644
index 0000000..46d7be8
--- /dev/null
+++ b/sdks/java/testing/jpms-tests/src/main/java/org/apache/beam/sdk/jpmstests/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/** Testing Java 9 modules. */
+package org.apache.beam.sdk.jpmstests;
diff --git a/sdks/java/testing/jpms-tests/src/test/java/org/apache/beam/sdk/jpmstests/JpmsIT.java b/sdks/java/testing/jpms-tests/src/test/java/org/apache/beam/sdk/jpmstests/JpmsIT.java
new file mode 100644
index 0000000..fe9e299
--- /dev/null
+++ b/sdks/java/testing/jpms-tests/src/test/java/org/apache/beam/sdk/jpmstests/JpmsIT.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.jpmstests;
+
+import static org.apache.beam.sdk.testing.FileChecksumMatcher.fileContentsHaveChecksum;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+import java.util.Date;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
+import org.apache.beam.sdk.jpmstests.WordCount.WordCountOptions;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestPipelineOptions;
+import org.apache.beam.sdk.util.NumberedShardedFile;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** End to end test of word count using JPMS. */
+@RunWith(JUnit4.class)
+public class JpmsIT {
+ private static final String DEFAULT_INPUT =
+ "gs://apache-beam-samples/shakespeare/winterstale-personae";
+ private static final String DEFAULT_OUTPUT_CHECKSUM = "ebf895e7324e8a3edc72e7bcc96fa2ba7f690def";
+ /** Options for the Jpms Integration test. */
+ public interface JpmsITOptions extends TestPipelineOptions, WordCountOptions {}
+
+ @BeforeClass
+ public static void setUp() {
+ PipelineOptionsFactory.register(TestPipelineOptions.class);
+ }
+
+ @Test
+ public void testE2EJpms() {
+ JpmsITOptions options = TestPipeline.testingPipelineOptions().as(JpmsITOptions.class);
+
+ options.setInputFile(DEFAULT_INPUT);
+ options.setOutput(
+ FileSystems.matchNewResource(options.getTempRoot(), true)
+ .resolve(
+ String.format("JpmsIT-%tF-%<tH-%<tM-%<tS-%<tL", new Date()),
+ StandardResolveOptions.RESOLVE_DIRECTORY)
+ .resolve("output", StandardResolveOptions.RESOLVE_DIRECTORY)
+ .resolve("results", StandardResolveOptions.RESOLVE_FILE)
+ .toString());
+ WordCount.runWordCount(options);
+ assertThat(
+ new NumberedShardedFile(options.getOutput() + "*-of-*"),
+ fileContentsHaveChecksum(DEFAULT_OUTPUT_CHECKSUM));
+ }
+}
diff --git a/settings.gradle b/settings.gradle
index b6d156b..f011970 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -164,13 +164,14 @@ include ":sdks:java:io:xml"
include ":sdks:java:io:synthetic"
include ":sdks:java:io:influxdb"
include ":sdks:java:javadoc"
-include ":sdks:java:testing:load-tests"
-include ":sdks:java:testing:test-utils"
include ":sdks:java:maven-archetypes:examples"
include ":sdks:java:maven-archetypes:starter"
include ":sdks:java:testing:nexmark"
include ":sdks:java:testing:expansion-service"
+include ":sdks:java:testing:jpms-tests"
include ":sdks:java:testing:kafka-service"
+include ":sdks:java:testing:load-tests"
+include ":sdks:java:testing:test-utils"
include ":sdks:java:testing:tpcds"
include ":sdks:python"
include ":sdks:python:apache_beam:testing:load_tests"