You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lc...@apache.org on 2018/01/25 00:10:43 UTC
[beam] 01/04: Initial post-release snapshot test
This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 93f4bd4560a6f14c9aa88c9903d60bd327f37c42
Author: Alan Myrvold <al...@comcast.net>
AuthorDate: Wed Dec 13 01:26:42 2017 +0000
Initial post-release snapshot test
---
.../job_beam_PostRelease_NightlySnapshot.groovy | 43 +++++++
release/TestHelper.groovy | 139 +++++++++++++++++++++
release/quickstart-java-direct.groovy | 64 ++++++++++
3 files changed, 246 insertions(+)
diff --git a/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
new file mode 100644
index 0000000..57fa631
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This verifies the nightly snapshot build.
+// From https://repository.apache.org/content/groups/snapshots/org/apache/beam.
+job('beam_PostRelease_NightlySnapshot') {
+ description('Runs post release verification of the nightly snapshot.')
+
+ // Execute concurrent builds if necessary.
+ concurrentBuild()
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // This is a post-commit job that runs once per day, not for every push.
+ common_job_properties.setPostCommit(
+ delegate,
+ '0 11 * * *',
+ false,
+ 'dev@beam.apache.org')
+
+ steps {
+ // Run a quickstart from https://beam.apache.org/get-started/quickstart-java/
+ shell('cd ' + common_job_properties.checkoutDir + '/release && groovy quickstart-java-direct.groovy')
+ }
+}
diff --git a/release/TestHelper.groovy b/release/TestHelper.groovy
new file mode 100644
index 0000000..2a44791
--- /dev/null
+++ b/release/TestHelper.groovy
@@ -0,0 +1,139 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Helper functions to make writing a test similar to the quickstart
+ * instructions from https://beam.apache.org/get-started/quickstart-java/
+ */
+class TestHelper {
+
+ // Global state to maintain when running the steps
+ class var {
+ static File startDir
+ static File curDir
+ static String lastText
+ }
+
+ // Both documents the overal scenario and creates a clean temp directory
+ def describe(String desc) {
+ var.startDir = File.createTempDir()
+ var.startDir.deleteOnExit()
+ var.curDir = var.startDir
+ print "*****\n* Scenario: ${desc}\n*****\n"
+ }
+
+ // Just document the intention of a set of steps
+ def it(String desc) {
+ print "\n*****\n* Test: ${desc}\n*****\n\n"
+ }
+
+
+ // Run a command
+ public void run(String cmd) {
+ if (cmd.startsWith("cd ")) {
+ _chdir(cmd.substring(3))
+ } else if (cmd.startsWith("mvn ")) {
+ _mvn(cmd.substring(4))
+ } else {
+ _execute(cmd)
+ }
+ }
+
+ // Check for expected results in stdout of the last command
+ public void see(String expected) {
+ if (!var.lastText.contains(expected)) {
+ var.startDir.deleteDir()
+ println "Cannot find ${expected} in ${var.lastText}"
+ _error("Cannot find expected text")
+ }
+ println "Verified $expected"
+ }
+
+ // Cleanup and print success
+ public void done() {
+ var.startDir.deleteDir()
+ println "[SUCCESS]"
+ System.exit(0)
+ }
+
+ // Run a single command, capture output, verify return code is 0
+ private void _execute(String cmd) {
+ def shell = "sh -c cmd".split(' ')
+ shell[2] = cmd
+ def pb = new ProcessBuilder(shell)
+ pb.directory(var.curDir)
+ def proc = pb.start()
+ var.lastText = ""
+ proc.inputStream.eachLine {
+ println it
+ var.lastText += it + "\n";
+ }
+ var.lastText = var.lastText.trim()
+ proc.waitFor()
+ if (proc.exitValue() != 0) {
+ println var.lastText
+ _error("Failed command")
+ }
+ }
+
+ // Change directory
+ private void _chdir(String subdir) {
+ var.curDir = new File(var.curDir.absolutePath, subdir)
+ if (!var.curDir.exists()) {
+ _error("Directory ${var.curDir} not found")
+ }
+ _execute("pwd")
+ if (var.lastText != var.curDir.absolutePath) {
+ _error("Directory mismatch, ${var.lastText} != ${var.curDir.absolutePath}")
+
+ }
+ }
+
+ // Run a maven command, setting up a new local repository and a settings.xml with the snapshot repository
+ private void _mvn(String args) {
+ def m2 = new File(var.startDir, ".m2/repository")
+ m2.mkdirs()
+ def settings = new File(var.startDir, "settings.xml")
+ settings.write """
+ <settings>
+ <localRepository>${m2.absolutePath}</localRepository>
+ <profiles>
+ <profile>
+ <id>snapshot</id>
+ <repositories>
+ <repository>
+ <id>apache.snapshots</id>
+ <url>https://repository.apache.org/content/repositories/snapshots</url>
+ </repository>
+ </repositories>
+ </profile>
+ </profiles>
+ </settings>
+ """
+ def cmd = "mvn ${args} -s${settings.absolutePath} -Psnapshot -B"
+ _execute(cmd)
+ }
+
+ // Clean up and report error
+ private void _error(String text) {
+ var.startDir.deleteDir()
+ println "[ERROR] $text"
+ System.exit(1)
+ }
+}
diff --git a/release/quickstart-java-direct.groovy b/release/quickstart-java-direct.groovy
new file mode 100644
index 0000000..e00ae2e
--- /dev/null
+++ b/release/quickstart-java-direct.groovy
@@ -0,0 +1,64 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+t = new TestHelper()
+
+/*
+ * Run the direct quickstart from https://beam.apache.org/get-started/quickstart-java/
+ */
+
+t.describe 'Run Apache Beam Java SDK Quickstart - Direct'
+
+ t.it 'Gets the WordCount Code'
+ ver = "2.3.0-SNAPSHOT"
+
+ // Generate a maven project from the snapshot repository
+ t.run """mvn archetype:generate \
+ -DarchetypeGroupId=org.apache.beam \
+ -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \
+ -DarchetypeVersion=$ver \
+ -DgroupId=org.example \
+ -DartifactId=word-count-beam \
+ -Dversion="0.1" \
+ -Dpackage=org.apache.beam.examples \
+ -DinteractiveMode=false"""
+
+ // Check if it was generated
+ t.see "[INFO] BUILD SUCCESS"
+ t.run "cd word-count-beam"
+ t.run "ls"
+ t.see "pom.xml"
+ t.see "src"
+ t.run "ls src/main/java/org/apache/beam/examples/"
+ t.see "WordCount.java"
+
+ t.it 'Runs the WordCount Code with Direct runner'
+
+ // Run the workcount example with the direct runner
+ t.run """mvn compile exec:java \
+ -Dexec.mainClass=org.apache.beam.examples.WordCount \
+ -Dexec.args="--inputFile=pom.xml --output=counts" \
+ -Pdirect-runner"""
+
+ // Verify text from the pom.xml input file
+ t.run "grep Foundation counts*"
+ t.see "Foundation: 1"
+
+ // Clean up
+ t.done()
--
To stop receiving notification emails like this one, please contact
lcwik@apache.org.