You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by "ASF GitHub Bot (JIRA)" <ji...@apache.org> on 2018/01/25 00:11:00 UTC
[jira] [Commented] (BEAM-3339) Create post-release testing of the
nightly snapshots
[ https://issues.apache.org/jira/browse/BEAM-3339?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16338459#comment-16338459 ]
ASF GitHub Bot commented on BEAM-3339:
--------------------------------------
lukecwik closed pull request #4252: [BEAM-3339] Create a post-release test of the nightly snapshots
URL: https://github.com/apache/beam/pull/4252
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
new file mode 100644
index 00000000000..60abf9e6464
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostRelease_NightlySnapshot.groovy
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This verifies the nightly snapshot build.
+// From https://repository.apache.org/content/groups/snapshots/org/apache/beam.
+job('beam_PostRelease_NightlySnapshot') {
+ description('Runs post release verification of the nightly snapshot.')
+
+ // Execute concurrent builds if necessary.
+ concurrentBuild()
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ parameters {
+ stringParam('snapshot_version',
+ '2.3.0-SNAPSHOT',
+ 'Version of the repository snapshot to install')
+ stringParam('snapshot_url',
+ 'https://repository.apache.org/content/repositories/snapshots',
+ 'Repository URL to install from')
+ }
+
+ // This is a post-commit job that runs once per day, not for every push.
+ common_job_properties.setPostCommit(
+ delegate,
+ '0 11 * * *',
+ false,
+ 'dev@beam.apache.org')
+
+ steps {
+ // Run a quickstart from https://beam.apache.org/get-started/quickstart-java/
+ shell('cd ' + common_job_properties.checkoutDir + '/release && groovy quickstart-java-direct.groovy')
+ }
+}
diff --git a/release/TestScripts.groovy b/release/TestScripts.groovy
new file mode 100644
index 00000000000..d8fdcec0065
--- /dev/null
+++ b/release/TestScripts.groovy
@@ -0,0 +1,140 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Scripting functions to make writing a test similar to the quickstart
+ * instructions from https://beam.apache.org/get-started/quickstart-java/
+ */
+class TestScripts {
+
+ // Global state to maintain when running the steps
+ class var {
+ static File startDir
+ static File curDir
+ static String lastText
+ }
+
+ // Both documents the overal scenario and creates a clean temp directory
+ def describe(String desc) {
+ var.startDir = File.createTempDir()
+ var.startDir.deleteOnExit()
+ var.curDir = var.startDir
+ print "*****\n* Scenario: ${desc}\n*****\n"
+ }
+
+ // Just document the intention of a set of steps
+ def intent(String desc) {
+ print "\n*****\n* Test: ${desc}\n*****\n\n"
+ }
+
+
+ // Run a command
+ public void run(String cmd) {
+ if (cmd.startsWith("cd ")) {
+ _chdir(cmd.substring(3))
+ } else if (cmd.startsWith("mvn ")) {
+ _mvn(cmd.substring(4))
+ } else {
+ _execute(cmd)
+ }
+ }
+
+ // Check for expected results in stdout of the last command
+ public void see(String expected) {
+ if (!var.lastText.contains(expected)) {
+ var.startDir.deleteDir()
+ println "Cannot find ${expected} in ${var.lastText}"
+ _error("Cannot find expected text")
+ }
+ println "Verified $expected"
+ }
+
+ // Cleanup and print success
+ public void done() {
+ var.startDir.deleteDir()
+ println "[SUCCESS]"
+ System.exit(0)
+ }
+
+ // Run a single command, capture output, verify return code is 0
+ private void _execute(String cmd) {
+ def shell = "sh -c cmd".split(' ')
+ shell[2] = cmd
+ def pb = new ProcessBuilder(shell)
+ pb.directory(var.curDir)
+ def proc = pb.start()
+ var.lastText = ""
+ proc.inputStream.eachLine {
+ println it
+ var.lastText += it + "\n";
+ }
+ var.lastText = var.lastText.trim()
+ proc.waitFor()
+ if (proc.exitValue() != 0) {
+ println var.lastText
+ _error("Failed command")
+ }
+ }
+
+ // Change directory
+ private void _chdir(String subdir) {
+ var.curDir = new File(var.curDir.absolutePath, subdir)
+ if (!var.curDir.exists()) {
+ _error("Directory ${var.curDir} not found")
+ }
+ _execute("pwd")
+ if (var.lastText != var.curDir.absolutePath) {
+ _error("Directory mismatch, ${var.lastText} != ${var.curDir.absolutePath}")
+
+ }
+ }
+
+ // Run a maven command, setting up a new local repository and a settings.xml with the snapshot repository
+ private void _mvn(String args) {
+ def m2 = new File(var.startDir, ".m2/repository")
+ m2.mkdirs()
+ def settings = new File(var.startDir, "settings.xml")
+ def repo = System.env.snapshot_repository ?: "https://repository.apache.org/content/repositories/snapshots"
+ settings.write """
+ <settings>
+ <localRepository>${m2.absolutePath}</localRepository>
+ <profiles>
+ <profile>
+ <id>snapshot</id>
+ <repositories>
+ <repository>
+ <id>apache.snapshots</id>
+ <url>${repo}</url>
+ </repository>
+ </repositories>
+ </profile>
+ </profiles>
+ </settings>
+ """
+ def cmd = "mvn ${args} -s${settings.absolutePath} -Psnapshot -B"
+ _execute(cmd)
+ }
+
+ // Clean up and report error
+ private void _error(String text) {
+ var.startDir.deleteDir()
+ println "[ERROR] $text"
+ System.exit(1)
+ }
+}
diff --git a/release/quickstart-java-direct.groovy b/release/quickstart-java-direct.groovy
new file mode 100644
index 00000000000..2ac1558e5cc
--- /dev/null
+++ b/release/quickstart-java-direct.groovy
@@ -0,0 +1,64 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+t = new TestScripts()
+
+/*
+ * Run the direct quickstart from https://beam.apache.org/get-started/quickstart-java/
+ */
+
+t.describe 'Run Apache Beam Java SDK Quickstart - Direct'
+
+ t.intent 'Gets the WordCount Code'
+ ver = System.env.snapshot_version ?: "2.3.0-SNAPSHOT"
+
+ // Generate a maven project from the snapshot repository
+ t.run """mvn archetype:generate \
+ -DarchetypeGroupId=org.apache.beam \
+ -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \
+ -DarchetypeVersion=$ver \
+ -DgroupId=org.example \
+ -DartifactId=word-count-beam \
+ -Dversion="0.1" \
+ -Dpackage=org.apache.beam.examples \
+ -DinteractiveMode=false"""
+
+ // Check if it was generated
+ t.see "[INFO] BUILD SUCCESS"
+ t.run "cd word-count-beam"
+ t.run "ls"
+ t.see "pom.xml"
+ t.see "src"
+ t.run "ls src/main/java/org/apache/beam/examples/"
+ t.see "WordCount.java"
+
+ t.intent 'Runs the WordCount Code with Direct runner'
+
+ // Run the workcount example with the direct runner
+ t.run """mvn compile exec:java \
+ -Dexec.mainClass=org.apache.beam.examples.WordCount \
+ -Dexec.args="--inputFile=pom.xml --output=counts" \
+ -Pdirect-runner"""
+
+ // Verify text from the pom.xml input file
+ t.run "grep Foundation counts*"
+ t.see "Foundation: 1"
+
+ // Clean up
+ t.done()
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
> Create post-release testing of the nightly snapshots
> ----------------------------------------------------
>
> Key: BEAM-3339
> URL: https://issues.apache.org/jira/browse/BEAM-3339
> Project: Beam
> Issue Type: Improvement
> Components: testing
> Reporter: Alan Myrvold
> Assignee: Jason Kuster
> Priority: Major
>
> The nightly java snapshots in https://repository.apache.org/content/groups/snapshots/org/apache/beam should be verified by following the https://beam.apache.org/get-started/quickstart-java/ instructions, to verify that the release is usable.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)