You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tg...@apache.org on 2017/04/12 20:52:45 UTC
[5/6] beam git commit: Move travis/jenkins folders in a test-infra
folder
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy b/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy
new file mode 100644
index 0000000..7c64f11
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PreCommit_Website_Stage.groovy
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// Defines a job.
+job('beam_PreCommit_Website_Stage') {
+ description('Stages the pull requests proposed for the Apache Beam ' +
+ 'website to a temporary location to ease reviews.')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelWebsiteJobProperties(delegate)
+
+ // Set pull request build trigger.
+ common_job_properties.setPreCommit(
+ delegate,
+ 'Automatic staging of pull requests',
+ '\nJenkins built the site at commit id ${ghprbActualCommit} with ' +
+ 'Jekyll and staged it [here](http://apache-beam-website-pull-' +
+ 'requests.storage.googleapis.com/${ghprbPullId}/index.html). ' +
+ 'Happy reviewing.\n\nNote that any previous site has been deleted. ' +
+ 'This staged site will be automatically deleted after its TTL ' +
+ 'expires. Push any commit to the pull request branch or re-trigger ' +
+ 'the build to get it staged again.')
+
+ steps {
+ // Run the following shell script as a build step.
+ shell '''
+ # Install RVM.
+ gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
+ 409B6B1796C275462A1703113804BB82D39DC0E3
+ \\curl -sSL https://get.rvm.io | bash
+ source /home/jenkins/.rvm/scripts/rvm
+
+ # Install Ruby.
+ RUBY_VERSION_NUM=2.3.0
+ rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
+
+ # Install Bundler gem
+ PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
+ GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
+ gem install bundler --user-install
+
+ # Install all needed gems.
+ bundle install --path ~/.gem/
+
+ # Remove current site if it exists.
+ GCS_PATH="gs://apache-beam-website-pull-requests/${ghprbPullId}/"
+ gsutil -m rm -r -f ${GCS_PATH} || true
+
+ # Build the new site with the baseurl specified.
+ rm -fr ./content/
+ bundle exec jekyll build --baseurl=/${ghprbPullId}
+
+ # Install BeautifulSoup HTML Parser for python.
+ pip install --user beautifulsoup4
+
+ # Fix links on staged website.
+ python .jenkins/append_index_html_to_internal_links.py
+
+ # Upload the new site.
+ gsutil -m cp -R ./content/* ${GCS_PATH}
+ '''.stripIndent().trim()
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy b/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy
new file mode 100644
index 0000000..421b58a
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PreCommit_Website_Test.groovy
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// Defines a job.
+job('beam_PreCommit_Website_Test') {
+ description('Runs tests on the pull requests proposed for the Apache Beam ' +
+ 'website.')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelWebsiteJobProperties(delegate)
+
+ // Execute concurrent builds. Multiple builds of this project may be executed
+ // in parallel. This is safe because this build does not require exclusive
+ // access to any shared resources.
+ concurrentBuild()
+
+ // Set pull request build trigger.
+ common_job_properties.setPreCommit(
+ delegate,
+ 'Test website (dead links, etc.)')
+
+ steps {
+ // Run the following shell script as a build step.
+ shell '''
+ # Install RVM.
+ gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
+ 409B6B1796C275462A1703113804BB82D39DC0E3
+ \\curl -sSL https://get.rvm.io | bash
+ source /home/jenkins/.rvm/scripts/rvm
+
+ # Install Ruby.
+ RUBY_VERSION_NUM=2.3.0
+ rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
+
+ # Install Bundler gem
+ PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
+ GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
+ gem install bundler --user-install
+
+ # Install all needed gems.
+ bundle install --path ~/.gem/
+
+ # Build the new site and test it.
+ rm -fr ./content/
+ bundle exec rake test
+ '''.stripIndent().trim()
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy b/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy
new file mode 100644
index 0000000..f2c3ff0
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_Release_NightlySnapshot.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the nightly snapshot build -- we use this to deploy a daily snapshot
+// to https://repository.apache.org/content/groups/snapshots/org/apache/beam.
+// Runs the postsubmit suite before deploying.
+mavenJob('beam_Release_NightlySnapshot') {
+ description('Runs a mvn clean deploy of the nightly snapshot.')
+
+ // Execute concurrent builds if necessary.
+ concurrentBuild()
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Set maven paramaters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // This is a post-commit job that runs once per day, not for every push.
+ common_job_properties.setPostCommit(
+ delegate,
+ '0 7 * * *',
+ false,
+ 'dev@beam.apache.org')
+
+ // Maven goals for this job.
+ goals('-B -e clean deploy -P release,dataflow-runner -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner" ]\'')
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_seed.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_seed.groovy b/.test-infra/jenkins/job_seed.groovy
new file mode 100644
index 0000000..2d1b07c
--- /dev/null
+++ b/.test-infra/jenkins/job_seed.groovy
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// Defines the seed job, which creates or updates all other Jenkins projects.
+job('beam_SeedJob') {
+ description('Automatically configures all Apache Beam Jenkins projects based' +
+ ' on Jenkins DSL groovy files checked into the code repository.')
+
+ previousNames('beam_SeedJob_Main')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // This is a post-commit job that runs once per day, not for every push.
+ common_job_properties.setPostCommit(
+ delegate,
+ '0 6 * * *',
+ false,
+ 'dev@beam.apache.org')
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Seed Job',
+ 'Run Seed Job')
+
+ steps {
+ dsl {
+ // A list or a glob of other groovy files to process.
+ external('.test-infra/jenkins/job_*.groovy')
+
+ // If a job is removed from the script, disable it (rather than deleting).
+ removeAction('DISABLE')
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/travis/README.md
----------------------------------------------------------------------
diff --git a/.test-infra/travis/README.md b/.test-infra/travis/README.md
new file mode 100644
index 0000000..526995a
--- /dev/null
+++ b/.test-infra/travis/README.md
@@ -0,0 +1,23 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Travis Scripts
+
+This directory contains scripts used for [Travis CI](https://travis-ci.org/apache/beam/)
+testing.
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/travis/settings.xml
----------------------------------------------------------------------
diff --git a/.test-infra/travis/settings.xml b/.test-infra/travis/settings.xml
new file mode 100644
index 0000000..e086aec
--- /dev/null
+++ b/.test-infra/travis/settings.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<settings>
+ <servers>
+ <server>
+ <id>central</id>
+ <configuration>
+ <httpConfiguration>
+ <all>
+ <connectionTimeout>1000</connectionTimeout>
+ </all>
+ </httpConfiguration>
+ <timeout>5000</timeout>
+ </configuration>
+ </server>
+ </servers>
+</settings>
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/travis/test_wordcount.sh
----------------------------------------------------------------------
diff --git a/.test-infra/travis/test_wordcount.sh b/.test-infra/travis/test_wordcount.sh
new file mode 100755
index 0000000..e059a35
--- /dev/null
+++ b/.test-infra/travis/test_wordcount.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script runs WordCount example locally in a few different ways.
+# Specifically, all combinations of:
+# a) using mvn exec, or java -cp with a bundled jar file;
+# b) input filename with no directory component, with a relative directory, or
+# with an absolute directory; AND
+# c) input filename containing wildcards or not.
+#
+# The one optional parameter is a path from the directory containing the script
+# to the directory containing the top-level (parent) pom.xml. If no parameter
+# is provided, the script assumes that directory is equal to the directory
+# containing the script itself.
+#
+# The exit-code of the script indicates success or a failure.
+
+set -e
+set -o pipefail
+
+PASS=1
+VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\[')
+JAR_FILE=examples/java/target/beam-examples-java-bundled-${VERSION}.jar
+
+function check_result_hash {
+ local name=$1
+ local outfile_prefix=$2
+ local expected=$3
+
+ local actual=$(LC_ALL=C sort $outfile_prefix-* | md5sum | awk '{print $1}' \
+ || LC_ALL=C sort $outfile_prefix-* | md5 -q) || exit 2 # OSX
+ if [[ "$actual" != "$expected" ]]
+ then
+ echo "FAIL $name: Output hash mismatch. Got $actual, expected $expected."
+ PASS=""
+ echo "head hexdump of actual:"
+ head $outfile_prefix-* | hexdump -c
+ else
+ echo "pass $name"
+ # Output files are left behind in /tmp
+ fi
+}
+
+function get_outfile_prefix {
+ local name=$1
+ # NOTE: mktemp on OSX doesn't support --tmpdir
+ mktemp -u "/tmp/$name.out.XXXXXXXXXX"
+}
+
+function run_via_mvn {
+ local name=$1
+ local input=$2
+ local expected_hash=$3
+
+ local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
+ local cmd='mvn exec:java -f pom.xml -pl examples/java \
+ -Dexec.mainClass=org.apache.beam.examples.WordCount \
+ -Dexec.args="--runner=DirectRunner --inputFile='"$input"' --output='"$outfile_prefix"'"'
+ echo "$name: Running $cmd" >&2
+ sh -c "$cmd"
+ check_result_hash "$name" "$outfile_prefix" "$expected_hash"
+}
+
+function run_bundled {
+ local name=$1
+ local input=$2
+ local expected_hash=$3
+
+ local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
+ local cmd='java -cp '"$JAR_FILE"' \
+ org.apache.beam.examples.WordCount \
+ --runner=DirectRunner \
+ --inputFile='"'$input'"' \
+ --output='"$outfile_prefix"
+ echo "$name: Running $cmd" >&2
+ sh -c "$cmd"
+ check_result_hash "$name" "$outfile_prefix" "$expected_hash"
+}
+
+function run_all_ways {
+ local name=$1
+ local input=$2
+ local expected_hash=$3
+
+ run_via_mvn ${name}a "$input" $expected_hash
+ check_for_jar_file
+ run_bundled ${name}b "$input" $expected_hash
+}
+
+function check_for_jar_file {
+ if [[ ! -f $JAR_FILE ]]
+ then
+ echo "Jar file $JAR_FILE not created" >&2
+ exit 2
+ fi
+}
+
+run_all_ways wordcount1 "LICENSE" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount2 "./LICENSE" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount3 "$PWD/LICENSE" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount4 "L*N?E*" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount5 "./LICE*N?E" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount6 "$PWD/*LIC?NSE" c5350a5ad4bb51e3e018612b4b044097
+
+if [[ ! "$PASS" ]]
+then
+ echo "One or more tests FAILED."
+ exit 1
+fi
+echo "All tests PASS"
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 2a75ebc..0b5d700 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -34,7 +34,7 @@ addons:
- python2.7
env:
global:
- - MAVEN_OVERRIDE="--settings=.travis/settings.xml"
+ - MAVEN_OVERRIDE="--settings=.test-infra/travis/settings.xml"
- MAVEN_CONTAINER_OVERRIDE="-DbeamSurefireArgline='-Xmx512m'"
matrix:
@@ -80,7 +80,7 @@ install:
script:
- if [ "$TEST_PYTHON" ]; then travis_retry $TOX_HOME/tox -c sdks/python/tox.ini; fi
- - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates --threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .travis/test_wordcount.sh; fi
+ - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates --threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .test-infra/travis/test_wordcount.sh; fi
cache:
directories:
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.travis/README.md
----------------------------------------------------------------------
diff --git a/.travis/README.md b/.travis/README.md
deleted file mode 100644
index 526995a..0000000
--- a/.travis/README.md
+++ /dev/null
@@ -1,23 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Travis Scripts
-
-This directory contains scripts used for [Travis CI](https://travis-ci.org/apache/beam/)
-testing.
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.travis/settings.xml
----------------------------------------------------------------------
diff --git a/.travis/settings.xml b/.travis/settings.xml
deleted file mode 100644
index e086aec..0000000
--- a/.travis/settings.xml
+++ /dev/null
@@ -1,33 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<settings>
- <servers>
- <server>
- <id>central</id>
- <configuration>
- <httpConfiguration>
- <all>
- <connectionTimeout>1000</connectionTimeout>
- </all>
- </httpConfiguration>
- <timeout>5000</timeout>
- </configuration>
- </server>
- </servers>
-</settings>
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.travis/test_wordcount.sh
----------------------------------------------------------------------
diff --git a/.travis/test_wordcount.sh b/.travis/test_wordcount.sh
deleted file mode 100755
index e059a35..0000000
--- a/.travis/test_wordcount.sh
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script runs WordCount example locally in a few different ways.
-# Specifically, all combinations of:
-# a) using mvn exec, or java -cp with a bundled jar file;
-# b) input filename with no directory component, with a relative directory, or
-# with an absolute directory; AND
-# c) input filename containing wildcards or not.
-#
-# The one optional parameter is a path from the directory containing the script
-# to the directory containing the top-level (parent) pom.xml. If no parameter
-# is provided, the script assumes that directory is equal to the directory
-# containing the script itself.
-#
-# The exit-code of the script indicates success or a failure.
-
-set -e
-set -o pipefail
-
-PASS=1
-VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\[')
-JAR_FILE=examples/java/target/beam-examples-java-bundled-${VERSION}.jar
-
-function check_result_hash {
- local name=$1
- local outfile_prefix=$2
- local expected=$3
-
- local actual=$(LC_ALL=C sort $outfile_prefix-* | md5sum | awk '{print $1}' \
- || LC_ALL=C sort $outfile_prefix-* | md5 -q) || exit 2 # OSX
- if [[ "$actual" != "$expected" ]]
- then
- echo "FAIL $name: Output hash mismatch. Got $actual, expected $expected."
- PASS=""
- echo "head hexdump of actual:"
- head $outfile_prefix-* | hexdump -c
- else
- echo "pass $name"
- # Output files are left behind in /tmp
- fi
-}
-
-function get_outfile_prefix {
- local name=$1
- # NOTE: mktemp on OSX doesn't support --tmpdir
- mktemp -u "/tmp/$name.out.XXXXXXXXXX"
-}
-
-function run_via_mvn {
- local name=$1
- local input=$2
- local expected_hash=$3
-
- local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
- local cmd='mvn exec:java -f pom.xml -pl examples/java \
- -Dexec.mainClass=org.apache.beam.examples.WordCount \
- -Dexec.args="--runner=DirectRunner --inputFile='"$input"' --output='"$outfile_prefix"'"'
- echo "$name: Running $cmd" >&2
- sh -c "$cmd"
- check_result_hash "$name" "$outfile_prefix" "$expected_hash"
-}
-
-function run_bundled {
- local name=$1
- local input=$2
- local expected_hash=$3
-
- local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
- local cmd='java -cp '"$JAR_FILE"' \
- org.apache.beam.examples.WordCount \
- --runner=DirectRunner \
- --inputFile='"'$input'"' \
- --output='"$outfile_prefix"
- echo "$name: Running $cmd" >&2
- sh -c "$cmd"
- check_result_hash "$name" "$outfile_prefix" "$expected_hash"
-}
-
-function run_all_ways {
- local name=$1
- local input=$2
- local expected_hash=$3
-
- run_via_mvn ${name}a "$input" $expected_hash
- check_for_jar_file
- run_bundled ${name}b "$input" $expected_hash
-}
-
-function check_for_jar_file {
- if [[ ! -f $JAR_FILE ]]
- then
- echo "Jar file $JAR_FILE not created" >&2
- exit 2
- fi
-}
-
-run_all_ways wordcount1 "LICENSE" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount2 "./LICENSE" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount3 "$PWD/LICENSE" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount4 "L*N?E*" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount5 "./LICE*N?E" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount6 "$PWD/*LIC?NSE" c5350a5ad4bb51e3e018612b4b044097
-
-if [[ ! "$PASS" ]]
-then
- echo "One or more tests FAILED."
- exit 1
-fi
-echo "All tests PASS"