You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2017/04/28 23:24:01 UTC
[1/2] beam git commit: Retire Travis-CI coverage in Apache Beam
Repository: beam
Updated Branches:
refs/heads/master 7478da997 -> 29b828235
Retire Travis-CI coverage in Apache Beam
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/15d8eacc
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/15d8eacc
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/15d8eacc
Branch: refs/heads/master
Commit: 15d8eacc8514197ca18289fe4b88ef245dd3379e
Parents: 7478da9
Author: Davor Bonaci <da...@google.com>
Authored: Fri Apr 28 14:19:09 2017 -0700
Committer: Davor Bonaci <da...@google.com>
Committed: Fri Apr 28 16:23:30 2017 -0700
----------------------------------------------------------------------
.test-infra/jenkins/test_wordcount.sh | 125 +++++++++++++++++++++++++++++
.test-infra/travis/README.md | 23 ------
.test-infra/travis/settings.xml | 33 --------
.test-infra/travis/test_wordcount.sh | 125 -----------------------------
.travis.yml | 93 ---------------------
README.md | 3 +-
6 files changed, 126 insertions(+), 276 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/15d8eacc/.test-infra/jenkins/test_wordcount.sh
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/test_wordcount.sh b/.test-infra/jenkins/test_wordcount.sh
new file mode 100755
index 0000000..e059a35
--- /dev/null
+++ b/.test-infra/jenkins/test_wordcount.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script runs WordCount example locally in a few different ways.
+# Specifically, all combinations of:
+# a) using mvn exec, or java -cp with a bundled jar file;
+# b) input filename with no directory component, with a relative directory, or
+# with an absolute directory; AND
+# c) input filename containing wildcards or not.
+#
+# The one optional parameter is a path from the directory containing the script
+# to the directory containing the top-level (parent) pom.xml. If no parameter
+# is provided, the script assumes that directory is equal to the directory
+# containing the script itself.
+#
+# The exit-code of the script indicates success or a failure.
+
+set -e
+set -o pipefail
+
+PASS=1
+VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\[')
+JAR_FILE=examples/java/target/beam-examples-java-bundled-${VERSION}.jar
+
+function check_result_hash {
+ local name=$1
+ local outfile_prefix=$2
+ local expected=$3
+
+ local actual=$(LC_ALL=C sort $outfile_prefix-* | md5sum | awk '{print $1}' \
+ || LC_ALL=C sort $outfile_prefix-* | md5 -q) || exit 2 # OSX
+ if [[ "$actual" != "$expected" ]]
+ then
+ echo "FAIL $name: Output hash mismatch. Got $actual, expected $expected."
+ PASS=""
+ echo "head hexdump of actual:"
+ head $outfile_prefix-* | hexdump -c
+ else
+ echo "pass $name"
+ # Output files are left behind in /tmp
+ fi
+}
+
+function get_outfile_prefix {
+ local name=$1
+ # NOTE: mktemp on OSX doesn't support --tmpdir
+ mktemp -u "/tmp/$name.out.XXXXXXXXXX"
+}
+
+function run_via_mvn {
+ local name=$1
+ local input=$2
+ local expected_hash=$3
+
+ local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
+ local cmd='mvn exec:java -f pom.xml -pl examples/java \
+ -Dexec.mainClass=org.apache.beam.examples.WordCount \
+ -Dexec.args="--runner=DirectRunner --inputFile='"$input"' --output='"$outfile_prefix"'"'
+ echo "$name: Running $cmd" >&2
+ sh -c "$cmd"
+ check_result_hash "$name" "$outfile_prefix" "$expected_hash"
+}
+
+function run_bundled {
+ local name=$1
+ local input=$2
+ local expected_hash=$3
+
+ local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
+ local cmd='java -cp '"$JAR_FILE"' \
+ org.apache.beam.examples.WordCount \
+ --runner=DirectRunner \
+ --inputFile='"'$input'"' \
+ --output='"$outfile_prefix"
+ echo "$name: Running $cmd" >&2
+ sh -c "$cmd"
+ check_result_hash "$name" "$outfile_prefix" "$expected_hash"
+}
+
+function run_all_ways {
+ local name=$1
+ local input=$2
+ local expected_hash=$3
+
+ run_via_mvn ${name}a "$input" $expected_hash
+ check_for_jar_file
+ run_bundled ${name}b "$input" $expected_hash
+}
+
+function check_for_jar_file {
+ if [[ ! -f $JAR_FILE ]]
+ then
+ echo "Jar file $JAR_FILE not created" >&2
+ exit 2
+ fi
+}
+
+run_all_ways wordcount1 "LICENSE" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount2 "./LICENSE" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount3 "$PWD/LICENSE" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount4 "L*N?E*" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount5 "./LICE*N?E" c5350a5ad4bb51e3e018612b4b044097
+run_all_ways wordcount6 "$PWD/*LIC?NSE" c5350a5ad4bb51e3e018612b4b044097
+
+if [[ ! "$PASS" ]]
+then
+ echo "One or more tests FAILED."
+ exit 1
+fi
+echo "All tests PASS"
http://git-wip-us.apache.org/repos/asf/beam/blob/15d8eacc/.test-infra/travis/README.md
----------------------------------------------------------------------
diff --git a/.test-infra/travis/README.md b/.test-infra/travis/README.md
deleted file mode 100644
index 526995a..0000000
--- a/.test-infra/travis/README.md
+++ /dev/null
@@ -1,23 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Travis Scripts
-
-This directory contains scripts used for [Travis CI](https://travis-ci.org/apache/beam/)
-testing.
http://git-wip-us.apache.org/repos/asf/beam/blob/15d8eacc/.test-infra/travis/settings.xml
----------------------------------------------------------------------
diff --git a/.test-infra/travis/settings.xml b/.test-infra/travis/settings.xml
deleted file mode 100644
index e086aec..0000000
--- a/.test-infra/travis/settings.xml
+++ /dev/null
@@ -1,33 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<settings>
- <servers>
- <server>
- <id>central</id>
- <configuration>
- <httpConfiguration>
- <all>
- <connectionTimeout>1000</connectionTimeout>
- </all>
- </httpConfiguration>
- <timeout>5000</timeout>
- </configuration>
- </server>
- </servers>
-</settings>
http://git-wip-us.apache.org/repos/asf/beam/blob/15d8eacc/.test-infra/travis/test_wordcount.sh
----------------------------------------------------------------------
diff --git a/.test-infra/travis/test_wordcount.sh b/.test-infra/travis/test_wordcount.sh
deleted file mode 100755
index e059a35..0000000
--- a/.test-infra/travis/test_wordcount.sh
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script runs WordCount example locally in a few different ways.
-# Specifically, all combinations of:
-# a) using mvn exec, or java -cp with a bundled jar file;
-# b) input filename with no directory component, with a relative directory, or
-# with an absolute directory; AND
-# c) input filename containing wildcards or not.
-#
-# The one optional parameter is a path from the directory containing the script
-# to the directory containing the top-level (parent) pom.xml. If no parameter
-# is provided, the script assumes that directory is equal to the directory
-# containing the script itself.
-#
-# The exit-code of the script indicates success or a failure.
-
-set -e
-set -o pipefail
-
-PASS=1
-VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\[')
-JAR_FILE=examples/java/target/beam-examples-java-bundled-${VERSION}.jar
-
-function check_result_hash {
- local name=$1
- local outfile_prefix=$2
- local expected=$3
-
- local actual=$(LC_ALL=C sort $outfile_prefix-* | md5sum | awk '{print $1}' \
- || LC_ALL=C sort $outfile_prefix-* | md5 -q) || exit 2 # OSX
- if [[ "$actual" != "$expected" ]]
- then
- echo "FAIL $name: Output hash mismatch. Got $actual, expected $expected."
- PASS=""
- echo "head hexdump of actual:"
- head $outfile_prefix-* | hexdump -c
- else
- echo "pass $name"
- # Output files are left behind in /tmp
- fi
-}
-
-function get_outfile_prefix {
- local name=$1
- # NOTE: mktemp on OSX doesn't support --tmpdir
- mktemp -u "/tmp/$name.out.XXXXXXXXXX"
-}
-
-function run_via_mvn {
- local name=$1
- local input=$2
- local expected_hash=$3
-
- local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
- local cmd='mvn exec:java -f pom.xml -pl examples/java \
- -Dexec.mainClass=org.apache.beam.examples.WordCount \
- -Dexec.args="--runner=DirectRunner --inputFile='"$input"' --output='"$outfile_prefix"'"'
- echo "$name: Running $cmd" >&2
- sh -c "$cmd"
- check_result_hash "$name" "$outfile_prefix" "$expected_hash"
-}
-
-function run_bundled {
- local name=$1
- local input=$2
- local expected_hash=$3
-
- local outfile_prefix="$(get_outfile_prefix "$name")" || exit 2
- local cmd='java -cp '"$JAR_FILE"' \
- org.apache.beam.examples.WordCount \
- --runner=DirectRunner \
- --inputFile='"'$input'"' \
- --output='"$outfile_prefix"
- echo "$name: Running $cmd" >&2
- sh -c "$cmd"
- check_result_hash "$name" "$outfile_prefix" "$expected_hash"
-}
-
-function run_all_ways {
- local name=$1
- local input=$2
- local expected_hash=$3
-
- run_via_mvn ${name}a "$input" $expected_hash
- check_for_jar_file
- run_bundled ${name}b "$input" $expected_hash
-}
-
-function check_for_jar_file {
- if [[ ! -f $JAR_FILE ]]
- then
- echo "Jar file $JAR_FILE not created" >&2
- exit 2
- fi
-}
-
-run_all_ways wordcount1 "LICENSE" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount2 "./LICENSE" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount3 "$PWD/LICENSE" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount4 "L*N?E*" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount5 "./LICE*N?E" c5350a5ad4bb51e3e018612b4b044097
-run_all_ways wordcount6 "$PWD/*LIC?NSE" c5350a5ad4bb51e3e018612b4b044097
-
-if [[ ! "$PASS" ]]
-then
- echo "One or more tests FAILED."
- exit 1
-fi
-echo "All tests PASS"
http://git-wip-us.apache.org/repos/asf/beam/blob/15d8eacc/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 0b5d700..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,93 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-language: java
-
-sudo: false
-
-notifications:
- email:
- # Group email notifications are disabled for now, since we cannot do it on a per-branch basis.
- # Right now, it would trigger a notification for each fork, which generates a lot of spam.
- # recipients:
- # - dataflow-sdk-build-notifications+travis@google.com
- on_success: change
- on_failure: always
-
-addons:
- apt:
- packages:
- - python2.7
-env:
- global:
- - MAVEN_OVERRIDE="--settings=.test-infra/travis/settings.xml"
- - MAVEN_CONTAINER_OVERRIDE="-DbeamSurefireArgline='-Xmx512m'"
-
-matrix:
- include:
- # On OSX, run with default JDK only.
- - os: osx
-
- # On Linux, run with specific JDKs only.
- - os: linux
- env: CUSTOM_JDK="oraclejdk8" MAVEN_OVERRIDE="$MAVEN_OVERRIDE $MAVEN_CONTAINER_OVERRIDE"
- - os: linux
- env: CUSTOM_JDK="oraclejdk7" MAVEN_OVERRIDE="$MAVEN_OVERRIDE $MAVEN_CONTAINER_OVERRIDE"
- - os: linux
- env: CUSTOM_JDK="openjdk7" MAVEN_OVERRIDE="$MAVEN_OVERRIDE $MAVEN_CONTAINER_OVERRIDE"
- - os: linux
- env: MAVEN_OVERRIDE="-Peclipse-jdt -DskipTests $MAVEN_OVERRIDE $MAVEN_CONTAINER_OVERRIDE" CUSTOM_JDK="oraclejdk8"
-
- # Python SDK tests.
- - os: osx
- env: TEST_PYTHON="1"
- - os: linux
- env: TEST_PYTHON="1"
-
-before_install:
- # The -XX:+TieredCompilation -XX:TieredStopAtLevel=1 JVM options enable
- # tiered compilation to make the JVM startup times faster during the tests.
- - echo 'MAVEN_OPTS="$MAVEN_OPTS -Xmx1024m -XX:MaxPermSize=512m -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+BytecodeVerificationLocal"' >> ~/.mavenrc
- - echo $'MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\'T\'HH:mm:ss.SSS"' >> ~/.mavenrc
- - cat ~/.mavenrc
- - if [ "$TRAVIS_OS_NAME" == "osx" ]; then export JAVA_HOME=$(/usr/libexec/java_home); fi
- - if [ "$TRAVIS_OS_NAME" == "linux" ]; then jdk_switcher use "$CUSTOM_JDK"; fi
- - export BEAM_SUREFIRE_ARGLINE="-Xmx512m"
- # Python SDK environment settings.
- - if [ "$TRAVIS_OS_NAME" == "osx" ]; then export TOX_HOME=$HOME/Library/Python/2.7/bin; fi
- - if [ "$TRAVIS_OS_NAME" == "linux" ]; then export TOX_HOME=$HOME/.local/bin; fi
-
-install:
- - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn -B install clean -U -DskipTests=true; fi
- - if [ "$TEST_PYTHON" ] && pip list | grep tox; then TOX_FILE=`which tox` ; export TOX_HOME=`dirname $TOX_FILE`; fi
- - if [ "$TEST_PYTHON" ] && ! pip list | grep tox; then travis_retry pip install tox --user; fi
- # Removing this here protects from inadvertent caching
- - rm -rf "$HOME/.m2/repository/org/apache/beam"
-
-script:
- - if [ "$TEST_PYTHON" ]; then travis_retry $TOX_HOME/tox -c sdks/python/tox.ini; fi
- - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn --batch-mode --update-snapshots --no-snapshot-updates --threads 1C $MAVEN_OVERRIDE install && travis_retry bash -ex .test-infra/travis/test_wordcount.sh; fi
-
-cache:
- directories:
- - $HOME/.m2/repository
-
-before_cache:
- # Removing here increases cache hits (makes the above
- # rm in `install` redundant unless our config has a bug,
- # but it will be idempotent)
- - rm -rf "$HOME/.m2/repository/org/apache/beam"
http://git-wip-us.apache.org/repos/asf/beam/blob/15d8eacc/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 23768ce..1202c54 100644
--- a/README.md
+++ b/README.md
@@ -23,8 +23,7 @@
## Status
-[](https://travis-ci.org/apache/beam?branch=master)
-[](https://builds.apache.org/job/beam_PostCommit_MavenVerify/)
+[](https://builds.apache.org/job/beam_PostCommit_Java_MavenInstall/)
[](https://coveralls.io/github/apache/beam?branch=master)
## Overview
[2/2] beam git commit: This closes #2769
Posted by da...@apache.org.
This closes #2769
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/29b82823
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/29b82823
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/29b82823
Branch: refs/heads/master
Commit: 29b828235cc4c901c0dc93ff96bcd3d8f494c18c
Parents: 7478da9 15d8eac
Author: Davor Bonaci <da...@google.com>
Authored: Fri Apr 28 16:23:52 2017 -0700
Committer: Davor Bonaci <da...@google.com>
Committed: Fri Apr 28 16:23:52 2017 -0700
----------------------------------------------------------------------
.test-infra/jenkins/test_wordcount.sh | 125 +++++++++++++++++++++++++++++
.test-infra/travis/README.md | 23 ------
.test-infra/travis/settings.xml | 33 --------
.test-infra/travis/test_wordcount.sh | 125 -----------------------------
.travis.yml | 93 ---------------------
README.md | 3 +-
6 files changed, 126 insertions(+), 276 deletions(-)
----------------------------------------------------------------------