You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/07/01 04:24:51 UTC
[1/2] incubator-beam git commit: Move Jenkins Python post commit
script to the repository.
Repository: incubator-beam
Updated Branches:
refs/heads/python-sdk fbe44ee83 -> df7ecd55f
Move Jenkins Python post commit script to the repository.
This is a copy of the script Jenkins post commit Python verify job is
running (with some cosmetic changes, license and instructions.) Moving
it to the repo so that we can track changes to it.
Jenkins will be updated to call this script instead of keeping a copy
of it as part of its configuration.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/dc319188
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/dc319188
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/dc319188
Branch: refs/heads/python-sdk
Commit: dc3191886e2440e3bae481548efa33eb8b474d0f
Parents: fbe44ee
Author: Ahmet Altay <al...@google.com>
Authored: Thu Jun 30 17:02:59 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Thu Jun 30 21:24:40 2016 -0700
----------------------------------------------------------------------
sdks/python/run_postcommit.sh | 87 ++++++++++++++++++++++++++++++++++++++
1 file changed, 87 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/dc319188/sdks/python/run_postcommit.sh
----------------------------------------------------------------------
diff --git a/sdks/python/run_postcommit.sh b/sdks/python/run_postcommit.sh
new file mode 100755
index 0000000..7af4e6c
--- /dev/null
+++ b/sdks/python/run_postcommit.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script will be run by Jenkins as a post commit test. In order to run
+# locally make the following changes:
+#
+# LOCAL_PATH -> Path of tox and virtualenv if you have them already installed.
+# GCS_LOCATION -> Temporary location to use for service tests.
+# PROJECT -> Project name to use for service jobs.
+#
+# Execute from the root of the repository: sdks/python/run_postcommit.sh
+
+set -e
+set -v
+
+# pip install --user installation location.
+LOCAL_PATH=$HOME/.local/bin/
+
+# INFRA does not install virtualenv
+pip install virtualenv --user
+
+# INFRA does not install tox
+pip install tox --user
+
+# Tox runs unit tests in a virtual environment
+# ${LOCAL_PATH}/tox -e py27 -c sdks/python/tox.ini
+
+# Virtualenv for the rest of the script to run setup & e2e tests
+${LOCAL_PATH}/virtualenv sdks/python
+. sdks/python/bin/activate
+cd sdks/python
+python setup.py install
+
+# Run wordcount in the Direct Runner and validate output.
+echo ">>> RUNNING DIRECT RUNNER py-wordcount"
+python -m apache_beam.examples.wordcount --output /tmp/py-wordcount-direct
+# TODO: check that output file is generated for Direct Runner.
+
+# Run wordcount on the service.
+
+# Where to store wordcount output.
+GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
+
+# Job name needs to be unique
+JOBNAME=py-wordcount-`date +%s`
+
+PROJECT=apache-beam-testing
+
+# Create a tarball
+python setup.py sdist
+
+# Run wordcount on the Google Cloud Dataflow service
+python -m apache_beam.examples.wordcount \
+--output $GCS_LOCATION/py-wordcount-cloud \
+--staging_location $GCS_LOCATION/staging-wordcount \
+--temp_location $GCS_LOCATION/temp-wordcount \
+--runner BlockingDataflowPipelineRunner \
+--job_name $JOBNAME \
+--project $PROJECT \
+--sdk_location dist/apache-beam-sdk-*.tar.gz \
+--num_workers 1 >> job_output 2>&1 || true;
+
+# Print full job output, validate correct, then remove it.
+echo ">>> JOB OUTPUT FOLLOWS"
+cat job_output
+
+# Grep will exit with status 1 if success message was not found.
+echo ">>> CHECKING JOB SUCCESS"
+grep JOB_STATE_DONE job_output
+
+# Remove the job output.
+rm job_output
[2/2] incubator-beam git commit: Closes #571
Posted by dh...@apache.org.
Closes #571
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/df7ecd55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/df7ecd55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/df7ecd55
Branch: refs/heads/python-sdk
Commit: df7ecd55f3cb42b1e81a863a7c3f544fa7fe4200
Parents: fbe44ee dc31918
Author: Dan Halperin <dh...@google.com>
Authored: Thu Jun 30 21:24:42 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Thu Jun 30 21:24:42 2016 -0700
----------------------------------------------------------------------
sdks/python/run_postcommit.sh | 87 ++++++++++++++++++++++++++++++++++++++
1 file changed, 87 insertions(+)
----------------------------------------------------------------------