[beam] branch master updated: Porting Python PostCommit to Gradle. (#5146)

[pa...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 96e82f9  Porting Python PostCommit to Gradle. (#5146)
96e82f9 is described below

commit 96e82f9e51e2fca415efb47d090992b6367de8a3
Author: Pablo <pa...@users.noreply.github.com>
AuthorDate: Wed May 9 17:11:23 2018 -0700

    Porting Python PostCommit to Gradle. (#5146)
    
    * Porting Python PostCommit to Gradle.
---
 build.gradle                  |  9 ++------
 sdks/python/build.gradle      | 51 +++++++++++++++++++++++++++++++++++++++++++
 sdks/python/run_postcommit.sh | 27 -----------------------
 sdks/python/setup.py          |  4 ++++
 4 files changed, 57 insertions(+), 34 deletions(-)

diff --git a/build.gradle b/build.gradle
index b00cfa2..5fbf00f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -159,16 +159,11 @@ task goPostCommit() {
 
 task pythonPreCommit() {
   dependsOn ":rat"
-  dependsOn ":beam-sdks-python:check"
+  dependsOn ":beam-sdks-python:preCommit"
 }
 
 task pythonPostCommit() {
-  doLast {
-    exec {
-      executable 'sh'
-      args '-c', './sdks/python/run_postcommit.sh'
-    }
-  }
+  dependsOn ":beam-sdks-python:postCommit"
 }
 
 apply plugin: 'net.researchgate.release'
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index 7f981a9..df5a098 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -160,3 +160,54 @@ task cover(dependsOn: 'setupVirtualenv') {
   }
 }
 
+task preCommit() {
+  dependsOn "docs"
+  dependsOn "testCython"
+  dependsOn "testPython"
+  dependsOn "testGcp"
+  dependsOn "lint"
+}
+
+task installGcpTest(dependsOn: 'setupVirtualenv') {
+  doLast {
+    exec {
+      executable 'sh'
+      args '-c', ". ${envdir}/bin/activate && pip install -e .[gcp,test]"
+    }
+  }
+}
+
+task localWordCount(dependsOn: 'installGcpTest') {
+  doLast {
+    exec {
+      executable 'sh'
+      args '-c', ". ${envdir}/bin/activate && python -m apache_beam.examples.wordcount --output /tmp/py-wordcount-direct"
+      // TODO: Check that the output file is generated and runs.
+    }
+  }
+}
+
+task validatesRunnerTests(dependsOn: 'installGcpTest') {
+  doLast {
+    exec {
+      executable 'sh'
+      args '-c', ". ${envdir}/bin/activate && ./run_postcommit.sh"
+    }
+  }
+}
+
+task hdfsIntegrationTest(dependsOn: 'installGcpTest') {
+  doLast {
+    exec {
+      executable 'sh'
+      args '-c', ". ${envdir}/bin/activate && ./apache_beam/io/hdfs_integration_test/hdfs_integration_test.sh"
+    }
+  }
+}
+
+task postCommit() {
+  dependsOn "preCommit"
+  dependsOn "localWordCount"
+  dependsOn "hdfsIntegrationTest"
+  dependsOn "validatesRunnerTests"
+}
diff --git a/sdks/python/run_postcommit.sh b/sdks/python/run_postcommit.sh
index 951579f..582c432 100755
--- a/sdks/python/run_postcommit.sh
+++ b/sdks/python/run_postcommit.sh
@@ -28,30 +28,6 @@
 set -e
 set -v
 
-# pip install --user installation location.
-LOCAL_PATH=$HOME/.local/bin/
-
-# Remove any tox cache from previous workspace
-# TODO(udim): Remove this line and add '-r' to tox invocation instead.
-rm -rf sdks/python/target/.tox
-
-# INFRA does not install these packages
-pip install --user --upgrade virtualenv tox
-
-# Tox runs unit tests in a virtual environment
-${LOCAL_PATH}/tox -e ALL -c sdks/python/tox.ini
-
-# Virtualenv for the rest of the script to run setup & e2e tests
-${LOCAL_PATH}/virtualenv sdks/python
-. sdks/python/bin/activate
-cd sdks/python
-pip install -e .[gcp,test]
-
-# Run wordcount in the Direct Runner and validate output.
-echo ">>> RUNNING DIRECT RUNNER py-wordcount"
-python -m apache_beam.examples.wordcount --output /tmp/py-wordcount-direct
-# TODO: check that output file is generated for Direct Runner.
-
 # Run tests on the service.
 
 # Where to store integration test outputs.
@@ -81,6 +57,3 @@ python setup.py nosetests \
     --sdk_location=$SDK_LOCATION \
     --num_workers=1 \
     --sleep_secs=20"
-
-echo ">>> RUNNING DIRECT RUNNER hdfs_integration_test"
-apache_beam/io/hdfs_integration_test/hdfs_integration_test.sh
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 3641483..7b87dc6 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -30,6 +30,8 @@ import setuptools
 from pkg_resources import DistributionNotFound
 from pkg_resources import get_distribution
 from setuptools.command.build_py import build_py
+from setuptools.command.develop import develop
+from setuptools.command.egg_info import egg_info
 from setuptools.command.sdist import sdist
 from setuptools.command.test import test
 
@@ -200,6 +202,8 @@ setuptools.setup(
         ]},
     cmdclass={
         'build_py': generate_protos_first(build_py),
+        'develop': generate_protos_first(develop),
+        'egg_info': generate_protos_first(egg_info),
         'sdist': generate_protos_first(sdist),
         'test': generate_protos_first(test),
     },

-- 
To stop receiving notification emails like this one, please contact
pabloem@apache.org.