You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by td...@apache.org on 2015/07/30 09:46:40 UTC
spark git commit: [SPARK-9335] [TESTS] Enable Kinesis tests only when
files in extras/kinesis-asl are changed
Repository: spark
Updated Branches:
refs/heads/master 1221849f9 -> 76f2e393a
[SPARK-9335] [TESTS] Enable Kinesis tests only when files in extras/kinesis-asl are changed
Author: zsxwing <zs...@gmail.com>
Closes #7711 from zsxwing/SPARK-9335-test and squashes the following commits:
c13ec2f [zsxwing] environs -> environ
69c2865 [zsxwing] Merge remote-tracking branch 'origin/master' into SPARK-9335-test
ef84a08 [zsxwing] Revert "Modify the Kinesis project to trigger ENABLE_KINESIS_TESTS"
f691028 [zsxwing] Modify the Kinesis project to trigger ENABLE_KINESIS_TESTS
7618205 [zsxwing] Enable Kinesis tests only when files in extras/kinesis-asl are changed
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/76f2e393
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/76f2e393
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/76f2e393
Branch: refs/heads/master
Commit: 76f2e393a5fad0db8b56c4b8dad5ef686bf140a4
Parents: 1221849
Author: zsxwing <zs...@gmail.com>
Authored: Thu Jul 30 00:46:36 2015 -0700
Committer: Tathagata Das <ta...@gmail.com>
Committed: Thu Jul 30 00:46:36 2015 -0700
----------------------------------------------------------------------
dev/run-tests.py | 16 ++++++++++++++++
dev/sparktestsupport/modules.py | 14 ++++++++++++--
2 files changed, 28 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/76f2e393/dev/run-tests.py
----------------------------------------------------------------------
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 1f0d218..29420da 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -85,6 +85,13 @@ def identify_changed_files_from_git_commits(patch_sha, target_branch=None, targe
return [f for f in raw_output.split('\n') if f]
+def setup_test_environ(environ):
+ print("[info] Setup the following environment variables for tests: ")
+ for (k, v) in environ.items():
+ print("%s=%s" % (k, v))
+ os.environ[k] = v
+
+
def determine_modules_to_test(changed_modules):
"""
Given a set of modules that have changed, compute the transitive closure of those modules'
@@ -455,6 +462,15 @@ def main():
print("[info] Found the following changed modules:",
", ".join(x.name for x in changed_modules))
+ # setup environment variables
+ # note - the 'root' module doesn't collect environment variables for all modules. Because the
+ # environment variables should not be set if a module is not changed, even if running the 'root'
+ # module. So here we should use changed_modules rather than test_modules.
+ test_environ = {}
+ for m in changed_modules:
+ test_environ.update(m.environ)
+ setup_test_environ(test_environ)
+
test_modules = determine_modules_to_test(changed_modules)
# license checks
http://git-wip-us.apache.org/repos/asf/spark/blob/76f2e393/dev/sparktestsupport/modules.py
----------------------------------------------------------------------
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 3073d48..030d982 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -29,7 +29,7 @@ class Module(object):
changed.
"""
- def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=(),
+ def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=(), environ={},
sbt_test_goals=(), python_test_goals=(), blacklisted_python_implementations=(),
should_run_r_tests=False):
"""
@@ -43,6 +43,8 @@ class Module(object):
filename strings.
:param build_profile_flags: A set of profile flags that should be passed to Maven or SBT in
order to build and test this module (e.g. '-PprofileName').
+ :param environ: A dict of environment variables that should be set when files in this
+ module are changed.
:param sbt_test_goals: A set of SBT test goals for testing this module.
:param python_test_goals: A set of Python test goals for testing this module.
:param blacklisted_python_implementations: A set of Python implementations that are not
@@ -55,6 +57,7 @@ class Module(object):
self.source_file_prefixes = source_file_regexes
self.sbt_test_goals = sbt_test_goals
self.build_profile_flags = build_profile_flags
+ self.environ = environ
self.python_test_goals = python_test_goals
self.blacklisted_python_implementations = blacklisted_python_implementations
self.should_run_r_tests = should_run_r_tests
@@ -126,15 +129,22 @@ streaming = Module(
)
+# Don't set the dependencies because changes in other modules should not trigger Kinesis tests.
+# Kinesis tests depends on external Amazon kinesis service. We should run these tests only when
+# files in streaming_kinesis_asl are changed, so that if Kinesis experiences an outage, we don't
+# fail other PRs.
streaming_kinesis_asl = Module(
name="kinesis-asl",
- dependencies=[streaming],
+ dependencies=[],
source_file_regexes=[
"extras/kinesis-asl/",
],
build_profile_flags=[
"-Pkinesis-asl",
],
+ environ={
+ "ENABLE_KINESIS_TESTS": "1"
+ },
sbt_test_goals=[
"kinesis-asl/test",
]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org