You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2019/05/20 14:53:57 UTC
[spark] branch master updated: [SPARK-27402][INFRA][FOLLOW-UP]
Exclude 'hive-thriftserver' in modules to test for hadoop3.2 for now
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b7bf4fd [SPARK-27402][INFRA][FOLLOW-UP] Exclude 'hive-thriftserver' in modules to test for hadoop3.2 for now
b7bf4fd is described below
commit b7bf4fd12350d6c556cdf37b13ad2609cd6c35d4
Author: HyukjinKwon <gu...@apache.org>
AuthorDate: Mon May 20 07:53:19 2019 -0700
[SPARK-27402][INFRA][FOLLOW-UP] Exclude 'hive-thriftserver' in modules to test for hadoop3.2 for now
## What changes were proposed in this pull request?
This PR excludes 'hive-thriftserver' in modules to test for hadoop3.2 for now as well
## How was this patch tested?
Manually tested via `run-tests.py`
Closes #24644 from HyukjinKwon/SPARK-27402.
Authored-by: HyukjinKwon <gu...@apache.org>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
dev/run-tests.py | 9 ++++++++-
dev/sparktestsupport/modules.py | 21 ++++++++++-----------
2 files changed, 18 insertions(+), 12 deletions(-)
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 5211aea..592fbee 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -112,6 +112,7 @@ def determine_modules_to_test(changed_modules):
['graphx', 'examples']
>>> x = [x.name for x in determine_modules_to_test([modules.sql])]
>>> x # doctest: +NORMALIZE_WHITESPACE
+ ... # doctest: +SKIP
['sql', 'avro', 'hive', 'mllib', 'sql-kafka-0-10', 'examples', 'hive-thriftserver',
'pyspark-sql', 'repl', 'sparkr', 'pyspark-mllib', 'pyspark-ml']
"""
@@ -122,9 +123,15 @@ def determine_modules_to_test(changed_modules):
# If we need to run all of the tests, then we should short-circuit and return 'root'
if modules.root in modules_to_test:
return [modules.root]
- return toposort_flatten(
+ changed_modules = toposort_flatten(
{m: set(m.dependencies).intersection(modules_to_test) for m in modules_to_test}, sort=True)
+ # TODO: Skip hive-thriftserver module for hadoop-3.2. remove this once hadoop-3.2 support it
+ if modules.hadoop_version == "hadoop3.2":
+ changed_modules = [m for m in changed_modules if m.name != "hive-thriftserver"]
+
+ return changed_modules
+
def determine_tags_to_exclude(changed_modules):
tags = []
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 812c2ef..7fe3b1b 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -21,12 +21,6 @@ import itertools
import re
import os
-if os.environ.get("AMPLAB_JENKINS"):
- hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop2.7")
-else:
- hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.7")
-print("[info] Choosing supported modules with Hadoop profile", hadoop_version)
-
all_modules = []
@@ -80,11 +74,7 @@ class Module(object):
self.dependent_modules = set()
for dep in dependencies:
dep.dependent_modules.add(self)
- # TODO: Skip hive-thriftserver module for hadoop-3.2. remove this once hadoop-3.2 support it
- if name == "hive-thriftserver" and hadoop_version == "hadoop3.2":
- print("[info] Skip unsupported module:", name)
- else:
- all_modules.append(self)
+ all_modules.append(self)
def contains_file(self, filename):
return any(re.match(p, filename) for p in self.source_file_prefixes)
@@ -568,6 +558,15 @@ spark_ganglia_lgpl = Module(
]
)
+# TODO: Skip hive-thriftserver module for hadoop-3.2. remove this once hadoop-3.2 support it
+if os.environ.get("AMPLAB_JENKINS"):
+ hadoop_version = os.environ.get("AMPLAB_JENKINS_BUILD_PROFILE", "hadoop2.7")
+else:
+ hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.7")
+if hadoop_version == "hadoop3.2":
+ print("[info] Skip unsupported module:", "hive-thriftserver")
+ all_modules = [m for m in all_modules if m.name != "hive-thriftserver"]
+
# The root module is a dummy module which is used to run all of the tests.
# No other modules should directly depend on this module.
root = Module(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org