You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by tg...@apache.org on 2020/07/15 16:41:38 UTC
[spark] branch master updated: [SPARK-32036] Replace references to
blacklist/whitelist language with more appropriate terminology,
excluding the blacklisting feature
This is an automated email from the ASF dual-hosted git repository.
tgraves pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new cf22d94 [SPARK-32036] Replace references to blacklist/whitelist language with more appropriate terminology, excluding the blacklisting feature
cf22d94 is described below
commit cf22d947fb8f37aa4d394b6633d6f08dbbf6dc1c
Author: Erik Krogen <ek...@linkedin.com>
AuthorDate: Wed Jul 15 11:40:55 2020 -0500
[SPARK-32036] Replace references to blacklist/whitelist language with more appropriate terminology, excluding the blacklisting feature
### What changes were proposed in this pull request?
This PR will remove references to these "blacklist" and "whitelist" terms besides the blacklisting feature as a whole, which can be handled in a separate JIRA/PR.
This touches quite a few files, but the changes are straightforward (variable/method/etc. name changes) and most quite self-contained.
### Why are the changes needed?
As per discussion on the Spark dev list, it will be beneficial to remove references to problematic language that can alienate potential community members. One such reference is "blacklist" and "whitelist". While it seems to me that there is some valid debate as to whether these terms have racist origins, the cultural connotations are inescapable in today's world.
### Does this PR introduce _any_ user-facing change?
In the test file `HiveQueryFileTest`, a developer has the ability to specify the system property `spark.hive.whitelist` to specify a list of Hive query files that should be tested. This system property has been renamed to `spark.hive.includelist`. The old property has been kept for compatibility, but will log a warning if used. I am open to feedback from others on whether keeping a deprecated property here is unnecessary given that this is just for developers running tests.
### How was this patch tested?
Existing tests should be suitable since no behavior changes are expected as a result of this PR.
Closes #28874 from xkrogen/xkrogen-SPARK-32036-rename-blacklists.
Authored-by: Erik Krogen <ek...@linkedin.com>
Signed-off-by: Thomas Graves <tg...@apache.org>
---
R/pkg/tests/fulltests/test_context.R | 2 +-
R/pkg/tests/fulltests/test_sparkSQL.R | 8 ++--
R/pkg/tests/run-all.R | 4 +-
.../java/org/apache/spark/network/crypto/README.md | 2 +-
.../spark/deploy/history/FsHistoryProvider.scala | 29 +++++++------
.../spark/deploy/rest/RestSubmissionClient.scala | 4 +-
.../spark/scheduler/OutputCommitCoordinator.scala | 2 +-
.../scala/org/apache/spark/util/JsonProtocol.scala | 4 +-
.../test/scala/org/apache/spark/ThreadAudit.scala | 4 +-
.../org/apache/spark/deploy/SparkSubmitSuite.scala | 22 +++++-----
.../deploy/history/FsHistoryProviderSuite.scala | 8 ++--
.../org/apache/spark/ui/UISeleniumSuite.scala | 14 +++---
dev/sparktestsupport/modules.py | 10 ++---
docs/streaming-programming-guide.md | 50 +++++++++++-----------
.../streaming/JavaRecoverableNetworkWordCount.java | 20 ++++-----
.../streaming/recoverable_network_wordcount.py | 16 +++----
.../streaming/RecoverableNetworkWordCount.scala | 16 +++----
.../scala/org/apache/spark/util/DockerUtils.scala | 6 +--
project/SparkBuild.scala | 4 +-
python/pylintrc | 2 +-
python/pyspark/cloudpickle.py | 6 +--
python/pyspark/sql/functions.py | 4 +-
python/pyspark/sql/pandas/typehints.py | 4 +-
python/run-tests.py | 2 +-
.../cluster/mesos/MesosClusterScheduler.scala | 4 +-
.../spark/deploy/yarn/YarnSparkHadoopUtil.scala | 2 +-
.../sql/catalyst/analysis/CheckAnalysis.scala | 2 +-
.../spark/sql/catalyst/json/JSONOptions.scala | 10 ++---
.../spark/sql/catalyst/optimizer/Optimizer.scala | 34 +++++++--------
.../spark/sql/catalyst/optimizer/expressions.scala | 2 +-
.../plans/logical/basicLogicalOperators.scala | 2 +-
.../spark/sql/catalyst/rules/RuleExecutor.scala | 6 +--
.../catalyst/optimizer/FilterPushdownSuite.scala | 2 +-
.../PullupCorrelatedPredicatesSuite.scala | 2 +-
.../datasources/json/JsonOutputWriter.scala | 2 +-
.../inputs/{blacklist.sql => ignored.sql} | 2 +-
.../org/apache/spark/sql/SQLQueryTestSuite.scala | 6 +--
.../org/apache/spark/sql/TPCDSQuerySuite.scala | 4 +-
.../sql/execution/datasources/json/JsonSuite.scala | 2 +-
.../thriftserver/ThriftServerQueryTestSuite.scala | 4 +-
.../hive/execution/HiveCompatibilitySuite.scala | 16 +++----
.../execution/HiveWindowFunctionQuerySuite.scala | 8 ++--
.../clientpositive/add_partition_no_includelist.q | 7 +++
.../clientpositive/add_partition_no_whitelist.q | 7 ---
.../add_partition_with_includelist.q | 7 +++
.../clientpositive/add_partition_with_whitelist.q | 9 ----
.../alter_partition_with_includelist.q | 9 ++++
.../alter_partition_with_whitelist.q | 9 ----
.../sql/hive/execution/AggregationQuerySuite.scala | 2 +-
.../sql/hive/execution/HiveQueryFileTest.scala | 39 ++++++++++-------
.../HiveHadoopDelegationTokenManagerSuite.scala | 4 +-
.../streaming/util/FileBasedWriteAheadLog.scala | 4 +-
52 files changed, 231 insertions(+), 219 deletions(-)
diff --git a/R/pkg/tests/fulltests/test_context.R b/R/pkg/tests/fulltests/test_context.R
index f86872d..1add5a9 100644
--- a/R/pkg/tests/fulltests/test_context.R
+++ b/R/pkg/tests/fulltests/test_context.R
@@ -139,7 +139,7 @@ test_that("utility function can be called", {
expect_true(TRUE)
})
-test_that("getClientModeSparkSubmitOpts() returns spark-submit args from whitelist", {
+test_that("getClientModeSparkSubmitOpts() returns spark-submit args from allowList", {
e <- new.env()
e[["spark.driver.memory"]] <- "512m"
ops <- getClientModeSparkSubmitOpts("sparkrmain", e)
diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R
index 611d905..e008bc5 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL.R
@@ -3921,14 +3921,14 @@ test_that("No extra files are created in SPARK_HOME by starting session and maki
# before creating a SparkSession with enableHiveSupport = T at the top of this test file
# (filesBefore). The test here is to compare that (filesBefore) against the list of files before
# any test is run in run-all.R (sparkRFilesBefore).
- # sparkRWhitelistSQLDirs is also defined in run-all.R, and should contain only 2 whitelisted dirs,
+ # sparkRAllowedSQLDirs is also defined in run-all.R, and should contain only 2 allowed dirs,
# here allow the first value, spark-warehouse, in the diff, everything else should be exactly the
# same as before any test is run.
- compare_list(sparkRFilesBefore, setdiff(filesBefore, sparkRWhitelistSQLDirs[[1]]))
+ compare_list(sparkRFilesBefore, setdiff(filesBefore, sparkRAllowedSQLDirs[[1]]))
# third, ensure only spark-warehouse and metastore_db are created when enableHiveSupport = T
# note: as the note above, after running all tests in this file while enableHiveSupport = T, we
- # check the list of files again. This time we allow both whitelisted dirs to be in the diff.
- compare_list(sparkRFilesBefore, setdiff(filesAfter, sparkRWhitelistSQLDirs))
+ # check the list of files again. This time we allow both dirs to be in the diff.
+ compare_list(sparkRFilesBefore, setdiff(filesAfter, sparkRAllowedSQLDirs))
})
unlink(parquetPath)
diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R
index bf02ecd..a46924a 100644
--- a/R/pkg/tests/run-all.R
+++ b/R/pkg/tests/run-all.R
@@ -35,8 +35,8 @@ if (identical(Sys.getenv("NOT_CRAN"), "true")) {
install.spark(overwrite = TRUE)
sparkRDir <- file.path(Sys.getenv("SPARK_HOME"), "R")
- sparkRWhitelistSQLDirs <- c("spark-warehouse", "metastore_db")
- invisible(lapply(sparkRWhitelistSQLDirs,
+ sparkRAllowedSQLDirs <- c("spark-warehouse", "metastore_db")
+ invisible(lapply(sparkRAllowedSQLDirs,
function(x) { unlink(file.path(sparkRDir, x), recursive = TRUE, force = TRUE)}))
sparkRFilesBefore <- list.files(path = sparkRDir, all.files = TRUE)
diff --git a/common/network-common/src/main/java/org/apache/spark/network/crypto/README.md b/common/network-common/src/main/java/org/apache/spark/network/crypto/README.md
index 14df703..7a9fa3a 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/crypto/README.md
+++ b/common/network-common/src/main/java/org/apache/spark/network/crypto/README.md
@@ -155,4 +155,4 @@ server will be able to understand. This will cause the server to close the conne
attacker tries to send any command to the server. The attacker can just hold the channel open for
some time, which will be closed when the server times out the channel. These issues could be
separately mitigated by adding a shorter timeout for the first message after authentication, and
-potentially by adding host blacklists if a possible attack is detected from a particular host.
+potentially by adding host reject-lists if a possible attack is detected from a particular host.
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index 891c5bf..a73a5e9 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -188,23 +188,24 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
processing.remove(path.getName)
}
- private val blacklist = new ConcurrentHashMap[String, Long]
+ private val inaccessibleList = new ConcurrentHashMap[String, Long]
// Visible for testing
- private[history] def isBlacklisted(path: Path): Boolean = {
- blacklist.containsKey(path.getName)
+ private[history] def isAccessible(path: Path): Boolean = {
+ !inaccessibleList.containsKey(path.getName)
}
- private def blacklist(path: Path): Unit = {
- blacklist.put(path.getName, clock.getTimeMillis())
+ private def markInaccessible(path: Path): Unit = {
+ inaccessibleList.put(path.getName, clock.getTimeMillis())
}
/**
- * Removes expired entries in the blacklist, according to the provided `expireTimeInSeconds`.
+ * Removes expired entries in the inaccessibleList, according to the provided
+ * `expireTimeInSeconds`.
*/
- private def clearBlacklist(expireTimeInSeconds: Long): Unit = {
+ private def clearInaccessibleList(expireTimeInSeconds: Long): Unit = {
val expiredThreshold = clock.getTimeMillis() - expireTimeInSeconds * 1000
- blacklist.asScala.retain((_, creationTime) => creationTime >= expiredThreshold)
+ inaccessibleList.asScala.retain((_, creationTime) => creationTime >= expiredThreshold)
}
private val activeUIs = new mutable.HashMap[(String, Option[String]), LoadedAppUI]()
@@ -470,7 +471,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
logDebug(s"Scanning $logDir with lastScanTime==$lastScanTime")
val updated = Option(fs.listStatus(new Path(logDir))).map(_.toSeq).getOrElse(Nil)
- .filter { entry => !isBlacklisted(entry.getPath) }
+ .filter { entry => isAccessible(entry.getPath) }
.filter { entry => !isProcessing(entry.getPath) }
.flatMap { entry => EventLogFileReader(fs, entry) }
.filter { reader =>
@@ -687,8 +688,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
case e: AccessControlException =>
// We don't have read permissions on the log file
logWarning(s"Unable to read log $rootPath", e)
- blacklist(rootPath)
- // SPARK-28157 We should remove this blacklisted entry from the KVStore
+ markInaccessible(rootPath)
+ // SPARK-28157 We should remove this inaccessible entry from the KVStore
// to handle permission-only changes with the same file sizes later.
listing.delete(classOf[LogInfo], rootPath.toString)
case e: Exception =>
@@ -956,8 +957,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
}
}
- // Clean the blacklist from the expired entries.
- clearBlacklist(CLEAN_INTERVAL_S)
+ // Clean the inaccessibleList from the expired entries.
+ clearInaccessibleList(CLEAN_INTERVAL_S)
}
private def deleteAttemptLogs(
@@ -1334,7 +1335,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
private def deleteLog(fs: FileSystem, log: Path): Boolean = {
var deleted = false
- if (isBlacklisted(log)) {
+ if (!isAccessible(log)) {
logDebug(s"Skipping deleting $log as we don't have permissions on it.")
} else {
try {
diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
index 89ef051..cc1d60a 100644
--- a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionClient.scala
@@ -411,7 +411,7 @@ private[spark] object RestSubmissionClient {
// SPARK_HOME and SPARK_CONF_DIR are filtered out because they are usually wrong
// on the remote machine (SPARK-12345) (SPARK-25934)
- private val BLACKLISTED_SPARK_ENV_VARS = Set("SPARK_ENV_LOADED", "SPARK_HOME", "SPARK_CONF_DIR")
+ private val EXCLUDED_SPARK_ENV_VARS = Set("SPARK_ENV_LOADED", "SPARK_HOME", "SPARK_CONF_DIR")
private val REPORT_DRIVER_STATUS_INTERVAL = 1000
private val REPORT_DRIVER_STATUS_MAX_TRIES = 10
val PROTOCOL_VERSION = "v1"
@@ -421,7 +421,7 @@ private[spark] object RestSubmissionClient {
*/
private[rest] def filterSystemEnvironment(env: Map[String, String]): Map[String, String] = {
env.filterKeys { k =>
- (k.startsWith("SPARK_") && !BLACKLISTED_SPARK_ENV_VARS.contains(k)) || k.startsWith("MESOS_")
+ (k.startsWith("SPARK_") && !EXCLUDED_SPARK_ENV_VARS.contains(k)) || k.startsWith("MESOS_")
}.toMap
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala b/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
index b382d62..a5858eb 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/OutputCommitCoordinator.scala
@@ -151,7 +151,7 @@ private[spark] class OutputCommitCoordinator(conf: SparkConf, isDriver: Boolean)
logInfo(s"Task was denied committing, stage: $stage.$stageAttempt, " +
s"partition: $partition, attempt: $attemptNumber")
case _ =>
- // Mark the attempt as failed to blacklist from future commit protocol
+ // Mark the attempt as failed to exclude from future commit protocol
val taskId = TaskIdentifier(stageAttempt, attemptNumber)
stageState.failures.getOrElseUpdate(partition, mutable.Set()) += taskId
if (stageState.authorizedCommitters(partition) == taskId) {
diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index 170ba2b..ceaddb4 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -328,11 +328,11 @@ private[spark] object JsonProtocol {
("Accumulables" -> accumulablesToJson(taskInfo.accumulables))
}
- private lazy val accumulableBlacklist = Set("internal.metrics.updatedBlockStatuses")
+ private lazy val accumulableExcludeList = Set("internal.metrics.updatedBlockStatuses")
def accumulablesToJson(accumulables: Iterable[AccumulableInfo]): JArray = {
JArray(accumulables
- .filterNot(_.name.exists(accumulableBlacklist.contains))
+ .filterNot(_.name.exists(accumulableExcludeList.contains))
.toList.map(accumulableInfoToJson))
}
diff --git a/core/src/test/scala/org/apache/spark/ThreadAudit.scala b/core/src/test/scala/org/apache/spark/ThreadAudit.scala
index 44d1f22..1e29176 100644
--- a/core/src/test/scala/org/apache/spark/ThreadAudit.scala
+++ b/core/src/test/scala/org/apache/spark/ThreadAudit.scala
@@ -26,7 +26,7 @@ import org.apache.spark.internal.Logging
*/
trait ThreadAudit extends Logging {
- val threadWhiteList = Set(
+ val threadExcludeList = Set(
/**
* Netty related internal threads.
* These are excluded because their lifecycle is handled by the netty itself
@@ -108,7 +108,7 @@ trait ThreadAudit extends Logging {
if (threadNamesSnapshot.nonEmpty) {
val remainingThreadNames = runningThreadNames().diff(threadNamesSnapshot)
- .filterNot { s => threadWhiteList.exists(s.matches(_)) }
+ .filterNot { s => threadExcludeList.exists(s.matches(_)) }
if (remainingThreadNames.nonEmpty) {
logWarning(s"\n\n===== POSSIBLE THREAD LEAK IN SUITE $shortSuiteName, " +
s"thread names: ${remainingThreadNames.mkString(", ")} =====\n")
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index fd2d1f5..fd3d4bc 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -1210,17 +1210,17 @@ class SparkSubmitSuite
testRemoteResources(enableHttpFs = true)
}
- test("force download from blacklisted schemes") {
- testRemoteResources(enableHttpFs = true, blacklistSchemes = Seq("http"))
+ test("force download from forced schemes") {
+ testRemoteResources(enableHttpFs = true, forceDownloadSchemes = Seq("http"))
}
test("force download for all the schemes") {
- testRemoteResources(enableHttpFs = true, blacklistSchemes = Seq("*"))
+ testRemoteResources(enableHttpFs = true, forceDownloadSchemes = Seq("*"))
}
private def testRemoteResources(
enableHttpFs: Boolean,
- blacklistSchemes: Seq[String] = Nil): Unit = {
+ forceDownloadSchemes: Seq[String] = Nil): Unit = {
val hadoopConf = new Configuration()
updateConfWithFakeS3Fs(hadoopConf)
if (enableHttpFs) {
@@ -1237,8 +1237,8 @@ class SparkSubmitSuite
val tmpHttpJar = TestUtils.createJarWithFiles(Map("test.resource" -> "USER"), tmpDir)
val tmpHttpJarPath = s"http://${new File(tmpHttpJar.toURI).getAbsolutePath}"
- val forceDownloadArgs = if (blacklistSchemes.nonEmpty) {
- Seq("--conf", s"spark.yarn.dist.forceDownloadSchemes=${blacklistSchemes.mkString(",")}")
+ val forceDownloadArgs = if (forceDownloadSchemes.nonEmpty) {
+ Seq("--conf", s"spark.yarn.dist.forceDownloadSchemes=${forceDownloadSchemes.mkString(",")}")
} else {
Nil
}
@@ -1256,19 +1256,19 @@ class SparkSubmitSuite
val jars = conf.get("spark.yarn.dist.jars").split(",").toSet
- def isSchemeBlacklisted(scheme: String) = {
- blacklistSchemes.contains("*") || blacklistSchemes.contains(scheme)
+ def isSchemeForcedDownload(scheme: String) = {
+ forceDownloadSchemes.contains("*") || forceDownloadSchemes.contains(scheme)
}
- if (!isSchemeBlacklisted("s3")) {
+ if (!isSchemeForcedDownload("s3")) {
assert(jars.contains(tmpS3JarPath))
}
- if (enableHttpFs && blacklistSchemes.isEmpty) {
+ if (enableHttpFs && forceDownloadSchemes.isEmpty) {
// If Http FS is supported by yarn service, the URI of remote http resource should
// still be remote.
assert(jars.contains(tmpHttpJarPath))
- } else if (!enableHttpFs || isSchemeBlacklisted("http")) {
+ } else if (!enableHttpFs || isSchemeForcedDownload("http")) {
// If Http FS is not supported by yarn service, or http scheme is configured to be force
// downloading, the URI of remote http resource should be changed to a local one.
val jarName = new File(tmpHttpJar.toURI).getName
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index c2f34fc..ade03a0 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -1117,7 +1117,7 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
}
}
- test("SPARK-24948: blacklist files we don't have read permission on") {
+ test("SPARK-24948: ignore files we don't have read permission on") {
val clock = new ManualClock(1533132471)
val provider = new FsHistoryProvider(createTestConf(), clock)
val accessDenied = newLogFile("accessDenied", None, inProgress = false)
@@ -1137,17 +1137,17 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
updateAndCheck(mockedProvider) { list =>
list.size should be(1)
}
- // Doing 2 times in order to check the blacklist filter too
+ // Doing 2 times in order to check the inaccessibleList filter too
updateAndCheck(mockedProvider) { list =>
list.size should be(1)
}
val accessDeniedPath = new Path(accessDenied.getPath)
- assert(mockedProvider.isBlacklisted(accessDeniedPath))
+ assert(!mockedProvider.isAccessible(accessDeniedPath))
clock.advance(24 * 60 * 60 * 1000 + 1) // add a bit more than 1d
isReadable = true
mockedProvider.cleanLogs()
updateAndCheck(mockedProvider) { list =>
- assert(!mockedProvider.isBlacklisted(accessDeniedPath))
+ assert(mockedProvider.isAccessible(accessDeniedPath))
assert(list.exists(_.name == "accessDenied"))
assert(list.exists(_.name == "accessGranted"))
list.size should be(2)
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index ecfdf48..4f808f0 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -48,24 +48,28 @@ import org.apache.spark.util.CallSite
private[spark] class SparkUICssErrorHandler extends DefaultCssErrorHandler {
- private val cssWhiteList = List("bootstrap.min.css", "vis-timeline-graph2d.min.css")
+ /**
+ * Some libraries have warn/error messages that are too noisy for the tests; exclude them from
+ * normal error handling to avoid logging these.
+ */
+ private val cssExcludeList = List("bootstrap.min.css", "vis-timeline-graph2d.min.css")
- private def isInWhileList(uri: String): Boolean = cssWhiteList.exists(uri.endsWith)
+ private def isInExcludeList(uri: String): Boolean = cssExcludeList.exists(uri.endsWith)
override def warning(e: CSSParseException): Unit = {
- if (!isInWhileList(e.getURI)) {
+ if (!isInExcludeList(e.getURI)) {
super.warning(e)
}
}
override def fatalError(e: CSSParseException): Unit = {
- if (!isInWhileList(e.getURI)) {
+ if (!isInExcludeList(e.getURI)) {
super.fatalError(e)
}
}
override def error(e: CSSParseException): Unit = {
- if (!isInWhileList(e.getURI)) {
+ if (!isInExcludeList(e.getURI)) {
super.error(e)
}
}
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 2e8b8b5..3c438e3 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -32,7 +32,7 @@ class Module(object):
"""
def __init__(self, name, dependencies, source_file_regexes, build_profile_flags=(), environ={},
- sbt_test_goals=(), python_test_goals=(), blacklisted_python_implementations=(),
+ sbt_test_goals=(), python_test_goals=(), excluded_python_implementations=(),
test_tags=(), should_run_r_tests=False, should_run_build_tests=False):
"""
Define a new module.
@@ -49,7 +49,7 @@ class Module(object):
module are changed.
:param sbt_test_goals: A set of SBT test goals for testing this module.
:param python_test_goals: A set of Python test goals for testing this module.
- :param blacklisted_python_implementations: A set of Python implementations that are not
+ :param excluded_python_implementations: A set of Python implementations that are not
supported by this module's Python components. The values in this set should match
strings returned by Python's `platform.python_implementation()`.
:param test_tags A set of tags that will be excluded when running unit tests if the module
@@ -64,7 +64,7 @@ class Module(object):
self.build_profile_flags = build_profile_flags
self.environ = environ
self.python_test_goals = python_test_goals
- self.blacklisted_python_implementations = blacklisted_python_implementations
+ self.excluded_python_implementations = excluded_python_implementations
self.test_tags = test_tags
self.should_run_r_tests = should_run_r_tests
self.should_run_build_tests = should_run_build_tests
@@ -524,7 +524,7 @@ pyspark_mllib = Module(
"pyspark.mllib.tests.test_streaming_algorithms",
"pyspark.mllib.tests.test_util",
],
- blacklisted_python_implementations=[
+ excluded_python_implementations=[
"PyPy" # Skip these tests under PyPy since they require numpy and it isn't available there
]
)
@@ -565,7 +565,7 @@ pyspark_ml = Module(
"pyspark.ml.tests.test_tuning",
"pyspark.ml.tests.test_wrapper",
],
- blacklisted_python_implementations=[
+ excluded_python_implementations=[
"PyPy" # Skip these tests under PyPy since they require numpy and it isn't available there
]
)
diff --git a/docs/streaming-programming-guide.md b/docs/streaming-programming-guide.md
index 587deeb..56a455a 100644
--- a/docs/streaming-programming-guide.md
+++ b/docs/streaming-programming-guide.md
@@ -1822,7 +1822,7 @@ This is shown in the following example.
<div data-lang="scala" markdown="1">
{% highlight scala %}
-object WordBlacklist {
+object WordExcludeList {
@volatile private var instance: Broadcast[Seq[String]] = null
@@ -1830,8 +1830,8 @@ object WordBlacklist {
if (instance == null) {
synchronized {
if (instance == null) {
- val wordBlacklist = Seq("a", "b", "c")
- instance = sc.broadcast(wordBlacklist)
+ val wordExcludeList = Seq("a", "b", "c")
+ instance = sc.broadcast(wordExcludeList)
}
}
}
@@ -1847,7 +1847,7 @@ object DroppedWordsCounter {
if (instance == null) {
synchronized {
if (instance == null) {
- instance = sc.longAccumulator("WordsInBlacklistCounter")
+ instance = sc.longAccumulator("DroppedWordsCounter")
}
}
}
@@ -1856,13 +1856,13 @@ object DroppedWordsCounter {
}
wordCounts.foreachRDD { (rdd: RDD[(String, Int)], time: Time) =>
- // Get or register the blacklist Broadcast
- val blacklist = WordBlacklist.getInstance(rdd.sparkContext)
+ // Get or register the excludeList Broadcast
+ val excludeList = WordExcludeList.getInstance(rdd.sparkContext)
// Get or register the droppedWordsCounter Accumulator
val droppedWordsCounter = DroppedWordsCounter.getInstance(rdd.sparkContext)
- // Use blacklist to drop words and use droppedWordsCounter to count them
+ // Use excludeList to drop words and use droppedWordsCounter to count them
val counts = rdd.filter { case (word, count) =>
- if (blacklist.value.contains(word)) {
+ if (excludeList.value.contains(word)) {
droppedWordsCounter.add(count)
false
} else {
@@ -1879,16 +1879,16 @@ See the full [source code]({{site.SPARK_GITHUB_URL}}/blob/v{{site.SPARK_VERSION_
<div data-lang="java" markdown="1">
{% highlight java %}
-class JavaWordBlacklist {
+class JavaWordExcludeList {
private static volatile Broadcast<List<String>> instance = null;
public static Broadcast<List<String>> getInstance(JavaSparkContext jsc) {
if (instance == null) {
- synchronized (JavaWordBlacklist.class) {
+ synchronized (JavaWordExcludeList.class) {
if (instance == null) {
- List<String> wordBlacklist = Arrays.asList("a", "b", "c");
- instance = jsc.broadcast(wordBlacklist);
+ List<String> wordExcludeList = Arrays.asList("a", "b", "c");
+ instance = jsc.broadcast(wordExcludeList);
}
}
}
@@ -1904,7 +1904,7 @@ class JavaDroppedWordsCounter {
if (instance == null) {
synchronized (JavaDroppedWordsCounter.class) {
if (instance == null) {
- instance = jsc.sc().longAccumulator("WordsInBlacklistCounter");
+ instance = jsc.sc().longAccumulator("DroppedWordsCounter");
}
}
}
@@ -1913,13 +1913,13 @@ class JavaDroppedWordsCounter {
}
wordCounts.foreachRDD((rdd, time) -> {
- // Get or register the blacklist Broadcast
- Broadcast<List<String>> blacklist = JavaWordBlacklist.getInstance(new JavaSparkContext(rdd.context()));
+ // Get or register the excludeList Broadcast
+ Broadcast<List<String>> excludeList = JavaWordExcludeList.getInstance(new JavaSparkContext(rdd.context()));
// Get or register the droppedWordsCounter Accumulator
LongAccumulator droppedWordsCounter = JavaDroppedWordsCounter.getInstance(new JavaSparkContext(rdd.context()));
- // Use blacklist to drop words and use droppedWordsCounter to count them
+ // Use excludeList to drop words and use droppedWordsCounter to count them
String counts = rdd.filter(wordCount -> {
- if (blacklist.value().contains(wordCount._1())) {
+ if (excludeList.value().contains(wordCount._1())) {
droppedWordsCounter.add(wordCount._2());
return false;
} else {
@@ -1935,10 +1935,10 @@ See the full [source code]({{site.SPARK_GITHUB_URL}}/blob/v{{site.SPARK_VERSION_
</div>
<div data-lang="python" markdown="1">
{% highlight python %}
-def getWordBlacklist(sparkContext):
- if ("wordBlacklist" not in globals()):
- globals()["wordBlacklist"] = sparkContext.broadcast(["a", "b", "c"])
- return globals()["wordBlacklist"]
+def getWordExcludeList(sparkContext):
+ if ("wordExcludeList" not in globals()):
+ globals()["wordExcludeList"] = sparkContext.broadcast(["a", "b", "c"])
+ return globals()["wordExcludeList"]
def getDroppedWordsCounter(sparkContext):
if ("droppedWordsCounter" not in globals()):
@@ -1946,14 +1946,14 @@ def getDroppedWordsCounter(sparkContext):
return globals()["droppedWordsCounter"]
def echo(time, rdd):
- # Get or register the blacklist Broadcast
- blacklist = getWordBlacklist(rdd.context)
+ # Get or register the excludeList Broadcast
+ excludeList = getWordExcludeList(rdd.context)
# Get or register the droppedWordsCounter Accumulator
droppedWordsCounter = getDroppedWordsCounter(rdd.context)
- # Use blacklist to drop words and use droppedWordsCounter to count them
+ # Use excludeList to drop words and use droppedWordsCounter to count them
def filterFunc(wordCount):
- if wordCount[0] in blacklist.value:
+ if wordCount[0] in excludeList.value:
droppedWordsCounter.add(wordCount[1])
False
else:
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
index 45a876d..c01a62b 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
@@ -41,16 +41,16 @@ import org.apache.spark.util.LongAccumulator;
/**
* Use this singleton to get or register a Broadcast variable.
*/
-class JavaWordBlacklist {
+class JavaWordExcludeList {
private static volatile Broadcast<List<String>> instance = null;
public static Broadcast<List<String>> getInstance(JavaSparkContext jsc) {
if (instance == null) {
- synchronized (JavaWordBlacklist.class) {
+ synchronized (JavaWordExcludeList.class) {
if (instance == null) {
- List<String> wordBlacklist = Arrays.asList("a", "b", "c");
- instance = jsc.broadcast(wordBlacklist);
+ List<String> wordExcludeList = Arrays.asList("a", "b", "c");
+ instance = jsc.broadcast(wordExcludeList);
}
}
}
@@ -69,7 +69,7 @@ class JavaDroppedWordsCounter {
if (instance == null) {
synchronized (JavaDroppedWordsCounter.class) {
if (instance == null) {
- instance = jsc.sc().longAccumulator("WordsInBlacklistCounter");
+ instance = jsc.sc().longAccumulator("DroppedWordsCounter");
}
}
}
@@ -133,15 +133,15 @@ public final class JavaRecoverableNetworkWordCount {
.reduceByKey((i1, i2) -> i1 + i2);
wordCounts.foreachRDD((rdd, time) -> {
- // Get or register the blacklist Broadcast
- Broadcast<List<String>> blacklist =
- JavaWordBlacklist.getInstance(new JavaSparkContext(rdd.context()));
+ // Get or register the excludeList Broadcast
+ Broadcast<List<String>> excludeList =
+ JavaWordExcludeList.getInstance(new JavaSparkContext(rdd.context()));
// Get or register the droppedWordsCounter Accumulator
LongAccumulator droppedWordsCounter =
JavaDroppedWordsCounter.getInstance(new JavaSparkContext(rdd.context()));
- // Use blacklist to drop words and use droppedWordsCounter to count them
+ // Use excludeList to drop words and use droppedWordsCounter to count them
String counts = rdd.filter(wordCount -> {
- if (blacklist.value().contains(wordCount._1())) {
+ if (excludeList.value().contains(wordCount._1())) {
droppedWordsCounter.add(wordCount._2());
return false;
} else {
diff --git a/examples/src/main/python/streaming/recoverable_network_wordcount.py b/examples/src/main/python/streaming/recoverable_network_wordcount.py
index 8424556..6ebe91a 100644
--- a/examples/src/main/python/streaming/recoverable_network_wordcount.py
+++ b/examples/src/main/python/streaming/recoverable_network_wordcount.py
@@ -43,10 +43,10 @@ from pyspark.streaming import StreamingContext
# Get or register a Broadcast variable
-def getWordBlacklist(sparkContext):
- if ('wordBlacklist' not in globals()):
- globals()['wordBlacklist'] = sparkContext.broadcast(["a", "b", "c"])
- return globals()['wordBlacklist']
+def getWordExcludeList(sparkContext):
+ if ('wordExcludeList' not in globals()):
+ globals()['wordExcludeList'] = sparkContext.broadcast(["a", "b", "c"])
+ return globals()['wordExcludeList']
# Get or register an Accumulator
@@ -72,14 +72,14 @@ def createContext(host, port, outputPath):
wordCounts = words.map(lambda x: (x, 1)).reduceByKey(lambda x, y: x + y)
def echo(time, rdd):
- # Get or register the blacklist Broadcast
- blacklist = getWordBlacklist(rdd.context)
+ # Get or register the excludeList Broadcast
+ excludeList = getWordExcludeList(rdd.context)
# Get or register the droppedWordsCounter Accumulator
droppedWordsCounter = getDroppedWordsCounter(rdd.context)
- # Use blacklist to drop words and use droppedWordsCounter to count them
+ # Use excludeList to drop words and use droppedWordsCounter to count them
def filterFunc(wordCount):
- if wordCount[0] in blacklist.value:
+ if wordCount[0] in excludeList.value:
droppedWordsCounter.add(wordCount[1])
return False
else:
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
index 243c22e..ee3bbe4 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
@@ -32,7 +32,7 @@ import org.apache.spark.util.{IntParam, LongAccumulator}
/**
* Use this singleton to get or register a Broadcast variable.
*/
-object WordBlacklist {
+object WordExcludeList {
@volatile private var instance: Broadcast[Seq[String]] = null
@@ -40,8 +40,8 @@ object WordBlacklist {
if (instance == null) {
synchronized {
if (instance == null) {
- val wordBlacklist = Seq("a", "b", "c")
- instance = sc.broadcast(wordBlacklist)
+ val wordExcludeList = Seq("a", "b", "c")
+ instance = sc.broadcast(wordExcludeList)
}
}
}
@@ -60,7 +60,7 @@ object DroppedWordsCounter {
if (instance == null) {
synchronized {
if (instance == null) {
- instance = sc.longAccumulator("WordsInBlacklistCounter")
+ instance = sc.longAccumulator("DroppedWordsCounter")
}
}
}
@@ -117,13 +117,13 @@ object RecoverableNetworkWordCount {
val words = lines.flatMap(_.split(" "))
val wordCounts = words.map((_, 1)).reduceByKey(_ + _)
wordCounts.foreachRDD { (rdd: RDD[(String, Int)], time: Time) =>
- // Get or register the blacklist Broadcast
- val blacklist = WordBlacklist.getInstance(rdd.sparkContext)
+ // Get or register the excludeList Broadcast
+ val excludeList = WordExcludeList.getInstance(rdd.sparkContext)
// Get or register the droppedWordsCounter Accumulator
val droppedWordsCounter = DroppedWordsCounter.getInstance(rdd.sparkContext)
- // Use blacklist to drop words and use droppedWordsCounter to count them
+ // Use excludeList to drop words and use droppedWordsCounter to count them
val counts = rdd.filter { case (word, count) =>
- if (blacklist.value.contains(word)) {
+ if (excludeList.value.contains(word)) {
droppedWordsCounter.add(count)
false
} else {
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala
index fda377e..5abca8d 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/util/DockerUtils.scala
@@ -36,7 +36,7 @@ private[spark] object DockerUtils {
.orElse(findFromDockerMachine())
.orElse(Try(Seq("/bin/bash", "-c", "boot2docker ip 2>/dev/null").!!.trim).toOption)
.getOrElse {
- // This block of code is based on Utils.findLocalInetAddress(), but is modified to blacklist
+ // This block of code is based on Utils.findLocalInetAddress(), but is modified to exclude
// certain interfaces.
val address = InetAddress.getLocalHost
// Address resolves to something like 127.0.1.1, which happens on Debian; try to find
@@ -44,12 +44,12 @@ private[spark] object DockerUtils {
// getNetworkInterfaces returns ifs in reverse order compared to ifconfig output order
// on unix-like system. On windows, it returns in index order.
// It's more proper to pick ip address following system output order.
- val blackListedIFs = Seq(
+ val excludedIFs = Seq(
"vboxnet0", // Mac
"docker0" // Linux
)
val activeNetworkIFs = NetworkInterface.getNetworkInterfaces.asScala.toSeq.filter { i =>
- !blackListedIFs.contains(i.getName)
+ !excludedIFs.contains(i.getName)
}
val reOrderedNetworkIFs = activeNetworkIFs.reverse
for (ni <- reOrderedNetworkIFs) {
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 5184fd8..d19b514 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -444,10 +444,10 @@ object SparkBuild extends PomBuild {
object SparkParallelTestGrouping {
// Settings for parallelizing tests. The basic strategy here is to run the slowest suites (or
// collections of suites) in their own forked JVMs, allowing us to gain parallelism within a
- // SBT project. Here, we take a whitelisting approach where the default behavior is to run all
+ // SBT project. Here, we take an opt-in approach where the default behavior is to run all
// tests sequentially in a single JVM, requiring us to manually opt-in to the extra parallelism.
//
- // There are a reasons why such a whitelist approach is good:
+ // There are a reasons why such an opt-in approach is good:
//
// 1. Launching one JVM per suite adds significant overhead for short-running suites. In
// addition to JVM startup time and JIT warmup, it appears that initialization of Derby
diff --git a/python/pylintrc b/python/pylintrc
index 6a675770..26d2741 100644
--- a/python/pylintrc
+++ b/python/pylintrc
@@ -27,7 +27,7 @@
# Profiled execution.
profile=no
-# Add files or directories to the blacklist. They should be base names, not
+# Add files or directories to the ignoreList. They should be base names, not
# paths.
ignore=pyspark.heapq3
diff --git a/python/pyspark/cloudpickle.py b/python/pyspark/cloudpickle.py
index 09d3a5e..af49c77 100644
--- a/python/pyspark/cloudpickle.py
+++ b/python/pyspark/cloudpickle.py
@@ -87,8 +87,8 @@ if sys.version_info[0] < 3: # pragma: no branch
PY2 = True
PY2_WRAPPER_DESCRIPTOR_TYPE = type(object.__init__)
PY2_METHOD_WRAPPER_TYPE = type(object.__eq__)
- PY2_CLASS_DICT_BLACKLIST = (PY2_METHOD_WRAPPER_TYPE,
- PY2_WRAPPER_DESCRIPTOR_TYPE)
+ PY2_CLASS_DICT_SKIP_PICKLE_METHOD_TYPE = (PY2_METHOD_WRAPPER_TYPE,
+ PY2_WRAPPER_DESCRIPTOR_TYPE)
else:
types.ClassType = type
from pickle import _Pickler as Pickler
@@ -327,7 +327,7 @@ def _extract_class_dict(cls):
if hasattr(value, "im_func"):
if value.im_func is getattr(base_value, "im_func", None):
to_remove.append(name)
- elif isinstance(value, PY2_CLASS_DICT_BLACKLIST):
+ elif isinstance(value, PY2_CLASS_DICT_SKIP_PICKLE_METHOD_TYPE):
# On Python 2 we have no way to pickle those specific
# methods types nor to check that they are actually
# inherited. So we assume that they are always inherited
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 63b0499..5a35210 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -3411,9 +3411,9 @@ def udf(f=None, returnType=StringType()):
evalType=PythonEvalType.SQL_BATCHED_UDF)
-blacklist = ['map', 'since']
+ignored_fns = ['map', 'since']
__all__ = [k for k, v in globals().items()
- if not k.startswith('_') and k[0].islower() and callable(v) and k not in blacklist]
+ if not k.startswith('_') and k[0].islower() and callable(v) and k not in ignored_fns]
__all__ += ["PandasUDFType"]
__all__.sort()
diff --git a/python/pyspark/sql/pandas/typehints.py b/python/pyspark/sql/pandas/typehints.py
index b0323ba..e696f67 100644
--- a/python/pyspark/sql/pandas/typehints.py
+++ b/python/pyspark/sql/pandas/typehints.py
@@ -98,8 +98,8 @@ def infer_eval_type(sig):
a,
parameter_check_func=lambda ua: ua == pd.Series or ua == pd.DataFrame)
for a in parameters_sig) and (
- # It's tricky to whitelist which types pd.Series constructor can take.
- # Simply blacklist common types used here for now (which becomes object
+ # It's tricky to include only types which pd.Series constructor can take.
+ # Simply exclude common types used here for now (which becomes object
# types Spark can't recognize).
return_annotation != pd.Series and
return_annotation != pd.DataFrame and
diff --git a/python/run-tests.py b/python/run-tests.py
index 357eb8f..db62f96 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -268,7 +268,7 @@ def main():
[python_exec, "--version"], stderr=subprocess.STDOUT, universal_newlines=True).strip())
if should_test_modules:
for module in modules_to_test:
- if python_implementation not in module.blacklisted_python_implementations:
+ if python_implementation not in module.excluded_python_implementations:
for test_goal in module.python_test_goals:
heavy_tests = ['pyspark.streaming.tests', 'pyspark.mllib.tests',
'pyspark.tests', 'pyspark.sql.tests', 'pyspark.ml.tests']
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
index edcdb92..26939ef 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosClusterScheduler.scala
@@ -539,14 +539,14 @@ private[spark] class MesosClusterScheduler(
options ++= Seq("--py-files", formattedFiles)
// --conf
- val replicatedOptionsBlacklist = Set(
+ val replicatedOptionsExcludeList = Set(
JARS.key, // Avoids duplicate classes in classpath
SUBMIT_DEPLOY_MODE.key, // this would be set to `cluster`, but we need client
"spark.master" // this contains the address of the dispatcher, not master
)
val defaultConf = conf.getAllWithPrefix(config.DISPATCHER_DRIVER_DEFAULT_PREFIX).toMap
val driverConf = desc.conf.getAll
- .filter { case (key, _) => !replicatedOptionsBlacklist.contains(key) }
+ .filter { case (key, _) => !replicatedOptionsExcludeList.contains(key) }
.toMap
(defaultConf ++ driverConf).foreach { case (key, value) =>
options ++= Seq("--conf", s"${key}=${value}") }
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala
index 9cefc40..9d6b776 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala
@@ -107,7 +107,7 @@ object YarnSparkHadoopUtil {
* Not killing the task leaves various aspects of the executor and (to some extent) the jvm in
* an inconsistent state.
* TODO: If the OOM is not recoverable by rescheduling it on different node, then do
- * 'something' to fail job ... akin to blacklisting trackers in mapred ?
+ * 'something' to fail job ... akin to unhealthy trackers in mapred ?
*
* The handler if an OOM Exception is thrown by the JVM must be configured on Windows
* differently: the 'taskkill' command should be used, whereas Unix-based systems use 'kill'.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 43dd097..351be32 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -867,7 +867,7 @@ trait CheckAnalysis extends PredicateHelper {
// Simplify the predicates before validating any unsupported correlation patterns in the plan.
AnalysisHelper.allowInvokingTransformsInAnalyzer { BooleanSimplification(sub).foreachUp {
- // Whitelist operators allowed in a correlated subquery
+ // Approve operators allowed in a correlated subquery
// There are 4 categories:
// 1. Operators that are allowed anywhere in a correlated subquery, and,
// by definition of the operators, they either do not contain
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
index 70a673b..c145f26 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
@@ -168,10 +168,10 @@ private[sql] class JSONOptionsInRead(
}
protected override def checkedEncoding(enc: String): String = {
- val isBlacklisted = JSONOptionsInRead.blacklist.contains(Charset.forName(enc))
- require(multiLine || !isBlacklisted,
- s"""The ${enc} encoding must not be included in the blacklist when multiLine is disabled:
- |Blacklist: ${JSONOptionsInRead.blacklist.mkString(", ")}""".stripMargin)
+ val isDenied = JSONOptionsInRead.denyList.contains(Charset.forName(enc))
+ require(multiLine || !isDenied,
+ s"""The $enc encoding must not be included in the denyList when multiLine is disabled:
+ |denylist: ${JSONOptionsInRead.denyList.mkString(", ")}""".stripMargin)
val isLineSepRequired =
multiLine || Charset.forName(enc) == StandardCharsets.UTF_8 || lineSeparator.nonEmpty
@@ -188,7 +188,7 @@ private[sql] object JSONOptionsInRead {
// only the first lines will have the BOM which leads to impossibility for reading
// the rest lines. Besides of that, the lineSep option must have the BOM in such
// encodings which can never present between lines.
- val blacklist = Seq(
+ val denyList = Seq(
Charset.forName("UTF-16"),
Charset.forName("UTF-32")
)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index e481cdb..33da482 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -48,7 +48,7 @@ abstract class Optimizer(catalogManager: CatalogManager)
plan.find(PlanHelper.specialExpressionsInUnsupportedOperator(_).nonEmpty).isEmpty)
}
- override protected val blacklistedOnceBatches: Set[String] =
+ override protected val excludedOnceBatches: Set[String] =
Set(
"PartitionPruning",
"Extract Python UDFs",
@@ -373,38 +373,38 @@ object RemoveRedundantAliases extends Rule[LogicalPlan] {
/**
* Remove the top-level alias from an expression when it is redundant.
*/
- private def removeRedundantAlias(e: Expression, blacklist: AttributeSet): Expression = e match {
+ private def removeRedundantAlias(e: Expression, excludeList: AttributeSet): Expression = e match {
// Alias with metadata can not be stripped, or the metadata will be lost.
// If the alias name is different from attribute name, we can't strip it either, or we
// may accidentally change the output schema name of the root plan.
case a @ Alias(attr: Attribute, name)
if a.metadata == Metadata.empty &&
name == attr.name &&
- !blacklist.contains(attr) &&
- !blacklist.contains(a) =>
+ !excludeList.contains(attr) &&
+ !excludeList.contains(a) =>
attr
case a => a
}
/**
- * Remove redundant alias expression from a LogicalPlan and its subtree. A blacklist is used to
- * prevent the removal of seemingly redundant aliases used to deduplicate the input for a (self)
- * join or to prevent the removal of top-level subquery attributes.
+ * Remove redundant alias expression from a LogicalPlan and its subtree. A set of excludes is used
+ * to prevent the removal of seemingly redundant aliases used to deduplicate the input for a
+ * (self) join or to prevent the removal of top-level subquery attributes.
*/
- private def removeRedundantAliases(plan: LogicalPlan, blacklist: AttributeSet): LogicalPlan = {
+ private def removeRedundantAliases(plan: LogicalPlan, excluded: AttributeSet): LogicalPlan = {
plan match {
// We want to keep the same output attributes for subqueries. This means we cannot remove
// the aliases that produce these attributes
case Subquery(child, correlated) =>
- Subquery(removeRedundantAliases(child, blacklist ++ child.outputSet), correlated)
+ Subquery(removeRedundantAliases(child, excluded ++ child.outputSet), correlated)
// A join has to be treated differently, because the left and the right side of the join are
- // not allowed to use the same attributes. We use a blacklist to prevent us from creating a
- // situation in which this happens; the rule will only remove an alias if its child
+ // not allowed to use the same attributes. We use an exclude list to prevent us from creating
+ // a situation in which this happens; the rule will only remove an alias if its child
// attribute is not on the black list.
case Join(left, right, joinType, condition, hint) =>
- val newLeft = removeRedundantAliases(left, blacklist ++ right.outputSet)
- val newRight = removeRedundantAliases(right, blacklist ++ newLeft.outputSet)
+ val newLeft = removeRedundantAliases(left, excluded ++ right.outputSet)
+ val newRight = removeRedundantAliases(right, excluded ++ newLeft.outputSet)
val mapping = AttributeMap(
createAttributeMapping(left, newLeft) ++
createAttributeMapping(right, newRight))
@@ -417,7 +417,7 @@ object RemoveRedundantAliases extends Rule[LogicalPlan] {
// Remove redundant aliases in the subtree(s).
val currentNextAttrPairs = mutable.Buffer.empty[(Attribute, Attribute)]
val newNode = plan.mapChildren { child =>
- val newChild = removeRedundantAliases(child, blacklist)
+ val newChild = removeRedundantAliases(child, excluded)
currentNextAttrPairs ++= createAttributeMapping(child, newChild)
newChild
}
@@ -430,9 +430,9 @@ object RemoveRedundantAliases extends Rule[LogicalPlan] {
// Create a an expression cleaning function for nodes that can actually produce redundant
// aliases, use identity otherwise.
val clean: Expression => Expression = plan match {
- case _: Project => removeRedundantAlias(_, blacklist)
- case _: Aggregate => removeRedundantAlias(_, blacklist)
- case _: Window => removeRedundantAlias(_, blacklist)
+ case _: Project => removeRedundantAlias(_, excluded)
+ case _: Aggregate => removeRedundantAlias(_, excluded)
+ case _: Window => removeRedundantAlias(_, excluded)
case _ => identity[Expression]
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index 759ce57..773ee77 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -677,7 +677,7 @@ object FoldablePropagation extends Rule[LogicalPlan] {
}
/**
- * Whitelist of all [[UnaryNode]]s for which allow foldable propagation.
+ * List of all [[UnaryNode]]s which allow foldable propagation.
*/
private def canPropagateFoldables(u: UnaryNode): Boolean = u match {
case _: Project => true
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 039fd93..f1a363c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -1028,7 +1028,7 @@ case class Deduplicate(
/**
* A trait to represent the commands that support subqueries.
- * This is used to whitelist such commands in the subquery-related checks.
+ * This is used to allow such commands in the subquery-related checks.
*/
trait SupportsSubquery extends LogicalPlan
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
index bff04d3..2109e8f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
@@ -80,8 +80,8 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
/** Defines a sequence of rule batches, to be overridden by the implementation. */
protected def batches: Seq[Batch]
- /** Once batches that are blacklisted in the idempotence checker */
- protected val blacklistedOnceBatches: Set[String] = Set.empty
+ /** Once batches that are excluded in the idempotence checker */
+ protected val excludedOnceBatches: Set[String] = Set.empty
/**
* Defines a check function that checks for structural integrity of the plan after the execution
@@ -189,7 +189,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
}
// Check idempotence for Once batches.
if (batch.strategy == Once &&
- Utils.isTesting && !blacklistedOnceBatches.contains(batch.name)) {
+ Utils.isTesting && !excludedOnceBatches.contains(batch.name)) {
checkBatchIdempotence(batch, curPlan)
}
continue = false
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
index bb8f5f9..bb7e9d0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
@@ -33,7 +33,7 @@ class FilterPushdownSuite extends PlanTest {
object Optimize extends RuleExecutor[LogicalPlan] {
- override protected val blacklistedOnceBatches: Set[String] =
+ override protected val excludedOnceBatches: Set[String] =
Set("Push CNF predicate through join")
val batches =
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala
index 2d86d5a..e777570 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.rules.RuleExecutor
class PullupCorrelatedPredicatesSuite extends PlanTest {
object Optimize extends RuleExecutor[LogicalPlan] {
- override protected val blacklistedOnceBatches = Set("PullupCorrelatedPredicates")
+ override protected val excludedOnceBatches = Set("PullupCorrelatedPredicates")
val batches =
Batch("PullupCorrelatedPredicates", Once,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonOutputWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonOutputWriter.scala
index dfd84e3..719d72f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonOutputWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonOutputWriter.scala
@@ -39,7 +39,7 @@ class JsonOutputWriter(
case None => StandardCharsets.UTF_8
}
- if (JSONOptionsInRead.blacklist.contains(encoding)) {
+ if (JSONOptionsInRead.denyList.contains(encoding)) {
logWarning(s"The JSON file ($path) was written in the encoding ${encoding.displayName()}" +
" which can be read back by Spark only if multiLine is enabled.")
}
diff --git a/sql/core/src/test/resources/sql-tests/inputs/blacklist.sql b/sql/core/src/test/resources/sql-tests/inputs/ignored.sql
similarity index 75%
rename from sql/core/src/test/resources/sql-tests/inputs/blacklist.sql
rename to sql/core/src/test/resources/sql-tests/inputs/ignored.sql
index d69f814..0b5331a 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/blacklist.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/ignored.sql
@@ -1,4 +1,4 @@
--- This is a query file that has been blacklisted.
+-- This is a query file that has been ignored.
-- It includes a query that should crash Spark.
-- If the test case is run, the whole suite would fail.
some random not working query that should crash Spark.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index f0522df..3324745 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -153,8 +153,8 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper
.set(SQLConf.SHUFFLE_PARTITIONS, 4)
/** List of test cases to ignore, in lower cases. */
- protected def blackList: Set[String] = Set(
- "blacklist.sql" // Do NOT remove this one. It is here to test the blacklist functionality.
+ protected def ignoreList: Set[String] = Set(
+ "ignored.sql" // Do NOT remove this one. It is here to test the ignore functionality.
)
// Create all the test cases.
@@ -222,7 +222,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper
name: String, inputFile: String, resultFile: String) extends TestCase with AnsiTest
protected def createScalaTestCase(testCase: TestCase): Unit = {
- if (blackList.exists(t =>
+ if (ignoreList.exists(t =>
testCase.name.toLowerCase(Locale.ROOT).contains(t.toLowerCase(Locale.ROOT)))) {
// Create a test case to ignore this case.
ignore(testCase.name) { /* Do nothing */ }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
index aacb625..d0d484e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
@@ -85,7 +85,7 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
// List up the known queries having too large code in a generated function.
// A JIRA file for `modified-q3` is as follows;
// [SPARK-29128] Split predicate code in OR expressions
- val blackListForMethodCodeSizeCheck = Set("modified-q3")
+ val excludeListForMethodCodeSizeCheck = Set("modified-q3")
modifiedTPCDSQueries.foreach { name =>
val queryString = resourceToString(s"tpcds-modifiedQueries/$name.sql",
@@ -94,7 +94,7 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
test(testName) {
// check the plans can be properly generated
val plan = sql(queryString).queryExecution.executedPlan
- checkGeneratedCode(plan, !blackListForMethodCodeSizeCheck.contains(testName))
+ checkGeneratedCode(plan, !excludeListForMethodCodeSizeCheck.contains(testName))
}
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index c7448b1..de01099 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -2488,7 +2488,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
.json(testFile("test-data/utf16LE.json"))
.count()
}
- assert(exception.getMessage.contains("encoding must not be included in the blacklist"))
+ assert(exception.getMessage.contains("encoding must not be included in the denyList"))
}
}
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
index 686dc1c..30d911b 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
@@ -67,7 +67,7 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServ
}
/** List of test cases to ignore, in lower cases. */
- override def blackList: Set[String] = super.blackList ++ Set(
+ override def ignoreList: Set[String] = super.ignoreList ++ Set(
// Missing UDF
"postgreSQL/boolean.sql",
"postgreSQL/case.sql",
@@ -208,7 +208,7 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServ
}
override def createScalaTestCase(testCase: TestCase): Unit = {
- if (blackList.exists(t =>
+ if (ignoreList.exists(t =>
testCase.name.toLowerCase(Locale.ROOT).contains(t.toLowerCase(Locale.ROOT)))) {
// Create a test case to ignore this case.
ignore(testCase.name) { /* Do nothing */ }
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index d1ed55a..b7ea063 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -85,7 +85,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
}
/** A list of tests deemed out of scope currently and thus completely disregarded. */
- override def blackList: Seq[String] = Seq(
+ override def excludeList: Seq[String] = Seq(
// These tests use hooks that are not on the classpath and thus break all subsequent execution.
"hook_order",
"hook_context_cs",
@@ -516,7 +516,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
// This test uses CREATE EXTERNAL TABLE without specifying LOCATION
"alter2",
- // [SPARK-16248][SQL] Whitelist the list of Hive fallback functions
+ // [SPARK-16248][SQL] Include the list of Hive fallback functions
"udf_field",
"udf_reflect2",
"udf_xpath",
@@ -604,7 +604,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
"udf_radians"
)
- private def commonWhiteList = Seq(
+ private def commonIncludeList = Seq(
"add_part_exist",
"add_part_multiple",
"add_partition_no_whitelist",
@@ -1142,14 +1142,14 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
)
/**
- * The set of tests that are believed to be working in catalyst. Tests not on whiteList or
- * blacklist are implicitly marked as ignored.
+ * The set of tests that are believed to be working in catalyst. Tests not on includeList or
+ * excludeList are implicitly marked as ignored.
*/
- override def whiteList: Seq[String] = if (HiveUtils.isHive23) {
- commonWhiteList ++ Seq(
+ override def includeList: Seq[String] = if (HiveUtils.isHive23) {
+ commonIncludeList ++ Seq(
"decimal_1_1"
)
} else {
- commonWhiteList
+ commonIncludeList
}
}
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
index 2c0970c..1b801ad 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
@@ -763,7 +763,7 @@ class HiveWindowFunctionQueryFileSuite
}
}
- override def blackList: Seq[String] = Seq(
+ override def excludeList: Seq[String] = Seq(
// Partitioned table functions are not supported.
"ptf*",
// tests of windowing.q are in HiveWindowFunctionQueryBaseSuite
@@ -791,12 +791,12 @@ class HiveWindowFunctionQueryFileSuite
"windowing_adjust_rowcontainer_sz"
)
- override def whiteList: Seq[String] = Seq(
+ override def includeList: Seq[String] = Seq(
"windowing_udaf2"
)
- // Only run those query tests in the realWhileList (do not try other ignored query files).
+ // Only run those query tests in the realIncludeList (do not try other ignored query files).
override def testCases: Seq[(String, File)] = super.testCases.filter {
- case (name, _) => realWhiteList.contains(name)
+ case (name, _) => realIncludeList.contains(name)
}
}
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_no_includelist.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_no_includelist.q
new file mode 100644
index 0000000..1767712
--- /dev/null
+++ b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_no_includelist.q
@@ -0,0 +1,7 @@
+SET hive.metastore.partition.name.whitelist.pattern=;
+-- Test with no partition name include-list pattern
+
+CREATE TABLE part_noincludelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
+SHOW PARTITIONS part_noincludelist_test;
+
+ALTER TABLE part_noincludelist_test ADD PARTITION (ds='1,2,3,4');
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_no_whitelist.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_no_whitelist.q
deleted file mode 100644
index f51c53c..0000000
--- a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_no_whitelist.q
+++ /dev/null
@@ -1,7 +0,0 @@
-SET hive.metastore.partition.name.whitelist.pattern=;
--- Test with no partition name whitelist pattern
-
-CREATE TABLE part_nowhitelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
-SHOW PARTITIONS part_nowhitelist_test;
-
-ALTER TABLE part_nowhitelist_test ADD PARTITION (ds='1,2,3,4');
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_with_includelist.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_with_includelist.q
new file mode 100644
index 0000000..7e7f30d
--- /dev/null
+++ b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_with_includelist.q
@@ -0,0 +1,7 @@
+SET hive.metastore.partition.name.whitelist.pattern=[A-Za-z]*;
+-- This pattern matches only letters.
+
+CREATE TABLE part_includelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
+SHOW PARTITIONS part_includelist_test;
+
+ALTER TABLE part_includelist_test ADD PARTITION (ds='Part');
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_with_whitelist.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_with_whitelist.q
deleted file mode 100644
index 009c761..0000000
--- a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/add_partition_with_whitelist.q
+++ /dev/null
@@ -1,9 +0,0 @@
-SET hive.metastore.partition.name.whitelist.pattern=[A-Za-z]*;
--- This pattern matches only letters.
-
-CREATE TABLE part_whitelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
-SHOW PARTITIONS part_whitelist_test;
-
-ALTER TABLE part_whitelist_test ADD PARTITION (ds='Part');
-
-
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/alter_partition_with_includelist.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/alter_partition_with_includelist.q
new file mode 100644
index 0000000..fcef12c
--- /dev/null
+++ b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/alter_partition_with_includelist.q
@@ -0,0 +1,9 @@
+SET hive.metastore.partition.name.whitelist.pattern=[A-Za-z]*;
+-- This pattern matches only letters.
+
+CREATE TABLE part_includelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
+SHOW PARTITIONS part_includelist_test;
+
+ALTER TABLE part_includelist_test ADD PARTITION (ds='Part');
+
+ALTER TABLE part_includelist_test PARTITION (ds='Part') rename to partition (ds='Apart');
diff --git a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/alter_partition_with_whitelist.q b/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/alter_partition_with_whitelist.q
deleted file mode 100644
index 301362a..0000000
--- a/sql/hive/src/test/resources/ql/src/test/queries/clientpositive/alter_partition_with_whitelist.q
+++ /dev/null
@@ -1,9 +0,0 @@
-SET hive.metastore.partition.name.whitelist.pattern=[A-Za-z]*;
--- This pattern matches only letters.
-
-CREATE TABLE part_whitelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
-SHOW PARTITIONS part_whitelist_test;
-
-ALTER TABLE part_whitelist_test ADD PARTITION (ds='Part');
-
-ALTER TABLE part_whitelist_test PARTITION (ds='Part') rename to partition (ds='Apart');
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
index 1dd2ad3..87771ee 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
@@ -781,7 +781,7 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
assert(math.abs(corr6 + 1.0) < 1e-12)
// Test for udaf_corr in HiveCompatibilitySuite
- // udaf_corr has been blacklisted due to numerical errors
+ // udaf_corr has been excluded due to numerical errors
// We test it here:
// SELECT corr(b, c) FROM covar_tab WHERE a < 1; => NULL
// SELECT corr(b, c) FROM covar_tab WHERE a < 3; => NULL
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala
index bb4ce6d..192fff2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala
@@ -25,18 +25,18 @@ import org.apache.spark.sql.catalyst.util._
* A framework for running the query tests that are listed as a set of text files.
*
* TestSuites that derive from this class must provide a map of testCaseName to testCaseFiles
- * that should be included. Additionally, there is support for whitelisting and blacklisting
+ * that should be included. Additionally, there is support for including and excluding
* tests as development progresses.
*/
abstract class HiveQueryFileTest extends HiveComparisonTest {
/** A list of tests deemed out of scope and thus completely disregarded */
- def blackList: Seq[String] = Nil
+ def excludeList: Seq[String] = Nil
/**
- * The set of tests that are believed to be working in catalyst. Tests not in whiteList
- * blacklist are implicitly marked as ignored.
+ * The set of tests that are believed to be working in catalyst. Tests not in includeList or
+ * excludeList are implicitly marked as ignored.
*/
- def whiteList: Seq[String] = ".*" :: Nil
+ def includeList: Seq[String] = ".*" :: Nil
def testCases: Seq[(String, File)]
@@ -45,25 +45,34 @@ abstract class HiveQueryFileTest extends HiveComparisonTest {
runOnlyDirectories.nonEmpty ||
skipDirectories.nonEmpty
- val whiteListProperty: String = "spark.hive.whitelist"
- // Allow the whiteList to be overridden by a system property
- val realWhiteList: Seq[String] =
- Option(System.getProperty(whiteListProperty)).map(_.split(",").toSeq).getOrElse(whiteList)
+ val deprecatedIncludeListProperty: String = "spark.hive.whitelist"
+ val includeListProperty: String = "spark.hive.includelist"
+ if (System.getProperty(deprecatedIncludeListProperty) != null) {
+ logWarning(s"System property `$deprecatedIncludeListProperty` is deprecated; please update " +
+ s"to use new property: $includeListProperty")
+ }
+ // Allow the includeList to be overridden by a system property
+ val realIncludeList: Seq[String] =
+ Option(System.getProperty(includeListProperty))
+ .orElse(Option(System.getProperty(deprecatedIncludeListProperty)))
+ .map(_.split(",").toSeq)
+ .getOrElse(includeList)
// Go through all the test cases and add them to scala test.
testCases.sorted.foreach {
case (testCaseName, testCaseFile) =>
- if (blackList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_)) {
- logDebug(s"Blacklisted test skipped $testCaseName")
- } else if (realWhiteList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_) ||
+ if (excludeList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_)) {
+ logDebug(s"Excluded test skipped $testCaseName")
+ } else if (
+ realIncludeList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_) ||
runAll) {
// Build a test case and submit it to scala test framework...
val queriesString = fileToString(testCaseFile)
createQueryTest(testCaseName, queriesString, reset = true, tryWithoutResettingFirst = true)
} else {
- // Only output warnings for the built in whitelist as this clutters the output when the user
- // trying to execute a single test from the commandline.
- if (System.getProperty(whiteListProperty) == null && !runAll) {
+ // Only output warnings for the built in includeList as this clutters the output when the
+ // user is trying to execute a single test from the commandline.
+ if (System.getProperty(includeListProperty) == null && !runAll) {
ignore(testCaseName) {}
}
}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/security/HiveHadoopDelegationTokenManagerSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/security/HiveHadoopDelegationTokenManagerSuite.scala
index 97eab4f..f8f5551 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/security/HiveHadoopDelegationTokenManagerSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/security/HiveHadoopDelegationTokenManagerSuite.scala
@@ -52,8 +52,8 @@ class HiveHadoopDelegationTokenManagerSuite extends SparkFunSuite {
throw new ClassNotFoundException(name)
}
- val prefixBlacklist = Seq("java", "scala", "com.sun.", "sun.")
- if (prefixBlacklist.exists(name.startsWith(_))) {
+ val prefixExcludeList = Seq("java", "scala", "com.sun.", "sun.")
+ if (prefixExcludeList.exists(name.startsWith(_))) {
return currentLoader.loadClass(name)
}
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/FileBasedWriteAheadLog.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/FileBasedWriteAheadLog.scala
index 6f3ee5c..2e50001 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/FileBasedWriteAheadLog.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/FileBasedWriteAheadLog.scala
@@ -277,10 +277,10 @@ private[streaming] object FileBasedWriteAheadLog {
}
def getCallerName(): Option[String] = {
- val blacklist = Seq("WriteAheadLog", "Logging", "java.lang", "scala.")
+ val ignoreList = Seq("WriteAheadLog", "Logging", "java.lang", "scala.")
Thread.currentThread.getStackTrace()
.map(_.getClassName)
- .find { c => !blacklist.exists(c.contains) }
+ .find { c => !ignoreList.exists(c.contains) }
.flatMap(_.split("\\.").lastOption)
.flatMap(_.split("\\$\\$").headOption)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org