You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/07/13 09:06:34 UTC
[5/5] spark git commit: [SPARK-19810][BUILD][CORE] Remove support for
Scala 2.10
[SPARK-19810][BUILD][CORE] Remove support for Scala 2.10
## What changes were proposed in this pull request?
- Remove Scala 2.10 build profiles and support
- Replace some 2.10 support in scripts with commented placeholders for 2.12 later
- Remove deprecated API calls from 2.10 support
- Remove usages of deprecated context bounds where possible
- Remove Scala 2.10 workarounds like ScalaReflectionLock
- Other minor Scala warning fixes
## How was this patch tested?
Existing tests
Author: Sean Owen <so...@cloudera.com>
Closes #17150 from srowen/SPARK-19810.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/425c4ada
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/425c4ada
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/425c4ada
Branch: refs/heads/master
Commit: 425c4ada4c24e338b45d0e9987071f05c5766fa5
Parents: e08d06b
Author: Sean Owen <so...@cloudera.com>
Authored: Thu Jul 13 17:06:24 2017 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Thu Jul 13 17:06:24 2017 +0800
----------------------------------------------------------------------
R/pkg/R/sparkR.R | 4 +-
R/pkg/tests/fulltests/test_client.R | 4 +-
bin/load-spark-env.cmd | 22 +-
bin/load-spark-env.sh | 22 +-
build/mvn | 6 +-
.../scala/org/apache/spark/Accumulable.scala | 3 +-
.../scala/org/apache/spark/SparkContext.scala | 15 +-
.../spark/rdd/SequenceFileRDDFunctions.scala | 54 +-
.../scala/org/apache/spark/rpc/RpcTimeout.scala | 4 +-
.../scala/org/apache/spark/ui/JettyUtils.scala | 6 +-
.../spark/util/logging/FileAppender.scala | 8 +-
.../test/scala/org/apache/spark/FileSuite.scala | 4 +-
.../spark/deploy/SparkSubmitUtilsSuite.scala | 6 +-
.../spark/deploy/master/MasterSuite.scala | 3 +-
.../apache/spark/executor/ExecutorSuite.scala | 1 +
.../apache/spark/rdd/LocalCheckpointSuite.scala | 1 +
.../OutputCommitCoordinatorSuite.scala | 2 +-
.../spark/scheduler/SparkListenerSuite.scala | 2 +-
.../spark/scheduler/TaskSetBlacklistSuite.scala | 4 +-
.../spark/serializer/KryoSerializerSuite.scala | 4 +-
.../spark/util/TimeStampedHashMapSuite.scala | 2 +-
.../apache/spark/util/VersionUtilsSuite.scala | 8 +-
.../util/collection/AppendOnlyMapSuite.scala | 4 +-
.../util/collection/ExternalSorterSuite.scala | 14 +-
.../util/collection/OpenHashMapSuite.scala | 5 +-
dev/change-scala-version.sh | 4 +-
dev/change-version-to-2.10.sh | 23 -
dev/change-version-to-2.11.sh | 23 -
dev/create-release/release-build.sh | 23 +-
docs/building-spark.md | 8 -
docs/index.md | 3 +-
.../graphx/AggregateMessagesExample.scala | 2 +-
.../spark/examples/mllib/AbstractParams.scala | 2 +-
.../spark/examples/sql/SparkSQLExample.scala | 2 -
.../apache/spark/streaming/kafka/KafkaRDD.scala | 4 +-
.../org/apache/spark/graphx/EdgeContext.scala | 2 +-
.../spark/launcher/AbstractCommandBuilder.java | 20 +-
.../org/apache/spark/ml/linalg/Matrices.scala | 2 +-
.../apache/spark/ml/linalg/MatricesSuite.scala | 20 +-
.../org/apache/spark/ml/tree/treeModels.scala | 2 +-
.../apache/spark/mllib/linalg/Matrices.scala | 2 +-
.../mllib/linalg/distributed/BlockMatrix.scala | 4 +-
.../spark/ml/recommendation/ALSSuite.scala | 2 +-
.../spark/mllib/clustering/LDASuite.scala | 2 +-
.../spark/mllib/linalg/MatricesSuite.scala | 20 +-
pom.xml | 68 +-
project/SparkBuild.scala | 23 +-
python/run-tests.py | 3 +-
repl/pom.xml | 26 +-
.../main/scala/org/apache/spark/repl/Main.scala | 37 -
.../apache/spark/repl/SparkCommandLine.scala | 46 -
.../org/apache/spark/repl/SparkExprTyper.scala | 114 --
.../org/apache/spark/repl/SparkHelper.scala | 39 -
.../org/apache/spark/repl/SparkILoop.scala | 1145 -----------
.../org/apache/spark/repl/SparkILoopInit.scala | 168 --
.../org/apache/spark/repl/SparkIMain.scala | 1808 ------------------
.../org/apache/spark/repl/SparkImports.scala | 239 ---
.../spark/repl/SparkJLineCompletion.scala | 403 ----
.../apache/spark/repl/SparkJLineReader.scala | 90 -
.../apache/spark/repl/SparkMemberHandlers.scala | 232 ---
.../apache/spark/repl/SparkRunnerSettings.scala | 31 -
.../scala/org/apache/spark/repl/ReplSuite.scala | 366 ----
.../sql/execution/UnsafeExternalRowSorter.java | 4 +-
.../spark/sql/catalyst/ScalaReflection.scala | 36 +-
.../spark/sql/catalyst/analysis/Analyzer.scala | 4 +-
.../catalyst/expressions/codegen/package.scala | 4 +-
.../catalyst/expressions/namedExpressions.scala | 2 +-
.../org/apache/spark/sql/catalyst/package.scala | 6 -
.../util/AbstractScalaRowIterator.scala | 30 -
.../org/apache/spark/sql/types/BinaryType.scala | 3 +-
.../apache/spark/sql/types/BooleanType.scala | 3 +-
.../org/apache/spark/sql/types/ByteType.scala | 3 +-
.../org/apache/spark/sql/types/DateType.scala | 4 +-
.../apache/spark/sql/types/DecimalType.scala | 3 +-
.../org/apache/spark/sql/types/DoubleType.scala | 3 +-
.../org/apache/spark/sql/types/FloatType.scala | 3 +-
.../apache/spark/sql/types/HiveStringType.scala | 5 +-
.../apache/spark/sql/types/IntegerType.scala | 4 +-
.../org/apache/spark/sql/types/LongType.scala | 3 +-
.../org/apache/spark/sql/types/ShortType.scala | 3 +-
.../org/apache/spark/sql/types/StringType.scala | 3 +-
.../apache/spark/sql/types/TimestampType.scala | 4 +-
.../optimizer/JoinOptimizationSuite.scala | 24 +-
.../optimizer/OptimizeCodegenSuite.scala | 4 +-
.../scala/org/apache/spark/sql/Column.scala | 2 +-
.../execution/OptimizeMetadataOnlyQuery.scala | 8 +-
.../spark/sql/execution/QueryExecution.scala | 4 +-
.../execution/python/ExtractPythonUDFs.scala | 2 +-
.../org/apache/spark/sql/DatasetSuite.scala | 4 +-
.../spark/sql/StatisticsCollectionSuite.scala | 20 +-
.../org/apache/spark/sql/SubquerySuite.scala | 30 +-
.../datasources/parquet/ParquetIOSuite.scala | 2 +-
.../sql/execution/metric/SQLMetricsSuite.scala | 60 +-
.../sql/execution/ui/SQLListenerSuite.scala | 2 +-
.../sql/expressions/ReduceAggregatorSuite.scala | 14 +-
.../streaming/StreamingQueryListenerSuite.scala | 2 -
.../org/apache/spark/sql/hive/HiveUtils.scala | 4 +-
.../sql/hive/HiveMetastoreCatalogSuite.scala | 8 +-
.../spark/sql/hive/HiveSparkSubmitSuite.scala | 2 +-
.../spark/sql/hive/execution/HiveDDLSuite.scala | 2 +
.../sql/hive/execution/HiveQuerySuite.scala | 2 +-
101 files changed, 311 insertions(+), 5231 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/R/pkg/R/sparkR.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R
index f2d2620..81507ea 100644
--- a/R/pkg/R/sparkR.R
+++ b/R/pkg/R/sparkR.R
@@ -113,7 +113,7 @@ sparkR.stop <- function() {
#' list(spark.executor.memory="4g"),
#' list(LD_LIBRARY_PATH="/directory of JVM libraries (libjvm.so) on workers/"),
#' c("one.jar", "two.jar", "three.jar"),
-#' c("com.databricks:spark-avro_2.10:2.0.1"))
+#' c("com.databricks:spark-avro_2.11:2.0.1"))
#'}
#' @note sparkR.init since 1.4.0
sparkR.init <- function(
@@ -357,7 +357,7 @@ sparkRHive.init <- function(jsc = NULL) {
#' sparkR.session("yarn-client", "SparkR", "/home/spark",
#' list(spark.executor.memory="4g"),
#' c("one.jar", "two.jar", "three.jar"),
-#' c("com.databricks:spark-avro_2.10:2.0.1"))
+#' c("com.databricks:spark-avro_2.11:2.0.1"))
#' sparkR.session(spark.master = "yarn-client", spark.executor.memory = "4g")
#'}
#' @note sparkR.session since 2.0.0
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/R/pkg/tests/fulltests/test_client.R
----------------------------------------------------------------------
diff --git a/R/pkg/tests/fulltests/test_client.R b/R/pkg/tests/fulltests/test_client.R
index 0cf25fe..de624b5 100644
--- a/R/pkg/tests/fulltests/test_client.R
+++ b/R/pkg/tests/fulltests/test_client.R
@@ -37,7 +37,7 @@ test_that("multiple packages don't produce a warning", {
test_that("sparkJars sparkPackages as character vectors", {
args <- generateSparkSubmitArgs("", "", c("one.jar", "two.jar", "three.jar"), "",
- c("com.databricks:spark-avro_2.10:2.0.1"))
+ c("com.databricks:spark-avro_2.11:2.0.1"))
expect_match(args, "--jars one.jar,two.jar,three.jar")
- expect_match(args, "--packages com.databricks:spark-avro_2.10:2.0.1")
+ expect_match(args, "--packages com.databricks:spark-avro_2.11:2.0.1")
})
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/bin/load-spark-env.cmd
----------------------------------------------------------------------
diff --git a/bin/load-spark-env.cmd b/bin/load-spark-env.cmd
index 0977025..993aa31 100644
--- a/bin/load-spark-env.cmd
+++ b/bin/load-spark-env.cmd
@@ -35,21 +35,21 @@ if [%SPARK_ENV_LOADED%] == [] (
rem Setting SPARK_SCALA_VERSION if not already set.
-set ASSEMBLY_DIR2="%SPARK_HOME%\assembly\target\scala-2.11"
-set ASSEMBLY_DIR1="%SPARK_HOME%\assembly\target\scala-2.10"
+rem set ASSEMBLY_DIR2="%SPARK_HOME%\assembly\target\scala-2.11"
+rem set ASSEMBLY_DIR1="%SPARK_HOME%\assembly\target\scala-2.12"
if [%SPARK_SCALA_VERSION%] == [] (
- if exist %ASSEMBLY_DIR2% if exist %ASSEMBLY_DIR1% (
- echo "Presence of build for both scala versions(SCALA 2.10 and SCALA 2.11) detected."
- echo "Either clean one of them or, set SPARK_SCALA_VERSION=2.11 in spark-env.cmd."
- exit 1
- )
- if exist %ASSEMBLY_DIR2% (
+ rem if exist %ASSEMBLY_DIR2% if exist %ASSEMBLY_DIR1% (
+ rem echo "Presence of build for multiple Scala versions detected."
+ rem echo "Either clean one of them or, set SPARK_SCALA_VERSION=2.11 in spark-env.cmd."
+ rem exit 1
+ rem )
+ rem if exist %ASSEMBLY_DIR2% (
set SPARK_SCALA_VERSION=2.11
- ) else (
- set SPARK_SCALA_VERSION=2.10
- )
+ rem ) else (
+ rem set SPARK_SCALA_VERSION=2.12
+ rem )
)
exit /b 0
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/bin/load-spark-env.sh
----------------------------------------------------------------------
diff --git a/bin/load-spark-env.sh b/bin/load-spark-env.sh
index 8a2f709..9de6203 100644
--- a/bin/load-spark-env.sh
+++ b/bin/load-spark-env.sh
@@ -46,18 +46,18 @@ fi
if [ -z "$SPARK_SCALA_VERSION" ]; then
- ASSEMBLY_DIR2="${SPARK_HOME}/assembly/target/scala-2.11"
- ASSEMBLY_DIR1="${SPARK_HOME}/assembly/target/scala-2.10"
+ #ASSEMBLY_DIR2="${SPARK_HOME}/assembly/target/scala-2.11"
+ #ASSEMBLY_DIR1="${SPARK_HOME}/assembly/target/scala-2.12"
- if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then
- echo -e "Presence of build for both scala versions(SCALA 2.10 and SCALA 2.11) detected." 1>&2
- echo -e 'Either clean one of them or, export SPARK_SCALA_VERSION=2.11 in spark-env.sh.' 1>&2
- exit 1
- fi
+ #if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then
+ # echo -e "Presence of build for multiple Scala versions detected." 1>&2
+ # echo -e 'Either clean one of them or, export SPARK_SCALA_VERSION=2.11 in spark-env.sh.' 1>&2
+ # exit 1
+ #fi
- if [ -d "$ASSEMBLY_DIR2" ]; then
+ #if [ -d "$ASSEMBLY_DIR2" ]; then
export SPARK_SCALA_VERSION="2.11"
- else
- export SPARK_SCALA_VERSION="2.10"
- fi
+ #else
+ # export SPARK_SCALA_VERSION="2.12"
+ #fi
fi
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/build/mvn
----------------------------------------------------------------------
diff --git a/build/mvn b/build/mvn
index 1e393c3..efa4f93 100755
--- a/build/mvn
+++ b/build/mvn
@@ -91,13 +91,13 @@ install_mvn() {
# Install zinc under the build/ folder
install_zinc() {
- local zinc_path="zinc-0.3.11/bin/zinc"
+ local zinc_path="zinc-0.3.15/bin/zinc"
[ ! -f "${_DIR}/${zinc_path}" ] && ZINC_INSTALL_FLAG=1
local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.typesafe.com}
install_app \
- "${TYPESAFE_MIRROR}/zinc/0.3.11" \
- "zinc-0.3.11.tgz" \
+ "${TYPESAFE_MIRROR}/zinc/0.3.15" \
+ "zinc-0.3.15.tgz" \
"${zinc_path}"
ZINC_BIN="${_DIR}/${zinc_path}"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/main/scala/org/apache/spark/Accumulable.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/Accumulable.scala b/core/src/main/scala/org/apache/spark/Accumulable.scala
index 5532931..3092074 100644
--- a/core/src/main/scala/org/apache/spark/Accumulable.scala
+++ b/core/src/main/scala/org/apache/spark/Accumulable.scala
@@ -201,7 +201,8 @@ trait AccumulableParam[R, T] extends Serializable {
@deprecated("use AccumulatorV2", "2.0.0")
private[spark] class
-GrowableAccumulableParam[R <% Growable[T] with TraversableOnce[T] with Serializable: ClassTag, T]
+GrowableAccumulableParam[R : ClassTag, T]
+ (implicit rg: R => Growable[T] with TraversableOnce[T] with Serializable)
extends AccumulableParam[R, T] {
def addAccumulator(growable: R, elem: T): R = {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index b2a26c5..e1ce66a 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -183,8 +183,6 @@ class SparkContext(config: SparkConf) extends Logging {
// log out Spark Version in Spark driver log
logInfo(s"Running Spark version $SPARK_VERSION")
- warnDeprecatedVersions()
-
/* ------------------------------------------------------------------------------------- *
| Private variables. These variables keep the internal state of the context, and are |
| not accessible by the outside world. They're mutable since we want to initialize all |
@@ -349,13 +347,6 @@ class SparkContext(config: SparkConf) extends Logging {
value
}
- private def warnDeprecatedVersions(): Unit = {
- val javaVersion = System.getProperty("java.version").split("[+.\\-]+", 3)
- if (scala.util.Properties.releaseVersion.exists(_.startsWith("2.10"))) {
- logWarning("Support for Scala 2.10 is deprecated as of Spark 2.1.0")
- }
- }
-
/** Control our logLevel. This overrides any user-defined log settings.
* @param logLevel The desired log level as a string.
* Valid log levels include: ALL, DEBUG, ERROR, FATAL, INFO, OFF, TRACE, WARN
@@ -1396,6 +1387,8 @@ class SparkContext(config: SparkConf) extends Logging {
@deprecated("use AccumulatorV2", "2.0.0")
def accumulableCollection[R <% Growable[T] with TraversableOnce[T] with Serializable: ClassTag, T]
(initialValue: R): Accumulable[R, T] = {
+ // TODO the context bound (<%) above should be replaced with simple type bound and implicit
+ // conversion but is a breaking change. This should be fixed in Spark 3.x.
val param = new GrowableAccumulableParam[R, T]
val acc = new Accumulable(initialValue, param)
cleaner.foreach(_.registerAccumulatorForCleanup(acc.newAcc))
@@ -2605,9 +2598,9 @@ object SparkContext extends Logging {
*/
private[spark] val LEGACY_DRIVER_IDENTIFIER = "<driver>"
- private implicit def arrayToArrayWritable[T <% Writable: ClassTag](arr: Traversable[T])
+ private implicit def arrayToArrayWritable[T <: Writable : ClassTag](arr: Traversable[T])
: ArrayWritable = {
- def anyToWritable[U <% Writable](u: U): Writable = u
+ def anyToWritable[U <: Writable](u: U): Writable = u
new ArrayWritable(classTag[T].runtimeClass.asInstanceOf[Class[Writable]],
arr.map(x => anyToWritable(x)).toArray)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
index 86a3327..02def89 100644
--- a/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
@@ -16,7 +16,7 @@
*/
package org.apache.spark.rdd
-import scala.reflect.{classTag, ClassTag}
+import scala.reflect.ClassTag
import org.apache.hadoop.io.Writable
import org.apache.hadoop.io.compress.CompressionCodec
@@ -39,40 +39,8 @@ class SequenceFileRDDFunctions[K <% Writable: ClassTag, V <% Writable : ClassTag
extends Logging
with Serializable {
- private val keyWritableClass =
- if (_keyWritableClass == null) {
- // pre 1.3.0, we need to use Reflection to get the Writable class
- getWritableClass[K]()
- } else {
- _keyWritableClass
- }
-
- private val valueWritableClass =
- if (_valueWritableClass == null) {
- // pre 1.3.0, we need to use Reflection to get the Writable class
- getWritableClass[V]()
- } else {
- _valueWritableClass
- }
-
- private def getWritableClass[T <% Writable: ClassTag](): Class[_ <: Writable] = {
- val c = {
- if (classOf[Writable].isAssignableFrom(classTag[T].runtimeClass)) {
- classTag[T].runtimeClass
- } else {
- // We get the type of the Writable class by looking at the apply method which converts
- // from T to Writable. Since we have two apply methods we filter out the one which
- // is not of the form "java.lang.Object apply(java.lang.Object)"
- implicitly[T => Writable].getClass.getDeclaredMethods().filter(
- m => m.getReturnType().toString != "class java.lang.Object" &&
- m.getName() == "apply")(0).getReturnType
-
- }
- // TODO: use something like WritableConverter to avoid reflection
- }
- c.asInstanceOf[Class[_ <: Writable]]
- }
-
+ // TODO the context bound (<%) above should be replaced with simple type bound and implicit
+ // conversion but is a breaking change. This should be fixed in Spark 3.x.
/**
* Output the RDD as a Hadoop SequenceFile using the Writable types we infer from the RDD's key
@@ -90,24 +58,24 @@ class SequenceFileRDDFunctions[K <% Writable: ClassTag, V <% Writable : ClassTag
// valueWritableClass at the compile time. To implement that, we need to add type parameters to
// SequenceFileRDDFunctions. however, SequenceFileRDDFunctions is a public class so it will be a
// breaking change.
- val convertKey = self.keyClass != keyWritableClass
- val convertValue = self.valueClass != valueWritableClass
+ val convertKey = self.keyClass != _keyWritableClass
+ val convertValue = self.valueClass != _valueWritableClass
- logInfo("Saving as sequence file of type (" + keyWritableClass.getSimpleName + "," +
- valueWritableClass.getSimpleName + ")" )
+ logInfo("Saving as sequence file of type " +
+ s"(${_keyWritableClass.getSimpleName},${_valueWritableClass.getSimpleName})" )
val format = classOf[SequenceFileOutputFormat[Writable, Writable]]
val jobConf = new JobConf(self.context.hadoopConfiguration)
if (!convertKey && !convertValue) {
- self.saveAsHadoopFile(path, keyWritableClass, valueWritableClass, format, jobConf, codec)
+ self.saveAsHadoopFile(path, _keyWritableClass, _valueWritableClass, format, jobConf, codec)
} else if (!convertKey && convertValue) {
self.map(x => (x._1, anyToWritable(x._2))).saveAsHadoopFile(
- path, keyWritableClass, valueWritableClass, format, jobConf, codec)
+ path, _keyWritableClass, _valueWritableClass, format, jobConf, codec)
} else if (convertKey && !convertValue) {
self.map(x => (anyToWritable(x._1), x._2)).saveAsHadoopFile(
- path, keyWritableClass, valueWritableClass, format, jobConf, codec)
+ path, _keyWritableClass, _valueWritableClass, format, jobConf, codec)
} else if (convertKey && convertValue) {
self.map(x => (anyToWritable(x._1), anyToWritable(x._2))).saveAsHadoopFile(
- path, keyWritableClass, valueWritableClass, format, jobConf, codec)
+ path, _keyWritableClass, _valueWritableClass, format, jobConf, codec)
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala b/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala
index 0557b7a..3dc41f7 100644
--- a/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala
@@ -125,9 +125,9 @@ private[spark] object RpcTimeout {
var foundProp: Option[(String, String)] = None
while (itr.hasNext && foundProp.isEmpty) {
val propKey = itr.next()
- conf.getOption(propKey).foreach { prop => foundProp = Some(propKey, prop) }
+ conf.getOption(propKey).foreach { prop => foundProp = Some((propKey, prop)) }
}
- val finalProp = foundProp.getOrElse(timeoutPropList.head, defaultValue)
+ val finalProp = foundProp.getOrElse((timeoutPropList.head, defaultValue))
val timeout = { Utils.timeStringAsSeconds(finalProp._2).seconds }
new RpcTimeout(timeout, finalProp._1)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index b9371c7..0fa9671 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -54,7 +54,7 @@ private[spark] object JettyUtils extends Logging {
// implicit conversion from many types of functions to jetty Handlers.
type Responder[T] = HttpServletRequest => T
- class ServletParams[T <% AnyRef](val responder: Responder[T],
+ class ServletParams[T <: AnyRef](val responder: Responder[T],
val contentType: String,
val extractFn: T => String = (in: Any) => in.toString) {}
@@ -68,7 +68,7 @@ private[spark] object JettyUtils extends Logging {
implicit def textResponderToServlet(responder: Responder[String]): ServletParams[String] =
new ServletParams(responder, "text/plain")
- def createServlet[T <% AnyRef](
+ def createServlet[T <: AnyRef](
servletParams: ServletParams[T],
securityMgr: SecurityManager,
conf: SparkConf): HttpServlet = {
@@ -113,7 +113,7 @@ private[spark] object JettyUtils extends Logging {
}
/** Create a context handler that responds to a request with the given path prefix */
- def createServletHandler[T <% AnyRef](
+ def createServletHandler[T <: AnyRef](
path: String,
servletParams: ServletParams[T],
securityMgr: SecurityManager,
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala b/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
index 8a0cc70..2f9ad4c 100644
--- a/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
+++ b/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
@@ -125,16 +125,16 @@ private[spark] object FileAppender extends Logging {
val validatedParams: Option[(Long, String)] = rollingInterval match {
case "daily" =>
logInfo(s"Rolling executor logs enabled for $file with daily rolling")
- Some(24 * 60 * 60 * 1000L, "--yyyy-MM-dd")
+ Some((24 * 60 * 60 * 1000L, "--yyyy-MM-dd"))
case "hourly" =>
logInfo(s"Rolling executor logs enabled for $file with hourly rolling")
- Some(60 * 60 * 1000L, "--yyyy-MM-dd--HH")
+ Some((60 * 60 * 1000L, "--yyyy-MM-dd--HH"))
case "minutely" =>
logInfo(s"Rolling executor logs enabled for $file with rolling every minute")
- Some(60 * 1000L, "--yyyy-MM-dd--HH-mm")
+ Some((60 * 1000L, "--yyyy-MM-dd--HH-mm"))
case IntParam(seconds) =>
logInfo(s"Rolling executor logs enabled for $file with rolling $seconds seconds")
- Some(seconds * 1000L, "--yyyy-MM-dd--HH-mm-ss")
+ Some((seconds * 1000L, "--yyyy-MM-dd--HH-mm-ss"))
case _ =>
logWarning(s"Illegal interval for rolling executor logs [$rollingInterval], " +
s"rolling logs not enabled")
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/FileSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/FileSuite.scala b/core/src/test/scala/org/apache/spark/FileSuite.scala
index 5be0121..0272818 100644
--- a/core/src/test/scala/org/apache/spark/FileSuite.scala
+++ b/core/src/test/scala/org/apache/spark/FileSuite.scala
@@ -113,11 +113,11 @@ class FileSuite extends SparkFunSuite with LocalSparkContext {
val normalFile = new File(normalDir, "part-00000")
val normalContent = sc.sequenceFile[String, String](normalDir).collect
- assert(normalContent === Array.fill(100)("abc", "abc"))
+ assert(normalContent === Array.fill(100)(("abc", "abc")))
val compressedFile = new File(compressedOutputDir, "part-00000" + codec.getDefaultExtension)
val compressedContent = sc.sequenceFile[String, String](compressedOutputDir).collect
- assert(compressedContent === Array.fill(100)("abc", "abc"))
+ assert(compressedContent === Array.fill(100)(("abc", "abc")))
assert(compressedFile.length < normalFile.length)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
index 5702478..88b77e5 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
@@ -93,8 +93,8 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
test("add dependencies works correctly") {
val md = SparkSubmitUtils.getModuleDescriptor
- val artifacts = SparkSubmitUtils.extractMavenCoordinates("com.databricks:spark-csv_2.10:0.1," +
- "com.databricks:spark-avro_2.10:0.1")
+ val artifacts = SparkSubmitUtils.extractMavenCoordinates("com.databricks:spark-csv_2.11:0.1," +
+ "com.databricks:spark-avro_2.11:0.1")
SparkSubmitUtils.addDependenciesToIvy(md, artifacts, "default")
assert(md.getDependencies.length === 2)
@@ -196,7 +196,7 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
SparkSubmitUtils.buildIvySettings(None, None),
isTest = true)
assert(path === "", "should return empty path")
- val main = MavenCoordinate("org.apache.spark", "spark-streaming-kafka-assembly_2.10", "1.2.0")
+ val main = MavenCoordinate("org.apache.spark", "spark-streaming-kafka-assembly_2.11", "1.2.0")
IvyTestUtils.withRepository(main, None, None) { repo =>
val files = SparkSubmitUtils.resolveMavenCoordinates(
coordinates + "," + main.toString,
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
index a223212..84b3a29 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
@@ -80,6 +80,7 @@ class MockWorker(master: RpcEndpointRef, conf: SparkConf = new SparkConf) extend
case Some(appId) =>
apps.remove(appId)
master.send(UnregisterApplication(appId))
+ case None =>
}
driverIdToAppId.remove(driverId)
}
@@ -575,7 +576,7 @@ class MasterSuite extends SparkFunSuite
override val rpcEnv: RpcEnv = master.rpcEnv
override def receive: PartialFunction[Any, Unit] = {
- case KillExecutor(_, appId, execId) => killedExecutors.add(appId, execId)
+ case KillExecutor(_, appId, execId) => killedExecutors.add((appId, execId))
case KillDriver(driverId) => killedDrivers.add(driverId)
}
})
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
index efcad14..601dde6 100644
--- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
@@ -25,6 +25,7 @@ import java.util.concurrent.{CountDownLatch, TimeUnit}
import scala.collection.mutable.Map
import scala.concurrent.duration._
+import scala.language.postfixOps
import org.mockito.ArgumentCaptor
import org.mockito.Matchers.{any, eq => meq}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala b/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala
index 9e204f5..478f069 100644
--- a/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala
@@ -18,6 +18,7 @@
package org.apache.spark.rdd
import scala.concurrent.duration._
+import scala.language.postfixOps
import org.scalatest.concurrent.Eventually.{eventually, interval, timeout}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorSuite.scala
index 1579b61..60b5955 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorSuite.scala
@@ -115,7 +115,7 @@ class OutputCommitCoordinatorSuite extends SparkFunSuite with BeforeAndAfter {
locality: TaskLocality.Value): Option[(Int, TaskLocality.Value)] = {
if (!hasDequeuedSpeculatedTask) {
hasDequeuedSpeculatedTask = true
- Some(0, TaskLocality.PROCESS_LOCAL)
+ Some((0, TaskLocality.PROCESS_LOCAL))
} else {
None
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
index f3d0bc1..481603b 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
@@ -299,7 +299,7 @@ class SparkListenerSuite extends SparkFunSuite with LocalSparkContext with Match
val d2 = d.map { i => w(i) -> i * 2 }.setName("shuffle input 1")
val d3 = d.map { i => w(i) -> (0 to (i % 5)) }.setName("shuffle input 2")
val d4 = d2.cogroup(d3, numSlices).map { case (k, (v1, v2)) =>
- w(k) -> (v1.size, v2.size)
+ (w(k), (v1.size, v2.size))
}
d4.setName("A Cogroup")
d4.collectAsMap()
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala
index 6b52c10..f1392e9 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala
@@ -86,8 +86,8 @@ class TaskSetBlacklistSuite extends SparkFunSuite {
Seq("exec1", "exec2").foreach { exec =>
assert(
execToFailures(exec).taskToFailureCountAndFailureTime === Map(
- 0 -> (1, 0),
- 1 -> (1, 0)
+ 0 -> ((1, 0)),
+ 1 -> ((1, 0))
)
)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index 7c3922e..eaec098 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -276,7 +276,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
}
test("kryo with collect for specialized tuples") {
- assert (sc.parallelize( Array((1, 11), (2, 22), (3, 33)) ).collect().head === (1, 11))
+ assert (sc.parallelize( Array((1, 11), (2, 22), (3, 33)) ).collect().head === ((1, 11)))
}
test("kryo with SerializableHyperLogLog") {
@@ -475,7 +475,7 @@ class KryoSerializerAutoResetDisabledSuite extends SparkFunSuite with SharedSpar
val deserializationStream = serInstance.deserializeStream(new ByteArrayInputStream(worldWorld))
assert(deserializationStream.readValue[Any]() === world)
deserializationStream.close()
- assert(serInstance.deserialize[Any](helloHello) === (hello, hello))
+ assert(serInstance.deserialize[Any](helloHello) === ((hello, hello)))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/util/TimeStampedHashMapSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/TimeStampedHashMapSuite.scala b/core/src/test/scala/org/apache/spark/util/TimeStampedHashMapSuite.scala
index fd9add7..dcae789 100644
--- a/core/src/test/scala/org/apache/spark/util/TimeStampedHashMapSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/TimeStampedHashMapSuite.scala
@@ -115,7 +115,7 @@ class TimeStampedHashMapSuite extends SparkFunSuite {
testMap2("k1") = "v1"
testMap2 --= keys
assert(testMap2.size === 1)
- assert(testMap2.iterator.toSeq.head === ("k1", "v1"))
+ assert(testMap2.iterator.toSeq.head === (("k1", "v1")))
// +
val testMap3 = testMap2 + (("k0", "v0"))
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
index aaf79eb..b36d6be 100644
--- a/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
@@ -58,10 +58,10 @@ class VersionUtilsSuite extends SparkFunSuite {
}
test("Parse Spark major and minor versions") {
- assert(majorMinorVersion("2.0") === (2, 0))
- assert(majorMinorVersion("12.10.11") === (12, 10))
- assert(majorMinorVersion("2.0.1-SNAPSHOT") === (2, 0))
- assert(majorMinorVersion("2.0.x") === (2, 0))
+ assert(majorMinorVersion("2.0") === ((2, 0)))
+ assert(majorMinorVersion("12.10.11") === ((12, 10)))
+ assert(majorMinorVersion("2.0.1-SNAPSHOT") === ((2, 0)))
+ assert(majorMinorVersion("2.0.x") === ((2, 0)))
withClue("majorMinorVersion parsing should fail for invalid major version number") {
intercept[IllegalArgumentException] {
majorMinorVersion("2z.0")
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/util/collection/AppendOnlyMapSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/collection/AppendOnlyMapSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/AppendOnlyMapSuite.scala
index a2a6d70..6b4e928 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/AppendOnlyMapSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/AppendOnlyMapSuite.scala
@@ -181,9 +181,9 @@ class AppendOnlyMapSuite extends SparkFunSuite {
// Should be sorted by key
assert(it.hasNext)
var previous = it.next()
- assert(previous == (null, "happy new year!"))
+ assert(previous == ((null, "happy new year!")))
previous = it.next()
- assert(previous == ("1", "2014"))
+ assert(previous == (("1", "2014")))
while (it.hasNext) {
val kv = it.next()
assert(kv._1.toInt > previous._1.toInt)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/util/collection/ExternalSorterSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/collection/ExternalSorterSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/ExternalSorterSuite.scala
index 6bcc601..47173b8 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/ExternalSorterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/ExternalSorterSuite.scala
@@ -388,13 +388,13 @@ class ExternalSorterSuite extends SparkFunSuite with LocalSparkContext {
sorter.insertAll(elements)
assert(sorter.numSpills > 0, "sorter did not spill")
val iter = sorter.partitionedIterator.map(p => (p._1, p._2.toList))
- assert(iter.next() === (0, Nil))
- assert(iter.next() === (1, List((1, 1))))
- assert(iter.next() === (2, (0 until 1000).map(x => (2, 2)).toList))
- assert(iter.next() === (3, Nil))
- assert(iter.next() === (4, Nil))
- assert(iter.next() === (5, List((5, 5))))
- assert(iter.next() === (6, Nil))
+ assert(iter.next() === ((0, Nil)))
+ assert(iter.next() === ((1, List((1, 1)))))
+ assert(iter.next() === ((2, (0 until 1000).map(x => (2, 2)).toList)))
+ assert(iter.next() === ((3, Nil)))
+ assert(iter.next() === ((4, Nil)))
+ assert(iter.next() === ((5, List((5, 5)))))
+ assert(iter.next() === ((6, Nil)))
sorter.stop()
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
index 335ecb9..08a3200 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
@@ -75,7 +75,7 @@ class OpenHashMapSuite extends SparkFunSuite with Matchers {
for ((k, v) <- map) {
set.add((k, v))
}
- val expected = (1 to 1000).map(x => (x.toString, x)) :+ (null.asInstanceOf[String], -1)
+ val expected = (1 to 1000).map(x => (x.toString, x)) :+ ((null.asInstanceOf[String], -1))
assert(set === expected.toSet)
}
@@ -103,7 +103,8 @@ class OpenHashMapSuite extends SparkFunSuite with Matchers {
for ((k, v) <- map) {
set.add((k, v))
}
- val expected = (1 to 1000).map(_.toString).map(x => (x, x)) :+ (null.asInstanceOf[String], "-1")
+ val expected =
+ (1 to 1000).map(_.toString).map(x => (x, x)) :+ ((null.asInstanceOf[String], "-1"))
assert(set === expected.toSet)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/dev/change-scala-version.sh
----------------------------------------------------------------------
diff --git a/dev/change-scala-version.sh b/dev/change-scala-version.sh
index d7975df..022e68c 100755
--- a/dev/change-scala-version.sh
+++ b/dev/change-scala-version.sh
@@ -19,7 +19,7 @@
set -e
-VALID_VERSIONS=( 2.10 2.11 )
+VALID_VERSIONS=( 2.11 2.12 )
usage() {
echo "Usage: $(basename $0) [-h|--help] <version>
@@ -45,7 +45,7 @@ check_scala_version() {
check_scala_version "$TO_VERSION"
if [ $TO_VERSION = "2.11" ]; then
- FROM_VERSION="2.10"
+ FROM_VERSION="2.12"
else
FROM_VERSION="2.11"
fi
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/dev/change-version-to-2.10.sh
----------------------------------------------------------------------
diff --git a/dev/change-version-to-2.10.sh b/dev/change-version-to-2.10.sh
deleted file mode 100755
index b718d94..0000000
--- a/dev/change-version-to-2.10.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script exists for backwards compatibility. Use change-scala-version.sh instead.
-echo "This script is deprecated. Please instead run: change-scala-version.sh 2.10"
-
-$(dirname $0)/change-scala-version.sh 2.10
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/dev/change-version-to-2.11.sh
----------------------------------------------------------------------
diff --git a/dev/change-version-to-2.11.sh b/dev/change-version-to-2.11.sh
deleted file mode 100755
index 9308795..0000000
--- a/dev/change-version-to-2.11.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script exists for backwards compatibility. Use change-scala-version.sh instead.
-echo "This script is deprecated. Please instead run: change-scala-version.sh 2.11"
-
-$(dirname $0)/change-scala-version.sh 2.11
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/dev/create-release/release-build.sh
----------------------------------------------------------------------
diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index a72307a..9bf2899 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -155,10 +155,10 @@ if [[ "$1" == "package" ]]; then
cd spark-$SPARK_VERSION-bin-$NAME
- # TODO There should probably be a flag to make-distribution to allow 2.10 support
- if [[ $FLAGS == *scala-2.10* ]]; then
- ./dev/change-scala-version.sh 2.10
- fi
+ # TODO There should probably be a flag to make-distribution to allow 2.12 support
+ #if [[ $FLAGS == *scala-2.12* ]]; then
+ # ./dev/change-scala-version.sh 2.12
+ #fi
export ZINC_PORT=$ZINC_PORT
echo "Creating distribution: $NAME ($FLAGS)"
@@ -305,9 +305,9 @@ if [[ "$1" == "publish-snapshot" ]]; then
export ZINC_PORT=$(python -S -c "import random; print random.randrange(3030,4030)")
$MVN -DzincPort=$ZINC_PORT --settings $tmp_settings -DskipTests $PUBLISH_PROFILES deploy
- ./dev/change-scala-version.sh 2.10
- $MVN -DzincPort=$ZINC_PORT -Dscala-2.10 --settings $tmp_settings \
- -DskipTests $PUBLISH_PROFILES clean deploy
+ #./dev/change-scala-version.sh 2.12
+ #$MVN -DzincPort=$ZINC_PORT -Pscala-2.12 --settings $tmp_settings \
+ # -DskipTests $PUBLISH_PROFILES clean deploy
# Clean-up Zinc nailgun process
/usr/sbin/lsof -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
@@ -342,16 +342,13 @@ if [[ "$1" == "publish-release" ]]; then
$MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -DskipTests $PUBLISH_PROFILES clean install
- ./dev/change-scala-version.sh 2.10
-
- $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Dscala-2.10 \
- -DskipTests $PUBLISH_PROFILES clean install
+ #./dev/change-scala-version.sh 2.12
+ #$MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Pscala-2.12 \
+ # -DskipTests $PUBLISH_PROFILES clean install
# Clean-up Zinc nailgun process
/usr/sbin/lsof -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
- ./dev/change-version-to-2.10.sh
-
pushd $tmp_repo/org/apache/spark
# Remove any extra files generated during install
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/docs/building-spark.md
----------------------------------------------------------------------
diff --git a/docs/building-spark.md b/docs/building-spark.md
index 815843c..69d8302 100644
--- a/docs/building-spark.md
+++ b/docs/building-spark.md
@@ -91,14 +91,6 @@ like ZooKeeper and Hadoop itself.
./build/mvn -Pmesos -DskipTests clean package
-## Building for Scala 2.10
-To produce a Spark package compiled with Scala 2.10, use the `-Dscala-2.10` property:
-
- ./dev/change-scala-version.sh 2.10
- ./build/mvn -Pyarn -Dscala-2.10 -DskipTests clean package
-
-Note that support for Scala 2.10 is deprecated as of Spark 2.1.0 and may be removed in Spark 2.3.0.
-
## Building submodules individually
It's possible to build Spark sub-modules using the `mvn -pl` option.
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/docs/index.md
----------------------------------------------------------------------
diff --git a/docs/index.md b/docs/index.md
index 81ed465..07b6b17 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -31,8 +31,7 @@ uses Scala {{site.SCALA_BINARY_VERSION}}. You will need to use a compatible Scal
({{site.SCALA_BINARY_VERSION}}.x).
Note that support for Java 7, Python 2.6 and old Hadoop versions before 2.6.5 were removed as of Spark 2.2.0.
-
-Note that support for Scala 2.10 is deprecated as of Spark 2.1.0, and may be removed in Spark 2.3.0.
+Support for Scala 2.10 was removed as of 2.3.0.
# Running the Examples and Shell
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala b/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala
index 8f8262d..8441b5a 100644
--- a/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala
@@ -52,7 +52,7 @@ object AggregateMessagesExample {
triplet => { // Map Function
if (triplet.srcAttr > triplet.dstAttr) {
// Send message to destination vertex containing counter and age
- triplet.sendToDst(1, triplet.srcAttr)
+ triplet.sendToDst((1, triplet.srcAttr))
}
},
// Add counter and age
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala
index ae60577..8985c85 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala
@@ -38,7 +38,7 @@ abstract class AbstractParams[T: TypeTag] {
*/
override def toString: String = {
val tpe = tag.tpe
- val allAccessors = tpe.declarations.collect {
+ val allAccessors = tpe.decls.collect {
case m: MethodSymbol if m.isCaseAccessor => m
}
val mirror = runtimeMirror(getClass.getClassLoader)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala b/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala
index b9a612d..958361a 100644
--- a/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala
@@ -29,8 +29,6 @@ import org.apache.spark.sql.types._
object SparkSQLExample {
// $example on:create_ds$
- // Note: Case classes in Scala 2.10 can support only up to 22 fields. To work around this limit,
- // you can use custom classes that implement the Product interface
case class Person(name: String, age: Long)
// $example off:create_ds$
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala
----------------------------------------------------------------------
diff --git a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala
index 2b92577..5ea52b6 100644
--- a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala
+++ b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala
@@ -259,8 +259,8 @@ object KafkaRDD {
messageHandler: MessageAndMetadata[K, V] => R
): KafkaRDD[K, V, U, T, R] = {
val leaders = untilOffsets.map { case (tp, lo) =>
- tp -> (lo.host, lo.port)
- }.toMap
+ tp -> ((lo.host, lo.port))
+ }
val offsetRanges = fromOffsets.map { case (tp, fo) =>
val uo = untilOffsets(tp)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/graphx/src/main/scala/org/apache/spark/graphx/EdgeContext.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/EdgeContext.scala b/graphx/src/main/scala/org/apache/spark/graphx/EdgeContext.scala
index 2343017..3b96a42 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/EdgeContext.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/EdgeContext.scala
@@ -63,5 +63,5 @@ object EdgeContext {
* }}}
*/
def unapply[VD, ED, A](edge: EdgeContext[VD, ED, A]): Some[(VertexId, VertexId, VD, VD, ED)] =
- Some(edge.srcId, edge.dstId, edge.srcAttr, edge.dstAttr, edge.attr)
+ Some((edge.srcId, edge.dstId, edge.srcAttr, edge.dstAttr, edge.attr))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
----------------------------------------------------------------------
diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
index 6c0c3eb..481ff20 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
@@ -229,17 +229,17 @@ abstract class AbstractCommandBuilder {
return scala;
}
String sparkHome = getSparkHome();
- File scala210 = new File(sparkHome, "launcher/target/scala-2.10");
+ //File scala212 = new File(sparkHome, "launcher/target/scala-2.12");
File scala211 = new File(sparkHome, "launcher/target/scala-2.11");
- checkState(!scala210.isDirectory() || !scala211.isDirectory(),
- "Presence of build for both scala versions (2.10 and 2.11) detected.\n" +
- "Either clean one of them or set SPARK_SCALA_VERSION in your environment.");
- if (scala210.isDirectory()) {
- return "2.10";
- } else {
- checkState(scala211.isDirectory(), "Cannot find any build directories.");
- return "2.11";
- }
+ //checkState(!scala210.isDirectory() || !scala211.isDirectory(),
+ // "Presence of build for multiple Scala versions detected.\n" +
+ // "Either clean one of them or set SPARK_SCALA_VERSION in your environment.");
+ //if (scala212.isDirectory()) {
+ // return "2.12";
+ //} else {
+ checkState(scala211.isDirectory(), "Cannot find any build directories.");
+ return "2.11";
+ //}
}
String getSparkHome() {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
----------------------------------------------------------------------
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
index 07f3bc2..66c5362 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
@@ -856,7 +856,7 @@ object SparseMatrix {
var prevRow = -1
var prevVal = 0.0
// Append a dummy entry to include the last one at the end of the loop.
- (sortedEntries.view :+ (numRows, numCols, 1.0)).foreach { case (i, j, v) =>
+ (sortedEntries.view :+ ((numRows, numCols, 1.0))).foreach { case (i, j, v) =>
if (v != 0) {
if (i == prevRow && j == prevCol) {
prevVal += v
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala
----------------------------------------------------------------------
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala
index 9f82020..7fb9034 100644
--- a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala
@@ -633,22 +633,22 @@ class MatricesSuite extends SparkMLFunSuite {
dnMap.put((i, j), value)
}
assert(dnMap.size === 6)
- assert(dnMap(0, 0) === 1.0)
- assert(dnMap(1, 0) === 2.0)
- assert(dnMap(2, 0) === 0.0)
- assert(dnMap(0, 1) === 0.0)
- assert(dnMap(1, 1) === 4.0)
- assert(dnMap(2, 1) === 5.0)
+ assert(dnMap((0, 0)) === 1.0)
+ assert(dnMap((1, 0)) === 2.0)
+ assert(dnMap((2, 0)) === 0.0)
+ assert(dnMap((0, 1)) === 0.0)
+ assert(dnMap((1, 1)) === 4.0)
+ assert(dnMap((2, 1)) === 5.0)
val spMap = MutableMap[(Int, Int), Double]()
sp.foreachActive { (i, j, value) =>
spMap.put((i, j), value)
}
assert(spMap.size === 4)
- assert(spMap(0, 0) === 1.0)
- assert(spMap(1, 0) === 2.0)
- assert(spMap(1, 1) === 4.0)
- assert(spMap(2, 1) === 5.0)
+ assert(spMap((0, 0)) === 1.0)
+ assert(spMap((1, 0)) === 2.0)
+ assert(spMap((1, 1)) === 4.0)
+ assert(spMap((2, 1)) === 5.0)
}
test("horzcat, vertcat, eye, speye") {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
index 0d6e903..4aa4c36 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
@@ -436,7 +436,7 @@ private[ml] object EnsembleModelReadWrite {
val treesMetadataRDD: RDD[(Int, (Metadata, Double))] = sql.read.parquet(treesMetadataPath)
.select("treeID", "metadata", "weights").as[(Int, String, Double)].rdd.map {
case (treeID: Int, json: String, weights: Double) =>
- treeID -> (DefaultParamsReader.parseMetadata(json, treeClassName), weights)
+ treeID -> ((DefaultParamsReader.parseMetadata(json, treeClassName), weights))
}
val treesMetadataWeights = treesMetadataRDD.sortByKey().values.collect()
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
index 2b2b5fe..bf9b4cf 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
@@ -797,7 +797,7 @@ object SparseMatrix {
var prevRow = -1
var prevVal = 0.0
// Append a dummy entry to include the last one at the end of the loop.
- (sortedEntries.view :+ (numRows, numCols, 1.0)).foreach { case (i, j, v) =>
+ (sortedEntries.view :+ ((numRows, numCols, 1.0))).foreach { case (i, j, v) =>
if (v != 0) {
if (i == prevRow && j == prevCol) {
prevVal += v
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala
index 20d68a3..7caacd1 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala
@@ -275,7 +275,7 @@ class BlockMatrix @Since("1.3.0") (
val rows = blocks.flatMap { case ((blockRowIdx, blockColIdx), mat) =>
mat.rowIter.zipWithIndex.map {
case (vector, rowIdx) =>
- blockRowIdx * rowsPerBlock + rowIdx -> (blockColIdx, vector.asBreeze)
+ blockRowIdx * rowsPerBlock + rowIdx -> ((blockColIdx, vector.asBreeze))
}
}.groupByKey().map { case (rowIdx, vectors) =>
val numberNonZeroPerRow = vectors.map(_._2.activeSize).sum.toDouble / cols.toDouble
@@ -286,7 +286,7 @@ class BlockMatrix @Since("1.3.0") (
BDV.zeros[Double](cols)
}
- vectors.foreach { case (blockColIdx: Int, vec: BV[Double]) =>
+ vectors.foreach { case (blockColIdx: Int, vec: BV[_]) =>
val offset = colsPerBlock * blockColIdx
wholeVector(offset until Math.min(cols, offset + colsPerBlock)) := vec
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
index 0a0fea2..45d3f9b 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
@@ -898,7 +898,7 @@ class ALSStorageSuite
// check final factor RDD default storage levels
val defaultFactorRDDs = sc.getPersistentRDDs.collect {
case (id, rdd) if rdd.name == "userFactors" || rdd.name == "itemFactors" =>
- rdd.name -> (id, rdd.getStorageLevel)
+ rdd.name -> ((id, rdd.getStorageLevel))
}.toMap
defaultFactorRDDs.foreach { case (_, (id, level)) =>
assert(level == StorageLevel.MEMORY_AND_DISK)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
index 086bb21..8906e52 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
@@ -151,7 +151,7 @@ class LDASuite extends SparkFunSuite with MLlibTestSparkContext {
// Check: topTopicAssignments
// Make sure it assigns a topic to each term appearing in each doc.
val topTopicAssignments: Map[Long, (Array[Int], Array[Int])] =
- model.topicAssignments.collect().map(x => x._1 -> (x._2, x._3)).toMap
+ model.topicAssignments.collect().map(x => x._1 -> ((x._2, x._3))).toMap
assert(topTopicAssignments.keys.max < tinyCorpus.length)
tinyCorpus.foreach { case (docID: Long, doc: Vector) =>
if (topTopicAssignments.contains(docID)) {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
index 93c00d8..6736e7d 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
@@ -241,22 +241,22 @@ class MatricesSuite extends SparkFunSuite {
dnMap.put((i, j), value)
}
assert(dnMap.size === 6)
- assert(dnMap(0, 0) === 1.0)
- assert(dnMap(1, 0) === 2.0)
- assert(dnMap(2, 0) === 0.0)
- assert(dnMap(0, 1) === 0.0)
- assert(dnMap(1, 1) === 4.0)
- assert(dnMap(2, 1) === 5.0)
+ assert(dnMap((0, 0)) === 1.0)
+ assert(dnMap((1, 0)) === 2.0)
+ assert(dnMap((2, 0)) === 0.0)
+ assert(dnMap((0, 1)) === 0.0)
+ assert(dnMap((1, 1)) === 4.0)
+ assert(dnMap((2, 1)) === 5.0)
val spMap = MutableMap[(Int, Int), Double]()
sp.foreachActive { (i, j, value) =>
spMap.put((i, j), value)
}
assert(spMap.size === 4)
- assert(spMap(0, 0) === 1.0)
- assert(spMap(1, 0) === 2.0)
- assert(spMap(1, 1) === 4.0)
- assert(spMap(2, 1) === 5.0)
+ assert(spMap((0, 0)) === 1.0)
+ assert(spMap((1, 0)) === 2.0)
+ assert(spMap((1, 1)) === 4.0)
+ assert(spMap((2, 1)) === 5.0)
}
test("horzcat, vertcat, eye, speye") {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f124ba4..bea2798 100644
--- a/pom.xml
+++ b/pom.xml
@@ -498,7 +498,7 @@
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
<version>${slf4j.version}</version>
- <!-- <scope>runtime</scope> --> <!-- more correct, but scalac 2.10.3 doesn't like it -->
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
@@ -1859,9 +1859,9 @@
<version>${antlr4.version}</version>
</dependency>
<dependency>
- <groupId>${jline.groupid}</groupId>
+ <groupId>jline</groupId>
<artifactId>jline</artifactId>
- <version>${jline.version}</version>
+ <version>2.12.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -1933,6 +1933,7 @@
-->
<exclude>org.jboss.netty</exclude>
<exclude>org.codehaus.groovy</exclude>
+ <exclude>*:*_2.10</exclude>
</excludes>
<searchTransitive>true</searchTransitive>
</bannedDependencies>
@@ -1987,6 +1988,8 @@
<arg>-unchecked</arg>
<arg>-deprecation</arg>
<arg>-feature</arg>
+ <arg>-explaintypes</arg>
+ <arg>-Yno-adapted-args</arg>
</args>
<jvmArgs>
<jvmArg>-Xms1024m</jvmArg>
@@ -2586,44 +2589,6 @@
</profile>
<profile>
- <id>scala-2.10</id>
- <activation>
- <property><name>scala-2.10</name></property>
- </activation>
- <properties>
- <scala.version>2.10.6</scala.version>
- <scala.binary.version>2.10</scala.binary.version>
- <jline.version>${scala.version}</jline.version>
- <jline.groupid>org.scala-lang</jline.groupid>
- </properties>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-enforcer-plugin</artifactId>
- <executions>
- <execution>
- <id>enforce-versions</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <configuration>
- <rules>
- <bannedDependencies>
- <excludes combine.children="append">
- <exclude>*:*_2.11</exclude>
- </excludes>
- </bannedDependencies>
- </rules>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
-
- <profile>
<id>test-java-home</id>
<activation>
<property><name>env.JAVA_HOME</name></property>
@@ -2633,16 +2598,18 @@
</properties>
</profile>
+ <!-- Exists for backwards compatibility; profile doesn't do anything -->
<profile>
<id>scala-2.11</id>
- <activation>
- <property><name>!scala-2.10</name></property>
- </activation>
+ </profile>
+
+ <!-- Draft of Scala 2.12 profile for later -->
+ <!--
+ <profile>
+ <id>scala-2.12</id>
<properties>
- <scala.version>2.11.8</scala.version>
- <scala.binary.version>2.11</scala.binary.version>
- <jline.version>2.12.1</jline.version>
- <jline.groupid>jline</jline.groupid>
+ <scala.version>2.12.1</scala.version>
+ <scala.binary.version>2.12</scala.binary.version>
</properties>
<build>
<plugins>
@@ -2659,7 +2626,7 @@
<rules>
<bannedDependencies>
<excludes combine.children="append">
- <exclude>*:*_2.10</exclude>
+ <exclude>*:*_2.11</exclude>
</excludes>
</bannedDependencies>
</rules>
@@ -2670,10 +2637,11 @@
</plugins>
</build>
</profile>
+ -->
<!--
This is a profile to enable the use of the ASF snapshot and staging repositories
- during a build. It is useful when testing againt nightly or RC releases of dependencies.
+ during a build. It is useful when testing against nightly or RC releases of dependencies.
It MUST NOT be used when building copies of Spark to use in production of for distribution,
-->
<profile>
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 89b0c7a..41f3a04 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -87,19 +87,11 @@ object SparkBuild extends PomBuild {
val projectsMap: Map[String, Seq[Setting[_]]] = Map.empty
override val profiles = {
- val profiles = Properties.envOrNone("SBT_MAVEN_PROFILES") match {
+ Properties.envOrNone("SBT_MAVEN_PROFILES") match {
case None => Seq("sbt")
case Some(v) =>
v.split("(\\s+|,)").filterNot(_.isEmpty).map(_.trim.replaceAll("-P", "")).toSeq
}
-
- if (System.getProperty("scala-2.10") == "") {
- // To activate scala-2.10 profile, replace empty property value to non-empty value
- // in the same way as Maven which handles -Dname as -Dname=true before executes build process.
- // see: https://github.com/apache/maven/blob/maven-3.0.4/maven-embedder/src/main/java/org/apache/maven/cli/MavenCli.java#L1082
- System.setProperty("scala-2.10", "true")
- }
- profiles
}
Properties.envOrNone("SBT_MAVEN_PROPERTIES") match {
@@ -234,9 +226,7 @@ object SparkBuild extends PomBuild {
},
javacJVMVersion := "1.8",
- // SBT Scala 2.10 build still doesn't support Java 8, because scalac 2.10 doesn't, but,
- // it also doesn't touch Java 8 code and it's OK to emit Java 7 bytecode in this case
- scalacJVMVersion := (if (System.getProperty("scala-2.10") == "true") "1.7" else "1.8"),
+ scalacJVMVersion := "1.8",
javacOptions in Compile ++= Seq(
"-encoding", "UTF-8",
@@ -477,7 +467,6 @@ object OldDeps {
def oldDepsSettings() = Defaults.coreDefaultSettings ++ Seq(
name := "old-deps",
- scalaVersion := "2.10.5",
libraryDependencies := allPreviousArtifactKeys.value.flatten
)
}
@@ -756,13 +745,7 @@ object CopyDependencies {
object TestSettings {
import BuildCommons._
- private val scalaBinaryVersion =
- if (System.getProperty("scala-2.10") == "true") {
- "2.10"
- } else {
- "2.11"
- }
-
+ private val scalaBinaryVersion = "2.11"
lazy val settings = Seq (
// Fork new JVMs for tests and set Java options for those
fork := true,
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/python/run-tests.py
----------------------------------------------------------------------
diff --git a/python/run-tests.py b/python/run-tests.py
index b2e5043..afd3d29 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -54,7 +54,8 @@ FAILURE_REPORTING_LOCK = Lock()
LOGGER = logging.getLogger()
# Find out where the assembly jars are located.
-for scala in ["2.11", "2.10"]:
+# Later, add back 2.12 to this list:
+for scala in ["2.11"]:
build_dir = os.path.join(SPARK_HOME, "assembly", "target", "scala-" + scala)
if os.path.isdir(build_dir):
SPARK_DIST_CLASSPATH = os.path.join(build_dir, "jars", "*")
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/pom.xml
----------------------------------------------------------------------
diff --git a/repl/pom.xml b/repl/pom.xml
index 6d133a3..51eb9b6 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -32,8 +32,8 @@
<properties>
<sbt.project.name>repl</sbt.project.name>
- <extra.source.dir>scala-2.10/src/main/scala</extra.source.dir>
- <extra.testsource.dir>scala-2.10/src/test/scala</extra.testsource.dir>
+ <extra.source.dir>scala-2.11/src/main/scala</extra.source.dir>
+ <extra.testsource.dir>scala-2.11/src/test/scala</extra.testsource.dir>
</properties>
<dependencies>
@@ -71,7 +71,7 @@
<version>${scala.version}</version>
</dependency>
<dependency>
- <groupId>${jline.groupid}</groupId>
+ <groupId>jline</groupId>
<artifactId>jline</artifactId>
</dependency>
<dependency>
@@ -170,23 +170,17 @@
</plugin>
</plugins>
</build>
+
+ <!--
<profiles>
<profile>
- <id>scala-2.10</id>
- <activation>
- <property><name>scala-2.10</name></property>
- </activation>
- </profile>
-
- <profile>
- <id>scala-2.11</id>
- <activation>
- <property><name>!scala-2.10</name></property>
- </activation>
+ <id>scala-2.12</id>
<properties>
- <extra.source.dir>scala-2.11/src/main/scala</extra.source.dir>
- <extra.testsource.dir>scala-2.11/src/test/scala</extra.testsource.dir>
+ <extra.source.dir>scala-2.12/src/main/scala</extra.source.dir>
+ <extra.testsource.dir>scala-2.12/src/test/scala</extra.testsource.dir>
</properties>
</profile>
</profiles>
+ -->
+
</project>
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/Main.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/Main.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/Main.scala
deleted file mode 100644
index fba321b..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/Main.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.repl
-
-import org.apache.spark.internal.Logging
-
-object Main extends Logging {
-
- initializeLogIfNecessary(true)
- Signaling.cancelOnInterrupt()
-
- private var _interp: SparkILoop = _
-
- def interp = _interp
-
- def interp_=(i: SparkILoop) { _interp = i }
-
- def main(args: Array[String]) {
- _interp = new SparkILoop
- _interp.process(args)
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala
deleted file mode 100644
index be9b790..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.repl
-
-import scala.tools.nsc.{CompilerCommand, Settings}
-
-import org.apache.spark.annotation.DeveloperApi
-
-/**
- * Command class enabling Spark-specific command line options (provided by
- * <i>org.apache.spark.repl.SparkRunnerSettings</i>).
- *
- * @example new SparkCommandLine(Nil).settings
- *
- * @param args The list of command line arguments
- * @param settings The underlying settings to associate with this set of
- * command-line options
- */
-@DeveloperApi
-class SparkCommandLine(args: List[String], override val settings: Settings)
- extends CompilerCommand(args, settings) {
- def this(args: List[String], error: String => Unit) {
- this(args, new SparkRunnerSettings(error))
- }
-
- def this(args: List[String]) {
- // scalastyle:off println
- this(args, str => Console.println("Error: " + str))
- // scalastyle:on println
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala
deleted file mode 100644
index 2b5d56a..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala
+++ /dev/null
@@ -1,114 +0,0 @@
-// scalastyle:off
-
-/* NSC -- new Scala compiler
- * Copyright 2005-2013 LAMP/EPFL
- * @author Paul Phillips
- */
-
-package org.apache.spark.repl
-
-import scala.tools.nsc._
-import scala.tools.nsc.interpreter._
-
-import scala.reflect.internal.util.BatchSourceFile
-import scala.tools.nsc.ast.parser.Tokens.EOF
-
-import org.apache.spark.internal.Logging
-
-private[repl] trait SparkExprTyper extends Logging {
- val repl: SparkIMain
-
- import repl._
- import global.{ reporter => _, Import => _, _ }
- import definitions._
- import syntaxAnalyzer.{ UnitParser, UnitScanner, token2name }
- import naming.freshInternalVarName
-
- object codeParser extends { val global: repl.global.type = repl.global } with CodeHandlers[Tree] {
- def applyRule[T](code: String, rule: UnitParser => T): T = {
- reporter.reset()
- val scanner = newUnitParser(code)
- val result = rule(scanner)
-
- if (!reporter.hasErrors)
- scanner.accept(EOF)
-
- result
- }
-
- def defns(code: String) = stmts(code) collect { case x: DefTree => x }
- def expr(code: String) = applyRule(code, _.expr())
- def stmts(code: String) = applyRule(code, _.templateStats())
- def stmt(code: String) = stmts(code).last // guaranteed nonempty
- }
-
- /** Parse a line into a sequence of trees. Returns None if the input is incomplete. */
- def parse(line: String): Option[List[Tree]] = debugging(s"""parse("$line")""") {
- var isIncomplete = false
- reporter.withIncompleteHandler((_, _) => isIncomplete = true) {
- val trees = codeParser.stmts(line)
- if (reporter.hasErrors) {
- Some(Nil)
- } else if (isIncomplete) {
- None
- } else {
- Some(trees)
- }
- }
- }
- // def parsesAsExpr(line: String) = {
- // import codeParser._
- // (opt expr line).isDefined
- // }
-
- def symbolOfLine(code: String): Symbol = {
- def asExpr(): Symbol = {
- val name = freshInternalVarName()
- // Typing it with a lazy val would give us the right type, but runs
- // into compiler bugs with things like existentials, so we compile it
- // behind a def and strip the NullaryMethodType which wraps the expr.
- val line = "def " + name + " = {\n" + code + "\n}"
-
- interpretSynthetic(line) match {
- case IR.Success =>
- val sym0 = symbolOfTerm(name)
- // drop NullaryMethodType
- val sym = sym0.cloneSymbol setInfo afterTyper(sym0.info.finalResultType)
- if (sym.info.typeSymbol eq UnitClass) NoSymbol else sym
- case _ => NoSymbol
- }
- }
- def asDefn(): Symbol = {
- val old = repl.definedSymbolList.toSet
-
- interpretSynthetic(code) match {
- case IR.Success =>
- repl.definedSymbolList filterNot old match {
- case Nil => NoSymbol
- case sym :: Nil => sym
- case syms => NoSymbol.newOverloaded(NoPrefix, syms)
- }
- case _ => NoSymbol
- }
- }
- beQuietDuring(asExpr()) orElse beQuietDuring(asDefn())
- }
-
- private var typeOfExpressionDepth = 0
- def typeOfExpression(expr: String, silent: Boolean = true): Type = {
- if (typeOfExpressionDepth > 2) {
- logDebug("Terminating typeOfExpression recursion for expression: " + expr)
- return NoType
- }
- typeOfExpressionDepth += 1
- // Don't presently have a good way to suppress undesirable success output
- // while letting errors through, so it is first trying it silently: if there
- // is an error, and errors are desired, then it re-evaluates non-silently
- // to induce the error message.
- try beSilentDuring(symbolOfLine(expr).tpe) match {
- case NoType if !silent => symbolOfLine(expr).tpe // generate error
- case tpe => tpe
- }
- finally typeOfExpressionDepth -= 1
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkHelper.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkHelper.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkHelper.scala
deleted file mode 100644
index 955be17..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkHelper.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package scala.tools.nsc
-
-import org.apache.spark.annotation.DeveloperApi
-
-// NOTE: Forced to be public (and in scala.tools.nsc package) to access the
-// settings "explicitParentLoader" method
-
-/**
- * Provides exposure for the explicitParentLoader method on settings instances.
- */
-@DeveloperApi
-object SparkHelper {
- /**
- * Retrieves the explicit parent loader for the provided settings.
- *
- * @param settings The settings whose explicit parent loader to retrieve
- *
- * @return The Optional classloader representing the explicit parent loader
- */
- @DeveloperApi
- def explicitParentLoader(settings: Settings) = settings.explicitParentLoader
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org