You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/07/13 09:06:30 UTC
[1/5] spark git commit: [SPARK-19810][BUILD][CORE] Remove support for
Scala 2.10
Repository: spark
Updated Branches:
refs/heads/master e08d06b37 -> 425c4ada4
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 8258407..69d110e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -359,7 +359,7 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
test("reduce") {
val ds = Seq(("a", 1), ("b", 2), ("c", 3)).toDS()
- assert(ds.reduce((a, b) => ("sum", a._2 + b._2)) == ("sum", 6))
+ assert(ds.reduce((a, b) => ("sum", a._2 + b._2)) == (("sum", 6)))
}
test("joinWith, flat schema") {
@@ -784,7 +784,7 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
test("SPARK-14000: case class with tuple type field") {
checkDataset(
Seq(TupleClass((1, "a"))).toDS(),
- TupleClass(1, "a")
+ TupleClass((1, "a"))
)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
index b80bd80..9e459ed 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
@@ -176,16 +176,16 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
test("number format in statistics") {
val numbers = Seq(
- BigInt(0) -> ("0.0 B", "0"),
- BigInt(100) -> ("100.0 B", "100"),
- BigInt(2047) -> ("2047.0 B", "2.05E+3"),
- BigInt(2048) -> ("2.0 KB", "2.05E+3"),
- BigInt(3333333) -> ("3.2 MB", "3.33E+6"),
- BigInt(4444444444L) -> ("4.1 GB", "4.44E+9"),
- BigInt(5555555555555L) -> ("5.1 TB", "5.56E+12"),
- BigInt(6666666666666666L) -> ("5.9 PB", "6.67E+15"),
- BigInt(1L << 10 ) * (1L << 60) -> ("1024.0 EB", "1.18E+21"),
- BigInt(1L << 11) * (1L << 60) -> ("2.36E+21 B", "2.36E+21")
+ BigInt(0) -> (("0.0 B", "0")),
+ BigInt(100) -> (("100.0 B", "100")),
+ BigInt(2047) -> (("2047.0 B", "2.05E+3")),
+ BigInt(2048) -> (("2.0 KB", "2.05E+3")),
+ BigInt(3333333) -> (("3.2 MB", "3.33E+6")),
+ BigInt(4444444444L) -> (("4.1 GB", "4.44E+9")),
+ BigInt(5555555555555L) -> (("5.1 TB", "5.56E+12")),
+ BigInt(6666666666666666L) -> (("5.9 PB", "6.67E+15")),
+ BigInt(1L << 10 ) * (1L << 60) -> (("1024.0 EB", "1.18E+21")),
+ BigInt(1L << 11) * (1L << 60) -> (("2.36E+21 B", "2.36E+21"))
)
numbers.foreach { case (input, (expectedSize, expectedRows)) =>
val stats = Statistics(sizeInBytes = input, rowCount = Some(input))
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
index 7bcb419..274694b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -27,23 +27,23 @@ class SubquerySuite extends QueryTest with SharedSQLContext {
val row = identity[(java.lang.Integer, java.lang.Double)](_)
lazy val l = Seq(
- row(1, 2.0),
- row(1, 2.0),
- row(2, 1.0),
- row(2, 1.0),
- row(3, 3.0),
- row(null, null),
- row(null, 5.0),
- row(6, null)).toDF("a", "b")
+ row((1, 2.0)),
+ row((1, 2.0)),
+ row((2, 1.0)),
+ row((2, 1.0)),
+ row((3, 3.0)),
+ row((null, null)),
+ row((null, 5.0)),
+ row((6, null))).toDF("a", "b")
lazy val r = Seq(
- row(2, 3.0),
- row(2, 3.0),
- row(3, 2.0),
- row(4, 1.0),
- row(null, null),
- row(null, 5.0),
- row(6, null)).toDF("c", "d")
+ row((2, 3.0)),
+ row((2, 3.0)),
+ row((3, 2.0)),
+ row((4, 1.0)),
+ row((null, null)),
+ row((null, 5.0)),
+ row((6, null))).toDF("c", "d")
lazy val t = r.filter($"c".isNotNull && $"d".isNotNull)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 94a2f9a..d76990b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -211,7 +211,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSQLContext {
}
testStandardAndLegacyModes("nested map with struct as value type") {
- val data = (1 to 4).map(i => Tuple1(Map(i -> (i, s"val_$i"))))
+ val data = (1 to 4).map(i => Tuple1(Map(i -> ((i, s"val_$i")))))
withParquetDataFrame(data) { df =>
checkAnswer(df, data.map { case Tuple1(m) =>
Row(m.mapValues(struct => Row(struct.productIterator.toSeq: _*)))
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
index 2911cbb..fd79323 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
@@ -146,8 +146,8 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
// PhysicalRDD(nodeId = 1) -> Filter(nodeId = 0)
val df = person.filter('age < 25)
testSparkPlanMetrics(df, 1, Map(
- 0L -> ("Filter", Map(
- "number of output rows" -> 1L)))
+ 0L -> (("Filter", Map(
+ "number of output rows" -> 1L))))
)
}
@@ -170,8 +170,8 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
Map("number of output rows" -> 1L,
"avg hash probe (min, med, max)" -> "\n(1, 1, 1)"))
testSparkPlanMetrics(df, 1, Map(
- 2L -> ("HashAggregate", expected1(0)),
- 0L -> ("HashAggregate", expected1(1)))
+ 2L -> (("HashAggregate", expected1(0))),
+ 0L -> (("HashAggregate", expected1(1))))
)
// 2 partitions and each partition contains 2 keys
@@ -182,8 +182,8 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
Map("number of output rows" -> 3L,
"avg hash probe (min, med, max)" -> "\n(1, 1, 1)"))
testSparkPlanMetrics(df2, 1, Map(
- 2L -> ("HashAggregate", expected2(0)),
- 0L -> ("HashAggregate", expected2(1)))
+ 2L -> (("HashAggregate", expected2(0))),
+ 0L -> (("HashAggregate", expected2(1))))
)
}
@@ -234,15 +234,15 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
// -> ObjectHashAggregate(nodeId = 0)
val df = testData2.groupBy().agg(collect_set('a)) // 2 partitions
testSparkPlanMetrics(df, 1, Map(
- 2L -> ("ObjectHashAggregate", Map("number of output rows" -> 2L)),
- 0L -> ("ObjectHashAggregate", Map("number of output rows" -> 1L)))
+ 2L -> (("ObjectHashAggregate", Map("number of output rows" -> 2L))),
+ 0L -> (("ObjectHashAggregate", Map("number of output rows" -> 1L))))
)
// 2 partitions and each partition contains 2 keys
val df2 = testData2.groupBy('a).agg(collect_set('a))
testSparkPlanMetrics(df2, 1, Map(
- 2L -> ("ObjectHashAggregate", Map("number of output rows" -> 4L)),
- 0L -> ("ObjectHashAggregate", Map("number of output rows" -> 3L)))
+ 2L -> (("ObjectHashAggregate", Map("number of output rows" -> 4L))),
+ 0L -> (("ObjectHashAggregate", Map("number of output rows" -> 3L))))
)
}
@@ -264,9 +264,9 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
val df = spark.sql(
"SELECT * FROM testData2 JOIN testDataForJoin ON testData2.a = testDataForJoin.a")
testSparkPlanMetrics(df, 1, Map(
- 0L -> ("SortMergeJoin", Map(
+ 0L -> (("SortMergeJoin", Map(
// It's 4 because we only read 3 rows in the first partition and 1 row in the second one
- "number of output rows" -> 4L)))
+ "number of output rows" -> 4L))))
)
}
}
@@ -282,17 +282,17 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
val df = spark.sql(
"SELECT * FROM testData2 left JOIN testDataForJoin ON testData2.a = testDataForJoin.a")
testSparkPlanMetrics(df, 1, Map(
- 0L -> ("SortMergeJoin", Map(
+ 0L -> (("SortMergeJoin", Map(
// It's 4 because we only read 3 rows in the first partition and 1 row in the second one
- "number of output rows" -> 8L)))
+ "number of output rows" -> 8L))))
)
val df2 = spark.sql(
"SELECT * FROM testDataForJoin right JOIN testData2 ON testData2.a = testDataForJoin.a")
testSparkPlanMetrics(df2, 1, Map(
- 0L -> ("SortMergeJoin", Map(
+ 0L -> (("SortMergeJoin", Map(
// It's 4 because we only read 3 rows in the first partition and 1 row in the second one
- "number of output rows" -> 8L)))
+ "number of output rows" -> 8L))))
)
}
}
@@ -304,9 +304,9 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
// ... -> BroadcastHashJoin(nodeId = 1) -> TungstenProject(nodeId = 0)
val df = df1.join(broadcast(df2), "key")
testSparkPlanMetrics(df, 2, Map(
- 1L -> ("BroadcastHashJoin", Map(
+ 1L -> (("BroadcastHashJoin", Map(
"number of output rows" -> 2L,
- "avg hash probe (min, med, max)" -> "\n(1, 1, 1)")))
+ "avg hash probe (min, med, max)" -> "\n(1, 1, 1)"))))
)
}
@@ -365,9 +365,9 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
val df = df1.join(df2, "key")
val metrics = getSparkPlanMetrics(df, 1, Set(1L))
testSparkPlanMetrics(df, 1, Map(
- 1L -> ("ShuffledHashJoin", Map(
+ 1L -> (("ShuffledHashJoin", Map(
"number of output rows" -> 2L,
- "avg hash probe (min, med, max)" -> "\n(1, 1, 1)")))
+ "avg hash probe (min, med, max)" -> "\n(1, 1, 1)"))))
)
}
}
@@ -426,14 +426,14 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
// ... -> BroadcastHashJoin(nodeId = 0)
val df = df1.join(broadcast(df2), $"key" === $"key2", "left_outer")
testSparkPlanMetrics(df, 2, Map(
- 0L -> ("BroadcastHashJoin", Map(
- "number of output rows" -> 5L)))
+ 0L -> (("BroadcastHashJoin", Map(
+ "number of output rows" -> 5L))))
)
val df3 = df1.join(broadcast(df2), $"key" === $"key2", "right_outer")
testSparkPlanMetrics(df3, 2, Map(
- 0L -> ("BroadcastHashJoin", Map(
- "number of output rows" -> 6L)))
+ 0L -> (("BroadcastHashJoin", Map(
+ "number of output rows" -> 6L))))
)
}
@@ -448,8 +448,8 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
"SELECT * FROM testData2 left JOIN testDataForJoin ON " +
"testData2.a * testDataForJoin.a != testData2.a + testDataForJoin.a")
testSparkPlanMetrics(df, 3, Map(
- 1L -> ("BroadcastNestedLoopJoin", Map(
- "number of output rows" -> 12L)))
+ 1L -> (("BroadcastNestedLoopJoin", Map(
+ "number of output rows" -> 12L))))
)
}
}
@@ -462,8 +462,8 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
// ... -> BroadcastHashJoin(nodeId = 0)
val df = df1.join(broadcast(df2), $"key" === $"key2", "leftsemi")
testSparkPlanMetrics(df, 2, Map(
- 0L -> ("BroadcastHashJoin", Map(
- "number of output rows" -> 2L)))
+ 0L -> (("BroadcastHashJoin", Map(
+ "number of output rows" -> 2L))))
)
}
@@ -477,7 +477,7 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
val df = spark.sql(
"SELECT * FROM testData2 JOIN testDataForJoin")
testSparkPlanMetrics(df, 1, Map(
- 0L -> ("CartesianProduct", Map("number of output rows" -> 12L)))
+ 0L -> (("CartesianProduct", Map("number of output rows" -> 12L))))
)
}
}
@@ -490,7 +490,7 @@ class SQLMetricsSuite extends SparkFunSuite with SharedSQLContext {
val df = spark.sql(
"SELECT * FROM testData2 ANTI JOIN antiData ON testData2.a = antiData.a")
testSparkPlanMetrics(df, 1, Map(
- 0L -> ("SortMergeJoin", Map("number of output rows" -> 4L)))
+ 0L -> (("SortMergeJoin", Map("number of output rows" -> 4L))))
)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLListenerSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLListenerSuite.scala
index 82eff5e..2c6763e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLListenerSuite.scala
@@ -395,7 +395,7 @@ class SQLListenerSuite extends SparkFunSuite with SharedSQLContext with JsonTest
}
// Listener tracks only SQL metrics, not other accumulators
assert(trackedAccums.size === 1)
- assert(trackedAccums.head === (sqlMetricInfo.id, sqlMetricInfo.update.get))
+ assert(trackedAccums.head === ((sqlMetricInfo.id, sqlMetricInfo.update.get)))
}
test("driver side SQL metrics") {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/core/src/test/scala/org/apache/spark/sql/expressions/ReduceAggregatorSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ReduceAggregatorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ReduceAggregatorSuite.scala
index d826d3f..f65dcdf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ReduceAggregatorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ReduceAggregatorSuite.scala
@@ -27,7 +27,7 @@ class ReduceAggregatorSuite extends SparkFunSuite {
val encoder: ExpressionEncoder[Int] = ExpressionEncoder()
val func = (v1: Int, v2: Int) => v1 + v2
val aggregator: ReduceAggregator[Int] = new ReduceAggregator(func)(Encoders.scalaInt)
- assert(aggregator.zero == (false, null))
+ assert(aggregator.zero == (false, null).asInstanceOf[(Boolean, Int)])
}
test("reduce, merge and finish") {
@@ -36,22 +36,22 @@ class ReduceAggregatorSuite extends SparkFunSuite {
val aggregator: ReduceAggregator[Int] = new ReduceAggregator(func)(Encoders.scalaInt)
val firstReduce = aggregator.reduce(aggregator.zero, 1)
- assert(firstReduce == (true, 1))
+ assert(firstReduce == ((true, 1)))
val secondReduce = aggregator.reduce(firstReduce, 2)
- assert(secondReduce == (true, 3))
+ assert(secondReduce == ((true, 3)))
val thirdReduce = aggregator.reduce(secondReduce, 3)
- assert(thirdReduce == (true, 6))
+ assert(thirdReduce == ((true, 6)))
val mergeWithZero1 = aggregator.merge(aggregator.zero, firstReduce)
- assert(mergeWithZero1 == (true, 1))
+ assert(mergeWithZero1 == ((true, 1)))
val mergeWithZero2 = aggregator.merge(secondReduce, aggregator.zero)
- assert(mergeWithZero2 == (true, 3))
+ assert(mergeWithZero2 == ((true, 3)))
val mergeTwoReduced = aggregator.merge(firstReduce, secondReduce)
- assert(mergeTwoReduced == (true, 4))
+ assert(mergeTwoReduced == ((true, 4)))
assert(aggregator.finish(firstReduce)== 1)
assert(aggregator.finish(secondReduce) == 3)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
index 59c6a6f..d21d1f1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
@@ -20,12 +20,10 @@ package org.apache.spark.sql.streaming
import java.util.UUID
import scala.collection.mutable
-import scala.concurrent.duration._
import scala.language.reflectiveCalls
import org.scalactic.TolerantNumerics
import org.scalatest.concurrent.AsyncAssertions.Waiter
-import org.scalatest.concurrent.Eventually._
import org.scalatest.concurrent.PatienceConfiguration.Timeout
import org.scalatest.BeforeAndAfter
import org.scalatest.PrivateMethodTester._
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index 3de60c7..a29d7a7 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -414,7 +414,7 @@ private[spark] object HiveUtils extends Logging {
protected[sql] def toHiveString(a: (Any, DataType)): String = a match {
case (struct: Row, StructType(fields)) =>
struct.toSeq.zip(fields).map {
- case (v, t) => s""""${t.name}":${toHiveStructString(v, t.dataType)}"""
+ case (v, t) => s""""${t.name}":${toHiveStructString((v, t.dataType))}"""
}.mkString("{", ",", "}")
case (seq: Seq[_], ArrayType(typ, _)) =>
seq.map(v => (v, typ)).map(toHiveStructString).mkString("[", ",", "]")
@@ -437,7 +437,7 @@ private[spark] object HiveUtils extends Logging {
protected def toHiveStructString(a: (Any, DataType)): String = a match {
case (struct: Row, StructType(fields)) =>
struct.toSeq.zip(fields).map {
- case (v, t) => s""""${t.name}":${toHiveStructString(v, t.dataType)}"""
+ case (v, t) => s""""${t.name}":${toHiveStructString((v, t.dataType))}"""
}.mkString("{", ",", "}")
case (seq: Seq[_], ArrayType(typ, _)) =>
seq.map(v => (v, typ)).map(toHiveStructString).mkString("[", ",", "]")
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index d8fd68b..8140f88 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -80,17 +80,17 @@ class DataSourceWithHiveMetastoreCatalogSuite
).coalesce(1)
Seq(
- "parquet" -> (
+ "parquet" -> ((
"org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
"org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
"org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
- ),
+ )),
- "orc" -> (
+ "orc" -> ((
"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
"org.apache.hadoop.hive.ql.io.orc.OrcSerde"
- )
+ ))
).foreach { case (provider, (inputFormat, outputFormat, serde)) =>
test(s"Persist non-partitioned $provider relation into metastore as managed table") {
withTable("t") {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
index 5f15a70..1771546 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
@@ -151,7 +151,7 @@ class HiveSparkSubmitSuite
// the HiveContext code mistakenly overrides the class loader that contains user classes.
// For more detail, see sql/hive/src/test/resources/regression-test-SPARK-8489/*scala.
val version = Properties.versionNumberString match {
- case v if v.startsWith("2.10") || v.startsWith("2.11") => v.substring(0, 4)
+ case v if v.startsWith("2.12") || v.startsWith("2.11") => v.substring(0, 4)
case x => throw new Exception(s"Unsupported Scala Version: $x")
}
val jarDir = getTestResourcePath("regression-test-SPARK-8489")
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 12daf3a..5b62e37 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.hive.execution
import java.io.File
import java.net.URI
+import scala.language.existentials
+
import org.apache.hadoop.fs.Path
import org.scalatest.BeforeAndAfterEach
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index cf33760..799abc1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -795,7 +795,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
sql("SELECT * FROM m").collect().zip(sql("SELECT * FROM src LIMIT 10").collect()).foreach {
case (Row(map: Map[_, _]), Row(key: Int, value: String)) =>
assert(map.size === 1)
- assert(map.head === (key, value))
+ assert(map.head === ((key, value)))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[5/5] spark git commit: [SPARK-19810][BUILD][CORE] Remove support for
Scala 2.10
Posted by we...@apache.org.
[SPARK-19810][BUILD][CORE] Remove support for Scala 2.10
## What changes were proposed in this pull request?
- Remove Scala 2.10 build profiles and support
- Replace some 2.10 support in scripts with commented placeholders for 2.12 later
- Remove deprecated API calls from 2.10 support
- Remove usages of deprecated context bounds where possible
- Remove Scala 2.10 workarounds like ScalaReflectionLock
- Other minor Scala warning fixes
## How was this patch tested?
Existing tests
Author: Sean Owen <so...@cloudera.com>
Closes #17150 from srowen/SPARK-19810.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/425c4ada
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/425c4ada
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/425c4ada
Branch: refs/heads/master
Commit: 425c4ada4c24e338b45d0e9987071f05c5766fa5
Parents: e08d06b
Author: Sean Owen <so...@cloudera.com>
Authored: Thu Jul 13 17:06:24 2017 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Thu Jul 13 17:06:24 2017 +0800
----------------------------------------------------------------------
R/pkg/R/sparkR.R | 4 +-
R/pkg/tests/fulltests/test_client.R | 4 +-
bin/load-spark-env.cmd | 22 +-
bin/load-spark-env.sh | 22 +-
build/mvn | 6 +-
.../scala/org/apache/spark/Accumulable.scala | 3 +-
.../scala/org/apache/spark/SparkContext.scala | 15 +-
.../spark/rdd/SequenceFileRDDFunctions.scala | 54 +-
.../scala/org/apache/spark/rpc/RpcTimeout.scala | 4 +-
.../scala/org/apache/spark/ui/JettyUtils.scala | 6 +-
.../spark/util/logging/FileAppender.scala | 8 +-
.../test/scala/org/apache/spark/FileSuite.scala | 4 +-
.../spark/deploy/SparkSubmitUtilsSuite.scala | 6 +-
.../spark/deploy/master/MasterSuite.scala | 3 +-
.../apache/spark/executor/ExecutorSuite.scala | 1 +
.../apache/spark/rdd/LocalCheckpointSuite.scala | 1 +
.../OutputCommitCoordinatorSuite.scala | 2 +-
.../spark/scheduler/SparkListenerSuite.scala | 2 +-
.../spark/scheduler/TaskSetBlacklistSuite.scala | 4 +-
.../spark/serializer/KryoSerializerSuite.scala | 4 +-
.../spark/util/TimeStampedHashMapSuite.scala | 2 +-
.../apache/spark/util/VersionUtilsSuite.scala | 8 +-
.../util/collection/AppendOnlyMapSuite.scala | 4 +-
.../util/collection/ExternalSorterSuite.scala | 14 +-
.../util/collection/OpenHashMapSuite.scala | 5 +-
dev/change-scala-version.sh | 4 +-
dev/change-version-to-2.10.sh | 23 -
dev/change-version-to-2.11.sh | 23 -
dev/create-release/release-build.sh | 23 +-
docs/building-spark.md | 8 -
docs/index.md | 3 +-
.../graphx/AggregateMessagesExample.scala | 2 +-
.../spark/examples/mllib/AbstractParams.scala | 2 +-
.../spark/examples/sql/SparkSQLExample.scala | 2 -
.../apache/spark/streaming/kafka/KafkaRDD.scala | 4 +-
.../org/apache/spark/graphx/EdgeContext.scala | 2 +-
.../spark/launcher/AbstractCommandBuilder.java | 20 +-
.../org/apache/spark/ml/linalg/Matrices.scala | 2 +-
.../apache/spark/ml/linalg/MatricesSuite.scala | 20 +-
.../org/apache/spark/ml/tree/treeModels.scala | 2 +-
.../apache/spark/mllib/linalg/Matrices.scala | 2 +-
.../mllib/linalg/distributed/BlockMatrix.scala | 4 +-
.../spark/ml/recommendation/ALSSuite.scala | 2 +-
.../spark/mllib/clustering/LDASuite.scala | 2 +-
.../spark/mllib/linalg/MatricesSuite.scala | 20 +-
pom.xml | 68 +-
project/SparkBuild.scala | 23 +-
python/run-tests.py | 3 +-
repl/pom.xml | 26 +-
.../main/scala/org/apache/spark/repl/Main.scala | 37 -
.../apache/spark/repl/SparkCommandLine.scala | 46 -
.../org/apache/spark/repl/SparkExprTyper.scala | 114 --
.../org/apache/spark/repl/SparkHelper.scala | 39 -
.../org/apache/spark/repl/SparkILoop.scala | 1145 -----------
.../org/apache/spark/repl/SparkILoopInit.scala | 168 --
.../org/apache/spark/repl/SparkIMain.scala | 1808 ------------------
.../org/apache/spark/repl/SparkImports.scala | 239 ---
.../spark/repl/SparkJLineCompletion.scala | 403 ----
.../apache/spark/repl/SparkJLineReader.scala | 90 -
.../apache/spark/repl/SparkMemberHandlers.scala | 232 ---
.../apache/spark/repl/SparkRunnerSettings.scala | 31 -
.../scala/org/apache/spark/repl/ReplSuite.scala | 366 ----
.../sql/execution/UnsafeExternalRowSorter.java | 4 +-
.../spark/sql/catalyst/ScalaReflection.scala | 36 +-
.../spark/sql/catalyst/analysis/Analyzer.scala | 4 +-
.../catalyst/expressions/codegen/package.scala | 4 +-
.../catalyst/expressions/namedExpressions.scala | 2 +-
.../org/apache/spark/sql/catalyst/package.scala | 6 -
.../util/AbstractScalaRowIterator.scala | 30 -
.../org/apache/spark/sql/types/BinaryType.scala | 3 +-
.../apache/spark/sql/types/BooleanType.scala | 3 +-
.../org/apache/spark/sql/types/ByteType.scala | 3 +-
.../org/apache/spark/sql/types/DateType.scala | 4 +-
.../apache/spark/sql/types/DecimalType.scala | 3 +-
.../org/apache/spark/sql/types/DoubleType.scala | 3 +-
.../org/apache/spark/sql/types/FloatType.scala | 3 +-
.../apache/spark/sql/types/HiveStringType.scala | 5 +-
.../apache/spark/sql/types/IntegerType.scala | 4 +-
.../org/apache/spark/sql/types/LongType.scala | 3 +-
.../org/apache/spark/sql/types/ShortType.scala | 3 +-
.../org/apache/spark/sql/types/StringType.scala | 3 +-
.../apache/spark/sql/types/TimestampType.scala | 4 +-
.../optimizer/JoinOptimizationSuite.scala | 24 +-
.../optimizer/OptimizeCodegenSuite.scala | 4 +-
.../scala/org/apache/spark/sql/Column.scala | 2 +-
.../execution/OptimizeMetadataOnlyQuery.scala | 8 +-
.../spark/sql/execution/QueryExecution.scala | 4 +-
.../execution/python/ExtractPythonUDFs.scala | 2 +-
.../org/apache/spark/sql/DatasetSuite.scala | 4 +-
.../spark/sql/StatisticsCollectionSuite.scala | 20 +-
.../org/apache/spark/sql/SubquerySuite.scala | 30 +-
.../datasources/parquet/ParquetIOSuite.scala | 2 +-
.../sql/execution/metric/SQLMetricsSuite.scala | 60 +-
.../sql/execution/ui/SQLListenerSuite.scala | 2 +-
.../sql/expressions/ReduceAggregatorSuite.scala | 14 +-
.../streaming/StreamingQueryListenerSuite.scala | 2 -
.../org/apache/spark/sql/hive/HiveUtils.scala | 4 +-
.../sql/hive/HiveMetastoreCatalogSuite.scala | 8 +-
.../spark/sql/hive/HiveSparkSubmitSuite.scala | 2 +-
.../spark/sql/hive/execution/HiveDDLSuite.scala | 2 +
.../sql/hive/execution/HiveQuerySuite.scala | 2 +-
101 files changed, 311 insertions(+), 5231 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/R/pkg/R/sparkR.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R
index f2d2620..81507ea 100644
--- a/R/pkg/R/sparkR.R
+++ b/R/pkg/R/sparkR.R
@@ -113,7 +113,7 @@ sparkR.stop <- function() {
#' list(spark.executor.memory="4g"),
#' list(LD_LIBRARY_PATH="/directory of JVM libraries (libjvm.so) on workers/"),
#' c("one.jar", "two.jar", "three.jar"),
-#' c("com.databricks:spark-avro_2.10:2.0.1"))
+#' c("com.databricks:spark-avro_2.11:2.0.1"))
#'}
#' @note sparkR.init since 1.4.0
sparkR.init <- function(
@@ -357,7 +357,7 @@ sparkRHive.init <- function(jsc = NULL) {
#' sparkR.session("yarn-client", "SparkR", "/home/spark",
#' list(spark.executor.memory="4g"),
#' c("one.jar", "two.jar", "three.jar"),
-#' c("com.databricks:spark-avro_2.10:2.0.1"))
+#' c("com.databricks:spark-avro_2.11:2.0.1"))
#' sparkR.session(spark.master = "yarn-client", spark.executor.memory = "4g")
#'}
#' @note sparkR.session since 2.0.0
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/R/pkg/tests/fulltests/test_client.R
----------------------------------------------------------------------
diff --git a/R/pkg/tests/fulltests/test_client.R b/R/pkg/tests/fulltests/test_client.R
index 0cf25fe..de624b5 100644
--- a/R/pkg/tests/fulltests/test_client.R
+++ b/R/pkg/tests/fulltests/test_client.R
@@ -37,7 +37,7 @@ test_that("multiple packages don't produce a warning", {
test_that("sparkJars sparkPackages as character vectors", {
args <- generateSparkSubmitArgs("", "", c("one.jar", "two.jar", "three.jar"), "",
- c("com.databricks:spark-avro_2.10:2.0.1"))
+ c("com.databricks:spark-avro_2.11:2.0.1"))
expect_match(args, "--jars one.jar,two.jar,three.jar")
- expect_match(args, "--packages com.databricks:spark-avro_2.10:2.0.1")
+ expect_match(args, "--packages com.databricks:spark-avro_2.11:2.0.1")
})
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/bin/load-spark-env.cmd
----------------------------------------------------------------------
diff --git a/bin/load-spark-env.cmd b/bin/load-spark-env.cmd
index 0977025..993aa31 100644
--- a/bin/load-spark-env.cmd
+++ b/bin/load-spark-env.cmd
@@ -35,21 +35,21 @@ if [%SPARK_ENV_LOADED%] == [] (
rem Setting SPARK_SCALA_VERSION if not already set.
-set ASSEMBLY_DIR2="%SPARK_HOME%\assembly\target\scala-2.11"
-set ASSEMBLY_DIR1="%SPARK_HOME%\assembly\target\scala-2.10"
+rem set ASSEMBLY_DIR2="%SPARK_HOME%\assembly\target\scala-2.11"
+rem set ASSEMBLY_DIR1="%SPARK_HOME%\assembly\target\scala-2.12"
if [%SPARK_SCALA_VERSION%] == [] (
- if exist %ASSEMBLY_DIR2% if exist %ASSEMBLY_DIR1% (
- echo "Presence of build for both scala versions(SCALA 2.10 and SCALA 2.11) detected."
- echo "Either clean one of them or, set SPARK_SCALA_VERSION=2.11 in spark-env.cmd."
- exit 1
- )
- if exist %ASSEMBLY_DIR2% (
+ rem if exist %ASSEMBLY_DIR2% if exist %ASSEMBLY_DIR1% (
+ rem echo "Presence of build for multiple Scala versions detected."
+ rem echo "Either clean one of them or, set SPARK_SCALA_VERSION=2.11 in spark-env.cmd."
+ rem exit 1
+ rem )
+ rem if exist %ASSEMBLY_DIR2% (
set SPARK_SCALA_VERSION=2.11
- ) else (
- set SPARK_SCALA_VERSION=2.10
- )
+ rem ) else (
+ rem set SPARK_SCALA_VERSION=2.12
+ rem )
)
exit /b 0
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/bin/load-spark-env.sh
----------------------------------------------------------------------
diff --git a/bin/load-spark-env.sh b/bin/load-spark-env.sh
index 8a2f709..9de6203 100644
--- a/bin/load-spark-env.sh
+++ b/bin/load-spark-env.sh
@@ -46,18 +46,18 @@ fi
if [ -z "$SPARK_SCALA_VERSION" ]; then
- ASSEMBLY_DIR2="${SPARK_HOME}/assembly/target/scala-2.11"
- ASSEMBLY_DIR1="${SPARK_HOME}/assembly/target/scala-2.10"
+ #ASSEMBLY_DIR2="${SPARK_HOME}/assembly/target/scala-2.11"
+ #ASSEMBLY_DIR1="${SPARK_HOME}/assembly/target/scala-2.12"
- if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then
- echo -e "Presence of build for both scala versions(SCALA 2.10 and SCALA 2.11) detected." 1>&2
- echo -e 'Either clean one of them or, export SPARK_SCALA_VERSION=2.11 in spark-env.sh.' 1>&2
- exit 1
- fi
+ #if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then
+ # echo -e "Presence of build for multiple Scala versions detected." 1>&2
+ # echo -e 'Either clean one of them or, export SPARK_SCALA_VERSION=2.11 in spark-env.sh.' 1>&2
+ # exit 1
+ #fi
- if [ -d "$ASSEMBLY_DIR2" ]; then
+ #if [ -d "$ASSEMBLY_DIR2" ]; then
export SPARK_SCALA_VERSION="2.11"
- else
- export SPARK_SCALA_VERSION="2.10"
- fi
+ #else
+ # export SPARK_SCALA_VERSION="2.12"
+ #fi
fi
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/build/mvn
----------------------------------------------------------------------
diff --git a/build/mvn b/build/mvn
index 1e393c3..efa4f93 100755
--- a/build/mvn
+++ b/build/mvn
@@ -91,13 +91,13 @@ install_mvn() {
# Install zinc under the build/ folder
install_zinc() {
- local zinc_path="zinc-0.3.11/bin/zinc"
+ local zinc_path="zinc-0.3.15/bin/zinc"
[ ! -f "${_DIR}/${zinc_path}" ] && ZINC_INSTALL_FLAG=1
local TYPESAFE_MIRROR=${TYPESAFE_MIRROR:-https://downloads.typesafe.com}
install_app \
- "${TYPESAFE_MIRROR}/zinc/0.3.11" \
- "zinc-0.3.11.tgz" \
+ "${TYPESAFE_MIRROR}/zinc/0.3.15" \
+ "zinc-0.3.15.tgz" \
"${zinc_path}"
ZINC_BIN="${_DIR}/${zinc_path}"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/main/scala/org/apache/spark/Accumulable.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/Accumulable.scala b/core/src/main/scala/org/apache/spark/Accumulable.scala
index 5532931..3092074 100644
--- a/core/src/main/scala/org/apache/spark/Accumulable.scala
+++ b/core/src/main/scala/org/apache/spark/Accumulable.scala
@@ -201,7 +201,8 @@ trait AccumulableParam[R, T] extends Serializable {
@deprecated("use AccumulatorV2", "2.0.0")
private[spark] class
-GrowableAccumulableParam[R <% Growable[T] with TraversableOnce[T] with Serializable: ClassTag, T]
+GrowableAccumulableParam[R : ClassTag, T]
+ (implicit rg: R => Growable[T] with TraversableOnce[T] with Serializable)
extends AccumulableParam[R, T] {
def addAccumulator(growable: R, elem: T): R = {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index b2a26c5..e1ce66a 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -183,8 +183,6 @@ class SparkContext(config: SparkConf) extends Logging {
// log out Spark Version in Spark driver log
logInfo(s"Running Spark version $SPARK_VERSION")
- warnDeprecatedVersions()
-
/* ------------------------------------------------------------------------------------- *
| Private variables. These variables keep the internal state of the context, and are |
| not accessible by the outside world. They're mutable since we want to initialize all |
@@ -349,13 +347,6 @@ class SparkContext(config: SparkConf) extends Logging {
value
}
- private def warnDeprecatedVersions(): Unit = {
- val javaVersion = System.getProperty("java.version").split("[+.\\-]+", 3)
- if (scala.util.Properties.releaseVersion.exists(_.startsWith("2.10"))) {
- logWarning("Support for Scala 2.10 is deprecated as of Spark 2.1.0")
- }
- }
-
/** Control our logLevel. This overrides any user-defined log settings.
* @param logLevel The desired log level as a string.
* Valid log levels include: ALL, DEBUG, ERROR, FATAL, INFO, OFF, TRACE, WARN
@@ -1396,6 +1387,8 @@ class SparkContext(config: SparkConf) extends Logging {
@deprecated("use AccumulatorV2", "2.0.0")
def accumulableCollection[R <% Growable[T] with TraversableOnce[T] with Serializable: ClassTag, T]
(initialValue: R): Accumulable[R, T] = {
+ // TODO the context bound (<%) above should be replaced with simple type bound and implicit
+ // conversion but is a breaking change. This should be fixed in Spark 3.x.
val param = new GrowableAccumulableParam[R, T]
val acc = new Accumulable(initialValue, param)
cleaner.foreach(_.registerAccumulatorForCleanup(acc.newAcc))
@@ -2605,9 +2598,9 @@ object SparkContext extends Logging {
*/
private[spark] val LEGACY_DRIVER_IDENTIFIER = "<driver>"
- private implicit def arrayToArrayWritable[T <% Writable: ClassTag](arr: Traversable[T])
+ private implicit def arrayToArrayWritable[T <: Writable : ClassTag](arr: Traversable[T])
: ArrayWritable = {
- def anyToWritable[U <% Writable](u: U): Writable = u
+ def anyToWritable[U <: Writable](u: U): Writable = u
new ArrayWritable(classTag[T].runtimeClass.asInstanceOf[Class[Writable]],
arr.map(x => anyToWritable(x)).toArray)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala b/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
index 86a3327..02def89 100644
--- a/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/SequenceFileRDDFunctions.scala
@@ -16,7 +16,7 @@
*/
package org.apache.spark.rdd
-import scala.reflect.{classTag, ClassTag}
+import scala.reflect.ClassTag
import org.apache.hadoop.io.Writable
import org.apache.hadoop.io.compress.CompressionCodec
@@ -39,40 +39,8 @@ class SequenceFileRDDFunctions[K <% Writable: ClassTag, V <% Writable : ClassTag
extends Logging
with Serializable {
- private val keyWritableClass =
- if (_keyWritableClass == null) {
- // pre 1.3.0, we need to use Reflection to get the Writable class
- getWritableClass[K]()
- } else {
- _keyWritableClass
- }
-
- private val valueWritableClass =
- if (_valueWritableClass == null) {
- // pre 1.3.0, we need to use Reflection to get the Writable class
- getWritableClass[V]()
- } else {
- _valueWritableClass
- }
-
- private def getWritableClass[T <% Writable: ClassTag](): Class[_ <: Writable] = {
- val c = {
- if (classOf[Writable].isAssignableFrom(classTag[T].runtimeClass)) {
- classTag[T].runtimeClass
- } else {
- // We get the type of the Writable class by looking at the apply method which converts
- // from T to Writable. Since we have two apply methods we filter out the one which
- // is not of the form "java.lang.Object apply(java.lang.Object)"
- implicitly[T => Writable].getClass.getDeclaredMethods().filter(
- m => m.getReturnType().toString != "class java.lang.Object" &&
- m.getName() == "apply")(0).getReturnType
-
- }
- // TODO: use something like WritableConverter to avoid reflection
- }
- c.asInstanceOf[Class[_ <: Writable]]
- }
-
+ // TODO the context bound (<%) above should be replaced with simple type bound and implicit
+ // conversion but is a breaking change. This should be fixed in Spark 3.x.
/**
* Output the RDD as a Hadoop SequenceFile using the Writable types we infer from the RDD's key
@@ -90,24 +58,24 @@ class SequenceFileRDDFunctions[K <% Writable: ClassTag, V <% Writable : ClassTag
// valueWritableClass at the compile time. To implement that, we need to add type parameters to
// SequenceFileRDDFunctions. however, SequenceFileRDDFunctions is a public class so it will be a
// breaking change.
- val convertKey = self.keyClass != keyWritableClass
- val convertValue = self.valueClass != valueWritableClass
+ val convertKey = self.keyClass != _keyWritableClass
+ val convertValue = self.valueClass != _valueWritableClass
- logInfo("Saving as sequence file of type (" + keyWritableClass.getSimpleName + "," +
- valueWritableClass.getSimpleName + ")" )
+ logInfo("Saving as sequence file of type " +
+ s"(${_keyWritableClass.getSimpleName},${_valueWritableClass.getSimpleName})" )
val format = classOf[SequenceFileOutputFormat[Writable, Writable]]
val jobConf = new JobConf(self.context.hadoopConfiguration)
if (!convertKey && !convertValue) {
- self.saveAsHadoopFile(path, keyWritableClass, valueWritableClass, format, jobConf, codec)
+ self.saveAsHadoopFile(path, _keyWritableClass, _valueWritableClass, format, jobConf, codec)
} else if (!convertKey && convertValue) {
self.map(x => (x._1, anyToWritable(x._2))).saveAsHadoopFile(
- path, keyWritableClass, valueWritableClass, format, jobConf, codec)
+ path, _keyWritableClass, _valueWritableClass, format, jobConf, codec)
} else if (convertKey && !convertValue) {
self.map(x => (anyToWritable(x._1), x._2)).saveAsHadoopFile(
- path, keyWritableClass, valueWritableClass, format, jobConf, codec)
+ path, _keyWritableClass, _valueWritableClass, format, jobConf, codec)
} else if (convertKey && convertValue) {
self.map(x => (anyToWritable(x._1), anyToWritable(x._2))).saveAsHadoopFile(
- path, keyWritableClass, valueWritableClass, format, jobConf, codec)
+ path, _keyWritableClass, _valueWritableClass, format, jobConf, codec)
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala b/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala
index 0557b7a..3dc41f7 100644
--- a/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/RpcTimeout.scala
@@ -125,9 +125,9 @@ private[spark] object RpcTimeout {
var foundProp: Option[(String, String)] = None
while (itr.hasNext && foundProp.isEmpty) {
val propKey = itr.next()
- conf.getOption(propKey).foreach { prop => foundProp = Some(propKey, prop) }
+ conf.getOption(propKey).foreach { prop => foundProp = Some((propKey, prop)) }
}
- val finalProp = foundProp.getOrElse(timeoutPropList.head, defaultValue)
+ val finalProp = foundProp.getOrElse((timeoutPropList.head, defaultValue))
val timeout = { Utils.timeStringAsSeconds(finalProp._2).seconds }
new RpcTimeout(timeout, finalProp._1)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index b9371c7..0fa9671 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -54,7 +54,7 @@ private[spark] object JettyUtils extends Logging {
// implicit conversion from many types of functions to jetty Handlers.
type Responder[T] = HttpServletRequest => T
- class ServletParams[T <% AnyRef](val responder: Responder[T],
+ class ServletParams[T <: AnyRef](val responder: Responder[T],
val contentType: String,
val extractFn: T => String = (in: Any) => in.toString) {}
@@ -68,7 +68,7 @@ private[spark] object JettyUtils extends Logging {
implicit def textResponderToServlet(responder: Responder[String]): ServletParams[String] =
new ServletParams(responder, "text/plain")
- def createServlet[T <% AnyRef](
+ def createServlet[T <: AnyRef](
servletParams: ServletParams[T],
securityMgr: SecurityManager,
conf: SparkConf): HttpServlet = {
@@ -113,7 +113,7 @@ private[spark] object JettyUtils extends Logging {
}
/** Create a context handler that responds to a request with the given path prefix */
- def createServletHandler[T <% AnyRef](
+ def createServletHandler[T <: AnyRef](
path: String,
servletParams: ServletParams[T],
securityMgr: SecurityManager,
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala b/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
index 8a0cc70..2f9ad4c 100644
--- a/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
+++ b/core/src/main/scala/org/apache/spark/util/logging/FileAppender.scala
@@ -125,16 +125,16 @@ private[spark] object FileAppender extends Logging {
val validatedParams: Option[(Long, String)] = rollingInterval match {
case "daily" =>
logInfo(s"Rolling executor logs enabled for $file with daily rolling")
- Some(24 * 60 * 60 * 1000L, "--yyyy-MM-dd")
+ Some((24 * 60 * 60 * 1000L, "--yyyy-MM-dd"))
case "hourly" =>
logInfo(s"Rolling executor logs enabled for $file with hourly rolling")
- Some(60 * 60 * 1000L, "--yyyy-MM-dd--HH")
+ Some((60 * 60 * 1000L, "--yyyy-MM-dd--HH"))
case "minutely" =>
logInfo(s"Rolling executor logs enabled for $file with rolling every minute")
- Some(60 * 1000L, "--yyyy-MM-dd--HH-mm")
+ Some((60 * 1000L, "--yyyy-MM-dd--HH-mm"))
case IntParam(seconds) =>
logInfo(s"Rolling executor logs enabled for $file with rolling $seconds seconds")
- Some(seconds * 1000L, "--yyyy-MM-dd--HH-mm-ss")
+ Some((seconds * 1000L, "--yyyy-MM-dd--HH-mm-ss"))
case _ =>
logWarning(s"Illegal interval for rolling executor logs [$rollingInterval], " +
s"rolling logs not enabled")
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/FileSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/FileSuite.scala b/core/src/test/scala/org/apache/spark/FileSuite.scala
index 5be0121..0272818 100644
--- a/core/src/test/scala/org/apache/spark/FileSuite.scala
+++ b/core/src/test/scala/org/apache/spark/FileSuite.scala
@@ -113,11 +113,11 @@ class FileSuite extends SparkFunSuite with LocalSparkContext {
val normalFile = new File(normalDir, "part-00000")
val normalContent = sc.sequenceFile[String, String](normalDir).collect
- assert(normalContent === Array.fill(100)("abc", "abc"))
+ assert(normalContent === Array.fill(100)(("abc", "abc")))
val compressedFile = new File(compressedOutputDir, "part-00000" + codec.getDefaultExtension)
val compressedContent = sc.sequenceFile[String, String](compressedOutputDir).collect
- assert(compressedContent === Array.fill(100)("abc", "abc"))
+ assert(compressedContent === Array.fill(100)(("abc", "abc")))
assert(compressedFile.length < normalFile.length)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
index 5702478..88b77e5 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala
@@ -93,8 +93,8 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
test("add dependencies works correctly") {
val md = SparkSubmitUtils.getModuleDescriptor
- val artifacts = SparkSubmitUtils.extractMavenCoordinates("com.databricks:spark-csv_2.10:0.1," +
- "com.databricks:spark-avro_2.10:0.1")
+ val artifacts = SparkSubmitUtils.extractMavenCoordinates("com.databricks:spark-csv_2.11:0.1," +
+ "com.databricks:spark-avro_2.11:0.1")
SparkSubmitUtils.addDependenciesToIvy(md, artifacts, "default")
assert(md.getDependencies.length === 2)
@@ -196,7 +196,7 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll {
SparkSubmitUtils.buildIvySettings(None, None),
isTest = true)
assert(path === "", "should return empty path")
- val main = MavenCoordinate("org.apache.spark", "spark-streaming-kafka-assembly_2.10", "1.2.0")
+ val main = MavenCoordinate("org.apache.spark", "spark-streaming-kafka-assembly_2.11", "1.2.0")
IvyTestUtils.withRepository(main, None, None) { repo =>
val files = SparkSubmitUtils.resolveMavenCoordinates(
coordinates + "," + main.toString,
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
index a223212..84b3a29 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
@@ -80,6 +80,7 @@ class MockWorker(master: RpcEndpointRef, conf: SparkConf = new SparkConf) extend
case Some(appId) =>
apps.remove(appId)
master.send(UnregisterApplication(appId))
+ case None =>
}
driverIdToAppId.remove(driverId)
}
@@ -575,7 +576,7 @@ class MasterSuite extends SparkFunSuite
override val rpcEnv: RpcEnv = master.rpcEnv
override def receive: PartialFunction[Any, Unit] = {
- case KillExecutor(_, appId, execId) => killedExecutors.add(appId, execId)
+ case KillExecutor(_, appId, execId) => killedExecutors.add((appId, execId))
case KillDriver(driverId) => killedDrivers.add(driverId)
}
})
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
index efcad14..601dde6 100644
--- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
@@ -25,6 +25,7 @@ import java.util.concurrent.{CountDownLatch, TimeUnit}
import scala.collection.mutable.Map
import scala.concurrent.duration._
+import scala.language.postfixOps
import org.mockito.ArgumentCaptor
import org.mockito.Matchers.{any, eq => meq}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala b/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala
index 9e204f5..478f069 100644
--- a/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/LocalCheckpointSuite.scala
@@ -18,6 +18,7 @@
package org.apache.spark.rdd
import scala.concurrent.duration._
+import scala.language.postfixOps
import org.scalatest.concurrent.Eventually.{eventually, interval, timeout}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorSuite.scala
index 1579b61..60b5955 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/OutputCommitCoordinatorSuite.scala
@@ -115,7 +115,7 @@ class OutputCommitCoordinatorSuite extends SparkFunSuite with BeforeAndAfter {
locality: TaskLocality.Value): Option[(Int, TaskLocality.Value)] = {
if (!hasDequeuedSpeculatedTask) {
hasDequeuedSpeculatedTask = true
- Some(0, TaskLocality.PROCESS_LOCAL)
+ Some((0, TaskLocality.PROCESS_LOCAL))
} else {
None
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
index f3d0bc1..481603b 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/SparkListenerSuite.scala
@@ -299,7 +299,7 @@ class SparkListenerSuite extends SparkFunSuite with LocalSparkContext with Match
val d2 = d.map { i => w(i) -> i * 2 }.setName("shuffle input 1")
val d3 = d.map { i => w(i) -> (0 to (i % 5)) }.setName("shuffle input 2")
val d4 = d2.cogroup(d3, numSlices).map { case (k, (v1, v2)) =>
- w(k) -> (v1.size, v2.size)
+ (w(k), (v1.size, v2.size))
}
d4.setName("A Cogroup")
d4.collectAsMap()
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala
index 6b52c10..f1392e9 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetBlacklistSuite.scala
@@ -86,8 +86,8 @@ class TaskSetBlacklistSuite extends SparkFunSuite {
Seq("exec1", "exec2").foreach { exec =>
assert(
execToFailures(exec).taskToFailureCountAndFailureTime === Map(
- 0 -> (1, 0),
- 1 -> (1, 0)
+ 0 -> ((1, 0)),
+ 1 -> ((1, 0))
)
)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index 7c3922e..eaec098 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -276,7 +276,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
}
test("kryo with collect for specialized tuples") {
- assert (sc.parallelize( Array((1, 11), (2, 22), (3, 33)) ).collect().head === (1, 11))
+ assert (sc.parallelize( Array((1, 11), (2, 22), (3, 33)) ).collect().head === ((1, 11)))
}
test("kryo with SerializableHyperLogLog") {
@@ -475,7 +475,7 @@ class KryoSerializerAutoResetDisabledSuite extends SparkFunSuite with SharedSpar
val deserializationStream = serInstance.deserializeStream(new ByteArrayInputStream(worldWorld))
assert(deserializationStream.readValue[Any]() === world)
deserializationStream.close()
- assert(serInstance.deserialize[Any](helloHello) === (hello, hello))
+ assert(serInstance.deserialize[Any](helloHello) === ((hello, hello)))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/util/TimeStampedHashMapSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/TimeStampedHashMapSuite.scala b/core/src/test/scala/org/apache/spark/util/TimeStampedHashMapSuite.scala
index fd9add7..dcae789 100644
--- a/core/src/test/scala/org/apache/spark/util/TimeStampedHashMapSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/TimeStampedHashMapSuite.scala
@@ -115,7 +115,7 @@ class TimeStampedHashMapSuite extends SparkFunSuite {
testMap2("k1") = "v1"
testMap2 --= keys
assert(testMap2.size === 1)
- assert(testMap2.iterator.toSeq.head === ("k1", "v1"))
+ assert(testMap2.iterator.toSeq.head === (("k1", "v1")))
// +
val testMap3 = testMap2 + (("k0", "v0"))
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
index aaf79eb..b36d6be 100644
--- a/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/VersionUtilsSuite.scala
@@ -58,10 +58,10 @@ class VersionUtilsSuite extends SparkFunSuite {
}
test("Parse Spark major and minor versions") {
- assert(majorMinorVersion("2.0") === (2, 0))
- assert(majorMinorVersion("12.10.11") === (12, 10))
- assert(majorMinorVersion("2.0.1-SNAPSHOT") === (2, 0))
- assert(majorMinorVersion("2.0.x") === (2, 0))
+ assert(majorMinorVersion("2.0") === ((2, 0)))
+ assert(majorMinorVersion("12.10.11") === ((12, 10)))
+ assert(majorMinorVersion("2.0.1-SNAPSHOT") === ((2, 0)))
+ assert(majorMinorVersion("2.0.x") === ((2, 0)))
withClue("majorMinorVersion parsing should fail for invalid major version number") {
intercept[IllegalArgumentException] {
majorMinorVersion("2z.0")
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/util/collection/AppendOnlyMapSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/collection/AppendOnlyMapSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/AppendOnlyMapSuite.scala
index a2a6d70..6b4e928 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/AppendOnlyMapSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/AppendOnlyMapSuite.scala
@@ -181,9 +181,9 @@ class AppendOnlyMapSuite extends SparkFunSuite {
// Should be sorted by key
assert(it.hasNext)
var previous = it.next()
- assert(previous == (null, "happy new year!"))
+ assert(previous == ((null, "happy new year!")))
previous = it.next()
- assert(previous == ("1", "2014"))
+ assert(previous == (("1", "2014")))
while (it.hasNext) {
val kv = it.next()
assert(kv._1.toInt > previous._1.toInt)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/util/collection/ExternalSorterSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/collection/ExternalSorterSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/ExternalSorterSuite.scala
index 6bcc601..47173b8 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/ExternalSorterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/ExternalSorterSuite.scala
@@ -388,13 +388,13 @@ class ExternalSorterSuite extends SparkFunSuite with LocalSparkContext {
sorter.insertAll(elements)
assert(sorter.numSpills > 0, "sorter did not spill")
val iter = sorter.partitionedIterator.map(p => (p._1, p._2.toList))
- assert(iter.next() === (0, Nil))
- assert(iter.next() === (1, List((1, 1))))
- assert(iter.next() === (2, (0 until 1000).map(x => (2, 2)).toList))
- assert(iter.next() === (3, Nil))
- assert(iter.next() === (4, Nil))
- assert(iter.next() === (5, List((5, 5))))
- assert(iter.next() === (6, Nil))
+ assert(iter.next() === ((0, Nil)))
+ assert(iter.next() === ((1, List((1, 1)))))
+ assert(iter.next() === ((2, (0 until 1000).map(x => (2, 2)).toList)))
+ assert(iter.next() === ((3, Nil)))
+ assert(iter.next() === ((4, Nil)))
+ assert(iter.next() === ((5, List((5, 5)))))
+ assert(iter.next() === ((6, Nil)))
sorter.stop()
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala b/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
index 335ecb9..08a3200 100644
--- a/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/collection/OpenHashMapSuite.scala
@@ -75,7 +75,7 @@ class OpenHashMapSuite extends SparkFunSuite with Matchers {
for ((k, v) <- map) {
set.add((k, v))
}
- val expected = (1 to 1000).map(x => (x.toString, x)) :+ (null.asInstanceOf[String], -1)
+ val expected = (1 to 1000).map(x => (x.toString, x)) :+ ((null.asInstanceOf[String], -1))
assert(set === expected.toSet)
}
@@ -103,7 +103,8 @@ class OpenHashMapSuite extends SparkFunSuite with Matchers {
for ((k, v) <- map) {
set.add((k, v))
}
- val expected = (1 to 1000).map(_.toString).map(x => (x, x)) :+ (null.asInstanceOf[String], "-1")
+ val expected =
+ (1 to 1000).map(_.toString).map(x => (x, x)) :+ ((null.asInstanceOf[String], "-1"))
assert(set === expected.toSet)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/dev/change-scala-version.sh
----------------------------------------------------------------------
diff --git a/dev/change-scala-version.sh b/dev/change-scala-version.sh
index d7975df..022e68c 100755
--- a/dev/change-scala-version.sh
+++ b/dev/change-scala-version.sh
@@ -19,7 +19,7 @@
set -e
-VALID_VERSIONS=( 2.10 2.11 )
+VALID_VERSIONS=( 2.11 2.12 )
usage() {
echo "Usage: $(basename $0) [-h|--help] <version>
@@ -45,7 +45,7 @@ check_scala_version() {
check_scala_version "$TO_VERSION"
if [ $TO_VERSION = "2.11" ]; then
- FROM_VERSION="2.10"
+ FROM_VERSION="2.12"
else
FROM_VERSION="2.11"
fi
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/dev/change-version-to-2.10.sh
----------------------------------------------------------------------
diff --git a/dev/change-version-to-2.10.sh b/dev/change-version-to-2.10.sh
deleted file mode 100755
index b718d94..0000000
--- a/dev/change-version-to-2.10.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script exists for backwards compatibility. Use change-scala-version.sh instead.
-echo "This script is deprecated. Please instead run: change-scala-version.sh 2.10"
-
-$(dirname $0)/change-scala-version.sh 2.10
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/dev/change-version-to-2.11.sh
----------------------------------------------------------------------
diff --git a/dev/change-version-to-2.11.sh b/dev/change-version-to-2.11.sh
deleted file mode 100755
index 9308795..0000000
--- a/dev/change-version-to-2.11.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script exists for backwards compatibility. Use change-scala-version.sh instead.
-echo "This script is deprecated. Please instead run: change-scala-version.sh 2.11"
-
-$(dirname $0)/change-scala-version.sh 2.11
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/dev/create-release/release-build.sh
----------------------------------------------------------------------
diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index a72307a..9bf2899 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -155,10 +155,10 @@ if [[ "$1" == "package" ]]; then
cd spark-$SPARK_VERSION-bin-$NAME
- # TODO There should probably be a flag to make-distribution to allow 2.10 support
- if [[ $FLAGS == *scala-2.10* ]]; then
- ./dev/change-scala-version.sh 2.10
- fi
+ # TODO There should probably be a flag to make-distribution to allow 2.12 support
+ #if [[ $FLAGS == *scala-2.12* ]]; then
+ # ./dev/change-scala-version.sh 2.12
+ #fi
export ZINC_PORT=$ZINC_PORT
echo "Creating distribution: $NAME ($FLAGS)"
@@ -305,9 +305,9 @@ if [[ "$1" == "publish-snapshot" ]]; then
export ZINC_PORT=$(python -S -c "import random; print random.randrange(3030,4030)")
$MVN -DzincPort=$ZINC_PORT --settings $tmp_settings -DskipTests $PUBLISH_PROFILES deploy
- ./dev/change-scala-version.sh 2.10
- $MVN -DzincPort=$ZINC_PORT -Dscala-2.10 --settings $tmp_settings \
- -DskipTests $PUBLISH_PROFILES clean deploy
+ #./dev/change-scala-version.sh 2.12
+ #$MVN -DzincPort=$ZINC_PORT -Pscala-2.12 --settings $tmp_settings \
+ # -DskipTests $PUBLISH_PROFILES clean deploy
# Clean-up Zinc nailgun process
/usr/sbin/lsof -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
@@ -342,16 +342,13 @@ if [[ "$1" == "publish-release" ]]; then
$MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -DskipTests $PUBLISH_PROFILES clean install
- ./dev/change-scala-version.sh 2.10
-
- $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Dscala-2.10 \
- -DskipTests $PUBLISH_PROFILES clean install
+ #./dev/change-scala-version.sh 2.12
+ #$MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Pscala-2.12 \
+ # -DskipTests $PUBLISH_PROFILES clean install
# Clean-up Zinc nailgun process
/usr/sbin/lsof -P |grep $ZINC_PORT | grep LISTEN | awk '{ print $2; }' | xargs kill
- ./dev/change-version-to-2.10.sh
-
pushd $tmp_repo/org/apache/spark
# Remove any extra files generated during install
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/docs/building-spark.md
----------------------------------------------------------------------
diff --git a/docs/building-spark.md b/docs/building-spark.md
index 815843c..69d8302 100644
--- a/docs/building-spark.md
+++ b/docs/building-spark.md
@@ -91,14 +91,6 @@ like ZooKeeper and Hadoop itself.
./build/mvn -Pmesos -DskipTests clean package
-## Building for Scala 2.10
-To produce a Spark package compiled with Scala 2.10, use the `-Dscala-2.10` property:
-
- ./dev/change-scala-version.sh 2.10
- ./build/mvn -Pyarn -Dscala-2.10 -DskipTests clean package
-
-Note that support for Scala 2.10 is deprecated as of Spark 2.1.0 and may be removed in Spark 2.3.0.
-
## Building submodules individually
It's possible to build Spark sub-modules using the `mvn -pl` option.
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/docs/index.md
----------------------------------------------------------------------
diff --git a/docs/index.md b/docs/index.md
index 81ed465..07b6b17 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -31,8 +31,7 @@ uses Scala {{site.SCALA_BINARY_VERSION}}. You will need to use a compatible Scal
({{site.SCALA_BINARY_VERSION}}.x).
Note that support for Java 7, Python 2.6 and old Hadoop versions before 2.6.5 were removed as of Spark 2.2.0.
-
-Note that support for Scala 2.10 is deprecated as of Spark 2.1.0, and may be removed in Spark 2.3.0.
+Support for Scala 2.10 was removed as of 2.3.0.
# Running the Examples and Shell
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala b/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala
index 8f8262d..8441b5a 100644
--- a/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala
@@ -52,7 +52,7 @@ object AggregateMessagesExample {
triplet => { // Map Function
if (triplet.srcAttr > triplet.dstAttr) {
// Send message to destination vertex containing counter and age
- triplet.sendToDst(1, triplet.srcAttr)
+ triplet.sendToDst((1, triplet.srcAttr))
}
},
// Add counter and age
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala
index ae60577..8985c85 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala
@@ -38,7 +38,7 @@ abstract class AbstractParams[T: TypeTag] {
*/
override def toString: String = {
val tpe = tag.tpe
- val allAccessors = tpe.declarations.collect {
+ val allAccessors = tpe.decls.collect {
case m: MethodSymbol if m.isCaseAccessor => m
}
val mirror = runtimeMirror(getClass.getClassLoader)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala b/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala
index b9a612d..958361a 100644
--- a/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala
@@ -29,8 +29,6 @@ import org.apache.spark.sql.types._
object SparkSQLExample {
// $example on:create_ds$
- // Note: Case classes in Scala 2.10 can support only up to 22 fields. To work around this limit,
- // you can use custom classes that implement the Product interface
case class Person(name: String, age: Long)
// $example off:create_ds$
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala
----------------------------------------------------------------------
diff --git a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala
index 2b92577..5ea52b6 100644
--- a/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala
+++ b/external/kafka-0-8/src/main/scala/org/apache/spark/streaming/kafka/KafkaRDD.scala
@@ -259,8 +259,8 @@ object KafkaRDD {
messageHandler: MessageAndMetadata[K, V] => R
): KafkaRDD[K, V, U, T, R] = {
val leaders = untilOffsets.map { case (tp, lo) =>
- tp -> (lo.host, lo.port)
- }.toMap
+ tp -> ((lo.host, lo.port))
+ }
val offsetRanges = fromOffsets.map { case (tp, fo) =>
val uo = untilOffsets(tp)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/graphx/src/main/scala/org/apache/spark/graphx/EdgeContext.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/EdgeContext.scala b/graphx/src/main/scala/org/apache/spark/graphx/EdgeContext.scala
index 2343017..3b96a42 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/EdgeContext.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/EdgeContext.scala
@@ -63,5 +63,5 @@ object EdgeContext {
* }}}
*/
def unapply[VD, ED, A](edge: EdgeContext[VD, ED, A]): Some[(VertexId, VertexId, VD, VD, ED)] =
- Some(edge.srcId, edge.dstId, edge.srcAttr, edge.dstAttr, edge.attr)
+ Some((edge.srcId, edge.dstId, edge.srcAttr, edge.dstAttr, edge.attr))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
----------------------------------------------------------------------
diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
index 6c0c3eb..481ff20 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
@@ -229,17 +229,17 @@ abstract class AbstractCommandBuilder {
return scala;
}
String sparkHome = getSparkHome();
- File scala210 = new File(sparkHome, "launcher/target/scala-2.10");
+ //File scala212 = new File(sparkHome, "launcher/target/scala-2.12");
File scala211 = new File(sparkHome, "launcher/target/scala-2.11");
- checkState(!scala210.isDirectory() || !scala211.isDirectory(),
- "Presence of build for both scala versions (2.10 and 2.11) detected.\n" +
- "Either clean one of them or set SPARK_SCALA_VERSION in your environment.");
- if (scala210.isDirectory()) {
- return "2.10";
- } else {
- checkState(scala211.isDirectory(), "Cannot find any build directories.");
- return "2.11";
- }
+ //checkState(!scala210.isDirectory() || !scala211.isDirectory(),
+ // "Presence of build for multiple Scala versions detected.\n" +
+ // "Either clean one of them or set SPARK_SCALA_VERSION in your environment.");
+ //if (scala212.isDirectory()) {
+ // return "2.12";
+ //} else {
+ checkState(scala211.isDirectory(), "Cannot find any build directories.");
+ return "2.11";
+ //}
}
String getSparkHome() {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
----------------------------------------------------------------------
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
index 07f3bc2..66c5362 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
@@ -856,7 +856,7 @@ object SparseMatrix {
var prevRow = -1
var prevVal = 0.0
// Append a dummy entry to include the last one at the end of the loop.
- (sortedEntries.view :+ (numRows, numCols, 1.0)).foreach { case (i, j, v) =>
+ (sortedEntries.view :+ ((numRows, numCols, 1.0))).foreach { case (i, j, v) =>
if (v != 0) {
if (i == prevRow && j == prevCol) {
prevVal += v
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala
----------------------------------------------------------------------
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala
index 9f82020..7fb9034 100644
--- a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/MatricesSuite.scala
@@ -633,22 +633,22 @@ class MatricesSuite extends SparkMLFunSuite {
dnMap.put((i, j), value)
}
assert(dnMap.size === 6)
- assert(dnMap(0, 0) === 1.0)
- assert(dnMap(1, 0) === 2.0)
- assert(dnMap(2, 0) === 0.0)
- assert(dnMap(0, 1) === 0.0)
- assert(dnMap(1, 1) === 4.0)
- assert(dnMap(2, 1) === 5.0)
+ assert(dnMap((0, 0)) === 1.0)
+ assert(dnMap((1, 0)) === 2.0)
+ assert(dnMap((2, 0)) === 0.0)
+ assert(dnMap((0, 1)) === 0.0)
+ assert(dnMap((1, 1)) === 4.0)
+ assert(dnMap((2, 1)) === 5.0)
val spMap = MutableMap[(Int, Int), Double]()
sp.foreachActive { (i, j, value) =>
spMap.put((i, j), value)
}
assert(spMap.size === 4)
- assert(spMap(0, 0) === 1.0)
- assert(spMap(1, 0) === 2.0)
- assert(spMap(1, 1) === 4.0)
- assert(spMap(2, 1) === 5.0)
+ assert(spMap((0, 0)) === 1.0)
+ assert(spMap((1, 0)) === 2.0)
+ assert(spMap((1, 1)) === 4.0)
+ assert(spMap((2, 1)) === 5.0)
}
test("horzcat, vertcat, eye, speye") {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
index 0d6e903..4aa4c36 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
@@ -436,7 +436,7 @@ private[ml] object EnsembleModelReadWrite {
val treesMetadataRDD: RDD[(Int, (Metadata, Double))] = sql.read.parquet(treesMetadataPath)
.select("treeID", "metadata", "weights").as[(Int, String, Double)].rdd.map {
case (treeID: Int, json: String, weights: Double) =>
- treeID -> (DefaultParamsReader.parseMetadata(json, treeClassName), weights)
+ treeID -> ((DefaultParamsReader.parseMetadata(json, treeClassName), weights))
}
val treesMetadataWeights = treesMetadataRDD.sortByKey().values.collect()
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
index 2b2b5fe..bf9b4cf 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
@@ -797,7 +797,7 @@ object SparseMatrix {
var prevRow = -1
var prevVal = 0.0
// Append a dummy entry to include the last one at the end of the loop.
- (sortedEntries.view :+ (numRows, numCols, 1.0)).foreach { case (i, j, v) =>
+ (sortedEntries.view :+ ((numRows, numCols, 1.0))).foreach { case (i, j, v) =>
if (v != 0) {
if (i == prevRow && j == prevCol) {
prevVal += v
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala
index 20d68a3..7caacd1 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala
@@ -275,7 +275,7 @@ class BlockMatrix @Since("1.3.0") (
val rows = blocks.flatMap { case ((blockRowIdx, blockColIdx), mat) =>
mat.rowIter.zipWithIndex.map {
case (vector, rowIdx) =>
- blockRowIdx * rowsPerBlock + rowIdx -> (blockColIdx, vector.asBreeze)
+ blockRowIdx * rowsPerBlock + rowIdx -> ((blockColIdx, vector.asBreeze))
}
}.groupByKey().map { case (rowIdx, vectors) =>
val numberNonZeroPerRow = vectors.map(_._2.activeSize).sum.toDouble / cols.toDouble
@@ -286,7 +286,7 @@ class BlockMatrix @Since("1.3.0") (
BDV.zeros[Double](cols)
}
- vectors.foreach { case (blockColIdx: Int, vec: BV[Double]) =>
+ vectors.foreach { case (blockColIdx: Int, vec: BV[_]) =>
val offset = colsPerBlock * blockColIdx
wholeVector(offset until Math.min(cols, offset + colsPerBlock)) := vec
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
index 0a0fea2..45d3f9b 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
@@ -898,7 +898,7 @@ class ALSStorageSuite
// check final factor RDD default storage levels
val defaultFactorRDDs = sc.getPersistentRDDs.collect {
case (id, rdd) if rdd.name == "userFactors" || rdd.name == "itemFactors" =>
- rdd.name -> (id, rdd.getStorageLevel)
+ rdd.name -> ((id, rdd.getStorageLevel))
}.toMap
defaultFactorRDDs.foreach { case (_, (id, level)) =>
assert(level == StorageLevel.MEMORY_AND_DISK)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
index 086bb21..8906e52 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
@@ -151,7 +151,7 @@ class LDASuite extends SparkFunSuite with MLlibTestSparkContext {
// Check: topTopicAssignments
// Make sure it assigns a topic to each term appearing in each doc.
val topTopicAssignments: Map[Long, (Array[Int], Array[Int])] =
- model.topicAssignments.collect().map(x => x._1 -> (x._2, x._3)).toMap
+ model.topicAssignments.collect().map(x => x._1 -> ((x._2, x._3))).toMap
assert(topTopicAssignments.keys.max < tinyCorpus.length)
tinyCorpus.foreach { case (docID: Long, doc: Vector) =>
if (topTopicAssignments.contains(docID)) {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
index 93c00d8..6736e7d 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
@@ -241,22 +241,22 @@ class MatricesSuite extends SparkFunSuite {
dnMap.put((i, j), value)
}
assert(dnMap.size === 6)
- assert(dnMap(0, 0) === 1.0)
- assert(dnMap(1, 0) === 2.0)
- assert(dnMap(2, 0) === 0.0)
- assert(dnMap(0, 1) === 0.0)
- assert(dnMap(1, 1) === 4.0)
- assert(dnMap(2, 1) === 5.0)
+ assert(dnMap((0, 0)) === 1.0)
+ assert(dnMap((1, 0)) === 2.0)
+ assert(dnMap((2, 0)) === 0.0)
+ assert(dnMap((0, 1)) === 0.0)
+ assert(dnMap((1, 1)) === 4.0)
+ assert(dnMap((2, 1)) === 5.0)
val spMap = MutableMap[(Int, Int), Double]()
sp.foreachActive { (i, j, value) =>
spMap.put((i, j), value)
}
assert(spMap.size === 4)
- assert(spMap(0, 0) === 1.0)
- assert(spMap(1, 0) === 2.0)
- assert(spMap(1, 1) === 4.0)
- assert(spMap(2, 1) === 5.0)
+ assert(spMap((0, 0)) === 1.0)
+ assert(spMap((1, 0)) === 2.0)
+ assert(spMap((1, 1)) === 4.0)
+ assert(spMap((2, 1)) === 5.0)
}
test("horzcat, vertcat, eye, speye") {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f124ba4..bea2798 100644
--- a/pom.xml
+++ b/pom.xml
@@ -498,7 +498,7 @@
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
<version>${slf4j.version}</version>
- <!-- <scope>runtime</scope> --> <!-- more correct, but scalac 2.10.3 doesn't like it -->
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
@@ -1859,9 +1859,9 @@
<version>${antlr4.version}</version>
</dependency>
<dependency>
- <groupId>${jline.groupid}</groupId>
+ <groupId>jline</groupId>
<artifactId>jline</artifactId>
- <version>${jline.version}</version>
+ <version>2.12.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -1933,6 +1933,7 @@
-->
<exclude>org.jboss.netty</exclude>
<exclude>org.codehaus.groovy</exclude>
+ <exclude>*:*_2.10</exclude>
</excludes>
<searchTransitive>true</searchTransitive>
</bannedDependencies>
@@ -1987,6 +1988,8 @@
<arg>-unchecked</arg>
<arg>-deprecation</arg>
<arg>-feature</arg>
+ <arg>-explaintypes</arg>
+ <arg>-Yno-adapted-args</arg>
</args>
<jvmArgs>
<jvmArg>-Xms1024m</jvmArg>
@@ -2586,44 +2589,6 @@
</profile>
<profile>
- <id>scala-2.10</id>
- <activation>
- <property><name>scala-2.10</name></property>
- </activation>
- <properties>
- <scala.version>2.10.6</scala.version>
- <scala.binary.version>2.10</scala.binary.version>
- <jline.version>${scala.version}</jline.version>
- <jline.groupid>org.scala-lang</jline.groupid>
- </properties>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-enforcer-plugin</artifactId>
- <executions>
- <execution>
- <id>enforce-versions</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <configuration>
- <rules>
- <bannedDependencies>
- <excludes combine.children="append">
- <exclude>*:*_2.11</exclude>
- </excludes>
- </bannedDependencies>
- </rules>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
-
- <profile>
<id>test-java-home</id>
<activation>
<property><name>env.JAVA_HOME</name></property>
@@ -2633,16 +2598,18 @@
</properties>
</profile>
+ <!-- Exists for backwards compatibility; profile doesn't do anything -->
<profile>
<id>scala-2.11</id>
- <activation>
- <property><name>!scala-2.10</name></property>
- </activation>
+ </profile>
+
+ <!-- Draft of Scala 2.12 profile for later -->
+ <!--
+ <profile>
+ <id>scala-2.12</id>
<properties>
- <scala.version>2.11.8</scala.version>
- <scala.binary.version>2.11</scala.binary.version>
- <jline.version>2.12.1</jline.version>
- <jline.groupid>jline</jline.groupid>
+ <scala.version>2.12.1</scala.version>
+ <scala.binary.version>2.12</scala.binary.version>
</properties>
<build>
<plugins>
@@ -2659,7 +2626,7 @@
<rules>
<bannedDependencies>
<excludes combine.children="append">
- <exclude>*:*_2.10</exclude>
+ <exclude>*:*_2.11</exclude>
</excludes>
</bannedDependencies>
</rules>
@@ -2670,10 +2637,11 @@
</plugins>
</build>
</profile>
+ -->
<!--
This is a profile to enable the use of the ASF snapshot and staging repositories
- during a build. It is useful when testing againt nightly or RC releases of dependencies.
+ during a build. It is useful when testing against nightly or RC releases of dependencies.
It MUST NOT be used when building copies of Spark to use in production of for distribution,
-->
<profile>
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 89b0c7a..41f3a04 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -87,19 +87,11 @@ object SparkBuild extends PomBuild {
val projectsMap: Map[String, Seq[Setting[_]]] = Map.empty
override val profiles = {
- val profiles = Properties.envOrNone("SBT_MAVEN_PROFILES") match {
+ Properties.envOrNone("SBT_MAVEN_PROFILES") match {
case None => Seq("sbt")
case Some(v) =>
v.split("(\\s+|,)").filterNot(_.isEmpty).map(_.trim.replaceAll("-P", "")).toSeq
}
-
- if (System.getProperty("scala-2.10") == "") {
- // To activate scala-2.10 profile, replace empty property value to non-empty value
- // in the same way as Maven which handles -Dname as -Dname=true before executes build process.
- // see: https://github.com/apache/maven/blob/maven-3.0.4/maven-embedder/src/main/java/org/apache/maven/cli/MavenCli.java#L1082
- System.setProperty("scala-2.10", "true")
- }
- profiles
}
Properties.envOrNone("SBT_MAVEN_PROPERTIES") match {
@@ -234,9 +226,7 @@ object SparkBuild extends PomBuild {
},
javacJVMVersion := "1.8",
- // SBT Scala 2.10 build still doesn't support Java 8, because scalac 2.10 doesn't, but,
- // it also doesn't touch Java 8 code and it's OK to emit Java 7 bytecode in this case
- scalacJVMVersion := (if (System.getProperty("scala-2.10") == "true") "1.7" else "1.8"),
+ scalacJVMVersion := "1.8",
javacOptions in Compile ++= Seq(
"-encoding", "UTF-8",
@@ -477,7 +467,6 @@ object OldDeps {
def oldDepsSettings() = Defaults.coreDefaultSettings ++ Seq(
name := "old-deps",
- scalaVersion := "2.10.5",
libraryDependencies := allPreviousArtifactKeys.value.flatten
)
}
@@ -756,13 +745,7 @@ object CopyDependencies {
object TestSettings {
import BuildCommons._
- private val scalaBinaryVersion =
- if (System.getProperty("scala-2.10") == "true") {
- "2.10"
- } else {
- "2.11"
- }
-
+ private val scalaBinaryVersion = "2.11"
lazy val settings = Seq (
// Fork new JVMs for tests and set Java options for those
fork := true,
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/python/run-tests.py
----------------------------------------------------------------------
diff --git a/python/run-tests.py b/python/run-tests.py
index b2e5043..afd3d29 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -54,7 +54,8 @@ FAILURE_REPORTING_LOCK = Lock()
LOGGER = logging.getLogger()
# Find out where the assembly jars are located.
-for scala in ["2.11", "2.10"]:
+# Later, add back 2.12 to this list:
+for scala in ["2.11"]:
build_dir = os.path.join(SPARK_HOME, "assembly", "target", "scala-" + scala)
if os.path.isdir(build_dir):
SPARK_DIST_CLASSPATH = os.path.join(build_dir, "jars", "*")
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/pom.xml
----------------------------------------------------------------------
diff --git a/repl/pom.xml b/repl/pom.xml
index 6d133a3..51eb9b6 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -32,8 +32,8 @@
<properties>
<sbt.project.name>repl</sbt.project.name>
- <extra.source.dir>scala-2.10/src/main/scala</extra.source.dir>
- <extra.testsource.dir>scala-2.10/src/test/scala</extra.testsource.dir>
+ <extra.source.dir>scala-2.11/src/main/scala</extra.source.dir>
+ <extra.testsource.dir>scala-2.11/src/test/scala</extra.testsource.dir>
</properties>
<dependencies>
@@ -71,7 +71,7 @@
<version>${scala.version}</version>
</dependency>
<dependency>
- <groupId>${jline.groupid}</groupId>
+ <groupId>jline</groupId>
<artifactId>jline</artifactId>
</dependency>
<dependency>
@@ -170,23 +170,17 @@
</plugin>
</plugins>
</build>
+
+ <!--
<profiles>
<profile>
- <id>scala-2.10</id>
- <activation>
- <property><name>scala-2.10</name></property>
- </activation>
- </profile>
-
- <profile>
- <id>scala-2.11</id>
- <activation>
- <property><name>!scala-2.10</name></property>
- </activation>
+ <id>scala-2.12</id>
<properties>
- <extra.source.dir>scala-2.11/src/main/scala</extra.source.dir>
- <extra.testsource.dir>scala-2.11/src/test/scala</extra.testsource.dir>
+ <extra.source.dir>scala-2.12/src/main/scala</extra.source.dir>
+ <extra.testsource.dir>scala-2.12/src/test/scala</extra.testsource.dir>
</properties>
</profile>
</profiles>
+ -->
+
</project>
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/Main.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/Main.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/Main.scala
deleted file mode 100644
index fba321b..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/Main.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.repl
-
-import org.apache.spark.internal.Logging
-
-object Main extends Logging {
-
- initializeLogIfNecessary(true)
- Signaling.cancelOnInterrupt()
-
- private var _interp: SparkILoop = _
-
- def interp = _interp
-
- def interp_=(i: SparkILoop) { _interp = i }
-
- def main(args: Array[String]) {
- _interp = new SparkILoop
- _interp.process(args)
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala
deleted file mode 100644
index be9b790..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkCommandLine.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.repl
-
-import scala.tools.nsc.{CompilerCommand, Settings}
-
-import org.apache.spark.annotation.DeveloperApi
-
-/**
- * Command class enabling Spark-specific command line options (provided by
- * <i>org.apache.spark.repl.SparkRunnerSettings</i>).
- *
- * @example new SparkCommandLine(Nil).settings
- *
- * @param args The list of command line arguments
- * @param settings The underlying settings to associate with this set of
- * command-line options
- */
-@DeveloperApi
-class SparkCommandLine(args: List[String], override val settings: Settings)
- extends CompilerCommand(args, settings) {
- def this(args: List[String], error: String => Unit) {
- this(args, new SparkRunnerSettings(error))
- }
-
- def this(args: List[String]) {
- // scalastyle:off println
- this(args, str => Console.println("Error: " + str))
- // scalastyle:on println
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala
deleted file mode 100644
index 2b5d56a..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala
+++ /dev/null
@@ -1,114 +0,0 @@
-// scalastyle:off
-
-/* NSC -- new Scala compiler
- * Copyright 2005-2013 LAMP/EPFL
- * @author Paul Phillips
- */
-
-package org.apache.spark.repl
-
-import scala.tools.nsc._
-import scala.tools.nsc.interpreter._
-
-import scala.reflect.internal.util.BatchSourceFile
-import scala.tools.nsc.ast.parser.Tokens.EOF
-
-import org.apache.spark.internal.Logging
-
-private[repl] trait SparkExprTyper extends Logging {
- val repl: SparkIMain
-
- import repl._
- import global.{ reporter => _, Import => _, _ }
- import definitions._
- import syntaxAnalyzer.{ UnitParser, UnitScanner, token2name }
- import naming.freshInternalVarName
-
- object codeParser extends { val global: repl.global.type = repl.global } with CodeHandlers[Tree] {
- def applyRule[T](code: String, rule: UnitParser => T): T = {
- reporter.reset()
- val scanner = newUnitParser(code)
- val result = rule(scanner)
-
- if (!reporter.hasErrors)
- scanner.accept(EOF)
-
- result
- }
-
- def defns(code: String) = stmts(code) collect { case x: DefTree => x }
- def expr(code: String) = applyRule(code, _.expr())
- def stmts(code: String) = applyRule(code, _.templateStats())
- def stmt(code: String) = stmts(code).last // guaranteed nonempty
- }
-
- /** Parse a line into a sequence of trees. Returns None if the input is incomplete. */
- def parse(line: String): Option[List[Tree]] = debugging(s"""parse("$line")""") {
- var isIncomplete = false
- reporter.withIncompleteHandler((_, _) => isIncomplete = true) {
- val trees = codeParser.stmts(line)
- if (reporter.hasErrors) {
- Some(Nil)
- } else if (isIncomplete) {
- None
- } else {
- Some(trees)
- }
- }
- }
- // def parsesAsExpr(line: String) = {
- // import codeParser._
- // (opt expr line).isDefined
- // }
-
- def symbolOfLine(code: String): Symbol = {
- def asExpr(): Symbol = {
- val name = freshInternalVarName()
- // Typing it with a lazy val would give us the right type, but runs
- // into compiler bugs with things like existentials, so we compile it
- // behind a def and strip the NullaryMethodType which wraps the expr.
- val line = "def " + name + " = {\n" + code + "\n}"
-
- interpretSynthetic(line) match {
- case IR.Success =>
- val sym0 = symbolOfTerm(name)
- // drop NullaryMethodType
- val sym = sym0.cloneSymbol setInfo afterTyper(sym0.info.finalResultType)
- if (sym.info.typeSymbol eq UnitClass) NoSymbol else sym
- case _ => NoSymbol
- }
- }
- def asDefn(): Symbol = {
- val old = repl.definedSymbolList.toSet
-
- interpretSynthetic(code) match {
- case IR.Success =>
- repl.definedSymbolList filterNot old match {
- case Nil => NoSymbol
- case sym :: Nil => sym
- case syms => NoSymbol.newOverloaded(NoPrefix, syms)
- }
- case _ => NoSymbol
- }
- }
- beQuietDuring(asExpr()) orElse beQuietDuring(asDefn())
- }
-
- private var typeOfExpressionDepth = 0
- def typeOfExpression(expr: String, silent: Boolean = true): Type = {
- if (typeOfExpressionDepth > 2) {
- logDebug("Terminating typeOfExpression recursion for expression: " + expr)
- return NoType
- }
- typeOfExpressionDepth += 1
- // Don't presently have a good way to suppress undesirable success output
- // while letting errors through, so it is first trying it silently: if there
- // is an error, and errors are desired, then it re-evaluates non-silently
- // to induce the error message.
- try beSilentDuring(symbolOfLine(expr).tpe) match {
- case NoType if !silent => symbolOfLine(expr).tpe // generate error
- case tpe => tpe
- }
- finally typeOfExpressionDepth -= 1
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkHelper.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkHelper.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkHelper.scala
deleted file mode 100644
index 955be17..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkHelper.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package scala.tools.nsc
-
-import org.apache.spark.annotation.DeveloperApi
-
-// NOTE: Forced to be public (and in scala.tools.nsc package) to access the
-// settings "explicitParentLoader" method
-
-/**
- * Provides exposure for the explicitParentLoader method on settings instances.
- */
-@DeveloperApi
-object SparkHelper {
- /**
- * Retrieves the explicit parent loader for the provided settings.
- *
- * @param settings The settings whose explicit parent loader to retrieve
- *
- * @return The Optional classloader representing the explicit parent loader
- */
- @DeveloperApi
- def explicitParentLoader(settings: Settings) = settings.explicitParentLoader
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[3/5] spark git commit: [SPARK-19810][BUILD][CORE] Remove support for
Scala 2.10
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkIMain.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkIMain.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkIMain.scala
deleted file mode 100644
index 74a04d5..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkIMain.scala
+++ /dev/null
@@ -1,1808 +0,0 @@
-// scalastyle:off
-
-/* NSC -- new Scala compiler
- * Copyright 2005-2013 LAMP/EPFL
- * @author Martin Odersky
- */
-
-package org.apache.spark.repl
-
-import java.io.File
-
-import scala.tools.nsc._
-import scala.tools.nsc.backend.JavaPlatform
-import scala.tools.nsc.interpreter._
-
-import Predef.{ println => _, _ }
-import scala.tools.nsc.util.{MergedClassPath, stringFromWriter, ScalaClassLoader, stackTraceString}
-import scala.reflect.internal.util._
-import java.net.URL
-import scala.sys.BooleanProp
-import io.{AbstractFile, PlainFile, VirtualDirectory}
-
-import reporters._
-import symtab.Flags
-import scala.reflect.internal.Names
-import scala.tools.util.PathResolver
-import ScalaClassLoader.URLClassLoader
-import scala.tools.nsc.util.Exceptional.unwrap
-import scala.collection.{ mutable, immutable }
-import scala.util.control.Exception.{ ultimately }
-import SparkIMain._
-import java.util.concurrent.Future
-import typechecker.Analyzer
-import scala.language.implicitConversions
-import scala.reflect.runtime.{ universe => ru }
-import scala.reflect.{ ClassTag, classTag }
-import scala.tools.reflect.StdRuntimeTags._
-import scala.util.control.ControlThrowable
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.internal.Logging
-import org.apache.spark.util.Utils
-import org.apache.spark.annotation.DeveloperApi
-
-// /** directory to save .class files to */
-// private class ReplVirtualDirectory(out: JPrintWriter) extends VirtualDirectory("((memory))", None) {
-// private def pp(root: AbstractFile, indentLevel: Int) {
-// val spaces = " " * indentLevel
-// out.println(spaces + root.name)
-// if (root.isDirectory)
-// root.toList sortBy (_.name) foreach (x => pp(x, indentLevel + 1))
-// }
-// // print the contents hierarchically
-// def show() = pp(this, 0)
-// }
-
- /** An interpreter for Scala code.
- *
- * The main public entry points are compile(), interpret(), and bind().
- * The compile() method loads a complete Scala file. The interpret() method
- * executes one line of Scala code at the request of the user. The bind()
- * method binds an object to a variable that can then be used by later
- * interpreted code.
- *
- * The overall approach is based on compiling the requested code and then
- * using a Java classloader and Java reflection to run the code
- * and access its results.
- *
- * In more detail, a single compiler instance is used
- * to accumulate all successfully compiled or interpreted Scala code. To
- * "interpret" a line of code, the compiler generates a fresh object that
- * includes the line of code and which has public member(s) to export
- * all variables defined by that code. To extract the result of an
- * interpreted line to show the user, a second "result object" is created
- * which imports the variables exported by the above object and then
- * exports members called "$eval" and "$print". To accommodate user expressions
- * that read from variables or methods defined in previous statements, "import"
- * statements are used.
- *
- * This interpreter shares the strengths and weaknesses of using the
- * full compiler-to-Java. The main strength is that interpreted code
- * behaves exactly as does compiled code, including running at full speed.
- * The main weakness is that redefining classes and methods is not handled
- * properly, because rebinding at the Java level is technically difficult.
- *
- * @author Moez A. Abdel-Gawad
- * @author Lex Spoon
- */
- @DeveloperApi
- class SparkIMain(
- initialSettings: Settings,
- val out: JPrintWriter,
- propagateExceptions: Boolean = false)
- extends SparkImports with Logging { imain =>
-
- private val conf = new SparkConf()
-
- private val SPARK_DEBUG_REPL: Boolean = (System.getenv("SPARK_DEBUG_REPL") == "1")
- /** Local directory to save .class files too */
- private[repl] val outputDir = {
- val rootDir = conf.getOption("spark.repl.classdir").getOrElse(Utils.getLocalDir(conf))
- Utils.createTempDir(root = rootDir, namePrefix = "repl")
- }
- if (SPARK_DEBUG_REPL) {
- echo("Output directory: " + outputDir)
- }
-
- /**
- * Returns the path to the output directory containing all generated
- * class files that will be served by the REPL class server.
- */
- @DeveloperApi
- lazy val getClassOutputDirectory = outputDir
-
- private val virtualDirectory = new PlainFile(outputDir) // "directory" for classfiles
- /** Jetty server that will serve our classes to worker nodes */
- private var currentSettings: Settings = initialSettings
- private var printResults = true // whether to print result lines
- private var totalSilence = false // whether to print anything
- private var _initializeComplete = false // compiler is initialized
- private var _isInitialized: Future[Boolean] = null // set up initialization future
- private var bindExceptions = true // whether to bind the lastException variable
- private var _executionWrapper = "" // code to be wrapped around all lines
-
- /** We're going to go to some trouble to initialize the compiler asynchronously.
- * It's critical that nothing call into it until it's been initialized or we will
- * run into unrecoverable issues, but the perceived repl startup time goes
- * through the roof if we wait for it. So we initialize it with a future and
- * use a lazy val to ensure that any attempt to use the compiler object waits
- * on the future.
- */
- private var _classLoader: AbstractFileClassLoader = null // active classloader
- private val _compiler: Global = newCompiler(settings, reporter) // our private compiler
-
- private trait ExposeAddUrl extends URLClassLoader { def addNewUrl(url: URL) = this.addURL(url) }
- private var _runtimeClassLoader: URLClassLoader with ExposeAddUrl = null // wrapper exposing addURL
-
- private val nextReqId = {
- var counter = 0
- () => { counter += 1 ; counter }
- }
-
- private def compilerClasspath: Seq[URL] = (
- if (isInitializeComplete) global.classPath.asURLs
- else new PathResolver(settings).result.asURLs // the compiler's classpath
- )
- // NOTE: Exposed to repl package since accessed indirectly from SparkIMain
- private[repl] def settings = currentSettings
- private def mostRecentLine = prevRequestList match {
- case Nil => ""
- case req :: _ => req.originalLine
- }
- // Run the code body with the given boolean settings flipped to true.
- private def withoutWarnings[T](body: => T): T = beQuietDuring {
- val saved = settings.nowarn.value
- if (!saved)
- settings.nowarn.value = true
-
- try body
- finally if (!saved) settings.nowarn.value = false
- }
-
- /** construct an interpreter that reports to Console */
- def this(settings: Settings) = this(settings, new NewLinePrintWriter(new ConsoleWriter, true))
- def this() = this(new Settings())
-
- private lazy val repllog: Logger = new Logger {
- val out: JPrintWriter = imain.out
- val isInfo: Boolean = BooleanProp keyExists "scala.repl.info"
- val isDebug: Boolean = BooleanProp keyExists "scala.repl.debug"
- val isTrace: Boolean = BooleanProp keyExists "scala.repl.trace"
- }
- private[repl] lazy val formatting: Formatting = new Formatting {
- val prompt = Properties.shellPromptString
- }
-
- // NOTE: Exposed to repl package since used by SparkExprTyper and SparkILoop
- private[repl] lazy val reporter: ConsoleReporter = new SparkIMain.ReplReporter(this)
-
- /**
- * Determines if errors were reported (typically during compilation).
- *
- * @note This is not for runtime errors
- *
- * @return True if had errors, otherwise false
- */
- @DeveloperApi
- def isReportingErrors = reporter.hasErrors
-
- import formatting._
- import reporter.{ printMessage, withoutTruncating }
-
- // This exists mostly because using the reporter too early leads to deadlock.
- private def echo(msg: String) { Console println msg }
- private def _initSources = List(new BatchSourceFile("<init>", "class $repl_$init { }"))
- private def _initialize() = {
- try {
- // todo. if this crashes, REPL will hang
- new _compiler.Run() compileSources _initSources
- _initializeComplete = true
- true
- }
- catch AbstractOrMissingHandler()
- }
- private def tquoted(s: String) = "\"\"\"" + s + "\"\"\""
-
- // argument is a thunk to execute after init is done
- // NOTE: Exposed to repl package since used by SparkILoop
- private[repl] def initialize(postInitSignal: => Unit) {
- synchronized {
- if (_isInitialized == null) {
- _isInitialized = io.spawn {
- try _initialize()
- finally postInitSignal
- }
- }
- }
- }
-
- /**
- * Initializes the underlying compiler/interpreter in a blocking fashion.
- *
- * @note Must be executed before using SparkIMain!
- */
- @DeveloperApi
- def initializeSynchronous(): Unit = {
- if (!isInitializeComplete) {
- _initialize()
- assert(global != null, global)
- }
- }
- private def isInitializeComplete = _initializeComplete
-
- /** the public, go through the future compiler */
-
- /**
- * The underlying compiler used to generate ASTs and execute code.
- */
- @DeveloperApi
- lazy val global: Global = {
- if (isInitializeComplete) _compiler
- else {
- // If init hasn't been called yet you're on your own.
- if (_isInitialized == null) {
- logWarning("Warning: compiler accessed before init set up. Assuming no postInit code.")
- initialize(())
- }
- // // blocks until it is ; false means catastrophic failure
- if (_isInitialized.get()) _compiler
- else null
- }
- }
- @deprecated("Use `global` for access to the compiler instance.", "2.9.0")
- private lazy val compiler: global.type = global
-
- import global._
- import definitions.{ScalaPackage, JavaLangPackage, termMember, typeMember}
- import rootMirror.{RootClass, getClassIfDefined, getModuleIfDefined, getRequiredModule, getRequiredClass}
-
- private implicit class ReplTypeOps(tp: Type) {
- def orElse(other: => Type): Type = if (tp ne NoType) tp else other
- def andAlso(fn: Type => Type): Type = if (tp eq NoType) tp else fn(tp)
- }
-
- // TODO: If we try to make naming a lazy val, we run into big time
- // scalac unhappiness with what look like cycles. It has not been easy to
- // reduce, but name resolution clearly takes different paths.
- // NOTE: Exposed to repl package since used by SparkExprTyper
- private[repl] object naming extends {
- val global: imain.global.type = imain.global
- } with Naming {
- // make sure we don't overwrite their unwisely named res3 etc.
- def freshUserTermName(): TermName = {
- val name = newTermName(freshUserVarName())
- if (definedNameMap contains name) freshUserTermName()
- else name
- }
- def isUserTermName(name: Name) = isUserVarName("" + name)
- def isInternalTermName(name: Name) = isInternalVarName("" + name)
- }
- import naming._
-
- // NOTE: Exposed to repl package since used by SparkILoop
- private[repl] object deconstruct extends {
- val global: imain.global.type = imain.global
- } with StructuredTypeStrings
-
- // NOTE: Exposed to repl package since used by SparkImports
- private[repl] lazy val memberHandlers = new {
- val intp: imain.type = imain
- } with SparkMemberHandlers
- import memberHandlers._
-
- /**
- * Suppresses overwriting print results during the operation.
- *
- * @param body The block to execute
- * @tparam T The return type of the block
- *
- * @return The result from executing the block
- */
- @DeveloperApi
- def beQuietDuring[T](body: => T): T = {
- val saved = printResults
- printResults = false
- try body
- finally printResults = saved
- }
-
- /**
- * Completely masks all output during the operation (minus JVM standard
- * out and error).
- *
- * @param operation The block to execute
- * @tparam T The return type of the block
- *
- * @return The result from executing the block
- */
- @DeveloperApi
- def beSilentDuring[T](operation: => T): T = {
- val saved = totalSilence
- totalSilence = true
- try operation
- finally totalSilence = saved
- }
-
- // NOTE: Exposed to repl package since used by SparkILoop
- private[repl] def quietRun[T](code: String) = beQuietDuring(interpret(code))
-
- private def logAndDiscard[T](label: String, alt: => T): PartialFunction[Throwable, T] = {
- case t: ControlThrowable => throw t
- case t: Throwable =>
- logDebug(label + ": " + unwrap(t))
- logDebug(stackTraceString(unwrap(t)))
- alt
- }
- /** takes AnyRef because it may be binding a Throwable or an Exceptional */
-
- private def withLastExceptionLock[T](body: => T, alt: => T): T = {
- assert(bindExceptions, "withLastExceptionLock called incorrectly.")
- bindExceptions = false
-
- try beQuietDuring(body)
- catch logAndDiscard("withLastExceptionLock", alt)
- finally bindExceptions = true
- }
-
- /**
- * Contains the code (in string form) representing a wrapper around all
- * code executed by this instance.
- *
- * @return The wrapper code as a string
- */
- @DeveloperApi
- def executionWrapper = _executionWrapper
-
- /**
- * Sets the code to use as a wrapper around all code executed by this
- * instance.
- *
- * @param code The wrapper code as a string
- */
- @DeveloperApi
- def setExecutionWrapper(code: String) = _executionWrapper = code
-
- /**
- * Clears the code used as a wrapper around all code executed by
- * this instance.
- */
- @DeveloperApi
- def clearExecutionWrapper() = _executionWrapper = ""
-
- /** interpreter settings */
- private lazy val isettings = new SparkISettings(this)
-
- /**
- * Instantiates a new compiler used by SparkIMain. Overridable to provide
- * own instance of a compiler.
- *
- * @param settings The settings to provide the compiler
- * @param reporter The reporter to use for compiler output
- *
- * @return The compiler as a Global
- */
- @DeveloperApi
- protected def newCompiler(settings: Settings, reporter: Reporter): ReplGlobal = {
- settings.outputDirs setSingleOutput virtualDirectory
- settings.exposeEmptyPackage.value = true
- new Global(settings, reporter) with ReplGlobal {
- override def toString: String = "<global>"
- }
- }
-
- /**
- * Adds any specified jars to the compile and runtime classpaths.
- *
- * @note Currently only supports jars, not directories
- * @param urls The list of items to add to the compile and runtime classpaths
- */
- @DeveloperApi
- def addUrlsToClassPath(urls: URL*): Unit = {
- new Run // Needed to force initialization of "something" to correctly load Scala classes from jars
- urls.foreach(_runtimeClassLoader.addNewUrl) // Add jars/classes to runtime for execution
- updateCompilerClassPath(urls: _*) // Add jars/classes to compile time for compiling
- }
-
- private def updateCompilerClassPath(urls: URL*): Unit = {
- require(!global.forMSIL) // Only support JavaPlatform
-
- val platform = global.platform.asInstanceOf[JavaPlatform]
-
- val newClassPath = mergeUrlsIntoClassPath(platform, urls: _*)
-
- // NOTE: Must use reflection until this is exposed/fixed upstream in Scala
- val fieldSetter = platform.getClass.getMethods
- .find(_.getName.endsWith("currentClassPath_$eq")).get
- fieldSetter.invoke(platform, Some(newClassPath))
-
- // Reload all jars specified into our compiler
- global.invalidateClassPathEntries(urls.map(_.getPath): _*)
- }
-
- private def mergeUrlsIntoClassPath(platform: JavaPlatform, urls: URL*): MergedClassPath[AbstractFile] = {
- // Collect our new jars/directories and add them to the existing set of classpaths
- val allClassPaths = (
- platform.classPath.asInstanceOf[MergedClassPath[AbstractFile]].entries ++
- urls.map(url => {
- platform.classPath.context.newClassPath(
- if (url.getProtocol == "file") {
- val f = new File(url.getPath)
- if (f.isDirectory)
- io.AbstractFile.getDirectory(f)
- else
- io.AbstractFile.getFile(f)
- } else {
- io.AbstractFile.getURL(url)
- }
- )
- })
- ).distinct
-
- // Combine all of our classpaths (old and new) into one merged classpath
- new MergedClassPath(allClassPaths, platform.classPath.context)
- }
-
- /**
- * Represents the parent classloader used by this instance. Can be
- * overridden to provide alternative classloader.
- *
- * @return The classloader used as the parent loader of this instance
- */
- @DeveloperApi
- protected def parentClassLoader: ClassLoader =
- SparkHelper.explicitParentLoader(settings).getOrElse( this.getClass.getClassLoader() )
-
- /* A single class loader is used for all commands interpreted by this Interpreter.
- It would also be possible to create a new class loader for each command
- to interpret. The advantages of the current approach are:
-
- - Expressions are only evaluated one time. This is especially
- significant for I/O, e.g. "val x = Console.readLine"
-
- The main disadvantage is:
-
- - Objects, classes, and methods cannot be rebound. Instead, definitions
- shadow the old ones, and old code objects refer to the old
- definitions.
- */
- private def resetClassLoader() = {
- logDebug("Setting new classloader: was " + _classLoader)
- _classLoader = null
- ensureClassLoader()
- }
- private final def ensureClassLoader() {
- if (_classLoader == null)
- _classLoader = makeClassLoader()
- }
-
- // NOTE: Exposed to repl package since used by SparkILoop
- private[repl] def classLoader: AbstractFileClassLoader = {
- ensureClassLoader()
- _classLoader
- }
- private class TranslatingClassLoader(parent: ClassLoader) extends AbstractFileClassLoader(virtualDirectory, parent) {
- /** Overridden here to try translating a simple name to the generated
- * class name if the original attempt fails. This method is used by
- * getResourceAsStream as well as findClass.
- */
- override protected def findAbstractFile(name: String): AbstractFile = {
- super.findAbstractFile(name) match {
- // deadlocks on startup if we try to translate names too early
- case null if isInitializeComplete =>
- generatedName(name) map (x => super.findAbstractFile(x)) orNull
- case file =>
- file
- }
- }
- }
- private def makeClassLoader(): AbstractFileClassLoader =
- new TranslatingClassLoader(parentClassLoader match {
- case null => ScalaClassLoader fromURLs compilerClasspath
- case p =>
- _runtimeClassLoader = new URLClassLoader(compilerClasspath, p) with ExposeAddUrl
- _runtimeClassLoader
- })
-
- private def getInterpreterClassLoader() = classLoader
-
- // Set the current Java "context" class loader to this interpreter's class loader
- // NOTE: Exposed to repl package since used by SparkILoopInit
- private[repl] def setContextClassLoader() = classLoader.setAsContext()
-
- /**
- * Returns the real name of a class based on its repl-defined name.
- *
- * ==Example==
- * Given a simple repl-defined name, returns the real name of
- * the class representing it, e.g. for "Bippy" it may return
- * {{{
- * $line19.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$Bippy
- * }}}
- *
- * @param simpleName The repl-defined name whose real name to retrieve
- *
- * @return Some real name if the simple name exists, else None
- */
- @DeveloperApi
- def generatedName(simpleName: String): Option[String] = {
- if (simpleName endsWith nme.MODULE_SUFFIX_STRING) optFlatName(simpleName.init) map (_ + nme.MODULE_SUFFIX_STRING)
- else optFlatName(simpleName)
- }
-
- // NOTE: Exposed to repl package since used by SparkILoop
- private[repl] def flatName(id: String) = optFlatName(id) getOrElse id
- // NOTE: Exposed to repl package since used by SparkILoop
- private[repl] def optFlatName(id: String) = requestForIdent(id) map (_ fullFlatName id)
-
- /**
- * Retrieves all simple names contained in the current instance.
- *
- * @return A list of sorted names
- */
- @DeveloperApi
- def allDefinedNames = definedNameMap.keys.toList.sorted
-
- private def pathToType(id: String): String = pathToName(newTypeName(id))
- // NOTE: Exposed to repl package since used by SparkILoop
- private[repl] def pathToTerm(id: String): String = pathToName(newTermName(id))
-
- /**
- * Retrieves the full code path to access the specified simple name
- * content.
- *
- * @param name The simple name of the target whose path to determine
- *
- * @return The full path used to access the specified target (name)
- */
- @DeveloperApi
- def pathToName(name: Name): String = {
- if (definedNameMap contains name)
- definedNameMap(name) fullPath name
- else name.toString
- }
-
- /** Most recent tree handled which wasn't wholly synthetic. */
- private def mostRecentlyHandledTree: Option[Tree] = {
- prevRequests.reverse foreach { req =>
- req.handlers.reverse foreach {
- case x: MemberDefHandler if x.definesValue && !isInternalTermName(x.name) => return Some(x.member)
- case _ => ()
- }
- }
- None
- }
-
- /** Stubs for work in progress. */
- private def handleTypeRedefinition(name: TypeName, old: Request, req: Request) = {
- for (t1 <- old.simpleNameOfType(name) ; t2 <- req.simpleNameOfType(name)) {
- logDebug("Redefining type '%s'\n %s -> %s".format(name, t1, t2))
- }
- }
-
- private def handleTermRedefinition(name: TermName, old: Request, req: Request) = {
- for (t1 <- old.compilerTypeOf get name ; t2 <- req.compilerTypeOf get name) {
- // Printing the types here has a tendency to cause assertion errors, like
- // assertion failed: fatal: <refinement> has owner value x, but a class owner is required
- // so DBG is by-name now to keep it in the family. (It also traps the assertion error,
- // but we don't want to unnecessarily risk hosing the compiler's internal state.)
- logDebug("Redefining term '%s'\n %s -> %s".format(name, t1, t2))
- }
- }
-
- private def recordRequest(req: Request) {
- if (req == null || referencedNameMap == null)
- return
-
- prevRequests += req
- req.referencedNames foreach (x => referencedNameMap(x) = req)
-
- // warning about serially defining companions. It'd be easy
- // enough to just redefine them together but that may not always
- // be what people want so I'm waiting until I can do it better.
- for {
- name <- req.definedNames filterNot (x => req.definedNames contains x.companionName)
- oldReq <- definedNameMap get name.companionName
- newSym <- req.definedSymbols get name
- oldSym <- oldReq.definedSymbols get name.companionName
- if Seq(oldSym, newSym).permutations exists { case Seq(s1, s2) => s1.isClass && s2.isModule }
- } {
- afterTyper(replwarn(s"warning: previously defined $oldSym is not a companion to $newSym."))
- replwarn("Companions must be defined together; you may wish to use :paste mode for this.")
- }
-
- // Updating the defined name map
- req.definedNames foreach { name =>
- if (definedNameMap contains name) {
- if (name.isTypeName) handleTypeRedefinition(name.toTypeName, definedNameMap(name), req)
- else handleTermRedefinition(name.toTermName, definedNameMap(name), req)
- }
- definedNameMap(name) = req
- }
- }
-
- private def replwarn(msg: => String) {
- if (!settings.nowarnings.value)
- printMessage(msg)
- }
-
- private def isParseable(line: String): Boolean = {
- beSilentDuring {
- try parse(line) match {
- case Some(xs) => xs.nonEmpty // parses as-is
- case None => true // incomplete
- }
- catch { case x: Exception => // crashed the compiler
- replwarn("Exception in isParseable(\"" + line + "\"): " + x)
- false
- }
- }
- }
-
- private def compileSourcesKeepingRun(sources: SourceFile*) = {
- val run = new Run()
- reporter.reset()
- run compileSources sources.toList
- (!reporter.hasErrors, run)
- }
-
- /**
- * Compiles specified source files.
- *
- * @param sources The sequence of source files to compile
- *
- * @return True if successful, otherwise false
- */
- @DeveloperApi
- def compileSources(sources: SourceFile*): Boolean =
- compileSourcesKeepingRun(sources: _*)._1
-
- /**
- * Compiles a string of code.
- *
- * @param code The string of code to compile
- *
- * @return True if successful, otherwise false
- */
- @DeveloperApi
- def compileString(code: String): Boolean =
- compileSources(new BatchSourceFile("<script>", code))
-
- /** Build a request from the user. `trees` is `line` after being parsed.
- */
- private def buildRequest(line: String, trees: List[Tree]): Request = {
- executingRequest = new Request(line, trees)
- executingRequest
- }
-
- // rewriting "5 // foo" to "val x = { 5 // foo }" creates broken code because
- // the close brace is commented out. Strip single-line comments.
- // ... but for error message output reasons this is not used, and rather than
- // enclosing in braces it is constructed like "val x =\n5 // foo".
- private def removeComments(line: String): String = {
- showCodeIfDebugging(line) // as we're about to lose our // show
- line.lines map (s => s indexOf "//" match {
- case -1 => s
- case idx => s take idx
- }) mkString "\n"
- }
-
- private def safePos(t: Tree, alt: Int): Int =
- try t.pos.startOrPoint
- catch { case _: UnsupportedOperationException => alt }
-
- // Given an expression like 10 * 10 * 10 we receive the parent tree positioned
- // at a '*'. So look at each subtree and find the earliest of all positions.
- private def earliestPosition(tree: Tree): Int = {
- var pos = Int.MaxValue
- tree foreach { t =>
- pos = math.min(pos, safePos(t, Int.MaxValue))
- }
- pos
- }
-
-
- private def requestFromLine(line: String, synthetic: Boolean): Either[IR.Result, Request] = {
- val content = indentCode(line)
- val trees = parse(content) match {
- case None => return Left(IR.Incomplete)
- case Some(Nil) => return Left(IR.Error) // parse error or empty input
- case Some(trees) => trees
- }
- logDebug(
- trees map (t => {
- // [Eugene to Paul] previously it just said `t map ...`
- // because there was an implicit conversion from Tree to a list of Trees
- // however Martin and I have removed the conversion
- // (it was conflicting with the new reflection API),
- // so I had to rewrite this a bit
- val subs = t collect { case sub => sub }
- subs map (t0 =>
- " " + safePos(t0, -1) + ": " + t0.shortClass + "\n"
- ) mkString ""
- }) mkString "\n"
- )
- // If the last tree is a bare expression, pinpoint where it begins using the
- // AST node position and snap the line off there. Rewrite the code embodied
- // by the last tree as a ValDef instead, so we can access the value.
- trees.last match {
- case _:Assign => // we don't want to include assignments
- case _:TermTree | _:Ident | _:Select => // ... but do want other unnamed terms.
- val varName = if (synthetic) freshInternalVarName() else freshUserVarName()
- val rewrittenLine = (
- // In theory this would come out the same without the 1-specific test, but
- // it's a cushion against any more sneaky parse-tree position vs. code mismatches:
- // this way such issues will only arise on multiple-statement repl input lines,
- // which most people don't use.
- if (trees.size == 1) "val " + varName + " =\n" + content
- else {
- // The position of the last tree
- val lastpos0 = earliestPosition(trees.last)
- // Oh boy, the parser throws away parens so "(2+2)" is mispositioned,
- // with increasingly hard to decipher positions as we move on to "() => 5",
- // (x: Int) => x + 1, and more. So I abandon attempts to finesse and just
- // look for semicolons and newlines, which I'm sure is also buggy.
- val (raw1, raw2) = content splitAt lastpos0
- logDebug("[raw] " + raw1 + " <---> " + raw2)
-
- val adjustment = (raw1.reverse takeWhile (ch => (ch != ';') && (ch != '\n'))).size
- val lastpos = lastpos0 - adjustment
-
- // the source code split at the laboriously determined position.
- val (l1, l2) = content splitAt lastpos
- logDebug("[adj] " + l1 + " <---> " + l2)
-
- val prefix = if (l1.trim == "") "" else l1 + ";\n"
- // Note to self: val source needs to have this precise structure so that
- // error messages print the user-submitted part without the "val res0 = " part.
- val combined = prefix + "val " + varName + " =\n" + l2
-
- logDebug(List(
- " line" -> line,
- " content" -> content,
- " was" -> l2,
- "combined" -> combined) map {
- case (label, s) => label + ": '" + s + "'"
- } mkString "\n"
- )
- combined
- }
- )
- // Rewriting "foo ; bar ; 123"
- // to "foo ; bar ; val resXX = 123"
- requestFromLine(rewrittenLine, synthetic) match {
- case Right(req) => return Right(req withOriginalLine line)
- case x => return x
- }
- case _ =>
- }
- Right(buildRequest(line, trees))
- }
-
- // normalize non-public types so we don't see protected aliases like Self
- private def normalizeNonPublic(tp: Type) = tp match {
- case TypeRef(_, sym, _) if sym.isAliasType && !sym.isPublic => tp.dealias
- case _ => tp
- }
-
- /**
- * Interpret one line of input. All feedback, including parse errors
- * and evaluation results, are printed via the supplied compiler's
- * reporter. Values defined are available for future interpreted strings.
- *
- * @note This assigns variables with user name structure like "res0"
- *
- * @param line The line representing the code to interpret
- *
- * @return Whether the line was interpreted successfully, or failed due to
- * incomplete code, compilation error, or runtime error
- */
- @DeveloperApi
- def interpret(line: String): IR.Result = interpret(line, false)
-
- /**
- * Interpret one line of input. All feedback, including parse errors
- * and evaluation results, are printed via the supplied compiler's
- * reporter. Values defined are available for future interpreted strings.
- *
- * @note This assigns variables with synthetic (generated) name structure
- * like "$ires0"
- *
- * @param line The line representing the code to interpret
- *
- * @return Whether the line was interpreted successfully, or failed due to
- * incomplete code, compilation error, or runtime error
- */
- @DeveloperApi
- def interpretSynthetic(line: String): IR.Result = interpret(line, true)
-
- private def interpret(line: String, synthetic: Boolean): IR.Result = {
- def loadAndRunReq(req: Request) = {
- classLoader.setAsContext()
- val (result, succeeded) = req.loadAndRun
-
- /** To our displeasure, ConsoleReporter offers only printMessage,
- * which tacks a newline on the end. Since that breaks all the
- * output checking, we have to take one off to balance.
- */
- if (succeeded) {
- if (printResults && result != "")
- printMessage(result stripSuffix "\n")
- else if (isReplDebug) // show quiet-mode activity
- printMessage(result.trim.lines map ("[quiet] " + _) mkString "\n")
-
- // Book-keeping. Have to record synthetic requests too,
- // as they may have been issued for information, e.g. :type
- recordRequest(req)
- IR.Success
- }
- else {
- // don't truncate stack traces
- withoutTruncating(printMessage(result))
- IR.Error
- }
- }
-
- if (global == null) IR.Error
- else requestFromLine(line, synthetic) match {
- case Left(result) => result
- case Right(req) =>
- // null indicates a disallowed statement type; otherwise compile and
- // fail if false (implying e.g. a type error)
- if (req == null || !req.compile) IR.Error
- else loadAndRunReq(req)
- }
- }
-
- /**
- * Bind a specified name to a specified value. The name may
- * later be used by expressions passed to interpret.
- *
- * @note This binds via compilation and interpretation
- *
- * @param name The variable name to bind
- * @param boundType The type of the variable, as a string
- * @param value The object value to bind to it
- *
- * @return An indication of whether the binding succeeded or failed
- * using interpreter results
- */
- @DeveloperApi
- def bind(name: String, boundType: String, value: Any, modifiers: List[String] = Nil): IR.Result = {
- val bindRep = new ReadEvalPrint()
- val run = bindRep.compile("""
- |object %s {
- | var value: %s = _
- | def set(x: Any) = value = x.asInstanceOf[%s]
- |}
- """.stripMargin.format(bindRep.evalName, boundType, boundType)
- )
- bindRep.callEither("set", value) match {
- case Left(ex) =>
- logDebug("Set failed in bind(%s, %s, %s)".format(name, boundType, value))
- logDebug(util.stackTraceString(ex))
- IR.Error
-
- case Right(_) =>
- val line = "%sval %s = %s.value".format(modifiers map (_ + " ") mkString, name, bindRep.evalPath)
- logDebug("Interpreting: " + line)
- interpret(line)
- }
- }
-
- /**
- * Bind a specified name to a specified value directly.
- *
- * @note This updates internal bound names directly
- *
- * @param name The variable name to bind
- * @param boundType The type of the variable, as a string
- * @param value The object value to bind to it
- *
- * @return An indication of whether the binding succeeded or failed
- * using interpreter results
- */
- @DeveloperApi
- def directBind(name: String, boundType: String, value: Any): IR.Result = {
- val result = bind(name, boundType, value)
- if (result == IR.Success)
- directlyBoundNames += newTermName(name)
- result
- }
-
- private def directBind(p: NamedParam): IR.Result = directBind(p.name, p.tpe, p.value)
- private def directBind[T: ru.TypeTag : ClassTag](name: String, value: T): IR.Result = directBind((name, value))
-
- /**
- * Overwrites previously-bound val with a new instance.
- *
- * @param p The named parameters used to provide the name, value, and type
- *
- * @return The results of rebinding the named val
- */
- @DeveloperApi
- def rebind(p: NamedParam): IR.Result = {
- val name = p.name
- val oldType = typeOfTerm(name) orElse { return IR.Error }
- val newType = p.tpe
- val tempName = freshInternalVarName()
-
- quietRun("val %s = %s".format(tempName, name))
- quietRun("val %s = %s.asInstanceOf[%s]".format(name, tempName, newType))
- }
- private def quietImport(ids: String*): IR.Result = beQuietDuring(addImports(ids: _*))
-
- /**
- * Executes an import statement per "id" provided
- *
- * @example addImports("org.apache.spark.SparkContext")
- *
- * @param ids The series of "id" strings used for import statements
- *
- * @return The results of importing the series of "id" strings
- */
- @DeveloperApi
- def addImports(ids: String*): IR.Result =
- if (ids.isEmpty) IR.Success
- else interpret("import " + ids.mkString(", "))
-
- // NOTE: Exposed to repl package since used by SparkILoop
- private[repl] def quietBind(p: NamedParam): IR.Result = beQuietDuring(bind(p))
- private def bind(p: NamedParam): IR.Result = bind(p.name, p.tpe, p.value)
- private def bind[T: ru.TypeTag : ClassTag](name: String, value: T): IR.Result = bind((name, value))
- private def bindSyntheticValue(x: Any): IR.Result = bindValue(freshInternalVarName(), x)
- private def bindValue(x: Any): IR.Result = bindValue(freshUserVarName(), x)
- private def bindValue(name: String, x: Any): IR.Result = bind(name, TypeStrings.fromValue(x), x)
-
- /**
- * Reset this interpreter, forgetting all user-specified requests.
- */
- @DeveloperApi
- def reset() {
- clearExecutionWrapper()
- resetClassLoader()
- resetAllCreators()
- prevRequests.clear()
- referencedNameMap.clear()
- definedNameMap.clear()
- virtualDirectory.delete()
- virtualDirectory.create()
- }
-
- /**
- * Stops the underlying REPL class server and flushes the reporter used
- * for compiler output.
- */
- @DeveloperApi
- def close() {
- reporter.flush()
- }
-
- /**
- * Captures the session names (which are set by system properties) once, instead of for each line.
- */
- @DeveloperApi
- object FixedSessionNames {
- val lineName = sessionNames.line
- val readName = sessionNames.read
- val evalName = sessionNames.eval
- val printName = sessionNames.print
- val resultName = sessionNames.result
- }
-
- /** Here is where we:
- *
- * 1) Read some source code, and put it in the "read" object.
- * 2) Evaluate the read object, and put the result in the "eval" object.
- * 3) Create a String for human consumption, and put it in the "print" object.
- *
- * Read! Eval! Print! Some of that not yet centralized here.
- */
- class ReadEvalPrint(val lineId: Int) {
- def this() = this(freshLineId())
-
- private var lastRun: Run = _
- private var evalCaught: Option[Throwable] = None
- private var conditionalWarnings: List[ConditionalWarning] = Nil
-
- val packageName = FixedSessionNames.lineName + lineId
- val readName = FixedSessionNames.readName
- val evalName = FixedSessionNames.evalName
- val printName = FixedSessionNames.printName
- val resultName = FixedSessionNames.resultName
-
- def bindError(t: Throwable) = {
- // Immediately throw the exception if we are asked to propagate them
- if (propagateExceptions) {
- throw unwrap(t)
- }
- if (!bindExceptions) // avoid looping if already binding
- throw t
-
- val unwrapped = unwrap(t)
- withLastExceptionLock[String]({
- directBind[Throwable]("lastException", unwrapped)(tagOfThrowable, classTag[Throwable])
- util.stackTraceString(unwrapped)
- }, util.stackTraceString(unwrapped))
- }
-
- // TODO: split it out into a package object and a regular
- // object and we can do that much less wrapping.
- def packageDecl = "package " + packageName
-
- def pathTo(name: String) = packageName + "." + name
- def packaged(code: String) = packageDecl + "\n\n" + code
-
- def readPath = pathTo(readName)
- def evalPath = pathTo(evalName)
- def printPath = pathTo(printName)
-
- def call(name: String, args: Any*): AnyRef = {
- val m = evalMethod(name)
- logDebug("Invoking: " + m)
- if (args.nonEmpty)
- logDebug(" with args: " + args.mkString(", "))
-
- m.invoke(evalClass, args.map(_.asInstanceOf[AnyRef]): _*)
- }
-
- def callEither(name: String, args: Any*): Either[Throwable, AnyRef] =
- try Right(call(name, args: _*))
- catch { case ex: Throwable => Left(ex) }
-
- def callOpt(name: String, args: Any*): Option[AnyRef] =
- try Some(call(name, args: _*))
- catch { case ex: Throwable => bindError(ex) ; None }
-
- class EvalException(msg: String, cause: Throwable) extends RuntimeException(msg, cause) { }
-
- private def evalError(path: String, ex: Throwable) =
- throw new EvalException("Failed to load '" + path + "': " + ex.getMessage, ex)
-
- private def load(path: String): Class[_] = {
- // scalastyle:off classforname
- try Class.forName(path, true, classLoader)
- catch { case ex: Throwable => evalError(path, unwrap(ex)) }
- // scalastyle:on classforname
- }
-
- lazy val evalClass = load(evalPath)
- lazy val evalValue = callEither(resultName) match {
- case Left(ex) => evalCaught = Some(ex) ; None
- case Right(result) => Some(result)
- }
-
- def compile(source: String): Boolean = compileAndSaveRun("<console>", source)
-
- /** The innermost object inside the wrapper, found by
- * following accessPath into the outer one.
- */
- def resolvePathToSymbol(accessPath: String): Symbol = {
- // val readRoot = getRequiredModule(readPath) // the outermost wrapper
- // MATEI: Changed this to getClass because the root object is no longer a module (Scala singleton object)
-
- val readRoot = rootMirror.getClassByName(newTypeName(readPath)) // the outermost wrapper
- (accessPath split '.').foldLeft(readRoot: Symbol) {
- case (sym, "") => sym
- case (sym, name) => afterTyper(termMember(sym, name))
- }
- }
- /** We get a bunch of repeated warnings for reasons I haven't
- * entirely figured out yet. For now, squash.
- */
- private def updateRecentWarnings(run: Run) {
- def loop(xs: List[(Position, String)]): List[(Position, String)] = xs match {
- case Nil => Nil
- case ((pos, msg)) :: rest =>
- val filtered = rest filter { case (pos0, msg0) =>
- (msg != msg0) || (pos.lineContent.trim != pos0.lineContent.trim) || {
- // same messages and same line content after whitespace removal
- // but we want to let through multiple warnings on the same line
- // from the same run. The untrimmed line will be the same since
- // there's no whitespace indenting blowing it.
- (pos.lineContent == pos0.lineContent)
- }
- }
- ((pos, msg)) :: loop(filtered)
- }
- // PRASHANT: This leads to a NoSuchMethodError for _.warnings. Yet to figure out its purpose.
- // val warnings = loop(run.allConditionalWarnings flatMap (_.warnings))
- // if (warnings.nonEmpty)
- // mostRecentWarnings = warnings
- }
- private def evalMethod(name: String) = evalClass.getMethods filter (_.getName == name) match {
- case Array(method) => method
- case xs => sys.error("Internal error: eval object " + evalClass + ", " + xs.mkString("\n", "\n", ""))
- }
- private def compileAndSaveRun(label: String, code: String) = {
- showCodeIfDebugging(code)
- val (success, run) = compileSourcesKeepingRun(new BatchSourceFile(label, packaged(code)))
- updateRecentWarnings(run)
- lastRun = run
- success
- }
- }
-
- /** One line of code submitted by the user for interpretation */
- // private
- class Request(val line: String, val trees: List[Tree]) {
- val reqId = nextReqId()
- val lineRep = new ReadEvalPrint()
-
- private var _originalLine: String = null
- def withOriginalLine(s: String): this.type = { _originalLine = s ; this }
- def originalLine = if (_originalLine == null) line else _originalLine
-
- /** handlers for each tree in this request */
- val handlers: List[MemberHandler] = trees map (memberHandlers chooseHandler _)
- def defHandlers = handlers collect { case x: MemberDefHandler => x }
-
- /** all (public) names defined by these statements */
- val definedNames = handlers flatMap (_.definedNames)
-
- /** list of names used by this expression */
- val referencedNames: List[Name] = handlers flatMap (_.referencedNames)
-
- /** def and val names */
- def termNames = handlers flatMap (_.definesTerm)
- def typeNames = handlers flatMap (_.definesType)
- def definedOrImported = handlers flatMap (_.definedOrImported)
- def definedSymbolList = defHandlers flatMap (_.definedSymbols)
-
- def definedTypeSymbol(name: String) = definedSymbols(newTypeName(name))
- def definedTermSymbol(name: String) = definedSymbols(newTermName(name))
-
- val definedClasses = handlers.exists {
- case _: ClassHandler => true
- case _ => false
- }
-
- /** Code to import bound names from previous lines - accessPath is code to
- * append to objectName to access anything bound by request.
- */
- val SparkComputedImports(importsPreamble, importsTrailer, accessPath) =
- importsCode(referencedNames.toSet, definedClasses)
-
- /** Code to access a variable with the specified name */
- def fullPath(vname: String) = {
- // lineRep.readPath + accessPath + ".`%s`".format(vname)
- lineRep.readPath + ".INSTANCE" + accessPath + ".`%s`".format(vname)
- }
- /** Same as fullpath, but after it has been flattened, so:
- * $line5.$iw.$iw.$iw.Bippy // fullPath
- * $line5.$iw$$iw$$iw$Bippy // fullFlatName
- */
- def fullFlatName(name: String) =
- // lineRep.readPath + accessPath.replace('.', '$') + nme.NAME_JOIN_STRING + name
- lineRep.readPath + ".INSTANCE" + accessPath.replace('.', '$') + nme.NAME_JOIN_STRING + name
-
- /** The unmangled symbol name, but supplemented with line info. */
- def disambiguated(name: Name): String = name + " (in " + lineRep + ")"
-
- /** Code to access a variable with the specified name */
- def fullPath(vname: Name): String = fullPath(vname.toString)
-
- /** the line of code to compute */
- def toCompute = line
-
- /** generate the source code for the object that computes this request */
- private object ObjectSourceCode extends CodeAssembler[MemberHandler] {
- def path = pathToTerm("$intp")
- def envLines = {
- if (!isReplPower) Nil // power mode only for now
- // $intp is not bound; punt, but include the line.
- else if (path == "$intp") List(
- "def $line = " + tquoted(originalLine),
- "def $trees = Nil"
- )
- else List(
- "def $line = " + tquoted(originalLine),
- "def $req = %s.requestForReqId(%s).orNull".format(path, reqId),
- "def $trees = if ($req eq null) Nil else $req.trees".format(lineRep.readName, path, reqId)
- )
- }
-
- val preamble = s"""
- |class ${lineRep.readName} extends Serializable {
- | ${envLines.map(" " + _ + ";\n").mkString}
- | $importsPreamble
- |
- | // If we need to construct any objects defined in the REPL on an executor we will need
- | // to pass the outer scope to the appropriate encoder.
- | org.apache.spark.sql.catalyst.encoders.OuterScopes.addOuterScope(this)
- | ${indentCode(toCompute)}
- """.stripMargin
- val postamble = importsTrailer + "\n}" + "\n" +
- "object " + lineRep.readName + " {\n" +
- " val INSTANCE = new " + lineRep.readName + "();\n" +
- "}\n"
- val generate = (m: MemberHandler) => m extraCodeToEvaluate Request.this
-
- /*
- val preamble = """
- |object %s extends Serializable {
- |%s%s%s
- """.stripMargin.format(lineRep.readName, envLines.map(" " + _ + ";\n").mkString, importsPreamble, indentCode(toCompute))
- val postamble = importsTrailer + "\n}"
- val generate = (m: MemberHandler) => m extraCodeToEvaluate Request.this
- */
-
- }
-
- private object ResultObjectSourceCode extends CodeAssembler[MemberHandler] {
- /** We only want to generate this code when the result
- * is a value which can be referred to as-is.
- */
- val evalResult =
- if (!handlers.last.definesValue) ""
- else handlers.last.definesTerm match {
- case Some(vname) if typeOf contains vname =>
- "lazy val %s = %s".format(lineRep.resultName, fullPath(vname))
- case _ => ""
- }
-
- // first line evaluates object to make sure constructor is run
- // initial "" so later code can uniformly be: + etc
- val preamble = """
- |object %s {
- | %s
- | val %s: String = %s {
- | %s
- | (""
- """.stripMargin.format(
- lineRep.evalName, evalResult, lineRep.printName,
- executionWrapper, lineRep.readName + ".INSTANCE" + accessPath
- )
- val postamble = """
- | )
- | }
- |}
- """.stripMargin
- val generate = (m: MemberHandler) => m resultExtractionCode Request.this
- }
-
- // get it
- def getEvalTyped[T] : Option[T] = getEval map (_.asInstanceOf[T])
- def getEval: Option[AnyRef] = {
- // ensure it has been compiled
- compile
- // try to load it and call the value method
- lineRep.evalValue filterNot (_ == null)
- }
-
- /** Compile the object file. Returns whether the compilation succeeded.
- * If all goes well, the "types" map is computed. */
- lazy val compile: Boolean = {
- // error counting is wrong, hence interpreter may overlook failure - so we reset
- reporter.reset()
-
- // compile the object containing the user's code
- lineRep.compile(ObjectSourceCode(handlers)) && {
- // extract and remember types
- typeOf
- typesOfDefinedTerms
-
- // Assign symbols to the original trees
- // TODO - just use the new trees.
- defHandlers foreach { dh =>
- val name = dh.member.name
- definedSymbols get name foreach { sym =>
- dh.member setSymbol sym
- logDebug("Set symbol of " + name + " to " + sym.defString)
- }
- }
-
- // compile the result-extraction object
- withoutWarnings(lineRep compile ResultObjectSourceCode(handlers))
- }
- }
-
- lazy val resultSymbol = lineRep.resolvePathToSymbol(accessPath)
- def applyToResultMember[T](name: Name, f: Symbol => T) = afterTyper(f(resultSymbol.info.nonPrivateDecl(name)))
-
- /* typeOf lookup with encoding */
- def lookupTypeOf(name: Name) = typeOf.getOrElse(name, typeOf(global.encode(name.toString)))
- def simpleNameOfType(name: TypeName) = (compilerTypeOf get name) map (_.typeSymbol.simpleName)
-
- private def typeMap[T](f: Type => T) =
- mapFrom[Name, Name, T](termNames ++ typeNames)(x => f(cleanMemberDecl(resultSymbol, x)))
-
- /** Types of variables defined by this request. */
- lazy val compilerTypeOf = typeMap[Type](x => x) withDefaultValue NoType
- /** String representations of same. */
- lazy val typeOf = typeMap[String](tp => afterTyper(tp.toString))
-
- // lazy val definedTypes: Map[Name, Type] = {
- // typeNames map (x => x -> afterTyper(resultSymbol.info.nonPrivateDecl(x).tpe)) toMap
- // }
- lazy val definedSymbols = (
- termNames.map(x => x -> applyToResultMember(x, x => x)) ++
- typeNames.map(x => x -> compilerTypeOf(x).typeSymbolDirect)
- ).toMap[Name, Symbol] withDefaultValue NoSymbol
-
- lazy val typesOfDefinedTerms = mapFrom[Name, Name, Type](termNames)(x => applyToResultMember(x, _.tpe))
-
- /** load and run the code using reflection */
- def loadAndRun: (String, Boolean) = {
- try { ("" + (lineRep call sessionNames.print), true) }
- catch { case ex: Throwable => (lineRep.bindError(ex), false) }
- }
-
- override def toString = "Request(line=%s, %s trees)".format(line, trees.size)
- }
-
- /**
- * Returns the name of the most recent interpreter result. Useful for
- * for extracting information regarding the previous result.
- *
- * @return The simple name of the result (such as res0)
- */
- @DeveloperApi
- def mostRecentVar: String =
- if (mostRecentlyHandledTree.isEmpty) ""
- else "" + (mostRecentlyHandledTree.get match {
- case x: ValOrDefDef => x.name
- case Assign(Ident(name), _) => name
- case ModuleDef(_, name, _) => name
- case _ => naming.mostRecentVar
- })
-
- private var mostRecentWarnings: List[(global.Position, String)] = Nil
-
- /**
- * Returns a list of recent warnings from compiler execution.
- *
- * @return The list of tuples (compiler position, warning)
- */
- @DeveloperApi
- def lastWarnings = mostRecentWarnings
-
- def treesForRequestId(id: Int): List[Tree] =
- requestForReqId(id).toList flatMap (_.trees)
-
- def requestForReqId(id: Int): Option[Request] =
- if (executingRequest != null && executingRequest.reqId == id) Some(executingRequest)
- else prevRequests find (_.reqId == id)
-
- def requestForName(name: Name): Option[Request] = {
- assert(definedNameMap != null, "definedNameMap is null")
- definedNameMap get name
- }
-
- def requestForIdent(line: String): Option[Request] =
- requestForName(newTermName(line)) orElse requestForName(newTypeName(line))
-
- def requestHistoryForName(name: Name): List[Request] =
- prevRequests.toList.reverse filter (_.definedNames contains name)
-
-
- def definitionForName(name: Name): Option[MemberHandler] =
- requestForName(name) flatMap { req =>
- req.handlers find (_.definedNames contains name)
- }
-
- /**
- * Retrieves the object representing the id (variable name, method name,
- * class name, etc) provided.
- *
- * @param id The id (variable name, method name, class name, etc) whose
- * associated content to retrieve
- *
- * @return Some containing term name (id) representation if exists, else None
- */
- @DeveloperApi
- def valueOfTerm(id: String): Option[AnyRef] =
- requestForName(newTermName(id)) flatMap (_.getEval)
-
- /**
- * Retrieves the class representing the id (variable name, method name,
- * class name, etc) provided.
- *
- * @param id The id (variable name, method name, class name, etc) whose
- * associated class to retrieve
- *
- * @return Some containing term name (id) class if exists, else None
- */
- @DeveloperApi
- def classOfTerm(id: String): Option[JClass] =
- valueOfTerm(id) map (_.getClass)
-
- /**
- * Retrieves the type representing the id (variable name, method name,
- * class name, etc) provided.
- *
- * @param id The id (variable name, method name, class name, etc) whose
- * associated type to retrieve
- *
- * @return The Type information about the term name (id) provided
- */
- @DeveloperApi
- def typeOfTerm(id: String): Type = newTermName(id) match {
- case nme.ROOTPKG => RootClass.tpe
- case name => requestForName(name).fold(NoType: Type)(_ compilerTypeOf name)
- }
-
- /**
- * Retrieves the symbol representing the id (variable name, method name,
- * class name, etc) provided.
- *
- * @param id The id (variable name, method name, class name, etc) whose
- * associated symbol to retrieve
- *
- * @return The Symbol information about the term name (id) provided
- */
- @DeveloperApi
- def symbolOfTerm(id: String): Symbol =
- requestForIdent(newTermName(id)).fold(NoSymbol: Symbol)(_ definedTermSymbol id)
-
- // TODO: No use yet, but could be exposed as a DeveloperApi
- private def symbolOfType(id: String): Symbol =
- requestForName(newTypeName(id)).fold(NoSymbol: Symbol)(_ definedTypeSymbol id)
-
- /**
- * Retrieves the runtime class and type representing the id (variable name,
- * method name, class name, etc) provided.
- *
- * @param id The id (variable name, method name, class name, etc) whose
- * associated runtime class and type to retrieve
- *
- * @return Some runtime class and Type information as a tuple for the
- * provided term name if it exists, else None
- */
- @DeveloperApi
- def runtimeClassAndTypeOfTerm(id: String): Option[(JClass, Type)] = {
- classOfTerm(id) flatMap { clazz =>
- new RichClass(clazz).supers find(c => !(new RichClass(c).isScalaAnonymous)) map { nonAnon =>
- (nonAnon, runtimeTypeOfTerm(id))
- }
- }
- }
-
- /**
- * Retrieves the runtime type representing the id (variable name,
- * method name, class name, etc) provided.
- *
- * @param id The id (variable name, method name, class name, etc) whose
- * associated runtime type to retrieve
- *
- * @return The runtime Type information about the term name (id) provided
- */
- @DeveloperApi
- def runtimeTypeOfTerm(id: String): Type = {
- typeOfTerm(id) andAlso { tpe =>
- val clazz = classOfTerm(id) getOrElse { return NoType }
- val staticSym = tpe.typeSymbol
- val runtimeSym = getClassIfDefined(clazz.getName)
-
- if ((runtimeSym != NoSymbol) && (runtimeSym != staticSym) && (runtimeSym isSubClass staticSym))
- runtimeSym.info
- else NoType
- }
- }
-
- private def cleanMemberDecl(owner: Symbol, member: Name): Type = afterTyper {
- normalizeNonPublic {
- owner.info.nonPrivateDecl(member).tpe match {
- case NullaryMethodType(tp) => tp
- case tp => tp
- }
- }
- }
-
- private object exprTyper extends {
- val repl: SparkIMain.this.type = imain
- } with SparkExprTyper { }
-
- /**
- * Constructs a list of abstract syntax trees representing the provided code.
- *
- * @param line The line of code to parse and construct into ASTs
- *
- * @return Some list of ASTs if the line is valid, else None
- */
- @DeveloperApi
- def parse(line: String): Option[List[Tree]] = exprTyper.parse(line)
-
- /**
- * Constructs a Symbol representing the final result of the expression
- * provided or representing the definition provided.
- *
- * @param code The line of code
- *
- * @return The Symbol or NoSymbol (found under scala.reflect.internal)
- */
- @DeveloperApi
- def symbolOfLine(code: String): Symbol =
- exprTyper.symbolOfLine(code)
-
- /**
- * Constructs type information based on the provided expression's final
- * result or the definition provided.
- *
- * @param expr The expression or definition
- *
- * @param silent Whether to output information while constructing the type
- *
- * @return The type information or an error
- */
- @DeveloperApi
- def typeOfExpression(expr: String, silent: Boolean = true): Type =
- exprTyper.typeOfExpression(expr, silent)
-
- protected def onlyTerms(xs: List[Name]) = xs collect { case x: TermName => x }
- protected def onlyTypes(xs: List[Name]) = xs collect { case x: TypeName => x }
-
- /**
- * Retrieves the defined, public names in the compiler.
- *
- * @return The list of matching "term" names
- */
- @DeveloperApi
- def definedTerms = onlyTerms(allDefinedNames) filterNot isInternalTermName
-
- /**
- * Retrieves the defined type names in the compiler.
- *
- * @return The list of matching type names
- */
- @DeveloperApi
- def definedTypes = onlyTypes(allDefinedNames)
-
- /**
- * Retrieves the defined symbols in the compiler.
- *
- * @return The set of matching Symbol instances
- */
- @DeveloperApi
- def definedSymbols = prevRequestList.flatMap(_.definedSymbols.values).toSet[Symbol]
-
- /**
- * Retrieves the list of public symbols in the compiler.
- *
- * @return The list of public Symbol instances
- */
- @DeveloperApi
- def definedSymbolList = prevRequestList flatMap (_.definedSymbolList) filterNot (s => isInternalTermName(s.name))
-
- // Terms with user-given names (i.e. not res0 and not synthetic)
-
- /**
- * Retrieves defined, public names that are not res0 or the result of a direct bind.
- *
- * @return The list of matching "term" names
- */
- @DeveloperApi
- def namedDefinedTerms = definedTerms filterNot (x => isUserVarName("" + x) || directlyBoundNames(x))
-
- private def findName(name: Name) = definedSymbols find (_.name == name) getOrElse NoSymbol
-
- /** Translate a repl-defined identifier into a Symbol.
- */
- private def apply(name: String): Symbol =
- types(name) orElse terms(name)
-
- private def types(name: String): Symbol = {
- val tpname = newTypeName(name)
- findName(tpname) orElse getClassIfDefined(tpname)
- }
- private def terms(name: String): Symbol = {
- val termname = newTypeName(name)
- findName(termname) orElse getModuleIfDefined(termname)
- }
- // [Eugene to Paul] possibly you could make use of TypeTags here
- private def types[T: ClassTag] : Symbol = types(classTag[T].runtimeClass.getName)
- private def terms[T: ClassTag] : Symbol = terms(classTag[T].runtimeClass.getName)
- private def apply[T: ClassTag] : Symbol = apply(classTag[T].runtimeClass.getName)
-
- /**
- * Retrieves the Symbols representing classes in the compiler.
- *
- * @return The list of matching ClassSymbol instances
- */
- @DeveloperApi
- def classSymbols = allDefSymbols collect { case x: ClassSymbol => x }
-
- /**
- * Retrieves the Symbols representing methods in the compiler.
- *
- * @return The list of matching MethodSymbol instances
- */
- @DeveloperApi
- def methodSymbols = allDefSymbols collect { case x: MethodSymbol => x }
-
- /** the previous requests this interpreter has processed */
- private var executingRequest: Request = _
- private val prevRequests = mutable.ListBuffer[Request]()
- private val referencedNameMap = mutable.Map[Name, Request]()
- private val definedNameMap = mutable.Map[Name, Request]()
- private val directlyBoundNames = mutable.Set[Name]()
-
- private def allHandlers = prevRequestList flatMap (_.handlers)
- private def allDefHandlers = allHandlers collect { case x: MemberDefHandler => x }
- private def allDefSymbols = allDefHandlers map (_.symbol) filter (_ ne NoSymbol)
-
- private def lastRequest = if (prevRequests.isEmpty) null else prevRequests.last
- // NOTE: Exposed to repl package since used by SparkImports
- private[repl] def prevRequestList = prevRequests.toList
- private def allSeenTypes = prevRequestList flatMap (_.typeOf.values.toList) distinct
- private def allImplicits = allHandlers filter (_.definesImplicit) flatMap (_.definedNames)
- // NOTE: Exposed to repl package since used by SparkILoop and SparkImports
- private[repl] def importHandlers = allHandlers collect { case x: ImportHandler => x }
-
- /**
- * Retrieves a list of unique defined and imported names in the compiler.
- *
- * @return The list of "term" names
- */
- def visibleTermNames: List[Name] = definedTerms ++ importedTerms distinct
-
- /** Another entry point for tab-completion, ids in scope */
- // NOTE: Exposed to repl package since used by SparkJLineCompletion
- private[repl] def unqualifiedIds = visibleTermNames map (_.toString) filterNot (_ contains "$") sorted
-
- /** Parse the ScalaSig to find type aliases */
- private def aliasForType(path: String) = ByteCode.aliasForType(path)
-
- private def withoutUnwrapping(op: => Unit): Unit = {
- val saved = isettings.unwrapStrings
- isettings.unwrapStrings = false
- try op
- finally isettings.unwrapStrings = saved
- }
-
- // NOTE: Exposed to repl package since used by SparkILoop
- private[repl] def symbolDefString(sym: Symbol) = {
- TypeStrings.quieter(
- afterTyper(sym.defString),
- sym.owner.name + ".this.",
- sym.owner.fullName + "."
- )
- }
-
- private def showCodeIfDebugging(code: String) {
- /** Secret bookcase entrance for repl debuggers: end the line
- * with "// show" and see what's going on.
- */
- def isShow = code.lines exists (_.trim endsWith "// show")
- def isShowRaw = code.lines exists (_.trim endsWith "// raw")
-
- // old style
- beSilentDuring(parse(code)) foreach { ts =>
- ts foreach { t =>
- if (isShow || isShowRaw)
- withoutUnwrapping(echo(asCompactString(t)))
- else
- withoutUnwrapping(logDebug(asCompactString(t)))
- }
- }
- }
-
- // debugging
- // NOTE: Exposed to repl package since accessed indirectly from SparkIMain
- // and SparkJLineCompletion
- private[repl] def debugging[T](msg: String)(res: T) = {
- logDebug(msg + " " + res)
- res
- }
-}
-
-/** Utility methods for the Interpreter. */
-object SparkIMain {
- // The two name forms this is catching are the two sides of this assignment:
- //
- // $line3.$read.$iw.$iw.Bippy =
- // $line3.$read$$iw$$iw$Bippy@4a6a00ca
- private def removeLineWrapper(s: String) = s.replaceAll("""\$line\d+[./]\$(read|eval|print)[$.]""", "")
- private def removeIWPackages(s: String) = s.replaceAll("""\$(iw|iwC|read|eval|print)[$.]""", "")
- private def removeSparkVals(s: String) = s.replaceAll("""\$VAL[0-9]+[$.]""", "")
-
- def stripString(s: String) = removeSparkVals(removeIWPackages(removeLineWrapper(s)))
-
- trait CodeAssembler[T] {
- def preamble: String
- def generate: T => String
- def postamble: String
-
- def apply(contributors: List[T]): String = stringFromWriter { code =>
- code println preamble
- contributors map generate foreach (code println _)
- code println postamble
- }
- }
-
- trait StrippingWriter {
- def isStripping: Boolean
- def stripImpl(str: String): String
- def strip(str: String): String = if (isStripping) stripImpl(str) else str
- }
- trait TruncatingWriter {
- def maxStringLength: Int
- def isTruncating: Boolean
- def truncate(str: String): String = {
- if (isTruncating && (maxStringLength != 0 && str.length > maxStringLength))
- (str take maxStringLength - 3) + "..."
- else str
- }
- }
- abstract class StrippingTruncatingWriter(out: JPrintWriter)
- extends JPrintWriter(out)
- with StrippingWriter
- with TruncatingWriter {
- self =>
-
- def clean(str: String): String = truncate(strip(str))
- override def write(str: String) = super.write(clean(str))
- }
- class ReplStrippingWriter(intp: SparkIMain) extends StrippingTruncatingWriter(intp.out) {
- import intp._
- def maxStringLength = isettings.maxPrintString
- def isStripping = isettings.unwrapStrings
- def isTruncating = reporter.truncationOK
-
- def stripImpl(str: String): String = naming.unmangle(str)
- }
-
- class ReplReporter(intp: SparkIMain) extends ConsoleReporter(intp.settings, null, new ReplStrippingWriter(intp)) {
- override def printMessage(msg: String) {
- // Avoiding deadlock when the compiler starts logging before
- // the lazy val is done.
- if (intp.isInitializeComplete) {
- if (intp.totalSilence) ()
- else super.printMessage(msg)
- }
- // scalastyle:off println
- else Console.println(msg)
- // scalastyle:on println
- }
- }
-}
-
-class SparkISettings(intp: SparkIMain) extends Logging {
- /** A list of paths where :load should look */
- var loadPath = List(".")
-
- /** Set this to true to see repl machinery under -Yrich-exceptions.
- */
- var showInternalStackTraces = false
-
- /** The maximum length of toString to use when printing the result
- * of an evaluation. 0 means no maximum. If a printout requires
- * more than this number of characters, then the printout is
- * truncated.
- */
- var maxPrintString = 800
-
- /** The maximum number of completion candidates to print for tab
- * completion without requiring confirmation.
- */
- var maxAutoprintCompletion = 250
-
- /** String unwrapping can be disabled if it is causing issues.
- * Settings this to false means you will see Strings like "$iw.$iw.".
- */
- var unwrapStrings = true
-
- def deprecation_=(x: Boolean) = {
- val old = intp.settings.deprecation.value
- intp.settings.deprecation.value = x
- if (!old && x) logDebug("Enabled -deprecation output.")
- else if (old && !x) logDebug("Disabled -deprecation output.")
- }
-
- def deprecation: Boolean = intp.settings.deprecation.value
-
- def allSettings = Map(
- "maxPrintString" -> maxPrintString,
- "maxAutoprintCompletion" -> maxAutoprintCompletion,
- "unwrapStrings" -> unwrapStrings,
- "deprecation" -> deprecation
- )
-
- private def allSettingsString =
- allSettings.toList sortBy (_._1) map { case (k, v) => " " + k + " = " + v + "\n" } mkString
-
- override def toString = """
- | SparkISettings {
- | %s
- | }""".stripMargin.format(allSettingsString)
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkImports.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkImports.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkImports.scala
deleted file mode 100644
index f227765..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkImports.scala
+++ /dev/null
@@ -1,239 +0,0 @@
-// scalastyle:off
-
-/* NSC -- new Scala compiler
- * Copyright 2005-2013 LAMP/EPFL
- * @author Paul Phillips
- */
-
-package org.apache.spark.repl
-
-import scala.tools.nsc._
-import scala.tools.nsc.interpreter._
-
-import scala.collection.{ mutable, immutable }
-
-private[repl] trait SparkImports {
- self: SparkIMain =>
-
- import global._
- import definitions.{ ScalaPackage, JavaLangPackage, PredefModule }
- import memberHandlers._
-
- def isNoImports = settings.noimports.value
- def isNoPredef = settings.nopredef.value
-
- /** Synthetic import handlers for the language defined imports. */
- private def makeWildcardImportHandler(sym: Symbol): ImportHandler = {
- val hd :: tl = sym.fullName.split('.').toList map newTermName
- val tree = Import(
- tl.foldLeft(Ident(hd): Tree)((x, y) => Select(x, y)),
- ImportSelector.wildList
- )
- tree setSymbol sym
- new ImportHandler(tree)
- }
-
- /** Symbols whose contents are language-defined to be imported. */
- def languageWildcardSyms: List[Symbol] = List(JavaLangPackage, ScalaPackage, PredefModule)
- def languageWildcards: List[Type] = languageWildcardSyms map (_.tpe)
- def languageWildcardHandlers = languageWildcardSyms map makeWildcardImportHandler
-
- def allImportedNames = importHandlers flatMap (_.importedNames)
- def importedTerms = onlyTerms(allImportedNames)
- def importedTypes = onlyTypes(allImportedNames)
-
- /** Types which have been wildcard imported, such as:
- * val x = "abc" ; import x._ // type java.lang.String
- * import java.lang.String._ // object java.lang.String
- *
- * Used by tab completion.
- *
- * XXX right now this gets import x._ and import java.lang.String._,
- * but doesn't figure out import String._. There's a lot of ad hoc
- * scope twiddling which should be swept away in favor of digging
- * into the compiler scopes.
- */
- def sessionWildcards: List[Type] = {
- importHandlers filter (_.importsWildcard) map (_.targetType) distinct
- }
- def wildcardTypes = languageWildcards ++ sessionWildcards
-
- def languageSymbols = languageWildcardSyms flatMap membersAtPickler
- def sessionImportedSymbols = importHandlers flatMap (_.importedSymbols)
- def importedSymbols = languageSymbols ++ sessionImportedSymbols
- def importedTermSymbols = importedSymbols collect { case x: TermSymbol => x }
- def importedTypeSymbols = importedSymbols collect { case x: TypeSymbol => x }
- def implicitSymbols = importedSymbols filter (_.isImplicit)
-
- def importedTermNamed(name: String): Symbol =
- importedTermSymbols find (_.name.toString == name) getOrElse NoSymbol
-
- /** Tuples of (source, imported symbols) in the order they were imported.
- */
- def importedSymbolsBySource: List[(Symbol, List[Symbol])] = {
- val lang = languageWildcardSyms map (sym => (sym, membersAtPickler(sym)))
- val session = importHandlers filter (_.targetType != NoType) map { mh =>
- (mh.targetType.typeSymbol, mh.importedSymbols)
- }
-
- lang ++ session
- }
- def implicitSymbolsBySource: List[(Symbol, List[Symbol])] = {
- importedSymbolsBySource map {
- case (k, vs) => (k, vs filter (_.isImplicit))
- } filterNot (_._2.isEmpty)
- }
-
- /** Compute imports that allow definitions from previous
- * requests to be visible in a new request. Returns
- * three pieces of related code:
- *
- * 1. An initial code fragment that should go before
- * the code of the new request.
- *
- * 2. A code fragment that should go after the code
- * of the new request.
- *
- * 3. An access path which can be traversed to access
- * any bindings inside code wrapped by #1 and #2 .
- *
- * The argument is a set of Names that need to be imported.
- *
- * Limitations: This method is not as precise as it could be.
- * (1) It does not process wildcard imports to see what exactly
- * they import.
- * (2) If it imports any names from a request, it imports all
- * of them, which is not really necessary.
- * (3) It imports multiple same-named implicits, but only the
- * last one imported is actually usable.
- */
- case class SparkComputedImports(prepend: String, append: String, access: String)
- def fallback = System.getProperty("spark.repl.fallback", "false").toBoolean
-
- protected def importsCode(wanted: Set[Name], definedClass: Boolean): SparkComputedImports = {
- /** Narrow down the list of requests from which imports
- * should be taken. Removes requests which cannot contribute
- * useful imports for the specified set of wanted names.
- */
- case class ReqAndHandler(req: Request, handler: MemberHandler) { }
-
- def reqsToUse: List[ReqAndHandler] = {
- /**
- * Loop through a list of MemberHandlers and select which ones to keep.
- * 'wanted' is the set of names that need to be imported.
- */
- def select(reqs: List[ReqAndHandler], wanted: Set[Name]): List[ReqAndHandler] = {
- // Single symbol imports might be implicits! See bug #1752. Rather than
- // try to finesse this, we will mimic all imports for now.
- def keepHandler(handler: MemberHandler) = handler match {
- /* This case clause tries to "precisely" import only what is required. And in this
- * it may miss out on some implicits, because implicits are not known in `wanted`. Thus
- * it is suitable for defining classes. AFAIK while defining classes implicits are not
- * needed.*/
- case h: ImportHandler if definedClass && !fallback =>
- h.importedNames.exists(x => wanted.contains(x))
- case _: ImportHandler => true
- case x => x.definesImplicit || (x.definedNames exists wanted)
- }
-
- reqs match {
- case Nil => Nil
- case rh :: rest if !keepHandler(rh.handler) => select(rest, wanted)
- case rh :: rest =>
- import rh.handler._
- val newWanted = wanted ++ referencedNames -- definedNames -- importedNames
- rh :: select(rest, newWanted)
- }
- }
-
- /** Flatten the handlers out and pair each with the original request */
- select(allReqAndHandlers reverseMap { case (r, h) => ReqAndHandler(r, h) }, wanted).reverse
- }
-
- val code, trailingBraces, accessPath = new StringBuilder
- val currentImps = mutable.HashSet[Name]()
-
- // add code for a new object to hold some imports
- def addWrapper() {
- val impname = nme.INTERPRETER_IMPORT_WRAPPER
- code append "class %sC extends Serializable {\n".format(impname)
- trailingBraces append "}\nval " + impname + " = new " + impname + "C;\n"
- accessPath append ("." + impname)
-
- currentImps.clear
- // code append "object %s {\n".format(impname)
- // trailingBraces append "}\n"
- // accessPath append ("." + impname)
-
- // currentImps.clear
- }
-
- addWrapper()
-
- // loop through previous requests, adding imports for each one
- for (ReqAndHandler(req, handler) <- reqsToUse) {
- handler match {
- // If the user entered an import, then just use it; add an import wrapping
- // level if the import might conflict with some other import
- case x: ImportHandler =>
- if (x.importsWildcard || currentImps.exists(x.importedNames contains _))
- addWrapper()
-
- code append (x.member + "\n")
-
- // give wildcard imports a import wrapper all to their own
- if (x.importsWildcard) addWrapper()
- else currentImps ++= x.importedNames
-
- // For other requests, import each defined name.
- // import them explicitly instead of with _, so that
- // ambiguity errors will not be generated. Also, quote
- // the name of the variable, so that we don't need to
- // handle quoting keywords separately.
- case x: ClassHandler if !fallback =>
- // I am trying to guess if the import is a defined class
- // This is an ugly hack, I am not 100% sure of the consequences.
- // Here we, let everything but "defined classes" use the import with val.
- // The reason for this is, otherwise the remote executor tries to pull the
- // classes involved and may fail.
- for (imv <- x.definedNames) {
- val objName = req.lineRep.readPath
- code.append("import " + objName + ".INSTANCE" + req.accessPath + ".`" + imv + "`\n")
- }
-
- case x =>
- for (imv <- x.definedNames) {
- if (currentImps contains imv) addWrapper()
- val objName = req.lineRep.readPath
- val valName = "$VAL" + newValId()
-
- if(!code.toString.endsWith(".`" + imv + "`;\n")) { // Which means already imported
- code.append("val " + valName + " = " + objName + ".INSTANCE;\n")
- code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n")
- }
- // code.append("val " + valName + " = " + objName + ".INSTANCE;\n")
- // code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n")
- // code append ("import " + (req fullPath imv) + "\n")
- currentImps += imv
- }
- }
- }
- // add one extra wrapper, to prevent warnings in the common case of
- // redefining the value bound in the last interpreter request.
- addWrapper()
- SparkComputedImports(code.toString, trailingBraces.toString, accessPath.toString)
- }
-
- private def allReqAndHandlers =
- prevRequestList flatMap (req => req.handlers map (req -> _))
-
- private def membersAtPickler(sym: Symbol): List[Symbol] =
- beforePickler(sym.info.nonPrivateMembers.toList)
-
- private var curValId = 0
-
- private def newValId(): Int = {
- curValId += 1
- curValId
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[2/5] spark git commit: [SPARK-19810][BUILD][CORE] Remove support for
Scala 2.10
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkJLineCompletion.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkJLineCompletion.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkJLineCompletion.scala
deleted file mode 100644
index 1ba17df..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkJLineCompletion.scala
+++ /dev/null
@@ -1,403 +0,0 @@
-// scalastyle:off
-
-/* NSC -- new Scala compiler
- * Copyright 2005-2013 LAMP/EPFL
- * @author Paul Phillips
- */
-
-package org.apache.spark.repl
-
-import scala.tools.nsc._
-import scala.tools.nsc.interpreter._
-
-import scala.tools.jline._
-import scala.tools.jline.console.completer._
-import Completion._
-import scala.collection.mutable.ListBuffer
-
-import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.Logging
-
-/**
- * Represents an auto-completion tool for the supplied interpreter that
- * utilizes supplied queries for valid completions based on the current
- * contents of the internal buffer.
- *
- * @param intp The interpreter to use for information retrieval to do with
- * auto completion
- */
-@DeveloperApi
-class SparkJLineCompletion(val intp: SparkIMain) extends Completion with CompletionOutput with Logging {
- // NOTE: Exposed in package as used in quite a few classes
- // NOTE: Must be public to override the global found in CompletionOutput
- val global: intp.global.type = intp.global
-
- import global._
- import definitions.{ PredefModule, AnyClass, AnyRefClass, ScalaPackage, JavaLangPackage }
- import rootMirror.{ RootClass, getModuleIfDefined }
- type ExecResult = Any
- import intp.{ debugging }
-
- /**
- * Represents the level of verbosity. Increments with consecutive tabs.
- */
- @DeveloperApi
- var verbosity: Int = 0
-
- /**
- * Resets the level of verbosity to zero.
- */
- @DeveloperApi
- def resetVerbosity() = verbosity = 0
-
- private def getSymbol(name: String, isModule: Boolean) = (
- if (isModule) getModuleIfDefined(name)
- else getModuleIfDefined(name)
- )
- private def getType(name: String, isModule: Boolean) = getSymbol(name, isModule).tpe
- private def typeOf(name: String) = getType(name, false)
- private def moduleOf(name: String) = getType(name, true)
-
- trait CompilerCompletion {
- def tp: Type
- def effectiveTp = tp match {
- case MethodType(Nil, resType) => resType
- case NullaryMethodType(resType) => resType
- case _ => tp
- }
-
- // for some reason any's members don't show up in subclasses, which
- // we need so 5.<tab> offers asInstanceOf etc.
- private def anyMembers = AnyClass.tpe.nonPrivateMembers
- def anyRefMethodsToShow = Set("isInstanceOf", "asInstanceOf", "toString")
-
- def tos(sym: Symbol): String = sym.decodedName
- def memberNamed(s: String) = afterTyper(effectiveTp member newTermName(s))
- def hasMethod(s: String) = memberNamed(s).isMethod
-
- // XXX we'd like to say "filterNot (_.isDeprecated)" but this causes the
- // compiler to crash for reasons not yet known.
- def members = afterTyper((effectiveTp.nonPrivateMembers.toList ++ anyMembers) filter (_.isPublic))
- def methods = members.toList filter (_.isMethod)
- def packages = members.toList filter (_.isPackage)
- def aliases = members.toList filter (_.isAliasType)
-
- def memberNames = members map tos
- def methodNames = methods map tos
- def packageNames = packages map tos
- def aliasNames = aliases map tos
- }
-
- object NoTypeCompletion extends TypeMemberCompletion(NoType) {
- override def memberNamed(s: String) = NoSymbol
- override def members = Nil
- override def follow(s: String) = None
- override def alternativesFor(id: String) = Nil
- }
-
- object TypeMemberCompletion {
- def apply(tp: Type, runtimeType: Type, param: NamedParam): TypeMemberCompletion = {
- new TypeMemberCompletion(tp) {
- var upgraded = false
- lazy val upgrade = {
- intp rebind param
- intp.reporter.printMessage("\nRebinding stable value %s from %s to %s".format(param.name, tp, param.tpe))
- upgraded = true
- new TypeMemberCompletion(runtimeType)
- }
- override def completions(verbosity: Int) = {
- super.completions(verbosity) ++ (
- if (verbosity == 0) Nil
- else upgrade.completions(verbosity)
- )
- }
- override def follow(s: String) = super.follow(s) orElse {
- if (upgraded) upgrade.follow(s)
- else None
- }
- override def alternativesFor(id: String) = super.alternativesFor(id) ++ (
- if (upgraded) upgrade.alternativesFor(id)
- else Nil
- ) distinct
- }
- }
- def apply(tp: Type): TypeMemberCompletion = {
- if (tp eq NoType) NoTypeCompletion
- else if (tp.typeSymbol.isPackageClass) new PackageCompletion(tp)
- else new TypeMemberCompletion(tp)
- }
- def imported(tp: Type) = new ImportCompletion(tp)
- }
-
- class TypeMemberCompletion(val tp: Type) extends CompletionAware
- with CompilerCompletion {
- def excludeEndsWith: List[String] = Nil
- def excludeStartsWith: List[String] = List("<") // <byname>, <repeated>, etc.
- def excludeNames: List[String] = (anyref.methodNames filterNot anyRefMethodsToShow) :+ "_root_"
-
- def methodSignatureString(sym: Symbol) = {
- IMain stripString afterTyper(new MethodSymbolOutput(sym).methodString())
- }
-
- def exclude(name: String): Boolean = (
- (name contains "$") ||
- (excludeNames contains name) ||
- (excludeEndsWith exists (name endsWith _)) ||
- (excludeStartsWith exists (name startsWith _))
- )
- def filtered(xs: List[String]) = xs filterNot exclude distinct
-
- def completions(verbosity: Int) =
- debugging(tp + " completions ==> ")(filtered(memberNames))
-
- override def follow(s: String): Option[CompletionAware] =
- debugging(tp + " -> '" + s + "' ==> ")(Some(TypeMemberCompletion(memberNamed(s).tpe)) filterNot (_ eq NoTypeCompletion))
-
- override def alternativesFor(id: String): List[String] =
- debugging(id + " alternatives ==> ") {
- val alts = members filter (x => x.isMethod && tos(x) == id) map methodSignatureString
-
- if (alts.nonEmpty) "" :: alts else Nil
- }
-
- override def toString = "%s (%d members)".format(tp, members.size)
- }
-
- class PackageCompletion(tp: Type) extends TypeMemberCompletion(tp) {
- override def excludeNames = anyref.methodNames
- }
-
- class LiteralCompletion(lit: Literal) extends TypeMemberCompletion(lit.value.tpe) {
- override def completions(verbosity: Int) = verbosity match {
- case 0 => filtered(memberNames)
- case _ => memberNames
- }
- }
-
- class ImportCompletion(tp: Type) extends TypeMemberCompletion(tp) {
- override def completions(verbosity: Int) = verbosity match {
- case 0 => filtered(members filterNot (_.isSetter) map tos)
- case _ => super.completions(verbosity)
- }
- }
-
- // not for completion but for excluding
- object anyref extends TypeMemberCompletion(AnyRefClass.tpe) { }
-
- // the unqualified vals/defs/etc visible in the repl
- object ids extends CompletionAware {
- override def completions(verbosity: Int) = intp.unqualifiedIds ++ List("classOf") //, "_root_")
- // now we use the compiler for everything.
- override def follow(id: String): Option[CompletionAware] = {
- if (!completions(0).contains(id))
- return None
-
- val tpe = intp typeOfExpression id
- if (tpe == NoType)
- return None
-
- def default = Some(TypeMemberCompletion(tpe))
-
- // only rebinding vals in power mode for now.
- if (!isReplPower) default
- else intp runtimeClassAndTypeOfTerm id match {
- case Some((clazz, runtimeType)) =>
- val sym = intp.symbolOfTerm(id)
- if (sym.isStable) {
- val param = new NamedParam.Untyped(id, intp valueOfTerm id getOrElse null)
- Some(TypeMemberCompletion(tpe, runtimeType, param))
- }
- else default
- case _ =>
- default
- }
- }
- override def toString = "<repl ids> (%s)".format(completions(0).size)
- }
-
- // user-issued wildcard imports like "import global._" or "import String._"
- private def imported = intp.sessionWildcards map TypeMemberCompletion.imported
-
- // literal Ints, Strings, etc.
- object literals extends CompletionAware {
- def simpleParse(code: String): Tree = newUnitParser(code).templateStats().last
- def completions(verbosity: Int) = Nil
-
- override def follow(id: String) = simpleParse(id) match {
- case x: Literal => Some(new LiteralCompletion(x))
- case _ => None
- }
- }
-
- // top level packages
- object rootClass extends TypeMemberCompletion(RootClass.tpe) {
- override def completions(verbosity: Int) = super.completions(verbosity) :+ "_root_"
- override def follow(id: String) = id match {
- case "_root_" => Some(this)
- case _ => super.follow(id)
- }
- }
- // members of Predef
- object predef extends TypeMemberCompletion(PredefModule.tpe) {
- override def excludeEndsWith = super.excludeEndsWith ++ List("Wrapper", "ArrayOps")
- override def excludeStartsWith = super.excludeStartsWith ++ List("wrap")
- override def excludeNames = anyref.methodNames
-
- override def exclude(name: String) = super.exclude(name) || (
- (name contains "2")
- )
-
- override def completions(verbosity: Int) = verbosity match {
- case 0 => Nil
- case _ => super.completions(verbosity)
- }
- }
- // members of scala.*
- object scalalang extends PackageCompletion(ScalaPackage.tpe) {
- def arityClasses = List("Product", "Tuple", "Function")
- def skipArity(name: String) = arityClasses exists (x => name != x && (name startsWith x))
- override def exclude(name: String) = super.exclude(name) || (
- skipArity(name)
- )
-
- override def completions(verbosity: Int) = verbosity match {
- case 0 => filtered(packageNames ++ aliasNames)
- case _ => super.completions(verbosity)
- }
- }
- // members of java.lang.*
- object javalang extends PackageCompletion(JavaLangPackage.tpe) {
- override lazy val excludeEndsWith = super.excludeEndsWith ++ List("Exception", "Error")
- override lazy val excludeStartsWith = super.excludeStartsWith ++ List("CharacterData")
-
- override def completions(verbosity: Int) = verbosity match {
- case 0 => filtered(packageNames)
- case _ => super.completions(verbosity)
- }
- }
-
- // the list of completion aware objects which should be consulted
- // for top level unqualified, it's too noisy to let much in.
- private lazy val topLevelBase: List[CompletionAware] = List(ids, rootClass, predef, scalalang, javalang, literals)
- private def topLevel = topLevelBase ++ imported
- private def topLevelThreshold = 50
-
- // the first tier of top level objects (doesn't include file completion)
- private def topLevelFor(parsed: Parsed): List[String] = {
- val buf = new ListBuffer[String]
- topLevel foreach { ca =>
- buf ++= (ca completionsFor parsed)
-
- if (buf.size > topLevelThreshold)
- return buf.toList.sorted
- }
- buf.toList
- }
-
- // the most recent result
- private def lastResult = Forwarder(() => ids follow intp.mostRecentVar)
-
- private def lastResultFor(parsed: Parsed) = {
- /** The logic is a little tortured right now because normally '.' is
- * ignored as a delimiter, but on .<tab> it needs to be propagated.
- */
- val xs = lastResult completionsFor parsed
- if (parsed.isEmpty) xs map ("." + _) else xs
- }
-
- // generic interface for querying (e.g. interpreter loop, testing)
- private def completions(buf: String): List[String] =
- topLevelFor(Parsed.dotted(buf + ".", buf.length + 1))
-
- /**
- * Constructs a new ScalaCompleter for auto completion.
- *
- * @return The new JLineTabCompletion instance
- */
- @DeveloperApi
- def completer(): ScalaCompleter = new JLineTabCompletion
-
- /** This gets a little bit hairy. It's no small feat delegating everything
- * and also keeping track of exactly where the cursor is and where it's supposed
- * to end up. The alternatives mechanism is a little hacky: if there is an empty
- * string in the list of completions, that means we are expanding a unique
- * completion, so don't update the "last" buffer because it'll be wrong.
- */
- class JLineTabCompletion extends ScalaCompleter {
- // For recording the buffer on the last tab hit
- private var lastBuf: String = ""
- private var lastCursor: Int = -1
-
- // Does this represent two consecutive tabs?
- def isConsecutiveTabs(buf: String, cursor: Int) =
- cursor == lastCursor && buf == lastBuf
-
- // Longest common prefix
- def commonPrefix(xs: List[String]): String = {
- if (xs.isEmpty || xs.contains("")) ""
- else xs.head.head match {
- case ch =>
- if (xs.tail forall (_.head == ch)) "" + ch + commonPrefix(xs map (_.tail))
- else ""
- }
- }
-
- // This is jline's entry point for completion.
- override def complete(buf: String, cursor: Int): Candidates = {
- verbosity = if (isConsecutiveTabs(buf, cursor)) verbosity + 1 else 0
- logDebug("\ncomplete(%s, %d) last = (%s, %d), verbosity: %s".format(buf, cursor, lastBuf, lastCursor, verbosity))
-
- // we don't try lower priority completions unless higher ones return no results.
- def tryCompletion(p: Parsed, completionFunction: Parsed => List[String]): Option[Candidates] = {
- val winners = completionFunction(p)
- if (winners.isEmpty)
- return None
- val newCursor =
- if (winners contains "") p.cursor
- else {
- val advance = commonPrefix(winners)
- lastCursor = p.position + advance.length
- lastBuf = (buf take p.position) + advance
- logDebug("tryCompletion(%s, _) lastBuf = %s, lastCursor = %s, p.position = %s".format(
- p, lastBuf, lastCursor, p.position))
- p.position
- }
-
- Some(Candidates(newCursor, winners))
- }
-
- def mkDotted = Parsed.dotted(buf, cursor) withVerbosity verbosity
- def mkUndelimited = Parsed.undelimited(buf, cursor) withVerbosity verbosity
-
- // a single dot is special cased to completion on the previous result
- def lastResultCompletion =
- if (!looksLikeInvocation(buf)) None
- else tryCompletion(Parsed.dotted(buf drop 1, cursor), lastResultFor)
-
- def tryAll = (
- lastResultCompletion
- orElse tryCompletion(mkDotted, topLevelFor)
- getOrElse Candidates(cursor, Nil)
- )
-
- /**
- * This is the kickoff point for all manner of theoretically
- * possible compiler unhappiness. The fault may be here or
- * elsewhere, but we don't want to crash the repl regardless.
- * The compiler makes it impossible to avoid catching Throwable
- * with its unfortunate tendency to throw java.lang.Errors and
- * AssertionErrors as the hats drop. We take two swings at it
- * because there are some spots which like to throw an assertion
- * once, then work after that. Yeah, what can I say.
- */
- try tryAll
- catch { case ex: Throwable =>
- logWarning("Error: complete(%s, %s) provoked".format(buf, cursor) + ex)
- Candidates(cursor,
- if (isReplDebug) List("<error:" + ex + ">")
- else Nil
- )
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkJLineReader.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkJLineReader.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkJLineReader.scala
deleted file mode 100644
index 016e0f0..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkJLineReader.scala
+++ /dev/null
@@ -1,90 +0,0 @@
-// scalastyle:off
-
-/* NSC -- new Scala compiler
- * Copyright 2005-2013 LAMP/EPFL
- * @author Stepan Koltsov
- */
-
-package org.apache.spark.repl
-
-import scala.reflect.io.{Path, File}
-import scala.tools.nsc._
-import scala.tools.nsc.interpreter._
-import scala.tools.nsc.interpreter.session.JLineHistory.JLineFileHistory
-
-import scala.tools.jline.console.ConsoleReader
-import scala.tools.jline.console.completer._
-import session._
-import scala.collection.JavaConverters._
-import Completion._
-import io.Streamable.slurp
-
-/**
- * Reads from the console using JLine.
- */
-private[repl] class SparkJLineReader(_completion: => Completion) extends InteractiveReader {
- val interactive = true
- val consoleReader = new JLineConsoleReader()
-
- lazy val completion = _completion
- lazy val history: JLineHistory = new SparkJLineHistory
-
- private def term = consoleReader.getTerminal()
- def reset() = term.reset()
- def init() = term.init()
-
- def scalaToJline(tc: ScalaCompleter): Completer = new Completer {
- def complete(_buf: String, cursor: Int, candidates: JList[CharSequence]): Int = {
- val buf = if (_buf == null) "" else _buf
- val Candidates(newCursor, newCandidates) = tc.complete(buf, cursor)
- newCandidates foreach (candidates add _)
- newCursor
- }
- }
-
- class JLineConsoleReader extends ConsoleReader with ConsoleReaderHelper {
- if ((history: History) ne NoHistory)
- this setHistory history
-
- // working around protected/trait/java insufficiencies.
- def goBack(num: Int): Unit = back(num)
- def readOneKey(prompt: String) = {
- this.print(prompt)
- this.flush()
- this.readVirtualKey()
- }
- def eraseLine() = consoleReader.resetPromptLine("", "", 0)
- def redrawLineAndFlush(): Unit = { flush() ; drawLine() ; flush() }
- // override def readLine(prompt: String): String
-
- // A hook for running code after the repl is done initializing.
- lazy val postInit: Unit = {
- this setBellEnabled false
-
- if (completion ne NoCompletion) {
- val argCompletor: ArgumentCompleter =
- new ArgumentCompleter(new JLineDelimiter, scalaToJline(completion.completer()))
- argCompletor setStrict false
-
- this addCompleter argCompletor
- this setAutoprintThreshold 400 // max completion candidates without warning
- }
- }
- }
-
- def currentLine = consoleReader.getCursorBuffer.buffer.toString
- def redrawLine() = consoleReader.redrawLineAndFlush()
- def eraseLine() = consoleReader.eraseLine()
- // Alternate implementation, not sure if/when I need this.
- // def eraseLine() = while (consoleReader.delete()) { }
- def readOneLine(prompt: String) = consoleReader readLine prompt
- def readOneKey(prompt: String) = consoleReader readOneKey prompt
-}
-
-/** Changes the default history file to not collide with the scala repl's. */
-private[repl] class SparkJLineHistory extends JLineFileHistory {
- import Properties.userHome
-
- def defaultFileName = ".spark_history"
- override protected lazy val historyFile = File(Path(userHome) / defaultFileName)
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkMemberHandlers.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkMemberHandlers.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkMemberHandlers.scala
deleted file mode 100644
index 4de9714..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkMemberHandlers.scala
+++ /dev/null
@@ -1,232 +0,0 @@
-// scalastyle:off
-
-/* NSC -- new Scala compiler
- * Copyright 2005-2013 LAMP/EPFL
- * @author Martin Odersky
- */
-
-package org.apache.spark.repl
-
-import scala.tools.nsc._
-import scala.tools.nsc.interpreter._
-
-import scala.collection.{ mutable, immutable }
-import scala.PartialFunction.cond
-import scala.reflect.internal.Chars
-import scala.reflect.internal.Flags._
-import scala.language.implicitConversions
-
-private[repl] trait SparkMemberHandlers {
- val intp: SparkIMain
-
- import intp.{ Request, global, naming }
- import global._
- import naming._
-
- private def codegenln(leadingPlus: Boolean, xs: String*): String = codegen(leadingPlus, (xs ++ Array("\n")): _*)
- private def codegenln(xs: String*): String = codegenln(true, xs: _*)
-
- private def codegen(xs: String*): String = codegen(true, xs: _*)
- private def codegen(leadingPlus: Boolean, xs: String*): String = {
- val front = if (leadingPlus) "+ " else ""
- front + (xs map string2codeQuoted mkString " + ")
- }
- private implicit def name2string(name: Name) = name.toString
-
- /** A traverser that finds all mentioned identifiers, i.e. things
- * that need to be imported. It might return extra names.
- */
- private class ImportVarsTraverser extends Traverser {
- val importVars = new mutable.HashSet[Name]()
-
- override def traverse(ast: Tree) = ast match {
- case Ident(name) =>
- // XXX this is obviously inadequate but it's going to require some effort
- // to get right.
- if (name.toString startsWith "x$") ()
- else importVars += name
- case _ => super.traverse(ast)
- }
- }
- private object ImportVarsTraverser {
- def apply(member: Tree) = {
- val ivt = new ImportVarsTraverser()
- ivt traverse member
- ivt.importVars.toList
- }
- }
-
- def chooseHandler(member: Tree): MemberHandler = member match {
- case member: DefDef => new DefHandler(member)
- case member: ValDef => new ValHandler(member)
- case member: Assign => new AssignHandler(member)
- case member: ModuleDef => new ModuleHandler(member)
- case member: ClassDef => new ClassHandler(member)
- case member: TypeDef => new TypeAliasHandler(member)
- case member: Import => new ImportHandler(member)
- case DocDef(_, documented) => chooseHandler(documented)
- case member => new GenericHandler(member)
- }
-
- sealed abstract class MemberDefHandler(override val member: MemberDef) extends MemberHandler(member) {
- def symbol = if (member.symbol eq null) NoSymbol else member.symbol
- def name: Name = member.name
- def mods: Modifiers = member.mods
- def keyword = member.keyword
- def prettyName = name.decode
-
- override def definesImplicit = member.mods.isImplicit
- override def definesTerm: Option[TermName] = Some(name.toTermName) filter (_ => name.isTermName)
- override def definesType: Option[TypeName] = Some(name.toTypeName) filter (_ => name.isTypeName)
- override def definedSymbols = if (symbol eq NoSymbol) Nil else List(symbol)
- }
-
- /** Class to handle one member among all the members included
- * in a single interpreter request.
- */
- sealed abstract class MemberHandler(val member: Tree) {
- def definesImplicit = false
- def definesValue = false
- def isLegalTopLevel = false
-
- def definesTerm = Option.empty[TermName]
- def definesType = Option.empty[TypeName]
-
- lazy val referencedNames = ImportVarsTraverser(member)
- def importedNames = List[Name]()
- def definedNames = definesTerm.toList ++ definesType.toList
- def definedOrImported = definedNames ++ importedNames
- def definedSymbols = List[Symbol]()
-
- def extraCodeToEvaluate(req: Request): String = ""
- def resultExtractionCode(req: Request): String = ""
-
- private def shortName = this.getClass.toString split '.' last
- override def toString = shortName + referencedNames.mkString(" (refs: ", ", ", ")")
- }
-
- class GenericHandler(member: Tree) extends MemberHandler(member)
-
- class ValHandler(member: ValDef) extends MemberDefHandler(member) {
- val maxStringElements = 1000 // no need to mkString billions of elements
- override def definesValue = true
-
- override def resultExtractionCode(req: Request): String = {
- val isInternal = isUserVarName(name) && req.lookupTypeOf(name) == "Unit"
- if (!mods.isPublic || isInternal) ""
- else {
- // if this is a lazy val we avoid evaluating it here
- val resultString =
- if (mods.isLazy) codegenln(false, "<lazy>")
- else any2stringOf(req fullPath name, maxStringElements)
-
- val vidString =
- if (replProps.vids) """" + " @ " + "%%8x".format(System.identityHashCode(%s)) + " """.trim.format(req fullPath name)
- else ""
-
- """ + "%s%s: %s = " + %s""".format(string2code(prettyName), vidString, string2code(req typeOf name), resultString)
- }
- }
- }
-
- class DefHandler(member: DefDef) extends MemberDefHandler(member) {
- private def vparamss = member.vparamss
- private def isMacro = member.symbol hasFlag MACRO
- // true if not a macro and 0-arity
- override def definesValue = !isMacro && flattensToEmpty(vparamss)
- override def resultExtractionCode(req: Request) =
- if (mods.isPublic) codegenln(name, ": ", req.typeOf(name)) else ""
- }
-
- class AssignHandler(member: Assign) extends MemberHandler(member) {
- val Assign(lhs, rhs) = member
- val name = newTermName(freshInternalVarName())
-
- override def definesTerm = Some(name)
- override def definesValue = true
- override def extraCodeToEvaluate(req: Request) =
- """val %s = %s""".format(name, lhs)
-
- /** Print out lhs instead of the generated varName */
- override def resultExtractionCode(req: Request) = {
- val lhsType = string2code(req lookupTypeOf name)
- val res = string2code(req fullPath name)
- """ + "%s: %s = " + %s + "\n" """.format(string2code(lhs.toString), lhsType, res) + "\n"
- }
- }
-
- class ModuleHandler(module: ModuleDef) extends MemberDefHandler(module) {
- override def definesTerm = Some(name)
- override def definesValue = true
- override def isLegalTopLevel = true
-
- override def resultExtractionCode(req: Request) = codegenln("defined module ", name)
- }
-
- class ClassHandler(member: ClassDef) extends MemberDefHandler(member) {
- override def definesType = Some(name.toTypeName)
- override def definesTerm = Some(name.toTermName) filter (_ => mods.isCase)
- override def isLegalTopLevel = true
-
- override def resultExtractionCode(req: Request) =
- codegenln("defined %s %s".format(keyword, name))
- }
-
- class TypeAliasHandler(member: TypeDef) extends MemberDefHandler(member) {
- private def isAlias = mods.isPublic && treeInfo.isAliasTypeDef(member)
- override def definesType = Some(name.toTypeName) filter (_ => isAlias)
-
- override def resultExtractionCode(req: Request) =
- codegenln("defined type alias ", name) + "\n"
- }
-
- class ImportHandler(imp: Import) extends MemberHandler(imp) {
- val Import(expr, selectors) = imp
- def targetType: Type = intp.typeOfExpression("" + expr)
- override def isLegalTopLevel = true
-
- def createImportForName(name: Name): String = {
- selectors foreach {
- case sel @ ImportSelector(old, _, `name`, _) => return "import %s.{ %s }".format(expr, sel)
- case _ => ()
- }
- "import %s.%s".format(expr, name)
- }
- // TODO: Need to track these specially to honor Predef masking attempts,
- // because they must be the leading imports in the code generated for each
- // line. We can use the same machinery as Contexts now, anyway.
- def isPredefImport = isReferenceToPredef(expr)
-
- // wildcard imports, e.g. import foo._
- private def selectorWild = selectors filter (_.name == nme.USCOREkw)
- // renamed imports, e.g. import foo.{ bar => baz }
- private def selectorRenames = selectors map (_.rename) filterNot (_ == null)
-
- /** Whether this import includes a wildcard import */
- val importsWildcard = selectorWild.nonEmpty
-
- /** Whether anything imported is implicit .*/
- def importsImplicit = implicitSymbols.nonEmpty
-
- def implicitSymbols = importedSymbols filter (_.isImplicit)
- def importedSymbols = individualSymbols ++ wildcardSymbols
-
- lazy val individualSymbols: List[Symbol] =
- beforePickler(individualNames map (targetType nonPrivateMember _))
-
- lazy val wildcardSymbols: List[Symbol] =
- if (importsWildcard) beforePickler(targetType.nonPrivateMembers.toList)
- else Nil
-
- /** Complete list of names imported by a wildcard */
- lazy val wildcardNames: List[Name] = wildcardSymbols map (_.name)
- lazy val individualNames: List[Name] = selectorRenames filterNot (_ == nme.USCOREkw) flatMap (_.bothNames)
-
- /** The names imported by this statement */
- override lazy val importedNames: List[Name] = wildcardNames ++ individualNames
- lazy val importsSymbolNamed: Set[String] = importedNames map (_.toString) toSet
-
- def importString = imp.toString
- override def resultExtractionCode(req: Request) = codegenln(importString) + "\n"
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala
deleted file mode 100644
index 94c801e..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkRunnerSettings.scala
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.repl
-
-import scala.tools.nsc.Settings
-
-/**
- * <i>scala.tools.nsc.Settings</i> implementation adding Spark-specific REPL
- * command line options.
- */
-private[repl] class SparkRunnerSettings(error: String => Unit) extends Settings(error) {
- val loadfiles = MultiStringSetting(
- "-i",
- "file",
- "load a file (assumes the code is given interactively)")
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/test/scala/org/apache/spark/repl/ReplSuite.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/scala-2.10/src/test/scala/org/apache/spark/repl/ReplSuite.scala
deleted file mode 100644
index b3688c9..0000000
--- a/repl/scala-2.10/src/test/scala/org/apache/spark/repl/ReplSuite.scala
+++ /dev/null
@@ -1,366 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.repl
-
-import java.io._
-import java.net.URLClassLoader
-
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.spark.{SparkContext, SparkFunSuite}
-import org.apache.commons.lang3.StringEscapeUtils
-import org.apache.spark.util.Utils
-
-
-class ReplSuite extends SparkFunSuite {
-
- def runInterpreter(master: String, input: String): String = {
- val CONF_EXECUTOR_CLASSPATH = "spark.executor.extraClassPath"
-
- val in = new BufferedReader(new StringReader(input + "\n"))
- val out = new StringWriter()
- val cl = getClass.getClassLoader
- var paths = new ArrayBuffer[String]
- if (cl.isInstanceOf[URLClassLoader]) {
- val urlLoader = cl.asInstanceOf[URLClassLoader]
- for (url <- urlLoader.getURLs) {
- if (url.getProtocol == "file") {
- paths += url.getFile
- }
- }
- }
- val classpath = paths.map(new File(_).getAbsolutePath).mkString(File.pathSeparator)
-
- val oldExecutorClasspath = System.getProperty(CONF_EXECUTOR_CLASSPATH)
- System.setProperty(CONF_EXECUTOR_CLASSPATH, classpath)
-
- val interp = new SparkILoop(in, new PrintWriter(out), master)
- org.apache.spark.repl.Main.interp = interp
- interp.process(Array("-classpath", classpath))
- org.apache.spark.repl.Main.interp = null
- if (interp.sparkContext != null) {
- interp.sparkContext.stop()
- }
- if (oldExecutorClasspath != null) {
- System.setProperty(CONF_EXECUTOR_CLASSPATH, oldExecutorClasspath)
- } else {
- System.clearProperty(CONF_EXECUTOR_CLASSPATH)
- }
- return out.toString
- }
-
- def assertContains(message: String, output: String) {
- val isContain = output.contains(message)
- assert(isContain,
- "Interpreter output did not contain '" + message + "':\n" + output)
- }
-
- def assertDoesNotContain(message: String, output: String) {
- val isContain = output.contains(message)
- assert(!isContain,
- "Interpreter output contained '" + message + "':\n" + output)
- }
-
- test("propagation of local properties") {
- // A mock ILoop that doesn't install the SIGINT handler.
- class ILoop(out: PrintWriter) extends SparkILoop(None, out, None) {
- settings = new scala.tools.nsc.Settings
- settings.usejavacp.value = true
- org.apache.spark.repl.Main.interp = this
- override def createInterpreter() {
- intp = new SparkILoopInterpreter
- intp.setContextClassLoader()
- }
- }
-
- val out = new StringWriter()
- val interp = new ILoop(new PrintWriter(out))
- interp.sparkContext = new SparkContext("local", "repl-test")
- interp.createInterpreter()
- interp.intp.initialize()
- interp.sparkContext.setLocalProperty("someKey", "someValue")
-
- // Make sure the value we set in the caller to interpret is propagated in the thread that
- // interprets the command.
- interp.interpret("org.apache.spark.repl.Main.interp.sparkContext.getLocalProperty(\"someKey\")")
- assert(out.toString.contains("someValue"))
-
- interp.sparkContext.stop()
- System.clearProperty("spark.driver.port")
- }
-
- test("simple foreach with accumulator") {
- val output = runInterpreter("local",
- """
- |val accum = sc.longAccumulator
- |sc.parallelize(1 to 10).foreach(x => accum.add(x))
- |accum.value
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- assertContains("res1: Long = 55", output)
- }
-
- test("external vars") {
- val output = runInterpreter("local",
- """
- |var v = 7
- |sc.parallelize(1 to 10).map(x => v).collect().reduceLeft(_+_)
- |v = 10
- |sc.parallelize(1 to 10).map(x => v).collect().reduceLeft(_+_)
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- assertContains("res0: Int = 70", output)
- assertContains("res1: Int = 100", output)
- }
-
- test("external classes") {
- val output = runInterpreter("local",
- """
- |class C {
- |def foo = 5
- |}
- |sc.parallelize(1 to 10).map(x => (new C).foo).collect().reduceLeft(_+_)
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- assertContains("res0: Int = 50", output)
- }
-
- test("external functions") {
- val output = runInterpreter("local",
- """
- |def double(x: Int) = x + x
- |sc.parallelize(1 to 10).map(x => double(x)).collect().reduceLeft(_+_)
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- assertContains("res0: Int = 110", output)
- }
-
- test("external functions that access vars") {
- val output = runInterpreter("local",
- """
- |var v = 7
- |def getV() = v
- |sc.parallelize(1 to 10).map(x => getV()).collect().reduceLeft(_+_)
- |v = 10
- |sc.parallelize(1 to 10).map(x => getV()).collect().reduceLeft(_+_)
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- assertContains("res0: Int = 70", output)
- assertContains("res1: Int = 100", output)
- }
-
- test("broadcast vars") {
- // Test that the value that a broadcast var had when it was created is used,
- // even if that variable is then modified in the driver program
- // TODO: This doesn't actually work for arrays when we run in local mode!
- val output = runInterpreter("local",
- """
- |var array = new Array[Int](5)
- |val broadcastArray = sc.broadcast(array)
- |sc.parallelize(0 to 4).map(x => broadcastArray.value(x)).collect()
- |array(0) = 5
- |sc.parallelize(0 to 4).map(x => broadcastArray.value(x)).collect()
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- assertContains("res0: Array[Int] = Array(0, 0, 0, 0, 0)", output)
- assertContains("res2: Array[Int] = Array(5, 0, 0, 0, 0)", output)
- }
-
- test("interacting with files") {
- val tempDir = Utils.createTempDir()
- val out = new FileWriter(tempDir + "/input")
- out.write("Hello world!\n")
- out.write("What's up?\n")
- out.write("Goodbye\n")
- out.close()
- val output = runInterpreter("local",
- """
- |var file = sc.textFile("%s").cache()
- |file.count()
- |file.count()
- |file.count()
- """.stripMargin.format(StringEscapeUtils.escapeJava(
- tempDir.getAbsolutePath + File.separator + "input")))
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- assertContains("res0: Long = 3", output)
- assertContains("res1: Long = 3", output)
- assertContains("res2: Long = 3", output)
- Utils.deleteRecursively(tempDir)
- }
-
- test("local-cluster mode") {
- val output = runInterpreter("local-cluster[1,1,1024]",
- """
- |var v = 7
- |def getV() = v
- |sc.parallelize(1 to 10).map(x => getV()).collect().reduceLeft(_+_)
- |v = 10
- |sc.parallelize(1 to 10).map(x => getV()).collect().reduceLeft(_+_)
- |var array = new Array[Int](5)
- |val broadcastArray = sc.broadcast(array)
- |sc.parallelize(0 to 4).map(x => broadcastArray.value(x)).collect()
- |array(0) = 5
- |sc.parallelize(0 to 4).map(x => broadcastArray.value(x)).collect()
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- assertContains("res0: Int = 70", output)
- assertContains("res1: Int = 100", output)
- assertContains("res2: Array[Int] = Array(0, 0, 0, 0, 0)", output)
- assertContains("res4: Array[Int] = Array(0, 0, 0, 0, 0)", output)
- }
-
- test("SPARK-1199 two instances of same class don't type check.") {
- val output = runInterpreter("local",
- """
- |case class Sum(exp: String, exp2: String)
- |val a = Sum("A", "B")
- |def b(a: Sum): String = a match { case Sum(_, _) => "Found Sum" }
- |b(a)
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- }
-
- test("SPARK-2452 compound statements.") {
- val output = runInterpreter("local",
- """
- |val x = 4 ; def f() = x
- |f()
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- }
-
- test("SPARK-2576 importing SQLContext.implicits._") {
- // We need to use local-cluster to test this case.
- val output = runInterpreter("local-cluster[1,1,1024]",
- """
- |val sqlContext = new org.apache.spark.sql.SQLContext(sc)
- |import sqlContext.implicits._
- |case class TestCaseClass(value: Int)
- |sc.parallelize(1 to 10).map(x => TestCaseClass(x)).toDF().collect()
- |
- |// Test Dataset Serialization in the REPL
- |Seq(TestCaseClass(1)).toDS().collect()
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- }
-
- test("SPARK-8461 SQL with codegen") {
- val output = runInterpreter("local",
- """
- |val sqlContext = new org.apache.spark.sql.SQLContext(sc)
- |sqlContext.setConf("spark.sql.codegen", "true")
- |sqlContext.range(0, 100).filter('id > 50).count()
- """.stripMargin)
- assertContains("Long = 49", output)
- assertDoesNotContain("java.lang.ClassNotFoundException", output)
- }
-
- test("Datasets and encoders") {
- val output = runInterpreter("local",
- """
- |import org.apache.spark.sql.functions._
- |import org.apache.spark.sql.{Encoder, Encoders}
- |import org.apache.spark.sql.expressions.Aggregator
- |import org.apache.spark.sql.TypedColumn
- |val simpleSum = new Aggregator[Int, Int, Int] {
- | def zero: Int = 0 // The initial value.
- | def reduce(b: Int, a: Int) = b + a // Add an element to the running total
- | def merge(b1: Int, b2: Int) = b1 + b2 // Merge intermediate values.
- | def finish(b: Int) = b // Return the final result.
- | def bufferEncoder: Encoder[Int] = Encoders.scalaInt
- | def outputEncoder: Encoder[Int] = Encoders.scalaInt
- |}.toColumn
- |
- |val ds = Seq(1, 2, 3, 4).toDS()
- |ds.select(simpleSum).collect
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- }
-
- test("SPARK-2632 importing a method from non serializable class and not using it.") {
- val output = runInterpreter("local-cluster[1,1,1024]",
- """
- |class TestClass() { def testMethod = 3 }
- |val t = new TestClass
- |import t.testMethod
- |case class TestCaseClass(value: Int)
- |sc.parallelize(1 to 10).map(x => TestCaseClass(x)).collect()
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- }
-
- if (System.getenv("MESOS_NATIVE_JAVA_LIBRARY") != null) {
- test("running on Mesos") {
- val output = runInterpreter("localquiet",
- """
- |var v = 7
- |def getV() = v
- |sc.parallelize(1 to 10).map(x => getV()).collect().reduceLeft(_+_)
- |v = 10
- |sc.parallelize(1 to 10).map(x => getV()).collect().reduceLeft(_+_)
- |var array = new Array[Int](5)
- |val broadcastArray = sc.broadcast(array)
- |sc.parallelize(0 to 4).map(x => broadcastArray.value(x)).collect()
- |array(0) = 5
- |sc.parallelize(0 to 4).map(x => broadcastArray.value(x)).collect()
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- assertContains("res0: Int = 70", output)
- assertContains("res1: Int = 100", output)
- assertContains("res2: Array[Int] = Array(0, 0, 0, 0, 0)", output)
- assertContains("res4: Array[Int] = Array(0, 0, 0, 0, 0)", output)
- }
- }
-
- test("collecting objects of class defined in repl") {
- val output = runInterpreter("local[2]",
- """
- |case class Foo(i: Int)
- |val ret = sc.parallelize((1 to 100).map(Foo), 10).collect()
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- assertContains("ret: Array[Foo] = Array(Foo(1),", output)
- }
-
- test("collecting objects of class defined in repl - shuffling") {
- val output = runInterpreter("local-cluster[1,1,1024]",
- """
- |case class Foo(i: Int)
- |val list = List((1, Foo(1)), (1, Foo(2)))
- |val ret = sc.parallelize(list).groupByKey().collect()
- """.stripMargin)
- assertDoesNotContain("error:", output)
- assertDoesNotContain("Exception", output)
- assertContains("ret: Array[(Int, Iterable[Foo])] = Array((1,", output)
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java
index c29b002..aadfcaa 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/execution/UnsafeExternalRowSorter.java
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution;
import java.io.IOException;
+import scala.collection.AbstractIterator;
import scala.collection.Iterator;
import scala.math.Ordering;
@@ -28,7 +29,6 @@ import org.apache.spark.SparkEnv;
import org.apache.spark.TaskContext;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.catalyst.expressions.UnsafeRow;
-import org.apache.spark.sql.catalyst.util.AbstractScalaRowIterator;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.unsafe.Platform;
import org.apache.spark.util.collection.unsafe.sort.PrefixComparator;
@@ -145,7 +145,7 @@ public final class UnsafeExternalRowSorter {
// here in order to prevent memory leaks.
cleanupResources();
}
- return new AbstractScalaRowIterator<UnsafeRow>() {
+ return new AbstractIterator<UnsafeRow>() {
private final int numFields = schema.length();
private UnsafeRow row = new UnsafeRow(numFields);
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 4d5401f..004b4ef 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -41,8 +41,7 @@ object ScalaReflection extends ScalaReflection {
// Since we are creating a runtime mirror using the class loader of current thread,
// we need to use def at here. So, every time we call mirror, it is using the
// class loader of the current thread.
- // SPARK-13640: Synchronize this because universe.runtimeMirror is not thread-safe in Scala 2.10.
- override def mirror: universe.Mirror = ScalaReflectionLock.synchronized {
+ override def mirror: universe.Mirror = {
universe.runtimeMirror(Thread.currentThread().getContextClassLoader)
}
@@ -62,7 +61,7 @@ object ScalaReflection extends ScalaReflection {
*/
def dataTypeFor[T : TypeTag]: DataType = dataTypeFor(localTypeOf[T])
- private def dataTypeFor(tpe: `Type`): DataType = ScalaReflectionLock.synchronized {
+ private def dataTypeFor(tpe: `Type`): DataType = {
tpe match {
case t if t <:< definitions.IntTpe => IntegerType
case t if t <:< definitions.LongTpe => LongType
@@ -94,7 +93,7 @@ object ScalaReflection extends ScalaReflection {
* Special handling is performed for primitive types to map them back to their raw
* JVM form instead of the Scala Array that handles auto boxing.
*/
- private def arrayClassFor(tpe: `Type`): ObjectType = ScalaReflectionLock.synchronized {
+ private def arrayClassFor(tpe: `Type`): ObjectType = {
val cls = tpe match {
case t if t <:< definitions.IntTpe => classOf[Array[Int]]
case t if t <:< definitions.LongTpe => classOf[Array[Long]]
@@ -141,7 +140,7 @@ object ScalaReflection extends ScalaReflection {
private def deserializerFor(
tpe: `Type`,
path: Option[Expression],
- walkedTypePath: Seq[String]): Expression = ScalaReflectionLock.synchronized {
+ walkedTypePath: Seq[String]): Expression = {
/** Returns the current path with a sub-field extracted. */
def addToPath(part: String, dataType: DataType, walkedTypePath: Seq[String]): Expression = {
@@ -329,8 +328,8 @@ object ScalaReflection extends ScalaReflection {
}
}
- val companion = t.normalize.typeSymbol.companionSymbol.typeSignature
- val cls = companion.member(newTermName("newBuilder")) match {
+ val companion = t.dealias.typeSymbol.companion.typeSignature
+ val cls = companion.member(TermName("newBuilder")) match {
case NoSymbol if t <:< localTypeOf[Seq[_]] => classOf[Seq[_]]
case NoSymbol if t <:< localTypeOf[scala.collection.Set[_]] =>
classOf[scala.collection.Set[_]]
@@ -349,7 +348,7 @@ object ScalaReflection extends ScalaReflection {
mirror.runtimeClass(t.typeSymbol.asClass)
)
- case t if t.typeSymbol.annotations.exists(_.tpe =:= typeOf[SQLUserDefinedType]) =>
+ case t if t.typeSymbol.annotations.exists(_.tree.tpe =:= typeOf[SQLUserDefinedType]) =>
val udt = getClassFromType(t).getAnnotation(classOf[SQLUserDefinedType]).udt().newInstance()
val obj = NewInstance(
udt.userClass.getAnnotation(classOf[SQLUserDefinedType]).udt(),
@@ -436,7 +435,7 @@ object ScalaReflection extends ScalaReflection {
inputObject: Expression,
tpe: `Type`,
walkedTypePath: Seq[String],
- seenTypeSet: Set[`Type`] = Set.empty): Expression = ScalaReflectionLock.synchronized {
+ seenTypeSet: Set[`Type`] = Set.empty): Expression = {
def toCatalystArray(input: Expression, elementType: `Type`): Expression = {
dataTypeFor(elementType) match {
@@ -591,7 +590,7 @@ object ScalaReflection extends ScalaReflection {
case t if t <:< localTypeOf[java.lang.Boolean] =>
Invoke(inputObject, "booleanValue", BooleanType)
- case t if t.typeSymbol.annotations.exists(_.tpe =:= typeOf[SQLUserDefinedType]) =>
+ case t if t.typeSymbol.annotations.exists(_.tree.tpe =:= typeOf[SQLUserDefinedType]) =>
val udt = getClassFromType(t)
.getAnnotation(classOf[SQLUserDefinedType]).udt().newInstance()
val obj = NewInstance(
@@ -643,7 +642,7 @@ object ScalaReflection extends ScalaReflection {
* Returns true if the given type is option of product type, e.g. `Option[Tuple2]`. Note that,
* we also treat [[DefinedByConstructorParams]] as product type.
*/
- def optionOfProductType(tpe: `Type`): Boolean = ScalaReflectionLock.synchronized {
+ def optionOfProductType(tpe: `Type`): Boolean = {
tpe match {
case t if t <:< localTypeOf[Option[_]] =>
val TypeRef(_, _, Seq(optType)) = t
@@ -705,9 +704,9 @@ object ScalaReflection extends ScalaReflection {
def schemaFor[T: TypeTag]: Schema = schemaFor(localTypeOf[T])
/** Returns a catalyst DataType and its nullability for the given Scala Type using reflection. */
- def schemaFor(tpe: `Type`): Schema = ScalaReflectionLock.synchronized {
+ def schemaFor(tpe: `Type`): Schema = {
tpe match {
- case t if t.typeSymbol.annotations.exists(_.tpe =:= typeOf[SQLUserDefinedType]) =>
+ case t if t.typeSymbol.annotations.exists(_.tree.tpe =:= typeOf[SQLUserDefinedType]) =>
val udt = getClassFromType(t).getAnnotation(classOf[SQLUserDefinedType]).udt().newInstance()
Schema(udt, nullable = true)
case t if UDTRegistration.exists(getClassNameFromType(t)) =>
@@ -814,10 +813,9 @@ trait ScalaReflection {
*
* @see SPARK-5281
*/
- // SPARK-13640: Synchronize this because TypeTag.tpe is not thread-safe in Scala 2.10.
- def localTypeOf[T: TypeTag]: `Type` = ScalaReflectionLock.synchronized {
+ def localTypeOf[T: TypeTag]: `Type` = {
val tag = implicitly[TypeTag[T]]
- tag.in(mirror).tpe.normalize
+ tag.in(mirror).tpe.dealias
}
/**
@@ -866,9 +864,9 @@ trait ScalaReflection {
}
protected def constructParams(tpe: Type): Seq[Symbol] = {
- val constructorSymbol = tpe.member(nme.CONSTRUCTOR)
+ val constructorSymbol = tpe.member(termNames.CONSTRUCTOR)
val params = if (constructorSymbol.isMethod) {
- constructorSymbol.asMethod.paramss
+ constructorSymbol.asMethod.paramLists
} else {
// Find the primary constructor, and use its parameter ordering.
val primaryConstructorSymbol: Option[Symbol] = constructorSymbol.asTerm.alternatives.find(
@@ -876,7 +874,7 @@ trait ScalaReflection {
if (primaryConstructorSymbol.isEmpty) {
sys.error("Internal SQL error: Product object did not have a primary constructor.")
} else {
- primaryConstructorSymbol.get.asMethod.paramss
+ primaryConstructorSymbol.get.asMethod.paramLists
}
}
params.flatten
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 5353649..7745709 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1518,9 +1518,9 @@ class Analyzer(
*/
def unapply(e: Expression): Option[(Generator, Seq[String], Boolean)] = e match {
case Alias(GeneratorOuter(g: Generator), name) if g.resolved => Some((g, name :: Nil, true))
- case MultiAlias(GeneratorOuter(g: Generator), names) if g.resolved => Some(g, names, true)
+ case MultiAlias(GeneratorOuter(g: Generator), names) if g.resolved => Some((g, names, true))
case Alias(g: Generator, name) if g.resolved => Some((g, name :: Nil, false))
- case MultiAlias(g: Generator, names) if g.resolved => Some(g, names, false)
+ case MultiAlias(g: Generator, names) if g.resolved => Some((g, names, false))
case _ => None
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/package.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/package.scala
index 41128fe..9ff5b87 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/package.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.catalyst.expressions
+import scala.reflect.internal.util.AbstractFileClassLoader
+
import org.apache.spark.sql.catalyst.rules
import org.apache.spark.util.Utils
@@ -51,7 +53,7 @@ package object codegen {
val classLoader =
generatedClass
.getClassLoader
- .asInstanceOf[scala.tools.nsc.interpreter.AbstractFileClassLoader]
+ .asInstanceOf[AbstractFileClassLoader]
val generatedBytes = classLoader.classBytes(generatedClass.getName)
val packageDir = new java.io.File(dumpDirectory, generatedClass.getPackage.getName)
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 29c3380..b898484 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -30,7 +30,7 @@ object NamedExpression {
private val curId = new java.util.concurrent.atomic.AtomicLong()
private[expressions] val jvmId = UUID.randomUUID()
def newExprId: ExprId = ExprId(curId.getAndIncrement(), jvmId)
- def unapply(expr: NamedExpression): Option[(String, DataType)] = Some(expr.name, expr.dataType)
+ def unapply(expr: NamedExpression): Option[(String, DataType)] = Some((expr.name, expr.dataType))
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/package.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/package.scala
index f9c88d4..a6b10cf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/package.scala
@@ -22,10 +22,4 @@ package org.apache.spark.sql
* considered an internal API to Spark SQL and are subject to change between minor releases.
*/
package object catalyst {
- /**
- * A JVM-global lock that should be used to prevent thread safety issues when using things in
- * scala.reflect.*. Note that Scala Reflection API is made thread-safe in 2.11, but not yet for
- * 2.10.* builds. See SI-6240 for more details.
- */
- protected[sql] object ScalaReflectionLock
}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/AbstractScalaRowIterator.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/AbstractScalaRowIterator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/AbstractScalaRowIterator.scala
deleted file mode 100644
index 0c7205b..0000000
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/AbstractScalaRowIterator.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.catalyst.util
-
-/**
- * Shim to allow us to implement [[scala.Iterator]] in Java. Scala 2.11+ has an AbstractIterator
- * class for this, but that class is `private[scala]` in 2.10. We need to explicitly fix this to
- * `Row` in order to work around a spurious IntelliJ compiler error. This cannot be an abstract
- * class because that leads to compilation errors under Scala 2.11.
- */
-class AbstractScalaRowIterator[T] extends Iterator[T] {
- override def hasNext: Boolean = throw new NotImplementedError
-
- override def next(): T = throw new NotImplementedError
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala
index 02c8318..032d6b5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BinaryType.scala
@@ -21,7 +21,6 @@ import scala.math.Ordering
import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.InterfaceStability
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
import org.apache.spark.sql.catalyst.util.TypeUtils
@@ -37,7 +36,7 @@ class BinaryType private() extends AtomicType {
private[sql] type InternalType = Array[Byte]
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
private[sql] val ordering = new Ordering[InternalType] {
def compare(x: Array[Byte], y: Array[Byte]): Int = {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala
index cee78f4..63f354d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/BooleanType.scala
@@ -21,7 +21,6 @@ import scala.math.Ordering
import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.InterfaceStability
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
/**
@@ -35,7 +34,7 @@ class BooleanType private() extends AtomicType {
// this type. Otherwise, the companion object would be of type "BooleanType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
private[sql] type InternalType = Boolean
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
private[sql] val ordering = implicitly[Ordering[InternalType]]
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala
index b1dd5ed..5854c3f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ByteType.scala
@@ -21,7 +21,6 @@ import scala.math.{Integral, Numeric, Ordering}
import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.InterfaceStability
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
/**
* The data type representing `Byte` values. Please use the singleton `DataTypes.ByteType`.
@@ -34,7 +33,7 @@ class ByteType private() extends IntegralType {
// this type. Otherwise, the companion object would be of type "ByteType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
private[sql] type InternalType = Byte
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
private[sql] val numeric = implicitly[Numeric[Byte]]
private[sql] val integral = implicitly[Integral[Byte]]
private[sql] val ordering = implicitly[Ordering[InternalType]]
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala
index 0c0574b..9e70dd4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DateType.scala
@@ -21,8 +21,6 @@ import scala.math.Ordering
import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.InterfaceStability
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
-
/**
* A date type, supporting "0001-01-01" through "9999-12-31".
@@ -40,7 +38,7 @@ class DateType private() extends AtomicType {
// Defined with a private constructor so the companion object is the only possible instantiation.
private[sql] type InternalType = Int
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
private[sql] val ordering = implicitly[Ordering[InternalType]]
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
index 5c4bc5e..6e050c1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
@@ -23,7 +23,6 @@ import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.InterfaceStability
import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
import org.apache.spark.sql.catalyst.expressions.Expression
@@ -57,7 +56,7 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType {
def this() = this(10)
private[sql] type InternalType = Decimal
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
private[sql] val numeric = Decimal.DecimalIsFractional
private[sql] val fractional = Decimal.DecimalIsFractional
private[sql] val ordering = Decimal.DecimalIsFractional
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
index 400f7ae..a5c79ff 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
@@ -22,7 +22,6 @@ import scala.math.Numeric.DoubleAsIfIntegral
import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.InterfaceStability
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
import org.apache.spark.util.Utils
/**
@@ -36,7 +35,7 @@ class DoubleType private() extends FractionalType {
// this type. Otherwise, the companion object would be of type "DoubleType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
private[sql] type InternalType = Double
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
private[sql] val numeric = implicitly[Numeric[Double]]
private[sql] val fractional = implicitly[Fractional[Double]]
private[sql] val ordering = new Ordering[Double] {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
index b9812b2..352147e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
@@ -22,7 +22,6 @@ import scala.math.Numeric.FloatAsIfIntegral
import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.InterfaceStability
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
import org.apache.spark.util.Utils
/**
@@ -36,7 +35,7 @@ class FloatType private() extends FractionalType {
// this type. Otherwise, the companion object would be of type "FloatType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
private[sql] type InternalType = Float
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
private[sql] val numeric = implicitly[Numeric[Float]]
private[sql] val fractional = implicitly[Fractional[Float]]
private[sql] val ordering = new Ordering[Float] {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala
index b319eb7..e0bca93 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.types
import scala.math.Ordering
import scala.reflect.runtime.universe.typeTag
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
import org.apache.spark.unsafe.types.UTF8String
/**
@@ -32,9 +31,7 @@ sealed abstract class HiveStringType extends AtomicType {
private[sql] val ordering = implicitly[Ordering[InternalType]]
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized {
- typeTag[InternalType]
- }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
override def defaultSize: Int = length
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala
index dca612e..a85e372 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/IntegerType.scala
@@ -21,8 +21,6 @@ import scala.math.{Integral, Numeric, Ordering}
import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.InterfaceStability
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
-
/**
* The data type representing `Int` values. Please use the singleton `DataTypes.IntegerType`.
@@ -35,7 +33,7 @@ class IntegerType private() extends IntegralType {
// this type. Otherwise, the companion object would be of type "IntegerType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
private[sql] type InternalType = Int
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
private[sql] val numeric = implicitly[Numeric[Int]]
private[sql] val integral = implicitly[Integral[Int]]
private[sql] val ordering = implicitly[Ordering[InternalType]]
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala
index 396c335..0997028 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/LongType.scala
@@ -21,7 +21,6 @@ import scala.math.{Integral, Numeric, Ordering}
import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.InterfaceStability
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
/**
* The data type representing `Long` values. Please use the singleton `DataTypes.LongType`.
@@ -34,7 +33,7 @@ class LongType private() extends IntegralType {
// this type. Otherwise, the companion object would be of type "LongType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
private[sql] type InternalType = Long
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
private[sql] val numeric = implicitly[Numeric[Long]]
private[sql] val integral = implicitly[Integral[Long]]
private[sql] val ordering = implicitly[Ordering[InternalType]]
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala
index 1410d5b..ee655c3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/ShortType.scala
@@ -21,7 +21,6 @@ import scala.math.{Integral, Numeric, Ordering}
import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.InterfaceStability
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
/**
* The data type representing `Short` values. Please use the singleton `DataTypes.ShortType`.
@@ -34,7 +33,7 @@ class ShortType private() extends IntegralType {
// this type. Otherwise, the companion object would be of type "ShortType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
private[sql] type InternalType = Short
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
private[sql] val numeric = implicitly[Numeric[Short]]
private[sql] val integral = implicitly[Integral[Short]]
private[sql] val ordering = implicitly[Ordering[InternalType]]
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala
index d1c0da3..59b124c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StringType.scala
@@ -21,7 +21,6 @@ import scala.math.Ordering
import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.InterfaceStability
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
import org.apache.spark.unsafe.types.UTF8String
/**
@@ -35,7 +34,7 @@ class StringType private() extends AtomicType {
// this type. Otherwise, the companion object would be of type "StringType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
private[sql] type InternalType = UTF8String
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
private[sql] val ordering = implicitly[Ordering[InternalType]]
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala
index 2875995..fdb91e0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/TimestampType.scala
@@ -21,8 +21,6 @@ import scala.math.Ordering
import scala.reflect.runtime.universe.typeTag
import org.apache.spark.annotation.InterfaceStability
-import org.apache.spark.sql.catalyst.ScalaReflectionLock
-
/**
* The data type representing `java.sql.Timestamp` values.
@@ -37,7 +35,7 @@ class TimestampType private() extends AtomicType {
// Defined with a private constructor so the companion object is the only possible instantiation.
private[sql] type InternalType = Long
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ @transient private[sql] lazy val tag = typeTag[InternalType]
private[sql] val ordering = implicitly[Ordering[InternalType]]
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinOptimizationSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinOptimizationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinOptimizationSuite.scala
index 2f30a78..97733a75 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinOptimizationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinOptimizationSuite.scala
@@ -70,27 +70,27 @@ class JoinOptimizationSuite extends PlanTest {
testExtract(x, None)
testExtract(x.where("x.b".attr === 1), None)
- testExtract(x.join(y), Some(Seq(x, y), Seq()))
+ testExtract(x.join(y), Some((Seq(x, y), Seq())))
testExtract(x.join(y, condition = Some("x.b".attr === "y.d".attr)),
- Some(Seq(x, y), Seq("x.b".attr === "y.d".attr)))
+ Some((Seq(x, y), Seq("x.b".attr === "y.d".attr))))
testExtract(x.join(y).where("x.b".attr === "y.d".attr),
- Some(Seq(x, y), Seq("x.b".attr === "y.d".attr)))
- testExtract(x.join(y).join(z), Some(Seq(x, y, z), Seq()))
+ Some((Seq(x, y), Seq("x.b".attr === "y.d".attr))))
+ testExtract(x.join(y).join(z), Some((Seq(x, y, z), Seq())))
testExtract(x.join(y).where("x.b".attr === "y.d".attr).join(z),
- Some(Seq(x, y, z), Seq("x.b".attr === "y.d".attr)))
- testExtract(x.join(y).join(x.join(z)), Some(Seq(x, y, x.join(z)), Seq()))
+ Some((Seq(x, y, z), Seq("x.b".attr === "y.d".attr))))
+ testExtract(x.join(y).join(x.join(z)), Some((Seq(x, y, x.join(z)), Seq())))
testExtract(x.join(y).join(x.join(z)).where("x.b".attr === "y.d".attr),
- Some(Seq(x, y, x.join(z)), Seq("x.b".attr === "y.d".attr)))
+ Some((Seq(x, y, x.join(z)), Seq("x.b".attr === "y.d".attr))))
- testExtractCheckCross(x.join(y, Cross), Some(Seq((x, Cross), (y, Cross)), Seq()))
+ testExtractCheckCross(x.join(y, Cross), Some((Seq((x, Cross), (y, Cross)), Seq())))
testExtractCheckCross(x.join(y, Cross).join(z, Cross),
- Some(Seq((x, Cross), (y, Cross), (z, Cross)), Seq()))
+ Some((Seq((x, Cross), (y, Cross), (z, Cross)), Seq())))
testExtractCheckCross(x.join(y, Cross, Some("x.b".attr === "y.d".attr)).join(z, Cross),
- Some(Seq((x, Cross), (y, Cross), (z, Cross)), Seq("x.b".attr === "y.d".attr)))
+ Some((Seq((x, Cross), (y, Cross), (z, Cross)), Seq("x.b".attr === "y.d".attr))))
testExtractCheckCross(x.join(y, Inner, Some("x.b".attr === "y.d".attr)).join(z, Cross),
- Some(Seq((x, Inner), (y, Inner), (z, Cross)), Seq("x.b".attr === "y.d".attr)))
+ Some((Seq((x, Inner), (y, Inner), (z, Cross)), Seq("x.b".attr === "y.d".attr))))
testExtractCheckCross(x.join(y, Cross, Some("x.b".attr === "y.d".attr)).join(z, Inner),
- Some(Seq((x, Cross), (y, Cross), (z, Inner)), Seq("x.b".attr === "y.d".attr)))
+ Some((Seq((x, Cross), (y, Cross), (z, Inner)), Seq("x.b".attr === "y.d".attr))))
}
test("reorder inner joins") {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeCodegenSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeCodegenSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeCodegenSuite.scala
index b71067c..2abf9fe 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeCodegenSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeCodegenSuite.scala
@@ -43,8 +43,8 @@ class OptimizeCodegenSuite extends PlanTest {
CaseWhen(Seq((TrueLiteral, Literal(1))), Literal(2)).toCodegen())
assertEquivalent(
- CaseWhen(List.fill(100)(TrueLiteral, Literal(1)), Literal(2)),
- CaseWhen(List.fill(100)(TrueLiteral, Literal(1)), Literal(2)))
+ CaseWhen(List.fill(100)((TrueLiteral, Literal(1))), Literal(2)),
+ CaseWhen(List.fill(100)((TrueLiteral, Literal(1))), Literal(2)))
}
test("Nested CaseWhen Codegen.") {
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index bd1669b..b546ccc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -505,7 +505,7 @@ class Column(val expr: Expression) extends Logging {
*/
def when(condition: Column, value: Any): Column = this.expr match {
case CaseWhen(branches, None) =>
- withExpr { CaseWhen(branches :+ (condition.expr, lit(value).expr)) }
+ withExpr { CaseWhen(branches :+ ((condition.expr, lit(value).expr))) }
case CaseWhen(branches, Some(_)) =>
throw new IllegalArgumentException(
"when() cannot be applied once otherwise() is applied")
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
index 5cfad91..02f45ab 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
@@ -133,20 +133,20 @@ case class OptimizeMetadataOnlyQuery(catalog: SessionCatalog) extends Rule[Logic
case l @ LogicalRelation(fsRelation: HadoopFsRelation, _, _)
if fsRelation.partitionSchema.nonEmpty =>
val partAttrs = getPartitionAttrs(fsRelation.partitionSchema.map(_.name), l)
- Some(AttributeSet(partAttrs), l)
+ Some((AttributeSet(partAttrs), l))
case relation: CatalogRelation if relation.tableMeta.partitionColumnNames.nonEmpty =>
val partAttrs = getPartitionAttrs(relation.tableMeta.partitionColumnNames, relation)
- Some(AttributeSet(partAttrs), relation)
+ Some((AttributeSet(partAttrs), relation))
case p @ Project(projectList, child) if projectList.forall(_.deterministic) =>
unapply(child).flatMap { case (partAttrs, relation) =>
- if (p.references.subsetOf(partAttrs)) Some(p.outputSet, relation) else None
+ if (p.references.subsetOf(partAttrs)) Some((p.outputSet, relation)) else None
}
case f @ Filter(condition, child) if condition.deterministic =>
unapply(child).flatMap { case (partAttrs, relation) =>
- if (f.references.subsetOf(partAttrs)) Some(partAttrs, relation) else None
+ if (f.references.subsetOf(partAttrs)) Some((partAttrs, relation)) else None
}
case _ => None
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 9533144..b56fbd4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -155,7 +155,7 @@ class QueryExecution(val sparkSession: SparkSession, val logical: LogicalPlan) {
def toHiveStructString(a: (Any, DataType)): String = a match {
case (struct: Row, StructType(fields)) =>
struct.toSeq.zip(fields).map {
- case (v, t) => s""""${t.name}":${toHiveStructString(v, t.dataType)}"""
+ case (v, t) => s""""${t.name}":${toHiveStructString((v, t.dataType))}"""
}.mkString("{", ",", "}")
case (seq: Seq[_], ArrayType(typ, _)) =>
seq.map(v => (v, typ)).map(toHiveStructString).mkString("[", ",", "]")
@@ -173,7 +173,7 @@ class QueryExecution(val sparkSession: SparkSession, val logical: LogicalPlan) {
a match {
case (struct: Row, StructType(fields)) =>
struct.toSeq.zip(fields).map {
- case (v, t) => s""""${t.name}":${toHiveStructString(v, t.dataType)}"""
+ case (v, t) => s""""${t.name}":${toHiveStructString((v, t.dataType))}"""
}.mkString("{", ",", "}")
case (seq: Seq[_], ArrayType(typ, _)) =>
seq.map(v => (v, typ)).map(toHiveStructString).mkString("[", ",", "]")
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
index 69b4b7b..2b3c5f0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/ExtractPythonUDFs.scala
@@ -133,7 +133,7 @@ object ExtractPythonUDFs extends Rule[SparkPlan] with PredicateHelper {
val validUdfs = udfs.filter { udf =>
// Check to make sure that the UDF can be evaluated with only the input of this child.
udf.references.subsetOf(child.outputSet)
- }.toArray // Turn it into an array since iterators cannot be serialized in Scala 2.10
+ }.toArray
if (validUdfs.nonEmpty) {
val resultAttrs = udfs.zipWithIndex.map { case (u, i) =>
AttributeReference(s"pythonUDF$i", u.dataType)()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[4/5] spark git commit: [SPARK-19810][BUILD][CORE] Remove support for
Scala 2.10
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala
deleted file mode 100644
index b7237a6..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoop.scala
+++ /dev/null
@@ -1,1145 +0,0 @@
-// scalastyle:off
-
-/* NSC -- new Scala compiler
- * Copyright 2005-2013 LAMP/EPFL
- * @author Alexander Spoon
- */
-
-package org.apache.spark.repl
-
-
-import java.net.URL
-
-import scala.reflect.io.AbstractFile
-import scala.tools.nsc._
-import scala.tools.nsc.backend.JavaPlatform
-import scala.tools.nsc.interpreter._
-import scala.tools.nsc.interpreter.{Results => IR}
-import Predef.{println => _, _}
-import java.io.{BufferedReader, FileReader}
-import java.net.URI
-import java.util.concurrent.locks.ReentrantLock
-import scala.sys.process.Process
-import scala.tools.nsc.interpreter.session._
-import scala.util.Properties.{jdkHome, javaVersion}
-import scala.tools.util.{Javap}
-import scala.annotation.tailrec
-import scala.collection.mutable.ListBuffer
-import scala.concurrent.ops
-import scala.tools.nsc.util._
-import scala.tools.nsc.interpreter._
-import scala.tools.nsc.io.{File, Directory}
-import scala.reflect.NameTransformer._
-import scala.tools.nsc.util.ScalaClassLoader._
-import scala.tools.util._
-import scala.language.{implicitConversions, existentials, postfixOps}
-import scala.reflect.{ClassTag, classTag}
-import scala.tools.reflect.StdRuntimeTags._
-
-import java.lang.{Class => jClass}
-import scala.reflect.api.{Mirror, TypeCreator, Universe => ApiUniverse}
-
-import org.apache.spark.SparkConf
-import org.apache.spark.SparkContext
-import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.util.Utils
-
-/** The Scala interactive shell. It provides a read-eval-print loop
- * around the Interpreter class.
- * After instantiation, clients should call the main() method.
- *
- * If no in0 is specified, then input will come from the console, and
- * the class will attempt to provide input editing feature such as
- * input history.
- *
- * @author Moez A. Abdel-Gawad
- * @author Lex Spoon
- * @version 1.2
- */
-@DeveloperApi
-class SparkILoop(
- private val in0: Option[BufferedReader],
- protected val out: JPrintWriter,
- val master: Option[String]
-) extends AnyRef with LoopCommands with SparkILoopInit with Logging {
- def this(in0: BufferedReader, out: JPrintWriter, master: String) = this(Some(in0), out, Some(master))
- def this(in0: BufferedReader, out: JPrintWriter) = this(Some(in0), out, None)
- def this() = this(None, new JPrintWriter(Console.out, true), None)
-
- private var in: InteractiveReader = _ // the input stream from which commands come
-
- // NOTE: Exposed in package for testing
- private[repl] var settings: Settings = _
-
- private[repl] var intp: SparkIMain = _
-
- @deprecated("Use `intp` instead.", "2.9.0") def interpreter = intp
- @deprecated("Use `intp` instead.", "2.9.0") def interpreter_= (i: SparkIMain): Unit = intp = i
-
- /** Having inherited the difficult "var-ness" of the repl instance,
- * I'm trying to work around it by moving operations into a class from
- * which it will appear a stable prefix.
- */
- private def onIntp[T](f: SparkIMain => T): T = f(intp)
-
- class IMainOps[T <: SparkIMain](val intp: T) {
- import intp._
- import global._
-
- def printAfterTyper(msg: => String) =
- intp.reporter printMessage afterTyper(msg)
-
- /** Strip NullaryMethodType artifacts. */
- private def replInfo(sym: Symbol) = {
- sym.info match {
- case NullaryMethodType(restpe) if sym.isAccessor => restpe
- case info => info
- }
- }
- def echoTypeStructure(sym: Symbol) =
- printAfterTyper("" + deconstruct.show(replInfo(sym)))
-
- def echoTypeSignature(sym: Symbol, verbose: Boolean) = {
- if (verbose) SparkILoop.this.echo("// Type signature")
- printAfterTyper("" + replInfo(sym))
-
- if (verbose) {
- SparkILoop.this.echo("\n// Internal Type structure")
- echoTypeStructure(sym)
- }
- }
- }
- implicit def stabilizeIMain(intp: SparkIMain) = new IMainOps[intp.type](intp)
-
- /** TODO -
- * -n normalize
- * -l label with case class parameter names
- * -c complete - leave nothing out
- */
- private def typeCommandInternal(expr: String, verbose: Boolean): Result = {
- onIntp { intp =>
- val sym = intp.symbolOfLine(expr)
- if (sym.exists) intp.echoTypeSignature(sym, verbose)
- else ""
- }
- }
-
- // NOTE: Must be public for visibility
- @DeveloperApi
- var sparkContext: SparkContext = _
-
- override def echoCommandMessage(msg: String) {
- intp.reporter printMessage msg
- }
-
- // def isAsync = !settings.Yreplsync.value
- private[repl] def isAsync = false
- // lazy val power = new Power(intp, new StdReplVals(this))(tagOfStdReplVals, classTag[StdReplVals])
- private def history = in.history
-
- /** The context class loader at the time this object was created */
- protected val originalClassLoader = Utils.getContextOrSparkClassLoader
-
- // classpath entries added via :cp
- private var addedClasspath: String = ""
-
- /** A reverse list of commands to replay if the user requests a :replay */
- private var replayCommandStack: List[String] = Nil
-
- /** A list of commands to replay if the user requests a :replay */
- private def replayCommands = replayCommandStack.reverse
-
- /** Record a command for replay should the user request a :replay */
- private def addReplay(cmd: String) = replayCommandStack ::= cmd
-
- private def savingReplayStack[T](body: => T): T = {
- val saved = replayCommandStack
- try body
- finally replayCommandStack = saved
- }
- private def savingReader[T](body: => T): T = {
- val saved = in
- try body
- finally in = saved
- }
-
-
- private def sparkCleanUp() {
- echo("Stopping spark context.")
- intp.beQuietDuring {
- command("sc.stop()")
- }
- }
- /** Close the interpreter and set the var to null. */
- private def closeInterpreter() {
- if (intp ne null) {
- sparkCleanUp()
- intp.close()
- intp = null
- }
- }
-
- class SparkILoopInterpreter extends SparkIMain(settings, out) {
- outer =>
-
- override private[repl] lazy val formatting = new Formatting {
- def prompt = SparkILoop.this.prompt
- }
- override protected def parentClassLoader = SparkHelper.explicitParentLoader(settings).getOrElse(classOf[SparkILoop].getClassLoader)
- }
-
- /**
- * Constructs a new interpreter.
- */
- protected def createInterpreter() {
- require(settings != null)
-
- if (addedClasspath != "") settings.classpath.append(addedClasspath)
- val addedJars =
- if (Utils.isWindows) {
- // Strip any URI scheme prefix so we can add the correct path to the classpath
- // e.g. file:/C:/my/path.jar -> C:/my/path.jar
- getAddedJars().map { jar => new URI(jar).getPath.stripPrefix("/") }
- } else {
- // We need new URI(jar).getPath here for the case that `jar` includes encoded white space (%20).
- getAddedJars().map { jar => new URI(jar).getPath }
- }
- // work around for Scala bug
- val totalClassPath = addedJars.foldLeft(
- settings.classpath.value)((l, r) => ClassPath.join(l, r))
- this.settings.classpath.value = totalClassPath
-
- intp = new SparkILoopInterpreter
- }
-
- /** print a friendly help message */
- private def helpCommand(line: String): Result = {
- if (line == "") helpSummary()
- else uniqueCommand(line) match {
- case Some(lc) => echo("\n" + lc.longHelp)
- case _ => ambiguousError(line)
- }
- }
- private def helpSummary() = {
- val usageWidth = commands map (_.usageMsg.length) max
- val formatStr = "%-" + usageWidth + "s %s %s"
-
- echo("All commands can be abbreviated, e.g. :he instead of :help.")
- echo("Those marked with a * have more detailed help, e.g. :help imports.\n")
-
- commands foreach { cmd =>
- val star = if (cmd.hasLongHelp) "*" else " "
- echo(formatStr.format(cmd.usageMsg, star, cmd.help))
- }
- }
- private def ambiguousError(cmd: String): Result = {
- matchingCommands(cmd) match {
- case Nil => echo(cmd + ": no such command. Type :help for help.")
- case xs => echo(cmd + " is ambiguous: did you mean " + xs.map(":" + _.name).mkString(" or ") + "?")
- }
- Result(true, None)
- }
- private def matchingCommands(cmd: String) = commands filter (_.name startsWith cmd)
- private def uniqueCommand(cmd: String): Option[LoopCommand] = {
- // this lets us add commands willy-nilly and only requires enough command to disambiguate
- matchingCommands(cmd) match {
- case List(x) => Some(x)
- // exact match OK even if otherwise appears ambiguous
- case xs => xs find (_.name == cmd)
- }
- }
- private var fallbackMode = false
-
- private def toggleFallbackMode() {
- val old = fallbackMode
- fallbackMode = !old
- System.setProperty("spark.repl.fallback", fallbackMode.toString)
- echo(s"""
- |Switched ${if (old) "off" else "on"} fallback mode without restarting.
- | If you have defined classes in the repl, it would
- |be good to redefine them incase you plan to use them. If you still run
- |into issues it would be good to restart the repl and turn on `:fallback`
- |mode as first command.
- """.stripMargin)
- }
-
- /** Show the history */
- private lazy val historyCommand = new LoopCommand("history", "show the history (optional num is commands to show)") {
- override def usage = "[num]"
- def defaultLines = 20
-
- def apply(line: String): Result = {
- if (history eq NoHistory)
- return "No history available."
-
- val xs = words(line)
- val current = history.index
- val count = try xs.head.toInt catch { case _: Exception => defaultLines }
- val lines = history.asStrings takeRight count
- val offset = current - lines.size + 1
-
- for ((line, index) <- lines.zipWithIndex)
- echo("%3d %s".format(index + offset, line))
- }
- }
-
- // When you know you are most likely breaking into the middle
- // of a line being typed. This softens the blow.
- private[repl] def echoAndRefresh(msg: String) = {
- echo("\n" + msg)
- in.redrawLine()
- }
- private[repl] def echo(msg: String) = {
- out println msg
- out.flush()
- }
- private def echoNoNL(msg: String) = {
- out print msg
- out.flush()
- }
-
- /** Search the history */
- private def searchHistory(_cmdline: String) {
- val cmdline = _cmdline.toLowerCase
- val offset = history.index - history.size + 1
-
- for ((line, index) <- history.asStrings.zipWithIndex ; if line.toLowerCase contains cmdline)
- echo("%d %s".format(index + offset, line))
- }
-
- private var currentPrompt = Properties.shellPromptString
-
- /**
- * Sets the prompt string used by the REPL.
- *
- * @param prompt The new prompt string
- */
- @DeveloperApi
- def setPrompt(prompt: String) = currentPrompt = prompt
-
- /**
- * Represents the current prompt string used by the REPL.
- *
- * @return The current prompt string
- */
- @DeveloperApi
- def prompt = currentPrompt
-
- import LoopCommand.{ cmd, nullary }
-
- /** Standard commands */
- private lazy val standardCommands = List(
- cmd("cp", "<path>", "add a jar or directory to the classpath", addClasspath),
- cmd("help", "[command]", "print this summary or command-specific help", helpCommand),
- historyCommand,
- cmd("h?", "<string>", "search the history", searchHistory),
- cmd("imports", "[name name ...]", "show import history, identifying sources of names", importsCommand),
- cmd("implicits", "[-v]", "show the implicits in scope", implicitsCommand),
- cmd("javap", "<path|class>", "disassemble a file or class name", javapCommand),
- cmd("load", "<path>", "load and interpret a Scala file", loadCommand),
- nullary("paste", "enter paste mode: all input up to ctrl-D compiled together", pasteCommand),
-// nullary("power", "enable power user mode", powerCmd),
- nullary("quit", "exit the repl", () => Result(false, None)),
- nullary("replay", "reset execution and replay all previous commands", replay),
- nullary("reset", "reset the repl to its initial state, forgetting all session entries", resetCommand),
- shCommand,
- nullary("silent", "disable/enable automatic printing of results", verbosity),
- nullary("fallback", """
- |disable/enable advanced repl changes, these fix some issues but may introduce others.
- |This mode will be removed once these fixes stablize""".stripMargin, toggleFallbackMode),
- cmd("type", "[-v] <expr>", "display the type of an expression without evaluating it", typeCommand),
- nullary("warnings", "show the suppressed warnings from the most recent line which had any", warningsCommand)
- )
-
- /** Power user commands */
- private lazy val powerCommands: List[LoopCommand] = List(
- // cmd("phase", "<phase>", "set the implicit phase for power commands", phaseCommand)
- )
-
- // private def dumpCommand(): Result = {
- // echo("" + power)
- // history.asStrings takeRight 30 foreach echo
- // in.redrawLine()
- // }
- // private def valsCommand(): Result = power.valsDescription
-
- private val typeTransforms = List(
- "scala.collection.immutable." -> "immutable.",
- "scala.collection.mutable." -> "mutable.",
- "scala.collection.generic." -> "generic.",
- "java.lang." -> "jl.",
- "scala.runtime." -> "runtime."
- )
-
- private def importsCommand(line: String): Result = {
- val tokens = words(line)
- val handlers = intp.languageWildcardHandlers ++ intp.importHandlers
- val isVerbose = tokens contains "-v"
-
- handlers.filterNot(_.importedSymbols.isEmpty).zipWithIndex foreach {
- case (handler, idx) =>
- val (types, terms) = handler.importedSymbols partition (_.name.isTypeName)
- val imps = handler.implicitSymbols
- val found = tokens filter (handler importsSymbolNamed _)
- val typeMsg = if (types.isEmpty) "" else types.size + " types"
- val termMsg = if (terms.isEmpty) "" else terms.size + " terms"
- val implicitMsg = if (imps.isEmpty) "" else imps.size + " are implicit"
- val foundMsg = if (found.isEmpty) "" else found.mkString(" // imports: ", ", ", "")
- val statsMsg = List(typeMsg, termMsg, implicitMsg) filterNot (_ == "") mkString ("(", ", ", ")")
-
- intp.reporter.printMessage("%2d) %-30s %s%s".format(
- idx + 1,
- handler.importString,
- statsMsg,
- foundMsg
- ))
- }
- }
-
- private def implicitsCommand(line: String): Result = onIntp { intp =>
- import intp._
- import global._
-
- def p(x: Any) = intp.reporter.printMessage("" + x)
-
- // If an argument is given, only show a source with that
- // in its name somewhere.
- val args = line split "\\s+"
- val filtered = intp.implicitSymbolsBySource filter {
- case (source, syms) =>
- (args contains "-v") || {
- if (line == "") (source.fullName.toString != "scala.Predef")
- else (args exists (source.name.toString contains _))
- }
- }
-
- if (filtered.isEmpty)
- return "No implicits have been imported other than those in Predef."
-
- filtered foreach {
- case (source, syms) =>
- p("/* " + syms.size + " implicit members imported from " + source.fullName + " */")
-
- // This groups the members by where the symbol is defined
- val byOwner = syms groupBy (_.owner)
- val sortedOwners = byOwner.toList sortBy { case (owner, _) => afterTyper(source.info.baseClasses indexOf owner) }
-
- sortedOwners foreach {
- case (owner, members) =>
- // Within each owner, we cluster results based on the final result type
- // if there are more than a couple, and sort each cluster based on name.
- // This is really just trying to make the 100 or so implicits imported
- // by default into something readable.
- val memberGroups: List[List[Symbol]] = {
- val groups = members groupBy (_.tpe.finalResultType) toList
- val (big, small) = groups partition (_._2.size > 3)
- val xss = (
- (big sortBy (_._1.toString) map (_._2)) :+
- (small flatMap (_._2))
- )
-
- xss map (xs => xs sortBy (_.name.toString))
- }
-
- val ownerMessage = if (owner == source) " defined in " else " inherited from "
- p(" /* " + members.size + ownerMessage + owner.fullName + " */")
-
- memberGroups foreach { group =>
- group foreach (s => p(" " + intp.symbolDefString(s)))
- p("")
- }
- }
- p("")
- }
- }
-
- private def findToolsJar() = {
- val jdkPath = Directory(jdkHome)
- val jar = jdkPath / "lib" / "tools.jar" toFile;
-
- if (jar isFile)
- Some(jar)
- else if (jdkPath.isDirectory)
- jdkPath.deepFiles find (_.name == "tools.jar")
- else None
- }
- private def addToolsJarToLoader() = {
- val cl = findToolsJar match {
- case Some(tools) => ScalaClassLoader.fromURLs(Seq(tools.toURL), intp.classLoader)
- case _ => intp.classLoader
- }
- if (Javap.isAvailable(cl)) {
- logDebug(":javap available.")
- cl
- }
- else {
- logDebug(":javap unavailable: no tools.jar at " + jdkHome)
- intp.classLoader
- }
- }
-
- private def newJavap() = new JavapClass(addToolsJarToLoader(), new SparkIMain.ReplStrippingWriter(intp)) {
- override def tryClass(path: String): Array[Byte] = {
- val hd :: rest = path split '.' toList;
- // If there are dots in the name, the first segment is the
- // key to finding it.
- if (rest.nonEmpty) {
- intp optFlatName hd match {
- case Some(flat) =>
- val clazz = flat :: rest mkString NAME_JOIN_STRING
- val bytes = super.tryClass(clazz)
- if (bytes.nonEmpty) bytes
- else super.tryClass(clazz + MODULE_SUFFIX_STRING)
- case _ => super.tryClass(path)
- }
- }
- else {
- // Look for Foo first, then Foo$, but if Foo$ is given explicitly,
- // we have to drop the $ to find object Foo, then tack it back onto
- // the end of the flattened name.
- def className = intp flatName path
- def moduleName = (intp flatName path.stripSuffix(MODULE_SUFFIX_STRING)) + MODULE_SUFFIX_STRING
-
- val bytes = super.tryClass(className)
- if (bytes.nonEmpty) bytes
- else super.tryClass(moduleName)
- }
- }
- }
- // private lazy val javap = substituteAndLog[Javap]("javap", NoJavap)(newJavap())
- private lazy val javap =
- try newJavap()
- catch { case _: Exception => null }
-
- // Still todo: modules.
- private def typeCommand(line0: String): Result = {
- line0.trim match {
- case "" => ":type [-v] <expression>"
- case s if s startsWith "-v " => typeCommandInternal(s stripPrefix "-v " trim, true)
- case s => typeCommandInternal(s, false)
- }
- }
-
- private def warningsCommand(): Result = {
- if (intp.lastWarnings.isEmpty)
- "Can't find any cached warnings."
- else
- intp.lastWarnings foreach { case (pos, msg) => intp.reporter.warning(pos, msg) }
- }
-
- private def javapCommand(line: String): Result = {
- if (javap == null)
- ":javap unavailable, no tools.jar at %s. Set JDK_HOME.".format(jdkHome)
- else if (javaVersion startsWith "1.7")
- ":javap not yet working with java 1.7"
- else if (line == "")
- ":javap [-lcsvp] [path1 path2 ...]"
- else
- javap(words(line)) foreach { res =>
- if (res.isError) return "Failed: " + res.value
- else res.show()
- }
- }
-
- private def wrapCommand(line: String): Result = {
- def failMsg = "Argument to :wrap must be the name of a method with signature [T](=> T): T"
- onIntp { intp =>
- import intp._
- import global._
-
- words(line) match {
- case Nil =>
- intp.executionWrapper match {
- case "" => "No execution wrapper is set."
- case s => "Current execution wrapper: " + s
- }
- case "clear" :: Nil =>
- intp.executionWrapper match {
- case "" => "No execution wrapper is set."
- case s => intp.clearExecutionWrapper() ; "Cleared execution wrapper."
- }
- case wrapper :: Nil =>
- intp.typeOfExpression(wrapper) match {
- case PolyType(List(targ), MethodType(List(arg), restpe)) =>
- intp setExecutionWrapper intp.pathToTerm(wrapper)
- "Set wrapper to '" + wrapper + "'"
- case tp =>
- failMsg + "\nFound: <unknown>"
- }
- case _ => failMsg
- }
- }
- }
-
- private def pathToPhaseWrapper = intp.pathToTerm("$r") + ".phased.atCurrent"
- // private def phaseCommand(name: String): Result = {
- // val phased: Phased = power.phased
- // import phased.NoPhaseName
-
- // if (name == "clear") {
- // phased.set(NoPhaseName)
- // intp.clearExecutionWrapper()
- // "Cleared active phase."
- // }
- // else if (name == "") phased.get match {
- // case NoPhaseName => "Usage: :phase <expr> (e.g. typer, erasure.next, erasure+3)"
- // case ph => "Active phase is '%s'. (To clear, :phase clear)".format(phased.get)
- // }
- // else {
- // val what = phased.parse(name)
- // if (what.isEmpty || !phased.set(what))
- // "'" + name + "' does not appear to represent a valid phase."
- // else {
- // intp.setExecutionWrapper(pathToPhaseWrapper)
- // val activeMessage =
- // if (what.toString.length == name.length) "" + what
- // else "%s (%s)".format(what, name)
-
- // "Active phase is now: " + activeMessage
- // }
- // }
- // }
-
- /**
- * Provides a list of available commands.
- *
- * @return The list of commands
- */
- @DeveloperApi
- def commands: List[LoopCommand] = standardCommands /*++ (
- if (isReplPower) powerCommands else Nil
- )*/
-
- private val replayQuestionMessage =
- """|That entry seems to have slain the compiler. Shall I replay
- |your session? I can re-run each line except the last one.
- |[y/n]
- """.trim.stripMargin
-
- private def crashRecovery(ex: Throwable): Boolean = {
- echo(ex.toString)
- ex match {
- case _: NoSuchMethodError | _: NoClassDefFoundError =>
- echo("\nUnrecoverable error.")
- throw ex
- case _ =>
- def fn(): Boolean =
- try in.readYesOrNo(replayQuestionMessage, { echo("\nYou must enter y or n.") ; fn() })
- catch { case _: RuntimeException => false }
-
- if (fn()) replay()
- else echo("\nAbandoning crashed session.")
- }
- true
- }
-
- /** The main read-eval-print loop for the repl. It calls
- * command() for each line of input, and stops when
- * command() returns false.
- */
- private def loop() {
- def readOneLine() = {
- out.flush()
- in readLine prompt
- }
- // return false if repl should exit
- def processLine(line: String): Boolean = {
- if (isAsync) {
- if (!awaitInitialized()) return false
- runThunks()
- }
- if (line eq null) false // assume null means EOF
- else command(line) match {
- case Result(false, _) => false
- case Result(_, Some(finalLine)) => addReplay(finalLine) ; true
- case _ => true
- }
- }
- def innerLoop() {
- val shouldContinue = try {
- processLine(readOneLine())
- } catch {case t: Throwable => crashRecovery(t)}
- if (shouldContinue)
- innerLoop()
- }
- innerLoop()
- }
-
- /** interpret all lines from a specified file */
- private def interpretAllFrom(file: File) {
- savingReader {
- savingReplayStack {
- file applyReader { reader =>
- in = SimpleReader(reader, out, false)
- echo("Loading " + file + "...")
- loop()
- }
- }
- }
- }
-
- /** create a new interpreter and replay the given commands */
- private def replay() {
- reset()
- if (replayCommandStack.isEmpty)
- echo("Nothing to replay.")
- else for (cmd <- replayCommands) {
- echo("Replaying: " + cmd) // flush because maybe cmd will have its own output
- command(cmd)
- echo("")
- }
- }
- private def resetCommand() {
- echo("Resetting repl state.")
- if (replayCommandStack.nonEmpty) {
- echo("Forgetting this session history:\n")
- replayCommands foreach echo
- echo("")
- replayCommandStack = Nil
- }
- if (intp.namedDefinedTerms.nonEmpty)
- echo("Forgetting all expression results and named terms: " + intp.namedDefinedTerms.mkString(", "))
- if (intp.definedTypes.nonEmpty)
- echo("Forgetting defined types: " + intp.definedTypes.mkString(", "))
-
- reset()
- }
-
- private def reset() {
- intp.reset()
- // unleashAndSetPhase()
- }
-
- /** fork a shell and run a command */
- private lazy val shCommand = new LoopCommand("sh", "run a shell command (result is implicitly => List[String])") {
- override def usage = "<command line>"
- def apply(line: String): Result = line match {
- case "" => showUsage()
- case _ =>
- val toRun = classOf[ProcessResult].getName + "(" + string2codeQuoted(line) + ")"
- intp interpret toRun
- ()
- }
- }
-
- private def withFile(filename: String)(action: File => Unit) {
- val f = File(filename)
-
- if (f.exists) action(f)
- else echo("That file does not exist")
- }
-
- private def loadCommand(arg: String) = {
- var shouldReplay: Option[String] = None
- withFile(arg)(f => {
- interpretAllFrom(f)
- shouldReplay = Some(":load " + arg)
- })
- Result(true, shouldReplay)
- }
-
- private def addAllClasspath(args: Seq[String]): Unit = {
- var added = false
- var totalClasspath = ""
- for (arg <- args) {
- val f = File(arg).normalize
- if (f.exists) {
- added = true
- addedClasspath = ClassPath.join(addedClasspath, f.path)
- totalClasspath = ClassPath.join(settings.classpath.value, addedClasspath)
- intp.addUrlsToClassPath(f.toURI.toURL)
- sparkContext.addJar(f.toURI.toURL.getPath)
- }
- }
- }
-
- private def addClasspath(arg: String): Unit = {
- val f = File(arg).normalize
- if (f.exists) {
- addedClasspath = ClassPath.join(addedClasspath, f.path)
- intp.addUrlsToClassPath(f.toURI.toURL)
- sparkContext.addJar(f.toURI.toURL.getPath)
- echo("Added '%s'. Your new classpath is:\n\"%s\"".format(f.path, intp.global.classPath.asClasspathString))
- }
- else echo("The path '" + f + "' doesn't seem to exist.")
- }
-
-
- private def powerCmd(): Result = {
- if (isReplPower) "Already in power mode."
- else enablePowerMode(false)
- }
-
- private[repl] def enablePowerMode(isDuringInit: Boolean) = {
- // replProps.power setValue true
- // unleashAndSetPhase()
- // asyncEcho(isDuringInit, power.banner)
- }
- // private def unleashAndSetPhase() {
-// if (isReplPower) {
-// // power.unleash()
-// // Set the phase to "typer"
-// intp beSilentDuring phaseCommand("typer")
-// }
-// }
-
- private def asyncEcho(async: Boolean, msg: => String) {
- if (async) asyncMessage(msg)
- else echo(msg)
- }
-
- private def verbosity() = {
- // val old = intp.printResults
- // intp.printResults = !old
- // echo("Switched " + (if (old) "off" else "on") + " result printing.")
- }
-
- /**
- * Run one command submitted by the user. Two values are returned:
- * (1) whether to keep running, (2) the line to record for replay,
- * if any.
- */
- private[repl] def command(line: String): Result = {
- if (line startsWith ":") {
- val cmd = line.tail takeWhile (x => !x.isWhitespace)
- uniqueCommand(cmd) match {
- case Some(lc) => lc(line.tail stripPrefix cmd dropWhile (_.isWhitespace))
- case _ => ambiguousError(cmd)
- }
- }
- else if (intp.global == null) Result(false, None) // Notice failure to create compiler
- else Result(true, interpretStartingWith(line))
- }
-
- private def readWhile(cond: String => Boolean) = {
- Iterator continually in.readLine("") takeWhile (x => x != null && cond(x))
- }
-
- private def pasteCommand(): Result = {
- echo("// Entering paste mode (ctrl-D to finish)\n")
- val code = readWhile(_ => true) mkString "\n"
- echo("\n// Exiting paste mode, now interpreting.\n")
- intp interpret code
- ()
- }
-
- private object paste extends Pasted {
- val ContinueString = " | "
- val PromptString = "scala> "
-
- def interpret(line: String): Unit = {
- echo(line.trim)
- intp interpret line
- echo("")
- }
-
- def transcript(start: String) = {
- echo("\n// Detected repl transcript paste: ctrl-D to finish.\n")
- apply(Iterator(start) ++ readWhile(_.trim != PromptString.trim))
- }
- }
- import paste.{ ContinueString, PromptString }
-
- /**
- * Interpret expressions starting with the first line.
- * Read lines until a complete compilation unit is available
- * or until a syntax error has been seen. If a full unit is
- * read, go ahead and interpret it. Return the full string
- * to be recorded for replay, if any.
- */
- private def interpretStartingWith(code: String): Option[String] = {
- // signal completion non-completion input has been received
- in.completion.resetVerbosity()
-
- def reallyInterpret = {
- val reallyResult = intp.interpret(code)
- (reallyResult, reallyResult match {
- case IR.Error => None
- case IR.Success => Some(code)
- case IR.Incomplete =>
- if (in.interactive && code.endsWith("\n\n")) {
- echo("You typed two blank lines. Starting a new command.")
- None
- }
- else in.readLine(ContinueString) match {
- case null =>
- // we know compilation is going to fail since we're at EOF and the
- // parser thinks the input is still incomplete, but since this is
- // a file being read non-interactively we want to fail. So we send
- // it straight to the compiler for the nice error message.
- intp.compileString(code)
- None
-
- case line => interpretStartingWith(code + "\n" + line)
- }
- })
- }
-
- /** Here we place ourselves between the user and the interpreter and examine
- * the input they are ostensibly submitting. We intervene in several cases:
- *
- * 1) If the line starts with "scala> " it is assumed to be an interpreter paste.
- * 2) If the line starts with "." (but not ".." or "./") it is treated as an invocation
- * on the previous result.
- * 3) If the Completion object's execute returns Some(_), we inject that value
- * and avoid the interpreter, as it's likely not valid scala code.
- */
- if (code == "") None
- else if (!paste.running && code.trim.startsWith(PromptString)) {
- paste.transcript(code)
- None
- }
- else if (Completion.looksLikeInvocation(code) && intp.mostRecentVar != "") {
- interpretStartingWith(intp.mostRecentVar + code)
- }
- else if (code.trim startsWith "//") {
- // line comment, do nothing
- None
- }
- else
- reallyInterpret._2
- }
-
- // runs :load `file` on any files passed via -i
- private def loadFiles(settings: Settings) = settings match {
- case settings: SparkRunnerSettings =>
- for (filename <- settings.loadfiles.value) {
- val cmd = ":load " + filename
- command(cmd)
- addReplay(cmd)
- echo("")
- }
- case _ =>
- }
-
- /** Tries to create a JLineReader, falling back to SimpleReader:
- * unless settings or properties are such that it should start
- * with SimpleReader.
- */
- private def chooseReader(settings: Settings): InteractiveReader = {
- if (settings.Xnojline.value || Properties.isEmacsShell)
- SimpleReader()
- else try new SparkJLineReader(
- if (settings.noCompletion.value) NoCompletion
- else new SparkJLineCompletion(intp)
- )
- catch {
- case ex @ (_: Exception | _: NoClassDefFoundError) =>
- echo("Failed to created SparkJLineReader: " + ex + "\nFalling back to SimpleReader.")
- SimpleReader()
- }
- }
-
- private val u: scala.reflect.runtime.universe.type = scala.reflect.runtime.universe
- private val m = u.runtimeMirror(Utils.getSparkClassLoader)
- private def tagOfStaticClass[T: ClassTag]: u.TypeTag[T] =
- u.TypeTag[T](
- m,
- new TypeCreator {
- def apply[U <: ApiUniverse with Singleton](m: Mirror[U]): U # Type =
- m.staticClass(classTag[T].runtimeClass.getName).toTypeConstructor.asInstanceOf[U # Type]
- })
-
- private def process(settings: Settings): Boolean = savingContextLoader {
- this.settings = settings
- createInterpreter()
-
- // sets in to some kind of reader depending on environmental cues
- in = in0 match {
- case Some(reader) => SimpleReader(reader, out, true)
- case None =>
- // some post-initialization
- chooseReader(settings) match {
- case x: SparkJLineReader => addThunk(x.consoleReader.postInit) ; x
- case x => x
- }
- }
- lazy val tagOfSparkIMain = tagOfStaticClass[org.apache.spark.repl.SparkIMain]
- // Bind intp somewhere out of the regular namespace where
- // we can get at it in generated code.
- addThunk(intp.quietBind(NamedParam[SparkIMain]("$intp", intp)(tagOfSparkIMain, classTag[SparkIMain])))
- addThunk({
- import scala.tools.nsc.io._
- import Properties.userHome
- import scala.compat.Platform.EOL
- val autorun = replProps.replAutorunCode.option flatMap (f => io.File(f).safeSlurp())
- if (autorun.isDefined) intp.quietRun(autorun.get)
- })
-
- addThunk(printWelcome())
- addThunk(initializeSpark())
-
- // it is broken on startup; go ahead and exit
- if (intp.reporter.hasErrors)
- return false
-
- // This is about the illusion of snappiness. We call initialize()
- // which spins off a separate thread, then print the prompt and try
- // our best to look ready. The interlocking lazy vals tend to
- // inter-deadlock, so we break the cycle with a single asynchronous
- // message to an rpcEndpoint.
- if (isAsync) {
- intp initialize initializedCallback()
- createAsyncListener() // listens for signal to run postInitialization
- }
- else {
- intp.initializeSynchronous()
- postInitialization()
- }
- // printWelcome()
-
- loadFiles(settings)
-
- try loop()
- catch AbstractOrMissingHandler()
- finally closeInterpreter()
-
- true
- }
-
- // NOTE: Must be public for visibility
- @DeveloperApi
- def createSparkSession(): SparkSession = {
- val execUri = System.getenv("SPARK_EXECUTOR_URI")
- val jars = getAddedJars()
- val conf = new SparkConf()
- .setMaster(getMaster())
- .setJars(jars)
- .setIfMissing("spark.app.name", "Spark shell")
- // SparkContext will detect this configuration and register it with the RpcEnv's
- // file server, setting spark.repl.class.uri to the actual URI for executors to
- // use. This is sort of ugly but since executors are started as part of SparkContext
- // initialization in certain cases, there's an initialization order issue that prevents
- // this from being set after SparkContext is instantiated.
- .set("spark.repl.class.outputDir", intp.outputDir.getAbsolutePath())
- if (execUri != null) {
- conf.set("spark.executor.uri", execUri)
- }
-
- val builder = SparkSession.builder.config(conf)
- val sparkSession = if (SparkSession.hiveClassesArePresent) {
- logInfo("Creating Spark session with Hive support")
- builder.enableHiveSupport().getOrCreate()
- } else {
- logInfo("Creating Spark session")
- builder.getOrCreate()
- }
- sparkContext = sparkSession.sparkContext
- sparkSession
- }
-
- private def getMaster(): String = {
- val master = this.master match {
- case Some(m) => m
- case None =>
- val envMaster = sys.env.get("MASTER")
- val propMaster = sys.props.get("spark.master")
- propMaster.orElse(envMaster).getOrElse("local[*]")
- }
- master
- }
-
- /** process command-line arguments and do as they request */
- def process(args: Array[String]): Boolean = {
- val command = new SparkCommandLine(args.toList, msg => echo(msg))
- def neededHelp(): String =
- (if (command.settings.help.value) command.usageMsg + "\n" else "") +
- (if (command.settings.Xhelp.value) command.xusageMsg + "\n" else "")
-
- // if they asked for no help and command is valid, we call the real main
- neededHelp() match {
- case "" => command.ok && process(command.settings)
- case help => echoNoNL(help) ; true
- }
- }
-
- @deprecated("Use `process` instead", "2.9.0")
- private def main(settings: Settings): Unit = process(settings)
-
- @DeveloperApi
- def getAddedJars(): Array[String] = {
- val conf = new SparkConf().setMaster(getMaster())
- val envJars = sys.env.get("ADD_JARS")
- if (envJars.isDefined) {
- logWarning("ADD_JARS environment variable is deprecated, use --jar spark submit argument instead")
- }
- val jars = {
- val userJars = Utils.getUserJars(conf, isShell = true)
- if (userJars.isEmpty) {
- envJars.getOrElse("")
- } else {
- userJars.mkString(",")
- }
- }
- Utils.resolveURIs(jars).split(",").filter(_.nonEmpty)
- }
-
-}
-
-object SparkILoop extends Logging {
- implicit def loopToInterpreter(repl: SparkILoop): SparkIMain = repl.intp
- private def echo(msg: String) = Console println msg
-
- // Designed primarily for use by test code: take a String with a
- // bunch of code, and prints out a transcript of what it would look
- // like if you'd just typed it into the repl.
- private[repl] def runForTranscript(code: String, settings: Settings): String = {
- import java.io.{ BufferedReader, StringReader, OutputStreamWriter }
-
- stringFromStream { ostream =>
- Console.withOut(ostream) {
- val output = new JPrintWriter(new OutputStreamWriter(ostream), true) {
- override def write(str: String) = {
- // completely skip continuation lines
- if (str forall (ch => ch.isWhitespace || ch == '|')) ()
- // print a newline on empty scala prompts
- else if ((str contains '\n') && (str.trim == "scala> ")) super.write("\n")
- else super.write(str)
- }
- }
- val input = new BufferedReader(new StringReader(code)) {
- override def readLine(): String = {
- val s = super.readLine()
- // helping out by printing the line being interpreted.
- if (s != null)
- // scalastyle:off println
- output.println(s)
- // scalastyle:on println
- s
- }
- }
- val repl = new SparkILoop(input, output)
-
- if (settings.classpath.isDefault)
- settings.classpath.value = sys.props("java.class.path")
-
- repl.getAddedJars().map(jar => new URI(jar).getPath).foreach(settings.classpath.append(_))
-
- repl process settings
- }
- }
- }
-
- /** Creates an interpreter loop with default settings and feeds
- * the given code to it as input.
- */
- private[repl] def run(code: String, sets: Settings = new Settings): String = {
- import java.io.{ BufferedReader, StringReader, OutputStreamWriter }
-
- stringFromStream { ostream =>
- Console.withOut(ostream) {
- val input = new BufferedReader(new StringReader(code))
- val output = new JPrintWriter(new OutputStreamWriter(ostream), true)
- val repl = new ILoop(input, output)
-
- if (sets.classpath.isDefault)
- sets.classpath.value = sys.props("java.class.path")
-
- repl process sets
- }
- }
- }
- private[repl] def run(lines: List[String]): String = run(lines map (_ + "\n") mkString)
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/425c4ada/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoopInit.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoopInit.scala b/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoopInit.scala
deleted file mode 100644
index 5f0d92b..0000000
--- a/repl/scala-2.10/src/main/scala/org/apache/spark/repl/SparkILoopInit.scala
+++ /dev/null
@@ -1,168 +0,0 @@
-// scalastyle:off
-
-/* NSC -- new Scala compiler
- * Copyright 2005-2013 LAMP/EPFL
- * @author Paul Phillips
- */
-
-package org.apache.spark.repl
-
-import scala.tools.nsc._
-import scala.tools.nsc.interpreter._
-
-import scala.tools.nsc.util.stackTraceString
-
-import org.apache.spark.SPARK_VERSION
-
-/**
- * Machinery for the asynchronous initialization of the repl.
- */
-private[repl] trait SparkILoopInit {
- self: SparkILoop =>
-
- /** Print a welcome message */
- def printWelcome() {
- echo("""Welcome to
- ____ __
- / __/__ ___ _____/ /__
- _\ \/ _ \/ _ `/ __/ '_/
- /___/ .__/\_,_/_/ /_/\_\ version %s
- /_/
-""".format(SPARK_VERSION))
- import Properties._
- val welcomeMsg = "Using Scala %s (%s, Java %s)".format(
- versionString, javaVmName, javaVersion)
- echo(welcomeMsg)
- echo("Type in expressions to have them evaluated.")
- echo("Type :help for more information.")
- }
-
- protected def asyncMessage(msg: String) {
- if (isReplInfo || isReplPower)
- echoAndRefresh(msg)
- }
-
- private val initLock = new java.util.concurrent.locks.ReentrantLock()
- private val initCompilerCondition = initLock.newCondition() // signal the compiler is initialized
- private val initLoopCondition = initLock.newCondition() // signal the whole repl is initialized
- private val initStart = System.nanoTime
-
- private def withLock[T](body: => T): T = {
- initLock.lock()
- try body
- finally initLock.unlock()
- }
- // a condition used to ensure serial access to the compiler.
- @volatile private var initIsComplete = false
- @volatile private var initError: String = null
- private def elapsed() = "%.3f".format((System.nanoTime - initStart).toDouble / 1000000000L)
-
- // the method to be called when the interpreter is initialized.
- // Very important this method does nothing synchronous (i.e. do
- // not try to use the interpreter) because until it returns, the
- // repl's lazy val `global` is still locked.
- protected def initializedCallback() = withLock(initCompilerCondition.signal())
-
- // Spins off a thread which awaits a single message once the interpreter
- // has been initialized.
- protected def createAsyncListener() = {
- io.spawn {
- withLock(initCompilerCondition.await())
- asyncMessage("[info] compiler init time: " + elapsed() + " s.")
- postInitialization()
- }
- }
-
- // called from main repl loop
- protected def awaitInitialized(): Boolean = {
- if (!initIsComplete)
- withLock { while (!initIsComplete) initLoopCondition.await() }
- if (initError != null) {
- // scalastyle:off println
- println("""
- |Failed to initialize the REPL due to an unexpected error.
- |This is a bug, please, report it along with the error diagnostics printed below.
- |%s.""".stripMargin.format(initError)
- )
- // scalastyle:on println
- false
- } else true
- }
- // private def warningsThunks = List(
- // () => intp.bind("lastWarnings", "" + typeTag[List[(Position, String)]], intp.lastWarnings _),
- // )
-
- protected def postInitThunks = List[Option[() => Unit]](
- Some(intp.setContextClassLoader _),
- if (isReplPower) Some(() => enablePowerMode(true)) else None
- ).flatten
- // ++ (
- // warningsThunks
- // )
- // called once after init condition is signalled
- protected def postInitialization() {
- try {
- postInitThunks foreach (f => addThunk(f()))
- runThunks()
- } catch {
- case ex: Throwable =>
- initError = stackTraceString(ex)
- throw ex
- } finally {
- initIsComplete = true
-
- if (isAsync) {
- asyncMessage("[info] total init time: " + elapsed() + " s.")
- withLock(initLoopCondition.signal())
- }
- }
- }
-
- def initializeSpark() {
- intp.beQuietDuring {
- command("""
- @transient val spark = org.apache.spark.repl.Main.interp.createSparkSession()
- @transient val sc = {
- val _sc = spark.sparkContext
- if (_sc.getConf.getBoolean("spark.ui.reverseProxy", false)) {
- val proxyUrl = _sc.getConf.get("spark.ui.reverseProxyUrl", null)
- if (proxyUrl != null) {
- println(s"Spark Context Web UI is available at ${proxyUrl}/proxy/${_sc.applicationId}")
- } else {
- println(s"Spark Context Web UI is available at Spark Master Public URL")
- }
- } else {
- _sc.uiWebUrl.foreach {
- webUrl => println(s"Spark context Web UI available at ${webUrl}")
- }
- }
- println("Spark context available as 'sc' " +
- s"(master = ${_sc.master}, app id = ${_sc.applicationId}).")
- println("Spark session available as 'spark'.")
- _sc
- }
- """)
- command("import org.apache.spark.SparkContext._")
- command("import spark.implicits._")
- command("import spark.sql")
- command("import org.apache.spark.sql.functions._")
- }
- }
-
- // code to be executed only after the interpreter is initialized
- // and the lazy val `global` can be accessed without risk of deadlock.
- private var pendingThunks: List[() => Unit] = Nil
- protected def addThunk(body: => Unit) = synchronized {
- pendingThunks :+= (() => body)
- }
- protected def runThunks(): Unit = synchronized {
- if (pendingThunks.nonEmpty)
- logDebug("Clearing " + pendingThunks.size + " thunks.")
-
- while (pendingThunks.nonEmpty) {
- val thunk = pendingThunks.head
- pendingThunks = pendingThunks.tail
- thunk()
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org