You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/04/09 05:35:33 UTC
[1/2] spark git commit: [SPARK-6765] Fix test code style for SQL
Repository: spark
Updated Branches:
refs/heads/master 891ada5be -> 1b2aab8d5
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala
index 02518d5..f7b37da 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala
@@ -24,8 +24,9 @@ import org.apache.spark.sql.catalyst.util._
/**
* A framework for running the query tests that are listed as a set of text files.
*
- * TestSuites that derive from this class must provide a map of testCaseName -> testCaseFiles that should be included.
- * Additionally, there is support for whitelisting and blacklisting tests as development progresses.
+ * TestSuites that derive from this class must provide a map of testCaseName -> testCaseFiles
+ * that should be included. Additionally, there is support for whitelisting and blacklisting
+ * tests as development progresses.
*/
abstract class HiveQueryFileTest extends HiveComparisonTest {
/** A list of tests deemed out of scope and thus completely disregarded */
@@ -54,15 +55,17 @@ abstract class HiveQueryFileTest extends HiveComparisonTest {
case (testCaseName, testCaseFile) =>
if (blackList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_)) {
logDebug(s"Blacklisted test skipped $testCaseName")
- } else if (realWhiteList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_) || runAll) {
+ } else if (realWhiteList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_) ||
+ runAll) {
// Build a test case and submit it to scala test framework...
val queriesString = fileToString(testCaseFile)
createQueryTest(testCaseName, queriesString)
} else {
// Only output warnings for the built in whitelist as this clutters the output when the user
// trying to execute a single test from the commandline.
- if(System.getProperty(whiteListProperty) == null && !runAll)
+ if (System.getProperty(whiteListProperty) == null && !runAll) {
ignore(testCaseName) {}
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index de140fc..af781a5 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -37,7 +37,8 @@ import org.apache.spark.sql.hive.test.TestHive._
case class TestData(a: Int, b: String)
/**
- * A set of test cases expressed in Hive QL that are not covered by the tests included in the hive distribution.
+ * A set of test cases expressed in Hive QL that are not covered by the tests
+ * included in the hive distribution.
*/
class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
private val originalTimeZone = TimeZone.getDefault
@@ -237,7 +238,8 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
}
createQueryTest("modulus",
- "SELECT 11 % 10, IF((101.1 % 100.0) BETWEEN 1.01 AND 1.11, \"true\", \"false\"), (101 / 2) % 10 FROM src LIMIT 1")
+ "SELECT 11 % 10, IF((101.1 % 100.0) BETWEEN 1.01 AND 1.11, \"true\", \"false\"), " +
+ "(101 / 2) % 10 FROM src LIMIT 1")
test("Query expressed in SQL") {
setConf("spark.sql.dialect", "sql")
@@ -309,7 +311,8 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
"SELECT * FROM src a JOIN src b ON a.key = b.key")
createQueryTest("small.cartesian",
- "SELECT a.key, b.key FROM (SELECT key FROM src WHERE key < 1) a JOIN (SELECT key FROM src WHERE key = 2) b")
+ "SELECT a.key, b.key FROM (SELECT key FROM src WHERE key < 1) a JOIN " +
+ "(SELECT key FROM src WHERE key = 2) b")
createQueryTest("length.udf",
"SELECT length(\"test\") FROM src LIMIT 1")
@@ -457,6 +460,7 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
createQueryTest("lateral view3",
"FROM src SELECT key, D.* lateral view explode(array(key+3, key+4)) D as CX")
+ // scalastyle:off
createQueryTest("lateral view4",
"""
|create table src_lv1 (key string, value string);
@@ -466,6 +470,7 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
|insert overwrite table src_lv1 SELECT key, D.* lateral view explode(array(key+3, key+4)) D as CX
|insert overwrite table src_lv2 SELECT key, D.* lateral view explode(array(key+3, key+4)) D as CX
""".stripMargin)
+ // scalastyle:on
createQueryTest("lateral view5",
"FROM src SELECT explode(array(key+3, key+4))")
@@ -584,7 +589,7 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
}
}
- def isExplanation(result: DataFrame) = {
+ def isExplanation(result: DataFrame): Boolean = {
val explanation = result.select('plan).collect().map { case Row(plan: String) => plan }
explanation.contains("== Physical Plan ==")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
index f4440e5..8ad3627 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveResolutionSuite.scala
@@ -25,7 +25,8 @@ case class Nested(a: Int, B: Int)
case class Data(a: Int, B: Int, n: Nested, nestedArray: Seq[Nested])
/**
- * A set of test cases expressed in Hive QL that are not covered by the tests included in the hive distribution.
+ * A set of test cases expressed in Hive QL that are not covered by the tests
+ * included in the hive distribution.
*/
class HiveResolutionSuite extends HiveComparisonTest {
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
index 7486bfa..d05e11f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
@@ -26,8 +26,9 @@ import org.apache.spark.sql.hive.test.TestHive
*/
class HiveSerDeSuite extends HiveComparisonTest with BeforeAndAfterAll {
- override def beforeAll() = {
+ override def beforeAll(): Unit = {
TestHive.cacheTables = false
+ super.beforeAll()
}
createQueryTest(
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala
index ab0e044..f0f04f8 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTypeCoercionSuite.scala
@@ -35,8 +35,10 @@ class HiveTypeCoercionSuite extends HiveComparisonTest {
val nullVal = "null"
baseTypes.init.foreach { i =>
- createQueryTest(s"case when then $i else $nullVal end ", s"SELECT case when true then $i else $nullVal end FROM src limit 1")
- createQueryTest(s"case when then $nullVal else $i end ", s"SELECT case when true then $nullVal else $i end FROM src limit 1")
+ createQueryTest(s"case when then $i else $nullVal end ",
+ s"SELECT case when true then $i else $nullVal end FROM src limit 1")
+ createQueryTest(s"case when then $nullVal else $i end ",
+ s"SELECT case when true then $nullVal else $i end FROM src limit 1")
}
test("[SPARK-2210] boolean cast on boolean value should be removed") {
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUdfSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUdfSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUdfSuite.scala
index d7c5d1a..7f49eac 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUdfSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUdfSuite.scala
@@ -123,9 +123,10 @@ class HiveUdfSuite extends QueryTest {
IntegerCaseClass(1) :: IntegerCaseClass(2) :: Nil).toDF()
testData.registerTempTable("integerTable")
- sql(s"CREATE TEMPORARY FUNCTION testUDFIntegerToString AS '${classOf[UDFIntegerToString].getName}'")
+ val udfName = classOf[UDFIntegerToString].getName
+ sql(s"CREATE TEMPORARY FUNCTION testUDFIntegerToString AS '$udfName'")
checkAnswer(
- sql("SELECT testUDFIntegerToString(i) FROM integerTable"), //.collect(),
+ sql("SELECT testUDFIntegerToString(i) FROM integerTable"),
Seq(Row("1"), Row("2")))
sql("DROP TEMPORARY FUNCTION IF EXISTS testUDFIntegerToString")
@@ -141,7 +142,7 @@ class HiveUdfSuite extends QueryTest {
sql(s"CREATE TEMPORARY FUNCTION testUDFListListInt AS '${classOf[UDFListListInt].getName}'")
checkAnswer(
- sql("SELECT testUDFListListInt(lli) FROM listListIntTable"), //.collect(),
+ sql("SELECT testUDFListListInt(lli) FROM listListIntTable"),
Seq(Row(0), Row(2), Row(13)))
sql("DROP TEMPORARY FUNCTION IF EXISTS testUDFListListInt")
@@ -156,7 +157,7 @@ class HiveUdfSuite extends QueryTest {
sql(s"CREATE TEMPORARY FUNCTION testUDFListString AS '${classOf[UDFListString].getName}'")
checkAnswer(
- sql("SELECT testUDFListString(l) FROM listStringTable"), //.collect(),
+ sql("SELECT testUDFListString(l) FROM listStringTable"),
Seq(Row("a,b,c"), Row("d,e")))
sql("DROP TEMPORARY FUNCTION IF EXISTS testUDFListString")
@@ -170,7 +171,7 @@ class HiveUdfSuite extends QueryTest {
sql(s"CREATE TEMPORARY FUNCTION testStringStringUdf AS '${classOf[UDFStringString].getName}'")
checkAnswer(
- sql("SELECT testStringStringUdf(\"hello\", s) FROM stringTable"), //.collect(),
+ sql("SELECT testStringStringUdf(\"hello\", s) FROM stringTable"),
Seq(Row("hello world"), Row("hello goodbye")))
sql("DROP TEMPORARY FUNCTION IF EXISTS testStringStringUdf")
@@ -187,7 +188,7 @@ class HiveUdfSuite extends QueryTest {
sql(s"CREATE TEMPORARY FUNCTION testUDFTwoListList AS '${classOf[UDFTwoListList].getName}'")
checkAnswer(
- sql("SELECT testUDFTwoListList(lli, lli) FROM TwoListTable"), //.collect(),
+ sql("SELECT testUDFTwoListList(lli, lli) FROM TwoListTable"),
Seq(Row("0, 0"), Row("2, 2"), Row("13, 13")))
sql("DROP TEMPORARY FUNCTION IF EXISTS testUDFTwoListList")
@@ -247,7 +248,8 @@ class PairUdf extends GenericUDF {
override def initialize(p1: Array[ObjectInspector]): ObjectInspector =
ObjectInspectorFactory.getStandardStructObjectInspector(
Seq("id", "value"),
- Seq(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaIntObjectInspector)
+ Seq(PrimitiveObjectInspectorFactory.javaIntObjectInspector,
+ PrimitiveObjectInspectorFactory.javaIntObjectInspector)
)
override def evaluate(args: Array[DeferredObject]): AnyRef = {
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruningSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruningSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruningSuite.scala
index 8474d85..067b577 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruningSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruningSuite.scala
@@ -143,7 +143,7 @@ class PruningSuite extends HiveComparisonTest with BeforeAndAfter {
sql: String,
expectedOutputColumns: Seq[String],
expectedScannedColumns: Seq[String],
- expectedPartValues: Seq[Seq[String]]) = {
+ expectedPartValues: Seq[Seq[String]]): Unit = {
test(s"$testCaseName - pruning test") {
val plan = new TestHive.HiveQLQueryExecution(sql).executedPlan
val actualOutputColumns = plan.output.map(_.name)
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 817b9dc..7811bd2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -422,7 +422,7 @@ class SQLQuerySuite extends QueryTest {
}
test("resolve udtf with single alias") {
- val rdd = sparkContext.makeRDD((1 to 5).map(i => s"""{"a":[$i, ${i+1}]}"""))
+ val rdd = sparkContext.makeRDD((1 to 5).map(i => s"""{"a":[$i, ${i + 1}]}"""))
jsonRDD(rdd).registerTempTable("data")
val df = sql("SELECT explode(a) AS val FROM data")
val col = df("val")
@@ -435,7 +435,7 @@ class SQLQuerySuite extends QueryTest {
// is not in a valid state (cannot be executed). Because of this bug, the analysis rule of
// PreInsertionCasts will actually start to work before ImplicitGenerate and then
// generates an invalid query plan.
- val rdd = sparkContext.makeRDD((1 to 5).map(i => s"""{"a":[$i, ${i+1}]}"""))
+ val rdd = sparkContext.makeRDD((1 to 5).map(i => s"""{"a":[$i, ${i + 1}]}"""))
jsonRDD(rdd).registerTempTable("data")
val originalConf = getConf("spark.sql.hive.convertCTAS", "false")
setConf("spark.sql.hive.convertCTAS", "false")
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala
index 5f71e1b..d5dd0bf 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala
@@ -1,4 +1,3 @@
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -887,7 +886,11 @@ abstract class ParquetPartitioningTest extends QueryTest with BeforeAndAfterAll
test(s"SPARK-5775 read struct from $table") {
checkAnswer(
- sql(s"SELECT p, structField.intStructField, structField.stringStructField FROM $table WHERE p = 1"),
+ sql(
+ s"""
+ |SELECT p, structField.intStructField, structField.stringStructField
+ |FROM $table WHERE p = 1
+ """.stripMargin),
(1 to 10).map(i => Row(1, i, f"${i}_string")))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[2/2] spark git commit: [SPARK-6765] Fix test code style for SQL
Posted by rx...@apache.org.
[SPARK-6765] Fix test code style for SQL
So we can turn style checker on for test code.
Author: Reynold Xin <rx...@databricks.com>
Closes #5412 from rxin/test-style-sql and squashes the following commits:
9098a31 [Reynold Xin] One more compilation error ...
8c7250a [Reynold Xin] Fix compilation.
82d0944 [Reynold Xin] Indentation.
0b03fbb [Reynold Xin] code review.
f2f4348 [Reynold Xin] oops.
ef4ec48 [Reynold Xin] Hive module.
7e0db5e [Reynold Xin] sql module
04ec7ac [Reynold Xin] catalyst module
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1b2aab8d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1b2aab8d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1b2aab8d
Branch: refs/heads/master
Commit: 1b2aab8d5b9cc2ff702506038bd71aa8debe7ca0
Parents: 891ada5
Author: Reynold Xin <rx...@databricks.com>
Authored: Wed Apr 8 20:35:29 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Wed Apr 8 20:35:29 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/DistributionSuite.scala | 3 +-
.../sql/catalyst/analysis/AnalysisSuite.scala | 10 +-
.../analysis/HiveTypeCoercionSuite.scala | 8 +-
.../expressions/ExpressionEvaluationSuite.scala | 134 +++++++++++++------
.../optimizer/ConstantFoldingSuite.scala | 51 ++++---
.../optimizer/FilterPushdownSuite.scala | 3 +-
.../catalyst/optimizer/OptimizeInSuite.scala | 2 +-
.../spark/sql/catalyst/plans/PlanTest.scala | 5 +-
.../sql/catalyst/plans/SameResultSuite.scala | 2 +-
.../sql/catalyst/trees/TreeNodeSuite.scala | 8 +-
.../org/apache/spark/sql/CachedTableSuite.scala | 3 +-
.../org/apache/spark/sql/DataFrameSuite.scala | 3 +-
.../scala/org/apache/spark/sql/QueryTest.scala | 2 +-
.../org/apache/spark/sql/SQLQuerySuite.scala | 30 +++--
.../sql/ScalaReflectionRelationSuite.scala | 5 +-
.../apache/spark/sql/UserDefinedTypeSuite.scala | 2 +-
.../spark/sql/columnar/ColumnarTestUtils.scala | 4 +-
.../columnar/NullableColumnAccessorSuite.scala | 3 +-
.../columnar/NullableColumnBuilderSuite.scala | 3 +-
.../TestCompressibleColumnBuilder.scala | 2 +-
.../sql/execution/debug/DebuggingSuite.scala | 2 +-
.../org/apache/spark/sql/jdbc/JDBCSuite.scala | 98 +++++++-------
.../org/apache/spark/sql/json/JsonSuite.scala | 17 ++-
.../spark/sql/parquet/ParquetIOSuite.scala | 2 +-
.../spark/sql/parquet/ParquetSchemaSuite.scala | 2 +
.../apache/spark/sql/sources/DDLTestSuite.scala | 8 +-
.../spark/sql/sources/FilteredScanSuite.scala | 3 +-
.../spark/sql/sources/PrunedScanSuite.scala | 5 +-
.../spark/sql/sources/SaveLoadSuite.scala | 2 +-
.../spark/sql/sources/TableScanSuite.scala | 9 +-
.../spark/sql/hive/ErrorPositionSuite.scala | 2 +-
.../spark/sql/hive/HiveInspectorSuite.scala | 33 +++--
.../sql/hive/InsertIntoHiveTableSuite.scala | 57 ++++++--
.../apache/spark/sql/hive/StatisticsSuite.scala | 2 +-
.../hive/execution/BigDataBenchmarkSuite.scala | 12 +-
.../sql/hive/execution/HiveComparisonTest.scala | 27 ++--
.../sql/hive/execution/HiveQueryFileTest.scala | 11 +-
.../sql/hive/execution/HiveQuerySuite.scala | 13 +-
.../hive/execution/HiveResolutionSuite.scala | 3 +-
.../sql/hive/execution/HiveSerDeSuite.scala | 3 +-
.../hive/execution/HiveTypeCoercionSuite.scala | 6 +-
.../spark/sql/hive/execution/HiveUdfSuite.scala | 16 ++-
.../spark/sql/hive/execution/PruningSuite.scala | 2 +-
.../sql/hive/execution/SQLQuerySuite.scala | 4 +-
.../apache/spark/sql/hive/parquetSuites.scala | 7 +-
45 files changed, 395 insertions(+), 234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/DistributionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/DistributionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/DistributionSuite.scala
index 46b2250..ea82cd2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/DistributionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/DistributionSuite.scala
@@ -30,7 +30,7 @@ class DistributionSuite extends FunSuite {
inputPartitioning: Partitioning,
requiredDistribution: Distribution,
satisfied: Boolean) {
- if (inputPartitioning.satisfies(requiredDistribution) != satisfied)
+ if (inputPartitioning.satisfies(requiredDistribution) != satisfied) {
fail(
s"""
|== Input Partitioning ==
@@ -40,6 +40,7 @@ class DistributionSuite extends FunSuite {
|== Does input partitioning satisfy required distribution? ==
|Expected $satisfied got ${inputPartitioning.satisfies(requiredDistribution)}
""".stripMargin)
+ }
}
test("HashPartitioning is the output partitioning") {
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index ee7b14c..6e3d6b9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -27,6 +27,8 @@ import org.apache.spark.sql.types._
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
+import scala.collection.immutable
+
class AnalysisSuite extends FunSuite with BeforeAndAfter {
val caseSensitiveCatalog = new SimpleCatalog(true)
val caseInsensitiveCatalog = new SimpleCatalog(false)
@@ -41,10 +43,10 @@ class AnalysisSuite extends FunSuite with BeforeAndAfter {
}
- def caseSensitiveAnalyze(plan: LogicalPlan) =
+ def caseSensitiveAnalyze(plan: LogicalPlan): Unit =
caseSensitiveAnalyzer.checkAnalysis(caseSensitiveAnalyzer(plan))
- def caseInsensitiveAnalyze(plan: LogicalPlan) =
+ def caseInsensitiveAnalyze(plan: LogicalPlan): Unit =
caseInsensitiveAnalyzer.checkAnalysis(caseInsensitiveAnalyzer(plan))
val testRelation = LocalRelation(AttributeReference("a", IntegerType, nullable = true)())
@@ -147,7 +149,7 @@ class AnalysisSuite extends FunSuite with BeforeAndAfter {
name: String,
plan: LogicalPlan,
errorMessages: Seq[String],
- caseSensitive: Boolean = true) = {
+ caseSensitive: Boolean = true): Unit = {
test(name) {
val error = intercept[AnalysisException] {
if(caseSensitive) {
@@ -202,7 +204,7 @@ class AnalysisSuite extends FunSuite with BeforeAndAfter {
case class UnresolvedTestPlan() extends LeafNode {
override lazy val resolved = false
- override def output = Nil
+ override def output: Seq[Attribute] = Nil
}
errorTest(
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
index 70aef1c..fcd745f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala
@@ -96,7 +96,9 @@ class HiveTypeCoercionSuite extends PlanTest {
widenTest(StringType, TimestampType, None)
// ComplexType
- widenTest(NullType, MapType(IntegerType, StringType, false), Some(MapType(IntegerType, StringType, false)))
+ widenTest(NullType,
+ MapType(IntegerType, StringType, false),
+ Some(MapType(IntegerType, StringType, false)))
widenTest(NullType, StructType(Seq()), Some(StructType(Seq())))
widenTest(StringType, MapType(IntegerType, StringType, true), None)
widenTest(ArrayType(IntegerType), StructType(Seq()), None)
@@ -113,7 +115,9 @@ class HiveTypeCoercionSuite extends PlanTest {
// Remove superflous boolean -> boolean casts.
ruleTest(Cast(Literal(true), BooleanType), Literal(true))
// Stringify boolean when casting to string.
- ruleTest(Cast(Literal(false), StringType), If(Literal(false), Literal("true"), Literal("false")))
+ ruleTest(
+ Cast(Literal(false), StringType),
+ If(Literal(false), Literal("true"), Literal("false")))
}
test("coalesce casts") {
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
index 3dbefa4..d2b1090 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
@@ -82,10 +82,13 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
assert(BitwiseNot(1.toByte).eval(EmptyRow).isInstanceOf[Byte])
}
+ // scalastyle:off
/**
* Checks for three-valued-logic. Based on:
* http://en.wikipedia.org/wiki/Null_(SQL)#Comparisons_with_NULL_and_the_three-valued_logic_.283VL.29
- * I.e. in flat cpo "False -> Unknown -> True", OR is lowest upper bound, AND is greatest lower bound.
+ * I.e. in flat cpo "False -> Unknown -> True",
+ * OR is lowest upper bound,
+ * AND is greatest lower bound.
* p q p OR q p AND q p = q
* True True True True True
* True False True False False
@@ -102,7 +105,7 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
* False True
* Unknown Unknown
*/
-
+ // scalastyle:on
val notTrueTable =
(true, false) ::
(false, true) ::
@@ -165,7 +168,9 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
checkEvaluation(In(Literal(1), Seq(Literal(1), Literal(2))), true)
checkEvaluation(In(Literal(2), Seq(Literal(1), Literal(2))), true)
checkEvaluation(In(Literal(3), Seq(Literal(1), Literal(2))), false)
- checkEvaluation(In(Literal(1), Seq(Literal(1), Literal(2))) && In(Literal(2), Seq(Literal(1), Literal(2))), true)
+ checkEvaluation(
+ In(Literal(1), Seq(Literal(1), Literal(2))) && In(Literal(2), Seq(Literal(1), Literal(2))),
+ true)
}
test("Divide") {
@@ -180,7 +185,8 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
checkEvaluation(Divide(Literal.create(null, IntegerType), Literal(0)), null)
checkEvaluation(Divide(Literal.create(null, DoubleType), Literal(0.0)), null)
checkEvaluation(Divide(Literal.create(null, IntegerType), Literal(1)), null)
- checkEvaluation(Divide(Literal.create(null, IntegerType), Literal.create(null, IntegerType)), null)
+ checkEvaluation(Divide(Literal.create(null, IntegerType), Literal.create(null, IntegerType)),
+ null)
}
test("Remainder") {
@@ -195,7 +201,8 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
checkEvaluation(Remainder(Literal.create(null, IntegerType), Literal(0)), null)
checkEvaluation(Remainder(Literal.create(null, DoubleType), Literal(0.0)), null)
checkEvaluation(Remainder(Literal.create(null, IntegerType), Literal(1)), null)
- checkEvaluation(Remainder(Literal.create(null, IntegerType), Literal.create(null, IntegerType)), null)
+ checkEvaluation(Remainder(Literal.create(null, IntegerType), Literal.create(null, IntegerType)),
+ null)
}
test("INSET") {
@@ -264,7 +271,8 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
checkEvaluation("ab" like regEx, true, new GenericRow(Array[Any]("a%b")))
checkEvaluation("a\nb" like regEx, true, new GenericRow(Array[Any]("a%b")))
- checkEvaluation(Literal.create(null, StringType) like regEx, null, new GenericRow(Array[Any]("bc%")))
+ checkEvaluation(Literal.create(null, StringType) like regEx, null,
+ new GenericRow(Array[Any]("bc%")))
}
test("RLIKE literal Regular Expression") {
@@ -507,8 +515,10 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
}
test("array casting") {
- val array = Literal.create(Seq("123", "abc", "", null), ArrayType(StringType, containsNull = true))
- val array_notNull = Literal.create(Seq("123", "abc", ""), ArrayType(StringType, containsNull = false))
+ val array = Literal.create(Seq("123", "abc", "", null),
+ ArrayType(StringType, containsNull = true))
+ val array_notNull = Literal.create(Seq("123", "abc", ""),
+ ArrayType(StringType, containsNull = false))
{
val cast = Cast(array, ArrayType(IntegerType, containsNull = true))
@@ -765,7 +775,8 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
checkEvaluation(Coalesce(Literal.create(null, StringType) :: Nil), null, row)
checkEvaluation(Coalesce(Literal.create(null, StringType) :: c1 :: c2 :: Nil), "^Ba*n", row)
- checkEvaluation(If(c3, Literal.create("a", StringType), Literal.create("b", StringType)), "a", row)
+ checkEvaluation(
+ If(c3, Literal.create("a", StringType), Literal.create("b", StringType)), "a", row)
checkEvaluation(If(c3, c1, c2), "^Ba*n", row)
checkEvaluation(If(c4, c2, c1), "^Ba*n", row)
checkEvaluation(If(Literal.create(null, BooleanType), c2, c1), "^Ba*n", row)
@@ -842,18 +853,20 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
checkEvaluation(GetItem(BoundReference(3, typeMap, true),
Literal("aa")), "bb", row)
checkEvaluation(GetItem(Literal.create(null, typeMap), Literal("aa")), null, row)
- checkEvaluation(GetItem(Literal.create(null, typeMap), Literal.create(null, StringType)), null, row)
+ checkEvaluation(
+ GetItem(Literal.create(null, typeMap), Literal.create(null, StringType)), null, row)
checkEvaluation(GetItem(BoundReference(3, typeMap, true),
Literal.create(null, StringType)), null, row)
checkEvaluation(GetItem(BoundReference(4, typeArray, true),
Literal(1)), "bb", row)
checkEvaluation(GetItem(Literal.create(null, typeArray), Literal(1)), null, row)
- checkEvaluation(GetItem(Literal.create(null, typeArray), Literal.create(null, IntegerType)), null, row)
+ checkEvaluation(
+ GetItem(Literal.create(null, typeArray), Literal.create(null, IntegerType)), null, row)
checkEvaluation(GetItem(BoundReference(4, typeArray, true),
Literal.create(null, IntegerType)), null, row)
- def quickBuildGetField(expr: Expression, fieldName: String) = {
+ def quickBuildGetField(expr: Expression, fieldName: String): StructGetField = {
expr.dataType match {
case StructType(fields) =>
val field = fields.find(_.name == fieldName).get
@@ -861,7 +874,9 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
}
}
- def quickResolve(u: UnresolvedGetField) = quickBuildGetField(u.child, u.fieldName)
+ def quickResolve(u: UnresolvedGetField): StructGetField = {
+ quickBuildGetField(u.child, u.fieldName)
+ }
checkEvaluation(quickBuildGetField(BoundReference(2, typeS, nullable = true), "a"), "aa", row)
checkEvaluation(quickBuildGetField(Literal.create(null, typeS), "a"), null, row)
@@ -872,7 +887,8 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
)
assert(quickBuildGetField(BoundReference(2,typeS, nullable = true), "a").nullable === true)
- assert(quickBuildGetField(BoundReference(2, typeS_notNullable, nullable = false), "a").nullable === false)
+ assert(quickBuildGetField(BoundReference(2, typeS_notNullable, nullable = false), "a").nullable
+ === false)
assert(quickBuildGetField(Literal.create(null, typeS), "a").nullable === true)
assert(quickBuildGetField(Literal.create(null, typeS_notNullable), "a").nullable === true)
@@ -896,7 +912,8 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
checkEvaluation(Add(c1, c2), 3, row)
checkEvaluation(Add(c1, Literal.create(null, IntegerType)), null, row)
checkEvaluation(Add(Literal.create(null, IntegerType), c2), null, row)
- checkEvaluation(Add(Literal.create(null, IntegerType), Literal.create(null, IntegerType)), null, row)
+ checkEvaluation(
+ Add(Literal.create(null, IntegerType), Literal.create(null, IntegerType)), null, row)
checkEvaluation(-c1, -1, row)
checkEvaluation(c1 + c2, 3, row)
@@ -919,7 +936,8 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
checkEvaluation(Add(c1, c2), 3.1, row)
checkEvaluation(Add(c1, Literal.create(null, DoubleType)), null, row)
checkEvaluation(Add(Literal.create(null, DoubleType), c2), null, row)
- checkEvaluation(Add(Literal.create(null, DoubleType), Literal.create(null, DoubleType)), null, row)
+ checkEvaluation(
+ Add(Literal.create(null, DoubleType), Literal.create(null, DoubleType)), null, row)
checkEvaluation(-c1, -1.1, row)
checkEvaluation(c1 + c2, 3.1, row)
@@ -942,7 +960,8 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
checkEvaluation(LessThan(c1, c2), true, row)
checkEvaluation(LessThan(c1, Literal.create(null, IntegerType)), null, row)
checkEvaluation(LessThan(Literal.create(null, IntegerType), c2), null, row)
- checkEvaluation(LessThan(Literal.create(null, IntegerType), Literal.create(null, IntegerType)), null, row)
+ checkEvaluation(
+ LessThan(Literal.create(null, IntegerType), Literal.create(null, IntegerType)), null, row)
checkEvaluation(c1 < c2, true, row)
checkEvaluation(c1 <= c2, true, row)
@@ -985,54 +1004,84 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
val s = 'a.string.at(0)
// substring from zero position with less-than-full length
- checkEvaluation(Substring(s, Literal.create(0, IntegerType), Literal.create(2, IntegerType)), "ex", row)
- checkEvaluation(Substring(s, Literal.create(1, IntegerType), Literal.create(2, IntegerType)), "ex", row)
+ checkEvaluation(
+ Substring(s, Literal.create(0, IntegerType), Literal.create(2, IntegerType)), "ex", row)
+ checkEvaluation(
+ Substring(s, Literal.create(1, IntegerType), Literal.create(2, IntegerType)), "ex", row)
// substring from zero position with full length
- checkEvaluation(Substring(s, Literal.create(0, IntegerType), Literal.create(7, IntegerType)), "example", row)
- checkEvaluation(Substring(s, Literal.create(1, IntegerType), Literal.create(7, IntegerType)), "example", row)
+ checkEvaluation(
+ Substring(s, Literal.create(0, IntegerType), Literal.create(7, IntegerType)), "example", row)
+ checkEvaluation(
+ Substring(s, Literal.create(1, IntegerType), Literal.create(7, IntegerType)), "example", row)
// substring from zero position with greater-than-full length
- checkEvaluation(Substring(s, Literal.create(0, IntegerType), Literal.create(100, IntegerType)), "example", row)
- checkEvaluation(Substring(s, Literal.create(1, IntegerType), Literal.create(100, IntegerType)), "example", row)
+ checkEvaluation(Substring(s, Literal.create(0, IntegerType), Literal.create(100, IntegerType)),
+ "example", row)
+ checkEvaluation(Substring(s, Literal.create(1, IntegerType), Literal.create(100, IntegerType)),
+ "example", row)
// substring from nonzero position with less-than-full length
- checkEvaluation(Substring(s, Literal.create(2, IntegerType), Literal.create(2, IntegerType)), "xa", row)
+ checkEvaluation(Substring(s, Literal.create(2, IntegerType), Literal.create(2, IntegerType)),
+ "xa", row)
// substring from nonzero position with full length
- checkEvaluation(Substring(s, Literal.create(2, IntegerType), Literal.create(6, IntegerType)), "xample", row)
+ checkEvaluation(Substring(s, Literal.create(2, IntegerType), Literal.create(6, IntegerType)),
+ "xample", row)
// substring from nonzero position with greater-than-full length
- checkEvaluation(Substring(s, Literal.create(2, IntegerType), Literal.create(100, IntegerType)), "xample", row)
+ checkEvaluation(Substring(s, Literal.create(2, IntegerType), Literal.create(100, IntegerType)),
+ "xample", row)
// zero-length substring (within string bounds)
- checkEvaluation(Substring(s, Literal.create(0, IntegerType), Literal.create(0, IntegerType)), "", row)
+ checkEvaluation(Substring(s, Literal.create(0, IntegerType), Literal.create(0, IntegerType)),
+ "", row)
// zero-length substring (beyond string bounds)
- checkEvaluation(Substring(s, Literal.create(100, IntegerType), Literal.create(4, IntegerType)), "", row)
+ checkEvaluation(Substring(s, Literal.create(100, IntegerType), Literal.create(4, IntegerType)),
+ "", row)
// substring(null, _, _) -> null
- checkEvaluation(Substring(s, Literal.create(100, IntegerType), Literal.create(4, IntegerType)), null, new GenericRow(Array[Any](null)))
+ checkEvaluation(Substring(s, Literal.create(100, IntegerType), Literal.create(4, IntegerType)),
+ null, new GenericRow(Array[Any](null)))
// substring(_, null, _) -> null
- checkEvaluation(Substring(s, Literal.create(null, IntegerType), Literal.create(4, IntegerType)), null, row)
+ checkEvaluation(Substring(s, Literal.create(null, IntegerType), Literal.create(4, IntegerType)),
+ null, row)
// substring(_, _, null) -> null
- checkEvaluation(Substring(s, Literal.create(100, IntegerType), Literal.create(null, IntegerType)), null, row)
+ checkEvaluation(
+ Substring(s, Literal.create(100, IntegerType), Literal.create(null, IntegerType)),
+ null,
+ row)
// 2-arg substring from zero position
- checkEvaluation(Substring(s, Literal.create(0, IntegerType), Literal.create(Integer.MAX_VALUE, IntegerType)), "example", row)
- checkEvaluation(Substring(s, Literal.create(1, IntegerType), Literal.create(Integer.MAX_VALUE, IntegerType)), "example", row)
+ checkEvaluation(
+ Substring(s, Literal.create(0, IntegerType), Literal.create(Integer.MAX_VALUE, IntegerType)),
+ "example",
+ row)
+ checkEvaluation(
+ Substring(s, Literal.create(1, IntegerType), Literal.create(Integer.MAX_VALUE, IntegerType)),
+ "example",
+ row)
// 2-arg substring from nonzero position
- checkEvaluation(Substring(s, Literal.create(2, IntegerType), Literal.create(Integer.MAX_VALUE, IntegerType)), "xample", row)
+ checkEvaluation(
+ Substring(s, Literal.create(2, IntegerType), Literal.create(Integer.MAX_VALUE, IntegerType)),
+ "xample",
+ row)
val s_notNull = 'a.string.notNull.at(0)
- assert(Substring(s, Literal.create(0, IntegerType), Literal.create(2, IntegerType)).nullable === true)
- assert(Substring(s_notNull, Literal.create(0, IntegerType), Literal.create(2, IntegerType)).nullable === false)
- assert(Substring(s_notNull, Literal.create(null, IntegerType), Literal.create(2, IntegerType)).nullable === true)
- assert(Substring(s_notNull, Literal.create(0, IntegerType), Literal.create(null, IntegerType)).nullable === true)
+ assert(Substring(s, Literal.create(0, IntegerType), Literal.create(2, IntegerType)).nullable
+ === true)
+ assert(
+ Substring(s_notNull, Literal.create(0, IntegerType), Literal.create(2, IntegerType)).nullable
+ === false)
+ assert(Substring(s_notNull,
+ Literal.create(null, IntegerType), Literal.create(2, IntegerType)).nullable === true)
+ assert(Substring(s_notNull,
+ Literal.create(0, IntegerType), Literal.create(null, IntegerType)).nullable === true)
checkEvaluation(s.substr(0, 2), "ex", row)
checkEvaluation(s.substr(0), "example", row)
@@ -1065,17 +1114,20 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
checkEvaluation(BitwiseAnd(c1, c4), null, row)
checkEvaluation(BitwiseAnd(c1, c2), 0, row)
checkEvaluation(BitwiseAnd(c1, Literal.create(null, IntegerType)), null, row)
- checkEvaluation(BitwiseAnd(Literal.create(null, IntegerType), Literal.create(null, IntegerType)), null, row)
+ checkEvaluation(
+ BitwiseAnd(Literal.create(null, IntegerType), Literal.create(null, IntegerType)), null, row)
checkEvaluation(BitwiseOr(c1, c4), null, row)
checkEvaluation(BitwiseOr(c1, c2), 3, row)
checkEvaluation(BitwiseOr(c1, Literal.create(null, IntegerType)), null, row)
- checkEvaluation(BitwiseOr(Literal.create(null, IntegerType), Literal.create(null, IntegerType)), null, row)
+ checkEvaluation(
+ BitwiseOr(Literal.create(null, IntegerType), Literal.create(null, IntegerType)), null, row)
checkEvaluation(BitwiseXor(c1, c4), null, row)
checkEvaluation(BitwiseXor(c1, c2), 3, row)
checkEvaluation(BitwiseXor(c1, Literal.create(null, IntegerType)), null, row)
- checkEvaluation(BitwiseXor(Literal.create(null, IntegerType), Literal.create(null, IntegerType)), null, row)
+ checkEvaluation(
+ BitwiseXor(Literal.create(null, IntegerType), Literal.create(null, IntegerType)), null, row)
checkEvaluation(BitwiseNot(c4), null, row)
checkEvaluation(BitwiseNot(c1), -2, row)
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala
index a0efe9e..4396bd0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala
@@ -176,40 +176,39 @@ class ConstantFoldingSuite extends PlanTest {
}
test("Constant folding test: expressions have null literals") {
- val originalQuery =
- testRelation
- .select(
- IsNull(Literal(null)) as 'c1,
- IsNotNull(Literal(null)) as 'c2,
+ val originalQuery = testRelation.select(
+ IsNull(Literal(null)) as 'c1,
+ IsNotNull(Literal(null)) as 'c2,
- GetItem(Literal.create(null, ArrayType(IntegerType)), 1) as 'c3,
- GetItem(Literal.create(Seq(1), ArrayType(IntegerType)), Literal.create(null, IntegerType)) as 'c4,
- UnresolvedGetField(
- Literal.create(null, StructType(Seq(StructField("a", IntegerType, true)))),
- "a") as 'c5,
+ GetItem(Literal.create(null, ArrayType(IntegerType)), 1) as 'c3,
+ GetItem(
+ Literal.create(Seq(1), ArrayType(IntegerType)), Literal.create(null, IntegerType)) as 'c4,
+ UnresolvedGetField(
+ Literal.create(null, StructType(Seq(StructField("a", IntegerType, true)))),
+ "a") as 'c5,
- UnaryMinus(Literal.create(null, IntegerType)) as 'c6,
- Cast(Literal(null), IntegerType) as 'c7,
- Not(Literal.create(null, BooleanType)) as 'c8,
+ UnaryMinus(Literal.create(null, IntegerType)) as 'c6,
+ Cast(Literal(null), IntegerType) as 'c7,
+ Not(Literal.create(null, BooleanType)) as 'c8,
- Add(Literal.create(null, IntegerType), 1) as 'c9,
- Add(1, Literal.create(null, IntegerType)) as 'c10,
+ Add(Literal.create(null, IntegerType), 1) as 'c9,
+ Add(1, Literal.create(null, IntegerType)) as 'c10,
- EqualTo(Literal.create(null, IntegerType), 1) as 'c11,
- EqualTo(1, Literal.create(null, IntegerType)) as 'c12,
+ EqualTo(Literal.create(null, IntegerType), 1) as 'c11,
+ EqualTo(1, Literal.create(null, IntegerType)) as 'c12,
- Like(Literal.create(null, StringType), "abc") as 'c13,
- Like("abc", Literal.create(null, StringType)) as 'c14,
+ Like(Literal.create(null, StringType), "abc") as 'c13,
+ Like("abc", Literal.create(null, StringType)) as 'c14,
- Upper(Literal.create(null, StringType)) as 'c15,
+ Upper(Literal.create(null, StringType)) as 'c15,
- Substring(Literal.create(null, StringType), 0, 1) as 'c16,
- Substring("abc", Literal.create(null, IntegerType), 1) as 'c17,
- Substring("abc", 0, Literal.create(null, IntegerType)) as 'c18,
+ Substring(Literal.create(null, StringType), 0, 1) as 'c16,
+ Substring("abc", Literal.create(null, IntegerType), 1) as 'c17,
+ Substring("abc", 0, Literal.create(null, IntegerType)) as 'c18,
- Contains(Literal.create(null, StringType), "abc") as 'c19,
- Contains("abc", Literal.create(null, StringType)) as 'c20
- )
+ Contains(Literal.create(null, StringType), "abc") as 'c19,
+ Contains("abc", Literal.create(null, StringType)) as 'c20
+ )
val optimized = Optimize(originalQuery.analyze)
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
index 55c6766..1448098 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
@@ -432,7 +432,8 @@ class FilterPushdownSuite extends PlanTest {
val originalQuery = {
z.join(x.join(y))
- .where(("x.b".attr === "y.b".attr) && ("x.a".attr === 1) && ("z.a".attr >= 3) && ("z.a".attr === "x.b".attr))
+ .where(("x.b".attr === "y.b".attr) && ("x.a".attr === 1) &&
+ ("z.a".attr >= 3) && ("z.a".attr === "x.b".attr))
}
val optimized = Optimize(originalQuery.analyze)
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala
index 233e329..966bc9a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeInSuite.scala
@@ -52,7 +52,7 @@ class OptimizeInSuite extends PlanTest {
val optimized = Optimize(originalQuery.analyze)
val correctAnswer =
testRelation
- .where(InSet(UnresolvedAttribute("a"), HashSet[Any]()+1+2))
+ .where(InSet(UnresolvedAttribute("a"), HashSet[Any]() + 1 + 2))
.analyze
comparePlans(optimized, correctAnswer)
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
index 129d091..e7cafcc 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/PlanTest.scala
@@ -45,12 +45,13 @@ class PlanTest extends FunSuite {
protected def comparePlans(plan1: LogicalPlan, plan2: LogicalPlan) {
val normalized1 = normalizeExprIds(plan1)
val normalized2 = normalizeExprIds(plan2)
- if (normalized1 != normalized2)
+ if (normalized1 != normalized2) {
fail(
s"""
|== FAIL: Plans do not match ===
|${sideBySide(normalized1.treeString, normalized2.treeString).mkString("\n")}
- """.stripMargin)
+ """.stripMargin)
+ }
}
/** Fails the test if the two expressions do not match */
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SameResultSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SameResultSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SameResultSuite.scala
index 11e6831..1273921 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SameResultSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SameResultSuite.scala
@@ -32,7 +32,7 @@ class SameResultSuite extends FunSuite {
val testRelation = LocalRelation('a.int, 'b.int, 'c.int)
val testRelation2 = LocalRelation('a.int, 'b.int, 'c.int)
- def assertSameResult(a: LogicalPlan, b: LogicalPlan, result: Boolean = true) = {
+ def assertSameResult(a: LogicalPlan, b: LogicalPlan, result: Boolean = true): Unit = {
val aAnalyzed = a.analyze
val bAnalyzed = b.analyze
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
index 274f3ed..4eb8708 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
@@ -25,12 +25,12 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.types.{StringType, NullType}
case class Dummy(optKey: Option[Expression]) extends Expression {
- def children = optKey.toSeq
- def nullable = true
- def dataType = NullType
+ def children: Seq[Expression] = optKey.toSeq
+ def nullable: Boolean = true
+ def dataType: NullType = NullType
override lazy val resolved = true
type EvaluatedType = Any
- def eval(input: Row) = null.asInstanceOf[Any]
+ def eval(input: Row): Any = null.asInstanceOf[Any]
}
class TreeNodeSuite extends FunSuite {
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index c240f2b..f7b5f08 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -92,7 +92,8 @@ class CachedTableSuite extends QueryTest {
test("too big for memory") {
val data = "*" * 10000
- sparkContext.parallelize(1 to 200000, 1).map(_ => BigData(data)).toDF().registerTempTable("bigData")
+ sparkContext.parallelize(1 to 200000, 1).map(_ => BigData(data)).toDF()
+ .registerTempTable("bigData")
table("bigData").persist(StorageLevel.MEMORY_AND_DISK)
assert(table("bigData").count() === 200000L)
table("bigData").unpersist(blocking = true)
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 1db0cf7..f5df8c6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -329,8 +329,9 @@ class DataFrameSuite extends QueryTest {
checkAnswer(
decimalData.agg(avg('a cast DecimalType(10, 2))),
Row(new java.math.BigDecimal(2.0)))
+ // non-partial
checkAnswer(
- decimalData.agg(avg('a cast DecimalType(10, 2)), sumDistinct('a cast DecimalType(10, 2))), // non-partial
+ decimalData.agg(avg('a cast DecimalType(10, 2)), sumDistinct('a cast DecimalType(10, 2))),
Row(new java.math.BigDecimal(2.0), new java.math.BigDecimal(6)) :: Nil)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
index 9b4dd6c..9a81fc5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
@@ -67,7 +67,7 @@ class QueryTest extends PlanTest {
checkAnswer(df, Seq(expectedAnswer))
}
- def sqlTest(sqlString: String, expectedAnswer: Seq[Row])(implicit sqlContext: SQLContext): Unit = {
+ def sqlTest(sqlString: String, expectedAnswer: Seq[Row])(implicit sqlContext: SQLContext) {
test(sqlString) {
checkAnswer(sqlContext.sql(sqlString), expectedAnswer)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 1ad92a3..1392b48 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -268,7 +268,10 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
Row(java.sql.Timestamp.valueOf("1969-12-31 16:00:00.002")))
checkAnswer(sql(
- "SELECT time FROM timestamps WHERE time IN ('1969-12-31 16:00:00.001','1969-12-31 16:00:00.002')"),
+ """
+ |SELECT time FROM timestamps
+ |WHERE time IN ('1969-12-31 16:00:00.001','1969-12-31 16:00:00.002')
+ """.stripMargin),
Seq(Row(java.sql.Timestamp.valueOf("1969-12-31 16:00:00.001")),
Row(java.sql.Timestamp.valueOf("1969-12-31 16:00:00.002"))))
@@ -334,7 +337,7 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
Row("1"))
}
- def sortTest() = {
+ def sortTest(): Unit = {
checkAnswer(
sql("SELECT * FROM testData2 ORDER BY a ASC, b ASC"),
Seq(Row(1,1), Row(1,2), Row(2,1), Row(2,2), Row(3,1), Row(3,2)))
@@ -413,7 +416,10 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
test("from follow multiple brackets") {
checkAnswer(sql(
- "select key from ((select * from testData limit 1) union all (select * from testData limit 1)) x limit 1"),
+ """
+ |select key from ((select * from testData limit 1)
+ | union all (select * from testData limit 1)) x limit 1
+ """.stripMargin),
Row(1)
)
@@ -423,7 +429,11 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
)
checkAnswer(sql(
- "select key from (select * from testData limit 1 union all select * from testData limit 1) x limit 1"),
+ """
+ |select key from
+ | (select * from testData limit 1 union all select * from testData limit 1) x
+ | limit 1
+ """.stripMargin),
Row(1)
)
}
@@ -470,7 +480,10 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
Seq(Row(1, 0), Row(2, 1)))
checkAnswer(
- sql("SELECT COUNT(a), COUNT(b), COUNT(1), COUNT(DISTINCT a), COUNT(DISTINCT b) FROM testData3"),
+ sql(
+ """
+ |SELECT COUNT(a), COUNT(b), COUNT(1), COUNT(DISTINCT a), COUNT(DISTINCT b) FROM testData3
+ """.stripMargin),
Row(2, 1, 2, 2, 1))
}
@@ -1083,7 +1096,8 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
}
test("SPARK-3483 Special chars in column names") {
- val data = sparkContext.parallelize(Seq("""{"key?number1": "value1", "key.number2": "value2"}"""))
+ val data = sparkContext.parallelize(
+ Seq("""{"key?number1": "value1", "key.number2": "value2"}"""))
jsonRDD(data).registerTempTable("records")
sql("SELECT `key?number1` FROM records")
}
@@ -1168,8 +1182,8 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
}
test("SPARK-6145: ORDER BY test for nested fields") {
- jsonRDD(sparkContext.makeRDD(
- """{"a": {"b": 1, "a": {"a": 1}}, "c": [{"d": 1}]}""" :: Nil)).registerTempTable("nestedOrder")
+ jsonRDD(sparkContext.makeRDD("""{"a": {"b": 1, "a": {"a": 1}}, "c": [{"d": 1}]}""" :: Nil))
+ .registerTempTable("nestedOrder")
checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY a.b"), Row(1))
checkAnswer(sql("SELECT a.b FROM nestedOrder ORDER BY a.b"), Row(1))
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
index 17e923c..3fa00fd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
@@ -80,7 +80,7 @@ class ScalaReflectionRelationSuite extends FunSuite {
test("query case class RDD") {
val data = ReflectData("a", 1, 1L, 1.toFloat, 1.toDouble, 1.toShort, 1.toByte, true,
- new java.math.BigDecimal(1), new Date(12345), new Timestamp(12345), Seq(1,2,3))
+ new java.math.BigDecimal(1), new Date(12345), new Timestamp(12345), Seq(1,2,3))
val rdd = sparkContext.parallelize(data :: Nil)
rdd.toDF().registerTempTable("reflectData")
@@ -103,7 +103,8 @@ class ScalaReflectionRelationSuite extends FunSuite {
val rdd = sparkContext.parallelize(data :: Nil)
rdd.toDF().registerTempTable("reflectOptionalData")
- assert(sql("SELECT * FROM reflectOptionalData").collect().head === Row.fromSeq(Seq.fill(7)(null)))
+ assert(sql("SELECT * FROM reflectOptionalData").collect().head ===
+ Row.fromSeq(Seq.fill(7)(null)))
}
// Equality is broken for Arrays, so we test that separately.
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
index fe618e0..902da5c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
@@ -63,7 +63,7 @@ private[sql] class MyDenseVectorUDT extends UserDefinedType[MyDenseVector] {
}
}
- override def userClass = classOf[MyDenseVector]
+ override def userClass: Class[MyDenseVector] = classOf[MyDenseVector]
private[spark] override def asNullable: MyDenseVectorUDT = this
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnarTestUtils.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnarTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnarTestUtils.scala
index c7a4084..b301818 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnarTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnarTestUtils.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.GenericMutableRow
import org.apache.spark.sql.types.{Decimal, DataType, NativeType}
object ColumnarTestUtils {
- def makeNullRow(length: Int) = {
+ def makeNullRow(length: Int): GenericMutableRow = {
val row = new GenericMutableRow(length)
(0 until length).foreach(row.setNullAt)
row
@@ -93,7 +93,7 @@ object ColumnarTestUtils {
def makeUniqueValuesAndSingleValueRows[T <: NativeType](
columnType: NativeColumnType[T],
- count: Int) = {
+ count: Int): (Seq[T#JvmType], Seq[GenericMutableRow]) = {
val values = makeUniqueRandomValues(columnType, count)
val rows = values.map { value =>
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/columnar/NullableColumnAccessorSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/NullableColumnAccessorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/NullableColumnAccessorSuite.scala
index bb30535..a070214 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/NullableColumnAccessorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/NullableColumnAccessorSuite.scala
@@ -31,7 +31,8 @@ class TestNullableColumnAccessor[T <: DataType, JvmType](
with NullableColumnAccessor
object TestNullableColumnAccessor {
- def apply[T <: DataType, JvmType](buffer: ByteBuffer, columnType: ColumnType[T, JvmType]) = {
+ def apply[T <: DataType, JvmType](buffer: ByteBuffer, columnType: ColumnType[T, JvmType])
+ : TestNullableColumnAccessor[T, JvmType] = {
// Skips the column type ID
buffer.getInt()
new TestNullableColumnAccessor(buffer, columnType)
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/columnar/NullableColumnBuilderSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/NullableColumnBuilderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/NullableColumnBuilderSuite.scala
index 75a4749..3a5605d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/NullableColumnBuilderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/NullableColumnBuilderSuite.scala
@@ -27,7 +27,8 @@ class TestNullableColumnBuilder[T <: DataType, JvmType](columnType: ColumnType[T
with NullableColumnBuilder
object TestNullableColumnBuilder {
- def apply[T <: DataType, JvmType](columnType: ColumnType[T, JvmType], initialSize: Int = 0) = {
+ def apply[T <: DataType, JvmType](columnType: ColumnType[T, JvmType], initialSize: Int = 0)
+ : TestNullableColumnBuilder[T, JvmType] = {
val builder = new TestNullableColumnBuilder(columnType)
builder.initialize(initialSize)
builder
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/TestCompressibleColumnBuilder.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/TestCompressibleColumnBuilder.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/TestCompressibleColumnBuilder.scala
index 0b18b41..fc8ff3b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/TestCompressibleColumnBuilder.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/TestCompressibleColumnBuilder.scala
@@ -35,7 +35,7 @@ object TestCompressibleColumnBuilder {
def apply[T <: NativeType](
columnStats: ColumnStats,
columnType: NativeColumnType[T],
- scheme: CompressionScheme) = {
+ scheme: CompressionScheme): TestCompressibleColumnBuilder[T] = {
val builder = new TestCompressibleColumnBuilder(columnStats, columnType, Seq(scheme))
builder.initialize(0, "", useCompression = true)
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
index 4e9472c..358d8cf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
@@ -30,4 +30,4 @@ class DebuggingSuite extends FunSuite {
test("DataFrame.typeCheck()") {
testData.typeCheck()
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index 592ed4b..3596b18 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -45,10 +45,12 @@ class JDBCSuite extends FunSuite with BeforeAndAfter {
conn = DriverManager.getConnection(url, properties)
conn.prepareStatement("create schema test").executeUpdate()
- conn.prepareStatement("create table test.people (name TEXT(32) NOT NULL, theid INTEGER NOT NULL)").executeUpdate()
+ conn.prepareStatement(
+ "create table test.people (name TEXT(32) NOT NULL, theid INTEGER NOT NULL)").executeUpdate()
conn.prepareStatement("insert into test.people values ('fred', 1)").executeUpdate()
conn.prepareStatement("insert into test.people values ('mary', 2)").executeUpdate()
- conn.prepareStatement("insert into test.people values ('joe ''foo'' \"bar\"', 3)").executeUpdate()
+ conn.prepareStatement(
+ "insert into test.people values ('joe ''foo'' \"bar\"', 3)").executeUpdate()
conn.commit()
sql(
@@ -132,25 +134,25 @@ class JDBCSuite extends FunSuite with BeforeAndAfter {
}
test("SELECT *") {
- assert(sql("SELECT * FROM foobar").collect().size == 3)
+ assert(sql("SELECT * FROM foobar").collect().size === 3)
}
test("SELECT * WHERE (simple predicates)") {
- assert(sql("SELECT * FROM foobar WHERE THEID < 1").collect().size == 0)
- assert(sql("SELECT * FROM foobar WHERE THEID != 2").collect().size == 2)
- assert(sql("SELECT * FROM foobar WHERE THEID = 1").collect().size == 1)
- assert(sql("SELECT * FROM foobar WHERE NAME = 'fred'").collect().size == 1)
- assert(sql("SELECT * FROM foobar WHERE NAME > 'fred'").collect().size == 2)
- assert(sql("SELECT * FROM foobar WHERE NAME != 'fred'").collect().size == 2)
+ assert(sql("SELECT * FROM foobar WHERE THEID < 1").collect().size === 0)
+ assert(sql("SELECT * FROM foobar WHERE THEID != 2").collect().size === 2)
+ assert(sql("SELECT * FROM foobar WHERE THEID = 1").collect().size === 1)
+ assert(sql("SELECT * FROM foobar WHERE NAME = 'fred'").collect().size === 1)
+ assert(sql("SELECT * FROM foobar WHERE NAME > 'fred'").collect().size === 2)
+ assert(sql("SELECT * FROM foobar WHERE NAME != 'fred'").collect().size === 2)
}
test("SELECT * WHERE (quoted strings)") {
- assert(sql("select * from foobar").where('NAME === "joe 'foo' \"bar\"").collect().size == 1)
+ assert(sql("select * from foobar").where('NAME === "joe 'foo' \"bar\"").collect().size === 1)
}
test("SELECT first field") {
val names = sql("SELECT NAME FROM foobar").collect().map(x => x.getString(0)).sortWith(_ < _)
- assert(names.size == 3)
+ assert(names.size === 3)
assert(names(0).equals("fred"))
assert(names(1).equals("joe 'foo' \"bar\""))
assert(names(2).equals("mary"))
@@ -158,10 +160,10 @@ class JDBCSuite extends FunSuite with BeforeAndAfter {
test("SELECT second field") {
val ids = sql("SELECT THEID FROM foobar").collect().map(x => x.getInt(0)).sortWith(_ < _)
- assert(ids.size == 3)
- assert(ids(0) == 1)
- assert(ids(1) == 2)
- assert(ids(2) == 3)
+ assert(ids.size === 3)
+ assert(ids(0) === 1)
+ assert(ids(1) === 2)
+ assert(ids(2) === 3)
}
test("SELECT * partitioned") {
@@ -169,46 +171,46 @@ class JDBCSuite extends FunSuite with BeforeAndAfter {
}
test("SELECT WHERE (simple predicates) partitioned") {
- assert(sql("SELECT * FROM parts WHERE THEID < 1").collect().size == 0)
- assert(sql("SELECT * FROM parts WHERE THEID != 2").collect().size == 2)
- assert(sql("SELECT THEID FROM parts WHERE THEID = 1").collect().size == 1)
+ assert(sql("SELECT * FROM parts WHERE THEID < 1").collect().size === 0)
+ assert(sql("SELECT * FROM parts WHERE THEID != 2").collect().size === 2)
+ assert(sql("SELECT THEID FROM parts WHERE THEID = 1").collect().size === 1)
}
test("SELECT second field partitioned") {
val ids = sql("SELECT THEID FROM parts").collect().map(x => x.getInt(0)).sortWith(_ < _)
- assert(ids.size == 3)
- assert(ids(0) == 1)
- assert(ids(1) == 2)
- assert(ids(2) == 3)
+ assert(ids.size === 3)
+ assert(ids(0) === 1)
+ assert(ids(1) === 2)
+ assert(ids(2) === 3)
}
test("Basic API") {
- assert(TestSQLContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE").collect.size == 3)
+ assert(TestSQLContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE").collect().size === 3)
}
test("Partitioning via JDBCPartitioningInfo API") {
assert(TestSQLContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE", "THEID", 0, 4, 3)
- .collect.size == 3)
+ .collect.size === 3)
}
test("Partitioning via list-of-where-clauses API") {
val parts = Array[String]("THEID < 2", "THEID >= 2")
- assert(TestSQLContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE", parts).collect.size == 3)
+ assert(TestSQLContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE", parts).collect().size === 3)
}
test("H2 integral types") {
val rows = sql("SELECT * FROM inttypes WHERE A IS NOT NULL").collect()
- assert(rows.size == 1)
- assert(rows(0).getInt(0) == 1)
- assert(rows(0).getBoolean(1) == false)
- assert(rows(0).getInt(2) == 3)
- assert(rows(0).getInt(3) == 4)
- assert(rows(0).getLong(4) == 1234567890123L)
+ assert(rows.size === 1)
+ assert(rows(0).getInt(0) === 1)
+ assert(rows(0).getBoolean(1) === false)
+ assert(rows(0).getInt(2) === 3)
+ assert(rows(0).getInt(3) === 4)
+ assert(rows(0).getLong(4) === 1234567890123L)
}
test("H2 null entries") {
val rows = sql("SELECT * FROM inttypes WHERE A IS NULL").collect()
- assert(rows.size == 1)
+ assert(rows.size === 1)
assert(rows(0).isNullAt(0))
assert(rows(0).isNullAt(1))
assert(rows(0).isNullAt(2))
@@ -230,27 +232,27 @@ class JDBCSuite extends FunSuite with BeforeAndAfter {
val rows = sql("SELECT * FROM timetypes").collect()
val cal = new GregorianCalendar(java.util.Locale.ROOT)
cal.setTime(rows(0).getAs[java.sql.Timestamp](0))
- assert(cal.get(Calendar.HOUR_OF_DAY) == 12)
- assert(cal.get(Calendar.MINUTE) == 34)
- assert(cal.get(Calendar.SECOND) == 56)
+ assert(cal.get(Calendar.HOUR_OF_DAY) === 12)
+ assert(cal.get(Calendar.MINUTE) === 34)
+ assert(cal.get(Calendar.SECOND) === 56)
cal.setTime(rows(0).getAs[java.sql.Timestamp](1))
- assert(cal.get(Calendar.YEAR) == 1996)
- assert(cal.get(Calendar.MONTH) == 0)
- assert(cal.get(Calendar.DAY_OF_MONTH) == 1)
+ assert(cal.get(Calendar.YEAR) === 1996)
+ assert(cal.get(Calendar.MONTH) === 0)
+ assert(cal.get(Calendar.DAY_OF_MONTH) === 1)
cal.setTime(rows(0).getAs[java.sql.Timestamp](2))
- assert(cal.get(Calendar.YEAR) == 2002)
- assert(cal.get(Calendar.MONTH) == 1)
- assert(cal.get(Calendar.DAY_OF_MONTH) == 20)
- assert(cal.get(Calendar.HOUR) == 11)
- assert(cal.get(Calendar.MINUTE) == 22)
- assert(cal.get(Calendar.SECOND) == 33)
- assert(rows(0).getAs[java.sql.Timestamp](2).getNanos == 543543543)
+ assert(cal.get(Calendar.YEAR) === 2002)
+ assert(cal.get(Calendar.MONTH) === 1)
+ assert(cal.get(Calendar.DAY_OF_MONTH) === 20)
+ assert(cal.get(Calendar.HOUR) === 11)
+ assert(cal.get(Calendar.MINUTE) === 22)
+ assert(cal.get(Calendar.SECOND) === 33)
+ assert(rows(0).getAs[java.sql.Timestamp](2).getNanos === 543543543)
}
test("H2 floating-point types") {
val rows = sql("SELECT * FROM flttypes").collect()
- assert(rows(0).getDouble(0) == 1.00000000000000022) // Yes, I meant ==.
- assert(rows(0).getDouble(1) == 1.00000011920928955) // Yes, I meant ==.
+ assert(rows(0).getDouble(0) === 1.00000000000000022) // Yes, I meant ==.
+ assert(rows(0).getDouble(1) === 1.00000011920928955) // Yes, I meant ==.
assert(rows(0).getAs[BigDecimal](2)
.equals(new BigDecimal("123456789012345.54321543215432100000")))
}
@@ -264,7 +266,7 @@ class JDBCSuite extends FunSuite with BeforeAndAfter {
| user 'testUser', password 'testPass')
""".stripMargin.replaceAll("\n", " "))
val rows = sql("SELECT * FROM hack").collect()
- assert(rows(0).getDouble(0) == 1.00000011920928955) // Yes, I meant ==.
+ assert(rows(0).getDouble(0) === 1.00000011920928955) // Yes, I meant ==.
// For some reason, H2 computes this square incorrectly...
assert(math.abs(rows(0).getDouble(1) - 1.00000023841859331) < 1e-12)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
index 706c966..1fe0b76 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
@@ -380,8 +380,10 @@ class JsonSuite extends QueryTest {
sql("select * from jsonTable"),
Row("true", 11L, null, 1.1, "13.1", "str1") ::
Row("12", null, new java.math.BigDecimal("21474836470.9"), null, null, "true") ::
- Row("false", 21474836470L, new java.math.BigDecimal("92233720368547758070"), 100, "str1", "false") ::
- Row(null, 21474836570L, new java.math.BigDecimal("1.1"), 21474836470L, "92233720368547758070", null) :: Nil
+ Row("false", 21474836470L,
+ new java.math.BigDecimal("92233720368547758070"), 100, "str1", "false") ::
+ Row(null, 21474836570L,
+ new java.math.BigDecimal("1.1"), 21474836470L, "92233720368547758070", null) :: Nil
)
// Number and Boolean conflict: resolve the type as number in this query.
@@ -404,7 +406,8 @@ class JsonSuite extends QueryTest {
// Widening to DecimalType
checkAnswer(
sql("select num_num_2 + 1.2 from jsonTable where num_num_2 > 1.1"),
- Row(new java.math.BigDecimal("21474836472.1")) :: Row(new java.math.BigDecimal("92233720368547758071.2")) :: Nil
+ Row(new java.math.BigDecimal("21474836472.1")) ::
+ Row(new java.math.BigDecimal("92233720368547758071.2")) :: Nil
)
// Widening to DoubleType
@@ -913,8 +916,10 @@ class JsonSuite extends QueryTest {
df1.registerTempTable("applySchema1")
val df2 = df1.toDF
val result = df2.toJSON.collect()
+ // scalastyle:off
assert(result(0) === "{\"f1\":1,\"f2\":\"A1\",\"f3\":true,\"f4\":[\"1\",\" A1\",\" true\",\" null\"]}")
assert(result(3) === "{\"f1\":4,\"f2\":\"D4\",\"f3\":true,\"f4\":[\"4\",\" D4\",\" true\",\" 2147483644\"],\"f5\":2147483644}")
+ // scalastyle:on
val schema2 = StructType(
StructField("f1", StructType(
@@ -968,7 +973,8 @@ class JsonSuite extends QueryTest {
// Access elements of a BigInteger array (we use DecimalType internally).
checkAnswer(
- sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] from complexTable"),
+ sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] " +
+ " from complexTable"),
Row(new java.math.BigDecimal("922337203685477580700"),
new java.math.BigDecimal("-922337203685477580800"), null)
)
@@ -1008,7 +1014,8 @@ class JsonSuite extends QueryTest {
// Access elements of an array field of a struct.
checkAnswer(
- sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] from complexTable"),
+ sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] " +
+ "from complexTable"),
Row(5, null)
)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetIOSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetIOSuite.scala
index 203bc79..97c0f43 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetIOSuite.scala
@@ -218,7 +218,7 @@ class ParquetIOSuiteBase extends QueryTest with ParquetTest {
}
test("compression codec") {
- def compressionCodecFor(path: String) = {
+ def compressionCodecFor(path: String): String = {
val codecs = ParquetTypesConverter
.readMetaData(new Path(path), Some(configuration))
.getBlocks
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala
index 61f1cf3..c964b6d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala
@@ -180,10 +180,12 @@ class ParquetSchemaSuite extends FunSuite with ParquetTest {
val caseClassString =
"StructType(List(StructField(c1,IntegerType,false), StructField(c2,BinaryType,true)))"
+ // scalastyle:off
val jsonString =
"""
|{"type":"struct","fields":[{"name":"c1","type":"integer","nullable":false,"metadata":{}},{"name":"c2","type":"binary","nullable":true,"metadata":{}}]}
""".stripMargin
+ // scalastyle:on
val fromCaseClassString = ParquetTypesConverter.convertFromString(caseClassString)
val fromJson = ParquetTypesConverter.convertFromString(jsonString)
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala
index 54af50c..3f24a49 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/DDLTestSuite.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql.sources
+import org.apache.spark.rdd.RDD
import org.apache.spark.sql._
import org.apache.spark.sql.types._
@@ -31,7 +32,7 @@ class DDLScanSource extends RelationProvider {
case class SimpleDDLScan(from: Int, to: Int)(@transient val sqlContext: SQLContext)
extends BaseRelation with TableScan {
- override def schema =
+ override def schema: StructType =
StructType(Seq(
StructField("intType", IntegerType, nullable = false,
new MetadataBuilder().putString("comment", "test comment").build()),
@@ -57,8 +58,9 @@ case class SimpleDDLScan(from: Int, to: Int)(@transient val sqlContext: SQLConte
))
- override def buildScan() = sqlContext.sparkContext.parallelize(from to to).
- map(e => Row(s"people$e", e * 2))
+ override def buildScan(): RDD[Row] = {
+ sqlContext.sparkContext.parallelize(from to to).map(e => Row(s"people$e", e * 2))
+ }
}
class DDLTestSuite extends DataSourceTest {
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala
index 773bd16..cb5e514 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/FilteredScanSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.sources
import scala.language.existentials
+import org.apache.spark.rdd.RDD
import org.apache.spark.sql._
import org.apache.spark.sql.types._
@@ -41,7 +42,7 @@ case class SimpleFilteredScan(from: Int, to: Int)(@transient val sqlContext: SQL
StructField("b", IntegerType, nullable = false) ::
StructField("c", StringType, nullable = false) :: Nil)
- override def buildScan(requiredColumns: Array[String], filters: Array[Filter]) = {
+ override def buildScan(requiredColumns: Array[String], filters: Array[Filter]): RDD[Row] = {
val rowBuilders = requiredColumns.map {
case "a" => (i: Int) => Seq(i)
case "b" => (i: Int) => Seq(i * 2)
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala
index 08fb538..6a1ddf2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/PrunedScanSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.sources
import scala.language.existentials
+import org.apache.spark.rdd.RDD
import org.apache.spark.sql._
import org.apache.spark.sql.types._
@@ -34,12 +35,12 @@ case class SimplePrunedScan(from: Int, to: Int)(@transient val sqlContext: SQLCo
extends BaseRelation
with PrunedScan {
- override def schema =
+ override def schema: StructType =
StructType(
StructField("a", IntegerType, nullable = false) ::
StructField("b", IntegerType, nullable = false) :: Nil)
- override def buildScan(requiredColumns: Array[String]) = {
+ override def buildScan(requiredColumns: Array[String]): RDD[Row] = {
val rowBuilders = requiredColumns.map {
case "a" => (i: Int) => Seq(i)
case "b" => (i: Int) => Seq(i * 2)
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala
index 43bc8eb..cb287ba 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala
@@ -114,4 +114,4 @@ class SaveLoadSuite extends DataSourceTest with BeforeAndAfterAll {
message.contains("Append mode is not supported"),
"We should complain that 'Append mode is not supported' for JSON source.")
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala
index 7928600..60c8c00 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.sources
import java.sql.{Timestamp, Date}
+import org.apache.spark.rdd.RDD
import org.apache.spark.sql._
import org.apache.spark.sql.types._
@@ -35,10 +36,10 @@ class SimpleScanSource extends RelationProvider {
case class SimpleScan(from: Int, to: Int)(@transient val sqlContext: SQLContext)
extends BaseRelation with TableScan {
- override def schema =
+ override def schema: StructType =
StructType(StructField("i", IntegerType, nullable = false) :: Nil)
- override def buildScan() = sqlContext.sparkContext.parallelize(from to to).map(Row(_))
+ override def buildScan(): RDD[Row] = sqlContext.sparkContext.parallelize(from to to).map(Row(_))
}
class AllDataTypesScanSource extends SchemaRelationProvider {
@@ -57,9 +58,9 @@ case class AllDataTypesScan(
extends BaseRelation
with TableScan {
- override def schema = userSpecifiedSchema
+ override def schema: StructType = userSpecifiedSchema
- override def buildScan() = {
+ override def buildScan(): RDD[Row] = {
sqlContext.sparkContext.parallelize(from to to).map { i =>
Row(
s"str_$i",
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/ErrorPositionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ErrorPositionSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ErrorPositionSuite.scala
index 968557c..d960a30 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ErrorPositionSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ErrorPositionSuite.scala
@@ -136,7 +136,7 @@ class ErrorPositionSuite extends QueryTest with BeforeAndAfter {
* @param query the query to analyze
* @param token a unique token in the string that should be indicated by the exception
*/
- def positionTest(name: String, query: String, token: String) = {
+ def positionTest(name: String, query: String, token: String): Unit = {
def parseTree =
Try(quietly(HiveQl.dumpTree(HiveQl.getAst(query)))).getOrElse("<failed to parse>")
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index c482c6d..2a7374c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -116,21 +116,20 @@ class HiveInspectorSuite extends FunSuite with HiveInspectors {
}
def checkDataType(dt1: Seq[DataType], dt2: Seq[DataType]): Unit = {
- dt1.zip(dt2).map {
- case (dd1, dd2) =>
- assert(dd1.getClass === dd2.getClass) // DecimalType doesn't has the default precision info
+ dt1.zip(dt2).foreach { case (dd1, dd2) =>
+ assert(dd1.getClass === dd2.getClass) // DecimalType doesn't has the default precision info
}
}
def checkValues(row1: Seq[Any], row2: Seq[Any]): Unit = {
- row1.zip(row2).map {
- case (r1, r2) => checkValue(r1, r2)
+ row1.zip(row2).foreach { case (r1, r2) =>
+ checkValue(r1, r2)
}
}
def checkValues(row1: Seq[Any], row2: Row): Unit = {
- row1.zip(row2.toSeq).map {
- case (r1, r2) => checkValue(r1, r2)
+ row1.zip(row2.toSeq).foreach { case (r1, r2) =>
+ checkValue(r1, r2)
}
}
@@ -141,7 +140,7 @@ class HiveInspectorSuite extends FunSuite with HiveInspectors {
assert(r1.compare(r2) === 0)
case (r1: Array[Byte], r2: Array[Byte])
if r1 != null && r2 != null && r1.length == r2.length =>
- r1.zip(r2).map { case (b1, b2) => assert(b1 === b2) }
+ r1.zip(r2).foreach { case (b1, b2) => assert(b1 === b2) }
case (r1, r2) => assert(r1 === r2)
}
}
@@ -166,7 +165,8 @@ class HiveInspectorSuite extends FunSuite with HiveInspectors {
val constantData = constantExprs.map(_.eval())
val constantNullData = constantData.map(_ => null)
val constantWritableOIs = constantExprs.map(e => toWritableInspector(e.dataType))
- val constantNullWritableOIs = constantExprs.map(e => toInspector(Literal.create(null, e.dataType)))
+ val constantNullWritableOIs =
+ constantExprs.map(e => toInspector(Literal.create(null, e.dataType)))
checkValues(constantData, constantData.zip(constantWritableOIs).map {
case (d, oi) => unwrap(wrap(d, oi), oi)
@@ -202,7 +202,8 @@ class HiveInspectorSuite extends FunSuite with HiveInspectors {
case (t, idx) => StructField(s"c_$idx", t)
})
- checkValues(row, unwrap(wrap(Row.fromSeq(row), toInspector(dt)), toInspector(dt)).asInstanceOf[Row])
+ checkValues(row,
+ unwrap(wrap(Row.fromSeq(row), toInspector(dt)), toInspector(dt)).asInstanceOf[Row])
checkValue(null, unwrap(wrap(null, toInspector(dt)), toInspector(dt)))
}
@@ -212,8 +213,10 @@ class HiveInspectorSuite extends FunSuite with HiveInspectors {
val d = row(0) :: row(0) :: Nil
checkValue(d, unwrap(wrap(d, toInspector(dt)), toInspector(dt)))
checkValue(null, unwrap(wrap(null, toInspector(dt)), toInspector(dt)))
- checkValue(d, unwrap(wrap(d, toInspector(Literal.create(d, dt))), toInspector(Literal.create(d, dt))))
- checkValue(d, unwrap(wrap(null, toInspector(Literal.create(d, dt))), toInspector(Literal.create(d, dt))))
+ checkValue(d,
+ unwrap(wrap(d, toInspector(Literal.create(d, dt))), toInspector(Literal.create(d, dt))))
+ checkValue(d,
+ unwrap(wrap(null, toInspector(Literal.create(d, dt))), toInspector(Literal.create(d, dt))))
}
test("wrap / unwrap Map Type") {
@@ -222,7 +225,9 @@ class HiveInspectorSuite extends FunSuite with HiveInspectors {
val d = Map(row(0) -> row(1))
checkValue(d, unwrap(wrap(d, toInspector(dt)), toInspector(dt)))
checkValue(null, unwrap(wrap(null, toInspector(dt)), toInspector(dt)))
- checkValue(d, unwrap(wrap(d, toInspector(Literal.create(d, dt))), toInspector(Literal.create(d, dt))))
- checkValue(d, unwrap(wrap(null, toInspector(Literal.create(d, dt))), toInspector(Literal.create(d, dt))))
+ checkValue(d,
+ unwrap(wrap(d, toInspector(Literal.create(d, dt))), toInspector(Literal.create(d, dt))))
+ checkValue(d,
+ unwrap(wrap(null, toInspector(Literal.create(d, dt))), toInspector(Literal.create(d, dt))))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
index 8011952..ecb990e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
@@ -115,11 +115,36 @@ class InsertIntoHiveTableSuite extends QueryTest with BeforeAndAfter {
test("SPARK-4203:random partition directory order") {
sql("CREATE TABLE tmp_table (key int, value string)")
val tmpDir = Utils.createTempDir()
- sql(s"CREATE TABLE table_with_partition(c1 string) PARTITIONED by (p1 string,p2 string,p3 string,p4 string,p5 string) location '${tmpDir.toURI.toString}' ")
- sql("INSERT OVERWRITE TABLE table_with_partition partition (p1='a',p2='b',p3='c',p4='c',p5='1') SELECT 'blarr' FROM tmp_table")
- sql("INSERT OVERWRITE TABLE table_with_partition partition (p1='a',p2='b',p3='c',p4='c',p5='2') SELECT 'blarr' FROM tmp_table")
- sql("INSERT OVERWRITE TABLE table_with_partition partition (p1='a',p2='b',p3='c',p4='c',p5='3') SELECT 'blarr' FROM tmp_table")
- sql("INSERT OVERWRITE TABLE table_with_partition partition (p1='a',p2='b',p3='c',p4='c',p5='4') SELECT 'blarr' FROM tmp_table")
+ sql(
+ s"""
+ |CREATE TABLE table_with_partition(c1 string)
+ |PARTITIONED by (p1 string,p2 string,p3 string,p4 string,p5 string)
+ |location '${tmpDir.toURI.toString}'
+ """.stripMargin)
+ sql(
+ """
+ |INSERT OVERWRITE TABLE table_with_partition
+ |partition (p1='a',p2='b',p3='c',p4='c',p5='1')
+ |SELECT 'blarr' FROM tmp_table
+ """.stripMargin)
+ sql(
+ """
+ |INSERT OVERWRITE TABLE table_with_partition
+ |partition (p1='a',p2='b',p3='c',p4='c',p5='2')
+ |SELECT 'blarr' FROM tmp_table
+ """.stripMargin)
+ sql(
+ """
+ |INSERT OVERWRITE TABLE table_with_partition
+ |partition (p1='a',p2='b',p3='c',p4='c',p5='3')
+ |SELECT 'blarr' FROM tmp_table
+ """.stripMargin)
+ sql(
+ """
+ |INSERT OVERWRITE TABLE table_with_partition
+ |partition (p1='a',p2='b',p3='c',p4='c',p5='4')
+ |SELECT 'blarr' FROM tmp_table
+ """.stripMargin)
def listFolders(path: File, acc: List[String]): List[List[String]] = {
val dir = path.listFiles()
val folders = dir.filter(_.isDirectory).toList
@@ -196,34 +221,42 @@ class InsertIntoHiveTableSuite extends QueryTest with BeforeAndAfter {
testData.registerTempTable("testData")
val testDatawithNull = TestHive.sparkContext.parallelize(
- (1 to 10).map(i => ThreeCloumntable(i, i.toString,null))).toDF()
+ (1 to 10).map(i => ThreeCloumntable(i, i.toString, null))).toDF()
val tmpDir = Utils.createTempDir()
- sql(s"CREATE TABLE table_with_partition(key int,value string) PARTITIONED by (ds string) location '${tmpDir.toURI.toString}' ")
- sql("INSERT OVERWRITE TABLE table_with_partition partition (ds='1') SELECT key,value FROM testData")
+ sql(
+ s"""
+ |CREATE TABLE table_with_partition(key int,value string)
+ |PARTITIONED by (ds string) location '${tmpDir.toURI.toString}'
+ """.stripMargin)
+ sql(
+ """
+ |INSERT OVERWRITE TABLE table_with_partition
+ |partition (ds='1') SELECT key,value FROM testData
+ """.stripMargin)
// test schema the same between partition and table
sql("ALTER TABLE table_with_partition CHANGE COLUMN key key BIGINT")
checkAnswer(sql("select key,value from table_with_partition where ds='1' "),
- testData.collect.toSeq
+ testData.collect().toSeq
)
// test difference type of field
sql("ALTER TABLE table_with_partition CHANGE COLUMN key key BIGINT")
checkAnswer(sql("select key,value from table_with_partition where ds='1' "),
- testData.collect.toSeq
+ testData.collect().toSeq
)
// add column to table
sql("ALTER TABLE table_with_partition ADD COLUMNS(key1 string)")
checkAnswer(sql("select key,value,key1 from table_with_partition where ds='1' "),
- testDatawithNull.collect.toSeq
+ testDatawithNull.collect().toSeq
)
// change column name to table
sql("ALTER TABLE table_with_partition CHANGE COLUMN key keynew BIGINT")
checkAnswer(sql("select keynew,value from table_with_partition where ds='1' "),
- testData.collect.toSeq
+ testData.collect().toSeq
)
sql("DROP TABLE table_with_partition")
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
index ccd0e5a..00a69de 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
@@ -142,7 +142,7 @@ class StatisticsSuite extends QueryTest with BeforeAndAfterAll {
after: () => Unit,
query: String,
expectedAnswer: Seq[Row],
- ct: ClassTag[_]) = {
+ ct: ClassTag[_]): Unit = {
before()
var df = sql(query)
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/BigDataBenchmarkSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/BigDataBenchmarkSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/BigDataBenchmarkSuite.scala
index 42a82c1..a3f5921 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/BigDataBenchmarkSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/BigDataBenchmarkSuite.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.hive.test.TestHive._
class BigDataBenchmarkSuite extends HiveComparisonTest {
val testDataDirectory = new File("target" + File.separator + "big-data-benchmark-testdata")
+ val userVisitPath = new File(testDataDirectory, "uservisits").getCanonicalPath
val testTables = Seq(
TestTable(
"rankings",
@@ -63,7 +64,7 @@ class BigDataBenchmarkSuite extends HiveComparisonTest {
| searchWord STRING,
| duration INT)
| ROW FORMAT DELIMITED FIELDS TERMINATED BY ","
- | STORED AS TEXTFILE LOCATION "${new File(testDataDirectory, "uservisits").getCanonicalPath}"
+ | STORED AS TEXTFILE LOCATION "$userVisitPath"
""".stripMargin.cmd),
TestTable(
"documents",
@@ -83,7 +84,10 @@ class BigDataBenchmarkSuite extends HiveComparisonTest {
"SELECT pageURL, pageRank FROM rankings WHERE pageRank > 1")
createQueryTest("query2",
- "SELECT SUBSTR(sourceIP, 1, 10), SUM(adRevenue) FROM uservisits GROUP BY SUBSTR(sourceIP, 1, 10)")
+ """
+ |SELECT SUBSTR(sourceIP, 1, 10), SUM(adRevenue) FROM uservisits
+ |GROUP BY SUBSTR(sourceIP, 1, 10)
+ """.stripMargin)
createQueryTest("query3",
"""
@@ -113,8 +117,8 @@ class BigDataBenchmarkSuite extends HiveComparisonTest {
|CREATE TABLE url_counts_total AS
| SELECT SUM(count) AS totalCount, destpage
| FROM url_counts_partial GROUP BY destpage
- |-- The following queries run, but generate different results in HIVE likely because the UDF is not deterministic
- |-- given different input splits.
+ |-- The following queries run, but generate different results in HIVE
+ |-- likely because the UDF is not deterministic given different input splits.
|-- SELECT CAST(SUM(count) AS INT) FROM url_counts_partial
|-- SELECT COUNT(*) FROM url_counts_partial
|-- SELECT * FROM url_counts_partial
http://git-wip-us.apache.org/repos/asf/spark/blob/1b2aab8d/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
index a5ec312..027056d 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
@@ -255,8 +255,9 @@ abstract class HiveComparisonTest
.filterNot(_ contains "hive.outerjoin.supports.filters")
.filterNot(_ contains "hive.exec.post.hooks")
- if (allQueries != queryList)
+ if (allQueries != queryList) {
logWarning(s"Simplifications made on unsupported operations for test $testCaseName")
+ }
lazy val consoleTestCase = {
val quotes = "\"\"\""
@@ -305,13 +306,16 @@ abstract class HiveComparisonTest
try {
// Hooks often break the harness and don't really affect our test anyway, don't
// even try running them.
- if (installHooksCommand.findAllMatchIn(queryString).nonEmpty)
+ if (installHooksCommand.findAllMatchIn(queryString).nonEmpty) {
sys.error("hive exec hooks not supported for tests.")
+ }
- logWarning(s"Running query ${i+1}/${queryList.size} with hive.")
+ logWarning(s"Running query ${i + 1}/${queryList.size} with hive.")
// Analyze the query with catalyst to ensure test tables are loaded.
val answer = hiveQuery.analyzed match {
- case _: ExplainCommand => Nil // No need to execute EXPLAIN queries as we don't check the output.
+ case _: ExplainCommand =>
+ // No need to execute EXPLAIN queries as we don't check the output.
+ Nil
case _ => TestHive.runSqlHive(queryString)
}
@@ -394,21 +398,24 @@ abstract class HiveComparisonTest
case tf: org.scalatest.exceptions.TestFailedException => throw tf
case originalException: Exception =>
if (System.getProperty("spark.hive.canarytest") != null) {
- // When we encounter an error we check to see if the environment is still okay by running a simple query.
- // If this fails then we halt testing since something must have gone seriously wrong.
+ // When we encounter an error we check to see if the environment is still
+ // okay by running a simple query. If this fails then we halt testing since
+ // something must have gone seriously wrong.
try {
new TestHive.HiveQLQueryExecution("SELECT key FROM src").stringResult()
TestHive.runSqlHive("SELECT key FROM src")
} catch {
case e: Exception =>
- logError(s"FATAL ERROR: Canary query threw $e This implies that the testing environment has likely been corrupted.")
- // The testing setup traps exits so wait here for a long time so the developer can see when things started
- // to go wrong.
+ logError(s"FATAL ERROR: Canary query threw $e This implies that the " +
+ "testing environment has likely been corrupted.")
+ // The testing setup traps exits so wait here for a long time so the developer
+ // can see when things started to go wrong.
Thread.sleep(1000000)
}
}
- // If the canary query didn't fail then the environment is still okay, so just throw the original exception.
+ // If the canary query didn't fail then the environment is still okay,
+ // so just throw the original exception.
throw originalException
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org