You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/06/09 04:02:55 UTC
spark git commit: [SPARK-20954][SQL][BRANCH-2.2][EXTENDED] DESCRIBE `
result should be compatible with previous Spark
Repository: spark
Updated Branches:
refs/heads/branch-2.2 02cf178bb -> 3f6812cf8
[SPARK-20954][SQL][BRANCH-2.2][EXTENDED] DESCRIBE ` result should be compatible with previous Spark
## What changes were proposed in this pull request?
After [SPARK-20067](https://issues.apache.org/jira/browse/SPARK-20067), `DESCRIBE` and `DESCRIBE EXTENDED` shows the following result. This is incompatible with Spark 2.1.1. This PR removes the column header line in case of those commands.
**MASTER** and **BRANCH-2.2**
```scala
scala> sql("desc t").show(false)
+----------+---------+-------+
|col_name |data_type|comment|
+----------+---------+-------+
|# col_name|data_type|comment|
|a |int |null |
+----------+---------+-------+
```
**SPARK 2.1.1** and **this PR**
```scala
scala> sql("desc t").show(false)
+--------+---------+-------+
|col_name|data_type|comment|
+--------+---------+-------+
|a |int |null |
+--------+---------+-------+
```
## How was this patch tested?
Pass the Jenkins with the updated test suites.
Author: Dongjoon Hyun <do...@apache.org>
Closes #18245 from dongjoon-hyun/SPARK-20954-BRANCH-2.2.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3f6812cf
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3f6812cf
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3f6812cf
Branch: refs/heads/branch-2.2
Commit: 3f6812cf84e90af982e423e22a83671ccb5aff22
Parents: 02cf178
Author: Dongjoon Hyun <do...@apache.org>
Authored: Thu Jun 8 21:02:51 2017 -0700
Committer: Wenchen Fan <we...@databricks.com>
Committed: Thu Jun 8 21:02:51 2017 -0700
----------------------------------------------------------------------
.../spark/sql/execution/command/tables.scala | 17 +++++++++++------
.../sql-tests/results/change-column.sql.out | 9 ---------
.../resources/sql-tests/results/describe.sql.out | 17 -----------------
.../spark/sql/hive/execution/HiveDDLSuite.scala | 2 +-
4 files changed, 12 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3f6812cf/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index ebf03e1..6348638 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -522,15 +522,15 @@ case class DescribeTableCommand(
throw new AnalysisException(
s"DESC PARTITION is not allowed on a temporary view: ${table.identifier}")
}
- describeSchema(catalog.lookupRelation(table).schema, result)
+ describeSchema(catalog.lookupRelation(table).schema, result, header = false)
} else {
val metadata = catalog.getTableMetadata(table)
if (metadata.schema.isEmpty) {
// In older version(prior to 2.1) of Spark, the table schema can be empty and should be
// inferred at runtime. We should still support it.
- describeSchema(sparkSession.table(metadata.identifier).schema, result)
+ describeSchema(sparkSession.table(metadata.identifier).schema, result, header = false)
} else {
- describeSchema(metadata.schema, result)
+ describeSchema(metadata.schema, result, header = false)
}
describePartitionInfo(metadata, result)
@@ -550,7 +550,7 @@ case class DescribeTableCommand(
private def describePartitionInfo(table: CatalogTable, buffer: ArrayBuffer[Row]): Unit = {
if (table.partitionColumnNames.nonEmpty) {
append(buffer, "# Partition Information", "", "")
- describeSchema(table.partitionSchema, buffer)
+ describeSchema(table.partitionSchema, buffer, header = true)
}
}
@@ -601,8 +601,13 @@ case class DescribeTableCommand(
table.storage.toLinkedHashMap.foreach(s => append(buffer, s._1, s._2, ""))
}
- private def describeSchema(schema: StructType, buffer: ArrayBuffer[Row]): Unit = {
- append(buffer, s"# ${output.head.name}", output(1).name, output(2).name)
+ private def describeSchema(
+ schema: StructType,
+ buffer: ArrayBuffer[Row],
+ header: Boolean): Unit = {
+ if (header) {
+ append(buffer, s"# ${output.head.name}", output(1).name, output(2).name)
+ }
schema.foreach { column =>
append(buffer, column.name, column.dataType.simpleString, column.getComment().orNull)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/3f6812cf/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/results/change-column.sql.out b/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
index 678a3f0..ba8bc93 100644
--- a/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
@@ -15,7 +15,6 @@ DESC test_change
-- !query 1 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 1 output
-# col_name data_type comment
a int
b string
c int
@@ -35,7 +34,6 @@ DESC test_change
-- !query 3 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 3 output
-# col_name data_type comment
a int
b string
c int
@@ -55,7 +53,6 @@ DESC test_change
-- !query 5 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 5 output
-# col_name data_type comment
a int
b string
c int
@@ -94,7 +91,6 @@ DESC test_change
-- !query 8 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 8 output
-# col_name data_type comment
a int
b string
c int
@@ -129,7 +125,6 @@ DESC test_change
-- !query 12 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 12 output
-# col_name data_type comment
a int this is column a
b string #*02?`
c int
@@ -148,7 +143,6 @@ DESC test_change
-- !query 14 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 14 output
-# col_name data_type comment
a int this is column a
b string #*02?`
c int
@@ -168,7 +162,6 @@ DESC test_change
-- !query 16 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 16 output
-# col_name data_type comment
a int this is column a
b string #*02?`
c int
@@ -193,7 +186,6 @@ DESC test_change
-- !query 18 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 18 output
-# col_name data_type comment
a int this is column a
b string #*02?`
c int
@@ -237,7 +229,6 @@ DESC test_change
-- !query 23 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 23 output
-# col_name data_type comment
a int this is column A
b string #*02?`
c int
http://git-wip-us.apache.org/repos/asf/spark/blob/3f6812cf/sql/core/src/test/resources/sql-tests/results/describe.sql.out
----------------------------------------------------------------------
diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
index de10b29..46d32bb 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
@@ -54,7 +54,6 @@ DESCRIBE t
-- !query 5 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 5 output
-# col_name data_type comment
a string
b int
c string
@@ -70,7 +69,6 @@ DESC default.t
-- !query 6 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 6 output
-# col_name data_type comment
a string
b int
c string
@@ -86,7 +84,6 @@ DESC TABLE t
-- !query 7 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 7 output
-# col_name data_type comment
a string
b int
c string
@@ -102,7 +99,6 @@ DESC FORMATTED t
-- !query 8 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 8 output
-# col_name data_type comment
a string
b int
c string
@@ -132,7 +128,6 @@ DESC EXTENDED t
-- !query 9 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 9 output
-# col_name data_type comment
a string
b int
c string
@@ -162,7 +157,6 @@ DESC t PARTITION (c='Us', d=1)
-- !query 10 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 10 output
-# col_name data_type comment
a string
b int
c string
@@ -178,7 +172,6 @@ DESC EXTENDED t PARTITION (c='Us', d=1)
-- !query 11 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 11 output
-# col_name data_type comment
a string
b int
c string
@@ -206,7 +199,6 @@ DESC FORMATTED t PARTITION (c='Us', d=1)
-- !query 12 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 12 output
-# col_name data_type comment
a string
b int
c string
@@ -268,7 +260,6 @@ DESC temp_v
-- !query 16 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 16 output
-# col_name data_type comment
a string
b int
c string
@@ -280,7 +271,6 @@ DESC TABLE temp_v
-- !query 17 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 17 output
-# col_name data_type comment
a string
b int
c string
@@ -292,7 +282,6 @@ DESC FORMATTED temp_v
-- !query 18 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 18 output
-# col_name data_type comment
a string
b int
c string
@@ -304,7 +293,6 @@ DESC EXTENDED temp_v
-- !query 19 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 19 output
-# col_name data_type comment
a string
b int
c string
@@ -316,7 +304,6 @@ DESC temp_Data_Source_View
-- !query 20 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 20 output
-# col_name data_type comment
intType int test comment test1
stringType string
dateType date
@@ -349,7 +336,6 @@ DESC v
-- !query 22 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 22 output
-# col_name data_type comment
a string
b int
c string
@@ -361,7 +347,6 @@ DESC TABLE v
-- !query 23 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 23 output
-# col_name data_type comment
a string
b int
c string
@@ -373,7 +358,6 @@ DESC FORMATTED v
-- !query 24 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 24 output
-# col_name data_type comment
a string
b int
c string
@@ -396,7 +380,6 @@ DESC EXTENDED v
-- !query 25 schema
struct<col_name:string,data_type:string,comment:string>
-- !query 25 output
-# col_name data_type comment
a string
b int
c string
http://git-wip-us.apache.org/repos/asf/spark/blob/3f6812cf/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index b282acf..6b19b5c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -786,7 +786,7 @@ class HiveDDLSuite
checkAnswer(
sql(s"DESC $tabName").select("col_name", "data_type", "comment"),
- Row("# col_name", "data_type", "comment") :: Row("a", "int", "test") :: Nil
+ Row("a", "int", "test") :: Nil
)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org