You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2016/09/03 16:10:42 UTC
spark git commit: [SPARK-16959][SQL] Rebuild Table Comment when
Retrieving Metadata from Hive Metastore
Repository: spark
Updated Branches:
refs/heads/branch-2.0 196d62eae -> a7f5e7066
[SPARK-16959][SQL] Rebuild Table Comment when Retrieving Metadata from Hive Metastore
### What changes were proposed in this pull request?
The `comment` in `CatalogTable` returned from Hive is always empty. We store it in the table property when creating a table. However, when we try to retrieve the table metadata from Hive metastore, we do not rebuild it. The `comment` is always empty.
This PR is to fix the issue.
### How was this patch tested?
Fixed the test case to verify the change.
Author: gatorsmile <ga...@gmail.com>
Closes #14550 from gatorsmile/tableComment.
(cherry picked from commit bdd537164dcfeec5e9c51d54791ef16997ff2597)
Signed-off-by: Wenchen Fan <we...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a7f5e706
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a7f5e706
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a7f5e706
Branch: refs/heads/branch-2.0
Commit: a7f5e7066f935d58d702a3e86b85aa175291d0fc
Parents: 196d62e
Author: gatorsmile <ga...@gmail.com>
Authored: Wed Aug 10 16:25:01 2016 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Sun Sep 4 00:10:27 2016 +0800
----------------------------------------------------------------------
.../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 3 ++-
.../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 7 +++++--
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a7f5e706/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index 7b4483a..ac78676 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -412,7 +412,8 @@ private[hive] class HiveClientImpl(
serdeProperties = Option(h.getTTable.getSd.getSerdeInfo.getParameters)
.map(_.asScala.toMap).orNull
),
- properties = properties,
+ properties = properties.filter(kv => kv._1 != "comment"),
+ comment = properties.get("comment"),
viewOriginalText = Option(h.getViewOriginalText),
viewText = Option(h.getViewExpandedText),
unsupportedFeatures = unsupportedFeatures)
http://git-wip-us.apache.org/repos/asf/spark/blob/a7f5e706/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index f31215a..1416409 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -135,8 +135,11 @@ class HiveDDLSuite
sql(s"CREATE VIEW $viewName COMMENT 'no comment' AS SELECT * FROM $tabName")
val tableMetadata = catalog.getTableMetadata(TableIdentifier(tabName, Some("default")))
val viewMetadata = catalog.getTableMetadata(TableIdentifier(viewName, Some("default")))
- assert(tableMetadata.properties.get("comment") == Option("BLABLA"))
- assert(viewMetadata.properties.get("comment") == Option("no comment"))
+ assert(tableMetadata.comment == Option("BLABLA"))
+ assert(viewMetadata.comment == Option("no comment"))
+ // Ensure that `comment` is removed from the table property
+ assert(tableMetadata.properties.get("comment").isEmpty)
+ assert(viewMetadata.properties.get("comment").isEmpty)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org