You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2016/08/10 08:25:12 UTC
spark git commit: [SPARK-16959][SQL] Rebuild Table Comment when
Retrieving Metadata from Hive Metastore
Repository: spark
Updated Branches:
refs/heads/master 1203c8415 -> bdd537164
[SPARK-16959][SQL] Rebuild Table Comment when Retrieving Metadata from Hive Metastore
### What changes were proposed in this pull request?
The `comment` in `CatalogTable` returned from Hive is always empty. We store it in the table property when creating a table. However, when we try to retrieve the table metadata from Hive metastore, we do not rebuild it. The `comment` is always empty.
This PR is to fix the issue.
### How was this patch tested?
Fixed the test case to verify the change.
Author: gatorsmile <ga...@gmail.com>
Closes #14550 from gatorsmile/tableComment.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bdd53716
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bdd53716
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bdd53716
Branch: refs/heads/master
Commit: bdd537164dcfeec5e9c51d54791ef16997ff2597
Parents: 1203c84
Author: gatorsmile <ga...@gmail.com>
Authored: Wed Aug 10 16:25:01 2016 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Wed Aug 10 16:25:01 2016 +0800
----------------------------------------------------------------------
.../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 3 ++-
.../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 7 +++++--
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/bdd53716/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index 3bf4ed5..f8204e1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -413,7 +413,8 @@ private[hive] class HiveClientImpl(
properties = Option(h.getTTable.getSd.getSerdeInfo.getParameters)
.map(_.asScala.toMap).orNull
),
- properties = properties,
+ properties = properties.filter(kv => kv._1 != "comment"),
+ comment = properties.get("comment"),
viewOriginalText = Option(h.getViewOriginalText),
viewText = Option(h.getViewExpandedText),
unsupportedFeatures = unsupportedFeatures)
http://git-wip-us.apache.org/repos/asf/spark/blob/bdd53716/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index e078b58..970b688 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -135,8 +135,11 @@ class HiveDDLSuite
sql(s"CREATE VIEW $viewName COMMENT 'no comment' AS SELECT * FROM $tabName")
val tableMetadata = catalog.getTableMetadata(TableIdentifier(tabName, Some("default")))
val viewMetadata = catalog.getTableMetadata(TableIdentifier(viewName, Some("default")))
- assert(tableMetadata.properties.get("comment") == Option("BLABLA"))
- assert(viewMetadata.properties.get("comment") == Option("no comment"))
+ assert(tableMetadata.comment == Option("BLABLA"))
+ assert(viewMetadata.comment == Option("no comment"))
+ // Ensure that `comment` is removed from the table property
+ assert(tableMetadata.properties.get("comment").isEmpty)
+ assert(viewMetadata.properties.get("comment").isEmpty)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org