You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/10/18 18:03:02 UTC
spark git commit: [SPARK-17899][SQL][FOLLOW-UP] debug mode should
work for corrupted table
Repository: spark
Updated Branches:
refs/heads/master a9e79a41e -> e59df62e6
[SPARK-17899][SQL][FOLLOW-UP] debug mode should work for corrupted table
## What changes were proposed in this pull request?
Debug mode should work for corrupted table, so that we can really debug
## How was this patch tested?
new test in `MetastoreDataSourcesSuite`
Author: Wenchen Fan <we...@databricks.com>
Closes #15528 from cloud-fan/debug.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e59df62e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e59df62e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e59df62e
Branch: refs/heads/master
Commit: e59df62e62ec4c5f8bd02a13f05fa3ec6f0fc694
Parents: a9e79a4
Author: Wenchen Fan <we...@databricks.com>
Authored: Tue Oct 18 11:03:10 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Tue Oct 18 11:03:10 2016 -0700
----------------------------------------------------------------------
.../spark/sql/hive/HiveExternalCatalog.scala | 9 ++-------
.../sql/hive/MetastoreDataSourcesSuite.scala | 18 +++++++++++++++---
2 files changed, 17 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e59df62e/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index ff59b54..2003ff4 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -448,7 +448,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
* properties, and filter out these special entries from table properties.
*/
private def restoreTableMetadata(table: CatalogTable): CatalogTable = {
- val catalogTable = if (table.tableType == VIEW) {
+ val catalogTable = if (table.tableType == VIEW || conf.get(DEBUG_MODE)) {
table
} else {
getProviderFromTableProperties(table).map { provider =>
@@ -467,18 +467,13 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
} else {
table.storage
}
- val tableProps = if (conf.get(DEBUG_MODE)) {
- table.properties
- } else {
- getOriginalTableProperties(table)
- }
table.copy(
storage = storage,
schema = getSchemaFromTableProperties(table),
provider = Some(provider),
partitionColumnNames = getPartitionColumnsFromTableProperties(table),
bucketSpec = getBucketSpecFromTableProperties(table),
- properties = tableProps)
+ properties = getOriginalTableProperties(table))
} getOrElse {
table.copy(provider = Some("hive"))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/e59df62e/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
index 7cc6179..eaa67d3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala
@@ -1321,20 +1321,32 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv
sharedState.externalCatalog.getTable("default", "t")
}.getMessage
assert(e.contains(s"Could not read schema from the hive metastore because it is corrupted"))
+
+ withDebugMode {
+ val tableMeta = sharedState.externalCatalog.getTable("default", "t")
+ assert(tableMeta.identifier == TableIdentifier("t", Some("default")))
+ assert(tableMeta.properties(DATASOURCE_PROVIDER) == "json")
+ }
} finally {
hiveClient.dropTable("default", "t", ignoreIfNotExists = true, purge = true)
}
}
test("should keep data source entries in table properties when debug mode is on") {
- val previousValue = sparkSession.sparkContext.conf.get(DEBUG_MODE)
- try {
- sparkSession.sparkContext.conf.set(DEBUG_MODE, true)
+ withDebugMode {
val newSession = sparkSession.newSession()
newSession.sql("CREATE TABLE abc(i int) USING json")
val tableMeta = newSession.sessionState.catalog.getTableMetadata(TableIdentifier("abc"))
assert(tableMeta.properties(DATASOURCE_SCHEMA_NUMPARTS).toInt == 1)
assert(tableMeta.properties(DATASOURCE_PROVIDER) == "json")
+ }
+ }
+
+ private def withDebugMode(f: => Unit): Unit = {
+ val previousValue = sparkSession.sparkContext.conf.get(DEBUG_MODE)
+ try {
+ sparkSession.sparkContext.conf.set(DEBUG_MODE, true)
+ f
} finally {
sparkSession.sparkContext.conf.set(DEBUG_MODE, previousValue)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org