You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2016/11/22 20:25:25 UTC
spark git commit: [SPARK-18507][SQL]
HiveExternalCatalog.listPartitions should only call getTable once
Repository: spark
Updated Branches:
refs/heads/master 45ea46b7b -> 702cd403f
[SPARK-18507][SQL] HiveExternalCatalog.listPartitions should only call getTable once
## What changes were proposed in this pull request?
HiveExternalCatalog.listPartitions should only call `getTable` once, instead of calling it for every partitions.
## How was this patch tested?
N/A
Author: Wenchen Fan <we...@databricks.com>
Closes #15978 from cloud-fan/perf.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/702cd403
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/702cd403
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/702cd403
Branch: refs/heads/master
Commit: 702cd403fc8e5ce8281fe8828197ead46bdb8832
Parents: 45ea46b
Author: Wenchen Fan <we...@databricks.com>
Authored: Tue Nov 22 15:25:22 2016 -0500
Committer: Andrew Or <an...@gmail.com>
Committed: Tue Nov 22 15:25:22 2016 -0500
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/702cd403/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index 5dbb402..ff0923f 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -907,8 +907,9 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
db: String,
table: String,
partialSpec: Option[TablePartitionSpec] = None): Seq[CatalogTablePartition] = withClient {
+ val actualPartColNames = getTable(db, table).partitionColumnNames
client.getPartitions(db, table, partialSpec.map(lowerCasePartitionSpec)).map { part =>
- part.copy(spec = restorePartitionSpec(part.spec, getTable(db, table).partitionColumnNames))
+ part.copy(spec = restorePartitionSpec(part.spec, actualPartColNames))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org