You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2016/12/21 11:39:08 UTC
spark git commit: [SPARK-18947][SQL] SQLContext.tableNames should not
call Catalog.listTables
Repository: spark
Updated Branches:
refs/heads/master ba4468bb2 -> b7650f11c
[SPARK-18947][SQL] SQLContext.tableNames should not call Catalog.listTables
## What changes were proposed in this pull request?
It's a huge waste to call `Catalog.listTables` in `SQLContext.tableNames`, which only need the table names, while `Catalog.listTables` will get the table metadata for each table name.
## How was this patch tested?
N/A
Author: Wenchen Fan <we...@databricks.com>
Closes #16352 from cloud-fan/minor.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b7650f11
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b7650f11
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b7650f11
Branch: refs/heads/master
Commit: b7650f11c7afbdffc6f5caaafb5dcfd54f7a25ff
Parents: ba4468b
Author: Wenchen Fan <we...@databricks.com>
Authored: Wed Dec 21 19:39:00 2016 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Wed Dec 21 19:39:00 2016 +0800
----------------------------------------------------------------------
.../src/main/scala/org/apache/spark/sql/SQLContext.scala | 4 ++--
.../main/scala/org/apache/spark/sql/api/r/SQLUtils.scala | 9 +++++----
2 files changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b7650f11/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 6554359..1a7fd68 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -747,7 +747,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @since 1.3.0
*/
def tableNames(): Array[String] = {
- sparkSession.catalog.listTables().collect().map(_.name)
+ tableNames(sparkSession.catalog.currentDatabase)
}
/**
@@ -757,7 +757,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
* @since 1.3.0
*/
def tableNames(databaseName: String): Array[String] = {
- sparkSession.catalog.listTables(databaseName).collect().map(_.name)
+ sessionState.catalog.listTables(databaseName).map(_.table).toArray
}
////////////////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/spark/blob/b7650f11/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
index 80bbad4..e56c33e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala
@@ -276,11 +276,12 @@ private[sql] object SQLUtils extends Logging {
}
def getTableNames(sparkSession: SparkSession, databaseName: String): Array[String] = {
- databaseName match {
- case n: String if n != null && n.trim.nonEmpty =>
- sparkSession.catalog.listTables(n).collect().map(_.name)
+ val db = databaseName match {
+ case _ if databaseName != null && databaseName.trim.nonEmpty =>
+ databaseName
case _ =>
- sparkSession.catalog.listTables().collect().map(_.name)
+ sparkSession.catalog.currentDatabase
}
+ sparkSession.sessionState.catalog.listTables(db).map(_.table).toArray
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org