You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/01/24 12:49:31 UTC
spark git commit: [SPARK-19246][SQL] CataLogTable's partitionSchema
order and exist check
Repository: spark
Updated Branches:
refs/heads/master 3c86fdddf -> 752502be0
[SPARK-19246][SQL] CataLogTable's partitionSchema order and exist check
## What changes were proposed in this pull request?
CataLogTable's partitionSchema should check if each column name in partitionColumnNames must match one and only one field in schema, if not we should throw an exception
and CataLogTable's partitionSchema should keep order with partitionColumnNames
## How was this patch tested?
N/A
Author: windpiger <so...@outlook.com>
Closes #16606 from windpiger/checkPartionColNameWithSchema.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/752502be
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/752502be
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/752502be
Branch: refs/heads/master
Commit: 752502be053c66a95b04204b4ae0e9574394bc58
Parents: 3c86fdd
Author: windpiger <so...@outlook.com>
Authored: Tue Jan 24 20:49:23 2017 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Tue Jan 24 20:49:23 2017 +0800
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/catalog/interface.scala | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/752502be/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index 7bbaf6e..b8dc5f9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -182,10 +182,15 @@ case class CatalogTable(
import CatalogTable._
- /** schema of this table's partition columns */
- def partitionSchema: StructType = StructType(schema.filter {
- c => partitionColumnNames.contains(c.name)
- })
+ /**
+ * schema of this table's partition columns
+ */
+ def partitionSchema: StructType = {
+ val partitionFields = schema.takeRight(partitionColumnNames.length)
+ assert(partitionFields.map(_.name) == partitionColumnNames)
+
+ StructType(partitionFields)
+ }
/** Return the database this table was specified to belong to, assuming it exists. */
def database: String = identifier.database.getOrElse {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org