You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/04/21 01:49:47 UTC
spark git commit: [SPARK-20367] Properly unescape column names of
partitioning columns parsed from paths.
Repository: spark
Updated Branches:
refs/heads/master 592f5c893 -> 0368eb9d8
[SPARK-20367] Properly unescape column names of partitioning columns parsed from paths.
## What changes were proposed in this pull request?
When infering partitioning schema from paths, the column in parsePartitionColumn should be unescaped with unescapePathName, just like it is being done in e.g. parsePathFragmentAsSeq.
## How was this patch tested?
Added a test to FileIndexSuite.
Author: Juliusz Sompolski <ju...@databricks.com>
Closes #17703 from juliuszsompolski/SPARK-20367.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0368eb9d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0368eb9d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0368eb9d
Branch: refs/heads/master
Commit: 0368eb9d86634c83b3140ce3190cb9e0d0b7fd86
Parents: 592f5c8
Author: Juliusz Sompolski <ju...@databricks.com>
Authored: Fri Apr 21 09:49:42 2017 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Fri Apr 21 09:49:42 2017 +0800
----------------------------------------------------------------------
.../sql/execution/datasources/PartitioningUtils.scala | 2 +-
.../sql/execution/datasources/FileIndexSuite.scala | 12 ++++++++++++
2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0368eb9d/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
index c358320..2d70172 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
@@ -243,7 +243,7 @@ object PartitioningUtils {
if (equalSignIndex == -1) {
None
} else {
- val columnName = columnSpec.take(equalSignIndex)
+ val columnName = unescapePathName(columnSpec.take(equalSignIndex))
assert(columnName.nonEmpty, s"Empty partition column name in '$columnSpec'")
val rawColumnValue = columnSpec.drop(equalSignIndex + 1)
http://git-wip-us.apache.org/repos/asf/spark/blob/0368eb9d/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
index a9511cb..b461682 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.{FileStatus, Path, RawLocalFileSystem}
import org.apache.spark.metrics.source.HiveCatalogMetrics
import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.functions.col
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSQLContext
import org.apache.spark.util.{KnownSizeEstimation, SizeEstimator}
@@ -236,6 +237,17 @@ class FileIndexSuite extends SharedSQLContext {
val fileStatusCache = FileStatusCache.getOrCreate(spark)
fileStatusCache.putLeafFiles(new Path("/tmp", "abc"), files.toArray)
}
+
+ test("SPARK-20367 - properly unescape column names in inferPartitioning") {
+ withTempPath { path =>
+ val colToUnescape = "Column/#%'?"
+ spark
+ .range(1)
+ .select(col("id").as(colToUnescape), col("id"))
+ .write.partitionBy(colToUnescape).parquet(path.getAbsolutePath)
+ assert(spark.read.parquet(path.getAbsolutePath).schema.exists(_.name == colToUnescape))
+ }
+ }
}
class FakeParentPathFileSystem extends RawLocalFileSystem {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org