You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/03/22 16:58:51 UTC
spark git commit: [SPARK-20018][SQL] Pivot with timestamp and count
should not print internal representation
Repository: spark
Updated Branches:
refs/heads/master 465818389 -> 80fd07038
[SPARK-20018][SQL] Pivot with timestamp and count should not print internal representation
## What changes were proposed in this pull request?
Currently, when we perform count with timestamp types, it prints the internal representation as the column name as below:
```scala
Seq(new java.sql.Timestamp(1)).toDF("a").groupBy("a").pivot("a").count().show()
```
```
+--------------------+----+
| a|1000|
+--------------------+----+
|1969-12-31 16:00:...| 1|
+--------------------+----+
```
This PR proposes to use external Scala value instead of the internal representation in the column names as below:
```
+--------------------+-----------------------+
| a|1969-12-31 16:00:00.001|
+--------------------+-----------------------+
|1969-12-31 16:00:...| 1|
+--------------------+-----------------------+
```
## How was this patch tested?
Unit test in `DataFramePivotSuite` and manual tests.
Author: hyukjinkwon <gu...@gmail.com>
Closes #17348 from HyukjinKwon/SPARK-20018.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/80fd0703
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/80fd0703
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/80fd0703
Branch: refs/heads/master
Commit: 80fd070389a9c8ffa342d7b11f1ab2ea92e0f562
Parents: 4658183
Author: hyukjinkwon <gu...@gmail.com>
Authored: Wed Mar 22 09:58:46 2017 -0700
Committer: Xiao Li <ga...@gmail.com>
Committed: Wed Mar 22 09:58:46 2017 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/Analyzer.scala | 6 ++++--
.../apache/spark/sql/DataFramePivotSuite.scala | 18 +++++++++++++++++-
2 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/80fd0703/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 574f91b..036ed06 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -486,14 +486,16 @@ class Analyzer(
case Pivot(groupByExprs, pivotColumn, pivotValues, aggregates, child) =>
val singleAgg = aggregates.size == 1
def outputName(value: Literal, aggregate: Expression): String = {
+ val utf8Value = Cast(value, StringType, Some(conf.sessionLocalTimeZone)).eval(EmptyRow)
+ val stringValue: String = Option(utf8Value).map(_.toString).getOrElse("null")
if (singleAgg) {
- value.toString
+ stringValue
} else {
val suffix = aggregate match {
case n: NamedExpression => n.name
case _ => toPrettySQL(aggregate)
}
- value + "_" + suffix
+ stringValue + "_" + suffix
}
}
if (aggregates.forall(a => PivotFirst.supportsDataType(a.dataType))) {
http://git-wip-us.apache.org/repos/asf/spark/blob/80fd0703/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
index ca3cb56..6ca9ee5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSQLContext
import org.apache.spark.sql.types._
-class DataFramePivotSuite extends QueryTest with SharedSQLContext{
+class DataFramePivotSuite extends QueryTest with SharedSQLContext {
import testImplicits._
test("pivot courses") {
@@ -230,4 +230,20 @@ class DataFramePivotSuite extends QueryTest with SharedSQLContext{
.groupBy($"a").pivot("a").agg(min($"b")),
Row(null, Seq(null, 7), null) :: Row(1, null, Seq(1, 7)) :: Nil)
}
+
+ test("pivot with timestamp and count should not print internal representation") {
+ val ts = "2012-12-31 16:00:10.011"
+ val tsWithZone = "2013-01-01 00:00:10.011"
+
+ withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") {
+ val df = Seq(java.sql.Timestamp.valueOf(ts)).toDF("a").groupBy("a").pivot("a").count()
+ val expected = StructType(
+ StructField("a", TimestampType) ::
+ StructField(tsWithZone, LongType) :: Nil)
+ assert(df.schema == expected)
+ // String representation of timestamp with timezone should take the time difference
+ // into account.
+ checkAnswer(df.select($"a".cast(StringType)), Row(tsWithZone))
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org