You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/12/08 19:16:02 UTC
spark git commit: [SPARK-12195][SQL] Adding BigDecimal,
Date and Timestamp into Encoder
Repository: spark
Updated Branches:
refs/heads/master 381f17b54 -> c0b13d556
[SPARK-12195][SQL] Adding BigDecimal, Date and Timestamp into Encoder
This PR is to add three more data types into Encoder, including `BigDecimal`, `Date` and `Timestamp`.
marmbrus cloud-fan rxin Could you take a quick look at these three types? Not sure if it can be merged to 1.6. Thank you very much!
Author: gatorsmile <ga...@gmail.com>
Closes #10188 from gatorsmile/dataTypesinEncoder.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c0b13d55
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c0b13d55
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c0b13d55
Branch: refs/heads/master
Commit: c0b13d5565c45ae2acbe8cfb17319c92b6a634e4
Parents: 381f17b
Author: gatorsmile <ga...@gmail.com>
Authored: Tue Dec 8 10:15:58 2015 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Tue Dec 8 10:15:58 2015 -0800
----------------------------------------------------------------------
.../main/scala/org/apache/spark/sql/Encoder.scala | 18 ++++++++++++++++++
.../org/apache/spark/sql/JavaDatasetSuite.java | 17 +++++++++++++++++
2 files changed, 35 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c0b13d55/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
index c40061a..3ca5ade 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
@@ -98,6 +98,24 @@ object Encoders {
def STRING: Encoder[java.lang.String] = ExpressionEncoder()
/**
+ * An encoder for nullable decimal type.
+ * @since 1.6.0
+ */
+ def DECIMAL: Encoder[java.math.BigDecimal] = ExpressionEncoder()
+
+ /**
+ * An encoder for nullable date type.
+ * @since 1.6.0
+ */
+ def DATE: Encoder[java.sql.Date] = ExpressionEncoder()
+
+ /**
+ * An encoder for nullable timestamp type.
+ * @since 1.6.0
+ */
+ def TIMESTAMP: Encoder[java.sql.Timestamp] = ExpressionEncoder()
+
+ /**
* Creates an encoder for Java Bean of type T.
*
* T must be publicly accessible.
http://git-wip-us.apache.org/repos/asf/spark/blob/c0b13d55/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
----------------------------------------------------------------------
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
index ae47f4f..383a2d0 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
@@ -18,6 +18,9 @@
package test.org.apache.spark.sql;
import java.io.Serializable;
+import java.math.BigDecimal;
+import java.sql.Date;
+import java.sql.Timestamp;
import java.util.*;
import scala.Tuple2;
@@ -386,6 +389,20 @@ public class JavaDatasetSuite implements Serializable {
}
@Test
+ public void testPrimitiveEncoder() {
+ Encoder<Tuple5<Double, BigDecimal, Date, Timestamp, Float>> encoder =
+ Encoders.tuple(Encoders.DOUBLE(), Encoders.DECIMAL(), Encoders.DATE(), Encoders.TIMESTAMP(),
+ Encoders.FLOAT());
+ List<Tuple5<Double, BigDecimal, Date, Timestamp, Float>> data =
+ Arrays.asList(new Tuple5<Double, BigDecimal, Date, Timestamp, Float>(
+ 1.7976931348623157E308, new BigDecimal("0.922337203685477589"),
+ Date.valueOf("1970-01-01"), new Timestamp(System.currentTimeMillis()), Float.MAX_VALUE));
+ Dataset<Tuple5<Double, BigDecimal, Date, Timestamp, Float>> ds =
+ context.createDataset(data, encoder);
+ Assert.assertEquals(data, ds.collectAsList());
+ }
+
+ @Test
public void testTypedAggregation() {
Encoder<Tuple2<String, Integer>> encoder = Encoders.tuple(Encoders.STRING(), Encoders.INT());
List<Tuple2<String, Integer>> data =
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org