You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2016/01/05 08:23:45 UTC
spark git commit: [SPARK-12568][SQL] Add BINARY to Encoders
Repository: spark
Updated Branches:
refs/heads/master 7058dc115 -> 53beddc5b
[SPARK-12568][SQL] Add BINARY to Encoders
Author: Michael Armbrust <mi...@databricks.com>
Closes #10516 from marmbrus/datasetCleanup.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/53beddc5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/53beddc5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/53beddc5
Branch: refs/heads/master
Commit: 53beddc5bf04a35ab73de99158919c2fdd5d4508
Parents: 7058dc1
Author: Michael Armbrust <mi...@databricks.com>
Authored: Mon Jan 4 23:23:41 2016 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Jan 4 23:23:41 2016 -0800
----------------------------------------------------------------------
.../src/main/scala/org/apache/spark/sql/Encoder.scala | 6 ++++++
.../spark/sql/catalyst/encoders/ExpressionEncoder.scala | 9 +++++++++
.../sql/catalyst/encoders/ExpressionEncoderSuite.scala | 6 +++---
3 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/53beddc5/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
index bb0fdc4..22b7e1e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
@@ -158,6 +158,12 @@ object Encoders {
def TIMESTAMP: Encoder[java.sql.Timestamp] = ExpressionEncoder()
/**
+ * An encoder for arrays of bytes.
+ * @since 1.6.1
+ */
+ def BINARY: Encoder[Array[Byte]] = ExpressionEncoder()
+
+ /**
* Creates an encoder for Java Bean of type T.
*
* T must be publicly accessible.
http://git-wip-us.apache.org/repos/asf/spark/blob/53beddc5/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
index ad4beda..6c05846 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
@@ -199,6 +199,15 @@ case class ExpressionEncoder[T](
private lazy val constructProjection = GenerateSafeProjection.generate(fromRowExpression :: Nil)
/**
+ * Returns this encoder where it has been bound to its own output (i.e. no remaping of columns
+ * is performed).
+ */
+ def defaultBinding: ExpressionEncoder[T] = {
+ val attrs = schema.toAttributes
+ resolve(attrs, OuterScopes.outerScopes).bind(attrs)
+ }
+
+ /**
* Returns an encoded version of `t` as a Spark SQL row. Note that multiple calls to
* toRow are allowed to return the same actual [[InternalRow]] object. Thus, the caller should
* copy the result before making another call if required.
http://git-wip-us.apache.org/repos/asf/spark/blob/53beddc5/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
index 666699e..3740dea 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
@@ -77,6 +77,8 @@ class JavaSerializable(val value: Int) extends Serializable {
}
class ExpressionEncoderSuite extends SparkFunSuite {
+ OuterScopes.outerScopes.put(getClass.getName, this)
+
implicit def encoder[T : TypeTag]: ExpressionEncoder[T] = ExpressionEncoder()
// test flat encoders
@@ -278,8 +280,6 @@ class ExpressionEncoderSuite extends SparkFunSuite {
}
}
- private val outers: ConcurrentMap[String, AnyRef] = new MapMaker().weakValues().makeMap()
- outers.put(getClass.getName, this)
private def encodeDecodeTest[T : ExpressionEncoder](
input: T,
testName: String): Unit = {
@@ -287,7 +287,7 @@ class ExpressionEncoderSuite extends SparkFunSuite {
val encoder = implicitly[ExpressionEncoder[T]]
val row = encoder.toRow(input)
val schema = encoder.schema.toAttributes
- val boundEncoder = encoder.resolve(schema, outers).bind(schema)
+ val boundEncoder = encoder.defaultBinding
val convertedBack = try boundEncoder.fromRow(row) catch {
case e: Exception =>
fail(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org