You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2019/03/22 17:10:28 UTC
[spark] branch master updated: [SPARK-27174][SQL] Add support for
casting integer types to binary
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8efc5ec [SPARK-27174][SQL] Add support for casting integer types to binary
8efc5ec is described below
commit 8efc5ec72e2f5899547941010e22c023d6cb86b3
Author: Martin Junghanns <ma...@neotechnology.com>
AuthorDate: Fri Mar 22 10:09:35 2019 -0700
[SPARK-27174][SQL] Add support for casting integer types to binary
Co-authored-by: Philip Stutz <philip.stutzgmail.com>
## What changes were proposed in this pull request?
This PR adds support for casting
* `ByteType`
* `ShortType`
* `IntegerType`
* `LongType`
to `BinaryType`.
## How was this patch tested?
We added unit tests for casting instances of the above types. For validation, we used Javas `DataOutputStream` to compare the resulting byte array with the result of `Cast`.
We state that the contribution is our original work and that we license the work to the project under the project’s open source license.
cloud-fan we'd appreciate a review if you find the time, thx
Closes #24107 from s1ck/cast_to_binary.
Authored-by: Martin Junghanns <ma...@neotechnology.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../spark/sql/catalyst/expressions/Cast.scala | 11 +-
.../spark/sql/catalyst/util/NumberConverter.scala | 35 ++++++
.../sql/catalyst/analysis/AnalysisErrorSuite.scala | 5 -
.../sql/catalyst/util/NumberConverterSuite.scala | 48 ++++++++-
.../src/test/resources/sql-tests/inputs/cast.sql | 13 +++
.../test/resources/sql-tests/results/cast.sql.out | 84 +++++++++++++--
.../typeCoercion/native/binaryComparison.sql.out | 120 +++++++++------------
.../native/windowFrameCoercion.sql.out | 2 +-
.../scala/org/apache/spark/sql/DatasetSuite.scala | 4 +-
9 files changed, 234 insertions(+), 88 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 72cb6b2..848195f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -44,6 +44,7 @@ object Cast {
case (_, StringType) => true
case (StringType, BinaryType) => true
+ case (_: IntegralType, BinaryType) => true
case (StringType, BooleanType) => true
case (DateType, BooleanType) => true
@@ -326,6 +327,10 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
// BinaryConverter
private[this] def castToBinary(from: DataType): Any => Any = from match {
case StringType => buildCast[UTF8String](_, _.getBytes)
+ case ByteType => buildCast[Byte](_, NumberConverter.toBinary)
+ case ShortType => buildCast[Short](_, NumberConverter.toBinary)
+ case IntegerType => buildCast[Int](_, NumberConverter.toBinary)
+ case LongType => buildCast[Long](_, NumberConverter.toBinary)
}
// UDFToBoolean
@@ -908,7 +913,11 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
private[this] def castToBinaryCode(from: DataType): CastFunction = from match {
case StringType =>
- (c, evPrim, evNull) => code"$evPrim = $c.getBytes();"
+ (c, evPrim, evNull) =>
+ code"$evPrim = $c.getBytes();"
+ case _: IntegralType =>
+ (c, evPrim, evNull) =>
+ code"$evPrim = ${NumberConverter.getClass.getName.stripSuffix("$")}.toBinary($c);"
}
private[this] def castToDateCode(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
index 9c3f6b7..7dbdd1e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/NumberConverter.scala
@@ -169,4 +169,39 @@ object NumberConverter {
}
UTF8String.fromBytes(java.util.Arrays.copyOfRange(temp, resultStartPos, temp.length))
}
+
+ def toBinary(l: Long): Array[Byte] = {
+ val result = new Array[Byte](8)
+ result(0) = (l >>> 56 & 0xFF).toByte
+ result(1) = (l >>> 48 & 0xFF).toByte
+ result(2) = (l >>> 40 & 0xFF).toByte
+ result(3) = (l >>> 32 & 0xFF).toByte
+ result(4) = (l >>> 24 & 0xFF).toByte
+ result(5) = (l >>> 16 & 0xFF).toByte
+ result(6) = (l >>> 8 & 0xFF).toByte
+ result(7) = (l & 0xFF).toByte
+ result
+ }
+
+ def toBinary(i: Int): Array[Byte] = {
+ val result = new Array[Byte](4)
+ result(0) = (i >>> 24 & 0xFF).toByte
+ result(1) = (i >>> 16 & 0xFF).toByte
+ result(2) = (i >>> 8 & 0xFF).toByte
+ result(3) = (i & 0xFF).toByte
+ result
+ }
+
+ def toBinary(s: Short): Array[Byte] = {
+ val result = new Array[Byte](2)
+ result(0) = (s >>> 8 & 0xFF).toByte
+ result(1) = (s & 0xFF).toByte
+ result
+ }
+
+ def toBinary(s: Byte): Array[Byte] = {
+ val result = new Array[Byte](1)
+ result(0) = s
+ result
+ }
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 129ce3b..ec71c1c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -216,11 +216,6 @@ class AnalysisErrorSuite extends AnalysisTest {
"Invalid usage of '*'" :: "in expression 'sum'" :: Nil)
errorTest(
- "bad casts",
- testRelation.select(Literal(1).cast(BinaryType).as('badCast)),
- "cannot cast" :: Literal(1).dataType.simpleString :: BinaryType.simpleString :: Nil)
-
- errorTest(
"sorting by unsupported column types",
mapRelation.orderBy('map.asc),
"sort" :: "type" :: "map<int,int>" :: Nil)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala
index 13265a1..ec73f45 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/NumberConverterSuite.scala
@@ -17,8 +17,11 @@
package org.apache.spark.sql.catalyst.util
+import java.nio.ByteBuffer
+import java.nio.ByteOrder.BIG_ENDIAN
+
import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.util.NumberConverter.convert
+import org.apache.spark.sql.catalyst.util.NumberConverter.{convert, toBinary}
import org.apache.spark.unsafe.types.UTF8String
class NumberConverterSuite extends SparkFunSuite {
@@ -37,4 +40,47 @@ class NumberConverterSuite extends SparkFunSuite {
checkConv("11abc", 10, 16, "B")
}
+ test("byte to binary") {
+ checkToBinary(0.toByte)
+ checkToBinary(1.toByte)
+ checkToBinary(-1.toByte)
+ checkToBinary(Byte.MaxValue)
+ checkToBinary(Byte.MinValue)
+ }
+
+ test("short to binary") {
+ checkToBinary(0.toShort)
+ checkToBinary(1.toShort)
+ checkToBinary(-1.toShort)
+ checkToBinary(Short.MaxValue)
+ checkToBinary(Short.MinValue)
+ }
+
+ test("integer to binary") {
+ checkToBinary(0)
+ checkToBinary(1)
+ checkToBinary(-1)
+ checkToBinary(Int.MaxValue)
+ checkToBinary(Int.MinValue)
+ }
+
+ test("long to binary") {
+ checkToBinary(0L)
+ checkToBinary(1L)
+ checkToBinary(-1L)
+ checkToBinary(Long.MaxValue)
+ checkToBinary(Long.MinValue)
+ }
+
+ def checkToBinary[T](in: T): Unit = in match {
+ case b: Byte =>
+ assert(toBinary(b) === ByteBuffer.allocate(1).order(BIG_ENDIAN).put(b).array())
+ case s: Short =>
+ assert(toBinary(s) === ByteBuffer.allocate(2).order(BIG_ENDIAN).putShort(s).array())
+ case i: Int =>
+ assert(toBinary(i) === ByteBuffer.allocate(4).order(BIG_ENDIAN).putInt(i).array())
+ case l: Long =>
+ assert(toBinary(l) === ByteBuffer.allocate(8).order(BIG_ENDIAN).putLong(l).array())
+ }
+
}
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cast.sql b/sql/core/src/test/resources/sql-tests/inputs/cast.sql
index 629df59..7244cd3 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cast.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cast.sql
@@ -40,6 +40,19 @@ SELECT CAST('-9223372036854775809' AS long);
SELECT CAST('9223372036854775807' AS long);
SELECT CAST('9223372036854775808' AS long);
+-- cast string to its binary representation
+SELECT HEX(CAST('abc' AS binary));
+
+-- cast integral values to their corresponding binary representation
+SELECT HEX(CAST(CAST(123 AS byte) AS binary));
+SELECT HEX(CAST(CAST(-123 AS byte) AS binary));
+SELECT HEX(CAST(123S AS binary));
+SELECT HEX(CAST(-123S AS binary));
+SELECT HEX(CAST(123 AS binary));
+SELECT HEX(CAST(-123 AS binary));
+SELECT HEX(CAST(123L AS binary));
+SELECT HEX(CAST(-123L AS binary));
+
DESC FUNCTION boolean;
DESC FUNCTION EXTENDED boolean;
-- TODO: migrate all cast tests here.
diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out
index 9c5f455..21b18e9 100644
--- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 24
+-- Number of queries: 33
-- !query 0
@@ -179,20 +179,92 @@ NULL
-- !query 22
-DESC FUNCTION boolean
+SELECT HEX(CAST('abc' AS binary))
-- !query 22 schema
-struct<function_desc:string>
+struct<hex(CAST(abc AS BINARY)):string>
-- !query 22 output
+616263
+
+
+-- !query 23
+SELECT HEX(CAST(CAST(123 AS byte) AS binary))
+-- !query 23 schema
+struct<hex(CAST(CAST(123 AS TINYINT) AS BINARY)):string>
+-- !query 23 output
+7B
+
+
+-- !query 24
+SELECT HEX(CAST(CAST(-123 AS byte) AS binary))
+-- !query 24 schema
+struct<hex(CAST(CAST(-123 AS TINYINT) AS BINARY)):string>
+-- !query 24 output
+85
+
+
+-- !query 25
+SELECT HEX(CAST(123S AS binary))
+-- !query 25 schema
+struct<hex(CAST(123 AS BINARY)):string>
+-- !query 25 output
+007B
+
+
+-- !query 26
+SELECT HEX(CAST(-123S AS binary))
+-- !query 26 schema
+struct<hex(CAST(-123 AS BINARY)):string>
+-- !query 26 output
+FF85
+
+
+-- !query 27
+SELECT HEX(CAST(123 AS binary))
+-- !query 27 schema
+struct<hex(CAST(123 AS BINARY)):string>
+-- !query 27 output
+0000007B
+
+
+-- !query 28
+SELECT HEX(CAST(-123 AS binary))
+-- !query 28 schema
+struct<hex(CAST(-123 AS BINARY)):string>
+-- !query 28 output
+FFFFFF85
+
+
+-- !query 29
+SELECT HEX(CAST(123L AS binary))
+-- !query 29 schema
+struct<hex(CAST(123 AS BINARY)):string>
+-- !query 29 output
+000000000000007B
+
+
+-- !query 30
+SELECT HEX(CAST(-123L AS binary))
+-- !query 30 schema
+struct<hex(CAST(-123 AS BINARY)):string>
+-- !query 30 output
+FFFFFFFFFFFFFF85
+
+
+-- !query 31
+DESC FUNCTION boolean
+-- !query 31 schema
+struct<function_desc:string>
+-- !query 31 output
Class: org.apache.spark.sql.catalyst.expressions.Cast
Function: boolean
Usage: boolean(expr) - Casts the value `expr` to the target data type `boolean`.
--- !query 23
+-- !query 32
DESC FUNCTION EXTENDED boolean
--- !query 23 schema
+-- !query 32 schema
struct<function_desc:string>
--- !query 23 output
+-- !query 32 output
Class: org.apache.spark.sql.catalyst.expressions.Cast
Extended Usage:
No example/argument for boolean.
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out
index 2914d60..b764ce4 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/binaryComparison.sql.out
@@ -13,217 +13,193 @@ struct<>
-- !query 1
SELECT cast(1 as binary) = '1' FROM t
-- !query 1 schema
-struct<>
+struct<(CAST(1 AS BINARY) = CAST(1 AS BINARY)):boolean>
-- !query 1 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7
+false
-- !query 2
SELECT cast(1 as binary) > '2' FROM t
-- !query 2 schema
-struct<>
+struct<(CAST(1 AS BINARY) > CAST(2 AS BINARY)):boolean>
-- !query 2 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7
+false
-- !query 3
SELECT cast(1 as binary) >= '2' FROM t
-- !query 3 schema
-struct<>
+struct<(CAST(1 AS BINARY) >= CAST(2 AS BINARY)):boolean>
-- !query 3 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7
+false
-- !query 4
SELECT cast(1 as binary) < '2' FROM t
-- !query 4 schema
-struct<>
+struct<(CAST(1 AS BINARY) < CAST(2 AS BINARY)):boolean>
-- !query 4 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7
+true
-- !query 5
SELECT cast(1 as binary) <= '2' FROM t
-- !query 5 schema
-struct<>
+struct<(CAST(1 AS BINARY) <= CAST(2 AS BINARY)):boolean>
-- !query 5 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7
+true
-- !query 6
SELECT cast(1 as binary) <> '2' FROM t
-- !query 6 schema
-struct<>
+struct<(NOT (CAST(1 AS BINARY) = CAST(2 AS BINARY))):boolean>
-- !query 6 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7
+true
-- !query 7
SELECT cast(1 as binary) = cast(null as string) FROM t
-- !query 7 schema
-struct<>
+struct<(CAST(1 AS BINARY) = CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
-- !query 7 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7
+NULL
-- !query 8
SELECT cast(1 as binary) > cast(null as string) FROM t
-- !query 8 schema
-struct<>
+struct<(CAST(1 AS BINARY) > CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
-- !query 8 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7
+NULL
-- !query 9
SELECT cast(1 as binary) >= cast(null as string) FROM t
-- !query 9 schema
-struct<>
+struct<(CAST(1 AS BINARY) >= CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
-- !query 9 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7
+NULL
-- !query 10
SELECT cast(1 as binary) < cast(null as string) FROM t
-- !query 10 schema
-struct<>
+struct<(CAST(1 AS BINARY) < CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
-- !query 10 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7
+NULL
-- !query 11
SELECT cast(1 as binary) <= cast(null as string) FROM t
-- !query 11 schema
-struct<>
+struct<(CAST(1 AS BINARY) <= CAST(CAST(NULL AS STRING) AS BINARY)):boolean>
-- !query 11 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7
+NULL
-- !query 12
SELECT cast(1 as binary) <> cast(null as string) FROM t
-- !query 12 schema
-struct<>
+struct<(NOT (CAST(1 AS BINARY) = CAST(CAST(NULL AS STRING) AS BINARY))):boolean>
-- !query 12 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 7
+NULL
-- !query 13
SELECT '1' = cast(1 as binary) FROM t
-- !query 13 schema
-struct<>
+struct<(CAST(1 AS BINARY) = CAST(1 AS BINARY)):boolean>
-- !query 13 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 13
+false
-- !query 14
SELECT '2' > cast(1 as binary) FROM t
-- !query 14 schema
-struct<>
+struct<(CAST(2 AS BINARY) > CAST(1 AS BINARY)):boolean>
-- !query 14 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 13
+true
-- !query 15
SELECT '2' >= cast(1 as binary) FROM t
-- !query 15 schema
-struct<>
+struct<(CAST(2 AS BINARY) >= CAST(1 AS BINARY)):boolean>
-- !query 15 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 14
+true
-- !query 16
SELECT '2' < cast(1 as binary) FROM t
-- !query 16 schema
-struct<>
+struct<(CAST(2 AS BINARY) < CAST(1 AS BINARY)):boolean>
-- !query 16 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 13
+false
-- !query 17
SELECT '2' <= cast(1 as binary) FROM t
-- !query 17 schema
-struct<>
+struct<(CAST(2 AS BINARY) <= CAST(1 AS BINARY)):boolean>
-- !query 17 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 14
+false
-- !query 18
SELECT '2' <> cast(1 as binary) FROM t
-- !query 18 schema
-struct<>
+struct<(NOT (CAST(2 AS BINARY) = CAST(1 AS BINARY))):boolean>
-- !query 18 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 14
+true
-- !query 19
SELECT cast(null as string) = cast(1 as binary) FROM t
-- !query 19 schema
-struct<>
+struct<(CAST(CAST(NULL AS STRING) AS BINARY) = CAST(1 AS BINARY)):boolean>
-- !query 19 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 30
+NULL
-- !query 20
SELECT cast(null as string) > cast(1 as binary) FROM t
-- !query 20 schema
-struct<>
+struct<(CAST(CAST(NULL AS STRING) AS BINARY) > CAST(1 AS BINARY)):boolean>
-- !query 20 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 30
+NULL
-- !query 21
SELECT cast(null as string) >= cast(1 as binary) FROM t
-- !query 21 schema
-struct<>
+struct<(CAST(CAST(NULL AS STRING) AS BINARY) >= CAST(1 AS BINARY)):boolean>
-- !query 21 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 31
+NULL
-- !query 22
SELECT cast(null as string) < cast(1 as binary) FROM t
-- !query 22 schema
-struct<>
+struct<(CAST(CAST(NULL AS STRING) AS BINARY) < CAST(1 AS BINARY)):boolean>
-- !query 22 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 30
+NULL
-- !query 23
SELECT cast(null as string) <= cast(1 as binary) FROM t
-- !query 23 schema
-struct<>
+struct<(CAST(CAST(NULL AS STRING) AS BINARY) <= CAST(1 AS BINARY)):boolean>
-- !query 23 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 31
+NULL
-- !query 24
SELECT cast(null as string) <> cast(1 as binary) FROM t
-- !query 24 schema
-struct<>
+struct<(NOT (CAST(CAST(NULL AS STRING) AS BINARY) = CAST(1 AS BINARY))):boolean>
-- !query 24 output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'CAST(1 AS BINARY)' due to data type mismatch: cannot cast int to binary; line 1 pos 31
+NULL
-- !query 25
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out
index 01d8393..4fa2032 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/windowFrameCoercion.sql.out
@@ -177,7 +177,7 @@ SELECT COUNT(*) OVER (PARTITION BY 1 ORDER BY cast('1' as binary) DESC RANGE BET
struct<>
-- !query 21 output
org.apache.spark.sql.AnalysisException
-cannot resolve '(PARTITION BY 1 ORDER BY CAST('1' AS BINARY) DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND 1 FOLLOWING)' due to data type mismatch: The data type 'binary' used in the order specification does not match the data type 'int' which is used in the range frame.; line 1 pos 21
+cannot resolve 'RANGE BETWEEN CURRENT ROW AND CAST(1 AS BINARY) FOLLOWING' due to data type mismatch: The data type of the upper bound 'binary' does not match the expected data type '(numeric or calendarinterval)'.; line 1 pos 21
-- !query 22
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index db9f251..050699d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -714,11 +714,11 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
test("Kryo encoder: check the schema mismatch when converting DataFrame to Dataset") {
implicit val kryoEncoder = Encoders.kryo[KryoData]
- val df = Seq((1)).toDF("a")
+ val df = Seq((1.0)).toDF("a")
val e = intercept[AnalysisException] {
df.as[KryoData]
}.message
- assert(e.contains("cannot cast int to binary"))
+ assert(e.contains("cannot cast double to binary"))
}
test("Java encoder") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org