You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by da...@apache.org on 2015/10/06 02:32:08 UTC
spark git commit: [SPARK-10934] [SQL] handle hashCode of unsafe array
correctly
Repository: spark
Updated Branches:
refs/heads/master c4871369d -> a609eb20d
[SPARK-10934] [SQL] handle hashCode of unsafe array correctly
`Murmur3_x86_32.hashUnsafeWords` only accepts word-aligned bytes, but unsafe array is not.
Author: Wenchen Fan <cl...@163.com>
Closes #8987 from cloud-fan/hash.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a609eb20
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a609eb20
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a609eb20
Branch: refs/heads/master
Commit: a609eb20d964a7b92f1066300443415f6db181e3
Parents: c487136
Author: Wenchen Fan <cl...@163.com>
Authored: Mon Oct 5 17:30:54 2015 -0700
Committer: Davies Liu <da...@gmail.com>
Committed: Mon Oct 5 17:31:54 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/expressions/UnsafeArrayData.java | 6 +++++-
.../src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala | 7 +++++++
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a609eb20/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
index da9538b..6a16d34 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java
@@ -285,7 +285,11 @@ public class UnsafeArrayData extends ArrayData {
@Override
public int hashCode() {
- return Murmur3_x86_32.hashUnsafeWords(baseObject, baseOffset, sizeInBytes, 42);
+ int result = 37;
+ for (int i = 0; i < sizeInBytes; i++) {
+ result = 37 * result + Platform.getByte(baseObject, baseOffset + i);
+ }
+ return result;
}
@Override
http://git-wip-us.apache.org/repos/asf/spark/blob/a609eb20/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
index 2476b10..944d4e1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
@@ -131,4 +131,11 @@ class UnsafeRowSuite extends SparkFunSuite {
assert(emptyRow.getInt(0) === unsafeRow.getInt(0))
assert(emptyRow.getUTF8String(1) === unsafeRow.getUTF8String(1))
}
+
+ test("calling hashCode on unsafe array returned by getArray(ordinal)") {
+ val row = InternalRow.apply(new GenericArrayData(Array(1L)))
+ val unsafeRow = UnsafeProjection.create(Array[DataType](ArrayType(LongType))).apply(row)
+ // Makes sure hashCode on unsafe array won't crash
+ unsafeRow.getArray(0).hashCode()
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org