You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/05/04 02:17:35 UTC
[1/2] hive git commit: HIVE-16581 : bug in HIVE-16523 (Sergey
Shelukhin, reviewed by Gopal Vijayaraghavan)
Repository: hive
Updated Branches:
refs/heads/branch-2 828418201 -> a1d54a5d3
refs/heads/master 740779f66 -> d769f35fb
HIVE-16581 : bug in HIVE-16523 (Sergey Shelukhin, reviewed by Gopal Vijayaraghavan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a1d54a5d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a1d54a5d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a1d54a5d
Branch: refs/heads/branch-2
Commit: a1d54a5d337cfd93ad3dbe63f1705b25250621bc
Parents: 8284182
Author: sergey <se...@apache.org>
Authored: Wed May 3 19:13:26 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Wed May 3 19:16:26 2017 -0700
----------------------------------------------------------------------
.../org/apache/hive/common/util/Murmur3.java | 8 +++----
.../apache/hive/common/util/TestMurmur3.java | 24 ++++++++++++++++++++
2 files changed, 28 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a1d54a5d/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hive/common/util/Murmur3.java b/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
index 1c56765..eb05082 100644
--- a/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
+++ b/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
@@ -52,7 +52,7 @@ public class Murmur3 {
private static final int N1 = 0x52dce729;
private static final int N2 = 0x38495ab5;
- private static final int DEFAULT_SEED = 104729;
+ public static final int DEFAULT_SEED = 104729;
/**
* Murmur3 32-bit variant.
@@ -358,13 +358,13 @@ public class Murmur3 {
int k = -1;
switch (tailLen) {
case 1:
- k = orBytes(tail[0], data[0], data[1], data[2]);
+ k = orBytes(tail[0], data[offset], data[offset + 1], data[offset + 2]);
break;
case 2:
- k = orBytes(tail[0], tail[1], data[0], data[1]);
+ k = orBytes(tail[0], tail[1], data[offset], data[offset + 1]);
break;
case 3:
- k = orBytes(tail[0], tail[1], tail[2], data[0]);
+ k = orBytes(tail[0], tail[1], tail[2], data[offset]);
break;
default: throw new AssertionError(tailLen);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a1d54a5d/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
----------------------------------------------------------------------
diff --git a/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java b/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
index 5facc7c..391ee42 100644
--- a/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
+++ b/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
@@ -19,6 +19,7 @@
package org.apache.hive.common.util;
import static org.junit.Assert.assertEquals;
+import org.apache.hive.common.util.Murmur3.IncrementalHash32;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
@@ -221,4 +222,27 @@ public class TestMurmur3 {
assertEquals(gl2, m2);
}
}
+
+
+ @Test
+ public void testIncremental() {
+ final int seed = 123, arraySize = 1023;
+ byte[] bytes = new byte[arraySize];
+ new Random(seed).nextBytes(bytes);
+ int expected = Murmur3.hash32(bytes);
+ Murmur3.IncrementalHash32 same = new IncrementalHash32(), diff = new IncrementalHash32();
+ for (int blockSize = 1; blockSize <= arraySize; ++blockSize) {
+ byte[] block = new byte[blockSize];
+ same.start(Murmur3.DEFAULT_SEED);
+ diff.start(Murmur3.DEFAULT_SEED);
+ for (int offset = 0; offset < arraySize; offset += blockSize) {
+ int length = Math.min(arraySize - offset, blockSize);
+ same.add(bytes, offset, length);
+ System.arraycopy(bytes, offset, block, 0, length);
+ diff.add(block, 0, length);
+ }
+ assertEquals("Block size " + blockSize, expected, same.end());
+ assertEquals("Block size " + blockSize, expected, diff.end());
+ }
+ }
}
[2/2] hive git commit: HIVE-16581 : bug in HIVE-16523 (Sergey
Shelukhin, reviewed by Gopal Vijayaraghavan)
Posted by se...@apache.org.
HIVE-16581 : bug in HIVE-16523 (Sergey Shelukhin, reviewed by Gopal Vijayaraghavan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d769f35f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d769f35f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d769f35f
Branch: refs/heads/master
Commit: d769f35fbc50ee6f8ce7b6994444246ea9138767
Parents: 740779f
Author: sergey <se...@apache.org>
Authored: Wed May 3 19:13:26 2017 -0700
Committer: sergey <se...@apache.org>
Committed: Wed May 3 19:17:17 2017 -0700
----------------------------------------------------------------------
.../org/apache/hive/common/util/Murmur3.java | 8 +++----
.../apache/hive/common/util/TestMurmur3.java | 24 ++++++++++++++++++++
2 files changed, 28 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d769f35f/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hive/common/util/Murmur3.java b/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
index 1c56765..eb05082 100644
--- a/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
+++ b/storage-api/src/java/org/apache/hive/common/util/Murmur3.java
@@ -52,7 +52,7 @@ public class Murmur3 {
private static final int N1 = 0x52dce729;
private static final int N2 = 0x38495ab5;
- private static final int DEFAULT_SEED = 104729;
+ public static final int DEFAULT_SEED = 104729;
/**
* Murmur3 32-bit variant.
@@ -358,13 +358,13 @@ public class Murmur3 {
int k = -1;
switch (tailLen) {
case 1:
- k = orBytes(tail[0], data[0], data[1], data[2]);
+ k = orBytes(tail[0], data[offset], data[offset + 1], data[offset + 2]);
break;
case 2:
- k = orBytes(tail[0], tail[1], data[0], data[1]);
+ k = orBytes(tail[0], tail[1], data[offset], data[offset + 1]);
break;
case 3:
- k = orBytes(tail[0], tail[1], tail[2], data[0]);
+ k = orBytes(tail[0], tail[1], tail[2], data[offset]);
break;
default: throw new AssertionError(tailLen);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d769f35f/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
----------------------------------------------------------------------
diff --git a/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java b/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
index 5facc7c..391ee42 100644
--- a/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
+++ b/storage-api/src/test/org/apache/hive/common/util/TestMurmur3.java
@@ -19,6 +19,7 @@
package org.apache.hive.common.util;
import static org.junit.Assert.assertEquals;
+import org.apache.hive.common.util.Murmur3.IncrementalHash32;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
@@ -221,4 +222,27 @@ public class TestMurmur3 {
assertEquals(gl2, m2);
}
}
+
+
+ @Test
+ public void testIncremental() {
+ final int seed = 123, arraySize = 1023;
+ byte[] bytes = new byte[arraySize];
+ new Random(seed).nextBytes(bytes);
+ int expected = Murmur3.hash32(bytes);
+ Murmur3.IncrementalHash32 same = new IncrementalHash32(), diff = new IncrementalHash32();
+ for (int blockSize = 1; blockSize <= arraySize; ++blockSize) {
+ byte[] block = new byte[blockSize];
+ same.start(Murmur3.DEFAULT_SEED);
+ diff.start(Murmur3.DEFAULT_SEED);
+ for (int offset = 0; offset < arraySize; offset += blockSize) {
+ int length = Math.min(arraySize - offset, blockSize);
+ same.add(bytes, offset, length);
+ System.arraycopy(bytes, offset, block, 0, length);
+ diff.add(block, 0, length);
+ }
+ assertEquals("Block size " + blockSize, expected, same.end());
+ assertEquals("Block size " + blockSize, expected, diff.end());
+ }
+ }
}