You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2018/09/12 20:26:55 UTC
[1/2] hbase git commit: BASE-21168 BloomFilterUtil uses hardcoded
randomness (Mike Drob)
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 d8790bed5 -> 93bffa40d
refs/heads/branch-1.3 b7dfb7462 -> 82f72b5be
BASE-21168 BloomFilterUtil uses hardcoded randomness (Mike Drob)
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/82f72b5b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/82f72b5b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/82f72b5b
Branch: refs/heads/branch-1.3
Commit: 82f72b5bec678cb3e531747f530ccf076bd5115a
Parents: b7dfb74
Author: Mingliang Liu <li...@apache.org>
Authored: Wed Sep 12 12:31:09 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Sep 12 13:26:09 2018 -0700
----------------------------------------------------------------------
.../hadoop/hbase/util/ByteBloomFilter.java | 54 +++++++++++---------
.../regionserver/TestCompoundBloomFilter.java | 6 ++-
2 files changed, 34 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/82f72b5b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
index b8ec4c3..723571d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
@@ -26,6 +26,7 @@ import java.nio.ByteBuffer;
import java.text.NumberFormat;
import java.util.Random;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
@@ -423,26 +424,26 @@ public class ByteBloomFilter implements BloomFilter, BloomFilterWriter {
int hashCount) {
int hash1 = hash.hash(buf, offset, length, 0);
- int hash2 = hash.hash(buf, offset, length, hash1);
int bloomBitSize = bloomSize << 3;
-
+
+ int hash2 = 0;
+ int compositeHash = 0;
+
if (randomGeneratorForTest == null) {
- // Production mode.
- int compositeHash = hash1;
- for (int i = 0; i < hashCount; i++) {
- int hashLoc = Math.abs(compositeHash % bloomBitSize);
- compositeHash += hash2;
- if (!get(hashLoc, bloomBuf, bloomOffset)) {
- return false;
- }
- }
- } else {
- // Test mode with "fake lookups" to estimate "ideal false positive rate".
- for (int i = 0; i < hashCount; i++) {
- int hashLoc = randomGeneratorForTest.nextInt(bloomBitSize);
- if (!get(hashLoc, bloomBuf, bloomOffset)){
- return false;
- }
+ // Production mode
+ compositeHash = hash1;
+ hash2 = hash.hash(buf, offset, length, hash1);
+ }
+
+ for (int i = 0; i < hashCount; i++) {
+ int hashLoc = (randomGeneratorForTest == null
+ // Production mode
+ ? Math.abs(compositeHash % bloomBitSize)
+ // Test mode with "fake look-ups" to estimate "ideal false positive rate"
+ : randomGeneratorForTest.nextInt(bloomBitSize));
+ compositeHash += hash2;
+ if (!get(hashLoc, bloomBuf, bloomOffset)) {
+ return false;
}
}
return true;
@@ -598,12 +599,17 @@ public class ByteBloomFilter implements BloomFilter, BloomFilterWriter {
return bloom != null;
}
- public static void setFakeLookupMode(boolean enabled) {
- if (enabled) {
- randomGeneratorForTest = new Random(283742987L);
- } else {
- randomGeneratorForTest = null;
- }
+ /**
+ * Sets a random generator to be used for look-ups instead of computing hashes. Can be used to
+ * simulate uniformity of accesses better in a test environment. Should not be set in a real
+ * environment where correctness matters!
+ * <p>
+ * This gets used in {@link #contains(byte[], int, int, ByteBuffer, int, int, Hash, int)}
+ * @param random The random number source to use, or null to compute actual hashes
+ */
+ @VisibleForTesting
+ public static void setRandomGeneratorForTest(Random random) {
+ randomGeneratorForTest = random;
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/82f72b5b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
index 0129fad..a0bf6c5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
@@ -218,7 +218,9 @@ public class TestCompoundBloomFilter {
// Test for false positives (some percentage allowed). We test in two modes:
// "fake lookup" which ignores the key distribution, and production mode.
for (boolean fakeLookupEnabled : new boolean[] { true, false }) {
- ByteBloomFilter.setFakeLookupMode(fakeLookupEnabled);
+ if (fakeLookupEnabled) {
+ ByteBloomFilter.setRandomGeneratorForTest(new Random(283742987L));
+ }
try {
String fakeLookupModeStr = ", fake lookup is " + (fakeLookupEnabled ?
"enabled" : "disabled");
@@ -268,7 +270,7 @@ public class TestCompoundBloomFilter {
validateFalsePosRate(falsePosRate, nTrials, -2.58, cbf,
fakeLookupModeStr);
} finally {
- ByteBloomFilter.setFakeLookupMode(false);
+ ByteBloomFilter.setRandomGeneratorForTest(null);
}
}
[2/2] hbase git commit: BASE-21168 BloomFilterUtil uses hardcoded
randomness (Mike Drob)
Posted by ap...@apache.org.
BASE-21168 BloomFilterUtil uses hardcoded randomness (Mike Drob)
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/93bffa40
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/93bffa40
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/93bffa40
Branch: refs/heads/branch-1.2
Commit: 93bffa40d1dd3d2f7dbe8f31e540ef3970d6e446
Parents: d8790be
Author: Mingliang Liu <li...@apache.org>
Authored: Wed Sep 12 12:31:09 2018 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Sep 12 13:26:12 2018 -0700
----------------------------------------------------------------------
.../hadoop/hbase/util/ByteBloomFilter.java | 54 +++++++++++---------
.../regionserver/TestCompoundBloomFilter.java | 6 ++-
2 files changed, 34 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/93bffa40/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
index b8ec4c3..723571d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java
@@ -26,6 +26,7 @@ import java.nio.ByteBuffer;
import java.text.NumberFormat;
import java.util.Random;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
@@ -423,26 +424,26 @@ public class ByteBloomFilter implements BloomFilter, BloomFilterWriter {
int hashCount) {
int hash1 = hash.hash(buf, offset, length, 0);
- int hash2 = hash.hash(buf, offset, length, hash1);
int bloomBitSize = bloomSize << 3;
-
+
+ int hash2 = 0;
+ int compositeHash = 0;
+
if (randomGeneratorForTest == null) {
- // Production mode.
- int compositeHash = hash1;
- for (int i = 0; i < hashCount; i++) {
- int hashLoc = Math.abs(compositeHash % bloomBitSize);
- compositeHash += hash2;
- if (!get(hashLoc, bloomBuf, bloomOffset)) {
- return false;
- }
- }
- } else {
- // Test mode with "fake lookups" to estimate "ideal false positive rate".
- for (int i = 0; i < hashCount; i++) {
- int hashLoc = randomGeneratorForTest.nextInt(bloomBitSize);
- if (!get(hashLoc, bloomBuf, bloomOffset)){
- return false;
- }
+ // Production mode
+ compositeHash = hash1;
+ hash2 = hash.hash(buf, offset, length, hash1);
+ }
+
+ for (int i = 0; i < hashCount; i++) {
+ int hashLoc = (randomGeneratorForTest == null
+ // Production mode
+ ? Math.abs(compositeHash % bloomBitSize)
+ // Test mode with "fake look-ups" to estimate "ideal false positive rate"
+ : randomGeneratorForTest.nextInt(bloomBitSize));
+ compositeHash += hash2;
+ if (!get(hashLoc, bloomBuf, bloomOffset)) {
+ return false;
}
}
return true;
@@ -598,12 +599,17 @@ public class ByteBloomFilter implements BloomFilter, BloomFilterWriter {
return bloom != null;
}
- public static void setFakeLookupMode(boolean enabled) {
- if (enabled) {
- randomGeneratorForTest = new Random(283742987L);
- } else {
- randomGeneratorForTest = null;
- }
+ /**
+ * Sets a random generator to be used for look-ups instead of computing hashes. Can be used to
+ * simulate uniformity of accesses better in a test environment. Should not be set in a real
+ * environment where correctness matters!
+ * <p>
+ * This gets used in {@link #contains(byte[], int, int, ByteBuffer, int, int, Hash, int)}
+ * @param random The random number source to use, or null to compute actual hashes
+ */
+ @VisibleForTesting
+ public static void setRandomGeneratorForTest(Random random) {
+ randomGeneratorForTest = random;
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/93bffa40/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
index 0129fad..a0bf6c5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
@@ -218,7 +218,9 @@ public class TestCompoundBloomFilter {
// Test for false positives (some percentage allowed). We test in two modes:
// "fake lookup" which ignores the key distribution, and production mode.
for (boolean fakeLookupEnabled : new boolean[] { true, false }) {
- ByteBloomFilter.setFakeLookupMode(fakeLookupEnabled);
+ if (fakeLookupEnabled) {
+ ByteBloomFilter.setRandomGeneratorForTest(new Random(283742987L));
+ }
try {
String fakeLookupModeStr = ", fake lookup is " + (fakeLookupEnabled ?
"enabled" : "disabled");
@@ -268,7 +270,7 @@ public class TestCompoundBloomFilter {
validateFalsePosRate(falsePosRate, nTrials, -2.58, cbf,
fakeLookupModeStr);
} finally {
- ByteBloomFilter.setFakeLookupMode(false);
+ ByteBloomFilter.setRandomGeneratorForTest(null);
}
}