You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2016/09/27 18:16:55 UTC
[1/4] hbase git commit: HBASE-16660 ArrayIndexOutOfBounds during the
majorCompactionCheck in DateTieredCompaction
Repository: hbase
Updated Branches:
refs/heads/0.98 2b6da527b -> 744b867c9
refs/heads/branch-1 a3485cc5a -> 96a8e8dce
refs/heads/branch-1.3 1441b7c79 -> e0066e713
refs/heads/master b644e0fb8 -> d127d6426
HBASE-16660 ArrayIndexOutOfBounds during the majorCompactionCheck in DateTieredCompaction
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d127d642
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d127d642
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d127d642
Branch: refs/heads/master
Commit: d127d64266baed275a15d39407585a3bca4f6f15
Parents: b644e0f
Author: Abhishek Singh Chouhan <ab...@gmail.com>
Authored: Tue Sep 27 20:32:52 2016 +0530
Committer: Andrew Purtell <ap...@apache.org>
Committed: Tue Sep 27 10:58:51 2016 -0700
----------------------------------------------------------------------
.../compactions/DateTieredCompactionPolicy.java | 3 +++
.../TestDateTieredCompactionPolicy.java | 16 ++++++++++++++++
2 files changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d127d642/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
index 037bc80..e37a7fe 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
@@ -153,6 +153,9 @@ public class DateTieredCompactionPolicy extends SortedCompactionPolicy {
minTimestamp == null ? (Long)Long.MAX_VALUE : minTimestamp);
int upperWindowIndex = Collections.binarySearch(boundaries,
file.getMaximumTimestamp() == null ? (Long)Long.MAX_VALUE : file.getMaximumTimestamp());
+ // Handle boundary conditions and negative values of binarySearch
+ lowerWindowIndex = (lowerWindowIndex < 0) ? Math.abs(lowerWindowIndex + 2) : lowerWindowIndex;
+ upperWindowIndex = (upperWindowIndex < 0) ? Math.abs(upperWindowIndex + 2) : upperWindowIndex;
if (lowerWindowIndex != upperWindowIndex) {
LOG.debug("Major compaction triggered on store " + this + "; because file "
+ file.getPath() + " has data with timestamps cross window boundaries");
http://git-wip-us.apache.org/repos/asf/hbase/blob/d127d642/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
index eb52a84..4f02d8a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
@@ -235,6 +235,22 @@ public class TestDateTieredCompactionPolicy extends AbstractTestDateTieredCompac
}
/**
+ * Major Compaction to check min max timestamp falling in the same window and also to check
+ * boundary condition in which case binary sort gives insertion point as length of the array
+ * @throws IOException
+ */
+ @Test
+ public void checkMinMaxTimestampSameBoundary() throws IOException {
+ long[] minTimestamps = new long[] { 0, 26, 50, 90, 98, 122, 145, 151, 158, 166 };
+ long[] maxTimestamps = new long[] { 12, 46, 70, 95, 100, 140, 148, 155, 162, 174 };
+ long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 };
+
+ compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
+ new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 },
+ new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
+ }
+
+ /**
* Major compaction with negative numbers
* @throws IOException with error
*/
[4/4] hbase git commit: HBASE-16660 ArrayIndexOutOfBounds during the
majorCompactionCheck in DateTieredCompaction
Posted by ap...@apache.org.
HBASE-16660 ArrayIndexOutOfBounds during the majorCompactionCheck in DateTieredCompaction
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/744b867c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/744b867c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/744b867c
Branch: refs/heads/0.98
Commit: 744b867c9b81d779605f3bf581d9b8db9be0e47c
Parents: 2b6da52
Author: Abhishek Singh Chouhan <ab...@gmail.com>
Authored: Tue Sep 27 20:32:52 2016 +0530
Committer: Andrew Purtell <ap...@apache.org>
Committed: Tue Sep 27 11:09:17 2016 -0700
----------------------------------------------------------------------
.../compactions/DateTieredCompactionPolicy.java | 4 +++-
.../TestDateTieredCompactionPolicy.java | 16 ++++++++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/744b867c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
index 2c1601f..e1905bf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
@@ -151,7 +151,9 @@ public class DateTieredCompactionPolicy extends SortedCompactionPolicy {
minTimestamp == null ? Long.MAX_VALUE : file.getMinimumTimestamp());
int upperWindowIndex = Collections.binarySearch(boundaries,
file.getMaximumTimestamp() == null ? Long.MAX_VALUE : file.getMaximumTimestamp());
-
+ // Handle boundary conditions and negative values of binarySearch
+ lowerWindowIndex = (lowerWindowIndex < 0) ? Math.abs(lowerWindowIndex + 2) : lowerWindowIndex;
+ upperWindowIndex = (upperWindowIndex < 0) ? Math.abs(upperWindowIndex + 2) : upperWindowIndex;
if (lowerWindowIndex != upperWindowIndex) {
LOG.debug("Major compaction triggered on store " + this + "; because file "
+ file.getPath() + " has data with timestamps cross window boundaries");
http://git-wip-us.apache.org/repos/asf/hbase/blob/744b867c/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
index 5a0b0b1..45c5a76 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
@@ -235,6 +235,22 @@ public class TestDateTieredCompactionPolicy extends AbstractTestDateTieredCompac
}
/**
+ * Major Compaction to check min max timestamp falling in the same window and also to check
+ * boundary condition in which case binary sort gives insertion point as length of the array
+ * @throws IOException
+ */
+ @Test
+ public void checkMinMaxTimestampSameBoundary() throws IOException {
+ long[] minTimestamps = new long[] { 0, 26, 50, 90, 98, 122, 145, 151, 158, 166 };
+ long[] maxTimestamps = new long[] { 12, 46, 70, 95, 100, 140, 148, 155, 162, 174 };
+ long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 };
+
+ compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
+ new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 },
+ new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
+ }
+
+ /**
* Major compaction with negative numbers
* @throws IOException with error
*/
[3/4] hbase git commit: HBASE-16660 ArrayIndexOutOfBounds during the
majorCompactionCheck in DateTieredCompaction
Posted by ap...@apache.org.
HBASE-16660 ArrayIndexOutOfBounds during the majorCompactionCheck in DateTieredCompaction
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e0066e71
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e0066e71
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e0066e71
Branch: refs/heads/branch-1.3
Commit: e0066e713d6bd4f499e479ea173677e3fc14d2a2
Parents: 1441b7c
Author: Abhishek Singh Chouhan <ab...@gmail.com>
Authored: Tue Sep 27 20:32:52 2016 +0530
Committer: Andrew Purtell <ap...@apache.org>
Committed: Tue Sep 27 11:08:10 2016 -0700
----------------------------------------------------------------------
.../compactions/DateTieredCompactionPolicy.java | 3 +++
.../TestDateTieredCompactionPolicy.java | 16 ++++++++++++++++
2 files changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e0066e71/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
index 037bc80..e37a7fe 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
@@ -153,6 +153,9 @@ public class DateTieredCompactionPolicy extends SortedCompactionPolicy {
minTimestamp == null ? (Long)Long.MAX_VALUE : minTimestamp);
int upperWindowIndex = Collections.binarySearch(boundaries,
file.getMaximumTimestamp() == null ? (Long)Long.MAX_VALUE : file.getMaximumTimestamp());
+ // Handle boundary conditions and negative values of binarySearch
+ lowerWindowIndex = (lowerWindowIndex < 0) ? Math.abs(lowerWindowIndex + 2) : lowerWindowIndex;
+ upperWindowIndex = (upperWindowIndex < 0) ? Math.abs(upperWindowIndex + 2) : upperWindowIndex;
if (lowerWindowIndex != upperWindowIndex) {
LOG.debug("Major compaction triggered on store " + this + "; because file "
+ file.getPath() + " has data with timestamps cross window boundaries");
http://git-wip-us.apache.org/repos/asf/hbase/blob/e0066e71/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
index eb52a84..4f02d8a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
@@ -235,6 +235,22 @@ public class TestDateTieredCompactionPolicy extends AbstractTestDateTieredCompac
}
/**
+ * Major Compaction to check min max timestamp falling in the same window and also to check
+ * boundary condition in which case binary sort gives insertion point as length of the array
+ * @throws IOException
+ */
+ @Test
+ public void checkMinMaxTimestampSameBoundary() throws IOException {
+ long[] minTimestamps = new long[] { 0, 26, 50, 90, 98, 122, 145, 151, 158, 166 };
+ long[] maxTimestamps = new long[] { 12, 46, 70, 95, 100, 140, 148, 155, 162, 174 };
+ long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 };
+
+ compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
+ new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 },
+ new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
+ }
+
+ /**
* Major compaction with negative numbers
* @throws IOException with error
*/
[2/4] hbase git commit: HBASE-16660 ArrayIndexOutOfBounds during the
majorCompactionCheck in DateTieredCompaction
Posted by ap...@apache.org.
HBASE-16660 ArrayIndexOutOfBounds during the majorCompactionCheck in DateTieredCompaction
Signed-off-by: Andrew Purtell <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/96a8e8dc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/96a8e8dc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/96a8e8dc
Branch: refs/heads/branch-1
Commit: 96a8e8dce4ea65a12af6c14d22ef4d34b52ebb6c
Parents: a3485cc
Author: Abhishek Singh Chouhan <ab...@gmail.com>
Authored: Tue Sep 27 20:32:52 2016 +0530
Committer: Andrew Purtell <ap...@apache.org>
Committed: Tue Sep 27 11:08:00 2016 -0700
----------------------------------------------------------------------
.../compactions/DateTieredCompactionPolicy.java | 3 +++
.../TestDateTieredCompactionPolicy.java | 16 ++++++++++++++++
2 files changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/96a8e8dc/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
index 037bc80..e37a7fe 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java
@@ -153,6 +153,9 @@ public class DateTieredCompactionPolicy extends SortedCompactionPolicy {
minTimestamp == null ? (Long)Long.MAX_VALUE : minTimestamp);
int upperWindowIndex = Collections.binarySearch(boundaries,
file.getMaximumTimestamp() == null ? (Long)Long.MAX_VALUE : file.getMaximumTimestamp());
+ // Handle boundary conditions and negative values of binarySearch
+ lowerWindowIndex = (lowerWindowIndex < 0) ? Math.abs(lowerWindowIndex + 2) : lowerWindowIndex;
+ upperWindowIndex = (upperWindowIndex < 0) ? Math.abs(upperWindowIndex + 2) : upperWindowIndex;
if (lowerWindowIndex != upperWindowIndex) {
LOG.debug("Major compaction triggered on store " + this + "; because file "
+ file.getPath() + " has data with timestamps cross window boundaries");
http://git-wip-us.apache.org/repos/asf/hbase/blob/96a8e8dc/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
index eb52a84..4f02d8a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDateTieredCompactionPolicy.java
@@ -235,6 +235,22 @@ public class TestDateTieredCompactionPolicy extends AbstractTestDateTieredCompac
}
/**
+ * Major Compaction to check min max timestamp falling in the same window and also to check
+ * boundary condition in which case binary sort gives insertion point as length of the array
+ * @throws IOException
+ */
+ @Test
+ public void checkMinMaxTimestampSameBoundary() throws IOException {
+ long[] minTimestamps = new long[] { 0, 26, 50, 90, 98, 122, 145, 151, 158, 166 };
+ long[] maxTimestamps = new long[] { 12, 46, 70, 95, 100, 140, 148, 155, 162, 174 };
+ long[] sizes = new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 };
+
+ compactEquals(161, sfCreate(minTimestamps, maxTimestamps, sizes),
+ new long[] { 0, 50, 51, 40, 41, 42, 33, 30, 31, 2 },
+ new long[] { Long.MIN_VALUE, 24, 48, 72, 96, 120, 144, 150, 156 }, true, true);
+ }
+
+ /**
* Major compaction with negative numbers
* @throws IOException with error
*/