You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bb...@apache.org on 2024/02/23 18:27:49 UTC
(hbase) branch branch-2.6 updated (ea1c057e0b1 -> 751b75e82fd)
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a change to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
from ea1c057e0b1 HBASE-28377 Fallback to simple is broken for blocking rpc client (#5690)
new aae26cbcfd6 HBASE-28370 Default user quotas are refreshing too frequently (#5686)
new 751b75e82fd HBASE-28390 WAL value compression fails for cells with large values (#5696)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../aircompressor/TestWALCompressionLz4.java | 13 ----
.../aircompressor/TestWALCompressionLzo.java | 14 ----
.../aircompressor/TestWALCompressionSnappy.java | 13 ----
.../aircompressor/TestWALCompressionZstd.java | 13 ----
.../compress/brotli/TestWALCompressionBrotli.java | 13 ----
.../io/compress/lz4/TestWALCompressionLz4.java | 14 ----
.../compress/xerial/TestWALCompressionSnappy.java | 13 ----
.../io/compress/xz/TestWALCompressionLzma.java | 13 ----
.../io/compress/zstd/TestWALCompressionZstd.java | 14 ----
.../org/apache/hadoop/hbase/quotas/QuotaCache.java | 12 ++-
.../org/apache/hadoop/hbase/quotas/QuotaUtil.java | 6 +-
.../hbase/regionserver/wal/CompressionContext.java | 17 +++++
...LDecompressionBoundedDelegatingInputStream.java | 16 ++--
.../apache/hadoop/hbase/quotas/TestQuotaCache.java | 89 ++++++++++++++++++++++
.../hadoop/hbase/quotas/ThrottleQuotaTestUtil.java | 12 +++
.../hadoop/hbase/wal/CompressedWALTestBase.java | 32 +++++++-
.../apache/hadoop/hbase/wal/TestCompressedWAL.java | 13 ----
.../wal/TestCompressedWALValueCompression.java | 13 ----
18 files changed, 170 insertions(+), 160 deletions(-)
create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java
(hbase) 02/02: HBASE-28390 WAL value compression fails for cells with large values (#5696)
Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 751b75e82fdfa9fd52993ccddb067cf051999ca6
Author: Bryan Beaudreault <bb...@apache.org>
AuthorDate: Fri Feb 23 12:29:45 2024 -0500
HBASE-28390 WAL value compression fails for cells with large values (#5696)
Signed-off-by: Andrew Purtell <ap...@apache.org>
---
.../aircompressor/TestWALCompressionLz4.java | 13 ---------
.../aircompressor/TestWALCompressionLzo.java | 14 ----------
.../aircompressor/TestWALCompressionSnappy.java | 13 ---------
.../aircompressor/TestWALCompressionZstd.java | 13 ---------
.../compress/brotli/TestWALCompressionBrotli.java | 13 ---------
.../io/compress/lz4/TestWALCompressionLz4.java | 14 ----------
.../compress/xerial/TestWALCompressionSnappy.java | 13 ---------
.../io/compress/xz/TestWALCompressionLzma.java | 13 ---------
.../io/compress/zstd/TestWALCompressionZstd.java | 14 ----------
.../hbase/regionserver/wal/CompressionContext.java | 17 ++++++++++++
...LDecompressionBoundedDelegatingInputStream.java | 16 ++++++-----
.../hadoop/hbase/wal/CompressedWALTestBase.java | 32 ++++++++++++++++++++--
.../apache/hadoop/hbase/wal/TestCompressedWAL.java | 13 ---------
.../wal/TestCompressedWALValueCompression.java | 13 ---------
14 files changed, 55 insertions(+), 156 deletions(-)
diff --git a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
index 34a7dcfedfc..1361754189a 100644
--- a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
+++ b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLz4 extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLz4 extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLz4.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionLz4 extends CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
index 9c5bc8838c0..dd91b0cac02 100644
--- a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
+++ b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLzo extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLzo extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLzo.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -59,11 +52,4 @@ public class TestWALCompressionLzo extends CompressedWALTestBase {
public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
-
- @Test
- public void test() throws Exception {
- TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionSnappy.java b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionSnappy.java
index 72813bcbd65..93345b924fa 100644
--- a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionSnappy.java
+++ b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionSnappy.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionSnappy extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionSnappy extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionSnappy.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionSnappy extends CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionZstd.java b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionZstd.java
index 0f5c80ce269..3b367a2db9b 100644
--- a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionZstd.java
+++ b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionZstd.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionZstd extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionZstd extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionZstd.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionZstd extends CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git a/hbase-compression/hbase-compression-brotli/src/test/java/org/apache/hadoop/hbase/io/compress/brotli/TestWALCompressionBrotli.java b/hbase-compression/hbase-compression-brotli/src/test/java/org/apache/hadoop/hbase/io/compress/brotli/TestWALCompressionBrotli.java
index e37276fed6d..3539c4280d9 100644
--- a/hbase-compression/hbase-compression-brotli/src/test/java/org/apache/hadoop/hbase/io/compress/brotli/TestWALCompressionBrotli.java
+++ b/hbase-compression/hbase-compression-brotli/src/test/java/org/apache/hadoop/hbase/io/compress/brotli/TestWALCompressionBrotli.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.brotli;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionBrotli extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionBrotli extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionBrotli.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionBrotli extends CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git a/hbase-compression/hbase-compression-lz4/src/test/java/org/apache/hadoop/hbase/io/compress/lz4/TestWALCompressionLz4.java b/hbase-compression/hbase-compression-lz4/src/test/java/org/apache/hadoop/hbase/io/compress/lz4/TestWALCompressionLz4.java
index 81b5d943dc6..b79fe094de0 100644
--- a/hbase-compression/hbase-compression-lz4/src/test/java/org/apache/hadoop/hbase/io/compress/lz4/TestWALCompressionLz4.java
+++ b/hbase-compression/hbase-compression-lz4/src/test/java/org/apache/hadoop/hbase/io/compress/lz4/TestWALCompressionLz4.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.lz4;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLz4 extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLz4 extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLz4.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -59,11 +52,4 @@ public class TestWALCompressionLz4 extends CompressedWALTestBase {
public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
-
- @Test
- public void test() throws Exception {
- TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java
index 7a2bbfe7b87..bb6dac1e452 100644
--- a/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java
+++ b/hbase-compression/hbase-compression-snappy/src/test/java/org/apache/hadoop/hbase/io/compress/xerial/TestWALCompressionSnappy.java
@@ -22,7 +22,6 @@ import static org.junit.Assume.assumeTrue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -31,10 +30,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionSnappy extends CompressedWALTestBase {
@@ -43,9 +39,6 @@ public class TestWALCompressionSnappy extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionSnappy.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
assumeTrue(SnappyCodec.isLoaded());
@@ -63,10 +56,4 @@ public class TestWALCompressionSnappy extends CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git a/hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestWALCompressionLzma.java b/hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestWALCompressionLzma.java
index ee937230cd2..aa74926cb81 100644
--- a/hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestWALCompressionLzma.java
+++ b/hbase-compression/hbase-compression-xz/src/test/java/org/apache/hadoop/hbase/io/compress/xz/TestWALCompressionLzma.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.xz;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLzma extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLzma extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLzma.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionLzma extends CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git a/hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestWALCompressionZstd.java b/hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestWALCompressionZstd.java
index 55d61cf83ec..23a37ab8659 100644
--- a/hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestWALCompressionZstd.java
+++ b/hbase-compression/hbase-compression-zstd/src/test/java/org/apache/hadoop/hbase/io/compress/zstd/TestWALCompressionZstd.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.zstd;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionZstd extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionZstd extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionZstd.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -59,11 +52,4 @@ public class TestWALCompressionZstd extends CompressedWALTestBase {
public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
-
- @Test
- public void test() throws Exception {
- TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
index 633decab0a8..2481753dfb0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.regionserver.wal;
import java.io.ByteArrayOutputStream;
+import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -129,9 +130,25 @@ public class CompressionContext {
} else {
lowerIn.reset(in, inLength);
IOUtils.readFully(compressedIn, outArray, outOffset, outLength);
+ // if the uncompressed size was larger than the configured buffer size for the codec,
+ // the BlockCompressorStream will have left an extra 4 bytes hanging. This represents a size
+ // for the next segment, and it should be 0. See HBASE-28390
+ if (lowerIn.available() == 4) {
+ int remaining = rawReadInt(lowerIn);
+ assert remaining == 0;
+ }
}
}
+ private int rawReadInt(InputStream in) throws IOException {
+ int b1 = in.read();
+ int b2 = in.read();
+ int b3 = in.read();
+ int b4 = in.read();
+ if ((b1 | b2 | b3 | b4) < 0) throw new EOFException();
+ return ((b1 << 24) + (b2 << 16) + (b3 << 8) + (b4 << 0));
+ }
+
public void clear() {
if (compressedOut != null) {
try {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALDecompressionBoundedDelegatingInputStream.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALDecompressionBoundedDelegatingInputStream.java
index 0f4fd78a0b8..5d876b97aa8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALDecompressionBoundedDelegatingInputStream.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALDecompressionBoundedDelegatingInputStream.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hbase.regionserver.wal;
-import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.io.IOUtils;
@@ -67,19 +66,22 @@ class WALDecompressionBoundedDelegatingInputStream extends InputStream {
if (pos >= limit) {
return -1;
}
- int readLen = (int) Math.min(len, limit - pos);
- try {
- IOUtils.readFully(in, b, off, readLen);
- } catch (EOFException e) {
+ int toRead = (int) Math.min(len, limit - pos);
+ int readBytes = IOUtils.read(in, b, off, toRead);
+ // increase pos by however many we actually read
+ pos += readBytes;
+
+ if (readBytes != toRead) {
// This is trick here, we will always try to read enough bytes to fill the buffer passed in,
// or we reach the end of this compression block, if there are not enough bytes, we just
// return -1 to let the upper layer fail with EOF
// In WAL value decompression this is OK as if we can not read all the data, we will finally
// get an EOF somewhere
- LOG.debug("Got EOF while we want to read {} bytes from stream", readLen, e);
+ LOG.debug("Got EOF while we want to read {} bytes from stream, but only read {}", toRead,
+ readBytes);
return -1;
}
- return readLen;
+ return toRead;
}
@Override
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java
index cb2e5daa987..ebf8856a821 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java
@@ -36,9 +36,13 @@ import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings("checkstyle:innerassignment")
-public class CompressedWALTestBase {
+public abstract class CompressedWALTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(CompressedWALTestBase.class);
protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@@ -67,14 +71,36 @@ public class CompressedWALTestBase {
Arrays.fill(VALUE, off, (off += 1597), (byte) 'Q');
}
- public void doTest(TableName tableName) throws Exception {
+ @Test
+ public void test() throws Exception {
+ testForSize(1000);
+ }
+
+ @Test
+ public void testLarge() throws Exception {
+ testForSize(1024 * 1024);
+ }
+
+ private void testForSize(int size) throws Exception {
+ TableName tableName = TableName.valueOf(getClass().getSimpleName() + "_testForSize_" + size);
+ doTest(tableName, size);
+ }
+
+ public void doTest(TableName tableName, int valueSize) throws Exception {
NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
scopes.put(tableName.getName(), 0);
RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
final int total = 1000;
final byte[] row = Bytes.toBytes("row");
final byte[] family = Bytes.toBytes("family");
- final byte[] value = VALUE;
+ final byte[] value = new byte[valueSize];
+
+ int offset = 0;
+ while (offset + VALUE.length < value.length) {
+ System.arraycopy(VALUE, 0, value, offset, VALUE.length);
+ offset += VALUE.length;
+ }
+
final WALFactory wals =
new WALFactory(TEST_UTIL.getConfiguration(), tableName.getNameAsString());
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWAL.java
index d1275650793..faf244b5db7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWAL.java
@@ -20,16 +20,12 @@ package org.apache.hadoop.hbase.wal;
import java.util.Arrays;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
@@ -43,9 +39,6 @@ public class TestCompressedWAL extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestCompressedWAL.class);
- @Rule
- public TestName name = new TestName();
-
@Parameter
public String walProvider;
@@ -66,10 +59,4 @@ public class TestCompressedWAL extends CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
- TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
-
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java
index 88c0c889ade..c3374b021b6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java
@@ -21,7 +21,6 @@ import java.util.List;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@@ -50,9 +46,6 @@ public class TestCompressedWALValueCompression extends CompressedWALTestBase {
return HBaseTestingUtility.COMPRESSION_ALGORITHMS_PARAMETERIZED;
}
- @Rule
- public TestName name = new TestName();
-
private final Compression.Algorithm compression;
public TestCompressedWALValueCompression(Compression.Algorithm algo) {
@@ -72,10 +65,4 @@ public class TestCompressedWALValueCompression extends CompressedWALTestBase {
public void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
-
- @Test
- public void test() throws Exception {
- TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
- doTest(tableName);
- }
}
(hbase) 01/02: HBASE-28370 Default user quotas are refreshing too frequently (#5686)
Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit aae26cbcfd6fe3b4e44367d7b76cd30d29c11a4d
Author: Ray Mattingly <rm...@gmail.com>
AuthorDate: Mon Feb 19 15:32:00 2024 -0500
HBASE-28370 Default user quotas are refreshing too frequently (#5686)
Signed-off-by: Bryan Beaudreault <bb...@apache.org>
---
.../org/apache/hadoop/hbase/quotas/QuotaCache.java | 12 ++-
.../org/apache/hadoop/hbase/quotas/QuotaUtil.java | 6 +-
.../apache/hadoop/hbase/quotas/TestQuotaCache.java | 89 ++++++++++++++++++++++
.../hadoop/hbase/quotas/ThrottleQuotaTestUtil.java | 12 +++
4 files changed, 115 insertions(+), 4 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
index 67b2aecc544..9b3498ff894 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
@@ -71,6 +71,8 @@ public class QuotaCache implements Stoppable {
// for testing purpose only, enforce the cache to be always refreshed
static boolean TEST_FORCE_REFRESH = false;
+ // for testing purpose only, block cache refreshes to reliably verify state
+ static boolean TEST_BLOCK_REFRESH = false;
private final ConcurrentMap<String, QuotaState> namespaceQuotaCache = new ConcurrentHashMap<>();
private final ConcurrentMap<TableName, QuotaState> tableQuotaCache = new ConcurrentHashMap<>();
@@ -138,7 +140,7 @@ public class QuotaCache implements Stoppable {
*/
public UserQuotaState getUserQuotaState(final UserGroupInformation ugi) {
return computeIfAbsent(userQuotaCache, getQuotaUserName(ugi),
- () -> QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration()),
+ () -> QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration(), 0L),
this::triggerCacheRefresh);
}
@@ -239,6 +241,14 @@ public class QuotaCache implements Stoppable {
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "GC_UNRELATED_TYPES",
justification = "I do not understand why the complaints, it looks good to me -- FIX")
protected void chore() {
+ while (TEST_BLOCK_REFRESH) {
+ LOG.info("TEST_BLOCK_REFRESH=true, so blocking QuotaCache refresh until it is false");
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
// Prefetch online tables/namespaces
for (TableName table : ((HRegionServer) QuotaCache.this.rsServices).getOnlineTables()) {
if (table.isSystemTable()) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java
index 831c0297785..8ced76e3963 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java
@@ -333,7 +333,7 @@ public class QuotaUtil extends QuotaTableUtil {
String user = getUserFromRowKey(key);
if (results[i].isEmpty()) {
- userQuotas.put(user, buildDefaultUserQuotaState(connection.getConfiguration()));
+ userQuotas.put(user, buildDefaultUserQuotaState(connection.getConfiguration(), nowTs));
continue;
}
@@ -373,7 +373,7 @@ public class QuotaUtil extends QuotaTableUtil {
return userQuotas;
}
- protected static UserQuotaState buildDefaultUserQuotaState(Configuration conf) {
+ protected static UserQuotaState buildDefaultUserQuotaState(Configuration conf, long nowTs) {
QuotaProtos.Throttle.Builder throttleBuilder = QuotaProtos.Throttle.newBuilder();
buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_READ_NUM)
@@ -389,7 +389,7 @@ public class QuotaUtil extends QuotaTableUtil {
buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_WRITE_SIZE)
.ifPresent(throttleBuilder::setWriteSize);
- UserQuotaState state = new UserQuotaState();
+ UserQuotaState state = new UserQuotaState(nowTs);
QuotaProtos.Quotas defaultQuotas =
QuotaProtos.Quotas.newBuilder().setThrottle(throttleBuilder.build()).build();
state.setQuotas(defaultQuotas);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java
new file mode 100644
index 00000000000..1c431858291
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.apache.hadoop.hbase.quotas.ThrottleQuotaTestUtil.waitMinuteQuota;
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ RegionServerTests.class, MediumTests.class })
+public class TestQuotaCache {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestQuotaCache.class);
+
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final int REFRESH_TIME = 30_000;
+
+ @After
+ public void tearDown() throws Exception {
+ ThrottleQuotaTestUtil.clearQuotaCache(TEST_UTIL);
+ EnvironmentEdgeManager.reset();
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
+ TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, REFRESH_TIME);
+ TEST_UTIL.getConfiguration().setInt(QuotaUtil.QUOTA_DEFAULT_USER_MACHINE_READ_NUM, 1000);
+
+ TEST_UTIL.startMiniCluster(1);
+ TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME);
+ }
+
+ @Test
+ public void testDefaultUserRefreshFrequency() throws Exception {
+ QuotaCache.TEST_BLOCK_REFRESH = true;
+
+ QuotaCache quotaCache =
+ ThrottleQuotaTestUtil.getQuotaCaches(TEST_UTIL).stream().findAny().get();
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+
+ UserQuotaState userQuotaState = quotaCache.getUserQuotaState(ugi);
+ assertEquals(userQuotaState.getLastUpdate(), 0);
+
+ QuotaCache.TEST_BLOCK_REFRESH = false;
+ // new user should have refreshed immediately
+ TEST_UTIL.waitFor(5_000, () -> userQuotaState.getLastUpdate() != 0);
+ long lastUpdate = userQuotaState.getLastUpdate();
+
+ // refresh should not apply to recently refreshed quota
+ quotaCache.triggerCacheRefresh();
+ Thread.sleep(250);
+ long newLastUpdate = userQuotaState.getLastUpdate();
+ assertEquals(lastUpdate, newLastUpdate);
+
+ quotaCache.triggerCacheRefresh();
+ waitMinuteQuota();
+ // should refresh after time has passed
+ TEST_UTIL.waitFor(5_000, () -> lastUpdate != userQuotaState.getLastUpdate());
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
index 0bee1bf7c7a..ec1b1d4bdcc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/ThrottleQuotaTestUtil.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.hbase.quotas;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Random;
+import java.util.Set;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter.ExplainingPredicate;
@@ -283,6 +285,16 @@ public final class ThrottleQuotaTestUtil {
}
}
+ static Set<QuotaCache> getQuotaCaches(HBaseTestingUtility testUtil) {
+ Set<QuotaCache> quotaCaches = new HashSet<>();
+ for (RegionServerThread rst : testUtil.getMiniHBaseCluster().getRegionServerThreads()) {
+ RegionServerRpcQuotaManager quotaManager =
+ rst.getRegionServer().getRegionServerRpcQuotaManager();
+ quotaCaches.add(quotaManager.getQuotaCache());
+ }
+ return quotaCaches;
+ }
+
static void waitMinuteQuota() {
envEdge.incValue(70000);
}