You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bb...@apache.org on 2024/02/05 13:28:41 UTC
(hbase) branch branch-3 updated (fdd8d560179 -> ee6332a79c4)
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a change to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
from fdd8d560179 HBASE-28336 Correctly cast array of Certificates to array of X509Certificates (#5658)
new 4194a4b0f49 HBASE-28334 Add comment around erasure coding policy in DEFAULT_VALUE map (#5662)
new ee6332a79c4 HBASE-28338 Bounded leak of FSDataInputStream buffers from checksum switching (#5660)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hbase/client/TableDescriptorBuilder.java | 1 +
.../hadoop/hbase/io/FSDataInputStreamWrapper.java | 46 +++++-----------------
.../hbase/io/TestFSDataInputStreamWrapper.java | 28 ++++++-------
3 files changed, 24 insertions(+), 51 deletions(-)
(hbase) 02/02: HBASE-28338 Bounded leak of FSDataInputStream buffers from checksum switching (#5660)
Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit ee6332a79c43dfdb179c8c8a73391967cbd08ad5
Author: Bryan Beaudreault <bb...@apache.org>
AuthorDate: Mon Feb 5 08:26:31 2024 -0500
HBASE-28338 Bounded leak of FSDataInputStream buffers from checksum switching (#5660)
Signed-off-by: Duo Zhang <zh...@apache.org>
---
.../hadoop/hbase/io/FSDataInputStreamWrapper.java | 46 +++++-----------------
.../hbase/io/TestFSDataInputStreamWrapper.java | 28 ++++++-------
2 files changed, 23 insertions(+), 51 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java
index cb9dc84b94b..33eace47d63 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java
@@ -19,17 +19,13 @@ package org.apache.hadoop.hbase.io;
import java.io.Closeable;
import java.io.IOException;
-import java.io.InputStream;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hadoop.fs.CanUnbuffer;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
@@ -40,8 +36,6 @@ import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
*/
@InterfaceAudience.Private
public class FSDataInputStreamWrapper implements Closeable {
- private static final Logger LOG = LoggerFactory.getLogger(FSDataInputStreamWrapper.class);
- private static final boolean isLogTraceEnabled = LOG.isTraceEnabled();
private final HFileSystem hfs;
private final Path path;
@@ -94,9 +88,6 @@ public class FSDataInputStreamWrapper implements Closeable {
long totalZeroCopyBytesRead;
}
- private Boolean instanceOfCanUnbuffer = null;
- private CanUnbuffer unbuffer = null;
-
protected Path readerPath;
public FSDataInputStreamWrapper(FileSystem fs, Path path) throws IOException {
@@ -314,41 +305,22 @@ public class FSDataInputStreamWrapper implements Closeable {
* stream, the current socket will be closed and a new socket will be opened to serve the
* requests.
*/
- @SuppressWarnings({ "rawtypes" })
public void unbuffer() {
+ // todo: it may make sense to always unbuffer both streams. we'd need to carefully
+ // research the usages to know if that is safe. for now just do the current.
FSDataInputStream stream = this.getStream(this.shouldUseHBaseChecksum());
if (stream != null) {
- InputStream wrappedStream = stream.getWrappedStream();
- // CanUnbuffer interface was added as part of HDFS-7694 and the fix is available in Hadoop
- // 2.6.4+ and 2.7.1+ versions only so check whether the stream object implements the
- // CanUnbuffer interface or not and based on that call the unbuffer api.
- final Class<? extends InputStream> streamClass = wrappedStream.getClass();
- if (this.instanceOfCanUnbuffer == null) {
- // To ensure we compute whether the stream is instance of CanUnbuffer only once.
- this.instanceOfCanUnbuffer = false;
- if (wrappedStream instanceof CanUnbuffer) {
- this.unbuffer = (CanUnbuffer) wrappedStream;
- this.instanceOfCanUnbuffer = true;
- }
- }
- if (this.instanceOfCanUnbuffer) {
- try {
- this.unbuffer.unbuffer();
- } catch (UnsupportedOperationException e) {
- if (isLogTraceEnabled) {
- LOG.trace("Failed to invoke 'unbuffer' method in class " + streamClass
- + " . So there may be the stream does not support unbuffering.", e);
- }
- }
- } else {
- if (isLogTraceEnabled) {
- LOG.trace("Failed to find 'unbuffer' method in class " + streamClass);
- }
- }
+ stream.unbuffer();
}
}
public Path getReaderPath() {
return readerPath;
}
+
+ // For tests
+ void setShouldUseHBaseChecksum() {
+ useHBaseChecksumConfigured = true;
+ useHBaseChecksum = true;
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestFSDataInputStreamWrapper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestFSDataInputStreamWrapper.java
index bb476670f10..77aa00ef91f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestFSDataInputStreamWrapper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestFSDataInputStreamWrapper.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.io;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
@@ -31,6 +32,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.HasEnhancedByteBufferAccess;
import org.apache.hadoop.fs.ReadOption;
+import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.io.ByteBufferPool;
@@ -48,22 +50,22 @@ public class TestFSDataInputStreamWrapper {
@Test
public void testUnbuffer() throws Exception {
InputStream pc = new ParentClass();
- FSDataInputStreamWrapper fsdisw1 = new FSDataInputStreamWrapper(new FSDataInputStream(pc));
+ InputStream noChecksumPc = new ParentClass();
+ FSDataInputStreamWrapper fsdisw1 =
+ new FSDataInputStreamWrapper(new FSDataInputStream(pc), new FSDataInputStream(noChecksumPc));
fsdisw1.unbuffer();
- // parent class should be true
+ // should have called main stream unbuffer, but not no-checksum
assertTrue(((ParentClass) pc).getIsCallUnbuffer());
+ assertFalse(((ParentClass) noChecksumPc).getIsCallUnbuffer());
+ // switch to checksums and call unbuffer again. should unbuffer the nochecksum stream now
+ fsdisw1.setShouldUseHBaseChecksum();
+ fsdisw1.unbuffer();
+ assertTrue(((ParentClass) noChecksumPc).getIsCallUnbuffer());
fsdisw1.close();
-
- InputStream cc1 = new ChildClass1();
- FSDataInputStreamWrapper fsdisw2 = new FSDataInputStreamWrapper(new FSDataInputStream(cc1));
- fsdisw2.unbuffer();
- // child1 class should be true
- assertTrue(((ChildClass1) cc1).getIsCallUnbuffer());
- fsdisw2.close();
}
private class ParentClass extends FSInputStream implements ByteBufferReadable, CanSetDropBehind,
- CanSetReadahead, HasEnhancedByteBufferAccess, CanUnbuffer {
+ CanSetReadahead, HasEnhancedByteBufferAccess, CanUnbuffer, StreamCapabilities {
public boolean isCallUnbuffer = false;
@@ -122,12 +124,10 @@ public class TestFSDataInputStreamWrapper {
public boolean seekToNewSource(long paramLong) throws IOException {
return false;
}
- }
- private class ChildClass1 extends ParentClass {
@Override
- public void unbuffer() {
- isCallUnbuffer = true;
+ public boolean hasCapability(String s) {
+ return s.equals(StreamCapabilities.UNBUFFER);
}
}
}
(hbase) 01/02: HBASE-28334 Add comment around erasure coding policy in DEFAULT_VALUE map (#5662)
Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4194a4b0f497a96eff9e6f9726c917c6eb7951a8
Author: Nirdosh Yadav <ni...@gmail.com>
AuthorDate: Mon Feb 5 18:46:43 2024 +0530
HBASE-28334 Add comment around erasure coding policy in DEFAULT_VALUE map (#5662)
Signed-off-by: Bryan Beaudreault <bb...@apache.org>
---
.../main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java
index 7cab90c7e37..fcdbe4e4ae6 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableDescriptorBuilder.java
@@ -234,6 +234,7 @@ public class TableDescriptorBuilder {
DEFAULT_VALUES.put(DURABILITY, DEFAULT_DURABLITY.name()); // use the enum name
DEFAULT_VALUES.put(REGION_REPLICATION, String.valueOf(DEFAULT_REGION_REPLICATION));
DEFAULT_VALUES.put(PRIORITY, String.valueOf(DEFAULT_PRIORITY));
+ // Setting ERASURE_CODING_POLICY to NULL so that it is not considered as metadata
DEFAULT_VALUES.put(ERASURE_CODING_POLICY, String.valueOf(DEFAULT_ERASURE_CODING_POLICY));
DEFAULT_VALUES.keySet().stream().map(s -> new Bytes(Bytes.toBytes(s)))
.forEach(RESERVED_KEYWORDS::add);