You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2023/02/15 04:16:06 UTC
[hbase] branch branch-2 updated: HBASE-27637 Zero length value would cause value compressor read nothing and not advance the position of the InputStream (#5025)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new ee618d5fdf8 HBASE-27637 Zero length value would cause value compressor read nothing and not advance the position of the InputStream (#5025)
ee618d5fdf8 is described below
commit ee618d5fdf8294ff98860b83c40e9ccba5b0f73b
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Wed Feb 15 12:00:08 2023 +0800
HBASE-27637 Zero length value would cause value compressor read nothing and not advance the position of the InputStream (#5025)
Signed-off-by: Xiaolin Ha <ha...@apache.org>
Signed-off-by: Andrew Purtell <ap...@apache.org>
(cherry picked from commit 2bbe036e2999f886008bb83988c541eb879e2431)
---
.../hbase/regionserver/wal/CompressionContext.java | 18 +++++---
.../hbase/regionserver/wal/WALCellCodec.java | 6 +--
.../hadoop/hbase/wal/CompressedWALTestBase.java | 52 +++++++++++++---------
.../wal/TestCompressedWALValueCompression.java | 3 +-
4 files changed, 46 insertions(+), 33 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
index 0ac0ce0d4fb..73cf4821db0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/CompressionContext.java
@@ -25,6 +25,7 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.EnumMap;
import java.util.Map;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.io.BoundedDelegatingInputStream;
@@ -105,7 +106,7 @@ public class CompressionContext {
return compressed;
}
- public int decompress(InputStream in, int inLength, byte[] outArray, int outOffset,
+ public void decompress(InputStream in, int inLength, byte[] outArray, int outOffset,
int outLength) throws IOException {
// Our input is a sequence of bounded byte ranges (call them segments), with
@@ -122,11 +123,16 @@ public class CompressionContext {
} else {
lowerIn.setDelegate(in, inLength);
}
-
- // Caller must handle short reads.
- // With current Hadoop compression codecs all 'outLength' bytes are read in here, so not
- // an issue for now.
- return compressedIn.read(outArray, outOffset, outLength);
+ if (outLength == 0) {
+ // The BufferedInputStream will return earlier and skip reading anything if outLength == 0,
+ // but in fact for an empty value, the compressed output still contains some metadata so the
+ // compressed size is not 0, so here we need to manually skip inLength bytes otherwise the
+ // next read on this stream will start from an invalid position and cause critical problem,
+ // such as data loss when splitting wal or replicating wal.
+ IOUtils.skipFully(in, inLength);
+ } else {
+ IOUtils.readFully(compressedIn, outArray, outOffset, outLength);
+ }
}
public void clear() {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java
index 816ce3ed45a..84709cbc58d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java
@@ -382,13 +382,9 @@ public class WALCellCodec implements Codec {
private void readCompressedValue(InputStream in, byte[] outArray, int outOffset,
int expectedLength) throws IOException {
int compressedLen = StreamUtils.readRawVarint32(in);
- int read = compression.getValueCompressor().decompress(in, compressedLen, outArray, outOffset,
+ compression.getValueCompressor().decompress(in, compressedLen, outArray, outOffset,
expectedLength);
- if (read != expectedLength) {
- throw new IOException("ValueCompressor state error: short read");
- }
}
-
}
public static class EnsureKvEncoder extends BaseEncoder {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java
index 2f577bfa78e..b5f7dc634c7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/CompressedWALTestBase.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.wal;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -26,8 +28,9 @@ import java.util.NavigableMap;
import java.util.TreeMap;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
@@ -82,33 +85,42 @@ public class CompressedWALTestBase {
for (int i = 0; i < total; i++) {
WALEdit kvs = new WALEdit();
- kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));
+ kvs.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setType(Cell.Type.Put)
+ .setRow(row).setFamily(family).setQualifier(Bytes.toBytes(i)).setValue(value).build());
+ kvs.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
+ .setType(Cell.Type.DeleteFamily).setRow(row).setFamily(family).build());
wal.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName,
System.currentTimeMillis(), mvcc, scopes), kvs);
+ wal.sync();
}
- wal.sync();
final Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);
wals.shutdown();
// Confirm the WAL can be read back
- WAL.Reader reader = wals.createReader(TEST_UTIL.getTestFileSystem(), walPath);
- int count = 0;
- WAL.Entry entry = new WAL.Entry();
- while (reader.next(entry) != null) {
- count++;
- List<Cell> cells = entry.getEdit().getCells();
- assertTrue("Should be one KV per WALEdit", cells.size() == 1);
- for (Cell cell : cells) {
- assertTrue("Incorrect row", Bytes.equals(cell.getRowArray(), cell.getRowOffset(),
- cell.getRowLength(), row, 0, row.length));
- assertTrue("Incorrect family", Bytes.equals(cell.getFamilyArray(), cell.getFamilyOffset(),
- cell.getFamilyLength(), family, 0, family.length));
- assertTrue("Incorrect value", Bytes.equals(cell.getValueArray(), cell.getValueOffset(),
- cell.getValueLength(), value, 0, value.length));
+ try (WAL.Reader reader = wals.createReader(TEST_UTIL.getTestFileSystem(), walPath)) {
+ int count = 0;
+ WAL.Entry entry = new WAL.Entry();
+ while (reader.next(entry) != null) {
+ count++;
+ List<Cell> cells = entry.getEdit().getCells();
+ assertThat("Should be two KVs per WALEdit", cells, hasSize(2));
+ Cell putCell = cells.get(0);
+ assertEquals(Cell.Type.Put, putCell.getType());
+ assertTrue("Incorrect row", Bytes.equals(putCell.getRowArray(), putCell.getRowOffset(),
+ putCell.getRowLength(), row, 0, row.length));
+ assertTrue("Incorrect family", Bytes.equals(putCell.getFamilyArray(),
+ putCell.getFamilyOffset(), putCell.getFamilyLength(), family, 0, family.length));
+ assertTrue("Incorrect value", Bytes.equals(putCell.getValueArray(),
+ putCell.getValueOffset(), putCell.getValueLength(), value, 0, value.length));
+
+ Cell deleteCell = cells.get(1);
+ assertEquals(Cell.Type.DeleteFamily, deleteCell.getType());
+ assertTrue("Incorrect row", Bytes.equals(deleteCell.getRowArray(),
+ deleteCell.getRowOffset(), deleteCell.getRowLength(), row, 0, row.length));
+ assertTrue("Incorrect family", Bytes.equals(deleteCell.getFamilyArray(),
+ deleteCell.getFamilyOffset(), deleteCell.getFamilyLength(), family, 0, family.length));
}
+ assertEquals("Should have read back as many KVs as written", total, count);
}
- assertEquals("Should have read back as many KVs as written", total, count);
- reader.close();
}
-
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java
index 2c35b0accc6..88c0c889ade 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestCompressedWALValueCompression.java
@@ -45,7 +45,7 @@ public class TestCompressedWALValueCompression extends CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestCompressedWALValueCompression.class);
- @Parameters
+ @Parameters(name = "{index}: compression={0}")
public static List<Object[]> params() {
return HBaseTestingUtility.COMPRESSION_ALGORITHMS_PARAMETERIZED;
}
@@ -78,5 +78,4 @@ public class TestCompressedWALValueCompression extends CompressedWALTestBase {
TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
doTest(tableName);
}
-
}