You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2017/08/20 21:30:43 UTC
[46/50] [abbrv] hbase git commit: HBASE-18471 The DeleteFamily cell
is skipped when StoreScanner seeks to next column
HBASE-18471 The DeleteFamily cell is skipped when StoreScanner seeks to next column
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/56f9e1a6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/56f9e1a6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/56f9e1a6
Branch: refs/heads/HBASE-18467
Commit: 56f9e1a60af224f8f3b0453932155ca46d44cf71
Parents: e9bafeb
Author: Chia-Ping Tsai <ch...@gmail.com>
Authored: Sat Aug 19 02:15:12 2017 +0800
Committer: Chia-Ping Tsai <ch...@gmail.com>
Committed: Sat Aug 19 02:15:12 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hbase/CellUtil.java | 53 ++++++++++
.../querymatcher/ScanQueryMatcher.java | 13 +++
.../hbase/client/TestFromClientSide3.java | 101 +++++++++++++++++++
3 files changed, 167 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/56f9e1a6/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
index 1146de4..03d8b70 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
@@ -3135,4 +3135,57 @@ public final class CellUtil {
return Type.DeleteFamily.getCode();
}
}
+
+ /**
+ * @return An new cell is located following input cell. If both of type and timestamp are
+ * minimum, the input cell will be returned directly.
+ */
+ @InterfaceAudience.Private
+ public static Cell createNextOnRowCol(Cell cell) {
+ long ts = cell.getTimestamp();
+ byte type = cell.getTypeByte();
+ if (type != Type.Minimum.getCode()) {
+ type = KeyValue.Type.values()[KeyValue.Type.codeToType(type).ordinal() - 1].getCode();
+ } else if (ts != HConstants.OLDEST_TIMESTAMP) {
+ ts = ts - 1;
+ type = Type.Maximum.getCode();
+ } else {
+ return cell;
+ }
+ return createNextOnRowCol(cell, ts, type);
+ }
+
+ private static Cell createNextOnRowCol(Cell cell, long ts, byte type) {
+ if (cell instanceof ByteBufferCell) {
+ return new LastOnRowColByteBufferCell(((ByteBufferCell) cell).getRowByteBuffer(),
+ ((ByteBufferCell) cell).getRowPosition(), cell.getRowLength(),
+ ((ByteBufferCell) cell).getFamilyByteBuffer(),
+ ((ByteBufferCell) cell).getFamilyPosition(), cell.getFamilyLength(),
+ ((ByteBufferCell) cell).getQualifierByteBuffer(),
+ ((ByteBufferCell) cell).getQualifierPosition(), cell.getQualifierLength()) {
+ @Override
+ public long getTimestamp() {
+ return ts;
+ }
+
+ @Override
+ public byte getTypeByte() {
+ return type;
+ }
+ };
+ }
+ return new LastOnRowColCell(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
+ cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
+ cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()) {
+ @Override
+ public long getTimestamp() {
+ return ts;
+ }
+
+ @Override
+ public byte getTypeByte() {
+ return type;
+ }
+ };
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/56f9e1a6/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ScanQueryMatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ScanQueryMatcher.java
index 524d3f7..f00a400 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ScanQueryMatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ScanQueryMatcher.java
@@ -290,6 +290,19 @@ public abstract class ScanQueryMatcher implements ShipperListener {
public abstract boolean moreRowsMayExistAfter(Cell cell);
public Cell getKeyForNextColumn(Cell cell) {
+ // We aren't sure whether any DeleteFamily cells exist, so we can't skip to next column.
+ // TODO: Current way disable us to seek to next column quickly. Is there any better solution?
+ // see HBASE-18471 for more details
+ // see TestFromClientSide3#testScanAfterDeletingSpecifiedRow
+ // see TestFromClientSide3#testScanAfterDeletingSpecifiedRowV2
+ if (cell.getQualifierLength() == 0) {
+ Cell nextKey = CellUtil.createNextOnRowCol(cell);
+ if (nextKey != cell) {
+ return nextKey;
+ }
+ // The cell is at the end of row/family/qualifier, so it is impossible to find any DeleteFamily cells.
+ // Let us seek to next column.
+ }
ColumnCount nextColumn = columns.getColumnHint();
if (nextColumn == null) {
return CellUtil.createLastOnRowCol(cell);
http://git-wip-us.apache.org/repos/asf/hbase/blob/56f9e1a6/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
index f20c050..7b6c9a8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java
@@ -34,6 +34,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
@@ -159,6 +160,106 @@ public class TestFromClientSide3 {
}
}
+ private static List<Cell> toList(ResultScanner scanner) {
+ try {
+ List<Cell> cells = new ArrayList<>();
+ for (Result r : scanner) {
+ cells.addAll(r.listCells());
+ }
+ return cells;
+ } finally {
+ scanner.close();
+ }
+ }
+
+ @Test
+ public void testScanAfterDeletingSpecifiedRow() throws IOException {
+ TableName tableName = TableName.valueOf(name.getMethodName());
+ TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
+ .addColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY))
+ .build();
+ TEST_UTIL.getAdmin().createTable(desc);
+ byte[] row = Bytes.toBytes("SpecifiedRow");
+ byte[] value0 = Bytes.toBytes("value_0");
+ byte[] value1 = Bytes.toBytes("value_1");
+ try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
+ Put put = new Put(row);
+ put.addColumn(FAMILY, QUALIFIER, VALUE);
+ t.put(put);
+ Delete d = new Delete(row);
+ t.delete(d);
+ put = new Put(row);
+ put.addColumn(FAMILY, null, value0);
+ t.put(put);
+ put = new Put(row);
+ put.addColumn(FAMILY, null, value1);
+ t.put(put);
+ List<Cell> cells = toList(t.getScanner(new Scan()));
+ assertEquals(1, cells.size());
+ assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
+
+ cells = toList(t.getScanner(new Scan().addFamily(FAMILY)));
+ assertEquals(1, cells.size());
+ assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
+
+ cells = toList(t.getScanner(new Scan().addColumn(FAMILY, QUALIFIER)));
+ assertEquals(0, cells.size());
+
+ TEST_UTIL.getAdmin().flush(tableName);
+ cells = toList(t.getScanner(new Scan()));
+ assertEquals(1, cells.size());
+ assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
+
+ cells = toList(t.getScanner(new Scan().addFamily(FAMILY)));
+ assertEquals(1, cells.size());
+ assertEquals("value_1", Bytes.toString(CellUtil.cloneValue(cells.get(0))));
+
+ cells = toList(t.getScanner(new Scan().addColumn(FAMILY, QUALIFIER)));
+ assertEquals(0, cells.size());
+ }
+ }
+
+ @Test
+ public void testScanAfterDeletingSpecifiedRowV2() throws IOException {
+ TableName tableName = TableName.valueOf(name.getMethodName());
+ TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
+ .addColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY))
+ .build();
+ TEST_UTIL.getAdmin().createTable(desc);
+ byte[] row = Bytes.toBytes("SpecifiedRow");
+ byte[] qual0 = Bytes.toBytes("qual0");
+ byte[] qual1 = Bytes.toBytes("qual1");
+ try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
+ Delete d = new Delete(row);
+ t.delete(d);
+
+ Put put = new Put(row);
+ put.addColumn(FAMILY, null, VALUE);
+ t.put(put);
+
+ put = new Put(row);
+ put.addColumn(FAMILY, qual1, qual1);
+ t.put(put);
+
+ put = new Put(row);
+ put.addColumn(FAMILY, qual0, qual0);
+ t.put(put);
+
+ Result r = t.get(new Get(row));
+ assertEquals(3, r.size());
+ assertEquals("testValue", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0])));
+ assertEquals("qual0", Bytes.toString(CellUtil.cloneValue(r.rawCells()[1])));
+ assertEquals("qual1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[2])));
+
+ TEST_UTIL.getAdmin().flush(tableName);
+ r = t.get(new Get(row));
+ assertEquals(3, r.size());
+ assertEquals("testValue", Bytes.toString(CellUtil.cloneValue(r.rawCells()[0])));
+ assertEquals("qual0", Bytes.toString(CellUtil.cloneValue(r.rawCells()[1])));
+ assertEquals("qual1", Bytes.toString(CellUtil.cloneValue(r.rawCells()[2])));
+ }
+ }
+
// override the config settings at the CF level and ensure priority
@Test(timeout = 60000)
public void testAdvancedConfigOverride() throws Exception {