You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bb...@apache.org on 2023/06/23 17:58:56 UTC
[hbase] branch branch-2.4 updated: HBASE-27950 Addendum to fix test for branch's scanner semantics
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 340b997d1b0 HBASE-27950 Addendum to fix test for branch's scanner semantics
340b997d1b0 is described below
commit 340b997d1b083f38610d5ad00f2db7367aef2ae0
Author: Bryan Beaudreault <bb...@hubspot.com>
AuthorDate: Fri Jun 23 12:28:56 2023 -0400
HBASE-27950 Addendum to fix test for branch's scanner semantics
---
.../hbase/client/TestClientSideRegionScanner.java | 77 ++++++++++------------
1 file changed, 35 insertions(+), 42 deletions(-)
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientSideRegionScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientSideRegionScanner.java
index e0e7187da91..dc67892dc04 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientSideRegionScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientSideRegionScanner.java
@@ -22,25 +22,20 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.mockito.ArgumentMatchers.anyList;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
import java.io.IOException;
-import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CompareOperator;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.IndexOnlyLruBlockCache;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.StoreScanner;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
@@ -61,6 +56,7 @@ public class TestClientSideRegionScanner {
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final TableName TABLE_NAME = TableName.valueOf("test");
private static final byte[] FAM_NAME = Bytes.toBytes("f");
+ private static final byte[] FAM_NAME_2 = Bytes.toBytes("f2");
private Configuration conf;
private Path rootDir;
@@ -130,21 +126,38 @@ public class TestClientSideRegionScanner {
@Test
public void testContinuesToScanIfHasMore() throws IOException {
- // Conditions for this test to set up RegionScannerImpl to bail on the scan
- // after a single iteration
- // 1. Configure preadMaxBytes to something small to trigger scannerContext#returnImmediately
- // 2. Configure a filter to filter out some rows, in this case rows with values < 5
- // 3. Configure the filter's hasFilterRow to return true so RegionScannerImpl sets
- // the limitScope to something with a depth of 0, so we bail on the scan after the first
- // iteration
+ // In order to hit this bug, we need RegionScanner to exit out early despite having retrieved
+ // no values. By default, RegionScanner will continue until it gets values or a limit is
+ // exceeded. In the context of this test, the easiest way to do that is to trigger the
+ // retryImmediately behavior of converting from PREAD to STREAM. To do that, we set PREAD max
+ // bytes to a small number. Then we need to actually check our limits, which would trigger
+ // retryImmediately. To do that, we use joinedContinuationRow (essential family) through
+ // a SingleColumnValueFilter with setFilterIfMissing(true). This causes FAM_NAME_2 to be
+ // non-essential, and only brought in when the filter passes. This causes us to go through the
+ // limit check in joinedContinuationRow and exit out early with hasMore=true but empty values.
+ // Prior to HBASE-27950, the first call to ClientSideRegionScanner.next() below would
+ // return null because of the empty values list, and there would further be nulls in
+ // between row 5 and 8. With the fix, the scanner appropriately continues iterating, only
+ // returning if there are values or finally when hasMore=false at the end. Thus, the first
+ // next returns row 5, then row 8, then null because the scanner is exhausted. The intermediate
+ // nulls from rows in between are correctly skipped.
Configuration copyConf = new Configuration(conf);
copyConf.setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES, 1);
+ int[] filteredRows = new int[] { 5, 8 };
+ FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
+ for (int filteredRow : filteredRows) {
+ SingleColumnValueFilter filter = new SingleColumnValueFilter(FAM_NAME,
+ Bytes.toBytes(filteredRow), CompareOperator.EQUAL, Bytes.toBytes(filteredRow));
+ filter.setFilterIfMissing(true);
+ filterList.addFilter(filter);
+ }
+
Scan scan = new Scan();
- scan.setFilter(new FiltersRowsLessThan5());
+ scan.setFilter(filterList);
scan.setLimit(1);
- try (Table table = TEST_UTIL.createTable(TABLE_NAME, FAM_NAME)) {
+ try (Table table = TEST_UTIL.createTable(TABLE_NAME, new byte[][] { FAM_NAME, FAM_NAME_2 })) {
TableDescriptor htd = TEST_UTIL.getAdmin().getDescriptor(TABLE_NAME);
RegionInfo hri = TEST_UTIL.getAdmin().getRegions(TABLE_NAME).get(0);
@@ -157,20 +170,14 @@ public class TestClientSideRegionScanner {
ClientSideRegionScanner clientSideRegionScanner =
new ClientSideRegionScanner(copyConf, fs, rootDir, htd, hri, scan, null);
- RegionScanner scannerSpy = spy(clientSideRegionScanner.scanner);
- clientSideRegionScanner.scanner = scannerSpy;
- Result result = clientSideRegionScanner.next();
- verify(scannerSpy, times(6)).nextRaw(anyList());
- assertNotNull(result);
- assertEquals(Bytes.toInt(result.getRow()), 5);
- assertTrue(clientSideRegionScanner.hasMore);
+ Result result;
- for (int i = 6; i < 10; ++i) {
+ for (int filteredRow : filteredRows) {
result = clientSideRegionScanner.next();
- verify(scannerSpy, times(i + 1)).nextRaw(anyList());
assertNotNull(result);
- assertEquals(Bytes.toInt(result.getRow()), i);
+ assertEquals(Bytes.toInt(result.getRow()), filteredRow);
+ assertTrue(clientSideRegionScanner.hasMore);
}
result = clientSideRegionScanner.next();
@@ -183,22 +190,8 @@ public class TestClientSideRegionScanner {
byte[] row = Bytes.toBytes(rowAsInt);
Put put = new Put(row);
put.addColumn(FAM_NAME, row, row);
+ put.addColumn(FAM_NAME_2, row, row);
return put;
}
- private static class FiltersRowsLessThan5 extends FilterBase {
-
- @Override
- public boolean filterRowKey(Cell cell) {
- byte[] rowKey = Arrays.copyOfRange(cell.getRowArray(), cell.getRowOffset(),
- cell.getRowLength() + cell.getRowOffset());
- int intValue = Bytes.toInt(rowKey);
- return intValue < 5;
- }
-
- @Override
- public boolean hasFilterRow() {
- return true;
- }
- }
}