You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bb...@apache.org on 2023/06/23 17:58:56 UTC

[hbase] branch branch-2.4 updated: HBASE-27950 Addendum to fix test for branch's scanner semantics

This is an automated email from the ASF dual-hosted git repository.

bbeaudreault pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
     new 340b997d1b0 HBASE-27950 Addendum to fix test for branch's scanner semantics
340b997d1b0 is described below

commit 340b997d1b083f38610d5ad00f2db7367aef2ae0
Author: Bryan Beaudreault <bb...@hubspot.com>
AuthorDate: Fri Jun 23 12:28:56 2023 -0400

    HBASE-27950 Addendum to fix test for branch's scanner semantics
---
 .../hbase/client/TestClientSideRegionScanner.java  | 77 ++++++++++------------
 1 file changed, 35 insertions(+), 42 deletions(-)

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientSideRegionScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientSideRegionScanner.java
index e0e7187da91..dc67892dc04 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientSideRegionScanner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientSideRegionScanner.java
@@ -22,25 +22,20 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.ArgumentMatchers.anyList;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
 
 import java.io.IOException;
-import java.util.Arrays;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CompareOperator;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.filter.FilterBase;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.io.hfile.BlockCache;
 import org.apache.hadoop.hbase.io.hfile.IndexOnlyLruBlockCache;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.regionserver.StoreScanner;
 import org.apache.hadoop.hbase.testclassification.ClientTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
@@ -61,6 +56,7 @@ public class TestClientSideRegionScanner {
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static final TableName TABLE_NAME = TableName.valueOf("test");
   private static final byte[] FAM_NAME = Bytes.toBytes("f");
+  private static final byte[] FAM_NAME_2 = Bytes.toBytes("f2");
 
   private Configuration conf;
   private Path rootDir;
@@ -130,21 +126,38 @@ public class TestClientSideRegionScanner {
 
   @Test
   public void testContinuesToScanIfHasMore() throws IOException {
-    // Conditions for this test to set up RegionScannerImpl to bail on the scan
-    // after a single iteration
-    // 1. Configure preadMaxBytes to something small to trigger scannerContext#returnImmediately
-    // 2. Configure a filter to filter out some rows, in this case rows with values < 5
-    // 3. Configure the filter's hasFilterRow to return true so RegionScannerImpl sets
-    // the limitScope to something with a depth of 0, so we bail on the scan after the first
-    // iteration
+    // In order to hit this bug, we need RegionScanner to exit out early despite having retrieved
+    // no values. By default, RegionScanner will continue until it gets values or a limit is
+    // exceeded. In the context of this test, the easiest way to do that is to trigger the
+    // retryImmediately behavior of converting from PREAD to STREAM. To do that, we set PREAD max
+    // bytes to a small number. Then we need to actually check our limits, which would trigger
+    // retryImmediately. To do that, we use joinedContinuationRow (essential family) through
+    // a SingleColumnValueFilter with setFilterIfMissing(true). This causes FAM_NAME_2 to be
+    // non-essential, and only brought in when the filter passes. This causes us to go through the
+    // limit check in joinedContinuationRow and exit out early with hasMore=true but empty values.
+    // Prior to HBASE-27950, the first call to ClientSideRegionScanner.next() below would
+    // return null because of the empty values list, and there would further be nulls in
+    // between row 5 and 8. With the fix, the scanner appropriately continues iterating, only
+    // returning if there are values or finally when hasMore=false at the end. Thus, the first
+    // next returns row 5, then row 8, then null because the scanner is exhausted. The intermediate
+    // nulls from rows in between are correctly skipped.
 
     Configuration copyConf = new Configuration(conf);
     copyConf.setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES, 1);
+    int[] filteredRows = new int[] { 5, 8 };
+    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
+    for (int filteredRow : filteredRows) {
+      SingleColumnValueFilter filter = new SingleColumnValueFilter(FAM_NAME,
+        Bytes.toBytes(filteredRow), CompareOperator.EQUAL, Bytes.toBytes(filteredRow));
+      filter.setFilterIfMissing(true);
+      filterList.addFilter(filter);
+    }
+
     Scan scan = new Scan();
-    scan.setFilter(new FiltersRowsLessThan5());
+    scan.setFilter(filterList);
     scan.setLimit(1);
 
-    try (Table table = TEST_UTIL.createTable(TABLE_NAME, FAM_NAME)) {
+    try (Table table = TEST_UTIL.createTable(TABLE_NAME, new byte[][] { FAM_NAME, FAM_NAME_2 })) {
       TableDescriptor htd = TEST_UTIL.getAdmin().getDescriptor(TABLE_NAME);
       RegionInfo hri = TEST_UTIL.getAdmin().getRegions(TABLE_NAME).get(0);
 
@@ -157,20 +170,14 @@ public class TestClientSideRegionScanner {
 
       ClientSideRegionScanner clientSideRegionScanner =
         new ClientSideRegionScanner(copyConf, fs, rootDir, htd, hri, scan, null);
-      RegionScanner scannerSpy = spy(clientSideRegionScanner.scanner);
-      clientSideRegionScanner.scanner = scannerSpy;
-      Result result = clientSideRegionScanner.next();
 
-      verify(scannerSpy, times(6)).nextRaw(anyList());
-      assertNotNull(result);
-      assertEquals(Bytes.toInt(result.getRow()), 5);
-      assertTrue(clientSideRegionScanner.hasMore);
+      Result result;
 
-      for (int i = 6; i < 10; ++i) {
+      for (int filteredRow : filteredRows) {
         result = clientSideRegionScanner.next();
-        verify(scannerSpy, times(i + 1)).nextRaw(anyList());
         assertNotNull(result);
-        assertEquals(Bytes.toInt(result.getRow()), i);
+        assertEquals(Bytes.toInt(result.getRow()), filteredRow);
+        assertTrue(clientSideRegionScanner.hasMore);
       }
 
       result = clientSideRegionScanner.next();
@@ -183,22 +190,8 @@ public class TestClientSideRegionScanner {
     byte[] row = Bytes.toBytes(rowAsInt);
     Put put = new Put(row);
     put.addColumn(FAM_NAME, row, row);
+    put.addColumn(FAM_NAME_2, row, row);
     return put;
   }
 
-  private static class FiltersRowsLessThan5 extends FilterBase {
-
-    @Override
-    public boolean filterRowKey(Cell cell) {
-      byte[] rowKey = Arrays.copyOfRange(cell.getRowArray(), cell.getRowOffset(),
-        cell.getRowLength() + cell.getRowOffset());
-      int intValue = Bytes.toInt(rowKey);
-      return intValue < 5;
-    }
-
-    @Override
-    public boolean hasFilterRow() {
-      return true;
-    }
-  }
 }