You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2018/08/21 20:31:26 UTC
hbase git commit: HBASE-21032 ScanResponses contain only one cell each
Repository: hbase
Updated Branches:
refs/heads/branch-2.1 5a40eae63 -> 5f03be467
HBASE-21032 ScanResponses contain only one cell each
Amending-Author: Duo Zhang <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5f03be46
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5f03be46
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5f03be46
Branch: refs/heads/branch-2.1
Commit: 5f03be46754a27649d4a4fe22c79d460a6c52025
Parents: 5a40eae
Author: Andrey Elenskiy <a...@pickle.vodka>
Authored: Mon Aug 20 17:21:10 2018 +0800
Committer: Michael Stack <st...@apache.org>
Committed: Tue Aug 21 13:31:18 2018 -0700
----------------------------------------------------------------------
.../hadoop/hbase/regionserver/StoreScanner.java | 18 ++--
.../hbase/client/TestGetScanPartialResult.java | 102 +++++++++++++++++++
2 files changed, 111 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5f03be46/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index f4cc24d..736c08a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -175,16 +175,16 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
this.oldestUnexpiredTS = scan.isRaw() ? 0L : now - scanInfo.getTtl();
this.minVersions = scanInfo.getMinVersions();
- // We look up row-column Bloom filters for multi-column queries as part of
- // the seek operation. However, we also look the row-column Bloom filter
- // for multi-row (non-"get") scans because this is not done in
- // StoreFile.passesBloomFilter(Scan, SortedSet<byte[]>).
- this.useRowColBloom = numColumns > 1 || (!get && numColumns == 1);
- this.maxRowSize = scanInfo.getTableMaxRowSize();
+ // We look up row-column Bloom filters for multi-column queries as part of
+ // the seek operation. However, we also look the row-column Bloom filter
+ // for multi-row (non-"get") scans because this is not done in
+ // StoreFile.passesBloomFilter(Scan, SortedSet<byte[]>).
+ this.useRowColBloom = numColumns > 1 || (!get && numColumns == 1);
+ this.maxRowSize = scanInfo.getTableMaxRowSize();
if (get) {
this.readType = Scan.ReadType.PREAD;
this.scanUsePread = true;
- } else if(scanType != ScanType.USER_SCAN) {
+ } else if (scanType != ScanType.USER_SCAN) {
// For compaction scanners never use Pread as already we have stream based scanners on the
// store files to be compacted
this.readType = Scan.ReadType.STREAM;
@@ -556,7 +556,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
// Or if the preadMaxBytes is reached and we may want to return so we can switch to stream in
// the shipped method below.
if (kvsScanned % cellsPerHeartbeatCheck == 0 || (scanUsePread &&
- scan.getReadType() == Scan.ReadType.DEFAULT && bytesRead > preadMaxBytes)) {
+ readType == Scan.ReadType.DEFAULT && bytesRead > preadMaxBytes)) {
if (scannerContext.checkTimeLimit(LimitScope.BETWEEN_CELLS)) {
return scannerContext.setScannerState(NextState.TIME_LIMIT_REACHED).hasMoreValues();
}
@@ -568,7 +568,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
checkScanOrder(prevCell, cell, comparator);
int cellSize = PrivateCellUtil.estimatedSerializedSizeOf(cell);
bytesRead += cellSize;
- if (scanUsePread && scan.getReadType() == Scan.ReadType.DEFAULT &&
+ if (scanUsePread && readType == Scan.ReadType.DEFAULT &&
bytesRead > preadMaxBytes) {
// return immediately if we want to switch from pread to stream. We need this because we can
// only switch in the shipped method, if user use a filter to filter out everything and rpc
http://git-wip-us.apache.org/repos/asf/hbase/blob/5f03be46/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestGetScanPartialResult.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestGetScanPartialResult.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestGetScanPartialResult.java
new file mode 100644
index 0000000..0925aee
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestGetScanPartialResult.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.testclassification.ClientTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Testcase for HBASE-21032, where use the wrong readType from a Scan instance which is actually a
+ * get scan and cause returning only 1 cell per rpc call.
+ */
+@Category({ ClientTests.class, MediumTests.class })
+public class TestGetScanPartialResult {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestGetScanPartialResult.class);
+
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final TableName TABLE = TableName.valueOf("table");
+ private static final byte[] CF = { 'c', 'f' };
+ private static final byte[] ROW = { 'r', 'o', 'w' };
+ private static final int VALUE_SIZE = 10000;
+ private static final int NUM_COLUMNS = 300;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ TEST_UTIL.startMiniCluster(1);
+ TEST_UTIL.createTable(TABLE, CF);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ private static byte[] makeLargeValue(int size) {
+ byte[] v = new byte[size];
+ for (int i = 0; i < size; i++) {
+ v[i] = 0;
+ }
+ return v;
+ }
+
+ @Test
+ public void test() throws IOException {
+ try (Table t = TEST_UTIL.getConnection().getTable(TABLE)) {
+ // populate a row with bunch of columns and large values
+ // to cause scan to return partials
+ byte[] val = makeLargeValue(VALUE_SIZE);
+ Put p = new Put(ROW);
+ for (int i = 0; i < NUM_COLUMNS; i++) {
+ p.addColumn(CF, Integer.toString(i).getBytes(), val);
+ }
+ t.put(p);
+
+ Scan scan = new Scan();
+ scan.withStartRow(ROW);
+ scan.withStopRow(ROW, true);
+ scan.setAllowPartialResults(true);
+ scan.setMaxResultSize(2L * 1024 * 1024);
+ scan.readVersions(1);
+ ResultScanner scanner = t.getScanner(scan);
+
+ int nResults = 0;
+ int nCells = 0;
+ for (Result result = scanner.next(); (result != null); result = scanner.next()) {
+ nResults++;
+ nCells += result.listCells().size();
+ }
+ assertEquals(NUM_COLUMNS, nCells);
+ assertTrue(nResults < 5);
+ }
+ }
+}