You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2024/05/17 14:18:55 UTC
(hbase) 03/03: HBASE-28579 Hide HFileScanner related methods in StoreFileReader (#5889)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 8fe524dc42e97cbbd8f284528c1d9a5eac50288c
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Fri May 17 22:07:40 2024 +0800
HBASE-28579 Hide HFileScanner related methods in StoreFileReader (#5889)
Signed-off-by: Xin Sun <su...@apache.org>
(cherry picked from commit b260199882ba9148e4b1fe6ae201259defcdd545)
---
.../hadoop/hbase/io/HalfStoreFileReader.java | 4 +-
.../hadoop/hbase/regionserver/StoreFileReader.java | 26 +--
.../hadoop/hbase/tool/BulkLoadHFilesTool.java | 99 ++++++-----
.../hadoop/hbase/io/TestHalfStoreFileReader.java | 42 +++--
.../hadoop/hbase/regionserver/TestCompaction.java | 14 +-
.../hbase/regionserver/TestFSErrorsExposed.java | 73 ++++----
.../hadoop/hbase/regionserver/TestHStoreFile.java | 194 +++++++++++----------
.../hbase/regionserver/TestMajorCompaction.java | 36 ++--
.../hbase/regionserver/TestRegionReplicas.java | 25 +--
9 files changed, 262 insertions(+), 251 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
index f4cccfd03b0..0989f73df0a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java
@@ -92,7 +92,7 @@ public class HalfStoreFileReader extends StoreFileReader {
}
@Override
- public HFileScanner getScanner(final boolean cacheBlocks, final boolean pread,
+ protected HFileScanner getScanner(final boolean cacheBlocks, final boolean pread,
final boolean isCompaction) {
final HFileScanner s = super.getScanner(cacheBlocks, pread, isCompaction);
return new HFileScanner() {
@@ -283,7 +283,7 @@ public class HalfStoreFileReader extends StoreFileReader {
return super.getLastKey();
}
// Get a scanner that caches the block and that uses pread.
- HFileScanner scanner = getScanner(true, true);
+ HFileScanner scanner = getScanner(true, true, false);
try {
if (scanner.seekBefore(this.splitCell)) {
return Optional.ofNullable(scanner.getKey());
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
index 09c379227bd..4f872d7084e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
@@ -183,31 +183,9 @@ public class StoreFileReader {
}
/**
- * @deprecated since 2.0.0 and will be removed in 3.0.0. Do not write further code which depends
- * on this call. Instead use getStoreFileScanner() which uses the StoreFileScanner
- * class/interface which is the preferred way to scan a store with higher level
- * concepts.
- * @param cacheBlocks should we cache the blocks?
- * @param pread use pread (for concurrent small readers)
- * @return the underlying HFileScanner
- * @see <a href="https://issues.apache.org/jira/browse/HBASE-15296">HBASE-15296</a>
+ * Will be overridden in HalfStoreFileReader
*/
- @Deprecated
- public HFileScanner getScanner(boolean cacheBlocks, boolean pread) {
- return getScanner(cacheBlocks, pread, false);
- }
-
- /**
- * @deprecated since 2.0.0 and will be removed in 3.0.0. Do not write further code which depends
- * on this call. Instead use getStoreFileScanner() which uses the StoreFileScanner
- * class/interface which is the preferred way to scan a store with higher level
- * concepts. should we cache the blocks? use pread (for concurrent small readers) is
- * scanner being used for compaction?
- * @return the underlying HFileScanner
- * @see <a href="https://issues.apache.org/jira/browse/HBASE-15296">HBASE-15296</a>
- */
- @Deprecated
- public HFileScanner getScanner(boolean cacheBlocks, boolean pread, boolean isCompaction) {
+ protected HFileScanner getScanner(boolean cacheBlocks, boolean pread, boolean isCompaction) {
return reader.getScanner(conf, cacheBlocks, pread, isCompaction);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
index 9b4e1aea906..24578417ef3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/BulkLoadHFilesTool.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.AsyncAdmin;
@@ -74,7 +75,6 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.io.HFileLink;
-import org.apache.hadoop.hbase.io.HalfStoreFileReader;
import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
@@ -83,11 +83,12 @@ import org.apache.hadoop.hbase.io.hfile.HFileContext;
import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
import org.apache.hadoop.hbase.io.hfile.HFileInfo;
-import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.io.hfile.ReaderContext;
import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.regionserver.StoreFileReader;
+import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
import org.apache.hadoop.hbase.regionserver.StoreUtils;
import org.apache.hadoop.hbase.security.UserProvider;
@@ -757,6 +758,41 @@ public class BulkLoadHFilesTool extends Configured implements BulkLoadHFiles, To
copyHFileHalf(conf, inFile, bottomOut, bottomReference, familyDesc, loc);
}
+ private static StoreFileWriter initStoreFileWriter(Configuration conf, Cell cell,
+ HFileContext hFileContext, CacheConfig cacheConf, BloomType bloomFilterType, FileSystem fs,
+ Path outFile, AsyncTableRegionLocator loc) throws IOException {
+ if (conf.getBoolean(LOCALITY_SENSITIVE_CONF_KEY, DEFAULT_LOCALITY_SENSITIVE)) {
+ byte[] rowKey = CellUtil.cloneRow(cell);
+ HRegionLocation hRegionLocation = FutureUtils.get(loc.getRegionLocation(rowKey));
+ InetSocketAddress[] favoredNodes = null;
+ if (null == hRegionLocation) {
+ LOG.warn("Failed get region location for rowkey {} , Using writer without favoured nodes.",
+ Bytes.toString(rowKey));
+ return new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
+ .withBloomType(bloomFilterType).withFileContext(hFileContext).build();
+ } else {
+ LOG.debug("First rowkey: [{}]", Bytes.toString(rowKey));
+ InetSocketAddress initialIsa =
+ new InetSocketAddress(hRegionLocation.getHostname(), hRegionLocation.getPort());
+ if (initialIsa.isUnresolved()) {
+ LOG.warn("Failed get location for region {} , Using writer without favoured nodes.",
+ hRegionLocation);
+ return new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
+ .withBloomType(bloomFilterType).withFileContext(hFileContext).build();
+ } else {
+ LOG.debug("Use favored nodes writer: {}", initialIsa.getHostString());
+ favoredNodes = new InetSocketAddress[] { initialIsa };
+ return new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
+ .withBloomType(bloomFilterType).withFileContext(hFileContext)
+ .withFavoredNodes(favoredNodes).build();
+ }
+ }
+ } else {
+ return new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
+ .withBloomType(bloomFilterType).withFileContext(hFileContext).build();
+ }
+ }
+
/**
* Copy half of an HFile into a new HFile with favored nodes.
*/
@@ -765,14 +801,14 @@ public class BulkLoadHFilesTool extends Configured implements BulkLoadHFiles, To
throws IOException {
FileSystem fs = inFile.getFileSystem(conf);
CacheConfig cacheConf = CacheConfig.DISABLED;
- HalfStoreFileReader halfReader = null;
+ StoreFileReader halfReader = null;
StoreFileWriter halfWriter = null;
try {
ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, inFile).build();
StoreFileInfo storeFileInfo =
new StoreFileInfo(conf, fs, fs.getFileStatus(inFile), reference);
storeFileInfo.initHFileInfo(context);
- halfReader = (HalfStoreFileReader) storeFileInfo.createReader(context, cacheConf);
+ halfReader = storeFileInfo.createReader(context, cacheConf);
storeFileInfo.getHFileInfo().initMetaAndIndex(halfReader.getHFileReader());
Map<byte[], byte[]> fileInfo = halfReader.loadFileInfo();
@@ -785,51 +821,22 @@ public class BulkLoadHFilesTool extends Configured implements BulkLoadHFiles, To
.withDataBlockEncoding(familyDescriptor.getDataBlockEncoding()).withIncludesTags(true)
.withCreateTime(EnvironmentEdgeManager.currentTime()).build();
- HFileScanner scanner = halfReader.getScanner(false, false, false);
- scanner.seekTo();
- do {
- final Cell cell = scanner.getCell();
- if (null != halfWriter) {
- halfWriter.append(cell);
- } else {
-
- // init halfwriter
- if (conf.getBoolean(LOCALITY_SENSITIVE_CONF_KEY, DEFAULT_LOCALITY_SENSITIVE)) {
- byte[] rowKey = CellUtil.cloneRow(cell);
- HRegionLocation hRegionLocation = FutureUtils.get(loc.getRegionLocation(rowKey));
- InetSocketAddress[] favoredNodes = null;
- if (null == hRegionLocation) {
- LOG.warn(
- "Failed get region location for rowkey {} , Using writer without favoured nodes.",
- Bytes.toString(rowKey));
- halfWriter = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
- .withBloomType(bloomFilterType).withFileContext(hFileContext).build();
- } else {
- LOG.debug("First rowkey: [{}]", Bytes.toString(rowKey));
- InetSocketAddress initialIsa =
- new InetSocketAddress(hRegionLocation.getHostname(), hRegionLocation.getPort());
- if (initialIsa.isUnresolved()) {
- LOG.warn("Failed get location for region {} , Using writer without favoured nodes.",
- hRegionLocation);
- halfWriter = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
- .withBloomType(bloomFilterType).withFileContext(hFileContext).build();
- } else {
- LOG.debug("Use favored nodes writer: {}", initialIsa.getHostString());
- favoredNodes = new InetSocketAddress[] { initialIsa };
- halfWriter = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
- .withBloomType(bloomFilterType).withFileContext(hFileContext)
- .withFavoredNodes(favoredNodes).build();
- }
- }
- } else {
- halfWriter = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
- .withBloomType(bloomFilterType).withFileContext(hFileContext).build();
+ try (StoreFileScanner scanner =
+ halfReader.getStoreFileScanner(false, false, false, Long.MAX_VALUE, 0, false)) {
+ scanner.seek(KeyValue.LOWESTKEY);
+ for (;;) {
+ Cell cell = scanner.next();
+ if (cell == null) {
+ break;
+ }
+ if (halfWriter == null) {
+ // init halfwriter
+ halfWriter = initStoreFileWriter(conf, cell, hFileContext, cacheConf, bloomFilterType,
+ fs, outFile, loc);
}
halfWriter.append(cell);
}
-
- } while (scanner.next());
-
+ }
for (Map.Entry<byte[], byte[]> entry : fileInfo.entrySet()) {
if (shouldCopyHFileMetaKey(entry.getKey())) {
halfWriter.appendFileInfo(entry.getKey(), entry.getValue());
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
index 7dd4cbe44f9..0a41159e3aa 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
@@ -124,21 +124,22 @@ public class TestHalfStoreFileReader {
(HalfStoreFileReader) storeFileInfo.createReader(context, cacheConf);
storeFileInfo.getHFileInfo().initMetaAndIndex(halfreader.getHFileReader());
halfreader.loadFileInfo();
- final HFileScanner scanner = halfreader.getScanner(false, false);
-
- scanner.seekTo();
- Cell curr;
- do {
- curr = scanner.getCell();
- KeyValue reseekKv = getLastOnCol(curr);
- int ret = scanner.reseekTo(reseekKv);
- assertTrue("reseek to returned: " + ret, ret > 0);
- // System.out.println(curr + ": " + ret);
- } while (scanner.next());
-
- int ret = scanner.reseekTo(getLastOnCol(curr));
- // System.out.println("Last reseek: " + ret);
- assertTrue(ret > 0);
+ try (HFileScanner scanner = halfreader.getScanner(false, false, false)) {
+
+ scanner.seekTo();
+ Cell curr;
+ do {
+ curr = scanner.getCell();
+ KeyValue reseekKv = getLastOnCol(curr);
+ int ret = scanner.reseekTo(reseekKv);
+ assertTrue("reseek to returned: " + ret, ret > 0);
+ // System.out.println(curr + ": " + ret);
+ } while (scanner.next());
+
+ int ret = scanner.reseekTo(getLastOnCol(curr));
+ // System.out.println("Last reseek: " + ret);
+ assertTrue(ret > 0);
+ }
halfreader.close(true);
}
@@ -222,9 +223,14 @@ public class TestHalfStoreFileReader {
(HalfStoreFileReader) storeFileInfo.createReader(context, cacheConfig);
storeFileInfo.getHFileInfo().initMetaAndIndex(halfreader.getHFileReader());
halfreader.loadFileInfo();
- final HFileScanner scanner = halfreader.getScanner(false, false);
- scanner.seekBefore(seekBefore);
- return scanner.getCell();
+ try (HFileScanner scanner = halfreader.getScanner(false, false, false)) {
+ scanner.seekBefore(seekBefore);
+ if (scanner.getCell() != null) {
+ return KeyValueUtil.copyToNewKeyValue(scanner.getCell());
+ } else {
+ return null;
+ }
+ }
}
private KeyValue getLastOnCol(Cell curr) {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index c0bc72079cb..9b43ab32c2c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTestConst;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
@@ -62,7 +63,6 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
-import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
@@ -316,13 +316,13 @@ public class TestCompaction {
private int count() throws IOException {
int count = 0;
for (HStoreFile f : this.r.stores.get(COLUMN_FAMILY_TEXT).getStorefiles()) {
- HFileScanner scanner = f.getReader().getScanner(false, false);
- if (!scanner.seekTo()) {
- continue;
+ f.initReader();
+ try (StoreFileScanner scanner = f.getPreadScanner(false, Long.MAX_VALUE, 0, false)) {
+ scanner.seek(KeyValue.LOWESTKEY);
+ while (scanner.next() != null) {
+ count++;
+ }
}
- do {
- count++;
- } while (scanner.next());
}
return count;
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
index 55320e94a9f..f4fff4f5cbe 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
@@ -99,26 +99,26 @@ public class TestFSErrorsExposed {
BloomType.NONE, true);
sf.initReader();
StoreFileReader reader = sf.getReader();
- HFileScanner scanner = reader.getScanner(false, true);
+ try (HFileScanner scanner = reader.getScanner(false, true, false)) {
+ FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
+ assertNotNull(inStream);
- FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
- assertNotNull(inStream);
+ scanner.seekTo();
+ // Do at least one successful read
+ assertTrue(scanner.next());
- scanner.seekTo();
- // Do at least one successful read
- assertTrue(scanner.next());
+ faultyfs.startFaults();
- faultyfs.startFaults();
-
- try {
- int scanned = 0;
- while (scanner.next()) {
- scanned++;
+ try {
+ int scanned = 0;
+ while (scanner.next()) {
+ scanned++;
+ }
+ fail("Scanner didn't throw after faults injected");
+ } catch (IOException ioe) {
+ LOG.info("Got expected exception", ioe);
+ assertTrue(ioe.getMessage().contains("Fault"));
}
- fail("Scanner didn't throw after faults injected");
- } catch (IOException ioe) {
- LOG.info("Got expected exception", ioe);
- assertTrue(ioe.getMessage().contains("Fault"));
}
reader.close(true); // end of test so evictOnClose
}
@@ -147,27 +147,32 @@ public class TestFSErrorsExposed {
Collections.singletonList(sf), false, true, false, false,
// 0 is passed as readpoint because this test operates on HStoreFile directly
0);
- KeyValueScanner scanner = scanners.get(0);
+ try {
+ KeyValueScanner scanner = scanners.get(0);
- FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
- assertNotNull(inStream);
+ FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
+ assertNotNull(inStream);
- scanner.seek(KeyValue.LOWESTKEY);
- // Do at least one successful read
- assertNotNull(scanner.next());
- faultyfs.startFaults();
+ scanner.seek(KeyValue.LOWESTKEY);
+ // Do at least one successful read
+ assertNotNull(scanner.next());
+ faultyfs.startFaults();
- try {
- int scanned = 0;
- while (scanner.next() != null) {
- scanned++;
+ try {
+ int scanned = 0;
+ while (scanner.next() != null) {
+ scanned++;
+ }
+ fail("Scanner didn't throw after faults injected");
+ } catch (IOException ioe) {
+ LOG.info("Got expected exception", ioe);
+ assertTrue(ioe.getMessage().contains("Could not iterate"));
+ }
+ } finally {
+ for (StoreFileScanner scanner : scanners) {
+ scanner.close();
}
- fail("Scanner didn't throw after faults injected");
- } catch (IOException ioe) {
- LOG.info("Got expected exception", ioe);
- assertTrue(ioe.getMessage().contains("Could not iterate"));
}
- scanner.close();
}
/**
@@ -202,13 +207,13 @@ public class TestFSErrorsExposed {
// Load some data
util.loadTable(table, fam, false);
util.flush();
- util.countRows(table);
+ HBaseTestingUtil.countRows(table);
// Kill the DFS cluster
util.getDFSCluster().shutdownDataNodes();
try {
- util.countRows(table);
+ HBaseTestingUtil.countRows(table);
fail("Did not fail to count after removing data");
} catch (Exception e) {
LOG.info("Got expected error", e);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
index aa7fb53566d..824c195fd0f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java
@@ -245,19 +245,20 @@ public class TestHStoreFile {
refHsf.initReader();
// Now confirm that I can read from the reference and that it only gets
// keys from top half of the file.
- HFileScanner s = refHsf.getReader().getScanner(false, false);
- Cell kv = null;
- for (boolean first = true; (!s.isSeeked() && s.seekTo()) || s.next();) {
- ByteBuffer bb = ByteBuffer.wrap(((KeyValue) s.getKey()).getKey());
- kv = KeyValueUtil.createKeyValueFromKey(bb);
- if (first) {
- assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), midRow, 0,
- midRow.length));
- first = false;
+ try (HFileScanner s = refHsf.getReader().getScanner(false, false, false)) {
+ Cell kv = null;
+ for (boolean first = true; (!s.isSeeked() && s.seekTo()) || s.next();) {
+ ByteBuffer bb = ByteBuffer.wrap(((KeyValue) s.getKey()).getKey());
+ kv = KeyValueUtil.createKeyValueFromKey(bb);
+ if (first) {
+ assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), midRow, 0,
+ midRow.length));
+ first = false;
+ }
}
+ assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), finalRow, 0,
+ finalRow.length));
}
- assertTrue(Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), finalRow, 0,
- finalRow.length));
}
@Test
@@ -333,11 +334,12 @@ public class TestHStoreFile {
hsf.initReader();
// Now confirm that I can read from the link
- int count = 1;
- HFileScanner s = hsf.getReader().getScanner(false, false);
- s.seekTo();
- while (s.next()) {
- count++;
+ int count = 0;
+ try (StoreFileScanner scanner = hsf.getPreadScanner(false, Long.MAX_VALUE, 0, false)) {
+ scanner.seek(KeyValue.LOWESTKEY);
+ while (scanner.next() != null) {
+ count++;
+ }
}
assertEquals((LAST_CHAR - FIRST_CHAR + 1) * (LAST_CHAR - FIRST_CHAR + 1), count);
}
@@ -395,26 +397,25 @@ public class TestHStoreFile {
hsfA.initReader();
// Now confirm that I can read from the ref to link
- int count = 1;
- HFileScanner s = hsfA.getReader().getScanner(false, false);
- s.seekTo();
- while (s.next()) {
- count++;
+ int count = 0;
+ try (StoreFileScanner scanner = hsfA.getPreadScanner(false, Long.MAX_VALUE, 0, false)) {
+ scanner.seek(KeyValue.LOWESTKEY);
+ while (scanner.next() != null) {
+ count++;
+ }
+ assertTrue(count > 0); // read some rows here
}
- assertTrue(count > 0); // read some rows here
// Try to open store file from link
HStoreFile hsfB = new HStoreFile(this.fs, pathB, testConf, cacheConf, BloomType.NONE, true);
hsfB.initReader();
// Now confirm that I can read from the ref to link
- HFileScanner sB = hsfB.getReader().getScanner(false, false);
- sB.seekTo();
-
- // count++ as seekTo() will advance the scanner
- count++;
- while (sB.next()) {
- count++;
+ try (StoreFileScanner scanner = hsfB.getPreadScanner(false, Long.MAX_VALUE, 0, false)) {
+ scanner.seek(KeyValue.LOWESTKEY);
+ while (scanner.next() != null) {
+ count++;
+ }
}
// read the rest of the rows
@@ -454,39 +455,41 @@ public class TestHStoreFile {
// Now test reading from the top.
boolean first = true;
ByteBuffer key = null;
- HFileScanner topScanner = top.getScanner(false, false);
- while (
- (!topScanner.isSeeked() && topScanner.seekTo())
- || (topScanner.isSeeked() && topScanner.next())
- ) {
- key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey());
-
- if (
- (PrivateCellUtil.compare(topScanner.getReader().getComparator(), midKV, key.array(),
- key.arrayOffset(), key.limit())) > 0
+ try (HFileScanner topScanner = top.getScanner(false, false, false)) {
+ while (
+ (!topScanner.isSeeked() && topScanner.seekTo())
+ || (topScanner.isSeeked() && topScanner.next())
) {
- fail("key=" + Bytes.toStringBinary(key) + " < midkey=" + midkey);
- }
- if (first) {
- first = false;
- LOG.info("First in top: " + Bytes.toString(Bytes.toBytes(key)));
+ key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey());
+
+ if (
+ (PrivateCellUtil.compare(topScanner.getReader().getComparator(), midKV, key.array(),
+ key.arrayOffset(), key.limit())) > 0
+ ) {
+ fail("key=" + Bytes.toStringBinary(key) + " < midkey=" + midkey);
+ }
+ if (first) {
+ first = false;
+ LOG.info("First in top: " + Bytes.toString(Bytes.toBytes(key)));
+ }
}
}
LOG.info("Last in top: " + Bytes.toString(Bytes.toBytes(key)));
first = true;
- HFileScanner bottomScanner = bottom.getScanner(false, false);
- while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) {
- previous = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());
- key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());
- if (first) {
- first = false;
- LOG.info("First in bottom: " + Bytes.toString(Bytes.toBytes(previous)));
+ try (HFileScanner bottomScanner = bottom.getScanner(false, false, false)) {
+ while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) {
+ previous = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());
+ key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());
+ if (first) {
+ first = false;
+ LOG.info("First in bottom: " + Bytes.toString(Bytes.toBytes(previous)));
+ }
+ assertTrue(key.compareTo(bbMidkeyBytes) < 0);
+ }
+ if (previous != null) {
+ LOG.info("Last in bottom: " + Bytes.toString(Bytes.toBytes(previous)));
}
- assertTrue(key.compareTo(bbMidkeyBytes) < 0);
- }
- if (previous != null) {
- LOG.info("Last in bottom: " + Bytes.toString(Bytes.toBytes(previous)));
}
// Remove references.
regionFs.cleanupDaughterRegion(topHri);
@@ -507,29 +510,31 @@ public class TestHStoreFile {
top = topF.getReader();
// Now read from the top.
first = true;
- topScanner = top.getScanner(false, false);
- KeyValue.KeyOnlyKeyValue keyOnlyKV = new KeyValue.KeyOnlyKeyValue();
- while ((!topScanner.isSeeked() && topScanner.seekTo()) || topScanner.next()) {
- key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey());
- keyOnlyKV.setKey(key.array(), 0 + key.arrayOffset(), key.limit());
- assertTrue(PrivateCellUtil.compare(topScanner.getReader().getComparator(), keyOnlyKV,
- badmidkey, 0, badmidkey.length) >= 0);
- if (first) {
- first = false;
- KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);
- LOG.info("First top when key < bottom: " + keyKV);
- String tmp =
- Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
- for (int i = 0; i < tmp.length(); i++) {
- assertTrue(tmp.charAt(i) == 'a');
+ try (HFileScanner topScanner = top.getScanner(false, false, false)) {
+ KeyValue.KeyOnlyKeyValue keyOnlyKV = new KeyValue.KeyOnlyKeyValue();
+ while ((!topScanner.isSeeked() && topScanner.seekTo()) || topScanner.next()) {
+ key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey());
+ keyOnlyKV.setKey(key.array(), 0 + key.arrayOffset(), key.limit());
+ assertTrue(PrivateCellUtil.compare(topScanner.getReader().getComparator(), keyOnlyKV,
+ badmidkey, 0, badmidkey.length) >= 0);
+ if (first) {
+ first = false;
+ KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);
+ LOG.info("First top when key < bottom: " + keyKV);
+ String tmp =
+ Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
+ for (int i = 0; i < tmp.length(); i++) {
+ assertTrue(tmp.charAt(i) == 'a');
+ }
}
}
- }
- KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);
- LOG.info("Last top when key < bottom: " + keyKV);
- String tmp = Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
- for (int i = 0; i < tmp.length(); i++) {
- assertTrue(tmp.charAt(i) == 'z');
+ KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);
+ LOG.info("Last top when key < bottom: " + keyKV);
+ String tmp =
+ Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
+ for (int i = 0; i < tmp.length(); i++) {
+ assertTrue(tmp.charAt(i) == 'z');
+ }
}
// Remove references.
regionFs.cleanupDaughterRegion(topHri);
@@ -545,25 +550,28 @@ public class TestHStoreFile {
bottomF.initReader();
bottom = bottomF.getReader();
first = true;
- bottomScanner = bottom.getScanner(false, false);
- while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) {
- key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());
- if (first) {
- first = false;
- keyKV = KeyValueUtil.createKeyValueFromKey(key);
- LOG.info("First bottom when key > top: " + keyKV);
- tmp = Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
- for (int i = 0; i < tmp.length(); i++) {
- assertTrue(tmp.charAt(i) == 'a');
+ try (HFileScanner bottomScanner = bottom.getScanner(false, false, false)) {
+ while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) {
+ key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey());
+ if (first) {
+ first = false;
+ KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);
+ LOG.info("First bottom when key > top: " + keyKV);
+ String tmp =
+ Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
+ for (int i = 0; i < tmp.length(); i++) {
+ assertTrue(tmp.charAt(i) == 'a');
+ }
}
}
- }
- keyKV = KeyValueUtil.createKeyValueFromKey(key);
- LOG.info("Last bottom when key > top: " + keyKV);
- for (int i = 0; i < tmp.length(); i++) {
- assertTrue(
- Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength()).charAt(i)
- == 'z');
+ KeyValue keyKV = KeyValueUtil.createKeyValueFromKey(key);
+ LOG.info("Last bottom when key > top: " + keyKV);
+ String tmp =
+ Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength());
+ for (int i = 0; i < tmp.length(); i++) {
+ assertTrue(Bytes.toString(keyKV.getRowArray(), keyKV.getRowOffset(), keyKV.getRowLength())
+ .charAt(i) == 'z');
+ }
}
} finally {
if (top != null) {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
index 4c96dc221ab..9ec1ed80931 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTestConst;
import org.apache.hadoop.hbase.KeepDeletedCells;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Delete;
@@ -51,7 +52,6 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
-import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
@@ -331,16 +331,21 @@ public class TestMajorCompaction {
int count1 = 0;
int count2 = 0;
for (HStoreFile f : r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) {
- HFileScanner scanner = f.getReader().getScanner(false, false);
- scanner.seekTo();
- do {
- byte[] row = CellUtil.cloneRow(scanner.getCell());
- if (Bytes.equals(row, STARTROW)) {
- count1++;
- } else if (Bytes.equals(row, secondRowBytes)) {
- count2++;
+ try (StoreFileScanner scanner = f.getPreadScanner(false, Long.MAX_VALUE, 0, false)) {
+ scanner.seek(KeyValue.LOWESTKEY);
+ for (Cell cell;;) {
+ cell = scanner.next();
+ if (cell == null) {
+ break;
+ }
+ byte[] row = CellUtil.cloneRow(cell);
+ if (Bytes.equals(row, STARTROW)) {
+ count1++;
+ } else if (Bytes.equals(row, secondRowBytes)) {
+ count2++;
+ }
}
- } while (scanner.next());
+ }
}
assertEquals(countRow1, count1);
assertEquals(countRow2, count2);
@@ -349,13 +354,12 @@ public class TestMajorCompaction {
private int count() throws IOException {
int count = 0;
for (HStoreFile f : r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) {
- HFileScanner scanner = f.getReader().getScanner(false, false);
- if (!scanner.seekTo()) {
- continue;
+ try (StoreFileScanner scanner = f.getPreadScanner(false, Long.MAX_VALUE, 0, false)) {
+ scanner.seek(KeyValue.LOWESTKEY);
+ while (scanner.next() != null) {
+ count++;
+ }
}
- do {
- count++;
- } while (scanner.next());
}
return count;
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
index f7c65b02d8b..68c6b6434c4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicas.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TestMetaTableAccessor;
import org.apache.hadoop.hbase.client.Consistency;
@@ -43,7 +44,6 @@ import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -501,16 +501,19 @@ public class TestRegionReplicas {
// Our file does not exist anymore. was moved by the compaction above.
LOG.debug(Boolean.toString(getRS().getFileSystem().exists(sf.getPath())));
Assert.assertFalse(getRS().getFileSystem().exists(sf.getPath()));
-
- HFileScanner scanner = sf.getReader().getScanner(false, false);
- scanner.seekTo();
- do {
- keys++;
-
- Cell cell = scanner.getCell();
- sum += Integer
- .parseInt(Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
- } while (scanner.next());
+ sf.initReader();
+ try (StoreFileScanner scanner = sf.getPreadScanner(false, Long.MAX_VALUE, 0, false)) {
+ scanner.seek(KeyValue.LOWESTKEY);
+ for (Cell cell;;) {
+ cell = scanner.next();
+ if (cell == null) {
+ break;
+ }
+ keys++;
+ sum += Integer.parseInt(
+ Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
+ }
+ }
}
Assert.assertEquals(3000, keys);
Assert.assertEquals(4498500, sum);