You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2020/01/02 03:32:47 UTC
[hbase] branch branch-2 updated: HBASE-23624 Add a tool to dump the
procedure info in HFile (#975)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new bee7f4e HBASE-23624 Add a tool to dump the procedure info in HFile (#975)
bee7f4e is described below
commit bee7f4e08c4edd896587c161a886a0c82711cf63
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Thu Jan 2 11:23:55 2020 +0800
HBASE-23624 Add a tool to dump the procedure info in HFile (#975)
Signed-off-by: stack <st...@apache.org>
---
.../java/org/apache/hadoop/hbase/CellUtil.java | 26 +--
.../org/apache/hadoop/hbase/io/hfile/HFile.java | 2 +-
.../store/region/HFileProcedurePrettyPrinter.java | 175 +++++++++++++++++++++
.../store/region/RegionProcedureStore.java | 39 +++--
.../store/region/WALProcedurePrettyPrinter.java | 2 +-
.../store/region/RegionProcedureStoreTestBase.java | 49 ++++++
.../region/TestHFileProcedurePrettyPrinter.java | 153 ++++++++++++++++++
.../store/region/TestRegionProcedureStore.java | 28 +---
.../region/TestWALProcedurePrettyPrinter.java | 28 +---
9 files changed, 416 insertions(+), 86 deletions(-)
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
index 3d4ae6c..1dbbe43 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
@@ -34,7 +34,7 @@ import java.util.List;
import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.Optional;
-
+import java.util.function.Function;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
@@ -1297,17 +1297,25 @@ public final class CellUtil {
* @return The Key portion of the passed <code>cell</code> as a String.
*/
public static String getCellKeyAsString(Cell cell) {
- StringBuilder sb = new StringBuilder(
- Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
+ return getCellKeyAsString(cell,
+ c -> Bytes.toStringBinary(c.getRowArray(), c.getRowOffset(), c.getRowLength()));
+ }
+
+ /**
+ * @param cell the cell to convert
+ * @param rowConverter used to convert the row of the cell to a string
+ * @return The Key portion of the passed <code>cell</code> as a String.
+ */
+ public static String getCellKeyAsString(Cell cell, Function<Cell, String> rowConverter) {
+ StringBuilder sb = new StringBuilder(rowConverter.apply(cell));
sb.append('/');
- sb.append(cell.getFamilyLength() == 0 ? ""
- : Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(),
- cell.getFamilyLength()));
+ sb.append(cell.getFamilyLength() == 0 ? "" :
+ Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()));
// KeyValue only added ':' if family is non-null. Do same.
if (cell.getFamilyLength() > 0) sb.append(':');
- sb.append(cell.getQualifierLength() == 0 ? ""
- : Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(),
- cell.getQualifierLength()));
+ sb.append(cell.getQualifierLength() == 0 ? "" :
+ Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(),
+ cell.getQualifierLength()));
sb.append('/');
sb.append(KeyValue.humanReadableTimestamp(cell.getTimestamp()));
sb.append('/');
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
index 3719611..cb6a352 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
@@ -634,7 +634,7 @@ public class HFile {
* @return The list of files found.
* @throws IOException When scanning the files fails.
*/
- static List<Path> getStoreFiles(FileSystem fs, Path regionDir)
+ public static List<Path> getStoreFiles(FileSystem fs, Path regionDir)
throws IOException {
List<Path> regionHFiles = new ArrayList<>();
PathFilter dirFilter = new FSUtils.DirFilter(fs);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/HFileProcedurePrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/HFileProcedurePrettyPrinter.java
new file mode 100644
index 0000000..110547f
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/HFileProcedurePrettyPrinter.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure2.store.region;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.PrivateCellUtil;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
+import org.apache.hadoop.hbase.util.AbstractHBaseTool;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+
+import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
+import org.apache.hbase.thirdparty.org.apache.commons.cli.Option;
+import org.apache.hbase.thirdparty.org.apache.commons.cli.OptionGroup;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+
+/**
+ * A tool to dump the procedures in the HFiles.
+ * <p/>
+ * The different between this and {@link org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter} is
+ * that, this class will decode the procedure in the cell for better debugging. You are free to use
+ * {@link org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter} to dump the same file as well.
+ */
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
+@InterfaceStability.Evolving
+public class HFileProcedurePrettyPrinter extends AbstractHBaseTool {
+
+ private Long procId;
+
+ private List<Path> files = new ArrayList<>();
+
+ private final PrintStream out;
+
+ public HFileProcedurePrettyPrinter() {
+ this(System.out);
+ }
+
+ public HFileProcedurePrettyPrinter(PrintStream out) {
+ this.out = out;
+ }
+
+ @Override
+ protected void addOptions() {
+ addOptWithArg("w", "seekToPid", "Seek to this procedure id and print this procedure only");
+ OptionGroup files = new OptionGroup();
+ files.addOption(new Option("f", "file", true,
+ "File to scan. Pass full-path; e.g. hdfs://a:9000/MasterProcs/master/procedure/p/xxx"));
+ files.addOption(new Option("a", "all", false, "Scan the whole procedure region."));
+ files.setRequired(true);
+ options.addOptionGroup(files);
+ }
+
+ private void addAllHFiles() throws IOException {
+ Path masterProcDir =
+ new Path(CommonFSUtils.getWALRootDir(conf), RegionProcedureStore.MASTER_PROCEDURE_DIR);
+ Path tableDir = CommonFSUtils.getTableDir(masterProcDir, RegionProcedureStore.TABLE_NAME);
+ FileSystem fs = tableDir.getFileSystem(conf);
+ Path regionDir =
+ fs.listStatus(tableDir, p -> RegionInfo.isEncodedRegionName(Bytes.toBytes(p.getName())))[0]
+ .getPath();
+ List<Path> regionFiles = HFile.getStoreFiles(fs, regionDir);
+ files.addAll(regionFiles);
+ }
+
+ @Override
+ protected void processOptions(CommandLine cmd) {
+ if (cmd.hasOption("w")) {
+ String key = cmd.getOptionValue("w");
+ if (key != null && key.length() != 0) {
+ procId = Long.parseLong(key);
+ } else {
+ throw new IllegalArgumentException("Invalid row is specified.");
+ }
+ }
+ if (cmd.hasOption("f")) {
+ files.add(new Path(cmd.getOptionValue("f")));
+ }
+ if (cmd.hasOption("a")) {
+ try {
+ addAllHFiles();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ }
+
+ private void printCell(Cell cell) throws IOException {
+ out.print("K: " + CellUtil.getCellKeyAsString(cell,
+ c -> Long.toString(Bytes.toLong(c.getRowArray(), c.getRowOffset(), c.getRowLength()))));
+ if (cell.getType() == Cell.Type.Put) {
+ if (cell.getValueLength() == 0) {
+ out.println(" V: mark deleted");
+ } else {
+ Procedure<?> proc = ProcedureUtil.convertToProcedure(ProcedureProtos.Procedure.parser()
+ .parseFrom(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
+ out.println(" V: " + proc.toStringDetails());
+ }
+ } else {
+ out.println();
+ }
+ }
+
+ private void processFile(Path file) throws IOException {
+ out.println("Scanning -> " + file);
+ FileSystem fs = file.getFileSystem(conf);
+ try (HFile.Reader reader = HFile.createReader(fs, file, CacheConfig.DISABLED, true, conf);
+ HFileScanner scanner = reader.getScanner(false, false, false)) {
+ if (procId != null) {
+ if (scanner
+ .seekTo(PrivateCellUtil.createFirstOnRow(Bytes.toBytes(procId.longValue()))) != -1) {
+ do {
+ Cell cell = scanner.getCell();
+ long currentProcId =
+ Bytes.toLong(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
+ if (currentProcId != procId.longValue()) {
+ break;
+ }
+ printCell(cell);
+ } while (scanner.next());
+ }
+ } else {
+ if (scanner.seekTo()) {
+ do {
+ Cell cell = scanner.getCell();
+ printCell(cell);
+ } while (scanner.next());
+ }
+ }
+ }
+ }
+
+ @Override
+ protected int doWork() throws Exception {
+ for (Path file : files) {
+ processFile(file);
+ }
+ return 0;
+ }
+
+ public static void main(String[] args) {
+ new HFileProcedurePrettyPrinter().doStaticMain(args);
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java
index ae0a54d..05a5059 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java
@@ -112,17 +112,15 @@ public class RegionProcedureStore extends ProcedureStoreBase {
static final String LOGCLEANER_PLUGINS = "hbase.procedure.store.region.logcleaner.plugins";
- private static final String DATA_DIR = "data";
-
- private static final String REPLAY_EDITS_DIR = "replay";
+ private static final String REPLAY_EDITS_DIR = "recovered.wals";
private static final String DEAD_WAL_DIR_SUFFIX = "-dead";
- private static final TableName TABLE_NAME = TableName.valueOf("master:procedure");
+ static final TableName TABLE_NAME = TableName.valueOf("master:procedure");
static final byte[] FAMILY = Bytes.toBytes("p");
- private static final byte[] PROC_QUALIFIER = Bytes.toBytes("d");
+ static final byte[] PROC_QUALIFIER = Bytes.toBytes("d");
private static final int REGION_ID = 1;
@@ -231,27 +229,26 @@ public class RegionProcedureStore extends ProcedureStoreBase {
return wal;
}
- private HRegion bootstrap(Configuration conf, FileSystem fs, Path rootDir, Path dataDir)
- throws IOException {
+ private HRegion bootstrap(Configuration conf, FileSystem fs, Path rootDir) throws IOException {
RegionInfo regionInfo = RegionInfoBuilder.newBuilder(TABLE_NAME).setRegionId(REGION_ID).build();
- Path tmpDataDir = new Path(dataDir.getParent(), dataDir.getName() + "-tmp");
- if (fs.exists(tmpDataDir) && !fs.delete(tmpDataDir, true)) {
- throw new IOException("Can not delete partial created proc region " + tmpDataDir);
+ Path tmpTableDir = CommonFSUtils.getTableDir(rootDir, TableName
+ .valueOf(TABLE_NAME.getNamespaceAsString(), TABLE_NAME.getQualifierAsString() + "-tmp"));
+ if (fs.exists(tmpTableDir) && !fs.delete(tmpTableDir, true)) {
+ throw new IOException("Can not delete partial created proc region " + tmpTableDir);
}
- Path tableDir = CommonFSUtils.getTableDir(tmpDataDir, TABLE_NAME);
- HRegion.createHRegion(conf, regionInfo, fs, tableDir, TABLE_DESC).close();
- if (!fs.rename(tmpDataDir, dataDir)) {
- throw new IOException("Can not rename " + tmpDataDir + " to " + dataDir);
+ HRegion.createHRegion(conf, regionInfo, fs, tmpTableDir, TABLE_DESC).close();
+ Path tableDir = CommonFSUtils.getTableDir(rootDir, TABLE_NAME);
+ if (!fs.rename(tmpTableDir, tableDir)) {
+ throw new IOException("Can not rename " + tmpTableDir + " to " + tableDir);
}
WAL wal = createWAL(fs, rootDir, regionInfo);
return HRegion.openHRegionFromTableDir(conf, fs, tableDir, regionInfo, TABLE_DESC, wal, null,
null);
}
- private HRegion open(Configuration conf, FileSystem fs, Path rootDir, Path dataDir)
- throws IOException {
+ private HRegion open(Configuration conf, FileSystem fs, Path rootDir) throws IOException {
String factoryId = server.getServerName().toString();
- Path tableDir = CommonFSUtils.getTableDir(dataDir, TABLE_NAME);
+ Path tableDir = CommonFSUtils.getTableDir(rootDir, TABLE_NAME);
Path regionDir =
fs.listStatus(tableDir, p -> RegionInfo.isEncodedRegionName(Bytes.toBytes(p.getName())))[0]
.getPath();
@@ -391,13 +388,13 @@ public class RegionProcedureStore extends ProcedureStoreBase {
walRoller.start();
walFactory = new WALFactory(conf, server.getServerName().toString());
- Path dataDir = new Path(rootDir, DATA_DIR);
- if (fs.exists(dataDir)) {
+ Path tableDir = CommonFSUtils.getTableDir(rootDir, TABLE_NAME);
+ if (fs.exists(tableDir)) {
// load the existing region.
- region = open(conf, fs, rootDir, dataDir);
+ region = open(conf, fs, rootDir);
} else {
// bootstrapping...
- region = bootstrap(conf, fs, rootDir, dataDir);
+ region = bootstrap(conf, fs, rootDir);
}
flusherAndCompactor = new RegionFlusherAndCompactor(conf, server, region);
walRoller.setFlusherAndCompactor(flusherAndCompactor);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/WALProcedurePrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/WALProcedurePrettyPrinter.java
index 9afd981..1e5c142 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/WALProcedurePrettyPrinter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/WALProcedurePrettyPrinter.java
@@ -49,7 +49,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
* <p/>
* The different between this and {@link WALPrettyPrinter} is that, this class will decode the
* procedure in the WALEdit for better debugging. You are free to use {@link WALPrettyPrinter} to
- * dump the safe file as well.
+ * dump the same file as well.
*/
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
@InterfaceStability.Evolving
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStoreTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStoreTestBase.java
new file mode 100644
index 0000000..6f07805
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStoreTestBase.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure2.store.region;
+
+import java.io.IOException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter;
+import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.junit.After;
+import org.junit.Before;
+
+public class RegionProcedureStoreTestBase {
+
+ protected HBaseCommonTestingUtility htu;
+
+ protected RegionProcedureStore store;
+
+ @Before
+ public void setUp() throws IOException {
+ htu = new HBaseCommonTestingUtility();
+ htu.getConfiguration().setBoolean(MemStoreLAB.USEMSLAB_KEY, false);
+ Path testDir = htu.getDataTestDir();
+ CommonFSUtils.setWALRootDir(htu.getConfiguration(), testDir);
+ store = RegionProcedureStoreTestHelper.createStore(htu.getConfiguration(), new LoadCounter());
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ store.stop(true);
+ htu.cleanupTestDir();
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestHFileProcedurePrettyPrinter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestHFileProcedurePrettyPrinter.java
new file mode 100644
index 0000000..8d42b03
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestHFileProcedurePrettyPrinter.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.procedure2.store.region;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.lang3.mutable.MutableLong;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestHFileProcedurePrettyPrinter extends RegionProcedureStoreTestBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestHFileProcedurePrettyPrinter.class);
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestHFileProcedurePrettyPrinter.class);
+
+ private List<String> checkOutput(BufferedReader reader, MutableLong putCount,
+ MutableLong deleteCount, MutableLong markDeletedCount) throws IOException {
+ putCount.setValue(0);
+ deleteCount.setValue(0);
+ markDeletedCount.setValue(0);
+ List<String> fileScanned = new ArrayList<>();
+ for (;;) {
+ String line = reader.readLine();
+ if (line == null) {
+ return fileScanned;
+ }
+ LOG.info(line);
+ if (line.contains("V: mark deleted")) {
+ markDeletedCount.increment();
+ } else if (line.contains("/Put/")) {
+ putCount.increment();
+ } else if (line.contains("/DeleteFamily/")) {
+ deleteCount.increment();
+ } else if (line.startsWith("Scanning -> ")) {
+ fileScanned.add(line.split(" -> ")[1]);
+ } else {
+ fail("Unrecognized output: " + line);
+ }
+ }
+ }
+
+ @Test
+ public void test() throws Exception {
+ HFileProcedurePrettyPrinter printer = new HFileProcedurePrettyPrinter();
+ // -a or -f is required so passing empty args will cause an error and return a non-zero value.
+ assertNotEquals(0, ToolRunner.run(htu.getConfiguration(), printer, new String[0]));
+ List<RegionProcedureStoreTestProcedure> procs = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ RegionProcedureStoreTestProcedure proc = new RegionProcedureStoreTestProcedure();
+ store.insert(proc, null);
+ procs.add(proc);
+ }
+ store.region.flush(true);
+ for (int i = 0; i < 5; i++) {
+ store.delete(procs.get(i).getProcId());
+ }
+ store.region.flush(true);
+ store.cleanup();
+ store.region.flush(true);
+ Path tableDir = CommonFSUtils.getTableDir(
+ new Path(htu.getDataTestDir(), RegionProcedureStore.MASTER_PROCEDURE_DIR),
+ RegionProcedureStore.TABLE_NAME);
+ FileSystem fs = tableDir.getFileSystem(htu.getConfiguration());
+ Path regionDir =
+ fs.listStatus(tableDir, p -> RegionInfo.isEncodedRegionName(Bytes.toBytes(p.getName())))[0]
+ .getPath();
+ List<Path> storefiles = HFile.getStoreFiles(fs, regionDir);
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(bos);
+ MutableLong putCount = new MutableLong();
+ MutableLong deleteCount = new MutableLong();
+ MutableLong markDeletedCount = new MutableLong();
+ for (Path file : storefiles) {
+ bos.reset();
+ printer = new HFileProcedurePrettyPrinter(out);
+ assertEquals(0,
+ ToolRunner.run(htu.getConfiguration(), printer, new String[] { "-f", file.toString() }));
+ try (BufferedReader reader =
+ new BufferedReader(new InputStreamReader(new ByteArrayInputStream(bos.toByteArray()),
+ StandardCharsets.UTF_8))) {
+ List<String> fileScanned = checkOutput(reader, putCount, deleteCount, markDeletedCount);
+ assertEquals(1, fileScanned.size());
+ assertEquals(file.toString(), fileScanned.get(0));
+ if (putCount.longValue() == 10) {
+ assertEquals(0, deleteCount.longValue());
+ assertEquals(0, markDeletedCount.longValue());
+ } else if (deleteCount.longValue() == 5) {
+ assertEquals(0, putCount.longValue());
+ assertEquals(0, markDeletedCount.longValue());
+ } else if (markDeletedCount.longValue() == 5) {
+ assertEquals(0, putCount.longValue());
+ assertEquals(0, deleteCount.longValue());
+ } else {
+ fail("Should have entered one of the above 3 branches");
+ }
+ }
+ }
+ bos.reset();
+ printer = new HFileProcedurePrettyPrinter(out);
+ assertEquals(0, ToolRunner.run(htu.getConfiguration(), printer, new String[] { "-a" }));
+ try (BufferedReader reader = new BufferedReader(
+ new InputStreamReader(new ByteArrayInputStream(bos.toByteArray()), StandardCharsets.UTF_8))) {
+ List<String> fileScanned = checkOutput(reader, putCount, deleteCount, markDeletedCount);
+ assertEquals(3, fileScanned.size());
+ assertEquals(10, putCount.longValue());
+ assertEquals(5, deleteCount.longValue());
+ assertEquals(5, markDeletedCount.longValue());
+ }
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStore.java
index b3daa89..6fb7d9b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStore.java
@@ -21,23 +21,16 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter;
-import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.CommonFSUtils;
-import org.junit.After;
-import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -45,7 +38,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Category({ MasterTests.class, MediumTests.class })
-public class TestRegionProcedureStore {
+public class TestRegionProcedureStore extends RegionProcedureStoreTestBase {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
@@ -53,25 +46,6 @@ public class TestRegionProcedureStore {
private static final Logger LOG = LoggerFactory.getLogger(TestRegionProcedureStore.class);
- private HBaseCommonTestingUtility htu;
-
- private RegionProcedureStore store;
-
- @Before
- public void setUp() throws IOException {
- htu = new HBaseCommonTestingUtility();
- htu.getConfiguration().setBoolean(MemStoreLAB.USEMSLAB_KEY, false);
- Path testDir = htu.getDataTestDir();
- CommonFSUtils.setWALRootDir(htu.getConfiguration(), testDir);
- store = RegionProcedureStoreTestHelper.createStore(htu.getConfiguration(), new LoadCounter());
- }
-
- @After
- public void tearDown() throws IOException {
- store.stop(true);
- htu.cleanupTestDir();
- }
-
private void verifyProcIdsOnRestart(final Set<Long> procIds) throws Exception {
LOG.debug("expected: " + procIds);
LoadCounter loader = new LoadCounter();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestWALProcedurePrettyPrinter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestWALProcedurePrettyPrinter.java
index f9b816e..194b508 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestWALProcedurePrettyPrinter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/procedure2/store/region/TestWALProcedurePrettyPrinter.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertEquals;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
@@ -31,16 +30,10 @@ import java.util.List;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter;
-import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.util.ToolRunner;
-import org.junit.After;
-import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -48,7 +41,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Category({ MasterTests.class, MediumTests.class })
-public class TestWALProcedurePrettyPrinter {
+public class TestWALProcedurePrettyPrinter extends RegionProcedureStoreTestBase {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
@@ -56,25 +49,6 @@ public class TestWALProcedurePrettyPrinter {
private static final Logger LOG = LoggerFactory.getLogger(TestWALProcedurePrettyPrinter.class);
- private HBaseCommonTestingUtility htu;
-
- private RegionProcedureStore store;
-
- @Before
- public void setUp() throws IOException {
- htu = new HBaseCommonTestingUtility();
- htu.getConfiguration().setBoolean(MemStoreLAB.USEMSLAB_KEY, false);
- Path testDir = htu.getDataTestDir();
- CommonFSUtils.setWALRootDir(htu.getConfiguration(), testDir);
- store = RegionProcedureStoreTestHelper.createStore(htu.getConfiguration(), new LoadCounter());
- }
-
- @After
- public void tearDown() throws IOException {
- store.stop(true);
- htu.cleanupTestDir();
- }
-
@Test
public void test() throws Exception {
List<RegionProcedureStoreTestProcedure> procs = new ArrayList<>();