You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2018/09/09 16:27:13 UTC
hbase git commit: HBASE-21171 [amv2] Tool to parse a directory of
MasterProcWALs standalone
Repository: hbase
Updated Branches:
refs/heads/branch-2.1 f85fba4a5 -> f755ded2d
HBASE-21171 [amv2] Tool to parse a directory of MasterProcWALs standalone
Signed-off-by: Mike Drob <md...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f755ded2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f755ded2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f755ded2
Branch: refs/heads/branch-2.1
Commit: f755ded2d24b870770406c80a47f61555ac6f8ab
Parents: f85fba4
Author: Michael Stack <st...@apache.org>
Authored: Fri Sep 7 17:07:11 2018 -0700
Committer: Michael Stack <st...@apache.org>
Committed: Sat Sep 8 20:35:15 2018 -0700
----------------------------------------------------------------------
.../store/wal/ProcedureWALPrettyPrinter.java | 1 +
.../procedure2/store/wal/WALProcedureStore.java | 38 ++++++++++++++++----
2 files changed, 33 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f755ded2/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
index c692365..a7712b1 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALPrettyPrinter.java
@@ -47,6 +47,7 @@ import org.apache.hbase.thirdparty.org.apache.commons.cli.PosixParser;
/**
* ProcedureWALPrettyPrinter prints the contents of a given ProcedureWAL file
+ * @see WALProcedureStore#main(String[]) if you want to check parse of a directory of WALs.
*/
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
@InterfaceStability.Evolving
http://git-wip-us.apache.org/repos/asf/hbase/blob/f755ded2/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
----------------------------------------------------------------------
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
index 975452a..70c71f5 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java
@@ -43,9 +43,11 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.log.HBaseMarkers;
import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreBase;
import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker;
import org.apache.hadoop.hbase.procedure2.util.ByteSlot;
@@ -64,6 +66,8 @@ import org.apache.hbase.thirdparty.org.apache.commons.collections4.queue.Circula
/**
* WAL implementation of the ProcedureStore.
+ * @see ProcedureWALPrettyPrinter for printing content of a single WAL.
+ * @see #main(String[]) to parse a directory of MasterWALProcs.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -424,7 +428,6 @@ public class WALProcedureStore extends ProcedureStoreBase {
it.next(); // Skip the current log
ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() {
- long count = 0;
@Override
public void setMaxProcId(long maxProcId) {
@@ -433,11 +436,6 @@ public class WALProcedureStore extends ProcedureStoreBase {
@Override
public void load(ProcedureIterator procIter) throws IOException {
- if ((++count % 1000) == 0) {
- // Log every 1000 procedures otherwise it looks like Master is dead if loads of WALs
- // and procedures to load.
- LOG.debug("Loaded {} procedures", this.count);
- }
loader.load(procIter);
}
@@ -1324,4 +1322,32 @@ public class WALProcedureStore extends ProcedureStoreBase {
log.close();
return log;
}
+
+ /**
+ * Parses a directory of WALs building up ProcedureState.
+ * For testing parse and profiling.
+ * @param args Include pointer to directory of WAL files for a store instance to parse & load.
+ */
+ public static void main(String [] args) throws IOException {
+ Configuration conf = HBaseConfiguration.create();
+ if (args == null || args.length != 1) {
+ System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR.");
+ System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR");
+ System.exit(-1);
+ }
+ WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null,
+ new WALProcedureStore.LeaseRecovery() {
+ @Override
+ public void recoverFileLease(FileSystem fs, Path path) throws IOException {
+ // no-op
+ }
+ });
+ try {
+ store.start(16);
+ ProcedureExecutor pe = new ProcedureExecutor(conf, new Object()/*Pass anything*/, store);
+ pe.init(1, true);
+ } finally {
+ store.stop(true);
+ }
+ }
}